From 46d3d09d49aded1d9f1b468c83fce75e07d631dc Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Thu, 27 Jun 2013 13:19:56 -0500 Subject: [PATCH] Consolidated lower/upper her[2]k blocked variants. Details: - Consolidated lower and upper blocked variants for herk and her2k, and renamed the resulting variants, according to the same changes recently made to trmm and trsm. - Implemented support for four new subpartitions types: BLIS_SUBPART1T BLIS_SUBPART1B BLIS_SUBPART1L BLIS_SUBPART1R which correspond to "merged" partitions that include the middle "1" partition as well as either the neighboring "0" or "2" partition. This is used to clean up code in herk/her2k var2 that attempts to partition away the strictly zero region above or below the diagonal of a matrix operand that is being marched through diagonally. - Added safeguards to herk macro-kernels that skip any leading or trailing zero region in the panel of C that is passed in. This is now needed given that herk/her2k var1 no longer partitions off this zero region before calling the macro-kernel (via bli_her[2]k_int()). - Updated comments and other whitespace changes to trmm/trsm macro-kernels. --- frame/3/her2k/bli_her2k.h | 11 +-- ...r2k_l_blk_var1.c => bli_her2k_blk_var1f.c} | 84 +++++++------------ ...r2k_l_blk_var2.h => bli_her2k_blk_var1f.h} | 18 ++-- ...r2k_l_blk_var2.c => bli_her2k_blk_var2f.c} | 64 +++++++------- ...r2k_l_blk_var1.h => bli_her2k_blk_var2f.h} | 18 ++-- ...her2k_blk_var3.c => bli_her2k_blk_var3f.c} | 18 ++-- ...her2k_blk_var3.h => bli_her2k_blk_var3f.h} | 18 ++-- frame/3/her2k/bli_her2k_int.c | 16 ++-- .../her2k/{ => other}/bli_her2k_l_blk_var4.c | 0 .../her2k/{ => other}/bli_her2k_l_blk_var4.h | 0 .../her2k/{ => other}/bli_her2k_u_blk_var1.c | 0 .../her2k/{ => other}/bli_her2k_u_blk_var1.h | 0 .../her2k/{ => other}/bli_her2k_u_blk_var2.c | 0 .../her2k/{ => other}/bli_her2k_u_blk_var2.h | 0 .../her2k/{ => other}/bli_her2k_u_blk_var4.c | 0 .../her2k/{ => other}/bli_her2k_u_blk_var4.h | 0 frame/3/herk/bli_herk.h | 11 +-- ...herk_l_blk_var1.c => bli_herk_blk_var1f.c} | 49 ++++------- ...i_herk_blk_var3.h => bli_herk_blk_var1f.h} | 12 +-- ...herk_l_blk_var2.c => bli_herk_blk_var2f.c} | 50 +++++------ ...herk_l_blk_var1.h => bli_herk_blk_var2f.h} | 12 +-- ...i_herk_blk_var3.c => bli_herk_blk_var3f.c} | 12 +-- ...herk_l_blk_var2.h => bli_herk_blk_var3f.h} | 12 +-- frame/3/herk/bli_herk_int.c | 16 ++-- frame/3/herk/bli_herk_l_ker_var2.c | 27 +++++- frame/3/herk/bli_herk_u_ker_var2.c | 27 +++++- .../3/herk/{ => other}/bli_herk_l_blk_var4.c | 0 .../3/herk/{ => other}/bli_herk_l_blk_var4.h | 0 .../3/herk/{ => other}/bli_herk_u_blk_var1.c | 30 ++----- .../3/herk/{ => other}/bli_herk_u_blk_var1.h | 0 .../3/herk/{ => other}/bli_herk_u_blk_var2.c | 21 ++--- .../3/herk/{ => other}/bli_herk_u_blk_var2.h | 0 .../3/herk/{ => other}/bli_herk_u_blk_var4.c | 0 .../3/herk/{ => other}/bli_herk_u_blk_var4.h | 0 frame/3/trmm/bli_trmm_ll_ker_var2.c | 10 +-- frame/3/trmm/bli_trmm_lu_ker_var2.c | 9 +- frame/3/trmm/bli_trmm_rl_ker_var2.c | 23 ++--- frame/3/trmm/bli_trmm_ru_ker_var2.c | 9 +- frame/3/trsm/bli_trsm_ll_ker_var2.c | 8 +- frame/3/trsm/bli_trsm_lu_ker_var2.c | 5 +- frame/3/trsm/bli_trsm_rl_ker_var2.c | 11 +-- frame/3/trsm/bli_trsm_ru_ker_var2.c | 9 +- frame/base/bli_check.c | 12 ++- frame/base/bli_part.c | 54 ++++++++---- frame/include/bli_type_defs.h | 8 +- 45 files changed, 353 insertions(+), 331 deletions(-) rename frame/3/her2k/{bli_her2k_l_blk_var1.c => bli_her2k_blk_var1f.c} (71%) rename frame/3/her2k/{bli_her2k_l_blk_var2.h => bli_her2k_blk_var1f.h} (83%) rename frame/3/her2k/{bli_her2k_l_blk_var2.c => bli_her2k_blk_var2f.c} (78%) rename frame/3/her2k/{bli_her2k_l_blk_var1.h => bli_her2k_blk_var2f.h} (83%) rename frame/3/her2k/{bli_her2k_blk_var3.c => bli_her2k_blk_var3f.c} (93%) rename frame/3/her2k/{bli_her2k_blk_var3.h => bli_her2k_blk_var3f.h} (82%) rename frame/3/her2k/{ => other}/bli_her2k_l_blk_var4.c (100%) rename frame/3/her2k/{ => other}/bli_her2k_l_blk_var4.h (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var1.c (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var1.h (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var2.c (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var2.h (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var4.c (100%) rename frame/3/her2k/{ => other}/bli_her2k_u_blk_var4.h (100%) rename frame/3/herk/{bli_herk_l_blk_var1.c => bli_herk_blk_var1f.c} (75%) rename frame/3/herk/{bli_herk_blk_var3.h => bli_herk_blk_var1f.h} (87%) rename frame/3/herk/{bli_herk_l_blk_var2.c => bli_herk_blk_var2f.c} (79%) rename frame/3/herk/{bli_herk_l_blk_var1.h => bli_herk_blk_var2f.h} (87%) rename frame/3/herk/{bli_herk_blk_var3.c => bli_herk_blk_var3f.c} (94%) rename frame/3/herk/{bli_herk_l_blk_var2.h => bli_herk_blk_var3f.h} (87%) rename frame/3/herk/{ => other}/bli_herk_l_blk_var4.c (100%) rename frame/3/herk/{ => other}/bli_herk_l_blk_var4.h (100%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var1.c (83%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var1.h (100%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var2.c (87%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var2.h (100%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var4.c (100%) rename frame/3/herk/{ => other}/bli_herk_u_blk_var4.h (100%) diff --git a/frame/3/her2k/bli_her2k.h b/frame/3/her2k/bli_her2k.h index b75188593..02afd30d0 100644 --- a/frame/3/her2k/bli_her2k.h +++ b/frame/3/her2k/bli_her2k.h @@ -37,16 +37,11 @@ #include "bli_her2k_int.h" #include "bli_her2k_target.h" -#include "bli_her2k_l_blk_var1.h" -#include "bli_her2k_u_blk_var1.h" +#include "bli_her2k_blk_var1f.h" -#include "bli_her2k_l_blk_var2.h" -#include "bli_her2k_u_blk_var2.h" +#include "bli_her2k_blk_var2f.h" -#include "bli_her2k_blk_var3.h" - -#include "bli_her2k_l_blk_var4.h" -#include "bli_her2k_u_blk_var4.h" +#include "bli_her2k_blk_var3f.h" #include "bli_her2k_l_ker_var2.h" #include "bli_her2k_u_ker_var2.h" diff --git a/frame/3/her2k/bli_her2k_l_blk_var1.c b/frame/3/her2k/bli_her2k_blk_var1f.c similarity index 71% rename from frame/3/her2k/bli_her2k_l_blk_var1.c rename to frame/3/her2k/bli_her2k_blk_var1f.c index a9bf32d2e..6463193c2 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var1.c +++ b/frame/3/her2k/bli_her2k_blk_var1f.c @@ -34,34 +34,32 @@ #include "blis.h" -void bli_her2k_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { obj_t a1, a1_pack; - obj_t bh_pack, bhL_pack; + obj_t bh_pack; obj_t b1, b1_pack; - obj_t ah_pack, ahL_pack; - obj_t c1; - obj_t c1L, c1L_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offL, nL; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &bh_pack ); bli_obj_init_pack( &b1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1L_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -88,30 +86,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, a, + b_alg = bli_determine_blocksize_f( i, m_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, a, &a1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Bh_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &bh_pack, &bhL_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -121,19 +108,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &bhL_pack, + &bh_pack, beta, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -159,30 +146,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, b, + b_alg = bli_determine_blocksize_f( i, m_trans, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, b, &b1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &ah_pack, &ahL_pack ); - // Initialize objects for packing B1 and C1. bli_packm_init( &b1, &b1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack B1 and scale by alpha_conj (if instructed). @@ -192,19 +168,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha_conj, &b1_pack, - &ahL_pack, + &ah_pack, &BLIS_ONE, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -212,6 +188,6 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &b1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1L_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/her2k/bli_her2k_l_blk_var2.h b/frame/3/her2k/bli_her2k_blk_var1f.h similarity index 83% rename from frame/3/her2k/bli_her2k_l_blk_var2.h rename to frame/3/her2k/bli_her2k_blk_var1f.h index 161b63a07..b96caf99b 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var2.h +++ b/frame/3/her2k/bli_her2k_blk_var1f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_l_blk_var2.c b/frame/3/her2k/bli_her2k_blk_var2f.c similarity index 78% rename from frame/3/her2k/bli_her2k_l_blk_var2.c rename to frame/3/her2k/bli_her2k_blk_var2f.c index b25069857..2c53aad77 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var2.c +++ b/frame/3/her2k/bli_her2k_blk_var2f.c @@ -34,34 +34,39 @@ #include "blis.h" -void bli_her2k_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { - obj_t a_pack, aB_pack; + obj_t a_pack, aS_pack; obj_t bh1, bh1_pack; - obj_t b_pack, bB_pack; + obj_t b_pack, bS_pack; obj_t ah1, ah1_pack; obj_t c1; - obj_t c1B, c1B_pack; + obj_t c1S, c1S_pack; dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offB, mB; + subpart_t stored_part; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); bli_obj_init_pack( &bh1_pack ); bli_obj_init_pack( &b_pack ); bli_obj_init_pack( &ah1_pack ); - bli_obj_init_pack( &c1B_pack ); + bli_obj_init_pack( &c1S_pack ); + + // The upper and lower variants are identical, except for which + // merged subpartition is acquired in the loop body. + if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; + else stored_part = BLIS_SUBPART1T; // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *c ); @@ -103,23 +108,20 @@ void bli_her2k_l_blk_var2( obj_t* alpha, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding regions - // of A_pack and B_pack. We compute the length of the subpartition - // taking the location of the diagonal into account. - offB = bli_max( 0, -bli_obj_diag_offset_after_trans( c1 ) ); - mB = bli_obj_length_after_trans( c1 ) - offB; - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &c1, &c1B ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &a_pack, &aB_pack ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &b_pack, &bB_pack ); + // of A_pack and B_pack. + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &c1, &c1S ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &a_pack, &aS_pack ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &b_pack, &bS_pack ); // Initialize objects for packing B1', A1', and C1. bli_packm_init( &bh1, &bh1_pack, cntl_sub_packm_b( cntl ) ); bli_packm_init( &ah1, &ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1B, &c1B_pack, + bli_packm_init( &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Pack B1' and scale by alpha (if instructed). @@ -134,22 +136,22 @@ void bli_her2k_l_blk_var2( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1B, &c1B_pack, + &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Perform her2k subproblem. bli_her2k_int( alpha, - &aB_pack, + &aS_pack, &bh1_pack, alpha_conj, - &bB_pack, + &bS_pack, &ah1_pack, beta, - &c1B_pack, + &c1S_pack, cntl_sub_her2k( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1B_pack, &c1B, + bli_unpackm_int( &c1S_pack, &c1S, cntl_sub_unpackm_c( cntl ) ); } @@ -159,6 +161,6 @@ void bli_her2k_l_blk_var2( obj_t* alpha, bli_obj_release_pack( &bh1_pack ); bli_obj_release_pack( &b_pack ); bli_obj_release_pack( &ah1_pack ); - bli_obj_release_pack( &c1B_pack ); + bli_obj_release_pack( &c1S_pack ); } diff --git a/frame/3/her2k/bli_her2k_l_blk_var1.h b/frame/3/her2k/bli_her2k_blk_var2f.h similarity index 83% rename from frame/3/her2k/bli_her2k_l_blk_var1.h rename to frame/3/her2k/bli_her2k_blk_var2f.h index 20e74fb46..e1cbcdb6b 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var1.h +++ b/frame/3/her2k/bli_her2k_blk_var2f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_blk_var3.c b/frame/3/her2k/bli_her2k_blk_var3f.c similarity index 93% rename from frame/3/her2k/bli_her2k_blk_var3.c rename to frame/3/her2k/bli_her2k_blk_var3f.c index 10d2522eb..58a9699c7 100644 --- a/frame/3/her2k/bli_her2k_blk_var3.c +++ b/frame/3/her2k/bli_her2k_blk_var3f.c @@ -34,15 +34,15 @@ #include "blis.h" -void bli_her2k_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { obj_t a1, a1_pack; obj_t bh1, bh1_pack; diff --git a/frame/3/her2k/bli_her2k_blk_var3.h b/frame/3/her2k/bli_her2k_blk_var3f.h similarity index 82% rename from frame/3/her2k/bli_her2k_blk_var3.h rename to frame/3/her2k/bli_her2k_blk_var3f.h index 844b435f5..0dbac4fc5 100644 --- a/frame/3/her2k/bli_her2k_blk_var3.h +++ b/frame/3/her2k/bli_her2k_blk_var3f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_int.c b/frame/3/her2k/bli_her2k_int.c index 2362bbfc6..db6dcd9b2 100644 --- a/frame/3/her2k/bli_her2k_int.c +++ b/frame/3/her2k/bli_her2k_int.c @@ -46,24 +46,22 @@ typedef void (*FUNCPTR_T)( obj_t* alpha, obj_t* c, her2k_t* cntl ); -static FUNCPTR_T vars[2][5][3] = +static FUNCPTR_T vars[2][4][3] = { // lower { // unblocked optimized unblocked blocked - { NULL, NULL, bli_her2k_l_blk_var1 }, - { NULL, bli_her2k_l_ker_var2, bli_her2k_l_blk_var2 }, - { NULL, NULL, bli_her2k_blk_var3 }, - { NULL, NULL, bli_her2k_l_blk_var4 }, + { NULL, NULL, bli_her2k_blk_var1f }, + { NULL, bli_her2k_l_ker_var2, bli_her2k_blk_var2f }, + { NULL, NULL, bli_her2k_blk_var3f }, { NULL, NULL, NULL }, }, // upper { // unblocked optimized unblocked blocked - { NULL, NULL, bli_her2k_u_blk_var1 }, - { NULL, bli_her2k_u_ker_var2, bli_her2k_u_blk_var2 }, - { NULL, NULL, bli_her2k_blk_var3 }, - { NULL, NULL, bli_her2k_u_blk_var4 }, + { NULL, NULL, bli_her2k_blk_var1f }, + { NULL, bli_her2k_u_ker_var2, bli_her2k_blk_var2f }, + { NULL, NULL, bli_her2k_blk_var3f }, { NULL, NULL, NULL }, } }; diff --git a/frame/3/her2k/bli_her2k_l_blk_var4.c b/frame/3/her2k/other/bli_her2k_l_blk_var4.c similarity index 100% rename from frame/3/her2k/bli_her2k_l_blk_var4.c rename to frame/3/her2k/other/bli_her2k_l_blk_var4.c diff --git a/frame/3/her2k/bli_her2k_l_blk_var4.h b/frame/3/her2k/other/bli_her2k_l_blk_var4.h similarity index 100% rename from frame/3/her2k/bli_her2k_l_blk_var4.h rename to frame/3/her2k/other/bli_her2k_l_blk_var4.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var1.c b/frame/3/her2k/other/bli_her2k_u_blk_var1.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var1.c rename to frame/3/her2k/other/bli_her2k_u_blk_var1.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var1.h b/frame/3/her2k/other/bli_her2k_u_blk_var1.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var1.h rename to frame/3/her2k/other/bli_her2k_u_blk_var1.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var2.c b/frame/3/her2k/other/bli_her2k_u_blk_var2.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var2.c rename to frame/3/her2k/other/bli_her2k_u_blk_var2.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var2.h b/frame/3/her2k/other/bli_her2k_u_blk_var2.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var2.h rename to frame/3/her2k/other/bli_her2k_u_blk_var2.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var4.c b/frame/3/her2k/other/bli_her2k_u_blk_var4.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var4.c rename to frame/3/her2k/other/bli_her2k_u_blk_var4.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var4.h b/frame/3/her2k/other/bli_her2k_u_blk_var4.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var4.h rename to frame/3/her2k/other/bli_her2k_u_blk_var4.h diff --git a/frame/3/herk/bli_herk.h b/frame/3/herk/bli_herk.h index 1174bbe9c..82ef49688 100644 --- a/frame/3/herk/bli_herk.h +++ b/frame/3/herk/bli_herk.h @@ -37,16 +37,11 @@ #include "bli_herk_int.h" #include "bli_herk_target.h" -#include "bli_herk_l_blk_var1.h" -#include "bli_herk_u_blk_var1.h" +#include "bli_herk_blk_var1f.h" -#include "bli_herk_l_blk_var2.h" -#include "bli_herk_u_blk_var2.h" +#include "bli_herk_blk_var2f.h" -#include "bli_herk_blk_var3.h" - -#include "bli_herk_l_blk_var4.h" -#include "bli_herk_u_blk_var4.h" +#include "bli_herk_blk_var3f.h" #include "bli_herk_l_ker_var2.h" #include "bli_herk_u_ker_var2.h" diff --git a/frame/3/herk/bli_herk_l_blk_var1.c b/frame/3/herk/bli_herk_blk_var1f.c similarity index 75% rename from frame/3/herk/bli_herk_l_blk_var1.c rename to frame/3/herk/bli_herk_blk_var1f.c index b76334042..dc137e1ce 100644 --- a/frame/3/herk/bli_herk_l_blk_var1.c +++ b/frame/3/herk/bli_herk_blk_var1f.c @@ -34,27 +34,25 @@ #include "blis.h" -void bli_herk_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { obj_t a1, a1_pack; - obj_t ah_pack, ahL_pack; - obj_t c1; - obj_t c1L, c1L_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offL, nL; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1L_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -77,30 +75,19 @@ void bli_herk_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, a, + b_alg = bli_determine_blocksize_f( i, m_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, a, &a1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &ah_pack, &ahL_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -110,19 +97,19 @@ void bli_herk_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &ahL_pack, + &ah_pack, beta, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -130,6 +117,6 @@ void bli_herk_l_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1L_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/herk/bli_herk_blk_var3.h b/frame/3/herk/bli_herk_blk_var1f.h similarity index 87% rename from frame/3/herk/bli_herk_blk_var3.h rename to frame/3/herk/bli_herk_blk_var1f.h index 812c5c23e..94900b772 100644 --- a/frame/3/herk/bli_herk_blk_var3.h +++ b/frame/3/herk/bli_herk_blk_var1f.h @@ -32,10 +32,10 @@ */ -void bli_herk_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_l_blk_var2.c b/frame/3/herk/bli_herk_blk_var2f.c similarity index 79% rename from frame/3/herk/bli_herk_l_blk_var2.c rename to frame/3/herk/bli_herk_blk_var2f.c index 42ad4d2f4..df39ce55d 100644 --- a/frame/3/herk/bli_herk_l_blk_var2.c +++ b/frame/3/herk/bli_herk_blk_var2f.c @@ -34,27 +34,32 @@ #include "blis.h" -void bli_herk_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { - obj_t a_pack, aB_pack; + obj_t a_pack, aS_pack; obj_t ah1, ah1_pack; obj_t c1; - obj_t c1B, c1B_pack; + obj_t c1S, c1S_pack; dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offB, mB; + subpart_t stored_part; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); bli_obj_init_pack( &ah1_pack ); - bli_obj_init_pack( &c1B_pack ); + bli_obj_init_pack( &c1S_pack ); + + // The upper and lower variants are identical, except for which + // merged subpartition is acquired in the loop body. + if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; + else stored_part = BLIS_SUBPART1T; // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *c ); @@ -87,19 +92,16 @@ void bli_herk_l_blk_var2( obj_t* alpha, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding region - // of A_pack. We compute the length of the subpartition taking the - // location of the diagonal into account. - offB = bli_max( 0, -bli_obj_diag_offset_after_trans( c1 ) ); - mB = bli_obj_length_after_trans( c1 ) - offB; - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &c1, &c1B ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &a_pack, &aB_pack ); + // of A_pack. + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &c1, &c1S ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &a_pack, &aS_pack ); // Initialize objects for packing A1' and C1. bli_packm_init( &ah1, &ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1B, &c1B_pack, + bli_packm_init( &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Pack A1' and scale by alpha (if instructed). @@ -109,19 +111,19 @@ void bli_herk_l_blk_var2( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1B, &c1B_pack, + &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, - &aB_pack, + &aS_pack, &ah1_pack, beta, - &c1B_pack, + &c1S_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1B_pack, &c1B, + bli_unpackm_int( &c1S_pack, &c1S, cntl_sub_unpackm_c( cntl ) ); } @@ -129,6 +131,6 @@ void bli_herk_l_blk_var2( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a_pack ); bli_obj_release_pack( &ah1_pack ); - bli_obj_release_pack( &c1B_pack ); + bli_obj_release_pack( &c1S_pack ); } diff --git a/frame/3/herk/bli_herk_l_blk_var1.h b/frame/3/herk/bli_herk_blk_var2f.h similarity index 87% rename from frame/3/herk/bli_herk_l_blk_var1.h rename to frame/3/herk/bli_herk_blk_var2f.h index 6cfdd1a4e..1a5034242 100644 --- a/frame/3/herk/bli_herk_l_blk_var1.h +++ b/frame/3/herk/bli_herk_blk_var2f.h @@ -32,10 +32,10 @@ */ -void bli_herk_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_blk_var3.c b/frame/3/herk/bli_herk_blk_var3f.c similarity index 94% rename from frame/3/herk/bli_herk_blk_var3.c rename to frame/3/herk/bli_herk_blk_var3f.c index ddf9d6536..a54e341b9 100644 --- a/frame/3/herk/bli_herk_blk_var3.c +++ b/frame/3/herk/bli_herk_blk_var3f.c @@ -34,12 +34,12 @@ #include "blis.h" -void bli_herk_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { obj_t a1, a1_pack; obj_t ah1, ah1_pack; diff --git a/frame/3/herk/bli_herk_l_blk_var2.h b/frame/3/herk/bli_herk_blk_var3f.h similarity index 87% rename from frame/3/herk/bli_herk_l_blk_var2.h rename to frame/3/herk/bli_herk_blk_var3f.h index 8205750f7..5e96900f7 100644 --- a/frame/3/herk/bli_herk_l_blk_var2.h +++ b/frame/3/herk/bli_herk_blk_var3f.h @@ -32,10 +32,10 @@ */ -void bli_herk_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_int.c b/frame/3/herk/bli_herk_int.c index 85a09ce34..afeef8d32 100644 --- a/frame/3/herk/bli_herk_int.c +++ b/frame/3/herk/bli_herk_int.c @@ -43,24 +43,22 @@ typedef void (*FUNCPTR_T)( obj_t* alpha, obj_t* c, herk_t* cntl ); -static FUNCPTR_T vars[2][5][3] = +static FUNCPTR_T vars[2][4][3] = { // lower { // unblocked optimized unblocked blocked - { NULL, NULL, bli_herk_l_blk_var1 }, - { NULL, bli_herk_l_ker_var2, bli_herk_l_blk_var2 }, - { NULL, NULL, bli_herk_blk_var3 }, - { NULL, NULL, bli_herk_l_blk_var4 }, + { NULL, NULL, bli_herk_blk_var1f }, + { NULL, bli_herk_l_ker_var2, bli_herk_blk_var2f }, + { NULL, NULL, bli_herk_blk_var3f }, { NULL, NULL, NULL }, }, // upper { // unblocked optimized unblocked blocked - { NULL, NULL, bli_herk_u_blk_var1 }, - { NULL, bli_herk_u_ker_var2, bli_herk_u_blk_var2 }, - { NULL, NULL, bli_herk_blk_var3 }, - { NULL, NULL, bli_herk_u_blk_var4 }, + { NULL, NULL, bli_herk_blk_var1f }, + { NULL, bli_herk_u_ker_var2, bli_herk_blk_var2f }, + { NULL, NULL, bli_herk_blk_var3f }, { NULL, NULL, NULL }, } }; diff --git a/frame/3/herk/bli_herk_l_ker_var2.c b/frame/3/herk/bli_herk_l_ker_var2.c index 8f326e52b..0e657032d 100644 --- a/frame/3/herk/bli_herk_l_ker_var2.c +++ b/frame/3/herk/bli_herk_l_ker_var2.c @@ -186,7 +186,7 @@ void PASTEMAC(ch,varname)( \ dim_t n_iter, n_left; \ dim_t m_cur; \ dim_t n_cur; \ - dim_t i, j; \ + dim_t i, j, ip; \ inc_t rstep_a; \ inc_t cstep_b; \ inc_t rstep_c, cstep_c; \ @@ -205,6 +205,31 @@ void PASTEMAC(ch,varname)( \ \ /* If any dimension is zero, return immediately. */ \ if ( bli_zero_dim3( m, n, k ) ) return; \ +\ + /* Safeguard: If the current panel of C is entirely above the diagonal, + it is not stored. So we do nothing. */ \ + if ( bli_is_strictly_above_diag_n( diagoffc, m, n ) ) return; \ +\ + /* If there is a zero region above where the diagonal of C intersects + the left edge of the panel, adjust the pointer to C and A and treat + this case as if the diagonal offset were zero. */ \ + if ( diagoffc < 0 ) \ + { \ + ip = -diagoffc / MR; \ + i = ip * MR; \ + m = m - i; \ + diagoffc = -diagoffc % MR; \ + c_cast = c_cast + (i )*rs_c; \ + a_cast = a_cast + (ip )*ps_a; \ + } \ +\ + /* If there is a zero region to the right of where the diagonal + of C intersects the bottom of the panel, shrink it to prevent + "no-op" iterations from executing. */ \ + if ( diagoffc + m < n ) \ + { \ + n = diagoffc + m; \ + } \ \ /* Clear the temporary C buffer in case it has any infs or NaNs. */ \ PASTEMAC(ch,set0s_mxn)( MR, NR, \ diff --git a/frame/3/herk/bli_herk_u_ker_var2.c b/frame/3/herk/bli_herk_u_ker_var2.c index bc40d0196..98c840941 100644 --- a/frame/3/herk/bli_herk_u_ker_var2.c +++ b/frame/3/herk/bli_herk_u_ker_var2.c @@ -186,7 +186,7 @@ void PASTEMAC(ch,varname)( \ dim_t n_iter, n_left; \ dim_t m_cur; \ dim_t n_cur; \ - dim_t i, j; \ + dim_t i, j, jp; \ inc_t rstep_a; \ inc_t cstep_b; \ inc_t rstep_c, cstep_c; \ @@ -205,6 +205,31 @@ void PASTEMAC(ch,varname)( \ \ /* If any dimension is zero, return immediately. */ \ if ( bli_zero_dim3( m, n, k ) ) return; \ +\ + /* Safeguard: If the current panel of C is entirely below the diagonal, + it is not stored. So we do nothing. */ \ + if ( bli_is_strictly_below_diag_n( diagoffc, m, n ) ) return; \ +\ + /* If there is a zero region to the left of where the diagonal of C + intersects the top edge of the panel, adjust the pointer to C and B + and treat this case as if the diagonal offset were zero. */ \ + if ( diagoffc > 0 ) \ + { \ + jp = diagoffc / NR; \ + j = jp * NR; \ + n = n - j; \ + diagoffc = diagoffc % NR; \ + c_cast = c_cast + (j )*cs_c; \ + b_cast = b_cast + (jp )*ps_b; \ + } \ +\ + /* If there is a zero region below where the diagonal of C intersects + the right edge of the panel, shrink it to prevent "no-op" iterations + from executing. */ \ + if ( -diagoffc + n < m ) \ + { \ + m = -diagoffc + n; \ + } \ \ /* Clear the temporary C buffer in case it has any infs or NaNs. */ \ PASTEMAC(ch,set0s_mxn)( MR, NR, \ diff --git a/frame/3/herk/bli_herk_l_blk_var4.c b/frame/3/herk/other/bli_herk_l_blk_var4.c similarity index 100% rename from frame/3/herk/bli_herk_l_blk_var4.c rename to frame/3/herk/other/bli_herk_l_blk_var4.c diff --git a/frame/3/herk/bli_herk_l_blk_var4.h b/frame/3/herk/other/bli_herk_l_blk_var4.h similarity index 100% rename from frame/3/herk/bli_herk_l_blk_var4.h rename to frame/3/herk/other/bli_herk_l_blk_var4.h diff --git a/frame/3/herk/bli_herk_u_blk_var1.c b/frame/3/herk/other/bli_herk_u_blk_var1.c similarity index 83% rename from frame/3/herk/bli_herk_u_blk_var1.c rename to frame/3/herk/other/bli_herk_u_blk_var1.c index 20b722bc8..e352a1b00 100644 --- a/frame/3/herk/bli_herk_u_blk_var1.c +++ b/frame/3/herk/other/bli_herk_u_blk_var1.c @@ -42,19 +42,17 @@ void bli_herk_u_blk_var1( obj_t* alpha, herk_t* cntl ) { obj_t a1, a1_pack; - obj_t ah_pack, ahR_pack; - obj_t c1; - obj_t c1R, c1R_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offR, nR; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1R_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -89,20 +87,10 @@ void bli_herk_u_blk_var1( obj_t* alpha, bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offR = bli_max( 0, bli_obj_diag_offset_after_trans( c1 ) ); - nR = bli_obj_width_after_trans( c1 ) - offR; - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offR, nR, &c1, &c1R ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offR, nR, &ah_pack, &ahR_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1R, &c1R_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -112,19 +100,19 @@ void bli_herk_u_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1R, &c1R_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &ahR_pack, + &ah_pack, beta, - &c1R_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1R_pack, &c1R, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -132,6 +120,6 @@ void bli_herk_u_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1R_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/herk/bli_herk_u_blk_var1.h b/frame/3/herk/other/bli_herk_u_blk_var1.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var1.h rename to frame/3/herk/other/bli_herk_u_blk_var1.h diff --git a/frame/3/herk/bli_herk_u_blk_var2.c b/frame/3/herk/other/bli_herk_u_blk_var2.c similarity index 87% rename from frame/3/herk/bli_herk_u_blk_var2.c rename to frame/3/herk/other/bli_herk_u_blk_var2.c index e240f50b2..ff038b37c 100644 --- a/frame/3/herk/bli_herk_u_blk_var2.c +++ b/frame/3/herk/other/bli_herk_u_blk_var2.c @@ -49,7 +49,6 @@ void bli_herk_u_blk_var2( obj_t* alpha, dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offT, mT; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); @@ -77,25 +76,21 @@ void bli_herk_u_blk_var2( obj_t* alpha, for ( i = 0; i < n_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, n_trans, a, + b_alg = bli_determine_blocksize_f( i, n_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1' and C1. - bli_acquire_mpart_r2l( BLIS_SUBPART1, + bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, ah, &ah1 ); - bli_acquire_mpart_r2l( BLIS_SUBPART1, + bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding region - // of A_pack. We compute the length of the subpartition taking the - // location of the diagonal into account. - offT = 0; - mT = bli_min( bli_obj_length_after_trans( c1 ), - -bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offT, mT, &c1, &c1T ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offT, mT, &a_pack, &aT_pack ); + // of A_pack. + bli_acquire_mpart_t2b( BLIS_SUBPART1T, + i, b_alg, &c1, &c1T ); + bli_acquire_mpart_t2b( BLIS_SUBPART1T, + i, b_alg, &a_pack, &aT_pack ); // Initialize objects for packing A1' and C1. bli_packm_init( &ah1, &ah1_pack, diff --git a/frame/3/herk/bli_herk_u_blk_var2.h b/frame/3/herk/other/bli_herk_u_blk_var2.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var2.h rename to frame/3/herk/other/bli_herk_u_blk_var2.h diff --git a/frame/3/herk/bli_herk_u_blk_var4.c b/frame/3/herk/other/bli_herk_u_blk_var4.c similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var4.c rename to frame/3/herk/other/bli_herk_u_blk_var4.c diff --git a/frame/3/herk/bli_herk_u_blk_var4.h b/frame/3/herk/other/bli_herk_u_blk_var4.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var4.h rename to frame/3/herk/other/bli_herk_u_blk_var4.h diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index 43701f54a..d07ed5989 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -205,11 +205,11 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is negative, adjust the pointer to C and - the effective diagonal offset of A accordingly to skip over the zero - region that was not packed. (Note we only skip over the region in - increments of PACKMR since that is how the region would have been - skipped by packm.) */ \ + /* If there is a zero region above where the diagonal of A intersects the + left edge of the block, adjust the pointer to C and treat this case as + if the diagonal offset were zero. This skips over the region (in + increments of MR) that was not packed. (Note we skip in increments of + MR since that is how the region would have been skipped by packm.) */ \ if ( diagoffa < 0 ) \ { \ i = ( -diagoffa / MR ) * MR; \ diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 78fae73f9..2619ffbd1 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -205,16 +205,17 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is positive, adjust the pointer to B and + /* If there is a zero region to the left of where the diagonal of A + intersects the top edge of the block, adjust the pointer to B and treat this case as if the diagonal offset were zero. Note that we don't need to adjust the pointer to A since packm would have simply skipped over the region that was not stored. */ \ if ( diagoffa > 0 ) \ { \ - j = diagoffa; \ - k = k - j; \ + i = diagoffa; \ + k = k - i; \ diagoffa = 0; \ - b_cast = b_cast + (j )*rs_b; \ + b_cast = b_cast + (i )*rs_b; \ } \ \ /* If there is a zero region below where the diagonal of A intersects the diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 5d60dadbc..cfcab09ed 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -206,17 +206,18 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % NR != 0 ) k += NR - ( k % NR ); \ \ - /* If the diagonal offset of B is negative, adjust the pointer to A and - treat this case as if the diagonal offset were zero. Note that we - don't need to adjust the pointer to B since packm would have simply - skipped over the region that was not stored. */ \ - if ( diagoffb < 0 ) \ - { \ - j = -diagoffb; \ - k = k - j; \ - diagoffb = 0; \ - a_cast = a_cast + (j )*cs_a; \ - } \ + /* If there is a zero region above where the diagonal of B intersects + the left edge of the panel, adjust the pointer to A and treat this + case as if the diagonal offset were zero. Note that we don't need to + adjust the pointer to B since packm would have simply skipped over + the region that was not stored. */ \ + if ( diagoffb < 0 ) \ + { \ + j = -diagoffb; \ + k = k - j; \ + diagoffb = 0; \ + a_cast = a_cast + (j )*cs_a; \ + } \ \ /* If there is a zero region to the right of where the diagonal of B intersects the bottom of the panel, shrink it to prevent diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index 50bfd8f43..9889ac008 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -206,9 +206,12 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % NR != 0 ) k += NR - ( k % NR ); \ \ - /* If the diagonal offset of B is positive, adjust the pointer to C and - the effective diagonal offset of B accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region to the left of where the diagonal of B + intersects the top edge of the panel, adjust the pointer to C and + treat this case as if the diagonal offset were zero. This skips over + the region (in increments of NR) that was not packed. (Note we skip + in increments of NR since that is how the region would have been + skipped by packm.) */ \ if ( diagoffb > 0 ) \ { \ j = ( diagoffb / NR ) * NR; \ diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 79bab87f7..37f8583d3 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -206,9 +206,11 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is negative, adjust the pointer to C and - the effective diagonal offset of A accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region above where the diagonal of A intersects the + left edge of the block, adjust the pointer to C and treat this case as + if the diagonal offset were zero. This skips over the region (in + increments of MR) that was not packed. (Note we skip in increments of + MR since that is how the region would have been skipped by packm.) */ \ if ( diagoffa < 0 ) \ { \ i = ( -diagoffa / MR ) * MR; \ diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index f9bfb0013..bb2972102 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -207,10 +207,11 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is positive, adjust the pointer to B and + /* If there is a zero region to the left of where the diagonal of A + intersects the top edge of the block, adjust the pointer to B and treat this case as if the diagonal offset were zero. Note that we don't need to adjust the pointer to A since packm would have simply - skipped over the panels that were not stored. */ \ + skipped over the region that was not stored. */ \ if ( diagoffa > 0 ) \ { \ i = diagoffa; \ diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index ed499876a..591cb27bd 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -196,16 +196,17 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of B is negative, adjust the pointer to C and - treat this case as if the diagonal offset were zero. Note that we - don't need to adjust the pointer to B since packm would have simply - skipped over the region that was not stored. */ \ + /* If there is a zero region above where the diagonal of B intersects + the left edge of the panel, adjust the pointer to A and treat this + case as if the diagonal offset were zero. Note that we don't need to + adjust the pointer to B since packm would have simply skipped over + the region that was not stored. */ \ if ( diagoffb < 0 ) \ { \ j = -diagoffb; \ k = k - j; \ diagoffb = 0; \ - a_cast = a_cast + (j )*cs_a; \ + a_cast = a_cast + (j )*cs_a; \ } \ \ /* If there is a zero region to the right of where the diagonal diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 9c2d68014..2ee513a21 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -195,9 +195,12 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of B is positive, adjust the pointer to C and - the effective diagonal offset of B accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region to the left of where the diagonal of B + intersects the top edge of the panel, adjust the pointer to C and + treat this case as if the diagonal offset were zero. This skips over + the region (in increments of NR) that was not packed. (Note we skip + in increments of NR since that is how the region would have been + skipped by packm.) */ \ if ( diagoffb > 0 ) \ { \ j = ( diagoffb / NR ) * NR; \ diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index d5241e0d9..4cc1abb28 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -606,8 +606,10 @@ err_t bli_check_valid_3x1_subpart( subpart_t part ) { err_t e_val = BLIS_SUCCESS; - if ( part != BLIS_SUBPART0 && - part != BLIS_SUBPART1 && + if ( part != BLIS_SUBPART0 && + part != BLIS_SUBPART1T && + part != BLIS_SUBPART1 && + part != BLIS_SUBPART1B && part != BLIS_SUBPART2 ) e_val = BLIS_INVALID_3x1_SUBPART; @@ -618,8 +620,10 @@ err_t bli_check_valid_1x3_subpart( subpart_t part ) { err_t e_val = BLIS_SUCCESS; - if ( part != BLIS_SUBPART0 && - part != BLIS_SUBPART1 && + if ( part != BLIS_SUBPART0 && + part != BLIS_SUBPART1L && + part != BLIS_SUBPART1 && + part != BLIS_SUBPART1R && part != BLIS_SUBPART2 ) e_val = BLIS_INVALID_1x3_SUBPART; diff --git a/frame/base/bli_part.c b/frame/base/bli_part.c index 482b0724b..ac1e4bf13 100644 --- a/frame/base/bli_part.c +++ b/frame/base/bli_part.c @@ -99,6 +99,15 @@ void bli_acquire_mpart_t2b( subpart_t requested_part, m_part = i; n_part = n; } + if ( requested_part == BLIS_SUBPART1T ) + { + // A1T (offm,offn) unchanged. + // A1T is (i+b) x n. + offm_inc = 0; + offn_inc = 0; + m_part = i + b; + n_part = n; + } else if ( requested_part == BLIS_SUBPART1 ) { // A1 (offm,offn) += (i,0). @@ -108,6 +117,15 @@ void bli_acquire_mpart_t2b( subpart_t requested_part, m_part = b; n_part = n; } + else if ( requested_part == BLIS_SUBPART1B ) + { + // A1B (offm,offn) += (i,0). + // A1B is (m-i) x n. + offm_inc = i; + offn_inc = 0; + m_part = m - i; + n_part = n; + } else // if ( requested_part == BLIS_SUBPART2 ) { // A2 (offm,offn) += (i+b,0). @@ -269,6 +287,15 @@ void bli_acquire_mpart_l2r( subpart_t requested_part, m_part = m; n_part = j; } + if ( requested_part == BLIS_SUBPART1L ) + { + // A1L (offm,offn) unchanged. + // A1L is m x (j+b). + offm_inc = 0; + offn_inc = 0; + m_part = m; + n_part = j + b; + } else if ( requested_part == BLIS_SUBPART1 ) { // A1 (offm,offn) += (0,j). @@ -278,6 +305,15 @@ void bli_acquire_mpart_l2r( subpart_t requested_part, m_part = m; n_part = b; } + else if ( requested_part == BLIS_SUBPART1R ) + { + // A1R (offm,offn) += (0,j). + // A1R is m x (n-j). + offm_inc = 0; + offn_inc = j; + m_part = m; + n_part = n - j; + } else // if ( requested_part == BLIS_SUBPART2 ) { // A2 (offm,offn) += (0,j+b). @@ -451,15 +487,6 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, m_part = b; n_part = ij; } - else if ( requested_part == BLIS_SUBPART10B ) - { - // A10B (offm,offn) += (ij,0). - // A10B is b x (ij + b). - offm_inc = ij; - offn_inc = 0; - m_part = b; - n_part = ij + b; - } else if ( requested_part == BLIS_SUBPART20 ) { // A20 (offm,offn) += (ij+b,0). @@ -518,15 +545,6 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, m_part = b; n_part = n - ij - b; } - else if ( requested_part == BLIS_SUBPART12B ) - { - // A12B (offm,offn) += (ij,ij). - // A12B is b x (n-ij). - offm_inc = ij; - offn_inc = ij; - m_part = b; - n_part = n - ij; - } else // if ( requested_part == BLIS_SUBPART22 ) { // A22 (offm,offn) += (ij+b,ij+b). diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 516b1c6d2..1f3defc18 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -469,6 +469,10 @@ typedef enum BLIS_SUBPART0, BLIS_SUBPART1, BLIS_SUBPART2, + BLIS_SUBPART1T, + BLIS_SUBPART1B, + BLIS_SUBPART1L, + BLIS_SUBPART1R, BLIS_SUBPART00, BLIS_SUBPART10, BLIS_SUBPART20, @@ -477,9 +481,7 @@ typedef enum BLIS_SUBPART21, BLIS_SUBPART02, BLIS_SUBPART12, - BLIS_SUBPART22, - BLIS_SUBPART10B, - BLIS_SUBPART12B + BLIS_SUBPART22 } subpart_t;