diff --git a/frame/3/her2k/bli_her2k.h b/frame/3/her2k/bli_her2k.h index b75188593..02afd30d0 100644 --- a/frame/3/her2k/bli_her2k.h +++ b/frame/3/her2k/bli_her2k.h @@ -37,16 +37,11 @@ #include "bli_her2k_int.h" #include "bli_her2k_target.h" -#include "bli_her2k_l_blk_var1.h" -#include "bli_her2k_u_blk_var1.h" +#include "bli_her2k_blk_var1f.h" -#include "bli_her2k_l_blk_var2.h" -#include "bli_her2k_u_blk_var2.h" +#include "bli_her2k_blk_var2f.h" -#include "bli_her2k_blk_var3.h" - -#include "bli_her2k_l_blk_var4.h" -#include "bli_her2k_u_blk_var4.h" +#include "bli_her2k_blk_var3f.h" #include "bli_her2k_l_ker_var2.h" #include "bli_her2k_u_ker_var2.h" diff --git a/frame/3/her2k/bli_her2k_l_blk_var1.c b/frame/3/her2k/bli_her2k_blk_var1f.c similarity index 71% rename from frame/3/her2k/bli_her2k_l_blk_var1.c rename to frame/3/her2k/bli_her2k_blk_var1f.c index a9bf32d2e..6463193c2 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var1.c +++ b/frame/3/her2k/bli_her2k_blk_var1f.c @@ -34,34 +34,32 @@ #include "blis.h" -void bli_her2k_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { obj_t a1, a1_pack; - obj_t bh_pack, bhL_pack; + obj_t bh_pack; obj_t b1, b1_pack; - obj_t ah_pack, ahL_pack; - obj_t c1; - obj_t c1L, c1L_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offL, nL; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &bh_pack ); bli_obj_init_pack( &b1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1L_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -88,30 +86,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, a, + b_alg = bli_determine_blocksize_f( i, m_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, a, &a1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Bh_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &bh_pack, &bhL_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -121,19 +108,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &bhL_pack, + &bh_pack, beta, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -159,30 +146,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, b, + b_alg = bli_determine_blocksize_f( i, m_trans, b, cntl_blocksize( cntl ) ); // Acquire partitions for B1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, b, &b1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &ah_pack, &ahL_pack ); - // Initialize objects for packing B1 and C1. bli_packm_init( &b1, &b1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack B1 and scale by alpha_conj (if instructed). @@ -192,19 +168,19 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha_conj, &b1_pack, - &ahL_pack, + &ah_pack, &BLIS_ONE, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -212,6 +188,6 @@ void bli_her2k_l_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &b1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1L_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/her2k/bli_her2k_l_blk_var2.h b/frame/3/her2k/bli_her2k_blk_var1f.h similarity index 83% rename from frame/3/her2k/bli_her2k_l_blk_var2.h rename to frame/3/her2k/bli_her2k_blk_var1f.h index 161b63a07..b96caf99b 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var2.h +++ b/frame/3/her2k/bli_her2k_blk_var1f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_l_blk_var2.c b/frame/3/her2k/bli_her2k_blk_var2f.c similarity index 78% rename from frame/3/her2k/bli_her2k_l_blk_var2.c rename to frame/3/her2k/bli_her2k_blk_var2f.c index b25069857..2c53aad77 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var2.c +++ b/frame/3/her2k/bli_her2k_blk_var2f.c @@ -34,34 +34,39 @@ #include "blis.h" -void bli_her2k_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { - obj_t a_pack, aB_pack; + obj_t a_pack, aS_pack; obj_t bh1, bh1_pack; - obj_t b_pack, bB_pack; + obj_t b_pack, bS_pack; obj_t ah1, ah1_pack; obj_t c1; - obj_t c1B, c1B_pack; + obj_t c1S, c1S_pack; dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offB, mB; + subpart_t stored_part; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); bli_obj_init_pack( &bh1_pack ); bli_obj_init_pack( &b_pack ); bli_obj_init_pack( &ah1_pack ); - bli_obj_init_pack( &c1B_pack ); + bli_obj_init_pack( &c1S_pack ); + + // The upper and lower variants are identical, except for which + // merged subpartition is acquired in the loop body. + if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; + else stored_part = BLIS_SUBPART1T; // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *c ); @@ -103,23 +108,20 @@ void bli_her2k_l_blk_var2( obj_t* alpha, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding regions - // of A_pack and B_pack. We compute the length of the subpartition - // taking the location of the diagonal into account. - offB = bli_max( 0, -bli_obj_diag_offset_after_trans( c1 ) ); - mB = bli_obj_length_after_trans( c1 ) - offB; - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &c1, &c1B ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &a_pack, &aB_pack ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &b_pack, &bB_pack ); + // of A_pack and B_pack. + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &c1, &c1S ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &a_pack, &aS_pack ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &b_pack, &bS_pack ); // Initialize objects for packing B1', A1', and C1. bli_packm_init( &bh1, &bh1_pack, cntl_sub_packm_b( cntl ) ); bli_packm_init( &ah1, &ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1B, &c1B_pack, + bli_packm_init( &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Pack B1' and scale by alpha (if instructed). @@ -134,22 +136,22 @@ void bli_her2k_l_blk_var2( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1B, &c1B_pack, + &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Perform her2k subproblem. bli_her2k_int( alpha, - &aB_pack, + &aS_pack, &bh1_pack, alpha_conj, - &bB_pack, + &bS_pack, &ah1_pack, beta, - &c1B_pack, + &c1S_pack, cntl_sub_her2k( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1B_pack, &c1B, + bli_unpackm_int( &c1S_pack, &c1S, cntl_sub_unpackm_c( cntl ) ); } @@ -159,6 +161,6 @@ void bli_her2k_l_blk_var2( obj_t* alpha, bli_obj_release_pack( &bh1_pack ); bli_obj_release_pack( &b_pack ); bli_obj_release_pack( &ah1_pack ); - bli_obj_release_pack( &c1B_pack ); + bli_obj_release_pack( &c1S_pack ); } diff --git a/frame/3/her2k/bli_her2k_l_blk_var1.h b/frame/3/her2k/bli_her2k_blk_var2f.h similarity index 83% rename from frame/3/her2k/bli_her2k_l_blk_var1.h rename to frame/3/her2k/bli_her2k_blk_var2f.h index 20e74fb46..e1cbcdb6b 100644 --- a/frame/3/her2k/bli_her2k_l_blk_var1.h +++ b/frame/3/her2k/bli_her2k_blk_var2f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_blk_var3.c b/frame/3/her2k/bli_her2k_blk_var3f.c similarity index 93% rename from frame/3/her2k/bli_her2k_blk_var3.c rename to frame/3/her2k/bli_her2k_blk_var3f.c index 10d2522eb..58a9699c7 100644 --- a/frame/3/her2k/bli_her2k_blk_var3.c +++ b/frame/3/her2k/bli_her2k_blk_var3f.c @@ -34,15 +34,15 @@ #include "blis.h" -void bli_her2k_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ) +void bli_her2k_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ) { obj_t a1, a1_pack; obj_t bh1, bh1_pack; diff --git a/frame/3/her2k/bli_her2k_blk_var3.h b/frame/3/her2k/bli_her2k_blk_var3f.h similarity index 82% rename from frame/3/her2k/bli_her2k_blk_var3.h rename to frame/3/her2k/bli_her2k_blk_var3f.h index 844b435f5..0dbac4fc5 100644 --- a/frame/3/her2k/bli_her2k_blk_var3.h +++ b/frame/3/her2k/bli_her2k_blk_var3f.h @@ -32,13 +32,13 @@ */ -void bli_her2k_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* bh, - obj_t* alpha_conj, - obj_t* b, - obj_t* ah, - obj_t* beta, - obj_t* c, - her2k_t* cntl ); +void bli_her2k_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* bh, + obj_t* alpha_conj, + obj_t* b, + obj_t* ah, + obj_t* beta, + obj_t* c, + her2k_t* cntl ); diff --git a/frame/3/her2k/bli_her2k_int.c b/frame/3/her2k/bli_her2k_int.c index 2362bbfc6..db6dcd9b2 100644 --- a/frame/3/her2k/bli_her2k_int.c +++ b/frame/3/her2k/bli_her2k_int.c @@ -46,24 +46,22 @@ typedef void (*FUNCPTR_T)( obj_t* alpha, obj_t* c, her2k_t* cntl ); -static FUNCPTR_T vars[2][5][3] = +static FUNCPTR_T vars[2][4][3] = { // lower { // unblocked optimized unblocked blocked - { NULL, NULL, bli_her2k_l_blk_var1 }, - { NULL, bli_her2k_l_ker_var2, bli_her2k_l_blk_var2 }, - { NULL, NULL, bli_her2k_blk_var3 }, - { NULL, NULL, bli_her2k_l_blk_var4 }, + { NULL, NULL, bli_her2k_blk_var1f }, + { NULL, bli_her2k_l_ker_var2, bli_her2k_blk_var2f }, + { NULL, NULL, bli_her2k_blk_var3f }, { NULL, NULL, NULL }, }, // upper { // unblocked optimized unblocked blocked - { NULL, NULL, bli_her2k_u_blk_var1 }, - { NULL, bli_her2k_u_ker_var2, bli_her2k_u_blk_var2 }, - { NULL, NULL, bli_her2k_blk_var3 }, - { NULL, NULL, bli_her2k_u_blk_var4 }, + { NULL, NULL, bli_her2k_blk_var1f }, + { NULL, bli_her2k_u_ker_var2, bli_her2k_blk_var2f }, + { NULL, NULL, bli_her2k_blk_var3f }, { NULL, NULL, NULL }, } }; diff --git a/frame/3/her2k/bli_her2k_l_blk_var4.c b/frame/3/her2k/other/bli_her2k_l_blk_var4.c similarity index 100% rename from frame/3/her2k/bli_her2k_l_blk_var4.c rename to frame/3/her2k/other/bli_her2k_l_blk_var4.c diff --git a/frame/3/her2k/bli_her2k_l_blk_var4.h b/frame/3/her2k/other/bli_her2k_l_blk_var4.h similarity index 100% rename from frame/3/her2k/bli_her2k_l_blk_var4.h rename to frame/3/her2k/other/bli_her2k_l_blk_var4.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var1.c b/frame/3/her2k/other/bli_her2k_u_blk_var1.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var1.c rename to frame/3/her2k/other/bli_her2k_u_blk_var1.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var1.h b/frame/3/her2k/other/bli_her2k_u_blk_var1.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var1.h rename to frame/3/her2k/other/bli_her2k_u_blk_var1.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var2.c b/frame/3/her2k/other/bli_her2k_u_blk_var2.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var2.c rename to frame/3/her2k/other/bli_her2k_u_blk_var2.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var2.h b/frame/3/her2k/other/bli_her2k_u_blk_var2.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var2.h rename to frame/3/her2k/other/bli_her2k_u_blk_var2.h diff --git a/frame/3/her2k/bli_her2k_u_blk_var4.c b/frame/3/her2k/other/bli_her2k_u_blk_var4.c similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var4.c rename to frame/3/her2k/other/bli_her2k_u_blk_var4.c diff --git a/frame/3/her2k/bli_her2k_u_blk_var4.h b/frame/3/her2k/other/bli_her2k_u_blk_var4.h similarity index 100% rename from frame/3/her2k/bli_her2k_u_blk_var4.h rename to frame/3/her2k/other/bli_her2k_u_blk_var4.h diff --git a/frame/3/herk/bli_herk.h b/frame/3/herk/bli_herk.h index 1174bbe9c..82ef49688 100644 --- a/frame/3/herk/bli_herk.h +++ b/frame/3/herk/bli_herk.h @@ -37,16 +37,11 @@ #include "bli_herk_int.h" #include "bli_herk_target.h" -#include "bli_herk_l_blk_var1.h" -#include "bli_herk_u_blk_var1.h" +#include "bli_herk_blk_var1f.h" -#include "bli_herk_l_blk_var2.h" -#include "bli_herk_u_blk_var2.h" +#include "bli_herk_blk_var2f.h" -#include "bli_herk_blk_var3.h" - -#include "bli_herk_l_blk_var4.h" -#include "bli_herk_u_blk_var4.h" +#include "bli_herk_blk_var3f.h" #include "bli_herk_l_ker_var2.h" #include "bli_herk_u_ker_var2.h" diff --git a/frame/3/herk/bli_herk_l_blk_var1.c b/frame/3/herk/bli_herk_blk_var1f.c similarity index 75% rename from frame/3/herk/bli_herk_l_blk_var1.c rename to frame/3/herk/bli_herk_blk_var1f.c index b76334042..dc137e1ce 100644 --- a/frame/3/herk/bli_herk_l_blk_var1.c +++ b/frame/3/herk/bli_herk_blk_var1f.c @@ -34,27 +34,25 @@ #include "blis.h" -void bli_herk_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { obj_t a1, a1_pack; - obj_t ah_pack, ahL_pack; - obj_t c1; - obj_t c1L, c1L_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offL, nL; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1L_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -77,30 +75,19 @@ void bli_herk_l_blk_var1( obj_t* alpha, for ( i = 0; i < m_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, m_trans, a, + b_alg = bli_determine_blocksize_f( i, m_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1 and C1. - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, a, &a1 ); - bli_acquire_mpart_b2t( BLIS_SUBPART1, + bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offL = 0; - nL = bli_min( bli_obj_width_after_trans( c1 ), - bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &c1, &c1L ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offL, nL, &ah_pack, &ahL_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1L, &c1L_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -110,19 +97,19 @@ void bli_herk_l_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1L, &c1L_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &ahL_pack, + &ah_pack, beta, - &c1L_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1L_pack, &c1L, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -130,6 +117,6 @@ void bli_herk_l_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1L_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/herk/bli_herk_blk_var3.h b/frame/3/herk/bli_herk_blk_var1f.h similarity index 87% rename from frame/3/herk/bli_herk_blk_var3.h rename to frame/3/herk/bli_herk_blk_var1f.h index 812c5c23e..94900b772 100644 --- a/frame/3/herk/bli_herk_blk_var3.h +++ b/frame/3/herk/bli_herk_blk_var1f.h @@ -32,10 +32,10 @@ */ -void bli_herk_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var1f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_l_blk_var2.c b/frame/3/herk/bli_herk_blk_var2f.c similarity index 79% rename from frame/3/herk/bli_herk_l_blk_var2.c rename to frame/3/herk/bli_herk_blk_var2f.c index 42ad4d2f4..df39ce55d 100644 --- a/frame/3/herk/bli_herk_l_blk_var2.c +++ b/frame/3/herk/bli_herk_blk_var2f.c @@ -34,27 +34,32 @@ #include "blis.h" -void bli_herk_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { - obj_t a_pack, aB_pack; + obj_t a_pack, aS_pack; obj_t ah1, ah1_pack; obj_t c1; - obj_t c1B, c1B_pack; + obj_t c1S, c1S_pack; dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offB, mB; + subpart_t stored_part; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); bli_obj_init_pack( &ah1_pack ); - bli_obj_init_pack( &c1B_pack ); + bli_obj_init_pack( &c1S_pack ); + + // The upper and lower variants are identical, except for which + // merged subpartition is acquired in the loop body. + if ( bli_obj_is_lower( *c ) ) stored_part = BLIS_SUBPART1B; + else stored_part = BLIS_SUBPART1T; // Query dimension in partitioning direction. n_trans = bli_obj_width_after_trans( *c ); @@ -87,19 +92,16 @@ void bli_herk_l_blk_var2( obj_t* alpha, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding region - // of A_pack. We compute the length of the subpartition taking the - // location of the diagonal into account. - offB = bli_max( 0, -bli_obj_diag_offset_after_trans( c1 ) ); - mB = bli_obj_length_after_trans( c1 ) - offB; - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &c1, &c1B ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offB, mB, &a_pack, &aB_pack ); + // of A_pack. + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &c1, &c1S ); + bli_acquire_mpart_t2b( stored_part, + i, b_alg, &a_pack, &aS_pack ); // Initialize objects for packing A1' and C1. bli_packm_init( &ah1, &ah1_pack, cntl_sub_packm_b( cntl ) ); - bli_packm_init( &c1B, &c1B_pack, + bli_packm_init( &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Pack A1' and scale by alpha (if instructed). @@ -109,19 +111,19 @@ void bli_herk_l_blk_var2( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1B, &c1B_pack, + &c1S, &c1S_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, - &aB_pack, + &aS_pack, &ah1_pack, beta, - &c1B_pack, + &c1S_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1B_pack, &c1B, + bli_unpackm_int( &c1S_pack, &c1S, cntl_sub_unpackm_c( cntl ) ); } @@ -129,6 +131,6 @@ void bli_herk_l_blk_var2( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a_pack ); bli_obj_release_pack( &ah1_pack ); - bli_obj_release_pack( &c1B_pack ); + bli_obj_release_pack( &c1S_pack ); } diff --git a/frame/3/herk/bli_herk_l_blk_var1.h b/frame/3/herk/bli_herk_blk_var2f.h similarity index 87% rename from frame/3/herk/bli_herk_l_blk_var1.h rename to frame/3/herk/bli_herk_blk_var2f.h index 6cfdd1a4e..1a5034242 100644 --- a/frame/3/herk/bli_herk_l_blk_var1.h +++ b/frame/3/herk/bli_herk_blk_var2f.h @@ -32,10 +32,10 @@ */ -void bli_herk_l_blk_var1( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var2f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_blk_var3.c b/frame/3/herk/bli_herk_blk_var3f.c similarity index 94% rename from frame/3/herk/bli_herk_blk_var3.c rename to frame/3/herk/bli_herk_blk_var3f.c index ddf9d6536..a54e341b9 100644 --- a/frame/3/herk/bli_herk_blk_var3.c +++ b/frame/3/herk/bli_herk_blk_var3f.c @@ -34,12 +34,12 @@ #include "blis.h" -void bli_herk_blk_var3( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ) +void bli_herk_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ) { obj_t a1, a1_pack; obj_t ah1, ah1_pack; diff --git a/frame/3/herk/bli_herk_l_blk_var2.h b/frame/3/herk/bli_herk_blk_var3f.h similarity index 87% rename from frame/3/herk/bli_herk_l_blk_var2.h rename to frame/3/herk/bli_herk_blk_var3f.h index 8205750f7..5e96900f7 100644 --- a/frame/3/herk/bli_herk_l_blk_var2.h +++ b/frame/3/herk/bli_herk_blk_var3f.h @@ -32,10 +32,10 @@ */ -void bli_herk_l_blk_var2( obj_t* alpha, - obj_t* a, - obj_t* ah, - obj_t* beta, - obj_t* c, - herk_t* cntl ); +void bli_herk_blk_var3f( obj_t* alpha, + obj_t* a, + obj_t* ah, + obj_t* beta, + obj_t* c, + herk_t* cntl ); diff --git a/frame/3/herk/bli_herk_int.c b/frame/3/herk/bli_herk_int.c index 85a09ce34..afeef8d32 100644 --- a/frame/3/herk/bli_herk_int.c +++ b/frame/3/herk/bli_herk_int.c @@ -43,24 +43,22 @@ typedef void (*FUNCPTR_T)( obj_t* alpha, obj_t* c, herk_t* cntl ); -static FUNCPTR_T vars[2][5][3] = +static FUNCPTR_T vars[2][4][3] = { // lower { // unblocked optimized unblocked blocked - { NULL, NULL, bli_herk_l_blk_var1 }, - { NULL, bli_herk_l_ker_var2, bli_herk_l_blk_var2 }, - { NULL, NULL, bli_herk_blk_var3 }, - { NULL, NULL, bli_herk_l_blk_var4 }, + { NULL, NULL, bli_herk_blk_var1f }, + { NULL, bli_herk_l_ker_var2, bli_herk_blk_var2f }, + { NULL, NULL, bli_herk_blk_var3f }, { NULL, NULL, NULL }, }, // upper { // unblocked optimized unblocked blocked - { NULL, NULL, bli_herk_u_blk_var1 }, - { NULL, bli_herk_u_ker_var2, bli_herk_u_blk_var2 }, - { NULL, NULL, bli_herk_blk_var3 }, - { NULL, NULL, bli_herk_u_blk_var4 }, + { NULL, NULL, bli_herk_blk_var1f }, + { NULL, bli_herk_u_ker_var2, bli_herk_blk_var2f }, + { NULL, NULL, bli_herk_blk_var3f }, { NULL, NULL, NULL }, } }; diff --git a/frame/3/herk/bli_herk_l_ker_var2.c b/frame/3/herk/bli_herk_l_ker_var2.c index 8f326e52b..0e657032d 100644 --- a/frame/3/herk/bli_herk_l_ker_var2.c +++ b/frame/3/herk/bli_herk_l_ker_var2.c @@ -186,7 +186,7 @@ void PASTEMAC(ch,varname)( \ dim_t n_iter, n_left; \ dim_t m_cur; \ dim_t n_cur; \ - dim_t i, j; \ + dim_t i, j, ip; \ inc_t rstep_a; \ inc_t cstep_b; \ inc_t rstep_c, cstep_c; \ @@ -205,6 +205,31 @@ void PASTEMAC(ch,varname)( \ \ /* If any dimension is zero, return immediately. */ \ if ( bli_zero_dim3( m, n, k ) ) return; \ +\ + /* Safeguard: If the current panel of C is entirely above the diagonal, + it is not stored. So we do nothing. */ \ + if ( bli_is_strictly_above_diag_n( diagoffc, m, n ) ) return; \ +\ + /* If there is a zero region above where the diagonal of C intersects + the left edge of the panel, adjust the pointer to C and A and treat + this case as if the diagonal offset were zero. */ \ + if ( diagoffc < 0 ) \ + { \ + ip = -diagoffc / MR; \ + i = ip * MR; \ + m = m - i; \ + diagoffc = -diagoffc % MR; \ + c_cast = c_cast + (i )*rs_c; \ + a_cast = a_cast + (ip )*ps_a; \ + } \ +\ + /* If there is a zero region to the right of where the diagonal + of C intersects the bottom of the panel, shrink it to prevent + "no-op" iterations from executing. */ \ + if ( diagoffc + m < n ) \ + { \ + n = diagoffc + m; \ + } \ \ /* Clear the temporary C buffer in case it has any infs or NaNs. */ \ PASTEMAC(ch,set0s_mxn)( MR, NR, \ diff --git a/frame/3/herk/bli_herk_u_ker_var2.c b/frame/3/herk/bli_herk_u_ker_var2.c index bc40d0196..98c840941 100644 --- a/frame/3/herk/bli_herk_u_ker_var2.c +++ b/frame/3/herk/bli_herk_u_ker_var2.c @@ -186,7 +186,7 @@ void PASTEMAC(ch,varname)( \ dim_t n_iter, n_left; \ dim_t m_cur; \ dim_t n_cur; \ - dim_t i, j; \ + dim_t i, j, jp; \ inc_t rstep_a; \ inc_t cstep_b; \ inc_t rstep_c, cstep_c; \ @@ -205,6 +205,31 @@ void PASTEMAC(ch,varname)( \ \ /* If any dimension is zero, return immediately. */ \ if ( bli_zero_dim3( m, n, k ) ) return; \ +\ + /* Safeguard: If the current panel of C is entirely below the diagonal, + it is not stored. So we do nothing. */ \ + if ( bli_is_strictly_below_diag_n( diagoffc, m, n ) ) return; \ +\ + /* If there is a zero region to the left of where the diagonal of C + intersects the top edge of the panel, adjust the pointer to C and B + and treat this case as if the diagonal offset were zero. */ \ + if ( diagoffc > 0 ) \ + { \ + jp = diagoffc / NR; \ + j = jp * NR; \ + n = n - j; \ + diagoffc = diagoffc % NR; \ + c_cast = c_cast + (j )*cs_c; \ + b_cast = b_cast + (jp )*ps_b; \ + } \ +\ + /* If there is a zero region below where the diagonal of C intersects + the right edge of the panel, shrink it to prevent "no-op" iterations + from executing. */ \ + if ( -diagoffc + n < m ) \ + { \ + m = -diagoffc + n; \ + } \ \ /* Clear the temporary C buffer in case it has any infs or NaNs. */ \ PASTEMAC(ch,set0s_mxn)( MR, NR, \ diff --git a/frame/3/herk/bli_herk_l_blk_var4.c b/frame/3/herk/other/bli_herk_l_blk_var4.c similarity index 100% rename from frame/3/herk/bli_herk_l_blk_var4.c rename to frame/3/herk/other/bli_herk_l_blk_var4.c diff --git a/frame/3/herk/bli_herk_l_blk_var4.h b/frame/3/herk/other/bli_herk_l_blk_var4.h similarity index 100% rename from frame/3/herk/bli_herk_l_blk_var4.h rename to frame/3/herk/other/bli_herk_l_blk_var4.h diff --git a/frame/3/herk/bli_herk_u_blk_var1.c b/frame/3/herk/other/bli_herk_u_blk_var1.c similarity index 83% rename from frame/3/herk/bli_herk_u_blk_var1.c rename to frame/3/herk/other/bli_herk_u_blk_var1.c index 20b722bc8..e352a1b00 100644 --- a/frame/3/herk/bli_herk_u_blk_var1.c +++ b/frame/3/herk/other/bli_herk_u_blk_var1.c @@ -42,19 +42,17 @@ void bli_herk_u_blk_var1( obj_t* alpha, herk_t* cntl ) { obj_t a1, a1_pack; - obj_t ah_pack, ahR_pack; - obj_t c1; - obj_t c1R, c1R_pack; + obj_t ah_pack; + obj_t c1, c1_pack; dim_t i; dim_t b_alg; dim_t m_trans; - dim_t offR, nR; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a1_pack ); bli_obj_init_pack( &ah_pack ); - bli_obj_init_pack( &c1R_pack ); + bli_obj_init_pack( &c1_pack ); // Query dimension in partitioning direction. m_trans = bli_obj_length_after_trans( *c ); @@ -89,20 +87,10 @@ void bli_herk_u_blk_var1( obj_t* alpha, bli_acquire_mpart_t2b( BLIS_SUBPART1, i, b_alg, c, &c1 ); - // Partition off the stored region of C1 and the corresponding region - // of Ah_pack. We compute the width of the subpartition taking the - // location of the diagonal into account. - offR = bli_max( 0, bli_obj_diag_offset_after_trans( c1 ) ); - nR = bli_obj_width_after_trans( c1 ) - offR; - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offR, nR, &c1, &c1R ); - bli_acquire_mpart_l2r( BLIS_SUBPART1, - offR, nR, &ah_pack, &ahR_pack ); - // Initialize objects for packing A1 and C1. bli_packm_init( &a1, &a1_pack, cntl_sub_packm_a( cntl ) ); - bli_packm_init( &c1R, &c1R_pack, + bli_packm_init( &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Pack A1 and scale by alpha (if instructed). @@ -112,19 +100,19 @@ void bli_herk_u_blk_var1( obj_t* alpha, // Pack C1 and scale by beta (if instructed). bli_packm_int( beta, - &c1R, &c1R_pack, + &c1, &c1_pack, cntl_sub_packm_c( cntl ) ); // Perform herk subproblem. bli_herk_int( alpha, &a1_pack, - &ahR_pack, + &ah_pack, beta, - &c1R_pack, + &c1_pack, cntl_sub_herk( cntl ) ); // Unpack C1 (if C1 was packed). - bli_unpackm_int( &c1R_pack, &c1R, + bli_unpackm_int( &c1_pack, &c1, cntl_sub_unpackm_c( cntl ) ); } @@ -132,6 +120,6 @@ void bli_herk_u_blk_var1( obj_t* alpha, // to the memory manager. bli_obj_release_pack( &a1_pack ); bli_obj_release_pack( &ah_pack ); - bli_obj_release_pack( &c1R_pack ); + bli_obj_release_pack( &c1_pack ); } diff --git a/frame/3/herk/bli_herk_u_blk_var1.h b/frame/3/herk/other/bli_herk_u_blk_var1.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var1.h rename to frame/3/herk/other/bli_herk_u_blk_var1.h diff --git a/frame/3/herk/bli_herk_u_blk_var2.c b/frame/3/herk/other/bli_herk_u_blk_var2.c similarity index 87% rename from frame/3/herk/bli_herk_u_blk_var2.c rename to frame/3/herk/other/bli_herk_u_blk_var2.c index e240f50b2..ff038b37c 100644 --- a/frame/3/herk/bli_herk_u_blk_var2.c +++ b/frame/3/herk/other/bli_herk_u_blk_var2.c @@ -49,7 +49,6 @@ void bli_herk_u_blk_var2( obj_t* alpha, dim_t i; dim_t b_alg; dim_t n_trans; - dim_t offT, mT; // Initialize all pack objects that are passed into packm_init(). bli_obj_init_pack( &a_pack ); @@ -77,25 +76,21 @@ void bli_herk_u_blk_var2( obj_t* alpha, for ( i = 0; i < n_trans; i += b_alg ) { // Determine the current algorithmic blocksize. - b_alg = bli_determine_blocksize_b( i, n_trans, a, + b_alg = bli_determine_blocksize_f( i, n_trans, a, cntl_blocksize( cntl ) ); // Acquire partitions for A1' and C1. - bli_acquire_mpart_r2l( BLIS_SUBPART1, + bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, ah, &ah1 ); - bli_acquire_mpart_r2l( BLIS_SUBPART1, + bli_acquire_mpart_l2r( BLIS_SUBPART1, i, b_alg, c, &c1 ); // Partition off the stored region of C1 and the corresponding region - // of A_pack. We compute the length of the subpartition taking the - // location of the diagonal into account. - offT = 0; - mT = bli_min( bli_obj_length_after_trans( c1 ), - -bli_obj_diag_offset_after_trans( c1 ) + b_alg ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offT, mT, &c1, &c1T ); - bli_acquire_mpart_t2b( BLIS_SUBPART1, - offT, mT, &a_pack, &aT_pack ); + // of A_pack. + bli_acquire_mpart_t2b( BLIS_SUBPART1T, + i, b_alg, &c1, &c1T ); + bli_acquire_mpart_t2b( BLIS_SUBPART1T, + i, b_alg, &a_pack, &aT_pack ); // Initialize objects for packing A1' and C1. bli_packm_init( &ah1, &ah1_pack, diff --git a/frame/3/herk/bli_herk_u_blk_var2.h b/frame/3/herk/other/bli_herk_u_blk_var2.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var2.h rename to frame/3/herk/other/bli_herk_u_blk_var2.h diff --git a/frame/3/herk/bli_herk_u_blk_var4.c b/frame/3/herk/other/bli_herk_u_blk_var4.c similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var4.c rename to frame/3/herk/other/bli_herk_u_blk_var4.c diff --git a/frame/3/herk/bli_herk_u_blk_var4.h b/frame/3/herk/other/bli_herk_u_blk_var4.h similarity index 100% rename from frame/3/herk/bli_herk_u_blk_var4.h rename to frame/3/herk/other/bli_herk_u_blk_var4.h diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index 43701f54a..d07ed5989 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -205,11 +205,11 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is negative, adjust the pointer to C and - the effective diagonal offset of A accordingly to skip over the zero - region that was not packed. (Note we only skip over the region in - increments of PACKMR since that is how the region would have been - skipped by packm.) */ \ + /* If there is a zero region above where the diagonal of A intersects the + left edge of the block, adjust the pointer to C and treat this case as + if the diagonal offset were zero. This skips over the region (in + increments of MR) that was not packed. (Note we skip in increments of + MR since that is how the region would have been skipped by packm.) */ \ if ( diagoffa < 0 ) \ { \ i = ( -diagoffa / MR ) * MR; \ diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 78fae73f9..2619ffbd1 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -205,16 +205,17 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is positive, adjust the pointer to B and + /* If there is a zero region to the left of where the diagonal of A + intersects the top edge of the block, adjust the pointer to B and treat this case as if the diagonal offset were zero. Note that we don't need to adjust the pointer to A since packm would have simply skipped over the region that was not stored. */ \ if ( diagoffa > 0 ) \ { \ - j = diagoffa; \ - k = k - j; \ + i = diagoffa; \ + k = k - i; \ diagoffa = 0; \ - b_cast = b_cast + (j )*rs_b; \ + b_cast = b_cast + (i )*rs_b; \ } \ \ /* If there is a zero region below where the diagonal of A intersects the diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 5d60dadbc..cfcab09ed 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -206,17 +206,18 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % NR != 0 ) k += NR - ( k % NR ); \ \ - /* If the diagonal offset of B is negative, adjust the pointer to A and - treat this case as if the diagonal offset were zero. Note that we - don't need to adjust the pointer to B since packm would have simply - skipped over the region that was not stored. */ \ - if ( diagoffb < 0 ) \ - { \ - j = -diagoffb; \ - k = k - j; \ - diagoffb = 0; \ - a_cast = a_cast + (j )*cs_a; \ - } \ + /* If there is a zero region above where the diagonal of B intersects + the left edge of the panel, adjust the pointer to A and treat this + case as if the diagonal offset were zero. Note that we don't need to + adjust the pointer to B since packm would have simply skipped over + the region that was not stored. */ \ + if ( diagoffb < 0 ) \ + { \ + j = -diagoffb; \ + k = k - j; \ + diagoffb = 0; \ + a_cast = a_cast + (j )*cs_a; \ + } \ \ /* If there is a zero region to the right of where the diagonal of B intersects the bottom of the panel, shrink it to prevent diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index 50bfd8f43..9889ac008 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -206,9 +206,12 @@ void PASTEMAC(ch,varname)( \ handle bottom-right corner edges of the triangle. */ \ if ( k % NR != 0 ) k += NR - ( k % NR ); \ \ - /* If the diagonal offset of B is positive, adjust the pointer to C and - the effective diagonal offset of B accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region to the left of where the diagonal of B + intersects the top edge of the panel, adjust the pointer to C and + treat this case as if the diagonal offset were zero. This skips over + the region (in increments of NR) that was not packed. (Note we skip + in increments of NR since that is how the region would have been + skipped by packm.) */ \ if ( diagoffb > 0 ) \ { \ j = ( diagoffb / NR ) * NR; \ diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 79bab87f7..37f8583d3 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -206,9 +206,11 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is negative, adjust the pointer to C and - the effective diagonal offset of A accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region above where the diagonal of A intersects the + left edge of the block, adjust the pointer to C and treat this case as + if the diagonal offset were zero. This skips over the region (in + increments of MR) that was not packed. (Note we skip in increments of + MR since that is how the region would have been skipped by packm.) */ \ if ( diagoffa < 0 ) \ { \ i = ( -diagoffa / MR ) * MR; \ diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index f9bfb0013..bb2972102 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -207,10 +207,11 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of A is positive, adjust the pointer to B and + /* If there is a zero region to the left of where the diagonal of A + intersects the top edge of the block, adjust the pointer to B and treat this case as if the diagonal offset were zero. Note that we don't need to adjust the pointer to A since packm would have simply - skipped over the panels that were not stored. */ \ + skipped over the region that was not stored. */ \ if ( diagoffa > 0 ) \ { \ i = diagoffa; \ diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index ed499876a..591cb27bd 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -196,16 +196,17 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of B is negative, adjust the pointer to C and - treat this case as if the diagonal offset were zero. Note that we - don't need to adjust the pointer to B since packm would have simply - skipped over the region that was not stored. */ \ + /* If there is a zero region above where the diagonal of B intersects + the left edge of the panel, adjust the pointer to A and treat this + case as if the diagonal offset were zero. Note that we don't need to + adjust the pointer to B since packm would have simply skipped over + the region that was not stored. */ \ if ( diagoffb < 0 ) \ { \ j = -diagoffb; \ k = k - j; \ diagoffb = 0; \ - a_cast = a_cast + (j )*cs_a; \ + a_cast = a_cast + (j )*cs_a; \ } \ \ /* If there is a zero region to the right of where the diagonal diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 9c2d68014..2ee513a21 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -195,9 +195,12 @@ void PASTEMAC(ch,varname)( \ values of the corresponding block of B. */ \ if ( k % MR != 0 ) k += MR - ( k % MR ); \ \ - /* If the diagonal offset of B is positive, adjust the pointer to C and - the effective diagonal offset of B accordingly to skip over the zero - region that was not packed. */ \ + /* If there is a zero region to the left of where the diagonal of B + intersects the top edge of the panel, adjust the pointer to C and + treat this case as if the diagonal offset were zero. This skips over + the region (in increments of NR) that was not packed. (Note we skip + in increments of NR since that is how the region would have been + skipped by packm.) */ \ if ( diagoffb > 0 ) \ { \ j = ( diagoffb / NR ) * NR; \ diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index d5241e0d9..4cc1abb28 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -606,8 +606,10 @@ err_t bli_check_valid_3x1_subpart( subpart_t part ) { err_t e_val = BLIS_SUCCESS; - if ( part != BLIS_SUBPART0 && - part != BLIS_SUBPART1 && + if ( part != BLIS_SUBPART0 && + part != BLIS_SUBPART1T && + part != BLIS_SUBPART1 && + part != BLIS_SUBPART1B && part != BLIS_SUBPART2 ) e_val = BLIS_INVALID_3x1_SUBPART; @@ -618,8 +620,10 @@ err_t bli_check_valid_1x3_subpart( subpart_t part ) { err_t e_val = BLIS_SUCCESS; - if ( part != BLIS_SUBPART0 && - part != BLIS_SUBPART1 && + if ( part != BLIS_SUBPART0 && + part != BLIS_SUBPART1L && + part != BLIS_SUBPART1 && + part != BLIS_SUBPART1R && part != BLIS_SUBPART2 ) e_val = BLIS_INVALID_1x3_SUBPART; diff --git a/frame/base/bli_part.c b/frame/base/bli_part.c index 482b0724b..ac1e4bf13 100644 --- a/frame/base/bli_part.c +++ b/frame/base/bli_part.c @@ -99,6 +99,15 @@ void bli_acquire_mpart_t2b( subpart_t requested_part, m_part = i; n_part = n; } + if ( requested_part == BLIS_SUBPART1T ) + { + // A1T (offm,offn) unchanged. + // A1T is (i+b) x n. + offm_inc = 0; + offn_inc = 0; + m_part = i + b; + n_part = n; + } else if ( requested_part == BLIS_SUBPART1 ) { // A1 (offm,offn) += (i,0). @@ -108,6 +117,15 @@ void bli_acquire_mpart_t2b( subpart_t requested_part, m_part = b; n_part = n; } + else if ( requested_part == BLIS_SUBPART1B ) + { + // A1B (offm,offn) += (i,0). + // A1B is (m-i) x n. + offm_inc = i; + offn_inc = 0; + m_part = m - i; + n_part = n; + } else // if ( requested_part == BLIS_SUBPART2 ) { // A2 (offm,offn) += (i+b,0). @@ -269,6 +287,15 @@ void bli_acquire_mpart_l2r( subpart_t requested_part, m_part = m; n_part = j; } + if ( requested_part == BLIS_SUBPART1L ) + { + // A1L (offm,offn) unchanged. + // A1L is m x (j+b). + offm_inc = 0; + offn_inc = 0; + m_part = m; + n_part = j + b; + } else if ( requested_part == BLIS_SUBPART1 ) { // A1 (offm,offn) += (0,j). @@ -278,6 +305,15 @@ void bli_acquire_mpart_l2r( subpart_t requested_part, m_part = m; n_part = b; } + else if ( requested_part == BLIS_SUBPART1R ) + { + // A1R (offm,offn) += (0,j). + // A1R is m x (n-j). + offm_inc = 0; + offn_inc = j; + m_part = m; + n_part = n - j; + } else // if ( requested_part == BLIS_SUBPART2 ) { // A2 (offm,offn) += (0,j+b). @@ -451,15 +487,6 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, m_part = b; n_part = ij; } - else if ( requested_part == BLIS_SUBPART10B ) - { - // A10B (offm,offn) += (ij,0). - // A10B is b x (ij + b). - offm_inc = ij; - offn_inc = 0; - m_part = b; - n_part = ij + b; - } else if ( requested_part == BLIS_SUBPART20 ) { // A20 (offm,offn) += (ij+b,0). @@ -518,15 +545,6 @@ void bli_acquire_mpart_tl2br( subpart_t requested_part, m_part = b; n_part = n - ij - b; } - else if ( requested_part == BLIS_SUBPART12B ) - { - // A12B (offm,offn) += (ij,ij). - // A12B is b x (n-ij). - offm_inc = ij; - offn_inc = ij; - m_part = b; - n_part = n - ij; - } else // if ( requested_part == BLIS_SUBPART22 ) { // A22 (offm,offn) += (ij+b,ij+b). diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 516b1c6d2..1f3defc18 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -469,6 +469,10 @@ typedef enum BLIS_SUBPART0, BLIS_SUBPART1, BLIS_SUBPART2, + BLIS_SUBPART1T, + BLIS_SUBPART1B, + BLIS_SUBPART1L, + BLIS_SUBPART1R, BLIS_SUBPART00, BLIS_SUBPART10, BLIS_SUBPART20, @@ -477,9 +481,7 @@ typedef enum BLIS_SUBPART21, BLIS_SUBPART02, BLIS_SUBPART12, - BLIS_SUBPART22, - BLIS_SUBPART10B, - BLIS_SUBPART12B + BLIS_SUBPART22 } subpart_t;