diff --git a/frame/1m/packm/bli_packm_blk_var1.c b/frame/1m/packm/bli_packm_blk_var1.c index 394613c2c..5f0ed38ae 100644 --- a/frame/1m/packm/bli_packm_blk_var1.c +++ b/frame/1m/packm/bli_packm_blk_var1.c @@ -358,6 +358,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ @@ -389,6 +390,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ @@ -418,6 +420,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ BLIS_DENSE, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ diff --git a/frame/1m/packm/bli_packm_blk_var2.c b/frame/1m/packm/bli_packm_blk_var2.c index a002ed95e..2c5a75e22 100644 --- a/frame/1m/packm/bli_packm_blk_var2.c +++ b/frame/1m/packm/bli_packm_blk_var2.c @@ -413,6 +413,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ @@ -444,6 +445,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ @@ -473,6 +475,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ BLIS_DENSE, \ conjc, \ + schema, \ invdiag, \ *m_panel_use, \ *n_panel_use, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk.c b/frame/1m/packm/bli_packm_struc_cxk.c index c482206a8..fdce961f5 100644 --- a/frame/1m/packm/bli_packm_struc_cxk.c +++ b/frame/1m/packm/bli_packm_struc_cxk.c @@ -43,6 +43,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -59,10 +60,9 @@ void PASTEMAC(ch,varname)( \ inc_t ldp; \ \ \ - /* If the strides of p indicate row storage, then we are packing to - column panels; otherwise, if the strides indicate column storage, - we are packing to row panels. */ \ - if ( bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ) ) \ + /* Determine the dimensions and relative strides of the micro-panel + based on its pack schema. */ \ + if ( bli_is_col_packed( schema ) ) \ { \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ @@ -71,7 +71,7 @@ void PASTEMAC(ch,varname)( \ ldc = rs_c; \ ldp = rs_p; \ } \ - else /* if ( bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ) ) */ \ + else /* if ( bli_is_row_packed( schema ) ) */ \ { \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ @@ -103,6 +103,7 @@ void PASTEMAC(ch,varname)( \ diagoffc, \ uploc, \ conjc, \ + schema, \ m_panel, \ n_panel, \ m_panel_max, \ @@ -124,6 +125,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ m_panel, \ n_panel, \ @@ -223,6 +225,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -242,12 +245,12 @@ void PASTEMAC(ch,varname)( \ bool_t col_stored; \ \ \ - /* Create flags to incidate row or column storage. Since we don't - have the schema in scope, we must use the dimensions and strides - of the micro-panel to determine whether it is row- or column- - stored. */ \ - row_stored = bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ); \ - col_stored = bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ); \ + /* Create flags to incidate row or column storage. Note that the + schema bit that encodes row or column is describing the form of + micro-panel, not the storage in the micro-panel. Hence the + mismatch in "row" and "column" semantics. */ \ + row_stored = bli_is_col_packed( schema ); \ + col_stored = bli_is_row_packed( schema ); \ \ /* Handle the case where the micro-panel does NOT intersect the diagonal separately from the case where it does intersect. */ \ @@ -437,6 +440,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk.h b/frame/1m/packm/bli_packm_struc_cxk.h index f1e669c77..4b300839f 100644 --- a/frame/1m/packm/bli_packm_struc_cxk.h +++ b/frame/1m/packm/bli_packm_struc_cxk.h @@ -41,6 +41,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -63,6 +64,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -89,6 +91,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_3m.c b/frame/1m/packm/bli_packm_struc_cxk_3m.c index cb90c1058..a09637f41 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_3m.c +++ b/frame/1m/packm/bli_packm_struc_cxk_3m.c @@ -43,6 +43,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -60,10 +61,9 @@ void PASTEMAC(ch,varname)( \ inc_t is_p, ldp; \ \ \ - /* If the strides of p indicate row storage, then we are packing to - column panels; otherwise, if the strides indicate column storage, - we are packing to row panels. */ \ - if ( bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ) ) \ + /* Determine the dimensions and relative strides of the micro-panel + based on its pack schema. */ \ + if ( bli_is_col_packed( schema ) ) \ { \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ @@ -73,7 +73,7 @@ void PASTEMAC(ch,varname)( \ ldc = rs_c; \ ldp = rs_p; \ } \ - else /* if ( bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ) ) */ \ + else /* if ( bli_is_row_packed( schema ) ) */ \ { \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ @@ -110,6 +110,7 @@ void PASTEMAC(ch,varname)( \ diagoffc, \ uploc, \ conjc, \ + schema, \ m_panel, \ n_panel, \ m_panel_max, \ @@ -131,6 +132,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ m_panel, \ n_panel, \ @@ -269,6 +271,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -288,12 +291,12 @@ void PASTEMAC(ch,varname)( \ bool_t col_stored; \ \ \ - /* Create flags to incidate row or column storage. Since we don't - have the schema in scope, we must use the dimensions and strides - of the micro-panel to determine whether it is row- or column- - stored. */ \ - row_stored = bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ); \ - col_stored = bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ); \ + /* Create flags to incidate row or column storage. Note that the + schema bit that encodes row or column is describing the form of + micro-panel, not the storage in the micro-panel. Hence the + mismatch in "row" and "column" semantics. */ \ + row_stored = bli_is_col_packed( schema ); \ + col_stored = bli_is_row_packed( schema ); \ \ \ /* Handle the case where the micro-panel does NOT intersect the @@ -545,6 +548,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -563,12 +567,12 @@ void PASTEMAC(ch,varname)( \ bool_t col_stored; \ \ \ - /* Create flags to incidate row or column storage. Since we don't - have the schema in scope, we must use the dimensions and strides - of the micro-panel to determine whether it is row- or column- - stored. */ \ - row_stored = bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ); \ - col_stored = bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ); \ + /* Create flags to incidate row or column storage. Note that the + schema bit that encodes row or column is describing the form of + micro-panel, not the storage in the micro-panel. Hence the + mismatch in "row" and "column" semantics. */ \ + row_stored = bli_is_col_packed( schema ); \ + col_stored = bli_is_row_packed( schema ); \ \ \ /* Pack the panel. */ \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_3m.h b/frame/1m/packm/bli_packm_struc_cxk_3m.h index e24a290d5..c5168864c 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_3m.h +++ b/frame/1m/packm/bli_packm_struc_cxk_3m.h @@ -41,6 +41,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -63,6 +64,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -89,6 +91,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_4m.c b/frame/1m/packm/bli_packm_struc_cxk_4m.c index a11ff2e10..40a92ea10 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_4m.c +++ b/frame/1m/packm/bli_packm_struc_cxk_4m.c @@ -43,6 +43,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -60,10 +61,9 @@ void PASTEMAC(ch,varname)( \ inc_t is_p, ldp; \ \ \ - /* If the strides of p indicate row storage, then we are packing to - column panels; otherwise, if the strides indicate column storage, - we are packing to row panels. */ \ - if ( bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ) ) \ + /* Determine the dimensions and relative strides of the micro-panel + based on its pack schema. */ \ + if ( bli_is_col_packed( schema ) ) \ { \ /* Prepare to pack to row-stored column panel. */ \ panel_dim = n_panel; \ @@ -73,7 +73,7 @@ void PASTEMAC(ch,varname)( \ ldc = rs_c; \ ldp = rs_p; \ } \ - else /* if ( bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ) ) */ \ + else /* if ( bli_is_row_packed( schema ) ) */ \ { \ /* Prepare to pack to column-stored row panel. */ \ panel_dim = m_panel; \ @@ -110,6 +110,7 @@ void PASTEMAC(ch,varname)( \ diagoffc, \ uploc, \ conjc, \ + schema, \ m_panel, \ n_panel, \ m_panel_max, \ @@ -131,6 +132,7 @@ void PASTEMAC(ch,varname)( \ diagc, \ uploc, \ conjc, \ + schema, \ invdiag, \ m_panel, \ n_panel, \ @@ -253,6 +255,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -272,12 +275,12 @@ void PASTEMAC(ch,varname)( \ bool_t col_stored; \ \ \ - /* Create flags to incidate row or column storage. Since we don't - have the schema in scope, we must use the dimensions and strides - of the micro-panel to determine whether it is row- or column- - stored. */ \ - row_stored = bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ); \ - col_stored = bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ); \ + /* Create flags to incidate row or column storage. Note that the + schema bit that encodes row or column is describing the form of + micro-panel, not the storage in the micro-panel. Hence the + mismatch in "row" and "column" semantics. */ \ + row_stored = bli_is_col_packed( schema ); \ + col_stored = bli_is_row_packed( schema ); \ \ \ /* Handle the case where the micro-panel does NOT intersect the @@ -517,6 +520,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -535,12 +539,12 @@ void PASTEMAC(ch,varname)( \ bool_t col_stored; \ \ \ - /* Create flags to incidate row or column storage. Since we don't - have the schema in scope, we must use the dimensions and strides - of the micro-panel to determine whether it is row- or column- - stored. */ \ - row_stored = bli_is_row_stored_f( m_panel, n_panel, rs_p, cs_p ); \ - col_stored = bli_is_col_stored_f( m_panel, n_panel, rs_p, cs_p ); \ + /* Create flags to incidate row or column storage. Note that the + schema bit that encodes row or column is describing the form of + micro-panel, not the storage in the micro-panel. Hence the + mismatch in "row" and "column" semantics. */ \ + row_stored = bli_is_col_packed( schema ); \ + col_stored = bli_is_row_packed( schema ); \ \ \ /* Pack the panel. */ \ diff --git a/frame/1m/packm/bli_packm_struc_cxk_4m.h b/frame/1m/packm/bli_packm_struc_cxk_4m.h index 4bffca572..320d0ddcc 100644 --- a/frame/1m/packm/bli_packm_struc_cxk_4m.h +++ b/frame/1m/packm/bli_packm_struc_cxk_4m.h @@ -41,6 +41,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ @@ -63,6 +64,7 @@ void PASTEMAC(ch,varname)( \ doff_t diagoffc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ dim_t m_panel, \ dim_t n_panel, \ dim_t m_panel_max, \ @@ -89,6 +91,7 @@ void PASTEMAC(ch,varname)( \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \ diff --git a/frame/include/bli_kernel_type_defs.h b/frame/include/bli_kernel_type_defs.h index ae9a72e1a..d576ca762 100644 --- a/frame/include/bli_kernel_type_defs.h +++ b/frame/include/bli_kernel_type_defs.h @@ -113,6 +113,7 @@ typedef void \ diag_t diagc, \ uplo_t uploc, \ conj_t conjc, \ + pack_t schema, \ bool_t invdiag, \ dim_t m_panel, \ dim_t n_panel, \