From 64a1f786d58001284aa4f7faf9fae17f0be7a018 Mon Sep 17 00:00:00 2001 From: Devin Matthews Date: Wed, 11 Aug 2021 17:53:12 -0500 Subject: [PATCH 1/4] Implement proposed new function pointer fields for obj_t. The added fields: 1. `pack_t schema`: storing the pack schema on the object allows the macrokernel to act accordingly without side-channel information from the rntm_t and cntx_t. The pack schema and "pack_[ab]" fields could be removed from those structs. 2. `void* user_data`: this field can be used to store any sort of additional information provided by the user. The pointer is propagated to submatrix objects and copies, but is otherwise ignored by the framework and the default implementations of the following three fields. User-specified pack, kernel, or ukr functions can do whatever they want with the data, and the user is 100% responsible for allocating, assigning, and freeing this buffer. 3. `obj_pack_fn_t pack`: the function called when a matrix is packed. This functions receives the expected arguments, as well as a mdim_t and mem_t* as memory must be allocated inside this function, and behavior may differ based on which matrix is being backed (i.e. transposition for B). This could also be achieved by passing a desired pack schema, but this would require additional information to travel down the control tree. 4. `obj_ker_fn_t ker`: the function called when we get to the "second loop", or the macro-kernel. Behavior may depend on the pack schemas of the input matrices. The default implementation would perform the inner two loops around the ukr, and then call either the default ukr or a user-supplied one (next field). 5. `obj_ukr_fn_t ukr`: the function called by the default macrokernel. This would replace the various current "virtual" microkernels, and could also be used to supply user-defined behavior. Users could supply both a custom kernel (above) and microkernel, although the user-specified kernel does **not** necessarily have to call the ukr function specified on the obj_t. Note that no macros or functions for accessing these new fields have been defined yet. That is next once these are finalized. Addresses https://github.com/flame/blis/projects/1#card-62357687. --- frame/include/bli_type_defs.h | 86 +++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index cba112256..566ad5f50 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -150,7 +150,7 @@ typedef uint32_t objbits_t; // object information bit field // interoperability with BLIS. #ifndef _DEFINED_SCOMPLEX #define _DEFINED_SCOMPLEX - typedef struct + typedef struct scomplex { float real; float imag; @@ -161,7 +161,7 @@ typedef uint32_t objbits_t; // object information bit field // interoperability with BLIS. #ifndef _DEFINED_DCOMPLEX #define _DEFINED_DCOMPLEX - typedef struct + typedef struct dcomplex { double real; double imag; @@ -1232,6 +1232,47 @@ typedef struct constdata_s // -- BLIS object type definitions --------------------------------------------- // +// Forward declarations for function pointer types +struct obj_s; +struct cntx_s; +struct rntm_s; +struct thrinfo_s; + +typedef void (*obj_pack_fn_t) + ( + mdim_t mat, + mem_t* mem, + struct obj_s* a, + struct obj_s* ap, + struct cntx_s* cntx, + struct rntm_s* rntm, + struct thrinfo_s* thread + ); + +typedef void (*obj_ker_fn_t) + ( + struct obj_s* a, + struct obj_s* b, + struct obj_s* c, + struct cntx_s* cntx, + struct rntm_s* rntm, + struct thrinfo_s* thread + ); + +typedef void (*obj_ukr_fn_t) + ( + dim_t m, + dim_t n, + dim_t k, + void* restrict alpha, + void* restrict a, + void* restrict b, + void* restrict beta, + void* restrict c, inc_t rs_c, inc_t cs_c, + auxinfo_t* restrict data, + struct cntx_s* restrict cntx + ); + typedef struct obj_s { // Basic fields @@ -1261,6 +1302,15 @@ typedef struct obj_s // usually MR or NR) dim_t m_panel; // m dimension of a "full" panel dim_t n_panel; // n dimension of a "full" panel + pack_t schema; // pack schema, which may be unpacked + + // User data pointer + void* user_data; + + // Function pointers + obj_pack_fn_t pack; + obj_ker_fn_t ker; + obj_ukr_fn_t ukr; } obj_t; // Pre-initializors. Things that must be set afterwards: @@ -1297,7 +1347,14 @@ typedef struct obj_s .ps = 0, \ .pd = 0, \ .m_panel = 0, \ - .n_panel = 0 \ + .n_panel = 0, \ + .schema = BLIS_NOT_PACKED, \ +\ + .user_data = NULL, \ +\ + .pack = NULL, \ + .ker = NULL, \ + .ukr = NULL \ } #define BLIS_OBJECT_INITIALIZER_1X1 \ @@ -1325,7 +1382,14 @@ typedef struct obj_s .ps = 0, \ .pd = 0, \ .m_panel = 0, \ - .n_panel = 0 \ + .n_panel = 0, \ + .schema = BLIS_NOT_PACKED, \ +\ + .user_data = NULL, \ +\ + .pack = NULL, \ + .ker = NULL, \ + .ukr = NULL \ } // Define these macros here since they must be updated if contents of @@ -1359,6 +1423,13 @@ BLIS_INLINE void bli_obj_init_full_shallow_copy_of( obj_t* a, obj_t* b ) b->pd = a->pd; b->m_panel = a->m_panel; b->n_panel = a->n_panel; + b->schema = a->schema; + + b->user_data = a->user_data; + + b->pack = a->pack; + b->ker = a->ker; + b->ukr = a->ukr; } BLIS_INLINE void bli_obj_init_subpart_from( obj_t* a, obj_t* b ) @@ -1392,6 +1463,13 @@ BLIS_INLINE void bli_obj_init_subpart_from( obj_t* a, obj_t* b ) b->pd = a->pd; b->m_panel = a->m_panel; b->n_panel = a->n_panel; + b->schema = a->schema; + + b->user_data = a->user_data; + + b->pack = a->pack; + b->ker = a->ker; + b->ukr = a->ukr; } // Initializors for global scalar constants. From 3cddce1e2a021be6064b90af30022b99cbfea986 Mon Sep 17 00:00:00 2001 From: Devin Matthews Date: Thu, 12 Aug 2021 22:32:34 -0500 Subject: [PATCH 2/4] Remove schema field on obj_t (redundant) and add new API functions. --- frame/include/bli_obj_macro_defs.h | 51 ++++++++++++++++++++++++++++++ frame/include/bli_type_defs.h | 23 ++++++-------- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index 855384425..bb7045099 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -1187,6 +1187,57 @@ BLIS_INLINE stor3_t bli_obj_stor3_from_strides( obj_t* c, obj_t* a, obj_t* b ) } +// -- User-provided information macros -- + +// User data query + +BLIS_INLINE void* bli_obj_user_data( obj_t* obj ) +{ + return obj->user_data; +} + +// User data modification + +BLIS_INLINE void bli_obj_set_user_data( void* data, obj_t* obj ) +{ + obj->user_data = data; +} + +// Function pointer query + +BLIS_INLINE obj_pack_fn_t bli_obj_pack_fn( obj_t* obj ) +{ + return obj->pack; +} + +BLIS_INLINE obj_ker_fn_t bli_obj_ker_fn( obj_t* obj ) +{ + return obj->ker; +} + +BLIS_INLINE obj_ukr_fn_t bli_obj_ukf_fn( obj_t* obj ) +{ + return obj->ukr; +} + +// Function pointer modification + +BLIS_INLINE void bli_obj_set_pack_fn( obj_pack_fn_t pack, obj_t* obj ) +{ + obj->pack = pack; +} + +BLIS_INLINE void bli_obj_set_ker_fn( obj_ker_fn_t ker, obj_t* obj ) +{ + obj->ker = ker; +} + +BLIS_INLINE void bli_obj_set_ukf_fn( obj_ukr_fn_t ukr, obj_t* obj ) +{ + obj->ukr = ukr; +} + + // -- Initialization-related macros -- // Finish the initialization started by the matrix-specific static initializer diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 566ad5f50..2abcf35fe 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -385,7 +385,7 @@ typedef void (*free_ft) ( void* p ); #define BLIS_BITVAL_SINGLE_PREC 0x0 #define BLIS_BITVAL_DOUBLE_PREC BLIS_PRECISION_BIT #define BLIS_BITVAL_FLOAT_TYPE 0x0 -#define BLIS_BITVAL_SCOMPLEX_TYPE BLIS_DOMAIN_BIT +#define BLIS_BITVAL_SCOMPLEX_TYPE BLIS_DOMAIN_BIT #define BLIS_BITVAL_DOUBLE_TYPE BLIS_PRECISION_BIT #define BLIS_BITVAL_DCOMPLEX_TYPE ( BLIS_DOMAIN_BIT | BLIS_PRECISION_BIT ) #define BLIS_BITVAL_INT_TYPE 0x04 @@ -395,10 +395,10 @@ typedef void (*free_ft) ( void* p ); #define BLIS_BITVAL_NO_CONJ 0x0 #define BLIS_BITVAL_CONJ BLIS_CONJ_BIT #define BLIS_BITVAL_CONJ_TRANS ( BLIS_CONJ_BIT | BLIS_TRANS_BIT ) -#define BLIS_BITVAL_ZEROS 0x0 +#define BLIS_BITVAL_ZEROS 0x0 #define BLIS_BITVAL_UPPER ( BLIS_UPPER_BIT | BLIS_DIAG_BIT ) #define BLIS_BITVAL_LOWER ( BLIS_LOWER_BIT | BLIS_DIAG_BIT ) -#define BLIS_BITVAL_DENSE BLIS_UPLO_BITS +#define BLIS_BITVAL_DENSE BLIS_UPLO_BITS #define BLIS_BITVAL_NONUNIT_DIAG 0x0 #define BLIS_BITVAL_UNIT_DIAG BLIS_UNIT_DIAG_BIT #define BLIS_BITVAL_INVERT_DIAG BLIS_INVERT_DIAG_BIT @@ -1242,7 +1242,7 @@ typedef void (*obj_pack_fn_t) ( mdim_t mat, mem_t* mem, - struct obj_s* a, + struct obj_s* a, struct obj_s* ap, struct cntx_s* cntx, struct rntm_s* rntm, @@ -1251,8 +1251,8 @@ typedef void (*obj_pack_fn_t) typedef void (*obj_ker_fn_t) ( - struct obj_s* a, - struct obj_s* b, + struct obj_s* a, + struct obj_s* b, struct obj_s* c, struct cntx_s* cntx, struct rntm_s* rntm, @@ -1302,7 +1302,6 @@ typedef struct obj_s // usually MR or NR) dim_t m_panel; // m dimension of a "full" panel dim_t n_panel; // n dimension of a "full" panel - pack_t schema; // pack schema, which may be unpacked // User data pointer void* user_data; @@ -1348,7 +1347,6 @@ typedef struct obj_s .pd = 0, \ .m_panel = 0, \ .n_panel = 0, \ - .schema = BLIS_NOT_PACKED, \ \ .user_data = NULL, \ \ @@ -1383,7 +1381,6 @@ typedef struct obj_s .pd = 0, \ .m_panel = 0, \ .n_panel = 0, \ - .schema = BLIS_NOT_PACKED, \ \ .user_data = NULL, \ \ @@ -1423,7 +1420,6 @@ BLIS_INLINE void bli_obj_init_full_shallow_copy_of( obj_t* a, obj_t* b ) b->pd = a->pd; b->m_panel = a->m_panel; b->n_panel = a->n_panel; - b->schema = a->schema; b->user_data = a->user_data; @@ -1463,7 +1459,6 @@ BLIS_INLINE void bli_obj_init_subpart_from( obj_t* a, obj_t* b ) b->pd = a->pd; b->m_panel = a->m_panel; b->n_panel = a->n_panel; - b->schema = a->schema; b->user_data = a->user_data; @@ -1624,13 +1619,13 @@ typedef enum BLIS_INVALID_COL_STRIDE = ( -51), BLIS_INVALID_DIM_STRIDE_COMBINATION = ( -52), - // Structure-specific errors + // Structure-specific errors BLIS_EXPECTED_GENERAL_OBJECT = ( -60), BLIS_EXPECTED_HERMITIAN_OBJECT = ( -61), BLIS_EXPECTED_SYMMETRIC_OBJECT = ( -62), BLIS_EXPECTED_TRIANGULAR_OBJECT = ( -63), - // Storage-specific errors + // Storage-specific errors BLIS_EXPECTED_UPPER_OR_LOWER_OBJECT = ( -70), // Partitioning-specific errors @@ -1644,7 +1639,7 @@ typedef enum // Packing-specific errors BLIS_PACK_SCHEMA_NOT_SUPPORTED_FOR_UNPACK = (-100), - // Buffer-specific errors + // Buffer-specific errors BLIS_EXPECTED_NONNULL_OBJECT_BUFFER = (-110), // Memory errors From 1772db029e10e0075b5a59d3fb098487b1ad542a Mon Sep 17 00:00:00 2001 From: Devin Matthews Date: Fri, 13 Aug 2021 14:46:35 -0500 Subject: [PATCH 3/4] Add row- and column-strides for A/B in obj_ukr_fn_t. --- frame/include/bli_obj_macro_defs.h | 4 ++-- frame/include/bli_type_defs.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index bb7045099..fbf979c8a 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -1215,7 +1215,7 @@ BLIS_INLINE obj_ker_fn_t bli_obj_ker_fn( obj_t* obj ) return obj->ker; } -BLIS_INLINE obj_ukr_fn_t bli_obj_ukf_fn( obj_t* obj ) +BLIS_INLINE obj_ukr_fn_t bli_obj_ukr_fn( obj_t* obj ) { return obj->ukr; } @@ -1232,7 +1232,7 @@ BLIS_INLINE void bli_obj_set_ker_fn( obj_ker_fn_t ker, obj_t* obj ) obj->ker = ker; } -BLIS_INLINE void bli_obj_set_ukf_fn( obj_ukr_fn_t ukr, obj_t* obj ) +BLIS_INLINE void bli_obj_set_ukr_fn( obj_ukr_fn_t ukr, obj_t* obj ) { obj->ukr = ukr; } diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 2abcf35fe..5a180e2e9 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -1265,8 +1265,8 @@ typedef void (*obj_ukr_fn_t) dim_t n, dim_t k, void* restrict alpha, - void* restrict a, - void* restrict b, + void* restrict a, inc_t rs_a, inc_t cs_a, + void* restrict b, inc_t rs_b, inc_t cs_b, void* restrict beta, void* restrict c, inc_t rs_c, inc_t cs_c, auxinfo_t* restrict data, From 4b8ed99d926876fbf54c15468feae4637268eb6b Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 13 Aug 2021 15:31:10 -0500 Subject: [PATCH 4/4] Whitespace tweaks. --- frame/include/bli_obj_macro_defs.h | 16 +++++----- frame/include/bli_type_defs.h | 51 +++++++++++++++--------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/frame/include/bli_obj_macro_defs.h b/frame/include/bli_obj_macro_defs.h index fbf979c8a..84c977289 100644 --- a/frame/include/bli_obj_macro_defs.h +++ b/frame/include/bli_obj_macro_defs.h @@ -1193,48 +1193,48 @@ BLIS_INLINE stor3_t bli_obj_stor3_from_strides( obj_t* c, obj_t* a, obj_t* b ) BLIS_INLINE void* bli_obj_user_data( obj_t* obj ) { - return obj->user_data; + return obj->user_data; } // User data modification BLIS_INLINE void bli_obj_set_user_data( void* data, obj_t* obj ) { - obj->user_data = data; + obj->user_data = data; } // Function pointer query BLIS_INLINE obj_pack_fn_t bli_obj_pack_fn( obj_t* obj ) { - return obj->pack; + return obj->pack; } BLIS_INLINE obj_ker_fn_t bli_obj_ker_fn( obj_t* obj ) { - return obj->ker; + return obj->ker; } BLIS_INLINE obj_ukr_fn_t bli_obj_ukr_fn( obj_t* obj ) { - return obj->ukr; + return obj->ukr; } // Function pointer modification BLIS_INLINE void bli_obj_set_pack_fn( obj_pack_fn_t pack, obj_t* obj ) { - obj->pack = pack; + obj->pack = pack; } BLIS_INLINE void bli_obj_set_ker_fn( obj_ker_fn_t ker, obj_t* obj ) { - obj->ker = ker; + obj->ker = ker; } BLIS_INLINE void bli_obj_set_ukr_fn( obj_ukr_fn_t ukr, obj_t* obj ) { - obj->ukr = ukr; + obj->ukr = ukr; } diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 5a180e2e9..f03fc72ac 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -1240,22 +1240,22 @@ struct thrinfo_s; typedef void (*obj_pack_fn_t) ( - mdim_t mat, - mem_t* mem, - struct obj_s* a, - struct obj_s* ap, - struct cntx_s* cntx, - struct rntm_s* rntm, + mdim_t mat, + mem_t* mem, + struct obj_s* a, + struct obj_s* ap, + struct cntx_s* cntx, + struct rntm_s* rntm, struct thrinfo_s* thread ); typedef void (*obj_ker_fn_t) ( - struct obj_s* a, - struct obj_s* b, - struct obj_s* c, - struct cntx_s* cntx, - struct rntm_s* rntm, + struct obj_s* a, + struct obj_s* b, + struct obj_s* c, + struct cntx_s* cntx, + struct rntm_s* rntm, struct thrinfo_s* thread ); @@ -1303,13 +1303,14 @@ typedef struct obj_s dim_t m_panel; // m dimension of a "full" panel dim_t n_panel; // n dimension of a "full" panel - // User data pointer - void* user_data; + // User data pointer + void* user_data; + + // Function pointers + obj_pack_fn_t pack; + obj_ker_fn_t ker; + obj_ukr_fn_t ukr; - // Function pointers - obj_pack_fn_t pack; - obj_ker_fn_t ker; - obj_ukr_fn_t ukr; } obj_t; // Pre-initializors. Things that must be set afterwards: @@ -1348,11 +1349,11 @@ typedef struct obj_s .m_panel = 0, \ .n_panel = 0, \ \ - .user_data = NULL, \ + .user_data = NULL, \ \ - .pack = NULL, \ - .ker = NULL, \ - .ukr = NULL \ + .pack = NULL, \ + .ker = NULL, \ + .ukr = NULL \ } #define BLIS_OBJECT_INITIALIZER_1X1 \ @@ -1382,11 +1383,11 @@ typedef struct obj_s .m_panel = 0, \ .n_panel = 0, \ \ - .user_data = NULL, \ + .user_data = NULL, \ \ - .pack = NULL, \ - .ker = NULL, \ - .ukr = NULL \ + .pack = NULL, \ + .ker = NULL, \ + .ukr = NULL \ } // Define these macros here since they must be updated if contents of