mirror of
https://github.com/amd/blis.git
synced 2026-04-29 12:01:12 +00:00
70640a3710
Details: - Defined two new functions in bli_init.c: bli_init_once() and bli_finalize_once(). Each is implemented with pthread_once(), which guarantees that, among the threads that pass in the same pthread_once_t data structure, exactly one thread will execute a user-defined function. (Thus, there is now a runtime dependency against libpthread even when multithreading is not enabled at configure-time.) - Added calls to bli_init_once() to top-level user APIs for all computational operations as well as many other functions in BLIS to all but guarantee that BLIS will self-initialize through the normal use of its functions. - Rewrote and simplified bli_init() and bli_finalize() and related functions. - Added -lpthread to LDFLAGS in common.mk. - Modified the bli_init_auto()/_finalize_auto() functions used by the BLAS compatibility layer to take and return no arguments. (The previous API that tracked whether BLIS was initialized, and then only finalized if it was initialized in the same function, was too cute by half and borderline useless because by default BLIS stays initialized when auto-initialized via the compatibility layer.) - Removed static variables that track initialization of the sub-APIs in bli_const.c, bli_error.c, bli_init.c, bli_memsys.c, bli_thread, and bli_ind.c. We don't need to track initialization at the sub-API level, especially now that BLIS can self-initialize. - Added a critical section around the changing of the error checking level in bli_error.c. - Deprecated bli_ind_oper_has_avail() as well as all functions bli_<opname>_ind_get_avail(), where <opname> is a level-3 operation name. These functions had no use cases within BLIS and likely none outside of BLIS. - Commented out calls to bli_init() and bli_finalize() in testsuite's main() function, and likewise for standalone test drivers in 'test' directory, so that self-initialization is exercised by default.
484 lines
13 KiB
C
484 lines
13 KiB
C
/*
|
|
|
|
BLIS
|
|
An object-based framework for developing high-performance BLAS-like
|
|
libraries.
|
|
|
|
Copyright (C) 2014, The University of Texas at Austin
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are
|
|
met:
|
|
- Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
- Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in the
|
|
documentation and/or other materials provided with the distribution.
|
|
- Neither the name of The University of Texas at Austin nor the names
|
|
of its contributors may be used to endorse or promote products
|
|
derived from this software without specific prior written permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
#include "blis.h"
|
|
|
|
//
|
|
// Define BLAS-like interfaces with typed operands.
|
|
//
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
trans_t transa, \
|
|
conj_t conjx, \
|
|
dim_t m, \
|
|
dim_t n, \
|
|
ctype* alpha, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* beta, \
|
|
ctype* y, inc_t incy, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
dim_t m_y, n_x; \
|
|
\
|
|
/* Determine the dimensions of y and x. */ \
|
|
bli_set_dims_with_trans( transa, m, n, m_y, n_x ); \
|
|
\
|
|
/* If y has zero elements, return early. */ \
|
|
if ( bli_zero_dim1( m_y ) ) return; \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* If x has zero elements, or if alpha is zero, scale y by beta and
|
|
return early. */ \
|
|
if ( bli_zero_dim1( n_x ) || PASTEMAC(ch,eq0)( *alpha ) ) \
|
|
{ \
|
|
PASTEMAC(ch,scalv) \
|
|
( \
|
|
BLIS_NO_CONJUGATE, \
|
|
m_y, \
|
|
beta, \
|
|
y, incy, \
|
|
cntx \
|
|
); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_does_notrans( transa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_does_trans( transa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
transa, \
|
|
conjx, \
|
|
m, \
|
|
n, \
|
|
alpha, \
|
|
a, rs_a, cs_a, \
|
|
x, incx, \
|
|
beta, \
|
|
y, incy, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC3( gemv, gemv, gemv_unf_var1, gemv_unf_var2 )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
conj_t conjx, \
|
|
conj_t conjy, \
|
|
dim_t m, \
|
|
dim_t n, \
|
|
ctype* alpha, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* y, inc_t incy, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
/* If x or y has zero elements, or if alpha is zero, return early. */ \
|
|
if ( bli_zero_dim2( m, n ) || PASTEMAC(ch,eq0)( *alpha ) ) return; \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
conjx, \
|
|
conjy, \
|
|
m, \
|
|
n, \
|
|
alpha, \
|
|
x, incx, \
|
|
y, incy, \
|
|
a, rs_a, cs_a, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC3( ger, ger, ger_unb_var1, ger_unb_var2 )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, conjh, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
conj_t conja, \
|
|
conj_t conjx, \
|
|
dim_t m, \
|
|
ctype* alpha, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* beta, \
|
|
ctype* y, inc_t incy, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* If x has zero elements, or if alpha is zero, scale y by beta and
|
|
return early. */ \
|
|
if ( bli_zero_dim1( m ) || PASTEMAC(ch,eq0)( *alpha ) ) \
|
|
{ \
|
|
PASTEMAC(ch,scalv) \
|
|
( \
|
|
BLIS_NO_CONJUGATE, \
|
|
m, \
|
|
beta, \
|
|
y, incy, \
|
|
cntx \
|
|
); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_is_lower( uploa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_is_upper( uploa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
uploa, \
|
|
conja, \
|
|
conjx, \
|
|
conjh, /* used by variants to distinguish hemv from symv */ \
|
|
m, \
|
|
alpha, \
|
|
a, rs_a, cs_a, \
|
|
x, incx, \
|
|
beta, \
|
|
y, incy, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC4( hemv, hemv, BLIS_CONJUGATE, hemv_unf_var1, hemv_unf_var3 )
|
|
INSERT_GENTFUNC_BASIC4( symv, hemv, BLIS_NO_CONJUGATE, hemv_unf_var1, hemv_unf_var3 )
|
|
|
|
|
|
#undef GENTFUNCR
|
|
#define GENTFUNCR( ctype, ctype_r, ch, chr, opname, ftname, conjh, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
conj_t conjx, \
|
|
dim_t m, \
|
|
ctype_r* alpha, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
ctype alpha_local; \
|
|
\
|
|
/* If x has zero elements, or if alpha is zero, return early. */ \
|
|
if ( bli_zero_dim1( m ) || PASTEMAC(chr,eq0)( *alpha ) ) return; \
|
|
\
|
|
/* Make a local copy of alpha, cast into the complex domain. This
|
|
allows us to use the same underlying her variants to implement
|
|
both her and syr operations. */ \
|
|
PASTEMAC2(chr,ch,copys)( *alpha, alpha_local ); \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_is_lower( uploa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_is_upper( uploa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
uploa, \
|
|
conjx, \
|
|
conjh, /* used by variants to distinguish her from syr */ \
|
|
m, \
|
|
&alpha_local, \
|
|
x, incx, \
|
|
a, rs_a, cs_a, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNCR_BASIC4( her, her, BLIS_CONJUGATE, her_unb_var1, her_unb_var2 )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, conjh, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
conj_t conjx, \
|
|
dim_t m, \
|
|
ctype* alpha, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
/* If x has zero elements, or if alpha is zero, return early. */ \
|
|
if ( bli_zero_dim1( m ) || PASTEMAC(ch,eq0)( *alpha ) ) return; \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_is_lower( uploa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_is_upper( uploa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
uploa, \
|
|
conjx, \
|
|
conjh, /* used by variants to distinguish her2 from syr2 */ \
|
|
m, \
|
|
alpha, \
|
|
x, incx, \
|
|
a, rs_a, cs_a, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC4( syr, her, BLIS_NO_CONJUGATE, her_unb_var1, her_unb_var2 )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, conjh, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
conj_t conjx, \
|
|
conj_t conjy, \
|
|
dim_t m, \
|
|
ctype* alpha, \
|
|
ctype* x, inc_t incx, \
|
|
ctype* y, inc_t incy, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
/* If x has zero elements, or if alpha is zero, return early. */ \
|
|
if ( bli_zero_dim1( m ) || PASTEMAC(ch,eq0)( *alpha ) ) return; \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_is_lower( uploa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_is_upper( uploa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
uploa, \
|
|
conjx, \
|
|
conjy, \
|
|
conjh, \
|
|
m, \
|
|
alpha, \
|
|
x, incx, \
|
|
y, incy, \
|
|
a, rs_a, cs_a, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC4( her2, her2, BLIS_CONJUGATE, her2_unf_var1, her2_unf_var4 )
|
|
INSERT_GENTFUNC_BASIC4( syr2, her2, BLIS_NO_CONJUGATE, her2_unf_var1, her2_unf_var4 )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, ftname, rvarname, cvarname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
trans_t transa, \
|
|
diag_t diaga, \
|
|
dim_t m, \
|
|
ctype* alpha, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
ctype* x, inc_t incx, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
bli_init_once(); \
|
|
\
|
|
/* If x has zero elements, return early. */ \
|
|
if ( bli_zero_dim1( m ) ) return; \
|
|
\
|
|
/* Obtain a valid context from the gks if necessary. */ \
|
|
if ( cntx == NULL ) cntx = bli_gks_query_cntx(); \
|
|
\
|
|
/* If alpha is zero, set x to zero and return early. */ \
|
|
if ( PASTEMAC(ch,eq0)( *alpha ) ) \
|
|
{ \
|
|
PASTEMAC(ch,setv) \
|
|
( \
|
|
BLIS_NO_CONJUGATE, \
|
|
m, \
|
|
alpha, \
|
|
x, incx, \
|
|
cntx \
|
|
); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Declare a void function pointer for the current operation. */ \
|
|
PASTECH2(ch,ftname,_ft) f; \
|
|
\
|
|
/* Choose the underlying implementation. */ \
|
|
if ( bli_does_notrans( transa ) ) \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,rvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,cvarname); \
|
|
} \
|
|
else /* if ( bli_does_trans( transa ) ) */ \
|
|
{ \
|
|
if ( bli_is_row_stored( rs_a, cs_a ) ) f = PASTEMAC(ch,cvarname); \
|
|
else /* column or general stored */ f = PASTEMAC(ch,rvarname); \
|
|
} \
|
|
\
|
|
/* Invoke the variant chosen above, which loops over a level-1v or
|
|
level-1f kernel to implement the current operation. */ \
|
|
f( \
|
|
uploa, \
|
|
transa, \
|
|
diaga, \
|
|
m, \
|
|
alpha, \
|
|
a, rs_a, cs_a, \
|
|
x, incx, \
|
|
cntx \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC3( trmv, trmv, trmv_unf_var1, trmv_unf_var2 )
|
|
INSERT_GENTFUNC_BASIC3( trsv, trmv, trsv_unf_var1, trsv_unf_var2 )
|