mirror of
https://github.com/amd/blis.git
synced 2026-05-11 17:50:00 +00:00
126482a3b6
Details: - Implemented the 1m method for inducing complex domain matrix multiplication. 1m support has been added to all level-3 operations, including trsm, and is now the default induced method when native complex domain gemm microkernels are omitted from the configuration. - Updated _cntx_init() operations to take a datatype parameter. This was needed for the corresponding function for 1m (because 1m requires us to choose between column-oriented or row-oriented execution, which requires us to query the context for the storage preference of the gemm microkernel, which requires knowing the datatype) but I decided that it made sense for consistency to add the parameter to all other cntx initialization functions as well, even though those functions don't use the parameter. - Updated bli_cntx_set_blkszs() and bli_gks_cntx_set_blkszs() to take a second scalar for each blocksize entry. The semantic meaning of the two scalars now is that the first will scale the default blocksize while the second will scale the maximum blocksize. This allows scaling the two independently, and was needed to support 1m, which requires scaling for a register blocksize but not the register storage blocksize (ie: "packdim") analogue. - Deprecated bli_blksz_reduce_dt_to() and defined two new functions, bli_blksz_reduce_def_to() and bli_blksz_reduce_max_to(), for reducing default and maximum blocksizes to some desired blocksize multiple. These functions are needed in the updated definitions of bli_cntx_set_blkszs() and bli_gks_cntx_set_blkszs(). - Added support for the 1e and 1r packing schemas to packm, including 1e/1r packing kernels. - Added a minor optimization to bli_gemm_ker_var2() that allows, under certain circumstances (specifically, real domain beta and row- or column-stored matrix C), the real domain macrokernel and microkernel to be called directly, rather than using the virtual microkernel via the complex domain macrokernel, which carries a slight additional amount of overhead. - Added 1m support to the testsuite. - Added 1m support to Makefile and runme.sh in test/3m4m. Also simplified some code in test_gemm.c driver.
382 lines
8.9 KiB
C
382 lines
8.9 KiB
C
/*
|
|
|
|
BLIS
|
|
An object-based framework for developing high-performance BLAS-like
|
|
libraries.
|
|
|
|
Copyright (C) 2014, The University of Texas at Austin
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are
|
|
met:
|
|
- Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
- Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in the
|
|
documentation and/or other materials provided with the distribution.
|
|
- Neither the name of The University of Texas at Austin nor the names
|
|
of its contributors may be used to endorse or promote products
|
|
derived from this software without specific prior written permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
#include "blis.h"
|
|
|
|
//
|
|
// Define BLAS-like interfaces with typed operands.
|
|
//
|
|
|
|
#undef GENTFUNCR
|
|
#define GENTFUNCR( ctype, ctype_r, ch, chr, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
dim_t n, \
|
|
ctype* x, inc_t incx, \
|
|
ctype_r* asum, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If the vector length is zero, set the absolute sum return value to
|
|
zero and return early. */ \
|
|
if ( bli_zero_dim1( n ) ) \
|
|
{ \
|
|
PASTEMAC(chr,set0s)( *asum ); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
n, \
|
|
x, incx, \
|
|
asum, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNCR_BASIC0( asumv )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
uplo_t uploa, \
|
|
dim_t m, \
|
|
ctype* a, inc_t rs_a, inc_t cs_a, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If either dimension is zero, return early. */ \
|
|
if ( bli_zero_dim2( m, m ) ) return; \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
uploa, \
|
|
m, \
|
|
a, rs_a, cs_a, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC0( mkherm )
|
|
INSERT_GENTFUNC_BASIC0( mksymm )
|
|
INSERT_GENTFUNC_BASIC0( mktrim )
|
|
|
|
|
|
#undef GENTFUNCR
|
|
#define GENTFUNCR( ctype, ctype_r, ch, chr, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
dim_t n, \
|
|
ctype* x, inc_t incx, \
|
|
ctype_r* norm, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If the vector length is zero, set the norm to zero and return
|
|
early. */ \
|
|
if ( bli_zero_dim1( n ) ) \
|
|
{ \
|
|
PASTEMAC(chr,set0s)( *norm ); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
n, \
|
|
x, incx, \
|
|
norm, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNCR_BASIC0( norm1v )
|
|
INSERT_GENTFUNCR_BASIC0( normfv )
|
|
INSERT_GENTFUNCR_BASIC0( normiv )
|
|
|
|
|
|
#undef GENTFUNCR
|
|
#define GENTFUNCR( ctype, ctype_r, ch, chr, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
doff_t diagoffx, \
|
|
diag_t diagx, \
|
|
uplo_t uplox, \
|
|
dim_t m, \
|
|
dim_t n, \
|
|
ctype* x, inc_t rs_x, inc_t cs_x, \
|
|
ctype_r* norm, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If either dimension is zero, set the norm to zero and return
|
|
early. */ \
|
|
if ( bli_zero_dim2( m, n ) ) \
|
|
{ \
|
|
PASTEMAC(chr,set0s)( *norm ); \
|
|
return; \
|
|
} \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
diagoffx, \
|
|
diagx, \
|
|
uplox, \
|
|
m, \
|
|
n, \
|
|
x, rs_x, cs_x, \
|
|
norm, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNCR_BASIC0( norm1m )
|
|
INSERT_GENTFUNCR_BASIC0( normfm )
|
|
INSERT_GENTFUNCR_BASIC0( normim )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, varname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
char* s1, \
|
|
dim_t n, \
|
|
void* x, inc_t incx, \
|
|
char* format, \
|
|
char* s2 \
|
|
) \
|
|
{ \
|
|
PASTEMAC(ch,varname) \
|
|
( \
|
|
stdout, \
|
|
s1, \
|
|
n, \
|
|
x, incx, \
|
|
format, \
|
|
s2 \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC_I( printv, fprintv )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname, varname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
char* s1, \
|
|
dim_t m, \
|
|
dim_t n, \
|
|
void* x, inc_t rs_x, inc_t cs_x, \
|
|
char* format, \
|
|
char* s2 \
|
|
) \
|
|
{ \
|
|
PASTEMAC(ch,varname) \
|
|
( \
|
|
stdout, \
|
|
s1, \
|
|
m, \
|
|
n, \
|
|
x, rs_x, cs_x, \
|
|
format, \
|
|
s2 \
|
|
); \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC_I( printm, fprintm )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
dim_t n, \
|
|
ctype* x, inc_t incx, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If the vector length is zero, return early. */ \
|
|
if ( bli_zero_dim1( n ) ) return; \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
n, \
|
|
x, incx, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC0( randv )
|
|
INSERT_GENTFUNC_BASIC0( randnv )
|
|
|
|
|
|
#undef GENTFUNC
|
|
#define GENTFUNC( ctype, ch, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
doff_t diagoffx, \
|
|
uplo_t uplox, \
|
|
dim_t m, \
|
|
dim_t n, \
|
|
ctype* x, inc_t rs_x, inc_t cs_x, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If either dimension is zero, return early. */ \
|
|
if ( bli_zero_dim2( m, n ) ) return; \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
diagoffx, \
|
|
uplox, \
|
|
m, \
|
|
n, \
|
|
x, rs_x, cs_x, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNC_BASIC0( randm )
|
|
INSERT_GENTFUNC_BASIC0( randnm )
|
|
|
|
|
|
#undef GENTFUNCR
|
|
#define GENTFUNCR( ctype, ctype_r, ch, chr, opname ) \
|
|
\
|
|
void PASTEMAC(ch,opname) \
|
|
( \
|
|
dim_t n, \
|
|
ctype* x, inc_t incx, \
|
|
ctype_r* scale, \
|
|
ctype_r* sumsq, \
|
|
cntx_t* cntx \
|
|
) \
|
|
{ \
|
|
cntx_t* cntx_p = cntx; \
|
|
\
|
|
/* If x is zero length, return with scale and sumsq unchanged. */ \
|
|
if ( bli_zero_dim1( n ) ) return; \
|
|
\
|
|
/* Initialize a local context if the given context is NULL. */ \
|
|
/*bli_cntx_init_local_if( opname, dt, cntx, cntx_p );*/ \
|
|
\
|
|
/* Invoke the helper variant, which loops over the appropriate kernel
|
|
to implement the current operation. */ \
|
|
PASTEMAC2(ch,opname,_unb_var1) \
|
|
( \
|
|
n, \
|
|
x, incx, \
|
|
scale, \
|
|
sumsq, \
|
|
cntx_p \
|
|
); \
|
|
\
|
|
/* Finalize the context if it was initialized locally. */ \
|
|
/*bli_cntx_finalize_local_if( opname, cntx );*/ \
|
|
}
|
|
|
|
INSERT_GENTFUNCR_BASIC0( sumsqv )
|
|
|
|
|