From 4ee986f0a74207f4ca29df077929134725d62b80 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Mon, 22 Oct 2018 14:09:44 -0500 Subject: [PATCH] Added mixed-datatype testing to Travis CI (#271). Details: - Modified .travis.yml to automatically test the mixed-datatype support of the gemm operation, with supporting changes to common.mk, the top-level Makefile, and travis/do_testsuite.sh. - Added a new pair of input files to testsuite directory with the '.mixed' suffix (similar to those with the '.fast' suffix) for testing mixed-datatype gemm. - Updated docs/BuildSystem.md to document the new make targets 'testblis-md' and 'checkblis-md'. --- .travis.yml | 10 +- Makefile | 31 ++- common.mk | 2 + docs/BuildSystem.md | 2 + testsuite/input.general.mixed | 49 +++++ testsuite/input.operations.mixed | 318 +++++++++++++++++++++++++++++++ travis/do_testsuite.sh | 5 +- 7 files changed, 410 insertions(+), 7 deletions(-) create mode 100644 testsuite/input.general.mixed create mode 100644 testsuite/input.operations.mixed diff --git a/.travis.yml b/.travis.yml index eeaee727b..31e787c1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,14 @@ env: secure: "Ty3PM1xGhXwxfJG6YyY9bUZyXzw98ekHxQEqU9VnrMXTZb28IxfocPCXHjL34r9HTGosO5Pmierhal1Cs3ZKE5ZAJqJhCfck+kwlH21Uay5CNYglDtSmy2qxtbbDG4AxpEZ1UKlIZr1pNh/x+pRemSmnMEnQp/E7QJqdkhm4+aMX2bWKyLPtrdL+B9QXLVT2nT6/Fw3i05aBhpcFJpSPfvYX2KoCZYdJOSKcKci4T8nAfP/c0olkz+jAkBZxZFgO9Ptrt/lvHtVPrkh5o29GvHg2i/4vucbsMltoxlV31/2eYpdr17Ngtt41MMVn2fHV4lVhLmENc04nlm084fBtg73T6b8hNy5JlcA44xI/UrPJsQAJ+0A0ds9BbBQKPxOmaF/O8WGXhwiwdKT6DGS9lj05f3S+yZfeNE3pQhLEcvwXLO5SW3VvKXMj0t/lZyG+XCkvFjD7KEPQV4g+BZc2zzD9TwDx3ydn8Uzd6zZlq1erQUzCnODP24wuwfrNP8nqxFYG0VtI8oZW62IC9U2hcnAF5QNXXW3yDYD65k3BHbigfI28gu9iO9G8RxOglR27J7Whdqkqw3AMRaqyHt2tdbz7tM2dLZ0EatT5m8esjC+LP4EshW9C59jP2U9vJ/94YEgOfwiqk8+e6fL/7dJvOumbwu1RclRI9DS88PPYb3Q=" matrix: include: - # full testsuite + # full testsuite (all tests except for mixed datatype) - os: linux compiler: gcc env: OOT=0 TEST=1 SDE=0 THR="none" CONF="auto" + # mixed-datatype testsuite (gemm_nn only) + - os: linux + compiler: gcc + env: OOT=0 TEST=MD SDE=0 THR="none" CONF="auto" # test x86_64 ukrs with SDE - os: linux compiler: gcc @@ -34,13 +38,13 @@ matrix: - os: osx compiler: clang env: OOT=0 TEST=1 SDE=0 THR="none" CONF="auto" - # cortexa15 build and test (qemu) + # cortexa15 build and fast testsuite (qemu) - os: linux compiler: arm-linux-gnueabihf-gcc env: OOT=0 TEST=FAST SDE=0 THR="none" CONF="cortexa15" \ PACKAGES="gcc-arm-linux-gnueabihf qemu-system-arm qemu-user" \ TESTSUITE_WRAPPER="qemu-arm -cpu cortex-a15 -L /usr/arm-linux-gnueabihf/" - # cortexa57 build and test (qemu) + # cortexa57 build and fast testsuite (qemu) - os: linux compiler: aarch64-linux-gnu-gcc env: OOT=0 TEST=FAST SDE=0 THR="none" CONF="cortexa57" \ diff --git a/Makefile b/Makefile index ec7dff9bb..b135e291c 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,8 @@ test \ testblas blastest-f2c blastest-bin blastest-run \ testblis testsuite testsuite-bin testsuite-run \ - testblis-fast testsuite-run-fast \ - check checkblas checkblis checkblis-fast \ + testblis-fast testblis-md testsuite-run-fast \ + check checkblas checkblis checkblis-fast checkblis-md \ install-headers install-libs install-lib-symlinks \ showconfig \ clean cleanmk cleanh cleanlib distclean \ @@ -332,6 +332,8 @@ TESTSUITE_CONF_GEN_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_CONF_GEN) TESTSUITE_CONF_OPS_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_CONF_OPS) TESTSUITE_FAST_GEN_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_FAST_GEN) TESTSUITE_FAST_OPS_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_FAST_OPS) +TESTSUITE_MIXD_GEN_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_MIXD_GEN) +TESTSUITE_MIXD_OPS_PATH := $(DIST_PATH)/$(TESTSUITE_DIR)/$(TESTSUITE_MIXD_OPS) # The locations of the test suite source directory and the local object # directory. @@ -760,6 +762,8 @@ testblis: testsuite testblis-fast: testsuite-run-fast +testblis-md: testsuite-run-md + testsuite: testsuite-run testsuite-bin: check-env $(TESTSUITE_BIN) @@ -811,6 +815,21 @@ else > $(TESTSUITE_OUT_FILE) endif +# A rule to run the testsuite using the input.*.md files, which +# run a set of tests designed to only exercise mixed-datatype gemm. +testsuite-run-md: testsuite-bin +ifeq ($(ENABLE_VERBOSE),yes) + $(TESTSUITE_WRAPPER) ./$(TESTSUITE_BIN) -g $(TESTSUITE_MIXD_GEN_PATH) \ + -o $(TESTSUITE_MIXD_OPS_PATH) \ + > $(TESTSUITE_OUT_FILE) + +else + @echo "Running $(TESTSUITE_BIN) (mixed dt) with output redirected to '$(TESTSUITE_OUT_FILE)'" + @$(TESTSUITE_WRAPPER) ./$(TESTSUITE_BIN) -g $(TESTSUITE_MIXD_GEN_PATH) \ + -o $(TESTSUITE_MIXD_OPS_PATH) \ + > $(TESTSUITE_OUT_FILE) +endif + # Check the results of the BLIS testsuite. checkblis: testsuite-run ifeq ($(ENABLE_VERBOSE),yes) @@ -827,6 +846,14 @@ else @- $(TESTSUITE_CHECK_PATH) $(TESTSUITE_OUT_FILE) endif +# Check the results of the BLIS testsuite (mixed-datatype). +checkblis-md: testsuite-run-md +ifeq ($(ENABLE_VERBOSE),yes) + - $(TESTSUITE_CHECK_PATH) $(TESTSUITE_OUT_FILE) +else + @- $(TESTSUITE_CHECK_PATH) $(TESTSUITE_OUT_FILE) +endif + # --- Install header rules --- install-headers: check-env $(MK_INCL_DIR_INST) diff --git a/common.mk b/common.mk index 4d2bdc62a..0fadb6131 100644 --- a/common.mk +++ b/common.mk @@ -321,6 +321,8 @@ TESTSUITE_CONF_GEN := input.general TESTSUITE_CONF_OPS := input.operations TESTSUITE_FAST_GEN := input.general.fast TESTSUITE_FAST_OPS := input.operations.fast +TESTSUITE_MIXD_GEN := input.general.mixed +TESTSUITE_MIXD_OPS := input.operations.mixed TESTSUITE_OUT_FILE := output.testsuite # CHANGELOG file. diff --git a/docs/BuildSystem.md b/docs/BuildSystem.md index 7ed8a972c..e7bad7c45 100644 --- a/docs/BuildSystem.md +++ b/docs/BuildSystem.md @@ -390,9 +390,11 @@ The BLIS `Makefile` implements many `make` targets. The table below lists most o | `check` | Execute `checkblis-fast` and `checkblas` targets. | | `checkblis` | Execute `testblis` and characterize the results to `stdout`. | | `checkblis-fast`| Execute `testblis-fast` and characterize the results to `stdout`. | +| `checkblis-md` | Execute `testblis-md` and characterize the results to `stdout`. | | `checkblas` | Execute `testblas` and characterize the results to `stdout`. | | `testblis` | Run the BLIS testsuite with default parameters (runs for 2-8 minutes). | | `testblis-fast` | Run the BLIS testsuite with "fast" parameters (runs for a few seconds). | +| `testblis-md` | Run the BLIS testsuite for `gemm` with full mixing of datatypes (runs for 10-30 seconds). | | `testsuite` | Same as `testblis`. | | `testblas` | Run the BLAS test drivers with default parameters (runs for a few seconds). | | `showconfig` | Show a summary of currently selected `configure` options. | diff --git a/testsuite/input.general.mixed b/testsuite/input.general.mixed new file mode 100644 index 000000000..55a3f56c7 --- /dev/null +++ b/testsuite/input.general.mixed @@ -0,0 +1,49 @@ +# ---------------------------------------------------------------------- +# +# input.general +# BLIS test suite +# +# This file contains input values that control how BLIS operations are +# tested. Comments explain the purpose of each parameter as well as +# accepted values. +# + +1 # Number of repeats per experiment (best result is reported) +rc # Matrix storage scheme(s) to test: + # 'c' = col-major storage; 'g' = general stride storage; + # 'r' = row-major storage +cj # Vector storage scheme(s) to test: + # 'c' = colvec / unit stride; 'j' = colvec / non-unit stride; + # 'r' = rowvec / unit stride; 'i' = rowvec / non-unit stride +0 # Test all combinations of storage schemes? +1 # Perform all tests with alignment? + # '0' = do NOT align buffers/ldims; '1' = align buffers/ldims +0 # Randomize vectors and matrices using: + # '0' = real values on [-1,1]; + # '1' = powers of 2 in narrow precision range +32 # General stride spacing (for cases when testing general stride) +sdcz # Datatype(s) to test: + # 's' = single real; 'c' = single complex; + # 'd' = double real; 'z' = double complex +1 # Test gemm with mixed-domain operands? +1 # Test gemm with mixed-precision operands? +100 # Problem size: first to test +500 # Problem size: maximum to test +100 # Problem size: increment between experiments + # Complex level-3 implementations to test: +0 # 3mh ('1' = enable; '0' = disable) +0 # 3m1 ('1' = enable; '0' = disable) +0 # 4mh ('1' = enable; '0' = disable) +0 # 4m1b ('1' = enable; '0' = disable) +0 # 4m1a ('1' = enable; '0' = disable) +1 # 1m ('1' = enable; '0' = disable) +1 # native ('1' = enable; '0' = disable) +1 # Simulate application-level threading: + # '1' = disable / use one testsuite thread; + # 'n' = enable and use n testsuite threads +1 # Error-checking level: + # '0' = disable error checking; '1' = full error checking +i # Reaction to test failure: + # 'i' = ignore; 's' = sleep() and continue; 'a' = abort +0 # Output results in matlab/octave format? ('1' = yes; '0' = no) +0 # Output results to stdout AND files? ('1' = yes; '0' = no) diff --git a/testsuite/input.operations.mixed b/testsuite/input.operations.mixed new file mode 100644 index 000000000..f99c6b871 --- /dev/null +++ b/testsuite/input.operations.mixed @@ -0,0 +1,318 @@ +# -------------------------------------------------------------------------- +# +# input.operations +# BLIS test suite +# +# This file contains input values that control which BLIS operations are +# tested as well as how those test runs are parameterized. We will now +# describe how each section or line type may be edited. +# +# ENABLING/DISABLING ENTIRE SECTIONS +# The values in the "Section overrides" section allow you to disable +# all operations in a given "level". Enabling a level here by itself +# does not enable every operation in that level; it simply means that +# the individual switches for each operation (in that level) determine +# whether or not the tests are executed. Use 1 to enable a section, or +# 0 to disable. +# +# ENABLING/DISABLING INDIVIDUAL OPERATION TESTS +# Given that an operation's section override switch is set to 1 +# (enabled), whether or not that operation will get tested is +# determined by its local switch. For example, if the level-1v section +# override is set to 1, and there is a 1 on the line marked "addv", +# then the addv operation will be tested. Similarly, a 0 would cause +# addv to not be tested. +# +# ENABLING ONLY SELECT OPERATIONS +# If you would like to enable just a few (or even just one) operation +# without adjusting any section overrides (or individual operation +# switches), change the desired operation switch(es) to 2. This will +# cause any operation that is not set to 2 to be disabled, regardless +# of section override values. For example, setting the axpyv and gemv +# operation switches to 2 will cause the test suite to test ONLY axpyv +# and gemv, even if all other sections and operations are set to 1. +# NOTE: As long as there is at least on operation switch set to 2, no +# other operations will be tested. When you are done testing your +# select operations, you should revert the operation switch(es) back +# to 1. +# +# CHANGING PROBLEM SIZE/SHAPES TESTED +# The problem sizes tested by an operation are determined by the +# dimension specifiers on the line marked "dimensions: ". +# If, for example, contains two dimension labels (e.g. +# "m n"), then the line should begin with two dimension specifiers. +# Dimension specifiers of -1 cause the corresponding dimension to be +# bound to the problem size, which is determined by values set in +# input.general. Positive values cause the corresponding dimension to +# be fixed to that value and held constant. +# +# Examples of dimension specifiers (where the dimensions are m and n): +# +# -1 -1 Dimensions m and n grow with problem size (resulting in +# square matrices). +# -1 150 Dimension m grows with problem size and n is fixed at +# 150. +# -1 -2 Dimension m grows with problem size and n grows +# proportional to half the problem size. +# +# CHANGING PARAMTER COMBINATIONS TESTED +# The parameter combinations tested by an operation are determined by +# the parameter specifier characters on the line marked "parameters: +# ". If, for example, contains two +# parameter labels (e.g. "transa conjx"), then the line should contain +# two parameter specifier characters. The '?' specifier character +# serves as a wildcard--it causes all possible values of that parameter +# to be tested. A character such as 'n' or 't' causes only that value +# to be tested. +# +# Examples of parameter specifiers (where the parameters are transa +# and conjx): +# +# ?? All combinations of the transa and conjx parameters are +# tested: nn, nc, tn, tc, cn, cc, hn, hc. +# ?n conjx is fixed to "no conjugate" but transa is allowed +# to vary: nn, tn, cn, hn. +# hc Only the case where transa is "Hermitian-transpose" and +# conjx is "conjugate" is tested. +# +# Here is a full list of the parameter types used by the various BLIS +# operations along with their possible character encodings: +# +# side: l,r left, right +# uplo: l,u lower, upper +# trans: n,t,c,h no transpose, transpose, conjugate, Hermitian- +# transpose (i.e. conjugate-transpose) +# conj: n,c no conjugate, conjugate +# diag: n,u non-unit diagonal, unit diagonal +# + +# --- Section overrides ---------------------------------------------------- + +1 # Utility +1 # Level-1v kernels +1 # Level-1m +1 # Level-1f kernels +1 # Level-2 +1 # Level-3 micro-kernels +1 # Level-3 + + +# --- Utility -------------------------------------------------------------- + +1 # randv +-1 # dimensions: m + +1 # randm +-1 -1 # dimensions: m n + + +# --- Level-1v ------------------------------------------------------------- + +1 # addv +-1 # dimensions: m +? # parameters: conjx + +1 # amaxv +-1 # dimensions: m + +1 # axpbyv +-1 # dimensions: m +? # parameters: conjx + +1 # axpyv +-1 # dimensions: m +? # parameters: conjx + +1 # copyv +-1 # dimensions: m +? # parameters: conjx + +1 # dotv +-1 # dimensions: m +?? # parameters: conjx conjy + +1 # dotxv +-1 # dimensions: m +?? # parameters: conjx conjy + +1 # normfv +-1 # dimensions: m + +1 # scalv +-1 # dimensions: m +? # parameters: conjbeta + +1 # scal2v +-1 # dimensions: m +? # parameters: conjx + +1 # setv +-1 # dimensions: m + +1 # subv +-1 # dimensions: m +? # parameters: conjx + +1 # xpbyv +-1 # dimensions: m +? # parameters: conjx + + +# --- Level-1m ------------------------------------------------------------- + +1 # addm +-1 -2 # dimensions: m n +? # parameters: transa + +1 # axpym +-1 -1 # dimensions: m n +? # parameters: transa + +1 # copym +-1 -2 # dimensions: m n +? # parameters: transa + +1 # normfm +-1 -2 # dimensions: m n + +1 # scalm +-1 -2 # dimensions: m n +? # parameters: conjbeta + +1 # scal2m +-1 -2 # dimensions: m n +? # parameters: transa + +1 # setm +-1 -2 # dimensions: m n + +1 # subm +-1 -2 # dimensions: m n +? # parameters: transa + +1 # xpbym +-1 -1 # dimensions: m n +? # parameters: transa + + +# --- Level-1f kernels ----------------------------------------------------- + +1 # axpy2v +-1 # dimensions: m +?? # parameters: conjx conjy + +1 # dotaxpyv +-1 # dimensions: m +??? # parameters: conjxt conjx conjy + +1 # axpyf +-1 # dimensions: m +?? # parameters: conja conjx + +1 # dotxf +-1 # dimensions: m +?? # parameters: conjat conjx + +1 # dotxaxpyf +-1 # dimensions: m +???? # parameters: conjat conja conjw conjx + + +# --- Level-2 -------------------------------------------------------------- + +1 # gemv +-1 -2 # dimensions: m n +?? # parameters: transa conjx + +1 # ger +-1 -2 # dimensions: m n +?? # parameters: conjx conjy + +1 # hemv +-1 # dimensions: m +??? # parameters: uploa conja conjx + +1 # her +-1 # dimensions: m +?? # parameters: uploc conjx + +1 # her2 +-1 # dimensions: m +??? # parameters: uploc conjx conjy + +1 # symv +-1 # dimensions: m +??? # parameters: uploa conja conjx + +1 # syr +-1 # dimensions: m +?? # parameters: uploc conjx + +1 # syr2 +-1 # dimensions: m +??? # parameters: uploc conjx conjy + +1 # trmv +-1 # dimensions: m +??? # parameters: uploa transa diaga + +1 # trsv +-1 # dimensions: m +??? # parameters: uploa transa diaga + + +# --- Level-3 micro-kernels ------------------------------------------------ + +1 # gemm +-1 # dimensions: k + +1 # trsm +? # parameters: uploa + +1 # gemmtrsm +-1 # dimensions: k +? # parameters: uploa + + +# --- Level-3 -------------------------------------------------------------- + +2 # gemm +-1 -1 -1 # dimensions: m n k +nn # parameters: transa transb + +1 # hemm +-1 -1 # dimensions: m n +???? # parameters: side uploa conja transb + +1 # herk +-1 -1 # dimensions: m k +?? # parameters: uploc transa + +1 # her2k +-1 -1 # dimensions: m k +??? # parameters: uploc transa transb + +1 # symm +-1 -1 # dimensions: m n +???? # parameters: side uploa conja transb + +1 # syrk +-1 -1 # dimensions: m k +?? # parameters: uploc transa + +1 # syr2k +-1 -1 # dimensions: m k +??? # parameters: uploc transa transb + +1 # trmm +-1 -1 # dimensions: m n +???? # parameters: side uploa transa diaga + +1 # trmm3 +-1 -1 # dimensions: m n +????n # parameters: side uploa transa diaga transb + +1 # trsm +-1 -1 # dimensions: m n +???? # parameters: side uploa transa diaga + diff --git a/travis/do_testsuite.sh b/travis/do_testsuite.sh index e5e5640c2..9a3b9ef14 100755 --- a/travis/do_testsuite.sh +++ b/travis/do_testsuite.sh @@ -8,9 +8,10 @@ export BLIS_JC_NT=1 export BLIS_IR_NT=1 export BLIS_JR_NT=1 -if [ "$TEST" = "FAST" ] -then +if [ "$TEST" = "FAST" ]; then make testblis-fast +elif [ "$TEST" = "MD" ]; then + make testblis-md else make testblis fi