Refactor CI performance tests. (#1726)

* merge the build and performance tests CI stages together

* add gemm performance test on gfx11/gfx12

* add suffices to distinguish gemm performance logs from different archs

* use smaller gemm set in CI for gfx10/gfx11/gfx12

* disable performance tests on gfx1030

* fix the shashing logic

* fix finding python3 for mha instances

script/process_perf_data.py

@@ -82,7 +82,7 @@ def parse_logfile(logfile):
     StrideA=[]
     StrideB=[]
     StrideC=[]
-    if 'perf_gemm.log' in logfile:
+    if 'perf_gemm' in logfile and 'gemm_bilinear' not in logfile:
         for line in open(logfile):
             if 'Best Perf' in line:
                 lst=line.split()
@@ -260,7 +260,7 @@ def main():
         conn = sqlEngine.connect()
 
         #save gemm performance tests:
-        if 'perf_gemm.log' in filename:
+        if 'perf_gemm' in filename and 'gemm_bilinear' not in filename:
             #write the ck_gemm_test_params table only needed once the test set changes
             #post_test_params(test_list,conn)
             for i in range(1,len(results)+1):

script/process_perf_data.sh

@@ -11,9 +11,22 @@
 
 #process results
 python3 process_perf_data.py perf_gemm.log
+python3 process_perf_data.py perf_onnx_gemm.log
 python3 process_perf_data.py perf_resnet50_N256.log
 python3 process_perf_data.py perf_resnet50_N4.log
 
+file=./perf_onnx_gemm_gfx10.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx10.log
+fi
+file=./perf_onnx_gemm_gfx11.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx11.log
+fi
+file=./perf_onnx_gemm_gfx12.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx12.log
+fi
 file=./perf_fmha_fwd_gfx942.log
 if [ -e "$file" ]; then
     python3 process_perf_data.py perf_fmha_fwd_gfx942.log

script/process_qa_data.sh

@@ -24,6 +24,18 @@ python3 process_perf_data.py perf_splitK_gemm.log
 python3 process_perf_data.py perf_onnx_gemm.log
 python3 process_perf_data.py perf_mixed_gemm.log
 
+file=./perf_onnx_gemm_gfx10.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx10.log
+fi
+file=./perf_onnx_gemm_gfx11.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx11.log
+fi
+file=./perf_onnx_gemm_gfx12.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx12.log
+fi
 file=./perf_fmha_fwd_gfx942.log
 if [ -e "$file" ]; then
     python3 process_perf_data.py perf_fmha_fwd_gfx942.log

script/run_full_performance_tests.sh

@@ -5,7 +5,7 @@
 # post your new test results to the database and compare them to the baseline
 # please contact Illia.Silin@amd.com for more details
 #
-# run the script as "./run_full_performance_tests.sh <verification> <tag for your test environment> <branch name> < node name>
+# run the script as "./run_full_performance_tests.sh <verification> <tag for your test environment> <branch name> <node name>
 # input arguments: 
 # verification = 0 : do not verify result correctness on CPU
 #              = 1 : verifuy correctness on CPU (may take a long time)

script/run_gemm_performance_tests.sh

@@ -0,0 +1,41 @@
+#!/bin/bash 
+#
+# in order to run this script you'd first need to build the ckProfiler executable in ../build/bin/
+# run the script as "./run_gemm_performance_tests.sh <verification> <tag for your test environment> <branch name> <node name> <arch>
+# input arguments: 
+# verification = 0 : do not verify result correctness on CPU
+#              = 1 : verify correctness on CPU (may take a long time)
+# environment tag  : a string describing the specifics of your test environment
+# branch name      : name of the branch in git repo (git status | grep -e 'On branch')
+# node name        : $hostname
+# arch             : GPU architecture, e.g. "gfx9" or "gfx1100"
+
+#get the command line arguments:
+export verify=$1
+echo 'Verification: ' $verify
+export env_type=$2
+echo 'Environment type: ' $env_type
+export branch=$3
+echo 'Branch name: ' $branch
+export host_name=$4
+echo 'Host name: ' $host_name
+export arch=$5
+echo 'GPU architecture: ' $arch
+
+function print_log_header(){
+	rm -f $1;
+	echo 'On branch ' $3 &> $1;
+	echo 'Node name: ' $4 >> $1;
+	#get GPU_arch and number of compute units from rocminfo
+	echo -n "GPU_arch: " >> $1; rocminfo | grep "Name:" | grep "gfx" >> $1;
+	rocminfo | grep "Compute Unit:" >> $1;
+	hipcc --version | grep -e 'HIP version'  >> $1;
+	echo 'Environment type: ' $2 >> $1;
+	/opt/rocm/bin/amdclang++ --version | grep -e 'InstalledDir' >> $1;
+}
+
+#run ONNX gemm tests
+export onnx_log="perf_onnx_gemm_$arch.log"
+print_log_header $onnx_log $env_type $branch $host_name
+./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
+./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log

script/run_performance_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 #
 # in order to run this script you'd first need to build the ckProfiler executable in ../build/bin/
-# run the script as "./run_performance_tests.sh <verification> <tag for your test environment> <branch name> < node name>
+# run the script as "./run_performance_tests.sh <verification> <tag for your test environment> <branch name> <node name>
 # input arguments: 
 # verification = 0 : do not verify result correctness on CPU
 #              = 1 : verify correctness on CPU (may take a long time)
@@ -51,20 +51,11 @@ print_log_header $gemm_log $env_type $branch $host_name
 ./profile_gemm.sh gemm 2 3 $verify 1 0 1 | tee -a $gemm_log
 ./profile_gemm.sh gemm 3 3 $verify 1 0 1 | tee -a $gemm_log
 
-#run grouped_fwd fp16 tests
-export grouped_conv_fwd_log="perf_grouped_conv_fwd_fp16.log"
-print_log_header $conv_fwd_log $env_type $branch $host_name
-./profile_grouped_conv_fwd.sh grouped_conv_fwd 1 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log
-
-#run grouped_bwd_data fp16 tests
-export grouped_conv_bwd_data_log="perf_grouped_conv_bwd_data_fp16.log"
-print_log_header $grouped_conv_bwd_data_log $env_type $branch $host_name
-./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 1 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log
-
-#run grouped_bwd_weight fp16 tests
-export grouped_conv_bwd_weight_log="perf_grouped_conv_bwd_weight_fp16.log"
-print_log_header $grouped_conv_bwd_weight_log $env_type $branch $host_name
-./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 1 1 $verify 1 0 1 256 1 2>&1 | tee -a $grouped_conv_bwd_weight_log
+#run ONNX gemm tests
+export onnx_log="perf_onnx_gemm.log"
+print_log_header $onnx_log $env_type $branch $host_name
+./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
+./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
 
 #run resnet50 tests
 export resnet256_log="perf_resnet50_N256.log"