Resolve issues with performance logs in CI. (#2733)

* update the performance test logic

* fix unstash perf logs logic

* untangle unstashing fmha logs for different archs

* run process stage after running fmha tests

* fix the processing of perf logs

* fix arguments for run_performance scripts

Jenkinsfile

@@ -578,31 +578,60 @@ def Build_CK(Map conf=[:]){
                         if (params.RUN_FULL_QA && arch == 1){
                             // run full tests on gfx90a
                             echo "Run full performance tests"
-                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
-                            archiveArtifacts "perf_gemm.log"
-                            archiveArtifacts "perf_resnet50_N256.log"
-                            archiveArtifacts "perf_resnet50_N4.log"
-                            archiveArtifacts "perf_batched_gemm.log"
-                            archiveArtifacts "perf_grouped_gemm.log"
-                            archiveArtifacts "perf_grouped_conv_fwd.log"
-                            archiveArtifacts "perf_grouped_conv_bwd_data.log"
-                            archiveArtifacts "perf_grouped_conv_bwd_weight.log"
-                            archiveArtifacts "perf_gemm_bilinear.log"
-                            archiveArtifacts "perf_reduction.log"
-                            archiveArtifacts "perf_splitK_gemm.log"
-                            archiveArtifacts "perf_onnx_gemm.log"
-                            archiveArtifacts "perf_mixed_gemm.log"
-                            stash includes: "perf_**.log", name: "perf_log"
+                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx90a"
+                            archiveArtifacts "perf_gemm_gfx90a.log"
+                            archiveArtifacts "perf_resnet50_N256_gfx90a.log"
+                            archiveArtifacts "perf_resnet50_N4_gfx90a.log"
+                            archiveArtifacts "perf_batched_gemm_gfx90a.log"
+                            archiveArtifacts "perf_grouped_gemm_gfx90a.log"
+                            archiveArtifacts "perf_grouped_conv_fwd_gfx90a.log"
+                            archiveArtifacts "perf_grouped_conv_bwd_data_gfx90a.log"
+                            archiveArtifacts "perf_grouped_conv_bwd_weight_gfx90a.log"
+                            archiveArtifacts "perf_gemm_bilinear_gfx90a.log"
+                            archiveArtifacts "perf_reduction_gfx90a.log"
+                            archiveArtifacts "perf_splitK_gemm_gfx90a.log"
+                            archiveArtifacts "perf_onnx_gemm_gfx90a.log"
+                            archiveArtifacts "perf_mixed_gemm_gfx90a.log"
+                            stash includes: "perf_**.log", name: "perf_log_gfx90a"
+                        }
+                        if (params.RUN_FULL_QA && arch == 2){
+                            // run full tests on gfx942
+                            echo "Run full performance tests"
+                            sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx942"
+                            archiveArtifacts "perf_gemm_gfx942.log"
+                            archiveArtifacts "perf_resnet50_N256_gfx942.log"
+                            archiveArtifacts "perf_resnet50_N4_gfx942.log"
+                            archiveArtifacts "perf_batched_gemm_gfx942.log"
+                            archiveArtifacts "perf_grouped_gemm_gfx942.log"
+                            archiveArtifacts "perf_grouped_conv_fwd_gfx942.log"
+                            archiveArtifacts "perf_grouped_conv_bwd_data_gfx942.log"
+                            archiveArtifacts "perf_grouped_conv_bwd_weight_gfx942.log"
+                            archiveArtifacts "perf_gemm_bilinear_gfx942.log"
+                            archiveArtifacts "perf_reduction_gfx942.log"
+                            archiveArtifacts "perf_splitK_gemm_gfx942.log"
+                            archiveArtifacts "perf_onnx_gemm_gfx942.log"
+                            archiveArtifacts "perf_mixed_gemm_gfx942.log"
+                            stash includes: "perf_**.log", name: "perf_log_gfx942"
                         }
                         else if ( arch == 1 ){
                             // run standard tests on gfx90a
                             echo "Run performance tests"
-                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}"
-                            archiveArtifacts "perf_gemm.log"
-                            archiveArtifacts "perf_onnx_gemm.log"
-                            archiveArtifacts "perf_resnet50_N256.log"
-                            archiveArtifacts "perf_resnet50_N4.log"
-                            stash includes: "perf_**.log", name: "perf_log"
+                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx90a"
+                            archiveArtifacts "perf_gemm_gfx90a.log"
+                            archiveArtifacts "perf_onnx_gemm_gfx90a.log"
+                            archiveArtifacts "perf_resnet50_N256_gfx90a.log"
+                            archiveArtifacts "perf_resnet50_N4_gfx90a.log"
+                            stash includes: "perf_**.log", name: "perf_log_gfx90a"
+                        }
+                        else if ( arch == 2 ){
+                            // run standard tests on gfx942
+                            echo "Run performance tests"
+                            sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx942"
+                            archiveArtifacts "perf_gemm_gfx942.log"
+                            archiveArtifacts "perf_onnx_gemm_gfx942.log"
+                            archiveArtifacts "perf_resnet50_N256_gfx942.log"
+                            archiveArtifacts "perf_resnet50_N4_gfx942.log"
+                            stash includes: "perf_**.log", name: "perf_log_gfx942"
                         }
                         // disable performance tests on gfx1030 for now.
                         //else if ( arch == 3){
@@ -720,10 +749,15 @@ def process_results(Map conf=[:]){
                     if (params.RUN_CK_TILE_FMHA_TESTS){
                         try{
                             unstash "perf_fmha_log_gfx942"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the FMHA performance logs for gfx942: ${err.getMessage()}."
+                        }
+                        try{
                             unstash "perf_fmha_log_gfx90a"
                         }
                         catch(Exception err){
-                            echo "could not locate the FMHA performance logs: ${err.getMessage()}."
+                            echo "could not locate the FMHA performance logs for gfx90a: ${err.getMessage()}."
                         }
                     }
                     if (params.BUILD_INSTANCES_ONLY){
@@ -733,16 +767,46 @@ def process_results(Map conf=[:]){
                     }
                     else{
                         // unstash perf files to master
-                        unstash "perf_log"
+                        try{
+                            unstash "perf_log_gfx90a"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the gfx90a performance logs: ${err.getMessage()}."
+                        }
+                        try{
+                            unstash "perf_log_gfx942"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the gfx942 performance logs: ${err.getMessage()}."
+                        }
+                        try{
+                            unstash "perf_log_gfx950"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the gfx950 performance logs: ${err.getMessage()}."
+                        }
+                        try{
+                            unstash "perf_log_gfx908"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the gfx908 performance logs: ${err.getMessage()}."
+                        }
                         try{
                             unstash "perf_log_gfx11"
+                        }
+                        catch(Exception err){
+                            echo "could not locate the gfx11 performance logs: ${err.getMessage()}."
+                        }
+                        try{
+
                             unstash "perf_log_gfx12"
                         }
                         catch(Exception err){
-                            echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}."
+                            echo "could not locate the gfx12 performance logs: ${err.getMessage()}."
                         }
-                        sh "./process_perf_data.sh"
                     }
+                    // process the logs
+                    sh "./process_perf_data.sh"
                 }
             }
             catch(e){
@@ -1505,7 +1569,7 @@ pipeline {
                 stage("Process results"){
                     when {
                         beforeAgent true
-                        expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() }
+                        expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean() || params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() }
                     }
                     agent { label 'mici' }
                     steps{

script/process_perf_data.sh

@@ -10,10 +10,39 @@
 # please contact Illia.Silin@amd.com for more details
 
 #process results
-python3 process_perf_data.py perf_gemm.log
-python3 process_perf_data.py perf_onnx_gemm.log
-python3 process_perf_data.py perf_resnet50_N256.log
-python3 process_perf_data.py perf_resnet50_N4.log
+file=./perf_gemm_gfx90a.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_gemm_gfx90a.log
+fi
+file=./perf_onnx_gemm_gfx90a.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx90a.log
+fi
+file=./perf_resnet50_N256_gfx90a.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_resnet50_N256_gfx90a.log
+fi
+file=./perf_resnet50_N4_gfx90a.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_resnet50_N4_gfx90a.log
+fi
+
+file=./perf_gemm_gfx942.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_gemm_gfx942.log
+fi
+file=./perf_onnx_gemm_gfx942.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_onnx_gemm_gfx942.log
+fi
+file=./perf_resnet50_N256_gfx942.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_resnet50_N256_gfx942.log
+fi
+file=./perf_resnet50_N4_gfx942.log
+if [ -e "$file" ]; then
+    python3 process_perf_data.py perf_resnet50_N4_gfx942.log
+fi
 
 file=./perf_onnx_gemm_gfx10.log
 if [ -e "$file" ]; then

script/run_full_performance_tests.sh

@@ -22,6 +22,9 @@ export branch=$3
 echo 'Branch name: ' $branch
 export host_name=$4
 echo 'Host name: ' $host_name
+export arch=$5
+echo 'GPU architecture: ' $arch
+
 function print_log_header(){
 	rm -f $1;
 	echo 'On branch ' $3 &> $1;
@@ -35,7 +38,7 @@ function print_log_header(){
 }
 
 #run gemm tests
-export gemm_log="perf_gemm.log"
+export gemm_log="perf_gemm_$arch.log"
 print_log_header $gemm_log $env_type $branch $host_name
 ./profile_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $gemm_log
 ./profile_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $gemm_log
@@ -55,7 +58,7 @@ print_log_header $gemm_log $env_type $branch $host_name
 ./profile_gemm.sh gemm 3 3 $verify 1 0 1 2>&1 | tee -a $gemm_log
 
 #run batched_gemm tests
-export batched_gemm_log="perf_batched_gemm.log"
+export batched_gemm_log="perf_batched_gemm_$arch.log"
 print_log_header $batched_gemm_log $env_type $branch $host_name
 ./profile_batched_gemm.sh batched_gemm 0 0 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log
 ./profile_batched_gemm.sh batched_gemm 0 1 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log
@@ -75,7 +78,7 @@ print_log_header $batched_gemm_log $env_type $branch $host_name
 ./profile_batched_gemm.sh batched_gemm 3 3 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log
 
 #run grouped_gemm tests
-export grouped_gemm_log="perf_grouped_gemm.log"
+export grouped_gemm_log="perf_grouped_gemm_$arch.log"
 print_log_header $grouped_gemm_log $env_type $branch $host_name
 ./profile_grouped_gemm.sh grouped_gemm 1 0 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log
 ./profile_grouped_gemm.sh grouped_gemm 1 1 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log
@@ -83,7 +86,7 @@ print_log_header $grouped_gemm_log $env_type $branch $host_name
 ./profile_grouped_gemm.sh grouped_gemm 1 3 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log
 
 #run GEMM+Bilinear tests
-export gemm_bilinear_log="perf_gemm_bilinear.log"
+export gemm_bilinear_log="perf_gemm_bilinear_$arch.log"
 print_log_header $gemm_bilinear_log $env_type $branch $host_name
 ./profile_gemm_bilinear.sh gemm_bilinear 1 0 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log
 ./profile_gemm_bilinear.sh gemm_bilinear 1 1 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log
@@ -91,21 +94,21 @@ print_log_header $gemm_bilinear_log $env_type $branch $host_name
 ./profile_gemm_bilinear.sh gemm_bilinear 1 3 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log
 
 #run grouped_fwd tests
-export grouped_conv_fwd_log="perf_grouped_conv_fwd.log"
+export grouped_conv_fwd_log="perf_grouped_conv_fwd_$arch.log"
 print_log_header $grouped_conv_fwd_log $env_type $branch $host_name
 ./profile_grouped_conv_fwd.sh grouped_conv_fwd 0 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log
 ./profile_grouped_conv_fwd.sh grouped_conv_fwd 1 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log
 ./profile_grouped_conv_fwd.sh grouped_conv_fwd 2 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log
 
 #run grouped_bwd_data tests
-export grouped_conv_bwd_data_log="perf_grouped_conv_bwd_data.log"
+export grouped_conv_bwd_data_log="perf_grouped_conv_bwd_data_$arch.log"
 print_log_header $grouped_conv_bwd_data_log $env_type $branch $host_name
 ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 0 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log
 ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 1 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log
 ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 2 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log
 
 #run grouped_bwd_weight tests
-export grouped_conv_bwd_weight_log="perf_grouped_conv_bwd_weight.log"
+export grouped_conv_bwd_weight_log="perf_grouped_conv_bwd_weight_$arch.log"
 print_log_header $grouped_conv_bwd_weight_log $env_type $branch $host_name
 ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 0 2 $verify 1 0 1 256 1 2>&1 | tee -a $grouped_conv_bwd_weight_log
 ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 1 2 $verify 1 0 1 256 1 2>&1 | tee -a $grouped_conv_bwd_weight_log
@@ -113,21 +116,21 @@ print_log_header $grouped_conv_bwd_weight_log $env_type $branch $host_name
 ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 1 2 $verify 1 0 1 256 4 2>&1 | tee -a $grouped_conv_bwd_weight_log
 
 #run resnet50 tests
-export resnet256_log="perf_resnet50_N256.log"
+export resnet256_log="perf_resnet50_N256_$arch.log"
 print_log_header $resnet256_log $env_type $branch $host_name
 ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 256 2>&1 | tee -a $resnet256_log
-export resnet4_log="perf_resnet50_N4.log"
+export resnet4_log="perf_resnet50_N4_$arch.log"
 print_log_header $resnet4_log $env_type $branch $host_name
 ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 4 2>&1 | tee -a $resnet4_log
 
 #run reduction tests
-export reduction_log="perf_reduction.log"
+export reduction_log="perf_reduction_$arch.log"
 print_log_header $reduction_log $env_type $branch $host_name
 ./profile_reduce_with_index.sh $verify 2 10 --half 2>&1 | tee -a $reduction_log
 ./profile_reduce_no_index.sh $verify 2 10 --half 2>&1 | tee -a $reduction_log
 
 #run splitK_gemm tests, first correctness verification, then performance
-export splitK_gemm_log="perf_splitK_gemm.log"
+export splitK_gemm_log="perf_splitK_gemm_$arch.log"
 print_log_header $splitK_gemm_log $env_type $branch $host_name
 ./profile_splitK_gemm.sh gemm_splitk 0 0 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log
 ./profile_splitK_gemm.sh gemm_splitk 0 1 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log
@@ -139,13 +142,13 @@ print_log_header $splitK_gemm_log $env_type $branch $host_name
 ./profile_splitK_gemm.sh gemm_splitk 1 3 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log
 
 #run ONNX gemm tests
-export onnx_log="perf_onnx_gemm.log"
+export onnx_log="perf_onnx_gemm_$arch.log"
 print_log_header $onnx_log $env_type $branch $host_name
 ./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
 ./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
 
 #run mixed fp16/fp8 and fp8/fp16 gemm tests
-export mixed_gemm_log="perf_mixed_gemm.log"
+export mixed_gemm_log="perf_mixed_gemm_$arch.log"
 print_log_header $mixed_gemm_log $env_type $branch $host_name
 ./profile_mixed_gemm.sh gemm_splitk 4 0 $verify 2 0 1 16 2>&1 | tee -a $mixed_gemm_log
 ./profile_mixed_gemm.sh gemm_splitk 5 0 $verify 2 0 1 16 2>&1 | tee -a $mixed_gemm_log

script/run_performance_tests.sh

@@ -18,6 +18,8 @@ export branch=$3
 echo 'Branch name: ' $branch
 export host_name=$4
 echo 'Host name: ' $host_name
+export arch=$5
+echo 'GPU architecture: ' $arch
 
 function print_log_header(){
 	rm -f $1;
@@ -32,7 +34,7 @@ function print_log_header(){
 }
 
 #run gemm tests
-export gemm_log="perf_gemm.log"
+export gemm_log="perf_gemm_$arch.log"
 print_log_header $gemm_log $env_type $branch $host_name
 ./profile_gemm.sh gemm 0 0 $verify 1 0 1 | tee -a $gemm_log
 ./profile_gemm.sh gemm 1 0 $verify 1 0 1 | tee -a $gemm_log
@@ -52,15 +54,15 @@ print_log_header $gemm_log $env_type $branch $host_name
 ./profile_gemm.sh gemm 3 3 $verify 1 0 1 | tee -a $gemm_log
 
 #run ONNX gemm tests
-export onnx_log="perf_onnx_gemm.log"
+export onnx_log="perf_onnx_gemm_$arch.log"
 print_log_header $onnx_log $env_type $branch $host_name
 ./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
 ./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log
 
 #run resnet50 tests
-export resnet256_log="perf_resnet50_N256.log"
+export resnet256_log="perf_resnet50_N256_$arch.log"
 print_log_header $resnet256_log $env_type $branch $host_name
 ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 256 | tee -a $resnet256_log
-export resnet4_log="perf_resnet50_N4.log"
+export resnet4_log="perf_resnet50_N4_$arch.log"
 print_log_header $resnet4_log $env_type $branch $host_name
 ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 4 | tee -a $resnet4_log