diff --git a/Jenkinsfile b/Jenkinsfile index b3b63098c2..6c79acb14b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -578,31 +578,60 @@ def Build_CK(Map conf=[:]){ if (params.RUN_FULL_QA && arch == 1){ // run full tests on gfx90a echo "Run full performance tests" - sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}" - archiveArtifacts "perf_gemm.log" - archiveArtifacts "perf_resnet50_N256.log" - archiveArtifacts "perf_resnet50_N4.log" - archiveArtifacts "perf_batched_gemm.log" - archiveArtifacts "perf_grouped_gemm.log" - archiveArtifacts "perf_grouped_conv_fwd.log" - archiveArtifacts "perf_grouped_conv_bwd_data.log" - archiveArtifacts "perf_grouped_conv_bwd_weight.log" - archiveArtifacts "perf_gemm_bilinear.log" - archiveArtifacts "perf_reduction.log" - archiveArtifacts "perf_splitK_gemm.log" - archiveArtifacts "perf_onnx_gemm.log" - archiveArtifacts "perf_mixed_gemm.log" - stash includes: "perf_**.log", name: "perf_log" + sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx90a" + archiveArtifacts "perf_gemm_gfx90a.log" + archiveArtifacts "perf_resnet50_N256_gfx90a.log" + archiveArtifacts "perf_resnet50_N4_gfx90a.log" + archiveArtifacts "perf_batched_gemm_gfx90a.log" + archiveArtifacts "perf_grouped_gemm_gfx90a.log" + archiveArtifacts "perf_grouped_conv_fwd_gfx90a.log" + archiveArtifacts "perf_grouped_conv_bwd_data_gfx90a.log" + archiveArtifacts "perf_grouped_conv_bwd_weight_gfx90a.log" + archiveArtifacts "perf_gemm_bilinear_gfx90a.log" + archiveArtifacts "perf_reduction_gfx90a.log" + archiveArtifacts "perf_splitK_gemm_gfx90a.log" + archiveArtifacts "perf_onnx_gemm_gfx90a.log" + archiveArtifacts "perf_mixed_gemm_gfx90a.log" + stash includes: "perf_**.log", name: "perf_log_gfx90a" + } + if (params.RUN_FULL_QA && arch == 2){ + // run full tests on gfx942 + echo "Run full performance tests" + sh "./run_full_performance_tests.sh 0 QA_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx942" + archiveArtifacts "perf_gemm_gfx942.log" + archiveArtifacts "perf_resnet50_N256_gfx942.log" + archiveArtifacts "perf_resnet50_N4_gfx942.log" + archiveArtifacts "perf_batched_gemm_gfx942.log" + archiveArtifacts "perf_grouped_gemm_gfx942.log" + archiveArtifacts "perf_grouped_conv_fwd_gfx942.log" + archiveArtifacts "perf_grouped_conv_bwd_data_gfx942.log" + archiveArtifacts "perf_grouped_conv_bwd_weight_gfx942.log" + archiveArtifacts "perf_gemm_bilinear_gfx942.log" + archiveArtifacts "perf_reduction_gfx942.log" + archiveArtifacts "perf_splitK_gemm_gfx942.log" + archiveArtifacts "perf_onnx_gemm_gfx942.log" + archiveArtifacts "perf_mixed_gemm_gfx942.log" + stash includes: "perf_**.log", name: "perf_log_gfx942" } else if ( arch == 1 ){ // run standard tests on gfx90a echo "Run performance tests" - sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME}" - archiveArtifacts "perf_gemm.log" - archiveArtifacts "perf_onnx_gemm.log" - archiveArtifacts "perf_resnet50_N256.log" - archiveArtifacts "perf_resnet50_N4.log" - stash includes: "perf_**.log", name: "perf_log" + sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx90a" + archiveArtifacts "perf_gemm_gfx90a.log" + archiveArtifacts "perf_onnx_gemm_gfx90a.log" + archiveArtifacts "perf_resnet50_N256_gfx90a.log" + archiveArtifacts "perf_resnet50_N4_gfx90a.log" + stash includes: "perf_**.log", name: "perf_log_gfx90a" + } + else if ( arch == 2 ){ + // run standard tests on gfx942 + echo "Run performance tests" + sh "./run_performance_tests.sh 0 CI_${params.COMPILER_VERSION} ${env.BRANCH_NAME} ${NODE_NAME} gfx942" + archiveArtifacts "perf_gemm_gfx942.log" + archiveArtifacts "perf_onnx_gemm_gfx942.log" + archiveArtifacts "perf_resnet50_N256_gfx942.log" + archiveArtifacts "perf_resnet50_N4_gfx942.log" + stash includes: "perf_**.log", name: "perf_log_gfx942" } // disable performance tests on gfx1030 for now. //else if ( arch == 3){ @@ -720,10 +749,15 @@ def process_results(Map conf=[:]){ if (params.RUN_CK_TILE_FMHA_TESTS){ try{ unstash "perf_fmha_log_gfx942" + } + catch(Exception err){ + echo "could not locate the FMHA performance logs for gfx942: ${err.getMessage()}." + } + try{ unstash "perf_fmha_log_gfx90a" } catch(Exception err){ - echo "could not locate the FMHA performance logs: ${err.getMessage()}." + echo "could not locate the FMHA performance logs for gfx90a: ${err.getMessage()}." } } if (params.BUILD_INSTANCES_ONLY){ @@ -733,16 +767,46 @@ def process_results(Map conf=[:]){ } else{ // unstash perf files to master - unstash "perf_log" + try{ + unstash "perf_log_gfx90a" + } + catch(Exception err){ + echo "could not locate the gfx90a performance logs: ${err.getMessage()}." + } + try{ + unstash "perf_log_gfx942" + } + catch(Exception err){ + echo "could not locate the gfx942 performance logs: ${err.getMessage()}." + } + try{ + unstash "perf_log_gfx950" + } + catch(Exception err){ + echo "could not locate the gfx950 performance logs: ${err.getMessage()}." + } + try{ + unstash "perf_log_gfx908" + } + catch(Exception err){ + echo "could not locate the gfx908 performance logs: ${err.getMessage()}." + } try{ unstash "perf_log_gfx11" + } + catch(Exception err){ + echo "could not locate the gfx11 performance logs: ${err.getMessage()}." + } + try{ + unstash "perf_log_gfx12" } catch(Exception err){ - echo "could not locate the GEMM gfx11/gfx12 performance logs: ${err.getMessage()}." + echo "could not locate the gfx12 performance logs: ${err.getMessage()}." } - sh "./process_perf_data.sh" } + // process the logs + sh "./process_perf_data.sh" } } catch(e){ @@ -1505,7 +1569,7 @@ pipeline { stage("Process results"){ when { beforeAgent true - expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() } + expression { (params.RUN_PERFORMANCE_TESTS.toBoolean() || params.BUILD_INSTANCES_ONLY.toBoolean() || params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) && !params.BUILD_LEGACY_OS.toBoolean() } } agent { label 'mici' } steps{ diff --git a/script/process_perf_data.sh b/script/process_perf_data.sh index fc44064874..50c84924f5 100755 --- a/script/process_perf_data.sh +++ b/script/process_perf_data.sh @@ -10,10 +10,39 @@ # please contact Illia.Silin@amd.com for more details #process results -python3 process_perf_data.py perf_gemm.log -python3 process_perf_data.py perf_onnx_gemm.log -python3 process_perf_data.py perf_resnet50_N256.log -python3 process_perf_data.py perf_resnet50_N4.log +file=./perf_gemm_gfx90a.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_gemm_gfx90a.log +fi +file=./perf_onnx_gemm_gfx90a.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_onnx_gemm_gfx90a.log +fi +file=./perf_resnet50_N256_gfx90a.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_resnet50_N256_gfx90a.log +fi +file=./perf_resnet50_N4_gfx90a.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_resnet50_N4_gfx90a.log +fi + +file=./perf_gemm_gfx942.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_gemm_gfx942.log +fi +file=./perf_onnx_gemm_gfx942.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_onnx_gemm_gfx942.log +fi +file=./perf_resnet50_N256_gfx942.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_resnet50_N256_gfx942.log +fi +file=./perf_resnet50_N4_gfx942.log +if [ -e "$file" ]; then + python3 process_perf_data.py perf_resnet50_N4_gfx942.log +fi file=./perf_onnx_gemm_gfx10.log if [ -e "$file" ]; then diff --git a/script/run_full_performance_tests.sh b/script/run_full_performance_tests.sh index ddc5c270b8..508200b21a 100755 --- a/script/run_full_performance_tests.sh +++ b/script/run_full_performance_tests.sh @@ -22,6 +22,9 @@ export branch=$3 echo 'Branch name: ' $branch export host_name=$4 echo 'Host name: ' $host_name +export arch=$5 +echo 'GPU architecture: ' $arch + function print_log_header(){ rm -f $1; echo 'On branch ' $3 &> $1; @@ -35,7 +38,7 @@ function print_log_header(){ } #run gemm tests -export gemm_log="perf_gemm.log" +export gemm_log="perf_gemm_$arch.log" print_log_header $gemm_log $env_type $branch $host_name ./profile_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $gemm_log ./profile_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $gemm_log @@ -55,7 +58,7 @@ print_log_header $gemm_log $env_type $branch $host_name ./profile_gemm.sh gemm 3 3 $verify 1 0 1 2>&1 | tee -a $gemm_log #run batched_gemm tests -export batched_gemm_log="perf_batched_gemm.log" +export batched_gemm_log="perf_batched_gemm_$arch.log" print_log_header $batched_gemm_log $env_type $branch $host_name ./profile_batched_gemm.sh batched_gemm 0 0 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log ./profile_batched_gemm.sh batched_gemm 0 1 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log @@ -75,7 +78,7 @@ print_log_header $batched_gemm_log $env_type $branch $host_name ./profile_batched_gemm.sh batched_gemm 3 3 $verify 1 0 1 2>&1 | tee -a $batched_gemm_log #run grouped_gemm tests -export grouped_gemm_log="perf_grouped_gemm.log" +export grouped_gemm_log="perf_grouped_gemm_$arch.log" print_log_header $grouped_gemm_log $env_type $branch $host_name ./profile_grouped_gemm.sh grouped_gemm 1 0 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log ./profile_grouped_gemm.sh grouped_gemm 1 1 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log @@ -83,7 +86,7 @@ print_log_header $grouped_gemm_log $env_type $branch $host_name ./profile_grouped_gemm.sh grouped_gemm 1 3 $verify 1 0 1 2>&1 | tee -a $grouped_gemm_log #run GEMM+Bilinear tests -export gemm_bilinear_log="perf_gemm_bilinear.log" +export gemm_bilinear_log="perf_gemm_bilinear_$arch.log" print_log_header $gemm_bilinear_log $env_type $branch $host_name ./profile_gemm_bilinear.sh gemm_bilinear 1 0 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log ./profile_gemm_bilinear.sh gemm_bilinear 1 1 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log @@ -91,21 +94,21 @@ print_log_header $gemm_bilinear_log $env_type $branch $host_name ./profile_gemm_bilinear.sh gemm_bilinear 1 3 $verify 1 0 1 2>&1 | tee -a $gemm_bilinear_log #run grouped_fwd tests -export grouped_conv_fwd_log="perf_grouped_conv_fwd.log" +export grouped_conv_fwd_log="perf_grouped_conv_fwd_$arch.log" print_log_header $grouped_conv_fwd_log $env_type $branch $host_name ./profile_grouped_conv_fwd.sh grouped_conv_fwd 0 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log ./profile_grouped_conv_fwd.sh grouped_conv_fwd 1 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log ./profile_grouped_conv_fwd.sh grouped_conv_fwd 2 1 0 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_fwd_log #run grouped_bwd_data tests -export grouped_conv_bwd_data_log="perf_grouped_conv_bwd_data.log" +export grouped_conv_bwd_data_log="perf_grouped_conv_bwd_data_$arch.log" print_log_header $grouped_conv_bwd_data_log $env_type $branch $host_name ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 0 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 1 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log ./profile_grouped_conv_bwd_data.sh grouped_conv_bwd_data 2 1 $verify 1 0 1 256 2>&1 | tee -a $grouped_conv_bwd_data_log #run grouped_bwd_weight tests -export grouped_conv_bwd_weight_log="perf_grouped_conv_bwd_weight.log" +export grouped_conv_bwd_weight_log="perf_grouped_conv_bwd_weight_$arch.log" print_log_header $grouped_conv_bwd_weight_log $env_type $branch $host_name ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 0 2 $verify 1 0 1 256 1 2>&1 | tee -a $grouped_conv_bwd_weight_log ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 1 2 $verify 1 0 1 256 1 2>&1 | tee -a $grouped_conv_bwd_weight_log @@ -113,21 +116,21 @@ print_log_header $grouped_conv_bwd_weight_log $env_type $branch $host_name ./profile_grouped_conv_bwd_weight.sh grouped_conv_bwd_weight 1 2 $verify 1 0 1 256 4 2>&1 | tee -a $grouped_conv_bwd_weight_log #run resnet50 tests -export resnet256_log="perf_resnet50_N256.log" +export resnet256_log="perf_resnet50_N256_$arch.log" print_log_header $resnet256_log $env_type $branch $host_name ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 256 2>&1 | tee -a $resnet256_log -export resnet4_log="perf_resnet50_N4.log" +export resnet4_log="perf_resnet50_N4_$arch.log" print_log_header $resnet4_log $env_type $branch $host_name ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 4 2>&1 | tee -a $resnet4_log #run reduction tests -export reduction_log="perf_reduction.log" +export reduction_log="perf_reduction_$arch.log" print_log_header $reduction_log $env_type $branch $host_name ./profile_reduce_with_index.sh $verify 2 10 --half 2>&1 | tee -a $reduction_log ./profile_reduce_no_index.sh $verify 2 10 --half 2>&1 | tee -a $reduction_log #run splitK_gemm tests, first correctness verification, then performance -export splitK_gemm_log="perf_splitK_gemm.log" +export splitK_gemm_log="perf_splitK_gemm_$arch.log" print_log_header $splitK_gemm_log $env_type $branch $host_name ./profile_splitK_gemm.sh gemm_splitk 0 0 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log ./profile_splitK_gemm.sh gemm_splitk 0 1 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log @@ -139,13 +142,13 @@ print_log_header $splitK_gemm_log $env_type $branch $host_name ./profile_splitK_gemm.sh gemm_splitk 1 3 $verify 1 0 1 4 2>&1 | tee -a $splitK_gemm_log #run ONNX gemm tests -export onnx_log="perf_onnx_gemm.log" +export onnx_log="perf_onnx_gemm_$arch.log" print_log_header $onnx_log $env_type $branch $host_name ./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log ./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log #run mixed fp16/fp8 and fp8/fp16 gemm tests -export mixed_gemm_log="perf_mixed_gemm.log" +export mixed_gemm_log="perf_mixed_gemm_$arch.log" print_log_header $mixed_gemm_log $env_type $branch $host_name ./profile_mixed_gemm.sh gemm_splitk 4 0 $verify 2 0 1 16 2>&1 | tee -a $mixed_gemm_log ./profile_mixed_gemm.sh gemm_splitk 5 0 $verify 2 0 1 16 2>&1 | tee -a $mixed_gemm_log \ No newline at end of file diff --git a/script/run_performance_tests.sh b/script/run_performance_tests.sh index c8a281dc07..4e13b59d34 100755 --- a/script/run_performance_tests.sh +++ b/script/run_performance_tests.sh @@ -18,6 +18,8 @@ export branch=$3 echo 'Branch name: ' $branch export host_name=$4 echo 'Host name: ' $host_name +export arch=$5 +echo 'GPU architecture: ' $arch function print_log_header(){ rm -f $1; @@ -32,7 +34,7 @@ function print_log_header(){ } #run gemm tests -export gemm_log="perf_gemm.log" +export gemm_log="perf_gemm_$arch.log" print_log_header $gemm_log $env_type $branch $host_name ./profile_gemm.sh gemm 0 0 $verify 1 0 1 | tee -a $gemm_log ./profile_gemm.sh gemm 1 0 $verify 1 0 1 | tee -a $gemm_log @@ -52,15 +54,15 @@ print_log_header $gemm_log $env_type $branch $host_name ./profile_gemm.sh gemm 3 3 $verify 1 0 1 | tee -a $gemm_log #run ONNX gemm tests -export onnx_log="perf_onnx_gemm.log" +export onnx_log="perf_onnx_gemm_$arch.log" print_log_header $onnx_log $env_type $branch $host_name ./profile_onnx_gemm.sh gemm 0 0 $verify 1 0 1 2>&1 | tee -a $onnx_log ./profile_onnx_gemm.sh gemm 1 0 $verify 1 0 1 2>&1 | tee -a $onnx_log #run resnet50 tests -export resnet256_log="perf_resnet50_N256.log" +export resnet256_log="perf_resnet50_N256_$arch.log" print_log_header $resnet256_log $env_type $branch $host_name ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 256 | tee -a $resnet256_log -export resnet4_log="perf_resnet50_N4.log" +export resnet4_log="perf_resnet50_N4_$arch.log" print_log_header $resnet4_log $env_type $branch $host_name ./profile_resnet50.sh conv_fwd_bias_relu 1 1 1 1 $verify 1 0 1 4 | tee -a $resnet4_log