Add performance tests as a stage of CI. (#247)

* modify ckProfiler_gemm output

* fix syntax

* change ckProfiler output and return 0

* fix syntax

* output datatype

* fix syntax

* output datatype in another way

* fix syntax

* fix syntax

* test return values of ckProfiler

* add layout info and tests, make sure ckprofiler returns 0

* fix syntax

* change layout output

* fix syntax

* fix syntax again

* update script to process perf results

* rearrange jenkins stages

* fix typo

* add python packages to Docker file

* adding setuptools-rust package

* modify parsing for new test parameters

* test db credentials on jenkins

* fix syntax

* update python script to handle incomplete lines

* ungrade python to 3.8 and write the gemm_params table

* add sqlalchemy package to docker

* move perf data processing to master node

* move the master node inside a steps region

* add new stage for result processing

* move results processing to separate stage

* reduce number of tests to speedup debugging

* pass config to processPerfResults stage

* run script on master in a docker container

* replace show_node_info

* try loading docker on master node again

* use ansible node instead of master

* get rid of pymysql package

* try ssh connection using paramiko

* put back pymysql

* put the perf data processing back on the gpu node

* put back artifact definition

* archive the perf_log before parsing

* clean up jenkinsfile, fix parsing

* fix typo

* enable all perf tests

* put all stages in original order, finalize script

* fix gpu_arch version

* update parsing script

* remove obsolete file causing merge conflict

[ROCm/composable_kernel commit: 1085794df3]

profiler/include/profile_gemm_impl.hpp

@@ -1,5 +1,7 @@
 #pragma once
 #include <iomanip>
+#include <iostream>
+#include <typeinfo>
 
 #include "check_err.hpp"
 #include "config.hpp"
@@ -527,8 +529,45 @@ void profile_gemm_impl(int do_verification,
         }
     }
 
-    std::cout << "Best Perf: " << best_ave_time << " ms, " << best_tflops << " TFlops, "
-              << best_gb_per_sec << " GB/s, " << best_gemm_name << std::endl;
+    if constexpr(is_same<CDataType, float>::value)
+    {
+        std::cout << "Best Perf for datatype = f32";
+    }
+    else if constexpr(is_same<CDataType, half_t>::value)
+    {
+        std::cout << "Best Perf for datatype = f16";
+    }
+    else if constexpr(is_same<CDataType, bhalf_t>::value)
+    {
+        std::cout << "Best Perf for datatype = bf16";
+    }
+    else if constexpr(is_same<CDataType, int8_t>::value)
+    {
+        std::cout << "Best Perf for datatype = int8";
+    }
+
+    if constexpr(is_same<ALayout, tensor_layout::gemm::RowMajor>::value)
+    {
+        std::cout << " ALayout =  RowMajor";
+    }
+    else if constexpr(is_same<ALayout, tensor_layout::gemm::ColumnMajor>::value)
+    {
+        std::cout << " ALayout =  ColumnMajor";
+    }
+
+    if constexpr(is_same<BLayout, tensor_layout::gemm::RowMajor>::value)
+    {
+        std::cout << " BLayout =  RowMajor";
+    }
+    else if constexpr(is_same<BLayout, tensor_layout::gemm::ColumnMajor>::value)
+    {
+        std::cout << " BLayout =  ColumnMajor";
+    }
+
+    std::cout << " M = " << M << " N = " << N << " K = " << K << " StrideA = " << StrideA
+              << " StrideB = " << StrideB << " StrideC = " << StrideC << " : " << best_ave_time
+              << " ms, " << best_tflops << " TFlops, " << best_gb_per_sec << " GB/s, "
+              << best_gemm_name << std::endl;
 }
 
 } // namespace profiler

profiler/src/profile_batched_gemm.cpp

@@ -396,5 +396,5 @@ int profile_batched_gemm(int argc, char* argv[])
         throw std::runtime_error("wrong! this GEMM data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_batched_gemm_reduce.cpp

@@ -149,5 +149,5 @@ int profile_batched_gemm_reduce(int argc, char* argv[])
         throw std::runtime_error("wrong! this data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_conv_bwd_weight.cpp

@@ -142,5 +142,5 @@ int profile_conv_bwd_weight(int argc, char* argv[])
         throw std::runtime_error("wrong! this Conv data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_conv_fwd_bias_relu.cpp

@@ -110,5 +110,5 @@ int profile_conv_fwd_bias_relu(int argc, char* argv[])
         throw std::runtime_error("wrong! data_type & layout for this operator is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_conv_fwd_bias_relu_add.cpp

@@ -111,5 +111,5 @@ int profile_conv_fwd_bias_relu_add(int argc, char* argv[])
         throw std::runtime_error("wrong! data_type & layout for this operator is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_conv_fwd_bias_relu_atomic_add.cpp

@@ -112,5 +112,5 @@ int profile_conv_fwd_bias_relu_atomic_add(int argc, char* argv[])
         throw std::runtime_error("wrong! data_type & layout for this operator is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_convnd_fwd.cpp

@@ -347,5 +347,5 @@ int ck::profiler::profile_convnd_fwd(int argc, char* argv[])
                                  std::to_string(num_dim_spatial));
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_gemm.cpp

@@ -388,5 +388,5 @@ int profile_gemm(int argc, char* argv[])
         throw std::runtime_error("wrong! this GEMM data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_gemm_bias_2d.cpp

@@ -252,5 +252,5 @@ int profile_gemm_bias_2d(int argc, char* argv[])
         throw std::runtime_error("wrong! this data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_gemm_bias_relu.cpp

@@ -139,5 +139,5 @@ int profile_gemm_bias_relu(int argc, char* argv[])
         throw std::runtime_error("wrong! this data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_gemm_bias_relu_add.cpp

@@ -144,5 +144,5 @@ int profile_gemm_bias_relu_add(int argc, char* argv[])
         throw std::runtime_error("wrong! this data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_gemm_reduce.cpp

@@ -142,5 +142,5 @@ int profile_gemm_reduce(int argc, char* argv[])
         throw std::runtime_error("wrong! this data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profile_grouped_gemm.cpp

@@ -153,5 +153,5 @@ int profile_grouped_gemm(int argc, char* argv[])
         throw std::runtime_error("wrong! this GEMM data_type & layout is not implemented");
     }
 
-    return 1;
+    return 0;
 }

profiler/src/profiler.cpp

@@ -25,7 +25,8 @@ int main(int argc, char* argv[])
 {
     if(strcmp(argv[1], "gemm") == 0)
     {
-        return profile_gemm(argc, argv);
+        int stat = profile_gemm(argc, argv);
+        return stat;
     }
     else if(strcmp(argv[1], "gemm_bias_2d") == 0)
     {