Group norm (#417)

* Add groupnorm example by layernorm
1.  Reference is not ready
2. shape of gamma and beta need to be fix

* Let shape of gamma and beta can be same as x

* Modify test, instance and client example

* [What] Fix bug of layernorm for greater than 2 dimension.
[Why] We need to get upper length from merge transform instead of embed transform.

* Add reference for groupnorm

* Fuse sigmoid after groupnorm

* [What] Rename original layernorm into layernorm2d
[Why] Prepare to add groupnorm using layernorm5d

* clang-format

* Add groupnorm test

* Refine error message

* Add groupnorm ckProfiler

* Test groupnorm kernel from device_instance

* update example

* upadte profiler

* Fix test naming

* Fix argc number

* Move descriptor and sweeponce to argument for quick debugging

Co-authored-by: Chao Liu <chao.liu2@amd.com>

[ROCm/composable_kernel commit: 4eba345f6e]

profiler/src/profile_groupnorm.cpp

@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#include <iostream>
+#include <vector>
+#include <unordered_map>
+
+#include "profiler/include/data_type_enum.hpp"
+#include "profiler/include/profile_groupnorm_impl.hpp"
+
+using ck::index_t;
+
+struct GroupnormArgParser
+{
+    std::unordered_map<std::string, std::vector<int>> long_opts = {{"length", {}}};
+
+    bool parse_opt(int argc, char* argv[], const std::string& key, int i)
+    {
+        if(std::string("--") + key == argv[i])
+        {
+            int pos = i;
+            while(++i < argc && argv[i][0] != '-') {}
+            int end = i;
+            for(int j = pos + 1; j < end; j++)
+            {
+                long_opts[key].push_back(std::stoi(argv[j]));
+            }
+            return true;
+        }
+        return false;
+    }
+
+    void operator()(int argc, char* argv[])
+    {
+        for(auto& kv : long_opts)
+        {
+            for(int i = 1; i < argc; i++)
+            {
+                if(parse_opt(argc, argv, kv.first, i))
+                    break;
+            }
+        }
+    }
+};
+
+void print_help_groupnorm()
+{
+    std::cout << "arg1: tensor operation (groupnorm: Group normalization)\n"
+              << "arg2: data type (0: fp16; 1: fp32)\n"
+              << "arg3: verification (0: no; 1: yes)\n"
+              << "arg4: initialization (0: no init; 1: integer value; 2: decimal value)\n"
+              << "arg5: print tensor value (0: no; 1: yes)\n"
+              << "arg6: time kernel (0=no, 1=yes)\n"
+              << "--length: tensor extents (e.g, --length 1 16 16 32 40) \n"
+              << std::endl;
+}
+
+int profile_groupnorm(int argc, char* argv[])
+{
+    ck::DataTypeEnum data_type  = ck::DataTypeEnum::Half;
+    bool do_verification        = false;
+    int init_method             = 0;
+    bool do_log                 = 0;
+    bool time_kernel            = 1;
+    std::vector<index_t> length = {64, 16, 16, 32, 40};
+
+    if(argc != 1 && argc != 13)
+    {
+        print_help_groupnorm();
+        return 0;
+    }
+
+    if(argc == 13)
+    {
+        data_type       = static_cast<ck::DataTypeEnum>(std::stoi(argv[2]));
+        do_verification = std::stoi(argv[3]);
+        init_method     = std::stoi(argv[4]);
+        do_log          = std::stoi(argv[5]);
+        time_kernel     = std::stoi(argv[6]);
+
+        // parse the long options
+        GroupnormArgParser arg_parser;
+        arg_parser(argc, argv);
+        length = arg_parser.long_opts["length"];
+    }
+
+    using F16 = ck::half_t;
+    using F32 = float;
+
+    if(data_type == ck::DataTypeEnum::Float)
+    {
+        ck::profiler::profile_groupnorm_impl<F32, F32, F32, F32, F32>(
+            do_verification, init_method, do_log, time_kernel, length);
+    }
+    else if(data_type == ck::DataTypeEnum::Half)
+    {
+        ck::profiler::profile_groupnorm_impl<F16, F16, F16, F32, F16>(
+            do_verification, init_method, do_log, time_kernel, length);
+    }
+    else
+    {
+        throw std::runtime_error("not implemented yet");
+    }
+
+    return 0;
+}

profiler/src/profile_layernorm.cpp

@@ -5,6 +5,7 @@
 #include <vector>
 #include <unordered_map>
 
+#include "profiler/include/data_type_enum.hpp"
 #include "profiler/include/profile_layernorm_impl.hpp"
 
 using ck::index_t;
@@ -49,7 +50,7 @@ void print_help_layernorm()
               << "arg2: verification (0: no; 1: yes)\n"
               << "arg3: initialization (0: no init; 1: integer value; 2: decimal value)\n"
               << "arg4: print tensor value (0: no; 1: yes)\n"
-              << "arg5: time kernel (0=n0, 1=yes)\n"
+              << "arg5: time kernel (0=no, 1=yes)\n"
               << "--length: tensor extents (e.g, --length 1024 1024) \n"
               << "--strideXY: tensor strides (e.g, --strideXY 1024 1)\n"
               << "--strideGamma: tensor strides (e.g, --strideGamma 1)\n"
@@ -114,10 +115,3 @@ int profile_layernorm(int argc, char* argv[])
 
     return 0;
 }
-
-// hijack main() for quick debugging
-// int main(int argc, char* argv[])
-// {
-//     profile_layernorm(argc, argv);
-//     return 0;
-// }

profiler/src/profiler.cpp

@@ -3,26 +3,27 @@
 
 #include <cstring>
 
-int profile_gemm(int, char*[]);
-int profile_gemm_splitk(int, char*[]);
-int profile_gemm_bilinear(int, char*[]);
-int profile_gemm_add_add_fastgelu(int, char*[]);
-int profile_gemm_reduce(int, char*[]);
-int profile_gemm_bias_add_reduce(int, char*[]);
-int profile_batched_gemm(int, char*[]);
-int profile_batched_gemm_gemm(int, char*[]);
-int profile_batched_gemm_add_relu_gemm_add(int, char*[]);
-int profile_batched_gemm_reduce(int, char*[]);
-int profile_grouped_gemm(int, char*[]);
-int profile_conv_fwd(int, char*[]);
-int profile_conv_fwd_bias_relu(int, char*[]);
-int profile_conv_fwd_bias_relu_add(int, char*[]);
-int profile_conv_bwd_data(int, char*[]);
-int profile_conv_bwd_weight(int, char*[]);
-int profile_grouped_conv_fwd(int, char*[]);
-int profile_normalization(int, char*[]);
+// int profile_gemm(int, char*[]);
+// int profile_gemm_splitk(int, char*[]);
+// int profile_gemm_bilinear(int, char*[]);
+// int profile_gemm_add_add_fastgelu(int, char*[]);
+// int profile_gemm_reduce(int, char*[]);
+// int profile_gemm_bias_add_reduce(int, char*[]);
+// int profile_batched_gemm(int, char*[]);
+// int profile_batched_gemm_gemm(int, char*[]);
+// int profile_batched_gemm_add_relu_gemm_add(int, char*[]);
+// int profile_batched_gemm_reduce(int, char*[]);
+// int profile_grouped_gemm(int, char*[]);
+// int profile_conv_fwd(int, char*[]);
+// int profile_conv_fwd_bias_relu(int, char*[]);
+// int profile_conv_fwd_bias_relu_add(int, char*[]);
+// int profile_conv_bwd_data(int, char*[]);
+// int profile_conv_bwd_weight(int, char*[]);
+// int profile_grouped_conv_fwd(int, char*[]);
+// int profile_normalization(int, char*[]);
 int profile_layernorm(int, char*[]);
-int profile_reduce(int, char*[]);
+int profile_groupnorm(int, char*[]);
+// int profile_reduce(int, char*[]);
 
 static void print_helper_message()
 {
@@ -56,6 +57,7 @@ int main(int argc, char* argv[])
 
         return 0;
     }
+#if 0
     else if(strcmp(argv[1], "gemm") == 0)
     {
         return profile_gemm(argc, argv);
@@ -132,10 +134,15 @@ int main(int argc, char* argv[])
     {
         return profile_normalization(argc, argv);
     }
+#endif
     else if(strcmp(argv[1], "layernorm") == 0)
     {
         return profile_layernorm(argc, argv);
     }
+    else if(strcmp(argv[1], "groupnorm") == 0)
+    {
+        return profile_groupnorm(argc, argv);
+    }
     else
     {
         print_helper_message();