Runner script for benchmarking.

2026-07-19 02:01:01 +00:00 · 2026-02-04 09:38:16 -05:00
parent 3e6415a8ea
commit 73b459c5a4
2 changed files with 102 additions and 0 deletions
--- a/benchmarking/plot-improvement.py
+++ b/benchmarking/plot-improvement.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import numpy as np
+import matplotlib
+matplotlib.use('Agg')  # Set non-interactive backend before importing pyplot
+import matplotlib.pyplot as plt
+
+tflops_baseline = [2.85567, 10.3518, 15.4196, 849.773, 990.991]
+tflops_improved = [8.09047, 19.8867, 20.4604, 952.806, 1091.89]
+
+# Plot tflops for both baseline and improved
+plt.figure(figsize=(10, 6))
+indices = np.arange(len(tflops_baseline))
+bar_width = 0.35
+plt.bar(indices, tflops_baseline, bar_width, label='Baseline', color='b')
+plt.bar(indices + bar_width, tflops_improved, bar_width, label='Improved', color='g')
+plt.xlabel('Test Cases')
+plt.ylabel('TFLOPS')
+plt.title('TFLOPS Comparison: Baseline vs Improved')
+plt.xticks(indices + bar_width / 2, ['Case 1', 'Case 2', 'Case 3', 'Case 4', 'Case 5'])
+plt.yscale('log')
+plt.legend()
+plt.grid(axis='y')
+plt.tight_layout()
+plt.savefig('tflops_comparison.png')
+plt.close()
+
+# plot improvement factor
+improvement_factor = [100 *(improved / baseline)for improved, baseline in zip(tflops_improved, tflops_baseline)]
+improvement_factor = [factor - 100 for factor in improvement_factor]  # Convert to percentage improvement
+plt.figure(figsize=(10, 6))
+plt.bar(indices, improvement_factor, color='orange')
+plt.xlabel('Test Cases')
+plt.ylabel('Improvement (%)')
+plt.title('Improvement')
+plt.xticks(indices, ['Case 1', 'Case 2', 'Case 3', 'Case 4', 'Case 5'])
+plt.grid(axis='y')
+plt.tight_layout()
+plt.savefig('improvement.png')
+plt.close()
--- a/benchmarking/run-best-instances.py
+++ b/benchmarking/run-best-instances.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import argparse
+
+profiler_commands = [
+  "1         1       1              0      1         0     1           2     32  32  4     4    3  3  200  200   1   1   1   1   1   1    1   1", 
+  "1         1       1              0      1         0     1           2     32  32  8     8    3  3  200  200   2   2   1   1   1   1    1   1", 
+  "1         1       1              0      1         0     1           2     32  32  8     8    3  3  100  100   1   2   1   1   1   1    1   1", 
+  "1         1       1              0      1         0     1           2     1   32  2376  256  3  3  100  100   1   1   1   1   1   1    1   1", 
+  "1         1       1              0      1         0     1           2     1   32  256   256  3  3  100  100   1   1   1   1   1   1    1   1"
+  ]
+
+baseline_instances = [
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 64, 64, 32, Default, 16, 16, 2, 2, 4, 4, 4, 1, 1, 1>", 
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_DirectLoad<128, 16, 64, 64, Default, 16, 16, 1, 2, 8, 8, 4, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1>",
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3_DirectLoad<256, 256, 32, 64, Default, 32, 32, 2, 1, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v1>", 
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 256, 128, 32, Default, 32, 32, 4, 2, 8, 8, 8, 1, 1, 1>", 
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 256, 128, 32, OddC, 32, 32, 4, 2, 8, 8, 8, 1, 1, 1>"
+]
+
+improved_instances = [
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 128, 32, 32, Default, 32, 32, 2, 1, 4, 4, 1, 1, 1, BlkGemmPipelineScheduler: Interwave, BlkGemmPipelineVersion: v1, 8>",
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 64, 32, Default, 32, 32, 4, 2, 8, 8, 1, 1, 1, 8>",
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle<256, 128, 64, 32, Default, 32, 32, 4, 2, 8, 8, 1, 1, 1, 8>",
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 256, 256, 64, Default, 32, 32, 4, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3, 1>",
+  "DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3<256, 256, 256, 64, Default, 32, 32, 4, 4, 8, 8, 8, 1, 1, BlkGemmPipelineScheduler: Intrawave, BlkGemmPipelineVersion: v3, 1>"
+]
+
+def main():
+  # Parse command-line arguments
+  parser = argparse.ArgumentParser(description='Run CK profiler with best instances for given conv shapes.')
+  parser.add_argument('--profiler-path', type=str, required=True, help='Path to the profiler binary')
+  parser.add_argument('--baseline', action='store_true', 
+                      help='Run baseline instances (default: run improved instances)')
+  args = parser.parse_args()
+
+  instances_to_run = baseline_instances if args.baseline else improved_instances
+  instance_type = "baseline" if args.baseline else "improved"
+
+  print(f"Running {instance_type} instances...\n")
+
+  ck_profiler_path = args.profiler_path
+  if not os.path.isfile(ck_profiler_path):
+      print(f"Error: Profiler binary not found at {ck_profiler_path}")
+      sys.exit(1)
+
+  for i in range(len(profiler_commands)):
+        command = profiler_commands[i]
+        instance = instances_to_run[i]
+        profiler_args = [x for x in command.split()]
+        profiler_args.append(instance)
+
+        print(f"Running profiler for {instance_type} instance {i+1}/{len(profiler_commands)}:")
+        print(instance)
+        subprocess.run([ck_profiler_path] + ["grouped_conv_fwd"] + profiler_args, check=True)
+        print()
+
+if __name__ == "__main__":
+    main()