diff --git a/script/analyze_conv_tests.py b/script/analyze_conv_tests.py index 9446b37470..9c32bba8d2 100644 --- a/script/analyze_conv_tests.py +++ b/script/analyze_conv_tests.py @@ -6,6 +6,8 @@ import sys import pandas as pd import csv import matplotlib +from collections import defaultdict + matplotlib.use('Agg') # Use a non-interactive backend from matplotlib import pyplot as plt @@ -64,7 +66,7 @@ def plot_local_ranking_bar_chart(best_split_k_ranking_numbers, file_name, explan # Check that there are not other ranks than 1-9 if any(rank < 1 or rank > max_rank for rank in rankings_count.keys()): - raise f"Error: Found ranks outside the range 1-9." + raise f"Error: Found ranks outside the range 1-9:" plt.figure(figsize=(10, 6)) @@ -161,210 +163,199 @@ def plot_local_performance_histogram(local_performance, file_name, explanation): plt.savefig(file_name) plt.close() -def main(): - args = parse_cli_args() +def plot_best_split_k_values(standard_counts, optimized_count, + standard_equal_optimized_counts, suffix, args): + # Prepare data for plotting + categories = list(standard_counts.keys()) + ['Optimized Split-K'] - csv.register_dialect('PipeDialect', delimiter=';') - with open(args.csv_file) as csvfile: - data = [row for row in csv.reader(csvfile, 'PipeDialect')] + # Calculate total counts (standard counts + cases where standard equals optimized) + total_standard_counts = [] + equal_counts = [] - df = pd.DataFrame(data = data) + # First, collect data for all standard values + for key in standard_counts.keys(): + # Get the count where standard equals optimized (default to 0 if key doesn't exist) + equal_count = standard_equal_optimized_counts.get(key, 0) + equal_counts.append(equal_count) + + # Total is the standard count + total_standard_counts.append(standard_counts[key]) - print(f"Loaded {len(df)} rows.") - print(df.head()) + # Add the optimized count as the last category + total_counts = total_standard_counts + [optimized_count] + equal_counts.append(0) # No "equals optimized" for the optimized category itself - best_ops = df[0] - best_times = df[1] - best_split_k = df[2] - best_split_k_ops = df[3] - best_split_k_times = df[4] - best_split_k_values = df[5] - best_split_k_ranks = df[6] - num_ops = df[7] + # Calculate the "non-equal" portion (what will show at the bottom of each stack) + non_equal_counts = [total - equal for total, equal in zip(total_counts, equal_counts)] - local_rankings = [] - local_peformance = [] - local_data_num_cols = 7 # Number of columns we expect in the local data - max_columns = df.shape[1] - local_data_num_cols - for i in range(8, max_columns, local_data_num_cols): - temp_df = pd.DataFrame({ - 'best_times': df[i + 1], - 'best_split_k': df[i + 2], - 'opt_split_k_times': df[i + 3], - 'opt_split_k_values': df[i + 4], - 'opt_split_k_rank': df[i + 5], - 'num_ops': df[i + 6] - }) - clean_df = temp_df.dropna() - local_opt_split_k_rank = clean_df['opt_split_k_rank'].astype(int).tolist() + # Create figure + plt.figure(figsize=(14, 7)) - # Filter out rows where opt_split_k equals best_split_k - filtered_df = clean_df[clean_df['opt_split_k_values'] != clean_df['best_split_k']] + # Create the base bars (non-equal counts) + base_bars = plt.bar( + range(len(categories)), # X positions + non_equal_counts, # Heights (counts without the "equals optimized" portion) + color='skyblue', # Base color + edgecolor='black', + alpha=0.8, + width=0.6, + label='Standard Split-K (1,2,4,8,16,32,64,128)' + ) - # Calculate performance metrics on filtered data - perf_factor = filtered_df['best_times'].astype(float) / filtered_df['opt_split_k_times'].astype(float) - local_perf = 100.0 * perf_factor + # Create the stacked bars for the "equals optimized" portion + equal_bars = plt.bar( + range(len(categories)), # X positions + equal_counts, # Heights (just the "equals optimized" counts) + bottom=non_equal_counts, # Start these bars where the base bars end + color='orange', # Different color to highlight this portion + edgecolor='black', + alpha=0.8, + width=0.6, + label='Standard = Optimized' + ) - local_peformance.extend(local_perf.tolist()) - local_rankings.extend(local_opt_split_k_rank) - - suffix = f"_{args.label}" if args.label else "" + # Add value labels for total height of each bar + for i, (total, equal) in enumerate(zip(total_counts, equal_counts)): + if total > 0: # Only add label if there's a value + # Position the text at the top of the stacked bar + plt.text( + i, # X position (bar index) + total + 0.5, # Y position (just above the top) + f'{int(total)}', # Total count as text + ha='center', + va='bottom', + fontweight='bold' + ) + + # If there's a significant "equals optimized" portion, add a label inside that section + if equal > 5: # Only add for larger values to avoid clutter + plt.text( + i, # X position (bar index) + non_equal_counts[i] + equal/2, # Y position (middle of orange section) + f'{int(equal)}', # Equal count as text + ha='center', + va='center', + fontweight='bold', + color='black' + ) - # Plot the local ranking numbers as a bar chart - explanation = """Each supported instance was benchmarked with split-K values ["optimized", 1, 2, 4, 8, 16, 32, 64, 128]. -Ranking 1 means that optimized split-K value was the best, and ranking 9 means that it was the worst""" - file_name = os.path.join(args.output_dir, f'local_ranking_chart{suffix}.png') - plot_local_ranking_bar_chart(local_rankings, file_name, explanation) + # Highlight the optimized category with a different color + base_bars[-1].set_color('green') + base_bars[-1].set_label('Optimized Split-K') - # Plot the local performance as a histogram - explanation = """Performance of the optimal split-K value compared to the best split-K value -when optimal split-K value was not the best.""" - file_name = os.path.join(args.output_dir, f'local_performance_histogram{suffix}.png') - plot_local_performance_histogram(local_peformance, file_name, explanation) + # Set x-tick positions and labels + plt.xticks( + range(len(categories)), # Positions + categories, # Labels + rotation=45 if len(categories) > 8 else 0, # Rotate if many categories + fontsize=11, + ha='right' if len(categories) > 8 else 'center' # Align rotated labels + ) - print(f"Column stats:") - print(f"- Best split-k values unique count: {best_split_k.nunique()}") - print(f"- Best split-k values: {', '.join(best_split_k.unique().tolist()[:10])}...") - - # Calculate ranking numbers - best_split_k_ranking_numbers = calculate_ranking_numbers(best_split_k_ranks, num_ops) + # Add labels, title, and legend + plt.title('Best Split-K Values', fontsize=16, fontweight='bold') + plt.xlabel('Split-K Value', fontsize=14) + plt.ylabel('Count', fontsize=14) + plt.grid(True, linestyle='--', alpha=0.7, axis='y') # Grid lines only on y-axis + plt.legend(fontsize=12) - # Plot the global ranking numbers as a historgram - explanation = """For each shape, all supported instances were benchmarked -with split-K values ["optimized", 1, 2, 4, 8, 16, 32, 64, 128]. -Ranking 100% means that best instance had optimized split-K value, -lower values mean that the best instance had one of the fixed split-K values.""" - file_name = os.path.join(args.output_dir, f'ranking_histogram{suffix}.png') - plot_ranking_histogram(best_split_k_ranking_numbers, file_name, explanation) - - # Find indices where split-k is not in the standard set - standard_split_k = ['1', '2', '4', '8', '16', '32', '64', '128'] - non_standard_indices = [i for i in range(len(best_split_k)) - if best_split_k.iloc[i] not in standard_split_k] - - print(f"Found {len(non_standard_indices)} cases with non-standard split-k values") - - if non_standard_indices: - # Calculate ranking for non-standard split-k values - non_standard_split_k_ranking_numbers = [] - non_standard_split_k_values = [] - - for i in non_standard_indices: - try: - rank = int(best_split_k_ranks.iloc[i]) - total_ops = int(num_ops.iloc[i]) - ranking = 100.0 * (total_ops - rank + 1) / total_ops - non_standard_split_k_ranking_numbers.append(ranking) - non_standard_split_k_values.append(best_split_k.iloc[i]) - except (ValueError, TypeError) as e: - print(f"Warning: Could not process non-standard row {i}: {e}") - - # Define standard split-K values - standard_split_k = ['1', '2', '4', '8', '16', '32', '64', '128'] + # Add explanation text for the orange portion + explanation = "Orange sections represent cases where optimized\nsplit-K equals to one of the fixed split-K values" + plt.text( + 0.02, 0.95, # Position in axes coordinates (top-left) + explanation, + transform=plt.gca().transAxes, # Use axes coordinates + fontsize=11, + verticalalignment='top', + bbox=dict(boxstyle='round', facecolor='white', alpha=0.7) + ) - # Count occurrences - standard_counts = {} - optimized_count = 0 + # Adjust layout to prevent label cutoff + plt.tight_layout() - # Initialize standard counts with zeros - for sk in standard_split_k: - standard_counts[sk] = 0 - - # Count occurrences in your data - for i in range(len(best_split_k)): - value = best_split_k.iloc[i] - if value in standard_split_k: - standard_counts[value] += 1 - else: - optimized_count += 1 - - # Create ordered categories for the plot - categories = list(standard_counts.keys()) + ['Optimized Split-K'] - counts = list(standard_counts.values()) + [optimized_count] - - # Create figure - plt.figure(figsize=(14, 7)) - - # Create bar chart with different colors for standard vs optimized - colors = ['skyblue'] * len(standard_counts) + ['crimson'] - bars = plt.bar( - range(len(categories)), # X positions - counts, # Heights (counts) - color=colors, - edgecolor='black', - alpha=0.8, - width=0.6 - ) - - # Add value labels on top of each bar - for bar in bars: - height = bar.get_height() - plt.text( - bar.get_x() + bar.get_width()/2., - height + 0.5, - f'{int(height)}', - ha='center', - va='bottom', - fontweight='bold' - ) - - # Set x-tick positions and labels - plt.xticks( - range(len(categories)), # Positions - categories, # Labels - rotation=0, # No rotation needed for few categories - fontsize=11 - ) - - # Add labels and title - plt.title('Distribution of Best Split-K Values', fontsize=16, fontweight='bold') - plt.xlabel('Split-K Value', fontsize=14) - plt.ylabel('Frequency (Count)', fontsize=14) - plt.grid(True, linestyle='--', alpha=0.7, axis='y') # Grid lines only on y-axis - - # Add a legend - from matplotlib.patches import Patch - legend_elements = [ - Patch(facecolor='skyblue', edgecolor='black', label='Standard Values'), - Patch(facecolor='crimson', edgecolor='black', label='Optimized Values') - ] - plt.legend(handles=legend_elements, loc='upper center', fontsize=12) - - # Adjust layout to prevent label cutoff - plt.tight_layout() - - # Save the plot - bar_plot_path = os.path.join(args.output_dir, f'best_split_k_distribution{suffix}.png') - plt.savefig(bar_plot_path) - print(f"Saved split-K distribution chart to: {bar_plot_path}") - print(f"You can view it with: \"$BROWSER\" {os.path.abspath(bar_plot_path)}") - - # Display the detailed breakdown - print("\nFrequency of Split-K values:") - for k, count in standard_counts.items(): - print(f" Split-K = {k}: {count} instances") - print(f" Optimized Split-K: {optimized_count} instances") - - # If optimized count is non-zero, show the distribution of optimized values - if optimized_count > 0: - non_standard_values = [best_split_k.iloc[i] for i in range(len(best_split_k)) - if best_split_k.iloc[i] not in standard_split_k] - non_standard_counts = {} - for val in non_standard_values: - non_standard_counts[val] = non_standard_counts.get(val, 0) + 1 - - print("\nBreakdown of optimized Split-K values:") - for k, count in sorted(non_standard_counts.items(), key=lambda x: int(x[0])): - print(f" Split-K = {k}: {count} instances") - - - if optimized_count > 0: - non_standard_values = [best_split_k.iloc[i] for i in range(len(best_split_k)) - if best_split_k.iloc[i] not in standard_split_k] - non_standard_counts = {} - for val in non_standard_values: - non_standard_counts[val] = non_standard_counts.get(val, 0) + 1 + # Save the figure + split_k_distribution_path = os.path.join(args.output_dir, f'best_split_k_values{suffix}.png') + plt.savefig(split_k_distribution_path) + print(f"Saved best split-K values chart to: {split_k_distribution_path}") + plt.close() + +def plot_perf(perf_difference, output_dir, suffix=""): + """Plot the performance differences as a histogram with statistics.""" + import numpy as np + + # Calculate statistics + mean_val = np.mean(perf_difference) + median_val = np.median(perf_difference) + std_val = np.std(perf_difference) + min_val = np.min(perf_difference) + max_val = np.max(perf_difference) + p25 = np.percentile(perf_difference, 25) + p75 = np.percentile(perf_difference, 75) + count = len(perf_difference) + + # Determine bin edges at 5% intervals + min_edge = np.floor(min_val / 5) * 5 + max_edge = np.ceil(max_val / 5) * 5 + bin_edges = np.arange(min_edge, max_edge + 5, 5) + + # Create figure + plt.figure(figsize=(12, 6)) + + # Split data into below and above 100% + below_100 = [x for x in perf_difference if x < 100] + above_100 = [x for x in perf_difference if x >= 100] + + # Plot histogram for values below 100% (red) + if below_100: + plt.hist(below_100, bins=bin_edges, color='red', + alpha=0.7, edgecolor='black', label='Below 100%') + + # Plot histogram for values above or equal to 100% (green) + if above_100: + plt.hist(above_100, bins=bin_edges, color='green', + alpha=0.7, edgecolor='black', label='Above 100%') + + # Create statistics text + stats_text = (f"Statistics:\n" + f"Count: {count}\n" + f"Mean: {mean_val:.2f}%\n" + f"Median: {median_val:.2f}%\n" + f"Std Dev: {std_val:.2f}%\n" + f"Min: {min_val:.2f}%\n" + f"Max: {max_val:.2f}%\n" + f"25th Percentile: {p25:.2f}%\n" + f"75th Percentile: {p75:.2f}%") + + plt.title('Performance of Optimized Split-K value vs Best Standard Split-K value', + fontsize=14, fontweight='bold') + plt.xlabel('Performance (%)', fontsize=12) + plt.ylabel('Count', fontsize=12) + + # Add gridlines aligned with bin edges + plt.grid(True, linestyle='--', alpha=0.7) + + # Ensure x-axis ticks align with bin edges + plt.xticks(bin_edges) + + # Add statistics text box + plt.text(0.02, 0.97, stats_text, transform=plt.gca().transAxes, fontsize=10, + verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + + # Add a vertical line at x=100 to highlight the threshold + plt.axvline(x=100, color='black', linestyle='--', alpha=0.9, linewidth=2, + label='100% Threshold') + + plt.legend(loc='upper center') + plt.tight_layout() + + file_name = os.path.join(output_dir, f'performance{suffix}.png') + plt.savefig(file_name, dpi=150) + print(f"Saved performance chart to: {file_name}") + + plt.close() + +def plot_split_k_distribution(non_standard_counts, optimized_count, args, suffix): # Sort the values numerically sorted_items = sorted(non_standard_counts.items(), key=lambda x: int(x[0])) opt_values = [x[0] for x in sorted_items] @@ -377,7 +368,7 @@ lower values mean that the best instance had one of the fixed split-K values.""" bars = plt.barh( range(len(opt_values)), # Y positions opt_counts, # Widths (counts) - color='crimson', + color='green', edgecolor='black', alpha=0.8, height=0.6 @@ -426,5 +417,105 @@ lower values mean that the best instance had one of the fixed split-K values.""" plt.savefig(opt_plot_path) print(f"Saved optimized split-K distribution chart to: {opt_plot_path}") +def main(): + args = parse_cli_args() + + csv.register_dialect('PipeDialect', delimiter=';') + with open(args.csv_file) as csvfile: + data = [row for row in csv.reader(csvfile, 'PipeDialect')] + + df = pd.DataFrame(data = data) + + print(f"Loaded {len(df)} rows.") + print(df.head()) + + non_opt_split_k_ops = df[0] + non_opt_split_k_times = df[1] + non_opt_split_k_value = df[2] + opt_split_k_ops = df[3] + opt_split_k_times = df[4] + opt_split_k_values = df[5] + + suffix = f"_{args.label}" if args.label else "" + + # Find indices where split-k is not in the standard set + standard_split_k = ['1', '2', '4', '8', '16', '32', '64', '128'] + non_standard_indices = [i for i in range(len(opt_split_k_values)) + if opt_split_k_values.iloc[i] not in standard_split_k] + + print(f"Found {len(non_standard_indices)} cases with non-standard split-k values") + + if non_standard_indices: + non_standard_split_k_values = [] + + for i in non_standard_indices: + try: + non_standard_split_k_values.append(opt_split_k_values.iloc[i]) + except (ValueError, TypeError) as e: + print(f"Warning: Could not process non-standard row {i}: {e}") + + standard_counts = defaultdict(int) + optimized_count = 0 + standard_equal_optimized_counts = defaultdict(int) + perf_change = [] + + # Initialize counts for standard split-k values + for sk in standard_split_k: + standard_counts[sk] = 0 + standard_equal_optimized_counts[sk] = 0 + + assert len(non_opt_split_k_value) == len(opt_split_k_values), \ + "Length of non-opt split-k values and optimized split-k values must match." + + for i in range(len(non_opt_split_k_value)): + non_opt_time = float(non_opt_split_k_times.iloc[i]) + opt_time = float(opt_split_k_times.iloc[i]) + non_opt_value = non_opt_split_k_value.iloc[i] + opt_value = opt_split_k_values.iloc[i] + non_opt_op = non_opt_split_k_ops.iloc[i] + opt_op = opt_split_k_ops.iloc[i] + + perf = 100.0 * (non_opt_time / opt_time) if opt_time > 1e-5 else 0.0 + perf_change.append(perf) + + if opt_value == non_opt_value and opt_op == non_opt_op: + standard_equal_optimized_counts[non_opt_value] += 1 + + elif opt_time < non_opt_time: + optimized_count += 1 + elif opt_time > non_opt_time: + standard_counts[non_opt_value] += 1 + + + plot_perf(perf_change, args.output_dir, suffix) + + plot_best_split_k_values( + standard_counts, optimized_count, + standard_equal_optimized_counts, suffix, args) + + # Display the detailed breakdown + print("\nFrequency of standard Split-K values:") + for k, count in standard_counts.items(): + print(f" Split-K = {k}: {count} instances") + + print("\nFrequency of standard = optimized Split-K values:") + for k, count in standard_equal_optimized_counts.items(): + print(f" Split-K = {k}: {count} instances") + + print(f"\nOptimized Split-K: {optimized_count} instances") + + # If optimized count is non-zero, show the distribution of optimized values + if optimized_count > 0: + non_standard_values = [opt_split_k_values.iloc[i] for i in non_standard_indices] + non_standard_counts = {} + for val in non_standard_values: + non_standard_counts[val] = non_standard_counts.get(val, 0) + 1 + + print("\nBreakdown of optimized Split-K values:") + for k, count in sorted(non_standard_counts.items(), key=lambda x: int(x[0])): + print(f" Split-K = {k}: {count} instances") + + plot_split_k_distribution(non_standard_counts, optimized_count, args, suffix) + if __name__ == "__main__": main() \ No newline at end of file