From cc17e96c3bbd9538344b4923d4ccbeb736cf65b3 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:02:51 -0500 Subject: [PATCH] Add nvbench-compare-legacy --- python/pyproject.toml | 1 + python/scripts/nvbench_compare_legacy.py | 1629 ++++++++++++++++++++++ 2 files changed, 1630 insertions(+) create mode 100644 python/scripts/nvbench_compare_legacy.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 948c2a5..d1cacb6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -52,6 +52,7 @@ tools = ["cuda-bench[compare,plot]"] [project.scripts] nvbench-compare = "cuda.bench.scripts.nvbench_compare:main" +nvbench-compare-legacy = "cuda.bench.scripts.nvbench_compare_legacy:main" nvbench-histogram = "cuda.bench.scripts.nvbench_histogram:main" nvbench-json-summary = "cuda.bench.scripts.nvbench_json_summary:main" nvbench-walltime = "cuda.bench.scripts.nvbench_walltime:main" diff --git a/python/scripts/nvbench_compare_legacy.py b/python/scripts/nvbench_compare_legacy.py new file mode 100644 index 0000000..6cc02e5 --- /dev/null +++ b/python/scripts/nvbench_compare_legacy.py @@ -0,0 +1,1629 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from __future__ import annotations + +import argparse +import math +import os +import sys +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Any + +if __package__: + from .nvbench_json import reader + from .nvbench_tooling_deps import ( + MissingToolingDependencyError, + ToolingDependency, + require_tooling_dependency, + ) +else: + from nvbench_json import reader # type: ignore[no-redef] + from nvbench_tooling_deps import ( # type: ignore[no-redef] + MissingToolingDependencyError, + ToolingDependency, + require_tooling_dependency, + ) + + +# Parse version string into tuple, "x.y.z" -> (x, y, z) +def version_tuple(v): + return tuple(map(int, (v.split(".")))) + + +Fore: Any = None + + +def load_nvbench_compare_tooling(*, load_color: bool = True) -> None: + global Fore + + if load_color and Fore is None: + colorama = require_tooling_dependency( + ToolingDependency("colorama", "colorama", "colored status output"), + tool_name="nvbench-compare-legacy", + ) + Fore = colorama.Fore + + +def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]: + tabulate_module = require_tooling_dependency( + ToolingDependency("tabulate", "tabulate", "table output"), + tool_name="nvbench-compare-legacy", + ) + return tabulate_module, version_tuple(tabulate_module.__version__) + + +def load_jsondiff_for_device_diff() -> Any: + return require_tooling_dependency( + ToolingDependency("jsondiff", "jsondiff", "device metadata diffs"), + tool_name="nvbench-compare-legacy", + ) + + +GPU_TIME_MIN_TAG = "nv/cold/time/gpu/min" +GPU_TIME_MAX_TAG = "nv/cold/time/gpu/max" +GPU_TIME_MEAN_TAG = "nv/cold/time/gpu/mean" +GPU_TIME_STDEV_TAG = "nv/cold/time/gpu/stdev/absolute" +GPU_TIME_STDEV_RELATIVE_TAG = "nv/cold/time/gpu/stdev/relative" + + +def read_nvbench_json_root(filename: str) -> Mapping[str, Any]: + try: + root = reader.read_file(filename) + except (KeyError, OSError, TypeError, ValueError) as exc: + raise ValueError( + f"failed to read NVBench JSON file {filename!r}: {exc}" + ) from exc + + if not isinstance(root, Mapping): + raise ValueError(f"NVBench JSON file {filename!r} root must be an object") + + missing_keys = [key for key in ("devices", "benchmarks") if key not in root] + if missing_keys: + missing = ", ".join(repr(key) for key in missing_keys) + raise ValueError( + f"NVBench JSON file {filename!r} is missing required root key(s): {missing}" + ) + + for key in ("devices", "benchmarks"): + value = root[key] + if not isinstance(value, list): + raise ValueError( + f"NVBench JSON file {filename!r} root key {key!r} must be an array" + ) + for index, entry in enumerate(value): + if not isinstance(entry, Mapping): + raise ValueError( + f"NVBench JSON file {filename!r} root key {key!r} entry " + f"{index} must be an object" + ) + + return root + + +def format_json_structure_error(ref: str, comp: str, exc: Exception) -> str: + if isinstance(exc, KeyError) and exc.args: + detail = f"missing key {exc.args[0]!r}" + else: + detail = str(exc) or exc.__class__.__name__ + return ( + f"invalid NVBench JSON structure while comparing {ref!r} and {comp!r}: {detail}" + ) + + +@dataclass(frozen=True) +class GpuTimingData: + mean: float | None + stdev: float | None + stdev_relative: float | None + + +@dataclass(frozen=True) +class TimeEstimate: + center: float | None + relative_dispersion: float | None + + +@dataclass(frozen=True) +class TimingInterval: + lower: float + upper: float + center: float + + +@dataclass(frozen=True) +class TimingComparisonInputs: + ref_estimate: TimeEstimate + cmp_estimate: TimeEstimate + ref_interval: TimingInterval | None + cmp_interval: TimingInterval | None + + +class ComparisonStatus(str, Enum): + UNKNOWN = "????" + SAME = "SAME" + FAST = "FAST" + SLOW = "SLOW" + + +@dataclass(frozen=True) +class DecisionReason: + code: str + message: str + severity: float = 0.0 + + +@dataclass(frozen=True) +class TimingDecision: + status: ComparisonStatus + reason: DecisionReason + + +@dataclass(frozen=True) +class SummaryComparison: + ref_interval: TimingInterval | None + cmp_interval: TimingInterval | None + ref_estimate: TimeEstimate + cmp_estimate: TimeEstimate + ref_time: float | None + cmp_time: float | None + ref_noise: float | None + cmp_noise: float | None + diff: float | None + frac_diff: float | None + diff_interval: tuple[float, float] | None + frac_diff_interval: tuple[float, float] | None + min_noise: float | None + status: ComparisonStatus + reason: DecisionReason + + +@dataclass +class ComparisonStats: + config_count: int = 0 + pass_count: int = 0 + improvement_count: int = 0 + regression_count: int = 0 + unknown_count: int = 0 + + def record( + self, status: ComparisonStatus, reason: DecisionReason | None = None + ) -> None: + self.config_count += 1 + if status == ComparisonStatus.UNKNOWN: + self.unknown_count += 1 + elif status == ComparisonStatus.SAME: + self.pass_count += 1 + elif status == ComparisonStatus.FAST: + self.improvement_count += 1 + else: + self.regression_count += 1 + + +DeviceInfo = Mapping[str, Any] + + +@dataclass(frozen=True) +class ComparisonRunData: + # Device metadata fields are treated as read-only; stats is intentionally + # mutable and accumulates counts across one comparison run. + stats: ComparisonStats + ref_devices: tuple[DeviceInfo, ...] + cmp_devices: tuple[DeviceInfo, ...] + + +@dataclass(frozen=True) +class BenchmarkFilterScope: + benchmark_name: str + axis_filters: list[dict] + + +@dataclass(frozen=True) +class BenchmarkFilterPlan: + global_axis_filters: list[dict] + benchmark_scopes: list[BenchmarkFilterScope] + + +class OrderedBenchmarkFilterAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + actions = getattr(namespace, self.dest, None) + actions = [] if actions is None else list(actions) + action_kind = "axis" if option_string in {"-a", "--axis"} else "benchmark" + actions.append((action_kind, values)) + setattr(namespace, self.dest, actions) + + +def state_match_key(state): + device_prefix = f"Device={state['device']}" + state_name = state["name"] + if state_name == device_prefix: + return "" + if state_name.startswith(f"{device_prefix} "): + return state_name[len(device_prefix) + 1 :] + return state_name + + +def normalized_axis_values(state): + axis_values = state.get("axis_values") or [] + return tuple( + sorted( + ( + axis_value.get("name"), + axis_value.get("type"), + repr(axis_value.get("value")), + ) + for axis_value in axis_values + ) + ) + + +def state_comparison_key(state): + return state_match_key(state), normalized_axis_values(state) + + +def group_states_by_match_key(states): + grouped = {} + for state in states: + grouped.setdefault(state_comparison_key(state), []).append(state) + return grouped + + +def state_group_counts(grouped_states): + return Counter( + {state_name: len(states) for state_name, states in grouped_states.items()} + ) + + +def format_device_ids(device_ids): + return ", ".join(str(device_id) for device_id in device_ids) + + +def parse_device_filter(device_arg, option_name): + device_arg = device_arg.strip() + if device_arg.lower() == "all": + return None + + values = [value.strip() for value in device_arg.split(",")] + if not all(values): + raise ValueError( + f"{option_name} must be 'all', a non-negative integer, " + "or comma-separated non-negative integers" + ) + + try: + device_ids = [int(value) for value in values] + except ValueError as exc: + raise ValueError( + f"{option_name} must be 'all', a non-negative integer, " + "or comma-separated non-negative integers" + ) from exc + if any(device_id < 0 for device_id in device_ids): + raise ValueError( + f"{option_name} must be 'all', a non-negative integer, " + "or comma-separated non-negative integers" + ) + return device_ids + + +def select_devices(all_devices, device_filter, option_name): + if device_filter is None: + return list(all_devices) + + devices_by_id = {device["id"]: device for device in all_devices} + missing_ids = [ + device_id for device_id in device_filter if device_id not in devices_by_id + ] + if missing_ids: + raise ValueError( + f"{option_name} requested device id(s) not present in input: " + f"{format_device_ids(missing_ids)}" + ) + + return [devices_by_id[device_id] for device_id in device_filter] + + +def resolve_benchmark_device_ids(bench, device_filter, option_name): + if device_filter is None: + return list(bench["devices"]) + + benchmark_device_ids = set(bench["devices"]) + missing_ids = [ + device_id + for device_id in device_filter + if device_id not in benchmark_device_ids + ] + if missing_ids: + raise ValueError( + f"benchmark {bench['name']!r} does not contain {option_name} " + f"device id(s): {format_device_ids(missing_ids)}" + ) + + return device_filter + + +def require_matching_device_sections(reference_device_filter, compare_device_filter): + return reference_device_filter is None and compare_device_filter is None + + +# TODO(opavlyk): replace with Emoji(StrEnum) after EOL of Python 3.10 +class Emoji(str, Enum): + YELLOW = "\U0001f7e1" + BLUE = "\U0001f535" + GREEN = "\U0001f7e2" + RED = "\U0001f534" + SHRUG = "\U0001f937" + NONE = "" + + +def colorize(msg: str, fore: str, emoji: Emoji, no_color: bool) -> str: + if no_color: + prefix = "" + if emoji_s := emoji.value: + prefix = f"{emoji_s} " + return f"{prefix}{msg}" + else: + return f"{fore}{msg}{Fore.RESET}" + + +def lookup_summary(summaries, tag): + return next((summary for summary in summaries if summary["tag"] == tag), None) + + +def extract_summary_data_value(summary, name, expected_type): + summary_tag = summary.get("tag", "") + for value_data in summary.get("data", []): + if value_data.get("name") != name: + continue + + value_type = value_data.get("type") + if value_type != expected_type: + raise ValueError( + f"summary {summary_tag!r} field {name!r} has type " + f"{value_type!r}; expected {expected_type!r}" + ) + if "value" not in value_data: + raise ValueError(f"summary {summary_tag!r} field {name!r} is missing value") + return value_data["value"] + + raise ValueError(f"summary {summary_tag!r} is missing field {name!r}") + + +def extract_summary_value(summary): + return extract_summary_data_value(summary, "value", "float64") + + +def normalize_float_value(value, *, null_value=None): + if value is None: + return null_value + if isinstance(value, bool): + return null_value + return float(value) + + +def extract_summary_float(summaries, tag, *, null_value=None): + summary = lookup_summary(summaries, tag) + if summary is None: + return None + return normalize_float_value(extract_summary_value(summary), null_value=null_value) + + +def extract_gpu_timing_data( + summaries, json_dir=None, float32_reader=None, json_path=None +): + mean = extract_summary_float(summaries, GPU_TIME_MEAN_TAG) + stdev = extract_summary_float(summaries, GPU_TIME_STDEV_TAG, null_value=math.inf) + stdev_relative = extract_summary_float( + summaries, GPU_TIME_STDEV_RELATIVE_TAG, null_value=math.inf + ) + if stdev is None: + stdev = derive_absolute_dispersion(stdev_relative, mean) + + return GpuTimingData( + mean=mean, + stdev=stdev, + stdev_relative=stdev_relative, + ) + + +def make_empty_gpu_timing_data(): + return GpuTimingData( + mean=None, + stdev=None, + stdev_relative=None, + ) + + +def compute_relative_dispersion(dispersion, center): + if ( + dispersion is None + or center is None + or center <= 0 + or not math.isfinite(center) + or dispersion < 0 + or math.isnan(dispersion) + ): + return None + return dispersion / center + + +def is_finite(value): + return value is not None and math.isfinite(value) + + +def is_positive_finite(value): + return is_finite(value) and value > 0.0 + + +def is_nonnegative_finite(value): + return is_finite(value) and value >= 0.0 + + +def derive_absolute_dispersion(relative_dispersion, center): + if is_nonnegative_finite(relative_dispersion) and is_positive_finite(center): + return relative_dispersion * center + return None + + +def parse_plot_axis_value(axis_name, axis_value): + try: + value = float(axis_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"--plot-along requires numeric axis values; " + f"axis {axis_name!r} has value {axis_value!r}" + ) from exc + if not is_positive_finite(value): + raise ValueError( + f"--plot-along requires positive finite axis values; " + f"axis {axis_name!r} has value {axis_value!r}" + ) + return value + + +def make_decision(status, code, message, *, severity=0.0): + return TimingDecision( + status=status, + reason=DecisionReason(code=code, message=message, severity=severity), + ) + + +def select_relative_dispersion(relative_dispersion, absolute_dispersion, center): + if relative_dispersion is not None: + return relative_dispersion + return compute_relative_dispersion(absolute_dispersion, center) + + +def unusable_timing_center_decision(ref_time, cmp_time): + if ref_time is None or cmp_time is None: + return make_decision( + ComparisonStatus.UNKNOWN, + "timing_center_missing", + "timing center is missing", + ) + if not math.isfinite(ref_time) or not math.isfinite(cmp_time): + return make_decision( + ComparisonStatus.UNKNOWN, + "timing_center_nonfinite", + "timing center is non-finite", + ) + if ref_time <= 0.0 or cmp_time <= 0.0: + return make_decision( + ComparisonStatus.UNKNOWN, + "timing_center_nonpositive", + "timing center is non-positive", + ) + return None + + +def make_unavailable_timing_comparison(decision, timing_inputs): + return SummaryComparison( + ref_interval=timing_inputs.ref_interval, + cmp_interval=timing_inputs.cmp_interval, + ref_estimate=timing_inputs.ref_estimate, + cmp_estimate=timing_inputs.cmp_estimate, + ref_time=timing_inputs.ref_estimate.center, + cmp_time=timing_inputs.cmp_estimate.center, + ref_noise=timing_inputs.ref_estimate.relative_dispersion, + cmp_noise=timing_inputs.cmp_estimate.relative_dispersion, + diff=None, + frac_diff=None, + diff_interval=None, + frac_diff_interval=None, + min_noise=None, + status=decision.status, + reason=decision.reason, + ) + + +def compute_legacy_timing_comparison_inputs(ref_timing, cmp_timing): + return TimingComparisonInputs( + ref_estimate=TimeEstimate( + center=ref_timing.mean, + relative_dispersion=select_relative_dispersion( + ref_timing.stdev_relative, ref_timing.stdev, ref_timing.mean + ), + ), + cmp_estimate=TimeEstimate( + center=cmp_timing.mean, + relative_dispersion=select_relative_dispersion( + cmp_timing.stdev_relative, cmp_timing.stdev, cmp_timing.mean + ), + ), + ref_interval=None, + cmp_interval=None, + ) + + +def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None): + timing_inputs = compute_legacy_timing_comparison_inputs(ref_timing, cmp_timing) + ref_estimate = timing_inputs.ref_estimate + cmp_estimate = timing_inputs.cmp_estimate + + cmp_time = cmp_estimate.center + ref_time = ref_estimate.center + + cmp_noise = cmp_estimate.relative_dispersion + ref_noise = ref_estimate.relative_dispersion + + unusable_center_decision = unusable_timing_center_decision(ref_time, cmp_time) + if unusable_center_decision is not None: + return make_unavailable_timing_comparison( + unusable_center_decision, timing_inputs + ) + + diff = cmp_time - ref_time + frac_diff = diff / ref_time + + if not is_usable_noise(ref_noise) or not is_usable_noise(cmp_noise): + decision = make_decision( + ComparisonStatus.UNKNOWN, + "noise_unavailable", + "relative standard deviation is unavailable, negative, or non-finite", + ) + min_noise = None + else: + min_noise = min(ref_noise, cmp_noise) + if abs(frac_diff) <= min_noise: + decision = make_decision( + ComparisonStatus.SAME, + "same", + "absolute fractional difference is within relative standard deviation", + ) + elif diff < 0: + decision = make_decision( + ComparisonStatus.FAST, + "fast", + "compare timing mean is lower than reference timing mean", + ) + else: + decision = make_decision( + ComparisonStatus.SLOW, + "slow", + "compare timing mean is higher than reference timing mean", + ) + + return SummaryComparison( + ref_interval=None, + cmp_interval=None, + ref_estimate=ref_estimate, + cmp_estimate=cmp_estimate, + ref_time=ref_time, + cmp_time=cmp_time, + ref_noise=ref_noise, + cmp_noise=cmp_noise, + diff=diff, + frac_diff=frac_diff, + diff_interval=None, + frac_diff_interval=None, + min_noise=min_noise, + status=decision.status, + reason=decision.reason, + ) + + +def get_state_summaries(state: Mapping[str, Any]) -> list[dict[str, Any]]: + summaries = state.get("summaries") + return summaries if summaries is not None else [] + + +def state_has_summaries(state): + return bool(state.get("summaries")) + + +def format_skipped_state_reason(side, state): + reason = state.get("skip_reason") + if reason: + return f"{side} state skipped: {reason}" + return f"{side} state skipped" + + +def missing_state_summaries_decision(ref_state, cmp_state): + skipped_messages = [] + if ref_state.get("is_skipped"): + skipped_messages.append(format_skipped_state_reason("reference", ref_state)) + if cmp_state.get("is_skipped"): + skipped_messages.append(format_skipped_state_reason("compare", cmp_state)) + if skipped_messages: + return make_decision( + ComparisonStatus.UNKNOWN, + "state_skipped", + "; ".join(skipped_messages), + ) + + missing_sides = [] + if not state_has_summaries(ref_state): + missing_sides.append("reference") + if not state_has_summaries(cmp_state): + missing_sides.append("compare") + if not missing_sides: + return None + if len(missing_sides) == 2: + message = "reference and compare GPU timing summaries are missing" + else: + message = f"{missing_sides[0]} GPU timing summaries are missing" + return make_decision( + ComparisonStatus.UNKNOWN, + "gpu_timing_summaries_missing", + message, + ) + + +def find_matching_bench(needle, haystack): + for hay in haystack: + if hay["name"] == needle["name"]: + return hay + return None + + +def find_device_by_id(device_id, all_devices): + for device in all_devices: + if device["id"] == device_id: + return device + return None + + +def format_int64_axis_value(axis_name, axis_value, axes): + axis = next(filter(lambda ax: ax["name"] == axis_name, axes)) + axis_flags = axis["flags"] + value = int(axis_value["value"]) + if axis_flags == "pow2": + value = math.log2(value) + return f"2^{value:.0f}" + return f"{value:d}" + + +def format_float64_axis_value(axis_name, axis_value, axes): + return "%.5g" % float(axis_value["value"]) + + +def format_type_axis_value(axis_name, axis_value, axes): + return f"{axis_value['value']}" + + +def format_string_axis_value(axis_name, axis_value, axes): + return f"{axis_value['value']}" + + +def format_axis_value(axis_name, axis_value, axes): + axis = next(filter(lambda ax: ax["name"] == axis_name, axes)) + axis_type = axis["type"] + if axis_type == "int64": + return format_int64_axis_value(axis_name, axis_value, axes) + elif axis_type == "float64": + return format_float64_axis_value(axis_name, axis_value, axes) + elif axis_type == "type": + return format_type_axis_value(axis_name, axis_value, axes) + elif axis_type == "string": + return format_string_axis_value(axis_name, axis_value, axes) + raise ValueError(f"unsupported axis type {axis_type!r} for axis {axis_name!r}") + + +def make_display(name: str, display_values: list[str]) -> str: + open_bracket, close_bracket = ("[", "]") if len(display_values) > 1 else ("", "") + joined_values = ",".join(display_values) + return f"{name}={open_bracket}{joined_values}{close_bracket}" + + +def parse_axis_filters(axis_args): + filters = [] + for axis_arg in axis_args: + if "=" not in axis_arg: + raise ValueError(f"Axis filter must be NAME=VALUE: {axis_arg}") + name, value = axis_arg.split("=", 1) + name = name.strip() + value = value.strip() + if not name or not value: + raise ValueError(f"Axis filter must be NAME=VALUE: {axis_arg}") + + values = [] + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1].strip() + values = [ + stripped for item in inner.split(",") if (stripped := item.strip()) + ] + else: + values = [value] + display_values = list(values) + + if name.endswith("[pow2]"): + name = name[: -len("[pow2]")].strip() + if not name: + raise ValueError(f"Axis filter missing name before [pow2]: {axis_arg}") + try: + exponents = [int(v) for v in values] + except ValueError as exc: + raise ValueError( + f"Axis filter [pow2] value must be integer: {axis_arg}" + ) from exc + values = [str(2**exponent) for exponent in exponents] + display_values = [f"2^{exponent}" for exponent in exponents] + + if not values: + raise ValueError(f"Axis filter must specify at least one value: {axis_arg}") + + display = make_display(name, display_values) + filters.append( + { + "name": name, + "values": values, + "display": display, + } + ) + return filters + + +def build_benchmark_filter_plan(filter_actions): + global_axis_args = [] + benchmark_scopes = [] + current_scope = None + + for action_kind, action_value in filter_actions or []: + if action_kind == "benchmark": + current_scope = {"benchmark_name": action_value, "axis_args": []} + benchmark_scopes.append(current_scope) + elif current_scope is None: + global_axis_args.append(action_value) + else: + current_scope["axis_args"].append(action_value) + + return BenchmarkFilterPlan( + global_axis_filters=parse_axis_filters(global_axis_args), + benchmark_scopes=[ + BenchmarkFilterScope( + benchmark_name=scope["benchmark_name"], + axis_filters=parse_axis_filters(scope["axis_args"]), + ) + for scope in benchmark_scopes + ], + ) + + +def benchmark_is_selected(benchmark_name, filter_plan): + return bool(axis_filter_groups_for_benchmark(benchmark_name, filter_plan)) + + +def axis_filter_groups_for_benchmark(benchmark_name, filter_plan): + if not filter_plan.benchmark_scopes: + return [filter_plan.global_axis_filters] + + matching_scopes = [ + scope + for scope in filter_plan.benchmark_scopes + if scope.benchmark_name == benchmark_name + ] + + if matching_scopes: + return [ + filter_plan.global_axis_filters + scope.axis_filters + for scope in matching_scopes + ] + return [] + + +def matches_axis_filters(state, axis_filters): + if not axis_filters: + return True + + axis_values = state.get("axis_values") or [] + for axis_filter in axis_filters: + filter_name = axis_filter["name"] + filter_values = axis_filter["values"] + matched = False + for axis_value in axis_values: + if axis_value.get("name") != filter_name: + continue + value = axis_value.get("value") + if value is None: + continue + if str(value) in filter_values: + matched = True + break + if not matched: + return False + return True + + +def matches_axis_filter_groups(state, axis_filter_groups): + return any( + matches_axis_filters(state, axis_filters) for axis_filters in axis_filter_groups + ) + + +def matching_axis_filters(state, axis_filter_groups): + return next( + ( + axis_filters + for axis_filters in axis_filter_groups + if matches_axis_filters(state, axis_filters) + ), + [], + ) + + +def is_finite_number(value): + return value is not None and math.isfinite(value) + + +def format_duration(seconds, *, allow_negative=False, allow_zero=False): + if ( + not is_finite_number(seconds) + or (seconds < 0.0 and not allow_negative) + or (seconds == 0.0 and not allow_zero) + ): + return "n/a" + + if seconds >= 1: + multiplier = 1.0 + units = "s" + elif seconds >= 1e-3: + multiplier = 1e3 + units = "ms" + elif seconds >= 1e-6: + multiplier = 1e6 + units = "us" + else: + multiplier = 1e6 + units = "us" + return f"{seconds * multiplier:0.3f} {units}" + + +def select_duration_units(*seconds_values): + seconds_values = [value for value in seconds_values if is_finite_number(value)] + if not seconds_values: + return 1e6, "us" + + max_abs_seconds = max(abs(value) for value in seconds_values) + if max_abs_seconds >= 1: + return 1.0, "s" + if max_abs_seconds >= 1e-3: + return 1e3, "ms" + return 1e6, "us" + + +def duration_precision_for_center(center, delta_multiplier): + if not is_finite_number(center): + return 3 + + center_multiplier, _ = select_duration_units(center) + center_quantum = 10.0**-3 * (delta_multiplier / center_multiplier) + if center_quantum >= 1.0: + return 0 + return int(math.ceil(-math.log10(center_quantum))) + + +def format_duration_range(bounds): + if bounds is None: + return "n/a" + lower, upper = bounds + if not is_finite_number(lower) or not is_finite_number(upper): + return "n/a" + + multiplier, units = select_duration_units(lower, upper) + return f"[{lower * multiplier:0.2f}, {upper * multiplier:0.2f}] {units}" + + +def format_percentage(percentage): + if percentage is None: + return "n/a" + if math.isnan(percentage): + return "n/a" + if math.isinf(percentage): + return "inf" + return f"{percentage * 100.0:0.2f}%" + + +def get_display_headers(): + return ( + [ + "Ref Time", + "Ref Noise", + "Cmp Time", + "Cmp Noise", + "Diff", + "%Diff", + "Status", + ], + ["right", "right", "right", "right", "right", "right", "center"], + ) + + +def append_display_row(row, comparison, no_color): + row.append(format_duration(comparison.ref_time)) + row.append(format_percentage(comparison.ref_noise)) + row.append(format_duration(comparison.cmp_time)) + row.append(format_percentage(comparison.cmp_noise)) + row.append(format_duration(comparison.diff, allow_negative=True, allow_zero=True)) + row.append(format_percentage(comparison.frac_diff)) + row.append(colorize_comparison_status(comparison.status, no_color)) + + +def is_usable_noise(noise): + return is_nonnegative_finite(noise) + + +def colorize_comparison_status(status, no_color): + if status == ComparisonStatus.UNKNOWN: + fore_name = "YELLOW" + emoji = Emoji.YELLOW + elif status == ComparisonStatus.SAME: + fore_name = "BLUE" + emoji = Emoji.BLUE + elif status == ComparisonStatus.FAST: + fore_name = "GREEN" + emoji = Emoji.GREEN + else: + fore_name = "RED" + emoji = Emoji.RED + + fore = "" if no_color else getattr(Fore, fore_name) + return colorize(status.value, fore, emoji, no_color) + + +def format_axis_values(axis_values, axes, axis_filters=None): + if not axis_values: + return "" + filtered_names = set() + if axis_filters: + filtered_names = { + axis_filter["name"] + for axis_filter in axis_filters + if len(axis_filter["values"]) == 1 + } + parts = [] + for axis_value in axis_values: + axis_name = axis_value["name"] + if axis_name in filtered_names: + continue + formatted = format_axis_value(axis_name, axis_value, axes) + parts.append(f"{axis_name}={formatted}") + return " ".join(parts) + + +def plot_comparison_entries(entries, title=None, dark=False): + if not entries: + print("No comparison data to plot.") + return 1 + + matplotlib = require_tooling_dependency( + ToolingDependency("matplotlib", "matplotlib", "plot rendering"), + tool_name="nvbench-compare-legacy", + ) + if not os.environ.get("DISPLAY"): + matplotlib.use("Agg") + + plt = require_tooling_dependency( + ToolingDependency("matplotlib.pyplot", "matplotlib", "plot rendering"), + tool_name="nvbench-compare-legacy", + ) + ticker = require_tooling_dependency( + ToolingDependency("matplotlib.ticker", "matplotlib", "plot axis formatting"), + tool_name="nvbench-compare-legacy", + ) + PercentFormatter = ticker.PercentFormatter + + labels, values, statuses, bench_names = map(list, zip(*entries)) + + status_colors = { + "SLOW": "red", + "FAST": "green", + "SAME": "blue", + } + colors = [status_colors.get(status, "gray") for status in statuses] + + fig_height = max(4.0, 0.3 * len(entries) + 1.5) + fig, ax = plt.subplots(figsize=(10, fig_height)) + if dark: + fig.patch.set_facecolor("black") + ax.set_facecolor("black") + ax.tick_params(colors="white") + ax.xaxis.label.set_color("white") + ax.yaxis.label.set_color("white") + ax.title.set_color("white") + for spine in ax.spines.values(): + spine.set_color("white") + + y_pos = range(len(labels)) + ax.barh(y_pos, values, color=colors) + ax.set_yticks(y_pos) + ax.set_yticklabels(labels) + ax.invert_yaxis() + ax.set_ylim(len(labels) - 0.5, -0.5) + + separator_color = "white" if dark else "gray" + ax.axvline(0, color=separator_color, linewidth=1, alpha=0.6) + for index in range(1, len(bench_names)): + if bench_names[index] != bench_names[index - 1]: + ax.axhline(index - 0.5, color=separator_color, linewidth=0.6, alpha=0.4) + ax.xaxis.set_major_formatter(PercentFormatter(1.0)) + + if title: + ax.set_title(title) + + min_val = min(values) + max_val = max(values) + if min_val == max_val: + pad = 0.05 if min_val == 0 else abs(min_val) * 0.1 + ax.set_xlim(min_val - pad, max_val + pad) + else: + pad = (max_val - min_val) * 0.1 + ax.set_xlim(min_val - pad, max_val + pad) + + fig.tight_layout() + + if not os.environ.get("DISPLAY"): + output = "nvbench_compare.png" + fig.savefig(output, dpi=150) + print(f"Saved comparison plot to {output}") + else: + plt.show() + return 0 + + +def compare_benches( + run_data: ComparisonRunData, + ref_benches, + cmp_benches, + threshold, + plot_along, + plot, + dark, + filter_plan, + no_color, + reference_device_filter=None, + compare_device_filter=None, + ref_json_dir=None, + cmp_json_dir=None, + ref_json_path=None, + cmp_json_path=None, +): + if plot_along: + plt = require_tooling_dependency( + ToolingDependency( + "matplotlib.pyplot", "matplotlib", "per-axis plot rendering" + ), + tool_name="nvbench-compare-legacy", + ) + sns = require_tooling_dependency( + ToolingDependency("seaborn", "seaborn", "per-axis plot styling"), + tool_name="nvbench-compare-legacy", + ) + + sns.set_theme() + + comparison_entries = [] + comparison_device_names = set() + for cmp_bench in cmp_benches: + ref_bench = find_matching_bench(cmp_bench, ref_benches) + if not ref_bench: + continue + if not benchmark_is_selected(cmp_bench["name"], filter_plan): + continue + axis_filter_groups = axis_filter_groups_for_benchmark( + cmp_bench["name"], filter_plan + ) + + cmp_device_ids = resolve_benchmark_device_ids( + cmp_bench, compare_device_filter, "--compare-devices" + ) + ref_device_ids = resolve_benchmark_device_ids( + ref_bench, reference_device_filter, "--reference-devices" + ) + if len(cmp_device_ids) != len(ref_device_ids): + raise ValueError( + f"benchmark {cmp_bench['name']!r} has {len(ref_device_ids)} " + f"reference device(s) but {len(cmp_device_ids)} compare device(s); " + "nvbench_compare pairs devices by position, so each compared " + "benchmark must contain the same number of devices" + ) + + print(f"""# {cmp_bench["name"]}\n""") + + axes = cmp_bench["axes"] + ref_states = ref_bench["states"] + cmp_states = cmp_bench["states"] + + axes = axes if axes else [] + + headers = [x["name"] for x in axes] + colalign = ["center"] * len(headers) + display_headers, display_colalign = get_display_headers() + headers.extend(display_headers) + colalign.extend(display_colalign) + + for cmp_device_index, cmp_device_id in enumerate(cmp_device_ids): + ref_device_id = ref_device_ids[cmp_device_index] + ref_device_states = [ + state + for state in ref_states + if state["device"] == ref_device_id + and matches_axis_filter_groups(state, axis_filter_groups) + ] + cmp_device_states = [ + state + for state in cmp_states + if state["device"] == cmp_device_id + and matches_axis_filter_groups(state, axis_filter_groups) + ] + ref_states_by_name = group_states_by_match_key(ref_device_states) + cmp_states_by_name = group_states_by_match_key(cmp_device_states) + ref_state_counts = state_group_counts(ref_states_by_name) + cmp_state_counts = state_group_counts(cmp_states_by_name) + if ref_state_counts != cmp_state_counts: + raise ValueError( + f"benchmark {cmp_bench['name']!r} device pair " + f"ref={ref_device_id} cmp={cmp_device_id} has mismatched " + f"state occurrences: ref={dict(ref_state_counts)}, " + f"cmp={dict(cmp_state_counts)}" + ) + + rows = [] + plot_data: dict[str, dict[str, dict[float, float | None]]] = { + "cmp": {}, + "ref": {}, + "cmp_noise": {}, + "ref_noise": {}, + } + counters: dict[Any, int] = {} + + for cmp_state in cmp_device_states: + cmp_state_key = state_comparison_key(cmp_state) + occurrence = counters.get(cmp_state_key, 0) + counters[cmp_state_key] = occurrence + 1 + # Duplicate state names with identical axis values are matched + # by occurrence order within the filtered device section. + ref_state = ref_states_by_name[cmp_state_key][occurrence] + axis_values = cmp_state["axis_values"] + if not axis_values: + axis_values = [] + + row = [] + for axis_value in axis_values: + axis_value_name = axis_value["name"] + row.append(format_axis_value(axis_value_name, axis_value, axes)) + + cmp_summaries = get_state_summaries(cmp_state) + ref_summaries = get_state_summaries(ref_state) + + # TODO: Use other timings, too. Maybe multiple rows, with a + # "Timing" column + values "CPU/GPU/Batch"? + missing_summaries_decision = missing_state_summaries_decision( + ref_state, cmp_state + ) + if missing_summaries_decision is not None: + continue + + cmp_gpu_time = extract_gpu_timing_data( + cmp_summaries, cmp_json_dir, json_path=cmp_json_path + ) + ref_gpu_time = extract_gpu_timing_data( + ref_summaries, ref_json_dir, json_path=ref_json_path + ) + comparison = compare_gpu_timings(ref_gpu_time, cmp_gpu_time) + if comparison is None: + continue + + if ( + plot_along + and is_positive_finite(comparison.ref_time) + and is_positive_finite(comparison.cmp_time) + ): + axis_name_parts = [] + axis_value = None + for av in axis_values: + if av["name"] != plot_along: + axis_name_parts.append(f"""{av["name"]} = {av["value"]}""") + else: + axis_value = parse_plot_axis_value(av["name"], av["value"]) + if axis_value is not None: + axis_name = ", ".join(axis_name_parts) + + if axis_name not in plot_data["cmp"]: + plot_data["cmp"][axis_name] = {} + plot_data["ref"][axis_name] = {} + plot_data["cmp_noise"][axis_name] = {} + plot_data["ref_noise"][axis_name] = {} + + plot_data["cmp"][axis_name][axis_value] = comparison.cmp_time + plot_data["ref"][axis_name][axis_value] = comparison.ref_time + plot_data["cmp_noise"][axis_name][axis_value] = ( + comparison.cmp_noise + ) + plot_data["ref_noise"][axis_name][axis_value] = ( + comparison.ref_noise + ) + + run_data.stats.record(comparison.status, comparison.reason) + if comparison.status == ComparisonStatus.UNKNOWN or ( + comparison.frac_diff is not None + and abs(comparison.frac_diff) >= threshold + ): + axis_filters = matching_axis_filters(cmp_state, axis_filter_groups) + append_display_row(row, comparison, no_color) + + rows.append(row) + if ( + plot + and comparison.frac_diff is not None + and math.isfinite(comparison.frac_diff) + ): + axis_label = format_axis_values(axis_values, axes, axis_filters) + if axis_label: + label = f"""{cmp_bench["name"]} | {axis_label}""" + else: + label = cmp_bench["name"] + cmp_device = find_device_by_id( + cmp_state["device"], run_data.cmp_devices + ) + if cmp_device: + comparison_device_names.add(cmp_device["name"]) + comparison_entries.append( + ( + label, + comparison.frac_diff, + comparison.status.value, + cmp_bench["name"], + ) + ) + + if len(rows) == 0: + continue + + cmp_device = find_device_by_id(cmp_device_id, run_data.cmp_devices) + ref_device = find_device_by_id(ref_device_id, run_data.ref_devices) + if ref_device is None or cmp_device is None: + raise ValueError( + f"benchmark {cmp_bench['name']!r} references device pair " + f"ref={ref_device_id} cmp={cmp_device_id}, but device metadata is missing" + ) + + if cmp_device == ref_device: + print(f"## [{cmp_device['id']}] {cmp_device['name']}\n") + else: + print( + f"## [{ref_device['id']}] {ref_device['name']} vs. " + f"[{cmp_device['id']}] {cmp_device['name']}\n" + ) + tabulate, tabulate_version = load_tabulate_for_table_output() + # colalign and github format require tabulate 0.8.3 + if tabulate_version >= (0, 8, 3): + print( + tabulate.tabulate( + rows, headers=headers, colalign=colalign, tablefmt="github" + ) + ) + else: + print(tabulate.tabulate(rows, headers=headers, tablefmt="markdown")) + + print("") + + if plot_along: + fig = plt.figure() + try: + plt.xscale("log") + plt.yscale("log") + plt.xlabel(plot_along) + plt.ylabel("time [s]") + plt.title(cmp_device["name"]) + + def plot_line(key, shape, label, data_axis, data=plot_data): + axis_times = data[key][data_axis] + if not axis_times: + return + axis_noise = data[key + "_noise"][data_axis] + series = sorted( + ( + ( + float(axis_value), + axis_times[axis_value], + axis_noise[axis_value], + ) + for axis_value in axis_times + ), + key=lambda item: item[0], + ) + x, y, noise = map(list, zip(*series, strict=True)) + + p = plt.plot(x, y, shape, marker="o", label=label) + + def plot_confidence_band(first, last): + if last - first < 2: + return + + band_x = x[first:last] + band_y = y[first:last] + band_noise = noise[first:last] + top = [ + band_y[i] + band_y[i] * band_noise[i] + for i in range(len(band_x)) + ] + bottom = [ + max( + band_y[i] - band_y[i] * band_noise[i], + band_y[i] * 0.001, + ) + for i in range(len(band_x)) + ] + plt.fill_between( + band_x, bottom, top, color=p[0].get_color(), alpha=0.1 + ) + + start = None + for i, noise_value in enumerate(noise): + if is_usable_noise(noise_value) and start is None: + start = i + if not is_usable_noise(noise_value) and start is not None: + plot_confidence_band(start, i) + start = None + + if start is not None: + plot_confidence_band(start, len(x)) + + for axis in plot_data["cmp"].keys(): + plot_line("cmp", "-", axis, axis) + plot_line("ref", "--", axis + " ref", axis) + + plt.legend() + plt.show() + finally: + plt.close(fig) + + if plot: + title = "%SOL Bandwidth change" + if len(comparison_device_names) == 1: + title = f"{title} - {next(iter(comparison_device_names))}" + if filter_plan.global_axis_filters: + axis_label = ", ".join( + axis_filter["display"] + for axis_filter in filter_plan.global_axis_filters + if len(axis_filter["values"]) == 1 + ) + if axis_label: + title = f"{title} ({axis_label})" + plot_comparison_entries(comparison_entries, title=title, dark=dark) + + +def main() -> int: + """ + Returns a process exit code. + - 0 means the comparison completed successfully. + - 1 signals an error has occurred. + + The number of detected regressions is reported in the summary output. + """ + help_text = "%(prog)s [reference.json compare.json | reference_dir/ compare_dir/]" + parser = argparse.ArgumentParser(usage=help_text) + parser.add_argument( + "--ignore-devices", + dest="ignore_devices", + default=False, + help="Ignore differences in the device sections and compare anyway", + action="store_true", + ) + parser.add_argument( + "--threshold-diff", + type=float, + dest="threshold", + default=0.0, + help="only show rows where abs(%%Diff) is >= THRESHOLD percent", + ) + parser.add_argument( + "--plot-along", type=str, dest="plot_along", default=None, help="plot results" + ) + parser.add_argument( + "--plot", + dest="plot", + default=False, + help="plot comparison summary", + action="store_true", + ) + parser.add_argument( + "--dark", + action="store_true", + help="Use dark theme (black background, white text)", + ) + parser.add_argument( + "--no-color", + dest="no_color", + action="store_true", + help="Use emoji instead of ANSI color codes (useful for GitHub issues/PRs)", + ) + parser.add_argument( + "--reference-devices", + default="all", + help="Reference devices to compare: all, a non-negative integer id, or comma-separated ids", + ) + parser.add_argument( + "--compare-devices", + default="all", + help="Compare devices to compare: all, a non-negative integer id, or comma-separated ids", + ) + parser.add_argument( + "-a", + "--axis", + dest="filter_actions", + action=OrderedBenchmarkFilterAction, + help=( + "Filter on axis value, e.g. -a 'Elements{io}[pow2]=20'. Applies to the " + "most recent --benchmark, or all benchmarks if specified before any " + "--benchmark arguments." + ), + ) + parser.add_argument( + "-b", + "--benchmark", + dest="filter_actions", + action=OrderedBenchmarkFilterAction, + help="Filter by benchmark name (can repeat)", + ) + parser.add_argument("files_or_dirs", nargs="*") + + args = parser.parse_args() + files_or_dirs = args.files_or_dirs + + try: + filter_plan = build_benchmark_filter_plan(args.filter_actions) + reference_device_filter = parse_device_filter( + args.reference_devices, "--reference-devices" + ) + compare_device_filter = parse_device_filter( + args.compare_devices, "--compare-devices" + ) + except ValueError as exc: + print(str(exc)) + return 1 + + if len(files_or_dirs) != 2: + parser.print_help() + return 1 + + try: + load_nvbench_compare_tooling(load_color=not args.no_color) + except MissingToolingDependencyError as exc: + print(str(exc), file=sys.stderr) + return 1 + + # if provided two directories, find all the exactly named files + # in both and treat them as the reference and compare + to_compare = [] + if os.path.isdir(files_or_dirs[0]) and os.path.isdir(files_or_dirs[1]): + for f in os.listdir(files_or_dirs[1]): + if os.path.splitext(f)[1] != ".json": + continue + r = os.path.join(files_or_dirs[0], f) + c = os.path.join(files_or_dirs[1], f) + if ( + os.path.isfile(r) + and os.path.isfile(c) + and os.path.getsize(r) > 0 + and os.path.getsize(c) > 0 + ): + to_compare.append((r, c)) + else: + to_compare = [(files_or_dirs[0], files_or_dirs[1])] + + stats = ComparisonStats() + + for ref, comp in to_compare: + try: + ref_root = read_nvbench_json_root(ref) + cmp_root = read_nvbench_json_root(comp) + selected_ref_devices = select_devices( + ref_root["devices"], reference_device_filter, "--reference-devices" + ) + selected_cmp_devices = select_devices( + cmp_root["devices"], compare_device_filter, "--compare-devices" + ) + except ValueError as exc: + print(str(exc)) + return 1 + except (KeyError, TypeError, IndexError) as exc: + print(format_json_structure_error(ref, comp, exc)) + return 1 + + if len(selected_ref_devices) != len(selected_cmp_devices): + print( + f"--reference-devices selected {len(selected_ref_devices)} device(s), " + f"but --compare-devices selected {len(selected_cmp_devices)} device(s)" + ) + return 1 + + if selected_ref_devices != selected_cmp_devices: + try: + jsondiff = load_jsondiff_for_device_diff() + except MissingToolingDependencyError as exc: + print(str(exc), file=sys.stderr) + return 1 + + if args.no_color: + warn_fore = "" + else: + warn_fore = Fore.YELLOW if args.ignore_devices else Fore.RED + msg_text = "Device sections do not match" + print(colorize(msg_text, warn_fore, Emoji.NONE, args.no_color), end="") + print(": ", end="") + + print( + jsondiff.diff( + selected_ref_devices, selected_cmp_devices, syntax="symmetric" + ) + ) + if not args.ignore_devices and require_matching_device_sections( + reference_device_filter, compare_device_filter + ): + return 1 + + run_data = ComparisonRunData( + stats=stats, + ref_devices=tuple(selected_ref_devices), + cmp_devices=tuple(selected_cmp_devices), + ) + + try: + compare_benches( + run_data, + ref_root["benchmarks"], + cmp_root["benchmarks"], + threshold=args.threshold / 100.0, + plot_along=args.plot_along, + plot=args.plot, + dark=args.dark, + filter_plan=filter_plan, + no_color=args.no_color, + reference_device_filter=reference_device_filter, + compare_device_filter=compare_device_filter, + ref_json_dir=os.path.dirname(ref), + cmp_json_dir=os.path.dirname(comp), + ref_json_path=ref, + cmp_json_path=comp, + ) + except MissingToolingDependencyError as exc: + print(str(exc), file=sys.stderr) + return 1 + except ValueError as exc: + print(str(exc)) + return 1 + except (KeyError, TypeError, IndexError) as exc: + print(format_json_structure_error(ref, comp, exc)) + return 1 + + print("# Summary\n") + print(f"- Total Matches: {stats.config_count}") + print(f" - Pass (abs(%Diff) <= min_noise): {stats.pass_count}") + print( + " - Improvement (abs(%Diff) > min_noise, %Diff < 0): " + f"{stats.improvement_count}" + ) + print( + f" - Regression (abs(%Diff) > min_noise, %Diff > 0): {stats.regression_count}" + ) + print(f" - Unknown (infinite or unavailable noise): {stats.unknown_count}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())