From 78f70b097ff90726c8df79dc02c840c4f106d224 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:55:42 -0500 Subject: [PATCH] Replaced UNDECIDED with AMBG, use Gray color/shrug emoji --- docs/nvbench_compare.md | 24 ++++++++++++------------ python/scripts/nvbench_compare.py | 11 +++++------ python/test/test_nvbench_compare.py | 16 +++++++++++++--- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/docs/nvbench_compare.md b/docs/nvbench_compare.md index 49ca96f..d0c0054 100644 --- a/docs/nvbench_compare.md +++ b/docs/nvbench_compare.md @@ -1,7 +1,7 @@ # NVBench Compare `nvbench-compare` compares two NVBench JSON outputs and classifies matching -benchmark states as `SAME`, `FAST`, `SLOW`, `UNDECIDED`, or `????`. +benchmark states as `SAME`, `FAST`, `SLOW`, `AMBG`, or `????`. NVBench treats benchmark performance data as describing a timing interval over which measured timings varied. The interval is not intended as a precise @@ -11,7 +11,7 @@ separated, clearly compatible, or ambiguous. The comparison is intentionally conservative. It reports `FAST` or `SLOW` only when the timing intervals have a clear gap and the gap is confirmed in cycle -space when clock information is available. Ambiguous cases stay `UNDECIDED` +space when clock information is available. Ambiguous cases stay `AMBG` instead of forcing a pass or regression. ## Common Invocations @@ -192,11 +192,11 @@ row = bulk_rows[0] arrays = load_bulk_data(row) ``` -Select the second undecided row: +Select the second ambiguous row: ```python -undecided = [row for row in bulk_rows if row["status"] == "UNDECIDED"] -row = undecided[1] +ambiguous = [row for row in bulk_rows if row["status"] == "AMBG"] +row = ambiguous[1] arrays = load_bulk_data(row) ``` @@ -224,7 +224,7 @@ centers are not compared. ## Decision Tree -The comparison logic starts from `UNDECIDED` and upgrades only when enough +The comparison logic starts from `AMBG` and upgrades only when enough evidence is available. ### 1. Check For A Clear Gap @@ -260,11 +260,11 @@ cycles = sample_time * sample_frequency It then builds cycle intervals from the bulk cycle samples and requires the cycle interval comparison to agree with the timing interval comparison. A timing -gap that is not confirmed by bulk cycle intervals is `UNDECIDED`. +gap that is not confirmed by bulk cycle intervals is `AMBG`. If bulk data are unavailable, `nvbench-compare` falls back to summary clock-rate confirmation using `sm_clock_rate/mean`. If that clock-rate summary is missing -or invalid, the clear-gap decision remains `UNDECIDED`. +or invalid, the clear-gap decision remains `AMBG`. ### 3. Check Bulk-Data Compatibility For SAME @@ -311,14 +311,14 @@ If `sm_clock_rate/mean` is available on both sides, the same check must also be confirmed in summary cycle space. If clock-rate summaries are unavailable, the summary timing decision can still report `SAME`. -### 5. Otherwise Report UNDECIDED +### 5. Otherwise Report AMBG If none of the clear-gap or same-result paths has enough evidence, -`nvbench-compare` reports `UNDECIDED` and records a reason in the summary. +`nvbench-compare` reports `AMBG` and records a reason in the summary. -## What To Do With UNDECIDED Results +## What To Do With AMBG Results -`UNDECIDED` does not mean a benchmark improved or regressed. It means +`AMBG` does not mean a benchmark improved or regressed. It means `nvbench-compare` did not find enough evidence to classify the result as `SAME`, `FAST`, or `SLOW`. diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 230e48d..28c7e6b 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -428,7 +428,7 @@ class TimingInterval: class ComparisonStatus(str, Enum): UNKNOWN = "????" - UNDECIDED = "UNDECIDED" + UNDECIDED = "AMBG" SAME = "SAME" FAST = "FAST" SLOW = "SLOW" @@ -702,6 +702,7 @@ class Emoji(str, Enum): BLUE = "\U0001f535" GREEN = "\U0001f7e2" RED = "\U0001f534" + SHRUG = "\U0001f937" NONE = "" @@ -1026,7 +1027,7 @@ def format_bulk_debug_python(bulk_rows: list[dict[str, Any]]) -> str: "# Examples:\n" "# row = bulk_rows[0]\n" "# arrays = load_bulk_data(row)\n" - "# undecided = [row for row in bulk_rows if row['status'] == 'UNDECIDED']\n" + "# ambiguous = [row for row in bulk_rows if row['status'] == 'AMBG']\n" ) @@ -2288,7 +2289,7 @@ def colorize_comparison_status(status, no_color): if status == ComparisonStatus.UNKNOWN: return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color) if status == ComparisonStatus.UNDECIDED: - return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color) + return colorize(status.value, Fore.LIGHTBLACK_EX, Emoji.SHRUG, no_color) if status == ComparisonStatus.SAME: return colorize(status.value, Fore.BLUE, Emoji.BLUE, no_color) if status == ComparisonStatus.FAST: @@ -2954,9 +2955,7 @@ def main() -> int: print(f" - Pass (centers close and intervals overlap): {stats.pass_count}") print(f" - Improvement (clear timing gap, %Diff < 0): {stats.improvement_count}") print(f" - Regression (clear timing gap, %Diff > 0): {stats.regression_count}") - print( - f" - Undecided (comparison requires more evidence): {stats.undecided_count}" - ) + print(f" - Ambiguous (comparison requires more evidence): {stats.undecided_count}") if stats.undecided_reasons: print(" - Reasons:") for code, reason_summary in sorted( diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index 9f0fdee..f60461f 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -55,6 +55,7 @@ def nvbench_compare(monkeypatch): Fore=types.SimpleNamespace( BLUE="", GREEN="", + LIGHTBLACK_EX="", RED="", RESET="", YELLOW="", @@ -900,6 +901,15 @@ def test_format_change_only_reports_fast_and_slow_rows(nvbench_compare): assert nvbench_compare.format_change(undecided) == "" +def test_ambiguous_status_uses_shrug_marker(nvbench_compare): + assert ( + nvbench_compare.colorize_comparison_status( + nvbench_compare.ComparisonStatus.UNDECIDED, no_color=True + ) + == "\U0001f937 AMBG" + ) + + def test_format_timing_with_interval(nvbench_compare): interval = nvbench_compare.TimingInterval( lower=0.002237, upper=0.002389, center=0.0023 @@ -1601,7 +1611,7 @@ def test_main_prints_undecided_reason_summary(monkeypatch, capsys, nvbench_compa assert nvbench_compare.main() == 0 output = capsys.readouterr().out - assert "Undecided (comparison requires more evidence): 1" in output + assert "Ambiguous (comparison requires more evidence): 1" in output assert "noise_too_high: 1" in output assert "Reason legend: noise-high = noise_too_high" in output @@ -1810,7 +1820,7 @@ def test_main_prints_bulk_debug_python_to_stdout(monkeypatch, capsys, nvbench_co kwargs["bulk_debug_rows"].append( { "row_index": 0, - "status": "UNDECIDED", + "status": "AMBG", "reference_sample_filename": None, "reference_sample_count": None, "reference_frequency_filename": None, @@ -1838,7 +1848,7 @@ def test_main_prints_bulk_debug_python_to_stdout(monkeypatch, capsys, nvbench_co assert nvbench_compare.main() == 0 output = capsys.readouterr().out assert "bulk_rows = [" in output - assert "'status': 'UNDECIDED'" in output + assert "'status': 'AMBG'" in output assert "def load_bulk_data(row):" in output