From 418ce129e9e4c39c459cbe878b431ff99f57dd46 Mon Sep 17 00:00:00 2001
From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com>
Date: Tue, 2 Jun 2026 15:23:41 -0500
Subject: [PATCH] Introduce UNDECIDED comparison status

It is not emitted just yet, but the code becomes ready for it
when it starts being emitted
---
 python/scripts/nvbench_compare.py   |  9 +++++++++
 python/test/test_nvbench_compare.py | 13 +++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py
index a2cd55c..180e42d 100644
--- a/python/scripts/nvbench_compare.py
+++ b/python/scripts/nvbench_compare.py
@@ -67,6 +67,7 @@ class TimeEstimate:
 
 class ComparisonStatus(str, Enum):
     UNKNOWN = "????"
+    UNDECIDED = "UNDECIDED"
     SAME = "SAME"
     FAST = "FAST"
     SLOW = "SLOW"
@@ -92,12 +93,15 @@ class ComparisonStats:
     pass_count: int = 0
     improvement_count: int = 0
     regression_count: int = 0
+    undecided_count: int = 0
     unknown_count: int = 0
 
     def record(self, status: ComparisonStatus) -> None:
         self.config_count += 1
         if status == ComparisonStatus.UNKNOWN:
             self.unknown_count += 1
+        elif status == ComparisonStatus.UNDECIDED:
+            self.undecided_count += 1
         elif status == ComparisonStatus.SAME:
             self.pass_count += 1
         elif status == ComparisonStatus.FAST:
@@ -753,6 +757,8 @@ def has_finite_noise(noise):
 def colorize_comparison_status(status, no_color):
     if status == ComparisonStatus.UNKNOWN:
         return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
+    if status == ComparisonStatus.UNDECIDED:
+        return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
     if status == ComparisonStatus.SAME:
         return colorize(status.value, Fore.BLUE, Emoji.BLUE, no_color)
     if status == ComparisonStatus.FAST:
@@ -1351,6 +1357,9 @@ def main() -> int:
     print(
         f"  - Regression  (abs(%Diff) > max_noise, %Diff > 0): {stats.regression_count}"
     )
+    print(
+        f"  - Undecided   (comparison requires more evidence): {stats.undecided_count}"
+    )
     print(f"  - Unknown     (infinite or unavailable noise): {stats.unknown_count}")
     return 0
 
diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py
index 6c0a224..2250ca1 100644
--- a/python/test/test_nvbench_compare.py
+++ b/python/test/test_nvbench_compare.py
@@ -423,6 +423,19 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare):
     assert unknown.status == nvbench_compare.ComparisonStatus.UNKNOWN
 
 
+def test_comparison_stats_records_undecided_status(nvbench_compare):
+    stats = nvbench_compare.ComparisonStats()
+
+    stats.record(nvbench_compare.ComparisonStatus.UNDECIDED)
+
+    assert stats.config_count == 1
+    assert stats.pass_count == 0
+    assert stats.improvement_count == 0
+    assert stats.regression_count == 0
+    assert stats.undecided_count == 1
+    assert stats.unknown_count == 0
+
+
 @pytest.mark.parametrize("ref_time, cmp_time", [(None, 1.0), (1.0, None), (0.0, 1.0)])
 def test_compare_gpu_timings_rejects_unusable_centers(
     nvbench_compare, ref_time, cmp_time