NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce UNDECIDED comparison status

Browse Source 
It is not emitted just yet, but the code becomes ready for it
when it starts being emitted
This commit is contained in:
Oleksandr Pavlyk
2026-06-02 15:23:41 -05:00
parent 737794f1e6
commit 418ce129e9
2 changed files with 22 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/scripts/nvbench_compare.py
View File

@@ -67,6 +67,7 @@ class TimeEstimate:
class ComparisonStatus(str, Enum):
UNKNOWN = "????"
UNDECIDED = "UNDECIDED"
SAME = "SAME"
FAST = "FAST"
SLOW = "SLOW"
@@ -92,12 +93,15 @@ class ComparisonStats:
pass_count: int = 0
improvement_count: int = 0
regression_count: int = 0
undecided_count: int = 0
unknown_count: int = 0
def record(self, status: ComparisonStatus) -> None:
self.config_count += 1
if status == ComparisonStatus.UNKNOWN:
self.unknown_count += 1
elif status == ComparisonStatus.UNDECIDED:
self.undecided_count += 1
elif status == ComparisonStatus.SAME:
self.pass_count += 1
elif status == ComparisonStatus.FAST:
@@ -753,6 +757,8 @@ def has_finite_noise(noise):
def colorize_comparison_status(status, no_color):
if status == ComparisonStatus.UNKNOWN:
return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
if status == ComparisonStatus.UNDECIDED:
return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
if status == ComparisonStatus.SAME:
return colorize(status.value, Fore.BLUE, Emoji.BLUE, no_color)
if status == ComparisonStatus.FAST:
@@ -1351,6 +1357,9 @@ def main() -> int:
print(
f" - Regression (abs(%Diff) > max_noise, %Diff > 0): {stats.regression_count}"
)
print(
f" - Undecided (comparison requires more evidence): {stats.undecided_count}"
)
print(f" - Unknown (infinite or unavailable noise): {stats.unknown_count}")
return 0