NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce UNDECIDED comparison status

Browse Source 
It is not emitted just yet, but the code becomes ready for it
when it starts being emitted
This commit is contained in:
Oleksandr Pavlyk
2026-06-02 15:23:41 -05:00
parent d4283f77a5
commit 6d8aa878cf
2 changed files with 22 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/scripts/nvbench_compare.py
View File

@@ -67,6 +67,7 @@ class TimeEstimate:
class ComparisonStatus(str, Enum):
UNKNOWN = "????"
UNDECIDED = "UNDECIDED"
SAME = "SAME"
FAST = "FAST"
SLOW = "SLOW"
@@ -92,12 +93,15 @@ class ComparisonStats:
pass_count: int = 0
improvement_count: int = 0
regression_count: int = 0
undecided_count: int = 0
unknown_count: int = 0
def record(self, status: ComparisonStatus) -> None:
self.config_count += 1
if status == ComparisonStatus.UNKNOWN:
self.unknown_count += 1
elif status == ComparisonStatus.UNDECIDED:
self.undecided_count += 1
elif status == ComparisonStatus.SAME:
self.pass_count += 1
elif status == ComparisonStatus.FAST:
@@ -753,6 +757,8 @@ def has_finite_noise(noise):
def colorize_comparison_status(status, no_color):
if status == ComparisonStatus.UNKNOWN:
return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
if status == ComparisonStatus.UNDECIDED:
return colorize(status.value, Fore.YELLOW, Emoji.YELLOW, no_color)
if status == ComparisonStatus.SAME:
return colorize(status.value, Fore.BLUE, Emoji.BLUE, no_color)
if status == ComparisonStatus.FAST:
@@ -1351,6 +1357,9 @@ def main() -> int:
print(
f" - Regression (abs(%Diff) > max_noise, %Diff > 0): {stats.regression_count}"
)
print(
f" - Undecided (comparison requires more evidence): {stats.undecided_count}"
)
print(f" - Unknown (infinite or unavailable noise): {stats.unknown_count}")
return 0