NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-04-20 14:58:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Report pass/fail status

Browse Source
This commit is contained in:
Ashwin Srinath
2021-05-26 13:26:56 -04:00
committed by Allison Vacanti
parent 38ec66dc0f
commit 0f63f86e9f
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/nvbench_compare.py
View File

@@ -1,9 +1,11 @@
#!/usr/bin/env python
from colorama import Style, Fore
import json
import keyword
import sys
import tabulate
if len(sys.argv) != 3:
@@ -68,13 +70,18 @@ def get_row(cmp_benches, ref_benches):
cmp_noise = cmp_noise_summary["value"]["value"]
ref_noise = ref_noise_summary["value"]["value"]
# pass/fail status
failed = (cmp_noise - ref_noise) > 2 * (((cmp_noise / 100.) * cmp_time) + ((ref_noise / 100.) * ref_time))
status = (Fore.RED + "FAIL" if failed else Fore.GREEN + "PASS") + Fore.RESET
# Relative time comparison
yield [cmp_bench['name'], cmp_state_description] + f"{cmp_time - ref_time} {cmp_time} {ref_time} {cmp_noise:0.6f}% {ref_noise:0.6f}%\n".split()
yield [cmp_bench['name'], cmp_state_description] + f"{cmp_time - ref_time} {cmp_time} {ref_time} {cmp_noise:0.6f}% {ref_noise:0.6f}% {status}\n".split()
print(tabulate.tabulate((row for row in get_row(cmp_benches, ref_benches)),
# TODO: Reduce precision once we have really different
# numbers for comparison.
floatfmt="0.12f",
headers=("Name", "Parameters", "Old - New", "New Time", "Old Time", "New Std", "Old Std"),
headers=("Name", "Parameters", "Old - New", "New Time", "Old Time", "New Std", "Old Std", "Status"),
))