mirror of
https://github.com/NVIDIA/nvbench.git
synced 2026-07-12 18:17:49 +00:00
Harden nvbench-compare plotting and filter docs
Skip UNKNOWN rows when collecting summary plot entries so non-numeric fractional differences cannot reach the plotting path. Add a regression test that exercises compare_benches(..., plot=True) with an UNKNOWN row. Document the supported pow2 axis-filter syntax and update the CLI help example to use NAME[pow2]=EXP, matching the parser behavior for axes displayed as 2^N. * Document when status ???? (UNKNOWN) is emitted * Clarify --no-color behavior * nvbench_compare.md: clarify --no-color behavior, fix example * Document display options in nvbench_compare.md * Small mention of plotting capabilities in nvbench_compare.md * Call out that example requires shell with process substitution capabilities
This commit is contained in:
@@ -2807,7 +2807,11 @@ def compare_benches(
|
||||
comparison=comparison,
|
||||
)
|
||||
)
|
||||
if plot:
|
||||
if (
|
||||
plot
|
||||
and comparison.frac_diff is not None
|
||||
and math.isfinite(comparison.frac_diff)
|
||||
):
|
||||
axis_label = format_axis_values(axis_values, axes, axis_filters)
|
||||
if axis_label:
|
||||
label = f"""{cmp_bench["name"]} | {axis_label}"""
|
||||
@@ -3038,7 +3042,7 @@ def main() -> int:
|
||||
dest="filter_actions",
|
||||
action=OrderedBenchmarkFilterAction,
|
||||
help=(
|
||||
"Filter on axis value, e.g. -a Elements{io}=2^20. Applies to the "
|
||||
"Filter on axis value, e.g. -a Elements{io}[pow2]=20. Applies to the "
|
||||
"most recent --benchmark, or all benchmarks if specified before any "
|
||||
"--benchmark arguments."
|
||||
),
|
||||
|
||||
@@ -2117,6 +2117,39 @@ def test_compare_benches_counts_missing_summaries_as_unknown(
|
||||
assert row[-1] == "\U0001f7e1 ????"
|
||||
|
||||
|
||||
def test_compare_benches_plot_skips_unknown_rows(monkeypatch, nvbench_compare):
|
||||
plotted_entries = []
|
||||
|
||||
def fake_plot_comparison_entries(entries, *args, **kwargs):
|
||||
plotted_entries.extend(entries)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
nvbench_compare, "plot_comparison_entries", fake_plot_comparison_entries
|
||||
)
|
||||
|
||||
run_data = make_comparison_run_data(nvbench_compare)
|
||||
ref_state = make_state(nvbench_compare, "state")
|
||||
del ref_state["summaries"]
|
||||
cmp_state = make_state(nvbench_compare, "state", mean="1.0")
|
||||
|
||||
nvbench_compare.compare_benches(
|
||||
run_data,
|
||||
[make_benchmark([ref_state])],
|
||||
[make_benchmark([cmp_state])],
|
||||
threshold=1.0,
|
||||
plot_along=None,
|
||||
plot=True,
|
||||
dark=False,
|
||||
filter_plan=make_filter_plan(nvbench_compare),
|
||||
no_color=True,
|
||||
)
|
||||
|
||||
assert run_data.stats.config_count == 1
|
||||
assert run_data.stats.unknown_count == 1
|
||||
assert plotted_entries == []
|
||||
|
||||
|
||||
def test_compare_benches_defaults_to_interval_display(monkeypatch, nvbench_compare):
|
||||
run_data = make_comparison_run_data(nvbench_compare)
|
||||
tabulate_calls = capture_tabulate_calls(monkeypatch, nvbench_compare)
|
||||
|
||||
Reference in New Issue
Block a user