Harden nvbench-compare plotting and filter docs

Skip UNKNOWN rows when collecting summary plot entries so non-numeric
fractional differences cannot reach the plotting path. Add a regression test
that exercises compare_benches(..., plot=True) with an UNKNOWN row.

Document the supported pow2 axis-filter syntax and update the CLI help example
to use NAME[pow2]=EXP, matching the parser behavior for axes displayed as 2^N.

* Document when status ???? (UNKNOWN) is emitted
* Clarify --no-color behavior

* nvbench_compare.md: clarify --no-color behavior, fix example

* Document display options in nvbench_compare.md

* Small mention of plotting capabilities in nvbench_compare.md
* Call out that example requires shell with process substitution capabilities
This commit is contained in:
Oleksandr Pavlyk
2026-06-25 09:07:29 -05:00
parent 118382daf1
commit 4fabe3fde9
3 changed files with 79 additions and 12 deletions

View File

@@ -2807,7 +2807,11 @@ def compare_benches(
comparison=comparison,
)
)
if plot:
if (
plot
and comparison.frac_diff is not None
and math.isfinite(comparison.frac_diff)
):
axis_label = format_axis_values(axis_values, axes, axis_filters)
if axis_label:
label = f"""{cmp_bench["name"]} | {axis_label}"""
@@ -3038,7 +3042,7 @@ def main() -> int:
dest="filter_actions",
action=OrderedBenchmarkFilterAction,
help=(
"Filter on axis value, e.g. -a Elements{io}=2^20. Applies to the "
"Filter on axis value, e.g. -a Elements{io}[pow2]=20. Applies to the "
"most recent --benchmark, or all benchmarks if specified before any "
"--benchmark arguments."
),

View File

@@ -2117,6 +2117,39 @@ def test_compare_benches_counts_missing_summaries_as_unknown(
assert row[-1] == "\U0001f7e1 ????"
def test_compare_benches_plot_skips_unknown_rows(monkeypatch, nvbench_compare):
plotted_entries = []
def fake_plot_comparison_entries(entries, *args, **kwargs):
plotted_entries.extend(entries)
return 0
monkeypatch.setattr(
nvbench_compare, "plot_comparison_entries", fake_plot_comparison_entries
)
run_data = make_comparison_run_data(nvbench_compare)
ref_state = make_state(nvbench_compare, "state")
del ref_state["summaries"]
cmp_state = make_state(nvbench_compare, "state", mean="1.0")
nvbench_compare.compare_benches(
run_data,
[make_benchmark([ref_state])],
[make_benchmark([cmp_state])],
threshold=1.0,
plot_along=None,
plot=True,
dark=False,
filter_plan=make_filter_plan(nvbench_compare),
no_color=True,
)
assert run_data.stats.config_count == 1
assert run_data.stats.unknown_count == 1
assert plotted_entries == []
def test_compare_benches_defaults_to_interval_display(monkeypatch, nvbench_compare):
run_data = make_comparison_run_data(nvbench_compare)
tabulate_calls = capture_tabulate_calls(monkeypatch, nvbench_compare)