Add TOML configuration for nvbench-compare thresholds

Add versioned TOML configuration support for nvbench-compare threshold
settings. The new --config option reads grouped settings for clear-gap,
same-result, bulk coverage, and rare-support filtering thresholds. The parser
validates the schema strictly so unknown tables, unknown keys, invalid types,
unsupported versions, and out-of-range values fail early.

Add --dump-config to print the effective configuration without requiring input
JSON files. This makes the currently selected preset and resolved threshold
values discoverable and gives users a starting point for custom configuration.

Preset resolution is:
  - default is used when neither TOML nor CLI selects a preset
  - [preset] name = "..." in TOML selects the base preset
  - --preset ... overrides the TOML preset selection
  - explicit threshold values in TOML override whichever base preset was selected

For example:
  - nvbench-compare --dump-config
    Prints the built-in default settings as grouped TOML.

  - nvbench-compare --preset permissive --dump-config
    Prints the permissive preset values as TOML.

  - nvbench-compare --config compare.toml ref.json cmp.json
    Compares using the preset named in compare.toml, plus any explicit TOML
    threshold overrides.

  - nvbench-compare --config compare.toml --preset strict ref.json cmp.json
    Uses the strict preset as the base, while preserving explicit threshold
    overrides from compare.toml.

Keep TOML parsing lazy: Python 3.11+ uses tomllib, while Python 3.10 only
requires tomli when --config is used. Add focused tests for grouped config
dumping, strict validation, preset/override precedence, and CLI dump behavior.
This commit is contained in:
Oleksandr Pavlyk
2026-06-04 09:55:58 -05:00
parent 2585842cf5
commit 732d227be1
2 changed files with 433 additions and 4 deletions

View File

@@ -1380,6 +1380,181 @@ def test_get_comparison_thresholds_returns_named_presets(nvbench_compare):
assert permissive.bulk_same_support_coverage < default.bulk_same_support_coverage
def test_dump_comparison_config_uses_grouped_toml(nvbench_compare):
config = nvbench_compare.dump_comparison_config(
"default", nvbench_compare.get_comparison_thresholds("default")
)
assert "version = 1\n" in config
assert '[preset]\nname = "default"\n' in config
assert "[clear_gap]\nrelative = 0.005\n" in config
assert "[same]\n" in config
assert "[bulk]\n" in config
assert "sample_coverage = 0.97\n" in config
assert "[bulk.rare_support]\n" in config
def test_resolve_comparison_thresholds_applies_config_overrides(
monkeypatch, nvbench_compare
):
def read_config(_):
return (
"strict",
{
"bulk_same_sample_coverage": 0.93,
"bulk_support_max_removed_sample_fraction": 0.02,
},
)
monkeypatch.setattr(nvbench_compare, "read_comparison_config_file", read_config)
preset, thresholds = nvbench_compare.resolve_comparison_thresholds(
None, "settings.toml"
)
assert preset == "strict"
assert thresholds.clear_gap_relative == pytest.approx(
nvbench_compare.get_comparison_thresholds("strict").clear_gap_relative
)
assert thresholds.bulk_same_sample_coverage == pytest.approx(0.93)
assert thresholds.bulk_support_max_removed_sample_fraction == pytest.approx(0.02)
preset, thresholds = nvbench_compare.resolve_comparison_thresholds(
"permissive", "settings.toml"
)
assert preset == "permissive"
assert thresholds.clear_gap_relative == pytest.approx(
nvbench_compare.get_comparison_thresholds("permissive").clear_gap_relative
)
assert thresholds.bulk_same_sample_coverage == pytest.approx(0.93)
assert thresholds.bulk_support_max_removed_sample_fraction == pytest.approx(0.02)
def test_parse_comparison_config_data_validates_grouped_thresholds(nvbench_compare):
preset, overrides = nvbench_compare.parse_comparison_config_data(
{
"version": 1,
"preset": {"name": "strict"},
"clear_gap": {"relative": 0.01},
"same": {
"center_relative": 0.002,
"overlap_fraction": 0.75,
"relative_dispersion_ceiling": 0.02,
},
"bulk": {
"sample_coverage": 0.99,
"support_coverage": 0.8,
"rare_support": {
"sample_fraction": 0.001,
"max_removed_sample_fraction": 0.01,
},
},
}
)
assert preset == "strict"
assert overrides == {
"clear_gap_relative": 0.01,
"same_center_relative": 0.002,
"same_overlap_fraction": 0.75,
"same_relative_dispersion_ceiling": 0.02,
"bulk_same_sample_coverage": 0.99,
"bulk_same_support_coverage": 0.8,
"bulk_support_rare_sample_fraction": 0.001,
"bulk_support_max_removed_sample_fraction": 0.01,
}
@pytest.mark.parametrize(
"config_data, match",
[
({}, "version"),
({"version": 2}, "unsupported"),
({"version": 1, "rare_support": {}}, "unknown top-level"),
({"version": 1, "bulk": {"unknown": 0.1}}, r"\[bulk\]"),
({"version": 1, "clear_gap": {"rare_support": {}}}, r"\[clear_gap\]"),
({"version": 1, "bulk": {"sample_coverage": 1.5}}, "<= 1"),
({"version": 1, "same": {"center_relative": "tight"}}, "finite number"),
({"version": 1, "preset": {"name": "aggressive"}}, "unknown comparison preset"),
],
)
def test_parse_comparison_config_data_rejects_invalid_config(
nvbench_compare, config_data, match
):
with pytest.raises(ValueError, match=match):
nvbench_compare.parse_comparison_config_data(config_data)
def test_read_comparison_config_file_parses_toml_when_parser_is_available(
tmp_path, nvbench_compare
):
parser_module = "tomllib" if sys.version_info >= (3, 11) else "tomli"
pytest.importorskip(parser_module)
config_path = tmp_path / "settings.toml"
config_path.write_text(
"""
version = 1
[preset]
name = "strict"
[bulk]
sample_coverage = 0.93
""",
encoding="utf-8",
)
preset, overrides = nvbench_compare.read_comparison_config_file(config_path)
assert preset == "strict"
assert overrides == {"bulk_same_sample_coverage": 0.93}
def test_main_dump_config_does_not_require_input_files(
monkeypatch, capsys, nvbench_compare
):
def read_file(_):
raise AssertionError("dump-config should not read JSON files")
monkeypatch.setattr(nvbench_compare.reader, "read_file", read_file)
monkeypatch.setattr(
sys,
"argv",
["nvbench_compare", "--preset", "strict", "--dump-config"],
)
assert nvbench_compare.main() == 0
output = capsys.readouterr().out
assert 'name = "strict"' in output
assert "[bulk.rare_support]" in output
def test_main_dump_config_merges_config_and_cli_preset(
monkeypatch, capsys, nvbench_compare
):
def read_config(_):
return ("strict", {"bulk_same_sample_coverage": 0.93})
monkeypatch.setattr(nvbench_compare, "read_comparison_config_file", read_config)
monkeypatch.setattr(
sys,
"argv",
[
"nvbench_compare",
"--config",
"settings.toml",
"--preset",
"permissive",
"--dump-config",
],
)
assert nvbench_compare.main() == 0
output = capsys.readouterr().out
assert 'name = "permissive"' in output
assert "relative = 0.0025" in output
assert "sample_coverage = 0.93" in output
def test_compare_benches_defaults_to_interval_display(monkeypatch, nvbench_compare):
run_data = make_comparison_run_data(nvbench_compare)
captured = {}