diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index c96a4d4..e75f158 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -102,6 +102,16 @@ def read_nvbench_json_root(filename: str) -> Mapping[str, Any]: return root +def format_json_structure_error(ref: str, comp: str, exc: Exception) -> str: + if isinstance(exc, KeyError) and exc.args: + detail = f"missing key {exc.args[0]!r}" + else: + detail = str(exc) or exc.__class__.__name__ + return ( + f"invalid NVBench JSON structure while comparing {ref!r} and {comp!r}: {detail}" + ) + + # These dataclasses are treated as parsed value objects. frozen=True prevents # accidental field reassignment but does not imply deep immutability. @@ -3186,6 +3196,9 @@ def main() -> int: except ValueError as exc: print(str(exc)) return 1 + except (KeyError, TypeError, IndexError) as exc: + print(format_json_structure_error(ref, comp, exc)) + return 1 if len(selected_ref_devices) != len(selected_cmp_devices): print( @@ -3240,6 +3253,9 @@ def main() -> int: except ValueError as exc: print(str(exc)) return 1 + except (KeyError, TypeError, IndexError) as exc: + print(format_json_structure_error(ref, comp, exc)) + return 1 print("# Summary\n") print(f"- Total Matches: {stats.config_count}") diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index 8ef9a05..0743544 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -2265,6 +2265,50 @@ def test_main_rejects_non_object_root_array_entries( ) +def test_main_reports_invalid_device_entry_structure( + monkeypatch, capsys, nvbench_compare +): + monkeypatch.setattr( + nvbench_compare.reader, + "read_file", + lambda _: {"devices": [{}], "benchmarks": []}, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--reference-devices", + "0", + "--compare-devices", + "0", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 1 + output = capsys.readouterr().out + assert "invalid NVBench JSON structure" in output + assert "missing key 'id'" in output + + +def test_main_reports_invalid_benchmark_entry_structure( + monkeypatch, capsys, nvbench_compare +): + monkeypatch.setattr( + nvbench_compare.reader, + "read_file", + lambda _: {"devices": [], "benchmarks": [{}]}, + ) + monkeypatch.setattr(sys, "argv", ["nvbench_compare", "ref.json", "cmp.json"]) + + assert nvbench_compare.main() == 1 + output = capsys.readouterr().out + assert "invalid NVBench JSON structure" in output + assert "missing key 'name'" in output + + def test_main_prints_bulk_debug_python_to_stdout(monkeypatch, capsys, nvbench_compare): devices = [{"id": 0, "name": "Test GPU"}] root = {