diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 50f5762..9c7bbaf 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -49,6 +49,8 @@ CLEAR_GAP_RELATIVE_THRESHOLD = 0.005 SAME_CENTER_RELATIVE_THRESHOLD = 0.005 SAME_OVERLAP_FRACTION_THRESHOLD = 0.5 SAME_RELATIVE_DISPERSION_CEILING = 0.02 +BULK_SAME_SAMPLE_COVERAGE_THRESHOLD = 0.99 +BULK_SAME_SUPPORT_COVERAGE_THRESHOLD = 0.80 # The reader returns an object supporting the buffer protocol. Python 3.10 does # not provide a standard Buffer type annotation. @@ -132,6 +134,7 @@ class ComparisonStatus(str, Enum): class DecisionReason: code: str message: str + severity: float = 0.0 @dataclass(frozen=True) @@ -155,6 +158,13 @@ class SummaryComparison: reason: DecisionReason +@dataclass +class DecisionReasonSummary: + count: int = 0 + message: str = "" + severity: float = 0.0 + + @dataclass class ComparisonStats: config_count: int = 0 @@ -163,7 +173,7 @@ class ComparisonStats: regression_count: int = 0 undecided_count: int = 0 unknown_count: int = 0 - undecided_reasons: Counter[DecisionReason] = field(default_factory=Counter) + undecided_reasons: dict[str, DecisionReasonSummary] = field(default_factory=dict) def record( self, status: ComparisonStatus, reason: DecisionReason | None = None @@ -174,7 +184,13 @@ class ComparisonStats: elif status == ComparisonStatus.UNDECIDED: self.undecided_count += 1 if reason is not None: - self.undecided_reasons[reason] += 1 + summary = self.undecided_reasons.setdefault( + reason.code, DecisionReasonSummary() + ) + if summary.count == 0 or reason.severity > summary.severity: + summary.message = reason.message + summary.severity = reason.severity + summary.count += 1 elif status == ComparisonStatus.SAME: self.pass_count += 1 elif status == ComparisonStatus.FAST: @@ -571,9 +587,10 @@ def compute_timing_interval(timing): return None -def make_decision(status, code, message): +def make_decision(status, code, message, *, severity=0.0): return TimingDecision( - status=status, reason=DecisionReason(code=code, message=message) + status=status, + reason=DecisionReason(code=code, message=message, severity=severity), ) @@ -634,6 +651,111 @@ def intervals_overlap_strongly(ref_interval, cmp_interval): ) +def nearest_distances_to_sorted(target, source): + pos = np.searchsorted(source, target, side="left") + left = np.clip(pos - 1, 0, len(source) - 1) + right = np.clip(pos, 0, len(source) - 1) + return np.minimum( + np.abs(target - source[left]), + np.abs(target - source[right]), + ) + + +def symmetric_nearest_distances(x, y): + # This is O(N log M + M log N), but runs in NumPy C code and operates on + # unique supports. If this becomes a bottleneck for very large supports, + # add an optional O(N + M) two-pass merge helper to cuda.bench and fall back + # to this implementation when cuda.bench is unavailable. + return nearest_distances_to_sorted(x, y), nearest_distances_to_sorted(y, x) + + +def symmetric_nearest_log_distances(x, y): + return symmetric_nearest_distances(np.log(x), np.log(y)) + + +def compute_nearest_neighbor_coverages(ref_values, cmp_values): + ref_unique, ref_counts = np.unique_counts(ref_values) + cmp_unique, cmp_counts = np.unique_counts(cmp_values) + if len(ref_unique) == 0 or len(cmp_unique) == 0: + return None + + ref_distances, cmp_distances = symmetric_nearest_log_distances( + ref_unique, cmp_unique + ) + tolerance = math.log1p(SAME_CENTER_RELATIVE_THRESHOLD) + ref_covered = ref_distances <= tolerance + cmp_covered = cmp_distances <= tolerance + + return { + "ref_sample": np.sum(ref_counts[ref_covered]) / np.sum(ref_counts), + "cmp_sample": np.sum(cmp_counts[cmp_covered]) / np.sum(cmp_counts), + "ref_support": np.mean(ref_covered), + "cmp_support": np.mean(cmp_covered), + } + + +def coverages_support_same(coverages): + return ( + coverages["ref_sample"] >= BULK_SAME_SAMPLE_COVERAGE_THRESHOLD + and coverages["cmp_sample"] >= BULK_SAME_SAMPLE_COVERAGE_THRESHOLD + and coverages["ref_support"] >= BULK_SAME_SUPPORT_COVERAGE_THRESHOLD + and coverages["cmp_support"] >= BULK_SAME_SUPPORT_COVERAGE_THRESHOLD + ) + + +def format_coverage_threshold(threshold): + return f"{threshold * 100.0:.1f}%" + + +def format_coverage(value): + return f"{value * 100.0:.1f}%" + + +def make_bulk_coverage_mismatch_decision(label, coverages): + sample_threshold = format_coverage_threshold(BULK_SAME_SAMPLE_COVERAGE_THRESHOLD) + support_threshold = format_coverage_threshold(BULK_SAME_SUPPORT_COVERAGE_THRESHOLD) + sample_deficit = max( + BULK_SAME_SAMPLE_COVERAGE_THRESHOLD - coverages["ref_sample"], + BULK_SAME_SAMPLE_COVERAGE_THRESHOLD - coverages["cmp_sample"], + 0.0, + ) + support_deficit = max( + BULK_SAME_SUPPORT_COVERAGE_THRESHOLD - coverages["ref_support"], + BULK_SAME_SUPPORT_COVERAGE_THRESHOLD - coverages["cmp_support"], + 0.0, + ) + severity = max(sample_deficit, support_deficit) + return make_decision( + ComparisonStatus.UNDECIDED, + f"bulk_{label}_support_mismatch", + f"sample ref={format_coverage(coverages['ref_sample'])} " + f"cmp={format_coverage(coverages['cmp_sample'])} >= {sample_threshold}; " + f"support ref={format_coverage(coverages['ref_support'])} " + f"cmp={format_coverage(coverages['cmp_support'])} >= {support_threshold}", + severity=severity, + ) + + +def positive_finite_array(values): + if values is None or len(values) == 0: + return None + + array = np.asarray(values, dtype=np.float64) + if np.all(np.isfinite(array) & (array > 0.0)): + return array + return None + + +def get_bulk_time_and_cycles(timing): + samples = positive_finite_array(timing.samples) + frequencies = positive_finite_array(timing.frequencies) + if samples is None or frequencies is None: + return None + if len(samples) != len(frequencies): + return None + return samples, samples * frequencies + + def scale_interval(interval, scale): if not is_positive_finite(scale): return None @@ -751,6 +873,51 @@ def confirm_same_with_clock_rate(ref_timing, cmp_timing, ref_interval, cmp_inter ) +def compare_values_for_bulk_same(ref_values, cmp_values, *, label): + coverages = compute_nearest_neighbor_coverages(ref_values, cmp_values) + if coverages is None: + return make_decision( + ComparisonStatus.UNDECIDED, + f"bulk_{label}_data_unusable", + f"bulk {label} data is empty or unusable", + ) + if coverages_support_same(coverages): + return make_decision( + ComparisonStatus.SAME, + f"bulk_{label}_same", + f"bulk {label} nearest-neighbor coverage supports same", + ) + return make_bulk_coverage_mismatch_decision(label, coverages) + + +def compare_timings_for_bulk_same(ref_timing, cmp_timing): + ref_bulk = get_bulk_time_and_cycles(ref_timing) + cmp_bulk = get_bulk_time_and_cycles(cmp_timing) + if ref_bulk is None or cmp_bulk is None: + return make_decision( + ComparisonStatus.UNDECIDED, + "bulk_data_unavailable", + "bulk sample time and frequency data are unavailable", + ) + + ref_times, ref_cycles = ref_bulk + cmp_times, cmp_cycles = cmp_bulk + + time_decision = compare_values_for_bulk_same(ref_times, cmp_times, label="time") + if time_decision.status != ComparisonStatus.SAME: + return time_decision + + cycle_decision = compare_values_for_bulk_same(ref_cycles, cmp_cycles, label="cycle") + if cycle_decision.status != ComparisonStatus.SAME: + return cycle_decision + + return make_decision( + ComparisonStatus.SAME, + "bulk_same", + "bulk time and cycle nearest-neighbor coverage both support same", + ) + + def compare_timings_for_same(ref_timing, cmp_timing, ref_noise, cmp_noise): if not has_finite_noise(ref_noise) or not has_finite_noise(cmp_noise): return make_decision( @@ -886,9 +1053,13 @@ def compare_gpu_timings(ref_timing, cmp_timing): "no_clear_gap", "missing_interval", }: - decision = compare_timings_for_same( - ref_timing, cmp_timing, ref_noise, cmp_noise - ) + bulk_decision = compare_timings_for_bulk_same(ref_timing, cmp_timing) + if bulk_decision.reason.code == "bulk_data_unavailable": + decision = compare_timings_for_same( + ref_timing, cmp_timing, ref_noise, cmp_noise + ) + else: + decision = bulk_decision return SummaryComparison( ref_estimate=ref_estimate, @@ -1728,8 +1899,12 @@ def main() -> int: ) if stats.undecided_reasons: print(" - Reasons:") - for reason, count in stats.undecided_reasons.most_common(): - print(f" - {reason.code}: {count} ({reason.message})") + for code, reason_summary in sorted( + stats.undecided_reasons.items(), + key=lambda item: item[1].count, + reverse=True, + ): + print(f" - {code}: {reason_summary.count} ({reason_summary.message})") print(f" - Unknown (infinite or unavailable noise): {stats.unknown_count}") return 0 diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index 9937042..14be5be 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -124,6 +124,8 @@ def make_gpu_timing_data( interquartile_range=None, interquartile_range_relative=None, sm_clock_rate_mean=None, + sample_values=None, + frequency_values=None, ): return nvbench_compare.GpuTimingData( minimum=minimum, @@ -137,6 +139,14 @@ def make_gpu_timing_data( interquartile_range=interquartile_range, interquartile_range_relative=interquartile_range_relative, sm_clock_rate_mean=sm_clock_rate_mean, + sample_source=None + if sample_values is None + else types.SimpleNamespace(values=np.asarray(sample_values, dtype=np.float32)), + frequency_source=None + if frequency_values is None + else types.SimpleNamespace( + values=np.asarray(frequency_values, dtype=np.float32) + ), ) @@ -652,6 +662,115 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare): assert missing_noise.reason.code == "noise_unavailable" +def test_compare_gpu_timings_uses_bulk_data_to_confirm_same(nvbench_compare): + ref_timing = make_gpu_timing_data( + nvbench_compare, + mean=1.0, + stdev_relative=0.05, + sample_values=[1.0] * 8 + [1.004] * 2, + frequency_values=[100.0] * 10, + ) + cmp_timing = make_gpu_timing_data( + nvbench_compare, + mean=1.0, + stdev_relative=0.05, + sample_values=[1.0] * 2 + [1.004] * 8, + frequency_values=[100.0] * 10, + ) + + comparison = nvbench_compare.compare_gpu_timings(ref_timing, cmp_timing) + + assert comparison is not None + assert comparison.status == nvbench_compare.ComparisonStatus.SAME + assert comparison.reason.code == "bulk_same" + + +def test_compare_gpu_timings_keeps_bulk_mismatch_undecided(nvbench_compare): + ref_timing = make_gpu_timing_data( + nvbench_compare, + minimum=1.0, + first_quartile=1.1, + median=1.2, + third_quartile=1.3, + mean=1.2, + interquartile_range_relative=0.01, + sample_values=[1.0, 1.0, 1.004, 1.004], + frequency_values=[100.0] * 4, + ) + cmp_timing = make_gpu_timing_data( + nvbench_compare, + minimum=1.02, + first_quartile=1.1, + median=1.204, + third_quartile=1.28, + mean=1.204, + interquartile_range_relative=0.01, + sample_values=[1.02, 1.02, 1.024, 1.024], + frequency_values=[100.0] * 4, + ) + + comparison = nvbench_compare.compare_gpu_timings(ref_timing, cmp_timing) + + assert comparison is not None + assert comparison.status == nvbench_compare.ComparisonStatus.UNDECIDED + assert comparison.reason.code == "bulk_time_support_mismatch" + assert "sample ref=" in comparison.reason.message + assert "support ref=" in comparison.reason.message + assert "99.0%" in comparison.reason.message + assert "80.0%" in comparison.reason.message + + +def test_compare_gpu_timings_requires_bulk_cycle_coverage(nvbench_compare): + ref_timing = make_gpu_timing_data( + nvbench_compare, + mean=1.0, + stdev_relative=0.01, + sample_values=[1.0, 1.0, 1.004, 1.004], + frequency_values=[100.0] * 4, + ) + cmp_timing = make_gpu_timing_data( + nvbench_compare, + mean=1.0, + stdev_relative=0.01, + sample_values=[1.0, 1.0, 1.004, 1.004], + frequency_values=[200.0] * 4, + ) + + comparison = nvbench_compare.compare_gpu_timings(ref_timing, cmp_timing) + + assert comparison is not None + assert comparison.status == nvbench_compare.ComparisonStatus.UNDECIDED + assert comparison.reason.code == "bulk_cycle_support_mismatch" + + +def test_bulk_same_reports_sample_weight_coverage_mismatch(nvbench_compare): + ref_values = [1.0, 1.001, 1.002, 1.003] + [1.02] * 100 + cmp_values = [1.0, 1.001, 1.002, 1.003] + + decision = nvbench_compare.compare_values_for_bulk_same( + ref_values, cmp_values, label="time" + ) + + assert decision.status == nvbench_compare.ComparisonStatus.UNDECIDED + assert decision.reason.code == "bulk_time_support_mismatch" + assert "sample ref=3.8%" in decision.reason.message + assert "support ref=80.0%" in decision.reason.message + + +def test_bulk_same_reports_unique_support_coverage_mismatch(nvbench_compare): + ref_values = [1.0] * 1000 + [1.02 + 0.01 * i for i in range(10)] + cmp_values = [1.0] + + decision = nvbench_compare.compare_values_for_bulk_same( + ref_values, cmp_values, label="time" + ) + + assert decision.status == nvbench_compare.ComparisonStatus.UNDECIDED + assert decision.reason.code == "bulk_time_support_mismatch" + assert "sample ref=99.0%" in decision.reason.message + assert "support ref=9.1%" in decision.reason.message + + def test_comparison_stats_records_undecided_status(nvbench_compare): stats = nvbench_compare.ComparisonStats() @@ -667,14 +786,23 @@ def test_comparison_stats_records_undecided_status(nvbench_compare): def test_comparison_stats_records_undecided_reason(nvbench_compare): stats = nvbench_compare.ComparisonStats() - reason = nvbench_compare.DecisionReason( + less_severe_reason = nvbench_compare.DecisionReason( code="test_reason", - message="test reason", + message="less severe reason", + severity=1.0, + ) + more_severe_reason = nvbench_compare.DecisionReason( + code="test_reason", + message="more severe reason", + severity=2.0, ) - stats.record(nvbench_compare.ComparisonStatus.UNDECIDED, reason) + stats.record(nvbench_compare.ComparisonStatus.UNDECIDED, less_severe_reason) + stats.record(nvbench_compare.ComparisonStatus.UNDECIDED, more_severe_reason) - assert stats.undecided_reasons[reason] == 1 + summary = stats.undecided_reasons["test_reason"] + assert summary.count == 2 + assert summary.message == "more severe reason" @pytest.mark.parametrize("ref_time, cmp_time", [(None, 1.0), (1.0, None), (0.0, 1.0)])