mirror of
https://github.com/NVIDIA/nvbench.git
synced 2026-06-29 18:57:44 +00:00
Implement clear-gap comparison for early FAST/SLOW decision
Implemented the clear-gap comparison, with the log-distance-equivalent
algebra and pessimistic SM-clock fallback.
What changed:
- Added TimingInterval and interval construction from summaries:
- robust interval: [min, q3], centered at median
- fallback interval: clipped [mean - stdev, mean + stdev] intersected with [min, max]
- Added CLEAR_GAP_RELATIVE_THRESHOLD = 0.005.
- FAST gap uses:
(ref.lower - cmp.upper) / cmp.upper >= delta
which is equivalent to log(ref.lower / cmp.upper) >= log(1 + delta).
- SLOW gap uses:
(cmp.lower - ref.upper) / ref.upper >= delta
- FAST/SLOW now requires SM clock summaries on both sides and the same clear-gap result after scaling intervals by sm_clock_rate_mean.
- If intervals are missing, overlap, fail the gap threshold, have missing/invalid clock summaries, or time/cycle comparison disagrees, status is UNDECIDED.
- Existing center/noise values are still computed and displayed, but no longer drive FAST/SLOW/SAME classification.
Updated tests to cover:
- center/noise-only comparisons becoming UNDECIDED
- clear FAST/SLOW with matching clock evidence
- missing clock fallback to UNDECIDED
- frequency-shift disagreement becoming UNDECIDED
- regression reporting with robust interval and clock evidence
This commit is contained in:
@@ -45,6 +45,7 @@ GPU_TIME_IR_RELATIVE_TAG = "nv/cold/time/gpu/ir/relative"
|
||||
GPU_SM_CLOCK_RATE_MEAN_TAG = "nv/cold/sm_clock_rate/mean"
|
||||
SAMPLE_TIMES_TAG = "nv/json/bin:nv/cold/sample_times"
|
||||
SAMPLE_FREQUENCIES_TAG = "nv/json/freqs-bin:nv/cold/sample_freqs"
|
||||
CLEAR_GAP_RELATIVE_THRESHOLD = 0.005
|
||||
|
||||
# The reader returns an object supporting the buffer protocol. Python 3.10 does
|
||||
# not provide a standard Buffer type annotation.
|
||||
@@ -109,6 +110,13 @@ class TimeEstimate:
|
||||
relative_dispersion: float | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TimingInterval:
|
||||
lower: float
|
||||
upper: float
|
||||
center: float
|
||||
|
||||
|
||||
class ComparisonStatus(str, Enum):
|
||||
UNKNOWN = "????"
|
||||
UNDECIDED = "UNDECIDED"
|
||||
@@ -493,6 +501,111 @@ def compute_relative_dispersion(dispersion, center):
|
||||
return dispersion / center
|
||||
|
||||
|
||||
def is_positive_finite(value):
|
||||
return value is not None and value > 0.0 and math.isfinite(value)
|
||||
|
||||
|
||||
def make_timing_interval(lower, upper, center):
|
||||
if (
|
||||
not is_positive_finite(lower)
|
||||
or not is_positive_finite(upper)
|
||||
or not is_positive_finite(center)
|
||||
or lower > center
|
||||
or center > upper
|
||||
):
|
||||
return None
|
||||
return TimingInterval(lower=lower, upper=upper, center=center)
|
||||
|
||||
|
||||
def compute_timing_interval(timing):
|
||||
if (
|
||||
is_positive_finite(timing.minimum)
|
||||
and is_positive_finite(timing.first_quartile)
|
||||
and is_positive_finite(timing.median)
|
||||
and is_positive_finite(timing.third_quartile)
|
||||
and timing.minimum <= timing.first_quartile
|
||||
and timing.first_quartile <= timing.median
|
||||
and timing.median <= timing.third_quartile
|
||||
):
|
||||
return make_timing_interval(
|
||||
lower=timing.minimum,
|
||||
upper=timing.third_quartile,
|
||||
center=timing.median,
|
||||
)
|
||||
|
||||
if (
|
||||
is_positive_finite(timing.minimum)
|
||||
and is_positive_finite(timing.maximum)
|
||||
and is_positive_finite(timing.mean)
|
||||
and is_positive_finite(timing.stdev)
|
||||
and timing.minimum <= timing.mean
|
||||
and timing.mean <= timing.maximum
|
||||
):
|
||||
return make_timing_interval(
|
||||
lower=max(timing.minimum, timing.mean - timing.stdev),
|
||||
upper=min(timing.maximum, timing.mean + timing.stdev),
|
||||
center=timing.mean,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def compare_intervals_for_clear_gap(ref_interval, cmp_interval):
|
||||
# These ratios are equivalent to log(ref/cmp) >= log(1 + delta), but avoid
|
||||
# evaluating logarithms on every comparison.
|
||||
if cmp_interval.upper < ref_interval.lower:
|
||||
gap = ref_interval.lower - cmp_interval.upper
|
||||
if gap / cmp_interval.upper >= CLEAR_GAP_RELATIVE_THRESHOLD:
|
||||
return ComparisonStatus.FAST
|
||||
if cmp_interval.lower > ref_interval.upper:
|
||||
gap = cmp_interval.lower - ref_interval.upper
|
||||
if gap / ref_interval.upper >= CLEAR_GAP_RELATIVE_THRESHOLD:
|
||||
return ComparisonStatus.SLOW
|
||||
return None
|
||||
|
||||
|
||||
def scale_interval(interval, scale):
|
||||
if not is_positive_finite(scale):
|
||||
return None
|
||||
return make_timing_interval(
|
||||
lower=interval.lower * scale,
|
||||
upper=interval.upper * scale,
|
||||
center=interval.center * scale,
|
||||
)
|
||||
|
||||
|
||||
def confirm_clear_gap_with_clock_rate(
|
||||
status, ref_timing, cmp_timing, ref_interval, cmp_interval
|
||||
):
|
||||
if ref_timing.sm_clock_rate_mean is None or cmp_timing.sm_clock_rate_mean is None:
|
||||
return ComparisonStatus.UNDECIDED
|
||||
|
||||
ref_cycles = scale_interval(ref_interval, ref_timing.sm_clock_rate_mean)
|
||||
cmp_cycles = scale_interval(cmp_interval, cmp_timing.sm_clock_rate_mean)
|
||||
if ref_cycles is None or cmp_cycles is None:
|
||||
return ComparisonStatus.UNDECIDED
|
||||
|
||||
cycle_status = compare_intervals_for_clear_gap(ref_cycles, cmp_cycles)
|
||||
if cycle_status == status:
|
||||
return status
|
||||
return ComparisonStatus.UNDECIDED
|
||||
|
||||
|
||||
def compare_timings_for_clear_gap(ref_timing, cmp_timing):
|
||||
ref_interval = compute_timing_interval(ref_timing)
|
||||
cmp_interval = compute_timing_interval(cmp_timing)
|
||||
if ref_interval is None or cmp_interval is None:
|
||||
return ComparisonStatus.UNDECIDED
|
||||
|
||||
status = compare_intervals_for_clear_gap(ref_interval, cmp_interval)
|
||||
if status is None:
|
||||
return ComparisonStatus.UNDECIDED
|
||||
|
||||
return confirm_clear_gap_with_clock_rate(
|
||||
status, ref_timing, cmp_timing, ref_interval, cmp_interval
|
||||
)
|
||||
|
||||
|
||||
def has_robust_estimate(summary):
|
||||
return summary.median is not None and (
|
||||
summary.interquartile_range_relative is not None
|
||||
@@ -588,15 +701,10 @@ def compare_gpu_timings(ref_timing, cmp_timing):
|
||||
|
||||
if not has_finite_noise(ref_noise) or not has_finite_noise(cmp_noise):
|
||||
max_noise = None
|
||||
status = ComparisonStatus.UNKNOWN
|
||||
else:
|
||||
max_noise = max(ref_noise, cmp_noise)
|
||||
if abs(frac_diff) <= max_noise:
|
||||
status = ComparisonStatus.SAME
|
||||
elif diff < 0:
|
||||
status = ComparisonStatus.FAST
|
||||
else:
|
||||
status = ComparisonStatus.SLOW
|
||||
|
||||
status = compare_timings_for_clear_gap(ref_timing, cmp_timing)
|
||||
|
||||
return SummaryComparison(
|
||||
ref_estimate=ref_estimate,
|
||||
|
||||
Reference in New Issue
Block a user