From e8d7340696400a26cc75aa62afd0310f69adde42 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:33:35 -0500 Subject: [PATCH] Implement sample-count stopping criterion with parameter target-samples --stopping-criterion sample-count --target-samples 100 would stop once max(--min-samples, --target-samples) samples are collected --- docs/cli_help.md | 11 ++++ nvbench/CMakeLists.txt | 1 + nvbench/criterion_manager.cuh | 1 + nvbench/criterion_manager.cxx | 1 + nvbench/detail/measure_cold.cu | 2 +- nvbench/detail/measure_cpu_only.cxx | 2 +- nvbench/detail/measure_hot.cuh | 2 +- nvbench/detail/sample_count_criterion.cuh | 50 +++++++++++++++++++ nvbench/detail/sample_count_criterion.cxx | 37 ++++++++++++++ nvbench/detail/stdrel_criterion.cxx | 5 ++ testing/CMakeLists.txt | 1 + testing/criterion_manager.cu | 2 + testing/option_parser.cu | 39 +++++++++++++++ testing/sample_count_criterion.cu | 61 +++++++++++++++++++++++ testing/stdrel_criterion.cu | 16 ++++++ 15 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 nvbench/detail/sample_count_criterion.cuh create mode 100644 nvbench/detail/sample_count_criterion.cxx create mode 100644 testing/sample_count_criterion.cu diff --git a/docs/cli_help.md b/docs/cli_help.md index 22b033b..4a89ff8 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -157,6 +157,7 @@ * "stdrel": (default) Converges to a minimal relative standard deviation, stdev / mean * "entropy": Converges based on the cumulative entropy of all samples. + * "sample-count": Stops after a target number of samples. * Each stopping criterion may provide additional parameters to customize behavior, as detailed below: @@ -192,3 +193,13 @@ * Default is 0.36. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. + +### "sample-count" Stopping Criterion Parameters + +* `--target-samples ` + * Stop after at least `` samples are collected. + * Default is 100 samples. + * The total number of collected samples is + `max(--min-samples, --target-samples)`. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. diff --git a/nvbench/CMakeLists.txt b/nvbench/CMakeLists.txt index 9ec62f6..1aa4131 100644 --- a/nvbench/CMakeLists.txt +++ b/nvbench/CMakeLists.txt @@ -27,6 +27,7 @@ set(srcs detail/measure_cold.cu detail/measure_cpu_only.cxx detail/measure_hot.cu + detail/sample_count_criterion.cxx detail/state_generator.cxx detail/stdrel_criterion.cxx detail/gpu_frequency.cxx diff --git a/nvbench/criterion_manager.cuh b/nvbench/criterion_manager.cuh index f9445be..ea65d7e 100644 --- a/nvbench/criterion_manager.cuh +++ b/nvbench/criterion_manager.cuh @@ -29,6 +29,7 @@ #endif #include +#include #include #include #include diff --git a/nvbench/criterion_manager.cxx b/nvbench/criterion_manager.cxx index 60c8bad..e5f8229 100644 --- a/nvbench/criterion_manager.cxx +++ b/nvbench/criterion_manager.cxx @@ -33,6 +33,7 @@ criterion_manager::criterion_manager() { m_map.emplace("stdrel", std::make_unique()); m_map.emplace("entropy", std::make_unique()); + m_map.emplace("sample-count", std::make_unique()); } criterion_manager &criterion_manager::get() diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index c3885d8..dd24b87 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -175,7 +175,7 @@ bool measure_cold_base::is_finished() } // Check that we've gathered enough samples: - if (m_total_samples > m_min_samples) + if (m_total_samples >= m_min_samples) { if (m_stopping_criterion.is_finished()) { diff --git a/nvbench/detail/measure_cpu_only.cxx b/nvbench/detail/measure_cpu_only.cxx index bacde6f..3cdda93 100644 --- a/nvbench/detail/measure_cpu_only.cxx +++ b/nvbench/detail/measure_cpu_only.cxx @@ -93,7 +93,7 @@ bool measure_cpu_only_base::is_finished() } // Check that we've gathered enough samples: - if (m_total_samples > m_min_samples) + if (m_total_samples >= m_min_samples) { if (m_stopping_criterion.is_finished()) { diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index 2a310d3..f0aa267 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -186,7 +186,7 @@ private: (m_total_cuda_time / static_cast(m_total_samples))); if (m_total_cuda_time > m_min_time && // min time okay - m_total_samples > m_min_samples) // min samples okay + m_total_samples >= m_min_samples) // min samples okay { break; // Stop iterating } diff --git a/nvbench/detail/sample_count_criterion.cuh b/nvbench/detail/sample_count_criterion.cuh new file mode 100644 index 0000000..d7f44f0 --- /dev/null +++ b/nvbench/detail/sample_count_criterion.cuh @@ -0,0 +1,50 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#if defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_GCC) +#pragma GCC system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_CLANG) +#pragma clang system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_MSVC) +#pragma system_header +#endif + +#include +#include + +namespace nvbench::detail +{ + +class sample_count_criterion final : public stopping_criterion_base +{ + nvbench::int64_t m_total_samples{}; + +public: + sample_count_criterion(); + +protected: + virtual void do_initialize() override; + virtual void do_add_measurement(nvbench::float64_t measurement) override; + virtual bool do_is_finished() override; +}; + +} // namespace nvbench::detail diff --git a/nvbench/detail/sample_count_criterion.cxx b/nvbench/detail/sample_count_criterion.cxx new file mode 100644 index 0000000..ab89e92 --- /dev/null +++ b/nvbench/detail/sample_count_criterion.cxx @@ -0,0 +1,37 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace nvbench::detail +{ + +sample_count_criterion::sample_count_criterion() + : stopping_criterion_base{"sample-count", {{"target-samples", nvbench::int64_t{100}}}} +{} + +void sample_count_criterion::do_initialize() { m_total_samples = 0; } + +void sample_count_criterion::do_add_measurement(nvbench::float64_t) { ++m_total_samples; } + +bool sample_count_criterion::do_is_finished() +{ + return m_total_samples >= m_params.get_int64("target-samples"); +} + +} // namespace nvbench::detail diff --git a/nvbench/detail/stdrel_criterion.cxx b/nvbench/detail/stdrel_criterion.cxx index 8960818..38d7948 100644 --- a/nvbench/detail/stdrel_criterion.cxx +++ b/nvbench/detail/stdrel_criterion.cxx @@ -60,6 +60,11 @@ bool stdrel_criterion::do_is_finished() return false; } + if (m_noise_tracker.empty()) + { + return false; + } + // Noise has dropped below threshold if (m_noise_tracker.back() < m_params.get_float64("max-noise")) { diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index bbf3e19..4f3bb72 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -20,6 +20,7 @@ set(test_srcs reset_error.cu ring_buffer.cu runner.cu + sample_count_criterion.cu state.cu statistics.cu state_generator.cu diff --git a/testing/criterion_manager.cu b/testing/criterion_manager.cu index b466c30..556763c 100644 --- a/testing/criterion_manager.cu +++ b/testing/criterion_manager.cu @@ -25,6 +25,8 @@ void test_standard_criteria_exist() { ASSERT(nvbench::criterion_manager::get().get_criterion("stdrel").get_name() == "stdrel"); ASSERT(nvbench::criterion_manager::get().get_criterion("entropy").get_name() == "entropy"); + ASSERT(nvbench::criterion_manager::get().get_criterion("sample-count").get_name() == + "sample-count"); } class custom_criterion : public nvbench::stopping_criterion_base diff --git a/testing/option_parser.cu b/testing/option_parser.cu index c2c0bd9..6169463 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1307,6 +1307,45 @@ void test_stopping_criterion() ASSERT(criterion_params.get_float64("max-angle") == 0.42); ASSERT(criterion_params.get_float64("min-r2") == 0.6); } + { // Sample-count criterion default params + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "sample-count", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "sample-count"); + + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("target-samples")); + ASSERT(criterion_params.get_int64("target-samples") == 100); + } + { // Sample-count criterion params are independent from min_samples + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--min-samples", + "7", + "--stopping-criterion", + "sample-count", + "--target-samples", + "123", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_min_samples() == 7); + ASSERT(states[0].get_stopping_criterion() == "sample-count"); + + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("target-samples")); + ASSERT(criterion_params.get_int64("target-samples") == 123); + } { // Unknown stopping criterion should throw bool exception_thrown = false; try diff --git a/testing/sample_count_criterion.cu b/testing/sample_count_criterion.cu new file mode 100644 index 0000000..5d9ecbf --- /dev/null +++ b/testing/sample_count_criterion.cu @@ -0,0 +1,61 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "test_asserts.cuh" + +void test_default_target_samples() +{ + nvbench::detail::sample_count_criterion criterion; + criterion.initialize(nvbench::criterion_params{}); + + for (int i = 0; i < 99; ++i) + { + criterion.add_measurement(1.0); + ASSERT(!criterion.is_finished()); + } + + criterion.add_measurement(1.0); + ASSERT(criterion.is_finished()); +} + +void test_custom_target_samples() +{ + nvbench::criterion_params params; + params.set_int64("target-samples", 3); + + nvbench::detail::sample_count_criterion criterion; + criterion.initialize(params); + + criterion.add_measurement(1.0); + ASSERT(!criterion.is_finished()); + + criterion.add_measurement(1.0); + ASSERT(!criterion.is_finished()); + + criterion.add_measurement(1.0); + ASSERT(criterion.is_finished()); +} + +int main() +{ + test_default_target_samples(); + test_custom_target_samples(); +} diff --git a/testing/stdrel_criterion.cu b/testing/stdrel_criterion.cu index fbe1dab..0e6b5af 100644 --- a/testing/stdrel_criterion.cu +++ b/testing/stdrel_criterion.cu @@ -78,8 +78,24 @@ void test_stdrel() ASSERT(!criterion.is_finished()); } +void test_stdrel_needs_enough_samples() +{ + nvbench::criterion_params params; + params.set_float64("min-time", 0.0); + + nvbench::detail::stdrel_criterion criterion; + criterion.initialize(params); + + for (int i = 0; i < 4; ++i) + { + criterion.add_measurement(42.0); + } + ASSERT(!criterion.is_finished()); +} + int main() { test_const(); test_stdrel(); + test_stdrel_needs_enough_samples(); }