From 4e83e048baf1171d98295c739c939403a3ba736a Mon Sep 17 00:00:00 2001 From: Allison Vacanti Date: Thu, 18 Mar 2021 13:42:43 -0400 Subject: [PATCH] Store percentages as ratios. Human-readable outputs (md) and CLI inputs still use percentages. In-memory and machine-readable outputs (csv, json) use ratios. This is the convention that spreadsheet apps expect. Fixes #2. --- docs/cli_help.md | 4 ++-- nvbench/benchmark_base.cuh | 6 +++--- nvbench/csv_printer.cu | 2 +- nvbench/detail/measure_cold.cu | 9 ++++----- nvbench/detail/measure_cold.cuh | 6 +++--- nvbench/detail/statistics.cuh | 5 ++--- nvbench/markdown_printer.cu | 2 +- nvbench/option_parser.cu | 4 ++-- nvbench/state.cuh | 4 ++-- nvbench/summary.cuh | 3 ++- testing/option_parser.cu | 4 ++-- 11 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/cli_help.md b/docs/cli_help.md index fd8fc6d..9a6e6fc 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -68,8 +68,8 @@ * `--max-noise ` * Gather samples until the error in the measurement drops below ``. - * Noise is computed as the percent relative standard deviation. - * Default is 0.5%. + * Noise is specified as the percent relative standard deviation. + * Default is 0.5% (`--max-noise 0.5`) * Only applies to Cold measurements. * If both GPU and CPU times are gathered, this applies to GPU noise only. * Applies to the most recent `--benchmark`, or all benchmarks if specified diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index 4fc81a5..7a1f594 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -185,8 +185,8 @@ struct benchmark_base /// @} /// Specify the maximum amount of noise if a measurement supports noise. - /// Noise is the relative standard deviation expressed as a percentage: - /// `noise = 100 * (stdev / mean_time)`. @{ + /// Noise is the relative standard deviation: + /// `noise = stdev / mean_time`. @{ [[nodiscard]] nvbench::float64_t get_max_noise() const { return m_max_noise; } benchmark_base &set_max_noise(nvbench::float64_t max_noise) { @@ -239,7 +239,7 @@ protected: nvbench::int64_t m_min_samples{10}; nvbench::float64_t m_min_time{0.5}; - nvbench::float64_t m_max_noise{0.5}; // 0.5% relative standard deviation + nvbench::float64_t m_max_noise{0.005}; // 0.5% relative standard deviation nvbench::float64_t m_skip_time{-1.}; nvbench::float64_t m_timeout{15.}; diff --git a/nvbench/csv_printer.cu b/nvbench/csv_printer.cu index efd6e57..340b109 100644 --- a/nvbench/csv_printer.cu +++ b/nvbench/csv_printer.cu @@ -142,7 +142,7 @@ void csv_printer::do_print_benchmark_results(const benchmark_vector &benches) } else if (hint == "percentage") { - table.add_cell(row, key, header + " (%)", std::move(value)); + table.add_cell(row, key, header, std::move(value)); } else { diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index f654d3a..2ac39e7 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -149,7 +149,7 @@ void measure_cold_base::generate_summaries() summ.set_string("description", "Global device memory throughput as a percentage of the " "device's peak bandwidth."); - summ.set_float64("value", avg_used_gmem_bw / peak_gmem_bw * 100.); + summ.set_float64("value", avg_used_gmem_bw / peak_gmem_bw); } } @@ -170,8 +170,8 @@ void measure_cold_base::generate_summaries() "while over noise threshold ({:0.2f}% > " "{:0.2f}%)", timeout, - m_cuda_noise, - m_max_noise)); + m_cuda_noise * 100, + m_max_noise * 100)); } if (m_total_samples < m_min_samples) { @@ -221,8 +221,7 @@ void measure_cold_base::check_skip_time(nvbench::float64_t warmup_time) void measure_cold_base::block_stream() { - m_blocker.block(m_launch.get_stream(), - m_state.get_blocking_kernel_timeout()); + m_blocker.block(m_launch.get_stream(), m_state.get_blocking_kernel_timeout()); } } // namespace nvbench::detail diff --git a/nvbench/detail/measure_cold.cuh b/nvbench/detail/measure_cold.cuh index 63e6467..43f1dc0 100644 --- a/nvbench/detail/measure_cold.cuh +++ b/nvbench/detail/measure_cold.cuh @@ -99,7 +99,7 @@ protected: nvbench::blocking_kernel m_blocker; nvbench::int64_t m_min_samples{}; - nvbench::float64_t m_max_noise{}; // % rel stdev + nvbench::float64_t m_max_noise{}; // rel stdev nvbench::float64_t m_min_time{}; nvbench::float64_t m_skip_time{}; @@ -108,8 +108,8 @@ protected: nvbench::int64_t m_total_samples{}; nvbench::float64_t m_total_cuda_time{}; nvbench::float64_t m_total_cpu_time{}; - nvbench::float64_t m_cuda_noise{}; // % rel stdev - nvbench::float64_t m_cpu_noise{}; // % rel stdev + nvbench::float64_t m_cuda_noise{}; // rel stdev + nvbench::float64_t m_cpu_noise{}; // rel stdev std::vector m_cuda_times; std::vector m_cpu_times; diff --git a/nvbench/detail/statistics.cuh b/nvbench/detail/statistics.cuh index b416d33..120475a 100644 --- a/nvbench/detail/statistics.cuh +++ b/nvbench/detail/statistics.cuh @@ -33,7 +33,7 @@ namespace nvbench::detail * vector, return a measure of the noise in the samples. * * The noise metric is the relative unbiased sample standard deviation - * expressed as a percentage: (std_dev / mean) * 100. + * (std_dev / mean). */ inline nvbench::float64_t compute_noise(const std::vector &data, @@ -58,8 +58,7 @@ compute_noise(const std::vector &data, }) / (num - 1); const auto abs_stdev = std::sqrt(variance); - const auto rel_stdev = abs_stdev / mean; - return rel_stdev * 100.; + return abs_stdev / mean; } } // namespace nvbench::detail diff --git a/nvbench/markdown_printer.cu b/nvbench/markdown_printer.cu index 7587216..184611a 100644 --- a/nvbench/markdown_printer.cu +++ b/nvbench/markdown_printer.cu @@ -459,7 +459,7 @@ std::string markdown_printer::do_format_sample_size(const summary &data) std::string markdown_printer::do_format_percentage(const summary &data) { const auto percentage = data.get_float64("value"); - return fmt::format("{:.2f}%", percentage); + return fmt::format("{:.2f}%", percentage * 100.); } } // namespace nvbench diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 3725975..4f402d6 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -807,8 +807,8 @@ try bench.set_min_time(value); } else if (prop_arg == "--max-noise") - { - bench.set_max_noise(value); + { // Specified as percentage, stored as ratio: + bench.set_max_noise(value / 100.); } else if (prop_arg == "--skip-time") { diff --git a/nvbench/state.cuh b/nvbench/state.cuh index 2b18d24..dd1ca76 100644 --- a/nvbench/state.cuh +++ b/nvbench/state.cuh @@ -151,8 +151,8 @@ struct state /// @} /// Specify the maximum amount of noise if a measurement supports noise. - /// Noise is the relative standard deviation expressed as a percentage: - /// `noise = 100 * (stdev / mean_time)`. @{ + /// Noise is the relative standard deviation: + /// `noise = stdev / mean_time`. @{ [[nodiscard]] nvbench::float64_t get_max_noise() const { return m_max_noise; } void set_max_noise(nvbench::float64_t max_noise) { m_max_noise = max_noise; } /// @} diff --git a/nvbench/summary.cuh b/nvbench/summary.cuh index dfd52af..7a6a2e6 100644 --- a/nvbench/summary.cuh +++ b/nvbench/summary.cuh @@ -48,7 +48,8 @@ namespace nvbench * - "bytes": "value" is an int64_t number of bytes. * - "byte_rate": "value" is a float64_t byte rate in bytes / second. * - "sample_size": "value" is an int64_t number of samples in a measurement. - * - "percentage": "value" is a float64_t percentage. + * - "percentage": "value" is a float64_t percentage (stored as a ratio, 1. = + * 100%). * * The key/value pair functionality is implemented by the * `nvbench::named_values` base class. diff --git a/testing/option_parser.cu b/testing/option_parser.cu index dbf951e..61f5f6b 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1200,11 +1200,11 @@ void test_max_noise() { nvbench::option_parser parser; parser.parse( - {"--benchmark", "DummyBench", "--max-noise", "12345e2"}); + {"--benchmark", "DummyBench", "--max-noise", "50.3"}); const auto& states = parser_to_states(parser); ASSERT(states.size() == 1); - ASSERT(std::abs(states[0].get_max_noise() - 12345e2) < 1.); + ASSERT(std::abs(states[0].get_max_noise() - 0.503) < 1.-4); } void test_skip_time()