NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-03-14 20:27:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Store percentages as ratios.

Browse Source 
Human-readable outputs (md) and CLI inputs still use percentages.
In-memory and machine-readable outputs (csv, json) use ratios.

This is the convention that spreadsheet apps expect. Fixes #2.
This commit is contained in:
Allison Vacanti
2021-03-18 13:42:43 -04:00
parent e62c786498
commit 4e83e048ba
11 changed files with 24 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/cli_help.md
View File

@@ -68,8 +68,8 @@
* `--max-noise <value>`
* Gather samples until the error in the measurement drops below `<value>`.
* Noise is computed as the percent relative standard deviation.
* Default is 0.5%.
* Noise is specified as the percent relative standard deviation.
* Default is 0.5% (`--max-noise 0.5`)
* Only applies to Cold measurements.
* If both GPU and CPU times are gathered, this applies to GPU noise only.
* Applies to the most recent `--benchmark`, or all benchmarks if specified

6
nvbench/benchmark_base.cuh
View File

@@ -185,8 +185,8 @@ struct benchmark_base
/// @}
/// Specify the maximum amount of noise if a measurement supports noise.
/// Noise is the relative standard deviation expressed as a percentage:
/// `noise = 100 * (stdev / mean_time)`. @{
/// Noise is the relative standard deviation:
/// `noise = stdev / mean_time`. @{
[[nodiscard]] nvbench::float64_t get_max_noise() const { return m_max_noise; }
benchmark_base &set_max_noise(nvbench::float64_t max_noise)
{
@@ -239,7 +239,7 @@ protected:
nvbench::int64_t m_min_samples{10};
nvbench::float64_t m_min_time{0.5};
nvbench::float64_t m_max_noise{0.5}; // 0.5% relative standard deviation
nvbench::float64_t m_max_noise{0.005}; // 0.5% relative standard deviation
nvbench::float64_t m_skip_time{-1.};
nvbench::float64_t m_timeout{15.};

2
nvbench/csv_printer.cu
View File

@@ -142,7 +142,7 @@ void csv_printer::do_print_benchmark_results(const benchmark_vector &benches)
}
else if (hint == "percentage")
{
table.add_cell(row, key, header + " (%)", std::move(value));
table.add_cell(row, key, header, std::move(value));
}
else
{

9
nvbench/detail/measure_cold.cu
View File

@@ -149,7 +149,7 @@ void measure_cold_base::generate_summaries()
summ.set_string("description",
"Global device memory throughput as a percentage of the "
"device's peak bandwidth.");
summ.set_float64("value", avg_used_gmem_bw / peak_gmem_bw * 100.);
summ.set_float64("value", avg_used_gmem_bw / peak_gmem_bw);
}
}
@@ -170,8 +170,8 @@ void measure_cold_base::generate_summaries()
"while over noise threshold ({:0.2f}% > "
"{:0.2f}%)",
timeout,
m_cuda_noise,
m_max_noise));
m_cuda_noise * 100,
m_max_noise * 100));
}
if (m_total_samples < m_min_samples)
{
@@ -221,8 +221,7 @@ void measure_cold_base::check_skip_time(nvbench::float64_t warmup_time)
void measure_cold_base::block_stream()
{
m_blocker.block(m_launch.get_stream(),
m_state.get_blocking_kernel_timeout());
m_blocker.block(m_launch.get_stream(), m_state.get_blocking_kernel_timeout());
}
} // namespace nvbench::detail

6
nvbench/detail/measure_cold.cuh
View File

@@ -99,7 +99,7 @@ protected:
nvbench::blocking_kernel m_blocker;
nvbench::int64_t m_min_samples{};
nvbench::float64_t m_max_noise{}; // % rel stdev
nvbench::float64_t m_max_noise{}; // rel stdev
nvbench::float64_t m_min_time{};
nvbench::float64_t m_skip_time{};
@@ -108,8 +108,8 @@ protected:
nvbench::int64_t m_total_samples{};
nvbench::float64_t m_total_cuda_time{};
nvbench::float64_t m_total_cpu_time{};
nvbench::float64_t m_cuda_noise{}; // % rel stdev
nvbench::float64_t m_cpu_noise{}; // % rel stdev
nvbench::float64_t m_cuda_noise{}; // rel stdev
nvbench::float64_t m_cpu_noise{}; // rel stdev
std::vector<nvbench::float64_t> m_cuda_times;
std::vector<nvbench::float64_t> m_cpu_times;

5
nvbench/detail/statistics.cuh
View File

@@ -33,7 +33,7 @@ namespace nvbench::detail
* vector, return a measure of the noise in the samples.
*
* The noise metric is the relative unbiased sample standard deviation
* expressed as a percentage: (std_dev / mean) * 100.
* (std_dev / mean).
*/
inline nvbench::float64_t
compute_noise(const std::vector<nvbench::float64_t> &data,
@@ -58,8 +58,7 @@ compute_noise(const std::vector<nvbench::float64_t> &data,
}) /
(num - 1);
const auto abs_stdev = std::sqrt(variance);
const auto rel_stdev = abs_stdev / mean;
return rel_stdev * 100.;
return abs_stdev / mean;
}
} // namespace nvbench::detail

2
nvbench/markdown_printer.cu
View File

@@ -459,7 +459,7 @@ std::string markdown_printer::do_format_sample_size(const summary &data)
std::string markdown_printer::do_format_percentage(const summary &data)
{
const auto percentage = data.get_float64("value");
return fmt::format("{:.2f}%", percentage);
return fmt::format("{:.2f}%", percentage * 100.);
}
} // namespace nvbench

4
nvbench/option_parser.cu
View File

@@ -807,8 +807,8 @@ try
bench.set_min_time(value);
}
else if (prop_arg == "--max-noise")
{
bench.set_max_noise(value);
{ // Specified as percentage, stored as ratio:
bench.set_max_noise(value / 100.);
}
else if (prop_arg == "--skip-time")
{

4
nvbench/state.cuh
View File

@@ -151,8 +151,8 @@ struct state
/// @}
/// Specify the maximum amount of noise if a measurement supports noise.
/// Noise is the relative standard deviation expressed as a percentage:
/// `noise = 100 * (stdev / mean_time)`. @{
/// Noise is the relative standard deviation:
/// `noise = stdev / mean_time`. @{
[[nodiscard]] nvbench::float64_t get_max_noise() const { return m_max_noise; }
void set_max_noise(nvbench::float64_t max_noise) { m_max_noise = max_noise; }
/// @}

3
nvbench/summary.cuh
View File

@@ -48,7 +48,8 @@ namespace nvbench
* - "bytes": "value" is an int64_t number of bytes.
* - "byte_rate": "value" is a float64_t byte rate in bytes / second.
* - "sample_size": "value" is an int64_t number of samples in a measurement.
* - "percentage": "value" is a float64_t percentage.
* - "percentage": "value" is a float64_t percentage (stored as a ratio, 1. =
* 100%).
*
* The key/value pair functionality is implemented by the
* `nvbench::named_values` base class.

4
testing/option_parser.cu
View File

@@ -1200,11 +1200,11 @@ void test_max_noise()
{
nvbench::option_parser parser;
parser.parse(
{"--benchmark", "DummyBench", "--max-noise", "12345e2"});
{"--benchmark", "DummyBench", "--max-noise", "50.3"});
const auto& states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(std::abs(states[0].get_max_noise() - 12345e2) < 1.);
ASSERT(std::abs(states[0].get_max_noise() - 0.503) < 1.-4);
}
void test_skip_time()