NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-05-13 17:55:39 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
pull-request/356
nvbench/testing/option_parser.cu
Oleksandr Pavlyk 9ea77bccaa Implement CLI option to control warmups for cold measurements (#339) 
* Implement warmup-runs count, supported as CLI

CLI option --warmup-runs implemented and documented.

The warm-up counts is enforced to always be positive.
This is necessary to ensure that JIT-ting has occurred,
and use of blocking kernel would not result in time-outs.

Test is option parser is added.

* Ensure that measure_cold::run_warmup instantiates blocking kernel

Because warm-up runs are executed without use of blocking kernel,
the blocking kernel was not jitted until actual measurements were
collected. The module loading cost incurred during the first run
shows as elevated CPU time noise value for the first measurement
as noted in https://github.com/NVIDIA/nvbench/pull/339

This PR adds `this->block_stream(); this->unblock_stream();` prior
to executing warm-up loop with use of blocking kernel disabled.

This ensures that blocking kernel is instantiated during the warm-up,
but it no other kernel is launched between its launch and stream sync
thus avoiding deadlocking.

* Rename --warmup-runs to --cold-warmup-runs, add --cold-max-warmup-walltime

Since configurable number of warmups only applies to measure_cold.cuh
rename the CLI option to reflect that.

Also add --cold-max-warmup-walltime (defaults to -1, i.e. disabled).
If enabled, exits warmup loop before request count is reached if
the wall-time expanded executign warmups exceeds this max-warmup-walltime
value.
2026-05-12 14:30:08 -05:00

1589 lines
59 KiB
Plaintext
Raw Permalink Blame History

/*
* Copyright 2021 NVIDIA Corporation
*
* Licensed under the Apache License, Version 2.0 with the LLVM exception
* (the "License"); you may not use this file except in compliance with
* the License.
*
* You may obtain a copy of the License at
*
* http://llvm.org/foundation/relicensing/LICENSE.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <nvbench/create.cuh>
#include <nvbench/option_parser.cuh>
#include <nvbench/type_list.cuh>
#include <fmt/format.h>
#include <chrono>
#include <iostream>
#include <stdexcept>
#include <system_error>
#if __has_include(<filesystem>)
#include <filesystem>
namespace fs = std::filesystem;
#elif __has_include(<experimental/filesystem>)
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
#else
#error "No <filesystem> or <experimental/filesystem> found."
#endif
#include "test_asserts.cuh"
//==============================================================================
// Declare a couple benchmarks for testing:
void DummyBench(nvbench::state &state) { state.skip("Skipping for testing."); }
NVBENCH_BENCH(DummyBench).clear_devices();
using Ts = nvbench::type_list<void, nvbench::int8_t, nvbench::uint8_t>;
using Us = nvbench::type_list<bool, nvbench::float32_t, nvbench::float64_t>;
template <typename T, typename U>
void TestBench(nvbench::state &state, nvbench::type_list<T, U>)
{
DummyBench(state);
}
NVBENCH_BENCH_TYPES(TestBench, NVBENCH_TYPE_AXES(Ts, Us))
.set_type_axes_names({"T", "U"})
.add_int64_axis("Ints", {42})
.add_int64_power_of_two_axis("PO2s", {3})
.add_float64_axis("Floats", {3.14})
.add_string_axis("Strings", {"S1"})
.clear_devices();
//==============================================================================
namespace
{
struct temp_tree
{
explicit temp_tree(fs::path root)
: root_path{std::move(root)}
{
std::error_code ec;
fs::remove_all(root_path, ec);
if (ec)
{
throw std::runtime_error{fmt::format("Failed to remove temporary directory `{}`: {}",
root_path.string(),
ec.message())};
}
}
~temp_tree()
{
std::error_code ec;
fs::remove_all(root_path, ec);
if (ec)
{
std::cerr << "Failed to remove temporary directory `" << root_path.string()
<< "`: " << ec.message() << "\n";
}
}
temp_tree(const temp_tree &) = delete;
temp_tree(temp_tree &&) = delete;
temp_tree &operator=(const temp_tree &) = delete;
temp_tree &operator=(temp_tree &&) = delete;
fs::path root_path;
};
[[nodiscard]] std::string states_to_string(const std::vector<nvbench::state> &states)
{
fmt::memory_buffer buffer;
std::string table_format = "| {:^5} | {:^10} | {:^4} | {:^4} | {:^4} "
"| {:^4} | {:^6} | {:^8} |\n";
fmt::format_to(std::back_inserter(buffer), "\n");
fmt::format_to(std::back_inserter(buffer),
table_format,
"State",
"TypeConfig",
"T",
"U",
"Ints",
"PO2s",
"Floats",
"Strings");
std::size_t config = 0;
for (const auto &state : states)
{
fmt::format_to(std::back_inserter(buffer),
table_format,
config++,
state.get_type_config_index(),
state.get_string("T"),
state.get_string("U"),
state.get_int64("Ints"),
state.get_int64("PO2s"),
state.get_float64("Floats"),
std::string{"\'"} + state.get_string("Strings") + "'");
}
return fmt::to_string(buffer);
}
// Expects the parser to have a single TestBench benchmark. Runs the benchmark
// and returns the resulting states.
[[nodiscard]] const auto &parser_to_states(nvbench::option_parser &parser)
{
const auto &benches = parser.get_benchmarks();
ASSERT(benches.size() == 1);
const auto &bench = benches.front();
ASSERT(bench != nullptr);
bench->run();
return bench->get_states();
}
// Expects the parser to have a single TestBench benchmark. Runs the benchmark
// and converts the generated states into a fingerprint string for regression
// testing.
[[nodiscard]] std::string parser_to_state_string(nvbench::option_parser &parser)
{
return states_to_string(parser_to_states(parser));
}
} // namespace
void test_empty()
{
{
nvbench::option_parser parser;
parser.parse({});
ASSERT(parser.get_benchmarks().size() == 2);
ASSERT(parser.get_args().empty());
}
{
nvbench::option_parser parser;
parser.parse(0, nullptr);
ASSERT(parser.get_benchmarks().size() == 2);
ASSERT(parser.get_args().empty());
}
}
void test_exec_name_tolerance()
{
nvbench::option_parser parser;
parser.parse({"TestExec"});
ASSERT(parser.get_benchmarks().size() == 2);
ASSERT(parser.get_args() == std::vector<std::string>{"TestExec"});
}
void test_argc_argv_parse()
{
char const *const argv[] = {"TestExec"};
{
nvbench::option_parser parser;
parser.parse(1, argv);
ASSERT(parser.get_benchmarks().size() == 2);
ASSERT(parser.get_args() == std::vector<std::string>{"TestExec"});
}
{
nvbench::option_parser parser;
parser.parse(0, nullptr);
ASSERT(parser.get_benchmarks().size() == 2);
ASSERT(parser.get_args().empty());
}
}
void test_invalid_option()
{
nvbench::option_parser parser;
ASSERT_THROWS_ANY(parser.parse({"--not-a-real-option"}));
}
void test_benchmark_long() // --benchmark
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 42 | 8 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "1"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_benchmark_short() // -b
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 42 | 8 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"-b", "TestBench"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"-b", "1"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 2 | 8 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 2 | 8 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 2 | 8 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 2 | 8 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 2 | 8 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 2 | 8 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 2 | 8 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 2 | 8 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 2 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ ] = 2 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints=2"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ ] = [ 2 ]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints=[2]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ ] = [ 2 : 2 : 1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints=[2:2]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 2 | 8 | 3.14 | 'S1' |
| 1 | 0 | void | bool | 7 | 8 | 3.14 | 'S1' |
| 2 | 1 | void | F32 | 2 | 8 | 3.14 | 'S1' |
| 3 | 1 | void | F32 | 7 | 8 | 3.14 | 'S1' |
| 4 | 2 | void | F64 | 2 | 8 | 3.14 | 'S1' |
| 5 | 2 | void | F64 | 7 | 8 | 3.14 | 'S1' |
| 6 | 3 | I8 | bool | 2 | 8 | 3.14 | 'S1' |
| 7 | 3 | I8 | bool | 7 | 8 | 3.14 | 'S1' |
| 8 | 4 | I8 | F32 | 2 | 8 | 3.14 | 'S1' |
| 9 | 4 | I8 | F32 | 7 | 8 | 3.14 | 'S1' |
| 10 | 5 | I8 | F64 | 2 | 8 | 3.14 | 'S1' |
| 11 | 5 | I8 | F64 | 7 | 8 | 3.14 | 'S1' |
| 12 | 6 | U8 | bool | 2 | 8 | 3.14 | 'S1' |
| 13 | 6 | U8 | bool | 7 | 8 | 3.14 | 'S1' |
| 14 | 7 | U8 | F32 | 2 | 8 | 3.14 | 'S1' |
| 15 | 7 | U8 | F32 | 7 | 8 | 3.14 | 'S1' |
| 16 | 8 | U8 | F64 | 2 | 8 | 3.14 | 'S1' |
| 17 | 8 | U8 | F64 | 7 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ ] = [ 2 , 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints=[2,7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ ] = [ 2 : 7 : 5 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints=[2:7:5]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_pow2_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 128 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 42 | 128 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 42 | 128 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 42 | 128 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 42 | 128 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 42 | 128 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 42 | 128 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 42 | 128 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 42 | 128 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] = 7 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]=7"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] = [ 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]=[7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] = [ 7 : 7 : 1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]=[7:7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_pow2_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 4 | 3.14 | 'S1' |
| 1 | 0 | void | bool | 42 | 128 | 3.14 | 'S1' |
| 2 | 1 | void | F32 | 42 | 4 | 3.14 | 'S1' |
| 3 | 1 | void | F32 | 42 | 128 | 3.14 | 'S1' |
| 4 | 2 | void | F64 | 42 | 4 | 3.14 | 'S1' |
| 5 | 2 | void | F64 | 42 | 128 | 3.14 | 'S1' |
| 6 | 3 | I8 | bool | 42 | 4 | 3.14 | 'S1' |
| 7 | 3 | I8 | bool | 42 | 128 | 3.14 | 'S1' |
| 8 | 4 | I8 | F32 | 42 | 4 | 3.14 | 'S1' |
| 9 | 4 | I8 | F32 | 42 | 128 | 3.14 | 'S1' |
| 10 | 5 | I8 | F64 | 42 | 4 | 3.14 | 'S1' |
| 11 | 5 | I8 | F64 | 42 | 128 | 3.14 | 'S1' |
| 12 | 6 | U8 | bool | 42 | 4 | 3.14 | 'S1' |
| 13 | 6 | U8 | bool | 42 | 128 | 3.14 | 'S1' |
| 14 | 7 | U8 | F32 | 42 | 4 | 3.14 | 'S1' |
| 15 | 7 | U8 | F32 | 42 | 128 | 3.14 | 'S1' |
| 16 | 8 | U8 | F64 | 42 | 4 | 3.14 | 'S1' |
| 17 | 8 | U8 | F64 | 42 | 128 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] = [ 2 , 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]=[2,7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] = [ 2 : 7 : 5 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]=[2:7:5]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_none_to_pow2_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 128 | 8 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 128 | 8 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 128 | 8 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 128 | 8 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 128 | 8 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 128 | 8 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 128 | 8 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 128 | 8 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 128 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] = 7 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]=7"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] = [ 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]=[7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] = [ 7 : 7 : 1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]=[7:7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_none_to_pow2_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 4 | 8 | 3.14 | 'S1' |
| 1 | 0 | void | bool | 128 | 8 | 3.14 | 'S1' |
| 2 | 1 | void | F32 | 4 | 8 | 3.14 | 'S1' |
| 3 | 1 | void | F32 | 128 | 8 | 3.14 | 'S1' |
| 4 | 2 | void | F64 | 4 | 8 | 3.14 | 'S1' |
| 5 | 2 | void | F64 | 128 | 8 | 3.14 | 'S1' |
| 6 | 3 | I8 | bool | 4 | 8 | 3.14 | 'S1' |
| 7 | 3 | I8 | bool | 128 | 8 | 3.14 | 'S1' |
| 8 | 4 | I8 | F32 | 4 | 8 | 3.14 | 'S1' |
| 9 | 4 | I8 | F32 | 128 | 8 | 3.14 | 'S1' |
| 10 | 5 | I8 | F64 | 4 | 8 | 3.14 | 'S1' |
| 11 | 5 | I8 | F64 | 128 | 8 | 3.14 | 'S1' |
| 12 | 6 | U8 | bool | 4 | 8 | 3.14 | 'S1' |
| 13 | 6 | U8 | bool | 128 | 8 | 3.14 | 'S1' |
| 14 | 7 | U8 | F32 | 4 | 8 | 3.14 | 'S1' |
| 15 | 7 | U8 | F32 | 128 | 8 | 3.14 | 'S1' |
| 16 | 8 | U8 | F64 | 4 | 8 | 3.14 | 'S1' |
| 17 | 8 | U8 | F64 | 128 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] = [ 2 , 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]=[2,7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] = [ 2 : 7 : 5 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]=[2:7:5]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_pow2_to_none_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 2 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 42 | 2 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 42 | 2 | 3.14 | 'S1' |
| 3 | 3 | I8 | bool | 42 | 2 | 3.14 | 'S1' |
| 4 | 4 | I8 | F32 | 42 | 2 | 3.14 | 'S1' |
| 5 | 5 | I8 | F64 | 42 | 2 | 3.14 | 'S1' |
| 6 | 6 | U8 | bool | 42 | 2 | 3.14 | 'S1' |
| 7 | 7 | U8 | F32 | 42 | 2 | 3.14 | 'S1' |
| 8 | 8 | U8 | F64 | 42 | 2 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ ] = 2 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s=2"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ ] = [ 2 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s=[2]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ ] = [ 2 : 2 : 1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s=[2:2]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_int64_axis_pow2_to_none_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 2 | 3.14 | 'S1' |
| 1 | 0 | void | bool | 42 | 7 | 3.14 | 'S1' |
| 2 | 1 | void | F32 | 42 | 2 | 3.14 | 'S1' |
| 3 | 1 | void | F32 | 42 | 7 | 3.14 | 'S1' |
| 4 | 2 | void | F64 | 42 | 2 | 3.14 | 'S1' |
| 5 | 2 | void | F64 | 42 | 7 | 3.14 | 'S1' |
| 6 | 3 | I8 | bool | 42 | 2 | 3.14 | 'S1' |
| 7 | 3 | I8 | bool | 42 | 7 | 3.14 | 'S1' |
| 8 | 4 | I8 | F32 | 42 | 2 | 3.14 | 'S1' |
| 9 | 4 | I8 | F32 | 42 | 7 | 3.14 | 'S1' |
| 10 | 5 | I8 | F64 | 42 | 2 | 3.14 | 'S1' |
| 11 | 5 | I8 | F64 | 42 | 7 | 3.14 | 'S1' |
| 12 | 6 | U8 | bool | 42 | 2 | 3.14 | 'S1' |
| 13 | 6 | U8 | bool | 42 | 7 | 3.14 | 'S1' |
| 14 | 7 | U8 | F32 | 42 | 2 | 3.14 | 'S1' |
| 15 | 7 | U8 | F32 | 42 | 7 | 3.14 | 'S1' |
| 16 | 8 | U8 | F64 | 42 | 2 | 3.14 | 'S1' |
| 17 | 8 | U8 | F64 | 42 | 7 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ ] = [ 2 , 7 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s=[2,7]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " PO2s [ ] = [ 2 : 7 : 5 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "PO2s=[2:7:5]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_float64_axis_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.5 | 'S1' |
| 1 | 1 | void | F32 | 42 | 8 | 3.5 | 'S1' |
| 2 | 2 | void | F64 | 42 | 8 | 3.5 | 'S1' |
| 3 | 3 | I8 | bool | 42 | 8 | 3.5 | 'S1' |
| 4 | 4 | I8 | F32 | 42 | 8 | 3.5 | 'S1' |
| 5 | 5 | I8 | F64 | 42 | 8 | 3.5 | 'S1' |
| 6 | 6 | U8 | bool | 42 | 8 | 3.5 | 'S1' |
| 7 | 7 | U8 | F32 | 42 | 8 | 3.5 | 'S1' |
| 8 | 8 | U8 | F64 | 42 | 8 | 3.5 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Floats [ ] = 3.5 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Floats=3.5"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Floats [ ] = [ 3.5 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Floats=[3.5]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Floats [ ] = [ 3.5 : 3.6 : 1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Floats=[3.5:3.6]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_float64_axis_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.5 | 'S1' |
| 1 | 0 | void | bool | 42 | 8 | 4.1 | 'S1' |
| 2 | 1 | void | F32 | 42 | 8 | 3.5 | 'S1' |
| 3 | 1 | void | F32 | 42 | 8 | 4.1 | 'S1' |
| 4 | 2 | void | F64 | 42 | 8 | 3.5 | 'S1' |
| 5 | 2 | void | F64 | 42 | 8 | 4.1 | 'S1' |
| 6 | 3 | I8 | bool | 42 | 8 | 3.5 | 'S1' |
| 7 | 3 | I8 | bool | 42 | 8 | 4.1 | 'S1' |
| 8 | 4 | I8 | F32 | 42 | 8 | 3.5 | 'S1' |
| 9 | 4 | I8 | F32 | 42 | 8 | 4.1 | 'S1' |
| 10 | 5 | I8 | F64 | 42 | 8 | 3.5 | 'S1' |
| 11 | 5 | I8 | F64 | 42 | 8 | 4.1 | 'S1' |
| 12 | 6 | U8 | bool | 42 | 8 | 3.5 | 'S1' |
| 13 | 6 | U8 | bool | 42 | 8 | 4.1 | 'S1' |
| 14 | 7 | U8 | F32 | 42 | 8 | 3.5 | 'S1' |
| 15 | 7 | U8 | F32 | 42 | 8 | 4.1 | 'S1' |
| 16 | 8 | U8 | F64 | 42 | 8 | 3.5 | 'S1' |
| 17 | 8 | U8 | F64 | 42 | 8 | 4.1 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Floats [ ] = [ 3.5 , 4.1 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Floats=[3.5,4.1]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Floats [ ] = [ 3.5 : 4.2 : 0.6 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Floats=[3.5:4.2:0.6]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_string_axis_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'fo br' |
| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'fo br' |
| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'fo br' |
| 3 | 3 | I8 | bool | 42 | 8 | 3.14 | 'fo br' |
| 4 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'fo br' |
| 5 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'fo br' |
| 6 | 6 | U8 | bool | 42 | 8 | 3.14 | 'fo br' |
| 7 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'fo br' |
| 8 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'fo br' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Strings [ ] = fo br "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Strings=fo br"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Strings [ ] = [ fo br ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Strings=[fo br]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_string_axis_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'fo br' |
| 1 | 0 | void | bool | 42 | 8 | 3.14 | 'baz' |
| 2 | 1 | void | F32 | 42 | 8 | 3.14 | 'fo br' |
| 3 | 1 | void | F32 | 42 | 8 | 3.14 | 'baz' |
| 4 | 2 | void | F64 | 42 | 8 | 3.14 | 'fo br' |
| 5 | 2 | void | F64 | 42 | 8 | 3.14 | 'baz' |
| 6 | 3 | I8 | bool | 42 | 8 | 3.14 | 'fo br' |
| 7 | 3 | I8 | bool | 42 | 8 | 3.14 | 'baz' |
| 8 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'fo br' |
| 9 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'baz' |
| 10 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'fo br' |
| 11 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'baz' |
| 12 | 6 | U8 | bool | 42 | 8 | 3.14 | 'fo br' |
| 13 | 6 | U8 | bool | 42 | 8 | 3.14 | 'baz' |
| 14 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'fo br' |
| 15 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'baz' |
| 16 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'fo br' |
| 17 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'baz' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " Strings [ ] = [ fo br , baz ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "Strings=[fo br,baz]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_type_axis_single()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' |
| 1 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 2 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " T [ ] = U8 "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "T=U8"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " T [ ] = [ U8 ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "T=[U8]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_type_axis_multi()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' |
| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' |
| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' |
| 3 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' |
| 4 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' |
| 5 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' |
)expected";
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", " T [ ] = [ U8, void ] "});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "TestBench", "--axis", "T=[void,U8]"});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
void test_multi_axis()
{
const std::string ref =
R"expected(
| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings |
| 0 | 0 | void | bool | 2 | 4 | 0.25 | 'foo' |
| 1 | 0 | void | bool | 5 | 4 | 0.25 | 'foo' |
| 2 | 0 | void | bool | 2 | 32 | 0.25 | 'foo' |
| 3 | 0 | void | bool | 5 | 32 | 0.25 | 'foo' |
| 4 | 0 | void | bool | 2 | 256 | 0.25 | 'foo' |
| 5 | 0 | void | bool | 5 | 256 | 0.25 | 'foo' |
| 6 | 0 | void | bool | 2 | 4 | 0.5 | 'foo' |
| 7 | 0 | void | bool | 5 | 4 | 0.5 | 'foo' |
| 8 | 0 | void | bool | 2 | 32 | 0.5 | 'foo' |
| 9 | 0 | void | bool | 5 | 32 | 0.5 | 'foo' |
| 10 | 0 | void | bool | 2 | 256 | 0.5 | 'foo' |
| 11 | 0 | void | bool | 5 | 256 | 0.5 | 'foo' |
| 12 | 0 | void | bool | 2 | 4 | 0.75 | 'foo' |
| 13 | 0 | void | bool | 5 | 4 | 0.75 | 'foo' |
| 14 | 0 | void | bool | 2 | 32 | 0.75 | 'foo' |
| 15 | 0 | void | bool | 5 | 32 | 0.75 | 'foo' |
| 16 | 0 | void | bool | 2 | 256 | 0.75 | 'foo' |
| 17 | 0 | void | bool | 5 | 256 | 0.75 | 'foo' |
| 18 | 0 | void | bool | 2 | 4 | 1 | 'foo' |
| 19 | 0 | void | bool | 5 | 4 | 1 | 'foo' |
| 20 | 0 | void | bool | 2 | 32 | 1 | 'foo' |
| 21 | 0 | void | bool | 5 | 32 | 1 | 'foo' |
| 22 | 0 | void | bool | 2 | 256 | 1 | 'foo' |
| 23 | 0 | void | bool | 5 | 256 | 1 | 'foo' |
| 24 | 0 | void | bool | 2 | 4 | 0.25 | 'bar' |
| 25 | 0 | void | bool | 5 | 4 | 0.25 | 'bar' |
| 26 | 0 | void | bool | 2 | 32 | 0.25 | 'bar' |
| 27 | 0 | void | bool | 5 | 32 | 0.25 | 'bar' |
| 28 | 0 | void | bool | 2 | 256 | 0.25 | 'bar' |
| 29 | 0 | void | bool | 5 | 256 | 0.25 | 'bar' |
| 30 | 0 | void | bool | 2 | 4 | 0.5 | 'bar' |
| 31 | 0 | void | bool | 5 | 4 | 0.5 | 'bar' |
| 32 | 0 | void | bool | 2 | 32 | 0.5 | 'bar' |
| 33 | 0 | void | bool | 5 | 32 | 0.5 | 'bar' |
| 34 | 0 | void | bool | 2 | 256 | 0.5 | 'bar' |
| 35 | 0 | void | bool | 5 | 256 | 0.5 | 'bar' |
| 36 | 0 | void | bool | 2 | 4 | 0.75 | 'bar' |
| 37 | 0 | void | bool | 5 | 4 | 0.75 | 'bar' |
| 38 | 0 | void | bool | 2 | 32 | 0.75 | 'bar' |
| 39 | 0 | void | bool | 5 | 32 | 0.75 | 'bar' |
| 40 | 0 | void | bool | 2 | 256 | 0.75 | 'bar' |
| 41 | 0 | void | bool | 5 | 256 | 0.75 | 'bar' |
| 42 | 0 | void | bool | 2 | 4 | 1 | 'bar' |
| 43 | 0 | void | bool | 5 | 4 | 1 | 'bar' |
| 44 | 0 | void | bool | 2 | 32 | 1 | 'bar' |
| 45 | 0 | void | bool | 5 | 32 | 1 | 'bar' |
| 46 | 0 | void | bool | 2 | 256 | 1 | 'bar' |
| 47 | 0 | void | bool | 5 | 256 | 1 | 'bar' |
| 48 | 0 | void | bool | 2 | 4 | 0.25 | 'baz' |
| 49 | 0 | void | bool | 5 | 4 | 0.25 | 'baz' |
| 50 | 0 | void | bool | 2 | 32 | 0.25 | 'baz' |
| 51 | 0 | void | bool | 5 | 32 | 0.25 | 'baz' |
| 52 | 0 | void | bool | 2 | 256 | 0.25 | 'baz' |
| 53 | 0 | void | bool | 5 | 256 | 0.25 | 'baz' |
| 54 | 0 | void | bool | 2 | 4 | 0.5 | 'baz' |
| 55 | 0 | void | bool | 5 | 4 | 0.5 | 'baz' |
| 56 | 0 | void | bool | 2 | 32 | 0.5 | 'baz' |
| 57 | 0 | void | bool | 5 | 32 | 0.5 | 'baz' |
| 58 | 0 | void | bool | 2 | 256 | 0.5 | 'baz' |
| 59 | 0 | void | bool | 5 | 256 | 0.5 | 'baz' |
| 60 | 0 | void | bool | 2 | 4 | 0.75 | 'baz' |
| 61 | 0 | void | bool | 5 | 4 | 0.75 | 'baz' |
| 62 | 0 | void | bool | 2 | 32 | 0.75 | 'baz' |
| 63 | 0 | void | bool | 5 | 32 | 0.75 | 'baz' |
| 64 | 0 | void | bool | 2 | 256 | 0.75 | 'baz' |
| 65 | 0 | void | bool | 5 | 256 | 0.75 | 'baz' |
| 66 | 0 | void | bool | 2 | 4 | 1 | 'baz' |
| 67 | 0 | void | bool | 5 | 4 | 1 | 'baz' |
| 68 | 0 | void | bool | 2 | 32 | 1 | 'baz' |
| 69 | 0 | void | bool | 5 | 32 | 1 | 'baz' |
| 70 | 0 | void | bool | 2 | 256 | 1 | 'baz' |
| 71 | 0 | void | bool | 5 | 256 | 1 | 'baz' |
| 72 | 6 | U8 | bool | 2 | 4 | 0.25 | 'foo' |
| 73 | 6 | U8 | bool | 5 | 4 | 0.25 | 'foo' |
| 74 | 6 | U8 | bool | 2 | 32 | 0.25 | 'foo' |
| 75 | 6 | U8 | bool | 5 | 32 | 0.25 | 'foo' |
| 76 | 6 | U8 | bool | 2 | 256 | 0.25 | 'foo' |
| 77 | 6 | U8 | bool | 5 | 256 | 0.25 | 'foo' |
| 78 | 6 | U8 | bool | 2 | 4 | 0.5 | 'foo' |
| 79 | 6 | U8 | bool | 5 | 4 | 0.5 | 'foo' |
| 80 | 6 | U8 | bool | 2 | 32 | 0.5 | 'foo' |
| 81 | 6 | U8 | bool | 5 | 32 | 0.5 | 'foo' |
| 82 | 6 | U8 | bool | 2 | 256 | 0.5 | 'foo' |
| 83 | 6 | U8 | bool | 5 | 256 | 0.5 | 'foo' |
| 84 | 6 | U8 | bool | 2 | 4 | 0.75 | 'foo' |
| 85 | 6 | U8 | bool | 5 | 4 | 0.75 | 'foo' |
| 86 | 6 | U8 | bool | 2 | 32 | 0.75 | 'foo' |
| 87 | 6 | U8 | bool | 5 | 32 | 0.75 | 'foo' |
| 88 | 6 | U8 | bool | 2 | 256 | 0.75 | 'foo' |
| 89 | 6 | U8 | bool | 5 | 256 | 0.75 | 'foo' |
| 90 | 6 | U8 | bool | 2 | 4 | 1 | 'foo' |
| 91 | 6 | U8 | bool | 5 | 4 | 1 | 'foo' |
| 92 | 6 | U8 | bool | 2 | 32 | 1 | 'foo' |
| 93 | 6 | U8 | bool | 5 | 32 | 1 | 'foo' |
| 94 | 6 | U8 | bool | 2 | 256 | 1 | 'foo' |
| 95 | 6 | U8 | bool | 5 | 256 | 1 | 'foo' |
| 96 | 6 | U8 | bool | 2 | 4 | 0.25 | 'bar' |
| 97 | 6 | U8 | bool | 5 | 4 | 0.25 | 'bar' |
| 98 | 6 | U8 | bool | 2 | 32 | 0.25 | 'bar' |
| 99 | 6 | U8 | bool | 5 | 32 | 0.25 | 'bar' |
| 100 | 6 | U8 | bool | 2 | 256 | 0.25 | 'bar' |
| 101 | 6 | U8 | bool | 5 | 256 | 0.25 | 'bar' |
| 102 | 6 | U8 | bool | 2 | 4 | 0.5 | 'bar' |
| 103 | 6 | U8 | bool | 5 | 4 | 0.5 | 'bar' |
| 104 | 6 | U8 | bool | 2 | 32 | 0.5 | 'bar' |
| 105 | 6 | U8 | bool | 5 | 32 | 0.5 | 'bar' |
| 106 | 6 | U8 | bool | 2 | 256 | 0.5 | 'bar' |
| 107 | 6 | U8 | bool | 5 | 256 | 0.5 | 'bar' |
| 108 | 6 | U8 | bool | 2 | 4 | 0.75 | 'bar' |
| 109 | 6 | U8 | bool | 5 | 4 | 0.75 | 'bar' |
| 110 | 6 | U8 | bool | 2 | 32 | 0.75 | 'bar' |
| 111 | 6 | U8 | bool | 5 | 32 | 0.75 | 'bar' |
| 112 | 6 | U8 | bool | 2 | 256 | 0.75 | 'bar' |
| 113 | 6 | U8 | bool | 5 | 256 | 0.75 | 'bar' |
| 114 | 6 | U8 | bool | 2 | 4 | 1 | 'bar' |
| 115 | 6 | U8 | bool | 5 | 4 | 1 | 'bar' |
| 116 | 6 | U8 | bool | 2 | 32 | 1 | 'bar' |
| 117 | 6 | U8 | bool | 5 | 32 | 1 | 'bar' |
| 118 | 6 | U8 | bool | 2 | 256 | 1 | 'bar' |
| 119 | 6 | U8 | bool | 5 | 256 | 1 | 'bar' |
| 120 | 6 | U8 | bool | 2 | 4 | 0.25 | 'baz' |
| 121 | 6 | U8 | bool | 5 | 4 | 0.25 | 'baz' |
| 122 | 6 | U8 | bool | 2 | 32 | 0.25 | 'baz' |
| 123 | 6 | U8 | bool | 5 | 32 | 0.25 | 'baz' |
| 124 | 6 | U8 | bool | 2 | 256 | 0.25 | 'baz' |
| 125 | 6 | U8 | bool | 5 | 256 | 0.25 | 'baz' |
| 126 | 6 | U8 | bool | 2 | 4 | 0.5 | 'baz' |
| 127 | 6 | U8 | bool | 5 | 4 | 0.5 | 'baz' |
| 128 | 6 | U8 | bool | 2 | 32 | 0.5 | 'baz' |
| 129 | 6 | U8 | bool | 5 | 32 | 0.5 | 'baz' |
| 130 | 6 | U8 | bool | 2 | 256 | 0.5 | 'baz' |
| 131 | 6 | U8 | bool | 5 | 256 | 0.5 | 'baz' |
| 132 | 6 | U8 | bool | 2 | 4 | 0.75 | 'baz' |
| 133 | 6 | U8 | bool | 5 | 4 | 0.75 | 'baz' |
| 134 | 6 | U8 | bool | 2 | 32 | 0.75 | 'baz' |
| 135 | 6 | U8 | bool | 5 | 32 | 0.75 | 'baz' |
| 136 | 6 | U8 | bool | 2 | 256 | 0.75 | 'baz' |
| 137 | 6 | U8 | bool | 5 | 256 | 0.75 | 'baz' |
| 138 | 6 | U8 | bool | 2 | 4 | 1 | 'baz' |
| 139 | 6 | U8 | bool | 5 | 4 | 1 | 'baz' |
| 140 | 6 | U8 | bool | 2 | 32 | 1 | 'baz' |
| 141 | 6 | U8 | bool | 5 | 32 | 1 | 'baz' |
| 142 | 6 | U8 | bool | 2 | 256 | 1 | 'baz' |
| 143 | 6 | U8 | bool | 5 | 256 | 1 | 'baz' |
)expected";
{
nvbench::option_parser parser;
parser.parse({
// clang-format off
"--benchmark", "TestBench",
"--axis", "T=[U8,void]",
"--axis", "U=bool",
"--axis", "Ints=[2:6:3]",
"--axis", "PO2s[pow2]=[2:10:3]",
"--axis", "Floats=[0.25:1:0.25]",
"--axis", "Strings=[foo,bar,baz]",
// clang-format on
});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
{
nvbench::option_parser parser;
parser.parse({
// clang-format off
"-b", "TestBench",
"-a", "Strings=[foo,bar,baz]",
"-a", "U=bool",
"-a", "Floats=[0.25:1:0.25]",
"-a", "Ints=[2:6:3]",
"-a", "PO2s[pow2]=[2:10:3]",
"-a", "T=[U8,void]",
// clang-format on
});
const auto test = parser_to_state_string(parser);
ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test);
}
}
// `--axis` affects the last `--benchmark`. An exception is thrown if there is
// no benchmark specified for an axis.
void test_axis_before_benchmark()
{
{
nvbench::option_parser parser;
ASSERT_THROWS_ANY(parser.parse({"--axis", "--benchmark"}));
}
{
nvbench::option_parser parser;
ASSERT_THROWS_ANY(parser.parse({"--axis", "-b"}));
}
{
nvbench::option_parser parser;
ASSERT_THROWS_ANY(parser.parse({"-a", "--benchmark"}));
}
{
nvbench::option_parser parser;
ASSERT_THROWS_ANY(parser.parse({"-a", "-b"}));
}
}
void test_min_samples()
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--min-samples", "12345"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_min_samples() == 12345);
}
void test_cold_warmup_runs()
{
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--cold-warmup-runs", "12345"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_cold_warmup_runs() == 12345);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--cold-warmup-runs", "0"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_cold_warmup_runs() == 1);
}
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--cold-warmup-runs", "-12345"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_cold_warmup_runs() == 1);
}
}
void test_skip_time()
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--skip-time", "12345e2"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(std::abs(states[0].get_skip_time() - 12345e2) < 1.);
}
void test_cold_max_warmup_walltime()
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--cold-max-warmup-walltime", "12345e2"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(std::abs(states[0].get_cold_max_warmup_walltime() - 12345e2) < 1.);
}
void test_timeout()
{
nvbench::option_parser parser;
parser.parse({"--benchmark", "DummyBench", "--timeout", "12345e2"});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(std::abs(states[0].get_timeout() - 12345e2) < 1.);
}
void test_output_parent_directories_created()
{
const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count();
const temp_tree temp{fs::temp_directory_path() /
fmt::format("nvbench_option_parser_test_{}", unique_suffix)};
const auto output_path = temp.root_path / "nested" / "results.json";
{
nvbench::option_parser parser;
parser.parse({"--json", output_path.string()});
}
ASSERT(fs::is_directory(output_path.parent_path()));
ASSERT(fs::exists(output_path));
}
void test_stopping_criterion()
{
{ // Per benchmark criterion
nvbench::option_parser parser;
parser.parse({
"--benchmark",
"DummyBench",
"--stopping-criterion",
"entropy",
"--max-angle",
"0.42",
"--min-r2",
"0.6",
});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_stopping_criterion() == "entropy");
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
ASSERT(criterion_params.has_value("max-angle"));
ASSERT(criterion_params.has_value("min-r2"));
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
}
{ // Global criterion
nvbench::option_parser parser;
parser.parse({
"--stopping-criterion",
"entropy",
"--max-angle",
"0.42",
"--min-r2",
"0.6",
"--benchmark",
"DummyBench",
});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_stopping_criterion() == "entropy");
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
ASSERT(criterion_params.has_value("max-angle"));
ASSERT(criterion_params.has_value("min-r2"));
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
}
{ // Global criterion, per-benchmark params
nvbench::option_parser parser;
parser.parse({
"--stopping-criterion",
"entropy",
"--benchmark",
"DummyBench",
"--max-angle",
"0.42",
"--min-r2",
"0.6",
});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_stopping_criterion() == "entropy");
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
ASSERT(criterion_params.has_value("max-angle"));
ASSERT(criterion_params.has_value("min-r2"));
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
}
{ // Global params to default criterion should work:
nvbench::option_parser parser;
parser.parse({
"--max-noise",
"0.5",
"--min-time",
"0.1",
"--benchmark",
"DummyBench",
"--stopping-criterion",
"entropy",
"--max-angle",
"0.42",
"--min-r2",
"0.6",
});
const auto &states = parser_to_states(parser);
ASSERT(states.size() == 1);
ASSERT(states[0].get_stopping_criterion() == "entropy");
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
ASSERT(criterion_params.has_value("max-angle"));
ASSERT(criterion_params.has_value("min-r2"));
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
}
{ // Unknown stopping criterion should throw
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--benchmark",
"DummyBench",
"--stopping-criterion",
"I_do_not_exist",
});
}
catch (const std::runtime_error &)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // Global criterion to non-default params without global --stopping-criterion should throw
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--max-angle",
"0.42",
"--min-r2",
"0.6",
"--benchmark",
"DummyBench",
"--stopping-criterion",
"entropy",
});
}
catch (const std::runtime_error &)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // Invalid global param throws exception:
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--max-angle",
"0.42",
"--benchmark",
"DummyBench",
"--stopping-criterion",
"entropy",
"--min-r2",
"0.6",
"--max-angle",
"0.42",
"--benchmark",
"TestBench",
"--stopping-criterion",
"stdrel",
});
}
catch (const std::runtime_error & /*ex*/)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // Invalid per-bench param throws exception:
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--benchmark",
"DummyBench",
"--stopping-criterion",
"entropy",
"--min-r2",
"0.6",
"--max-angle",
"0.42",
"--benchmark",
"TestBench",
"--stopping-criterion",
"stdrel",
"--max-angle",
"0.42",
});
}
catch (const std::runtime_error & /*ex*/)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // global param-before-criterion throws exception:
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--min-r2", //
"0.6",
"--stopping-criterion",
"entropy",
"--benchmark",
"DummyBench",
});
}
catch (const std::runtime_error & /*ex*/)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // per-benchmark param-before-criterion throws exception:
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--benchmark", //
"DummyBench",
"--min-r2",
"0.6",
"--stopping-criterion",
"entropy",
});
}
catch (const std::runtime_error & /*ex*/)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
{ // Invalid param type throws exception:
bool exception_thrown = false;
try
{
nvbench::option_parser parser;
parser.parse({
"--benchmark", //
"DummyBench",
"--stopping-criterion",
"entropy",
"--min-r2",
"\"foo\"",
});
}
catch (const std::runtime_error &)
{
exception_thrown = true;
}
ASSERT(exception_thrown);
}
}
int main()
try
{
test_empty();
test_exec_name_tolerance();
test_argc_argv_parse();
test_invalid_option();
test_benchmark_long();
test_benchmark_short();
test_int64_axis_single();
test_int64_axis_multi();
test_int64_axis_pow2_single();
test_int64_axis_pow2_multi();
test_int64_axis_none_to_pow2_single();
test_int64_axis_none_to_pow2_multi();
test_int64_axis_pow2_to_none_single();
test_int64_axis_pow2_to_none_multi();
test_float64_axis_single();
test_float64_axis_multi();
test_string_axis_single();
test_string_axis_multi();
test_type_axis_single();
test_type_axis_multi();
test_multi_axis();
test_axis_before_benchmark();
test_min_samples();
test_cold_warmup_runs();
test_skip_time();
test_cold_max_warmup_walltime();
test_timeout();
test_output_parent_directories_created();
test_stopping_criterion();
return 0;
}
catch (std::exception &err)
{
fmt::print(stderr, "Unexpected exception:\n{}\n", err.what());
return 1;
}
Reference in New Issue View Git Blame Copy Permalink