diff --git a/nvbench/CMakeLists.txt b/nvbench/CMakeLists.txt index 421e09c..5c996a8 100644 --- a/nvbench/CMakeLists.txt +++ b/nvbench/CMakeLists.txt @@ -7,6 +7,7 @@ set(srcs float64_axis.cu int64_axis.cu named_values.cu + option_parser.cu state.cu string_axis.cu type_axis.cu diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu new file mode 100644 index 0000000..6f2775b --- /dev/null +++ b/nvbench/option_parser.cu @@ -0,0 +1,469 @@ +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +//============================================================================== +// helpers types for using std::string_view with std::regex +using sv_citer = std::string_view::const_iterator; +using sv_match = std::match_results; +using sv_submatch = std::sub_match; +using sv_regex_iterator = std::regex_iterator; +std::string_view submatch_to_sv(const sv_submatch &in) +{ + // This will be much easier in C++20, but this string_view constructor is + // painfully absent until then: + // return {in.first, in.second}; + + // C++17 version: + if (in.first == in.second) + { + return {}; + } + + // We have to use the (ptr, len) ctor + return {&*in.first, static_cast(in.length())}; +} +//============================================================================== + +template +void parse(std::string_view input, T &val) +{ + // std::from_chars requires const char *, not iterators, grumble grumble + auto [_, err] = std::from_chars(&*input.cbegin(), &*input.cend(), val); + if (err != std::errc()) + { + throw std::runtime_error(fmt::format("{}:{}: Error parsing value from " + "string '{}'", + __FILE__, + __LINE__, + input)); + } +} + +void parse(std::string_view input, std::string &val) { val = input; } + +// Parses a list of values ", , , ..." into a vector: +template +std::vector parse_list_values(std::string_view list_spec) +{ + std::vector result; + + static const std::regex value_regex{ + "\\s*" // Whitespace + "([^,]+?)" // Single value + "\\s*" // Whitespace + "(?:,|$)" // Delimiters + }; + + auto values_begin = + sv_regex_iterator(list_spec.cbegin(), list_spec.cend(), value_regex); + auto values_end = sv_regex_iterator{}; + while (values_begin != values_end) + { + auto match = *values_begin++; + std::string_view sv = submatch_to_sv(match[1]); + ; + T val; + parse(sv, val); + result.push_back(std::move(val)); + } + + return result; +} + +// Parses a range specification " : [ : ]" and returns +// a vector filled with the specified range. +template +std::vector parse_range_values(std::string_view range_spec, + nvbench::wrapped_type) +{ + std::vector range_params; + + static const std::regex value_regex{ + "\\s*" // Whitespace + "([^:]+?)" // Single value + "\\s*" // Whitespace + "(?:$|:)" // Delimiters + }; + + auto values_begin = + sv_regex_iterator(range_spec.cbegin(), range_spec.cend(), value_regex); + auto values_end = sv_regex_iterator{}; + for (; values_begin != values_end; ++values_begin) + { + auto match = *values_begin; + std::string_view sv = submatch_to_sv(match[1]); + T val; + parse(sv, val); + range_params.push_back(std::move(val)); + } + + // Convert the parsed values into a range: + if (range_params.size() != 2 && range_params.size() != 3) + { + throw std::runtime_error(fmt::format("{}:{}: Expected 2 or 3 values for " + "range specification: {}", + __FILE__, + __LINE__, + range_spec)); + } + + const T first = range_params[0]; + const T last = range_params[1]; + const T stride = range_params.size() == 3 ? range_params[2] : T{1}; + + return nvbench::range(first, last, stride); +} + +// Disable range parsing for string types +std::vector parse_range_values(std::string_view range_spec, + nvbench::wrapped_type) +{ + throw std::runtime_error(fmt::format("{}:{}: Cannot use range syntax for " + "string axis specification: `{}`.", + __FILE__, + __LINE__, + range_spec)); +} + +template +std::vector parse_values(std::string_view value_spec) +{ + static const std::regex list_regex{"\\{" // Literal { + "\\s*" // Whitespace + "([^\\}]+?)" // list of values + "\\s*" // Whitespace + "\\}"}; // Literal } + + static const std::regex range_regex{"\\(" // Literal ( + "\\s*" // Whitespace + "([^\\)]+?)" // range spec + "\\s*" // Whitespace + "\\)"}; // Literal ) + sv_match match; + if (std::regex_search(value_spec.cbegin(), + value_spec.cend(), + match, + list_regex)) + { + return parse_list_values(submatch_to_sv(match[1])); + } + else if (std::regex_search(value_spec.cbegin(), + value_spec.cend(), + match, + range_regex)) + { + return parse_range_values(submatch_to_sv(match[1]), + nvbench::wrapped_type{}); + } + else + { + throw std::runtime_error(fmt::format("{}:{}: Invalid axis value spec: {}", + __FILE__, + __LINE__, + value_spec)); + } +} + +// Parse an axis specification into a 3-tuple of string_views containing the +// axis name, flags, and values. +auto parse_axis_key_flag_value_spec(const std::string &spec) +{ + static const std::regex spec_regex{ + "\\s*" // Optional Whitespace + "([^\\[:]+?)" // Axis name + "\\s*" // Optional Whitespace + + "(?:" // Start optional non-capture group for tag + "\\[" // - Literal [ + "\\s*" // - Optional Whitespace + "([^\\]]*?)" // - Flag spec + "\\s*" // - Optional Whitespace + "\\]" // - Literal ] + ")?" // End optional tag group + + "\\s*" // Optional Whitespace + ":" // Literal : + "\\s*" // Optional Whitespace + "(.+?)" // Value spec + "\\s*" // Optional Whitespace + "$" // end + }; + + sv_match match; + const std::string_view spec_sv = spec; + if (!std::regex_search(spec_sv.cbegin(), spec_sv.cend(), match, spec_regex)) + { + throw std::runtime_error( + fmt::format("{}:{}: Bad format.", __FILE__, __LINE__)); + } + + // Extract the matches: + const auto name = submatch_to_sv(match[1]); + const auto flag = submatch_to_sv(match[2]); + const auto vals = submatch_to_sv(match[3]); + return std::tie(name, flag, vals); +} + +} // namespace + +namespace nvbench +{ + +void option_parser::parse(int argc, char const *const *argv) +{ + m_args.clear(); + m_args.reserve(static_cast(argc)); + for (int i = 0; i < argc; ++i) + { + m_args.emplace_back(argv[i]); + } + + parse_impl(); +} + +void option_parser::parse(std::vector args) +{ + m_args = std::move(args); + parse_impl(); +} + +void option_parser::parse_impl() +{ + auto cur_arg = m_args.cbegin(); + const auto arg_end = m_args.cend(); + + // The first arg may be the executable name: + if (cur_arg != arg_end && !cur_arg->empty() && cur_arg->front() != '-') + { + cur_arg++; + } + + auto check_params = [&cur_arg, &arg_end](std::size_t num_params) { + const std::size_t rem_args = std::distance(cur_arg, arg_end) - 1; + if (rem_args < num_params) + { + throw std::runtime_error(fmt::format("{}:{}: Option '{}' requires {} " + "parameters, {} provided.", + __FILE__, + __LINE__, + *cur_arg, + num_params, + rem_args)); + } + }; + + while (cur_arg < arg_end) + { + const auto &arg = *cur_arg; + + if (arg == "--benchmark" || arg == "-b") + { + check_params(1); + this->add_benchmark(cur_arg[1]); + cur_arg += 2; + } + else if (arg == "--axis" || arg == "-a") + { + check_params(1); + this->update_axis(cur_arg[1]); + cur_arg += 2; + } + else + { + throw std::runtime_error(fmt::format("{}:{}: Unrecognized command-line " + "argument: `{}`.", + __FILE__, + __LINE__, + arg)); + } + } +} + +void option_parser::add_benchmark(const std::string &name) +{ + const auto &mgr = nvbench::benchmark_manager::get(); + m_benchmarks.push_back(mgr.get_benchmark(name).clone()); +} + +void option_parser::update_axis(const std::string &spec) +{ + // Valid examples: + // - "NumInputs [pow2] : (10 : 30 : 5)" <- Range specification (::) + // - "UniqueKeys [] : { 10, 15, 20, 25, 30 }" <- List spec {,,...} + // - "Quality : (0.0 : 1.0 : 0.1)" + // - "ValueType : { I32, F32, U64 }" + // - "RNG [] : { Uniform, Gaussian }" + // + // Generally: " [] : " + // + // Axis/Flag spec: "" (no flags) + // Axis/Flag spec: " []" (no flags) + // Axis/Flag spec: " [pow2]" (flags=`pow2`) + // Value spec: "{ , ... }" <- Explicit values + // Value spec: "( : )" <- Range, inclusive start/stop + // Value spec: "( : : )" <- Range, explicit stride + + // Check that an active benchmark exists: + if (m_benchmarks.empty()) + { + throw std::runtime_error(fmt::format("{}:{}: \"--axis <...>\" must follow " + "\"--benchmark <...>\".", + __FILE__, + __LINE__)); + } + benchmark_base &bench = *m_benchmarks.back(); + + try + { + const auto [name, flags, values] = parse_axis_key_flag_value_spec(spec); + nvbench::axis_base &axis = bench.get_axes().get_axis(name); + switch (axis.get_type()) + { + case axis_type::type: + this->update_type_axis(static_cast(axis), + values, + flags); + break; + + case axis_type::int64: + this->update_int64_axis(static_cast(axis), + values, + flags); + break; + + case axis_type::float64: + this->update_float64_axis(static_cast(axis), + values, + flags); + + break; + + case axis_type::string: + this->update_string_axis(static_cast(axis), + values, + flags); + + break; + + default: + // Internal error, this should never happen: + throw std::runtime_error( + fmt::format("{}:{}: Internal error: invalid axis type enum '{}'", + __FILE__, + __LINE__, + static_cast(axis.get_type()))); + } + } + catch (std::runtime_error &err) + { + throw std::runtime_error(fmt::format("{}:{}: Error parsing `--axis` " + "specification `{}`.\n{}", + __FILE__, + __LINE__, + spec, + err.what())); + } +} + +void option_parser::update_int64_axis(int64_axis &axis, + std::string_view value_spec, + std::string_view flag_spec) +{ + // Validate flags: + int64_axis_flags flags; + if (flag_spec.empty()) + { + flags = int64_axis_flags::none; + } + else if (flag_spec == "pow2") + { + flags = int64_axis_flags::power_of_two; + } + else + { + throw std::runtime_error(fmt::format("{}:{}: Invalid flag for int64 axis: " + "`{}`", + __FILE__, + __LINE__, + flag_spec)); + } + + auto input_values = parse_values(value_spec); + + axis.set_inputs(std::move(input_values), flags); +} + +void option_parser::update_float64_axis(float64_axis &axis, + std::string_view value_spec, + std::string_view flag_spec) +{ + // Validate flags: + if (!flag_spec.empty()) + { + throw std::runtime_error(fmt::format("{}:{}: Invalid flag for float64 " + "axis: `{}`", + __FILE__, + __LINE__, + flag_spec)); + } + + auto input_values = parse_values(value_spec); + + axis.set_inputs(std::move(input_values)); +} + +void option_parser::update_string_axis(string_axis &axis, + std::string_view value_spec, + std::string_view flag_spec) +{ + // Validate flags: + if (!flag_spec.empty()) + { + throw std::runtime_error(fmt::format("{}:{}: Invalid flag for string " + "axis: `{}`", + __FILE__, + __LINE__, + flag_spec)); + } + + auto input_values = parse_values(value_spec); + + axis.set_inputs(std::move(input_values)); +} + +void option_parser::update_type_axis(type_axis &axis, + std::string_view value_spec, + std::string_view flag_spec) +{ + // Validate flags: + if (!flag_spec.empty()) + { + throw std::runtime_error(fmt::format("{}:{}: Invalid flag for type axis: " + "`{}`", + __FILE__, + __LINE__, + flag_spec)); + } + + auto input_values = parse_values(value_spec); + + axis.set_active_inputs(input_values); +} + +} // namespace nvbench diff --git a/nvbench/option_parser.cuh b/nvbench/option_parser.cuh new file mode 100644 index 0000000..e94322e --- /dev/null +++ b/nvbench/option_parser.cuh @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include +#include +#include + +namespace nvbench +{ + +/** + * Parses command-line args into a set of benchmarks. + */ +struct option_parser +{ + using benchmark_vector = + std::vector>; + + void parse(int argc, char const *const argv[]); + void parse(std::vector args); + + [[nodiscard]] benchmark_vector &get_benchmarks() { return m_benchmarks; }; + [[nodiscard]] const benchmark_vector &get_benchmarks() const + { + return m_benchmarks; + }; + + [[nodiscard]] const std::vector &get_args() const + { + return m_args; + } + +private: + void parse_impl(); + void add_benchmark(const std::string &name); + void update_axis(const std::string &spec); + static void update_int64_axis(int64_axis &axis, + std::string_view value_spec, + std::string_view flag_spec); + static void update_float64_axis(float64_axis &axis, + std::string_view value_spec, + std::string_view flag_spec); + static void update_string_axis(string_axis &axis, + std::string_view value_spec, + std::string_view flag_spec); + static void update_type_axis(type_axis &axis, + std::string_view value_spec, + std::string_view flag_spec); + + std::vector m_args; + benchmark_vector m_benchmarks; +}; + +} // namespace nvbench diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 07cc100..364b64c 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -7,6 +7,7 @@ set(test_srcs float64_axis.cu int64_axis.cu named_values.cu + option_parser.cu runner.cu state.cu state_generator.cu diff --git a/testing/option_parser.cu b/testing/option_parser.cu new file mode 100644 index 0000000..2102646 --- /dev/null +++ b/testing/option_parser.cu @@ -0,0 +1,798 @@ +#include + +#include +#include + +#include "test_asserts.cuh" + +#include + +//============================================================================== +// Declare a benchmark for testing: +using Ts = nvbench::type_list; +using Us = nvbench::type_list; + +template +void TestBench(nvbench::state &state, nvbench::type_list) +{ + state.skip("Test"); +} +NVBENCH_CREATE_TEMPLATE(TestBench, NVBENCH_TYPE_AXES(Ts, Us)) + .set_type_axes_names({"T", "U"}) + .add_int64_axis("Ints", {42}) + .add_int64_power_of_two_axis("PO2s", {3}) + .add_float64_axis("Floats", {3.14}) + .add_string_axis("Strings", {"S1"}); +//============================================================================== + +namespace +{ + +[[nodiscard]] std::string +states_to_string(const std::vector> &states) +{ + fmt::memory_buffer buffer; + std::string table_format = "| {:^5} | {:^10} | {:^4} | {:^4} | {:^4} " + "| {:^4} | {:^6} | {:^8} |\n"; + + fmt::format_to(buffer, "\n"); + fmt::format_to(buffer, + table_format, + "State", + "TypeConfig", + "T", + "U", + "Ints", + "PO2s", + "Floats", + "Strings"); + + std::size_t type_config = 0; + std::size_t config = 0; + for (const auto &inner_states : states) + { + for (const nvbench::state &state : inner_states) + { + fmt::format_to(buffer, + table_format, + config++, + type_config, + state.get_string("T"), + state.get_string("U"), + state.get_int64("Ints"), + state.get_int64("PO2s"), + state.get_float64("Floats"), + std::string{"\'"} + state.get_string("Strings") + "'"); + } + type_config++; + } + return fmt::to_string(buffer); +} + +// Expects the parser to have a single TestBench benchmark. Runs the benchmark +// and converts the generated states into a fingerprint string for regression +// testing. +[[nodiscard]] std::string parser_to_state_string(nvbench::option_parser &parser) +{ + const auto &benches = parser.get_benchmarks(); + ASSERT(benches.size() == 1); + const auto &bench = benches.front(); + ASSERT(bench != nullptr); + + bench->run(); + return states_to_string(bench->get_states()); +} + +} // namespace + +void test_empty() +{ + { + nvbench::option_parser parser; + parser.parse({}); + ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_args().empty()); + } + + { + nvbench::option_parser parser; + parser.parse(0, nullptr); + ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_args().empty()); + } +} + +void test_exec_name_tolerance() +{ + nvbench::option_parser parser; + parser.parse({"TestExec"}); + ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_args() == std::vector{"TestExec"}); +} + +void test_argc_argv_parse() +{ + char const *const argv[] = {"TestExec"}; + { + nvbench::option_parser parser; + parser.parse(1, argv); + ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_args() == std::vector{"TestExec"}); + } + + { + nvbench::option_parser parser; + parser.parse(0, nullptr); + ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_args().empty()); + } +} + +void test_invalid_option() +{ + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--not-a-real-option"})); +} + +void test_benchmark_long() // --benchmark +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' | +| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' | +| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' | +| 3 | 3 | I8 | bool | 42 | 8 | 3.14 | 'S1' | +| 4 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'S1' | +| 5 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'S1' | +| 6 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' | +| 7 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' | +| 8 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' | +)expected"; + + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); +} + +void test_benchmark_short() // -b +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' | +| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' | +| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' | +| 3 | 3 | I8 | bool | 42 | 8 | 3.14 | 'S1' | +| 4 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'S1' | +| 5 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'S1' | +| 6 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' | +| 7 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' | +| 8 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' | +)expected"; + + nvbench::option_parser parser; + parser.parse({"-b", "TestBench"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); +} + +void test_int64_axis() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 2 | 8 | 3.14 | 'S1' | +| 1 | 0 | void | bool | 7 | 8 | 3.14 | 'S1' | +| 2 | 1 | void | F32 | 2 | 8 | 3.14 | 'S1' | +| 3 | 1 | void | F32 | 7 | 8 | 3.14 | 'S1' | +| 4 | 2 | void | F64 | 2 | 8 | 3.14 | 'S1' | +| 5 | 2 | void | F64 | 7 | 8 | 3.14 | 'S1' | +| 6 | 3 | I8 | bool | 2 | 8 | 3.14 | 'S1' | +| 7 | 3 | I8 | bool | 7 | 8 | 3.14 | 'S1' | +| 8 | 4 | I8 | F32 | 2 | 8 | 3.14 | 'S1' | +| 9 | 4 | I8 | F32 | 7 | 8 | 3.14 | 'S1' | +| 10 | 5 | I8 | F64 | 2 | 8 | 3.14 | 'S1' | +| 11 | 5 | I8 | F64 | 7 | 8 | 3.14 | 'S1' | +| 12 | 6 | U8 | bool | 2 | 8 | 3.14 | 'S1' | +| 13 | 6 | U8 | bool | 7 | 8 | 3.14 | 'S1' | +| 14 | 7 | U8 | F32 | 2 | 8 | 3.14 | 'S1' | +| 15 | 7 | U8 | F32 | 7 | 8 | 3.14 | 'S1' | +| 16 | 8 | U8 | F64 | 2 | 8 | 3.14 | 'S1' | +| 17 | 8 | U8 | F64 | 7 | 8 | 3.14 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Ints [ ] : { 2 , 7 } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Ints:{2,7}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Ints [ ] : ( 2 : 7 : 5 ) "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Ints:(2:7:5)"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_int64_axis_pow2() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 4 | 3.14 | 'S1' | +| 1 | 0 | void | bool | 42 | 128 | 3.14 | 'S1' | +| 2 | 1 | void | F32 | 42 | 4 | 3.14 | 'S1' | +| 3 | 1 | void | F32 | 42 | 128 | 3.14 | 'S1' | +| 4 | 2 | void | F64 | 42 | 4 | 3.14 | 'S1' | +| 5 | 2 | void | F64 | 42 | 128 | 3.14 | 'S1' | +| 6 | 3 | I8 | bool | 42 | 4 | 3.14 | 'S1' | +| 7 | 3 | I8 | bool | 42 | 128 | 3.14 | 'S1' | +| 8 | 4 | I8 | F32 | 42 | 4 | 3.14 | 'S1' | +| 9 | 4 | I8 | F32 | 42 | 128 | 3.14 | 'S1' | +| 10 | 5 | I8 | F64 | 42 | 4 | 3.14 | 'S1' | +| 11 | 5 | I8 | F64 | 42 | 128 | 3.14 | 'S1' | +| 12 | 6 | U8 | bool | 42 | 4 | 3.14 | 'S1' | +| 13 | 6 | U8 | bool | 42 | 128 | 3.14 | 'S1' | +| 14 | 7 | U8 | F32 | 42 | 4 | 3.14 | 'S1' | +| 15 | 7 | U8 | F32 | 42 | 128 | 3.14 | 'S1' | +| 16 | 8 | U8 | F64 | 42 | 4 | 3.14 | 'S1' | +| 17 | 8 | U8 | F64 | 42 | 128 | 3.14 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] : { 2 , 7 } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]:{2,7}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " PO2s [ pow2 ] : ( 2 : 7 : 5 ) "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "PO2s[pow2]:(2:7:5)"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_int64_axis_none_to_pow2() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 4 | 8 | 3.14 | 'S1' | +| 1 | 0 | void | bool | 128 | 8 | 3.14 | 'S1' | +| 2 | 1 | void | F32 | 4 | 8 | 3.14 | 'S1' | +| 3 | 1 | void | F32 | 128 | 8 | 3.14 | 'S1' | +| 4 | 2 | void | F64 | 4 | 8 | 3.14 | 'S1' | +| 5 | 2 | void | F64 | 128 | 8 | 3.14 | 'S1' | +| 6 | 3 | I8 | bool | 4 | 8 | 3.14 | 'S1' | +| 7 | 3 | I8 | bool | 128 | 8 | 3.14 | 'S1' | +| 8 | 4 | I8 | F32 | 4 | 8 | 3.14 | 'S1' | +| 9 | 4 | I8 | F32 | 128 | 8 | 3.14 | 'S1' | +| 10 | 5 | I8 | F64 | 4 | 8 | 3.14 | 'S1' | +| 11 | 5 | I8 | F64 | 128 | 8 | 3.14 | 'S1' | +| 12 | 6 | U8 | bool | 4 | 8 | 3.14 | 'S1' | +| 13 | 6 | U8 | bool | 128 | 8 | 3.14 | 'S1' | +| 14 | 7 | U8 | F32 | 4 | 8 | 3.14 | 'S1' | +| 15 | 7 | U8 | F32 | 128 | 8 | 3.14 | 'S1' | +| 16 | 8 | U8 | F64 | 4 | 8 | 3.14 | 'S1' | +| 17 | 8 | U8 | F64 | 128 | 8 | 3.14 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] : { 2 , 7 } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]:{2,7}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Ints [ pow2 ] : ( 2 : 7 : 5 ) "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Ints[pow2]:(2:7:5)"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_int64_axis_pow2_to_none() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 2 | 3.14 | 'S1' | +| 1 | 0 | void | bool | 42 | 7 | 3.14 | 'S1' | +| 2 | 1 | void | F32 | 42 | 2 | 3.14 | 'S1' | +| 3 | 1 | void | F32 | 42 | 7 | 3.14 | 'S1' | +| 4 | 2 | void | F64 | 42 | 2 | 3.14 | 'S1' | +| 5 | 2 | void | F64 | 42 | 7 | 3.14 | 'S1' | +| 6 | 3 | I8 | bool | 42 | 2 | 3.14 | 'S1' | +| 7 | 3 | I8 | bool | 42 | 7 | 3.14 | 'S1' | +| 8 | 4 | I8 | F32 | 42 | 2 | 3.14 | 'S1' | +| 9 | 4 | I8 | F32 | 42 | 7 | 3.14 | 'S1' | +| 10 | 5 | I8 | F64 | 42 | 2 | 3.14 | 'S1' | +| 11 | 5 | I8 | F64 | 42 | 7 | 3.14 | 'S1' | +| 12 | 6 | U8 | bool | 42 | 2 | 3.14 | 'S1' | +| 13 | 6 | U8 | bool | 42 | 7 | 3.14 | 'S1' | +| 14 | 7 | U8 | F32 | 42 | 2 | 3.14 | 'S1' | +| 15 | 7 | U8 | F32 | 42 | 7 | 3.14 | 'S1' | +| 16 | 8 | U8 | F64 | 42 | 2 | 3.14 | 'S1' | +| 17 | 8 | U8 | F64 | 42 | 7 | 3.14 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " PO2s [ ] : { 2 , 7 } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "PO2s:{2,7}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " PO2s [ ] : ( 2 : 7 : 5 ) "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "PO2s:(2:7:5)"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_float64_axis() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 8 | 3.5 | 'S1' | +| 1 | 0 | void | bool | 42 | 8 | 4.1 | 'S1' | +| 2 | 1 | void | F32 | 42 | 8 | 3.5 | 'S1' | +| 3 | 1 | void | F32 | 42 | 8 | 4.1 | 'S1' | +| 4 | 2 | void | F64 | 42 | 8 | 3.5 | 'S1' | +| 5 | 2 | void | F64 | 42 | 8 | 4.1 | 'S1' | +| 6 | 3 | I8 | bool | 42 | 8 | 3.5 | 'S1' | +| 7 | 3 | I8 | bool | 42 | 8 | 4.1 | 'S1' | +| 8 | 4 | I8 | F32 | 42 | 8 | 3.5 | 'S1' | +| 9 | 4 | I8 | F32 | 42 | 8 | 4.1 | 'S1' | +| 10 | 5 | I8 | F64 | 42 | 8 | 3.5 | 'S1' | +| 11 | 5 | I8 | F64 | 42 | 8 | 4.1 | 'S1' | +| 12 | 6 | U8 | bool | 42 | 8 | 3.5 | 'S1' | +| 13 | 6 | U8 | bool | 42 | 8 | 4.1 | 'S1' | +| 14 | 7 | U8 | F32 | 42 | 8 | 3.5 | 'S1' | +| 15 | 7 | U8 | F32 | 42 | 8 | 4.1 | 'S1' | +| 16 | 8 | U8 | F64 | 42 | 8 | 3.5 | 'S1' | +| 17 | 8 | U8 | F64 | 42 | 8 | 4.1 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Floats [ ] : { 3.5 , 4.1 } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Floats:{3.5,4.1}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", + "TestBench", + "--axis", + " Floats [ ] : ( 3.5 : 4.2 : 0.6 ) "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", "Floats:(3.5:4.2:0.6)"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_string_axis() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'fo br' | +| 1 | 0 | void | bool | 42 | 8 | 3.14 | 'baz' | +| 2 | 1 | void | F32 | 42 | 8 | 3.14 | 'fo br' | +| 3 | 1 | void | F32 | 42 | 8 | 3.14 | 'baz' | +| 4 | 2 | void | F64 | 42 | 8 | 3.14 | 'fo br' | +| 5 | 2 | void | F64 | 42 | 8 | 3.14 | 'baz' | +| 6 | 3 | I8 | bool | 42 | 8 | 3.14 | 'fo br' | +| 7 | 3 | I8 | bool | 42 | 8 | 3.14 | 'baz' | +| 8 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'fo br' | +| 9 | 4 | I8 | F32 | 42 | 8 | 3.14 | 'baz' | +| 10 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'fo br' | +| 11 | 5 | I8 | F64 | 42 | 8 | 3.14 | 'baz' | +| 12 | 6 | U8 | bool | 42 | 8 | 3.14 | 'fo br' | +| 13 | 6 | U8 | bool | 42 | 8 | 3.14 | 'baz' | +| 14 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'fo br' | +| 15 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'baz' | +| 16 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'fo br' | +| 17 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'baz' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " Strings [ ] : { fo br , baz } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "Strings:{fo br,baz}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_type_axis() +{ + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 42 | 8 | 3.14 | 'S1' | +| 1 | 1 | void | F32 | 42 | 8 | 3.14 | 'S1' | +| 2 | 2 | void | F64 | 42 | 8 | 3.14 | 'S1' | +| 3 | 6 | U8 | bool | 42 | 8 | 3.14 | 'S1' | +| 4 | 7 | U8 | F32 | 42 | 8 | 3.14 | 'S1' | +| 5 | 8 | U8 | F64 | 42 | 8 | 3.14 | 'S1' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse( + {"--benchmark", "TestBench", "--axis", " T [ ] : { U8, void } "}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "TestBench", "--axis", "T:{void,U8}"}); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +void test_multi_axis() +{ + + const std::string ref = + R"expected( +| State | TypeConfig | T | U | Ints | PO2s | Floats | Strings | +| 0 | 0 | void | bool | 2 | 4 | 0.25 | 'foo' | +| 1 | 0 | void | bool | 5 | 4 | 0.25 | 'foo' | +| 2 | 0 | void | bool | 2 | 32 | 0.25 | 'foo' | +| 3 | 0 | void | bool | 5 | 32 | 0.25 | 'foo' | +| 4 | 0 | void | bool | 2 | 256 | 0.25 | 'foo' | +| 5 | 0 | void | bool | 5 | 256 | 0.25 | 'foo' | +| 6 | 0 | void | bool | 2 | 4 | 0.5 | 'foo' | +| 7 | 0 | void | bool | 5 | 4 | 0.5 | 'foo' | +| 8 | 0 | void | bool | 2 | 32 | 0.5 | 'foo' | +| 9 | 0 | void | bool | 5 | 32 | 0.5 | 'foo' | +| 10 | 0 | void | bool | 2 | 256 | 0.5 | 'foo' | +| 11 | 0 | void | bool | 5 | 256 | 0.5 | 'foo' | +| 12 | 0 | void | bool | 2 | 4 | 0.75 | 'foo' | +| 13 | 0 | void | bool | 5 | 4 | 0.75 | 'foo' | +| 14 | 0 | void | bool | 2 | 32 | 0.75 | 'foo' | +| 15 | 0 | void | bool | 5 | 32 | 0.75 | 'foo' | +| 16 | 0 | void | bool | 2 | 256 | 0.75 | 'foo' | +| 17 | 0 | void | bool | 5 | 256 | 0.75 | 'foo' | +| 18 | 0 | void | bool | 2 | 4 | 1 | 'foo' | +| 19 | 0 | void | bool | 5 | 4 | 1 | 'foo' | +| 20 | 0 | void | bool | 2 | 32 | 1 | 'foo' | +| 21 | 0 | void | bool | 5 | 32 | 1 | 'foo' | +| 22 | 0 | void | bool | 2 | 256 | 1 | 'foo' | +| 23 | 0 | void | bool | 5 | 256 | 1 | 'foo' | +| 24 | 0 | void | bool | 2 | 4 | 0.25 | 'bar' | +| 25 | 0 | void | bool | 5 | 4 | 0.25 | 'bar' | +| 26 | 0 | void | bool | 2 | 32 | 0.25 | 'bar' | +| 27 | 0 | void | bool | 5 | 32 | 0.25 | 'bar' | +| 28 | 0 | void | bool | 2 | 256 | 0.25 | 'bar' | +| 29 | 0 | void | bool | 5 | 256 | 0.25 | 'bar' | +| 30 | 0 | void | bool | 2 | 4 | 0.5 | 'bar' | +| 31 | 0 | void | bool | 5 | 4 | 0.5 | 'bar' | +| 32 | 0 | void | bool | 2 | 32 | 0.5 | 'bar' | +| 33 | 0 | void | bool | 5 | 32 | 0.5 | 'bar' | +| 34 | 0 | void | bool | 2 | 256 | 0.5 | 'bar' | +| 35 | 0 | void | bool | 5 | 256 | 0.5 | 'bar' | +| 36 | 0 | void | bool | 2 | 4 | 0.75 | 'bar' | +| 37 | 0 | void | bool | 5 | 4 | 0.75 | 'bar' | +| 38 | 0 | void | bool | 2 | 32 | 0.75 | 'bar' | +| 39 | 0 | void | bool | 5 | 32 | 0.75 | 'bar' | +| 40 | 0 | void | bool | 2 | 256 | 0.75 | 'bar' | +| 41 | 0 | void | bool | 5 | 256 | 0.75 | 'bar' | +| 42 | 0 | void | bool | 2 | 4 | 1 | 'bar' | +| 43 | 0 | void | bool | 5 | 4 | 1 | 'bar' | +| 44 | 0 | void | bool | 2 | 32 | 1 | 'bar' | +| 45 | 0 | void | bool | 5 | 32 | 1 | 'bar' | +| 46 | 0 | void | bool | 2 | 256 | 1 | 'bar' | +| 47 | 0 | void | bool | 5 | 256 | 1 | 'bar' | +| 48 | 0 | void | bool | 2 | 4 | 0.25 | 'baz' | +| 49 | 0 | void | bool | 5 | 4 | 0.25 | 'baz' | +| 50 | 0 | void | bool | 2 | 32 | 0.25 | 'baz' | +| 51 | 0 | void | bool | 5 | 32 | 0.25 | 'baz' | +| 52 | 0 | void | bool | 2 | 256 | 0.25 | 'baz' | +| 53 | 0 | void | bool | 5 | 256 | 0.25 | 'baz' | +| 54 | 0 | void | bool | 2 | 4 | 0.5 | 'baz' | +| 55 | 0 | void | bool | 5 | 4 | 0.5 | 'baz' | +| 56 | 0 | void | bool | 2 | 32 | 0.5 | 'baz' | +| 57 | 0 | void | bool | 5 | 32 | 0.5 | 'baz' | +| 58 | 0 | void | bool | 2 | 256 | 0.5 | 'baz' | +| 59 | 0 | void | bool | 5 | 256 | 0.5 | 'baz' | +| 60 | 0 | void | bool | 2 | 4 | 0.75 | 'baz' | +| 61 | 0 | void | bool | 5 | 4 | 0.75 | 'baz' | +| 62 | 0 | void | bool | 2 | 32 | 0.75 | 'baz' | +| 63 | 0 | void | bool | 5 | 32 | 0.75 | 'baz' | +| 64 | 0 | void | bool | 2 | 256 | 0.75 | 'baz' | +| 65 | 0 | void | bool | 5 | 256 | 0.75 | 'baz' | +| 66 | 0 | void | bool | 2 | 4 | 1 | 'baz' | +| 67 | 0 | void | bool | 5 | 4 | 1 | 'baz' | +| 68 | 0 | void | bool | 2 | 32 | 1 | 'baz' | +| 69 | 0 | void | bool | 5 | 32 | 1 | 'baz' | +| 70 | 0 | void | bool | 2 | 256 | 1 | 'baz' | +| 71 | 0 | void | bool | 5 | 256 | 1 | 'baz' | +| 72 | 6 | U8 | bool | 2 | 4 | 0.25 | 'foo' | +| 73 | 6 | U8 | bool | 5 | 4 | 0.25 | 'foo' | +| 74 | 6 | U8 | bool | 2 | 32 | 0.25 | 'foo' | +| 75 | 6 | U8 | bool | 5 | 32 | 0.25 | 'foo' | +| 76 | 6 | U8 | bool | 2 | 256 | 0.25 | 'foo' | +| 77 | 6 | U8 | bool | 5 | 256 | 0.25 | 'foo' | +| 78 | 6 | U8 | bool | 2 | 4 | 0.5 | 'foo' | +| 79 | 6 | U8 | bool | 5 | 4 | 0.5 | 'foo' | +| 80 | 6 | U8 | bool | 2 | 32 | 0.5 | 'foo' | +| 81 | 6 | U8 | bool | 5 | 32 | 0.5 | 'foo' | +| 82 | 6 | U8 | bool | 2 | 256 | 0.5 | 'foo' | +| 83 | 6 | U8 | bool | 5 | 256 | 0.5 | 'foo' | +| 84 | 6 | U8 | bool | 2 | 4 | 0.75 | 'foo' | +| 85 | 6 | U8 | bool | 5 | 4 | 0.75 | 'foo' | +| 86 | 6 | U8 | bool | 2 | 32 | 0.75 | 'foo' | +| 87 | 6 | U8 | bool | 5 | 32 | 0.75 | 'foo' | +| 88 | 6 | U8 | bool | 2 | 256 | 0.75 | 'foo' | +| 89 | 6 | U8 | bool | 5 | 256 | 0.75 | 'foo' | +| 90 | 6 | U8 | bool | 2 | 4 | 1 | 'foo' | +| 91 | 6 | U8 | bool | 5 | 4 | 1 | 'foo' | +| 92 | 6 | U8 | bool | 2 | 32 | 1 | 'foo' | +| 93 | 6 | U8 | bool | 5 | 32 | 1 | 'foo' | +| 94 | 6 | U8 | bool | 2 | 256 | 1 | 'foo' | +| 95 | 6 | U8 | bool | 5 | 256 | 1 | 'foo' | +| 96 | 6 | U8 | bool | 2 | 4 | 0.25 | 'bar' | +| 97 | 6 | U8 | bool | 5 | 4 | 0.25 | 'bar' | +| 98 | 6 | U8 | bool | 2 | 32 | 0.25 | 'bar' | +| 99 | 6 | U8 | bool | 5 | 32 | 0.25 | 'bar' | +| 100 | 6 | U8 | bool | 2 | 256 | 0.25 | 'bar' | +| 101 | 6 | U8 | bool | 5 | 256 | 0.25 | 'bar' | +| 102 | 6 | U8 | bool | 2 | 4 | 0.5 | 'bar' | +| 103 | 6 | U8 | bool | 5 | 4 | 0.5 | 'bar' | +| 104 | 6 | U8 | bool | 2 | 32 | 0.5 | 'bar' | +| 105 | 6 | U8 | bool | 5 | 32 | 0.5 | 'bar' | +| 106 | 6 | U8 | bool | 2 | 256 | 0.5 | 'bar' | +| 107 | 6 | U8 | bool | 5 | 256 | 0.5 | 'bar' | +| 108 | 6 | U8 | bool | 2 | 4 | 0.75 | 'bar' | +| 109 | 6 | U8 | bool | 5 | 4 | 0.75 | 'bar' | +| 110 | 6 | U8 | bool | 2 | 32 | 0.75 | 'bar' | +| 111 | 6 | U8 | bool | 5 | 32 | 0.75 | 'bar' | +| 112 | 6 | U8 | bool | 2 | 256 | 0.75 | 'bar' | +| 113 | 6 | U8 | bool | 5 | 256 | 0.75 | 'bar' | +| 114 | 6 | U8 | bool | 2 | 4 | 1 | 'bar' | +| 115 | 6 | U8 | bool | 5 | 4 | 1 | 'bar' | +| 116 | 6 | U8 | bool | 2 | 32 | 1 | 'bar' | +| 117 | 6 | U8 | bool | 5 | 32 | 1 | 'bar' | +| 118 | 6 | U8 | bool | 2 | 256 | 1 | 'bar' | +| 119 | 6 | U8 | bool | 5 | 256 | 1 | 'bar' | +| 120 | 6 | U8 | bool | 2 | 4 | 0.25 | 'baz' | +| 121 | 6 | U8 | bool | 5 | 4 | 0.25 | 'baz' | +| 122 | 6 | U8 | bool | 2 | 32 | 0.25 | 'baz' | +| 123 | 6 | U8 | bool | 5 | 32 | 0.25 | 'baz' | +| 124 | 6 | U8 | bool | 2 | 256 | 0.25 | 'baz' | +| 125 | 6 | U8 | bool | 5 | 256 | 0.25 | 'baz' | +| 126 | 6 | U8 | bool | 2 | 4 | 0.5 | 'baz' | +| 127 | 6 | U8 | bool | 5 | 4 | 0.5 | 'baz' | +| 128 | 6 | U8 | bool | 2 | 32 | 0.5 | 'baz' | +| 129 | 6 | U8 | bool | 5 | 32 | 0.5 | 'baz' | +| 130 | 6 | U8 | bool | 2 | 256 | 0.5 | 'baz' | +| 131 | 6 | U8 | bool | 5 | 256 | 0.5 | 'baz' | +| 132 | 6 | U8 | bool | 2 | 4 | 0.75 | 'baz' | +| 133 | 6 | U8 | bool | 5 | 4 | 0.75 | 'baz' | +| 134 | 6 | U8 | bool | 2 | 32 | 0.75 | 'baz' | +| 135 | 6 | U8 | bool | 5 | 32 | 0.75 | 'baz' | +| 136 | 6 | U8 | bool | 2 | 256 | 0.75 | 'baz' | +| 137 | 6 | U8 | bool | 5 | 256 | 0.75 | 'baz' | +| 138 | 6 | U8 | bool | 2 | 4 | 1 | 'baz' | +| 139 | 6 | U8 | bool | 5 | 4 | 1 | 'baz' | +| 140 | 6 | U8 | bool | 2 | 32 | 1 | 'baz' | +| 141 | 6 | U8 | bool | 5 | 32 | 1 | 'baz' | +| 142 | 6 | U8 | bool | 2 | 256 | 1 | 'baz' | +| 143 | 6 | U8 | bool | 5 | 256 | 1 | 'baz' | +)expected"; + + { + nvbench::option_parser parser; + parser.parse({ + // clang-format off + "--benchmark", "TestBench", + "--axis", "T:{U8,void}", + "--axis", "U:{bool}", + "--axis", "Ints:(2:6:3)", + "--axis", "PO2s[pow2]:(2:10:3)", + "--axis", "Floats:(0.25:1:0.25)", + "--axis", "Strings:{foo,bar,baz}", + // clang-format on + }); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } + + { + nvbench::option_parser parser; + parser.parse({ + // clang-format off + "-b", "TestBench", + "-a", "Strings:{foo,bar,baz}", + "-a", "U:{bool}", + "-a", "Floats:(0.25:1:0.25)", + "-a", "Ints:(2:6:3)", + "-a", "PO2s[pow2]:(2:10:3)", + "-a", "T:{U8,void}", + // clang-format on + }); + const auto test = parser_to_state_string(parser); + ASSERT_MSG(test == ref, + fmt::format("Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test)); + } +} + +// `--axis` affects the last `--benchmark`. An exception is thrown if there is +// no benchmark specified for an axis. +void test_axis_before_benchmark() +{ + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--axis", "--benchmark"})); + } + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--axis", "-b"})); + } + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"-a", "--benchmark"})); + } + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"-a", "-b"})); + } +} + +int main() +{ + try + { + test_empty(); + test_exec_name_tolerance(); + test_argc_argv_parse(); + test_invalid_option(); + test_benchmark_long(); + test_benchmark_short(); + test_int64_axis(); + test_int64_axis_pow2(); + test_int64_axis_none_to_pow2(); + test_int64_axis_pow2_to_none(); + test_float64_axis(); + test_string_axis(); + test_type_axis(); + test_multi_axis(); + test_axis_before_benchmark(); + } + catch (std::exception &err) + { + fmt::print(stderr, "{}", err.what()); + return 1; + } + + return 0; +}