Add an nvbench-ctl executable.

This will provide functionality such as clock locking (--lgm),
persistance mode (--pm), device querying (--list), version checking
(--version), and documentation (--help).

This is possible already with any nvbench executable, but having
one with a reliable name will be helpful for scripting and writing
documentation.

CMakeLists.txt

@@ -51,6 +51,8 @@ if (NVBench_ENABLE_EXAMPLES OR NVBench_ENABLE_TESTING)
   enable_testing()
 endif()
 
+add_subdirectory(exec)
+
 if (NVBench_ENABLE_EXAMPLES)
   add_subdirectory(examples)
 endif()

cmake/NVBenchExports.cmake

@@ -23,14 +23,14 @@ macro(nvbench_generate_exports)
   rapids_export(BUILD NVBench
     EXPORT_SET nvbench-targets
     NAMESPACE "nvbench::"
-    GLOBAL_TARGETS nvbench main internal_build_interface
+    GLOBAL_TARGETS nvbench main ctl internal_build_interface
     LANGUAGES CUDA CXX
     FINAL_CODE_BLOCK nvbench_build_export_code_block
   )
   rapids_export(INSTALL NVBench
     EXPORT_SET nvbench-targets
     NAMESPACE "nvbench::"
-    GLOBAL_TARGETS nvbench main internal_build_interface
+    GLOBAL_TARGETS nvbench main ctl internal_build_interface
     LANGUAGES CUDA CXX
     FINAL_CODE_BLOCK nvbench_install_export_code_block
   )

cmake/NVBenchInstallRules.cmake

@@ -54,3 +54,8 @@ function(nvbench_install_libraries)
     EXPORT nvbench-targets
   )
 endfunction()
+
+# Call with a list of executables to generate install rules:
+function(nvbench_install_executables)
+  install(TARGETS ${ARGN} EXPORT nvbench-targets)
+endfunction()

exec/CMakeLists.txt

@@ -0,0 +1,44 @@
+add_executable(nvbench.ctl nvbench-ctl.cu)
+nvbench_config_target(nvbench.ctl)
+target_link_libraries(nvbench.ctl PRIVATE nvbench)
+set_target_properties(nvbench.ctl PROPERTIES
+  OUTPUT_NAME nvbench-ctl
+  EXPORT_NAME ctl
+)
+add_dependencies(nvbench.all nvbench.ctl)
+nvbench_setup_dep_dlls(nvbench.ctl)
+nvbench_install_executables(nvbench.ctl)
+
+if (NVBench_ENABLE_TESTING)
+  # Test: nvbench
+  add_test(NAME nvbench.ctl.no_args COMMAND "$<TARGET_FILE:nvbench.ctl>")
+  # Should print the version without any args:
+  set_property(TEST nvbench.ctl.no_args
+    PROPERTY PASS_REGULAR_EXPRESSION "NVBench v"
+  )
+
+  # Test: nvbench --version
+  add_test(NAME nvbench.ctl.version COMMAND "$<TARGET_FILE:nvbench.ctl>" --version)
+  # Should print the version without any args:
+  set_property(TEST nvbench.ctl.version
+    PROPERTY PASS_REGULAR_EXPRESSION "NVBench v"
+  )
+
+  # Test: nvbench --list
+  add_test(NAME nvbench.ctl.list COMMAND "$<TARGET_FILE:nvbench.ctl>" --list)
+
+  # Test: nvbench -l
+  add_test(NAME nvbench.ctl.l COMMAND "$<TARGET_FILE:nvbench.ctl>" -l)
+
+  # Test: nvbench --help
+  add_test(NAME nvbench.ctl.help COMMAND "$<TARGET_FILE:nvbench.ctl>" --help)
+
+  # Test: nvbench -h
+  add_test(NAME nvbench.ctl.h COMMAND "$<TARGET_FILE:nvbench.ctl>" -h)
+
+  # Test: nvbench --help-axes
+  add_test(NAME nvbench.ctl.help_axes COMMAND "$<TARGET_FILE:nvbench.ctl>" --help-axes)
+
+  # Test: nvbench --help-axis
+  add_test(NAME nvbench.ctl.help_axis COMMAND "$<TARGET_FILE:nvbench.ctl>" --help-axis)
+endif()

exec/nvbench-ctl.cu

@@ -0,0 +1,48 @@
+/*
+*  Copyright 2021 NVIDIA Corporation
+*
+*  Licensed under the Apache License, Version 2.0 with the LLVM exception
+*  (the "License"); you may not use this file except in compliance with
+*  the License.
+*
+*  You may obtain a copy of the License at
+*
+*      http://llvm.org/foundation/relicensing/LICENSE.txt
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*/
+
+#include <nvbench/main.cuh>
+
+#include <vector>
+
+int main(int argc, char const *const *argv)
+try
+{
+  // If no args, substitute a new argv that prints the version
+  std::vector<const char*> alt_argv;
+  if (argc == 1)
+  {
+    alt_argv.push_back("--version");
+    alt_argv.push_back(nullptr);
+    argv = alt_argv.data();
+  }
+
+  NVBENCH_MAIN_PARSE(argc, argv);
+  NVBENCH_CUDA_CALL(cudaDeviceReset());
+  return 0;
+}
+catch (std::exception & e)
+{
+  std::cerr << "\nNVBench encountered an error:\n\n" << e.what() << "\n";
+  return 1;
+}
+catch (...)
+{
+  std::cerr << "\nNVBench encountered an unknown error.\n";
+  return 1;
+}

nvbench/main.cuh

@@ -47,17 +47,22 @@
   }
 
 #ifdef NVBENCH_HAS_CUPTI
-#define NVBENCH_INITIALIZE_DRIVER_API NVBENCH_DRIVER_API_CALL(cuInit(0));
+#define NVBENCH_INITIALIZE_DRIVER_API NVBENCH_DRIVER_API_CALL(cuInit(0))
 #else
-#define NVBENCH_INITIALIZE_DRIVER_API
+// clang-format off
+#define NVBENCH_INITIALIZE_DRIVER_API do {} while (false)
+// clang-format on
 #endif
 
+#define NVBENCH_MAIN_PARSE(argc, argv)                                         \
+  nvbench::option_parser parser;                                               \
+  parser.parse(argc, argv)
+
 #define NVBENCH_MAIN_BODY(argc, argv)                                          \
   do                                                                           \
   {                                                                            \
-    NVBENCH_INITIALIZE_DRIVER_API                                              \
-    nvbench::option_parser parser;                                             \
-    parser.parse(argc, argv);                                                  \
+    NVBENCH_INITIALIZE_DRIVER_API;                                             \
+    NVBENCH_MAIN_PARSE(argc, argv);                                            \
     auto &printer = parser.get_printer();                                      \
                                                                                \
     printer.print_device_info();                                               \

nvbench/markdown_printer.cu

@@ -163,6 +163,11 @@ void markdown_printer::do_log_run_state(const nvbench::state &exec_state)
 void markdown_printer::do_print_benchmark_list(
   const printer_base::benchmark_vector &benches)
 {
+  if (benches.empty())
+  {
+    return;
+  }
+
   fmt::memory_buffer buffer;
   fmt::format_to(buffer, "# Benchmarks\n\n");
   std::size_t benchmark_id{0};

testing/cmake/CMakeLists.txt

@@ -47,6 +47,7 @@ nvbench_add_compile_test(test_name
   test_export
   build_tree
   -D "NVBench_DIR=${NVBench_BINARY_DIR}"
+  -D TEST_TYPE=BUILD_TREE
 )
 
 ################################################################################
@@ -57,6 +58,7 @@ nvbench_add_compile_test(test_name
   install_tree
   # "rapids_export() always installs to lib" per rapids_export docs
   -D "NVBench_DIR=${tmp_install_prefix}/lib/cmake/nvbench/"
+  -D TEST_TYPE=INSTALL_TREE
 )
 set_tests_properties(${test_name} PROPERTIES FIXTURES_REQUIRED install_tree)
 

testing/cmake/test_export/CMakeLists.txt

@@ -8,11 +8,47 @@ add_executable(test_bench test_bench.cu)
 target_link_libraries(test_bench PRIVATE nvbench::main)
 enable_testing()
 add_test(NAME test_bench COMMAND "$<TARGET_FILE:test_bench>" --timeout 1)
+add_test(NAME nvbench_ctl COMMAND "$<TARGET_FILE:nvbench::ctl>")
+
+# Setup LD_LIBRARY_PATH for testing
+if (UNIX)
+  set(ctl_lib_path "")
+  set(cupti_lib_path "")
+
+  # Need to find installed libnvbench.so for installed nvbench-ctl.
+  # Not needed for build_tree test because of RUNPATH.
+  if (TEST_TYPE STREQUAL "INSTALL_TREE")
+    get_property(nvbench_config TARGET nvbench::nvbench
+      PROPERTY IMPORTED_CONFIGURATIONS
+    )
+
+    list(LENGTH nvbench_config num_configs)
+    if (num_configs GREATER 1)
+      message(WARNING
+        "Multiple IMPORTED_CONFIGURATIONS for nvbench::nvbench. "
+        "Picking the first one. This may cause issues."
+      )
+      list(GET nvbench_config 0 nvbench_config)
+    endif()
+
+    get_property(ctl_lib_path TARGET nvbench::nvbench
+      PROPERTY IMPORTED_LOCATION_${nvbench_config}
+    )
+    cmake_path(GET ctl_lib_path PARENT_PATH ctl_lib_path)
+  endif()
+
+  # Need to add the CUPTI path to LD_LIBRARY_PATH to make sure CUPTI libraries
+  # are found at runtime:
+  if (TARGET nvbench::cupti)
+    get_property(cupti_lib_path TARGET nvbench::cupti PROPERTY IMPORTED_LOCATION)
+    cmake_path(GET cupti_lib_path PARENT_PATH cupti_lib_path)
+  endif()
+
+  set_property(TEST test_bench PROPERTY
+    ENVIRONMENT "LD_LIBRARY_PATH=${cupti_lib_path}"
+  )
+  set_property(TEST nvbench_ctl PROPERTY
+    ENVIRONMENT "LD_LIBRARY_PATH=${ctl_lib_path}:${cupti_lib_path}"
+  )
 
-# Need to add the CUPTI path to LD_LIBRARY_PATH to make sure CUPTI libraries
-# are found at runtime:
-if (UNIX AND TARGET nvbench::cupti)
-  get_property(cupti_lib_path TARGET nvbench::cupti PROPERTY IMPORTED_LOCATION)
-  cmake_path(GET cupti_lib_path PARENT_PATH cupti_lib_path)
-  set_property(TEST test_bench PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${cupti_lib_path}")
 endif()