Add NVML support for persistence mode, locking clocks.

Locking clocks is currently only implemented for Volta+ devices.

Example usage:

my_bench -d [0,1,3] --persistence-mode 1 --lock-gpu-clocks base

See the cli_help.md docs for more info.
This commit is contained in:
Allison Vacanti
2021-10-21 01:01:16 -04:00
parent d0c90ff920
commit b948e79cab
18 changed files with 656 additions and 19 deletions

View File

@@ -57,3 +57,10 @@ rapids_find_package(CUDAToolkit REQUIRED
# Append CTK targets to this as we add optional deps (NMVL, CUPTI, ...)
set(ctk_libraries CUDA::toolkit)
################################################################################
# CUDAToolkit -> NVML
if (NVBench_ENABLE_NVML)
include("${CMAKE_CURRENT_LIST_DIR}/NVBenchNVML.cmake")
list(APPEND ctk_libraries nvbench::nvml)
endif()

View File

@@ -0,0 +1,36 @@
# By default, add dependent DLLs to the build dir on MSVC. This avoids
# a variety of runtime issues when using NVML, etc.
# This behavior can be disabled using the following options:
if (WIN32)
option(NVBench_ADD_DEPENDENT_DLLS_TO_BUILD
"Copy dependent dlls to NVBench library build location (MSVC only)."
ON
)
else()
# These are forced off for non-MSVC builds, as $<TARGET_RUNTIME_DLLS:...>
# will always be empty on non-dll platforms.
set(NVBench_ADD_DEPENDENT_DLLS_TO_BUILD OFF)
endif()
if (NVBench_ADD_DEPENDENT_DLLS_TO_BUILD)
message(STATUS
"CMake 3.21.0 is required when NVBench_ADD_DEPENDENT_DLLS_TO_BUILD "
"is enabled."
)
cmake_minimum_required(VERSION 3.21.0)
endif()
function(nvbench_setup_dep_dlls target_name)
# The custom command below fails when there aren't any runtime DLLs to copy,
# so only enable it when a relevant dependency is enabled:
if (NVBench_ADD_DEPENDENT_DLLS_TO_BUILD AND NVBench_ENABLE_NVML)
add_custom_command(TARGET ${target_name}
POST_BUILD
COMMAND
"${CMAKE_COMMAND}" -E copy
"$<TARGET_RUNTIME_DLLS:${target_name}>"
"$<TARGET_FILE_DIR:${target_name}>"
COMMAND_EXPAND_LISTS
)
endif()
endfunction()

View File

@@ -1,14 +1,28 @@
macro(nvbench_generate_exports)
set(nvbench_build_export_code_block "")
set(nvbench_install_export_code_block "")
if (NVBench_ENABLE_NVML)
string(APPEND nvbench_build_export_code_block
"include(\"${NVBench_SOURCE_DIR}/cmake/NVBenchNVML.cmake\")\n"
)
string(APPEND nvbench_install_export_code_block
"include(\"\${CMAKE_CURRENT_LIST_DIR}/NVBenchNVML.cmake\")\n"
)
endif()
rapids_export(BUILD NVBench
EXPORT_SET nvbench-targets
NAMESPACE "nvbench::"
GLOBAL_TARGETS nvbench main
LANGUAGES CUDA CXX
FINAL_CODE_BLOCK nvbench_build_export_code_block
)
rapids_export(INSTALL NVBench
EXPORT_SET nvbench-targets
NAMESPACE "nvbench::"
GLOBAL_TARGETS nvbench main
LANGUAGES CUDA CXX
FINAL_CODE_BLOCK nvbench_install_export_code_block
)
endmacro()

View File

@@ -10,13 +10,35 @@ install(DIRECTORY "${NVBench_SOURCE_DIR}/nvbench"
)
# generated headers from build dir:
install(FILES
"${NVBench_BINARY_DIR}/nvbench/detail/version.cuh"
"${NVBench_BINARY_DIR}/nvbench/detail/git_revision.cuh"
install(
FILES
"${NVBench_BINARY_DIR}/nvbench/config.cuh"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/nvbench"
)
install(
FILES
"${NVBench_BINARY_DIR}/nvbench/detail/version.cuh"
"${NVBench_BINARY_DIR}/nvbench/detail/git_revision.cuh"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/nvbench/detail"
)
#
# Install CMake files needed by consumers to locate dependencies:
#
# Borrowing this logic from rapids_cmake's export logic to make sure these end
# up in the same location as nvbench-config.cmake:
rapids_cmake_install_lib_dir(config_install_location)
set(config_install_location "${config_install_location}/cmake/nvbench")
if (NVBench_ENABLE_NVML)
install(
FILES
"${NVBench_SOURCE_DIR}/cmake/NVBenchNVML.cmake"
DESTINATION "${config_install_location}"
)
endif()
# Call with a list of library targets to generate install rules:
function(nvbench_install_libraries)
install(TARGETS ${ARGN}

37
cmake/NVBenchNVML.cmake Normal file
View File

@@ -0,0 +1,37 @@
# Since this file is installed, we need to make sure that the CUDAToolkit has
# been found by consumers:
if (NOT TARGET CUDA::toolkit)
find_package(CUDAToolkit REQUIRED)
endif()
if (WIN32)
# The CUDA:: targets currently don't provide dll locations through the
# `IMPORTED_LOCATION` property, nor are they marked as `SHARED` libraries
# (they're currently `UNKNOWN`). This prevents the `nvbench_setup_dep_dlls`
# CMake function from copying the dlls to the build / install directories.
# This is discussed in https://gitlab.kitware.com/cmake/cmake/-/issues/22845
# and the other CMake issues it links to.
#
# We create a nvbench-specific target that configures the nvml interface as
# described here:
# https://gitlab.kitware.com/cmake/cmake/-/issues/22845#note_1077538
#
# Use find_file instead of find_library, which would search for a .lib file.
# This is also nice because find_file searches recursively (find_library
# does not) and some versions of CTK nest nvml.dll several directories deep
# under C:\Windows\System32.
find_file(NVBench_NVML_DLL nvml.dll REQUIRED
DOC "The full path to nvml.dll. Usually somewhere under C:/Windows/System32."
PATHS "C:/Windows/System32"
)
mark_as_advanced(NVBench_NVML_DLL)
add_library(nvbench::nvml SHARED IMPORTED)
target_link_libraries(nvbench::nvml INTERFACE CUDA::toolkit)
set_target_properties(nvbench::nvml PROPERTIES
IMPORTED_LOCATION "${NVBench_NVML_DLL}"
IMPORTED_IMPLIB "${CUDA_nvml_LIBRARY}"
)
else()
# Linux is much easier...
add_library(nvbench::nvml ALIAS CUDA::nvml)
endif()

View File

@@ -0,0 +1,7 @@
function(nvbench_write_config_header filepath)
if (NVBench_ENABLE_NVML)
set(NVBENCH_HAS_NVML 1)
endif()
configure_file("${NVBench_SOURCE_DIR}/cmake/config.cuh.in" "${filepath}")
endfunction()

22
cmake/config.cuh.in Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright 2021 NVIDIA Corporation
*
* Licensed under the Apache License, Version 2.0 with the LLVM exception
* (the "License"); you may not use this file except in compliance with
* the License.
*
* You may obtain a copy of the License at
*
* http://llvm.org/foundation/relicensing/LICENSE.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
// Defined if NVBench has been built with NVML support.
#cmakedefine NVBENCH_HAS_NVML