NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-31 05:29:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

515.76

Browse Source
This commit is contained in:
Andy Ritger
2022-09-20 13:54:59 -07:00
parent 9855350159
commit fe0728787f
50 changed files with 822 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
kernel-open/nvidia-uvm/uvm_channel.c
View File

@@ -35,10 +35,6 @@
#include "nv_uvm_interface.h"
#include "clb06f.h"
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT 1024
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MIN 32
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (1024 * 1024)
static unsigned uvm_channel_num_gpfifo_entries = UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT;
#define UVM_CHANNEL_GPFIFO_LOC_DEFAULT "auto"
@@ -86,6 +82,12 @@ static NvU32 uvm_channel_update_progress_with_max(uvm_channel_t *channel,
uvm_spin_lock(&channel->pool->lock);
// Completed value should never exceed the queued value
UVM_ASSERT_MSG_RELEASE(completed_value <= channel->tracking_sem.queued_value,
"GPU %s channel %s unexpected completed_value 0x%llx > queued_value 0x%llx\n",
channel->pool->manager->gpu->parent->name, channel->name, completed_value,
channel->tracking_sem.queued_value);
cpu_put = channel->cpu_put;
gpu_get = channel->gpu_get;
@@ -395,6 +397,14 @@ static void uvm_channel_semaphore_release(uvm_push_t *push, NvU64 semaphore_va,
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
// We used to skip the membar or use membar GPU for the semaphore release
// for a few pushes, but that doesn't provide sufficient ordering guarantees
// in some cases (e.g. ga100 with an LCE with PCEs from both HSHUBs) for the
// semaphore writes. To be safe, just always uses a membar sys for now.
// TODO bug 3770539: Optimize membars used by end of push semaphore releases
(void)uvm_push_get_and_reset_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_GPU);
(void)uvm_push_get_and_reset_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_NONE);
if (uvm_channel_is_ce(push->channel))
gpu->parent->ce_hal->semaphore_release(push, semaphore_va, new_payload);
@@ -1562,6 +1572,7 @@ static void uvm_channel_print_info(uvm_channel_t *channel, struct seq_file *s)
UVM_SEQ_OR_DBG_PRINT(s, "get %u\n", channel->gpu_get);
UVM_SEQ_OR_DBG_PRINT(s, "put %u\n", channel->cpu_put);
UVM_SEQ_OR_DBG_PRINT(s, "Semaphore GPU VA 0x%llx\n", uvm_channel_tracking_semaphore_get_gpu_va(channel));
UVM_SEQ_OR_DBG_PRINT(s, "Semaphore CPU VA 0x%llx\n", (NvU64)(uintptr_t)channel->tracking_sem.semaphore.payload);
uvm_spin_unlock(&channel->pool->lock);
}

15
kernel-open/nvidia-uvm/uvm_channel.h
View File

@@ -46,6 +46,21 @@
// wait for a GPFIFO entry to free up.
//
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT 1024
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MIN 32
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (1024 * 1024)
// Semaphore payloads cannot advance too much between calls to
// uvm_gpu_tracking_semaphore_update_completed_value(). In practice the jumps
// are bound by gpfifo sizing as we have to update the completed value to
// reclaim gpfifo entries. Set a limit based on the max gpfifo entries we could
// ever see.
//
// Logically this define belongs to uvm_gpu_semaphore.h but it depends on the
// channel GPFIFO sizing defined here so it's easiest to just have it here as
// uvm_channel.h includes uvm_gpu_semaphore.h.
#define UVM_GPU_SEMAPHORE_MAX_JUMP (2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX)
// Channel types
typedef enum
{

39
kernel-open/nvidia-uvm/uvm_channel_test.c
View File

@@ -151,6 +151,37 @@ done:
return status;
}
static NV_STATUS test_unexpected_completed_values(uvm_va_space_t *va_space)
{
NV_STATUS status;
uvm_gpu_t *gpu;
for_each_va_space_gpu(gpu, va_space) {
uvm_channel_t *channel;
NvU64 completed_value;
// The GPU channel manager is destroyed and then re-created after
// the test, so this test requires exclusive access to the GPU.
TEST_CHECK_RET(uvm_gpu_retained_count(gpu) == 1);
channel = &gpu->channel_manager->channel_pools[0].channels[0];
completed_value = uvm_channel_update_completed_value(channel);
uvm_gpu_semaphore_set_payload(&channel->tracking_sem.semaphore, (NvU32)completed_value + 1);
TEST_CHECK_RET(uvm_global_get_status() == NV_OK);
uvm_channel_update_progress_all(channel);
TEST_CHECK_RET(uvm_global_reset_fatal_error() == NV_ERR_INVALID_STATE);
uvm_channel_manager_destroy(gpu->channel_manager);
// Destruction will hit the error again, so clear one more time.
uvm_global_reset_fatal_error();
TEST_NV_CHECK_RET(uvm_channel_manager_create(gpu, &gpu->channel_manager));
}
return NV_OK;
}
static NV_STATUS uvm_test_rc_for_gpu(uvm_gpu_t *gpu)
{
uvm_push_t push;
@@ -712,6 +743,14 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct
g_uvm_global.disable_fatal_error_assert = true;
uvm_release_asserts_set_global_error_for_tests = true;
status = test_unexpected_completed_values(va_space);
uvm_release_asserts_set_global_error_for_tests = false;
g_uvm_global.disable_fatal_error_assert = false;
if (status != NV_OK)
goto done;
if (g_uvm_global.num_simulated_devices == 0) {
status = test_rc(va_space);
if (status != NV_OK)

27
kernel-open/nvidia-uvm/uvm_common.c
View File

@@ -48,6 +48,33 @@ module_param(uvm_enable_builtin_tests, int, S_IRUGO);
MODULE_PARM_DESC(uvm_enable_builtin_tests,
"Enable the UVM built-in tests. (This is a security risk)");
// Default to release asserts being enabled.
int uvm_release_asserts __read_mostly = 1;
// Make the module param writable so that release asserts can be enabled or
// disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts, "Enable uvm asserts included in release builds.");
// Default to failed release asserts not dumping stack.
int uvm_release_asserts_dump_stack __read_mostly = 0;
// Make the module param writable so that dumping the stack can be enabled and
// disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts_dump_stack, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts_dump_stack, "dump_stack() on failed UVM release asserts.");
// Default to failed release asserts not setting the global UVM error.
int uvm_release_asserts_set_global_error __read_mostly = 0;
// Make the module param writable so that setting the global fatal error can be
// enabled and disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts_set_global_error, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts_set_global_error, "Set UVM global fatal error on failed release asserts.");
// A separate flag to enable setting global error, to be used by tests only.
bool uvm_release_asserts_set_global_error_for_tests __read_mostly = false;
//
// Convert kernel errno codes to corresponding NV_STATUS
//

43
kernel-open/nvidia-uvm/uvm_common.h
View File

@@ -80,6 +80,9 @@ bool uvm_debug_prints_enabled(void);
#define UVM_ASSERT_PRINT(fmt, ...) \
UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_PRINT_RL(fmt, ...) \
UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
#define UVM_ERR_PRINT(fmt, ...) \
UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
@@ -146,9 +149,7 @@ void on_uvm_test_fail(void);
// Unlike on_uvm_test_fail it provides 'panic' coverity semantics
void on_uvm_assert(void);
// UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always enabled, even on
// release builds.
#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \
#define _UVM_ASSERT_MSG(expr, cond, fmt, ...) \
do { \
if (unlikely(!(expr))) { \
UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
@@ -157,9 +158,6 @@ void on_uvm_assert(void);
} \
} while (0)
#define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n")
// Prevent function calls in expr and the print argument list from being
// evaluated.
#define UVM_ASSERT_MSG_IGNORE(expr, fmt, ...) \
@@ -170,13 +168,42 @@ void on_uvm_assert(void);
// UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds
#if UVM_IS_DEBUG() || defined __COVERITY__
#define UVM_ASSERT_MSG UVM_ASSERT_MSG_RELEASE
#define UVM_ASSERT UVM_ASSERT_RELEASE
#define UVM_ASSERT_MSG(expr, fmt, ...) _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT(expr) _UVM_ASSERT_MSG(expr, #expr, "\n")
#else
#define UVM_ASSERT_MSG(expr, fmt, ...) UVM_ASSERT_MSG_IGNORE(expr, fmt, ##__VA_ARGS__)
#define UVM_ASSERT(expr) UVM_ASSERT_MSG_IGNORE(expr, "\n")
#endif
// UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always included in the
// build, even on release builds. They are skipped at runtime if
// uvm_release_asserts is 0.
// Whether release asserts are enabled and whether they should dump the stack
// and set the global error.
extern int uvm_release_asserts;
extern int uvm_release_asserts_dump_stack;
extern int uvm_release_asserts_set_global_error;
extern bool uvm_release_asserts_set_global_error_for_tests;
// Given these are enabled for release builds, we need to be more cautious than
// in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module
// param is enabled.
#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \
do { \
if (uvm_release_asserts && unlikely(!(expr))) { \
UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \
uvm_global_set_fatal_error(NV_ERR_INVALID_STATE); \
if (uvm_release_asserts_dump_stack) \
dump_stack(); \
on_uvm_assert(); \
} \
} while (0)
#define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n")
// Provide a short form of UUID's, typically for use in debug printing:
#define ABBREV_UUID(uuid) (unsigned)(uuid)

10
kernel-open/nvidia-uvm/uvm_gpu_semaphore.c
View File

@@ -25,6 +25,7 @@
#include "uvm_lock.h"
#include "uvm_global.h"
#include "uvm_kvmalloc.h"
#include "uvm_channel.h" // For UVM_GPU_SEMAPHORE_MAX_JUMP
#define UVM_SEMAPHORE_SIZE 4
#define UVM_SEMAPHORE_PAGE_SIZE PAGE_SIZE
@@ -467,9 +468,16 @@ static NvU64 update_completed_value_locked(uvm_gpu_tracking_semaphore_t *trackin
// push, it's easily guaranteed because of the small number of GPFIFO
// entries available per channel (there could be at most as many pending
// pushes as GPFIFO entries).
if (new_sem_value < old_sem_value)
if (unlikely(new_sem_value < old_sem_value))
new_value += 1ULL << 32;
// Check for unexpected large jumps of the semaphore value
UVM_ASSERT_MSG_RELEASE(new_value - old_value <= UVM_GPU_SEMAPHORE_MAX_JUMP,
"GPU %s unexpected semaphore (CPU VA 0x%llx) jump from 0x%llx to 0x%llx\n",
tracking_semaphore->semaphore.page->pool->gpu->parent->name,
(NvU64)(uintptr_t)tracking_semaphore->semaphore.payload,
old_value, new_value);
// Use an atomic write even though the spinlock is held so that the value can
// be (carefully) read atomically outside of the lock.
//

78
kernel-open/nvidia-uvm/uvm_gpu_semaphore_test.c
View File

@@ -27,6 +27,18 @@
#include "uvm_va_space.h"
#include "uvm_kvmalloc.h"
static NV_STATUS set_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 new_value)
{
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem) == new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value - 1));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value + 1));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
return NV_OK;
}
static NV_STATUS add_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU32 increment_by)
{
NvU64 new_value;
@@ -43,13 +55,45 @@ static NV_STATUS add_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU32
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem) == new_value);
TEST_NV_CHECK_RET(set_and_test(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, completed));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value - 1));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value + 1));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
return NV_OK;
}
// Set the current state of the sema, avoiding UVM_GPU_SEMAPHORE_MAX_JUMP
// detection.
static void manual_set(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 value)
{
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)value);
atomic64_set(&tracking_sem->completed_value, value);
tracking_sem->queued_value = value;
}
// Set the starting value and payload and expect a global error
static NV_STATUS set_and_expect_error(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 starting_value, NvU32 payload)
{
manual_set(tracking_sem, starting_value);
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, payload);
TEST_CHECK_RET(uvm_global_get_status() == NV_OK);
uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem);
TEST_CHECK_RET(uvm_global_reset_fatal_error() == NV_ERR_INVALID_STATE);
return NV_OK;
}
static NV_STATUS test_invalid_jumps(uvm_gpu_tracking_semaphore_t *tracking_sem)
{
int i;
for (i = 0; i < 10; ++i) {
NvU64 base = (1ULL<<32) * i;
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base, UVM_GPU_SEMAPHORE_MAX_JUMP + 1));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base, UINT_MAX));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + i + 1, i));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + UINT_MAX / 2, UINT_MAX / 2 + UVM_GPU_SEMAPHORE_MAX_JUMP + 1));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + UINT_MAX / 2, UINT_MAX / 2 - i - 1));
}
return NV_OK;
}
@@ -73,11 +117,31 @@ static NV_STATUS test_tracking(uvm_va_space_t *va_space)
goto done;
for (i = 0; i < 100; ++i) {
status = add_and_test(&tracking_sem, UINT_MAX - 1);
status = add_and_test(&tracking_sem, UVM_GPU_SEMAPHORE_MAX_JUMP - i);
if (status != NV_OK)
goto done;
}
// Test wrap-around cases
for (i = 0; i < 100; ++i) {
// Start with a value right before wrap-around
NvU64 starting_value = (1ULL<<32) * (i + 1) - i - 1;
manual_set(&tracking_sem, starting_value);
// And set payload to after wrap-around
status = set_and_test(&tracking_sem, (1ULL<<32) * (i + 1) + i);
if (status != NV_OK)
goto done;
}
g_uvm_global.disable_fatal_error_assert = true;
uvm_release_asserts_set_global_error_for_tests = true;
status = test_invalid_jumps(&tracking_sem);
uvm_release_asserts_set_global_error_for_tests = false;
g_uvm_global.disable_fatal_error_assert = false;
if (status != NV_OK)
goto done;
done:
uvm_gpu_tracking_semaphore_free(&tracking_sem);
return status;

10
kernel-open/nvidia-uvm/uvm_push.h
View File

@@ -52,11 +52,21 @@ typedef enum
// By default all operations include a membar sys after any transfer and
// before a semaphore operation.
// This flag indicates that next operation should use no membar at all.
//
// For end of push semaphore release, this flag indicates that the push
// itself does not need a membar to be used (membar sys is the default). A
// membar may still be used, if needed to order the semaphore release
// write. See comments in uvm_channel_end_push().
UVM_PUSH_FLAG_NEXT_MEMBAR_NONE,
// By default all operations include a membar sys after any transfer and
// before a semaphore operation.
// This flag indicates that next operation should use a membar gpu instead.
//
// For end of push semaphore release, this flag indicates that the push
// itself only needs a membar gpu (the default is membar sys). A membar sys
// may still be used, if needed to order the semaphore release write. See
// comments in uvm_channel_end_push().
UVM_PUSH_FLAG_NEXT_MEMBAR_GPU,
UVM_PUSH_FLAG_COUNT,