NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-08 00:59:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

535.43.08

Browse Source
This commit is contained in:
russellcnv
2023-08-17 13:03:06 -07:00
parent eb5c7665a1
commit 18b7303c54
378 changed files with 124043 additions and 161949 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
View File

@@ -108,5 +108,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_device_range
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_mm_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_pt_regs_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg

10
kernel-open/nvidia-uvm/uvm_ampere_ce.c
View File

@@ -121,6 +121,8 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
return true;
if (uvm_channel_is_proxy(push->channel)) {
uvm_pushbuffer_t *pushbuffer;
if (dst.is_virtual) {
UVM_ERR_PRINT("Destination address of memcopy must be physical, not virtual\n");
return false;
@@ -142,7 +144,8 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
return false;
}
push_begin_gpu_va = uvm_pushbuffer_get_gpu_va_for_push(push->channel->pool->manager->pushbuffer, push);
pushbuffer = uvm_channel_get_pushbuffer(push->channel);
push_begin_gpu_va = uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
if ((src.address < push_begin_gpu_va) || (src.address >= push_begin_gpu_va + uvm_push_get_size(push))) {
UVM_ERR_PRINT("Source address of memcopy must point to pushbuffer\n");
@@ -177,10 +180,13 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
// irrespective of the virtualization mode.
void uvm_hal_ampere_ce_memcopy_patch_src_c6b5(uvm_push_t *push, uvm_gpu_address_t *src)
{
uvm_pushbuffer_t *pushbuffer;
if (!uvm_channel_is_proxy(push->channel))
return;
src->address -= uvm_pushbuffer_get_gpu_va_for_push(push->channel->pool->manager->pushbuffer, push);
pushbuffer = uvm_channel_get_pushbuffer(push->channel);
src->address -= uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
}
bool uvm_hal_ampere_ce_memset_is_valid_c6b5(uvm_push_t *push,

111
kernel-open/nvidia-uvm/uvm_ats_faults.c
View File

@@ -24,6 +24,7 @@
#include "uvm_va_range.h"
#include "uvm_ats_faults.h"
#include "uvm_migrate_pageable.h"
#include <linux/mempolicy.h>
// TODO: Bug 2103669: Implement a real prefetching policy and remove or adapt
// these experimental parameters. These are intended to help guide that policy.
@@ -79,7 +80,7 @@ static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,
NvU64 start,
size_t length,
uvm_fault_access_type_t access_type,
uvm_fault_client_type_t client_type)
uvm_ats_fault_context_t *ats_context)
{
uvm_va_space_t *va_space = gpu_va_space->va_space;
struct mm_struct *mm = va_space->va_space_mm.mm;
@@ -95,17 +96,18 @@ static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,
// 2) guest physical -> host physical
//
// The overall ATS translation will fault if either of those translations is
// invalid. The get_user_pages() call above handles translation #1, but not
// #2. We don't know if we're running as a guest, but in case we are we can
// force that translation to be valid by touching the guest physical address
// from the CPU. If the translation is not valid then the access will cause
// a hypervisor fault. Note that dma_map_page() can't establish mappings
// used by GPU ATS SVA translations. GPU accesses to host physical addresses
// obtained as a result of the address translation request uses the CPU
// address space instead of the IOMMU address space since the translated
// host physical address isn't necessarily an IOMMU address. The only way to
// establish guest physical to host physical mapping in the CPU address
// space is to touch the page from the CPU.
// invalid. The pin_user_pages() call within uvm_migrate_pageable() call
// below handles translation #1, but not #2. We don't know if we're running
// as a guest, but in case we are we can force that translation to be valid
// by touching the guest physical address from the CPU. If the translation
// is not valid then the access will cause a hypervisor fault. Note that
// dma_map_page() can't establish mappings used by GPU ATS SVA translations.
// GPU accesses to host physical addresses obtained as a result of the
// address translation request uses the CPU address space instead of the
// IOMMU address space since the translated host physical address isn't
// necessarily an IOMMU address. The only way to establish guest physical to
// host physical mapping in the CPU address space is to touch the page from
// the CPU.
//
// We assume that the hypervisor mappings are all VM_PFNMAP, VM_SHARED, and
// VM_WRITE, meaning that the mappings are all granted write access on any
@@ -116,20 +118,26 @@ static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,
uvm_migrate_args_t uvm_migrate_args =
{
.va_space = va_space,
.mm = mm,
.dst_id = gpu_va_space->gpu->parent->id,
.dst_node_id = -1,
.populate_permissions = write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY,
.touch = true,
.skip_mapped = true,
.user_space_start = &user_space_start,
.user_space_length = &user_space_length,
.va_space = va_space,
.mm = mm,
.dst_id = ats_context->residency_id,
.dst_node_id = ats_context->residency_node,
.populate_permissions = write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY,
.touch = true,
.skip_mapped = true,
.populate_on_cpu_alloc_failures = true,
.user_space_start = &user_space_start,
.user_space_length = &user_space_length,
};
UVM_ASSERT(uvm_ats_can_service_faults(gpu_va_space, mm));
expand_fault_region(vma, start, length, client_type, &uvm_migrate_args.start, &uvm_migrate_args.length);
expand_fault_region(vma,
start,
length,
ats_context->client_type,
&uvm_migrate_args.start,
&uvm_migrate_args.length);
// We are trying to use migrate_vma API in the kernel (if it exists) to
// populate and map the faulting region on the GPU. We want to do this only
@@ -165,6 +173,58 @@ static void flush_tlb_write_faults(uvm_gpu_va_space_t *gpu_va_space,
uvm_tlb_batch_invalidate(&ats_invalidate->write_faults_tlb_batch, addr, size, PAGE_SIZE, UVM_MEMBAR_NONE);
}
static void ats_batch_select_residency(uvm_gpu_va_space_t *gpu_va_space,
struct vm_area_struct *vma,
uvm_ats_fault_context_t *ats_context)
{
uvm_gpu_t *gpu = gpu_va_space->gpu;
int residency = uvm_gpu_numa_node(gpu);
#if defined(NV_MEMPOLICY_HAS_UNIFIED_NODES)
struct mempolicy *vma_policy = vma_policy(vma);
unsigned short mode;
if (!vma_policy)
goto done;
mode = vma_policy->mode;
if ((mode == MPOL_BIND) || (mode == MPOL_PREFERRED_MANY) || (mode == MPOL_PREFERRED)) {
int home_node = NUMA_NO_NODE;
#if defined(NV_MEMPOLICY_HAS_HOME_NODE)
if ((mode != MPOL_PREFERRED) && (vma_policy->home_node != NUMA_NO_NODE))
home_node = vma_policy->home_node;
#endif
// Prefer home_node if set. Otherwise, prefer the faulting GPU if it's
// in the list of preferred nodes, else prefer the closest_cpu_numa_node
// to the GPU if closest_cpu_numa_node is in the list of preferred
// nodes. Fallback to the faulting GPU if all else fails.
if (home_node != NUMA_NO_NODE) {
residency = home_node;
}
else if (!node_isset(residency, vma_policy->nodes)) {
int closest_cpu_numa_node = gpu->parent->closest_cpu_numa_node;
if ((closest_cpu_numa_node != NUMA_NO_NODE) && node_isset(closest_cpu_numa_node, vma_policy->nodes))
residency = gpu->parent->closest_cpu_numa_node;
else
residency = first_node(vma_policy->nodes);
}
}
// Update gpu if residency is not the faulting gpu.
if (residency != uvm_gpu_numa_node(gpu))
gpu = uvm_va_space_find_gpu_with_memory_node_id(gpu_va_space->va_space, residency);
done:
#endif
ats_context->residency_id = gpu ? gpu->parent->id : UVM_ID_CPU;
ats_context->residency_node = residency;
}
NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
struct vm_area_struct *vma,
NvU64 base,
@@ -205,6 +265,8 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
uvm_page_mask_zero(write_fault_mask);
}
ats_batch_select_residency(gpu_va_space, vma, ats_context);
for_each_va_block_subregion_in_mask(subregion, write_fault_mask, region) {
NvU64 start = base + (subregion.first * PAGE_SIZE);
size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
@@ -215,7 +277,7 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
UVM_ASSERT(start >= vma->vm_start);
UVM_ASSERT((start + length) <= vma->vm_end);
status = service_ats_faults(gpu_va_space, vma, start, length, access_type, client_type);
status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context);
if (status != NV_OK)
return status;
@@ -244,11 +306,12 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
for_each_va_block_subregion_in_mask(subregion, read_fault_mask, region) {
NvU64 start = base + (subregion.first * PAGE_SIZE);
size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
uvm_fault_access_type_t access_type = UVM_FAULT_ACCESS_TYPE_READ;
UVM_ASSERT(start >= vma->vm_start);
UVM_ASSERT((start + length) <= vma->vm_end);
status = service_ats_faults(gpu_va_space, vma, start, length, UVM_FAULT_ACCESS_TYPE_READ, client_type);
status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context);
if (status != NV_OK)
return status;

58
kernel-open/nvidia-uvm/uvm_ce_test.c
View File

@@ -338,11 +338,6 @@ static NV_STATUS test_memcpy_and_memset_inner(uvm_gpu_t *gpu,
return NV_OK;
}
if (!gpu->parent->ce_hal->memcopy_is_valid(&push, dst, src)) {
TEST_NV_CHECK_RET(uvm_push_end_and_wait(&push));
return NV_OK;
}
// The input virtual addresses exist in UVM's internal address space, not
// the proxy address space
if (uvm_channel_is_proxy(push.channel)) {
@@ -401,7 +396,7 @@ static NV_STATUS test_memcpy_and_memset_inner(uvm_gpu_t *gpu,
static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
{
NV_STATUS status = NV_OK;
bool is_proxy_va_space;
bool is_proxy_va_space = false;
uvm_gpu_address_t gpu_verif_addr;
void *cpu_verif_addr;
uvm_mem_t *verif_mem = NULL;
@@ -437,6 +432,34 @@ static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
}
}
// Virtual address (in UVM's internal address space) backed by sysmem
TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_SYS, size, 0, &sys_rm_mem), done);
gpu_addresses[0] = uvm_rm_mem_get_gpu_va(sys_rm_mem, gpu, is_proxy_va_space);
if (uvm_conf_computing_mode_enabled(gpu)) {
for (i = 0; i < iterations; ++i) {
for (s = 0; s < ARRAY_SIZE(element_sizes); s++) {
TEST_NV_CHECK_GOTO(test_memcpy_and_memset_inner(gpu,
gpu_addresses[0],
gpu_addresses[0],
size,
element_sizes[s],
gpu_verif_addr,
cpu_verif_addr,
i),
done);
}
}
// Because gpu_verif_addr is in sysmem, when the Confidential
// Computing feature is enabled, only the previous cases are valid.
// TODO: Bug 3839176: the test partially waived on Confidential
// Computing because it assumes that GPU can access system memory
// without using encryption.
goto done;
}
// Using a page size equal to the allocation size ensures that the UVM
// memories about to be allocated are physically contiguous. And since the
// size is a valid GPU page size, the memories can be virtually mapped on
@@ -448,37 +471,22 @@ static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
// Physical address in sysmem
TEST_NV_CHECK_GOTO(uvm_mem_alloc(&mem_params, &sys_uvm_mem), done);
TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_phys(sys_uvm_mem, gpu), done);
gpu_addresses[0] = uvm_mem_gpu_address_physical(sys_uvm_mem, gpu, 0, size);
gpu_addresses[1] = uvm_mem_gpu_address_physical(sys_uvm_mem, gpu, 0, size);
// Physical address in vidmem
mem_params.backing_gpu = gpu;
TEST_NV_CHECK_GOTO(uvm_mem_alloc(&mem_params, &gpu_uvm_mem), done);
gpu_addresses[1] = uvm_mem_gpu_address_physical(gpu_uvm_mem, gpu, 0, size);
gpu_addresses[2] = uvm_mem_gpu_address_physical(gpu_uvm_mem, gpu, 0, size);
// Virtual address (in UVM's internal address space) backed by vidmem
TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_GPU, size, 0, &gpu_rm_mem), done);
is_proxy_va_space = false;
gpu_addresses[2] = uvm_rm_mem_get_gpu_va(gpu_rm_mem, gpu, is_proxy_va_space);
gpu_addresses[3] = uvm_rm_mem_get_gpu_va(gpu_rm_mem, gpu, is_proxy_va_space);
// Virtual address (in UVM's internal address space) backed by sysmem
TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_SYS, size, 0, &sys_rm_mem), done);
gpu_addresses[3] = uvm_rm_mem_get_gpu_va(sys_rm_mem, gpu, is_proxy_va_space);
for (i = 0; i < iterations; ++i) {
for (j = 0; j < ARRAY_SIZE(gpu_addresses); ++j) {
for (k = 0; k < ARRAY_SIZE(gpu_addresses); ++k) {
for (s = 0; s < ARRAY_SIZE(element_sizes); s++) {
// Because gpu_verif_addr is in sysmem, when the Confidential
// Computing feature is enabled, only the following cases are
// valid.
//
// TODO: Bug 3839176: the test partially waived on
// Confidential Computing because it assumes that GPU can
// access system memory without using encryption.
if (uvm_conf_computing_mode_enabled(gpu) &&
!(gpu_addresses[k].is_unprotected && gpu_addresses[j].is_unprotected)) {
continue;
}
TEST_NV_CHECK_GOTO(test_memcpy_and_memset_inner(gpu,
gpu_addresses[k],
gpu_addresses[j],
@@ -752,7 +760,7 @@ static NV_STATUS alloc_vidmem_protected(uvm_gpu_t *gpu, uvm_mem_t **mem, size_t
*mem = NULL;
TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem_protected(size, gpu, mem));
TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem(size, gpu, mem));
TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
TEST_NV_CHECK_GOTO(zero_vidmem(*mem), err);

881
kernel-open/nvidia-uvm/uvm_channel.c
View File

File diff suppressed because it is too large Load Diff

81
kernel-open/nvidia-uvm/uvm_channel.h
View File

@@ -104,16 +104,14 @@ typedef enum
// ----------------------------------
// Channel type with fixed schedules
// Work Launch Channel (WLC) is a specialized channel
// for launching work on other channels when
// Confidential Computing is enabled.
// It is paired with LCIC (below)
// Work Launch Channel (WLC) is a specialized channel for launching work on
// other channels when the Confidential Computing is feature enabled. It is
// paired with LCIC (below)
UVM_CHANNEL_TYPE_WLC,
// Launch Confirmation Indicator Channel (LCIC) is a
// specialized channel with fixed schedule. It gets
// triggered by executing WLC work, and makes sure that
// WLC get/put pointers are up-to-date.
// Launch Confirmation Indicator Channel (LCIC) is a specialized channel
// with fixed schedule. It gets triggered by executing WLC work, and makes
// sure that WLC get/put pointers are up-to-date.
UVM_CHANNEL_TYPE_LCIC,
UVM_CHANNEL_TYPE_COUNT,
@@ -242,11 +240,9 @@ typedef struct
DECLARE_BITMAP(push_locks, UVM_CHANNEL_MAX_NUM_CHANNELS_PER_POOL);
// Counting semaphore for available and unlocked channels, it must be
// acquired before submitting work to a secure channel.
// acquired before submitting work to a channel when the Confidential
// Computing feature is enabled.
uvm_semaphore_t push_sem;
// See uvm_channel_is_secure() documentation.
bool secure;
} uvm_channel_pool_t;
struct uvm_channel_struct
@@ -304,8 +300,9 @@ struct uvm_channel_struct
// its internal operation and each push may modify this state.
uvm_mutex_t push_lock;
// Every secure channel has cryptographic state in HW, which is
// mirrored here for CPU-side operations.
// When the Confidential Computing feature is enabled, every channel has
// cryptographic state in HW, which is mirrored here for CPU-side
// operations.
UvmCslContext ctx;
bool is_ctx_initialized;
@@ -355,6 +352,13 @@ struct uvm_channel_struct
// Encryption auth tags have to be located in unprotected sysmem.
void *launch_auth_tag_cpu;
NvU64 launch_auth_tag_gpu_va;
// Used to decrypt the push back to protected sysmem.
// This happens when profilers register callbacks for migration data.
uvm_push_crypto_bundle_t *push_crypto_bundles;
// Accompanying authentication tags for the crypto bundles
uvm_rm_mem_t *push_crypto_bundle_auth_tags;
} conf_computing;
// RM channel information
@@ -452,46 +456,28 @@ struct uvm_channel_manager_struct
// Create a channel manager for the GPU
NV_STATUS uvm_channel_manager_create(uvm_gpu_t *gpu, uvm_channel_manager_t **manager_out);
static bool uvm_channel_pool_is_ce(uvm_channel_pool_t *pool);
// A channel is secure if it has HW encryption capabilities.
//
// Secure channels are treated differently in the UVM driver. Each secure
// channel has a unique CSL context associated with it, has relatively
// restrictive reservation policies (in comparison with non-secure channels),
// it is requested to be allocated differently by RM, etc.
static bool uvm_channel_pool_is_secure(uvm_channel_pool_t *pool)
static bool uvm_pool_type_is_valid(uvm_channel_pool_type_t pool_type)
{
return pool->secure;
}
static bool uvm_channel_is_secure(uvm_channel_t *channel)
{
return uvm_channel_pool_is_secure(channel->pool);
return (is_power_of_2(pool_type) && (pool_type < UVM_CHANNEL_POOL_TYPE_MASK));
}
static bool uvm_channel_pool_is_sec2(uvm_channel_pool_t *pool)
{
UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));
return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_SEC2);
}
static bool uvm_channel_pool_is_secure_ce(uvm_channel_pool_t *pool)
{
return uvm_channel_pool_is_secure(pool) && uvm_channel_pool_is_ce(pool);
}
static bool uvm_channel_pool_is_wlc(uvm_channel_pool_t *pool)
{
UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));
return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_WLC);
}
static bool uvm_channel_pool_is_lcic(uvm_channel_pool_t *pool)
{
UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));
return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_LCIC);
}
@@ -501,11 +487,6 @@ static bool uvm_channel_is_sec2(uvm_channel_t *channel)
return uvm_channel_pool_is_sec2(channel->pool);
}
static bool uvm_channel_is_secure_ce(uvm_channel_t *channel)
{
return uvm_channel_pool_is_secure_ce(channel->pool);
}
static bool uvm_channel_is_wlc(uvm_channel_t *channel)
{
return uvm_channel_pool_is_wlc(channel->pool);
@@ -516,12 +497,9 @@ static bool uvm_channel_is_lcic(uvm_channel_t *channel)
return uvm_channel_pool_is_lcic(channel->pool);
}
bool uvm_channel_type_requires_secure_pool(uvm_gpu_t *gpu, uvm_channel_type_t channel_type);
NV_STATUS uvm_channel_secure_init(uvm_gpu_t *gpu, uvm_channel_t *channel);
static bool uvm_channel_pool_is_proxy(uvm_channel_pool_t *pool)
{
UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));
return pool->pool_type == UVM_CHANNEL_POOL_TYPE_CE_PROXY;
}
@@ -533,11 +511,7 @@ static bool uvm_channel_is_proxy(uvm_channel_t *channel)
static bool uvm_channel_pool_is_ce(uvm_channel_pool_t *pool)
{
UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
if (uvm_channel_pool_is_wlc(pool) || uvm_channel_pool_is_lcic(pool))
return true;
return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_CE) || uvm_channel_pool_is_proxy(pool);
return !uvm_channel_pool_is_sec2(pool);
}
static bool uvm_channel_is_ce(uvm_channel_t *channel)
@@ -679,6 +653,11 @@ static uvm_gpu_t *uvm_channel_get_gpu(uvm_channel_t *channel)
return channel->pool->manager->gpu;
}
static uvm_pushbuffer_t *uvm_channel_get_pushbuffer(uvm_channel_t *channel)
{
return channel->pool->manager->pushbuffer;
}
// Index of a channel within the owning pool
static unsigned uvm_channel_index_in_pool(const uvm_channel_t *channel)
{

12
kernel-open/nvidia-uvm/uvm_channel_test.c
View File

@@ -681,9 +681,10 @@ done:
}
// The following test is inspired by uvm_push_test.c:test_concurrent_pushes.
// This test verifies that concurrent pushes using the same secure channel pool
// select different channels.
NV_STATUS test_secure_channel_selection(uvm_va_space_t *va_space)
// This test verifies that concurrent pushes using the same channel pool
// select different channels, when the Confidential Computing feature is
// enabled.
NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
{
NV_STATUS status = NV_OK;
uvm_channel_pool_t *pool;
@@ -703,9 +704,6 @@ NV_STATUS test_secure_channel_selection(uvm_va_space_t *va_space)
uvm_channel_type_t channel_type;
for (channel_type = 0; channel_type < UVM_CHANNEL_TYPE_COUNT; channel_type++) {
if (!uvm_channel_type_requires_secure_pool(gpu, channel_type))
continue;
pool = gpu->channel_manager->pool_to_use.default_for_type[channel_type];
TEST_CHECK_RET(pool != NULL);
@@ -997,7 +995,7 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct
if (status != NV_OK)
goto done;
status = test_secure_channel_selection(va_space);
status = test_conf_computing_channel_selection(va_space);
if (status != NV_OK)
goto done;

68
kernel-open/nvidia-uvm/uvm_conf_computing.c
View File

@@ -26,6 +26,7 @@
#include "uvm_conf_computing.h"
#include "uvm_kvmalloc.h"
#include "uvm_gpu.h"
#include "uvm_hal.h"
#include "uvm_mem.h"
#include "uvm_processors.h"
#include "uvm_tracker.h"
@@ -60,8 +61,7 @@ NV_STATUS uvm_conf_computing_init_parent_gpu(const uvm_parent_gpu_t *parent)
uvm_assert_mutex_locked(&g_uvm_global.global_lock);
// TODO: Bug 2844714.
// Since we have no routine to traverse parent gpus,
// TODO: Bug 2844714: since we have no routine to traverse parent GPUs,
// find first child GPU and get its parent.
first = uvm_global_processor_mask_find_first_gpu(&g_uvm_global.retained_gpus);
if (!first)
@@ -378,11 +378,12 @@ void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv)
NV_STATUS status;
uvm_mutex_lock(&channel->csl.ctx_lock);
status = nvUvmInterfaceCslLogDeviceEncryption(&channel->csl.ctx, iv);
status = nvUvmInterfaceCslIncrementIv(&channel->csl.ctx, UVM_CSL_OPERATION_DECRYPT, 1, iv);
uvm_mutex_unlock(&channel->csl.ctx_lock);
// nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
// overflows. This is not supposed to happen on CC.
// TODO: Bug 4014720: If nvUvmInterfaceCslIncrementIv returns with
// NV_ERR_INSUFFICIENT_RESOURCES then the IV needs to be rotated via
// nvUvmInterfaceCslRotateIv.
UVM_ASSERT(status == NV_OK);
}
@@ -391,11 +392,12 @@ void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *
NV_STATUS status;
uvm_mutex_lock(&channel->csl.ctx_lock);
status = nvUvmInterfaceCslAcquireEncryptionIv(&channel->csl.ctx, iv);
status = nvUvmInterfaceCslIncrementIv(&channel->csl.ctx, UVM_CSL_OPERATION_ENCRYPT, 1, iv);
uvm_mutex_unlock(&channel->csl.ctx_lock);
// nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
// overflows. This is not supposed to happen on CC.
// TODO: Bug 4014720: If nvUvmInterfaceCslIncrementIv returns with
// NV_ERR_INSUFFICIENT_RESOURCES then the IV needs to be rotated via
// nvUvmInterfaceCslRotateIv.
UVM_ASSERT(status == NV_OK);
}
@@ -439,8 +441,58 @@ NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
(const NvU8 *) src_cipher,
src_iv,
(NvU8 *) dst_plain,
NULL,
0,
(const NvU8 *) auth_tag_buffer);
uvm_mutex_unlock(&channel->csl.ctx_lock);
return status;
}
NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu,
void *dst_plain,
const void *src_cipher,
const void *auth_tag_buffer,
NvU8 valid)
{
NV_STATUS status;
// There is no dedicated lock for the CSL context associated with replayable
// faults. The mutual exclusion required by the RM CSL API is enforced by
// relying on the GPU replayable service lock (ISR lock), since fault
// decryption is invoked as part of fault servicing.
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
status = nvUvmInterfaceCslDecrypt(&parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx,
parent_gpu->fault_buffer_hal->entry_size(parent_gpu),
(const NvU8 *) src_cipher,
NULL,
(NvU8 *) dst_plain,
&valid,
sizeof(valid),
(const NvU8 *) auth_tag_buffer);
if (status != NV_OK)
UVM_ERR_PRINT("nvUvmInterfaceCslDecrypt() failed: %s, GPU %s\n", nvstatusToString(status), parent_gpu->name);
return status;
}
void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu, NvU64 increment)
{
NV_STATUS status;
// See comment in uvm_conf_computing_fault_decrypt
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
status = nvUvmInterfaceCslIncrementIv(&parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx,
UVM_CSL_OPERATION_DECRYPT,
increment,
NULL);
UVM_ASSERT(status == NV_OK);
}

32
kernel-open/nvidia-uvm/uvm_conf_computing.h
View File

@@ -42,9 +42,11 @@
// Use sizeof(UvmCslIv) to refer to the IV size.
#define UVM_CONF_COMPUTING_IV_ALIGNMENT 16
// SEC2 decrypt operation buffers are required to be 16-bytes aligned. CE
// encrypt/decrypt can be unaligned if the buffer lies in a single 32B segment.
// Otherwise, they need to be 32B aligned.
// SEC2 decrypt operation buffers are required to be 16-bytes aligned.
#define UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT 16
// CE encrypt/decrypt can be unaligned if the entire buffer lies in a single
// 32B segment. Otherwise, it needs to be 32B aligned.
#define UVM_CONF_COMPUTING_BUF_ALIGNMENT 32
#define UVM_CONF_COMPUTING_DMA_BUFFER_SIZE UVM_VA_BLOCK_SIZE
@@ -175,4 +177,28 @@ NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
const UvmCslIv *src_iv,
size_t size,
const void *auth_tag_buffer);
// CPU decryption of a single replayable fault, encrypted by GSP-RM.
//
// Replayable fault decryption depends not only on the encrypted fault contents,
// and the authentication tag, but also on the plaintext valid bit associated
// with the fault.
//
// When decrypting data previously encrypted by the Copy Engine, use
// uvm_conf_computing_cpu_decrypt instead.
//
// Locking: this function must be invoked while holding the replayable ISR lock.
NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu,
void *dst_plain,
const void *src_cipher,
const void *auth_tag_buffer,
NvU8 valid);
// Increment the CPU-side decrypt IV of the CSL context associated with
// replayable faults. The function is a no-op if the given increment is zero.
//
// The IV associated with a fault CSL context is a 64-bit counter.
//
// Locking: this function must be invoked while holding the replayable ISR lock.
void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu, NvU64 increment);
#endif // __UVM_CONF_COMPUTING_H__

1
kernel-open/nvidia-uvm/uvm_forward_decl.h
View File

@@ -50,6 +50,7 @@ typedef struct uvm_channel_struct uvm_channel_t;
typedef struct uvm_user_channel_struct uvm_user_channel_t;
typedef struct uvm_push_struct uvm_push_t;
typedef struct uvm_push_info_struct uvm_push_info_t;
typedef struct uvm_push_crypto_bundle_struct uvm_push_crypto_bundle_t;
typedef struct uvm_push_acquire_info_struct uvm_push_acquire_info_t;
typedef struct uvm_pushbuffer_struct uvm_pushbuffer_t;
typedef struct uvm_gpfifo_entry_struct uvm_gpfifo_entry_t;

6
kernel-open/nvidia-uvm/uvm_gpu.h
View File

@@ -198,6 +198,12 @@ typedef struct
// Client type of the service requestor.
uvm_fault_client_type_t client_type;
// New residency ID of the faulting region.
uvm_processor_id_t residency_id;
// New residency NUMA node ID of the faulting region.
int residency_node;
} uvm_ats_fault_context_t;
struct uvm_fault_service_batch_context_struct

58
kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c
View File

@@ -177,31 +177,34 @@ bool uvm_gpu_non_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu)
return has_pending_faults == NV_TRUE;
}
static NvU32 fetch_non_replayable_fault_buffer_entries(uvm_gpu_t *gpu)
static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *parent_gpu, NvU32 *cached_faults)
{
NV_STATUS status;
NvU32 i = 0;
NvU32 cached_faults = 0;
uvm_fault_buffer_entry_t *fault_cache;
NvU32 entry_size = gpu->parent->fault_buffer_hal->entry_size(gpu->parent);
uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
NvU32 i;
NvU32 entry_size = parent_gpu->fault_buffer_hal->entry_size(parent_gpu);
uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &parent_gpu->fault_buffer_info.non_replayable;
char *current_hw_entry = (char *)non_replayable_faults->shadow_buffer_copy;
uvm_fault_buffer_entry_t *fault_entry = non_replayable_faults->fault_cache;
fault_cache = non_replayable_faults->fault_cache;
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.non_replayable_faults.service_lock));
UVM_ASSERT(parent_gpu->non_replayable_faults_supported);
UVM_ASSERT(uvm_sem_is_locked(&gpu->parent->isr.non_replayable_faults.service_lock));
UVM_ASSERT(gpu->parent->non_replayable_faults_supported);
status = nvUvmInterfaceGetNonReplayableFaults(&parent_gpu->fault_buffer_info.rm_info,
current_hw_entry,
cached_faults);
status = nvUvmInterfaceGetNonReplayableFaults(&gpu->parent->fault_buffer_info.rm_info,
non_replayable_faults->shadow_buffer_copy,
&cached_faults);
UVM_ASSERT(status == NV_OK);
if (status != NV_OK) {
UVM_ERR_PRINT("nvUvmInterfaceGetNonReplayableFaults() failed: %s, GPU %s\n",
nvstatusToString(status),
parent_gpu->name);
uvm_global_set_fatal_error(status);
return status;
}
// Parse all faults
for (i = 0; i < cached_faults; ++i) {
uvm_fault_buffer_entry_t *fault_entry = &non_replayable_faults->fault_cache[i];
gpu->parent->fault_buffer_hal->parse_non_replayable_entry(gpu->parent, current_hw_entry, fault_entry);
for (i = 0; i < *cached_faults; ++i) {
parent_gpu->fault_buffer_hal->parse_non_replayable_entry(parent_gpu, current_hw_entry, fault_entry);
// The GPU aligns the fault addresses to 4k, but all of our tracking is
// done in PAGE_SIZE chunks which might be larger.
@@ -226,9 +229,10 @@ static NvU32 fetch_non_replayable_fault_buffer_entries(uvm_gpu_t *gpu)
}
current_hw_entry += entry_size;
fault_entry++;
}
return cached_faults;
return NV_OK;
}
// In SRIOV, the UVM (guest) driver does not have access to the privileged
@@ -705,21 +709,28 @@ exit_no_channel:
uvm_va_space_up_read(va_space);
uvm_va_space_mm_release_unlock(va_space, mm);
if (status != NV_OK)
UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n", uvm_gpu_name(gpu));
return status;
}
void uvm_gpu_service_non_replayable_fault_buffer(uvm_gpu_t *gpu)
{
NV_STATUS status = NV_OK;
NvU32 cached_faults;
// If this handler is modified to handle fewer than all of the outstanding
// faults, then special handling will need to be added to uvm_suspend()
// to guarantee that fault processing has completed before control is
// returned to the RM.
while ((cached_faults = fetch_non_replayable_fault_buffer_entries(gpu)) > 0) {
do {
NV_STATUS status;
NvU32 i;
status = fetch_non_replayable_fault_buffer_entries(gpu->parent, &cached_faults);
if (status != NV_OK)
return;
// Differently to replayable faults, we do not batch up and preprocess
// non-replayable faults since getting multiple faults on the same
// memory region is not very likely
@@ -728,10 +739,7 @@ void uvm_gpu_service_non_replayable_fault_buffer(uvm_gpu_t *gpu)
for (i = 0; i < cached_faults; ++i) {
status = service_fault(gpu, &gpu->parent->fault_buffer_info.non_replayable.fault_cache[i]);
if (status != NV_OK)
break;
return;
}
}
if (status != NV_OK)
UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n", uvm_gpu_name(gpu));
} while (cached_faults > 0);
}

42
kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.c
View File

@@ -486,7 +486,9 @@ static NV_STATUS cancel_fault_precise_va(uvm_gpu_t *gpu,
return status;
}
static NV_STATUS push_replay_on_gpu(uvm_gpu_t *gpu, uvm_fault_replay_type_t type, uvm_fault_service_batch_context_t *batch_context)
static NV_STATUS push_replay_on_gpu(uvm_gpu_t *gpu,
uvm_fault_replay_type_t type,
uvm_fault_service_batch_context_t *batch_context)
{
NV_STATUS status;
uvm_push_t push;
@@ -572,6 +574,19 @@ static NV_STATUS hw_fault_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu)
return status;
}
static void fault_buffer_skip_replayable_entry(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UVM_ASSERT(parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, index));
// Flushed faults are never decrypted, but the decryption IV associated with
// replayable faults still requires manual adjustment so it is kept in sync
// with the encryption IV on the GSP-RM's side.
if (!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu))
uvm_conf_computing_fault_increment_decrypt_iv(parent_gpu, 1);
parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, index);
}
static NV_STATUS fault_buffer_flush_locked(uvm_gpu_t *gpu,
uvm_gpu_buffer_flush_mode_t flush_mode,
uvm_fault_replay_type_t fault_replay,
@@ -610,7 +625,7 @@ static NV_STATUS fault_buffer_flush_locked(uvm_gpu_t *gpu,
// Wait until valid bit is set
UVM_SPIN_WHILE(!parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, get), &spin);
parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, get);
fault_buffer_skip_replayable_entry(parent_gpu, get);
++get;
if (get == replayable_faults->max_faults)
get = 0;
@@ -785,9 +800,9 @@ static bool fetch_fault_buffer_try_merge_entry(uvm_fault_buffer_entry_t *current
// This optimization cannot be performed during fault cancel on Pascal GPUs
// (fetch_mode == FAULT_FETCH_MODE_ALL) since we need accurate tracking of all
// the faults in each uTLB in order to guarantee precise fault attribution.
static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
uvm_fault_service_batch_context_t *batch_context,
fault_fetch_mode_t fetch_mode)
static NV_STATUS fetch_fault_buffer_entries(uvm_gpu_t *gpu,
uvm_fault_service_batch_context_t *batch_context,
fault_fetch_mode_t fetch_mode)
{
NvU32 get;
NvU32 put;
@@ -796,6 +811,7 @@ static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
NvU32 utlb_id;
uvm_fault_buffer_entry_t *fault_cache;
uvm_spin_loop_t spin;
NV_STATUS status = NV_OK;
uvm_replayable_fault_buffer_info_t *replayable_faults = &gpu->parent->fault_buffer_info.replayable;
const bool in_pascal_cancel_path = (!gpu->parent->fault_cancel_va_supported && fetch_mode == FAULT_FETCH_MODE_ALL);
const bool may_filter = uvm_perf_fault_coalesce && !in_pascal_cancel_path;
@@ -851,7 +867,9 @@ static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
smp_mb__after_atomic();
// Got valid bit set. Let's cache.
gpu->parent->fault_buffer_hal->parse_entry(gpu->parent, get, current_entry);
status = gpu->parent->fault_buffer_hal->parse_replayable_entry(gpu->parent, get, current_entry);
if (status != NV_OK)
goto done;
// The GPU aligns the fault addresses to 4k, but all of our tracking is
// done in PAGE_SIZE chunks which might be larger.
@@ -918,6 +936,8 @@ done:
batch_context->num_cached_faults = fault_index;
batch_context->num_coalesced_faults = num_coalesced_faults;
return status;
}
// Sort comparator for pointers to fault buffer entries that sorts by
@@ -2475,7 +2495,10 @@ static NV_STATUS cancel_faults_precise_tlb(uvm_gpu_t *gpu, uvm_fault_service_bat
batch_context->has_throttled_faults = false;
// 5) Fetch all faults from buffer
fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_ALL);
status = fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_ALL);
if (status != NV_OK)
break;
++batch_context->batch_id;
UVM_ASSERT(batch_context->num_cached_faults == batch_context->num_coalesced_faults);
@@ -2612,7 +2635,10 @@ void uvm_gpu_service_replayable_faults(uvm_gpu_t *gpu)
batch_context->has_fatal_faults = false;
batch_context->has_throttled_faults = false;
fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_BATCH_READY);
status = fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_BATCH_READY);
if (status != NV_OK)
break;
if (batch_context->num_cached_faults == 0)
break;

4
kernel-open/nvidia-uvm/uvm_gpu_semaphore.c
View File

@@ -579,8 +579,10 @@ static void uvm_gpu_semaphore_encrypted_payload_update(uvm_channel_t *channel, u
void *auth_tag_cpu_addr = uvm_rm_mem_get_cpu_va(semaphore->conf_computing.auth_tag);
NvU32 *gpu_notifier_cpu_addr = (NvU32 *)uvm_rm_mem_get_cpu_va(semaphore->conf_computing.notifier);
NvU32 *payload_cpu_addr = (NvU32 *)uvm_rm_mem_get_cpu_va(semaphore->conf_computing.encrypted_payload);
uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);
UVM_ASSERT(uvm_channel_is_secure_ce(channel));
UVM_ASSERT(uvm_conf_computing_mode_enabled(gpu));
UVM_ASSERT(uvm_channel_is_ce(channel));
last_observed_notifier = semaphore->conf_computing.last_observed_notifier;
gpu_notifier = UVM_READ_ONCE(*gpu_notifier_cpu_addr);

6
kernel-open/nvidia-uvm/uvm_gpu_semaphore.h
View File

@@ -91,9 +91,9 @@ struct uvm_gpu_tracking_semaphore_struct
// Create a semaphore pool for a GPU.
NV_STATUS uvm_gpu_semaphore_pool_create(uvm_gpu_t *gpu, uvm_gpu_semaphore_pool_t **pool_out);
// When the Confidential Computing feature is enabled, pools associated with
// secure CE channels are allocated in the CPR of vidmem and as such have
// all the associated access restrictions. Because of this, they're called
// When the Confidential Computing feature is enabled, semaphore pools
// associated with CE channels are allocated in the CPR of vidmem and as such
// have all the associated access restrictions. Because of this, they're called
// secure pools and secure semaphores are allocated out of said secure pools.
NV_STATUS uvm_gpu_semaphore_secure_pool_create(uvm_gpu_t *gpu, uvm_gpu_semaphore_pool_t **pool_out);

6
kernel-open/nvidia-uvm/uvm_hal.c
View File

@@ -373,7 +373,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.read_get = uvm_hal_maxwell_fault_buffer_read_get_unsupported,
.write_get = uvm_hal_maxwell_fault_buffer_write_get_unsupported,
.get_ve_id = uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported,
.parse_entry = uvm_hal_maxwell_fault_buffer_parse_entry_unsupported,
.parse_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported,
.entry_is_valid = uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported,
.entry_clear_valid = uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported,
.entry_size = uvm_hal_maxwell_fault_buffer_entry_size_unsupported,
@@ -396,7 +396,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.read_put = uvm_hal_pascal_fault_buffer_read_put,
.read_get = uvm_hal_pascal_fault_buffer_read_get,
.write_get = uvm_hal_pascal_fault_buffer_write_get,
.parse_entry = uvm_hal_pascal_fault_buffer_parse_entry,
.parse_replayable_entry = uvm_hal_pascal_fault_buffer_parse_replayable_entry,
.entry_is_valid = uvm_hal_pascal_fault_buffer_entry_is_valid,
.entry_clear_valid = uvm_hal_pascal_fault_buffer_entry_clear_valid,
.entry_size = uvm_hal_pascal_fault_buffer_entry_size,
@@ -411,7 +411,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.read_get = uvm_hal_volta_fault_buffer_read_get,
.write_get = uvm_hal_volta_fault_buffer_write_get,
.get_ve_id = uvm_hal_volta_fault_buffer_get_ve_id,
.parse_entry = uvm_hal_volta_fault_buffer_parse_entry,
.parse_replayable_entry = uvm_hal_volta_fault_buffer_parse_replayable_entry,
.parse_non_replayable_entry = uvm_hal_volta_fault_buffer_parse_non_replayable_entry,
.get_fault_type = uvm_hal_volta_fault_buffer_get_fault_type,
}

36
kernel-open/nvidia-uvm/uvm_hal.h
View File

@@ -485,11 +485,24 @@ typedef NvU32 (*uvm_hal_fault_buffer_read_get_t)(uvm_parent_gpu_t *parent_gpu);
typedef void (*uvm_hal_fault_buffer_write_get_t)(uvm_parent_gpu_t *parent_gpu, NvU32 get);
typedef NvU8 (*uvm_hal_fault_buffer_get_ve_id_t)(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
// Parse the entry on the given buffer index. This also clears the valid bit of
// the entry in the buffer.
typedef void (*uvm_hal_fault_buffer_parse_entry_t)(uvm_parent_gpu_t *gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
// Parse the replayable entry at the given buffer index. This also clears the
// valid bit of the entry in the buffer.
typedef NV_STATUS (*uvm_hal_fault_buffer_parse_replayable_entry_t)(uvm_parent_gpu_t *gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_volta_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
typedef bool (*uvm_hal_fault_buffer_entry_is_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
typedef void (*uvm_hal_fault_buffer_entry_clear_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
typedef NvU32 (*uvm_hal_fault_buffer_entry_size_t)(uvm_parent_gpu_t *parent_gpu);
@@ -508,9 +521,6 @@ NvU32 uvm_hal_maxwell_fault_buffer_read_put_unsupported(uvm_parent_gpu_t *parent
NvU32 uvm_hal_maxwell_fault_buffer_read_get_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_fault_buffer_write_get_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
void uvm_hal_maxwell_fault_buffer_parse_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry);
void uvm_hal_pascal_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
@@ -519,18 +529,14 @@ void uvm_hal_pascal_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32
NvU32 uvm_hal_pascal_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_pascal_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
uvm_fault_type_t uvm_hal_pascal_fault_buffer_get_fault_type(const NvU32 *fault_entry);
NvU32 uvm_hal_volta_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_volta_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_volta_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU8 uvm_hal_volta_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
void uvm_hal_volta_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
uvm_fault_type_t uvm_hal_volta_fault_buffer_get_fault_type(const NvU32 *fault_entry);
void uvm_hal_turing_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
@@ -772,7 +778,7 @@ struct uvm_fault_buffer_hal_struct
uvm_hal_fault_buffer_read_get_t read_get;
uvm_hal_fault_buffer_write_get_t write_get;
uvm_hal_fault_buffer_get_ve_id_t get_ve_id;
uvm_hal_fault_buffer_parse_entry_t parse_entry;
uvm_hal_fault_buffer_parse_replayable_entry_t parse_replayable_entry;
uvm_hal_fault_buffer_entry_is_valid_t entry_is_valid;
uvm_hal_fault_buffer_entry_clear_valid_t entry_clear_valid;
uvm_hal_fault_buffer_entry_size_t entry_size;

7
kernel-open/nvidia-uvm/uvm_hal_types.h
View File

@@ -128,6 +128,13 @@ static uvm_gpu_address_t uvm_gpu_address_virtual(NvU64 va)
return address;
}
static uvm_gpu_address_t uvm_gpu_address_virtual_unprotected(NvU64 va)
{
uvm_gpu_address_t address = uvm_gpu_address_virtual(va);
address.is_unprotected = true;
return address;
}
// Create a physical GPU address
static uvm_gpu_address_t uvm_gpu_address_physical(uvm_aperture_t aperture, NvU64 pa)
{

2
kernel-open/nvidia-uvm/uvm_hmm.c
View File

@@ -2575,7 +2575,7 @@ static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block,
continue;
}
if (folio_test_swapcache(page_folio(src_page))) {
if (PageSwapCache(src_page)) {
// TODO: Bug 4050579: Remove this when swap cached pages can be
// migrated.
if (service_context) {

6
kernel-open/nvidia-uvm/uvm_hopper.c
View File

@@ -61,7 +61,11 @@ void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
// GH180.
parent_gpu->ce_phys_vidmem_write_supported = !uvm_gpu_is_coherent(parent_gpu);
parent_gpu->peer_copy_mode = g_uvm_global.peer_copy_mode;
// TODO: Bug 4174553: [HGX-SkinnyJoe][GH180] channel errors discussion/debug
// portion for the uvm tests became nonresponsive after
// some time and then failed even after reboot
parent_gpu->peer_copy_mode = uvm_gpu_is_coherent(parent_gpu) ?
UVM_GPU_PEER_COPY_MODE_VIRTUAL : g_uvm_global.peer_copy_mode;
// All GR context buffers may be mapped to 57b wide VAs. All "compute" units
// accessing GR context buffers support the 57-bit VA range.

2
kernel-open/nvidia-uvm/uvm_hopper_ce.c
View File

@@ -491,7 +491,6 @@ void uvm_hal_hopper_ce_encrypt(uvm_push_t *push,
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
UVM_ASSERT(uvm_conf_computing_mode_is_hcc(gpu));
UVM_ASSERT(uvm_push_is_fake(push) || uvm_channel_is_secure(push->channel));
UVM_ASSERT(IS_ALIGNED(auth_tag.address, UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT));
if (!src.is_virtual)
@@ -540,7 +539,6 @@ void uvm_hal_hopper_ce_decrypt(uvm_push_t *push,
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
UVM_ASSERT(uvm_conf_computing_mode_is_hcc(gpu));
UVM_ASSERT(!push->channel || uvm_channel_is_secure(push->channel));
UVM_ASSERT(IS_ALIGNED(auth_tag.address, UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT));
// The addressing mode (and aperture, if applicable) of the source and

1
kernel-open/nvidia-uvm/uvm_hopper_sec2.c
View File

@@ -166,6 +166,7 @@ void uvm_hal_hopper_sec2_decrypt(uvm_push_t *push, NvU64 dst_va, NvU64 src_va, N
NvU32 *csl_sign_init = push->next;
// Check that the provided alignment matches HW
BUILD_BUG_ON(UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT != (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)));
BUILD_BUG_ON(UVM_CONF_COMPUTING_BUF_ALIGNMENT < (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)));
BUILD_BUG_ON(UVM_CONF_COMPUTING_BUF_ALIGNMENT % (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)) != 0);

4
kernel-open/nvidia-uvm/uvm_linux.h
View File

@@ -153,6 +153,10 @@ static inline const struct cpumask *uvm_cpumask_of_node(int node)
#define VM_MIXEDMAP 0x00000000
#endif
#if !defined(MPOL_PREFERRED_MANY)
#define MPOL_PREFERRED_MANY 5
#endif
//
// printk.h already defined pr_fmt, so we have to redefine it so the pr_*
// routines pick up our version

52
kernel-open/nvidia-uvm/uvm_lock.h
View File

@@ -279,13 +279,14 @@
// Operations not allowed while holding the lock:
// - GPU memory allocation which can evict memory (would require nesting
// block locks)
//
// - GPU DMA Allocation pool lock (gpu->conf_computing.dma_buffer_pool.lock)
// Order: UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL
// Condition: The Confidential Computing feature is enabled
// Exclusive lock (mutex)
//
// Protects:
// - Protect the state of the uvm_conf_computing_dma_buffer_pool_t
// when the Confidential Computing feature is enabled on the system.
//
// - Chunk mapping lock (gpu->root_chunk_mappings.bitlocks and
// gpu->sysmem_mappings.bitlock)
@@ -321,22 +322,25 @@
// Operations not allowed while holding this lock
// - GPU memory allocation which can evict
//
// - Secure channel CSL channel pool semaphore
// - CE channel CSL channel pool semaphore
// Order: UVM_LOCK_ORDER_CSL_PUSH
// Semaphore per SEC2 channel pool
// Condition: The Confidential Computing feature is enabled
// Semaphore per CE channel pool
//
// The semaphore controls concurrent pushes to secure channels. Secure work
// submission depends on channel availability in GPFIFO entries (as in any
// other channel type) but also on channel locking. Each secure channel has a
// lock to enforce ordering of pushes. The channel's CSL lock is taken on
// channel reservation until uvm_push_end. Secure channels are stateful
// channels and the CSL lock protects their CSL state/context.
// The semaphore controls concurrent pushes to CE channels that are not WCL
// channels. Secure work submission depends on channel availability in
// GPFIFO entries (as in any other channel type) but also on channel
// locking. Each channel has a lock to enforce ordering of pushes. The
// channel's CSL lock is taken on channel reservation until uvm_push_end.
// When the Confidential Computing feature is enabled, channels are
// stateful, and the CSL lock protects their CSL state/context.
//
// Operations allowed while holding this lock
// - Pushing work to CE secure channels
// - Pushing work to CE channels (except for WLC channels)
//
// - WLC CSL channel pool semaphore
// Order: UVM_LOCK_ORDER_CSL_WLC_PUSH
// Condition: The Confidential Computing feature is enabled
// Semaphore per WLC channel pool
//
// The semaphore controls concurrent pushes to WLC channels. WLC work
@@ -346,8 +350,8 @@
// channel reservation until uvm_push_end. SEC2 channels are stateful
// channels and the CSL lock protects their CSL state/context.
//
// This lock ORDER is different and sits below generic secure channel CSL
// lock and above SEC2 CSL lock. This reflects the dual nature of WLC
// This lock ORDER is different and sits below the generic channel CSL
// lock and above the SEC2 CSL lock. This reflects the dual nature of WLC
// channels; they use SEC2 indirect work launch during initialization,
// and after their schedule is initialized they provide indirect launch
// functionality to other CE channels.
@@ -357,6 +361,7 @@
//
// - SEC2 CSL channel pool semaphore
// Order: UVM_LOCK_ORDER_SEC2_CSL_PUSH
// Condition: The Confidential Computing feature is enabled
// Semaphore per SEC2 channel pool
//
// The semaphore controls concurrent pushes to SEC2 channels. SEC2 work
@@ -366,9 +371,9 @@
// channel reservation until uvm_push_end. SEC2 channels are stateful
// channels and the CSL lock protects their CSL state/context.
//
// This lock ORDER is different and lower than the generic secure channel
// lock to allow secure work submission to use a SEC2 channel to submit
// work before releasing the CSL lock of the originating secure channel.
// This lock ORDER is different and lower than UVM_LOCK_ORDER_CSL_PUSH
// to allow secure work submission to use a SEC2 channel to submit
// work before releasing the CSL lock of the originating channel.
//
// Operations allowed while holding this lock
// - Pushing work to SEC2 channels
@@ -408,16 +413,18 @@
//
// - WLC Channel lock
// Order: UVM_LOCK_ORDER_WLC_CHANNEL
// Condition: The Confidential Computing feature is enabled
// Spinlock (uvm_spinlock_t)
//
// Lock protecting the state of WLC channels in a channel pool. This lock
// is separate from the above generic channel lock to allow for indirect
// worklaunch pushes while holding the main channel lock.
// (WLC pushes don't need any of the pushbuffer locks described above)
// is separate from the generic channel lock (UVM_LOCK_ORDER_CHANNEL)
// to allow for indirect worklaunch pushes while holding the main channel
// lock (WLC pushes don't need any of the pushbuffer locks described
// above)
//
// - Tools global VA space list lock (g_tools_va_space_list_lock)
// Order: UVM_LOCK_ORDER_TOOLS_VA_SPACE_LIST
// Reader/writer lock (rw_sempahore)
// Reader/writer lock (rw_semaphore)
//
// This lock protects the list of VA spaces used when broadcasting
// UVM profiling events.
@@ -437,9 +444,10 @@
//
// - Tracking semaphores
// Order: UVM_LOCK_ORDER_SECURE_SEMAPHORE
// When the Confidential Computing feature is enabled, CE semaphores are
// encrypted, and require to take the CSL lock (UVM_LOCK_ORDER_LEAF) to
// decrypt the payload.
// Condition: The Confidential Computing feature is enabled
//
// CE semaphore payloads are encrypted, and require to take the CSL lock
// (UVM_LOCK_ORDER_LEAF) to decrypt the payload.
//
// - Leaf locks
// Order: UVM_LOCK_ORDER_LEAF

7
kernel-open/nvidia-uvm/uvm_maxwell_fault_buffer.c
View File

@@ -68,11 +68,12 @@ uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const N
return UVM_FAULT_TYPE_COUNT;
}
void uvm_hal_maxwell_fault_buffer_parse_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false, "fault_buffer_parse_entry is not supported on GPU: %s.\n", parent_gpu->name);
return NV_ERR_NOT_SUPPORTED;
}
bool uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index)

6
kernel-open/nvidia-uvm/uvm_mem.h
View File

@@ -392,12 +392,6 @@ static NV_STATUS uvm_mem_alloc_vidmem(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **me
return uvm_mem_alloc(&params, mem_out);
}
// Helper for allocating protected vidmem with the default page size
static NV_STATUS uvm_mem_alloc_vidmem_protected(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **mem_out)
{
return uvm_mem_alloc_vidmem(size, gpu, mem_out);
}
// Helper for allocating sysmem and mapping it on the CPU
static NV_STATUS uvm_mem_alloc_sysmem_and_map_cpu_kernel(NvU64 size, struct mm_struct *mm, uvm_mem_t **mem_out)
{

39
kernel-open/nvidia-uvm/uvm_migrate.c
View File

@@ -134,6 +134,22 @@ static NV_STATUS block_migrate_map_unmapped_pages(uvm_va_block_t *va_block,
// first map operation
uvm_page_mask_complement(&va_block_context->caller_page_mask, &va_block->maybe_mapped_pages);
if (uvm_va_block_is_hmm(va_block) && !UVM_ID_IS_CPU(dest_id)) {
// Do not map pages that are already resident on the CPU. This is in
// order to avoid breaking system-wide atomic operations on HMM. HMM's
// implementation of system-side atomic operations involves restricting
// mappings to one processor (CPU or a GPU) at a time. If we were to
// grant a GPU a mapping to system memory, this gets into trouble
// because, on the CPU side, Linux can silently upgrade PTE permissions
// (move from read-only, to read-write, without any MMU notifiers
// firing), thus breaking the model by allowing simultaneous read-write
// access from two separate processors. To avoid that, just don't map
// such pages at all, when migrating.
uvm_page_mask_andnot(&va_block_context->caller_page_mask,
&va_block_context->caller_page_mask,
uvm_va_block_resident_mask_get(va_block, UVM_ID_CPU));
}
// Only map those pages that are not mapped anywhere else (likely due
// to a first touch or a migration). We pass
// UvmEventMapRemoteCauseInvalid since the destination processor of a
@@ -944,17 +960,18 @@ NV_STATUS uvm_api_migrate(UVM_MIGRATE_PARAMS *params, struct file *filp)
if (type == UVM_API_RANGE_TYPE_ATS) {
uvm_migrate_args_t uvm_migrate_args =
{
.va_space = va_space,
.mm = mm,
.start = params->base,
.length = params->length,
.dst_id = (dest_gpu ? dest_gpu->id : UVM_ID_CPU),
.dst_node_id = (int)params->cpuNumaNode,
.populate_permissions = UVM_POPULATE_PERMISSIONS_INHERIT,
.touch = false,
.skip_mapped = false,
.user_space_start = &params->userSpaceStart,
.user_space_length = &params->userSpaceLength,
.va_space = va_space,
.mm = mm,
.start = params->base,
.length = params->length,
.dst_id = (dest_gpu ? dest_gpu->id : UVM_ID_CPU),
.dst_node_id = (int)params->cpuNumaNode,
.populate_permissions = UVM_POPULATE_PERMISSIONS_INHERIT,
.touch = false,
.skip_mapped = false,
.populate_on_cpu_alloc_failures = false,
.user_space_start = &params->userSpaceStart,
.user_space_length = &params->userSpaceLength,
};
status = uvm_migrate_pageable(&uvm_migrate_args);

33
kernel-open/nvidia-uvm/uvm_migrate_pageable.c
View File

@@ -507,6 +507,22 @@ static NV_STATUS migrate_vma_copy_pages(struct vm_area_struct *vma,
return NV_OK;
}
void migrate_vma_cleanup_pages(unsigned long *dst, unsigned long npages)
{
unsigned long i;
for (i = 0; i < npages; i++) {
struct page *dst_page = migrate_pfn_to_page(dst[i]);
if (!dst_page)
continue;
unlock_page(dst_page);
__free_page(dst_page);
dst[i] = 0;
}
}
void uvm_migrate_vma_alloc_and_copy(struct migrate_vma *args, migrate_vma_state_t *state)
{
struct vm_area_struct *vma = args->vma;
@@ -531,6 +547,10 @@ void uvm_migrate_vma_alloc_and_copy(struct migrate_vma *args, migrate_vma_state_
if (state->status == NV_OK)
state->status = tracker_status;
// Mark all pages as not migrating if we're failing
if (state->status != NV_OK)
migrate_vma_cleanup_pages(args->dst, state->num_pages);
}
void uvm_migrate_vma_alloc_and_copy_helper(struct vm_area_struct *vma,
@@ -802,7 +822,7 @@ static NV_STATUS migrate_pageable_vma_region(struct vm_area_struct *vma,
// If the destination is the CPU, signal user-space to retry with a
// different node. Otherwise, just try to populate anywhere in the
// system
if (UVM_ID_IS_CPU(uvm_migrate_args->dst_id)) {
if (UVM_ID_IS_CPU(uvm_migrate_args->dst_id) && !uvm_migrate_args->populate_on_cpu_alloc_failures) {
*next_addr = start + find_first_bit(state->scratch2_mask, num_pages) * PAGE_SIZE;
return NV_ERR_MORE_PROCESSING_REQUIRED;
}
@@ -961,13 +981,10 @@ NV_STATUS uvm_migrate_pageable(uvm_migrate_args_t *uvm_migrate_args)
// We only check that dst_node_id is a valid node in the system and it
// doesn't correspond to a GPU node. This is fine because
// alloc_pages_node will clamp the allocation to
// cpuset_current_mems_allowed, and uvm_migrate_pageable is only called
// from process context (uvm_migrate) when dst_id is CPU. UVM bottom
// half never calls uvm_migrate_pageable when dst_id is CPU. So, assert
// that we're in a user thread. However, this would need to change if we
// wanted to call this function from a bottom half with CPU dst_id.
UVM_ASSERT(!(current->flags & PF_KTHREAD));
// cpuset_current_mems_allowed when uvm_migrate_pageable is called from
// process context (uvm_migrate) when dst_id is CPU. UVM bottom half
// calls uvm_migrate_pageable with CPU dst_id only when the VMA memory
// policy is set to dst_node_id and dst_node_id is not NUMA_NO_NODE.
if (!nv_numa_node_has_memory(dst_node_id) ||
uvm_va_space_find_gpu_with_memory_node_id(va_space, dst_node_id) != NULL)
return NV_ERR_INVALID_ARGUMENT;

1
kernel-open/nvidia-uvm/uvm_migrate_pageable.h
View File

@@ -43,6 +43,7 @@ typedef struct
uvm_populate_permissions_t populate_permissions;
bool touch : 1;
bool skip_mapped : 1;
bool populate_on_cpu_alloc_failures : 1;
NvU64 *user_space_start;
NvU64 *user_space_length;
} uvm_migrate_args_t;

8
kernel-open/nvidia-uvm/uvm_pascal_fault_buffer.c
View File

@@ -214,9 +214,9 @@ static UvmFaultMetadataPacket *get_fault_buffer_entry_metadata(uvm_parent_gpu_t
return fault_entry_metadata + index;
}
void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
{
NvU32 *fault_entry;
NvU64 addr_hi, addr_lo;
@@ -280,6 +280,8 @@ void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
// Automatically clear valid bit for the entry in the fault buffer
uvm_hal_pascal_fault_buffer_entry_clear_valid(parent_gpu, index);
return NV_OK;
}
bool uvm_hal_pascal_fault_buffer_entry_is_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index)

13
kernel-open/nvidia-uvm/uvm_perf_thrashing.c
View File

@@ -1455,7 +1455,18 @@ static uvm_perf_thrashing_hint_t get_hint_for_migration_thrashing(va_space_thras
hint.type = UVM_PERF_THRASHING_HINT_TYPE_NONE;
closest_resident_id = uvm_va_block_page_get_closest_resident(va_block, page_index, requester);
UVM_ASSERT(UVM_ID_IS_VALID(closest_resident_id));
if (uvm_va_block_is_hmm(va_block)) {
// HMM pages always start out resident on the CPU but may not be
// recorded in the va_block state because hmm_range_fault() or
// similar functions haven't been called to get an accurate snapshot
// of the Linux state. We can assume pages are CPU resident for the
// purpose of deciding where to migrate to reduce thrashing.
if (UVM_ID_IS_INVALID(closest_resident_id))
closest_resident_id = UVM_ID_CPU;
}
else {
UVM_ASSERT(UVM_ID_IS_VALID(closest_resident_id));
}
if (thrashing_processors_can_access(va_space, page_thrashing, preferred_location)) {
// The logic in uvm_va_block_select_residency chooses the preferred

8
kernel-open/nvidia-uvm/uvm_push.c
View File

@@ -391,11 +391,13 @@ uvm_gpu_address_t uvm_push_inline_data_end(uvm_push_inline_data_t *data)
inline_data_address = (NvU64) (uintptr_t)(push->next + 1);
}
else {
uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
// Offset of the inlined data within the push.
inline_data_address = (push->next - push->begin + 1) * UVM_METHOD_SIZE;
// Add GPU VA of the push begin
inline_data_address += uvm_pushbuffer_get_gpu_va_for_push(channel->pool->manager->pushbuffer, push);
inline_data_address += uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
}
// This will place a noop right before the inline data that was written.
@@ -438,10 +440,8 @@ NvU64 *uvm_push_timestamp(uvm_push_t *push)
if (uvm_channel_is_ce(push->channel))
gpu->parent->ce_hal->semaphore_timestamp(push, address.address);
else if (uvm_channel_is_sec2(push->channel))
gpu->parent->sec2_hal->semaphore_timestamp(push, address.address);
else
UVM_ASSERT_MSG(0, "Semaphore release timestamp on an unsupported channel.\n");
gpu->parent->sec2_hal->semaphore_timestamp(push, address.address);
return timestamp;
}

13
kernel-open/nvidia-uvm/uvm_push.h
View File

@@ -64,6 +64,14 @@ typedef enum
UVM_PUSH_FLAG_COUNT,
} uvm_push_flag_t;
struct uvm_push_crypto_bundle_struct {
// Initialization vector used to decrypt the push
UvmCslIv iv;
// Size of the pushbuffer that is encrypted/decrypted
NvU32 push_size;
};
struct uvm_push_struct
{
// Location of the first method of the push
@@ -369,11 +377,6 @@ static bool uvm_push_has_space(uvm_push_t *push, NvU32 free_space)
NV_STATUS uvm_push_begin_fake(uvm_gpu_t *gpu, uvm_push_t *push);
void uvm_push_end_fake(uvm_push_t *push);
static bool uvm_push_is_fake(uvm_push_t *push)
{
return !push->channel;
}
// Begin an inline data fragment in the push
//
// The inline data will be ignored by the GPU, but can be referenced from

99
kernel-open/nvidia-uvm/uvm_push_test.c
View File

@@ -40,10 +40,9 @@
static NvU32 get_push_begin_size(uvm_channel_t *channel)
{
if (uvm_channel_is_sec2(channel)) {
// SEC2 channels allocate CSL signature buffer at the beginning.
// SEC2 channels allocate CSL signature buffer at the beginning.
if (uvm_channel_is_sec2(channel))
return UVM_CONF_COMPUTING_SIGN_BUF_MAX_SIZE + UVM_METHOD_SIZE;
}
return 0;
}
@@ -51,10 +50,14 @@ static NvU32 get_push_begin_size(uvm_channel_t *channel)
// This is the storage required by a semaphore release.
static NvU32 get_push_end_min_size(uvm_channel_t *channel)
{
if (uvm_channel_is_ce(channel)) {
if (uvm_channel_is_wlc(channel)) {
// Space (in bytes) used by uvm_push_end() on a Secure CE channel.
// Note that Secure CE semaphore release pushes two memset and one
uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);
if (uvm_conf_computing_mode_enabled(gpu)) {
if (uvm_channel_is_ce(channel)) {
// Space (in bytes) used by uvm_push_end() on a CE channel when
// the Confidential Computing feature is enabled.
//
// Note that CE semaphore release pushes two memset and one
// encryption method on top of the regular release.
// Memset size
// -------------
@@ -75,43 +78,44 @@ static NvU32 get_push_end_min_size(uvm_channel_t *channel)
//
// TOTAL : 144 Bytes
// Same as CE + LCIC GPPut update + LCIC doorbell
return 24 + 144 + 24 + 24;
}
else if (uvm_channel_is_secure_ce(channel)) {
if (uvm_channel_is_wlc(channel)) {
// Same as CE + LCIC GPPut update + LCIC doorbell
return 24 + 144 + 24 + 24;
}
return 24 + 144;
}
// Space (in bytes) used by uvm_push_end() on a CE channel.
return 24;
}
else if (uvm_channel_is_sec2(channel)) {
UVM_ASSERT(uvm_channel_is_sec2(channel));
// A perfectly aligned inline buffer in SEC2 semaphore release.
// We add UVM_METHOD_SIZE because of the NOP method to reserve
// UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES (the inline buffer.)
return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_METHOD_SIZE;
}
return 0;
UVM_ASSERT(uvm_channel_is_ce(channel));
// Space (in bytes) used by uvm_push_end() on a CE channel.
return 24;
}
static NvU32 get_push_end_max_size(uvm_channel_t *channel)
{
if (uvm_channel_is_ce(channel)) {
if (uvm_channel_is_wlc(channel)) {
// WLC pushes are always padded to UVM_MAX_WLC_PUSH_SIZE
return UVM_MAX_WLC_PUSH_SIZE;
}
// Space (in bytes) used by uvm_push_end() on a CE channel.
return get_push_end_min_size(channel);
}
else if (uvm_channel_is_sec2(channel)) {
// Space (in bytes) used by uvm_push_end() on a SEC2 channel.
// Note that SEC2 semaphore release uses an inline buffer with alignment
// requirements. This is the "worst" case semaphore_release storage.
return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT;
}
// WLC pushes are always padded to UVM_MAX_WLC_PUSH_SIZE
if (uvm_channel_is_wlc(channel))
return UVM_MAX_WLC_PUSH_SIZE;
return 0;
// Space (in bytes) used by uvm_push_end() on a SEC2 channel.
// Note that SEC2 semaphore release uses an inline buffer with alignment
// requirements. This is the "worst" case semaphore_release storage.
if (uvm_channel_is_sec2(channel))
return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT;
UVM_ASSERT(uvm_channel_is_ce(channel));
// Space (in bytes) used by uvm_push_end() on a CE channel.
return get_push_end_min_size(channel);
}
static NV_STATUS test_push_end_size(uvm_va_space_t *va_space)
@@ -294,10 +298,19 @@ static NV_STATUS test_concurrent_pushes(uvm_va_space_t *va_space)
{
NV_STATUS status = NV_OK;
uvm_gpu_t *gpu;
NvU32 i;
uvm_push_t *pushes;
uvm_tracker_t tracker = UVM_TRACKER_INIT();
uvm_channel_type_t channel_type = UVM_CHANNEL_TYPE_GPU_INTERNAL;
uvm_tracker_t tracker;
// When the Confidential Computing feature is enabled, a channel reserved at
// the start of a push cannot be reserved again until that push ends. The
// test is waived, because the number of pushes it starts per pool exceeds
// the number of channels in the pool, so it would block indefinitely.
gpu = uvm_va_space_find_first_gpu(va_space);
if ((gpu != NULL) && uvm_conf_computing_mode_enabled(gpu))
return NV_OK;
uvm_tracker_init(&tracker);
// As noted above, this test does unsafe things that would be detected by
// lock tracking, opt-out.
@@ -310,16 +323,11 @@ static NV_STATUS test_concurrent_pushes(uvm_va_space_t *va_space)
}
for_each_va_space_gpu(gpu, va_space) {
NvU32 i;
// A secure channels reserved at the start of a push cannot be reserved
// again until that push ends. The test would block indefinitely
// if secure pools are not skipped, because the number of pushes started
// per pool exceeds the number of channels in the pool.
if (uvm_channel_type_requires_secure_pool(gpu, channel_type))
goto done;
for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
uvm_push_t *push = &pushes[i];
status = uvm_push_begin(gpu->channel_manager, channel_type, push, "concurrent push %u", i);
status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, push, "concurrent push %u", i);
TEST_CHECK_GOTO(status == NV_OK, done);
}
for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
@@ -776,15 +784,6 @@ static NV_STATUS test_timestamp_on_gpu(uvm_gpu_t *gpu)
NvU32 i;
NvU64 last_stamp = 0;
// TODO: Bug 3988992: [UVM][HCC] RFE - Support encrypted semaphore for secure CE channels
// This test is waived when Confidential Computing is enabled because it
// assumes that CPU can directly read the result of a semaphore timestamp
// operation. Instead the operation needs to be follower up by an encrypt
// -decrypt trip to be accessible to CPU. This will be cleaner and simpler
// once encrypted semaphores are available.
if (uvm_conf_computing_mode_enabled(gpu))
return NV_OK;
for (i = 0; i < 10; ++i) {
status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, &push, "Releasing a timestamp");
if (status != NV_OK)

59
kernel-open/nvidia-uvm/uvm_pushbuffer.c
View File

@@ -449,21 +449,68 @@ static uvm_pushbuffer_chunk_t *gpfifo_to_chunk(uvm_pushbuffer_t *pushbuffer, uvm
return chunk;
}
void uvm_pushbuffer_mark_completed(uvm_pushbuffer_t *pushbuffer, uvm_gpfifo_entry_t *gpfifo)
static void decrypt_push(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo)
{
NV_STATUS status;
NvU32 auth_tag_offset;
void *auth_tag_cpu_va;
void *push_protected_cpu_va;
void *push_unprotected_cpu_va;
NvU32 pushbuffer_offset = gpfifo->pushbuffer_offset;
NvU32 push_info_index = gpfifo->push_info - channel->push_infos;
uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
uvm_push_crypto_bundle_t *crypto_bundle = channel->conf_computing.push_crypto_bundles + push_info_index;
if (channel->conf_computing.push_crypto_bundles == NULL)
return;
// When the crypto bundle is used, the push size cannot be zero
if (crypto_bundle->push_size == 0)
return;
UVM_ASSERT(!uvm_channel_is_wlc(channel));
UVM_ASSERT(!uvm_channel_is_lcic(channel));
push_protected_cpu_va = (char *)get_base_cpu_va(pushbuffer) + pushbuffer_offset;
push_unprotected_cpu_va = (char *)uvm_rm_mem_get_cpu_va(pushbuffer->memory_unprotected_sysmem) + pushbuffer_offset;
auth_tag_offset = push_info_index * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
auth_tag_cpu_va = (char *)uvm_rm_mem_get_cpu_va(channel->conf_computing.push_crypto_bundle_auth_tags) +
auth_tag_offset;
status = uvm_conf_computing_cpu_decrypt(channel,
push_protected_cpu_va,
push_unprotected_cpu_va,
&crypto_bundle->iv,
crypto_bundle->push_size,
auth_tag_cpu_va);
// A decryption failure here is not fatal because it does not
// prevent UVM from running fine in the future and cannot be used
// maliciously to leak information or otherwise derail UVM from its
// regular duties.
UVM_ASSERT_MSG_RELEASE(status == NV_OK, "Pushbuffer decryption failure: %s\n", nvstatusToString(status));
// Avoid reusing the bundle across multiple pushes
crypto_bundle->push_size = 0;
}
void uvm_pushbuffer_mark_completed(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo)
{
uvm_pushbuffer_chunk_t *chunk;
uvm_push_info_t *push_info = gpfifo->push_info;
bool need_to_update_chunk = false;
uvm_push_info_t *push_info = gpfifo->push_info;
uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
UVM_ASSERT(gpfifo->type == UVM_GPFIFO_ENTRY_TYPE_NORMAL);
chunk = gpfifo_to_chunk(pushbuffer, gpfifo);
if (push_info->on_complete != NULL)
if (push_info->on_complete != NULL) {
decrypt_push(channel, gpfifo);
push_info->on_complete(push_info->on_complete_data);
push_info->on_complete = NULL;
push_info->on_complete_data = NULL;
push_info->on_complete = NULL;
push_info->on_complete_data = NULL;
}
uvm_spin_lock(&pushbuffer->lock);

14
kernel-open/nvidia-uvm/uvm_pushbuffer.h
View File

@@ -161,22 +161,22 @@
// * WFI: 8B
// Total: 64B
//
// Push space needed for secure work launch is 224B. The push is constructed
// Push space needed for secure work launch is 364B. The push is constructed
// in 'internal_channel_submit_work_indirect' and 'uvm_channel_end_push'
// * CE decrypt (of indirect PB): 56B
// * 2*semaphore release (indirect GPFIFO entry): 2*24B
// * memset_8 (indirect GPFIFO entry): 44B
// * semaphore release (indirect GPPUT): 24B
// * semaphore release (indirect doorbell): 24B
// Appendix added in 'uvm_channel_end_push':
// * semaphore release (WLC tracking): 168B
// * semaphore increment (memcopy): 24B
// * semaphore release (payload): 24B
// * notifier memset: 40B
// * payload encryption: 64B
// * notifier memset: 40B
// * semaphore increment (LCIC GPPUT): 24B
// * semaphore release (LCIC doorbell): 24B
// Total: 368B
#define UVM_MAX_WLC_PUSH_SIZE (368)
// Total: 364B
#define UVM_MAX_WLC_PUSH_SIZE (364)
// Push space needed for static LCIC schedule, as initialized in
// 'setup_lcic_schedule':
@@ -184,7 +184,7 @@
// * semaphore increment (WLC GPPUT): 24B
// * semaphore increment (WLC GPPUT): 24B
// * semaphore increment (LCIC tracking): 160B
// * semaphore increment (memcopy): 24B
// * semaphore increment (payload): 24B
// * notifier memcopy: 36B
// * payload encryption: 64B
// * notifier memcopy: 36B
@@ -258,7 +258,7 @@ NV_STATUS uvm_pushbuffer_begin_push(uvm_pushbuffer_t *pushbuffer, uvm_push_t *pu
// Complete a pending push
// Updates the chunk state the pending push used
void uvm_pushbuffer_mark_completed(uvm_pushbuffer_t *pushbuffer, uvm_gpfifo_entry_t *gpfifo);
void uvm_pushbuffer_mark_completed(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo);
// Get the GPU VA for an ongoing push
NvU64 uvm_pushbuffer_get_gpu_va_for_push(uvm_pushbuffer_t *pushbuffer, uvm_push_t *push);

83
kernel-open/nvidia-uvm/uvm_sec2_test.c
View File

@@ -213,6 +213,7 @@ done:
typedef enum
{
MEM_ALLOC_TYPE_SYSMEM_DMA,
MEM_ALLOC_TYPE_SYSMEM_PROTECTED,
MEM_ALLOC_TYPE_VIDMEM_PROTECTED
} mem_alloc_type_t;
@@ -269,14 +270,20 @@ static NV_STATUS alloc_and_init_mem(uvm_gpu_t *gpu, uvm_mem_t **mem, size_t size
*mem = NULL;
if (type == MEM_ALLOC_TYPE_VIDMEM_PROTECTED) {
TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem_protected(size, gpu, mem));
TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem(size, gpu, mem));
TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
TEST_NV_CHECK_GOTO(ce_memset_gpu(gpu, *mem, size, 0xdead), err);
}
else {
TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem_dma(size, gpu, NULL, mem));
if (type == MEM_ALLOC_TYPE_SYSMEM_DMA) {
TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem_dma(size, gpu, NULL, mem));
TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
}
else {
TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem(size, NULL, mem));
}
TEST_NV_CHECK_GOTO(uvm_mem_map_cpu_kernel(*mem), err);
TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
write_range_cpu(*mem, size, 0xdeaddead);
}
@@ -341,9 +348,9 @@ static NV_STATUS cpu_decrypt(uvm_channel_t *channel,
return NV_OK;
}
// gpu_encrypt uses a secure CE for encryption (instead of SEC2). SEC2 does not
// support encryption. The following function is copied from uvm_ce_test.c and
// adapted to SEC2 tests.
// gpu_encrypt uses the Copy Engine for encryption, instead of SEC2. SEC2 does
// not support encryption. The following function is copied from uvm_ce_test.c
// and adapted to SEC2 tests.
static void gpu_encrypt(uvm_push_t *push,
uvm_mem_t *dst_mem,
uvm_mem_t *src_mem,
@@ -405,48 +412,6 @@ static void gpu_decrypt(uvm_push_t *push,
}
}
// This test only uses sysmem so that we can use the CPU for encryption and SEC2
// for decryption, i.e., the test doesn't depend on any other GPU engine for
// the encryption operation (refer to test_cpu_to_gpu_roundtrip()). This is not
// how SEC2 is used in the driver. The intended SEC2 usage is to decrypt from
// unprotected sysmem to protected vidmem, which is tested in
// test_cpu_to_gpu_roundtrip().
static NV_STATUS test_cpu_to_gpu_sysmem(uvm_gpu_t *gpu, size_t copy_size, size_t size)
{
NV_STATUS status = NV_OK;
uvm_mem_t *src_plain = NULL;
uvm_mem_t *cipher = NULL;
uvm_mem_t *dst_plain = NULL;
uvm_mem_t *auth_tag_mem = NULL;
size_t auth_tag_buffer_size = (size / copy_size) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
uvm_push_t push;
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &auth_tag_mem, auth_tag_buffer_size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
write_range_cpu(src_plain, size, uvm_get_stale_thread_id());
write_range_cpu(dst_plain, size, 0xA5A5A5A5);
TEST_NV_CHECK_GOTO(uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_SEC2, &push, "enc(cpu)_dec(gpu)"), out);
cpu_encrypt(push.channel, cipher, src_plain, auth_tag_mem, size, copy_size);
gpu_decrypt(&push, dst_plain, cipher, auth_tag_mem, size, copy_size);
uvm_push_end_and_wait(&push);
TEST_CHECK_GOTO(mem_match(src_plain, dst_plain), out);
out:
uvm_mem_free(auth_tag_mem);
uvm_mem_free(cipher);
uvm_mem_free(dst_plain);
uvm_mem_free(src_plain);
return status;
}
// This test depends on the CE for the encryption, so we assume tests from
// uvm_ce_test.c have successfully passed.
static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, size_t size)
@@ -461,19 +426,16 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, siz
size_t auth_tag_buffer_size = (size / copy_size) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
uvm_push_t push;
UvmCslIv *decrypt_iv;
uvm_tracker_t tracker;
decrypt_iv = uvm_kvmalloc_zero((size / copy_size) * sizeof(UvmCslIv));
if (!decrypt_iv)
return NV_ERR_NO_MEMORY;
uvm_tracker_init(&tracker);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_PROTECTED), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain, size, MEM_ALLOC_TYPE_VIDMEM_PROTECTED), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain_cpu, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain_cpu, size, MEM_ALLOC_TYPE_SYSMEM_PROTECTED), out);
TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &auth_tag_mem, auth_tag_buffer_size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
write_range_cpu(src_plain, size, uvm_get_stale_thread_id());
@@ -483,14 +445,12 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, siz
cpu_encrypt(push.channel, src_cipher, src_plain, auth_tag_mem, size, copy_size);
gpu_decrypt(&push, dst_plain, src_cipher, auth_tag_mem, size, copy_size);
uvm_push_end(&push);
TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, &push), out);
// Wait for SEC2 before launching the CE part.
// SEC2 is only allowed to release semaphores in unprotected sysmem,
// and CE can only acquire semaphores in protected vidmem.
TEST_NV_CHECK_GOTO(uvm_push_end_and_wait(&push), out);
TEST_NV_CHECK_GOTO(uvm_push_begin_acquire(gpu->channel_manager,
UVM_CHANNEL_TYPE_GPU_TO_CPU,
&tracker,
&push,
"enc(gpu)_dec(cpu)"),
TEST_NV_CHECK_GOTO(uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_TO_CPU, &push, "enc(gpu)_dec(cpu)"),
out);
gpu_encrypt(&push, dst_cipher, dst_plain, decrypt_iv, auth_tag_mem, size, copy_size);
@@ -521,8 +481,6 @@ out:
uvm_kvfree(decrypt_iv);
uvm_tracker_deinit(&tracker);
return status;
}
@@ -545,7 +503,6 @@ static NV_STATUS test_encryption_decryption(uvm_gpu_t *gpu)
UVM_ASSERT(size % copy_sizes[i] == 0);
TEST_NV_CHECK_RET(test_cpu_to_gpu_sysmem(gpu, copy_sizes[i], size));
TEST_NV_CHECK_RET(test_cpu_to_gpu_roundtrip(gpu, copy_sizes[i], size));
}

44
kernel-open/nvidia-uvm/uvm_tools.c
View File

@@ -229,6 +229,24 @@ static void unmap_user_pages(struct page **pages, void *addr, NvU64 size)
uvm_kvfree(pages);
}
// This must be called with the mmap_lock held in read mode or better.
static NV_STATUS check_vmas(struct mm_struct *mm, NvU64 start_va, NvU64 size)
{
struct vm_area_struct *vma;
NvU64 addr = start_va;
NvU64 region_end = start_va + size;
do {
vma = find_vma(mm, addr);
if (!vma || !(addr >= vma->vm_start) || uvm_file_is_nvidia_uvm(vma->vm_file))
return NV_ERR_INVALID_ARGUMENT;
addr = vma->vm_end;
} while (addr < region_end);
return NV_OK;
}
// Map virtual memory of data from [user_va, user_va + size) of current process into kernel.
// Sets *addr to kernel mapping and *pages to the array of struct pages that contain the memory.
static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct page ***pages)
@@ -237,7 +255,6 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
long ret = 0;
long num_pages;
long i;
struct vm_area_struct **vmas = NULL;
*addr = NULL;
*pages = NULL;
@@ -254,22 +271,30 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
goto fail;
}
vmas = uvm_kvmalloc(sizeof(struct vm_area_struct *) * num_pages);
if (vmas == NULL) {
status = NV_ERR_NO_MEMORY;
// Although uvm_down_read_mmap_lock() is preferable due to its participation
// in the UVM lock dependency tracker, it cannot be used here. That's
// because pin_user_pages() may fault in HMM pages which are GPU-resident.
// When that happens, the UVM page fault handler would record another
// mmap_read_lock() on the same thread as this one, leading to a false
// positive lock dependency report.
//
// Therefore, use the lower level nv_mmap_read_lock() here.
nv_mmap_read_lock(current->mm);
status = check_vmas(current->mm, user_va, size);
if (status != NV_OK) {
nv_mmap_read_unlock(current->mm);
goto fail;
}
nv_mmap_read_lock(current->mm);
ret = NV_PIN_USER_PAGES(user_va, num_pages, FOLL_WRITE, *pages, vmas);
ret = NV_PIN_USER_PAGES(user_va, num_pages, FOLL_WRITE, *pages, NULL);
nv_mmap_read_unlock(current->mm);
if (ret != num_pages) {
status = NV_ERR_INVALID_ARGUMENT;
goto fail;
}
for (i = 0; i < num_pages; i++) {
if (page_count((*pages)[i]) > MAX_PAGE_COUNT || uvm_file_is_nvidia_uvm(vmas[i]->vm_file)) {
if (page_count((*pages)[i]) > MAX_PAGE_COUNT) {
status = NV_ERR_INVALID_ARGUMENT;
goto fail;
}
@@ -279,15 +304,12 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
if (*addr == NULL)
goto fail;
uvm_kvfree(vmas);
return NV_OK;
fail:
if (*pages == NULL)
return status;
uvm_kvfree(vmas);
if (ret > 0)
uvm_put_user_pages_dirty(*pages, ret);
else if (ret < 0)

8
kernel-open/nvidia-uvm/uvm_tracker_test.c
View File

@@ -69,6 +69,14 @@ static NV_STATUS test_tracker_completion(uvm_va_space_t *va_space)
gpu = uvm_va_space_find_first_gpu(va_space);
TEST_CHECK_RET(gpu != NULL);
// TODO: Bug 4008734: [UVM][HCC] Extend secure tracking semaphore mechanism
// to all semaphore
// This test allocates semaphore in vidmem and then releases it from the CPU
// SEC2 channels cannot target semaphores in vidmem. Moreover, CPU cannot
// directly release values to vidmem for CE channels.
if (uvm_conf_computing_mode_enabled(gpu))
return NV_OK;
TEST_NV_CHECK_RET(uvm_gpu_semaphore_alloc(gpu->semaphore_pool, &sema));
uvm_tracker_init(&tracker);

57
kernel-open/nvidia-uvm/uvm_va_block.c
View File

@@ -2083,12 +2083,6 @@ static uvm_processor_id_t block_page_get_closest_resident_in_mask(uvm_va_block_t
return id;
}
// HMM va_blocks don't know if a page is CPU resident until either
// migrate_vma_setup() or hmm_range_fault() is called. If a page isn't
// resident anywhere, assume it is CPU resident.
if (uvm_va_block_is_hmm(va_block))
return UVM_ID_CPU;
return UVM_ID_INVALID;
}
@@ -2888,7 +2882,7 @@ static uvm_va_block_region_t block_phys_contig_region(uvm_va_block_t *block,
{
if (UVM_ID_IS_CPU(resident_id)) {
uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_chunk_for_page(block, page_index);
return uvm_va_block_region(page_index, page_index + uvm_cpu_chunk_num_pages(chunk));
return uvm_cpu_chunk_block_region(block, chunk, page_index);
}
else {
uvm_chunk_size_t chunk_size;
@@ -3061,7 +3055,7 @@ static NV_STATUS conf_computing_copy_pages_finish(uvm_va_block_t *block,
void *auth_tag_buffer_base = uvm_mem_get_cpu_addr_kernel(dma_buffer->auth_tag);
void *staging_buffer_base = uvm_mem_get_cpu_addr_kernel(dma_buffer->alloc);
UVM_ASSERT(uvm_channel_is_secure(push->channel));
UVM_ASSERT(uvm_conf_computing_mode_enabled(push->gpu));
if (UVM_ID_IS_GPU(copy_state->dst.id))
return NV_OK;
@@ -3112,7 +3106,7 @@ static void block_copy_push(uvm_va_block_t *block,
uvm_push_set_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_NONE);
if (uvm_channel_is_secure(push->channel)) {
if (uvm_conf_computing_mode_enabled(gpu)) {
if (UVM_ID_IS_CPU(copy_state->src.id))
conf_computing_block_copy_push_cpu_to_gpu(block, copy_state, region, push);
else
@@ -3140,19 +3134,18 @@ static NV_STATUS block_copy_end_push(uvm_va_block_t *block,
// at that point.
uvm_push_end(push);
if ((push_status == NV_OK) && uvm_channel_is_secure(push->channel))
if ((push_status == NV_OK) && uvm_conf_computing_mode_enabled(push->gpu))
push_status = conf_computing_copy_pages_finish(block, copy_state, push);
tracker_status = uvm_tracker_add_push_safe(copy_tracker, push);
if (push_status == NV_OK)
push_status = tracker_status;
if (uvm_channel_is_secure(push->channel)) {
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
if (uvm_conf_computing_mode_enabled(push->gpu)) {
uvm_tracker_t local_tracker = UVM_TRACKER_INIT();
uvm_tracker_overwrite_with_push(&local_tracker, push);
uvm_conf_computing_dma_buffer_free(&gpu->conf_computing.dma_buffer_pool,
uvm_conf_computing_dma_buffer_free(&push->gpu->conf_computing.dma_buffer_pool,
copy_state->dma_buffer,
&local_tracker);
copy_state->dma_buffer = NULL;
@@ -7189,6 +7182,7 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
}
static void map_get_allowed_destinations(uvm_va_block_t *block,
uvm_va_block_context_t *va_block_context,
const uvm_va_policy_t *policy,
uvm_processor_id_t id,
uvm_processor_mask_t *allowed_mask)
@@ -7200,7 +7194,10 @@ static void map_get_allowed_destinations(uvm_va_block_t *block,
uvm_processor_mask_zero(allowed_mask);
uvm_processor_mask_set(allowed_mask, policy->preferred_location);
}
else if ((uvm_va_policy_is_read_duplicate(policy, va_space) || uvm_id_equal(policy->preferred_location, id)) &&
else if ((uvm_va_policy_is_read_duplicate(policy, va_space) ||
(uvm_id_equal(policy->preferred_location, id) &&
!is_uvm_fault_force_sysmem_set() &&
!uvm_hmm_must_use_sysmem(block, va_block_context))) &&
uvm_va_space_processor_has_memory(va_space, id)) {
// When operating under read-duplication we should only map the local
// processor to cause fault-and-duplicate of remote pages.
@@ -7285,7 +7282,7 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
// Map per resident location so we can more easily detect physically-
// contiguous mappings.
map_get_allowed_destinations(va_block, va_block_context->policy, id, &allowed_destinations);
map_get_allowed_destinations(va_block, va_block_context, va_block_context->policy, id, &allowed_destinations);
for_each_closest_id(resident_id, &allowed_destinations, id, va_space) {
if (UVM_ID_IS_CPU(id)) {
@@ -9614,15 +9611,9 @@ static uvm_prot_t compute_new_permission(uvm_va_block_t *va_block,
if (uvm_processor_mask_empty(&revoke_processors))
new_prot = UVM_PROT_READ_WRITE;
}
if (logical_prot == UVM_PROT_READ_WRITE_ATOMIC) {
// HMM allocations with logical read/write/atomic permission can be
// upgraded without notifying the driver so assume read/write/atomic
// even if the fault is only for reading.
if (new_prot == UVM_PROT_READ_WRITE ||
(UVM_ID_IS_CPU(fault_processor_id) && uvm_va_block_is_hmm(va_block))) {
if (uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(new_residency)], fault_processor_id))
new_prot = UVM_PROT_READ_WRITE_ATOMIC;
}
if (logical_prot == UVM_PROT_READ_WRITE_ATOMIC && new_prot == UVM_PROT_READ_WRITE) {
if (uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(new_residency)], fault_processor_id))
new_prot = UVM_PROT_READ_WRITE_ATOMIC;
}
return new_prot;
@@ -9859,8 +9850,6 @@ out:
return status == NV_OK ? tracker_status : status;
}
// TODO: Bug 1750144: check logical permissions from HMM to know what's the
// maximum allowed.
uvm_prot_t uvm_va_block_page_compute_highest_permission(uvm_va_block_t *va_block,
uvm_processor_id_t processor_id,
uvm_page_index_t page_index)
@@ -9937,14 +9926,18 @@ uvm_prot_t uvm_va_block_page_compute_highest_permission(uvm_va_block_t *va_block
// Exclude the processor for which the mapping protections are being computed
uvm_processor_mask_clear(&write_mappings, processor_id);
// At this point, any processor with atomic mappings either has native atomics support to the
// processor with the resident copy or has disabled system-wide atomics. If the requesting
// processor has disabled system-wide atomics or has native atomics to that processor, we can
// map with ATOMIC privileges. Likewise, if there are no other processors with WRITE or ATOMIC
// mappings, we can map with ATOMIC privileges.
// At this point, any processor with atomic mappings either has native
// atomics support to the processor with the resident copy or has
// disabled system-wide atomics. If the requesting processor has
// disabled system-wide atomics or has native atomics to that processor,
// we can map with ATOMIC privileges. Likewise, if there are no other
// processors with WRITE or ATOMIC mappings, we can map with ATOMIC
// privileges. For HMM, don't allow GPU atomic access to remote mapped
// system memory even if there are no write mappings since CPU access
// can be upgraded without notification.
if (!uvm_processor_mask_test(&va_space->system_wide_atomics_enabled_processors, processor_id) ||
uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(residency)], processor_id) ||
uvm_processor_mask_empty(&write_mappings)) {
(uvm_processor_mask_empty(&write_mappings) && !uvm_va_block_is_hmm(va_block))) {
return UVM_PROT_READ_WRITE_ATOMIC;
}

18
kernel-open/nvidia-uvm/uvm_va_space.c
View File

@@ -418,15 +418,6 @@ void uvm_va_space_destroy(uvm_va_space_t *va_space)
uvm_global_processor_mask_t retained_gpus;
LIST_HEAD(deferred_free_list);
// Normally we'd expect this to happen as part of uvm_mm_release()
// but if userspace never initialized uvm_mm_fd that won't happen.
// We don't have to take the va_space_mm spinlock and update state
// here because we know no other thread can be in or subsequently
// call uvm_api_mm_initialize successfully because the UVM
// file-descriptor has been released.
if (va_space->va_space_mm.state == UVM_VA_SPACE_MM_STATE_UNINITIALIZED)
uvm_va_space_mm_unregister(va_space);
// Remove the VA space from the global list before we start tearing things
// down so other threads can't see the VA space in a partially-valid state.
uvm_mutex_lock(&g_uvm_global.va_spaces.lock);
@@ -532,7 +523,14 @@ void uvm_va_space_destroy(uvm_va_space_t *va_space)
uvm_deferred_free_object_list(&deferred_free_list);
// MM FD teardown should already have destroyed va_space_mm
// Normally we'd expect this to happen as part of uvm_mm_release()
// but if userspace never initialized uvm_mm_fd that won't happen.
// We don't have to take the va_space_mm spinlock and update state
// here because we know no other thread can be in or subsequently
// call uvm_api_mm_initialize successfully because the UVM
// file-descriptor has been released.
if (va_space->va_space_mm.state == UVM_VA_SPACE_MM_STATE_UNINITIALIZED)
uvm_va_space_mm_unregister(va_space);
UVM_ASSERT(!uvm_va_space_mm_alive(&va_space->va_space_mm));
uvm_mutex_lock(&g_uvm_global.global_lock);

48
kernel-open/nvidia-uvm/uvm_volta_fault_buffer.c
View File

@@ -25,7 +25,8 @@
#include "uvm_global.h"
#include "uvm_gpu.h"
#include "uvm_hal.h"
#include "uvm_push.h"
#include "uvm_conf_computing.h"
#include "nv_uvm_types.h"
#include "hwref/volta/gv100/dev_fault.h"
#include "hwref/volta/gv100/dev_fb.h"
#include "clc369.h"
@@ -246,6 +247,20 @@ static NvU32 *get_fault_buffer_entry(uvm_parent_gpu_t *parent_gpu, NvU32 index)
return fault_entry;
}
// See uvm_pascal_fault_buffer.c::get_fault_buffer_entry_metadata
static UvmFaultMetadataPacket *get_fault_buffer_entry_metadata(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UvmFaultMetadataPacket *fault_entry_metadata;
UVM_ASSERT(index < parent_gpu->fault_buffer_info.replayable.max_faults);
UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
fault_entry_metadata = parent_gpu->fault_buffer_info.rm_info.replayable.bufferMetadata;
UVM_ASSERT(fault_entry_metadata != NULL);
return fault_entry_metadata + index;
}
static void parse_fault_entry_common(uvm_parent_gpu_t *parent_gpu,
NvU32 *fault_entry,
uvm_fault_buffer_entry_t *buffer_entry)
@@ -323,24 +338,47 @@ static void parse_fault_entry_common(uvm_parent_gpu_t *parent_gpu,
UVM_ASSERT_MSG(replayable_fault_enabled, "Fault with REPLAYABLE_FAULT_EN bit unset\n");
}
void uvm_hal_volta_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
NV_STATUS uvm_hal_volta_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
{
fault_buffer_entry_c369_t entry;
NvU32 *fault_entry;
BUILD_BUG_ON(NVC369_BUF_SIZE > UVM_GPU_MMU_MAX_FAULT_PACKET_SIZE);
BUILD_BUG_ON(sizeof(entry) > UVM_GPU_MMU_MAX_FAULT_PACKET_SIZE);
// Valid bit must be set before this function is called
UVM_ASSERT(parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, index));
fault_entry = get_fault_buffer_entry(parent_gpu, index);
// When Confidential Computing is enabled, faults are encrypted by RM, so
// they need to be decrypted before they can be parsed
if (!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu)) {
NV_STATUS status;
UvmFaultMetadataPacket *fault_entry_metadata = get_fault_buffer_entry_metadata(parent_gpu, index);
status = uvm_conf_computing_fault_decrypt(parent_gpu,
&entry,
fault_entry,
fault_entry_metadata->authTag,
fault_entry_metadata->valid);
if (status != NV_OK) {
uvm_global_set_fatal_error(status);
return status;
}
fault_entry = (NvU32 *) &entry;
}
parse_fault_entry_common(parent_gpu, fault_entry, buffer_entry);
UVM_ASSERT(buffer_entry->is_replayable);
// Automatically clear valid bit for the entry in the fault buffer
parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, index);
return NV_OK;
}
void uvm_hal_volta_fault_buffer_parse_non_replayable_entry(uvm_parent_gpu_t *parent_gpu,