NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-07 16:49:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

560.28.03

Browse Source
This commit is contained in:
Gaurav Juvekar
2024-07-19 15:45:15 -07:00
parent 5fdf5032fb
commit 448d5cc656
859 changed files with 165424 additions and 91129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

623
kernel-open/nvidia-uvm/uvm_va_block.c
View File

@@ -725,8 +725,9 @@ bool uvm_va_block_cpu_is_region_resident_on(uvm_va_block_t *va_block, int nid, u
}
// Return the preferred NUMA node ID for the block's policy.
// If the preferred node ID is NUMA_NO_NODE, the current NUMA node ID
// is returned.
// If the preferred node ID is NUMA_NO_NODE, the nearest NUMA node ID
// with memory is returned. In most cases, this should be the current
// NUMA node.
static int uvm_va_block_context_get_node(uvm_va_block_context_t *va_block_context)
{
if (va_block_context->make_resident.dest_nid != NUMA_NO_NODE)
@@ -1410,102 +1411,6 @@ error:
return status;
}
static NV_STATUS block_sysmem_mappings_add_gpu_chunk(uvm_va_block_t *block,
uvm_gpu_t *local_gpu,
uvm_gpu_chunk_t *chunk,
uvm_gpu_t *accessing_gpu)
{
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&local_gpu->pmm, chunk, accessing_gpu);
return uvm_pmm_sysmem_mappings_add_gpu_chunk_mapping(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr,
block->start + chunk->va_block_page_index * PAGE_SIZE,
uvm_gpu_chunk_get_size(chunk),
block,
local_gpu->id);
}
static void block_sysmem_mappings_remove_gpu_chunk(uvm_gpu_t *local_gpu,
uvm_gpu_chunk_t *chunk,
uvm_gpu_t *accessing_gpu)
{
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&local_gpu->pmm, chunk, accessing_gpu);
uvm_pmm_sysmem_mappings_remove_gpu_chunk_mapping(&accessing_gpu->pmm_reverse_sysmem_mappings, peer_addr);
}
static NV_STATUS block_gpu_map_all_chunks_indirect_peer(uvm_va_block_t *block,
uvm_gpu_t *local_gpu,
uvm_gpu_t *accessing_gpu)
{
uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(block, local_gpu->id);
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
size_t num_chunks, i;
NV_STATUS status;
UVM_ASSERT(uvm_processor_mask_test(&va_space->indirect_peers[uvm_id_value(local_gpu->id)],
accessing_gpu->id));
// If no chunks are allocated currently, the mappings will be created later
// at chunk allocation.
if (!gpu_state || !gpu_state->chunks)
return NV_OK;
num_chunks = block_num_gpu_chunks(block, local_gpu);
for (i = 0; i < num_chunks; i++) {
uvm_gpu_chunk_t *chunk = gpu_state->chunks[i];
if (!chunk)
continue;
status = uvm_pmm_gpu_indirect_peer_map(&local_gpu->pmm, chunk, accessing_gpu);
if (status != NV_OK)
goto error;
status = block_sysmem_mappings_add_gpu_chunk(block, local_gpu, chunk, accessing_gpu);
if (status != NV_OK)
goto error;
}
return NV_OK;
error:
while (i-- > 0) {
uvm_gpu_chunk_t *chunk = gpu_state->chunks[i];
if (chunk) {
// Indirect peer mappings are removed lazily by PMM, so if an error
// occurs the mappings established above will be removed when the
// chunk is freed later on. We only need to remove the sysmem
// reverse mappings.
block_sysmem_mappings_remove_gpu_chunk(local_gpu, chunk, accessing_gpu);
}
}
return status;
}
// Mappings for indirect peers are removed lazily by PMM, but we need to remove
// the entries from the reverse map.
static void block_gpu_unmap_all_chunks_indirect_peer(uvm_va_block_t *block,
uvm_gpu_t *local_gpu,
uvm_gpu_t *accessing_gpu)
{
uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(block, local_gpu->id);
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
size_t num_chunks, i;
UVM_ASSERT(uvm_processor_mask_test(&va_space->indirect_peers[uvm_id_value(local_gpu->id)],
accessing_gpu->id));
// Exit if no chunks are allocated currently.
if (!gpu_state || !gpu_state->chunks)
return;
num_chunks = block_num_gpu_chunks(block, local_gpu);
for (i = 0; i < num_chunks; i++) {
uvm_gpu_chunk_t *chunk = gpu_state->chunks[i];
if (chunk)
block_sysmem_mappings_remove_gpu_chunk(local_gpu, chunk, accessing_gpu);
}
}
// Retrieves the gpu_state for the given GPU. The returned pointer is
// internally managed and will be allocated (and freed) automatically,
// rather than by the caller.
@@ -1628,65 +1533,6 @@ void uvm_va_block_remove_cpu_chunks(uvm_va_block_t *va_block, uvm_va_block_regio
uvm_processor_mask_clear(&va_block->resident, UVM_ID_CPU);
}
// Create physical mappings to allow other GPUs to access this chunk.
static NV_STATUS block_map_indirect_peers_to_gpu_chunk(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_gpu_chunk_t *chunk)
{
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_gpu_t *accessing_gpu, *remove_gpu;
NV_STATUS status;
// Unlike uvm_va_block_map_cpu_chunk_on_gpus, this function isn't called on
// the eviction path, so we can assume that the VA space is locked.
//
// TODO: Bug 2007346: In the future we may want to enable eviction to peers,
// meaning we may need to allocate peer memory and map it on the
// eviction path. That will require making sure that peers can't be
// enabled or disabled either in the VA space or globally within this
// function.
uvm_assert_rwsem_locked(&va_space->lock);
uvm_assert_mutex_locked(&block->lock);
for_each_va_space_gpu_in_mask(accessing_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
status = uvm_pmm_gpu_indirect_peer_map(&gpu->pmm, chunk, accessing_gpu);
if (status != NV_OK)
goto error;
status = block_sysmem_mappings_add_gpu_chunk(block, gpu, chunk, accessing_gpu);
if (status != NV_OK)
goto error;
}
return NV_OK;
error:
for_each_va_space_gpu_in_mask(remove_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
if (remove_gpu == accessing_gpu)
break;
// Indirect peer mappings are removed lazily by PMM, so if an error
// occurs the mappings established above will be removed when the
// chunk is freed later on. We only need to remove the sysmem
// reverse mappings.
block_sysmem_mappings_remove_gpu_chunk(gpu, chunk, remove_gpu);
}
return status;
}
static void block_unmap_indirect_peers_from_gpu_chunk(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_gpu_chunk_t *chunk)
{
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_gpu_t *peer_gpu;
uvm_assert_rwsem_locked(&va_space->lock);
uvm_assert_mutex_locked(&block->lock);
// Indirect peer mappings are removed lazily by PMM, so we only need to
// remove the sysmem reverse mappings.
for_each_va_space_gpu_in_mask(peer_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)])
block_sysmem_mappings_remove_gpu_chunk(gpu, chunk, peer_gpu);
}
// Mark a CPU page as dirty.
static void block_mark_cpu_page_dirty(uvm_va_block_t *block, uvm_page_index_t page_index, int nid)
{
@@ -1711,33 +1557,6 @@ static bool block_cpu_page_is_dirty(uvm_va_block_t *block, uvm_page_index_t page
return uvm_cpu_chunk_is_dirty(chunk, page_index - chunk_region.first);
}
static NV_STATUS block_alloc_cpu_chunk_inject_error(uvm_va_block_t *block,
uvm_chunk_size_t alloc_size,
uvm_cpu_chunk_alloc_flags_t flags,
int nid,
uvm_cpu_chunk_t **chunk)
{
uvm_va_block_test_t *block_test = uvm_va_block_get_test(block);
if (block_test) {
// Return out of memory error if the tests have requested it. As opposed
// to other error injection settings, this one fails N times and then
// succeeds.
// TODO: Bug 3701182: This will print a warning in Linux kernels newer
// than 5.16.0-rc1+.
if (block_test->inject_cpu_pages_allocation_error_count) {
if (block_test->inject_cpu_pages_allocation_error_count != ~(NvU32)0)
block_test->inject_cpu_pages_allocation_error_count--;
return NV_ERR_NO_MEMORY;
}
if (block_test->cpu_chunk_allocation_actual_id != NUMA_NO_NODE)
nid = block_test->cpu_chunk_allocation_actual_id;
}
return uvm_cpu_chunk_alloc(alloc_size, flags, nid, chunk);
}
// Allocate a CPU chunk with the given properties. This may involve retrying if
// allocations fail. Allocating larger chunk sizes takes priority over
// allocating on the specified node in the following manner:
@@ -1754,19 +1573,36 @@ static NV_STATUS block_alloc_cpu_chunk(uvm_va_block_t *block,
int nid,
uvm_cpu_chunk_t **chunk)
{
uvm_va_block_test_t *block_test = uvm_va_block_get_test(block);
NV_STATUS status = NV_ERR_NO_MEMORY;
uvm_chunk_size_t alloc_size;
bool numa_fallback = false;
if (block_test) {
// Return out of memory error if the tests have requested it. As opposed
// to other error injection settings, this one fails N times and then
// succeeds.
// TODO: Bug 3701182: This will print a warning in Linux kernels newer
// than 5.16.0-rc1+.
if (block_test->inject_cpu_chunk_allocation_error_count) {
if (block_test->inject_cpu_chunk_allocation_error_count != ~(NvU32)0)
block_test->inject_cpu_chunk_allocation_error_count--;
return NV_ERR_NO_MEMORY;
}
if (block_test->cpu_chunk_allocation_actual_id != NUMA_NO_NODE)
nid = block_test->cpu_chunk_allocation_actual_id;
}
for_each_chunk_size_rev(alloc_size, cpu_allocation_sizes) {
status = block_alloc_cpu_chunk_inject_error(block, alloc_size, flags, nid, chunk);
status = uvm_cpu_chunk_alloc(alloc_size, flags, nid, chunk);
if (status == NV_OK)
break;
if (flags & UVM_CPU_CHUNK_ALLOC_FLAGS_STRICT) {
flags &= ~UVM_CPU_CHUNK_ALLOC_FLAGS_STRICT;
numa_fallback = true;
status = block_alloc_cpu_chunk_inject_error(block, alloc_size, flags, NUMA_NO_NODE, chunk);
status = uvm_cpu_chunk_alloc(alloc_size, flags, NUMA_NO_NODE, chunk);
if (status == NV_OK)
break;
}
@@ -2066,6 +1902,7 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block,
uvm_page_mask_t *allocated_mask;
uvm_cpu_chunk_alloc_flags_t alloc_flags = UVM_CPU_CHUNK_ALLOC_FLAGS_NONE;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
const uvm_va_policy_t *policy = uvm_va_policy_get_region(block, populate_region);
uvm_page_index_t page_index;
uvm_gpu_id_t id;
int preferred_nid = block_context->make_resident.dest_nid;
@@ -2073,6 +1910,10 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block,
if (block_test && block_test->cpu_chunk_allocation_target_id != NUMA_NO_NODE)
preferred_nid = block_test->cpu_chunk_allocation_target_id;
// If the VA range has a preferred NUMA node, use it.
if (preferred_nid == NUMA_NO_NODE)
preferred_nid = policy->preferred_nid;
// TODO: Bug 4158598: Using NUMA_NO_NODE for staging allocations is sub-optimal.
if (preferred_nid != NUMA_NO_NODE) {
uvm_va_block_cpu_node_state_t *node_state = block_node_state_get(block, preferred_nid);
@@ -2123,13 +1964,12 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block,
uvm_page_mask_t *node_pages_mask = &block_context->make_resident.node_pages_mask;
uvm_chunk_sizes_mask_t allocation_sizes;
if (uvm_page_mask_test(allocated_mask, page_index)) {
if (uvm_page_mask_test(allocated_mask, page_index) ||
uvm_va_block_cpu_is_page_resident_on(block, preferred_nid, page_index)) {
page_index = uvm_va_block_next_unset_page_in_mask(populate_region, allocated_mask, page_index) - 1;
continue;
}
UVM_ASSERT(!uvm_va_block_cpu_is_page_resident_on(block, preferred_nid, page_index));
allocation_sizes = block_calculate_largest_alloc_size(block,
page_index,
allocated_mask,
@@ -3129,17 +2969,13 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block,
// compile-time that it can store VA Block page indexes.
BUILD_BUG_ON(PAGES_PER_UVM_VA_BLOCK >= PAGE_SIZE);
status = block_map_indirect_peers_to_gpu_chunk(block, gpu, chunk);
if (status != NV_OK)
goto chunk_unmap;
if (block_test && block_test->inject_populate_error) {
block_test->inject_populate_error = false;
// Use NV_ERR_MORE_PROCESSING_REQUIRED to force a retry rather than
// causing a fatal OOM failure.
status = NV_ERR_MORE_PROCESSING_REQUIRED;
goto chunk_unmap_indirect_peers;
goto chunk_unmap;
}
// Record the used chunk so that it can be unpinned at the end of the whole
@@ -3149,9 +2985,6 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block,
return NV_OK;
chunk_unmap_indirect_peers:
block_unmap_indirect_peers_from_gpu_chunk(block, gpu, chunk);
chunk_unmap:
uvm_mmu_chunk_unmap(chunk, &block->tracker);
@@ -3326,8 +3159,9 @@ static uvm_gpu_chunk_t *block_phys_page_chunk(uvm_va_block_t *block, block_phys_
return chunk;
}
// Get the physical GPU address of a block's page from the POV of the specified GPU
// This is the address that should be used for making PTEs for the specified GPU.
// Get the physical GPU address of a block's page from the POV of the specified
// GPU. This is the address that should be used for making PTEs for the
// specified GPU.
static uvm_gpu_phys_address_t block_phys_page_address(uvm_va_block_t *block,
block_phys_page_t block_page,
uvm_gpu_t *gpu)
@@ -4033,7 +3867,7 @@ static NV_STATUS block_copy_pages(uvm_va_block_t *va_block,
UVM_ASSERT(dst_chunk);
UVM_ASSERT(uvm_cpu_chunk_get_size(src_chunk) >= uvm_va_block_region_size(region));
UVM_ASSERT(uvm_cpu_chunk_get_size(src_chunk) <= uvm_cpu_chunk_get_size(dst_chunk));
UVM_ASSERT(uvm_va_block_region_size(region) <= uvm_cpu_chunk_get_size(dst_chunk));
// CPU-to-CPU copies using memcpy() don't have any inherent ordering with
// copies using GPU CEs. So, we have to make sure that all previously
@@ -5128,7 +4962,7 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
uvm_page_mask_t *dst_resident_mask;
uvm_page_mask_t *migrated_pages;
uvm_page_mask_t *staged_pages;
uvm_page_mask_t *first_touch_mask;
uvm_page_mask_t *scratch_residency_mask;
// TODO: Bug 3660922: need to implement HMM read duplication support.
UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
@@ -5147,6 +4981,10 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
uvm_assert_mutex_locked(&va_block->lock);
UVM_ASSERT(!uvm_va_block_is_dead(va_block));
scratch_residency_mask = kmem_cache_alloc(g_uvm_page_mask_cache, NV_UVM_GFP_FLAGS);
if (!scratch_residency_mask)
return NV_ERR_NO_MEMORY;
// For pages that are entering read-duplication we need to unmap remote
// mappings and revoke RW and higher access permissions.
//
@@ -5173,12 +5011,12 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
status = block_prep_read_duplicate_mapping(va_block, va_block_context, src_id, region, preprocess_page_mask);
if (status != NV_OK)
return status;
goto out;
}
status = block_populate_pages(va_block, va_block_retry, va_block_context, dest_id, region, page_mask);
if (status != NV_OK)
return status;
goto out;
status = block_copy_resident_pages(va_block,
va_block_context,
@@ -5188,22 +5026,17 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
prefetch_page_mask,
UVM_VA_BLOCK_TRANSFER_MODE_COPY);
if (status != NV_OK)
return status;
goto out;
// Pages that weren't resident anywhere else were populated at the
// destination directly. Mark them as resident now, since there were no
// errors from block_copy_resident_pages() above.
// Note that va_block_context->scratch_page_mask is passed to
// block_copy_set_first_touch_residency() which is generally unsafe but in
// this case, block_copy_set_first_touch_residency() copies page_mask
// before scratch_page_mask could be clobbered.
migrated_pages = &va_block_context->make_resident.pages_migrated;
first_touch_mask = &va_block_context->scratch_page_mask;
uvm_page_mask_init_from_region(first_touch_mask, region, page_mask);
uvm_page_mask_andnot(first_touch_mask, first_touch_mask, migrated_pages);
uvm_page_mask_init_from_region(scratch_residency_mask, region, page_mask);
uvm_page_mask_andnot(scratch_residency_mask, scratch_residency_mask, migrated_pages);
if (!uvm_page_mask_empty(first_touch_mask))
block_copy_set_first_touch_residency(va_block, va_block_context, dest_id, region, first_touch_mask);
if (!uvm_page_mask_empty(scratch_residency_mask))
block_copy_set_first_touch_residency(va_block, va_block_context, dest_id, region, scratch_residency_mask);
staged_pages = &va_block_context->make_resident.pages_staged;
if (!UVM_ID_IS_CPU(dest_id) && !uvm_page_mask_empty(staged_pages)) {
@@ -5215,6 +5048,18 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
if (!uvm_page_mask_empty(migrated_pages)) {
if (UVM_ID_IS_CPU(dest_id)) {
// Check if the CPU is already in the resident set of processors.
// We need to do this since we can't have multiple NUMA nodes with
// resident pages.
// If any of the migrate pages were already resident on the CPU, the
// residency has to be switched to the destination NUMA node.
if (uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU) &&
uvm_page_mask_and(scratch_residency_mask,
uvm_va_block_resident_mask_get(va_block, UVM_ID_CPU, NUMA_NO_NODE),
migrated_pages)) {
uvm_va_block_cpu_clear_resident_all_chunks(va_block, va_block_context, scratch_residency_mask);
}
uvm_va_block_cpu_set_resident_all_chunks(va_block, va_block_context, migrated_pages);
}
else {
@@ -5243,7 +5088,9 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
// Check state of all chunks after residency change.
// TODO: Bug 4207783: Check both CPU and GPU chunks.
UVM_ASSERT(block_check_cpu_chunks(va_block));
return NV_OK;
out:
kmem_cache_free(g_uvm_page_mask_cache, scratch_residency_mask);
return status;
}
// Looks up the current CPU mapping state of page from the
@@ -5322,35 +5169,6 @@ static bool block_has_valid_mapping_cpu(uvm_va_block_t *block, uvm_va_block_regi
return true;
}
static bool block_check_chunk_indirect_peers(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_gpu_chunk_t *chunk)
{
uvm_gpu_t *accessing_gpu;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
if (!uvm_pmm_sysmem_mappings_indirect_supported())
return true;
for_each_va_space_gpu_in_mask(accessing_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&gpu->pmm, chunk, accessing_gpu);
uvm_reverse_map_t reverse_map;
size_t num_mappings;
num_mappings = uvm_pmm_sysmem_mappings_dma_to_virt(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr,
uvm_gpu_chunk_get_size(chunk),
&reverse_map,
1);
UVM_ASSERT(num_mappings == 1);
UVM_ASSERT(reverse_map.va_block == block);
UVM_ASSERT(reverse_map.region.first == chunk->va_block_page_index);
UVM_ASSERT(uvm_va_block_region_size(reverse_map.region) == uvm_gpu_chunk_get_size(chunk));
uvm_va_block_release_no_destroy(reverse_map.va_block);
}
return true;
}
// Sanity check the given GPU's chunks array
static bool block_check_gpu_chunks(uvm_va_block_t *block, uvm_gpu_id_t id)
{
@@ -5408,8 +5226,6 @@ static bool block_check_gpu_chunks(uvm_va_block_t *block, uvm_gpu_id_t id)
UVM_ASSERT(chunk->va_block == block);
UVM_ASSERT(chunk->va_block_page_index == page_index);
UVM_ASSERT(block_check_chunk_indirect_peers(block, gpu, chunk));
}
page_index += chunk_size / PAGE_SIZE;
@@ -5528,13 +5344,15 @@ static bool block_check_mappings_page(uvm_va_block_t *block,
*block->read_duplicated_pages.bitmap);
// Test read_duplicated_pages mask
UVM_ASSERT_MSG((uvm_processor_mask_get_count(resident_processors) <= 1 &&
!uvm_page_mask_test(&block->read_duplicated_pages, page_index)) ||
(uvm_processor_mask_get_count(resident_processors) > 1 &&
uvm_page_mask_test(&block->read_duplicated_pages, page_index)),
UVM_ASSERT_MSG((!uvm_page_mask_test(&block->read_duplicated_pages, page_index) &&
uvm_processor_mask_get_count(resident_processors) <= 1) ||
(uvm_page_mask_test(&block->read_duplicated_pages, page_index) &&
uvm_processor_mask_get_count(resident_processors) >= 1),
"Resident: 0x%lx - Mappings R: 0x%lx W: 0x%lx A: 0x%lx - SWA: 0x%lx - RD: 0x%lx\n",
*resident_processors->bitmap,
*read_mappings->bitmap, *write_mappings->bitmap, *atomic_mappings->bitmap,
*read_mappings->bitmap,
*write_mappings->bitmap,
*atomic_mappings->bitmap,
*va_space->system_wide_atomics_enabled_processors.bitmap,
*block->read_duplicated_pages.bitmap);
@@ -5613,17 +5431,6 @@ static bool block_check_mappings_page(uvm_va_block_t *block,
*va_space->system_wide_atomics_enabled_processors.bitmap);
for_each_id_in_mask(id, read_mappings) {
UVM_ASSERT(uvm_processor_mask_test(&va_space->can_access[uvm_id_value(id)], residency));
if (uvm_processor_mask_test(&va_space->indirect_peers[uvm_id_value(residency)], id)) {
uvm_gpu_t *resident_gpu = uvm_va_space_get_gpu(va_space, residency);
uvm_gpu_t *mapped_gpu = uvm_va_space_get_gpu(va_space, id);
uvm_gpu_chunk_t *chunk = block_phys_page_chunk(block,
block_phys_page(residency, NUMA_NO_NODE, page_index),
NULL);
// This function will assert if no mapping exists
(void)uvm_pmm_gpu_indirect_peer_addr(&resident_gpu->pmm, chunk, mapped_gpu);
}
}
}
}
@@ -6018,7 +5825,7 @@ static bool block_has_remote_mapping_gpu(uvm_va_block_t *block,
if (uvm_page_mask_empty(mapped_pages))
return false;
return !uvm_id_equal(uvm_va_range_get_policy(block->va_range)->preferred_location, gpu_id);
return !uvm_va_policy_preferred_location_equal(uvm_va_range_get_policy(block->va_range), gpu_id, NUMA_NO_NODE);
}
// Remote pages are pages which are mapped but not resident locally
@@ -8361,6 +8168,7 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
uvm_va_block_context_t *block_context,
uvm_gpu_t *gpu,
uvm_processor_id_t resident_id,
int resident_nid,
uvm_page_mask_t *map_page_mask,
uvm_prot_t new_prot,
uvm_tracker_t *out_tracker)
@@ -8370,7 +8178,7 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
uvm_push_t push;
NV_STATUS status;
uvm_page_mask_t *pages_to_map = &block_context->mapping.page_mask;
const uvm_page_mask_t *resident_mask = uvm_va_block_resident_mask_get(va_block, resident_id, NUMA_NO_NODE);
const uvm_page_mask_t *resident_mask = uvm_va_block_resident_mask_get(va_block, resident_id, resident_nid);
uvm_pte_bits_gpu_t pte_bit;
uvm_pte_bits_gpu_t prot_pte_bit = get_gpu_pte_bit_index(new_prot);
uvm_va_block_new_pte_state_t *new_pte_state = &block_context->mapping.new_pte_state;
@@ -8379,8 +8187,10 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
UVM_ASSERT(map_page_mask);
UVM_ASSERT(uvm_processor_mask_test(&va_space->accessible_from[uvm_id_value(resident_id)], gpu->id));
if (uvm_processor_mask_test(block_get_uvm_lite_gpus(va_block), gpu->id))
UVM_ASSERT(uvm_id_equal(resident_id, uvm_va_range_get_policy(va_block->va_range)->preferred_location));
if (uvm_processor_mask_test(block_get_uvm_lite_gpus(va_block), gpu->id)) {
uvm_va_policy_t *policy = uvm_va_range_get_policy(va_block->va_range);
UVM_ASSERT(uvm_va_policy_preferred_location_equal(policy, resident_id, policy->preferred_nid));
}
UVM_ASSERT(!uvm_page_mask_and(&block_context->scratch_page_mask,
map_page_mask,
@@ -8482,18 +8292,27 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
return uvm_tracker_add_push_safe(out_tracker, &push);
}
// allowed_nid_mask is only valid if the CPU is set in allowed_mask.
static void map_get_allowed_destinations(uvm_va_block_t *block,
uvm_va_block_context_t *va_block_context,
const uvm_va_policy_t *policy,
uvm_processor_id_t id,
uvm_processor_mask_t *allowed_mask)
uvm_processor_mask_t *allowed_mask,
nodemask_t *allowed_nid_mask)
{
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
*allowed_nid_mask = node_possible_map;
if (uvm_processor_mask_test(block_get_uvm_lite_gpus(block), id)) {
// UVM-Lite can only map resident pages on the preferred location
uvm_processor_mask_zero(allowed_mask);
uvm_processor_mask_set(allowed_mask, policy->preferred_location);
if (UVM_ID_IS_CPU(policy->preferred_location) &&
!uvm_va_policy_preferred_location_equal(policy, UVM_ID_CPU, NUMA_NO_NODE)) {
nodes_clear(*allowed_nid_mask);
node_set(policy->preferred_nid, *allowed_nid_mask);
}
}
else if ((uvm_va_policy_is_read_duplicate(policy, va_space) ||
(uvm_id_equal(policy->preferred_location, id) &&
@@ -8536,6 +8355,7 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
uvm_page_mask_t *running_page_mask = &va_block_context->mapping.map_running_page_mask;
NV_STATUS status = NV_OK;
const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, region);
nodemask_t *allowed_nid_destinations;
va_block_context->mapping.cause = cause;
@@ -8585,10 +8405,20 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
if (!allowed_destinations)
return NV_ERR_NO_MEMORY;
allowed_nid_destinations = uvm_kvmalloc(sizeof(*allowed_nid_destinations));
if (!allowed_nid_destinations) {
uvm_processor_mask_cache_free(allowed_destinations);
return NV_ERR_NO_MEMORY;
}
// Map per resident location so we can more easily detect physically-
// contiguous mappings.
map_get_allowed_destinations(va_block, va_block_context, policy, id, allowed_destinations);
map_get_allowed_destinations(va_block,
va_block_context,
policy,
id,
allowed_destinations,
allowed_nid_destinations);
for_each_closest_id(resident_id, allowed_destinations, id, va_space) {
if (UVM_ID_IS_CPU(id)) {
status = block_map_cpu_to(va_block,
@@ -8599,11 +8429,30 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
new_prot,
out_tracker);
}
else if (UVM_ID_IS_CPU(resident_id)) {
int nid;
// map_get_allowed_distinations() will set the mask of CPU NUMA
// nodes that should be mapped.
for_each_node_mask(nid, *allowed_nid_destinations) {
status = block_map_gpu_to(va_block,
va_block_context,
gpu,
resident_id,
nid,
running_page_mask,
new_prot,
out_tracker);
if (status != NV_OK)
break;
}
}
else {
status = block_map_gpu_to(va_block,
va_block_context,
gpu,
resident_id,
NUMA_NO_NODE,
running_page_mask,
new_prot,
out_tracker);
@@ -8618,6 +8467,7 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
}
uvm_processor_mask_cache_free(allowed_destinations);
uvm_kvfree(allowed_nid_destinations);
return status;
}
@@ -9031,7 +8881,7 @@ static void block_destroy_gpu_state(uvm_va_block_t *block, uvm_va_block_context_
uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(block, id);
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_gpu_va_space_t *gpu_va_space;
uvm_gpu_t *gpu, *other_gpu;
uvm_gpu_t *gpu;
if (!gpu_state)
return;
@@ -9041,32 +8891,14 @@ static void block_destroy_gpu_state(uvm_va_block_t *block, uvm_va_block_context_
// Unmap PTEs and free page tables
gpu = uvm_va_space_get_gpu(va_space, id);
gpu_va_space = uvm_gpu_va_space_get(va_space, gpu);
if (gpu_va_space) {
if (gpu_va_space)
uvm_va_block_remove_gpu_va_space(block, gpu_va_space, block_context);
}
UVM_ASSERT(!uvm_processor_mask_test(&block->mapped, id));
// No processor should have this GPU mapped at this point
UVM_ASSERT(block_check_processor_not_mapped(block, block_context, id));
// We need to remove the mappings of the indirect peers from the reverse
// map when the GPU state is being destroyed (for example, on
// unregister_gpu) and when peer access between indirect peers is disabled.
// However, we need to avoid double mapping removals. There are two
// possible scenarios:
// - Disable peer access first. This will remove all mappings between A and
// B GPUs, and the indirect_peers bit is cleared. Thus, the later call to
// unregister_gpu will not operate on that pair of GPUs.
// - Unregister GPU first. This will remove all mappings from all indirect
// peers to the GPU being unregistered. It will also destroy its GPU state.
// Subsequent calls to disable peers will remove the mappings from the GPU
// being unregistered, but never to the GPU being unregistered (since it no
// longer has a valid GPU state).
for_each_va_space_gpu_in_mask(other_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)])
block_gpu_unmap_all_chunks_indirect_peer(block, gpu, other_gpu);
if (gpu_state->chunks) {
size_t i, num_chunks;
@@ -9212,33 +9044,6 @@ void uvm_va_block_remove_gpu_va_space(uvm_va_block_t *va_block,
UVM_ASSERT(block_check_mappings(va_block, block_context));
}
NV_STATUS uvm_va_block_enable_peer(uvm_va_block_t *va_block, uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
{
NV_STATUS status;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
UVM_ASSERT(uvm_gpu_peer_caps(gpu0, gpu1)->link_type != UVM_GPU_LINK_INVALID);
uvm_assert_rwsem_locked_write(&va_space->lock);
uvm_assert_mutex_locked(&va_block->lock);
if (uvm_processor_mask_test(&va_space->indirect_peers[uvm_id_value(gpu0->id)], gpu1->id)) {
status = block_gpu_map_all_chunks_indirect_peer(va_block, gpu0, gpu1);
if (status != NV_OK)
return status;
status = block_gpu_map_all_chunks_indirect_peer(va_block, gpu1, gpu0);
if (status != NV_OK) {
block_gpu_unmap_all_chunks_indirect_peer(va_block, gpu0, gpu1);
return status;
}
}
// TODO: Bug 1767224: Refactor the uvm_va_block_set_accessed_by logic so we
// call it here.
return NV_OK;
}
void uvm_va_block_disable_peer(uvm_va_block_t *va_block, uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
{
uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
@@ -9251,12 +9056,6 @@ void uvm_va_block_disable_peer(uvm_va_block_t *va_block, uvm_gpu_t *gpu0, uvm_gp
uvm_assert_mutex_locked(&va_block->lock);
// See comment in block_destroy_gpu_state
if (uvm_processor_mask_test(&va_space->indirect_peers[uvm_id_value(gpu0->id)], gpu1->id)) {
block_gpu_unmap_all_chunks_indirect_peer(va_block, gpu0, gpu1);
block_gpu_unmap_all_chunks_indirect_peer(va_block, gpu1, gpu0);
}
// If either of the GPUs doesn't have GPU state then nothing could be mapped
// between them.
if (!uvm_va_block_gpu_state_get(va_block, gpu0->id) || !uvm_va_block_gpu_state_get(va_block, gpu1->id))
@@ -9588,8 +9387,6 @@ static void block_gpu_release_region(uvm_va_block_t *va_block,
if (!gpu_chunk)
continue;
// TODO: Bug 3898467: unmap indirect peers when freeing GPU chunks
uvm_mmu_chunk_unmap(gpu_chunk, &va_block->tracker);
// The GPU chunk will be freed when the device private reference drops.
@@ -9777,29 +9574,11 @@ static void block_gpu_chunk_get_split_state(uvm_va_block_t *block,
state->chunk_index = block_gpu_chunk_index_range(block, start, size, gpu, page_index, &state->chunk_size);
}
static void block_merge_chunk(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_gpu_chunk_t *chunk)
{
uvm_gpu_t *accessing_gpu;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_pmm_gpu_merge_chunk(&gpu->pmm, chunk);
for_each_va_space_gpu_in_mask(accessing_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&gpu->pmm, chunk, accessing_gpu);
uvm_pmm_sysmem_mappings_merge_gpu_chunk_mappings(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr,
uvm_gpu_chunk_get_size(chunk));
}
}
// Perform any chunk splitting and array growing required for this block split,
// but don't actually move chunk pointers anywhere.
static NV_STATUS block_presplit_gpu_chunks(uvm_va_block_t *existing, uvm_va_block_t *new, uvm_gpu_t *gpu)
{
uvm_va_block_gpu_state_t *existing_gpu_state = uvm_va_block_gpu_state_get(existing, gpu->id);
uvm_gpu_t *accessing_gpu;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(existing);
uvm_gpu_chunk_t **temp_chunks;
uvm_gpu_chunk_t *original_chunk, *curr_chunk;
uvm_page_index_t split_page_index = uvm_va_block_cpu_page_index(existing, new->start);
@@ -9864,17 +9643,6 @@ static NV_STATUS block_presplit_gpu_chunks(uvm_va_block_t *existing, uvm_va_bloc
if (status != NV_OK)
goto error;
// Split physical GPU mappings for indirect peers
for_each_va_space_gpu_in_mask(accessing_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&gpu->pmm, curr_chunk, accessing_gpu);
status = uvm_pmm_sysmem_mappings_split_gpu_chunk_mappings(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr,
subchunk_size);
if (status != NV_OK)
goto error;
}
if (subchunk_size == new_state.chunk_size)
break;
@@ -9900,7 +9668,7 @@ static NV_STATUS block_presplit_gpu_chunks(uvm_va_block_t *existing, uvm_va_bloc
error:
// On error we need to leave the chunk in its initial state
block_merge_chunk(existing, gpu, original_chunk);
uvm_pmm_gpu_merge_chunk(&gpu->pmm, original_chunk);
return status;
}
@@ -10338,7 +10106,7 @@ error:
if (!chunk || chunk->state != UVM_PMM_GPU_CHUNK_STATE_IS_SPLIT)
continue;
block_merge_chunk(existing, gpu, chunk);
uvm_pmm_gpu_merge_chunk(&gpu->pmm, chunk);
// We could attempt to shrink the chunks array back down, but it doesn't
// hurt much to have it larger than necessary, and we'd have to handle
@@ -10628,11 +10396,9 @@ static void block_split_gpu(uvm_va_block_t *existing, uvm_va_block_t *new, uvm_g
uvm_va_space_t *va_space = uvm_va_block_get_va_space(existing);
uvm_gpu_va_space_t *gpu_va_space;
uvm_gpu_t *gpu;
uvm_gpu_t *accessing_gpu;
size_t new_pages = uvm_va_block_num_cpu_pages(new);
size_t existing_pages, existing_pages_4k, existing_pages_big, new_pages_big;
uvm_pte_bits_gpu_t pte_bit;
size_t num_chunks, i;
uvm_cpu_chunk_t *cpu_chunk;
uvm_page_index_t page_index;
int nid;
@@ -10659,23 +10425,6 @@ static void block_split_gpu(uvm_va_block_t *existing, uvm_va_block_t *new, uvm_g
block_copy_split_gpu_chunks(existing, new, gpu);
num_chunks = block_num_gpu_chunks(new, gpu);
// Reparent GPU mappings for indirect peers
for (i = 0; i < num_chunks; ++i) {
uvm_gpu_chunk_t *chunk = new_gpu_state->chunks[i];
if (!chunk)
continue;
for_each_va_space_gpu_in_mask(accessing_gpu, va_space, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&gpu->pmm, chunk, accessing_gpu);
uvm_pmm_sysmem_mappings_reparent_gpu_chunk_mapping(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr,
new);
}
}
block_split_page_mask(&existing_gpu_state->resident,
existing_pages,
&new_gpu_state->resident,
@@ -11180,8 +10929,8 @@ NV_STATUS uvm_va_block_add_mappings_after_migration(uvm_va_block_t *va_block,
// so uvm_va_block_map will be a no-op.
uvm_processor_mask_and(map_uvm_lite_gpus, map_other_processors, block_get_uvm_lite_gpus(va_block));
if (!uvm_processor_mask_empty(map_uvm_lite_gpus) &&
uvm_id_equal(new_residency, preferred_location)) {
for_each_id_in_mask(map_processor_id, map_uvm_lite_gpus) {
uvm_va_policy_preferred_location_equal(policy, new_residency, va_block_context->make_resident.dest_nid)) {
for_each_id_in_mask (map_processor_id, map_uvm_lite_gpus) {
status = uvm_va_block_map(va_block,
va_block_context,
map_processor_id,
@@ -11642,6 +11391,10 @@ static int block_select_node_residency(uvm_va_block_t *va_block,
// For GPU faults, the bottom half is pinned to CPUs closest to their GPU.
// Therefore, in both cases, we can use numa_mem_id() to get the NUMA node
// ID of the faulting processor.
// Note that numa_mem_id() returns the nearest node with memory. In most
// cases, this will be the current NUMA node. However, in the case that the
// current node does not have any memory, we probably want the nearest node
// with memory, anyway.
int current_nid = numa_mem_id();
bool may_read_duplicate = can_read_duplicate(va_block, page_index, policy, thrashing_hint);
@@ -11665,7 +11418,12 @@ static int block_select_node_residency(uvm_va_block_t *va_block,
// If read duplication is enabled and the page is also resident on the CPU,
// keep its current NUMA node residency.
if (may_read_duplicate && uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index))
return block_get_page_node_residency(va_block, page_index);
return NUMA_NO_NODE;
// The new_residency processor is the CPU and the preferred location is not
// the CPU. If the page is resident on the CPU, keep its current residency.
if (uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index))
return NUMA_NO_NODE;
return current_nid;
}
@@ -13132,7 +12890,6 @@ NV_STATUS uvm_va_block_evict_chunks(uvm_va_block_t *va_block,
goto out;
for (i = 0; i < num_gpu_chunks; ++i) {
uvm_gpu_id_t accessing_gpu_id;
uvm_gpu_chunk_t *chunk = gpu_state->chunks[i];
if (!chunk)
@@ -13140,45 +12897,6 @@ NV_STATUS uvm_va_block_evict_chunks(uvm_va_block_t *va_block,
if (!uvm_gpu_chunk_same_root(chunk, root_chunk))
continue;
// Remove the mappings of indirect peers from the reverse map. We
// access the indirect peer mask from the VA space without holding the
// VA space lock. Therefore, we can race with enable_peer/disable_peer
// operations. However this is fine:
//
// The enable_peer sequence is as follows:
//
// set_bit in va_space->indirect_peers
// uvm_va_block_enable_peer;
//
// - If we read the mask BEFORE it is set or AFTER the mapping has
// been added to the map there is no race.
// - If we read the mask AFTER it is set but BEFORE adding the mapping
// to the reverse map, we will try to remove it although it is not
// there yet. Therefore, we use
// uvm_pmm_sysmem_mappings_remove_gpu_mapping_on_eviction, which does
// not check if the mapping is present in the reverse map.
//
// The disable_peer sequence is as follows:
//
// uvm_va_block_disable_peer;
// clear_bit in va_space->indirect_peers
//
// - If we read the mask BEFORE the mapping has been added to the map
// or AFTER the bit has been cleared, there is no race.
// - If we read the mask AFTER the mapping has been removed and BEFORE
// the bit is cleared, we will try to remove the mapping, too.
// Again, uvm_pmm_sysmem_mappings_remove_gpu_mapping_on_eviction works
// in this scenario.
// Obtain the uvm_gpu_t directly via the parent GPU's id since indirect
// peers are not supported when SMC is enabled.
for_each_gpu_id_in_mask(accessing_gpu_id, &va_space->indirect_peers[uvm_id_value(gpu->id)]) {
uvm_gpu_t *accessing_gpu = uvm_va_space_get_gpu(va_space, accessing_gpu_id);
NvU64 peer_addr = uvm_pmm_gpu_indirect_peer_addr(&gpu->pmm, chunk, accessing_gpu);
uvm_pmm_sysmem_mappings_remove_gpu_mapping_on_eviction(&accessing_gpu->pmm_reverse_sysmem_mappings,
peer_addr);
}
uvm_mmu_chunk_unmap(chunk, tracker);
uvm_pmm_gpu_mark_chunk_evicted(&gpu->pmm, gpu_state->chunks[i]);
@@ -13311,8 +13029,8 @@ NV_STATUS uvm_test_va_block_inject_error(UVM_TEST_VA_BLOCK_INJECT_ERROR_PARAMS *
if (params->eviction_error)
va_block_test->inject_eviction_error = params->eviction_error;
if (params->cpu_pages_allocation_error_count)
va_block_test->inject_cpu_pages_allocation_error_count = params->cpu_pages_allocation_error_count;
if (params->cpu_chunk_allocation_error_count)
va_block_test->inject_cpu_chunk_allocation_error_count = params->cpu_chunk_allocation_error_count;
if (params->populate_error)
va_block_test->inject_populate_error = params->populate_error;
@@ -13651,63 +13369,24 @@ NV_STATUS uvm_test_va_residency_info(UVM_TEST_VA_RESIDENCY_INFO_PARAMS *params,
++count;
}
if (params->resident_on_count == 1) {
if (uvm_processor_mask_test(resident_on_mask, UVM_ID_CPU)) {
if (uvm_pmm_sysmem_mappings_indirect_supported()) {
for_each_gpu_id(id) {
NvU64 page_size = uvm_va_block_page_size_processor(block, id, page_index);
uvm_reverse_map_t sysmem_page;
uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_chunk_for_page_resident(block, page_index);
size_t num_pages;
uvm_gpu_t *gpu;
if (params->resident_on_count == 1 && !uvm_processor_mask_test(resident_on_mask, UVM_ID_CPU)) {
uvm_gpu_id_t id = uvm_processor_mask_find_first_id(resident_on_mask);
uvm_reverse_map_t gpu_mapping;
size_t num_pages;
uvm_gpu_t *gpu = uvm_va_space_get_gpu(va_space, id);
uvm_gpu_phys_address_t phys_addr;
if (!uvm_va_block_gpu_state_get(block, id))
continue;
phys_addr = uvm_va_block_gpu_phys_page_address(block, page_index, gpu);
num_pages = uvm_pmm_gpu_phys_to_virt(&gpu->pmm, phys_addr.address, PAGE_SIZE, &gpu_mapping);
gpu = uvm_va_space_get_gpu(va_space, id);
// Chunk may be in TEMP_PINNED state so it may not have a VA block
// assigned. In that case, we don't get a valid translation.
if (num_pages > 0) {
UVM_ASSERT(num_pages == 1);
UVM_ASSERT(gpu_mapping.va_block == block);
UVM_ASSERT(uvm_reverse_map_start(&gpu_mapping) == addr);
if (!gpu->parent->access_counters_can_use_physical_addresses)
continue;
num_pages = uvm_pmm_sysmem_mappings_dma_to_virt(&gpu->pmm_reverse_sysmem_mappings,
uvm_cpu_chunk_get_gpu_phys_addr(chunk, gpu),
uvm_cpu_chunk_get_size(chunk),
&sysmem_page,
1);
if (page_size > 0)
UVM_ASSERT(num_pages == 1);
else
UVM_ASSERT(num_pages <= 1);
if (num_pages == 1) {
UVM_ASSERT(sysmem_page.va_block == block);
UVM_ASSERT(uvm_reverse_map_start(&sysmem_page) <= addr);
UVM_ASSERT(uvm_reverse_map_end(&sysmem_page) > addr);
++release_block_count;
}
}
}
}
else {
uvm_gpu_id_t id = uvm_processor_mask_find_first_id(resident_on_mask);
uvm_reverse_map_t gpu_mapping;
size_t num_pages;
uvm_gpu_t *gpu = uvm_va_space_get_gpu(va_space, id);
uvm_gpu_phys_address_t phys_addr;
phys_addr = uvm_va_block_gpu_phys_page_address(block, page_index, gpu);
num_pages = uvm_pmm_gpu_phys_to_virt(&gpu->pmm, phys_addr.address, PAGE_SIZE, &gpu_mapping);
// Chunk may be in TEMP_PINNED state so it may not have a VA block
// assigned. In that case, we don't get a valid translation.
if (num_pages > 0) {
UVM_ASSERT(num_pages == 1);
UVM_ASSERT(gpu_mapping.va_block == block);
UVM_ASSERT(uvm_reverse_map_start(&gpu_mapping) == addr);
++release_block_count;
}
++release_block_count;
}
}