NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-02 14:37:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

575.51.02

Browse Source
This commit is contained in:
Bernhard Stoeckner
2025-04-17 19:35:38 +02:00
parent e8113f665d
commit 4159579888
1142 changed files with 309085 additions and 272273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
kernel-open/nvidia-uvm/uvm_gpu.c
View File

@@ -149,7 +149,7 @@ static NV_STATUS get_gpu_caps(uvm_gpu_t *gpu)
}
// Return a PASID to use with the internal address space (AS), or -1 if not
// supported. This PASID is needed to enable ATS in the internal AS, but it is
// supported. This PASID is needed to enable ATS in the internal AS, but it is
// not used in address translation requests, which only translate GPA->SPA.
// The buffer management thus remains the same: DMA mapped GPA addresses can
// be accessed by the GPU, while unmapped addresses can not and any access is
@@ -358,10 +358,11 @@ bool uvm_gpu_can_address(uvm_gpu_t *gpu, NvU64 addr, NvU64 size)
addr_shift = gpu_addr_shift;
// Pascal+ GPUs are capable of accessing kernel pointers in various modes
// by applying the same upper-bit checks that x86, ARM, and Power
// processors do. x86 and ARM use canonical form addresses. For ARM, even
// by applying the same upper-bit checks that x86 or ARM CPU processors do.
// The x86 and ARM platforms use canonical form addresses. For ARM, even
// with Top-Byte Ignore enabled, the following logic validates addresses
// from the kernel VA range. PowerPC does not use canonical form address.
// from the kernel VA range.
//
// The following diagram illustrates the valid (V) VA regions that can be
// mapped (or addressed) by the GPU/CPU when the CPU uses canonical form.
// (C) regions are only accessible by the CPU. Similarly, (G) regions
@@ -389,8 +390,8 @@ bool uvm_gpu_can_address(uvm_gpu_t *gpu, NvU64 addr, NvU64 size)
// |VVVVVVVVVVVVVVVV| |VVVVVVVVVVVVVVVV|
// 0 +----------------+ 0 +----------------+
// On canonical form address platforms and Pascal+ GPUs.
if (uvm_platform_uses_canonical_form_address() && gpu_addr_shift > 40) {
// On Pascal+ GPUs.
if (gpu_addr_shift > 40) {
// On x86, when cpu_addr_shift > gpu_addr_shift, it means the CPU uses
// 5-level paging and the GPU is pre-Hopper. On Pascal-Ada GPUs (49b
// wide VA) we set addr_shift to match a 4-level paging x86 (48b wide).
@@ -431,30 +432,28 @@ NvU64 uvm_parent_gpu_canonical_address(uvm_parent_gpu_t *parent_gpu, NvU64 addr)
NvU8 addr_shift;
NvU64 input_addr = addr;
if (uvm_platform_uses_canonical_form_address()) {
// When the CPU VA width is larger than GPU's, it means that:
// On ARM: the CPU is on LVA mode and the GPU is pre-Hopper.
// On x86: the CPU uses 5-level paging and the GPU is pre-Hopper.
// We sign-extend on the 48b on ARM and on the 47b on x86 to mirror the
// behavior of CPUs with smaller (than GPU) VA widths.
gpu_addr_shift = parent_gpu->arch_hal->mmu_mode_hal(UVM_PAGE_SIZE_64K)->num_va_bits();
cpu_addr_shift = uvm_cpu_num_va_bits();
// When the CPU VA width is larger than GPU's, it means that:
// On ARM: the CPU is on LVA mode and the GPU is pre-Hopper.
// On x86: the CPU uses 5-level paging and the GPU is pre-Hopper.
// We sign-extend on the 48b on ARM and on the 47b on x86 to mirror the
// behavior of CPUs with smaller (than GPU) VA widths.
gpu_addr_shift = parent_gpu->arch_hal->mmu_mode_hal(UVM_PAGE_SIZE_64K)->num_va_bits();
cpu_addr_shift = uvm_cpu_num_va_bits();
if (cpu_addr_shift > gpu_addr_shift)
addr_shift = NVCPU_IS_X86_64 ? 48 : 49;
else if (gpu_addr_shift == 57)
addr_shift = gpu_addr_shift;
else
addr_shift = cpu_addr_shift;
if (cpu_addr_shift > gpu_addr_shift)
addr_shift = NVCPU_IS_X86_64 ? 48 : 49;
else if (gpu_addr_shift == 57)
addr_shift = gpu_addr_shift;
else
addr_shift = cpu_addr_shift;
addr = (NvU64)((NvS64)(addr << (64 - addr_shift)) >> (64 - addr_shift));
addr = (NvU64)((NvS64)(addr << (64 - addr_shift)) >> (64 - addr_shift));
// This protection acts on when the address is not covered by the GPU's
// OOR_ADDR_CHECK. This can only happen when OOR_ADDR_CHECK is in
// permissive (NO_CHECK) mode.
if ((addr << (64 - gpu_addr_shift)) != (input_addr << (64 - gpu_addr_shift)))
return input_addr;
}
// This protection acts on when the address is not covered by the GPU's
// OOR_ADDR_CHECK. This can only happen when OOR_ADDR_CHECK is in
// permissive (NO_CHECK) mode.
if ((addr << (64 - gpu_addr_shift)) != (input_addr << (64 - gpu_addr_shift)))
return input_addr;
return addr;
}
@@ -485,7 +484,7 @@ static void gpu_info_print_ce_caps(uvm_gpu_t *gpu, struct seq_file *s)
continue;
UVM_SEQ_OR_DBG_PRINT(s, " ce %u pce mask 0x%08x grce %u shared %u sysmem read %u sysmem write %u sysmem %u "
"nvlink p2p %u p2p %u\n",
"nvlink p2p %u p2p %u secure %u\n",
i,
ce_caps->cePceMask,
ce_caps->grce,
@@ -494,7 +493,8 @@ static void gpu_info_print_ce_caps(uvm_gpu_t *gpu, struct seq_file *s)
ce_caps->sysmemWrite,
ce_caps->sysmem,
ce_caps->nvlinkP2p,
ce_caps->p2p);
ce_caps->p2p,
ce_caps->secure);
}
out:
@@ -595,9 +595,6 @@ static void gpu_info_print_common(uvm_gpu_t *gpu, struct seq_file *s)
window_size / (1024 * 1024));
}
if (gpu->parent->npu)
UVM_SEQ_OR_DBG_PRINT(s, "npu_domain %d\n", gpu->parent->npu->pci_domain);
UVM_SEQ_OR_DBG_PRINT(s, "interrupts %llu\n", gpu->parent->isr.interrupt_count);
if (gpu->parent->isr.replayable_faults.handling) {
@@ -1041,7 +1038,7 @@ static NV_STATUS init_procfs_dirs(uvm_gpu_t *gpu)
{
struct proc_dir_entry *gpu_base_dir_entry;
char symlink_name[16]; // Hold a uvm_gpu_id_t value in decimal.
char uuid_buffer[max(UVM_PARENT_GPU_UUID_STRING_LENGTH, UVM_GPU_UUID_STRING_LENGTH)];
char uuid_buffer[NV_MAX(UVM_PARENT_GPU_UUID_STRING_LENGTH, UVM_GPU_UUID_STRING_LENGTH)];
char gpu_dir_name[sizeof(symlink_name) + sizeof(uuid_buffer) + 1];
if (!uvm_procfs_is_enabled())
@@ -1249,13 +1246,15 @@ static NV_STATUS configure_address_space(uvm_gpu_t *gpu)
NvU32 num_entries;
NvU64 va_size;
NvU64 va_per_entry;
NvU64 physical_address;
NvU64 dma_address;
uvm_mmu_page_table_alloc_t *tree_alloc;
status = uvm_page_tree_init(gpu,
NULL,
UVM_PAGE_TREE_TYPE_KERNEL,
gpu->big_page.internal_size,
uvm_get_page_tree_location(gpu->parent),
uvm_get_page_tree_location(gpu),
&gpu->address_space_tree);
if (status != NV_OK) {
UVM_ERR_PRINT("Initializing the page tree failed: %s, GPU %s\n", nvstatusToString(status), uvm_gpu_name(gpu));
@@ -1279,12 +1278,17 @@ static NV_STATUS configure_address_space(uvm_gpu_t *gpu)
gpu->parent->rm_va_size,
va_per_entry);
tree_alloc = uvm_page_tree_pdb(&gpu->address_space_tree);
tree_alloc = uvm_page_tree_pdb_internal(&gpu->address_space_tree);
if (tree_alloc->addr.aperture == UVM_APERTURE_VID)
physical_address = tree_alloc->addr.address;
else
physical_address = page_to_phys(tree_alloc->handle.page);
status = uvm_rm_locked_call(nvUvmInterfaceSetPageDirectory(gpu->rm_address_space,
tree_alloc->addr.address,
physical_address,
num_entries,
tree_alloc->addr.aperture == UVM_APERTURE_VID,
gpu_get_internal_pasid(gpu)));
gpu_get_internal_pasid(gpu),
&dma_address));
if (status != NV_OK) {
UVM_ERR_PRINT("nvUvmInterfaceSetPageDirectory() failed: %s, GPU %s\n",
nvstatusToString(status),
@@ -1292,6 +1296,9 @@ static NV_STATUS configure_address_space(uvm_gpu_t *gpu)
return status;
}
if (tree_alloc->addr.aperture == UVM_APERTURE_SYS)
gpu->address_space_tree.pdb_rm_dma_address = uvm_gpu_phys_address(UVM_APERTURE_SYS, dma_address);
gpu->rm_address_space_moved_to_page_tree = true;
return NV_OK;
@@ -1404,13 +1411,12 @@ static NV_STATUS init_parent_gpu(uvm_parent_gpu_t *parent_gpu,
parent_gpu->egm.enabled = gpu_info->egmEnabled;
parent_gpu->egm.local_peer_id = gpu_info->egmPeerId;
parent_gpu->egm.base_address = gpu_info->egmBaseAddr;
parent_gpu->access_counters_supported = (gpu_info->accessCntrBufferCount != 0);
status = uvm_rm_locked_call(nvUvmInterfaceGetFbInfo(parent_gpu->rm_device, &fb_info));
if (status != NV_OK)
return status;
parent_gpu->sli_enabled = (gpu_info->subdeviceCount > 1);
if (!fb_info.bZeroFb)
parent_gpu->max_allocatable_address = fb_info.maxAllocatableAddress;
@@ -1559,6 +1565,12 @@ static NV_STATUS init_gpu(uvm_gpu_t *gpu, const UvmGpuInfo *gpu_info)
return status;
}
status = uvm_pmm_sysmem_mappings_init(gpu, &gpu->pmm_reverse_sysmem_mappings);
if (status != NV_OK) {
UVM_ERR_PRINT("CPU PMM MMIO initialization failed: %s, GPU %s\n", nvstatusToString(status), uvm_gpu_name(gpu));
return status;
}
uvm_pmm_gpu_device_p2p_init(gpu);
status = init_semaphore_pools(gpu);
@@ -1757,6 +1769,8 @@ static void deinit_gpu(uvm_gpu_t *gpu)
uvm_pmm_gpu_device_p2p_deinit(gpu);
uvm_pmm_sysmem_mappings_deinit(&gpu->pmm_reverse_sysmem_mappings);
uvm_pmm_gpu_deinit(&gpu->pmm);
if (gpu->rm_address_space != 0)
@@ -3147,12 +3161,15 @@ bool uvm_gpu_address_is_peer(uvm_gpu_t *gpu, uvm_gpu_address_t address)
return false;
}
uvm_aperture_t uvm_get_page_tree_location(const uvm_parent_gpu_t *parent_gpu)
uvm_aperture_t uvm_get_page_tree_location(const uvm_gpu_t *gpu)
{
// See comment in page_tree_set_location
if (uvm_parent_gpu_is_virt_mode_sriov_heavy(parent_gpu) || g_uvm_global.conf_computing_enabled)
// See comments in page_tree_set_location
if (uvm_parent_gpu_is_virt_mode_sriov_heavy(gpu->parent) || g_uvm_global.conf_computing_enabled)
return UVM_APERTURE_VID;
if (!gpu->mem_info.size)
return UVM_APERTURE_SYS;
return UVM_APERTURE_DEFAULT;
}
@@ -3610,7 +3627,7 @@ static NvU64 gpu_addr_to_dma_addr(uvm_parent_gpu_t *parent_gpu, NvU64 gpu_addr)
// dma_addressable_start (in bifSetupDmaWindow_IMPL()) and hence when
// referencing sysmem from the GPU, dma_addressable_start should be
// subtracted from the DMA address we get from the OS.
static NvU64 dma_addr_to_gpu_addr(uvm_parent_gpu_t *parent_gpu, NvU64 dma_addr)
NvU64 uvm_parent_gpu_dma_addr_to_gpu_addr(uvm_parent_gpu_t *parent_gpu, NvU64 dma_addr)
{
NvU64 gpu_addr = dma_addr - parent_gpu->dma_addressable_start;
UVM_ASSERT(dma_addr >= gpu_addr);
@@ -3618,32 +3635,40 @@ static NvU64 dma_addr_to_gpu_addr(uvm_parent_gpu_t *parent_gpu, NvU64 dma_addr)
return gpu_addr;
}
void *uvm_parent_gpu_dma_alloc_page(uvm_parent_gpu_t *parent_gpu, gfp_t gfp_flags, NvU64 *dma_address_out)
static void *parent_gpu_dma_alloc_page(uvm_parent_gpu_t *parent_gpu, gfp_t gfp_flags, NvU64 *dma_address_out)
{
NvU64 dma_addr;
void *cpu_addr;
cpu_addr = dma_alloc_coherent(&parent_gpu->pci_dev->dev, PAGE_SIZE, &dma_addr, gfp_flags);
if (!cpu_addr)
return cpu_addr;
*dma_address_out = dma_addr_to_gpu_addr(parent_gpu, dma_addr);
*dma_address_out = uvm_parent_gpu_dma_addr_to_gpu_addr(parent_gpu, dma_addr);
atomic64_add(PAGE_SIZE, &parent_gpu->mapped_cpu_pages_size);
return cpu_addr;
}
void uvm_parent_gpu_dma_free_page(uvm_parent_gpu_t *parent_gpu, void *va, NvU64 dma_address)
NV_STATUS uvm_gpu_dma_alloc_page(uvm_gpu_t *gpu, gfp_t gfp_flags, void **cpu_addr_out, NvU64 *dma_address_out)
{
void *cpu_addr = parent_gpu_dma_alloc_page(gpu->parent, gfp_flags, dma_address_out);
if (!cpu_addr)
return NV_ERR_NO_MEMORY;
// TODO: Bug 4868590: Issue GPA invalidate here
*cpu_addr_out = cpu_addr;
return NV_OK;
}
void uvm_parent_gpu_dma_free_page(uvm_parent_gpu_t *parent_gpu, void *cpu_addr, NvU64 dma_address)
{
dma_address = gpu_addr_to_dma_addr(parent_gpu, dma_address);
dma_free_coherent(&parent_gpu->pci_dev->dev, PAGE_SIZE, va, dma_address);
dma_free_coherent(&parent_gpu->pci_dev->dev, PAGE_SIZE, cpu_addr, dma_address);
atomic64_sub(PAGE_SIZE, &parent_gpu->mapped_cpu_pages_size);
}
NV_STATUS uvm_parent_gpu_map_cpu_pages(uvm_parent_gpu_t *parent_gpu,
struct page *page,
size_t size,
NvU64 *dma_address_out)
static NV_STATUS parent_gpu_map_cpu_pages(uvm_parent_gpu_t *parent_gpu, struct page *page, size_t size, NvU64 *dma_address_out)
{
NvU64 dma_addr;
@@ -3666,11 +3691,20 @@ NV_STATUS uvm_parent_gpu_map_cpu_pages(uvm_parent_gpu_t *parent_gpu,
}
atomic64_add(size, &parent_gpu->mapped_cpu_pages_size);
*dma_address_out = dma_addr_to_gpu_addr(parent_gpu, dma_addr);
*dma_address_out = uvm_parent_gpu_dma_addr_to_gpu_addr(parent_gpu, dma_addr);
return NV_OK;
}
NV_STATUS uvm_gpu_map_cpu_pages(uvm_gpu_t *gpu, struct page *page, size_t size, NvU64 *dma_address_out)
{
NV_STATUS status = parent_gpu_map_cpu_pages(gpu->parent, page, size, dma_address_out);
// TODO: Bug 4868590: Issue GPA invalidate here
return status;
}
void uvm_parent_gpu_unmap_cpu_pages(uvm_parent_gpu_t *parent_gpu, NvU64 dma_address, size_t size)
{
UVM_ASSERT(PAGE_ALIGNED(size));