NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-09 17:50:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

580.105.08

Browse Source
This commit is contained in:
Maneet Singh
2025-11-04 12:45:59 -08:00
parent 2b436058a6
commit 2af9f1f0f7
112 changed files with 56415 additions and 54596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
kernel-open/nvidia-uvm/uvm_ampere_host.c
View File

@@ -461,3 +461,29 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
if (params->membar == UvmInvalidateTlbMemBarLocal)
uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
}
void uvm_hal_ampere_host_l2_invalidate(uvm_push_t *push, uvm_aperture_t aperture)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
NvU32 aperture_value;
if (aperture == UVM_APERTURE_SYS) {
aperture_value = HWCONST(C56F, MEM_OP_D, OPERATION, L2_SYSMEM_INVALIDATE);
}
else if (uvm_aperture_is_peer(aperture)) {
aperture_value = HWCONST(C56F, MEM_OP_D, OPERATION, L2_PEERMEM_INVALIDATE);
}
else {
UVM_ASSERT_MSG(false, "Invalid aperture_type %d\n", aperture);
return;
}
uvm_hal_membar(gpu, push, UVM_MEMBAR_SYS);
NV_PUSH_4U(C56F, MEM_OP_A, 0,
MEM_OP_B, 0,
MEM_OP_C, 0,
MEM_OP_D, aperture_value);
uvm_hal_membar(gpu, push, UVM_MEMBAR_SYS);
}

7
kernel-open/nvidia-uvm/uvm_ats_sva.h
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2018-2023 NVIDIA Corporation
Copyright (c) 2018-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -43,8 +43,11 @@
// commit cb4789b0d19ff231ce9f73376a023341300aed96 (11/23/2020). Commit
// 701fac40384f07197b106136012804c3cae0b3de (02/15/2022) removed ioasid_get()
// and added mm_pasid_drop().
//
// CONFIG_IOMMU_SVA_LIB was renamed to CONFIG_IOMMU_SVA with commit 7ba5647
// (02/07/2022).
#if UVM_CAN_USE_MMU_NOTIFIERS() && (defined(NV_IOASID_GET_PRESENT) || defined(NV_MM_PASID_DROP_PRESENT))
#if defined(CONFIG_IOMMU_SVA)
#if defined(CONFIG_IOMMU_SVA) || defined(CONFIG_IOMMU_SVA_LIB)
#define UVM_ATS_SVA_SUPPORTED() 1
#else
#define UVM_ATS_SVA_SUPPORTED() 0

19
kernel-open/nvidia-uvm/uvm_blackwell_host.c
View File

@@ -347,10 +347,23 @@ uvm_hal_blackwell_access_counter_query_clear_op_gb20x(uvm_parent_gpu_t *parent_g
return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
}
// Host-specific L2 cache invalidate for non-coherent sysmem
void uvm_hal_blackwell_host_l2_invalidate_noncoh_sysmem(uvm_push_t *push)
void uvm_hal_blackwell_host_l2_invalidate(uvm_push_t *push, uvm_aperture_t aperture)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
NvU32 aperture_value;
if (!gpu->parent->is_integrated_gpu) {
return uvm_hal_ampere_host_l2_invalidate(push, aperture);
}
switch (aperture) {
case UVM_APERTURE_SYS:
aperture_value = HWCONST(C96F, MEM_OP_D, OPERATION, L2_SYSMEM_NCOH_INVALIDATE);
break;
default:
UVM_ASSERT_MSG(false, "Invalid aperture_type %d\n", aperture);
return;
}
// First sysmembar
uvm_hal_membar(gpu, push, UVM_MEMBAR_SYS);
@@ -363,7 +376,7 @@ void uvm_hal_blackwell_host_l2_invalidate_noncoh_sysmem(uvm_push_t *push)
NV_PUSH_4U(C96F, MEM_OP_A, 0,
MEM_OP_B, 0,
MEM_OP_C, 0,
MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, L2_SYSMEM_NCOH_INVALIDATE));
MEM_OP_D, aperture_value);
// Final sysmembar
uvm_hal_membar(gpu, push, UVM_MEMBAR_SYS);
}

11
kernel-open/nvidia-uvm/uvm_get_rm_ptes_test.c
View File

@@ -144,10 +144,13 @@ static NV_STATUS verify_mapping_info(uvm_va_space_t *va_space,
phys_offset = mapping_offset;
// Add the physical offset for nvswitch connected peer mappings
if (uvm_aperture_is_peer(aperture) &&
uvm_parent_gpus_are_nvswitch_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
// Add the physical offset for peer mappings
if (uvm_aperture_is_peer(aperture)) {
if (uvm_parent_gpus_are_direct_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->peer_address_info.peer_gpa_memory_window_start;
else if (uvm_parent_gpus_are_nvswitch_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
}
for (index = 0; index < ext_mapping_info->numWrittenPtes; index++) {

18
kernel-open/nvidia-uvm/uvm_gpu.c
View File

@@ -107,6 +107,7 @@ static void fill_parent_gpu_info(uvm_parent_gpu_t *parent_gpu, const UvmGpuInfo
}
parent_gpu->nvswitch_info.is_nvswitch_connected = gpu_info->connectedToSwitch;
parent_gpu->peer_address_info.is_direct_connected = gpu_info->nvlDirectConnect;
// nvswitch is routed via physical pages, where the upper 13-bits of the
// 47-bit address space holds the routing information for each peer.
@@ -115,6 +116,9 @@ static void fill_parent_gpu_info(uvm_parent_gpu_t *parent_gpu, const UvmGpuInfo
parent_gpu->nvswitch_info.fabric_memory_window_start = gpu_info->nvswitchMemoryWindowStart;
parent_gpu->nvswitch_info.egm_fabric_memory_window_start = gpu_info->nvswitchEgmMemoryWindowStart;
}
else if (parent_gpu->peer_address_info.is_direct_connected) {
parent_gpu->peer_address_info.peer_gpa_memory_window_start = gpu_info->nvlDirectConnectMemoryWindowStart;
}
parent_gpu->ats.non_pasid_ats_enabled = gpu_info->nonPasidAtsSupport;
@@ -2110,6 +2114,16 @@ bool uvm_parent_gpus_are_nvswitch_connected(const uvm_parent_gpu_t *parent_gpu0,
return false;
}
bool uvm_parent_gpus_are_direct_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1)
{
if (parent_gpu0 != parent_gpu1 &&
parent_gpu0->peer_address_info.is_direct_connected &&
parent_gpu1->peer_address_info.is_direct_connected)
return true;
return false;
}
NV_STATUS uvm_gpu_check_ecc_error_no_rm(uvm_gpu_t *gpu)
{
// We may need to call service_interrupts() which cannot be done in the top
@@ -3068,7 +3082,9 @@ uvm_gpu_phys_address_t uvm_gpu_peer_phys_address(uvm_gpu_t *owning_gpu, NvU64 ad
{
uvm_aperture_t aperture = uvm_gpu_peer_aperture(accessing_gpu, owning_gpu);
if (uvm_parent_gpus_are_nvswitch_connected(accessing_gpu->parent, owning_gpu->parent))
if (uvm_parent_gpus_are_direct_connected(accessing_gpu->parent, owning_gpu->parent))
address += owning_gpu->parent->peer_address_info.peer_gpa_memory_window_start;
else if (uvm_parent_gpus_are_nvswitch_connected(accessing_gpu->parent, owning_gpu->parent))
address += owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
return uvm_gpu_phys_address(aperture, address);

16
kernel-open/nvidia-uvm/uvm_gpu.h
View File

@@ -1365,6 +1365,20 @@ struct uvm_parent_gpu_struct
NvU64 base_address;
} egm;
// Peer VIDMEM base offset used when creating GPA PTEs for
// peer mappings. RM will set this offset on systems where
// peer accesses are not zero-based (NVLINK 5+).
struct
{
// Is the GPU directly connected to peer GPUs.
bool is_direct_connected;
// 48-bit fabric memory physical offset that peer gpus need in order
// access to be rounted to the correct peer.
// Each memory window is 4TB, so the upper 6 bits are used for rounting.
NvU64 peer_gpa_memory_window_start;
} peer_address_info;
uvm_test_parent_gpu_inject_error_t test;
// PASID ATS
@@ -1619,6 +1633,8 @@ uvm_aperture_t uvm_gpu_egm_peer_aperture(uvm_parent_gpu_t *local_gpu, uvm_parent
bool uvm_parent_gpus_are_nvswitch_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1);
bool uvm_parent_gpus_are_direct_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1);
static bool uvm_gpus_are_smc_peers(const uvm_gpu_t *gpu0, const uvm_gpu_t *gpu1)
{
UVM_ASSERT(gpu0 != gpu1);

10
kernel-open/nvidia-uvm/uvm_hal.c
View File

@@ -221,7 +221,7 @@ static uvm_hal_class_ops_t host_table[] =
.access_counter_clear_all = uvm_hal_maxwell_access_counter_clear_all_unsupported,
.access_counter_clear_targeted = uvm_hal_maxwell_access_counter_clear_targeted_unsupported,
.access_counter_query_clear_op = uvm_hal_maxwell_access_counter_query_clear_op_unsupported,
.l2_invalidate_noncoh_sysmem = uvm_hal_host_l2_invalidate_noncoh_sysmem_unsupported,
.l2_invalidate = uvm_hal_host_l2_invalidate_unsupported,
.get_time = uvm_hal_maxwell_get_time,
}
},
@@ -287,6 +287,7 @@ static uvm_hal_class_ops_t host_table[] =
.tlb_invalidate_all = uvm_hal_ampere_host_tlb_invalidate_all,
.tlb_invalidate_va = uvm_hal_ampere_host_tlb_invalidate_va,
.tlb_invalidate_test = uvm_hal_ampere_host_tlb_invalidate_test,
.l2_invalidate = uvm_hal_ampere_host_l2_invalidate,
}
},
{
@@ -315,8 +316,8 @@ static uvm_hal_class_ops_t host_table[] =
.tlb_invalidate_phys = uvm_hal_blackwell_host_tlb_invalidate_phys,
.tlb_invalidate_test = uvm_hal_blackwell_host_tlb_invalidate_test,
.tlb_flush_prefetch = uvm_hal_blackwell_host_tlb_flush_prefetch,
.l2_invalidate_noncoh_sysmem = uvm_hal_blackwell_host_l2_invalidate_noncoh_sysmem,
.access_counter_query_clear_op = uvm_hal_blackwell_access_counter_query_clear_op_gb100,
.l2_invalidate = uvm_hal_blackwell_host_l2_invalidate,
}
},
{
@@ -1162,10 +1163,11 @@ void uvm_hal_ce_memcopy_patch_src_stub(uvm_push_t *push, uvm_gpu_address_t *src)
{
}
void uvm_hal_host_l2_invalidate_noncoh_sysmem_unsupported(uvm_push_t *push)
void uvm_hal_host_l2_invalidate_unsupported(uvm_push_t *push, uvm_aperture_t aperture)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
UVM_ERR_PRINT("L2 cache invalidation: Called on unsupported GPU %s (arch: 0x%x, impl: 0x%x)\n",
uvm_gpu_name(gpu), gpu->parent->rm_info.gpuArch, gpu->parent->rm_info.gpuImplementation);
UVM_ASSERT_MSG(false, "host l2_invalidate_noncoh_sysmem called on unsupported GPU\n");
UVM_ASSERT_MSG(false, "L2 invalidate is not supported on %s",
uvm_parent_gpu_name(gpu->parent));
}

13
kernel-open/nvidia-uvm/uvm_hal.h
View File

@@ -248,11 +248,12 @@ typedef void (*uvm_hal_host_tlb_flush_prefetch_t)(uvm_push_t *push);
void uvm_hal_maxwell_host_tlb_flush_prefetch_unsupported(uvm_push_t *push);
void uvm_hal_blackwell_host_tlb_flush_prefetch(uvm_push_t *push);
// L2 cache invalidate for non-coherent sysmem for systems with write back cache.
// These are iGPUs as of now.
typedef void (*uvm_hal_host_l2_invalidate_noncoh_sysmem_t)(uvm_push_t *push);
void uvm_hal_blackwell_host_l2_invalidate_noncoh_sysmem(uvm_push_t *push);
void uvm_hal_host_l2_invalidate_noncoh_sysmem_unsupported(uvm_push_t *push);
// Performs L2 cache invalidation for peer or system memory.
typedef void (*uvm_hal_host_l2_invalidate_t)(uvm_push_t *push, uvm_aperture_t aperture);
void uvm_hal_blackwell_host_l2_invalidate(uvm_push_t *push, uvm_aperture_t aperture);
void uvm_hal_ampere_host_l2_invalidate(uvm_push_t *push, uvm_aperture_t aperture);
void uvm_hal_host_l2_invalidate_unsupported(uvm_push_t *push, uvm_aperture_t aperture);
// By default all semaphore release operations include a membar sys before the
// operation. This can be affected by using UVM_PUSH_FLAG_NEXT_* flags with
@@ -822,7 +823,7 @@ struct uvm_host_hal_struct
uvm_hal_host_tlb_invalidate_phys_t tlb_invalidate_phys;
uvm_hal_host_tlb_invalidate_test_t tlb_invalidate_test;
uvm_hal_host_tlb_flush_prefetch_t tlb_flush_prefetch;
uvm_hal_host_l2_invalidate_noncoh_sysmem_t l2_invalidate_noncoh_sysmem;
uvm_hal_host_l2_invalidate_t l2_invalidate;
uvm_hal_fault_buffer_replay_t replay_faults;
uvm_hal_fault_cancel_global_t cancel_faults_global;
uvm_hal_fault_cancel_targeted_t cancel_faults_targeted;

17
kernel-open/nvidia-uvm/uvm_map_external.c
View File

@@ -1276,11 +1276,20 @@ void uvm_ext_gpu_map_destroy(uvm_va_range_external_t *external_range,
range_tree = uvm_ext_gpu_range_tree(external_range, mapped_gpu);
// Perform L2 cache invalidation for noncoherent sysmem mappings.
// This is done only on systems with write-back cache which is iGPUs as of now.
// Perform L2 cache invalidation for cached peer and sysmem mappings.
if (ext_gpu_map->need_l2_invalidate_at_unmap) {
UVM_ASSERT(ext_gpu_map->gpu->parent->is_integrated_gpu);
status = uvm_mmu_l2_invalidate_noncoh_sysmem(mapped_gpu);
uvm_aperture_t aperture;
// Peer cache invalidation is not targeted to a specific peer, so we
// just use UVM_APERTURE_PEER(0).
if (ext_gpu_map->is_egm)
aperture = UVM_APERTURE_PEER(0);
else if (ext_gpu_map->is_sysmem)
aperture = UVM_APERTURE_SYS;
else
aperture = UVM_APERTURE_PEER(0);
status = uvm_mmu_l2_invalidate(mapped_gpu, aperture);
UVM_ASSERT(status == NV_OK);
}

10
kernel-open/nvidia-uvm/uvm_mmu.c
View File

@@ -2974,25 +2974,21 @@ NV_STATUS uvm_mmu_tlb_invalidate_phys(uvm_gpu_t *gpu)
return uvm_push_end_and_wait(&push);
}
NV_STATUS uvm_mmu_l2_invalidate_noncoh_sysmem(uvm_gpu_t *gpu)
NV_STATUS uvm_mmu_l2_invalidate(uvm_gpu_t *gpu, uvm_aperture_t aperture)
{
uvm_push_t push;
NV_STATUS status;
// L2 cache invalidation is only done for systems with write-back
// cache which is iGPUs as of now.
UVM_ASSERT(gpu->parent->is_integrated_gpu);
status = uvm_push_begin(gpu->channel_manager,
UVM_CHANNEL_TYPE_MEMOPS,
&push,
"L2 cache invalidate for sysmem");
"L2 cache invalidate");
if (status != NV_OK) {
UVM_ERR_PRINT("L2 cache invalidation: Failed to begin push, status: %s\n", nvstatusToString(status));
return status;
}
gpu->parent->host_hal->l2_invalidate_noncoh_sysmem(&push);
gpu->parent->host_hal->l2_invalidate(&push, aperture);
status = uvm_push_end_and_wait(&push);
if (status != NV_OK)

5
kernel-open/nvidia-uvm/uvm_mmu.h
View File

@@ -722,9 +722,8 @@ uvm_gpu_address_t uvm_mmu_gpu_address(uvm_gpu_t *gpu, uvm_gpu_phys_address_t phy
// dma addresses, IOVAs, and GPAs). See uvm_dma_map_invalidation_t.
NV_STATUS uvm_mmu_tlb_invalidate_phys(uvm_gpu_t *gpu);
// Invalidate L2 cache when noncoherent sysmem mappings are unmapped.
// This is done for systems with write-back cache i.e. iGPUs as of now.
NV_STATUS uvm_mmu_l2_invalidate_noncoh_sysmem(uvm_gpu_t *gpu);
// Invalidate L2 cache for peer or system memory.
NV_STATUS uvm_mmu_l2_invalidate(uvm_gpu_t *gpu, uvm_aperture_t aperture);
NV_STATUS uvm_test_invalidate_tlb(UVM_TEST_INVALIDATE_TLB_PARAMS *params, struct file *filp);

8
kernel-open/nvidia-uvm/uvm_va_range.h
View File

@@ -204,8 +204,12 @@ typedef struct
uvm_deferred_free_object_t deferred_free;
// Flag indicating whether L2 cache invalidation is needed at unmap time.
// This is set by RM during mapping and used during unmap to determine
// if L2 cache invalidation should be performed for non coherent sysmem.
// This is set by RM during mapping and used during unmap to determine if L2
// cache invalidation should be performed. For GPU cached system memory
// allocations on systems a write-back cache this is required for
// correctness. For GPU cached peer and system memory on systems with a
// write-through cache the invalidation could be done by RM at map time
// however this introduces overhead during performance sensitive sections.
bool need_l2_invalidate_at_unmap;
} uvm_ext_gpu_map_t;