NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-08 00:59:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

535.43.02

Browse Source
This commit is contained in:
Andy Ritger
2023-05-30 10:11:36 -07:00
parent 6dd092ddb7
commit eb5c7665a1
1403 changed files with 295367 additions and 86235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
kernel-open/nvidia-uvm/uvm_hal.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2022 NVIDIA Corporation
Copyright (c) 2015-2023 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -53,6 +53,7 @@ MODULE_PARM_DESC(uvm_downgrade_force_membar_sys, "Force all TLB invalidation dow
#define ARCH_OP_COUNT (sizeof(uvm_arch_hal_t) / sizeof(void *))
#define FAULT_BUFFER_OP_COUNT (sizeof(uvm_fault_buffer_hal_t) / sizeof(void *))
#define ACCESS_COUNTER_BUFFER_OP_COUNT (sizeof(uvm_access_counter_buffer_hal_t) / sizeof(void *))
#define SEC2_OP_COUNT (sizeof(uvm_sec2_hal_t) / sizeof(void *))
// Table for copy engine functions.
// Each entry is associated with a copy engine class through the 'class' field.
@@ -73,6 +74,7 @@ static uvm_hal_class_ops_t ce_table[] =
.offset_in_out = uvm_hal_maxwell_ce_offset_in_out,
.phys_mode = uvm_hal_maxwell_ce_phys_mode,
.plc_mode = uvm_hal_maxwell_ce_plc_mode,
.memcopy_copy_type = uvm_hal_maxwell_ce_memcopy_copy_type,
.memcopy_is_valid = uvm_hal_ce_memcopy_is_valid_stub,
.memcopy_patch_src = uvm_hal_ce_memcopy_patch_src_stub,
.memcopy = uvm_hal_maxwell_ce_memcopy,
@@ -82,6 +84,8 @@ static uvm_hal_class_ops_t ce_table[] =
.memset_4 = uvm_hal_maxwell_ce_memset_4,
.memset_8 = uvm_hal_maxwell_ce_memset_8,
.memset_v_4 = uvm_hal_maxwell_ce_memset_v_4,
.encrypt = uvm_hal_maxwell_ce_encrypt_unsupported,
.decrypt = uvm_hal_maxwell_ce_decrypt_unsupported,
}
},
{
@@ -149,11 +153,14 @@ static uvm_hal_class_ops_t ce_table[] =
.semaphore_reduction_inc = uvm_hal_hopper_ce_semaphore_reduction_inc,
.offset_out = uvm_hal_hopper_ce_offset_out,
.offset_in_out = uvm_hal_hopper_ce_offset_in_out,
.memcopy_copy_type = uvm_hal_hopper_ce_memcopy_copy_type,
.memset_1 = uvm_hal_hopper_ce_memset_1,
.memset_4 = uvm_hal_hopper_ce_memset_4,
.memset_8 = uvm_hal_hopper_ce_memset_8,
.memcopy_is_valid = uvm_hal_hopper_ce_memcopy_is_valid,
.memset_is_valid = uvm_hal_hopper_ce_memset_is_valid,
.encrypt = uvm_hal_hopper_ce_encrypt,
.decrypt = uvm_hal_hopper_ce_decrypt,
},
},
};
@@ -371,6 +378,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.entry_clear_valid = uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported,
.entry_size = uvm_hal_maxwell_fault_buffer_entry_size_unsupported,
.parse_non_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_non_replayable_entry_unsupported,
.get_fault_type = uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported,
}
},
{
@@ -392,6 +400,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.entry_is_valid = uvm_hal_pascal_fault_buffer_entry_is_valid,
.entry_clear_valid = uvm_hal_pascal_fault_buffer_entry_clear_valid,
.entry_size = uvm_hal_pascal_fault_buffer_entry_size,
.get_fault_type = uvm_hal_pascal_fault_buffer_get_fault_type,
}
},
{
@@ -404,6 +413,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
.get_ve_id = uvm_hal_volta_fault_buffer_get_ve_id,
.parse_entry = uvm_hal_volta_fault_buffer_parse_entry,
.parse_non_replayable_entry = uvm_hal_volta_fault_buffer_parse_non_replayable_entry,
.get_fault_type = uvm_hal_volta_fault_buffer_get_fault_type,
}
},
{
@@ -495,6 +505,59 @@ static uvm_hal_class_ops_t access_counter_buffer_table[] =
},
};
static uvm_hal_class_ops_t sec2_table[] =
{
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.sec2_ops = {
.init = uvm_hal_maxwell_sec2_init_noop,
.decrypt = uvm_hal_maxwell_sec2_decrypt_unsupported,
.semaphore_release = uvm_hal_maxwell_sec2_semaphore_release_unsupported,
.semaphore_timestamp = uvm_hal_maxwell_sec2_semaphore_timestamp_unsupported,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100,
.u.sec2_ops = {
.init = uvm_hal_hopper_sec2_init,
.semaphore_release = uvm_hal_hopper_sec2_semaphore_release,
.semaphore_timestamp = uvm_hal_hopper_sec2_semaphore_timestamp_unsupported,
.decrypt = uvm_hal_hopper_sec2_decrypt,
}
},
};
static inline uvm_hal_class_ops_t *ops_find_by_id(uvm_hal_class_ops_t *table, NvU32 row_count, NvU32 id)
{
NvLength i;
@@ -598,6 +661,15 @@ NV_STATUS uvm_hal_init_table(void)
return status;
}
status = ops_init_from_parent(sec2_table,
ARRAY_SIZE(sec2_table),
SEC2_OP_COUNT,
offsetof(uvm_hal_class_ops_t, u.sec2_ops));
if (status != NV_OK) {
UVM_ERR_PRINT("ops_init_from_parent(sec2_table) failed: %s\n", nvstatusToString(status));
return status;
}
return NV_OK;
}
@@ -648,6 +720,14 @@ NV_STATUS uvm_hal_init_gpu(uvm_parent_gpu_t *parent_gpu)
parent_gpu->access_counter_buffer_hal = &class_ops->u.access_counter_buffer_ops;
class_ops = ops_find_by_id(sec2_table, ARRAY_SIZE(sec2_table), gpu_info->gpuArch);
if (class_ops == NULL) {
UVM_ERR_PRINT("SEC2 HAL not found, GPU %s, arch: 0x%X\n", parent_gpu->name, gpu_info->gpuArch);
return NV_ERR_INVALID_CLASS;
}
parent_gpu->sec2_hal = &class_ops->u.sec2_ops;
return NV_OK;
}
@@ -658,6 +738,9 @@ static void hal_override_properties(uvm_parent_gpu_t *parent_gpu)
// TODO: Bug 200692962: Add support for access counters in vGPU
if (parent_gpu->virt_mode != UVM_VIRT_MODE_NONE)
parent_gpu->access_counters_supported = false;
// Access counters are not supported in CC.
else if (uvm_conf_computing_mode_enabled_parent(parent_gpu))
parent_gpu->access_counters_supported = false;
}
void uvm_hal_init_properties(uvm_parent_gpu_t *parent_gpu)
@@ -711,7 +794,7 @@ uvm_membar_t uvm_hal_downgrade_membar_type(uvm_gpu_t *gpu, bool is_local_vidmem)
// memory, including those from other processors like the CPU or peer GPUs,
// must come through this GPU's L2. In all current architectures, MEMBAR_GPU
// is sufficient to resolve ordering at the L2 level.
if (is_local_vidmem && !gpu->parent->numa_info.enabled && !uvm_downgrade_force_membar_sys)
if (is_local_vidmem && !uvm_gpu_is_coherent(gpu->parent) && !uvm_downgrade_force_membar_sys)
return UVM_MEMBAR_GPU;
// If the mapped memory was remote, or if a coherence protocol can cache
@@ -895,7 +978,7 @@ void uvm_hal_ce_memcopy_patch_src_stub(uvm_push_t *push, uvm_gpu_address_t *src)
{
}
bool uvm_hal_ce_memset_is_valid_stub(uvm_push_t *push, uvm_gpu_address_t dst, size_t element_size)
bool uvm_hal_ce_memset_is_valid_stub(uvm_push_t *push, uvm_gpu_address_t dst, size_t num_elements, size_t element_size)
{
return true;
}