NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-30 21:19:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

580.94.16

Browse Source
This commit is contained in:
russellcnv
2026-01-22 14:46:29 -08:00
parent a3af2867b7
commit c273d84a8b
68 changed files with 160855 additions and 160151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
kernel-open/nvidia/nv-dma.c
View File

@@ -718,7 +718,11 @@ static NvBool nv_dma_use_map_resource
#endif
}
#if defined(NV_DMA_MAP_OPS_HAS_MAP_PHYS)
return (ops->map_phys != NULL);
#else
return (ops->map_resource != NULL);
#endif
}
/* DMA-map a peer device's C2C aperture for peer access. */

37
kernel-open/nvidia/nv-dmabuf.c
View File

@@ -253,7 +253,7 @@ nv_inc_and_check_one_phys_refcount(
return is_one;
}
// Must be called with RMAPI lock and GPU lock taken
// Must be called with RMAPI lock and all-GPU lock taken
static void
nv_dma_buf_undup_mem_handles_unlocked(
nvidia_stack_t *sp,
@@ -284,6 +284,28 @@ nv_dma_buf_undup_mem_handles_unlocked(
}
}
// Must be called with RMAPI lock taken
static void
nv_dma_buf_undup_mem_handles_gpus_locked(
nvidia_stack_t *sp,
NvU32 start_index,
NvU32 num_objects,
nv_dma_buf_file_private_t *priv
)
{
NV_STATUS status;
status = rm_acquire_all_gpus_lock(sp);
if (WARN_ON(status != NV_OK))
{
return;
}
nv_dma_buf_undup_mem_handles_unlocked(sp, start_index, num_objects, priv);
rm_release_all_gpus_lock(sp);
}
static void
nv_dma_buf_undup_mem_handles(
nvidia_stack_t *sp,
@@ -300,17 +322,8 @@ nv_dma_buf_undup_mem_handles(
return;
}
status = rm_acquire_all_gpus_lock(sp);
if (WARN_ON(status != NV_OK))
{
goto unlock_api_lock;
}
nv_dma_buf_undup_mem_handles_gpus_locked(sp, index, num_objects, priv);
nv_dma_buf_undup_mem_handles_unlocked(sp, index, num_objects, priv);
rm_release_all_gpus_lock(sp);
unlock_api_lock:
rm_release_api_lock(sp);
}
@@ -478,7 +491,7 @@ failed:
//
nv_dma_buf_release_gpu_lock(sp, priv);
nv_dma_buf_undup_mem_handles(sp, params->index, count, priv);
nv_dma_buf_undup_mem_handles_gpus_locked(sp, params->index, count, priv);
}
else
{

18
kernel-open/nvidia/nv-mmap.c
View File

@@ -72,7 +72,7 @@ nvidia_vma_open(struct vm_area_struct *vma)
if (at != NULL)
{
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
}
@@ -128,6 +128,9 @@ nvidia_vma_access(
void *kernel_mapping;
const nv_alloc_mapping_context_t *mmap_context = &nvlfp->mmap_context;
NvU64 offsInVma = addr - vma->vm_start;
NvBool bIsNuma = NV_FALSE;
bIsNuma = pfn_valid(mmap_context->access_start >> PAGE_SHIFT);
pageIndex = (offsInVma >> PAGE_SHIFT);
pageOffset = (offsInVma & ~PAGE_MASK);
@@ -164,10 +167,17 @@ nvidia_vma_access(
pageIndex = nv_array_index_no_speculate(pageIndex, at->num_pages);
kernel_mapping = (void *)(at->page_table[pageIndex].virt_addr + pageOffset);
}
else if (bIsNuma)
{
struct page *pPage = NV_GET_PAGE_STRUCT(mmap_context->page_array[pageIndex]);
NvU8 *pPagePtr = (NvU8 *) page_address(pPage);
kernel_mapping = &pPagePtr[pageOffset];
}
else
{
NvU64 idx = 0;
NvU64 curOffs = 0;
for(; idx < mmap_context->memArea.numRanges; idx++)
{
NvU64 nextOffs = mmap_context->memArea.pRanges[idx].size + curOffs;
@@ -195,7 +205,7 @@ found:
else
memcpy(buffer, kernel_mapping, length);
if (at == NULL)
if (at == NULL && !bIsNuma)
{
kernel_mapping = ((char *)kernel_mapping - pageOffset);
os_unmap_kernel_space(kernel_mapping, PAGE_SIZE);
@@ -408,7 +418,7 @@ static int nvidia_mmap_sysmem(
int ret = 0;
unsigned long start = 0;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
start = vma->vm_start;
for (j = page_index; j < (page_index + pages); j++)
@@ -444,7 +454,7 @@ static int nvidia_mmap_sysmem(
if (ret)
{
NV_ATOMIC_DEC(at->usage_count);
atomic64_dec(&at->usage_count);
nv_printf(NV_DBG_ERRORS,
"NVRM: Userspace mapping creation failed [%d]!\n", ret);
return -EAGAIN;

18
kernel-open/nvidia/nv-pci.c
View File

@@ -229,7 +229,11 @@ static int nv_resize_pcie_bars(struct pci_dev *pci_dev) {
resize:
/* Attempt to resize BAR1 to the largest supported size */
#if defined(NV_PCI_RESIZE_RESOURCE_HAS_EXCLUDE_BARS_ARG)
r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size, 0);
#else
r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size);
#endif
if (r) {
if (r == -ENOSPC)
@@ -420,6 +424,12 @@ nv_init_coherent_link_info
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-base-pa", &pa) == 0)
{
nvl->coherent_link_info.gpu_mem_pa = pa;
NvU64 gpu_mem_size;
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-size", &gpu_mem_size) == 0)
{
nvl->coherent_link_info.gpu_mem_size = gpu_mem_size;
}
}
else
{
@@ -1774,7 +1784,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
* For eGPU, fall off the bus along with clients active is a valid scenario.
* Hence skipping the sanity check for eGPU.
*/
if ((NV_ATOMIC_READ(nvl->usage_count) != 0) && !(nv->is_external_gpu))
if ((atomic64_read(&nvl->usage_count) != 0) && !(nv->is_external_gpu))
{
nv_printf(NV_DBG_ERRORS,
"NVRM: Attempting to remove device %04x:%02x:%02x.%x with non-zero usage count!\n",
@@ -1785,7 +1795,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
* We can't return from this function without corrupting state, so we wait for
* the usage count to go to zero.
*/
while (NV_ATOMIC_READ(nvl->usage_count) != 0)
while (atomic64_read(&nvl->usage_count) != 0)
{
/*
@@ -1863,7 +1873,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
nvl->sysfs_config_file = NULL;
}
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv_lock_destroy_locks(sp, nv);
}
@@ -1879,7 +1889,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
num_nv_devices--;
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
NV_PCI_DISABLE_DEVICE(pci_dev);
NV_KFREE(nvl, sizeof(nv_linux_state_t));

2
kernel-open/nvidia/nv-procfs.c
View File

@@ -889,7 +889,7 @@ nv_procfs_close_unbind_lock(
down(&nvl->ldata_lock);
if ((value == 1) && !(nv->flags & NV_FLAG_UNBIND_LOCK))
{
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
rm_unbind_lock(sp, nv);
if (nv->flags & NV_FLAG_UNBIND_LOCK)

44
kernel-open/nvidia/nv.c
View File

@@ -413,7 +413,7 @@ nv_alloc_t *nvos_create_alloc(
return NULL;
}
NV_ATOMIC_SET(at->usage_count, 0);
atomic64_set(&at->usage_count, 0);
at->pid = os_get_current_process();
at->dev = dev;
@@ -428,7 +428,7 @@ int nvos_free_alloc(
if (at == NULL)
return -1;
if (NV_ATOMIC_READ(at->usage_count))
if (atomic64_read(&at->usage_count))
return 1;
kvfree(at->page_table);
@@ -1646,13 +1646,10 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
return -ENODEV;
}
if (unlikely(NV_ATOMIC_READ(nvl->usage_count) >= NV_S32_MAX))
return -EMFILE;
if ( ! (nv->flags & NV_FLAG_OPEN))
{
/* Sanity check: !NV_FLAG_OPEN requires usage_count == 0 */
if (NV_ATOMIC_READ(nvl->usage_count) != 0)
if (atomic64_read(&nvl->usage_count) != 0)
{
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
"Minor device %u is referenced without being open!\n",
@@ -1674,7 +1671,7 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
nv_assert_not_in_gpu_exclusion_list(sp, nv);
NV_ATOMIC_INC(nvl->usage_count);
atomic64_inc(&nvl->usage_count);
return 0;
}
@@ -2102,7 +2099,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv_printf(NV_DBG_ERRORS,
"NVRM: Attempting to close unopened minor device %u!\n",
@@ -2111,7 +2108,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
return;
}
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
if (atomic64_dec_and_test(&nvl->usage_count))
nv_stop_device(nv, sp);
}
@@ -2156,7 +2153,7 @@ nvidia_close_callback(
nv_close_device(nv, sp);
bRemove = (!NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv)) &&
(NV_ATOMIC_READ(nvl->usage_count) == 0) &&
(atomic64_read(&nvl->usage_count) == 0) &&
rm_get_device_remove_flag(sp, nv->gpu_id);
nv_free_file_private(nvlfp);
@@ -2175,7 +2172,7 @@ nvidia_close_callback(
* any cleanup related to linux layer locks and nv linux state struct.
* nvidia_pci_remove when scheduled will do necessary cleanup.
*/
if ((NV_ATOMIC_READ(nvl->usage_count) == 0) && nv->removed)
if ((atomic64_read(&nvl->usage_count) == 0) && nv->removed)
{
nv_lock_destroy_locks(sp, nv);
NV_KFREE(nvl, sizeof(nv_linux_state_t));
@@ -2694,7 +2691,7 @@ nvidia_ioctl(
* Only the current client should have an open file
* descriptor for the device, to allow safe offlining.
*/
if (NV_ATOMIC_READ(nvl->usage_count) > 1)
if (atomic64_read(&nvl->usage_count) > 1)
{
status = -EBUSY;
goto unlock;
@@ -3083,12 +3080,12 @@ nvidia_ctl_open(
/* save the nv away in file->private_data */
nvlfp->nvptr = nvl;
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
}
NV_ATOMIC_INC(nvl->usage_count);
atomic64_inc(&nvl->usage_count);
up(&nvl->ldata_lock);
return 0;
@@ -3113,7 +3110,7 @@ nvidia_ctl_close(
nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_close\n");
down(&nvl->ldata_lock);
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
if (atomic64_dec_and_test(&nvl->usage_count))
{
nv->flags &= ~NV_FLAG_OPEN;
}
@@ -3276,7 +3273,7 @@ nv_alias_pages(
at->guest_id = guest_id;
*priv_data = at;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
@@ -3589,7 +3586,7 @@ NV_STATUS NV_API_CALL nv_register_sgt(
at->order = get_order(at->num_pages * PAGE_SIZE);
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
*priv_data = at;
@@ -3620,7 +3617,7 @@ void NV_API_CALL nv_unregister_sgt(
*import_priv = at->import_priv;
}
if (NV_ATOMIC_DEC_AND_TEST(at->usage_count))
if (atomic64_dec_and_test(&at->usage_count))
{
nvos_free_alloc(at);
}
@@ -3893,7 +3890,7 @@ NV_STATUS NV_API_CALL nv_alloc_pages(
}
*priv_data = at;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
@@ -3929,7 +3926,7 @@ NV_STATUS NV_API_CALL nv_free_pages(
* This is described in greater detail in the comments above the
* nvidia_vma_(open|release)() callbacks in nv-mmap.c.
*/
if (!NV_ATOMIC_DEC_AND_TEST(at->usage_count))
if (!atomic64_dec_and_test(&at->usage_count))
return NV_OK;
if (!at->flags.guest && !at->import_sgt)
@@ -3958,7 +3955,7 @@ NvBool nv_lock_init_locks
NV_INIT_MUTEX(&nvl->mmap_lock);
NV_INIT_MUTEX(&nvl->open_q_lock);
NV_ATOMIC_SET(nvl->usage_count, 0);
atomic64_set(&nvl->usage_count, 0);
if (!rm_init_event_locks(sp, nv))
return NV_FALSE;
@@ -5342,6 +5339,7 @@ NV_STATUS NV_API_CALL nv_get_device_memory_config(
nv_state_t *nv,
NvU64 *compr_addr_sys_phys,
NvU64 *addr_guest_phys,
NvU64 *size_guest_phys,
NvU64 *rsvd_phys,
NvU32 *addr_width,
NvS32 *node_id
@@ -5365,6 +5363,10 @@ NV_STATUS NV_API_CALL nv_get_device_memory_config(
{
*addr_guest_phys = nvl->coherent_link_info.gpu_mem_pa;
}
if (size_guest_phys)
{
*size_guest_phys = nvl->coherent_link_info.gpu_mem_size;
}
if (rsvd_phys)
{
*rsvd_phys = nvl->coherent_link_info.rsvd_mem_pa;

5
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -145,6 +145,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += devm_of_icc_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_put
NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_set_bw
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_map_ops_has_map_phys
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map_atomic
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_attachment_has_peer2peer
@@ -158,6 +159,8 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += assign_str
NV_CONFTEST_FUNCTION_COMPILE_TESTS += ioasid_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += mm_pasid_drop
NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vma_flags_set_word
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_flags_set
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_gpl_sme_active
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_swiotlb_map_sg_attrs
@@ -205,7 +208,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += remove_memory_has_nid_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_driver_has_driver_managed_dma
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += memory_failure_queue_has_trapno_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += foll_longterm_present
NV_CONFTEST_TYPE_COMPILE_TESTS += bus_type_has_iommu_ops
@@ -215,6 +217,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += class_create_has_no_owner_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += class_devnode_has_const_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += devfreq_has_freq_table
NV_CONFTEST_TYPE_COMPILE_TESTS += has_enum_pidtype_tgid
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_resize_resource_has_exclude_bars_arg
NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build