NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-03-13 17:19:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

565.57.01

Browse Source
This commit is contained in:
Bernhard Stoeckner
2024-10-22 17:38:58 +02:00
parent ed4be64962
commit d5a0858f90
1049 changed files with 209491 additions and 167508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
kernel-open/nvidia/linux_nvswitch.c
View File

@@ -31,6 +31,7 @@
#include "nvCpuUuid.h"
#include "nv-time.h"
#include "nvlink_caps.h"
#include "nvlink_proto.h"
#include <linux/module.h>
#include <linux/interrupt.h>

1
kernel-open/nvidia/nv-caps-imex.c
View File

@@ -22,6 +22,7 @@
*/
#include "nv-linux.h"
#include "nv-caps-imex.h"
extern int NVreg_ImexChannelCount;
extern int NVreg_CreateImexChannel0;

2
kernel-open/nvidia/nv-caps.c
View File

@@ -269,7 +269,7 @@ static void nv_cap_procfs_exit(void)
nv_cap_procfs_dir = NULL;
}
int nv_cap_procfs_init(void)
static int nv_cap_procfs_init(void)
{
static struct proc_dir_entry *file_entry;

6
kernel-open/nvidia/nv-dma.c
View File

@@ -290,7 +290,7 @@ void nv_destroy_dma_map_scatterlist(nv_dma_map_t *dma_map)
os_free_mem(dma_map->mapping.discontig.submaps);
}
void nv_load_dma_map_scatterlist(
static void nv_load_dma_map_scatterlist(
nv_dma_map_t *dma_map,
NvU64 *va_array
)
@@ -486,7 +486,7 @@ NV_STATUS NV_API_CALL nv_dma_map_sgt(
return status;
}
NV_STATUS NV_API_CALL nv_dma_unmap_sgt(
static NV_STATUS NV_API_CALL nv_dma_unmap_sgt(
nv_dma_device_t *dma_dev,
void **priv
)
@@ -854,7 +854,7 @@ NV_STATUS NV_API_CALL nv_dma_map_peer
* convert to a bus address.
*/
NvU64 offset = *va - res->start;
*va = nv_pci_bus_address(peer_pci_dev, bar_index) + offset;
*va = pci_bus_address(peer_pci_dev, bar_index) + offset;
status = NV_OK;
}

55
kernel-open/nvidia/nv-dmabuf.c
View File

@@ -33,19 +33,16 @@ typedef struct nv_dma_buf_mem_handle
NvU64 size;
// RM memdesc specific data
void *static_mem_info;
void *mem_info;
//
// Refcount for phys addresses
// If refcount > 0, phys address ranges in phys_range are reused.
// If refcount > 0, phys address ranges in memArea are reused.
//
NvU64 phys_refcount;
// Number of scatterlist entries in phys_range[] array
NvU32 phys_range_count;
// List of phys address ranges to be used to dma map
nv_phys_addr_range_t *phys_range;
// Scatterlist of all the memory ranges associated with the buf
MemoryArea memArea;
} nv_dma_buf_mem_handle_t;
typedef struct nv_dma_buf_file_private
@@ -331,7 +328,7 @@ nv_dma_buf_dup_mem_handles(
priv->handles[index].h_memory = h_memory_duped;
priv->handles[index].offset = params->offsets[i];
priv->handles[index].size = params->sizes[i];
priv->handles[index].static_mem_info = mem_info;
priv->handles[index].mem_info = mem_info;
priv->num_objects++;
index++;
count++;
@@ -380,14 +377,14 @@ nv_put_phys_addresses(
continue;
}
// Per-handle phys_range is freed by RM
// Per-handle memArea is freed by RM
rm_dma_buf_unmap_mem_handle(sp, priv->nv, priv->h_client,
priv->handles[index].h_memory,
priv->handles[index].size,
&priv->handles[index].phys_range,
priv->handles[index].phys_range_count);
priv->handles[index].mem_info,
priv->static_phys_addrs,
priv->handles[index].memArea);
priv->handles[index].phys_range_count = 0;
priv->handles[index].memArea.numRanges = 0;
}
}
@@ -506,14 +503,14 @@ nv_dma_buf_get_phys_addresses (
continue;
}
// Per-handle phys_range is allocated by RM
// Per-handle memArea is allocated by RM
status = rm_dma_buf_map_mem_handle(sp, priv->nv, priv->h_client,
priv->handles[index].h_memory,
priv->handles[index].offset,
priv->handles[index].size,
priv->handles[index].static_mem_info,
&priv->handles[index].phys_range,
&priv->handles[index].phys_range_count);
mrangeMake(priv->handles[index].offset,
priv->handles[index].size),
priv->handles[index].mem_info,
priv->static_phys_addrs,
&priv->handles[index].memArea);
if (status != NV_OK)
{
goto unmap_handles;
@@ -602,7 +599,7 @@ nv_dma_buf_map_pages (
// Calculate nents needed to allocate sg_table
for (i = 0; i < priv->num_objects; i++)
{
nents += priv->handles[i].phys_range_count;
nents += priv->handles[i].memArea.numRanges;
}
NV_KZALLOC(sgt, sizeof(struct sg_table));
@@ -621,12 +618,12 @@ nv_dma_buf_map_pages (
for (i = 0; i < priv->num_objects; i++)
{
NvU32 range_count = priv->handles[i].phys_range_count;
NvU32 range_count = priv->handles[i].memArea.numRanges;
NvU32 index = 0;
for (index = 0; index < range_count; index++)
{
NvU64 addr = priv->handles[i].phys_range[index].addr;
NvU64 len = priv->handles[i].phys_range[index].len;
NvU64 addr = priv->handles[i].memArea.pRanges[index].start;
NvU64 len = priv->handles[i].memArea.pRanges[index].size;
struct page *page = NV_GET_PAGE_STRUCT(addr);
if ((page == NULL) || (sg == NULL))
@@ -634,7 +631,7 @@ nv_dma_buf_map_pages (
goto free_table;
}
sg_set_page(sg, page, len, 0);
sg_set_page(sg, page, len, NV_GET_OFFSET_IN_PAGE(addr));
sg = sg_next(sg);
}
}
@@ -687,12 +684,12 @@ nv_dma_buf_map_pfns (
// Calculate nents needed to allocate sg_table
for (i = 0; i < priv->num_objects; i++)
{
NvU32 range_count = priv->handles[i].phys_range_count;
NvU32 range_count = priv->handles[i].memArea.numRanges;
NvU32 index;
for (index = 0; index < range_count; index++)
{
NvU64 length = priv->handles[i].phys_range[index].len;
NvU64 length = priv->handles[i].memArea.pRanges[index].size;
NvU64 count = length + dma_max_seg_size - 1;
do_div(count, dma_max_seg_size);
nents += count;
@@ -714,13 +711,13 @@ nv_dma_buf_map_pfns (
sg = sgt->sgl;
for (i = 0; i < priv->num_objects; i++)
{
NvU32 range_count = priv->handles[i].phys_range_count;
NvU32 range_count = priv->handles[i].memArea.numRanges;
NvU32 index = 0;
for (index = 0; index < range_count; index++)
{
NvU64 dma_addr = priv->handles[i].phys_range[index].addr;
NvU64 dma_len = priv->handles[i].phys_range[index].len;
NvU64 dma_addr = priv->handles[i].memArea.pRanges[index].start;
NvU64 dma_len = priv->handles[i].memArea.pRanges[index].size;
// Break the scatterlist into dma_max_seg_size chunks
while(dma_len != 0)

2
kernel-open/nvidia/nv-i2c.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2005-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2005-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a

4
kernel-open/nvidia/nv-ibmnpu.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2017-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -25,9 +25,9 @@
* nv-ibmnpu.c - interface with the ibmnpu (IBM NVLink Processing Unit) "module"
*/
#include "nv-linux.h"
#include "nv-ibmnpu.h"
#if defined(NVCPU_PPC64LE)
#include "nv-ibmnpu.h"
#include "nv-rsync.h"
/*

2
kernel-open/nvidia/nv-kthread-q.c
View File

@@ -201,7 +201,7 @@ static struct task_struct *thread_create_on_node(int (*threadfn)(void *data),
// Ran out of attempts - return thread even if its stack may not be
// allocated on the preferred node
if ((i == (attempts - 1)))
if (i == (attempts - 1))
break;
// Get the NUMA node where the first page of the stack is resident. If

103
kernel-open/nvidia/nv-mmap.c
View File

@@ -127,10 +127,10 @@ nvidia_vma_access(
NvU32 pageIndex, pageOffset;
void *kernel_mapping;
const nv_alloc_mapping_context_t *mmap_context = &nvlfp->mmap_context;
NvU64 offset;
NvU64 offsInVma = addr - vma->vm_start;
pageIndex = ((addr - vma->vm_start) >> PAGE_SHIFT);
pageOffset = (addr & ~PAGE_MASK);
pageIndex = (offsInVma >> PAGE_SHIFT);
pageOffset = (offsInVma & ~PAGE_MASK);
if (length < 0)
{
@@ -143,8 +143,6 @@ nvidia_vma_access(
return -EINVAL;
}
offset = mmap_context->mmap_start;
if (nv->flags & NV_FLAG_CONTROL)
{
at = NV_VMA_PRIVATE(vma);
@@ -170,17 +168,29 @@ nvidia_vma_access(
#endif
kernel_mapping = (void *)(at->page_table[pageIndex]->virt_addr + pageOffset);
}
else if (IS_FB_OFFSET(nv, offset, length))
else
{
addr = (offset & PAGE_MASK);
NvU64 idx = 0;
NvU64 curOffs = 0;
for(; idx < mmap_context->memArea.numRanges; idx++)
{
NvU64 nextOffs = mmap_context->memArea.pRanges[idx].size + curOffs;
if (curOffs <= offsInVma && nextOffs > offsInVma)
{
NvU64 realAddr = offsInVma - curOffs + mmap_context->memArea.pRanges[idx].start;
addr = realAddr & PAGE_MASK;
goto found;
}
curOffs = nextOffs;
}
return -EINVAL;
found:
kernel_mapping = os_map_kernel_space(addr, PAGE_SIZE, NV_MEMORY_UNCACHED);
if (kernel_mapping == NULL)
return -ENOMEM;
kernel_mapping = ((char *)kernel_mapping + pageOffset);
}
else
return -EINVAL;
length = NV_MIN(length, (int)(PAGE_SIZE - pageOffset));
@@ -213,10 +223,6 @@ static vm_fault_t nvidia_fault(
nv_state_t *nv = NV_STATE_PTR(nvl);
vm_fault_t ret = VM_FAULT_NOPAGE;
NvU64 page;
NvU64 num_pages = NV_VMA_SIZE(vma) >> PAGE_SHIFT;
NvU64 pfn_start = (nvlfp->mmap_context.mmap_start >> PAGE_SHIFT);
if (vma->vm_pgoff != 0)
{
return VM_FAULT_SIGBUS;
@@ -228,6 +234,7 @@ static vm_fault_t nvidia_fault(
return VM_FAULT_SIGBUS;
}
// Wake up GPU and reinstate mappings only if we are not in S3/S4 entry
if (!down_read_trylock(&nv_system_pm_lock))
{
@@ -269,24 +276,35 @@ static vm_fault_t nvidia_fault(
up_read(&nv_system_pm_lock);
return VM_FAULT_NOPAGE;
}
// Safe to mmap, map all pages in this VMA.
for (page = 0; page < num_pages; page++)
{
NvU64 virt_addr = vma->vm_start + (page << PAGE_SHIFT);
NvU64 pfn = pfn_start + page;
ret = nv_insert_pfn(vma, virt_addr, pfn,
nvlfp->mmap_context.remap_prot_extra);
if (ret != VM_FAULT_NOPAGE)
NvU64 idx;
NvU64 curOffs = 0;
NvBool bRevoked = NV_TRUE;
nv_alloc_mapping_context_t *mmap_context = &nvlfp->mmap_context;
for(idx = 0; idx < mmap_context->memArea.numRanges; idx++)
{
nv_printf(NV_DBG_ERRORS,
"NVRM: VM: nv_insert_pfn failed: %x\n", ret);
break;
NvU64 nextOffs = curOffs + mmap_context->memArea.pRanges[idx].size;
NvU64 pfn = mmap_context->memArea.pRanges[idx].start >> PAGE_SHIFT;
NvU64 numPages = mmap_context->memArea.pRanges[idx].size >> PAGE_SHIFT;
while (numPages != 0)
{
ret = nv_insert_pfn(vma, curOffs + vma->vm_start, pfn,
mmap_context->remap_prot_extra);
if (ret != VM_FAULT_NOPAGE)
{
goto err;
}
bRevoked = NV_FALSE;
curOffs += PAGE_SIZE;
pfn++;
numPages--;
}
curOffs = nextOffs;
}
nvl->all_mappings_revoked = NV_FALSE;
err:
nvl->all_mappings_revoked &= bRevoked;
}
up(&nvl->mmap_lock);
up_read(&nv_system_pm_lock);
@@ -384,7 +402,7 @@ static int nvidia_mmap_peer_io(
start = at->page_table[page_index]->phys_addr;
size = pages * PAGE_SIZE;
ret = nv_io_remap_page_range(vma, start, size, 0);
ret = nv_io_remap_page_range(vma, start, size, 0, vma->vm_start);
return ret;
}
@@ -416,14 +434,16 @@ static int nvidia_mmap_sysmem(
nv_speculation_barrier();
#endif
if (
#if defined(NV_VGPU_KVM_BUILD)
if (at->flags.guest)
at->flags.guest ||
#endif
at->flags.carveout)
{
ret = nv_remap_page_range(vma, start, at->page_table[j]->phys_addr,
PAGE_SIZE, vma->vm_page_prot);
}
else
#endif
{
vma->vm_page_prot = nv_adjust_pgprot(vma->vm_page_prot, 0);
ret = vm_insert_page(vma, start,
@@ -525,13 +545,11 @@ int nvidia_mmap_helper(
if (!NV_IS_CTL_DEVICE(nv))
{
NvU32 remap_prot_extra = mmap_context->remap_prot_extra;
NvU64 mmap_start = mmap_context->mmap_start;
NvU64 mmap_length = mmap_context->mmap_size;
NvU64 access_start = mmap_context->access_start;
NvU64 access_len = mmap_context->access_size;
// validate the size
if (NV_VMA_SIZE(vma) != mmap_length)
// Ensure size is correct.
if (NV_VMA_SIZE(vma) != memareaSize(mmap_context->memArea))
{
return -ENXIO;
}
@@ -590,11 +608,20 @@ int nvidia_mmap_helper(
}
else
{
if (nv_io_remap_page_range(vma, mmap_start, mmap_length,
remap_prot_extra) != 0)
NvU64 idx = 0;
NvU64 curOffs = 0;
for(; idx < mmap_context->memArea.numRanges; idx++)
{
up(&nvl->mmap_lock);
return -EAGAIN;
NvU64 nextOffs = curOffs + mmap_context->memArea.pRanges[idx].size;
if (nv_io_remap_page_range(vma,
mmap_context->memArea.pRanges[idx].start,
mmap_context->memArea.pRanges[idx].size,
remap_prot_extra, vma->vm_start + curOffs) != 0)
{
up(&nvl->mmap_lock);
return -EAGAIN;
}
curOffs = nextOffs;
}
}
}

4
kernel-open/nvidia/nv-modeset-interface.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2015 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -143,4 +143,4 @@ NV_STATUS nvidia_get_rm_ops(nvidia_modeset_rm_ops_t *rm_ops)
return NV_OK;
}
EXPORT_SYMBOL(nvidia_get_rm_ops);
NV_EXPORT_SYMBOL(nvidia_get_rm_ops);

42
kernel-open/nvidia/nv-pci.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -37,9 +37,14 @@
#include <linux/kernfs.h>
#endif
#include "detect-self-hosted.h"
#if !defined(NV_BUS_TYPE_HAS_IOMMU_OPS)
#include <linux/iommu.h>
#endif
#if NV_IS_EXPORT_SYMBOL_GPL_pci_ats_supported
#include <linux/pci-ats.h>
#endif
static void
nv_check_and_exclude_gpu(
@@ -434,22 +439,28 @@ nv_init_coherent_link_info
nvl->coherent_link_info.gpu_mem_pa =
NV_PCI_RESOURCE_START(nvl->pci_dev, gpu_bar2_offset);
if ((NV_PCI_RESOURCE_VALID(nvl->pci_dev, gpu_bar1_offset)) &&
if ((pci_devid_is_self_hosted_hopper(nv->pci_info.device_id)) &&
(NV_PCI_RESOURCE_VALID(nvl->pci_dev, gpu_bar1_offset)) &&
(NV_PCI_RESOURCE_FLAGS(nvl->pci_dev, gpu_bar1_offset) & PCI_BASE_ADDRESS_SPACE)
== PCI_BASE_ADDRESS_SPACE_MEMORY)
{
// Present only in passthrough case
// Present only in passthrough case for self-hosted hopper.
nvl->coherent_link_info.rsvd_mem_pa = NV_PCI_RESOURCE_START(nvl->pci_dev, gpu_bar1_offset);
//
// Unset nv->bars[1] only for self-hosted Hopper as BAR1 in virtualization case
// for hopper is used to convey RM reserved memory information and doesn't contain
// the traditional GPU BAR2. Starting from Blackwell BAR1 will be the real BAR1.
//
memset(&nv->bars[1], 0, sizeof(nv->bars[1]));
}
//
// Unset nv->bars[] as the BARs in the virtualization case are used
// only to convey the coherent GPU memory information and doesn't
// contain the traditional GPU BAR1/BAR2. This is to ensure the
// coherent FB addresses don't inadvertently pass the IS_FB_OFFSET
// or IS_IMEM_OFFSET checks.
// Unset nv->bars[2] for all self-hosted systems as BAR2 in the virtualization case
// is used only to convey the coherent GPU memory information and doesn't contain
// the traditional GPU BAR2. This is to ensure the coherent FB addresses don't
// inadvertently pass the IS_FB_OFFSET or IS_IMEM_OFFSET checks.
//
memset(&nv->bars[1], 0, sizeof(nv->bars[1]));
memset(&nv->bars[2], 0, sizeof(nv->bars[2]));
}
@@ -457,8 +468,11 @@ nv_init_coherent_link_info
NV_DEV_PRINTF(NV_DBG_INFO, nv, "DSD properties: \n");
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU memory PA: 0x%lx \n",
nvl->coherent_link_info.gpu_mem_pa);
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU reserved memory PA: 0x%lx \n",
nvl->coherent_link_info.rsvd_mem_pa);
if (pci_devid_is_self_hosted_hopper(nv->pci_info.device_id))
{
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU reserved memory PA: 0x%lx \n",
nvl->coherent_link_info.rsvd_mem_pa);
}
if (!gi_found)
{
@@ -780,8 +794,12 @@ next_bar:
// PPC64LE platform where ATS is currently supported (IBM P9).
nv_ats_supported &= nv_platform_supports_numa(nvl);
#else
#if defined(NV_PCI_DEV_HAS_ATS_ENABLED)
#if NV_IS_EXPORT_SYMBOL_GPL_pci_ats_supported
nv_ats_supported &= pci_ats_supported(pci_dev);
#elif defined(NV_PCI_DEV_HAS_ATS_ENABLED)
nv_ats_supported &= pci_dev->ats_enabled;
#else
nv_ats_supported = NV_FALSE;
#endif
#endif
if (nv_ats_supported)

4
kernel-open/nvidia/nv-procfs.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -691,7 +691,7 @@ static nv_proc_ops_t nv_procfs_suspend_fops = {
/*
* Forwards error to nv_log_error which exposes data to vendor callback
*/
void
static void
exercise_error_forwarding_va(
nv_state_t *nv,
NvU32 err,

14
kernel-open/nvidia/nv-reg.h
View File

@@ -802,6 +802,18 @@
#define NV_DMA_REMAP_PEER_MMIO_DISABLE 0x00000000
#define NV_DMA_REMAP_PEER_MMIO_ENABLE 0x00000001
/*
* Option: NVreg_RmNvlinkBandwidthLinkCount
*
* Description:
*
* This option allows user to reduce the GPU nvlink bandwidth to save power.
*
* This option is only for Blackwell+ GPU with NVLINK version 5.0.
*/
#define __NV_RM_NVLINK_BW_LINK_COUNT RmNvlinkBandwidthLinkCount
#define NV_RM_NVLINK_BW_LINK_COUNT NV_REG_STRING(__NV_RM_NVLINK_BW_LINK_COUNT)
/*
* Option: NVreg_RmNvlinkBandwidth
*
@@ -951,6 +963,7 @@ NV_DEFINE_REG_STRING_ENTRY(__NV_TEMPORARY_FILE_PATH, NULL);
NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL);
NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE);
NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL);
NV_DEFINE_REG_ENTRY(__NV_RM_NVLINK_BW_LINK_COUNT, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_CREATE_IMEX_CHANNEL_0, 0);
@@ -996,6 +1009,7 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_RESIZABLE_BAR),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_GPU_FIRMWARE_LOGS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_RM_NVLINK_BW_LINK_COUNT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_DBG_BREAKPOINT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO),

40
kernel-open/nvidia/nv-report-err.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2017-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,7 +29,7 @@
nv_report_error_cb_t nv_error_cb_handle = NULL;
int nv_register_error_cb(nv_report_error_cb_t report_error_cb)
int nvidia_register_error_cb(nv_report_error_cb_t report_error_cb)
{
if (report_error_cb == NULL)
return -EINVAL;
@@ -41,9 +41,9 @@ int nv_register_error_cb(nv_report_error_cb_t report_error_cb)
return 0;
}
EXPORT_SYMBOL(nv_register_error_cb);
EXPORT_SYMBOL(nvidia_register_error_cb);
int nv_unregister_error_cb(void)
int nvidia_unregister_error_cb(void)
{
if (nv_error_cb_handle == NULL)
return -EPERM;
@@ -52,9 +52,7 @@ int nv_unregister_error_cb(void)
return 0;
}
EXPORT_SYMBOL(nv_unregister_error_cb);
struct pci_dev;
EXPORT_SYMBOL(nvidia_unregister_error_cb);
void nv_report_error(
struct pci_dev *dev,
@@ -63,27 +61,17 @@ void nv_report_error(
va_list ap
)
{
va_list ap_copy;
char *buffer;
int length = 0;
int status = NV_OK;
gfp_t gfp = NV_MAY_SLEEP() ? NV_GFP_NO_OOM : NV_GFP_ATOMIC;
if (nv_error_cb_handle != NULL)
{
va_copy(ap_copy, ap);
length = vsnprintf(NULL, 0, format, ap);
va_end(ap_copy);
if (nv_error_cb_handle == NULL)
return;
if (length > 0)
{
status = os_alloc_mem((void *)&buffer, (length + 1)*sizeof(char));
buffer = kvasprintf(gfp, format, ap);
if (status == NV_OK)
{
vsnprintf(buffer, length, format, ap);
nv_error_cb_handle(dev, error_number, buffer, length + 1);
os_free_mem(buffer);
}
}
}
if (buffer == NULL)
return;
nv_error_cb_handle(dev, error_number, buffer, strlen(buffer) + 1);
kfree(buffer);
}

8
kernel-open/nvidia/nv-report-err.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2017-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -37,7 +37,7 @@
* @param[in] int
* Length of error string.
*/
typedef void (*nv_report_error_cb_t)(struct pci_dev *, uint32_t, char *, int);
typedef void (*nv_report_error_cb_t)(struct pci_dev *, uint32_t, char *, size_t);
/*
* @brief
@@ -51,7 +51,7 @@ typedef void (*nv_report_error_cb_t)(struct pci_dev *, uint32_t, char *, int);
* -EINVAL callback handle is NULL.
* -EBUSY callback handle is already registered.
*/
int nv_register_error_cb(nv_report_error_cb_t report_error_cb);
int nvidia_register_error_cb(nv_report_error_cb_t report_error_cb);
/*
* @brief
@@ -61,6 +61,6 @@ int nv_register_error_cb(nv_report_error_cb_t report_error_cb);
* 0 upon successful completion.
* -EPERM unregister not permitted on NULL callback handle.
*/
int nv_unregister_error_cb(void);
int nvidia_unregister_error_cb(void);
#endif /* _NV_REPORT_ERR_H_ */

21
kernel-open/nvidia/nv-usermap.c
View File

@@ -55,6 +55,8 @@ NV_STATUS NV_API_CALL nv_add_mapping_context_to_file(
status = NV_ERR_STATE_IN_USE;
goto done;
}
os_mem_set((void*) nvamc, 0, sizeof(nv_alloc_mapping_context_t));
if (NV_IS_CTL_DEVICE(nv))
{
@@ -69,8 +71,18 @@ NV_STATUS NV_API_CALL nv_add_mapping_context_to_file(
goto done;
}
nvamc->mmap_start = nvuap->mmap_start;
nvamc->mmap_size = nvuap->mmap_size;
status = os_alloc_mem((void**) &nvamc->memArea.pRanges,
sizeof(MemoryRange) * nvuap->memArea.numRanges);
if (status != NV_OK)
{
nvamc->memArea.pRanges = NULL;
goto done;
}
nvamc->memArea.numRanges = nvuap->memArea.numRanges;
os_mem_copy(nvamc->memArea.pRanges, nvuap->memArea.pRanges,
sizeof(MemoryRange) * nvuap->memArea.numRanges);
if (nv_get_numa_status(nvl) == NV_NUMA_STATUS_ONLINE)
{
nvamc->page_array = nvuap->page_array;
@@ -148,8 +160,9 @@ NV_STATUS NV_API_CALL nv_get_usermap_access_params(
nvuap->remap_prot_extra = NV_PROT_4K_PAGE_ISOLATION;
nvuap->access_start = (NvU64)NV_4K_PAGE_ISOLATION_ACCESS_START(addr);
nvuap->access_size = NV_4K_PAGE_ISOLATION_ACCESS_LEN(addr, size);
nvuap->mmap_start = (NvU64)NV_4K_PAGE_ISOLATION_MMAP_ADDR(addr);
nvuap->mmap_size = NV_4K_PAGE_ISOLATION_MMAP_LEN(size);
nvuap->memArea.pRanges[0].start = (NvU64)NV_4K_PAGE_ISOLATION_MMAP_ADDR(addr);
nvuap->memArea.pRanges[0].size = NV_4K_PAGE_ISOLATION_MMAP_LEN(size);
#else
NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "4K page isolation required but not available!\n");
return NV_ERR_OPERATING_SYSTEM;

89
kernel-open/nvidia/nv.c
View File

@@ -96,6 +96,10 @@
#include <linux/cc_platform.h>
#endif
#if defined(NV_ASM_MSHYPERV_H_PRESENT) && defined(NVCPU_X86_64)
#include <asm/mshyperv.h>
#endif
#if defined(NV_ASM_CPUFEATURE_H_PRESENT)
#include <asm/cpufeature.h>
#endif
@@ -184,11 +188,7 @@ struct semaphore nv_linux_devices_lock;
// True if all the successfully probed devices support ATS
// Assigned at device probe (module init) time
NvBool nv_ats_supported = NVCPU_IS_PPC64LE
#if defined(NV_PCI_DEV_HAS_ATS_ENABLED)
|| NV_TRUE
#endif
;
NvBool nv_ats_supported = NV_TRUE;
// allow an easy way to convert all debug printfs related to events
// back and forth between 'info' and 'errors'
@@ -285,6 +285,17 @@ void nv_detect_conf_compute_platform(
#if defined(NV_CC_PLATFORM_PRESENT)
os_cc_enabled = cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT);
#if defined(NV_CC_ATTR_SEV_SNP)
os_cc_sev_snp_enabled = cc_platform_has(CC_ATTR_GUEST_SEV_SNP);
#endif
#if defined(NV_HV_GET_ISOLATION_TYPE) && IS_ENABLED(CONFIG_HYPERV) && defined(NVCPU_X86_64)
if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP)
{
os_cc_snp_vtom_enabled = NV_TRUE;
}
#endif
#if defined(X86_FEATURE_TDX_GUEST)
if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST))
{
@@ -293,8 +304,10 @@ void nv_detect_conf_compute_platform(
#endif
#else
os_cc_enabled = NV_FALSE;
os_cc_sev_snp_enabled = NV_FALSE;
os_cc_snp_vtom_enabled = NV_FALSE;
os_cc_tdx_enabled = NV_FALSE;
#endif
#endif //NV_CC_PLATFORM_PRESENT
}
static
@@ -1018,9 +1031,16 @@ static void nv_free_file_private(nv_linux_file_private_t *nvlfp)
NV_KFREE(nvet, sizeof(nvidia_event_t));
}
if (nvlfp->mmap_context.page_array != NULL)
if (nvlfp->mmap_context.valid)
{
os_free_mem(nvlfp->mmap_context.page_array);
if (nvlfp->mmap_context.page_array != NULL)
{
os_free_mem(nvlfp->mmap_context.page_array);
}
if (nvlfp->mmap_context.memArea.pRanges != NULL)
{
os_free_mem(nvlfp->mmap_context.memArea.pRanges);
}
}
NV_KFREE(nvlfp, sizeof(nv_linux_file_private_t));
@@ -1251,12 +1271,6 @@ static int validate_numa_start_state(nv_linux_state_t *nvl)
return rc;
}
NV_STATUS NV_API_CALL nv_get_num_dpaux_instances(nv_state_t *nv, NvU32 *num_instances)
{
*num_instances = nv->num_dpaux_instance;
return NV_OK;
}
void NV_API_CALL
nv_schedule_uvm_isr(nv_state_t *nv)
{
@@ -1265,6 +1279,24 @@ nv_schedule_uvm_isr(nv_state_t *nv)
#endif
}
NV_STATUS NV_API_CALL
nv_schedule_uvm_drain_p2p(NvU8 *pUuid)
{
#if defined(NV_UVM_ENABLE)
return nv_uvm_drain_P2P(pUuid);
#else
return NV_ERR_NOT_SUPPORTED;
#endif
}
void NV_API_CALL
nv_schedule_uvm_resume_p2p(NvU8 *pUuid)
{
#if defined(NV_UVM_ENABLE)
nv_uvm_resume_P2P(pUuid);
#endif
}
/*
* Brings up the device on the first file open. Assumes nvl->ldata_lock is held.
*/
@@ -1623,9 +1655,6 @@ static void nv_init_mapping_revocation(nv_linux_state_t *nvl,
address_space_init_once(&nvlfp->mapping);
nvlfp->mapping.host = inode;
nvlfp->mapping.a_ops = inode->i_mapping->a_ops;
#if defined(NV_ADDRESS_SPACE_HAS_BACKING_DEV_INFO)
nvlfp->mapping.backing_dev_info = inode->i_mapping->backing_dev_info;
#endif
file->f_mapping = &nvlfp->mapping;
/* Add nvlfp to list of open files in nvl for mapping revocation */
@@ -3136,6 +3165,7 @@ nv_alias_pages(
NvU32 cache_type,
NvU64 guest_id,
NvU64 *pte_array,
NvBool carveout,
void **priv_data
)
{
@@ -3167,6 +3197,7 @@ nv_alias_pages(
#endif
at->flags.guest = NV_TRUE;
at->flags.carveout = carveout;
for (i=0; i < at->num_pages; ++i)
{
@@ -4411,15 +4442,7 @@ nvidia_suspend(
status = nv_power_management(nv, pm_action);
if (status != NV_OK)
{
nvidia_modeset_resume(nv->gpu_id);
goto done;
}
else
{
nv->flags |= NV_FLAG_SUSPENDED;
}
nv->flags |= NV_FLAG_SUSPENDED;
pci_pm:
/*
@@ -4606,13 +4629,17 @@ done:
{
if (resume_devices)
{
nvidia_resume(nvl->dev, pm_action);
nvidia_resume(nvl->dev, NV_PM_ACTION_RESUME);
}
nv_restore_user_channels(NV_STATE_PTR(nvl));
}
UNLOCK_NV_LINUX_DEVICES();
nv_uvm_resume();
nvidia_modeset_resume(0);
}
return status;
@@ -4689,6 +4716,10 @@ int nv_pmops_suspend(
NV_STATUS status;
status = nvidia_suspend(dev, NV_PM_ACTION_STANDBY, NV_FALSE);
if (status != NV_OK)
nvidia_resume(dev, NV_PM_ACTION_RESUME);
return (status == NV_OK) ? 0 : -EIO;
}
@@ -4709,6 +4740,10 @@ int nv_pmops_freeze(
NV_STATUS status;
status = nvidia_suspend(dev, NV_PM_ACTION_HIBERNATE, NV_FALSE);
if (status != NV_OK)
nvidia_resume(dev, NV_PM_ACTION_RESUME);
return (status == NV_OK) ? 0 : -EIO;
}

15
kernel-open/nvidia/nv_gpu_ops.h
View File

@@ -45,6 +45,11 @@ typedef struct gpuObject *gpuObjectHandle;
typedef struct gpuRetainedChannel_struct gpuRetainedChannel;
NV_STATUS calculatePCIELinkRateMBps(NvU32 lanes,
NvU32 pciLinkMaxSpeed,
NvU32 *pcieLinkRate);
NV_STATUS nvGpuOpsCreateSession(struct gpuSession **session);
NV_STATUS nvGpuOpsDestroySession(struct gpuSession *session);
@@ -180,6 +185,8 @@ NV_STATUS nvGpuOpsGetFbInfo(struct gpuDevice *device, gpuFbInfo * fbInfo);
NV_STATUS nvGpuOpsGetEccInfo(struct gpuDevice *device, gpuEccInfo * eccInfo);
NV_STATUS nvGpuOpsGetNvlinkInfo(struct gpuDevice *device, gpuNvlinkInfo * nvlinkInfo);
NV_STATUS nvGpuOpsInitFaultInfo(struct gpuDevice *device, gpuFaultInfo *pFaultInfo);
NV_STATUS nvGpuOpsDestroyFaultInfo(struct gpuDevice *device,
@@ -227,6 +234,12 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace,
NvU64 size,
gpuExternalMappingInfo *pGpuExternalMappingInfo);
NV_STATUS nvGpuOpsGetExternalAllocPhysAddrs(struct gpuAddressSpace *vaSpace,
NvHandle hDupedMemory,
NvU64 offset,
NvU64 size,
gpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo);
NV_STATUS nvGpuOpsRetainChannel(struct gpuAddressSpace *vaSpace,
NvHandle hClient,
NvHandle hChannel,
@@ -285,6 +298,8 @@ NV_STATUS nvGpuOpsFlushReplayableFaultBuffer(gpuFaultInfo *pFaultInfo,
NV_STATUS nvGpuOpsTogglePrefetchFaults(gpuFaultInfo *pFaultInfo,
NvBool bEnable);
void nvGpuOpsReportFatalError(NV_STATUS error);
// Interface used for CCSL
NV_STATUS nvGpuOpsCcslContextInit(struct ccslContext_t **ctx,
gpuChannelHandle channel);

101
kernel-open/nvidia/nv_uvm_interface.c
View File

@@ -1219,6 +1219,73 @@ NV_STATUS nv_uvm_event_interrupt(const NvU8 *pUuid)
return NV_ERR_NO_INTR_PENDING;
}
NV_STATUS nvUvmInterfaceGetNvlinkInfo(uvmGpuDeviceHandle device,
UvmGpuNvlinkInfo *nvlinkInfo)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
if (nv_kmem_cache_alloc_stack(&sp) != 0)
{
return NV_ERR_NO_MEMORY;
}
status = rm_gpu_ops_get_nvlink_info(sp, (gpuDeviceHandle)device, nvlinkInfo);
nv_kmem_cache_free_stack(sp);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetNvlinkInfo);
NV_STATUS nv_uvm_drain_P2P(const NvU8 *uuid)
{
NvProcessorUuid uvmUuid;
struct UvmOpsUvmEvents *events;
NV_STATUS ret = NV_ERR_NOT_SUPPORTED;
memcpy(uvmUuid.uuid, uuid, UVM_UUID_LEN);
// Synchronize callbacks with unregistration
down(&g_pNvUvmEventsLock);
// It's not strictly necessary to use a cached local copy of the events
// pointer here since it can't change under the lock, but we'll do it for
// consistency.
events = getUvmEvents();
if(events && events->drainP2P)
{
ret = events->drainP2P(&uvmUuid);
}
up(&g_pNvUvmEventsLock);
return ret;
}
NV_STATUS nv_uvm_resume_P2P(const NvU8 *uuid)
{
NvProcessorUuid uvmUuid;
struct UvmOpsUvmEvents *events;
NV_STATUS ret = NV_ERR_NOT_SUPPORTED;
memcpy(uvmUuid.uuid, uuid, UVM_UUID_LEN);
// Synchronize callbacks with unregistration
down(&g_pNvUvmEventsLock);
// It's not strictly necessary to use a cached local copy of the events
// pointer here since it can't change under the lock, but we'll do it for
// consistency.
events = getUvmEvents();
if(events && events->resumeP2P)
{
ret = events->resumeP2P(&uvmUuid);
}
up(&g_pNvUvmEventsLock);
return ret;
}
NV_STATUS nvUvmInterfaceP2pObjectCreate(uvmGpuDeviceHandle device1,
uvmGpuDeviceHandle device2,
NvHandle *hP2pObject)
@@ -1277,6 +1344,32 @@ NV_STATUS nvUvmInterfaceGetExternalAllocPtes(uvmGpuAddressSpaceHandle vaSpace,
}
EXPORT_SYMBOL(nvUvmInterfaceGetExternalAllocPtes);
NV_STATUS nvUvmInterfaceGetExternalAllocPhysAddrs(uvmGpuAddressSpaceHandle vaSpace,
NvHandle hDupedMemory,
NvU64 offset,
NvU64 size,
UvmGpuExternalPhysAddrInfo *gpuExternalPhysAddrInfo)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
if (nv_kmem_cache_alloc_stack(&sp) != 0)
{
return NV_ERR_NO_MEMORY;
}
status = rm_gpu_ops_get_external_alloc_phys_addrs(sp,
(gpuAddressSpaceHandle)vaSpace,
hDupedMemory,
offset,
size,
gpuExternalPhysAddrInfo);
nv_kmem_cache_free_stack(sp);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetExternalAllocPhysAddrs);
NV_STATUS nvUvmInterfaceRetainChannel(uvmGpuAddressSpaceHandle vaSpace,
NvHandle hClient,
NvHandle hChannel,
@@ -1478,6 +1571,14 @@ NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channe
}
EXPORT_SYMBOL(nvUvmInterfacePagingChannelPushStream);
void nvUvmInterfaceReportFatalError(NV_STATUS error)
{
nvidia_stack_t *sp = nvUvmGetSafeStack();
rm_gpu_ops_report_fatal_error(sp, error);
nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceReportFatalError);
NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext,
uvmGpuChannelHandle channel)
{

4
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -136,7 +136,6 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += of_find_node_by_phandle
NV_CONFTEST_FUNCTION_COMPILE_TESTS += of_node_to_nid
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pnv_pci_get_npu_dev
NV_CONFTEST_FUNCTION_COMPILE_TESTS += of_get_ibm_chip_id
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_bus_address
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_stop_and_remove_bus_device
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_rebar_get_possible_sizes
NV_CONFTEST_FUNCTION_COMPILE_TESTS += wait_for_random_bytes
@@ -160,6 +159,8 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += full_name_hash
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_enable_atomic_ops_to_root
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vga_tryget
NV_CONFTEST_FUNCTION_COMPILE_TESTS += cc_platform_has
NV_CONFTEST_FUNCTION_COMPILE_TESTS += cc_attr_guest_sev_snp
NV_CONFTEST_FUNCTION_COMPILE_TESTS += hv_get_isolation_type
NV_CONFTEST_FUNCTION_COMPILE_TESTS += seq_read_iter
NV_CONFTEST_FUNCTION_COMPILE_TESTS += follow_pfn
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_gem_object_get
@@ -230,6 +231,7 @@ NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_free_gsc
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_memory_block_size_bytes
NV_CONFTEST_SYMBOL_COMPILE_TESTS += crypto
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_follow_pte
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_gpl_pci_ats_supported
NV_CONFTEST_TYPE_COMPILE_TESTS += dma_ops
NV_CONFTEST_TYPE_COMPILE_TESTS += swiotlb_dma_ops

1
kernel-open/nvidia/nvlink_linux.c
View File

@@ -27,6 +27,7 @@
#include "nvlink_linux.h"
#include "nvlink_errors.h"
#include "nvlink_export.h"
#include "nvlink_proto.h"
#include "nv-linux.h"
#include "nv-procfs.h"
#include "nv-time.h"

1
kernel-open/nvidia/nvlink_proto.h
View File

@@ -45,7 +45,6 @@ void nvswitch_exit (void);
*/
int tegrashim_init (void);
void tegrashim_exit (void);
NvlStatus tegrashim_init_device (struct pci_dev *);
#endif
#endif /* _NVLINK_PROTO_H_ */

39
kernel-open/nvidia/os-interface.c
View File

@@ -52,6 +52,8 @@ NvU32 os_page_size = PAGE_SIZE;
NvU64 os_page_mask = NV_PAGE_MASK;
NvU8 os_page_shift = PAGE_SHIFT;
NvBool os_cc_enabled = 0;
NvBool os_cc_sev_snp_enabled = 0;
NvBool os_cc_snp_vtom_enabled = 0;
NvBool os_cc_tdx_enabled = 0;
#if defined(CONFIG_DMA_SHARED_BUFFER)
@@ -370,9 +372,27 @@ NvBool NV_API_CALL os_is_administrator(void)
return NV_IS_SUSER();
}
NvBool NV_API_CALL os_allow_priority_override(void)
NvBool NV_API_CALL os_check_access(RsAccessRight accessRight)
{
return capable(CAP_SYS_NICE);
switch (accessRight)
{
case RS_ACCESS_PERFMON:
{
#if defined(CAP_PERFMON)
return capable(CAP_PERFMON);
#else
return os_is_administrator();
#endif
}
case RS_ACCESS_NICE:
{
return capable(CAP_SYS_NICE);
}
default:
{
return NV_FALSE;
}
}
}
char* NV_API_CALL os_string_copy(
@@ -400,7 +420,7 @@ NvS32 NV_API_CALL os_string_compare(const char *str1, const char *str2)
return strcmp(str1, str2);
}
void *os_mem_copy_custom(
static void *os_mem_copy_custom(
void *dstPtr,
const void *srcPtr,
NvU32 length
@@ -1173,7 +1193,7 @@ NV_STATUS NV_API_CALL os_queue_work_item(struct os_work_queue *queue, void *data
return NV_OK;
}
NV_STATUS NV_API_CALL os_flush_work_queue(struct os_work_queue *queue)
NV_STATUS NV_API_CALL os_flush_work_queue(struct os_work_queue *queue, NvBool is_unload)
{
nv_kthread_q_t *kthread;
@@ -1182,9 +1202,13 @@ NV_STATUS NV_API_CALL os_flush_work_queue(struct os_work_queue *queue)
if (NV_MAY_SLEEP())
{
kthread->is_unload_flush_ongoing = is_unload;
if (kthread->q_kthread)
nv_kthread_q_flush(kthread);
kthread->is_unload_flush_ongoing = NV_FALSE;
return NV_OK;
}
else
@@ -1196,6 +1220,13 @@ NV_STATUS NV_API_CALL os_flush_work_queue(struct os_work_queue *queue)
}
}
NvBool NV_API_CALL os_is_queue_flush_ongoing(struct os_work_queue *queue)
{
nv_kthread_q_t *kthread = queue ? &queue->nvk : &nv_kthread_q;
return kthread->is_unload_flush_ongoing;
}
extern NvU32 NVreg_EnableDbgBreakpoint;
void NV_API_CALL os_dbg_breakpoint(void)

19
kernel-open/nvidia/os-usermap.c
View File

@@ -26,25 +26,6 @@
#include "os-interface.h"
#include "nv-linux.h"
void* NV_API_CALL os_map_user_space(
NvU64 start,
NvU64 size_bytes,
NvU32 mode,
NvU32 protect,
void **priv_data
)
{
return (void *)(NvUPtr)start;
}
void NV_API_CALL os_unmap_user_space(
void *address,
NvU64 size,
void *priv_data
)
{
}
NV_STATUS NV_API_CALL os_match_mmap_offset(
void *pAllocPrivate,
NvU64 offset,