NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-30 04:59:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

525.105.17

Browse Source
This commit is contained in:
Andy Ritger
2023-03-30 10:16:11 -07:00
parent e598191e8e
commit ebcc6656ff
100 changed files with 2912 additions and 1117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
kernel-open/nvidia/nv-mmap.c
View File

@@ -452,7 +452,7 @@ static int nvidia_mmap_numa(
}
// Needed for the linux kernel for mapping compound pages
vma->vm_flags |= VM_MIXEDMAP;
nv_vm_flags_set(vma, VM_MIXEDMAP);
for (i = 0, addr = mmap_context->page_array[0]; i < pages;
addr = mmap_context->page_array[++i], start += PAGE_SIZE)
@@ -596,7 +596,7 @@ int nvidia_mmap_helper(
}
up(&nvl->mmap_lock);
vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND;
nv_vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND);
}
else
{
@@ -663,15 +663,15 @@ int nvidia_mmap_helper(
NV_PRINT_AT(NV_DBG_MEMINFO, at);
vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED);
vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
nv_vm_flags_set(vma, VM_IO | VM_LOCKED | VM_RESERVED);
nv_vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
}
if ((prot & NV_PROTECT_WRITEABLE) == 0)
{
vma->vm_page_prot = NV_PGPROT_READ_ONLY(vma->vm_page_prot);
vma->vm_flags &= ~VM_WRITE;
vma->vm_flags &= ~VM_MAYWRITE;
nv_vm_flags_clear(vma, VM_WRITE);
nv_vm_flags_clear(vma, VM_MAYWRITE);
}
vma->vm_ops = &nv_vm_ops;

147
kernel-open/nvidia/nv-p2p.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2011-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2011-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -31,6 +31,11 @@
#include "nv-p2p.h"
#include "rmp2pdefines.h"
typedef enum nv_p2p_page_table_type {
NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT = 0,
NV_P2P_PAGE_TABLE_TYPE_PERSISTENT,
} nv_p2p_page_table_type_t;
typedef struct nv_p2p_dma_mapping {
struct list_head list_node;
struct nvidia_p2p_dma_mapping *dma_mapping;
@@ -44,13 +49,9 @@ typedef struct nv_p2p_mem_info {
struct list_head list_head;
struct semaphore lock;
} dma_mapping_list;
NvBool bPersistent;
void *private;
} nv_p2p_mem_info_t;
int nvidia_p2p_cap_persistent_pages = 1;
EXPORT_SYMBOL(nvidia_p2p_cap_persistent_pages);
// declared and created in nv.c
extern void *nvidia_p2p_page_t_cache;
@@ -238,6 +239,7 @@ static void nv_p2p_free_page_table(
}
static NV_STATUS nv_p2p_put_pages(
nv_p2p_page_table_type_t pt_type,
nvidia_stack_t * sp,
uint64_t p2p_token,
uint32_t va_space,
@@ -246,9 +248,6 @@ static NV_STATUS nv_p2p_put_pages(
)
{
NV_STATUS status;
struct nv_p2p_mem_info *mem_info = NULL;
mem_info = container_of(*page_table, nv_p2p_mem_info_t, page_table);
/*
* rm_p2p_put_pages returns NV_OK if the page_table was found and
@@ -258,8 +257,15 @@ static NV_STATUS nv_p2p_put_pages(
* rm_p2p_put_pages returns NV_ERR_OBJECT_NOT_FOUND if the page_table
* was already unlinked.
*/
if (mem_info->bPersistent)
if (pt_type == NV_P2P_PAGE_TABLE_TYPE_PERSISTENT)
{
struct nv_p2p_mem_info *mem_info = NULL;
/*
* It is safe to access persistent page_table as there is no async
* callback which can free it unlike non-persistent page_table.
*/
mem_info = container_of(*page_table, nv_p2p_mem_info_t, page_table);
status = rm_p2p_put_pages_persistent(sp, mem_info->private, *page_table);
}
else
@@ -273,7 +279,8 @@ static NV_STATUS nv_p2p_put_pages(
nv_p2p_free_page_table(*page_table);
*page_table = NULL;
}
else if (!mem_info->bPersistent && (status == NV_ERR_OBJECT_NOT_FOUND))
else if ((pt_type == NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT) &&
(status == NV_ERR_OBJECT_NOT_FOUND))
{
status = NV_OK;
*page_table = NULL;
@@ -327,7 +334,8 @@ static void nv_p2p_mem_info_free_callback(void *data)
nv_p2p_free_platform_data(&mem_info->page_table);
}
int nvidia_p2p_get_pages(
static int nv_p2p_get_pages(
nv_p2p_page_table_type_t pt_type,
uint64_t p2p_token,
uint32_t va_space,
uint64_t virtual_address,
@@ -376,9 +384,10 @@ int nvidia_p2p_get_pages(
*page_table = &(mem_info->page_table);
mem_info->bPersistent = (free_callback == NULL);
//asign length to temporary variable since do_div macro does in-place division
/*
* assign length to temporary variable since do_div macro does in-place
* division
*/
temp_length = length;
do_div(temp_length, page_size);
page_count = temp_length;
@@ -405,7 +414,7 @@ int nvidia_p2p_get_pages(
goto failed;
}
if (mem_info->bPersistent)
if (pt_type == NV_P2P_PAGE_TABLE_TYPE_PERSISTENT)
{
void *gpu_info = NULL;
@@ -415,7 +424,8 @@ int nvidia_p2p_get_pages(
goto failed;
}
status = rm_p2p_get_gpu_info(sp, virtual_address, length, &gpu_uuid, &gpu_info);
status = rm_p2p_get_gpu_info(sp, virtual_address, length,
&gpu_uuid, &gpu_info);
if (status != NV_OK)
{
goto failed;
@@ -432,8 +442,10 @@ int nvidia_p2p_get_pages(
bGetUuid = NV_TRUE;
status = rm_p2p_get_pages_persistent(sp, virtual_address, length, &mem_info->private,
physical_addresses, &entries, *page_table, gpu_info);
status = rm_p2p_get_pages_persistent(sp, virtual_address, length,
&mem_info->private,
physical_addresses, &entries,
*page_table, gpu_info);
if (status != NV_OK)
{
goto failed;
@@ -516,10 +528,12 @@ failed:
{
os_free_mem(physical_addresses);
}
if (wreqmb_h != NULL)
{
os_free_mem(wreqmb_h);
}
if (rreqmb_h != NULL)
{
os_free_mem(rreqmb_h);
@@ -527,7 +541,7 @@ failed:
if (bGetPages)
{
(void)nv_p2p_put_pages(sp, p2p_token, va_space,
(void)nv_p2p_put_pages(pt_type, sp, p2p_token, va_space,
virtual_address, page_table);
}
@@ -546,8 +560,45 @@ failed:
return nvidia_p2p_map_status(status);
}
int nvidia_p2p_get_pages(
uint64_t p2p_token,
uint32_t va_space,
uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void * data),
void *data
)
{
if (free_callback == NULL)
{
return -EINVAL;
}
return nv_p2p_get_pages(NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT,
p2p_token, va_space, virtual_address,
length, page_table, free_callback, data);
}
EXPORT_SYMBOL(nvidia_p2p_get_pages);
int nvidia_p2p_get_pages_persistent(
uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
uint32_t flags
)
{
if (flags != 0)
{
return -EINVAL;
}
return nv_p2p_get_pages(NV_P2P_PAGE_TABLE_TYPE_PERSISTENT, 0, 0,
virtual_address, length, page_table,
NULL, NULL);
}
EXPORT_SYMBOL(nvidia_p2p_get_pages_persistent);
/*
* This function is a no-op, but is left in place (for now), in order to allow
* third-party callers to build and run without errors or warnings. This is OK,
@@ -568,15 +619,14 @@ int nvidia_p2p_put_pages(
struct nvidia_p2p_page_table *page_table
)
{
struct nv_p2p_mem_info *mem_info = NULL;
NvU8 uuid[NVIDIA_P2P_GPU_UUID_LEN] = {0};
NV_STATUS status;
nvidia_stack_t *sp = NULL;
int rc = 0;
os_mem_copy(uuid, page_table->gpu_uuid, NVIDIA_P2P_GPU_UUID_LEN);
mem_info = container_of(page_table, nv_p2p_mem_info_t, page_table);
if (page_table == NULL)
{
return 0;
}
rc = nv_kmem_cache_alloc_stack(&sp);
if (rc != 0)
@@ -584,21 +634,56 @@ int nvidia_p2p_put_pages(
return -ENOMEM;
}
status = nv_p2p_put_pages(sp, p2p_token, va_space,
status = nv_p2p_put_pages(NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT,
sp, p2p_token, va_space,
virtual_address, &page_table);
if (mem_info->bPersistent)
{
nvidia_dev_put_uuid(uuid, sp);
}
nv_kmem_cache_free_stack(sp);
return nvidia_p2p_map_status(status);
}
EXPORT_SYMBOL(nvidia_p2p_put_pages);
int nvidia_p2p_put_pages_persistent(
uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table,
uint32_t flags
)
{
NvU8 uuid[NVIDIA_P2P_GPU_UUID_LEN] = {0};
NV_STATUS status;
nvidia_stack_t *sp = NULL;
int rc = 0;
if (flags != 0)
{
return -EINVAL;
}
if (page_table == NULL)
{
return 0;
}
rc = nv_kmem_cache_alloc_stack(&sp);
if (rc != 0)
{
return -ENOMEM;
}
os_mem_copy(uuid, page_table->gpu_uuid, NVIDIA_P2P_GPU_UUID_LEN);
status = nv_p2p_put_pages(NV_P2P_PAGE_TABLE_TYPE_PERSISTENT,
sp, 0, 0, virtual_address, &page_table);
nvidia_dev_put_uuid(uuid, sp);
nv_kmem_cache_free_stack(sp);
return nvidia_p2p_map_status(status);
}
EXPORT_SYMBOL(nvidia_p2p_put_pages_persistent);
int nvidia_p2p_dma_map_pages(
struct pci_dev *peer,
struct nvidia_p2p_page_table *page_table,

85
kernel-open/nvidia/nv-p2p.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2011-2016 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2011-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -94,11 +94,10 @@ struct nvidia_p2p_params {
} nvidia_p2p_params_t;
/*
* Capability flag for users to detect
* Macro for users to detect
* driver support for persistent pages.
*/
extern int nvidia_p2p_cap_persistent_pages;
#define NVIDIA_P2P_CAP_PERSISTENT_PAGES
#define NVIDIA_P2P_CAP_GET_PAGES_PERSISTENT_API
/*
* This API is not supported.
@@ -173,11 +172,6 @@ struct nvidia_p2p_page_table {
* A pointer to the function to be invoked when the pages
* underlying the virtual address range are freed
* implicitly.
* If NULL, persistent pages will be returned.
* This means the pages underlying the range of GPU virtual memory
* will persist until explicitly freed by nvidia_p2p_put_pages().
* Persistent GPU memory mappings are not supported on PowerPC,
* MIG-enabled devices and vGPU.
* @param[in] data
* A non-NULL opaque pointer to private data to be passed to the
* callback function.
@@ -190,12 +184,48 @@ struct nvidia_p2p_page_table {
* insufficient resources were available to complete the operation.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_get_pages(uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address,
int nvidia_p2p_get_pages( uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address, uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void *data), void *data);
/*
* @brief
* Pin and make the pages underlying a range of GPU virtual memory
* accessible to a third-party device. The pages will persist until
* explicitly freed by nvidia_p2p_put_pages_persistent().
*
* Persistent GPU memory mappings are not supported on PowerPC,
* MIG-enabled devices and vGPU.
*
* This API only supports pinned, GPU-resident memory, such as that provided
* by cudaMalloc().
*
* This API may sleep.
*
* @param[in] virtual_address
* The start address in the specified virtual address space.
* Address must be aligned to the 64KB boundary.
* @param[in] length
* The length of the requested P2P mapping.
* Length must be a multiple of 64KB.
* @param[out] page_table
* A pointer to an array of structures with P2P PTEs.
* @param[in] flags
* Must be set to zero for now.
*
* @return
* 0 upon successful completion.
* -EINVAL if an invalid argument was supplied.
* -ENOTSUPP if the requested operation is not supported.
* -ENOMEM if the driver failed to allocate memory or if
* insufficient resources were available to complete the operation.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_get_pages_persistent(uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void *data),
void *data);
uint32_t flags);
#define NVIDIA_P2P_DMA_MAPPING_VERSION 0x00020003
@@ -268,6 +298,8 @@ int nvidia_p2p_dma_unmap_pages(struct pci_dev *peer,
* Release a set of pages previously made accessible to
* a third-party device.
*
* This API may sleep.
*
* @param[in] p2p_token
* A token that uniquely identifies the P2P mapping.
* @param[in] va_space
@@ -282,10 +314,33 @@ int nvidia_p2p_dma_unmap_pages(struct pci_dev *peer,
* -EINVAL if an invalid argument was supplied.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_put_pages(uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address,
int nvidia_p2p_put_pages(uint64_t p2p_token,
uint32_t va_space, uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table);
/*
* @brief
* Release a set of persistent pages previously made accessible to
* a third-party device.
*
* This API may sleep.
*
* @param[in] virtual_address
* The start address in the specified virtual address space.
* @param[in] page_table
* A pointer to the array of structures with P2P PTEs.
* @param[in] flags
* Must be set to zero for now.
*
* @return
* 0 upon successful completion.
* -EINVAL if an invalid argument was supplied.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_put_pages_persistent(uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table,
uint32_t flags);
/*
* @brief
* Free a third-party P2P page table. (This function is a no-op.)

28
kernel-open/nvidia/nv.c
View File

@@ -165,7 +165,7 @@ NvBool nv_ats_supported = NVCPU_IS_PPC64LE
/* nvos_ functions.. do not take a state device parameter */
static int nvos_count_devices(void);
static nv_alloc_t *nvos_create_alloc(struct device *, int);
static nv_alloc_t *nvos_create_alloc(struct device *, NvU64);
static int nvos_free_alloc(nv_alloc_t *);
/***
@@ -280,11 +280,12 @@ void nv_sev_init(
static
nv_alloc_t *nvos_create_alloc(
struct device *dev,
int num_pages
NvU64 num_pages
)
{
nv_alloc_t *at;
unsigned int pt_size, i;
nv_alloc_t *at;
NvU64 pt_size;
unsigned int i;
NV_KZALLOC(at, sizeof(nv_alloc_t));
if (at == NULL)
@@ -295,6 +296,24 @@ nv_alloc_t *nvos_create_alloc(
at->dev = dev;
pt_size = num_pages * sizeof(nvidia_pte_t *);
//
// Check for multiplication overflow and check whether num_pages value can fit in at->num_pages.
//
if ((num_pages != 0) && ((pt_size / num_pages) != sizeof(nvidia_pte_t*)))
{
nv_printf(NV_DBG_ERRORS, "NVRM: Invalid page table allocation - Number of pages exceeds max value.\n");
NV_KFREE(at, sizeof(nv_alloc_t));
return NULL;
}
at->num_pages = num_pages;
if (at->num_pages != num_pages)
{
nv_printf(NV_DBG_ERRORS, "NVRM: Invalid page table allocation - requested size overflows.\n");
NV_KFREE(at, sizeof(nv_alloc_t));
return NULL;
}
if (os_alloc_mem((void **)&at->page_table, pt_size) != NV_OK)
{
nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate page table\n");
@@ -303,7 +322,6 @@ nv_alloc_t *nvos_create_alloc(
}
memset(at->page_table, 0, pt_size);
at->num_pages = num_pages;
NV_ATOMIC_SET(at->usage_count, 0);
for (i = 0; i < at->num_pages; i++)

1
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -230,6 +230,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += remove_memory_has_nid_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_driver_has_driver_managed_dma
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build