NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-01 14:09:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

535.43.02

Browse Source
This commit is contained in:
Andy Ritger
2023-05-30 10:11:36 -07:00
parent 6dd092ddb7
commit eb5c7665a1
1403 changed files with 295367 additions and 86235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
kernel-open/nvidia/nv-acpi.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -98,9 +98,6 @@ static int nv_acpi_notifier_call_chain_handler(
)
{
struct acpi_bus_event *info = data;
nv_stack_t *sp = NULL;
nv_linux_state_t *nvl = container_of(nb, nv_linux_state_t, acpi_nb);
nv_state_t *nv = NV_STATE_PTR(nvl);
/*
* The ACPI_VIDEO_NOTIFY_PROBE will be sent for display hot-plug/unplug.
@@ -1062,7 +1059,10 @@ static NV_STATUS nv_acpi_wmmx_method(
union acpi_object mmx_params[3];
if (!wmmx_handle)
{
*outDataSize = 0;
return NV_ERR_NOT_SUPPORTED;
}
if (!NV_MAY_SLEEP())
{

843
kernel-open/nvidia/nv-dmabuf.c
View File

File diff suppressed because it is too large Load Diff

2
kernel-open/nvidia/nv-kthread-q.c
View File

@@ -301,7 +301,7 @@ static void _q_flush_function(void *args)
static void _raw_q_flush(nv_kthread_q_t *q)
{
nv_kthread_q_item_t q_item;
DECLARE_COMPLETION(completion);
DECLARE_COMPLETION_ONSTACK(completion);
nv_kthread_q_item_init(&q_item, _q_flush_function, &completion);

6
kernel-open/nvidia/nv-nano-timer.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -174,7 +174,7 @@ void NV_API_CALL nv_create_nano_timer(
*
* @param[in] nv Per gpu linux state
* @param[in] nv_nstimer Pointer to nv_nano_timer_t object
* @param[in] timens time in nano seconds
* @param[in] time_ns Relative time in nano seconds
*/
void NV_API_CALL nv_start_nano_timer(
nv_state_t *nv,
@@ -213,7 +213,7 @@ void NV_API_CALL nv_cancel_nano_timer(
#if NV_NANO_TIMER_USE_HRTIMER
hrtimer_cancel(&nv_nstimer->hr_timer);
#else
del_timer(&nv_nstimer->jiffy_timer);
del_timer_sync(&nv_nstimer->jiffy_timer);
#endif
}

152
kernel-open/nvidia/nv-p2p.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2011-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2011-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -31,6 +31,11 @@
#include "nv-p2p.h"
#include "rmp2pdefines.h"
typedef enum nv_p2p_page_table_type {
NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT = 0,
NV_P2P_PAGE_TABLE_TYPE_PERSISTENT,
} nv_p2p_page_table_type_t;
typedef struct nv_p2p_dma_mapping {
struct list_head list_node;
struct nvidia_p2p_dma_mapping *dma_mapping;
@@ -44,13 +49,9 @@ typedef struct nv_p2p_mem_info {
struct list_head list_head;
struct semaphore lock;
} dma_mapping_list;
NvBool bPersistent;
void *private;
} nv_p2p_mem_info_t;
int nvidia_p2p_cap_persistent_pages = 1;
EXPORT_SYMBOL(nvidia_p2p_cap_persistent_pages);
// declared and created in nv.c
extern void *nvidia_p2p_page_t_cache;
@@ -238,6 +239,7 @@ static void nv_p2p_free_page_table(
}
static NV_STATUS nv_p2p_put_pages(
nv_p2p_page_table_type_t pt_type,
nvidia_stack_t * sp,
uint64_t p2p_token,
uint32_t va_space,
@@ -246,9 +248,6 @@ static NV_STATUS nv_p2p_put_pages(
)
{
NV_STATUS status;
struct nv_p2p_mem_info *mem_info = NULL;
mem_info = container_of(*page_table, nv_p2p_mem_info_t, page_table);
/*
* rm_p2p_put_pages returns NV_OK if the page_table was found and
@@ -258,8 +257,15 @@ static NV_STATUS nv_p2p_put_pages(
* rm_p2p_put_pages returns NV_ERR_OBJECT_NOT_FOUND if the page_table
* was already unlinked.
*/
if (mem_info->bPersistent)
if (pt_type == NV_P2P_PAGE_TABLE_TYPE_PERSISTENT)
{
struct nv_p2p_mem_info *mem_info = NULL;
/*
* It is safe to access persistent page_table as there is no async
* callback which can free it unlike non-persistent page_table.
*/
mem_info = container_of(*page_table, nv_p2p_mem_info_t, page_table);
status = rm_p2p_put_pages_persistent(sp, mem_info->private, *page_table);
}
else
@@ -273,7 +279,8 @@ static NV_STATUS nv_p2p_put_pages(
nv_p2p_free_page_table(*page_table);
*page_table = NULL;
}
else if (!mem_info->bPersistent && (status == NV_ERR_OBJECT_NOT_FOUND))
else if ((pt_type == NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT) &&
(status == NV_ERR_OBJECT_NOT_FOUND))
{
status = NV_OK;
*page_table = NULL;
@@ -327,7 +334,8 @@ static void nv_p2p_mem_info_free_callback(void *data)
nv_p2p_free_platform_data(&mem_info->page_table);
}
int nvidia_p2p_get_pages(
static int nv_p2p_get_pages(
nv_p2p_page_table_type_t pt_type,
uint64_t p2p_token,
uint32_t va_space,
uint64_t virtual_address,
@@ -376,9 +384,10 @@ int nvidia_p2p_get_pages(
*page_table = &(mem_info->page_table);
mem_info->bPersistent = (free_callback == NULL);
//asign length to temporary variable since do_div macro does in-place division
/*
* assign length to temporary variable since do_div macro does in-place
* division
*/
temp_length = length;
do_div(temp_length, page_size);
page_count = temp_length;
@@ -405,7 +414,7 @@ int nvidia_p2p_get_pages(
goto failed;
}
if (mem_info->bPersistent)
if (pt_type == NV_P2P_PAGE_TABLE_TYPE_PERSISTENT)
{
void *gpu_info = NULL;
@@ -415,12 +424,15 @@ int nvidia_p2p_get_pages(
goto failed;
}
status = rm_p2p_get_gpu_info(sp, virtual_address, length, &gpu_uuid, &gpu_info);
status = rm_p2p_get_gpu_info(sp, virtual_address, length,
&gpu_uuid, &gpu_info);
if (status != NV_OK)
{
goto failed;
}
(*page_table)->gpu_uuid = gpu_uuid;
rc = nvidia_dev_get_uuid(gpu_uuid, sp);
if (rc != 0)
{
@@ -432,8 +444,10 @@ int nvidia_p2p_get_pages(
bGetUuid = NV_TRUE;
status = rm_p2p_get_pages_persistent(sp, virtual_address, length, &mem_info->private,
physical_addresses, &entries, *page_table, gpu_info);
status = rm_p2p_get_pages_persistent(sp, virtual_address, length,
&mem_info->private,
physical_addresses, &entries,
*page_table, gpu_info);
if (status != NV_OK)
{
goto failed;
@@ -449,10 +463,11 @@ int nvidia_p2p_get_pages(
{
goto failed;
}
(*page_table)->gpu_uuid = gpu_uuid;
}
bGetPages = NV_TRUE;
(*page_table)->gpu_uuid = gpu_uuid;
status = os_alloc_mem((void *)&(*page_table)->pages,
(entries * sizeof(page)));
@@ -516,10 +531,12 @@ failed:
{
os_free_mem(physical_addresses);
}
if (wreqmb_h != NULL)
{
os_free_mem(wreqmb_h);
}
if (rreqmb_h != NULL)
{
os_free_mem(rreqmb_h);
@@ -527,7 +544,7 @@ failed:
if (bGetPages)
{
(void)nv_p2p_put_pages(sp, p2p_token, va_space,
(void)nv_p2p_put_pages(pt_type, sp, p2p_token, va_space,
virtual_address, page_table);
}
@@ -546,8 +563,45 @@ failed:
return nvidia_p2p_map_status(status);
}
int nvidia_p2p_get_pages(
uint64_t p2p_token,
uint32_t va_space,
uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void * data),
void *data
)
{
if (free_callback == NULL)
{
return -EINVAL;
}
return nv_p2p_get_pages(NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT,
p2p_token, va_space, virtual_address,
length, page_table, free_callback, data);
}
EXPORT_SYMBOL(nvidia_p2p_get_pages);
int nvidia_p2p_get_pages_persistent(
uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
uint32_t flags
)
{
if (flags != 0)
{
return -EINVAL;
}
return nv_p2p_get_pages(NV_P2P_PAGE_TABLE_TYPE_PERSISTENT, 0, 0,
virtual_address, length, page_table,
NULL, NULL);
}
EXPORT_SYMBOL(nvidia_p2p_get_pages_persistent);
/*
* This function is a no-op, but is left in place (for now), in order to allow
* third-party callers to build and run without errors or warnings. This is OK,
@@ -568,15 +622,14 @@ int nvidia_p2p_put_pages(
struct nvidia_p2p_page_table *page_table
)
{
struct nv_p2p_mem_info *mem_info = NULL;
NvU8 uuid[NVIDIA_P2P_GPU_UUID_LEN] = {0};
NV_STATUS status;
nvidia_stack_t *sp = NULL;
int rc = 0;
os_mem_copy(uuid, page_table->gpu_uuid, NVIDIA_P2P_GPU_UUID_LEN);
mem_info = container_of(page_table, nv_p2p_mem_info_t, page_table);
if (page_table == NULL)
{
return 0;
}
rc = nv_kmem_cache_alloc_stack(&sp);
if (rc != 0)
@@ -584,21 +637,56 @@ int nvidia_p2p_put_pages(
return -ENOMEM;
}
status = nv_p2p_put_pages(sp, p2p_token, va_space,
status = nv_p2p_put_pages(NV_P2P_PAGE_TABLE_TYPE_NON_PERSISTENT,
sp, p2p_token, va_space,
virtual_address, &page_table);
if (mem_info->bPersistent)
{
nvidia_dev_put_uuid(uuid, sp);
}
nv_kmem_cache_free_stack(sp);
return nvidia_p2p_map_status(status);
}
EXPORT_SYMBOL(nvidia_p2p_put_pages);
int nvidia_p2p_put_pages_persistent(
uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table,
uint32_t flags
)
{
NvU8 uuid[NVIDIA_P2P_GPU_UUID_LEN] = {0};
NV_STATUS status;
nvidia_stack_t *sp = NULL;
int rc = 0;
if (flags != 0)
{
return -EINVAL;
}
if (page_table == NULL)
{
return 0;
}
rc = nv_kmem_cache_alloc_stack(&sp);
if (rc != 0)
{
return -ENOMEM;
}
os_mem_copy(uuid, page_table->gpu_uuid, NVIDIA_P2P_GPU_UUID_LEN);
status = nv_p2p_put_pages(NV_P2P_PAGE_TABLE_TYPE_PERSISTENT,
sp, 0, 0, virtual_address, &page_table);
nvidia_dev_put_uuid(uuid, sp);
nv_kmem_cache_free_stack(sp);
return nvidia_p2p_map_status(status);
}
EXPORT_SYMBOL(nvidia_p2p_put_pages_persistent);
int nvidia_p2p_dma_map_pages(
struct pci_dev *peer,
struct nvidia_p2p_page_table *page_table,

85
kernel-open/nvidia/nv-p2p.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2011-2016 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2011-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -94,11 +94,10 @@ struct nvidia_p2p_params {
} nvidia_p2p_params_t;
/*
* Capability flag for users to detect
* Macro for users to detect
* driver support for persistent pages.
*/
extern int nvidia_p2p_cap_persistent_pages;
#define NVIDIA_P2P_CAP_PERSISTENT_PAGES
#define NVIDIA_P2P_CAP_GET_PAGES_PERSISTENT_API
/*
* This API is not supported.
@@ -173,11 +172,6 @@ struct nvidia_p2p_page_table {
* A pointer to the function to be invoked when the pages
* underlying the virtual address range are freed
* implicitly.
* If NULL, persistent pages will be returned.
* This means the pages underlying the range of GPU virtual memory
* will persist until explicitly freed by nvidia_p2p_put_pages().
* Persistent GPU memory mappings are not supported on PowerPC,
* MIG-enabled devices and vGPU.
* @param[in] data
* A non-NULL opaque pointer to private data to be passed to the
* callback function.
@@ -190,12 +184,48 @@ struct nvidia_p2p_page_table {
* insufficient resources were available to complete the operation.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_get_pages(uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address,
int nvidia_p2p_get_pages( uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address, uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void *data), void *data);
/*
* @brief
* Pin and make the pages underlying a range of GPU virtual memory
* accessible to a third-party device. The pages will persist until
* explicitly freed by nvidia_p2p_put_pages_persistent().
*
* Persistent GPU memory mappings are not supported on PowerPC,
* MIG-enabled devices and vGPU.
*
* This API only supports pinned, GPU-resident memory, such as that provided
* by cudaMalloc().
*
* This API may sleep.
*
* @param[in] virtual_address
* The start address in the specified virtual address space.
* Address must be aligned to the 64KB boundary.
* @param[in] length
* The length of the requested P2P mapping.
* Length must be a multiple of 64KB.
* @param[out] page_table
* A pointer to an array of structures with P2P PTEs.
* @param[in] flags
* Must be set to zero for now.
*
* @return
* 0 upon successful completion.
* -EINVAL if an invalid argument was supplied.
* -ENOTSUPP if the requested operation is not supported.
* -ENOMEM if the driver failed to allocate memory or if
* insufficient resources were available to complete the operation.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_get_pages_persistent(uint64_t virtual_address,
uint64_t length,
struct nvidia_p2p_page_table **page_table,
void (*free_callback)(void *data),
void *data);
uint32_t flags);
#define NVIDIA_P2P_DMA_MAPPING_VERSION 0x00020003
@@ -268,6 +298,8 @@ int nvidia_p2p_dma_unmap_pages(struct pci_dev *peer,
* Release a set of pages previously made accessible to
* a third-party device.
*
* This API may sleep.
*
* @param[in] p2p_token
* A token that uniquely identifies the P2P mapping.
* @param[in] va_space
@@ -282,10 +314,33 @@ int nvidia_p2p_dma_unmap_pages(struct pci_dev *peer,
* -EINVAL if an invalid argument was supplied.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_put_pages(uint64_t p2p_token, uint32_t va_space,
uint64_t virtual_address,
int nvidia_p2p_put_pages(uint64_t p2p_token,
uint32_t va_space, uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table);
/*
* @brief
* Release a set of persistent pages previously made accessible to
* a third-party device.
*
* This API may sleep.
*
* @param[in] virtual_address
* The start address in the specified virtual address space.
* @param[in] page_table
* A pointer to the array of structures with P2P PTEs.
* @param[in] flags
* Must be set to zero for now.
*
* @return
* 0 upon successful completion.
* -EINVAL if an invalid argument was supplied.
* -EIO if an unknown error occurred.
*/
int nvidia_p2p_put_pages_persistent(uint64_t virtual_address,
struct nvidia_p2p_page_table *page_table,
uint32_t flags);
/*
* @brief
* Free a third-party P2P page table. (This function is a no-op.)

78
kernel-open/nvidia/nv-pci.c
View File

@@ -269,6 +269,72 @@ resize:
#endif /* NV_PCI_REBAR_GET_POSSIBLE_SIZES_PRESENT */
}
static void
nv_init_coherent_link_info
(
nv_state_t *nv
)
{
#if defined(NV_DEVICE_PROPERTY_READ_U64_PRESENT) && \
defined(CONFIG_ACPI_NUMA) && \
NV_IS_EXPORT_SYMBOL_PRESENT_pxm_to_node
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
NvU64 pa = 0;
NvU64 pxm_start = 0;
NvU64 pxm_count = 0;
NvU32 pxm;
if (!NVCPU_IS_AARCH64)
return;
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-base-pa", &pa) != 0)
goto failed;
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-pxm-start", &pxm_start) != 0)
goto failed;
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-pxm-count", &pxm_count) != 0)
goto failed;
NV_DEV_PRINTF(NV_DBG_INFO, nv, "DSD properties: \n");
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU memory PA: 0x%lx \n", pa);
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU memory PXM start: %u \n", pxm_start);
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tGPU memory PXM count: %u \n", pxm_count);
nvl->coherent_link_info.gpu_mem_pa = pa;
for (pxm = pxm_start; pxm < (pxm_start + pxm_count); pxm++)
{
NvU32 node = pxm_to_node(pxm);
if (node != NUMA_NO_NODE)
{
set_bit(node, nvl->coherent_link_info.free_node_bitmap);
}
}
if (NVreg_EnableUserNUMAManagement)
{
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_OFFLINE);
nvl->numa_info.use_auto_online = NV_TRUE;
if (!bitmap_empty(nvl->coherent_link_info.free_node_bitmap, MAX_NUMNODES))
{
nvl->numa_info.node_id = find_first_bit(nvl->coherent_link_info.free_node_bitmap, MAX_NUMNODES);
}
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "GPU NUMA information: node id: %u PA: 0x%llx\n",
nvl->numa_info.node_id, nvl->coherent_link_info.gpu_mem_pa);
}
else
{
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "User-mode NUMA onlining disabled.\n");
}
return;
failed:
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Cannot get coherent link info.\n");
#endif
return;
}
/* find nvidia devices and set initial state */
static int
nv_pci_probe
@@ -463,6 +529,13 @@ next_bar:
NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
// With GH180 C2C, VF BAR1/2 are disabled and therefore expected to be 0.
if (j != NV_GPU_BAR_INDEX_REGS)
{
nv_printf(NV_DBG_INFO, "NVRM: ignore invalid BAR failure for BAR%d\n", j);
continue;
}
goto failed;
}
@@ -547,11 +620,16 @@ next_bar:
nv_init_ibmnpu_info(nv);
nv_init_coherent_link_info(nv);
#if defined(NVCPU_PPC64LE)
// Use HW NUMA support as a proxy for ATS support. This is true in the only
// PPC64LE platform where ATS is currently supported (IBM P9).
nv_ats_supported &= nv_platform_supports_numa(nvl);
#else
#if defined(NV_PCI_DEV_HAS_ATS_ENABLED)
nv_ats_supported &= pci_dev->ats_enabled;
#endif
#endif
if (nv_ats_supported)
{

17
kernel-open/nvidia/nv-procfs.c
View File

@@ -1016,6 +1016,23 @@ numa_status_read(
rm_status = rm_get_gpu_numa_info(sp, nv,
nid, numa_mem_addr, numa_mem_size,
list->addresses, &list->numEntries);
if (rm_status == NV_OK && *nid == NUMA_NO_NODE)
{
//
// RM returns NUMA_NO_NODE when running MIG instances because
// this rmClient is not subscribed to any MIG partition since
// it was subscribed to whole GPU only during RMInit and is not
// updated when MIG partitions are created.
// Returning error here so that numa_status results in EIO
// because of missing support in numa_status to use it for multiple
// numa nodes.
//
// TODO: add support for multiple numa nodes in numa_status interface
// and remove this check, bug 4006012
//
rm_status = NV_ERR_NOT_SUPPORTED;
}
*status = nv_get_numa_status(nvl);
done:

52
kernel-open/nvidia/nv-reg.h
View File

@@ -21,10 +21,15 @@
* DEALINGS IN THE SOFTWARE.
*/
//
// This file holds Unix-specific NVIDIA driver options
//
#ifndef _RM_REG_H_
#define _RM_REG_H_
#include "nvtypes.h"
#include "nv-firmware-registry.h"
/*
* use NV_REG_STRING to stringify a registry key when using that registry key
@@ -723,72 +728,33 @@
* When this option is enabled, the NVIDIA driver will enable use of GPU
* firmware.
*
* Possible mode values:
* 0 - Do not enable GPU firmware
* 1 - Enable GPU firmware
* 2 - (Default) Use the default enablement policy for GPU firmware
*
* Setting this to anything other than 2 will alter driver firmware-
* enablement policies, possibly disabling GPU firmware where it would
* have otherwise been enabled by default.
*
* If this key is set globally to the system, the driver may still attempt
* to apply some policies to maintain uniform firmware modes across all
* GPUS. This may result in the driver failing initialization on some GPUs
* to maintain such a policy.
*
*
* If this key is set using NVreg_RegistryDwordsPerDevice, then the driver
* will attempt to honor whatever configuration is specified without applying
* additional policies. This may also result in failed GPU initialzations if
* the configuration is not possible (for example if the firmware is missing
* from the filesystem, or the GPU is not capable).
*
* Policy bits:
*
* POLICY_ALLOW_FALLBACK:
* As the normal behavior is to fail GPU initialization if this registry
* entry is set in such a way that results in an invalid configuration, if
* instead the user would like the driver to automatically try to fallback
* to initializing the failing GPU with firmware disabled, then this bit can
* be set (ex: 0x11 means try to enable GPU firmware but fall back if needed).
* Note that this can result in a mixed mode configuration (ex: GPU0 has
* firmware enabled, but GPU1 does not).
* from the filesystem, or the GPU is not capable).
*
* NOTE: More details for this regkey can be found in nv-firmware-registry.h
*/
#define __NV_ENABLE_GPU_FIRMWARE EnableGpuFirmware
#define NV_REG_ENABLE_GPU_FIRMWARE NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE)
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_MASK 0x0000000F
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DISABLED 0x00000000
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_ENABLED 0x00000001
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DEFAULT 0x00000002
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_MASK 0x000000F0
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK 0x00000010
#define NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE 0x00000012
#define NV_REG_ENABLE_GPU_FIRMWARE_INVALID_VALUE 0xFFFFFFFF
/*
* Option: EnableGpuFirmwareLogs
*
* When this option is enabled, the NVIDIA driver will send GPU firmware logs
* to the system log, when possible.
*
* Possible values:
* 0 - Do not send GPU firmware logs to the system log
* 1 - Enable sending of GPU firmware logs to the system log
* 2 - (Default) Enable sending of GPU firmware logs to the system log for
* the debug kernel driver build only
* NOTE: More details for this regkey can be found in nv-firmware-registry.h
*/
#define __NV_ENABLE_GPU_FIRMWARE_LOGS EnableGpuFirmwareLogs
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS NV_REG_STRING(__NV_ENABLE_GPU_FIRMWARE_LOGS)
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_DISABLE 0x00000000
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE 0x00000001
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG 0x00000002
/*
* Option: EnableDbgBreakpoint
*

134
kernel-open/nvidia/nv.c
View File

@@ -139,9 +139,14 @@ struct semaphore nv_linux_devices_lock;
static NvTristate nv_chipset_is_io_coherent = NV_TRISTATE_INDETERMINATE;
NvU64 nv_shared_gpa_boundary = 0;
// True if all the successfully probed devices support ATS
// Assigned at device probe (module init) time
NvBool nv_ats_supported = NVCPU_IS_PPC64LE
#if defined(NV_PCI_DEV_HAS_ATS_ENABLED)
|| NV_TRUE
#endif
;
// allow an easy way to convert all debug printfs related to events
@@ -232,6 +237,22 @@ struct dev_pm_ops nv_pm_ops = {
#if defined(NVCPU_X86_64)
#define NV_AMD_SEV_BIT BIT(1)
#define NV_GENMASK_ULL(h, l) \
(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
static
void get_shared_gpa_boundary(
void
)
{
NvU32 priv_high = cpuid_ebx(0x40000003);
if (priv_high & BIT(22))
{
NvU32 isolation_config_b = cpuid_ebx(0x4000000C);
nv_shared_gpa_boundary = ((NvU64)1) << ((isolation_config_b & NV_GENMASK_ULL(11, 6)) >> 6);
}
}
static
NvBool nv_is_sev_supported(
void
@@ -246,6 +267,11 @@ NvBool nv_is_sev_supported(
if (eax < 0x8000001f)
return NV_FALSE;
/* By design, a VM using vTOM doesn't see the SEV setting */
get_shared_gpa_boundary();
if (nv_shared_gpa_boundary != 0)
return NV_TRUE;
eax = 0x8000001f;
ecx = 0;
native_cpuid(&eax, &ebx, &ecx, &edx);
@@ -274,6 +300,11 @@ void nv_sev_init(
#if defined(MSR_AMD64_SEV_ENABLED)
os_sev_enabled = (os_sev_status & MSR_AMD64_SEV_ENABLED);
#endif
/* By design, a VM using vTOM doesn't see the SEV setting */
if (nv_shared_gpa_boundary != 0)
os_sev_enabled = NV_TRUE;
#endif
}
@@ -1174,6 +1205,7 @@ static int nv_start_device(nv_state_t *nv, nvidia_stack_t *sp)
#endif
int rc = 0;
NvBool kthread_init = NV_FALSE;
NvBool remove_numa_memory_kthread_init = NV_FALSE;
NvBool power_ref = NV_FALSE;
rc = nv_get_rsync_info();
@@ -1311,6 +1343,15 @@ static int nv_start_device(nv_state_t *nv, nvidia_stack_t *sp)
if (rc)
goto failed;
nv->queue = &nvl->queue;
if (nv_platform_use_auto_online(nvl))
{
rc = nv_kthread_q_init(&nvl->remove_numa_memory_q,
"nv_remove_numa_memory");
if (rc)
goto failed;
remove_numa_memory_kthread_init = NV_TRUE;
}
}
if (!rm_init_adapter(sp, nv))
@@ -1399,6 +1440,12 @@ failed:
if (kthread_init && !(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
nv_kthread_q_stop(&nvl->bottom_half_q);
if (remove_numa_memory_kthread_init &&
!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
{
nv_kthread_q_stop(&nvl->remove_numa_memory_q);
}
if (nvl->isr_bh_unlocked_mutex)
{
os_free_mutex(nvl->isr_bh_unlocked_mutex);
@@ -1635,7 +1682,9 @@ void nv_shutdown_adapter(nvidia_stack_t *sp,
nv_state_t *nv,
nv_linux_state_t *nvl)
{
#if defined(NVCPU_PPC64LE)
validate_numa_shutdown_state(nvl);
#endif
rm_disable_adapter(sp, nv);
@@ -1687,6 +1736,9 @@ void nv_shutdown_adapter(nvidia_stack_t *sp,
}
rm_shutdown_adapter(sp, nv);
if (nv_platform_use_auto_online(nvl))
nv_kthread_q_stop(&nvl->remove_numa_memory_q);
}
/*
@@ -2241,6 +2293,7 @@ nvidia_ioctl(
}
api->status = nv_get_numa_status(nvl);
api->use_auto_online = nv_platform_use_auto_online(nvl);
api->memblock_size = nv_ctl_device.numa_memblock_size;
break;
}
@@ -4913,6 +4966,28 @@ NV_STATUS NV_API_CALL nv_get_device_memory_config(
status = NV_OK;
#endif
#if defined(NVCPU_AARCH64)
if (node_id != NULL)
{
*node_id = nvl->numa_info.node_id;
}
if (compr_addr_sys_phys)
{
*compr_addr_sys_phys = nvl->coherent_link_info.gpu_mem_pa;
}
if (addr_guest_phys)
{
*addr_guest_phys = nvl->coherent_link_info.gpu_mem_pa;
}
if (addr_width)
{
// TH500 PA width - NV_PFB_PRI_MMU_ATS_ADDR_RANGE_GRANULARITY
*addr_width = 48 - 37;
}
status = NV_OK;
#endif
return status;
}
@@ -5551,3 +5626,62 @@ void NV_API_CALL nv_get_updated_emu_seg(
}
}
NV_STATUS NV_API_CALL nv_get_egm_info(
nv_state_t *nv,
NvU64 *phys_addr,
NvU64 *size,
NvS32 *egm_node_id
)
{
#if defined(NV_DEVICE_PROPERTY_READ_U64_PRESENT) && \
defined(CONFIG_ACPI_NUMA) && \
NV_IS_EXPORT_SYMBOL_PRESENT_pxm_to_node
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
NvU64 pa, sz, pxm;
if (device_property_read_u64(nvl->dev, "nvidia,egm-pxm", &pxm) != 0)
{
goto failed;
}
if (device_property_read_u64(nvl->dev, "nvidia,egm-base-pa", &pa) != 0)
{
goto failed;
}
if (device_property_read_u64(nvl->dev, "nvidia,egm-size", &sz) != 0)
{
goto failed;
}
NV_DEV_PRINTF(NV_DBG_INFO, nv, "DSD properties: \n");
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tEGM base PA: 0x%llx \n", pa);
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tEGM size: 0x%llx \n", sz);
NV_DEV_PRINTF(NV_DBG_INFO, nv, "\tEGM _PXM: 0x%llx \n", pxm);
if (egm_node_id != NULL)
{
*egm_node_id = pxm_to_node(pxm);
nv_printf(NV_DBG_INFO, "EGM node id: %d\n", *egm_node_id);
}
if (phys_addr != NULL)
{
*phys_addr = pa;
nv_printf(NV_DBG_INFO, "EGM base addr: 0x%llx\n", *phys_addr);
}
if (size != NULL)
{
*size = sz;
nv_printf(NV_DBG_INFO, "EGM size: 0x%llx\n", *size);
}
return NV_OK;
failed:
#endif // NV_DEVICE_PROPERTY_READ_U64_PRESENT
NV_DEV_PRINTF(NV_DBG_INFO, nv, "Cannot get EGM info\n");
return NV_ERR_NOT_SUPPORTED;
}

49
kernel-open/nvidia/nv_gpu_ops.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2013-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -39,6 +39,7 @@
typedef struct gpuSession *gpuSessionHandle;
typedef struct gpuDevice *gpuDeviceHandle;
typedef struct gpuAddressSpace *gpuAddressSpaceHandle;
typedef struct gpuTsg *gpuTsgHandle;
typedef struct gpuChannel *gpuChannelHandle;
typedef struct gpuObject *gpuObjectHandle;
@@ -97,7 +98,11 @@ NV_STATUS nvGpuOpsPmaUnpinPages(void *pPma,
NvLength pageCount,
NvU64 pageSize);
NV_STATUS nvGpuOpsChannelAllocate(gpuAddressSpaceHandle vaSpace,
NV_STATUS nvGpuOpsTsgAllocate(gpuAddressSpaceHandle vaSpace,
const gpuTsgAllocParams *params,
gpuTsgHandle *tsgHandle);
NV_STATUS nvGpuOpsChannelAllocate(const gpuTsgHandle tsgHandle,
const gpuChannelAllocParams *params,
gpuChannelHandle *channelHandle,
gpuChannelInfo *channelInfo);
@@ -105,6 +110,8 @@ NV_STATUS nvGpuOpsChannelAllocate(gpuAddressSpaceHandle vaSpace,
NV_STATUS nvGpuOpsMemoryReopen(struct gpuAddressSpace *vaSpace,
NvHandle hSrcClient, NvHandle hSrcAllocation, NvLength length, NvU64 *gpuOffset);
void nvGpuOpsTsgDestroy(struct gpuTsg *tsg);
void nvGpuOpsChannelDestroy(struct gpuChannel *channel);
void nvGpuOpsMemoryFree(gpuAddressSpaceHandle vaSpace,
@@ -196,7 +203,7 @@ NV_STATUS nvGpuOpsGetPmaObject(struct gpuDevice *device,
void **pPma,
const UvmPmaStatistics **pPmaPubStats);
NV_STATUS nvGpuOpsInitAccessCntrInfo(struct gpuDevice *device, gpuAccessCntrInfo *pAccessCntrInfo);
NV_STATUS nvGpuOpsInitAccessCntrInfo(struct gpuDevice *device, gpuAccessCntrInfo *pAccessCntrInfo, NvU32 accessCntrIndex);
NV_STATUS nvGpuOpsDestroyAccessCntrInfo(struct gpuDevice *device,
gpuAccessCntrInfo *pAccessCntrInfo);
@@ -278,4 +285,40 @@ NV_STATUS nvGpuOpsPagingChannelPushStream(UvmGpuPagingChannel *channel,
NV_STATUS nvGpuOpsFlushReplayableFaultBuffer(struct gpuDevice *device);
// Interface used for CCSL
NV_STATUS nvGpuOpsCcslContextInit(struct ccslContext_t **ctx,
gpuChannelHandle channel);
NV_STATUS nvGpuOpsCcslContextClear(struct ccslContext_t *ctx);
NV_STATUS nvGpuOpsCcslLogDeviceEncryption(struct ccslContext_t *ctx,
NvU8 *decryptIv);
NV_STATUS nvGpuOpsCcslAcquireEncryptionIv(struct ccslContext_t *ctx,
NvU8 *encryptIv);
NV_STATUS nvGpuOpsCcslRotateIv(struct ccslContext_t *ctx,
NvU8 direction);
NV_STATUS nvGpuOpsCcslEncrypt(struct ccslContext_t *ctx,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 *outputBuffer,
NvU8 *authTagBuffer);
NV_STATUS nvGpuOpsCcslEncryptWithIv(struct ccslContext_t *ctx,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 *encryptIv,
NvU8 *outputBuffer,
NvU8 *authTagBuffer);
NV_STATUS nvGpuOpsCcslDecrypt(struct ccslContext_t *ctx,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 const *decryptIv,
NvU8 *outputBuffer,
NvU8 const *authTagBuffer);
NV_STATUS nvGpuOpsCcslSign(struct ccslContext_t *ctx,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 *authTagBuffer);
NV_STATUS nvGpuOpsQueryMessagePool(struct ccslContext_t *ctx,
NvU8 direction,
NvU64 *messageNum);
#endif /* _NV_GPU_OPS_H_*/

198
kernel-open/nvidia/nv_uvm_interface.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2013-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -435,7 +435,7 @@ EXPORT_SYMBOL(nvUvmInterfacePmaUnregisterEvictionCallbacks);
NV_STATUS nvUvmInterfacePmaAllocPages(void *pPma,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
UvmPmaAllocationOptions *pPmaAllocOptions,
NvU64 *pPages)
{
@@ -462,7 +462,7 @@ EXPORT_SYMBOL(nvUvmInterfacePmaAllocPages);
NV_STATUS nvUvmInterfacePmaPinPages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
NvU32 flags)
{
nvidia_stack_t *sp = NULL;
@@ -483,7 +483,7 @@ EXPORT_SYMBOL(nvUvmInterfacePmaPinPages);
NV_STATUS nvUvmInterfacePmaUnpinPages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize)
NvU64 pageSize)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
@@ -516,7 +516,7 @@ EXPORT_SYMBOL(nvUvmInterfaceMemoryFree);
void nvUvmInterfacePmaFreePages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
NvU32 flags)
{
nvidia_stack_t *sp = nvUvmGetSafeStack();
@@ -529,7 +529,7 @@ EXPORT_SYMBOL(nvUvmInterfacePmaFreePages);
NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace,
UvmGpuPointer gpuPointer, NvLength length, void **cpuPtr,
NvU32 pageSize)
NvU64 pageSize)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
@@ -557,7 +557,39 @@ void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace,
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryCpuUnMap);
NV_STATUS nvUvmInterfaceChannelAllocate(uvmGpuAddressSpaceHandle vaSpace,
NV_STATUS nvUvmInterfaceTsgAllocate(uvmGpuAddressSpaceHandle vaSpace,
const UvmGpuTsgAllocParams *allocParams,
uvmGpuTsgHandle *tsg)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
if (nv_kmem_cache_alloc_stack(&sp) != 0)
{
return NV_ERR_NO_MEMORY;
}
status = rm_gpu_ops_tsg_allocate(sp,
(gpuAddressSpaceHandle)vaSpace,
allocParams,
(gpuTsgHandle *)tsg);
nv_kmem_cache_free_stack(sp);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceTsgAllocate);
void nvUvmInterfaceTsgDestroy(uvmGpuTsgHandle tsg)
{
nvidia_stack_t *sp = nvUvmGetSafeStack();
rm_gpu_ops_tsg_destroy(sp, (gpuTsgHandle)tsg);
nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceTsgDestroy);
NV_STATUS nvUvmInterfaceChannelAllocate(const uvmGpuTsgHandle tsg,
const UvmGpuChannelAllocParams *allocParams,
uvmGpuChannelHandle *channel,
UvmGpuChannelInfo *channelInfo)
@@ -571,7 +603,7 @@ NV_STATUS nvUvmInterfaceChannelAllocate(uvmGpuAddressSpaceHandle vaSpace,
}
status = rm_gpu_ops_channel_allocate(sp,
(gpuAddressSpaceHandle)vaSpace,
(gpuTsgHandle)tsg,
allocParams,
(gpuChannelHandle *)channel,
channelInfo);
@@ -868,7 +900,8 @@ NV_STATUS nvUvmInterfaceInitFaultInfo(uvmGpuDeviceHandle device,
EXPORT_SYMBOL(nvUvmInterfaceInitFaultInfo);
NV_STATUS nvUvmInterfaceInitAccessCntrInfo(uvmGpuDeviceHandle device,
UvmGpuAccessCntrInfo *pAccessCntrInfo)
UvmGpuAccessCntrInfo *pAccessCntrInfo,
NvU32 accessCntrIndex)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
@@ -880,7 +913,8 @@ NV_STATUS nvUvmInterfaceInitAccessCntrInfo(uvmGpuDeviceHandle device,
status = rm_gpu_ops_init_access_cntr_info(sp,
(gpuDeviceHandle)device,
pAccessCntrInfo);
pAccessCntrInfo,
accessCntrIndex);
nv_kmem_cache_free_stack(sp);
return status;
@@ -1432,6 +1466,150 @@ NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channe
}
EXPORT_SYMBOL(nvUvmInterfacePagingChannelPushStream);
NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext,
uvmGpuChannelHandle channel)
{
nvidia_stack_t *sp = NULL;
NV_STATUS status;
if (nv_kmem_cache_alloc_stack(&sp) != 0)
{
return NV_ERR_NO_MEMORY;
}
status = rm_gpu_ops_ccsl_context_init(sp, &uvmCslContext->ctx, (gpuChannelHandle)channel);
// Saving the stack in the context allows UVM to safely use the CSL layer
// in interrupt context without making new allocations. UVM serializes CSL
// API usage for a given context so the stack pointer does not need
// additional protection.
if (status != NV_OK)
{
nv_kmem_cache_free_stack(sp);
}
else
{
uvmCslContext->nvidia_stack = sp;
}
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslInitContext);
void nvUvmInterfaceDeinitCslContext(UvmCslContext *uvmCslContext)
{
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
rm_gpu_ops_ccsl_context_clear(sp, uvmCslContext->ctx);
nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceDeinitCslContext);
NV_STATUS nvUvmInterfaceCslLogDeviceEncryption(UvmCslContext *uvmCslContext,
UvmCslIv *decryptIv)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
status = rm_gpu_ops_ccsl_log_device_encryption(sp, uvmCslContext->ctx, (NvU8 *)decryptIv);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslLogDeviceEncryption);
NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext,
UvmCslDirection direction)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
status = rm_gpu_ops_ccsl_rotate_iv(sp, uvmCslContext->ctx, direction);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslRotateIv);
NV_STATUS nvUvmInterfaceCslAcquireEncryptionIv(UvmCslContext *uvmCslContext,
UvmCslIv *encryptIv)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
BUILD_BUG_ON(NV_OFFSETOF(UvmCslIv, fresh) != sizeof(encryptIv->iv));
status = rm_gpu_ops_ccsl_acquire_encryption_iv(sp, uvmCslContext->ctx, (NvU8*)encryptIv);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslAcquireEncryptionIv);
NV_STATUS nvUvmInterfaceCslEncrypt(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
UvmCslIv *encryptIv,
NvU8 *outputBuffer,
NvU8 *authTagBuffer)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
if (encryptIv != NULL)
status = rm_gpu_ops_ccsl_encrypt_with_iv(sp, uvmCslContext->ctx, bufferSize, inputBuffer, (NvU8*)encryptIv, outputBuffer, authTagBuffer);
else
status = rm_gpu_ops_ccsl_encrypt(sp, uvmCslContext->ctx, bufferSize, inputBuffer, outputBuffer, authTagBuffer);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslEncrypt);
NV_STATUS nvUvmInterfaceCslDecrypt(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
UvmCslIv const *decryptIv,
NvU8 *outputBuffer,
NvU8 const *authTagBuffer)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
status = rm_gpu_ops_ccsl_decrypt(sp,
uvmCslContext->ctx,
bufferSize,
inputBuffer,
(NvU8 *)decryptIv,
outputBuffer,
authTagBuffer);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslDecrypt);
NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 *authTagBuffer)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
status = rm_gpu_ops_ccsl_sign(sp, uvmCslContext->ctx, bufferSize, inputBuffer, authTagBuffer);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslSign);
NV_STATUS nvUvmInterfaceCslQueryMessagePool(UvmCslContext *uvmCslContext,
UvmCslDirection direction,
NvU64 *messageNum)
{
NV_STATUS status;
nvidia_stack_t *sp = uvmCslContext->nvidia_stack;
status = rm_gpu_ops_ccsl_query_message_pool(sp, uvmCslContext->ctx, direction, messageNum);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCslQueryMessagePool);
#else // NV_UVM_ENABLE
NV_STATUS nv_uvm_suspend(void)

9
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -136,6 +136,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += of_get_ibm_chip_id
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_bus_address
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_stop_and_remove_bus_device
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_rebar_get_possible_sizes
NV_CONFTEST_FUNCTION_COMPILE_TESTS += wait_for_random_bytes
NV_CONFTEST_FUNCTION_COMPILE_TESTS += register_cpu_notifier
NV_CONFTEST_FUNCTION_COMPILE_TESTS += cpuhp_setup_state
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_map_resource
@@ -216,6 +217,11 @@ NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_i2c_bus_status
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tegra_fuse_control_read
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tegra_get_platform
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_pci_find_host_bridge
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_send_cmd
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_set_init_cb
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_clear_init_cb
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_alloc_mem_from_gscco
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tsec_comms_free_gscco_mem
NV_CONFTEST_TYPE_COMPILE_TESTS += dma_ops
NV_CONFTEST_TYPE_COMPILE_TESTS += swiotlb_dma_ops
@@ -237,6 +243,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_driver_has_driver_managed_dma
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += memory_failure_has_trapno_arg
NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build
@@ -251,5 +258,7 @@ NV_CONFTEST_GENERIC_COMPILE_TESTS += vm_fault_t
NV_CONFTEST_GENERIC_COMPILE_TESTS += pci_class_multimedia_hd_audio
NV_CONFTEST_GENERIC_COMPILE_TESTS += drm_available
NV_CONFTEST_GENERIC_COMPILE_TESTS += vfio_pci_core_available
NV_CONFTEST_GENERIC_COMPILE_TESTS += mdev_available
NV_CONFTEST_GENERIC_COMPILE_TESTS += cmd_uphy_display_port_init
NV_CONFTEST_GENERIC_COMPILE_TESTS += cmd_uphy_display_port_off
NV_CONFTEST_GENERIC_COMPILE_TESTS += memory_failure_mf_sw_simulated_defined

212
kernel-open/nvidia/os-interface.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -173,7 +173,7 @@ void* NV_API_CALL os_alloc_semaphore
return NULL;
}
NV_INIT_SEMA(os_sema, initialValue);
sema_init(os_sema, initialValue);
return (void *)os_sema;
}
@@ -1422,8 +1422,7 @@ NV_STATUS NV_API_CALL os_get_euid(NvU32 *pSecToken)
return NV_OK;
}
// These functions are needed only on x86_64 platforms.
#if defined(NVCPU_X86_64)
#if defined(NVCPU_X86_64) || defined(NVCPU_AARCH64)
static NvBool os_verify_checksum(const NvU8 *pMappedAddr, NvU32 length)
{
@@ -1461,6 +1460,9 @@ static NvBool os_verify_checksum(const NvU8 *pMappedAddr, NvU32 length)
static NV_STATUS os_get_smbios_header_legacy(NvU64 *pSmbsAddr)
{
#if !defined(NVCPU_X86_64)
return NV_ERR_NOT_SUPPORTED;
#else
NV_STATUS status = NV_ERR_OPERATING_SYSTEM;
NvU8 *pMappedAddr = NULL;
NvU8 *pIterAddr = NULL;
@@ -1495,6 +1497,7 @@ static NV_STATUS os_get_smbios_header_legacy(NvU64 *pSmbsAddr)
os_unmap_kernel_space(pMappedAddr, SMBIOS_LEGACY_SIZE);
return status;
#endif
}
// This function is needed only if "efi" is enabled.
@@ -1571,13 +1574,13 @@ static NV_STATUS os_get_smbios_header_uefi(NvU64 *pSmbsAddr)
return status;
}
#endif // defined(NVCPU_X86_64)
#endif // defined(NVCPU_X86_64) || defined(NVCPU_AARCH64)
// The function locates the SMBIOS entry point.
NV_STATUS NV_API_CALL os_get_smbios_header(NvU64 *pSmbsAddr)
{
#if !defined(NVCPU_X86_64)
#if !defined(NVCPU_X86_64) && !defined(NVCPU_AARCH64)
return NV_ERR_NOT_SUPPORTED;
#else
NV_STATUS status = NV_OK;
@@ -1998,13 +2001,22 @@ NvBool NV_API_CALL os_is_nvswitch_present(void)
return !!pci_dev_present(nvswitch_pci_table);
}
void NV_API_CALL os_get_random_bytes
/*
* This function may sleep (interruptible).
*/
NV_STATUS NV_API_CALL os_get_random_bytes
(
NvU8 *bytes,
NvU16 numBytes
)
{
#if defined NV_WAIT_FOR_RANDOM_BYTES_PRESENT
if (wait_for_random_bytes() < 0)
return NV_ERR_NOT_READY;
#endif
get_random_bytes(bytes, numBytes);
return NV_OK;
}
NV_STATUS NV_API_CALL os_alloc_wait_queue
@@ -2106,3 +2118,189 @@ void NV_API_CALL os_nv_cap_close_fd
nv_cap_close_fd(fd);
}
NV_STATUS NV_API_CALL os_numa_add_gpu_memory
(
void *handle,
NvU64 offset,
NvU64 size,
NvU32 *nodeId
)
{
#if defined(NV_ADD_MEMORY_DRIVER_MANAGED_PRESENT)
int node = 0;
nv_linux_state_t *nvl = pci_get_drvdata(handle);
NvU64 base = offset + nvl->coherent_link_info.gpu_mem_pa;
int ret;
if (nodeId == NULL)
{
return NV_ERR_INVALID_ARGUMENT;
}
if (bitmap_empty(nvl->coherent_link_info.free_node_bitmap, MAX_NUMNODES))
{
return NV_ERR_IN_USE;
}
node = find_first_bit(nvl->coherent_link_info.free_node_bitmap, MAX_NUMNODES);
if (node == MAX_NUMNODES)
{
return NV_ERR_INVALID_STATE;
}
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_ONLINE_IN_PROGRESS);
#ifdef NV_ADD_MEMORY_DRIVER_MANAGED_HAS_MHP_FLAGS_ARG
ret = add_memory_driver_managed(node, base, size, "System RAM (NVIDIA)", MHP_NONE);
#else
ret = add_memory_driver_managed(node, base, size, "System RAM (NVIDIA)");
#endif
if (ret == 0)
{
struct zone *zone = &NODE_DATA(node)->node_zones[ZONE_MOVABLE];
NvU64 start_pfn = base >> PAGE_SHIFT;
NvU64 end_pfn = (base + size) >> PAGE_SHIFT;
if (zone->zone_start_pfn != start_pfn ||
zone_end_pfn(zone) != end_pfn)
{
nv_printf(NV_DBG_ERRORS, "GPU memory zone movable auto onlining failed!\n");
#ifdef NV_OFFLINE_AND_REMOVE_MEMORY_PRESENT
#ifdef NV_REMOVE_MEMORY_HAS_NID_ARG
if (offline_and_remove_memory(node, base, size) != 0)
#else
if (offline_and_remove_memory(base, size) != 0)
#endif
{
nv_printf(NV_DBG_ERRORS, "offline_and_remove_memory failed\n");
}
#endif
goto failed;
}
*nodeId = node;
clear_bit(node, nvl->coherent_link_info.free_node_bitmap);
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_ONLINE);
return NV_OK;
}
nv_printf(NV_DBG_ERRORS, "NVRM: Memory add failed. base: 0x%lx size: 0x%lx ret: %d\n",
base, size, ret);
failed:
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_ONLINE_FAILED);
return NV_ERR_OPERATING_SYSTEM;
#endif
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS NV_API_CALL os_numa_remove_gpu_memory
(
void *handle,
NvU64 offset,
NvU64 size,
NvU32 nodeId
)
{
#ifdef NV_ADD_MEMORY_DRIVER_MANAGED_PRESENT
nv_linux_state_t *nvl = pci_get_drvdata(handle);
#ifdef NV_OFFLINE_AND_REMOVE_MEMORY_PRESENT
NvU64 base = offset + nvl->coherent_link_info.gpu_mem_pa;
remove_numa_memory_info_t numa_info;
nv_kthread_q_item_t remove_numa_memory_q_item;
int ret;
#endif
if (nodeId >= MAX_NUMNODES)
{
return NV_ERR_INVALID_ARGUMENT;
}
if ((nodeId == NUMA_NO_NODE) || test_bit(nodeId, nvl->coherent_link_info.free_node_bitmap))
{
return NV_ERR_INVALID_ARGUMENT;
}
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_OFFLINE_IN_PROGRESS);
#ifdef NV_OFFLINE_AND_REMOVE_MEMORY_PRESENT
numa_info.base = base;
numa_info.size = size;
numa_info.nodeId = nodeId;
numa_info.ret = 0;
nv_kthread_q_item_init(&remove_numa_memory_q_item,
offline_numa_memory_callback,
&numa_info);
nv_kthread_q_schedule_q_item(&nvl->remove_numa_memory_q,
&remove_numa_memory_q_item);
nv_kthread_q_flush(&nvl->remove_numa_memory_q);
ret = numa_info.ret;
if (ret == 0)
{
set_bit(nodeId, nvl->coherent_link_info.free_node_bitmap);
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_OFFLINE);
return NV_OK;
}
nv_printf(NV_DBG_ERRORS, "NVRM: Memory remove failed. base: 0x%lx size: 0x%lx ret: %d\n",
base, size, ret);
#endif
NV_ATOMIC_SET(nvl->numa_info.status, NV_IOCTL_NUMA_STATUS_OFFLINE_FAILED);
return NV_ERR_OPERATING_SYSTEM;
#endif
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS NV_API_CALL os_offline_page_at_address
(
NvU64 address
)
{
#if defined(CONFIG_MEMORY_FAILURE)
int flags = 0;
int ret;
NvU64 pfn;
struct page *page = NV_GET_PAGE_STRUCT(address);
if (page == NULL)
{
nv_printf(NV_DBG_ERRORS, "NVRM: Failed to get page struct for address: 0x%llx\n",
address);
return NV_ERR_INVALID_ARGUMENT;
}
pfn = page_to_pfn(page);
#ifdef NV_MEMORY_FAILURE_MF_SW_SIMULATED_DEFINED
//
// Set MF_SW_SIMULATED flag so Linux kernel can differentiate this from a HW
// memory failure. HW memory failures cannot be unset via unpoison_memory() API.
//
// Currently, RM does not use unpoison_memory(), so it makes no difference
// whether or not MF_SW_SIMULATED is set. Regardless, it is semantically more
// correct to set MF_SW_SIMULATED.
//
flags |= MF_SW_SIMULATED;
#endif
#ifdef NV_MEMORY_FAILURE_HAS_TRAPNO_ARG
ret = memory_failure(pfn, 0, flags);
#else
ret = memory_failure(pfn, flags);
#endif
if (ret != 0)
{
nv_printf(NV_DBG_ERRORS, "NVRM: page offlining failed. address: 0x%llx pfn: 0x%llx ret: %d\n",
address, pfn, ret);
return NV_ERR_OPERATING_SYSTEM;
}
return NV_OK;
#else // !defined(CONFIG_MEMORY_FAILURE)
nv_printf(NV_DBG_ERRORS, "NVRM: memory_failure() not supported by kernel. page offlining failed. address: 0x%llx\n",
address);
return NV_ERR_NOT_SUPPORTED;
#endif
}