NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-31 05:29:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

535.43.02

Browse Source
This commit is contained in:
Andy Ritger
2023-05-30 10:11:36 -07:00
parent 6dd092ddb7
commit eb5c7665a1
1403 changed files with 295367 additions and 86235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
kernel-open/common/inc/nv-firmware-registry.h Normal file
View File

@@ -0,0 +1,83 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
//
// This file holds GPU firmware related registry key definitions that are
// shared between Windows and Unix
//
#ifndef NV_FIRMWARE_REGISTRY_H
#define NV_FIRMWARE_REGISTRY_H
//
// Registry key that when enabled, will enable use of GPU firmware.
//
// Possible mode values:
// 0 - Do not enable GPU firmware
// 1 - Enable GPU firmware
// 2 - (Default) Use the default enablement policy for GPU firmware
//
// Setting this to anything other than 2 will alter driver firmware-
// enablement policies, possibly disabling GPU firmware where it would
// have otherwise been enabled by default.
//
// Policy bits:
//
// POLICY_ALLOW_FALLBACK:
// As the normal behavior is to fail GPU initialization if this registry
// entry is set in such a way that results in an invalid configuration, if
// instead the user would like the driver to automatically try to fallback
// to initializing the failing GPU with firmware disabled, then this bit can
// be set (ex: 0x11 means try to enable GPU firmware but fall back if needed).
// Note that this can result in a mixed mode configuration (ex: GPU0 has
// firmware enabled, but GPU1 does not).
//
#define NV_REG_STR_ENABLE_GPU_FIRMWARE "EnableGpuFirmware"
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_MASK 0x0000000F
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DISABLED 0x00000000
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_ENABLED 0x00000001
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DEFAULT 0x00000002
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_MASK 0x000000F0
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK 0x00000010
#define NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE 0x00000012
//
// Registry key that when enabled, will send GPU firmware logs
// to the system log, when possible.
//
// Possible values:
// 0 - Do not send GPU firmware logs to the system log
// 1 - Enable sending of GPU firmware logs to the system log
// 2 - (Default) Enable sending of GPU firmware logs to the system log for
// the debug kernel driver build only
//
#define NV_REG_STR_ENABLE_GPU_FIRMWARE_LOGS "EnableGpuFirmwareLogs"
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_DISABLE 0x00000000
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE 0x00000001
#define NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG 0x00000002
#endif // NV_FIRMWARE_REGISTRY_H

6
kernel-open/common/inc/nv-firmware.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -81,11 +81,11 @@ static inline const char *nv_firmware_path(
{
switch (fw_chip_family)
{
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_AD10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA10X:
return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_ga10x.bin");
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_TU11X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_TU10X:
@@ -100,11 +100,11 @@ static inline const char *nv_firmware_path(
{
switch (fw_chip_family)
{
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_AD10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA10X:
return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_ga10x.bin");
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_TU11X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_TU10X:

2
kernel-open/common/inc/nv-hypervisor.h
View File

@@ -41,6 +41,7 @@ typedef enum _HYPERVISOR_TYPE
#define CMD_VGPU_VFIO_INJECT_INTERRUPT 1
#define CMD_VGPU_VFIO_REGISTER_MDEV 2
#define CMD_VGPU_VFIO_PRESENT 3
#define CMD_VFIO_PCI_CORE_PRESENT 4
#define MAX_VF_COUNT_PER_GPU 64
@@ -57,6 +58,7 @@ typedef struct
void *waitQueue;
void *nv;
NvU32 *vgpuTypeIds;
NvU8 **vgpuNames;
NvU32 numVgpuTypes;
NvU32 domain;
NvU8 bus;

1
kernel-open/common/inc/nv-ioctl-numa.h
View File

@@ -62,6 +62,7 @@ typedef struct nv_ioctl_numa_info
uint64_t memblock_size __aligned(8);
uint64_t numa_mem_addr __aligned(8);
uint64_t numa_mem_size __aligned(8);
uint8_t use_auto_online;
nv_offline_addresses_t offline_addresses __aligned(8);
} nv_ioctl_numa_info_t;

7
kernel-open/common/inc/nv-kthread-q.h
View File

@@ -28,15 +28,10 @@
#include <linux/list.h> // list
#include <linux/sched.h> // task_struct
#include <linux/numa.h> // NUMA_NO_NODE
#include <linux/semaphore.h>
#include "conftest.h"
#if defined(NV_LINUX_SEMAPHORE_H_PRESENT)
#include <linux/semaphore.h>
#else
#include <asm/semaphore.h>
#endif
////////////////////////////////////////////////////////////////////////////////
// nv_kthread_q:
//

64
kernel-open/common/inc/nv-linux.h
View File

@@ -633,6 +633,8 @@ static NvBool nv_numa_node_has_memory(int node_id)
free_pages(ptr, order); \
}
extern NvU64 nv_shared_gpa_boundary;
static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot, NvU32 extra)
{
pgprot_t prot = __pgprot(pgprot_val(vm_prot) | extra);
@@ -644,7 +646,18 @@ static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot, NvU32 extra)
* If cc_mkdec() is present, then pgprot_decrypted() can't be used.
*/
#if defined(NV_CC_MKDEC_PRESENT)
prot = __pgprot(__sme_clr(pgprot_val(vm_prot)));
if (nv_shared_gpa_boundary != 0)
{
/*
* By design, a VM using vTOM doesn't see the SEV setting and
* for AMD with vTOM, *set* means decrypted.
*/
prot = __pgprot(nv_shared_gpa_boundary | (pgprot_val(vm_prot)));
}
else
{
prot = __pgprot(__sme_clr(pgprot_val(vm_prot)));
}
#else
prot = pgprot_decrypted(prot);
#endif
@@ -1438,6 +1451,17 @@ struct nv_dma_device {
NvBool nvlink;
};
/* Properties of the coherent link */
typedef struct coherent_link_info_s {
/* Physical Address of the GPU memory in SOC AMAP. In the case of
* baremetal OS environment it is System Physical Address(SPA) and in the case
* of virutalized OS environment it is Intermediate Physical Address(IPA) */
NvU64 gpu_mem_pa;
/* Bitmap of NUMA node ids, corresponding to the reserved PXMs,
* available for adding GPU memory to the kernel as system RAM */
DECLARE_BITMAP(free_node_bitmap, MAX_NUMNODES);
} coherent_link_info_t;
#if defined(NV_LINUX_ACPI_EVENTS_SUPPORTED)
/*
* acpi data storage structure
@@ -1471,6 +1495,13 @@ typedef struct nv_linux_state_s {
/* IBM-NPU info associated with this GPU */
nv_ibmnpu_info_t *npu;
/* coherent link information */
coherent_link_info_t coherent_link_info;
/* Dedicated queue to be used for removing FB memory which is onlined
* to kernel as a NUMA node. Refer Bug : 3879845*/
nv_kthread_q_t remove_numa_memory_q;
/* NUMA node information for the platforms where GPU memory is presented
* as a NUMA node to the kernel */
struct {
@@ -1481,6 +1512,7 @@ typedef struct nv_linux_state_s {
/* NUMA online/offline status for platforms that support GPU memory as
* NUMA node */
atomic_t status;
NvBool use_auto_online;
} numa_info;
nvidia_stack_t *sp[NV_DEV_STACK_COUNT];
@@ -1946,6 +1978,36 @@ static inline int nv_set_numa_status(nv_linux_state_t *nvl, int status)
return 0;
}
static inline NvBool nv_platform_use_auto_online(nv_linux_state_t *nvl)
{
return nvl->numa_info.use_auto_online;
}
typedef struct {
NvU64 base;
NvU64 size;
NvU32 nodeId;
int ret;
} remove_numa_memory_info_t;
static void offline_numa_memory_callback
(
void *args
)
{
#ifdef NV_OFFLINE_AND_REMOVE_MEMORY_PRESENT
remove_numa_memory_info_t *pNumaInfo = (remove_numa_memory_info_t *)args;
#ifdef NV_REMOVE_MEMORY_HAS_NID_ARG
pNumaInfo->ret = offline_and_remove_memory(pNumaInfo->nodeId,
pNumaInfo->base,
pNumaInfo->size);
#else
pNumaInfo->ret = offline_and_remove_memory(pNumaInfo->base,
pNumaInfo->size);
#endif
#endif
}
typedef enum
{
NV_NUMA_STATUS_DISABLED = 0,

22
kernel-open/common/inc/nv-lock.h
View File

@@ -29,17 +29,12 @@
#include <linux/spinlock.h>
#include <linux/rwsem.h>
#include <linux/sched.h> /* signal_pending, cond_resched */
#include <linux/semaphore.h>
#if defined(NV_LINUX_SCHED_SIGNAL_H_PRESENT)
#include <linux/sched/signal.h> /* signal_pending for kernels >= 4.11 */
#endif
#if defined(NV_LINUX_SEMAPHORE_H_PRESENT)
#include <linux/semaphore.h>
#else
#include <asm/semaphore.h>
#endif
#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_PREEMPT_RT_FULL)
typedef raw_spinlock_t nv_spinlock_t;
#define NV_SPIN_LOCK_INIT(lock) raw_spin_lock_init(lock)
@@ -62,20 +57,7 @@ typedef spinlock_t nv_spinlock_t;
#define NV_SPIN_UNLOCK_WAIT(lock) spin_unlock_wait(lock)
#endif
#if defined(NV_CONFIG_PREEMPT_RT)
#define NV_INIT_SEMA(sema, val) sema_init(sema,val)
#else
#if !defined(__SEMAPHORE_INITIALIZER) && defined(__COMPAT_SEMAPHORE_INITIALIZER)
#define __SEMAPHORE_INITIALIZER __COMPAT_SEMAPHORE_INITIALIZER
#endif
#define NV_INIT_SEMA(sema, val) \
{ \
struct semaphore __sema = \
__SEMAPHORE_INITIALIZER(*(sema), val); \
*(sema) = __sema; \
}
#endif
#define NV_INIT_MUTEX(mutex) NV_INIT_SEMA(mutex, 1)
#define NV_INIT_MUTEX(mutex) sema_init(mutex, 1)
static inline int nv_down_read_interruptible(struct rw_semaphore *lock)
{

31
kernel-open/common/inc/nv.h
View File

@@ -347,6 +347,12 @@ typedef struct nv_soc_irq_info_s {
/* DMA-capable device data, defined by kernel interface layer */
typedef struct nv_dma_device nv_dma_device_t;
typedef struct nv_phys_addr_range
{
NvU64 addr;
NvU64 len;
} nv_phys_addr_range_t;
typedef struct nv_state_t
{
void *priv; /* private data */
@@ -466,6 +472,9 @@ typedef struct nv_state_t
/* Bool to check if ISO iommu enabled */
NvBool iso_iommu_present;
/* Bool to check if NISO iommu enabled */
NvBool niso_iommu_present;
/* Bool to check if dma-buf is supported */
NvBool dma_buf_supported;
@@ -477,6 +486,8 @@ typedef struct nv_state_t
/* Bool to check if the device received a shutdown notification */
NvBool is_shutdown;
/* Bool to check if the GPU has a coherent sysmem link */
NvBool coherent;
} nv_state_t;
// These define need to be in sync with defines in system.h
@@ -505,6 +516,8 @@ struct nv_file_private_t
typedef struct gpuSession *nvgpuSessionHandle_t;
typedef struct gpuDevice *nvgpuDeviceHandle_t;
typedef struct gpuAddressSpace *nvgpuAddressSpaceHandle_t;
typedef struct gpuTsg *nvgpuTsgHandle_t;
typedef struct UvmGpuTsgAllocParams_tag nvgpuTsgAllocParams_t;
typedef struct gpuChannel *nvgpuChannelHandle_t;
typedef struct UvmGpuChannelInfo_tag *nvgpuChannelInfo_t;
typedef struct UvmGpuChannelAllocParams_tag nvgpuChannelAllocParams_t;
@@ -531,7 +544,7 @@ typedef struct UvmGpuPagingChannelAllocParams_tag nvgpuPagingChannelAllocPara
typedef struct UvmGpuPagingChannel_tag *nvgpuPagingChannelHandle_t;
typedef struct UvmGpuPagingChannelInfo_tag *nvgpuPagingChannelInfo_t;
typedef enum UvmPmaGpuMemoryType_tag nvgpuGpuMemoryType_t;
typedef NV_STATUS (*nvPmaEvictPagesCallback)(void *, NvU32, NvU64 *, NvU32, NvU64, NvU64, nvgpuGpuMemoryType_t);
typedef NV_STATUS (*nvPmaEvictPagesCallback)(void *, NvU64, NvU64 *, NvU32, NvU64, NvU64, nvgpuGpuMemoryType_t);
typedef NV_STATUS (*nvPmaEvictRangeCallback)(void *, NvU64, NvU64, nvgpuGpuMemoryType_t);
/*
@@ -599,6 +612,8 @@ typedef enum
#define NV_SOC_IS_ISO_IOMMU_PRESENT(nv) \
((nv)->iso_iommu_present)
#define NV_SOC_IS_NISO_IOMMU_PRESENT(nv) \
((nv)->niso_iommu_present)
/*
* GPU add/remove events
*/
@@ -813,6 +828,7 @@ nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **);
void NV_API_CALL nv_put_file_private(void *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_ibmnpu_genreg_info(nv_state_t *, NvU64 *, NvU64 *, void**);
NV_STATUS NV_API_CALL nv_get_ibmnpu_relaxed_ordering_mode(nv_state_t *nv, NvBool *mode);
@@ -920,6 +936,7 @@ NV_STATUS NV_API_CALL rm_write_registry_string (nvidia_stack_t *, nv_state_t *
void NV_API_CALL rm_parse_option_string (nvidia_stack_t *, const char *);
char* NV_API_CALL rm_remove_spaces (const char *);
char* NV_API_CALL rm_string_token (char **, const char);
void NV_API_CALL rm_vgpu_vfio_set_driver_vm(nvidia_stack_t *, NvBool);
NV_STATUS NV_API_CALL rm_run_rc_callback (nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_execute_work_item (nvidia_stack_t *, void *);
@@ -951,12 +968,12 @@ NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, N
NV_STATUS NV_API_CALL rm_p2p_register_callback (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *);
NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU32, NvU32, NvU64 *, void **);
NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *);
NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **);
NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *, void **);
void NV_API_CALL rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle);
NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64, NvU64 *);
NV_STATUS NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64);
NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle *, NvHandle *, NvHandle *, void **);
NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64, void *, nv_phys_addr_range_t **, NvU32 *);
void NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, nv_phys_addr_range_t **, NvU32);
NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle *, NvHandle *, NvHandle *, void **, NvBool *);
void NV_API_CALL rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *);
NV_STATUS NV_API_CALL rm_log_gpu_crash (nv_stack_t *, nv_state_t *);
@@ -994,7 +1011,7 @@ void NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);
NvBool NV_API_CALL rm_is_altstack_in_use(void);
/* vGPU VFIO specific functions */
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32, NvBool *);
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32);
NV_STATUS NV_API_CALL nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16);
NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool);
NV_STATUS NV_API_CALL nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8);

34
kernel-open/common/inc/nv_firmware_types.h
View File

@@ -1,34 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef NV_FIRMWARE_TYPES_H
#define NV_FIRMWARE_TYPES_H
typedef enum {
NV_FIRMWARE_MODE_DISABLED = 0,
NV_FIRMWARE_MODE_ENABLED = 1,
NV_FIRMWARE_MODE_DEFAULT = 2,
NV_FIRMWARE_MODE_INVALID = 0xFF
} NvFirmwareMode;
#endif // NV_FIRMWARE_TYPES_H

330
kernel-open/common/inc/nv_uvm_interface.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2013-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -327,7 +327,7 @@ NV_STATUS nvUvmInterfaceGetPmaObject(uvmGpuDeviceHandle device,
// Mirrors pmaEvictPagesCb_t, see its documentation in pma.h.
typedef NV_STATUS (*uvmPmaEvictPagesCallback)(void *callbackData,
NvU32 pageSize,
NvU64 pageSize,
NvU64 *pPages,
NvU32 count,
NvU64 physBegin,
@@ -390,7 +390,7 @@ void nvUvmInterfacePmaUnregisterEvictionCallbacks(void *pPma);
*/
NV_STATUS nvUvmInterfacePmaAllocPages(void *pPma,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
UvmPmaAllocationOptions *pPmaAllocOptions,
NvU64 *pPages);
@@ -419,7 +419,7 @@ NV_STATUS nvUvmInterfacePmaAllocPages(void *pPma,
NV_STATUS nvUvmInterfacePmaPinPages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
NvU32 flags);
/*******************************************************************************
@@ -447,7 +447,7 @@ NV_STATUS nvUvmInterfacePmaPinPages(void *pPma,
NV_STATUS nvUvmInterfacePmaUnpinPages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize);
NvU64 pageSize);
/*******************************************************************************
nvUvmInterfaceMemoryFree
@@ -488,7 +488,7 @@ void nvUvmInterfaceMemoryFree(uvmGpuAddressSpaceHandle vaSpace,
void nvUvmInterfacePmaFreePages(void *pPma,
NvU64 *pPages,
NvLength pageCount,
NvU32 pageSize,
NvU64 pageSize,
NvU32 flags);
/*******************************************************************************
@@ -507,7 +507,7 @@ void nvUvmInterfacePmaFreePages(void *pPma,
NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace,
UvmGpuPointer gpuPointer,
NvLength length, void **cpuPtr,
NvU32 pageSize);
NvU64 pageSize);
/*******************************************************************************
uvmGpuMemoryCpuUnmap
@@ -517,16 +517,59 @@ NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace,
void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace,
void *cpuPtr);
/*******************************************************************************
nvUvmInterfaceTsgAllocate
This function allocates a Time-Slice Group (TSG).
allocParams must contain an engineIndex as TSGs need to be bound to an
engine type at allocation time. The possible values are [0,
UVM_COPY_ENGINE_COUNT_MAX) for CE engine type. Notably only the copy engines
that have UvmGpuCopyEngineCaps::supported set to true can be allocated.
Note that TSG is not supported on all GPU architectures for all engine
types, e.g., pre-Volta GPUs only support TSG for the GR/Compute engine type.
On devices that do not support HW TSGs on the requested engine, this API is
still required, i.e., a TSG handle is required in
nvUvmInterfaceChannelAllocate(), due to information stored in it necessary
for channel allocation. However, when HW TSGs aren't supported, a TSG handle
is essentially a "fake" TSG with no HW scheduling impact.
tsg is filled with the address of the corresponding TSG handle.
Arguments:
vaSpace[IN] - VA space linked to a client and a device under which
the TSG is allocated.
allocParams[IN] - structure with allocation settings.
tsg[OUT] - pointer to the new TSG handle.
Error codes:
NV_ERR_GENERIC
NV_ERR_INVALID_ARGUMENT
NV_ERR_NO_MEMORY
NV_ERR_NOT_SUPPORTED
*/
NV_STATUS nvUvmInterfaceTsgAllocate(uvmGpuAddressSpaceHandle vaSpace,
const UvmGpuTsgAllocParams *allocParams,
uvmGpuTsgHandle *tsg);
/*******************************************************************************
nvUvmInterfaceTsgDestroy
This function destroys a given TSG.
Arguments:
tsg[IN] - Tsg handle
*/
void nvUvmInterfaceTsgDestroy(uvmGpuTsgHandle tsg);
/*******************************************************************************
nvUvmInterfaceChannelAllocate
This function will allocate a channel bound to a copy engine
This function will allocate a channel bound to a copy engine(CE) or a SEC2
engine.
allocParams must contain an engineIndex as channels need to be bound to an
engine type at allocation time. The possible values are [0,
UVM_COPY_ENGINE_COUNT_MAX), but notably only the copy engines that have
UvmGpuCopyEngineCaps::supported set to true can be allocated. This struct
also contains information relative to GPFIFO and GPPut.
allocParams contains information relative to GPFIFO and GPPut.
channel is filled with the address of the corresponding channel handle.
@@ -536,17 +579,18 @@ void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace,
Host channel submission doorbell.
Arguments:
vaSpace[IN] - VA space linked to a client and a device under which
the channel will be allocated
tsg[IN] - Time-Slice Group that the channel will be a member.
allocParams[IN] - structure with allocation settings
channel[OUT] - pointer to the new channel handle
channelInfo[OUT] - structure filled with channel information
Error codes:
NV_ERR_GENERIC
NV_ERR_INVALID_ARGUMENT
NV_ERR_NO_MEMORY
NV_ERR_NOT_SUPPORTED
*/
NV_STATUS nvUvmInterfaceChannelAllocate(uvmGpuAddressSpaceHandle vaSpace,
NV_STATUS nvUvmInterfaceChannelAllocate(const uvmGpuTsgHandle tsg,
const UvmGpuChannelAllocParams *allocParams,
uvmGpuChannelHandle *channel,
UvmGpuChannelInfo *channelInfo);
@@ -554,7 +598,7 @@ NV_STATUS nvUvmInterfaceChannelAllocate(uvmGpuAddressSpaceHandle vaSpace,
/*******************************************************************************
nvUvmInterfaceChannelDestroy
This function destroys a given channel
This function destroys a given channel.
Arguments:
channel[IN] - channel handle
@@ -575,7 +619,7 @@ void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel);
NV_ERR_NO_MEMORY
*/
NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuDeviceHandle device,
UvmGpuCaps * caps);
UvmGpuCaps *caps);
/*******************************************************************************
nvUvmInterfaceQueryCopyEnginesCaps
@@ -946,13 +990,15 @@ NV_STATUS nvUvmInterfaceFlushReplayableFaultBuffer(uvmGpuDeviceHandle device);
Arguments:
device[IN] - Device handle associated with the gpu
pAccessCntrInfo[OUT] - Information provided by RM for access counter handling
accessCntrIndex[IN] - Access counter index
Error codes:
NV_ERR_GENERIC
NV_ERR_INVALID_ARGUMENT
*/
NV_STATUS nvUvmInterfaceInitAccessCntrInfo(uvmGpuDeviceHandle device,
UvmGpuAccessCntrInfo *pAccessCntrInfo);
UvmGpuAccessCntrInfo *pAccessCntrInfo,
NvU32 accessCntrIndex);
/*******************************************************************************
nvUvmInterfaceDestroyAccessCntrInfo
@@ -1402,4 +1448,250 @@ NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channe
char *methodStream,
NvU32 methodStreamSize);
/*******************************************************************************
CSL Interface and Locking
The following functions do not acquire the RM API or GPU locks and must not be called
concurrently with the same UvmCslContext parameter in different threads. The caller must
guarantee this exclusion.
* nvUvmInterfaceCslLogDeviceEncryption
* nvUvmInterfaceCslRotateIv
* nvUvmInterfaceCslEncrypt
* nvUvmInterfaceCslDecrypt
* nvUvmInterfaceCslSign
* nvUvmInterfaceCslQueryMessagePool
*/
/*******************************************************************************
nvUvmInterfaceCslInitContext
Allocates and initializes a CSL context for a given secure channel.
The lifetime of the context is the same as the lifetime of the secure channel
it is paired with.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
channel[IN] - Handle to a secure channel.
Error codes:
NV_ERR_INVALID_STATE - The system is not operating in Confidential Compute mode.
NV_ERR_INVALID_CHANNEL - The associated channel is not a secure channel.
NV_ERR_IN_USE - The context has already been initialized.
*/
NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext,
uvmGpuChannelHandle channel);
/*******************************************************************************
nvUvmInterfaceDeinitCslContext
Securely deinitializes and clears the contents of a context.
If context is already deinitialized then function returns immediately.
Arguments:
uvmCslContext[IN] - The CSL context.
*/
void nvUvmInterfaceDeinitCslContext(UvmCslContext *uvmCslContext);
/*******************************************************************************
nvUvmInterfaceCslLogDeviceEncryption
Returns an IV that can be later used in the nvUvmInterfaceCslEncrypt
method. The IV contains a "freshness bit" which value is set by this method
and subsequently dirtied by nvUvmInterfaceCslEncrypt to prevent
non-malicious reuse of the IV.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
encryptIv[OUT] - Parameter that is stored before a successful
device encryption. It is used as an input to
nvUvmInterfaceCslEncrypt.
Error codes:
NV_ERR_INSUFFICIENT_RESOURCES - New IV would cause a counter to overflow.
*/
NV_STATUS nvUvmInterfaceCslAcquireEncryptionIv(UvmCslContext *uvmCslContext,
UvmCslIv *encryptIv);
/*******************************************************************************
nvUvmInterfaceCslLogDeviceEncryption
Logs and checks information about device encryption.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
decryptIv[OUT] - Parameter that is stored before a successful
device encryption. It is used as an input to
nvUvmInterfaceCslDecrypt.
Error codes:
NV_ERR_INSUFFICIENT_RESOURCES - The device encryption would cause a counter
to overflow.
*/
NV_STATUS nvUvmInterfaceCslLogDeviceEncryption(UvmCslContext *uvmCslContext,
UvmCslIv *decryptIv);
/*******************************************************************************
nvUvmInterfaceCslRotateIv
Rotates the IV for a given channel and direction.
This function will rotate the IV on both the CPU and the GPU.
Outstanding messages that have been encrypted by the GPU should first be
decrypted before calling this function with direction equal to
UVM_CSL_DIR_GPU_TO_CPU. Similiarly, outstanding messages that have been
encrypted by the CPU should first be decrypted before calling this function
with direction equal to UVM_CSL_DIR_CPU_TO_GPU. For a given direction
the channel must be idle before calling this function. This function can be
called regardless of the value of the IV's message counter.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
direction[IN] - Either
- UVM_CSL_DIR_CPU_TO_GPU
- UVM_CSL_DIR_GPU_TO_CPU
Error codes:
NV_ERR_INSUFFICIENT_RESOURCES - The rotate operation would cause a counter
to overflow.
NV_ERR_INVALID_ARGUMENT - Invalid value for direction.
*/
NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext,
UvmCslDirection direction);
/*******************************************************************************
nvUvmInterfaceCslEncrypt
Encrypts data and produces an authentication tag.
Auth, input, and output buffers must not overlap. If they do then calling
this function produces undefined behavior. Performance is typically
maximized when the input and output buffers are 16-byte aligned. This is
natural alignment for AES block.
The encryptIV can be obtained from nvUvmInterfaceCslAcquireEncryptionIv.
However, it is optional. If it is NULL, the next IV in line will be used.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
bufferSize[IN] - Size of the input and output buffers in
units of bytes. Value can range from 1 byte
to (2^32) - 1 bytes.
inputBuffer[IN] - Address of plaintext input buffer.
encryptIv[IN/OUT] - IV to use for encryption. Can be NULL.
outputBuffer[OUT] - Address of ciphertext output buffer.
authTagBuffer[OUT] - Address of authentication tag buffer.
Its size is UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES.
Error codes:
NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes.
- The encryptIv has already been used.
*/
NV_STATUS nvUvmInterfaceCslEncrypt(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
UvmCslIv *encryptIv,
NvU8 *outputBuffer,
NvU8 *authTagBuffer);
/*******************************************************************************
nvUvmInterfaceCslDecrypt
Verifies the authentication tag and decrypts data.
Auth, input, and output buffers must not overlap. If they do then calling
this function produces undefined behavior. Performance is typically
maximized when the input and output buffers are 16-byte aligned. This is
natural alignment for AES block.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
bufferSize[IN] - Size of the input and output buffers in
units of bytes. Value can range from 1 byte
to (2^32) - 1 bytes.
decryptIv[IN] - Parameter given by nvUvmInterfaceCslLogDeviceEncryption.
inputBuffer[IN] - Address of ciphertext input buffer.
outputBuffer[OUT] - Address of plaintext output buffer.
authTagBuffer[IN] - Address of authentication tag buffer.
Its size is UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES.
Error codes:
NV_ERR_INSUFFICIENT_RESOURCES - The decryption operation would cause a
counter overflow to occur.
NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes.
NV_ERR_INVALID_DATA - Verification of the authentication tag fails.
*/
NV_STATUS nvUvmInterfaceCslDecrypt(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
UvmCslIv const *decryptIv,
NvU8 *outputBuffer,
NvU8 const *authTagBuffer);
/*******************************************************************************
nvUvmInterfaceCslSign
Generates an authentication tag for secure work launch.
Auth and input buffers must not overlap. If they do then calling this function produces
undefined behavior.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
bufferSize[IN] - Size of the input buffer in units of bytes.
Value can range from 1 byte to (2^32) - 1 bytes.
inputBuffer[IN] - Address of plaintext input buffer.
authTagBuffer[OUT] - Address of authentication tag buffer.
Its size is UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES.
Error codes:
NV_ERR_INSUFFICIENT_RESOURCES - The signing operation would cause a counter overflow to occur.
NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes.
*/
NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 *authTagBuffer);
/*******************************************************************************
nvUvmInterfaceCslQueryMessagePool
Returns the number of messages that can be encrypted before the message counter will overflow.
See "CSL Interface and Locking" for locking requirements.
This function does not perform dynamic memory allocation.
Arguments:
uvmCslContext[IN/OUT] - The CSL context.
direction[IN] - Either UVM_CSL_DIR_CPU_TO_GPU or UVM_CSL_DIR_GPU_TO_CPU.
messageNum[OUT] - Number of messages left before overflow.
Error codes:
NV_ERR_INVALID_ARGUMENT - The value of the direction parameter is illegal.
*/
NV_STATUS nvUvmInterfaceCslQueryMessagePool(UvmCslContext *uvmCslContext,
UvmCslDirection direction,
NvU64 *messageNum);
#endif // _NV_UVM_INTERFACE_H_

215
kernel-open/common/inc/nv_uvm_types.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2014-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -92,6 +92,7 @@ typedef unsigned long long UvmGpuPointer;
typedef struct uvmGpuSession_tag *uvmGpuSessionHandle; // gpuSessionHandle
typedef struct uvmGpuDevice_tag *uvmGpuDeviceHandle; // gpuDeviceHandle
typedef struct uvmGpuAddressSpace_tag *uvmGpuAddressSpaceHandle; // gpuAddressSpaceHandle
typedef struct uvmGpuTsg_tag *uvmGpuTsgHandle; // gpuTsgHandle
typedef struct uvmGpuChannel_tag *uvmGpuChannelHandle; // gpuChannelHandle
typedef struct uvmGpuCopyEngine_tag *uvmGpuCopyEngineHandle; // gpuObjectHandle
@@ -280,6 +281,15 @@ typedef struct UvmGpuChannelInfo_tag
// to kick off the new work.
//
volatile NvU32 *pWorkSubmissionToken;
// GPU VAs of both GPFIFO and GPPUT are needed in Confidential Computing
// so a channel can be controlled via another channel (SEC2 or WLC/LCIC)
NvU64 gpFifoGpuVa;
NvU64 gpPutGpuVa;
// GPU VA of work submission offset is needed in Confidential Computing
// so CE channels can ring doorbell of other channels as required for
// WLC/LCIC work submission
NvU64 workSubmissionOffsetGpuVa;
} UvmGpuChannelInfo;
typedef enum
@@ -292,6 +302,17 @@ typedef enum
UVM_BUFFER_LOCATION_VID = 2,
} UVM_BUFFER_LOCATION;
typedef struct UvmGpuTsgAllocParams_tag
{
// Interpreted as UVM_GPU_CHANNEL_ENGINE_TYPE
NvU32 engineType;
// Index of the engine the TSG is bound to.
// Ignored if engineType is anything other than
// UVM_GPU_CHANNEL_ENGINE_TYPE_CE.
NvU32 engineIndex;
} UvmGpuTsgAllocParams;
typedef struct UvmGpuChannelAllocParams_tag
{
NvU32 numGpFifoEntries;
@@ -300,13 +321,9 @@ typedef struct UvmGpuChannelAllocParams_tag
NvU32 gpFifoLoc;
NvU32 gpPutLoc;
// Index of the engine the channel will be bound to
// ignored if engineType is anything other than UVM_GPU_CHANNEL_ENGINE_TYPE_CE
NvU32 engineIndex;
// interpreted as UVM_GPU_CHANNEL_ENGINE_TYPE
NvU32 engineType;
// Allocate the channel as secure. This flag should only be set when
// Confidential Compute is enabled.
NvBool secure;
} UvmGpuChannelAllocParams;
typedef struct UvmGpuPagingChannelAllocParams_tag
@@ -350,6 +367,9 @@ typedef struct
// True if the CE can be used for P2P transactions
NvBool p2p:1;
// True if the CE supports encryption
NvBool secure:1;
// Mask of physical CEs assigned to this LCE
//
// The value returned by RM for this field may change when a GPU is
@@ -372,39 +392,16 @@ typedef enum
UVM_LINK_TYPE_NVLINK_2,
UVM_LINK_TYPE_NVLINK_3,
UVM_LINK_TYPE_NVLINK_4,
UVM_LINK_TYPE_C2C,
} UVM_LINK_TYPE;
typedef struct UvmGpuCaps_tag
{
NvU32 sysmemLink; // UVM_LINK_TYPE
NvU32 sysmemLinkRateMBps; // See UvmGpuP2PCapsParams::totalLinkLineRateMBps
// If numaEnabled is NV_TRUE, then the system address of allocated GPU
// memory can be converted to struct pages. See
// UvmGpuInfo::systemMemoryWindowStart.
NvBool numaEnabled;
NvU32 numaNodeId;
// On ATS systems, GPUs connected to different CPU sockets can have peer
// traffic. They are called indirect peers. However, indirect peers are
// mapped using sysmem aperture. In order to disambiguate the location of a
// specific memory address, each GPU maps its memory to a different window
// in the System Physical Address (SPA) space. The following fields contain
// the base + size of such window for the GPU. systemMemoryWindowSize
// different than 0 indicates that the window is valid.
//
// - If the window is valid, then we can map GPU memory to the CPU as
// cache-coherent by adding the GPU address to the window start.
// - If numaEnabled is NV_TRUE, then we can also convert the system
// addresses of allocated GPU memory to struct pages.
//
// TODO: Bug 1986868: fix window start computation for SIMICS
NvU64 systemMemoryWindowStart;
NvU64 systemMemoryWindowSize;
// This tells if the GPU is connected to NVSwitch. On systems with NVSwitch
// all GPUs are connected to it. If connectedToSwitch is NV_TRUE,
// nvswitchMemoryWindowStart tells the base address for the GPU in the
// NVSwitch address space. It is used when creating PTEs of memory mappings
// to NVSwitch peers.
NvBool connectedToSwitch;
NvU64 nvswitchMemoryWindowStart;
} UvmGpuCaps;
typedef struct UvmGpuAddressSpaceInfo_tag
@@ -436,6 +433,8 @@ typedef struct UvmGpuAllocInfo_tag
NvBool bMemGrowsDown; // Causes RM to reserve physical heap from top of FB
NvBool bPersistentVidmem; // Causes RM to allocate persistent video memory
NvHandle hPhysHandle; // Handle for phys allocation either provided or retrieved
NvBool bUnprotected; // Allocation to be made in unprotected memory whenever
// SEV or GPU CC modes are enabled. Ignored otherwise
} UvmGpuAllocInfo;
typedef enum
@@ -584,6 +583,20 @@ typedef struct UvmGpuClientInfo_tag
NvHandle hSmcPartRef;
} UvmGpuClientInfo;
typedef enum
{
UVM_GPU_CONF_COMPUTE_MODE_NONE,
UVM_GPU_CONF_COMPUTE_MODE_APM,
UVM_GPU_CONF_COMPUTE_MODE_HCC,
UVM_GPU_CONF_COMPUTE_MODE_COUNT
} UvmGpuConfComputeMode;
typedef struct UvmGpuConfComputeCaps_tag
{
// Out: GPU's confidential compute mode
UvmGpuConfComputeMode mode;
} UvmGpuConfComputeCaps;
#define UVM_GPU_NAME_LENGTH 0x40
typedef struct UvmGpuInfo_tag
@@ -648,6 +661,31 @@ typedef struct UvmGpuInfo_tag
UvmGpuClientInfo smcUserClientInfo;
// Confidential Compute capabilities of this GPU
UvmGpuConfComputeCaps gpuConfComputeCaps;
// UVM_LINK_TYPE
NvU32 sysmemLink;
// See UvmGpuP2PCapsParams::totalLinkLineRateMBps
NvU32 sysmemLinkRateMBps;
// On coherent systems each GPU maps its memory to a window in the System
// Physical Address (SPA) space. The following fields describe that window.
//
// systemMemoryWindowSize > 0 indicates that the window is valid. meaning
// that GPU memory can be mapped by the CPU as cache-coherent by adding the
// GPU address to the window start.
NvU64 systemMemoryWindowStart;
NvU64 systemMemoryWindowSize;
// This tells if the GPU is connected to NVSwitch. On systems with NVSwitch
// all GPUs are connected to it. If connectedToSwitch is NV_TRUE,
// nvswitchMemoryWindowStart tells the base address for the GPU in the
// NVSwitch address space. It is used when creating PTEs of memory mappings
// to NVSwitch peers.
NvBool connectedToSwitch;
NvU64 nvswitchMemoryWindowStart;
} UvmGpuInfo;
typedef struct UvmGpuFbInfo_tag
@@ -690,6 +728,9 @@ typedef struct UvmPmaStatistics_tag
volatile NvU64 numPages2m; // PMA-wide 2MB pages count across all regions
volatile NvU64 numFreePages64k; // PMA-wide free 64KB page count across all regions
volatile NvU64 numFreePages2m; // PMA-wide free 2MB pages count across all regions
volatile NvU64 numPages2mProtected; // PMA-wide 2MB pages count in protected memory
volatile NvU64 numFreePages64kProtected; // PMA-wide free 64KB page count in protected memory
volatile NvU64 numFreePages2mProtected; // PMA-wide free 2MB pages count in protected memory
} UvmPmaStatistics;
/*******************************************************************************
@@ -797,24 +838,80 @@ struct UvmOpsUvmEvents
#endif
};
#define UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES 32
#define UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES 16
typedef union UvmFaultMetadataPacket_tag
{
struct {
NvU8 authTag[UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES];
NvBool valid;
};
// padding to 32Bytes
NvU8 _padding[32];
} UvmFaultMetadataPacket;
typedef struct UvmGpuFaultInfo_tag
{
struct
{
// Register mappings obtained from RM
// Fault buffer GET register mapping.
//
// When Confidential Computing is enabled, GET refers to the shadow
// buffer (see bufferAddress below), and not to the actual HW buffer.
// In this setup, writes of GET (by UVM) do not result on re-evaluation
// of any interrupt condition.
volatile NvU32* pFaultBufferGet;
// Fault buffer PUT register mapping.
//
// When Confidential Computing is enabled, PUT refers to the shadow
// buffer (see bufferAddress below), and not to the actual HW buffer.
// In this setup, writes of PUT (by GSP-RM) do not result on
// re-evaluation of any interrupt condition.
volatile NvU32* pFaultBufferPut;
// Note: this variable is deprecated since buffer overflow is not a separate
// register from future chips.
// Note: this variable is deprecated since buffer overflow is not a
// separate register from future chips.
volatile NvU32* pFaultBufferInfo;
// Register mapping used to clear a replayable fault interrupt in
// Turing+ GPUs.
volatile NvU32* pPmcIntr;
// Register mapping used to enable replayable fault interrupts.
volatile NvU32* pPmcIntrEnSet;
// Register mapping used to disable replayable fault interrupts.
volatile NvU32* pPmcIntrEnClear;
// Register used to enable, or disable, faults on prefetches.
volatile NvU32* pPrefetchCtrl;
// Replayable fault interrupt mask identifier.
NvU32 replayableFaultMask;
// fault buffer cpu mapping and size
void* bufferAddress;
// Fault buffer CPU mapping
void* bufferAddress;
//
// When Confidential Computing is disabled, the mapping points to the
// actual HW fault buffer.
//
// When Confidential Computing is enabled, the mapping points to a
// copy of the HW fault buffer. This "shadow buffer" is maintained
// by GSP-RM.
// Size, in bytes, of the fault buffer pointed by bufferAddress.
NvU32 bufferSize;
// Mapping pointing to the start of the fault buffer metadata containing
// a 16Byte authentication tag and a valid byte. Always NULL when
// Confidential Computing is disabled.
UvmFaultMetadataPacket *bufferMetadata;
// Indicates whether UVM owns the replayable fault buffer.
// The value of this field is always NV_TRUE When Confidential Computing
// is disabled.
NvBool bUvmOwnsHwFaultBuffer;
} replayable;
struct
{
@@ -834,8 +931,19 @@ typedef struct UvmGpuFaultInfo_tag
// Preallocated stack for functions called from the UVM isr bottom half
void *isr_bh_sp;
// Used only when Hopper Confidential Compute is enabled
// Register mappings obtained from RM
volatile NvU32* pFaultBufferPut;
// Used only when Hopper Confidential Compute is enabled
// Cached get index of the non-replayable shadow buffer
NvU32 shadowBufferGet;
// See replayable.bufferMetadata
UvmFaultMetadataPacket *shadowBufferMetadata;
} nonReplayable;
NvHandle faultBufferHandle;
struct Device *pDevice;
} UvmGpuFaultInfo;
struct Device;
@@ -871,12 +979,6 @@ typedef struct UvmGpuAccessCntrInfo_tag
void* bufferAddress;
NvU32 bufferSize;
NvHandle accessCntrBufferHandle;
// The Notification address in the access counter notification msg does not
// contain the correct upper bits 63-47 for GPA-based notifications. RM
// provides us with the correct offset to be added.
// See Bug 1803015
NvU64 baseDmaSysmemAddr;
} UvmGpuAccessCntrInfo;
typedef enum
@@ -919,6 +1021,7 @@ typedef enum UvmPmaGpuMemoryType_tag
} UVM_PMA_GPU_MEMORY_TYPE;
typedef UvmGpuChannelInfo gpuChannelInfo;
typedef UvmGpuTsgAllocParams gpuTsgAllocParams;
typedef UvmGpuChannelAllocParams gpuChannelAllocParams;
typedef UvmGpuCaps gpuCaps;
typedef UvmGpuCopyEngineCaps gpuCeCaps;
@@ -943,4 +1046,24 @@ typedef UvmGpuPagingChannelInfo gpuPagingChannelInfo;
typedef UvmGpuPagingChannelAllocParams gpuPagingChannelAllocParams;
typedef UvmPmaAllocationOptions gpuPmaAllocationOptions;
// This struct shall not be accessed nor modified directly by UVM as it is
// entirely managed by the RM layer
typedef struct UvmCslContext_tag
{
struct ccslContext_t *ctx;
void *nvidia_stack;
} UvmCslContext;
typedef struct UvmCslIv
{
NvU8 iv[12];
NvU8 fresh;
} UvmCslIv;
typedef enum UvmCslDirection
{
UVM_CSL_DIR_CPU_TO_GPU,
UVM_CSL_DIR_GPU_TO_CPU
} UvmCslDirection;
#endif // _NV_UVM_TYPES_H_

18
kernel-open/common/inc/nvkms-kapi.h
View File

@@ -165,8 +165,6 @@ struct NvKmsKapiConnectorInfo {
NvU32 physicalIndex;
NvU32 headMask;
NvKmsConnectorSignalFormat signalFormat;
NvKmsConnectorType type;
@@ -194,6 +192,7 @@ struct NvKmsKapiStaticDisplayInfo {
NvU32 numPossibleClones;
NvKmsKapiDisplay possibleCloneHandles[NVKMS_KAPI_MAX_CLONE_DISPLAYS];
NvU32 headMask;
};
struct NvKmsKapiSyncpt {
@@ -1074,6 +1073,21 @@ struct NvKmsKapiFunctionsTable {
NvU64 *pPages
);
/*!
* Check if this memory object can be scanned out for display.
*
* \param [in] device A device allocated using allocateDevice().
*
* \param [in] memory The memory object to check for display support.
*
* \return NV_TRUE if this memory can be displayed, NV_FALSE if not.
*/
NvBool (*isMemoryValidForDisplay)
(
const struct NvKmsKapiDevice *device,
const struct NvKmsKapiMemory *memory
);
/*
* Import SGT as a memory handle.
*

2
kernel-open/common/inc/nvlimits.h
View File

@@ -25,7 +25,7 @@
//
// This file was generated with FINN, an NVIDIA coding tool.
// Source file: nvlimits.finn
// Source file: nvlimits.finn
//

1
kernel-open/common/inc/nvstatuscodes.h
View File

@@ -149,6 +149,7 @@ NV_STATUS_CODE(NV_ERR_NVLINK_TRAINING_ERROR, 0x00000077, "Nvlink Train
NV_STATUS_CODE(NV_ERR_NVLINK_CONFIGURATION_ERROR, 0x00000078, "Nvlink Configuration Error")
NV_STATUS_CODE(NV_ERR_RISCV_ERROR, 0x00000079, "Generic RISC-V assert or halt")
NV_STATUS_CODE(NV_ERR_FABRIC_MANAGER_NOT_PRESENT, 0x0000007A, "Fabric Manager is not loaded")
NV_STATUS_CODE(NV_ERR_ALREADY_SIGNALLED, 0x0000007B, "Semaphore Surface value already >= requested wait value")
// Warnings:
NV_STATUS_CODE(NV_WARN_HOT_SWITCH, 0x00010001, "WARNING Hot switch")

14
kernel-open/common/inc/nvtypes.h
View File

@@ -513,6 +513,12 @@ typedef struct
// place to re-locate these from nvos.h which cannot be included by a number
// of builds that need them
#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)
#define NV_ATTRIBUTE_UNUSED __attribute__((__unused__))
#else
#define NV_ATTRIBUTE_UNUSED
#endif
#if defined(_MSC_VER)
#if _MSC_VER >= 1310
@@ -536,8 +542,6 @@ typedef struct
#define NV_FORCERESULTCHECK
#define NV_ATTRIBUTE_UNUSED
#define NV_FORMAT_PRINTF(_f, _a)
#else // ! defined(_MSC_VER)
@@ -635,12 +639,6 @@ typedef struct
#define NV_FORCERESULTCHECK
#endif
#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)
#define NV_ATTRIBUTE_UNUSED __attribute__((__unused__))
#else
#define NV_ATTRIBUTE_UNUSED
#endif
/*
* Functions decorated with NV_FORMAT_PRINTF(f, a) have a format string at
* parameter number 'f' and variadic arguments start at parameter number 'a'.

7
kernel-open/common/inc/os-interface.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -188,7 +188,7 @@ NV_STATUS NV_API_CALL os_read_file (void *, NvU8 *, NvU64, NvU
NV_STATUS NV_API_CALL os_open_readonly_file (const char *, void **);
NV_STATUS NV_API_CALL os_open_and_read_file (const char *, NvU8 *, NvU64);
NvBool NV_API_CALL os_is_nvswitch_present (void);
void NV_API_CALL os_get_random_bytes (NvU8 *, NvU16);
NV_STATUS NV_API_CALL os_get_random_bytes (NvU8 *, NvU16);
NV_STATUS NV_API_CALL os_alloc_wait_queue (os_wait_queue **);
void NV_API_CALL os_free_wait_queue (os_wait_queue *);
void NV_API_CALL os_wait_uninterruptible (os_wait_queue *);
@@ -207,6 +207,9 @@ enum os_pci_req_atomics_type {
OS_INTF_PCIE_REQ_ATOMICS_128BIT
};
NV_STATUS NV_API_CALL os_enable_pci_req_atomics (void *, enum os_pci_req_atomics_type);
NV_STATUS NV_API_CALL os_numa_add_gpu_memory (void *, NvU64, NvU64, NvU32 *);
NV_STATUS NV_API_CALL os_numa_remove_gpu_memory (void *, NvU64, NvU64, NvU32);
NV_STATUS NV_API_CALL os_offline_page_at_address(NvU64 address);
extern NvU32 os_page_size;
extern NvU64 os_page_mask;

19
kernel-open/common/inc/rm-gpu-ops.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -56,7 +56,9 @@ NV_STATUS NV_API_CALL rm_gpu_ops_get_p2p_caps(nvidia_stack_t *, nvgpuDeviceHan
NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_map(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64, NvLength, void **, NvU32);
NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_ummap(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, void*);
NV_STATUS NV_API_CALL rm_gpu_ops_channel_allocate(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, const nvgpuChannelAllocParams_t *, nvgpuChannelHandle_t *, nvgpuChannelInfo_t);
NV_STATUS NV_API_CALL rm_gpu_ops_tsg_allocate(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, const nvgpuTsgAllocParams_t *, nvgpuTsgHandle_t *);
NV_STATUS NV_API_CALL rm_gpu_ops_tsg_destroy(nvidia_stack_t *, nvgpuTsgHandle_t);
NV_STATUS NV_API_CALL rm_gpu_ops_channel_allocate(nvidia_stack_t *, const nvgpuTsgHandle_t, const nvgpuChannelAllocParams_t *, nvgpuChannelHandle_t *, nvgpuChannelInfo_t);
NV_STATUS NV_API_CALL rm_gpu_ops_channel_destroy(nvidia_stack_t *, nvgpuChannelHandle_t);
NV_STATUS NV_API_CALL rm_gpu_ops_memory_free(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64);
NV_STATUS NV_API_CALL rm_gpu_ops_query_caps(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuCaps_t);
@@ -76,7 +78,7 @@ NV_STATUS NV_API_CALL rm_gpu_ops_destroy_fault_info(nvidia_stack_t *, nvgpuDevi
NV_STATUS NV_API_CALL rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *, nvgpuFaultInfo_t, void *, NvU32 *);
NV_STATUS NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t *, nvgpuDeviceHandle_t);
NV_STATUS NV_API_CALL rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack_t *, nvgpuFaultInfo_t, NvBool *);
NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t);
NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t, NvU32);
NV_STATUS NV_API_CALL rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t);
NV_STATUS NV_API_CALL rm_gpu_ops_own_access_cntr_intr(nvidia_stack_t *, nvgpuSessionHandle_t, nvgpuAccessCntrInfo_t, NvBool);
NV_STATUS NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t, nvgpuAccessCntrConfig_t);
@@ -99,4 +101,15 @@ NV_STATUS NV_API_CALL rm_gpu_ops_paging_channels_map(nvidia_stack_t *, nvgpuAdd
void NV_API_CALL rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64, nvgpuDeviceHandle_t);
NV_STATUS NV_API_CALL rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *, nvgpuPagingChannelHandle_t, char *, NvU32);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *, struct ccslContext_t **, nvgpuChannelHandle_t);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *, struct ccslContext_t *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_log_device_encryption(nvidia_stack_t *, struct ccslContext_t *, NvU8 *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_iv(nvidia_stack_t *, struct ccslContext_t *, NvU8);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_acquire_encryption_iv(nvidia_stack_t *, struct ccslContext_t *, NvU8 *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 *, NvU8 *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_encrypt_with_iv(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8*, NvU8 *, NvU8 *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 const *, NvU8 *, NvU8 const *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_sign(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 *);
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_query_message_pool(nvidia_stack_t *, struct ccslContext_t *, NvU8, NvU64 *);
#endif