NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-07 08:39:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

550.54.14

Browse Source
This commit is contained in:
Bernhard Stoeckner
2024-02-23 16:37:56 +01:00
parent 91676d6628
commit 476bd34534
186 changed files with 42509 additions and 37629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/nvidia/arch/nvalloc/unix/include/nv-reg.h
View File

@@ -839,6 +839,45 @@
#define __NV_ENABLE_NONBLOCKING_OPEN EnableNonblockingOpen
#define NV_ENABLE_NONBLOCKING_OPEN NV_REG_STRING(__NV_ENABLE_NONBLOCKING_OPEN)
/*
* Option: NVreg_ImexChannelCount
*
* Description:
*
* This option allows users to specify the number of IMEX (import/export)
* channels. Within an IMEX domain, the channels allow sharing memory
* securely in a multi-user environment using the CUDA driver's fabric handle
* based APIs.
*
* An IMEX domain is either an OS instance or a group of securely
* connected OS instances using the NVIDIA IMEX daemon. The option must
* be set to the same value on each OS instance within the IMEX domain.
*
* An IMEX channel is a logical entity that is represented by a /dev node.
* The IMEX channels are global resources within the IMEX domain. When
* exporter and importer CUDA processes have been granted access to the
* same IMEX channel, they can securely share memory.
*
* Note that the NVIDIA driver will not attempt to create the /dev nodes. Thus,
* the related CUDA APIs will fail with an insufficient permission error until
* the /dev nodes are set up. The creation of these /dev nodes,
* /dev/nvidia-caps-imex-channels/channelN, must be handled by the
* administrator, where N is the minor number. The major number can be
* queried from /proc/devices.
*
* nvidia-modprobe CLI support is available to set up the /dev nodes.
* NVreg_ModifyDeviceFiles, NVreg_DeviceFileGID, NVreg_DeviceFileUID
* and NVreg_DeviceFileMode will be honored by nvidia-modprobe.
*
* Possible values:
* 0 - Disable IMEX using CUDA driver's fabric handles.
* N - N IMEX channels will be enabled in the driver to facilitate N
* concurrent users. Default value is 2048 channels, and the current
* maximum value is 20-bit, same as Linux dev_t's minor number limit.
*/
#define __NV_IMEX_CHANNEL_COUNT ImexChannelCount
#define NV_REG_IMEX_CHANNEL_COUNT NV_REG_STRING(__NV_IMEX_CHANNEL_COUNT)
#if defined(NV_DEFINE_REGISTRY_KEY_TABLE)
/*
@@ -887,6 +926,7 @@ NV_DEFINE_REG_STRING_ENTRY(__NV_TEMPORARY_FILE_PATH, NULL);
NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL);
NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE);
NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048);
/*
*----------------registry database definition----------------------
@@ -933,6 +973,7 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_DBG_BREAKPOINT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IMEX_CHANNEL_COUNT),
{NULL, NULL}
};

10
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -621,6 +621,14 @@ typedef enum
#define NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv) \
(((nv)->flags & NV_FLAG_IN_SURPRISE_REMOVAL) != 0)
/*
* For console setup by EFI GOP, the base address is BAR1.
* For console setup by VBIOS, the base address is BAR2 + 16MB.
*/
#define NV_IS_CONSOLE_MAPPED(nv, addr) \
(((addr) == (nv)->bars[NV_GPU_BAR_INDEX_FB].cpu_address) || \
((addr) == ((nv)->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000)))
#define NV_SOC_IS_ISO_IOMMU_PRESENT(nv) \
((nv)->iommus.iso_iommu_present)
@@ -878,6 +886,8 @@ NvBool NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *);
NvU32 NV_API_CALL nv_get_os_type(void);
void NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end);
void NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *);
struct dma_buf;
typedef struct nv_dma_buf nv_dma_buf_t;
struct drm_gem_object;

8
src/nvidia/arch/nvalloc/unix/include/os-interface.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -160,10 +160,9 @@ NvBool NV_API_CALL os_is_vgx_hyper (void);
NV_STATUS NV_API_CALL os_inject_vgx_msi (NvU16, NvU64, NvU32);
NvBool NV_API_CALL os_is_grid_supported (void);
NvU32 NV_API_CALL os_get_grid_csp_support (void);
void NV_API_CALL os_get_screen_info (NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64, NvU64);
void NV_API_CALL os_bug_check (NvU32, const char *);
NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32);
NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **, void**);
NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **);
NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *);
NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *);
NV_STATUS NV_API_CALL os_get_euid (NvU32 *);
@@ -198,6 +197,8 @@ nv_cap_t* NV_API_CALL os_nv_cap_create_file_entry (nv_cap_t *, const char *,
void NV_API_CALL os_nv_cap_destroy_entry (nv_cap_t *);
int NV_API_CALL os_nv_cap_validate_and_dup_fd(const nv_cap_t *, int);
void NV_API_CALL os_nv_cap_close_fd (int);
NvS32 NV_API_CALL os_imex_channel_get (NvU64);
NvS32 NV_API_CALL os_imex_channel_count (void);
enum os_pci_req_atomics_type {
OS_INTF_PCIE_REQ_ATOMICS_32BIT,
@@ -219,6 +220,7 @@ extern NvU8 os_page_shift;
extern NvBool os_cc_enabled;
extern NvBool os_cc_tdx_enabled;
extern NvBool os_dma_buf_enabled;
extern NvBool os_imex_channel_is_supported;
/*
* ---------------------------------------------------------------------------

22
src/nvidia/arch/nvalloc/unix/src/escape.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -167,25 +167,11 @@ static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secI
}
else if (rmStatus == NV_ERR_INVALID_ADDRESS)
{
rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount,
&pPteArray, &pPageArray);
rmStatus = os_lookup_user_io_memory(pDescriptor, pageCount, &pPteArray);
if (rmStatus == NV_OK)
{
if (pPageArray != NULL)
{
pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPageArray;
pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY;
}
else if (pPteArray != NULL)
{
pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray;
pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY;
}
else
{
NV_ASSERT_FAILED("unknown memory import type");
rmStatus = NV_ERR_NOT_SUPPORTED;
}
pApi->data.AllocOsDesc.descriptor = (NvP64)(NvUPtr)pPteArray;
pApi->data.AllocOsDesc.descriptorType = NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY;
}
}
if (rmStatus != NV_OK)

5
src/nvidia/arch/nvalloc/unix/src/os-hypervisor.c
View File

@@ -1165,6 +1165,11 @@ NV_STATUS osIsVfioPciCorePresent(void)
return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VFIO_PCI_CORE_PRESENT);
}
NvU32 osGetGridCspSupport(void)
{
return os_get_grid_csp_support();
}
void initVGXSpecificRegistry(OBJGPU *pGpu)
{
NvU32 data32;

33
src/nvidia/arch/nvalloc/unix/src/os.c
View File

@@ -4828,6 +4828,39 @@ osRmCapInitDescriptor
*pCapDescriptor = NV_U64_MAX;
}
/*
* @brief Checks if IMEX channel support is present.
*/
NvBool
osImexChannelIsSupported(void)
{
return os_imex_channel_is_supported;
}
/*
* @brief Returns IMEX channel count.
*/
NvS32
osImexChannelCount
(
void
)
{
return os_imex_channel_count();
}
/*
* @brief Returns IMEX channel number.
*
* @param[in] descriptor OS specific descriptor to query channel number.
*
*/
NvS32
osImexChannelGet(NvU64 descriptor)
{
return os_imex_channel_get(descriptor);
}
/*
* @brief Generates random bytes which can be used as a universally unique
* identifier.

67
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -2153,6 +2153,7 @@ static NV_STATUS RmCreateMmapContextLocked(
RsClient *pClient = staticCast(pRmClient, RsClient);
KernelMemorySystem *pKernelMemorySystem = NULL;
NvBool bCoherentAtsCpuOffset = NV_FALSE;
NvBool bSriovHostCoherentFbOffset = NV_FALSE;
nv_state_t *pNv = NULL;
NvU64 addr = (NvU64)address;
NvU32 prot = 0;
@@ -2200,6 +2201,8 @@ static NV_STATUS RmCreateMmapContextLocked(
pNv = NV_GET_NV_STATE(pGpu);
pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
bCoherentAtsCpuOffset = IS_COHERENT_CPU_ATS_OFFSET(pKernelMemorySystem, addr, size);
bSriovHostCoherentFbOffset = os_is_vgx_hyper() &&
IS_COHERENT_FB_OFFSET(pKernelMemorySystem, addr, size);
}
//
@@ -2210,7 +2213,7 @@ static NV_STATUS RmCreateMmapContextLocked(
if ((pNv == NULL) ||
(!IS_REG_OFFSET(pNv, addr, size) &&
!IS_FB_OFFSET(pNv, addr, size) &&
!bCoherentAtsCpuOffset &&
!(bCoherentAtsCpuOffset || bSriovHostCoherentFbOffset) &&
!IS_IMEM_OFFSET(pNv, addr, size)))
{
pNv = nv_get_ctl_state();
@@ -2240,6 +2243,38 @@ static NV_STATUS RmCreateMmapContextLocked(
goto done;
}
}
else if (bSriovHostCoherentFbOffset)
{
status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap);
if (status != NV_OK)
{
goto done;
}
//
// nvuap->page_array(allocated in RmGetMmapPteArray) is not assigned
// to nvamc->page_array if onlining status is false(which is the case with
// bSriovHostCoherentFbOffset) and so doesn't get freed if not done here.
// The call to RmGetMmapPteArray is for getting the contig and num
// pages of the allocation.
//
os_free_mem(nvuap->page_array);
nvuap->page_array = NULL;
//
// This path is taken in the case of self-hosted SRIOV host where
// the coherent GPU memory is not onlined but the CPU mapping to
// the coherent GPU memory is done via C2C(instead of BAR1) and so
// only contig can be supported for now.
//
if (!nvuap->contig && (nvuap->num_pages > 1))
{
NV_PRINTF(LEVEL_ERROR, "Mapping of Non-contig allocation for "
"not onlined coherent GPU memory not supported\n");
status = NV_ERR_NOT_SUPPORTED;
goto done;
}
}
if (RmSetUserMapAccessRange(nvuap) != NV_OK)
{
@@ -5377,16 +5412,11 @@ NvBool rm_get_uefi_console_status(
NvU64 fbBaseAddress = 0;
NvBool bConsoleDevice = NV_FALSE;
// os_get_screen_info() will return dimensions and an address for
// any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a
// UEFI console check the fbBaseAddress: if it was set up by the EFI GOP
// driver, it will point into BAR1 (FB); if it was set up by the VBIOS,
// it will point to BAR2 + 16MB.
os_get_screen_info(&fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch,
nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
fbSize = (NvU64)fbHeight * (NvU64)fbPitch;
//
// nv_get_screen_info() will return dimensions and an address for
// any fbdev driver (e.g., efifb, vesafb, etc).
//
nv_get_screen_info(nv, &fbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize);
bConsoleDevice = (fbSize != 0);
@@ -5403,16 +5433,11 @@ NvU64 rm_get_uefi_console_size(
fbSize = fbWidth = fbHeight = fbDepth = fbPitch = 0;
// os_get_screen_info() will return dimensions and an address for
// any fbdev driver (e.g., efifb, vesafb, etc). To find if this is a
// UEFI console check the fbBaseAddress: if it was set up by the EFI GOP
// driver, it will point into BAR1 (FB); if it was set up by the VBIOS,
// it will point to BAR2 + 16MB.
os_get_screen_info(pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch,
nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
fbSize = (NvU64)fbHeight * (NvU64)fbPitch;
//
// nv_get_screen_info() will return dimensions and an address for
// any fbdev driver (e.g., efifb, vesafb, etc).
//
nv_get_screen_info(nv, pFbBaseAddress, &fbWidth, &fbHeight, &fbDepth, &fbPitch, &fbSize);
return fbSize;
}

7
src/nvidia/arch/nvalloc/unix/src/rm-gpu-ops.c
View File

@@ -582,13 +582,14 @@ NV_STATUS NV_API_CALL rm_gpu_ops_get_non_replayable_faults(nvidia_stack_t *sp,
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t *sp,
gpuDeviceHandle device)
NV_STATUS NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t *sp,
gpuFaultInfo *pFaultInfo,
NvBool bCopyAndFlush)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsFlushReplayableFaultBuffer(device);
rmStatus = nvGpuOpsFlushReplayableFaultBuffer(pFaultInfo, bCopyAndFlush);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}

7
src/nvidia/arch/nvalloc/unix/src/unix_console.c
View File

@@ -348,7 +348,7 @@ NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL
if ((memmgrGetReservedConsoleMemDesc(pGpu, pMemoryManager) != NULL) && bContinue)
{
NvU64 baseAddr;
NvU64 baseAddr, size;
NvU32 width, height, depth, pitch;
// There should only be one.
@@ -357,9 +357,8 @@ NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL
pParams->subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
// Console is either mapped to BAR1 or BAR2 + 16 MB
os_get_screen_info(&baseAddr, &width, &height, &depth, &pitch,
nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address,
nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + 0x1000000);
nv_get_screen_info(nv, &baseAddr, &width, &height, &depth,
&pitch, &size);
pParams->width = (NvU16)width;
pParams->height = (NvU16)height;

4
src/nvidia/arch/nvalloc/unix/src/unix_intr.c
View File

@@ -622,7 +622,7 @@ NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(
// Copies all valid packets in RM's and client's shadow buffer
status = kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, &threadState, faultsCopied,
NON_REPLAYABLE_FAULT_BUFFER);
NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE);
threadStateFreeISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR);
tlsIsrDestroy(pIsrAllocator);
@@ -659,7 +659,7 @@ static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked(
// Copies all valid packets in RM's and client's shadow buffer
return kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, pThreadState, pFaultsCopied,
NON_REPLAYABLE_FAULT_BUFFER);
NON_REPLAYABLE_FAULT_BUFFER, NV_FALSE);
return NV_OK;
}