NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-06 08:09:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

550.40.07

Browse Source
This commit is contained in:
Bernhard Stoeckner
2024-01-24 17:51:53 +01:00
parent bb2dac1f20
commit 91676d6628
1411 changed files with 261367 additions and 145959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/nvidia/arch/nvalloc/common/inc/flcnretval.h
View File

@@ -272,6 +272,7 @@ typedef NvU8 FLCN_STATUS;
#define FLCN_ERR_QUEUE_TASK_INVALID_EVENT_TYPE (0xE2U)
#define FLCN_ERR_QUEUE_TASK_INVALID_UNIT_ID (0xE3U)
#define FLCN_ERR_QUEUE_TASK_INVALID_CMD_TYPE (0xE4U)
#define FLCN_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE (0xE5U)
// Posted write errors
#define FLCN_ERR_POSTED_WRITE_FAILURE (0xF0U)

9
src/nvidia/arch/nvalloc/common/inc/fsp/fsp_nvdm_format.h
View File

@@ -21,8 +21,8 @@
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _FSP_NVDM_FORMAT_H_
#define _FSP_NVDM_FORMAT_H_
#ifndef FSP_NVDM_FORMAT_H
#define FSP_NVDM_FORMAT_H
#include "fsp/fsp_mctp_format.h"
#include "fsp/fsp_emem_channels.h"
@@ -35,12 +35,15 @@
#define NVDM_TYPE_HULK 0x11
#define NVDM_TYPE_FIRMWARE_UPDATE 0x12
#define NVDM_TYPE_PRC 0x13
#define NVDM_TYPE_COT 0x14
#define NVDM_TYPE_FSP_RESPONSE 0x15
#define NVDM_TYPE_CAPS_QUERY 0x16
#define NVDM_TYPE_INFOROM 0x17
#define NVDM_TYPE_SMBPBI 0x18
#define NVDM_TYPE_ROMREAD 0x1A
#define NVDM_TYPE_UEFI_RM 0x1C
#define NVDM_TYPE_UEFI_XTL_DEBUG_INTR 0x1D
#define NVDM_TYPE_TNVL 0x1F
#endif // _FSP_NVDM_FORMAT_H_
#endif // FSP_NVDM_FORMAT_H

51
src/nvidia/arch/nvalloc/common/inc/fsp/fsp_tnvl_rpc.h Normal file
View File

@@ -0,0 +1,51 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _FSP_TNVL_RPC_H_
#define _FSP_TNVL_RPC_H_
#define TNVL_CAPS_SUBMESSAGE_ID 0xFF
#pragma pack(1)
/*!
* @brief TNVL payload to query FSP capabilities
*/
typedef struct
{
NvU8 subMessageId;
} TNVL_RPC_CAPS_PAYLOAD;
/*!
* @brief TNVL response payload for FSP capability query
*/
typedef struct
{
NvU8 nvdmType;
NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse;
NvU8 rspPayload[40];
} TNVL_RPC_CAPS_RSP_PAYLOAD;
#pragma pack()
#endif // _FSP_TNVL_RPC_H_

4
src/nvidia/arch/nvalloc/common/inc/gsp/gsp_proxy_reg.h
View File

@@ -36,4 +36,8 @@
#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_DISABLE 0x00000000
#define NVGSP_PROXY_REG_CONF_COMPUTE_DEV_MODE_ENABLE 0x00000001
#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE 4:3
#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_NONE 0x00000000
#define NVGSP_PROXY_REG_CONF_COMPUTE_MULTI_GPU_MODE_PROTECTED_PCIE 0x00000001
#endif // NVGSP_PROXY_REG_H

21
src/nvidia/arch/nvalloc/common/inc/gsp/gspifpub.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -83,6 +83,24 @@ typedef struct GSP_RM_PARAMS
NvU64 bootArgsOffset;
} GSP_RM_PARAMS;
/*!
* @brief GSP-SPDM Parameters
*
* Parameters required to set up a communication mechanism between Kernel-RM
* and SPDM partition inside GSP.
*/
typedef struct GSP_SPDM_PARAMS
{
// Physical Memory Aperture through which all addresses are accessed
GSP_DMA_TARGET target;
// Physical offset in the memory aperture where SPDM payload is stored
NvU64 payloadBufferOffset;
// Size of the above payload buffer
NvU32 payloadBufferSize;
} GSP_SPDM_PARAMS;
/*!
* @brief GSP-CC Microcode Parameters for Boot Partitions
*/
@@ -91,6 +109,7 @@ typedef struct GSP_FMC_BOOT_PARAMS
GSP_FMC_INIT_PARAMS initParams;
GSP_ACR_BOOT_GSP_RM_PARAMS bootGspRmParams;
GSP_RM_PARAMS gspRmParams;
GSP_SPDM_PARAMS gspSpdmParams;
} GSP_FMC_BOOT_PARAMS;
#endif // GSPIFPUB_H

23
src/nvidia/arch/nvalloc/common/inc/nvcst.h
View File

@@ -89,9 +89,12 @@ CHIPSET_SETUP_FUNC(Amazon_Gravitron2_setupFunc)
CHIPSET_SETUP_FUNC(Fujitsu_A64FX_setupFunc)
CHIPSET_SETUP_FUNC(Ampere_Altra_setupFunc)
CHIPSET_SETUP_FUNC(Arm_NeoverseN1_setupFunc)
CHIPSET_SETUP_FUNC(Ampere_AmpereOne_setupFunc)
CHIPSET_SETUP_FUNC(Nvidia_T210_setupFunc)
CHIPSET_SETUP_FUNC(Nvidia_T194_setupFunc)
CHIPSET_SETUP_FUNC(Nvidia_TH500_setupFunc)
CHIPSET_SETUP_FUNC(PLDA_XpressRichAXI_setupFunc)
CHIPSET_SETUP_FUNC(Riscv_generic_setupFunc)
// Keep string length <=32 (including termination) to avoid string copy overflow
@@ -265,6 +268,16 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_HYGON, 0x790E, CS_HYGON_C86, "Hygon-C86-7151", NULL},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX, "Marvell Octeon CN96xx", ARMV8_generic_setupFunc},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX, "Marvell Octeon CN98xx", ARMV8_generic_setupFunc},
{PCI_VENDOR_ID_SIFIVE, 0x0000, CS_SIFIVE_FU740_C000, "SiFive FU740-000", Riscv_generic_setupFunc},
{PCI_VENDOR_ID_PLDA, 0x1111, CS_PLDA_XPRESSRICH_AXI_REF, "XpressRich-AXI Ref Design", PLDA_XpressRichAXI_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE200, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE201, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE202, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE203, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE204, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE205, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE206, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE207, CS_AMPERE_AMPEREONE, "Ampere AmpereOne", Ampere_AmpereOne_setupFunc},
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -300,6 +313,8 @@ VENDORNAME vendorName[] =
{PCI_VENDOR_ID_CADENCE, "Cadence"},
{PCI_VENDOR_ID_ARM, "ARM"},
{PCI_VENDOR_ID_ALIBABA, "Alibaba"},
{PCI_VENDOR_ID_SIFIVE, "SiFive"},
{PCI_VENDOR_ID_PLDA, "PLDA"},
{0, "Unknown"} // Indicates end of the table
};
@@ -363,6 +378,14 @@ ARMCSALLOWLISTINFO armChipsetAllowListInfo[] =
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX}, // Marvell OCTEON CN96xx
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX}, // Marvell OCTEON CN98xx
{PCI_VENDOR_ID_ALIBABA, 0x8000, CS_ALIBABA_YITIAN}, // Alibaba Yitian
{PCI_VENDOR_ID_AMPERE, 0xE200, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE201, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE202, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE203, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE204, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE205, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE206, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
{PCI_VENDOR_ID_AMPERE, 0xE207, CS_AMPERE_AMPEREONE}, // Ampere AmpereOne
// last element must have chipset CS_UNKNOWN (zero)
{0, 0, CS_UNKNOWN}

19
src/nvidia/arch/nvalloc/common/inc/oob/smbpbi.h
View File

@@ -238,6 +238,8 @@
0x00000013
#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_PMGR_PWR_MODULE_LIMIT_INFO_GET \
0x00000014
#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_MEMORY_CAPACITY_UTILIZATION_GET \
0x00000015
#define NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_POLL 0x000000ff
@@ -635,7 +637,7 @@
/* MSGBOX data, capability dword structure */
#define NV_MSGBOX_DATA_REG 31:0
#define NV_MSGBOX_DATA_CAP_COUNT 5
#define NV_MSGBOX_DATA_CAP_COUNT 6
#define NV_MSGBOX_DATA_CAP_0 0
#define NV_MSGBOX_DATA_CAP_0_TEMP_GPU_0 0:0
@@ -961,6 +963,11 @@
#define NV_MSGBOX_DATA_CAP_4_MAX_DRAM_CAPACITY_V1_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_4_MAX_DRAM_CAPACITY_V1_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_5 5
#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION 6:6
#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_AVAILABLE 0x00000001
/* ECC counters */
#define NV_MSGBOX_DATA_ECC_CNT_16BIT_DBE 31:16
#define NV_MSGBOX_DATA_ECC_CNT_16BIT_SBE 16:0
@@ -2035,6 +2042,16 @@ typedef struct
NvU32 clkMaxFreqMHz;
} NV_MSGBOX_OOB_CLOCK_LIMIT_CTRL_PARAMS;
/*!
* This structure is used to hold parameters for
* NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_MEMORY_CAPACITY_UTILIZATION_GET
*/
typedef struct
{
NvU32 capacityReservedMiB; //<! Device memory (in MiB) reserved for system use.
NvU32 capacityUsedMiB; //<! Allocated device memory (in MiB).
} NV_MSGBOX_MEMORY_CAPACITY_UTILIZATION_PARAMS;
/*
* Parameters for NV_MSGBOX_CMD_ARG1_ASYNC_REQUEST_TEST_MESSAGE_SEND
*/

6
src/nvidia/arch/nvalloc/common/inc/rmlsfm.h
View File

@@ -192,7 +192,11 @@
* to uniquely identify it.
* @note this macro should be updated as needed whenever LSF_FALCON_ID* defines are added. See Bug: 3833461
*/
#define LSF_FALCON_USES_INSTANCE(falconId) ((falconId == LSF_FALCON_ID_NVDEC_RISCV_EB) || (falconId == LSF_FALCON_ID_NVJPG) || (falconId == LSF_FALCON_ID_NVJPG_RISCV_EB) || (falconId == LSF_FALCON_ID_NVENC_RISCV_EB))
#define LSF_FALCON_USES_INSTANCE(falconId) ((falconId == LSF_FALCON_ID_NVDEC_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVJPG) || \
(falconId == LSF_FALCON_ID_NVJPG_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVENC_RISCV_EB) || \
(falconId == LSF_FALCON_ID_OFA_RISCV_EB))
/*!
* Size in entries of the ucode descriptor's dependency map.

81
src/nvidia/arch/nvalloc/common/inc/spdm/rmspdmtransport.h
View File

@@ -24,51 +24,70 @@
#ifndef _RM_SPDM_TRANSPORT_H_
#define _RM_SPDM_TRANSPORT_H_
#ifdef NVRM
#include "gpu/mem_mgr/mem_desc.h"
#else
#ifndef ADDR_SYSMEM
#define ADDR_SYSMEM (1) // System memory (PCI)
#endif // ADDR_SYSMEM
#endif //NVRM
/* ------------------------- Macros and Defines ----------------------------- */
// TODO CONFCOMP-1277: All these defines should be reviewed and simplified or removed.
//
// The following defines and macros are shared for any message-related constants
// shared between RM and endpoint.
//
// Defines used for polling-based communication between Kernel-RM and GSP.
#define NV_SPDM_PARTITION_BOOT_SUCCESS (0x59D3B007)
#define NV_SPDM_REQUESTER_SECRETS_DERIVED (0x59D35DED)
#define NV_SPDM_NO_MESSAGE_PENDING_TOKEN (0x00000000)
#define NV_SPDM_REQUESTER_MESSAGE_PENDING_TOKEN (0xAFAFAFAF)
#define NV_SPDM_RESPONDER_MESSAGE_PENDING_TOKEN (0xFAFAFAFA)
#define NV_SPDM_SECRET_TEARDOWN_SUCCESS (0x59D35EDE)
#define NV_SPDM_SECRET_TEARDOWN_FAILURE (0x59D35EFA)
#define NV_SPDM_SECRET_TEARDOWN_ACK (0x59D35EAC)
//
// The SPDM communication between Kernel-RM and GSP utilizes a buffer in sysmem accessible
// to both endpoints. The sysmem buffer will contain both a custom NV-defined header
// describing the message type and size, and the SPDM payload itself.
//
//
// Size of the full transportation buffer (header + SPDM payload) allocated in sysmem.
// We need to align sysmem allocations with page size, but need less space. So we
// over-allocate and tell GSP we only have the size we actually need.
//
#define NV_SPDM_SYSMEM_SURFACE_SIZE_PAGE_ALIGNED (0x2000)
#define NV_SPDM_SYSMEM_SURFACE_SIZE_IN_BYTES (0x1200)
#define NV_SPDM_SYSMEM_SURFACE_ALIGNMENT_IN_BYTES (256)
// Struct and defines for the header which prepends the SPDM payload.
#pragma pack(1)
#define NV_SPDM_MESSAGE_TYPE_NORMAL (0)
#define NV_SPDM_MESSAGE_TYPE_SECURED (1)
#define CC_SPDM_ENDPOINT_ID_INVALID (0xFFFFFFFF)
#define CC_SPDM_GUEST_ID_INVALID (0xFFFFFFFF)
#define NV_SPDM_DESC_HEADER_SIZE_IN_BYTE (NvU32)sizeof(NV_SPDM_DESC_HEADER)
#define NV_SPDM_DESC_HEADER_VERSION_1_0 (0x10)
#define NV_SPDM_DESC_HEADER_VERSION_CURRENT NV_SPDM_DESC_HEADER_VERSION_1_0
#define NV_SPDM_DESC_HEADER_ALIGNMENT (256)
#define NV_SPDM_RM_SURFACE_SIZE_IN_BYTE (0x2000)
#define NV_RM_BUFFER_SIZE_IN_BYTE (NV_SPDM_RM_SURFACE_SIZE_IN_BYTE - NV_SPDM_DESC_HEADER_SIZE_IN_BYTE)
// Limited by the transport size, do not increase without increasing transport buffer.
#define NV_SPDM_MAX_RANDOM_MSG_BYTES (0x80)
typedef struct _NV_SPDM_DESC_HEADER
{
NvU32 msgType;
NvU32 msgSizeByte;
NvU32 version;
} NV_SPDM_DESC_HEADER, *PNV_SPDM_DESC_HEADER;
#pragma pack()
#define NV_SPDM_MESSAGE_TYPE_NORMAL (0)
#define NV_SPDM_MESSAGE_TYPE_SECURED (1)
#define NV_SPDM_DESC_HEADER_VERSION_1_0 (0x10)
#define NV_SPDM_DESC_HEADER_VERSION_CURRENT (NV_SPDM_DESC_HEADER_VERSION_1_0)
#define NV_SPDM_DESC_HEADER_ALIGNMENT_IN_BYTES (NV_SPDM_SYSMEM_SURFACE_ALIGNMENT_IN_BYTES)
// Define utilized to determine size available for SPDM payload
#define NV_SPDM_MAX_SPDM_PAYLOAD_SIZE (NV_SPDM_SYSMEM_SURFACE_SIZE_IN_BYTES - sizeof(NV_SPDM_DESC_HEADER))
#define NV_SPDM_MAX_TRANSCRIPT_BUFFER_SIZE (2 * NV_SPDM_MAX_SPDM_PAYLOAD_SIZE)
// Limited by the transport size, do not increase without increasing transport buffer.
#define NV_SPDM_MAX_RANDOM_MSG_BYTES (0x80)
#ifdef NVRM
#include "gpu/mem_mgr/mem_desc.h"
#else
#ifndef ADDR_SYSMEM
// System memory (PCI)
#define ADDR_SYSMEM (1)
#endif // ADDR_SYSMEM
#endif //NVRM
#define NV_SPDM_DMA_ADDR_SPACE_DEFAULT (ADDR_SYSMEM)
#define NV_SPDM_DMA_REGION_ID_DEFAULT (0)
#pragma pack()
#endif // _RM_SPDM_TRANSPORT_H_

21
src/nvidia/arch/nvalloc/common/inc/spdm/rmspdmvendordef.h
View File

@@ -33,16 +33,23 @@
/* ------------------------- Macros ----------------------------------------- */
#define SPDM_ATTESTATION_REPORT_MAX_SIZE (0x2000)
#define SPDM_MAX_MESSAGE_BUFFER_SIZE (0x1000)
#define SPDM_MAX_EXCHANGE_BUFFER_SIZE (2 * SPDM_MAX_MESSAGE_BUFFER_SIZE)
#define SPDM_MAX_CERT_CHAIN_SIZE (SPDM_MAX_MESSAGE_BUFFER_SIZE)
#define SPDM_CERT_DEFAULT_SLOT_ID (0)
#define SPDM_CAPABILITIES_CT_EXPONENT_MAX (0xFF)
#define BIN_STR_CONCAT_BUFFER_MAX_BYTES (128)
#define SPDM_MAX_MEASUREMENT_BLOCK_COUNT (64)
#define SPDM_CERT_DEFAULT_SLOT_ID (0)
#define SPDM_CAPABILITIES_CT_EXPONENT_MAX (0x1F)
#define SPDM_DEFAULT_HEARTBEAT_PERIOD_IN_SEC (4)
#define NV_BYTE_TO_BIT_OVERFLOW_MASK_UINT32 (0xE0000000)
#define IS_BYTE_TO_BIT_OVERFLOW_UINT32(a) \
((a & NV_BYTE_TO_BIT_OVERFLOW_MASK_UINT32) != 0)
/* ---------------------- Public defines ------------------------------------- */
#define NV_KEY_UPDATE_TRIGGER_THRESHOLD (0x10000000)
#define NV_KEY_UPDATE_TRIGGER_ID_INVALID (0)
#define NV_KEY_UPDATE_TRIGGER_ID_GET_MEASUREMENTS (1)
#define NV_KEY_UPDATE_TRIGGER_ID_HEARTBEAT (2)
#define NV_KEY_UPDATE_TRIGGER_ID_LAST (3)
#define NV_SPDM_ENCAP_CERT_SIZE_IN_BYTE (578)
#endif // _RMSPDMVENDORDEF_H_

3
src/nvidia/arch/nvalloc/unix/include/nv-ioctl-numbers.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -39,5 +39,6 @@
#define NV_ESC_QUERY_DEVICE_INTR (NV_IOCTL_BASE + 13)
#define NV_ESC_SYS_PARAMS (NV_IOCTL_BASE + 14)
#define NV_ESC_EXPORT_TO_DMABUF_FD (NV_IOCTL_BASE + 17)
#define NV_ESC_WAIT_OPEN_COMPLETE (NV_IOCTL_BASE + 18)
#endif

8
src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -142,4 +142,10 @@ typedef struct nv_ioctl_export_to_dma_buf_fd
NvU32 status;
} nv_ioctl_export_to_dma_buf_fd_t;
typedef struct nv_ioctl_wait_open_complete
{
int rc;
NvU32 adapterStatus;
} nv_ioctl_wait_open_complete_t;
#endif

1
src/nvidia/arch/nvalloc/unix/include/nv-priv.h
View File

@@ -336,6 +336,7 @@ typedef struct
*/
NvU64 s0ix_gcoff_max_fb_size;
NvU32 pmc_boot_1;
NvU32 pmc_boot_42;
} nv_priv_t;

18
src/nvidia/arch/nvalloc/unix/include/nv-reg.h
View File

@@ -822,6 +822,23 @@
#define __NV_RM_NVLINK_BW RmNvlinkBandwidth
#define NV_RM_NVLINK_BW NV_REG_STRING(__NV_RM_NVLINK_BW)
/*
* Option: NVreg_EnableNonblockingOpen
*
* Description:
*
* When this option is enabled, the NVIDIA driver will try to perform any
* required device initialization in the background when /dev/nvidiaN devices
* are opened with the flag O_NONBLOCK.
*
* Possible Values:
* 0 = O_NONBLOCK flag when opening devices is ignored
* 1 = O_NONBLOCK flag when opening devices results in background device
* initialization (default)
*/
#define __NV_ENABLE_NONBLOCKING_OPEN EnableNonblockingOpen
#define NV_ENABLE_NONBLOCKING_OPEN NV_REG_STRING(__NV_ENABLE_NONBLOCKING_OPEN)
#if defined(NV_DEFINE_REGISTRY_KEY_TABLE)
/*
@@ -860,6 +877,7 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1);
NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL);
NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS_PER_DEVICE, NULL);

41
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -221,7 +221,6 @@ typedef struct
#define NV_RM_PAGE_MASK (NV_RM_PAGE_SIZE - 1)
#define NV_RM_TO_OS_PAGE_SHIFT (os_page_shift - NV_RM_PAGE_SHIFT)
#define NV_RM_PAGES_PER_OS_PAGE (1U << NV_RM_TO_OS_PAGE_SHIFT)
#define NV_RM_PAGES_TO_OS_PAGES(count) \
((((NvUPtr)(count)) >> NV_RM_TO_OS_PAGE_SHIFT) + \
((((count) & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) != 0) ? 1 : 0))
@@ -467,12 +466,6 @@ typedef struct nv_state_t
NvHandle hDisp;
} rmapi;
/* Bool to check if ISO iommu enabled */
NvBool iso_iommu_present;
/* Bool to check if NISO iommu enabled */
NvBool niso_iommu_present;
/* Bool to check if dma-buf is supported */
NvBool dma_buf_supported;
@@ -484,6 +477,22 @@ typedef struct nv_state_t
/* Bool to check if the GPU has a coherent sysmem link */
NvBool coherent;
/*
* NUMA node ID of the CPU to which the GPU is attached.
* Holds NUMA_NO_NODE on platforms that don't support NUMA configuration.
*/
NvS32 cpu_numa_node_id;
struct {
/* Bool to check if ISO iommu enabled */
NvBool iso_iommu_present;
/* Bool to check if NISO iommu enabled */
NvBool niso_iommu_present;
/* Display SMMU Stream IDs */
NvU32 dispIsoStreamId;
NvU32 dispNisoStreamId;
} iommus;
} nv_state_t;
// These define need to be in sync with defines in system.h
@@ -613,10 +622,10 @@ typedef enum
(((nv)->flags & NV_FLAG_IN_SURPRISE_REMOVAL) != 0)
#define NV_SOC_IS_ISO_IOMMU_PRESENT(nv) \
((nv)->iso_iommu_present)
((nv)->iommus.iso_iommu_present)
#define NV_SOC_IS_NISO_IOMMU_PRESENT(nv) \
((nv)->niso_iommu_present)
((nv)->iommus.niso_iommu_present)
/*
* GPU add/remove events
*/
@@ -779,8 +788,6 @@ NV_STATUS NV_API_CALL nv_register_phys_pages (nv_state_t *, NvU64 *, NvU64,
void NV_API_CALL nv_unregister_phys_pages (nv_state_t *, void *);
NV_STATUS NV_API_CALL nv_dma_map_sgt (nv_dma_device_t *, NvU64, NvU64 *, NvU32, void **);
NV_STATUS NV_API_CALL nv_dma_map_pages (nv_dma_device_t *, NvU64, NvU64 *, NvBool, NvU32, void **);
NV_STATUS NV_API_CALL nv_dma_unmap_pages (nv_dma_device_t *, NvU64, NvU64 *, void **);
NV_STATUS NV_API_CALL nv_dma_map_alloc (nv_dma_device_t *, NvU64, NvU64 *, NvBool, void **);
NV_STATUS NV_API_CALL nv_dma_unmap_alloc (nv_dma_device_t *, NvU64, NvU64 *, void **);
@@ -830,7 +837,7 @@ void NV_API_CALL nv_put_firmware(const void *);
nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **);
void NV_API_CALL nv_put_file_private(void *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_ibmnpu_genreg_info(nv_state_t *, NvU64 *, NvU64 *, void**);
@@ -877,9 +884,9 @@ struct drm_gem_object;
NV_STATUS NV_API_CALL nv_dma_import_sgt (nv_dma_device_t *, struct sg_table *, struct drm_gem_object *);
void NV_API_CALL nv_dma_release_sgt(struct sg_table *, struct drm_gem_object *);
NV_STATUS NV_API_CALL nv_dma_import_dma_buf (nv_dma_device_t *, struct dma_buf *, NvU32 *, void **, struct sg_table **, nv_dma_buf_t **);
NV_STATUS NV_API_CALL nv_dma_import_from_fd (nv_dma_device_t *, NvS32, NvU32 *, void **, struct sg_table **, nv_dma_buf_t **);
void NV_API_CALL nv_dma_release_dma_buf (void *, nv_dma_buf_t *);
NV_STATUS NV_API_CALL nv_dma_import_dma_buf (nv_dma_device_t *, struct dma_buf *, NvU32 *, struct sg_table **, nv_dma_buf_t **);
NV_STATUS NV_API_CALL nv_dma_import_from_fd (nv_dma_device_t *, NvS32, NvU32 *, struct sg_table **, nv_dma_buf_t **);
void NV_API_CALL nv_dma_release_dma_buf (nv_dma_buf_t *);
void NV_API_CALL nv_schedule_uvm_isr (nv_state_t *);
@@ -895,6 +902,8 @@ typedef void (*nvTegraDceClientIpcCallback)(NvU32, NvU32, NvU32, void *, void *)
NV_STATUS NV_API_CALL nv_get_num_phys_pages (void *, NvU32 *);
NV_STATUS NV_API_CALL nv_get_phys_pages (void *, void *, NvU32 *);
void NV_API_CALL nv_get_disp_smmu_stream_ids (nv_state_t *, NvU32 *, NvU32 *);
/*
* ---------------------------------------------------------------------------
*
@@ -921,6 +930,7 @@ NV_STATUS NV_API_CALL rm_ioctl (nvidia_stack_t *, nv_state_t *
NvBool NV_API_CALL rm_isr (nvidia_stack_t *, nv_state_t *, NvU32 *);
void NV_API_CALL rm_isr_bh (nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_isr_bh_unlocked (nvidia_stack_t *, nv_state_t *);
NvBool NV_API_CALL rm_is_msix_allowed (nvidia_stack_t *, nv_state_t *);
NV_STATUS NV_API_CALL rm_power_management (nvidia_stack_t *, nv_state_t *, nv_pm_action_t);
NV_STATUS NV_API_CALL rm_stop_user_channels (nvidia_stack_t *, nv_state_t *);
NV_STATUS NV_API_CALL rm_restart_user_channels (nvidia_stack_t *, nv_state_t *);
@@ -940,6 +950,7 @@ void NV_API_CALL rm_parse_option_string (nvidia_stack_t *, const char *
char* NV_API_CALL rm_remove_spaces (const char *);
char* NV_API_CALL rm_string_token (char **, const char);
void NV_API_CALL rm_vgpu_vfio_set_driver_vm(nvidia_stack_t *, NvBool);
NV_STATUS NV_API_CALL rm_get_adapter_status_external(nvidia_stack_t *, nv_state_t *);
NV_STATUS NV_API_CALL rm_run_rc_callback (nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_execute_work_item (nvidia_stack_t *, void *);

10
src/nvidia/arch/nvalloc/unix/include/os-interface.h
View File

@@ -67,7 +67,6 @@ typedef struct os_wait_queue os_wait_queue;
* ---------------------------------------------------------------------------
*/
NvU64 NV_API_CALL os_get_num_phys_pages (void);
NV_STATUS NV_API_CALL os_alloc_mem (void **, NvU64);
void NV_API_CALL os_free_mem (void *);
NV_STATUS NV_API_CALL os_get_current_time (NvU32 *, NvU32 *);
@@ -105,7 +104,6 @@ void* NV_API_CALL os_map_kernel_space (NvU64, NvU64, NvU32);
void NV_API_CALL os_unmap_kernel_space (void *, NvU64);
void* NV_API_CALL os_map_user_space (NvU64, NvU64, NvU32, NvU32, void **);
void NV_API_CALL os_unmap_user_space (void *, NvU64, void *);
NV_STATUS NV_API_CALL os_flush_cpu_cache (void);
NV_STATUS NV_API_CALL os_flush_cpu_cache_all (void);
NV_STATUS NV_API_CALL os_flush_user_cache (void);
void NV_API_CALL os_flush_cpu_write_combine_buffer(void);
@@ -230,14 +228,12 @@ extern NvBool os_dma_buf_enabled;
* ---------------------------------------------------------------------------
*/
#define NV_DBG_INFO 0x1
#define NV_DBG_SETUP 0x2
#define NV_DBG_INFO 0x0
#define NV_DBG_SETUP 0x1
#define NV_DBG_USERERRORS 0x2
#define NV_DBG_WARNINGS 0x3
#define NV_DBG_ERRORS 0x4
#define NV_DBG_HW_ERRORS 0x5
#define NV_DBG_FATAL 0x6
#define NV_DBG_FORCE_LEVEL(level) ((level) | (1 << 8))
void NV_API_CALL out_string(const char *str);
int NV_API_CALL nv_printf(NvU32 debuglevel, const char *printf_format, ...);

46
src/nvidia/arch/nvalloc/unix/src/dynamic-power.c
View File

@@ -81,6 +81,7 @@
#include <gpu/mem_mgr/mem_mgr.h>
#include <gpu/kern_gpu_power.h>
#include <gpu_mgr/gpu_mgr.h>
#include <core/locks.h>
#include "kernel/gpu/intr/intr.h"
@@ -1129,8 +1130,8 @@ os_ref_dynamic_power(
{
default:
case NV_DYNAMIC_POWER_STATE_IN_USE:
NV_PRINTF(LEVEL_ERROR, "NVRM: %s: Unexpected dynamic power state 0x%x\n",
__FUNCTION__, old_state);
NV_PRINTF(LEVEL_ERROR, "Unexpected dynamic power state 0x%x\n",
old_state);
/* fallthrough */
case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
status = nv_indicate_not_idle(nv);
@@ -1482,8 +1483,8 @@ static void timerCallbackForIdlePreConditions(
NV_ASSERT(0);
/* fallthrough */
case NV_DYNAMIC_POWER_STATE_IDLE_INDICATED:
NV_PRINTF(LEVEL_ERROR, "NVRM: %s: unexpected dynamic power state 0x%x\n",
__FUNCTION__, nvp->dynamic_power.state);
NV_PRINTF(LEVEL_ERROR, "unexpected dynamic power state 0x%x\n",
nvp->dynamic_power.state);
/* fallthrough */
case NV_DYNAMIC_POWER_STATE_IN_USE:
break;
@@ -1740,7 +1741,7 @@ static NV_STATUS CreateDynamicPowerCallbacks(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"NVRM: Error creating dynamic power precondition check callback\n");
"Error creating dynamic power precondition check callback\n");
nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
@@ -1760,7 +1761,7 @@ static NV_STATUS CreateDynamicPowerCallbacks(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"NVRM: Error creating callback to indicate GPU idle\n");
"Error creating callback to indicate GPU idle\n");
nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
@@ -1780,7 +1781,7 @@ static NV_STATUS CreateDynamicPowerCallbacks(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"NVRM: Error creating callback to decrease kernel refcount\n");
"Error creating callback to decrease kernel refcount\n");
nvp->dynamic_power.idle_precondition_check_event = NvP64_NULL;
nvp->dynamic_power.indicate_idle_event = NvP64_NULL;
nvp->dynamic_power.remove_idle_holdoff = NvP64_NULL;
@@ -1831,7 +1832,7 @@ static void RmScheduleCallbackForIdlePreConditionsUnderGpuLock(
}
else
{
NV_PRINTF(LEVEL_ERROR, "NVRM: Error scheduling precondition callback\n");
NV_PRINTF(LEVEL_ERROR, "Error scheduling precondition callback\n");
}
}
}
@@ -1893,7 +1894,7 @@ static void RmScheduleCallbackToIndicateIdle(
status = tmrCtrlCmdEventSchedule(pGpu, &scheduleEventParams);
if (status != NV_OK)
NV_PRINTF(LEVEL_ERROR, "NVRM: Error scheduling indicate idle callback\n");
NV_PRINTF(LEVEL_ERROR, "Error scheduling indicate idle callback\n");
}
}
@@ -1928,7 +1929,7 @@ static void RmScheduleCallbackToRemoveIdleHoldoff(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"NVRM: Error scheduling kernel refcount decrement callback\n");
"Error scheduling kernel refcount decrement callback\n");
}
else
{
@@ -1956,7 +1957,7 @@ static NvBool RmCheckRtd3GcxSupport(
if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED))
{
NV_PRINTF(LEVEL_NOTICE, "NVRM: RTD3/GC6 is not supported for this arch\n");
NV_PRINTF(LEVEL_NOTICE, "RTD3/GC6 is not supported for this arch\n");
return NV_FALSE;
}
@@ -1974,7 +1975,7 @@ static NvBool RmCheckRtd3GcxSupport(
if (!bGC6Support && !bGCOFFSupport)
{
NV_PRINTF(LEVEL_NOTICE,
"NVRM: Disabling RTD3. [GC6 support=%d GCOFF support=%d]\n",
"Disabling RTD3. [GC6 support=%d GCOFF support=%d]\n",
bGC6Support, bGCOFFSupport);
return NV_FALSE;
}
@@ -1986,7 +1987,7 @@ static NvBool RmCheckRtd3GcxSupport(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"NVRM: Failed to get Virtualization mode, status=0x%x\n",
"Failed to get Virtualization mode, status=0x%x\n",
status);
return NV_FALSE;
}
@@ -1994,7 +1995,7 @@ static NvBool RmCheckRtd3GcxSupport(
if ((virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NONE) &&
(virtModeParams.virtualizationMode != NV0080_CTRL_GPU_VIRTUALIZATION_MODE_NMOS))
{
NV_PRINTF(LEVEL_NOTICE, "NVRM: RTD3/GC6 is not supported on VM\n");
NV_PRINTF(LEVEL_NOTICE, "RTD3/GC6 is not supported on VM\n");
return NV_FALSE;
}
@@ -2025,7 +2026,7 @@ void RmInitDeferredDynamicPowerManagement(
{
nvp->dynamic_power.mode = NV_DYNAMIC_PM_NEVER;
nvp->dynamic_power.b_fine_not_supported = NV_TRUE;
NV_PRINTF(LEVEL_NOTICE, "NVRM: RTD3/GC6 is not supported\n");
NV_PRINTF(LEVEL_NOTICE, "RTD3/GC6 is not supported\n");
goto unlock;
}
osAddGpuDynPwrSupported(gpuGetInstance(pGpu));
@@ -2047,7 +2048,7 @@ unlock:
}
if (status != NV_OK)
NV_PRINTF(LEVEL_ERROR, "NVRM: Failed to register for dynamic power callbacks\n");
NV_PRINTF(LEVEL_ERROR, "Failed to register for dynamic power callbacks\n");
}
/*!
@@ -2262,7 +2263,7 @@ NV_STATUS RmGcxPowerManagement(
// GC6 3.0 is possible without AML override but due to changes required
// for GCOFF-1.0 in SBIOS & HW, AML override is needed for GC6-3.0 also.
//
NV_PRINTF(LEVEL_INFO,"NVRM: AML overrides present in Desktop");
NV_PRINTF(LEVEL_INFO,"AML overrides present in Desktop");
}
nv->d0_state_in_suspend = NV_FALSE;
@@ -2382,9 +2383,8 @@ NV_STATUS RmGcxPowerManagement(
{
status = NV_ERR_NOT_SUPPORTED;
NV_PRINTF(LEVEL_ERROR,
"NVRM: %s: System suspend failed with current system suspend configuration. "
"Please change the system suspend configuration to s2idle in /sys/power/mem_sleep.\n",
__FUNCTION__);
"System suspend failed with current system suspend configuration. "
"Please change the system suspend configuration to s2idle in /sys/power/mem_sleep.\n");
}
}
}
@@ -2625,8 +2625,7 @@ void RmHandleDisplayChange(
}
else
NV_PRINTF(LEVEL_ERROR,
"NVRM: %s: Failed to increment dynamic power refcount\n",
__FUNCTION__);
"Failed to increment dynamic power refcount\n");
}
}
@@ -2727,8 +2726,7 @@ static void RmQueueIdleSustainedWorkitem(
if (status != NV_OK)
{
NV_PRINTF(LEVEL_WARNING,
"NVRM: %s: Failed to queue RmHandleIdleSustained() workitem.\n",
__FUNCTION__);
"Failed to queue RmHandleIdleSustained() workitem.\n");
RmScheduleCallbackForIdlePreConditionsUnderGpuLock(pGpu);
return;
}

14
src/nvidia/arch/nvalloc/unix/src/escape.c
View File

@@ -52,6 +52,8 @@
#include <ctrl/ctrl00fd.h>
#include <ctrl/ctrl00e0.h>
#define NV_CTL_DEVICE_ONLY(nv) \
{ \
if (((nv)->flags & NV_FLAG_CONTROL) == 0) \
@@ -86,6 +88,9 @@ static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd,
case NV00FD_CTRL_CMD_ATTACH_GPU:
paramSize = sizeof(NV00FD_CTRL_ATTACH_GPU_PARAMS);
break;
case NV00E0_CTRL_CMD_EXPORT_MEM:
paramSize = sizeof(NV00E0_CTRL_EXPORT_MEM_PARAMS);
break;
default:
return NV_OK;
}
@@ -106,6 +111,15 @@ static NV_STATUS RmGetDeviceFd(NVOS54_PARAMETERS *pApi, NvS32 *pFd,
*pFd = (NvS32)pAttachGpuParams->devDescriptor;
}
break;
case NV00E0_CTRL_CMD_EXPORT_MEM:
{
NV00E0_CTRL_EXPORT_MEM_PARAMS *pExportMemParams = pKernelParams;
// If hParent is client, no need to reference device.
*pSkipDeviceRef = (pExportMemParams->hParent == pApi->hClient);
*pFd = (NvS32)pExportMemParams->devDescriptor;
}
break;
default:
NV_ASSERT(0);
break;

86
src/nvidia/arch/nvalloc/unix/src/os-hypervisor.c
View File

@@ -46,6 +46,8 @@
#include <nv_ref.h> // NV_PMC_BOOT_1_VGPU
#include "nvdevid.h"
#include "g_vgpu_chip_flags.h" // vGPU device names
#define NV_VFIO_PCI_BAR0_REGION_INDEX 0
#define NV_VFIO_PCI_BAR1_REGION_INDEX 1
#define NV_VFIO_PCI_BAR2_REGION_INDEX 2
@@ -63,16 +65,6 @@ NvBool hypervisorIsVgxHyper_IMPL(void)
return os_is_vgx_hyper();
}
NvBool hypervisorIsAC_IMPL(void)
{
return NV_FALSE;
}
void hypervisorSetACSupported_IMPL(OBJHYPERVISOR *pHypervisor)
{
pHypervisor->bIsACSupported = NV_TRUE;
}
NV_STATUS hypervisorInjectInterrupt_IMPL
(
OBJHYPERVISOR *pHypervisor,
@@ -208,7 +200,7 @@ static NV_STATUS get_available_instances(
if (!(pKernelVgpuMgr->pgpuInfo[pgpuIndex].createdVfMask & NVBIT64(fnId)))
{
if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
if (kvgpumgrCheckVgpuTypeCreatable(pGpu, &pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
*avail_instances = 1;
}
}
@@ -216,7 +208,7 @@ static NV_STATUS get_available_instances(
}
else
{
if (kvgpumgrCheckVgpuTypeCreatable(&pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
if (kvgpumgrCheckVgpuTypeCreatable(pGpu, &pKernelVgpuMgr->pgpuInfo[pgpuIndex], vgpuTypeInfo) == NV_OK)
*avail_instances = vgpuTypeInfo->maxInstance - pKernelVgpuMgr->pgpuInfo[pgpuIndex].numCreatedVgpu;
}
@@ -620,6 +612,16 @@ NV_STATUS NV_API_CALL nv_vgpu_get_bar_info(
if (status == NV_OK && value)
bOverrideBar1Size = NV_TRUE;
}
if (gpuIsVfResizableBAR1Supported(pGpu))
{
if ((*size > pGpu->sriovState.vfBarSize[1]) ||
(!portStringCompare("Compute", (const char *)vgpuTypeInfo->vgpuClass, 7)))
{
*size = pGpu->sriovState.vfBarSize[1];
}
}
if (bOverrideBar1Size) {
NvU64 bar1SizeInBytes, guestBar1;
NvU64 gpuBar1LowerLimit = 256 * 1024 * 1024; // bar1 lower limit for override_bar1_length parameter
@@ -856,8 +858,17 @@ NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap(
numAreas, NULL, NULL);
if (rmStatus == NV_OK)
{
os_alloc_mem((void **)&vfRegionOffsets, sizeof(NvU64) * (*numAreas));
os_alloc_mem((void **)&vfRegionSizes, sizeof (NvU64) * (*numAreas));
rmStatus = os_alloc_mem((void **)&vfRegionOffsets, sizeof(NvU64) * (*numAreas));
if (rmStatus != NV_OK)
goto cleanup;
rmStatus = os_alloc_mem((void **)&vfRegionSizes, sizeof (NvU64) * (*numAreas));
if (rmStatus != NV_OK)
{
os_free_mem(vfRegionOffsets);
goto cleanup;
}
if (vfRegionOffsets && vfRegionSizes)
{
rmStatus = kbifGetVFSparseMmapRegions_HAL(pGpu, pKernelBif, pKernelHostVgpuDevice, os_page_size,
@@ -867,6 +878,11 @@ NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap(
*offsets = vfRegionOffsets;
*sizes = vfRegionSizes;
}
else
{
os_free_mem(vfRegionOffsets);
os_free_mem(vfRegionSizes);
}
}
else
{
@@ -916,8 +932,16 @@ NV_STATUS NV_API_CALL nv_vgpu_get_sparse_mmap(
{
NvU32 i = 0;
NvU64 *tmpOffset, *tmpSize;
os_alloc_mem((void **)offsets, sizeof(NvU64) * (*numAreas));
os_alloc_mem((void **)sizes, sizeof (NvU64) * (*numAreas));
rmStatus = os_alloc_mem((void **)offsets, sizeof(NvU64) * (*numAreas));
if (rmStatus != NV_OK)
goto cleanup;
rmStatus = os_alloc_mem((void **)sizes, sizeof (NvU64) * (*numAreas));
if (rmStatus != NV_OK)
{
os_free_mem(*offsets);
goto cleanup;
}
tmpOffset = *offsets;
tmpSize = *sizes;
@@ -1141,7 +1165,6 @@ NV_STATUS osIsVfioPciCorePresent(void)
return os_call_vgpu_vfio((void *) &vgpu_info, CMD_VFIO_PCI_CORE_PRESENT);
}
void initVGXSpecificRegistry(OBJGPU *pGpu)
{
NvU32 data32;
@@ -1165,12 +1188,14 @@ void initVGXSpecificRegistry(OBJGPU *pGpu)
NV_STATUS rm_is_vgpu_supported_device(
OS_GPU_INFO *pOsGpuInfo,
NvU32 pmc_boot_1
NvU32 pmc_boot_1,
NvU32 pmc_boot_42
)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
NvBool is_sriov_enabled = FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1);
NvU32 i;
// if not running in vGPU mode (guest VM) return NV_OK
if (!(pHypervisor && pHypervisor->bIsHVMGuest &&
@@ -1180,5 +1205,28 @@ NV_STATUS rm_is_vgpu_supported_device(
return NV_OK;
}
return NV_OK;
if (!is_sriov_enabled)
{
return NV_OK;
}
if (os_is_grid_supported() && !gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42))
{
return NV_ERR_NOT_SUPPORTED;
}
if (os_is_grid_supported())
{
for (i = 0; i < NV_ARRAY_ELEMENTS(sVgpuUsmTypes); i++)
{
if (pOsGpuInfo->pci_info.device_id == sVgpuUsmTypes[i].ulDevID &&
pOsGpuInfo->subsystem_vendor == sVgpuUsmTypes[i].ulSubSystemVendorID &&
pOsGpuInfo->subsystem_id == sVgpuUsmTypes[i].ulSubID)
{
return NV_OK;
}
}
}
return NV_ERR_NOT_SUPPORTED;
}

189
src/nvidia/arch/nvalloc/unix/src/os.c
View File

@@ -34,6 +34,7 @@
#include <gpu/device/device.h>
#include "gpu/gpu.h"
#include <gpu_mgr/gpu_mgr.h>
#include <osfuncs.h>
#include <platform/chipset/chipset.h>
@@ -337,65 +338,6 @@ void osUnmapKernelSpace(
os_unmap_kernel_space((void *)ptr, Size);
}
void* osMapIOSpace(
RmPhysAddr Start,
NvU64 Size,
void ** pData,
NvU32 User,
NvU32 Mode,
NvU32 Protect
)
{
NvU64 offset;
NvU8 *addr;
if (0 == Size)
{
NV_ASSERT(Size != 0);
return NULL;
}
offset = (Start & ~os_page_mask);
Start &= os_page_mask;
Size = ((Size + offset + ~os_page_mask) & os_page_mask);
if (User)
addr = os_map_user_space(Start, Size, Mode, Protect, pData);
else
addr = os_map_kernel_space(Start, Size, Mode);
if (addr != NULL)
return (addr + offset);
return addr;
}
void osUnmapIOSpace(
void *pAddress,
NvU64 Size,
void *pData,
NvU32 User
)
{
NvU64 offset;
NvUPtr addr = (NvUPtr)pAddress;
if (0 == Size)
{
NV_ASSERT(Size != 0);
return;
}
offset = (addr & ~os_page_mask);
addr &= os_page_mask;
Size = ((Size + offset + ~os_page_mask) & os_page_mask);
if (User)
os_unmap_user_space((void *)addr, Size, pData);
else
os_unmap_kernel_space((void *)addr, Size);
}
static NV_STATUS setNumaPrivData
(
KernelMemorySystem *pKernelMemorySystem,
@@ -406,36 +348,33 @@ static NV_STATUS setNumaPrivData
NV_STATUS rmStatus = NV_OK;
void *pAllocPrivate = NULL;
NvU64 *addrArray = NULL;
NvU64 numOsPages = pMemDesc->PageCount;
NvU64 numPages = pMemDesc->PageCount;
NvU64 i;
addrArray = pMemDesc->_pteArray;
addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64));
if (addrArray == NULL)
{
return NV_ERR_NO_MEMORY;
}
portMemCopy((void*)addrArray,
(numPages * sizeof(NvU64)),
(void*)memdescGetPteArray(pMemDesc, AT_CPU),
(numPages * sizeof(NvU64)));
if (NV_RM_PAGE_SIZE < os_page_size)
{
NvU64 numPages;
NvU64 i;
numPages = pMemDesc->PageCount;
addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64));
if (addrArray == NULL)
{
return NV_ERR_NO_MEMORY;
}
portMemCopy((void*)addrArray,
(numPages * sizeof(NvU64)), (void*)pMemDesc->_pteArray,
(numPages * sizeof(NvU64)));
RmDeflateRmToOsPageArray(addrArray, numPages);
numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
for (i = 0; i < numOsPages; i++)
{
// Update GPA to system physical address
addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
}
numPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
}
rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate);
for (i = 0; i < numPages; i++)
{
// Update GPA to system physical address
addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
}
rmStatus = nv_register_phys_pages(nv, addrArray, numPages, NV_MEMORY_CACHED, &pAllocPrivate);
if (rmStatus != NV_OK)
{
goto errors;
@@ -444,10 +383,7 @@ static NV_STATUS setNumaPrivData
memdescSetMemData(pMemDesc, pAllocPrivate, NULL);
errors:
if (NV_RM_PAGE_SIZE < os_page_size)
{
portMemFree(addrArray);
}
portMemFree(addrArray);
return rmStatus;
}
@@ -851,32 +787,6 @@ static inline nv_dma_device_t* osGetDmaDeviceForMemDesc(
pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev;
}
NV_STATUS osDmaMapPages(
OS_GPU_INFO *pOsGpuInfo,
MEMORY_DESCRIPTOR *pMemDesc
)
{
return nv_dma_map_pages(
osGetDmaDeviceForMemDesc(pOsGpuInfo, pMemDesc),
NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
memdescGetPteArray(pMemDesc, AT_CPU),
memdescGetContiguity(pMemDesc, AT_CPU),
memdescGetCpuCacheAttrib(pMemDesc),
NULL);
}
NV_STATUS osDmaUnmapPages(
OS_GPU_INFO *pOsGpuInfo,
MEMORY_DESCRIPTOR *pMemDesc
)
{
return nv_dma_unmap_pages(
pOsGpuInfo->dma_dev,
NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
memdescGetPteArray(pMemDesc, AT_CPU),
NULL);
}
//
// Set the DMA address size for the given GPU
//
@@ -1061,14 +971,21 @@ NV_STATUS osMapPciMemoryUser(
)
{
void *addr;
void *priv = NULL;
NvU64 offset = 0;
addr = osMapIOSpace(busAddress, length, &priv, NV_TRUE, modeFlag, Protect);
NV_ASSERT_OR_RETURN(length != 0, NV_ERR_INVALID_ARGUMENT);
*pPriv = NV_PTR_TO_NvP64(priv);
*pVirtualAddress = NV_PTR_TO_NvP64(addr);
offset = busAddress & (os_page_size - 1llu);
busAddress = NV_ALIGN_DOWN64(busAddress, os_page_size);
length = NV_ALIGN_UP64(busAddress + offset + length, os_page_size) - busAddress;
return (addr != NULL) ? NV_OK : NV_ERR_GENERIC;
addr = os_map_user_space(busAddress, length, modeFlag, Protect, (void **) pPriv);
NV_ASSERT_OR_RETURN(addr != NULL, NV_ERR_INVALID_ADDRESS);
*pVirtualAddress = (NvP64)(((NvU64) addr) + offset);
return NV_OK;
}
void osUnmapPciMemoryUser(
@@ -1078,12 +995,17 @@ void osUnmapPciMemoryUser(
NvP64 pPriv
)
{
void *addr, *priv;
NvU64 addr;
void *priv;
addr = NvP64_VALUE(virtualAddress);
priv = NvP64_VALUE(pPriv);
addr = (NvU64)(virtualAddress);
priv = (void*)(pPriv);
osUnmapIOSpace(addr, length, priv, NV_TRUE);
length = NV_ALIGN_UP64(addr + length, os_page_size);
addr = NV_ALIGN_DOWN64(addr, os_page_size);
length -= addr;
os_unmap_user_space((void *)addr, length, priv);
}
NV_STATUS osMapPciMemoryKernelOld
@@ -3819,7 +3741,7 @@ osGetAtsTargetAddressRange
}
else
{
NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL,
NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL, NULL,
pAddrWidth, NULL);
if (status == NV_OK)
{
@@ -3856,6 +3778,7 @@ osGetFbNumaInfo
(
OBJGPU *pGpu,
NvU64 *pAddrPhys,
NvU64 *pAddrRsvdPhys,
NvS32 *pNodeId
)
{
@@ -3871,7 +3794,8 @@ osGetFbNumaInfo
nv = NV_GET_NV_STATE(pGpu);
NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys, NULL, pNodeId);
NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys,
pAddrRsvdPhys, NULL, pNodeId);
return status;
#endif
@@ -4146,7 +4070,7 @@ osNumaOnliningEnabled
// Note that this numaNodeId value fetched from Linux layer might not be
// accurate since it is possible to overwrite it with regkey on some configs
//
if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL,
if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL, NULL,
&numaNodeId) != NV_OK)
{
return NV_FALSE;
@@ -4724,7 +4648,8 @@ osRmCapRelease
#define OS_RM_CAP_SYS_MIG_DIR 0
#define OS_RM_CAP_SYS_SMC_CONFIG_FILE 1
#define OS_RM_CAP_SYS_SMC_MONITOR_FILE 2
#define OS_RM_CAP_SYS_COUNT 3
#define OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE 3
#define OS_RM_CAP_SYS_COUNT 4
NV_STATUS
osRmCapRegisterSys
@@ -4776,6 +4701,15 @@ osRmCapRegisterSys
}
ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap;
cap = os_nv_cap_create_file_entry(nvidia_caps_root, "fabric-imex-mgmt", OS_RUSR);
if (cap == NULL)
{
NV_PRINTF(LEVEL_ERROR, "Failed to create imex file\n");
status = NV_ERR_OPERATING_SYSTEM;
goto failed;
}
ppCaps[OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE] = cap;
return NV_OK;
failed:
@@ -4844,6 +4778,11 @@ osRmCapAcquire
index = OS_RM_CAP_SYS_SMC_MONITOR_FILE;
break;
}
case NV_RM_CAP_SYS_FABRIC_IMEX_MGMT:
{
index = OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE;
break;
}
default:
{
return NV_ERR_INVALID_ARGUMENT;

310
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -737,7 +737,10 @@ static NV_STATUS RmAccessRegistry(
RM_LOCK_MODULES_GPU,
&gpuMask);
if (RmStatus != NV_OK)
return RmStatus;
{
gpuMask = 0;
goto done;
}
GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
pGpu = GPU_RES_GET_GPU(pSubdevice);
@@ -750,7 +753,10 @@ static NV_STATUS RmAccessRegistry(
RM_LOCK_MODULES_GPU,
&gpuMask);
if (RmStatus != NV_OK)
return RmStatus;
{
gpuMask = 0;
goto done;
}
GPU_RES_SET_THREAD_BC_STATE(pDevice);
pGpu = GPU_RES_GET_GPU(pDevice);
@@ -1442,6 +1448,117 @@ RmDmabufPutClientAndDevice(
NV_ASSERT_OK(kmigmgrDecRefCount(pKernelMIGGpuInstance->pShare));
}
static NV_STATUS RmP2PDmaMapPagesCoherent(
nv_dma_device_t *peer,
NvU64 pageSize,
NvU32 pageCount,
NvU64 *pDmaAddresses,
void **ppPriv
)
{
NV_STATUS status;
NvU64 *pOsDmaAddresses = NULL;
NvU32 osPagesPerP2PPage, osPageCount, count;
NvBool bDmaMapped = NV_FALSE;
NvU32 i, j, index;
NV_ASSERT_OR_RETURN((pageSize >= os_page_size), NV_ERR_INVALID_ARGUMENT);
if (pageSize == os_page_size)
{
return nv_dma_map_alloc(peer, pageCount, pDmaAddresses, NV_FALSE, ppPriv);
}
//
// If P2P page_size returned from nvidia_p2p_get_pages is bigger than
// OS page size, the page array need to be inflated to correctly do the DMA mapping.
// For example, if page_size = 64K and OS page size = 4K,
// there are 16 OS pages to this 64K P2P page.
//
osPagesPerP2PPage = pageSize / os_page_size;
osPageCount = osPagesPerP2PPage * pageCount;
status = os_alloc_mem((void **)&pOsDmaAddresses,
(osPageCount * sizeof(NvU64)));
if (status != NV_OK)
{
goto failed;
}
index = 0;
for (i = 0; i < pageCount; i++)
{
pOsDmaAddresses[index] = pDmaAddresses[i];
index++;
for (j = 1; j < osPagesPerP2PPage; j++)
{
pOsDmaAddresses[index] = pOsDmaAddresses[index - 1] + os_page_size;
index++;
}
}
status = nv_dma_map_alloc(peer, osPageCount, pOsDmaAddresses, NV_FALSE, ppPriv);
if (status != NV_OK)
{
goto failed;
}
bDmaMapped = NV_TRUE;
//
// The driver breaks down the size into submaps and dma-maps those individually.
// This may result in non-contiguous IOVA segments allocated to each submap.
// We check for OS page contiguity within the bigger pageSize since we need to
// return IOVA chunks at pageSize granularity.
//
count = 0;
for (i = 1; i < osPageCount; i++)
{
count++;
if (count == osPagesPerP2PPage)
{
count = 0;
continue;
}
if (pOsDmaAddresses[i] != (pOsDmaAddresses[i - 1] + os_page_size))
{
status = NV_ERR_INVALID_STATE;
goto failed;
}
}
//
// Fill pDmaAddresses at pageSize granularity
// from expanded pOsDmaAddresses.
//
index = 0;
for (i = 0; i < pageCount; i++)
{
pDmaAddresses[i] = pOsDmaAddresses[index];
index += osPagesPerP2PPage;
}
os_free_mem(pOsDmaAddresses);
return NV_OK;
failed:
if (bDmaMapped)
{
NV_ASSERT_OK(nv_dma_unmap_alloc(peer, osPageCount,
pOsDmaAddresses, ppPriv));
}
if (pOsDmaAddresses != NULL)
{
os_free_mem(pOsDmaAddresses);
}
return status;
}
static void
RmHandleNvpcfEvents(
nv_state_t *pNv
@@ -1913,13 +2030,13 @@ static NV_STATUS RmSetUserMapAccessRange(
return status;
}
static NV_STATUS RmGetAllocPrivate(NvU32, NvU32, NvU64, NvU64, NvU32 *, void **,
static NV_STATUS RmGetAllocPrivate(RmClient *, NvU32, NvU64, NvU64, NvU32 *, void **,
NvU64 *);
static NV_STATUS RmValidateMmapRequest(nv_state_t *, NvU64, NvU64, NvU32 *);
static NV_STATUS RmGetMmapPteArray(
KernelMemorySystem *pKernelMemorySystem,
NvHandle hClient,
RsClient *pClient,
NvHandle hMemory,
nv_usermap_access_params_t *nvuap
)
@@ -1942,12 +2059,14 @@ static NV_STATUS RmGetMmapPteArray(
// If we're mapping a memory handle, we can get the data from the
// descriptor (required if the allocation being mapped is discontiguous).
//
if (serverutilGetResourceRefWithType(hClient, hMemory, classId(Memory),
&pResourceRef) == NV_OK)
if (clientGetResourceRef(pClient, hMemory, &pResourceRef) == NV_OK)
{
pMemory = dynamicCast(pResourceRef->pResource, Memory);
pMemDesc = pMemory->pMemDesc;
nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU);
if (pMemory != NULL)
{
pMemDesc = pMemory->pMemDesc;
nvuap->contig = memdescGetContiguity(pMemDesc, AT_CPU);
}
}
//
@@ -2018,7 +2137,7 @@ static NV_STATUS RmGetMmapPteArray(
/* Must be called with the API lock and the GPU locks */
static NV_STATUS RmCreateMmapContextLocked(
NvHandle hClient,
RmClient *pRmClient,
NvHandle hDevice,
NvHandle hMemory,
NvP64 address,
@@ -2031,6 +2150,7 @@ static NV_STATUS RmCreateMmapContextLocked(
NV_STATUS status = NV_OK;
void *pAllocPriv = NULL;
OBJGPU *pGpu = NULL;
RsClient *pClient = staticCast(pRmClient, RsClient);
KernelMemorySystem *pKernelMemorySystem = NULL;
NvBool bCoherentAtsCpuOffset = NV_FALSE;
nv_state_t *pNv = NULL;
@@ -2038,19 +2158,14 @@ static NV_STATUS RmCreateMmapContextLocked(
NvU32 prot = 0;
NvU64 pageIndex = 0;
nv_usermap_access_params_t *nvuap = NULL;
NvBool bClientMap = (hClient == hDevice);
NvBool bClientMap = (pClient->hClient == hDevice);
if (!bClientMap)
{
if (CliSetGpuContext(hClient, hDevice, &pGpu, NULL) != NV_OK)
if (CliSetGpuContext(pClient->hClient, hDevice, &pGpu, NULL) != NV_OK)
{
RsClient *pClient;
Subdevice *pSubdevice;
status = serverGetClientUnderLock(&g_resServ, hClient, &pClient);
if (status != NV_OK)
return status;
status = subdeviceGetByHandle(pClient, hDevice, &pSubdevice);
if (status != NV_OK)
return status;
@@ -2104,7 +2219,7 @@ static NV_STATUS RmCreateMmapContextLocked(
// Validate the mapping request by looking up the underlying sysmem
// allocation.
//
status = RmGetAllocPrivate(hClient, hMemory, addr, size, &prot, &pAllocPriv,
status = RmGetAllocPrivate(pRmClient, hMemory, addr, size, &prot, &pAllocPriv,
&pageIndex);
if (status != NV_OK)
@@ -2119,7 +2234,7 @@ static NV_STATUS RmCreateMmapContextLocked(
//
if (bCoherentAtsCpuOffset)
{
status = RmGetMmapPteArray(pKernelMemorySystem, hClient, hMemory, nvuap);
status = RmGetMmapPteArray(pKernelMemorySystem, pClient, hMemory, nvuap);
if (status != NV_OK)
{
goto done;
@@ -2170,25 +2285,30 @@ NV_STATUS rm_create_mmap_context(
// LOCK: acquire API lock
if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK)
{
RmClient *pClient;
RmClient *pRmClient;
if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient))
if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pRmClient))
{
// UNLOCK: release API lock
rmapiLockRelease();
return NV_ERR_INVALID_CLIENT;
}
if (pClient->ProcID != osGetCurrentProcess())
if (pRmClient->ProcID != osGetCurrentProcess())
{
rmStatus = NV_ERR_INVALID_CLIENT;
}
// LOCK: acquire GPUs lock
else if ((rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI)) == NV_OK)
{
rmStatus = RmCreateMmapContextLocked(hClient, hDevice, hMemory,
address, size, offset, cachingType, fd);
rmStatus = RmCreateMmapContextLocked(pRmClient, hDevice,
hMemory, address, size, offset,
cachingType, fd);
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
}
serverutilReleaseClient(LOCK_ACCESS_READ, pClient);
serverutilReleaseClient(LOCK_ACCESS_READ, pRmClient);
// UNLOCK: release API lock
rmapiLockRelease();
@@ -2198,7 +2318,7 @@ NV_STATUS rm_create_mmap_context(
}
static NV_STATUS RmGetAllocPrivate(
NvU32 hClient,
RmClient *pRmClient,
NvU32 hMemory,
NvU64 offset,
NvU64 length,
@@ -2207,7 +2327,6 @@ static NV_STATUS RmGetAllocPrivate(
NvU64 *pPageIndex
)
{
RmClient *pClient;
NV_STATUS rmStatus;
PMEMORY_DESCRIPTOR pMemDesc;
NvU32 pageOffset;
@@ -2226,10 +2345,8 @@ static NV_STATUS RmGetAllocPrivate(
NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE);
if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClient))
return NV_ERR_INVALID_CLIENT;
rmStatus = clientGetResourceRef(staticCast(pClient, RsClient), hMemory, &pResourceRef);
rmStatus = clientGetResourceRef(staticCast(pRmClient, RsClient),
hMemory, &pResourceRef);
if (rmStatus != NV_OK)
goto done;
@@ -2301,8 +2418,6 @@ static NV_STATUS RmGetAllocPrivate(
*ppPrivate = pMemData;
done:
serverutilReleaseClient(LOCK_ACCESS_READ, pClient);
return rmStatus;
}
@@ -2358,6 +2473,26 @@ NV_STATUS rm_get_adapter_status(
return rmStatus;
}
NV_STATUS NV_API_CALL rm_get_adapter_status_external(
nvidia_stack_t *sp,
nv_state_t *pNv
)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
if (rm_get_adapter_status(pNv, &rmStatus) != NV_OK)
{
rmStatus = NV_ERR_OPERATING_SYSTEM;
}
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NvBool NV_API_CALL rm_init_rm(
nvidia_stack_t *sp
)
@@ -3388,27 +3523,55 @@ NV_STATUS NV_API_CALL rm_is_supported_device(
NULL);
if (!bIsFirmwareCapable)
{
nv_printf(NV_DBG_ERRORS,
"NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
"NVRM: installed in this system is not supported by open\n"
"NVRM: nvidia.ko because it does not include the required GPU\n"
"NVRM: System Processor (GSP).\n"
"NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
"NVRM: Firmware' sections in the driver README, available on\n"
"NVRM: the Linux graphics driver download page at\n"
"NVRM: www.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
if (hypervisorIsVgxHyper())
{
nv_printf(NV_DBG_ERRORS,
"NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
"NVRM: installed in this vGPU host system is not supported by\n"
"NVRM: open nvidia.ko.\n"
"NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
"NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n"
"NVRM: Software documentation, available at docs.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
}
else
{
nv_printf(NV_DBG_ERRORS,
"NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
"NVRM: installed in this system is not supported by open\n"
"NVRM: nvidia.ko because it does not include the required GPU\n"
"NVRM: System Processor (GSP).\n"
"NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
"NVRM: Firmware' sections in the driver README, available on\n"
"NVRM: the Linux graphics driver download page at\n"
"NVRM: www.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
}
goto threadfree;
}
goto print_unsupported;
}
rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1);
rmStatus = rm_is_vgpu_supported_device(pNv, pmc_boot_1, pmc_boot_42);
if (rmStatus != NV_OK)
goto print_unsupported;
{
nv_printf(NV_DBG_ERRORS,
"NVRM: The NVIDIA vGPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
"NVRM: installed in this system is not supported by open\n"
"NVRM: nvidia.ko.\n"
"NVRM: Please see the 'Open Linux Kernel Modules' and 'GSP\n"
"NVRM: Firmware' sections in the NVIDIA Virtual GPU (vGPU)\n"
"NVRM: Software documentation, available at docs.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
goto threadfree;
}
goto threadfree;
print_unsupported:
@@ -3855,8 +4018,12 @@ void RmI2cAddGpuPorts(nv_state_t * pNv)
RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
NvU32 displayMask;
NV_STATUS status;
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
NV0073_CTRL_SYSTEM_GET_SUPPORTED_PARAMS systemGetSupportedParams = { 0 };
if (IS_VIRTUAL(pGpu))
return;
// Make displayId as Invalid.
for (x = 0; x < MAX_I2C_ADAPTERS; x++)
{
@@ -4233,10 +4400,8 @@ NV_STATUS NV_API_CALL rm_p2p_dma_map_pages(
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING))
{
NV_ASSERT(pageSize == os_page_size);
rmStatus = nv_dma_map_alloc(peer, pageCount, pDmaAddresses,
NV_FALSE, ppPriv);
rmStatus = RmP2PDmaMapPagesCoherent(peer, pageSize, pageCount,
pDmaAddresses, ppPriv);
}
else
{
@@ -4611,6 +4776,47 @@ static void rm_set_firmware_logs(
NV_EXIT_RM_RUNTIME(sp,fp);
}
static NvBool rm_get_is_gsp_capable_vgpu(
nvidia_stack_t *sp,
nv_state_t *nv
)
{
void *fp;
THREAD_STATE_NODE threadState;
NvBool isVgpu = NV_FALSE;
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
if ((pHypervisor != NULL) && pHypervisor->bIsHVMGuest)
{
GPUHWREG *reg_mapping;
reg_mapping = osMapKernelSpace(nv->regs->cpu_address,
os_page_size,
NV_MEMORY_UNCACHED,
NV_PROTECT_READABLE);
if (reg_mapping != NULL)
{
NvU32 pmc_boot_1 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_1);
NvU32 pmc_boot_42 = NV_PRIV_REG_RD32(reg_mapping, NV_PMC_BOOT_42);
osUnmapKernelSpace(reg_mapping, os_page_size);
if (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmc_boot_1) &&
gpumgrIsVgxRmFirmwareCapableChip(pmc_boot_42))
{
isVgpu = NV_TRUE;
}
}
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
return isVgpu;
}
void NV_API_CALL rm_set_rm_firmware_requested(
nvidia_stack_t *sp,
nv_state_t *nv
@@ -4618,6 +4824,10 @@ void NV_API_CALL rm_set_rm_firmware_requested(
{
nv->request_firmware = NV_TRUE;
nv->allow_fallback_to_monolithic_rm = NV_FALSE;
if (rm_get_is_gsp_capable_vgpu(sp, nv))
{
nv->request_firmware = NV_FALSE;
}
// Check if we want firmware logs
if (nv->request_firmware)

243
src/nvidia/arch/nvalloc/unix/src/osinit.c
View File

@@ -103,6 +103,7 @@ typedef enum
RM_INIT_GPU_LOAD_FAILED,
RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED,
RM_INIT_GPU_DMA_CONFIGURATION_FAILED,
RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED,
/* vbios errors */
RM_INIT_VBIOS_FAILED = 0x30,
@@ -190,7 +191,6 @@ static void initUnixSpecificRegistry(OBJGPU *);
NvBool osRmInitRm(OBJOS *pOS)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
NvU64 system_memory_size = (NvU64)-1;
NV_PRINTF(LEVEL_INFO, "init rm\n");
@@ -274,13 +274,6 @@ NvBool osRmInitRm(OBJOS *pOS)
return NV_FALSE;
}
system_memory_size = NV_RM_PAGES_PER_OS_PAGE * os_get_num_phys_pages();
// if known, relay the number of system memory pages (in terms of RM page
// size) to the RM; this is needed for e.g. TurboCache parts.
if (system_memory_size != (NvU64)-1)
pOS->SystemMemorySize = system_memory_size;
// Setup any ThreadState defaults
threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED |
THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED |
@@ -420,6 +413,126 @@ osHandleGpuLost
return NV_OK;
}
/*!
* @brief Traverse bus topology till Gpu's root port.
* If any of the intermediate bridge has TB3 supported vendorId and hotplug
* capability(not necessarily same bridge), mark the Gpu as External Gpu.
*
* @params[in] pGpu OBJGPU pointer
* @params[in] pCl OBJCL pointer
*
* @return NV_OK
* Identified to be eGPU
* @return others
* Not an eGPU / error on identfying
*
*/
NvBool
RmCheckForExternalGpu
(
OBJGPU *pGpu,
OBJCL *pCl
)
{
NvU8 bus;
NvU32 domain;
void *handleUp;
NvU8 busUp, devUp, funcUp;
NvU16 vendorIdUp, deviceIdUp;
NvU32 portCaps, pciCaps, slotCaps;
NvU32 PCIECapPtr;
RM_API *pRmApi;
NV_STATUS status, rmStatus;
NvBool bTb3Bridge = NV_FALSE, bSlotHotPlugSupport = NV_FALSE;
NvBool iseGPUBridge = NV_FALSE;
pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
domain = gpuGetDomain(pGpu);
bus = gpuGetBus(pGpu);
do
{
// Find the upstream bridge
handleUp = clFindP2PBrdg(pCl, domain, bus, &busUp, &devUp, &funcUp, &vendorIdUp, &deviceIdUp);
if (!handleUp)
{
return iseGPUBridge;
}
if (vendorIdUp == PCI_VENDOR_ID_INTEL)
{
// Check for the supported TB3(ThunderBolt 3) bridges.
NV2080_CTRL_INTERNAL_GET_EGPU_BRIDGE_INFO_PARAMS params = { 0 };
// LOCK: acquire GPUs lock
rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
RM_LOCK_MODULES_INIT);
if (rmStatus != NV_OK)
{
return iseGPUBridge;
}
params.pciDeviceId = deviceIdUp;
status = pRmApi->Control(pRmApi,
pGpu->hInternalClient,
pGpu->hInternalSubdevice,
NV2080_CTRL_CMD_INTERNAL_GET_EGPU_BRIDGE_INFO,
&params,
sizeof(params));
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"Error 0x%08x on eGPU Approval for Bridge ID: 0x%08x\n", status, deviceIdUp);
DBG_BREAKPOINT();
return iseGPUBridge;
}
else
{
// Check for the approved eGPU BUS TB3
if (params.iseGPUBridge &&
params.approvedBusType == NV2080_CTRL_INTERNAL_EGPU_BUS_TYPE_TB3)
{
bTb3Bridge = NV_TRUE;
}
}
}
if (NV_OK != clSetPortPcieCapOffset(pCl, handleUp, &PCIECapPtr))
{
// PCIE bridge but no cap pointer.
break;
}
// Get the PCIE capabilities.
pciCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr);
if (CL_PCIE_CAP_SLOT & pciCaps)
{
// Get the slot capabilities.
slotCaps = osPciReadDword(handleUp, CL_PCIE_SLOT_CAP - CL_PCIE_BEGIN + PCIECapPtr);
if ((CL_PCIE_SLOT_CAP_HOTPLUG_CAPABLE & slotCaps) &&
(CL_PCIE_SLOT_CAP_HOTPLUG_SURPRISE & slotCaps))
{
bSlotHotPlugSupport = NV_TRUE;
}
}
if (bTb3Bridge && bSlotHotPlugSupport)
{
iseGPUBridge = NV_TRUE;
break;
}
bus = busUp;
// Get port caps to check if PCIE bridge is the root port
portCaps = osPciReadDword(handleUp, CL_PCIE_CAP - CL_PCIE_BEGIN + PCIECapPtr);
} while (!CL_IS_ROOT_PORT(portCaps));
return iseGPUBridge;
}
/*
* Initialize the required GPU information by doing RMAPI control calls
* and store the same in the UNIX specific data structures.
@@ -532,6 +645,8 @@ osInitNvMapping(
nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
NvU32 deviceInstance;
NvU32 data = 0;
NvU32 dispIsoStreamId;
NvU32 dispNisoStreamId;
NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n");
@@ -550,8 +665,6 @@ osInitNvMapping(
{
NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n");
RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED);
// RM_BASIC_LOCK_MODEL: free GPU lock
rmGpuLockFree(*pDeviceReference);
return;
}
@@ -610,6 +723,7 @@ osInitNvMapping(
gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size;
gpuAttachArg->iovaspaceId = nv->iovaspace_id;
gpuAttachArg->cpuNumaNodeId = nv->cpu_numa_node_id;
}
//
@@ -703,6 +817,17 @@ osInitNvMapping(
nv->preserve_vidmem_allocations = NV_TRUE;
}
// Check if SMMU can be enabled on PushBuffer Aperture
nv_get_disp_smmu_stream_ids(nv, &dispIsoStreamId, &dispNisoStreamId);
if (dispNisoStreamId != NV_U32_MAX)
{
pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_FALSE);
}
else
{
pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_TRUE);
}
}
void osInitScalabilityOptions
@@ -746,21 +871,6 @@ osTeardownScalability(
return clTeardownPcie(pGpu, pCl);
}
static void
populateDeviceAttributes(
OBJGPU *pGpu,
nv_state_t *nv
)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJCL *pCl = SYS_GET_CL(pSys);
if ((pCl != NULL) && pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU))
{
nv->is_external_gpu = NV_TRUE;
}
}
static void
RmSetConsolePreservationParams(OBJGPU *pGpu)
{
@@ -883,6 +993,8 @@ RmInitNvDevice(
{
// set the device context
OBJGPU *pGpu = gpumgrGetGpu(deviceReference);
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJCL *pCl = SYS_GET_CL(pSys);
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
@@ -900,14 +1012,17 @@ RmInitNvDevice(
return;
}
os_disable_console_access();
// Configure eGPU setting
if (RmCheckForExternalGpu(pGpu, pCl))
{
pCl->setProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU, NV_TRUE);
nv->is_external_gpu = NV_TRUE;
}
status->rmStatus = gpumgrStateInitGpu(pGpu);
if (status->rmStatus != NV_OK)
{
NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n");
RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED);
os_enable_console_access();
return;
}
nvp->flags |= NV_INIT_FLAG_GPU_STATE;
@@ -937,13 +1052,10 @@ RmInitNvDevice(
NV_PRINTF(LEVEL_ERROR,
"*** Cannot load state into the device\n");
RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED);
os_enable_console_access();
return;
}
nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD;
os_enable_console_access();
status->rmStatus = gpuPerformUniversalValidation_HAL(pGpu);
if (status->rmStatus != NV_OK)
{
@@ -1160,6 +1272,7 @@ NvBool RmInitPrivateState(
nv_priv_t *nvp;
NvU32 gpuId;
NvU32 pmc_boot_0 = 0;
NvU32 pmc_boot_1 = 0;
NvU32 pmc_boot_42 = 0;
NV_SET_NV_PRIV(pNv, NULL);
@@ -1177,6 +1290,7 @@ NvBool RmInitPrivateState(
}
pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0);
pmc_boot_1 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_1);
pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42);
os_unmap_kernel_space(pNv->regs->map_u, os_page_size);
@@ -1204,6 +1318,7 @@ NvBool RmInitPrivateState(
pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId :
NV_IOVA_DOMAIN_NONE;
pNv->cpu_numa_node_id = NV0000_CTRL_NO_NUMA_NODE;
kvgpumgrAttachGpu(pNv->gpu_id);
@@ -1216,6 +1331,7 @@ NvBool RmInitPrivateState(
os_mem_set(nvp, 0, sizeof(*nvp));
nvp->status = NV_ERR_INVALID_STATE;
nvp->pmc_boot_0 = pmc_boot_0;
nvp->pmc_boot_1 = pmc_boot_1;
nvp->pmc_boot_42 = pmc_boot_42;
NV_SET_NV_PRIV(pNv, nvp);
@@ -1234,7 +1350,7 @@ void RmClearPrivateState(
nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS];
nv_dynamic_power_t dynamicPowerCopy;
NvU32 x = 0;
NvU32 pmc_boot_0, pmc_boot_42;
NvU32 pmc_boot_0, pmc_boot_1, pmc_boot_42;
//
// Do not clear private state after GPU resets, it is used while
@@ -1252,6 +1368,7 @@ void RmClearPrivateState(
pRegistryCopy = nvp->pRegistry;
dynamicPowerCopy = nvp->dynamic_power;
pmc_boot_0 = nvp->pmc_boot_0;
pmc_boot_1 = nvp->pmc_boot_1;
pmc_boot_42 = nvp->pmc_boot_42;
for (x = 0; x < MAX_I2C_ADAPTERS; x++)
@@ -1267,6 +1384,7 @@ void RmClearPrivateState(
nvp->pRegistry = pRegistryCopy;
nvp->dynamic_power = dynamicPowerCopy;
nvp->pmc_boot_0 = pmc_boot_0;
nvp->pmc_boot_1 = pmc_boot_1;
nvp->pmc_boot_42 = pmc_boot_42;
for (x = 0; x < MAX_I2C_ADAPTERS; x++)
@@ -1537,6 +1655,7 @@ NvBool RmInitAdapter(
KernelDisplay *pKernelDisplay;
const void *gspFwHandle = NULL;
const void *gspFwLogHandle = NULL;
NvBool consoleDisabled = NV_FALSE;
GSP_FIRMWARE gspFw = {0};
PORT_UNREFERENCED_VARIABLE(gspFw);
@@ -1580,6 +1699,17 @@ NvBool RmInitAdapter(
}
}
//
// Initialization path requires expanded GPU visibility in GPUMGR
// in order to access the GPU undergoing initialization.
//
status.rmStatus = gpumgrThreadEnableExpandedGpuVisibility();
if (status.rmStatus != NV_OK)
{
RM_SET_ERROR(status, RM_INIT_GPU_GPUMGR_EXPANDED_VISIBILITY_FAILED);
goto shutdown;
}
// initialize the RM device register mapping
osInitNvMapping(nv, &devicereference, &status);
if (! RM_INIT_SUCCESS(status.initStatus) )
@@ -1616,7 +1746,7 @@ NvBool RmInitAdapter(
KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);
if ((pKernelFsp != NULL) && !IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu))
{
status.rmStatus = kfspSendBootCommands_HAL(pGpu, pKernelFsp);
status.rmStatus = kfspPrepareAndSendBootCommands_HAL(pGpu, pKernelFsp);
if (status.rmStatus != NV_OK)
{
NV_PRINTF(LEVEL_ERROR, "FSP boot command failed.\n");
@@ -1629,6 +1759,16 @@ NvBool RmInitAdapter(
RmInitAcpiMethods(pOS, pSys, pGpu);
//
// For GPU driving console, disable console access here, to ensure no console
// writes through BAR1 can interfere with physical RM's setup of BAR1
//
if (rm_get_uefi_console_status(nv))
{
os_disable_console_access();
consoleDisabled = NV_TRUE;
}
//
// If GSP fw RM support is enabled then start the GSP microcode
// (including the task running the full instance of the RM) and
@@ -1668,10 +1808,8 @@ NvBool RmInitAdapter(
if (IS_PASSTHRU(pGpu))
nv->flags |= NV_FLAG_PASSTHRU;
populateDeviceAttributes(pGpu, nv);
initVendorSpecificRegistry(pGpu, nv->pci_info.device_id);
if (!IS_VIRTUAL(pGpu) && !IS_GSP_CLIENT(pGpu))
if (!IS_VIRTUAL(pGpu))
{
initNbsiTable(pGpu);
}
@@ -1704,6 +1842,18 @@ NvBool RmInitAdapter(
goto shutdown;
}
if (consoleDisabled)
{
os_enable_console_access();
consoleDisabled = NV_FALSE;
}
//
// Expanded GPU visibility in GPUMGR is no longer needed once the
// GPU is initialized.
//
gpumgrThreadDisableExpandedGpuVisibility();
// LOCK: acquire GPUs lock
status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE,
RM_LOCK_MODULES_INIT);
@@ -1867,6 +2017,13 @@ NvBool RmInitAdapter(
shutdown:
nv->flags &= ~NV_FLAG_IN_RECOVERY;
gpumgrThreadDisableExpandedGpuVisibility();
if (consoleDisabled)
{
os_enable_console_access();
}
// call ShutdownAdapter to undo anything we've done above
RmShutdownAdapter(nv);
@@ -1903,6 +2060,14 @@ void RmShutdownAdapter(
// LOCK: acquire GPUs lock
if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
{
//
// Shutdown path requires expanded GPU visibility in GPUMGR in order
// to access the GPU undergoing shutdown which may not be fully
// initialized, and to continue accessing the GPU undergoing shutdown
// after state destroy.
//
NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility());
RmDestroyDeferredDynamicPowerManagement(nv);
freeNbsiTable(pGpu);
@@ -1952,6 +2117,12 @@ void RmShutdownAdapter(
gpumgrDetachGpu(gpuInstance);
gpumgrDestroyDevice(deviceInstance);
//
// Expanded GPU visibility in GPUMGR is no longer needed once the
// GPU is removed from GPUMGR.
//
gpumgrThreadDisableExpandedGpuVisibility();
if (nvp->flags & NV_INIT_FLAG_DMA)
{
RmTeardownDeviceDma(nv);

48
src/nvidia/arch/nvalloc/unix/src/osmemdesc.c
View File

@@ -791,7 +791,6 @@ _createMemdescFromDmaBuf
OBJGPU *pGpu,
NvU32 flags,
nv_dma_buf_t *pImportPriv,
void *pUserPages,
struct sg_table *pImportSgt,
NvU32 size,
MEMORY_DESCRIPTOR **ppMemDesc,
@@ -802,12 +801,9 @@ _createMemdescFromDmaBuf
_createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt,
size, ppMemDesc, ppPrivate,
osDestroyOsDescriptorFromDmaBuf);
NV_ASSERT(pUserPages == NULL);
if (rmStatus != NV_OK)
{
nv_dma_release_dma_buf(NULL, pImportPriv);
nv_dma_release_dma_buf(pImportPriv);
}
return rmStatus;
@@ -838,6 +834,23 @@ _createMemdescFromSgt
return rmStatus;
}
static nv_dma_device_t *GetDmaDeviceForImport
(
nv_state_t *nv,
NvU32 flags
)
{
if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags) &&
(nv->niso_dma_dev != NULL))
{
return nv->niso_dma_dev;
}
else
{
return nv->dma_dev;
}
}
static NV_STATUS
osCreateOsDescriptorFromFileHandle
(
@@ -852,8 +865,8 @@ osCreateOsDescriptorFromFileHandle
{
NV_STATUS rmStatus = NV_OK;
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
nv_dma_device_t *dma_dev = NULL;
NvU32 size = 0;
void *pUserPages = NULL;
nv_dma_buf_t *pImportPriv = NULL;
struct sg_table *pImportSgt = NULL;
NvS32 fd;
@@ -867,8 +880,9 @@ osCreateOsDescriptorFromFileHandle
return NV_ERR_INVALID_ARGUMENT;
}
rmStatus = nv_dma_import_from_fd(nv->dma_dev, fd, &size,
&pUserPages, &pImportSgt, &pImportPriv);
dma_dev = GetDmaDeviceForImport(nv, flags);
rmStatus = nv_dma_import_from_fd(dma_dev, fd, &size,
&pImportSgt, &pImportPriv);
if (rmStatus != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
@@ -878,7 +892,7 @@ osCreateOsDescriptorFromFileHandle
}
return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
pUserPages, pImportSgt,
pImportSgt,
size, ppMemDesc, ppPrivate);
}
@@ -929,24 +943,24 @@ osCreateOsDescriptorFromDmaBufPtr
{
NV_STATUS rmStatus = NV_OK;
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
nv_dma_device_t *dma_dev = NULL;
NvU32 size = 0;
void *pUserPages = NULL;
nv_dma_buf_t *pImportPriv = NULL;
struct sg_table *pImportSgt = NULL;
void *dmaBuf = (void*)((NvUPtr)pDescriptor);
rmStatus = nv_dma_import_dma_buf(nv->dma_dev, dmaBuf, &size,
&pUserPages, &pImportSgt, &pImportPriv);
dma_dev = GetDmaDeviceForImport(nv, flags);
rmStatus = nv_dma_import_dma_buf(dma_dev, dmaBuf, &size,
&pImportSgt, &pImportPriv);
if (rmStatus != NV_OK)
{
NV_PRINTF(LEVEL_ERROR,
"%s(): Error (%d) while trying to import dma_buf!\n",
__FUNCTION__, rmStatus);
NV_PRINTF_COND(rmStatus == NV_ERR_NOT_SUPPORTED, LEVEL_INFO, LEVEL_ERROR,
"Error (%d) while trying to import dma_buf!\n", rmStatus);
return rmStatus;
}
return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
pUserPages, pImportSgt,
pImportSgt,
size, ppMemDesc, ppPrivate);
}
@@ -1061,7 +1075,7 @@ osDestroyOsDescriptorFromDmaBuf
* SGT.
*/
nv_dma_release_dma_buf(NULL, pImportPriv);
nv_dma_release_dma_buf(pImportPriv);
}
static void

11
src/nvidia/arch/nvalloc/unix/src/osnvlink.c
View File

@@ -58,6 +58,7 @@ extern NvlStatus knvlinkCoreReadDiscoveryTokenCallback(struct nvlink_link *, NvU
extern NvlStatus knvlinkCoreWriteDiscoveryTokenCallback(struct nvlink_link *, NvU64);
extern void knvlinkCoreTrainingCompleteCallback(struct nvlink_link *);
extern void knvlinkCoreGetUphyLoadCallback(struct nvlink_link *, NvBool*);
extern NvlStatus knvlinkCoreGetCciLinkModeCallback(struct nvlink_link *, NvU64 *);
/*!
* @brief Helper to allocate an alternate stack from within core RM.
@@ -665,6 +666,15 @@ static NvlStatus NV_API_CALL rm_nvlink_ops_ali_training
return status;
}
static NvlStatus NV_API_CALL rm_nvlink_ops_get_cci_link_mode
(
struct nvlink_link *link,
NvU64 *mode
)
{
return NVL_SUCCESS;
}
#endif /* defined(INCLUDE_NVLINK_LIB) */
const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void)
@@ -691,6 +701,7 @@ const struct nvlink_link_handlers* osGetNvlinkLinkCallbacks(void)
.read_discovery_token = rm_nvlink_ops_read_link_discovery_token,
.training_complete = rm_nvlink_ops_training_complete,
.get_uphy_load = rm_nvlink_get_uphy_load,
.get_cci_link_mode = rm_nvlink_ops_get_cci_link_mode,
.ali_training = rm_nvlink_ops_ali_training,
};

52
src/nvidia/arch/nvalloc/unix/src/rm-gpu-ops.c
View File

@@ -84,14 +84,15 @@ NV_STATUS NV_API_CALL rm_gpu_ops_address_space_create (
gpuDeviceHandle device,
NvU64 vaBase,
NvU64 vaSize,
NvU32 enableAts,
gpuAddressSpaceHandle *vaSpace,
gpuAddressSpaceInfo *vaSpaceInfo)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, vaSpace,
vaSpaceInfo);
rmStatus = nvGpuOpsAddressSpaceCreate(device, vaBase, vaSize, enableAts,
vaSpace, vaSpaceInfo);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
@@ -239,18 +240,6 @@ NV_STATUS NV_API_CALL rm_gpu_ops_pma_pin_pages(
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_pma_unpin_pages(
nvidia_stack_t *sp, void *pPma,
NvU64 *pPages, NvLength pageCount, NvU64 pageSize)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsPmaUnpinPages(pPma, pPages, pageCount, pageSize);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_memory_cpu_map(
nvidia_stack_t *sp, gpuAddressSpaceHandle vaspace,
NvU64 gpuOffset, NvLength length, void **cpuPtr, NvU64 pageSize)
@@ -604,6 +593,18 @@ NV_STATUS NV_API_CALL rm_gpu_ops_flush_replayable_fault_buffer(nvidia_stack_t
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_toggle_prefetch_faults(nvidia_stack_t *sp,
gpuFaultInfo *pFaultInfo,
NvBool bEnable)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsTogglePrefetchFaults(pFaultInfo, bEnable);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *sp,
gpuDeviceHandle device,
gpuAccessCntrInfo *accessCntrInfo,
@@ -869,6 +870,17 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *sp,
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_update(nvidia_stack_t *sp,
struct ccslContext_t *ctx)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsCcslContextUpdate(ctx);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_iv(nvidia_stack_t *sp,
struct ccslContext_t *ctx,
NvU8 direction)
@@ -972,3 +984,15 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_increment_iv(nvidia_stack_t *sp,
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_log_device_encryption(nvidia_stack_t *sp,
struct ccslContext_t *ctx,
NvU32 bufferSize)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsLogDeviceEncryption(ctx, bufferSize);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}

5
src/nvidia/arch/nvalloc/unix/src/rmobjexportimport.c
View File

@@ -48,7 +48,6 @@
#include "rmobjexportimport.h"
#include "nvlimits.h"
#include "gpu/device/device.h"
#include "gpu/subdevice/subdevice.h"
#include "gpu/mig_mgr/kernel_mig_manager.h"
#include "gpu/mig_mgr/gpu_instance_subscription.h"
@@ -331,8 +330,8 @@ NV_STATUS RmExportObject(NvHandle hSrcClient, NvHandle hSrcObject,
NV_STATUS status;
RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
MIG_INSTANCE_REF ref = kmigmgrMakeNoMIGReference();
NvU64 deviceMapIdx;
RmObjExportDevice *pObjExportDevice;
NvU64 deviceMapIdx = 0;
RmObjExportDevice *pObjExportDevice = NULL;
RsResourceRef *pSrcResourceRef;
RsResourceRef *pDeviceRef;

70
src/nvidia/arch/nvalloc/unix/src/unix_console.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,6 +29,7 @@
#include <nv.h>
#include <osapi.h>
#include <gpu/mem_mgr/mem_mgr.h>
#include <platform/sli/sli.h>
#include <vgpu/rpc.h>
#include "vgpu/vgpu_events.h"
@@ -83,28 +84,28 @@ RmSaveDisplayState
NvBool use_vbios = NV_PRIMARY_VGA(nv) && RmGpuHasIOSpaceEnabled(nv);
NvU32 eax, ebx;
NV_STATUS status;
NV2080_CTRL_CMD_INTERNAL_DISPLAY_UNIX_CONSOLE_PARAMS unixConsoleParams = {0};
NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS preUnixConsoleParams = {0};
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS postUnixConsoleParams = {0};
if (IS_VIRTUAL(pGpu) || pKernelDisplay == NULL)
{
return;
}
os_disable_console_access();
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
{
NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support saving of efifb console\n");
goto done;
return;
}
unixConsoleParams.bSaveOrRestore = NV_TRUE;
unixConsoleParams.bUseVbios = use_vbios;
os_disable_console_access();
preUnixConsoleParams.bSave = NV_TRUE;
preUnixConsoleParams.bUseVbios = use_vbios;
NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice,
NV2080_CTRL_CMD_INTERNAL_DISPLAY_UNIX_CONSOLE,
&unixConsoleParams, sizeof(unixConsoleParams)), done);
NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE,
&preUnixConsoleParams, sizeof(preUnixConsoleParams)), done);
if (use_vbios)
{
@@ -124,6 +125,14 @@ RmSaveDisplayState
}
}
postUnixConsoleParams.bSave = NV_TRUE;
postUnixConsoleParams.bUseVbios = use_vbios;
NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient,
nv->rmapi.hSubDevice,
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE,
&postUnixConsoleParams, sizeof(postUnixConsoleParams)), done);
done:
os_enable_console_access();
}
@@ -140,8 +149,8 @@ static void RmRestoreDisplayState
KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
NV_STATUS status;
NvU32 eax, ebx;
NV2080_CTRL_CMD_INTERNAL_DISPLAY_UNIX_CONSOLE_PARAMS unixConsoleParams = {0};
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_RESTORE_PARAMS restoreParams = {0};
NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE_PARAMS preUnixConsoleParams = {0};
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE_PARAMS postUnixConsoleParams = {0};
NV_ASSERT_OR_RETURN_VOID(pKernelDisplay != NULL);
@@ -160,8 +169,6 @@ static void RmRestoreDisplayState
return;
}
os_disable_console_access();
//
// Fix up DCB index VBIOS scratch registers.
// The strategies employed are:
@@ -175,34 +182,39 @@ static void RmRestoreDisplayState
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
{
NV_PRINTF(LEVEL_INFO, "RM fallback doesn't support efifb console restore\n");
goto done;
return;
}
unixConsoleParams.bUseVbios = use_vbios;
unixConsoleParams.bSaveOrRestore = NV_FALSE;
os_disable_console_access();
preUnixConsoleParams.bUseVbios = use_vbios;
preUnixConsoleParams.bSave = NV_FALSE;
NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient,
nv->rmapi.hSubDevice,
NV2080_CTRL_CMD_INTERNAL_DISPLAY_UNIX_CONSOLE,
&unixConsoleParams, sizeof(unixConsoleParams)), done);
NV2080_CTRL_CMD_INTERNAL_DISPLAY_PRE_UNIX_CONSOLE,
&preUnixConsoleParams, sizeof(preUnixConsoleParams)), done);
eax = 0x4f02;
ebx = nvp->vga.vesaMode;
if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx))
if (use_vbios)
{
restoreParams.bWriteCr = NV_TRUE;
eax = 0x4f02;
ebx = nvp->vga.vesaMode;
if (NV_OK == unixCallVideoBIOS(pGpu, &eax, &ebx))
{
postUnixConsoleParams.bVbiosCallSuccessful = NV_TRUE;
}
}
postUnixConsoleParams.bSave = NV_FALSE;
postUnixConsoleParams.bUseVbios = use_vbios;
NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, pRmApi->Control(pRmApi, nv->rmapi.hClient,
nv->rmapi.hSubDevice,
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_RESTORE,
&restoreParams, sizeof(restoreParams)), done);
NV2080_CTRL_CMD_INTERNAL_DISPLAY_POST_UNIX_CONSOLE,
&postUnixConsoleParams, sizeof(postUnixConsoleParams)), done);
done:
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
{
}
os_enable_console_access();
}

132
src/nvidia/arch/nvalloc/unix/src/unix_intr.c
View File

@@ -26,6 +26,7 @@
#include <osapi.h>
#include <core/thread_state.h>
#include <core/locks.h>
#include <gpu_mgr/gpu_mgr.h>
#include <gpu/gpu.h>
#include "kernel/gpu/intr/intr.h"
#include "gpu/bif/kernel_bif.h"
@@ -34,7 +35,6 @@
#include <nv_sriov_defines.h>
#include "objtmr.h"
static NvBool osInterruptPending(
OBJGPU *pGpu,
NvBool *serviced,
@@ -107,8 +107,11 @@ static NvBool osInterruptPending(
pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
tlsIsrInit(pIsrAllocator);
//
// For SWRL granular locking process the countdown timer interrupt.
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING))
// GSP-RM handles SWRL interrupts if GSP-RM is active
//
if ((!IS_GSP_CLIENT(pGpu)) && (pGpu->getProperty(pGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING)))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
@@ -127,15 +130,43 @@ static NvBool osInterruptPending(
continue;
}
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
*serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &intr0Pending, &threadState);
*serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &threadState);
}
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
//
// Service nonstall interrupts before possibly acquiring GPUs lock
// so that we don't unnecesarily hold the lock while servicing them.
//
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
gpuInstance = 0;
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pIntr = GPU_GET_INTR(pGpu);
if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
{
NvBool bCtxswLog = NV_FALSE;
intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
if (!bitVectorTestAllCleared(&intr1Pending))
{
intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
*serviced = NV_TRUE;
}
}
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
// LOCK: try to acquire GPUs lock
if (rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK)
{
@@ -222,31 +253,6 @@ static NvBool osInterruptPending(
}
}
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
gpuInstance = 0;
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pIntr = GPU_GET_INTR(pGpu);
if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
{
NvBool bCtxswLog = NV_FALSE;
intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
if (!bitVectorTestAllCleared(&intr1Pending))
{
intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
*serviced = NV_TRUE;
}
}
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
tlsIsrDestroy(pIsrAllocator);
portMemAllocatorRelease(pIsrAllocator);
@@ -593,6 +599,36 @@ NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(
status = NV_OK;
goto done;
}
else
{
KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
PORT_MEM_ALLOCATOR *pIsrAllocator;
THREAD_STATE_NODE threadState;
NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
if (pKernelGmmu == NULL)
{
status = NV_ERR_OBJECT_NOT_FOUND;
goto done;
}
// If MMU fault buffer is not enabled, return early
if (!gpuIsVoltaHubIntrSupported(pGpu))
goto done;
pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
tlsIsrInit(pIsrAllocator);
threadStateInitISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR);
// Copies all valid packets in RM's and client's shadow buffer
status = kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, &threadState, faultsCopied,
NON_REPLAYABLE_FAULT_BUFFER);
threadStateFreeISRAndDeferredIntHandler(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR);
tlsIsrDestroy(pIsrAllocator);
portMemAllocatorRelease(pIsrAllocator);
}
done:
NV_EXIT_RM_RUNTIME(sp,fp);
@@ -610,6 +646,20 @@ static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked(
THREAD_STATE_NODE *pThreadState
)
{
KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
if (pKernelGmmu == NULL)
{
return NV_ERR_OBJECT_NOT_FOUND;
}
// If MMU fault buffer is not enabled, return early
if (!gpuIsVoltaHubIntrSupported(pGpu))
return NV_OK;
// Copies all valid packets in RM's and client's shadow buffer
return kgmmuCopyMmuFaults_HAL(pGpu, pKernelGmmu, pThreadState, pFaultsCopied,
NON_REPLAYABLE_FAULT_BUFFER);
return NV_OK;
}
@@ -693,3 +743,27 @@ NV_STATUS rm_gpu_handle_mmu_faults(
return status;
}
NvBool NV_API_CALL rm_is_msix_allowed(
nvidia_stack_t *sp,
nv_state_t *nv
)
{
nv_priv_t *pNvp = NV_GET_NV_PRIV(nv);
THREAD_STATE_NODE threadState;
void *fp;
NvBool ret = NV_FALSE;
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
if (rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT) == NV_OK)
{
ret = gpumgrIsDeviceMsixAllowed(nv->regs->cpu_address,
pNvp->pmc_boot_1, pNvp->pmc_boot_42);
rmapiLockRelease();
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
return ret;
}