NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-06 16:19:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

570.86.15

Browse Source
This commit is contained in:
Bernhard Stoeckner
2025-01-27 19:36:56 +01:00
parent 9d0b0414a5
commit 54d69484da
1166 changed files with 318863 additions and 182687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/nvidia/arch/nvalloc/common/inc/conf_compute/cc_keystore.h
View File

@@ -474,4 +474,9 @@ enum
#define CC_EXPORT_MASTER_KEY_SIZE_BYTES 32
#define CC_KEYID_PERCHANNEL_INITIAL_KEYSEED "ce_channel"
#define CC_KEYID_PERCHANNEL_UPDATE_KEYSEED "update_keyseeed"
#define CC_KEYID_PERCHANNEL_CPU_GPU_UPDATE_KEY "update_key1"
#define CC_KEYID_PERCHANNEL_GPU_CPU_UPDATE_KEY "update_key2"
#endif // CC_KEYSTORE_H

1
src/nvidia/arch/nvalloc/common/inc/flcnretval.h
View File

@@ -185,6 +185,7 @@ typedef NvU8 FLCN_STATUS;
#define FLCN_ERR_NVRNG_GENERATE_FAILED (0x97U)
#define FLCN_ERR_NVPKA_ECDSA_INIT_FAILED (0x98U)
#define FLCN_ERR_NVPKA_ECDSA_SIGN_FAILED (0x99U)
#define FLCN_ERR_NVPKA_GET_ECC_KEY_PAIR_FAILED (0x9AU)
// Heavy Secure Errors
#define FLCN_ERR_HS_CHK_INVALID_INPUT (0xA0U)

1
src/nvidia/arch/nvalloc/common/inc/fsp/fsp_nvdm_format.h
View File

@@ -46,5 +46,6 @@
#define NVDM_TYPE_UEFI_XTL_DEBUG_INTR 0x1D
#define NVDM_TYPE_TNVL 0x1F
#define NVDM_TYPE_CLOCK_BOOST 0x20
#define NVDM_TYPE_FSP_GSP_COMM 0x21
#endif // FSP_NVDM_FORMAT_H

41
src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,61 +29,46 @@
#define GSP_FW_SR_META_MAGIC 0x8a3bb9e6c6c39d93ULL
#define GSP_FW_SR_META_REVISION 2
#define GSP_FW_SR_META_INTERNAL_SIZE 128
/*!
* GSP firmware SR metadata
*
* Initialized by CPU-RM and kept in Sysmem.
* Verified by Booter.
*
* Consumed by secure ucode (Booter or ACR_RISCV).
*/
typedef struct
{
//
// Magic
// Use for verification by Booter
//
// Magic for verification by secure ucode
NvU64 magic; // = GSP_FW_SR_META_MAGIC;
//
// Revision number
// Bumped up when we change this interface so it is not backward compatible.
// Bumped up when we revoke GSP-RM ucode
//
NvU64 revision; // = GSP_FW_SR_META_MAGIC_REVISION;
//
// ---- Members regarding data in SYSMEM ----------------------------
// Consumed by Booter for DMA
//
// Members regarding data in SYSMEM
NvU64 sysmemAddrOfSuspendResumeData;
NvU64 sizeOfSuspendResumeData;
// ---- Members for crypto ops across S/R ---------------------------
//
// HMAC over the entire GspFwSRMeta structure (including padding)
// with the hmac field itself zeroed.
// Internal members for use by secure ucode
// Must be exactly GSP_FW_SR_META_INTERNAL_SIZE bytes.
//
NvU8 hmac[32];
// Hash over GspFwWprMeta structure
NvU8 wprMetaHash[32];
// Hash over GspFwHeapFreeList structure. All zeros signifies no free list.
NvU8 heapFreeListHash[32];
// Hash over data in WPR2 (skipping over free heap chunks; see Booter for details)
NvU8 dataHash[32];
NvU32 internal[32];
// Same as flags of GspFwWprMeta
NvU32 flags;
// Subrevision number used by secure ucode
NvU32 subrevision;
//
// Pad structure to exactly 256 bytes (1 DMA chunk).
// Padding initialized to zero.
//
NvU32 padding[23];
NvU32 padding[22];
} GspFwSRMeta;
#endif // GSP_FW_SR_META_H_

1
src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h
View File

@@ -235,5 +235,6 @@ typedef struct
#define GSP_FW_FLAGS 8:0
#define GSP_FW_FLAGS_CLOCK_BOOST NVBIT(0)
#define GSP_FW_FLAGS_RECOVERY_MARGIN_PRESENT NVBIT(1)
#define GSP_FW_FLAGS_PPCIE_ENABLED NVBIT(2)
#endif // GSP_FW_WPR_META_H_

5
src/nvidia/arch/nvalloc/common/inc/nv-firmware-chip-family-select.h
View File

@@ -54,9 +54,12 @@ static inline nv_firmware_chip_family_t nv_firmware_get_chip_family(
case GPU_ARCHITECTURE_HOPPER:
return NV_FIRMWARE_CHIP_FAMILY_GH100;
case GPU_ARCHITECTURE_BLACKWELL:
case GPU_ARCHITECTURE_BLACKWELL_GB1XX:
return NV_FIRMWARE_CHIP_FAMILY_GB10X;
case GPU_ARCHITECTURE_BLACKWELL_GB2XX:
return NV_FIRMWARE_CHIP_FAMILY_GB20X;
}
return NV_FIRMWARE_CHIP_FAMILY_NULL;

7
src/nvidia/arch/nvalloc/common/inc/nv-firmware.h
View File

@@ -32,7 +32,7 @@
typedef enum
{
NV_FIRMWARE_TYPE_GSP,
NV_FIRMWARE_TYPE_GSP_LOG
NV_FIRMWARE_TYPE_GSP_LOG,
} nv_firmware_type_t;
typedef enum
@@ -45,6 +45,7 @@ typedef enum
NV_FIRMWARE_CHIP_FAMILY_AD10X = 5,
NV_FIRMWARE_CHIP_FAMILY_GH100 = 6,
NV_FIRMWARE_CHIP_FAMILY_GB10X = 8,
NV_FIRMWARE_CHIP_FAMILY_GB20X = 9,
NV_FIRMWARE_CHIP_FAMILY_END,
} nv_firmware_chip_family_t;
@@ -54,6 +55,7 @@ static inline const char *nv_firmware_chip_family_to_string(
{
switch (fw_chip_family) {
case NV_FIRMWARE_CHIP_FAMILY_GB10X: return "gb10x";
case NV_FIRMWARE_CHIP_FAMILY_GB20X: return "gb20x";
case NV_FIRMWARE_CHIP_FAMILY_GH100: return "gh100";
case NV_FIRMWARE_CHIP_FAMILY_AD10X: return "ad10x";
case NV_FIRMWARE_CHIP_FAMILY_GA10X: return "ga10x";
@@ -84,6 +86,7 @@ static inline const char *nv_firmware_for_chip_family(
switch (fw_chip_family)
{
case NV_FIRMWARE_CHIP_FAMILY_GB10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GB20X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_AD10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA10X:
@@ -104,6 +107,7 @@ static inline const char *nv_firmware_for_chip_family(
switch (fw_chip_family)
{
case NV_FIRMWARE_CHIP_FAMILY_GB10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GB20X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GH100: // fall through
case NV_FIRMWARE_CHIP_FAMILY_AD10X: // fall through
case NV_FIRMWARE_CHIP_FAMILY_GA10X:
@@ -119,7 +123,6 @@ static inline const char *nv_firmware_for_chip_family(
return "";
}
}
return "";
}
#endif // defined(NV_FIRMWARE_FOR_NAME)

20
src/nvidia/arch/nvalloc/common/inc/nvcst.h
View File

@@ -65,6 +65,8 @@ CHIPSET_SETUP_FUNC(Intel_0685_setupFunc)
CHIPSET_SETUP_FUNC(Intel_4381_setupFunc)
CHIPSET_SETUP_FUNC(Intel_7A82_setupFunc)
CHIPSET_SETUP_FUNC(Intel_7A04_setupFunc)
CHIPSET_SETUP_FUNC(Intel_5795_setupFunc)
CHIPSET_SETUP_FUNC(Intel_1B81_setupFunc)
CHIPSET_SETUP_FUNC(SiS_656_setupFunc)
CHIPSET_SETUP_FUNC(ATI_RS400_setupFunc)
CHIPSET_SETUP_FUNC(ATI_RS480_setupFunc)
@@ -95,6 +97,8 @@ CHIPSET_SETUP_FUNC(Nvidia_T194_setupFunc)
CHIPSET_SETUP_FUNC(Nvidia_TH500_setupFunc)
CHIPSET_SETUP_FUNC(PLDA_XpressRichAXI_setupFunc)
CHIPSET_SETUP_FUNC(Riscv_generic_setupFunc)
CHIPSET_SETUP_FUNC(Intel_A70D_setupFunc)
CHIPSET_SETUP_FUNC(AMD_14D8_setupFunc)
// Keep string length <=32 (including termination) to avoid string copy overflow
@@ -188,11 +192,12 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_INTEL, 0x4385, CS_INTEL_4381, "Intel-RocketLake", Intel_4381_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x7A82, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x7A84, CS_INTEL_7A82, "Intel-AlderLake", Intel_7A82_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x1B81, CS_INTEL_1B81, "Intel-SapphireRapids", NULL},
{PCI_VENDOR_ID_INTEL, 0x7A8A, CS_INTEL_1B81, "Intel-SapphireRapids", NULL},
{PCI_VENDOR_ID_INTEL, 0x1B81, CS_INTEL_1B81, "Intel-SapphireRapids", Intel_1B81_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x7A8A, CS_INTEL_1B81, "Intel-SapphireRapids", Intel_1B81_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x18DC, CS_INTEL_18DC, "Intel-IceLake", NULL},
{PCI_VENDOR_ID_INTEL, 0x7A04, CS_INTEL_7A04, "Intel-RaptorLake", Intel_7A04_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x5795, CS_INTEL_5795, "Intel-GraniteRapids", NULL},
{PCI_VENDOR_ID_INTEL, 0x5795, CS_INTEL_5795, "Intel-GraniteRapids", Intel_5795_setupFunc},
{PCI_VENDOR_ID_INTEL, 0xA70D, CS_INTEL_B660, "Intel-B660", Intel_A70D_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc},
@@ -212,6 +217,10 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_NVIDIA, 0x22B4, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x22B8, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x22B9, CS_NVIDIA_TH500, "TH500", Nvidia_TH500_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x22D6, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x22D8, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x22DA, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x2B00, CS_NVIDIA_T264, "T264", Nvidia_T194_setupFunc},
{PCI_VENDOR_ID_SIS, 0x0649, CS_SIS_649, "649", SiS_656_setupFunc},
{PCI_VENDOR_ID_SIS, 0x0656, CS_SIS_656, "656", SiS_656_setupFunc},
@@ -231,6 +240,7 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_AMD, 0x9601, CS_AMD_GX890, "GX890" ,AMD_FX890_setupFunc},
{PCI_VENDOR_ID_AMD, 0x9600, CS_AMD_RS780, "RS780" ,AMD_RS780_setupFunc},
{PCI_VENDOR_ID_AMD, 0x790e, CS_AMD_X370, "X370/X399/X470/ TRX40/X570/WRX80", AMD_X370_setupFunc },
{PCI_VENDOR_ID_AMD, 0x14d8, CS_AMD_RPH, "AMD-Raphael", AMD_14D8_setupFunc},
{PCI_VENDOR_ID_VIA, 0x0308, CS_VIA_VT8369B, "VT8369B", NULL},
{PCI_VENDOR_ID_VIA, 0x0410, CS_VIA_VX900, "VX900", VIA_VX900_setupFunc},
@@ -347,6 +357,10 @@ ARMCSALLOWLISTINFO armChipsetAllowListInfo[] =
{PCI_VENDOR_ID_NVIDIA, 0x22B4, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x1
{PCI_VENDOR_ID_NVIDIA, 0x22B8, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x8
{PCI_VENDOR_ID_NVIDIA, 0x22B9, CS_NVIDIA_TH500}, // NVIDIA TH500 RP x2
{PCI_VENDOR_ID_NVIDIA, 0x22D6, CS_NVIDIA_T264}, // NVIDIA Tegra RP x8
{PCI_VENDOR_ID_NVIDIA, 0x22D8, CS_NVIDIA_T264}, // NVIDIA Tegra RP x4
{PCI_VENDOR_ID_NVIDIA, 0x22DA, CS_NVIDIA_T264}, // NVIDIA Tegra RP x2
{PCI_VENDOR_ID_NVIDIA, 0x2B00, CS_NVIDIA_T264}, // NVIDIA Tegra RP iGPU
{PCI_VENDOR_ID_APM, 0xe004, CS_APM_STORM}, // Applied Micro X-Gene "Storm"
{PCI_VENDOR_ID_MARVELL, 0xAF00, CS_MARVELL_THUNDERX2}, // Marvell ThunderX2

27
src/nvidia/arch/nvalloc/common/inc/nvpcie.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2000-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2000-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -211,18 +211,21 @@
// to any specific hardware.
//
//
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0 0x000000C8
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_ID 7:0
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_NEXT 15:8
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_LENGTH 23:16
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_SIG_LO 31:24
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1 0x000000CC
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_SIG_HI 15:0
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_VERSION 18:16
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_PEER_CLIQUE_ID 22:19
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RSVD 31:23
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0 0x000000C8
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_ID 7:0
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_NEXT 15:8
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_LENGTH 23:16
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_0_SIG_LO 31:24
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1 0x000000CC
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_SIG_HI 15:0
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_VERSION 18:16
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_PEER_CLIQUE_ID 22:19
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RELAXED_ORDERING 23:23
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RELAXED_ORDERING_DEFAULT 0x00000000
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RELAXED_ORDERING_DISABLE 0x00000001
#define NV_PCI_VIRTUAL_P2P_APPROVAL_CAP_1_RSVD 31:24
#define NV_PCI_VIRTUAL_P2P_APPROVAL_SIGNATURE 0x00503250
#define NV_PCI_VIRTUAL_P2P_APPROVAL_SIGNATURE 0x00503250
// Chipset-specific definitions.
// Intel SantaRosa definitions

2
src/nvidia/arch/nvalloc/common/inc/rmpbicmdif.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2009-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2009-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a

6
src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -128,6 +128,9 @@ typedef struct nv_ioctl_register_fd
#define NV_DMABUF_EXPORT_MAX_HANDLES 128
#define NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT 0
#define NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE 1
typedef struct nv_ioctl_export_to_dma_buf_fd
{
int fd;
@@ -136,6 +139,7 @@ typedef struct nv_ioctl_export_to_dma_buf_fd
NvU32 numObjects;
NvU32 index;
NvU64 totalSize NV_ALIGN_BYTES(8);
NvU8 mappingType;
NvHandle handles[NV_DMABUF_EXPORT_MAX_HANDLES];
NvU64 offsets[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8);
NvU64 sizes[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8);

3
src/nvidia/arch/nvalloc/unix/include/nv-priv.h
View File

@@ -330,6 +330,9 @@ typedef struct
/* Flag to check if S0ix-based power management is enabled. */
NvBool s0ix_pm_enabled;
/* Variable to track Dynamic Boost support */
int db_supported;
/*
* Maximum FB allocation size which can be saved in system memory
* during system supened with S0ix-based power management.

52
src/nvidia/arch/nvalloc/unix/include/nv-reg.h
View File

@@ -386,32 +386,6 @@
#define __NV_IGNORE_MMIO_CHECK IgnoreMMIOCheck
#define NV_REG_IGNORE_MMIO_CHECK NV_REG_STRING(__NV_IGNORE_MMIO_CHECK)
/*
* Option: TCEBypassMode
*
* Description:
*
* When this option is enabled, the NVIDIA kernel module will attempt to setup
* all GPUs in "TCE bypass mode", in which DMA mappings of system memory bypass
* the IOMMU/TCE remapping hardware on IBM POWER systems. This is typically
* necessary for CUDA applications in which large system memory mappings may
* exceed the default TCE remapping capacity when operated in non-bypass mode.
*
* This option has no effect on non-POWER platforms.
*
* Possible Values:
*
* 0: system default TCE mode on all GPUs
* 1: enable TCE bypass mode on all GPUs
* 2: disable TCE bypass mode on all GPUs
*/
#define __NV_TCE_BYPASS_MODE TCEBypassMode
#define NV_REG_TCE_BYPASS_MODE NV_REG_STRING(__NV_TCE_BYPASS_MODE)
#define NV_TCE_BYPASS_MODE_DEFAULT 0
#define NV_TCE_BYPASS_MODE_ENABLE 1
#define NV_TCE_BYPASS_MODE_DISABLE 2
/*
* Option: pci
*
@@ -899,7 +873,7 @@
*
* This option allows users to specify whether the NVIDIA driver must create
* the IMEX channel 0 by default. The channel will be created automatically
* when an application (e.g. nvidia-smi, nvidia-persistenced) is run.
* when the NVIDIA open GPU kernel module is loaded.
*
* Note that users are advised to enable this option only in trusted
* environments where it is acceptable for applications to share the same
@@ -915,6 +889,26 @@
#define __NV_CREATE_IMEX_CHANNEL_0 CreateImexChannel0
#define NV_CREATE_IMEX_CHANNEL_0 NV_REG_STRING(__CREATE_IMEX_CHANNEL_0)
/*
* Option: NVreg_GrdmaPciTopoCheckOverride
*
* Description:
*
* This option is applicable only on coherent systems with BAR1 enabled to allow
* maximum bandwidth between GPU and a third party device over a dedicated
* PCIe link instead of over C2C for GPUDirect RDMA use-cases.
* Such a config is only supported for a specific topology which is checked by
* the GPU driver's dma-buf and nv-p2p subsystems.
*
* This option allows the user to override the driver's topology check.
*
* Possible values:
* 0 - Do not override topology check (default).
* 1 - Override topology check.
*/
#define __NV_GRDMA_PCI_TOPO_CHECK_OVERRIDE GrdmaPciTopoCheckOverride
#define NV_GRDMA_PCI_TOPO_CHECK_OVERRIDE NV_REG_STRING(__NV_GRDMA_PCI_TOPO_CHECK_OVERRIDE)
#if defined(NV_DEFINE_REGISTRY_KEY_TABLE)
/*
@@ -931,7 +925,6 @@ NV_DEFINE_REG_ENTRY(__NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, 1);
NV_DEFINE_REG_ENTRY(__NV_USE_PAGE_ATTRIBUTE_TABLE, ~0);
NV_DEFINE_REG_ENTRY(__NV_ENABLE_PCIE_GEN3, 0);
NV_DEFINE_REG_ENTRY(__NV_ENABLE_MSI, 1);
NV_DEFINE_REG_ENTRY(__NV_TCE_BYPASS_MODE, NV_TCE_BYPASS_MODE_DEFAULT);
NV_DEFINE_REG_ENTRY(__NV_ENABLE_STREAM_MEMOPS, 0);
NV_DEFINE_REG_ENTRY(__NV_RM_PROFILING_ADMIN_ONLY_PARAMETER, 1);
NV_DEFINE_REG_ENTRY(__NV_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, 0);
@@ -966,6 +959,7 @@ NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL);
NV_DEFINE_REG_ENTRY(__NV_RM_NVLINK_BW_LINK_COUNT, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_CREATE_IMEX_CHANNEL_0, 0);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_GRDMA_PCI_TOPO_CHECK_OVERRIDE, 0);
/*
*----------------registry database definition----------------------
@@ -993,7 +987,6 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_KMALLOC_HEAP_MAX_SIZE),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_VMALLOC_HEAP_MAX_SIZE),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IGNORE_MMIO_CHECK),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_TCE_BYPASS_MODE),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_STREAM_MEMOPS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_USER_NUMA_MANAGEMENT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_NVLINK_DISABLE),
@@ -1015,6 +1008,7 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IMEX_CHANNEL_COUNT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_CREATE_IMEX_CHANNEL_0),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_GRDMA_PCI_TOPO_CHECK_OVERRIDE),
{NULL, NULL}
};

44
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -368,6 +368,8 @@ typedef struct nv_state_t
{
NvBool valid;
NvU8 uuid[GPU_UUID_LEN];
NvBool pci_uuid_read_attempted;
NV_STATUS pci_uuid_status;
} nv_uuid_cache;
void *handle;
@@ -479,6 +481,8 @@ typedef struct nv_state_t
/* Bool to check if the GPU has a coherent sysmem link */
NvBool coherent;
/* OS detected GPU has ATS capability */
NvBool ats_support;
/*
* NUMA node ID of the CPU to which the GPU is attached.
* Holds NUMA_NO_NODE on platforms that don't support NUMA configuration.
@@ -570,7 +574,8 @@ typedef NV_STATUS (*nvPmaEvictRangeCallback)(void *, NvU64, NvU64, nvgpuGpuMemor
#define NV_FLAG_PASSTHRU 0x0080
#define NV_FLAG_SUSPENDED 0x0100
#define NV_FLAG_SOC_IGPU 0x0200
// Unused 0x0400
/* To be set when an FLR needs to be triggered after device shut down. */
#define NV_FLAG_TRIGGER_FLR 0x0400
#define NV_FLAG_PERSISTENT_SW_STATE 0x0800
#define NV_FLAG_IN_RECOVERY 0x1000
// Unused 0x2000
@@ -613,6 +618,7 @@ typedef struct
const char *gc6_support;
const char *gcoff_support;
const char *s0ix_status;
const char *db_support;
} nv_power_info_t;
#define NV_PRIMARY_VGA(nv) ((nv)->primary_vga)
@@ -758,6 +764,7 @@ static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
#define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1))
#endif
/*
* driver internal interfaces
*/
@@ -813,7 +820,6 @@ NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU6
void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64);
void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *);
void NV_API_CALL nv_dma_enable_nvlink (nv_dma_device_t *);
NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *);
NvS32 NV_API_CALL nv_stop_rc_timer (nv_state_t *);
@@ -840,9 +846,7 @@ NV_STATUS NV_API_CALL nv_acpi_mux_method (nv_state_t *, NvU32 *, NvU32,
NV_STATUS NV_API_CALL nv_log_error (nv_state_t *, NvU32, const char *, va_list);
NvU64 NV_API_CALL nv_get_dma_start_address (nv_state_t *);
NV_STATUS NV_API_CALL nv_set_primary_vga_status(nv_state_t *);
NV_STATUS NV_API_CALL nv_pci_trigger_recovery (nv_state_t *);
NvBool NV_API_CALL nv_requires_dma_remap (nv_state_t *);
NvBool NV_API_CALL nv_is_rm_firmware_active(nv_state_t *);
@@ -855,19 +859,8 @@ void NV_API_CALL nv_put_file_private(void *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_ibmnpu_genreg_info(nv_state_t *, NvU64 *, NvU64 *, void**);
NV_STATUS NV_API_CALL nv_get_ibmnpu_relaxed_ordering_mode(nv_state_t *nv, NvBool *mode);
void NV_API_CALL nv_wait_for_ibmnpu_rsync(nv_state_t *nv);
void NV_API_CALL nv_ibmnpu_cache_flush_range(nv_state_t *nv, NvU64, NvU64);
void NV_API_CALL nv_p2p_free_platform_data(void *data);
#if defined(NVCPU_PPC64LE)
NV_STATUS NV_API_CALL nv_get_nvlink_line_rate (nv_state_t *, NvU32 *);
#endif
NV_STATUS NV_API_CALL nv_revoke_gpu_mappings (nv_state_t *);
void NV_API_CALL nv_acquire_mmap_lock (nv_state_t *);
void NV_API_CALL nv_release_mmap_lock (nv_state_t *);
@@ -998,18 +991,24 @@ NV_STATUS NV_API_CALL rm_p2p_init_mapping (nvidia_stack_t *, NvU64, NvU6
NV_STATUS NV_API_CALL rm_p2p_destroy_mapping (nvidia_stack_t *, NvU64);
NV_STATUS NV_API_CALL rm_p2p_get_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *);
NV_STATUS NV_API_CALL rm_p2p_get_gpu_info (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **);
NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *, void **);
NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, NvBool, void *, void *, void **);
NV_STATUS NV_API_CALL rm_p2p_register_callback (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *, void *);
NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **);
NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *, void **);
void NV_API_CALL rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle);
NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, MemoryRange, void *, NvBool, MemoryArea *);
void NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, void *, NvBool, MemoryArea);
NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle *, NvHandle *, NvHandle *, void **, NvBool *);
NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *,
NvHandle, NvHandle, MemoryRange,
NvU8, void *, NvBool, MemoryArea *);
void NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *,
NvHandle, NvHandle, NvU8, void *,
NvBool, MemoryArea);
NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *,
nv_state_t *, NvHandle, NvHandle,
NvU8, NvHandle *, NvHandle *,
NvHandle *, void **, NvBool *);
void NV_API_CALL rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *);
NV_STATUS NV_API_CALL rm_log_gpu_crash (nv_stack_t *, nv_state_t *);
void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd);
NvBool NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id);
@@ -1026,7 +1025,6 @@ NvBool NV_API_CALL rm_is_device_sequestered(nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_check_for_gpu_surprise_removal(nvidia_stack_t *, nv_state_t *);
NV_STATUS NV_API_CALL rm_set_external_kernel_client_count(nvidia_stack_t *, nv_state_t *, NvBool);
NV_STATUS NV_API_CALL rm_schedule_gpu_wakeup(nvidia_stack_t *, nv_state_t *);
NvBool NV_API_CALL rm_is_iommu_needed_for_sriov(nvidia_stack_t *, nv_state_t *);
NvBool NV_API_CALL rm_disable_iomap_wc(void);
void NV_API_CALL rm_init_dynamic_power_management(nvidia_stack_t *, nv_state_t *, NvBool);
@@ -1043,12 +1041,14 @@ void NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);
NvBool NV_API_CALL rm_is_altstack_in_use(void);
/* vGPU VFIO specific functions */
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32);
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *,
NvU32 *, NvU32 *, NvU32);
NV_STATUS NV_API_CALL nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16);
NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool);
NV_STATUS NV_API_CALL nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8);
NV_STATUS NV_API_CALL nv_vgpu_get_bar_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *,
NvU64 *, NvU64 *, NvU32 *, NvBool *, NvU8 *);
NV_STATUS NV_API_CALL nv_vgpu_update_sysfs_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU32);
NV_STATUS NV_API_CALL nv_vgpu_get_hbm_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *, NvU64 *);
NV_STATUS NV_API_CALL nv_vgpu_process_vf_info(nvidia_stack_t *, nv_state_t *, NvU8, NvU32, NvU8, NvU8, NvU8, NvBool, void *);
NV_STATUS NV_API_CALL nv_gpu_bind_event(nvidia_stack_t *, NvU32, NvBool *);

5
src/nvidia/arch/nvalloc/unix/include/os-interface.h
View File

@@ -166,7 +166,7 @@ NvU32 NV_API_CALL os_get_grid_csp_support (void);
void NV_API_CALL os_bug_check (NvU32, const char *);
NV_STATUS NV_API_CALL os_lock_user_pages (void *, NvU64, void **, NvU32);
NV_STATUS NV_API_CALL os_lookup_user_io_memory (void *, NvU64, NvU64 **);
NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *);
NV_STATUS NV_API_CALL os_unlock_user_pages (NvU64, void *, NvU32);
NV_STATUS NV_API_CALL os_match_mmap_offset (void *, NvU64, NvU64 *);
NV_STATUS NV_API_CALL os_get_euid (NvU32 *);
NV_STATUS NV_API_CALL os_get_smbios_header (NvU64 *pSmbsAddr);
@@ -174,6 +174,7 @@ NV_STATUS NV_API_CALL os_get_acpi_rsdp_from_uefi (NvU32 *);
void NV_API_CALL os_add_record_for_crashLog (void *, NvU32);
void NV_API_CALL os_delete_record_for_crashLog (void *);
NV_STATUS NV_API_CALL os_call_vgpu_vfio (void *, NvU32);
NV_STATUS NV_API_CALL os_device_vm_present (void);
NV_STATUS NV_API_CALL os_numa_memblock_size (NvU64 *);
NV_STATUS NV_API_CALL os_alloc_pages_node (NvS32, NvU32, NvU32, NvU64 *);
NV_STATUS NV_API_CALL os_get_page (NvU64 address);
@@ -209,6 +210,7 @@ enum os_pci_req_atomics_type {
OS_INTF_PCIE_REQ_ATOMICS_128BIT
};
NV_STATUS NV_API_CALL os_enable_pci_req_atomics (void *, enum os_pci_req_atomics_type);
void NV_API_CALL os_pci_trigger_flr(void *handle);
NV_STATUS NV_API_CALL os_get_numa_node_memory_usage (NvS32, NvU64 *, NvU64 *);
NV_STATUS NV_API_CALL os_numa_add_gpu_memory (void *, NvU64, NvU64, NvU32 *);
NV_STATUS NV_API_CALL os_numa_remove_gpu_memory (void *, NvU64, NvU64, NvU32);
@@ -216,6 +218,7 @@ NV_STATUS NV_API_CALL os_offline_page_at_address(NvU64 address);
void* NV_API_CALL os_get_pid_info(void);
void NV_API_CALL os_put_pid_info(void *pid_info);
NV_STATUS NV_API_CALL os_find_ns_pid(void *pid_info, NvU32 *ns_pid);
NvBool NV_API_CALL os_is_init_ns(void);
extern NvU32 os_page_size;
extern NvU64 os_page_mask;

2
src/nvidia/arch/nvalloc/unix/include/os_custom.h
View File

@@ -49,7 +49,7 @@
#define OS_XUGO (OS_XUSR | OS_XGRP | OS_XOTH)
// Trigger for collecting GPU state for later extraction.
NV_STATUS RmLogGpuCrash(OBJGPU *);
void RmLogGpuCrash(OBJGPU *);
// This is callback function in the miniport.
// The argument is a device extension, and must be cast as such to be useful.

93
src/nvidia/arch/nvalloc/unix/src/dynamic-power.c
View File

@@ -1074,6 +1074,63 @@ os_ref_dynamic_power(
return status;
}
/*
* @brief This function queries the dynamic boost support.
* This function returns -1 in case it failed to query the
* dynamic boost support. It returns 1 if dynamic boost is
* supported and returns 0 if dynamic boost is not supported.
*
* The caller must hold the API lock before calling this
* function.
*
* @param[in] nv nv_state_t pointer.
*/
static int os_get_dynamic_boost_support(
nv_state_t *nv
)
{
NV_STATUS status = NV_OK;
RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
NV0000_CTRL_CMD_SYSTEM_NVPCF_GET_POWER_MODE_INFO_PARAMS *pNvpcfParams;
OBJGPU *pGpu;
int ret;
status = os_alloc_mem((void**)&pNvpcfParams, sizeof(*pNvpcfParams));
if (status != NV_OK)
{
NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory\n");
ret = -1;
goto end;
}
pGpu = NV_GET_NV_PRIV_PGPU(nv);
pNvpcfParams->gpuId = pGpu->gpuId;
pNvpcfParams->subFunc =
NVPCF0100_CTRL_CONFIG_DSM_2X_FUNC_GET_SUPPORTED_CASE;
status = pRmApi->Control(pRmApi,
nv->rmapi.hClient,
nv->rmapi.hClient,
NV0000_CTRL_CMD_SYSTEM_NVPCF_GET_POWER_MODE_INFO,
(void*)pNvpcfParams,
sizeof(*pNvpcfParams));
if (status == NV_OK)
{
ret = 1;
}
else if (status == NV_ERR_NOT_SUPPORTED)
{
ret = 0;
}
else
{
ret = -1;
}
os_free_mem(pNvpcfParams);
end:
return ret;
}
/*!
* @brief Wrapper around os_ref_dynamic_power() suitable for use from the
* per-OS layers.
@@ -2546,6 +2603,38 @@ void RmHandleDisplayChange(
}
}
/*!
* @brief RmGetDynamicBoostSupport() gets the notebook DB
* support. DB support is static information for vast majority
* of notebooks and hence we fetch it only once (during the first
* call to this function). This way, the GPU remains in RTD3 state
* when procfs read is done.
*
* @param[in] nv nv_state_t pointer.
*/
static const char* RmGetDynamicBoostSupport(
nv_state_t *pNv
)
{
nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
const char* DbStatus = "?";
if (pNvp->db_supported == -1)
{
pNvp->db_supported = os_get_dynamic_boost_support(pNv);
}
if (pNvp->db_supported == 0)
{
DbStatus = "Not Supported";
}
else if (pNvp->db_supported == 1)
{
DbStatus = "Supported";
}
return DbStatus;
}
/*!
* @brief: Function to query Dynamic Power Management
*
@@ -2715,6 +2804,7 @@ void NV_API_CALL rm_get_power_info(
const char *pGcoffSupported = "?";
const char *pDynamicPowerStatus = "?";
const char *pS0ixStatus = "?";
const char *pDbStatus = "?";
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
@@ -2738,7 +2828,7 @@ void NV_API_CALL rm_get_power_info(
pGc6Supported = RmGetGpuGcxSupport(pGpu, NV_TRUE);
pGcoffSupported = RmGetGpuGcxSupport(pGpu, NV_FALSE);
pS0ixStatus = pNvp->s0ix_pm_enabled ? "Enabled" : "Disabled";
pDbStatus = RmGetDynamicBoostSupport(pNv);
// UNLOCK: release per device lock
rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
}
@@ -2755,5 +2845,6 @@ void NV_API_CALL rm_get_power_info(
powerInfo->gc6_support = pGc6Supported;
powerInfo->gcoff_support = pGcoffSupported;
powerInfo->s0ix_status = pS0ixStatus;
powerInfo->db_support = pDbStatus;
}

2
src/nvidia/arch/nvalloc/unix/src/escape.c
View File

@@ -186,7 +186,7 @@ static void RmCreateOsDescriptor(NVOS32_PARAMETERS *pApi, API_SECURITY_INFO secI
default:
break;
case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
os_unlock_user_pages(pageCount, pPageArray);
os_unlock_user_pages(pageCount, pPageArray, flags);
break;
}
}

65
src/nvidia/arch/nvalloc/unix/src/os-hypervisor.c
View File

@@ -75,7 +75,7 @@ NV_STATUS hypervisorInjectInterrupt_IMPL
{
NV_STATUS status = NV_ERR_NOT_SUPPORTED;
if (osIsVgpuVfioPresent() == NV_TRUE)
if (osIsVgpuVfioPresent() == NV_OK)
return NV_ERR_NOT_SUPPORTED;
else
{
@@ -328,6 +328,8 @@ NV_STATUS NV_API_CALL nv_vgpu_create_request(
const NvU8 *pVgpuDevName,
NvU32 vgpuTypeId,
NvU16 *vgpuId,
NvU32 *gpu_instance_id,
NvU32 *placement_id,
NvU32 gpuPciBdf
)
{
@@ -342,7 +344,7 @@ NV_STATUS NV_API_CALL nv_vgpu_create_request(
if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR)) == NV_OK)
{
rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pVgpuDevName,
vgpuTypeId, vgpuId, gpuPciBdf);
vgpuTypeId, vgpuId, gpu_instance_id, placement_id, gpuPciBdf);
// UNLOCK: release API lock
rmapiLockRelease();
@@ -534,6 +536,61 @@ exit:
return rmStatus;
}
NV_STATUS NV_API_CALL nv_vgpu_update_sysfs_info
(
nvidia_stack_t *sp,
nv_state_t *pNv,
const NvU8 *pVgpuDevName,
NvU32 mode,
NvU32 sysfs_val
)
{
THREAD_STATE_NODE threadState;
void *fp = NULL;
OBJSYS *pSys = SYS_GET_INSTANCE();
KernelVgpuMgr *pKernelVgpuMgr = SYS_GET_KERNEL_VGPUMGR(pSys);
REQUEST_VGPU_INFO_NODE *pRequestVgpu = NULL;
NV_STATUS rmStatus = NV_OK;
VGPU_SYSFS_OP updateMode = (VGPU_SYSFS_OP)mode;
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
// LOCK: acquire API lock
NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_SILENT,
rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_HYPERVISOR), exit);
// Check the corresponding REQUEST_VGPU_INFO_NODE
for (pRequestVgpu = listHead(&pKernelVgpuMgr->listRequestVgpuHead);
pRequestVgpu != NULL;
pRequestVgpu = listNext(&pKernelVgpuMgr->listRequestVgpuHead, pRequestVgpu))
{
if (portMemCmp(pRequestVgpu->vgpuDevName, pVgpuDevName, VM_UUID_SIZE) == 0)
break;
}
if (pRequestVgpu == NULL)
{
rmStatus = NV_ERR_OBJECT_NOT_FOUND;
goto release_lock;
}
if (updateMode == SET_GPU_INSTANCE_ID)
rmStatus = kvgpumgrSetGpuInstanceId(pRequestVgpu, sysfs_val);
else if (updateMode == SET_PLACEMENT_ID)
rmStatus = kvgpumgrSetPlacementId(pRequestVgpu, sysfs_val);
release_lock:
// UNLOCK: release API lock
rmapiLockRelease();
exit:
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL nv_vgpu_get_hbm_info(
nvidia_stack_t *sp,
nv_state_t *pNv,
@@ -925,3 +982,7 @@ NV_STATUS rm_is_vgpu_supported_device(
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS osIsVgpuDeviceVmPresent(void)
{
return os_device_vm_present();
}

155
src/nvidia/arch/nvalloc/unix/src/os.c
View File

@@ -463,6 +463,10 @@ NV_STATUS osMapSystemMemory
{
OBJGPU *pGpu = pMemDesc->pGpu;
NvU64 rootOffset = 0;
pMemDesc = memdescGetRootMemDesc(pMemDesc, &rootOffset);
Offset += rootOffset;
RmPhysAddr userAddress;
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
NV_STATUS rmStatus = NV_OK;
@@ -548,10 +552,13 @@ void osUnmapSystemMemory
)
{
NV_STATUS status;
void *pAllocPrivate = memdescGetMemData(pMemDesc);
void *pAllocPrivate;
OBJGPU *pGpu = pMemDesc->pGpu;
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
pMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
pAllocPrivate = memdescGetMemData(pMemDesc);
if (Kernel)
{
status = nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress),
@@ -663,6 +670,11 @@ NV_STATUS osFindNsPid(void *pOsPidInfo, NvU32 *pNsPid)
return os_find_ns_pid(pOsPidInfo, pNsPid);
}
NvBool osIsInitNs(void)
{
return os_is_init_ns();
}
NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId)
{
//
@@ -742,6 +754,8 @@ NV_STATUS osQueueWorkItemWithFlags(
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RO)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RO;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE)
@@ -806,6 +820,15 @@ void osQueueMMUFaultHandler(OBJGPU *pGpu)
nv_schedule_uvm_isr(nv);
}
NvBool osGpuSupportsAts(OBJGPU *pGpu)
{
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
// Checks ATS support from both OS side and RM side.
return nv->ats_support &&
pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED);
}
NV_STATUS osQueueDrainP2PHandler(NvU8 *pUuid)
{
return nv_schedule_uvm_drain_p2p(pUuid);
@@ -854,11 +877,17 @@ NV_STATUS osAllocPagesInternal(
NvS32 nodeId = NV0000_CTRL_NO_NUMA_NODE;
NV_ADDRESS_SPACE addrSpace = memdescGetAddressSpace(pMemDesc);
NvU64 pageSize = osGetPageSize();
NvU64 osPageCount = NV_ALIGN_UP(pMemDesc->Size, pageSize) >> BIT_IDX_32(pageSize); // TODO: Switch out for macro before submission.
NvU64 rmPageCount = NV_ALIGN_UP(pMemDesc->Size, RM_PAGE_SIZE) >> RM_PAGE_SHIFT;
memdescSetAddress(pMemDesc, NvP64_NULL);
memdescSetMemData(pMemDesc, NULL, NULL);
NV_ASSERT_OR_RETURN(pMemDesc->PageCount > 0, NV_ERR_INVALID_ARGUMENT);
// In the non-contig case need to protect against page array overflows.
if (!memdescGetContiguity(pMemDesc, AT_CPU))
{
NV_ASSERT_OR_RETURN(rmPageCount <= pMemDesc->pageArraySize, NV_ERR_INVALID_ARGUMENT);
}
//
// For carveout, the memory is already reserved so we don't have
@@ -867,16 +896,24 @@ NV_STATUS osAllocPagesInternal(
if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_FROM_SCANOUT_CARVEOUT) ||
memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
{
// We only support scanout carveout with contiguous memory.
if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_FROM_SCANOUT_CARVEOUT) &&
!memdescGetContiguity(pMemDesc, AT_CPU))
{
status = NV_ERR_NOT_SUPPORTED;
goto done;
}
if (NV_RM_PAGE_SIZE < os_page_size &&
!memdescGetContiguity(pMemDesc, AT_CPU))
{
RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
pMemDesc->PageCount);
rmPageCount);
}
status = nv_alias_pages(
NV_GET_NV_STATE(pGpu),
NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
osPageCount,
pageSize,
memdescGetContiguity(pMemDesc, AT_CPU),
memdescGetCpuCacheAttrib(pMemDesc),
@@ -910,43 +947,49 @@ NV_STATUS osAllocPagesInternal(
nodeId = GPU_GET_MEMORY_MANAGER(pGpu)->localEgmNodeId;
}
if (NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount) > NV_U32_MAX)
if (osPageCount > NV_U32_MAX || rmPageCount > NV_U32_MAX)
{
status = NV_ERR_INVALID_LIMIT;
goto done;
}
else
{
//
// Bug 4270864: Only non-contig EGM memory needs to specify order. Contig memory
// calculates it within nv_alloc_pages. The long term goal is to expand the ability
// to request large page size for all of sysmem.
//
if (memdescIsEgm(pMemDesc) && !memdescGetContiguity(pMemDesc, AT_CPU))
{
pageSize = memdescGetPageSize(pMemDesc, AT_GPU);
}
status = nv_alloc_pages(
NV_GET_NV_STATE(pGpu),
NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
pageSize,
memdescGetContiguity(pMemDesc, AT_CPU),
memdescGetCpuCacheAttrib(pMemDesc),
pSys->getProperty(pSys,
PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS),
unencrypted,
nodeId,
memdescGetPteArray(pMemDesc, AT_CPU),
&pMemData);
//
// Bug 4270864: Only non-contig EGM memory needs to specify order. Contig memory
// calculates it within nv_alloc_pages. The long term goal is to expand the ability
// to request large page size for all of sysmem.
//
if (memdescIsEgm(pMemDesc) && !memdescGetContiguity(pMemDesc, AT_CPU))
{
pageSize = memdescGetPageSize(pMemDesc, AT_GPU);
}
status = nv_alloc_pages(
NV_GET_NV_STATE(pGpu),
osPageCount, // TODO: This call needs to receive the page count param at the requested page size.
pageSize,
memdescGetContiguity(pMemDesc, AT_CPU),
memdescGetCpuCacheAttrib(pMemDesc),
pSys->getProperty(pSys,
PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS),
unencrypted,
nodeId,
memdescGetPteArray(pMemDesc, AT_CPU),
&pMemData);
if (nv && nv->force_dma32_alloc)
nv->force_dma32_alloc = NV_FALSE;
}
if (status != NV_OK)
{
return status;
goto done;
}
// Guest allocated memory is already initialized
if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
{
NV_ASSERT_OK_OR_RETURN(memdescSetAllocSizeFields(pMemDesc, rmPageCount * RM_PAGE_SIZE, RM_PAGE_SIZE));
}
//
@@ -964,7 +1007,7 @@ NV_STATUS osAllocPagesInternal(
if ((pGpu != NULL) && IS_VIRTUAL(pGpu))
NV_ASSERT_OK_OR_RETURN(vgpuUpdateGuestSysmemPfnBitMap(pGpu, pMemDesc, NV_TRUE));
done:
return status;
}
@@ -4242,54 +4285,6 @@ osCountTailPages
return os_count_tail_pages(pAddress);
}
/*
* @brief Upon success, gets NPU register address range.
*
* @param[in] pOsGpuInfo OS specific GPU information pointer
* @param[out] pBase base (physical) of NPU register address range
* @param[out] pSize size of NPU register address range
*/
NV_STATUS
osGetIbmnpuGenregInfo
(
OS_GPU_INFO *pOsGpuInfo,
NvU64 *pBase,
NvU64 *pSize
)
{
return nv_get_ibmnpu_genreg_info(pOsGpuInfo, pBase, pSize, NULL);
}
/*
* @brief Upon success, gets NPU's relaxed ordering mode.
*
* @param[in] pOsGpuInfo OS specific GPU information pointer
* @param[out] pMode relaxed ordering mode
*/
NV_STATUS
osGetIbmnpuRelaxedOrderingMode
(
OS_GPU_INFO *pOsGpuInfo,
NvBool *pMode
)
{
return nv_get_ibmnpu_relaxed_ordering_mode(pOsGpuInfo, pMode);
}
/*
* @brief Waits for NVLink HW flush on an NPU associated with a GPU.
*
* @param[in] pOsGpuInfo OS specific GPU information pointer
*/
void
osWaitForIbmnpuRsync
(
OS_GPU_INFO *pOsGpuInfo
)
{
nv_wait_for_ibmnpu_rsync(pOsGpuInfo);
}
NvU64
osGetPageSize(void)
{
@@ -5038,7 +5033,7 @@ osReadPFPciConfigInVF
* components of the tegra_bpmp_message struct, which BPMP uses to receive
* MRQs.
*
* @param[in] pOsGpuInfo OS specific GPU information pointer
* @param[in] pGpu OBJGPU pointer
* @param[in] mrq MRQ_xxx ID specifying what is requested
* @param[in] pRequestData Pointer to request input data
* @param[in] requestDataSize Size of structure pointed to by pRequestData
@@ -5058,7 +5053,7 @@ osReadPFPciConfigInVF
NV_STATUS
osTegraSocBpmpSendMrq
(
OS_GPU_INFO *pOsGpuInfo,
OBJGPU *pGpu,
NvU32 mrq,
const void *pRequestData,
NvU32 requestDataSize,

198
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -50,6 +50,7 @@
#include <core/thread_state.h>
#include <platform/acpi_common.h>
#include <core/locks.h>
#include "platform/sli/sli.h"
#include <mem_mgr/p2p.h>
@@ -207,7 +208,16 @@ const NvU8 * RmGetGpuUuidRaw(
}
else
{
rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid);
if (!pNv->nv_uuid_cache.pci_uuid_read_attempted)
{
rmStatus = pciPbiReadUuid(pNv->handle, pNv->nv_uuid_cache.uuid);
pNv->nv_uuid_cache.pci_uuid_read_attempted = NV_TRUE;
pNv->nv_uuid_cache.pci_uuid_status = rmStatus;
}
else
{
rmStatus = pNv->nv_uuid_cache.pci_uuid_status;
}
}
if (rmStatus == NV_OK)
@@ -293,17 +303,11 @@ static NV_STATUS RmGpuUuidRawToString(
}
// This function should be called with the API and GPU locks already acquired.
NV_STATUS
void
RmLogGpuCrash(OBJGPU *pGpu)
{
NV_STATUS status = NV_OK;
NvBool bGpuIsLost, bGpuIsConnected;
if (pGpu == NULL)
{
return NV_ERR_INVALID_ARGUMENT;
}
//
// Re-evaluate whether or not the GPU is accessible. This could be called
// from a recovery context where the OS has re-enabled MMIO for the device.
@@ -325,31 +329,32 @@ RmLogGpuCrash(OBJGPU *pGpu)
}
}
//
// Log the engine data to the Journal object, to be pulled out later. This
// will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be
// deferred to a passive IRQL. We still log the crash dump as being created
// in that case since it (should) be created shortly thereafter, and
// there's currently not a good way to print the below notification
// publicly from the core RM when the DPC completes.
//
status = rcdbAddRmGpuDump(pGpu);
if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED)
{
NV_PRINTF(LEVEL_ERROR,
"%s: failed to save GPU crash data\n", __FUNCTION__);
}
else
{
status = NV_OK;
nv_printf(NV_DBG_ERRORS,
"NVRM: A GPU crash dump has been created. If possible, please run\n"
"NVRM: nvidia-bug-report.sh as root to collect this data before\n"
"NVRM: the NVIDIA kernel module is unloaded.\n");
if (hypervisorIsVgxHyper())
//
// Log the engine data to the Journal object, to be pulled out later. This
// will return NV_WARN_MORE_PROCESSING_REQUIRED if the dump needed to be
// deferred to a passive IRQL. We still log the crash dump as being created
// in that case since it (should) be created shortly thereafter, and
// there's currently not a good way to print the below notification
// publicly from the core RM when the DPC completes.
//
NV_STATUS status = rcdbAddRmGpuDump(pGpu);
if (status != NV_OK && status != NV_WARN_MORE_PROCESSING_REQUIRED)
{
nv_printf(NV_DBG_ERRORS, "NVRM: Dumping nvlogs buffers\n");
nvlogDumpToKernelLog(NV_FALSE);
NV_PRINTF(LEVEL_ERROR,
"%s: failed to save GPU crash data\n", __FUNCTION__);
}
else
{
nv_printf(NV_DBG_ERRORS,
"NVRM: A GPU crash dump has been created. If possible, please run\n"
"NVRM: nvidia-bug-report.sh as root to collect this data before\n"
"NVRM: the NVIDIA kernel module is unloaded.\n");
if (hypervisorIsVgxHyper())
{
nv_printf(NV_DBG_ERRORS, "NVRM: Dumping nvlogs buffers\n");
nvlogDumpToKernelLog(NV_FALSE);
}
}
}
@@ -360,8 +365,6 @@ RmLogGpuCrash(OBJGPU *pGpu)
// Restore persistence mode to the way it was prior to the crash
osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo,
pGpu->getProperty(pGpu, PDB_PROP_GPU_PERSISTENT_SW_STATE));
return status;
}
static void free_os_event_under_lock(nv_event_t *event)
@@ -1353,6 +1356,7 @@ RmDmabufGetClientAndDevice(
OBJGPU *pGpu,
NvHandle hClient,
NvHandle hMemory,
NvU8 mappingType,
NvHandle *phClient,
NvHandle *phDevice,
NvHandle *phSubdevice,
@@ -1361,6 +1365,29 @@ RmDmabufGetClientAndDevice(
{
MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
// No dma-buf support for SLI-enabled GPU
if (IsSLIEnabled(pGpu))
{
return NV_ERR_NOT_SUPPORTED;
}
//
// MAPPING_TYPE_FORCE_PCIE is to be used only on coherent systems with a
// direct PCIe connection between the exporter and importer.
// MIG is not a supported use-case for dma-buf on these systems.
//
if (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE)
{
KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
pKernelBus->bBar1Disabled ||
IS_MIG_ENABLED(pGpu))
{
return NV_ERR_NOT_SUPPORTED;
}
}
if (IS_MIG_ENABLED(pGpu))
{
MIG_INSTANCE_REF ref;
@@ -3772,6 +3799,7 @@ static NV_STATUS RmNonDPAuxI2CTransfer
case NV_I2C_CMD_SMBUS_BLOCK_WRITE:
if (pData[0] >= len) {
portMemFree(params);
return NV_ERR_INVALID_ARGUMENT;
}
params->transData.smbusBlockData.bWrite = NV_TRUE;
@@ -4504,6 +4532,7 @@ NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
void **p2pObject,
NvU64 *pPhysicalAddresses,
NvU32 *pEntries,
NvBool bForcePcie,
void *pPlatformData,
void *pGpuInfo,
void **ppMigInfo
@@ -4524,6 +4553,7 @@ NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
p2pObject,
pPhysicalAddresses,
pEntries,
bForcePcie,
pPlatformData,
pGpuInfo,
ppMigInfo);
@@ -4927,38 +4957,6 @@ void NV_API_CALL rm_disable_gpu_state_persistence(nvidia_stack_t *sp, nv_state_t
NV_EXIT_RM_RUNTIME(sp,fp);
}
NV_STATUS NV_API_CALL rm_log_gpu_crash(
nv_stack_t *sp,
nv_state_t *nv
)
{
THREAD_STATE_NODE threadState;
NV_STATUS status;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
if ((status = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK)
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv);
if ((pGpu != NULL) &&
((status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DIAG)) == NV_OK))
{
status = RmLogGpuCrash(pGpu);
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
}
rmapiLockRelease();
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
return status;
}
void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd)
{
nvidia_kernel_rmapi_ops_t *ops = ops_cmd;
@@ -5427,35 +5425,6 @@ NvU64 rm_get_uefi_console_size(
return fbSize;
}
/*
* IOMMU needs to be present on the server to support SR-IOV vGPU, unless
* we have SR-IOV enabled for remote GPU.
*/
NvBool NV_API_CALL rm_is_iommu_needed_for_sriov(
nvidia_stack_t *sp,
nv_state_t * nv
)
{
OBJGPU *pGpu;
NvU32 data;
NvBool ret = NV_TRUE;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
pGpu = NV_GET_NV_PRIV_PGPU(nv);
if (osReadRegistryDword(pGpu, NV_REG_STR_RM_REMOTE_GPU, &data) == NV_OK)
{
if (data == NV_REG_STR_RM_REMOTE_GPU_ENABLE)
ret = NV_FALSE;
}
NV_EXIT_RM_RUNTIME(sp,fp);
return ret;
}
NvBool NV_API_CALL rm_disable_iomap_wc(void)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
@@ -5577,8 +5546,9 @@ void NV_API_CALL rm_dma_buf_undup_mem_handle(
//
// Maps a handle to system physical addresses:
// C2C for coherent platforms
// BAR1(static & dynamic) for non-coherent platforms
// C2C for coherent platforms with DEFAULT mapping type
// BAR1(static & dynamic) for non-coherent platforms and for
// coherent platforms with mapping type FORCE_PCIE
// Must be called with API lock and GPU lock held for dynamic BAR1.
//
NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
@@ -5587,6 +5557,7 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
NvHandle hClient,
NvHandle hMemory,
MemoryRange memRange,
NvU8 mappingType,
void *pMemInfo,
NvBool bStaticPhysAddrs,
MemoryArea *pMemArea
@@ -5608,8 +5579,9 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
pGpu = NV_GET_NV_PRIV_PGPU(nv);
pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo;
if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) ||
(pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)))
if (((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) ||
(pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB))) &&
(mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT))
{
KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
NvBool contiguity = memdescCheckContiguity(pMemDesc, AT_CPU);
@@ -5660,6 +5632,7 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
NvU64 idx;
NvU64 barOffset;
KernelBus *pKernelBus;
NvBool bForcePcie;
pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
@@ -5693,7 +5666,12 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle(
pDevice),
Done);
barOffset = gpumgrGetGpuPhysFbAddr(pGpu);
bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE);
NV_ASSERT_OK_OR_GOTO(rmStatus,
kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus,
bForcePcie, &barOffset),
Done);
for (idx = 0; idx < pMemArea->numRanges; idx++)
{
@@ -5717,6 +5695,7 @@ void NV_API_CALL rm_dma_buf_unmap_mem_handle(
nv_state_t *nv,
NvHandle hClient,
NvHandle hMemory,
NvU8 mappingType,
void *pMemInfo,
NvBool bStaticPhysAddrs,
MemoryArea memArea
@@ -5731,8 +5710,9 @@ void NV_API_CALL rm_dma_buf_unmap_mem_handle(
pGpu = NV_GET_NV_PRIV_PGPU(nv);
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB))
if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)) &&
(mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT))
{
os_free_mem(memArea.pRanges);
}
@@ -5742,9 +5722,12 @@ void NV_API_CALL rm_dma_buf_unmap_mem_handle(
NvU64 idx;
NvU64 barOffset;
MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR *) pMemInfo;
NvBool bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE);
pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
barOffset = gpumgrGetGpuPhysFbAddr(pGpu);
NV_ASSERT_OK(kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus,
bForcePcie, &barOffset));
for (idx = 0; idx < memArea.numRanges; idx++)
{
@@ -5776,6 +5759,7 @@ NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(
nv_state_t *nv,
NvHandle hClient,
NvHandle hMemory,
NvU8 mappingType,
NvHandle *phClient,
NvHandle *phDevice,
NvHandle *phSubdevice,
@@ -5799,15 +5783,15 @@ NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(
rmStatus = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_OSAPI);
if (rmStatus == NV_OK)
{
rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, hMemory,
rmStatus = RmDmabufGetClientAndDevice(pGpu, hClient, hMemory, mappingType,
phClient, phDevice,
phSubdevice, ppGpuInstanceInfo);
if (rmStatus == NV_OK)
{
*pbStaticPhysAddrs = ((pGpu->getProperty(pGpu,
PDB_PROP_GPU_COHERENT_CPU_MAPPING)) ||
kbusIsStaticBar1Enabled(pGpu, GPU_GET_KERNEL_BUS(pGpu)) ||
(pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)));
*pbStaticPhysAddrs = ((pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) ||
pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)) &&
(mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) ||
kbusIsStaticBar1Enabled(pGpu, GPU_GET_KERNEL_BUS(pGpu));
}
rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);

39
src/nvidia/arch/nvalloc/unix/src/osinit.c
View File

@@ -386,26 +386,8 @@ osHandleGpuLost
kgspRcAndNotifyAllChannels(pGpu, pKernelGsp, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, NV_FALSE);
}
// Trigger the OS's PCI recovery mechanism
if (nv_pci_trigger_recovery(nv) != NV_OK)
{
//
// Initiate a crash dump immediately, since the OS doesn't appear
// to have a mechanism wired up for attempted recovery.
//
(void) RmLogGpuCrash(pGpu);
}
else
{
//
// Make the SW state stick around until the recovery can start, but
// don't change the PDB property: this is only used to report to
// clients whether or not persistence mode is enabled, and we'll
// need it after the recovery callbacks to restore the correct
// persistence mode for the GPU.
//
osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_TRUE);
}
// Initiate a crash dump immediately.
RmLogGpuCrash(pGpu);
// Set SURPRISE_REMOVAL flag for eGPU to help in device removal.
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_EXTERNAL_GPU))
@@ -732,6 +714,13 @@ osInitNvMapping(
gpuAttachArg->iovaspaceId = nv->iovaspace_id;
gpuAttachArg->cpuNumaNodeId = nv->cpu_numa_node_id;
if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
{
// Default - PCIe GPUs are connected via NISO IOMMU
nv->iommus.iso_iommu_present = NV_FALSE;
nv->iommus.niso_iommu_present = NV_TRUE;
}
}
//
@@ -774,7 +763,6 @@ osInitNvMapping(
//
nv_set_dma_address_size(nv, gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM));
pGpu->dmaStartAddress = (RmPhysAddr)nv_get_dma_start_address(nv);
if (nv->fb != NULL)
{
pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map;
@@ -836,6 +824,11 @@ osInitNvMapping(
{
pGpu->setProperty(pGpu, PDB_PROP_GPU_DISP_PB_REQUIRES_SMMU_BYPASS, NV_TRUE);
}
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TRIGGER_PCIE_FLR))
{
nv->flags |= NV_FLAG_TRIGGER_FLR;
}
}
void osInitScalabilityOptions
@@ -1407,6 +1400,7 @@ NvBool RmInitPrivateState(
nvp->pmc_boot_0 = pmc_boot_0;
nvp->pmc_boot_1 = pmc_boot_1;
nvp->pmc_boot_42 = pmc_boot_42;
nvp->db_supported = -1;
NV_SET_NV_PRIV(pNv, nvp);
return NV_TRUE;
@@ -1426,6 +1420,7 @@ void RmClearPrivateState(
NvU32 x = 0;
NvU32 pmc_boot_0, pmc_boot_1, pmc_boot_42;
NvBool pr3_acpi_method_present = 0;
int db_supported;
//
// Do not clear private state after GPU resets, it is used while
@@ -1446,6 +1441,7 @@ void RmClearPrivateState(
pmc_boot_1 = nvp->pmc_boot_1;
pmc_boot_42 = nvp->pmc_boot_42;
pr3_acpi_method_present = nvp->pr3_acpi_method_present;
db_supported = nvp->db_supported;
for (x = 0; x < MAX_I2C_ADAPTERS; x++)
{
@@ -1463,6 +1459,7 @@ void RmClearPrivateState(
nvp->pmc_boot_1 = pmc_boot_1;
nvp->pmc_boot_42 = pmc_boot_42;
nvp->pr3_acpi_method_present = pr3_acpi_method_present;
nvp->db_supported = db_supported;
for (x = 0; x < MAX_I2C_ADAPTERS; x++)
{

16
src/nvidia/arch/nvalloc/unix/src/osmemdesc.c
View File

@@ -1030,14 +1030,22 @@ osDestroyOsDescriptorPageArray
nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount,
NULL /* import_priv */, &pPrivate);
if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE) == NV_FALSE)
if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE))
{
status = os_unlock_user_pages(osPageCount, pPrivate);
NV_ASSERT(status == NV_OK);
os_free_mem(pPrivate);
}
else
{
os_free_mem(pPrivate);
//
// We use MEMDESC_FLAGS_USER_READ_ONLY because this reflects the
// NVOS02_FLAGS_ALLOC_USER_READ_ONLY flag value passed into
// os_lock_user_pages(). That flag also results in
// MEMDESC_FLAGS_DEVICE_READ_ONLY being set.
//
NvBool writable = !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_USER_READ_ONLY);
NvU32 flags = DRF_NUM(_LOCK_USER_PAGES, _FLAGS, _WRITE, writable);
status = os_unlock_user_pages(osPageCount, pPrivate, flags);
NV_ASSERT(status == NV_OK);
}
}

27
src/nvidia/arch/nvalloc/unix/src/osnvlink.c
View File

@@ -778,31 +778,6 @@ osGetPlatformNvlinkLinerate
NvU32 *lineRate
)
{
#if defined(NVCPU_PPC64LE)
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
if (!pKernelNvlink)
return NV_ERR_INVALID_ARGUMENT;
return nv_get_nvlink_line_rate(nv, lineRate);
#else
//TODO : FRU based method to be filled out by Bug 200285656
//*lineRate = 0;
//return NV_OK;
// TODO: Remove this function
return NV_ERR_NOT_SUPPORTED;
#endif
}
void
osSetNVLinkSysmemLinkState
(
OBJGPU *pGpu,
NvBool enabled
)
{
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
NV_ASSERT(enabled);
if (enabled)
nv_dma_enable_nvlink(nv->dma_dev);
}

4
src/nvidia/arch/nvalloc/unix/src/unix_console.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -370,6 +370,8 @@ NV_STATUS deviceCtrlCmdOsUnixVTGetFBInfo_IMPL
nv_get_screen_info(nv, &baseAddr, &width, &height, &depth,
&pitch, &size);
pParams->baseAddress = baseAddr;
pParams->size = size;
pParams->width = (NvU16)width;
pParams->height = (NvU16)height;
pParams->depth = (NvU16)depth;

216
src/nvidia/arch/nvalloc/unix/src/unix_intr.c
View File

@@ -35,6 +35,96 @@
#include "gpu/disp/head/kernel_head.h"
#include <nv_sriov_defines.h>
#include "platform/sli/sli.h"
//
// Service Interrupts that are not Masked when using
// INTR_USE_INTR_MASK_FOR_LOCKING
// Note that everything done here is without holding the GPUs Lock.
// Returning NV_OK means an interrupt has been serviced
//
static NV_STATUS _osIsrIntrMask_GpusUnlocked(OBJGPU *pGpu, THREAD_STATE_NODE *pThreadState)
{
NV_STATUS rmStatus = NV_ERR_NO_INTR_PENDING;
Intr *pIntr = GPU_GET_INTR(pGpu);
MC_ENGINE_BITVECTOR intrPending, intrEngMask;
NvU32 vblankIntrServicedHeadMask = 0;
KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
if (gpuIsGpuFullPower(pGpu) == NV_FALSE)
{
return rmStatus;
}
intrGetPendingDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState);
// Only disp interrupt remains in INTR_MASK
if ((pKernelDisplay != NULL) &&
(( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP_LOW)) ||
(!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP))))
{
NV_ASSERT_OR_RETURN(pKernelDisplay->pLowLatencySpinLock != NULL, NV_ERR_INVALID_STATE);
portSyncSpinlockAcquire(pKernelDisplay->pLowLatencySpinLock);
kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
(VBLANK_STATE_PROCESS_LOW_LATENCY |
VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
pThreadState,
&vblankIntrServicedHeadMask,
&intrPending);
portSyncSpinlockRelease(pKernelDisplay->pLowLatencySpinLock);
//
// If an unmasked intr is still pending after the above, that means a
// intr other than the vblank, so we'll MSK this off
// for now and handle once the GPUs Lock is released.
//
intrGetPendingDisplayIntr_HAL(pGpu, pIntr, &intrPending, pThreadState);
// Only disp interrupt remains in INTR_MASK
if (!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intrPending, MC_ENGINE_IDX_DISP))
{
NvU32 intrMaskFlags;
NvU64 oldIrql;
oldIrql = rmIntrMaskLockAcquire(pGpu);
// Skip the IntrMask update if told to by another thread.
intrMaskFlags = intrGetIntrMaskFlags(pIntr);
if ((intrMaskFlags & INTR_MASK_FLAGS_ISR_SKIP_MASK_UPDATE) == 0)
{
if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING))
{
intrGetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState);
// Only disp interrupt remains in INTR_MASK
bitVectorClr(&intrEngMask, MC_ENGINE_IDX_DISP);
if (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_TOP_EN_FOR_VBLANK_HANDLING))
{
intrSetDisplayInterruptEnable_HAL(pGpu, pIntr, NV_FALSE, pThreadState);
}
else
{
intrSetIntrMask_HAL(pGpu, pIntr, &intrEngMask, pThreadState);
}
}
}
rmIntrMaskLockRelease(pGpu, oldIrql);
}
if (vblankIntrServicedHeadMask)
{
rmStatus = NV_OK;
}
}
return rmStatus;
}
static NvBool osInterruptPending(
OBJGPU *pGpu,
NvBool *serviced,
@@ -124,7 +214,7 @@ static NvBool osInterruptPending(
//
NvBool bIsAnyStallIntrPending = NV_TRUE;
NvBool bIsAnyBottomHalfStallPending = NV_FALSE;
NvU32 isDispPendingPerGpu = 0;
NvU32 isDispPendingPerGpu = 0, isDispLowLatencyPendingPerGpu = 0;
NvU32 isTmrPendingPerGpu = 0;
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
@@ -138,6 +228,8 @@ static NvBool osInterruptPending(
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pIntr = GPU_GET_INTR(pGpu);
KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu);
if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)))
{
NvBool bCtxswLog = NV_FALSE;
@@ -158,13 +250,26 @@ static NvBool osInterruptPending(
//
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
if (!bitVectorTestAllCleared(&intr0Pending))
{
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
{
if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE))
{
isDispPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW))
{
isDispLowLatencyPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
// We will attempt to handle this separately from the other stall interrupts
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
// We will attempt to handle this separately from the other stall interrupts
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
}
}
else
{
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
{
isDispPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
// We will attempt to handle this separately from the other stall interrupts
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
}
}
if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
@@ -233,9 +338,21 @@ static NvBool osInterruptPending(
pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
bitVectorClrAll(&intr0Pending);
if ((isDispPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
if ((pKernelDisplay != NULL) && pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE))
{
bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP);
if ((isDispLowLatencyPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
{
bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
}
}
else
{
if ((isDispPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
{
bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP);
}
}
if ((isTmrPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
@@ -248,23 +365,43 @@ static NvBool osInterruptPending(
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
}
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
if ((pKernelDisplay != NULL) &&
(( pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP_LOW)) ||
(!pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))))
{
if (pKernelDisplay != NULL)
{
MC_ENGINE_BITVECTOR intrDispPending;
MC_ENGINE_BITVECTOR intrDispPending;
NvBool bIsLowLatencyIntrPending;
kdispServiceVblank_HAL(pGpu, pKernelDisplay, 0,
(VBLANK_STATE_PROCESS_LOW_LATENCY |
VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
&threadState);
NV_ASSERT_OR_RETURN(pKernelDisplay->pLowLatencySpinLock != NULL,
NV_ERR_INVALID_STATE);
*serviced = NV_TRUE;
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
portSyncSpinlockAcquire(pKernelDisplay->pLowLatencySpinLock);
intrGetPendingDisplayIntr_HAL(pGpu, pIntr, &intrDispPending, &threadState);
bitVectorOr(&intr0Pending, &intr0Pending, &intrDispPending);
// If this is true, then we're here because it is true
bIsLowLatencyIntrPending = pKernelDisplay->getProperty(pKernelDisplay, PDB_PROP_KDISP_HAS_SEPARATE_LOW_LATENCY_LINE);
kdispServiceLowLatencyIntrs_HAL(pGpu, pKernelDisplay, 0,
(VBLANK_STATE_PROCESS_LOW_LATENCY |
VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
&threadState,
NULL,
&intr0Pending);
portSyncSpinlockRelease(pKernelDisplay->pLowLatencySpinLock);
*serviced = NV_TRUE;
if (bIsLowLatencyIntrPending)
{
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP_LOW);
}
else
{
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
}
intrGetPendingDisplayIntr_HAL(pGpu, pIntr, &intrDispPending, &threadState);
bitVectorOr(&intr0Pending, &intr0Pending, &intrDispPending);
}
if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
@@ -320,6 +457,21 @@ static NvBool osInterruptPending(
threadStateFreeISRAndDeferredIntHandler(&threadState,
pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR);
}
else if (bIsAnyStallIntrPending)
{
pIntr = GPU_GET_INTR(pDeviceLockGpu);
if ((pIntr != NULL) && (pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
if (_osIsrIntrMask_GpusUnlocked(pDeviceLockGpu, &threadState) == NV_OK)
{
*serviced = NV_TRUE;
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
}
tlsIsrDestroy(pIsrAllocator);
portMemAllocatorRelease(pIsrAllocator);
@@ -461,7 +613,6 @@ static void RmIsrBottomHalf(
)
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
THREAD_STATE_NODE threadState;
OS_THREAD_HANDLE threadId;
NvU32 gpuMask, gpuInstance;
OBJGPU *pDeviceLockGpu = pGpu;
@@ -482,13 +633,18 @@ static void RmIsrBottomHalf(
rmDeviceGpuLockSetOwner(pDeviceLockGpu, threadId);
gpuMask = gpumgrGetGpuMask(pGpu);
gpuInstance = 0;
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
threadStateInitISRAndDeferredIntHandler(pGpu->pDpcThreadState, pGpu,
THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
}
threadStateInitISRAndDeferredIntHandler(&threadState,
pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
gpuInstance = 0;
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pIntr = GPU_GET_INTR(pGpu);
pDisp = GPU_GET_DISP(pGpu);
@@ -509,25 +665,17 @@ static void RmIsrBottomHalf(
!pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
MC_ENGINE_BITVECTOR intrPending;
intrServiceNonStall_HAL(pGpu, pIntr, &intrPending, &threadState);
intrServiceNonStall_HAL(pGpu, pIntr, &intrPending, pGpu->pDpcThreadState);
}
}
threadStateFreeISRAndDeferredIntHandler(&threadState,
pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
}
gpuInstance = 0;
pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
threadStateInitISRAndDeferredIntHandler(&threadState,
pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
// UNLOCK: release GPUs lock
rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL);
threadStateFreeISRAndDeferredIntHandler(&threadState,
pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER);
rmDeviceGpuLocksReleaseAndThreadStateFreeDeferredIntHandlerOptimized(pGpu, GPUS_LOCK_FLAGS_NONE, NULL);
tlsIsrDestroy(pIsrAllocator);
portMemAllocatorRelease(pIsrAllocator);