NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-07 08:39:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

555.42.02

Browse Source 
(cherry picked from commit 5a1c474040)
This commit is contained in:
Bernhard Stoeckner
2024-05-21 15:11:46 +02:00
committed by Gaurav Juvekar
parent caa2dd11a0
commit 3084c04453
1004 changed files with 172522 additions and 150960 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/nvidia/arch/nvalloc/common/inc/conf_compute/cc_keyrotation.h
View File

@@ -1,34 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef CC_KEYROTATION_H
#define CC_KEYROTATION_H
//
// Default threshold value derived from SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT
// Minimum threshold defined based on minimum in confComputeSetKeyRotation.
//
#define KEY_ROTATION_MINIMUM_INTERNAL_THRESHOLD (134217727u)
#define KEY_ROTATION_DEFAULT_INTERNAL_THRESHOLD (24296003999ull)
#endif // CC_KEYROTATION_H

118
src/nvidia/arch/nvalloc/common/inc/conf_compute/cc_keystore.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -55,8 +55,8 @@ enum
CC_LKEYID_GSP_CPU_REPLAYABLE_FAULT,
CC_LKEYID_CPU_GSP_RESERVED2,
CC_LKEYID_GSP_CPU_NON_REPLAYABLE_FAULT,
CC_LKEYID_SEC2_GSP_LOCKED_RPC,
CC_LKEYID_GSP_SEC2_LOCKED_RPC,
CC_LKEYID_SEC2_GSP_LOCKED_RPC,
CC_KEYSPACE_GSP_SIZE // This is always the last element.
};
// The fault buffers only support GPU-to-CPU encryption, so the CPU-to-GPU encryption slot
@@ -75,17 +75,13 @@ enum
CC_LKEYID_CPU_SEC2_HMAC_USER,
CC_LKEYID_CPU_SEC2_DATA_KERN,
CC_LKEYID_CPU_SEC2_HMAC_KERN,
CC_LKEYID_CPU_SEC2_DATA_SCRUBBER,
CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER,
CC_KEYSPACE_SEC2_SIZE // This is always the last element.
};
#define CC_LKEYID_CPU_SEC2_DATA_USER_STR "cpu_sec2_data_user"
#define CC_LKEYID_CPU_SEC2_HMAC_USER_STR "cpu_sec2_hmac_user"
#define CC_LKEYID_CPU_SEC2_DATA_KERN_STR "cpu_sec2_data_kernel"
#define CC_LKEYID_CPU_SEC2_HMAC_KERN_STR "cpu_sec2_hmac_kernel"
#define CC_LKEYID_CPU_SEC2_DATA_SCRUBBER_STR "cpu_sec2_data_scrubber"
#define CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER_STR "cpu_sec2_hmac_scrubber"
#define CC_LKEYID_CPU_SEC2_DATA_USER_STR "cpu_sec2_data_user"
#define CC_LKEYID_CPU_SEC2_HMAC_USER_STR "cpu_sec2_hmac_user"
#define CC_LKEYID_CPU_SEC2_DATA_KERN_STR "cpu_sec2_data_kernel"
#define CC_LKEYID_CPU_SEC2_HMAC_KERN_STR "cpu_sec2_hmac_kernel"
enum
{
@@ -100,61 +96,61 @@ enum
#define CC_KEYSPACE_TOTAL_SIZE (CC_KEYSPACE_GSP_SIZE + CC_KEYSPACE_SEC2_SIZE + (8 * CC_KEYSPACE_LCE_SIZE))
#define CC_LKEYID_LCE0_H2D_USER_STR "Lce00_h2d_user"
#define CC_LKEYID_LCE0_D2H_USER_STR "Lce00_d2h_user"
#define CC_LKEYID_LCE0_H2D_KERN_STR "Lce00_h2d_kernel"
#define CC_LKEYID_LCE0_D2H_KERN_STR "Lce00_d2h_kernel"
#define CC_LKEYID_LCE0_H2D_P2P_STR "Lce00_h2d_p2p"
#define CC_LKEYID_LCE0_D2H_P2P_STR "Lce00_d2h_p2p"
#define CC_LKEYID_LCE0_H2D_USER_STR "Lce0_h2d_user"
#define CC_LKEYID_LCE0_D2H_USER_STR "Lce0_d2h_user"
#define CC_LKEYID_LCE0_H2D_KERN_STR "Lce0_h2d_kernel"
#define CC_LKEYID_LCE0_D2H_KERN_STR "Lce0_d2h_kernel"
#define CC_LKEYID_LCE0_H2D_P2P_STR "Lce0_h2d_p2p"
#define CC_LKEYID_LCE0_D2H_P2P_STR "Lce0_d2h_p2p"
#define CC_LKEYID_LCE1_H2D_USER_STR "Lce01_h2d_user"
#define CC_LKEYID_LCE1_D2H_USER_STR "Lce01_d2h_user"
#define CC_LKEYID_LCE1_H2D_KERN_STR "Lce01_h2d_kernel"
#define CC_LKEYID_LCE1_D2H_KERN_STR "Lce01_d2h_kernel"
#define CC_LKEYID_LCE1_H2D_P2P_STR "Lce01_h2d_p2p"
#define CC_LKEYID_LCE1_D2H_P2P_STR "Lce01_d2h_p2p"
#define CC_LKEYID_LCE1_H2D_USER_STR "Lce1_h2d_user"
#define CC_LKEYID_LCE1_D2H_USER_STR "Lce1_d2h_user"
#define CC_LKEYID_LCE1_H2D_KERN_STR "Lce1_h2d_kernel"
#define CC_LKEYID_LCE1_D2H_KERN_STR "Lce1_d2h_kernel"
#define CC_LKEYID_LCE1_H2D_P2P_STR "Lce1_h2d_p2p"
#define CC_LKEYID_LCE1_D2H_P2P_STR "Lce1_d2h_p2p"
#define CC_LKEYID_LCE2_H2D_USER_STR "Lce02_h2d_user"
#define CC_LKEYID_LCE2_D2H_USER_STR "Lce02_d2h_user"
#define CC_LKEYID_LCE2_H2D_KERN_STR "Lce02_h2d_kernel"
#define CC_LKEYID_LCE2_D2H_KERN_STR "Lce02_d2h_kernel"
#define CC_LKEYID_LCE2_H2D_P2P_STR "Lce02_h2d_p2p"
#define CC_LKEYID_LCE2_D2H_P2P_STR "Lce02_d2h_p2p"
#define CC_LKEYID_LCE2_H2D_USER_STR "Lce2_h2d_user"
#define CC_LKEYID_LCE2_D2H_USER_STR "Lce2_d2h_user"
#define CC_LKEYID_LCE2_H2D_KERN_STR "Lce2_h2d_kernel"
#define CC_LKEYID_LCE2_D2H_KERN_STR "Lce2_d2h_kernel"
#define CC_LKEYID_LCE2_H2D_P2P_STR "Lce2_h2d_p2p"
#define CC_LKEYID_LCE2_D2H_P2P_STR "Lce2_d2h_p2p"
#define CC_LKEYID_LCE3_H2D_USER_STR "Lce03_h2d_user"
#define CC_LKEYID_LCE3_D2H_USER_STR "Lce03_d2h_user"
#define CC_LKEYID_LCE3_H2D_KERN_STR "Lce03_h2d_kernel"
#define CC_LKEYID_LCE3_D2H_KERN_STR "Lce03_d2h_kernel"
#define CC_LKEYID_LCE3_H2D_P2P_STR "Lce03_h2d_p2p"
#define CC_LKEYID_LCE3_D2H_P2P_STR "Lce03_d2h_p2p"
#define CC_LKEYID_LCE3_H2D_USER_STR "Lce3_h2d_user"
#define CC_LKEYID_LCE3_D2H_USER_STR "Lce3_d2h_user"
#define CC_LKEYID_LCE3_H2D_KERN_STR "Lce3_h2d_kernel"
#define CC_LKEYID_LCE3_D2H_KERN_STR "Lce3_d2h_kernel"
#define CC_LKEYID_LCE3_H2D_P2P_STR "Lce3_h2d_p2p"
#define CC_LKEYID_LCE3_D2H_P2P_STR "Lce3_d2h_p2p"
#define CC_LKEYID_LCE4_H2D_USER_STR "Lce04_h2d_user"
#define CC_LKEYID_LCE4_D2H_USER_STR "Lce04_d2h_user"
#define CC_LKEYID_LCE4_H2D_KERN_STR "Lce04_h2d_kernel"
#define CC_LKEYID_LCE4_D2H_KERN_STR "Lce04_d2h_kernel"
#define CC_LKEYID_LCE4_H2D_P2P_STR "Lce04_h2d_p2p"
#define CC_LKEYID_LCE4_D2H_P2P_STR "Lce04_d2h_p2p"
#define CC_LKEYID_LCE4_H2D_USER_STR "Lce4_h2d_user"
#define CC_LKEYID_LCE4_D2H_USER_STR "Lce4_d2h_user"
#define CC_LKEYID_LCE4_H2D_KERN_STR "Lce4_h2d_kernel"
#define CC_LKEYID_LCE4_D2H_KERN_STR "Lce4_d2h_kernel"
#define CC_LKEYID_LCE4_H2D_P2P_STR "Lce4_h2d_p2p"
#define CC_LKEYID_LCE4_D2H_P2P_STR "Lce4_d2h_p2p"
#define CC_LKEYID_LCE5_H2D_USER_STR "Lce05_h2d_user"
#define CC_LKEYID_LCE5_D2H_USER_STR "Lce05_d2h_user"
#define CC_LKEYID_LCE5_H2D_KERN_STR "Lce05_h2d_kernel"
#define CC_LKEYID_LCE5_D2H_KERN_STR "Lce05_d2h_kernel"
#define CC_LKEYID_LCE5_H2D_P2P_STR "Lce05_h2d_p2p"
#define CC_LKEYID_LCE5_D2H_P2P_STR "Lce05_d2h_p2p"
#define CC_LKEYID_LCE5_H2D_USER_STR "Lce5_h2d_user"
#define CC_LKEYID_LCE5_D2H_USER_STR "Lce5_d2h_user"
#define CC_LKEYID_LCE5_H2D_KERN_STR "Lce5_h2d_kernel"
#define CC_LKEYID_LCE5_D2H_KERN_STR "Lce5_d2h_kernel"
#define CC_LKEYID_LCE5_H2D_P2P_STR "Lce5_h2d_p2p"
#define CC_LKEYID_LCE5_D2H_P2P_STR "Lce5_d2h_p2p"
#define CC_LKEYID_LCE6_H2D_USER_STR "Lce06_h2d_user"
#define CC_LKEYID_LCE6_D2H_USER_STR "Lce06_d2h_user"
#define CC_LKEYID_LCE6_H2D_KERN_STR "Lce06_h2d_kernel"
#define CC_LKEYID_LCE6_D2H_KERN_STR "Lce06_d2h_kernel"
#define CC_LKEYID_LCE6_H2D_P2P_STR "Lce06_h2d_p2p"
#define CC_LKEYID_LCE6_D2H_P2P_STR "Lce06_d2h_p2p"
#define CC_LKEYID_LCE6_H2D_USER_STR "Lce6_h2d_user"
#define CC_LKEYID_LCE6_D2H_USER_STR "Lce6_d2h_user"
#define CC_LKEYID_LCE6_H2D_KERN_STR "Lce6_h2d_kernel"
#define CC_LKEYID_LCE6_D2H_KERN_STR "Lce6_d2h_kernel"
#define CC_LKEYID_LCE6_H2D_P2P_STR "Lce6_h2d_p2p"
#define CC_LKEYID_LCE6_D2H_P2P_STR "Lce6_d2h_p2p"
#define CC_LKEYID_LCE7_H2D_USER_STR "Lce07_h2d_user"
#define CC_LKEYID_LCE7_D2H_USER_STR "Lce07_d2h_user"
#define CC_LKEYID_LCE7_H2D_KERN_STR "Lce07_h2d_kernel"
#define CC_LKEYID_LCE7_D2H_KERN_STR "Lce07_d2h_kernel"
#define CC_LKEYID_LCE7_H2D_P2P_STR "Lce07_h2d_p2p"
#define CC_LKEYID_LCE7_D2H_P2P_STR "Lce07_d2h_p2p"
#define CC_LKEYID_LCE7_H2D_USER_STR "Lce7_h2d_user"
#define CC_LKEYID_LCE7_D2H_USER_STR "Lce7_d2h_user"
#define CC_LKEYID_LCE7_H2D_KERN_STR "Lce7_h2d_kernel"
#define CC_LKEYID_LCE7_D2H_KERN_STR "Lce7_d2h_kernel"
#define CC_LKEYID_LCE7_H2D_P2P_STR "Lce7_h2d_p2p"
#define CC_LKEYID_LCE7_D2H_P2P_STR "Lce7_d2h_p2p"
// Generate a global key ID from a keyspace (a) and local key ID (b).
#define CC_GKEYID_GEN(a, b) (NvU32)(((a) << 16) | (b))
@@ -192,11 +188,7 @@ enum
(CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_DATA_KERN) ? \
CC_LKEYID_CPU_SEC2_DATA_KERN_STR : \
(CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_HMAC_KERN) ? \
CC_LKEYID_CPU_SEC2_HMAC_KERN_STR : \
(CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_DATA_SCRUBBER) ? \
CC_LKEYID_CPU_SEC2_DATA_SCRUBBER_STR : \
(CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER) ? \
CC_LKEYID_CPU_SEC2_HMAC_SCRUBBER_STR : NULL : \
CC_LKEYID_CPU_SEC2_HMAC_KERN_STR : NULL : \
(CC_GKEYID_GET_KEYSPACE(a) == CC_KEYSPACE_LCE0) ? \
(CC_GKEYID_GET_LKEYID(a) == CC_LKEYID_LCE_H2D_USER) ? \
CC_LKEYID_LCE0_H2D_USER_STR : \

5
src/nvidia/arch/nvalloc/common/inc/dev_ctrl_defines.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -77,9 +77,6 @@
#define NV_CTRL_INTR_GPU_VECTOR_TO_SUBTREE(i) \
((NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(i)) / 2)
// First index of doorbell which is controlled by VF
#define NV_CTRL_INTR_GPU_DOORBELL_INDEX_VF_START 2048
// The max number of leaf registers we expect
#define NV_MAX_INTR_LEAVES 16

6
src/nvidia/arch/nvalloc/common/inc/flcnifcmn.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2014-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -31,10 +31,6 @@
#include "nvtypes.h"
#include "flcnretval.h"
#ifndef NV_ARRAY_ELEMENTS
#define NV_ARRAY_ELEMENTS(x) ((sizeof(x)/sizeof((x)[0])))
#endif
/*!
* @ref NvU64_ALIGN32 , NvU64_ALIGN32_PACK, NvU64_ALIGN32_UNPACK
* TODO: Use NV types directly

14
src/nvidia/arch/nvalloc/common/inc/fsp/fsp_tnvl_rpc.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,10 +29,6 @@
#define TNVL_GET_ATT_REPORT_SUBMESSAGE_ID 0x1
#define TNVL_LOCK_CONFIG_SUBMESSAGE_ID 0x2
#define NVSWITCH_IK_HASH_LENGTH (48)
#define NVSWITCH_ATT_CERT_SIZE_FIELD_LENGTH (2)
#define NVSWITCH_ATT_RSVD1_FIELD_LENGTH (2)
#pragma pack(1)
/*!
@@ -65,11 +61,10 @@ typedef struct
} TNVL_GET_ATT_CERTS_CMD_PAYLOAD;
/*!
* @brief TNVL response payload for attestation cert chain
*/
* @brief TNVL response payload for attestation cert chain
*/
typedef struct
{
NvU8 nvdmType;
NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse;
NvU8 subMessageId;
NvU8 rsvd0;
@@ -77,7 +72,6 @@ typedef struct
NvU8 majorVersion;
NvU16 certChainLength;
NvU16 rsvd1;
NvU8 devIkHash[NVSWITCH_IK_HASH_LENGTH];
NvU8 certChain[NVSWITCH_ATTESTATION_CERT_CHAIN_MAX_SIZE];
} TNVL_GET_ATT_CERTS_RSP_PAYLOAD;
@@ -98,7 +92,6 @@ typedef struct
*/
typedef struct
{
NvU8 nvdmType;
NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse;
NvU8 subMessageId;
NvU8 rsvd0;
@@ -124,7 +117,6 @@ typedef struct
*/
typedef struct
{
NvU8 nvdmType;
NVDM_PAYLOAD_COMMAND_RESPONSE cmdResponse;
NvU8 subMessageId;
NvU8 rsvd0;

13
src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h
View File

@@ -39,7 +39,9 @@
* | VGA WORKSPACE |
* ---------------------------- <- vbiosReservedOffset (64K? aligned)
* | (potential align. gap) |
* ---------------------------- <- gspFwWprEnd (128K aligned)
* ---------------------------- <- gspFwWprEnd + frtsSize + pmuReservedSize
* | PMU mem reservation |
* ---------------------------- <- gspFwWprEnd (128K aligned) + frtsSize
* | FRTS data | (frtsSize is 0 on GA100)
* | ------------------------ | <- frtsOffset
* | BOOT BIN (e.g. SK + BL) |
@@ -192,7 +194,14 @@ typedef struct
// Pad structure to exactly 256 bytes. Can replace padding with additional
// fields without incrementing revision. Padding initialized to 0.
NvU8 padding[6];
NvU8 padding[2];
//
// Starts at gspFwWprEnd+frtsSize b/c FRTS is positioned
// to end where this allocation starts (when RM requests FSP to create
// FRTS).
//
NvU32 pmuReservedSize;
// BL to use for verification (i.e. Booter says OK to boot)
NvU64 verified; // 0x0 -> unverified, 0xa0a0a0a0a0a0a0a0 -> verified

2
src/nvidia/arch/nvalloc/common/inc/inforom/ifrecc.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2017-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a

1
src/nvidia/arch/nvalloc/common/inc/nvcst.h
View File

@@ -192,7 +192,6 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_INTEL, 0x7A8A, CS_INTEL_1B81, "Intel-SapphireRapids", NULL},
{PCI_VENDOR_ID_INTEL, 0x18DC, CS_INTEL_18DC, "Intel-IceLake", NULL},
{PCI_VENDOR_ID_INTEL, 0x7A04, CS_INTEL_7A04, "Intel-RaptorLake", Intel_7A04_setupFunc},
{PCI_VENDOR_ID_INTEL, 0x5795, CS_INTEL_5795, "Intel-GraniteRapids", NULL},
{PCI_VENDOR_ID_NVIDIA, 0x0FAE, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc},
{PCI_VENDOR_ID_NVIDIA, 0x0FAF, CS_NVIDIA_T210, "T210", Nvidia_T210_setupFunc},

12
src/nvidia/arch/nvalloc/common/inc/oob/smbpbi.h
View File

@@ -344,7 +344,6 @@
#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE 15:8
#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_CORRECTABLE_ERROR 0
#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_UNCORRECTABLE_ERROR 1
#define NV_MSGBOX_CMD_ARG1_ECC_V6_ERROR_TYPE_ECC_STATE_FLAGS 2
#define NV_MSGBOX_CMD_ARG1_ENERGY_COUNTER_GPU 0x00000000
#define NV_MSGBOX_CMD_ARG1_ENERGY_COUNTER_MODULE 0x00000003
@@ -969,10 +968,6 @@
#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_5_MEMORY_CAPACITY_UTILIZATION_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED 9:9
#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_5_SRAM_ERROR_THRESHOLD_EXCEEDED_AVAILABLE 0x00000001
/* ECC counters */
#define NV_MSGBOX_DATA_ECC_CNT_16BIT_DBE 31:16
#define NV_MSGBOX_DATA_ECC_CNT_16BIT_SBE 16:0
@@ -1007,13 +1002,6 @@
#define NV_MSGBOX_DATA_ECC_V5_METADATA_LOCATION_ID 26:22
#define NV_MSGBOX_DATA_ECC_V5_METADATA_SUBLOCATION_ID 31:27
/* ECC state flags */
#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS 31:0
#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED 0:0
#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED_FALSE 0
#define NV_MSGBOX_DATA_ECC_V6_STATE_FLAGS_SRAM_ERROR_THRESHOLD_EXCEEDED_TRUE 1
/* NV_MSGBOX_CMD_OPCODE_SCRATCH_COPY src offset argument */
#define NV_MSGBOX_DATA_COPY_SRC_OFFSET 7:0

22
src/nvidia/arch/nvalloc/common/inc/rmlsfm.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2011-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2011-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -151,7 +151,10 @@
#define LSF_FALCON_ID_OFA_RISCV_EB (24U)
#define LSF_FALCON_ID_NVENC_RISCV_EB (25U)
#define LSF_FALCON_ID_PMU_RISCV_EB (26U)
#define LSF_FALCON_ID_END (27U)
#define LSF_FALCON_ID_NVDEC0_RISCV_EB (27U)
#define LSF_FALCON_ID_GSPLITE_RISCV_EB (28U)
#define LSF_FALCON_ID_DISPLAY_RISCV_EB (29U)
#define LSF_FALCON_ID_END (30U)
#define LSF_FALCON_ID_INVALID (0xFFFFFFFFU)
@@ -176,6 +179,7 @@
#define LSF_FALCON_ID_END_17 (17U)
#define LSF_FALCON_ID_END_18 (18U)
#define LSF_FALCON_ID_END_21 (21U)
#define LSF_FALCON_ID_END_30 (30U)
#define LSF_FALCON_INSTANCE_DEFAULT_0 (0x0U)
#define LSF_FALCON_INSTANCE_COUNT_DEFAULT_1 (0x1U)
@@ -192,12 +196,16 @@
* to uniquely identify it.
* @note this macro should be updated as needed whenever LSF_FALCON_ID* defines are added. See Bug: 3833461
*/
#define LSF_FALCON_USES_INSTANCE(falconId) ((falconId == LSF_FALCON_ID_NVDEC_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVJPG) || \
(falconId == LSF_FALCON_ID_NVJPG_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVENC_RISCV_EB) || \
#define LSF_FALCON_USES_INSTANCE(falconId) ((falconId == LSF_FALCON_ID_NVDEC_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVJPG) || \
(falconId == LSF_FALCON_ID_NVJPG_RISCV_EB) || \
(falconId == LSF_FALCON_ID_NVENC_RISCV_EB) || \
(falconId == LSF_FALCON_ID_FECS_RISCV_EB) || \
(falconId == LSF_FALCON_ID_GPCCS_RISCV_EB) || \
(falconId == LSF_FALCON_ID_GSPLITE_RISCV_EB) || \
(falconId == LSF_FALCON_ID_OFA_RISCV_EB))
/*!
* Size in entries of the ucode descriptor's dependency map.
* This used to be LSF_FALCON_ID_END, but since that had to grow and we did not want to break any
@@ -205,7 +213,7 @@
*
* Increasing this number should be done with care.
*/
#define LSF_FALCON_DEPMAP_SIZE (11)
#define LSF_FALCON_DEPMAP_SIZE (11U)
/*!
* Falcon Binaries version defines

29
src/nvidia/arch/nvalloc/unix/include/nv-reg.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2006-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2006-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -869,6 +869,8 @@
* NVreg_ModifyDeviceFiles, NVreg_DeviceFileGID, NVreg_DeviceFileUID
* and NVreg_DeviceFileMode will be honored by nvidia-modprobe.
*
* Also, refer to the NVreg_CreateImexChannel0 option.
*
* Possible values:
* 0 - Disable IMEX using CUDA driver's fabric handles.
* N - N IMEX channels will be enabled in the driver to facilitate N
@@ -878,6 +880,29 @@
#define __NV_IMEX_CHANNEL_COUNT ImexChannelCount
#define NV_REG_IMEX_CHANNEL_COUNT NV_REG_STRING(__NV_IMEX_CHANNEL_COUNT)
/*
* Option: NVreg_CreateImexChannel0
*
* Description:
*
* This option allows users to specify whether the NVIDIA driver must create
* the IMEX channel 0 by default. The channel will be created automatically
* when an application (e.g. nvidia-smi, nvidia-persistenced) is run.
*
* Note that users are advised to enable this option only in trusted
* environments where it is acceptable for applications to share the same
* IMEX channel.
*
* For more details on IMEX channels, refer to the NVreg_ImexChannelCount
* option.
*
* Possible values:
* 0 - Do not create IMEX channel 0 (default).
* 1 - Create IMEX channel 0.
*/
#define __NV_CREATE_IMEX_CHANNEL_0 CreateImexChannel0
#define NV_CREATE_IMEX_CHANNEL_0 NV_REG_STRING(__CREATE_IMEX_CHANNEL_0)
#if defined(NV_DEFINE_REGISTRY_KEY_TABLE)
/*
@@ -927,6 +952,7 @@ NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL);
NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE);
NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_CREATE_IMEX_CHANNEL_0, 0);
/*
*----------------registry database definition----------------------
@@ -974,6 +1000,7 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IMEX_CHANNEL_COUNT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_CREATE_IMEX_CHANNEL_0),
{NULL, NULL}
};

22
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -609,6 +609,15 @@ typedef enum
NV_POWER_STATE_RUNNING
} nv_power_state_t;
typedef struct
{
const char *vidmem_power_status;
const char *dynamic_power_status;
const char *gc6_support;
const char *gcoff_support;
const char *s0ix_status;
} nv_power_info_t;
#define NV_PRIMARY_VGA(nv) ((nv)->primary_vga)
#define NV_IS_CTL_DEVICE(nv) ((nv)->flags & NV_FLAG_CONTROL)
@@ -778,7 +787,7 @@ nv_state_t* NV_API_CALL nv_get_ctl_state (void);
void NV_API_CALL nv_set_dma_address_size (nv_state_t *, NvU32 );
NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU32, NvU32, NvU64, NvU64 *, void **);
NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, void **);
NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **);
NV_STATUS NV_API_CALL nv_free_pages (nv_state_t *, NvU32, NvBool, NvU32, void *);
@@ -822,6 +831,7 @@ void NV_API_CALL nv_acpi_methods_init (NvU32 *);
void NV_API_CALL nv_acpi_methods_uninit (void);
NV_STATUS NV_API_CALL nv_acpi_method (NvU32, NvU32, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
NV_STATUS NV_API_CALL nv_acpi_d3cold_dsm_for_upstream_port (nv_state_t *, NvU8 *, NvU32, NvU32, NvU32 *);
NV_STATUS NV_API_CALL nv_acpi_dsm_method (nv_state_t *, NvU8 *, NvU32, NvBool, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
NV_STATUS NV_API_CALL nv_acpi_ddc_method (nv_state_t *, void *, NvU32 *, NvBool);
NV_STATUS NV_API_CALL nv_acpi_dod_method (nv_state_t *, NvU32 *, NvU32 *);
@@ -990,10 +1000,10 @@ NV_STATUS NV_API_CALL rm_p2p_init_mapping (nvidia_stack_t *, NvU64, NvU6
NV_STATUS NV_API_CALL rm_p2p_destroy_mapping (nvidia_stack_t *, NvU64);
NV_STATUS NV_API_CALL rm_p2p_get_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *);
NV_STATUS NV_API_CALL rm_p2p_get_gpu_info (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **);
NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *);
NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent (nvidia_stack_t *, NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *, void **);
NV_STATUS NV_API_CALL rm_p2p_register_callback (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages (nvidia_stack_t *, NvU64, NvU32, NvU64, void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *);
NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *, void *);
NV_STATUS NV_API_CALL rm_p2p_dma_map_pages (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **);
NV_STATUS NV_API_CALL rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *, void **);
void NV_API_CALL rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle);
@@ -1027,9 +1037,7 @@ void NV_API_CALL rm_enable_dynamic_power_management(nvidia_stack_t *, nv_s
NV_STATUS NV_API_CALL rm_ref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
void NV_API_CALL rm_unref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
NV_STATUS NV_API_CALL rm_transition_dynamic_power(nvidia_stack_t *, nv_state_t *, NvBool, NvBool *);
const char* NV_API_CALL rm_get_vidmem_power_status(nvidia_stack_t *, nv_state_t *);
const char* NV_API_CALL rm_get_dynamic_power_management_status(nvidia_stack_t *, nv_state_t *);
const char* NV_API_CALL rm_get_gpu_gcx_support(nvidia_stack_t *, nv_state_t *, NvBool);
void NV_API_CALL rm_get_power_info(nvidia_stack_t *, nv_state_t *, nv_power_info_t *);
void NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32);
void NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);

2
src/nvidia/arch/nvalloc/unix/include/osapi.h
View File

@@ -68,7 +68,7 @@
if ((i * sizeof(NvU32)) < ((sp)->size / 8)) \
{ \
nv_printf(NV_DBG_ERRORS, "NVRM: altstack: used %d of %d bytes!\n", \
((sp)->size - (i * sizeof(NvU32))), (sp)->size); \
((sp)->size - (i * (NvU32)sizeof(NvU32))), (sp)->size); \
NV_ASSERT_PRECOMP((i * sizeof(NvU32)) >= ((sp)->size / 8)); \
} \
}

1
src/nvidia/arch/nvalloc/unix/include/osfuncs.h
View File

@@ -34,7 +34,6 @@
#if defined(NVCPU_X86_64)
OSnv_rdcr4 nv_rdcr4;
NvU64 nv_rdcr3(OBJOS *);
OSnv_cpuid nv_cpuid;
#endif

7
src/nvidia/arch/nvalloc/unix/src/asm/x86/nv_rdcr.c
View File

@@ -34,11 +34,4 @@ NvU32 nv_rdcr4(OBJOS *pOS)
return (NvU32)val;
}
NvU64 nv_rdcr3(OBJOS *pOS)
{
NvU64 val;
asm volatile ("movq %%cr3,%0" : "=r" (val));
return val;
}
#endif

273
src/nvidia/arch/nvalloc/unix/src/dynamic-power.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -564,135 +564,43 @@ subdeviceCtrlCmdOsUnixAudioDynamicPower_IMPL
return NV_OK;
}
/*!
* @brief: Function to indicate if Video Memory is powered off or not by
* checking if GPU is in GCOFF state.
*
* @param[in] sp nvidia_stack_t pointer.
* @param[in] pNv nv_state_t pointer.
*
* @return String indicating Video Memory power status.
*/
const char* NV_API_CALL rm_get_vidmem_power_status(
nvidia_stack_t *sp,
nv_state_t *pNv
)
{
THREAD_STATE_NODE threadState;
void *fp;
GPU_MASK gpuMask;
const char *pVidmemPowerStatus = "?";
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
// LOCK: acquire API lock
if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
// LOCK: acquire per device lock
if ((pGpu != NULL) &&
((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
&gpuMask)) == NV_OK))
{
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED))
{
pVidmemPowerStatus = "Off";
}
else
{
pVidmemPowerStatus = "Active";
}
// UNLOCK: release per device lock
rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
}
// UNLOCK: release API lock
rmapiLockRelease();
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
return pVidmemPowerStatus;
}
/*!
* @brief: Function to indicate if GC6/GC-OFF is supported
* or not on the SKU.
*
* @param[in] sp nvidia_stack_t pointer.
* @param[in] pNv nv_state_t pointer.
* @param[in] pGpu OBJGPU pointer.
* @param[in] bGcxTypeGc6 If true, returns string indicating GC6 support
* otherwise returns GC-OFF support.
*
* @return String indicating GC6/GC-OFF support status.
*/
const char* NV_API_CALL rm_get_gpu_gcx_support(
nvidia_stack_t *sp,
nv_state_t *pNv,
NvBool bGcxTypeGC6
static const char* RmGetGpuGcxSupport(
OBJGPU *pGpu,
NvBool bGcxTypeGC6
)
{
THREAD_STATE_NODE threadState;
void *fp;
GPU_MASK gpuMask;
const char *pSupported = "?";
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
// LOCK: acquire API lock
if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED))
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
// LOCK: acquire per device lock
if ((pGpu != NULL) &&
((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
&gpuMask)) == NV_OK))
{
pSupported = "Not Supported";
if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_UNIX_DYNAMIC_POWER_SUPPORTED))
{
goto done;
}
if (bGcxTypeGC6)
{
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED))
{
pSupported = "Supported";
}
}
else
{
if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED)) ||
(pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED)))
{
pSupported = "Supported";
}
}
done:
// UNLOCK: release per device lock
rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
}
//UNLOCK: release API lock
rmapiLockRelease();
return "Not Supported";
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
if (bGcxTypeGC6)
{
if (pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GC6_SUPPORTED))
{
return "Supported";
}
}
else
{
if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_RTD3_GCOFF_SUPPORTED)) ||
(pGpu->getProperty(pGpu, PDB_PROP_GPU_LEGACY_GCOFF_SUPPORTED)))
{
return "Supported";
}
}
return pSupported;
return "Not Supported";
}
/*!
@@ -2555,8 +2463,12 @@ NV_STATUS NV_API_CALL rm_transition_dynamic_power(
status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER);
if (status == NV_OK)
{
rmapiEnterRtd3PmPath();
status = RmTransitionDynamicPower(pGpu, bEnter, bTryAgain);
rmapiLeaveRtd3PmPath();
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
}
@@ -2632,64 +2544,40 @@ void RmHandleDisplayChange(
/*!
* @brief: Function to query Dynamic Power Management
*
* @param[in] sp nvidia_stack_t pointer.
* @param[in] pNv nv_state_t pointer.
* @param[in] nvp nv_priv_t pointer.
*
* @return String indicating Dynamic Power Management status.
*/
const char* NV_API_CALL rm_get_dynamic_power_management_status(
nvidia_stack_t *sp,
nv_state_t *pNv
static const char* RmGetDynamicPowerManagementStatus(
nv_priv_t *nvp
)
{
void *fp;
nv_priv_t *nvp = NV_GET_NV_PRIV(pNv);
const char *returnString = "?";
NV_ENTER_RM_RUNTIME(sp,fp);
// LOCK: acquire API lock
if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
/*
* If the platform/driver does not support Dynamic Power Management,
* we set mode as NV_DYNAMIC_PM_NEVER. Hence, after RmInit if the
* mode is still NV_DYNAMIC_PM_FINE, we are sure that it is
* supported and enabled. Also see NOTE.
*/
if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE)
{
return "Enabled (fine-grained)";
}
else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_COARSE)
{
return "Enabled (coarse-grained)";
}
else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER)
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
// Check if RM is inited
if (pGpu != NULL)
{
/*
* If the platform/driver does not support Dynamic Power Management,
* we set mode as NV_DYNAMIC_PM_NEVER. Hence, after RmInit if the
* mode is still NV_DYNAMIC_PM_FINE, we are sure that it is
* supported and enabled. Also see NOTE.
*/
if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_FINE)
{
returnString = "Enabled (fine-grained)";
}
else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_COARSE)
{
returnString = "Enabled (coarse-grained)";
}
else if (nvp->dynamic_power.mode == NV_DYNAMIC_PM_NEVER)
{
if (nvp->dynamic_power.dynamic_power_regkey ==
NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER)
returnString = "Disabled";
else if (nvp->dynamic_power.dynamic_power_regkey ==
NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)
returnString = "Disabled by default";
else
returnString = "Not supported";
}
}
//UNLOCK: release API lock
rmapiLockRelease();
if (nvp->dynamic_power.dynamic_power_regkey ==
NV_REG_DYNAMIC_POWER_MANAGEMENT_NEVER)
return "Disabled";
else if (nvp->dynamic_power.dynamic_power_regkey ==
NV_REG_DYNAMIC_POWER_MANAGEMENT_DEFAULT)
return "Disabled by default";
}
NV_EXIT_RM_RUNTIME(sp,fp);
return returnString;
return "Not supported";
}
static void RmHandleIdleSustained(
@@ -2789,3 +2677,60 @@ RmInitS0ixPowerManagement(
}
}
}
void NV_API_CALL rm_get_power_info(
nvidia_stack_t *sp,
nv_state_t *pNv,
nv_power_info_t *powerInfo
)
{
THREAD_STATE_NODE threadState;
void *fp;
GPU_MASK gpuMask;
const char *pVidmemPowerStatus = "?";
const char *pGc6Supported = "?";
const char *pGcoffSupported = "?";
const char *pDynamicPowerStatus = "?";
const char *pS0ixStatus = "?";
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
// LOCK: acquire API lock
if ((rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER)) == NV_OK)
{
OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv);
// LOCK: acquire per device lock
if ((pGpu != NULL) &&
((rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DYN_POWER,
&gpuMask)) == NV_OK))
{
nv_priv_t *pNvp = NV_GET_NV_PRIV(pNv);
pVidmemPowerStatus = pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERED) ?
"Off" : "Active";
pDynamicPowerStatus = RmGetDynamicPowerManagementStatus(pNvp);
pGc6Supported = RmGetGpuGcxSupport(pGpu, NV_TRUE);
pGcoffSupported = RmGetGpuGcxSupport(pGpu, NV_FALSE);
pS0ixStatus = pNvp->s0ix_pm_enabled ? "Enabled" : "Disabled";
// UNLOCK: release per device lock
rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
}
// UNLOCK: release API lock
rmapiLockRelease();
}
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);
powerInfo->vidmem_power_status = pVidmemPowerStatus;
powerInfo->dynamic_power_status = pDynamicPowerStatus;
powerInfo->gc6_support = pGc6Supported;
powerInfo->gcoff_support = pGcoffSupported;
powerInfo->s0ix_status = pS0ixStatus;
}

160
src/nvidia/arch/nvalloc/unix/src/os.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -345,11 +345,13 @@ static NV_STATUS setNumaPrivData
MEMORY_DESCRIPTOR *pMemDesc
)
{
NV_STATUS rmStatus = NV_OK;
void *pAllocPrivate = NULL;
NvU64 *addrArray = NULL;
NvU64 numPages = pMemDesc->PageCount;
NvU64 i;
NV_STATUS rmStatus = NV_OK;
void *pAllocPrivate = NULL;
NvU64 *addrArray = NULL;
NvU64 numPages = pMemDesc->PageCount;
NvU64 numOsPages = numPages;
RmPhysAddr *pteArray = memdescGetPteArray(pMemDesc, AT_CPU);
NvU64 i;
addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64));
if (addrArray == NULL)
@@ -357,24 +359,45 @@ static NV_STATUS setNumaPrivData
return NV_ERR_NO_MEMORY;
}
portMemCopy((void*)addrArray,
(numPages * sizeof(NvU64)),
(void*)memdescGetPteArray(pMemDesc, AT_CPU),
(numPages * sizeof(NvU64)));
if (NV_RM_PAGE_SIZE < os_page_size)
{
RmDeflateRmToOsPageArray(addrArray, numPages);
numPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
numOsPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
}
for (i = 0; i < numPages; i++)
if (!memdescGetContiguity(pMemDesc, AT_CPU))
{
// Update GPA to system physical address
addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
portMemCopy((void*)addrArray,
(numPages * sizeof(NvU64)),
(void*)pteArray,
(numPages * sizeof(NvU64)));
if (NV_RM_PAGE_SIZE < os_page_size)
{
RmDeflateRmToOsPageArray(addrArray, numPages);
}
for (i = 0; i < numOsPages; i++)
{
// Update GPA to system physical address
addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
}
}
else
{
//
// Original PTE array in contiguous memdesc only holds start address.
// We need to fill the OS page array with adjacent page addresses to
// map contiguously.
//
NvU64 offset = pteArray[0] + pKernelMemorySystem->coherentCpuFbBase;
for (i = 0; i < numOsPages; i++, offset += os_page_size)
{
addrArray[i] = offset;
}
}
rmStatus = nv_register_phys_pages(nv, addrArray, numPages, NV_MEMORY_CACHED, &pAllocPrivate);
rmStatus = nv_register_phys_pages(nv, addrArray, numOsPages, NV_MEMORY_CACHED, &pAllocPrivate);
if (rmStatus != NV_OK)
{
goto errors;
@@ -445,7 +468,7 @@ NV_STATUS osMapSystemMemory
void *pAddress;
void *pPrivate = NULL;
NvU64 pageIndex;
NvU32 pageOffset;
NvU32 pageOffset = (Offset & ~os_page_mask);
if (pGpu != NULL &&
pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
@@ -456,6 +479,8 @@ NV_STATUS osMapSystemMemory
rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc);
if (rmStatus != NV_OK)
return rmStatus;
pageOffset = memdescGetPhysAddr(pMemDesc, FORCE_VMMU_TRANSLATION(pMemDesc, AT_GPU), Offset) & ~os_page_mask;
}
*ppAddress = NvP64_NULL;
@@ -467,7 +492,6 @@ NV_STATUS osMapSystemMemory
return NV_ERR_INVALID_ARGUMENT;
pageIndex = (Offset >> os_page_shift);
pageOffset = (Offset & ~os_page_mask);
pAllocPrivate = memdescGetMemData(pMemDesc);
if (!pAllocPrivate)
@@ -716,8 +740,8 @@ NV_STATUS osQueueWorkItemWithFlags(
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW)
pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW;
if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW)
@@ -814,6 +838,7 @@ NV_STATUS osAllocPagesInternal(
NV_STATUS status;
NvS32 nodeId = NV0000_CTRL_NO_NUMA_NODE;
NV_ADDRESS_SPACE addrSpace = memdescGetAddressSpace(pMemDesc);
NvU64 pageSize = osGetPageSize();
memdescSetAddress(pMemDesc, NvP64_NULL);
memdescSetMemData(pMemDesc, NULL, NULL);
@@ -832,6 +857,7 @@ NV_STATUS osAllocPagesInternal(
status = nv_alias_pages(
NV_GET_NV_STATE(pGpu),
NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
pageSize,
memdescGetContiguity(pMemDesc, AT_CPU),
memdescGetCpuCacheAttrib(pMemDesc),
memdescGetGuestId(pMemDesc),
@@ -869,8 +895,6 @@ NV_STATUS osAllocPagesInternal(
}
else
{
NvU64 pageSize = osGetPageSize();
//
// Bug 4270864: Only non-contig EGM memory needs to specify order. Contig memory
// calculates it within nv_alloc_pages. The long term goal is to expand the ability
@@ -1809,6 +1833,7 @@ void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *p
{
*pScale = 60; // 1s -> 1m
}
return;
}
@@ -2712,24 +2737,6 @@ void osModifyGpuSwStatePersistence
}
}
NV_STATUS
osSystemGetBatteryDrain(NvS32 *pChargeRate)
{
NV_PRINTF(LEVEL_WARNING, "%s: Platform not supported!\n", __FUNCTION__);
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS
osPexRecoveryCallback
(
OS_GPU_INFO *pOsGpuInfo,
OS_PEX_RECOVERY_STATUS Status
)
{
NV_ASSERT_FAILED("Not supported");
return NV_ERR_NOT_SUPPORTED;
}
//
//osCallACPI_MXDS
//
@@ -2864,67 +2871,6 @@ NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16
return rmStatus;
}
NV_STATUS
osGetSystemCpuLogicalCoreCounts
(
NvU32 *pCpuCoreCount
)
{
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS
osGetSystemCpuC0AndAPerfCounters
(
NvU32 coreIndex,
POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
)
{
return NV_ERR_NOT_SUPPORTED;
}
void
osEnableCpuPerformanceCounters
(
OBJOS *pOS
)
{
NV_ASSERT_FAILED("Not supported");
return;
}
NV_STATUS
osCpuDpcObjInit
(
void **ppCpuDpcObj,
OBJGPU *pGpu,
NvU32 coreCount
)
{
NV_ASSERT_FAILED("Not supported");
return NV_ERR_NOT_SUPPORTED;
}
void
osCpuDpcObjQueue
(
void **ppCpuDpcObj,
NvU32 coreCount,
POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
)
{
NV_ASSERT_FAILED("Not supported");
}
void
osCpuDpcObjFree
(
void **ppCpuDpcObj
)
{
NV_ASSERT_FAILED("Not supported");
}
NV_STATUS
osGetCarveoutInfo
(
@@ -3240,10 +3186,16 @@ osIovaMap
KernelMemorySystem *pRootKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pRootMemDesc->pGpu);
if (bIsIndirectPeerMapping)
{
//
// If the first page from the memdesc is in FB then the remaining pages
// should also be in FB. If the memdesc is contiguous, check that it is
// contained within the coherent CPU FB range. memdescGetNvLinkGpa()
// will check that each page is in FB to handle the discontiguous case.
//
NvU64 atsBase = base + pRootKernelMemorySystem->coherentCpuFbBase;
NvU64 atsEnd = bIsContig ? (atsBase + pIovaMapping->pPhysMemDesc->Size) : atsBase;
if ((atsBase >= pRootKernelMemorySystem->coherentCpuFbBase) &&
(atsBase + pIovaMapping->pPhysMemDesc->Size <=
pRootKernelMemorySystem->coherentCpuFbEnd))
(atsEnd <= pRootKernelMemorySystem->coherentCpuFbEnd))
{
bIsFbOffset = NV_TRUE;
}

153
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -606,7 +606,7 @@ static void RmExecuteWorkItem(
NvU32 releaseLocks = 0;
if (!(pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU) &&
((pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW) ||
((pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS) ||
(pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW) ||
(pWi->flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW) ||
(pWi->flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY)))
@@ -626,17 +626,27 @@ static void RmExecuteWorkItem(
// Some work items may not require a valid GPU instance
if (pWi->flags & NV_WORK_ITEM_FLAGS_REQUIRES_GPU)
{
pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData);
// Make sure that pGpu is present
OBJGPU *pGpu = gpumgrGetGpu(pWi->gpuInstance);
if (pGpu != NULL)
{
pWi->func.pGpuFunction(pWi->gpuInstance, pWi->pData);
}
else
{
NV_PRINTF(LEVEL_ERROR, "Invalid GPU instance for workitem\n");
goto done;
}
}
else
{
pWi->func.pSystemFunction(pWi->pData);
}
done:
// Release any locks taken
workItemLocksRelease(releaseLocks, gpuMask);
done:
if ((pWi->pData != NULL) &&
!(pWi->flags & NV_WORK_ITEM_FLAGS_DONT_FREE_DATA))
{
@@ -694,13 +704,13 @@ static NV_STATUS RmAccessRegistry(
NvU32 *Entry
)
{
NvU32 gpuMask = 0, gpuInstance = 0;
OBJGPU *pGpu;
NvBool isDevice = NV_FALSE;
NV_STATUS RmStatus = NV_ERR_OPERATING_SYSTEM;
RsClient *pClient;
Device *pDevice;
Subdevice *pSubdevice;
NvU32 gpuMask = 0, gpuInstance = 0;
OBJGPU *pGpu;
NvBool isDevice = NV_FALSE;
NV_STATUS RmStatus = NV_ERR_OPERATING_SYSTEM;
CLIENT_ENTRY *pClientEntry;
Device *pDevice;
Subdevice *pSubdevice;
RMAPI_PARAM_COPY devNodeParamCopy;
NvU8 *tmpDevNode = NULL;
@@ -715,8 +725,11 @@ static NV_STATUS RmAccessRegistry(
NvU32 BinaryDataLength = 0;
NvU32 copyOutBinaryDataLength = 0;
if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient))
if (NV_OK != serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE,
&pClientEntry))
{
return NV_ERR_INVALID_CLIENT;
}
if (hClient == hObject)
{
@@ -724,6 +737,8 @@ static NV_STATUS RmAccessRegistry(
}
else
{
RsClient *pClient = pClientEntry->pClient;
RmStatus = deviceGetByHandle(pClient, hObject, &pDevice);
if (RmStatus != NV_OK)
{
@@ -939,7 +954,7 @@ static NV_STATUS RmAccessRegistry(
*pBinaryDataLength = copyOutBinaryDataLength;
}
serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient);
serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry);
return RmStatus;
}
@@ -952,6 +967,7 @@ static NV_STATUS RmUpdateDeviceMappingInfo(
)
{
NV_STATUS status;
CLIENT_ENTRY *pClientEntry;
RsClient *pClient;
RsResourceRef *pMappableRef;
RsCpuMapping *pCpuMapping;
@@ -959,10 +975,12 @@ static NV_STATUS RmUpdateDeviceMappingInfo(
Subdevice *pSubdevice;
NvU32 gpuMask = 0;
status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClient);
status = serverAcquireClient(&g_resServ, hClient, LOCK_ACCESS_WRITE, &pClientEntry);
if (status != NV_OK)
return status;
pClient = pClientEntry->pClient;
status = deviceGetByHandle(pClient, hDevice, &pDevice);
if (status != NV_OK)
{
@@ -1018,7 +1036,7 @@ done:
if (gpuMask != 0)
rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClient);
serverReleaseClient(&g_resServ, LOCK_ACCESS_WRITE, pClientEntry);
return status;
}
@@ -1345,32 +1363,6 @@ RmDmabufVerifyMemHandle(
return NV_OK;
}
static NV_STATUS
RmDmabufGetParentDevice(
OBJGPU *pGpu,
NvHandle hClient,
NvHandle hMemory,
Device **ppDevice
)
{
RsClient *pClient;
RsResourceRef *pDeviceRef;
RsResourceRef *pMemoryRef;
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
serverGetClientUnderLock(&g_resServ, hClient, &pClient));
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
clientGetResourceRef(pClient, hMemory, &pMemoryRef));
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
refFindAncestorOfType(pMemoryRef, classId(Device), &pDeviceRef));
*ppDevice = dynamicCast(pDeviceRef->pResource, Device);
return NV_OK;
}
static NV_STATUS
RmDmabufGetClientAndDevice(
OBJGPU *pGpu,
@@ -1386,30 +1378,23 @@ RmDmabufGetClientAndDevice(
if (IS_MIG_ENABLED(pGpu))
{
NV_STATUS status;
MIG_INSTANCE_REF ref;
Device *pParentDevice;
RsClient *pClient;
Memory *pMemory;
KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
status = RmDmabufGetParentDevice(pGpu, hClient, hMemory,
&pParentDevice);
if (status != NV_OK)
{
return status;
}
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
serverGetClientUnderLock(&g_resServ, hClient, &pClient));
status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
pParentDevice, &ref);
if (status != NV_OK)
{
return status;
}
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
memGetByHandle(pClient, hMemory, &pMemory));
status = kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare);
if (status != NV_OK)
{
return status;
}
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
pMemory->pDevice, &ref));
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kmigmgrIncRefCount(ref.pKernelMIGGpuInstance->pShare));
*phClient = ref.pKernelMIGGpuInstance->instanceHandles.hClient;
*phDevice = ref.pKernelMIGGpuInstance->instanceHandles.hDevice;
@@ -2320,9 +2305,11 @@ NV_STATUS rm_create_mmap_context(
// LOCK: acquire API lock
if ((rmStatus = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_OSAPI)) == NV_OK)
{
CLIENT_ENTRY *pClientEntry;
RmClient *pRmClient;
if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pRmClient))
if (NV_OK != serverutilAcquireClient(hClient, LOCK_ACCESS_READ, &pClientEntry,
&pRmClient))
{
// UNLOCK: release API lock
rmapiLockRelease();
@@ -2343,7 +2330,7 @@ NV_STATUS rm_create_mmap_context(
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
}
serverutilReleaseClient(LOCK_ACCESS_READ, pRmClient);
serverutilReleaseClient(LOCK_ACCESS_READ, pClientEntry);
// UNLOCK: release API lock
rmapiLockRelease();
@@ -3552,10 +3539,10 @@ NV_STATUS NV_API_CALL rm_is_supported_device(
{
NvBool bIsFirmwareCapable;
bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pNv->pci_info.device_id,
pmc_boot_42,
bIsFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(pmc_boot_42,
NV_IS_SOC_DISPLAY_DEVICE(pNv),
NULL);
NULL,
NV_FALSE /* isMCDM */);
if (!bIsFirmwareCapable)
{
if (hypervisorIsVgxHyper())
@@ -3569,7 +3556,7 @@ NV_STATUS NV_API_CALL rm_is_supported_device(
"NVRM: Software documentation, available at docs.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
pNv->pci_info.device_id);
}
else
{
@@ -3584,7 +3571,7 @@ NV_STATUS NV_API_CALL rm_is_supported_device(
"NVRM: www.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
pNv->pci_info.device_id);
}
goto threadfree;
}
@@ -3604,7 +3591,7 @@ NV_STATUS NV_API_CALL rm_is_supported_device(
"NVRM: Software documentation, available at docs.nvidia.com.\n",
pNv->pci_info.domain, pNv->pci_info.bus, pNv->pci_info.slot,
pNv->pci_info.function, pNv->pci_info.vendor_id,
pNv->pci_info.device_id, NV_VERSION_STRING);
pNv->pci_info.device_id);
goto threadfree;
}
goto threadfree;
@@ -4532,7 +4519,8 @@ NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
NvU64 *pPhysicalAddresses,
NvU32 *pEntries,
void *pPlatformData,
void *pGpuInfo
void *pGpuInfo,
void **ppMigInfo
)
{
THREAD_STATE_NODE threadState;
@@ -4551,7 +4539,8 @@ NV_STATUS NV_API_CALL rm_p2p_get_pages_persistent(
pPhysicalAddresses,
pEntries,
pPlatformData,
pGpuInfo);
pGpuInfo,
ppMigInfo);
// UNLOCK: release API lock
rmapiLockRelease();
}
@@ -4664,7 +4653,8 @@ NV_STATUS NV_API_CALL rm_p2p_register_callback(
NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(
nvidia_stack_t *sp,
void *p2pObject,
void *pKey
void *pKey,
void *pMigInfo
)
{
THREAD_STATE_NODE threadState;
@@ -4677,7 +4667,7 @@ NV_STATUS NV_API_CALL rm_p2p_put_pages_persistent(
// LOCK: acquire API lock
if ((rmStatus = rmapiLockAcquire(API_LOCK_FLAGS_NONE, RM_LOCK_MODULES_P2P)) == NV_OK)
{
rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey);
rmStatus = RmP2PPutPagesPersistent(p2pObject, pKey, pMigInfo);
// UNLOCK: release API lock
rmapiLockRelease();
@@ -5321,18 +5311,13 @@ NvBool NV_API_CALL rm_is_device_sequestered(
NV_ENTER_RM_RUNTIME(sp,fp);
threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE);
if (rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_GPU) == NV_OK)
{
//
// If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed.
// If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume
// that if core RM can't tell us the drain state, it must not be
// attached and the "sequestered" question is not relevant.
//
(void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL);
rmapiLockRelease();
}
//
// If gpumgrQueryGpuDrainState succeeds, bDrain will be set as needed.
// If gpumgrQueryGpuDrainState fails, bDrain will stay false; we assume
// that if core RM can't tell us the drain state, it must not be
// attached and the "sequestered" question is not relevant.
//
(void) gpumgrQueryGpuDrainState(pNv->gpu_id, &bDrain, NULL);
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
NV_EXIT_RM_RUNTIME(sp,fp);

404
src/nvidia/arch/nvalloc/unix/src/osinit.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -80,8 +80,6 @@
#include <class/cl2080.h>
#include <class/cl402c.h>
#include <gpu/conf_compute/conf_compute.h>
#include <gpu/dce_client/dce_client.h>
// RMCONFIG: need definition of REGISTER_ALL_HALS()
#include "g_hal_register.h"
@@ -188,11 +186,10 @@ static void initVendorSpecificRegistry(OBJGPU *, NvU16);
static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *);
static void initUnixSpecificRegistry(OBJGPU *);
NvBool osRmInitRm(OBJOS *pOS)
NV_STATUS osRmInitRm(void)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
NV_PRINTF(LEVEL_INFO, "init rm\n");
NV_STATUS status;
if (os_is_efi_enabled())
{
@@ -268,10 +265,11 @@ NvBool osRmInitRm(OBJOS *pOS)
NVLOG_UPDATE();
// Register all supported hals
if (REGISTER_ALL_HALS() != NV_OK)
status = REGISTER_ALL_HALS();
if (status != NV_OK)
{
RmDestroyRegistry(NULL);
return NV_FALSE;
return status;
}
// Setup any ThreadState defaults
@@ -280,7 +278,7 @@ NvBool osRmInitRm(OBJOS *pOS)
THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED |
THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED);
return NV_TRUE;
return NV_OK;
}
void RmShutdownRm(void)
@@ -367,6 +365,8 @@ osHandleGpuLost
nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS,
"GPU has fallen off the bus.");
gpuNotifySubDeviceEvent(pGpu, NV2080_NOTIFIERS_RC_ERROR, NULL, 0, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, 0);
NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n");
if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0')
@@ -871,6 +871,91 @@ osTeardownScalability(
return clTeardownPcie(pGpu, pCl);
}
#define NV_DBG_PRINT_VGA_STATUS(nv, src) \
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "%s reports GPU is %s VGA\n", \
src, NV_PRIMARY_VGA(nv) ? "primary" : "not primary");
static void
RmAssignPrimaryVga(
nv_state_t *nv,
OBJGPU *pGpu
)
{
//
// Check with the OS for the primary VGA status of the adapter. If it knows
// definitively (nv_set_primary_vga_status() returns NV_OK), then we should
// use that value.
//
// Otherwise, check the I/O access and VGA decoding along the path from the
// adapter to the root. We expect that the primary VGA will be the only
// non-3D controller with these properties enabled along the entire path.
//
if (nv_set_primary_vga_status(nv) != NV_OK)
{
KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJCL *pCl = SYS_GET_CL(pSys);
nv->primary_vga = (kbifIsPciIoAccessEnabled_HAL(pGpu, pKernelBif) &&
!kbifIs3dController_HAL(pGpu, pKernelBif) && (pCl != NULL) &&
clUpstreamVgaDecodeEnabled(pGpu, pCl));
NV_DBG_PRINT_VGA_STATUS(nv, "PCI config space");
}
else
{
NV_DBG_PRINT_VGA_STATUS(nv, "OS");
}
}
static void
RmDeterminePrimaryDevice(OBJGPU *pGpu)
{
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
NvBool bFrameBufferConsoleDevice = NV_FALSE;
// Skip updating nv->primary_vga while RM is recovering after GPU reset
if (nv->flags & NV_FLAG_IN_RECOVERY)
{
return;
}
nv->primary_vga = NV_FALSE;
//
// In case of Passthru, GPU will always be secondary
//
if (IS_PASSTHRU(pGpu))
{
return;
}
//
// In case of VIRTUAL GPU, there is no display, hence it will be secondary
//
if (IS_VIRTUAL(pGpu))
{
return;
}
RmAssignPrimaryVga(nv, pGpu);
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " is %s VGA\n",
!!nv->primary_vga ? "primary" : "not primary");
//
// If GPU is driving any frame buffer console(vesafb, efifb etc)
// mark the GPU as Primary.
//
bFrameBufferConsoleDevice = rm_get_uefi_console_status(nv);
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " is %s UEFI console device\n",
bFrameBufferConsoleDevice ? "primary" : "not primary");
pGpu->setProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE,
(bFrameBufferConsoleDevice || !!nv->primary_vga));
}
static void
RmSetConsolePreservationParams(OBJGPU *pGpu)
{
@@ -910,29 +995,8 @@ RmSetConsolePreservationParams(OBJGPU *pGpu)
// mode, so we need to create a BAR1 mapping for the console
// driver to use.
//
// If the console driver is not using _either_ of those
// mappings, then the console driver will not be able to
// access the memory we reserve. This happens on some
// UEFI systems with multiple GPUs, because the firmware
// initializes the GOP driver on more than one GPU. In
// that case, just skip reserving anything for GPUs where
// neither BAR1 nor BAR2 match the console's base
// address.
//
fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress);
if (fbConsoleSize == 0)
{
if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI))
{
fbConsoleSize = 0x40000;
}
else
{
NV_PRINTF(LEVEL_INFO, "No Frame Buffer Present\n");
}
}
if ((fbConsoleSize > 0) && (fbBaseAddress != 0))
{
KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
@@ -940,6 +1004,16 @@ RmSetConsolePreservationParams(OBJGPU *pGpu)
(fbBaseAddress == nv->fb->cpu_address);
}
//
// Case for legacy console, where OS will not give console size
// and primary_vga will be set for the device.
// Hence, reserving default size for VGA text modes.
//
if ((fbConsoleSize == 0) && nv->primary_vga)
{
fbConsoleSize = 0x40000;
}
pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000);
}
@@ -1209,12 +1283,12 @@ RmSetupRegisters(
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n",
nv->pci_info.vendor_id, nv->pci_info.device_id);
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n");
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: " NvP64_fmt " " NvP64_fmt,
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx,
nv->regs->cpu_address, nv->regs->size);
if (nv->fb != NULL)
{
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: " NvP64_fmt " " NvP64_fmt,
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx,
nv->fb->cpu_address, nv->fb->size);
}
@@ -1231,7 +1305,7 @@ RmSetupRegisters(
}
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n");
NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n");
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: " NvP64_fmt " " NvP64_fmt " 0x%p\n",
NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: 0x%" NvU64_fmtx " 0x%" NvU64_fmtx " 0x%p\n",
nv->regs->cpu_address, nv->regs->size, nv->regs->map);
ret = RmSetupDpauxRegisters(nv, status);
@@ -1755,6 +1829,8 @@ NvBool RmInitAdapter(
}
}
RmDeterminePrimaryDevice(pGpu);
RmSetConsolePreservationParams(pGpu);
RmInitAcpiMethods(pOS, pSys, pGpu);
@@ -1808,23 +1884,21 @@ NvBool RmInitAdapter(
if (IS_PASSTHRU(pGpu))
nv->flags |= NV_FLAG_PASSTHRU;
status.rmStatus = RmInitX86Emu(pGpu);
if (status.rmStatus != NV_OK)
{
RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED);
NV_PRINTF(LEVEL_ERROR,
"RmInitX86Emu failed, bailing out of RmInitAdapter\n");
goto shutdown;
}
initVendorSpecificRegistry(pGpu, nv->pci_info.device_id);
if (!IS_VIRTUAL(pGpu))
{
initNbsiTable(pGpu);
}
//
// Load GSP proxy if early init is required. We need to do this
// before we trigger a full gpuStateInit and gpuStateLoad in
// RmInitNvDevice
// TODO: Check bug 200744430
//
if (gpuIsCCFeatureEnabled(pGpu))
{
confComputeEarlyInit(pGpu, GPU_GET_CONF_COMPUTE(pGpu));
}
// finally, initialize the device
RmInitNvDevice(devicereference, &status);
if (! RM_INIT_SUCCESS(status.initStatus) )
@@ -1937,15 +2011,6 @@ NvBool RmInitAdapter(
goto shutdown;
}
status.rmStatus = RmInitX86Emu(pGpu);
if (status.rmStatus != NV_OK)
{
RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED);
NV_PRINTF(LEVEL_ERROR,
"RmInitX86Emu failed, bailing out of RmInitAdapter\n");
goto shutdown;
}
// i2c only on master device??
RmI2cAddGpuPorts(nv);
nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C;
@@ -2057,86 +2122,97 @@ void RmShutdownAdapter(
nv->ud.cpu_address = 0;
nv->ud.size = 0;
// LOCK: acquire GPUs lock
if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
//
// LOCK: lock all clients in case of eGPU hot unplug, which
// will not wait for all existing RM clients to stop using the GPU.
//
if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK)
{
//
// Shutdown path requires expanded GPU visibility in GPUMGR in order
// to access the GPU undergoing shutdown which may not be fully
// initialized, and to continue accessing the GPU undergoing shutdown
// after state destroy.
//
NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility());
RmDestroyDeferredDynamicPowerManagement(nv);
freeNbsiTable(pGpu);
gpuFreeEventHandle(pGpu);
OBJCL *pCl = SYS_GET_CL(pSys);
OBJOS *pOS = GPU_GET_OS(pGpu);
if (pCl != NULL)
// LOCK: acquire GPUs lock
if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
{
if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC)
//
// Shutdown path requires expanded GPU visibility in GPUMGR in order
// to access the GPU undergoing shutdown which may not be fully
// initialized, and to continue accessing the GPU undergoing shutdown
// after state destroy.
//
NV_ASSERT_OK(gpumgrThreadEnableExpandedGpuVisibility());
RmDestroyDeferredDynamicPowerManagement(nv);
freeNbsiTable(pGpu);
gpuFreeEventHandle(pGpu);
OBJCL *pCl = SYS_GET_CL(pSys);
OBJOS *pOS = GPU_GET_OS(pGpu);
if (pCl != NULL)
{
teardownCoreLogic(pOS, pGpu);
if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC)
{
teardownCoreLogic(pOS, pGpu);
}
else if (nvp->flags & NV_INIT_FLAG_SCALABILITY)
{
osTeardownScalability(pGpu);
}
}
else if (nvp->flags & NV_INIT_FLAG_SCALABILITY)
rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance));
rmapiDelPendingDevices(NVBIT(gpuInstance));
os_disable_console_access();
if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
{
osTeardownScalability(pGpu);
rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
NV_ASSERT(rmStatus == NV_OK);
}
}
rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance));
rmapiDelPendingDevices(NVBIT(gpuInstance));
if (nvp->flags & NV_INIT_FLAG_GPU_STATE)
{
rmStatus = gpuStateDestroy(pGpu);
NV_ASSERT(rmStatus == NV_OK);
}
os_disable_console_access();
os_enable_console_access();
if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
{
rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
NV_ASSERT(rmStatus == NV_OK);
}
if (nvp->flags & NV_INIT_FLAG_GPU_STATE)
{
rmStatus = gpuStateDestroy(pGpu);
NV_ASSERT(rmStatus == NV_OK);
}
os_enable_console_access();
//if (nvp->flags & NV_INIT_FLAG_HAL)
// destroyHal(pDev);
//if (nvp->flags & NV_INIT_FLAG_HAL)
// destroyHal(pDev);
#if NVCPU_IS_X86_64
RmFreeX86EmuState(pGpu);
RmFreeX86EmuState(pGpu);
#endif
gpumgrDetachGpu(gpuInstance);
gpumgrDestroyDevice(deviceInstance);
gpumgrDetachGpu(gpuInstance);
gpumgrDestroyDevice(deviceInstance);
//
// Expanded GPU visibility in GPUMGR is no longer needed once the
// GPU is removed from GPUMGR.
//
gpumgrThreadDisableExpandedGpuVisibility();
//
// Expanded GPU visibility in GPUMGR is no longer needed once the
// GPU is removed from GPUMGR.
//
gpumgrThreadDisableExpandedGpuVisibility();
if (nvp->flags & NV_INIT_FLAG_DMA)
{
RmTeardownDeviceDma(nv);
if (nvp->flags & NV_INIT_FLAG_DMA)
{
RmTeardownDeviceDma(nv);
}
RmClearPrivateState(nv);
RmUnInitAcpiMethods(pSys);
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
// RM_BASIC_LOCK_MODEL: free GPU lock
rmGpuLockFree(deviceInstance);
}
RmClearPrivateState(nv);
RmUnInitAcpiMethods(pSys);
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
// RM_BASIC_LOCK_MODEL: free GPU lock
rmGpuLockFree(deviceInstance);
// UNLOCK: unlock all clients for eGPU hot unplug path
if (nv->is_external_gpu)
serverUnlockAllClients(&g_resServ);
}
}
else
@@ -2189,54 +2265,66 @@ void RmDisableAdapter(
}
//
// Free the client allocated resources.
// LOCK: lock all clients in case of eGPU hot unplug, which
// will not wait for all existing RM clients to stop using the GPU.
//
// This needs to happen prior to tearing down SLI state when SLI is enabled.
//
// Note this doesn't free RM internal resource allocations. Those are
// freed during (gpumgrUpdateSLIConfig->...->)gpuStateUnload.
//
// We need to free resources for all GPUs linked in a group as
// gpumgrUpdateSLIConfig will teardown GPU state for the entire set.
//
gpuMask = gpumgrGetGpuMask(pGpu);
rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask);
rmapiDelPendingDevices(gpuMask);
// LOCK: acquire GPUs lock
if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
if (!nv->is_external_gpu || serverLockAllClients(&g_resServ) == NV_OK)
{
nv_stop_rc_timer(nv);
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJCL *pCl = SYS_GET_CL(pSys);
OBJOS *pOS = SYS_GET_OS(pSys);
if (pCl != NULL)
// LOCK: acquire GPUs lock
if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK)
{
teardownCoreLogic(pOS, pGpu);
nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC;
//
// Free the client allocated resources.
//
// This needs to happen prior to tearing down SLI state when SLI is enabled.
//
// Note this doesn't free RM internal resource allocations. Those are
// freed during (gpumgrUpdateSLIConfig->...->)gpuStateUnload.
//
// We need to free resources for all GPUs linked in a group as
// gpumgrUpdateSLIConfig will teardown GPU state for the entire set.
//
gpuMask = gpumgrGetGpuMask(pGpu);
rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask);
rmapiDelPendingDevices(gpuMask);
nv_stop_rc_timer(nv);
OBJSYS *pSys = SYS_GET_INSTANCE();
OBJCL *pCl = SYS_GET_CL(pSys);
OBJOS *pOS = SYS_GET_OS(pSys);
if (pCl != NULL)
{
teardownCoreLogic(pOS, pGpu);
nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC;
}
if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG)
{
krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu));
nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG;
}
os_disable_console_access();
if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
{
rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
NV_ASSERT(rmStatus == NV_OK);
nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD;
}
os_enable_console_access();
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
}
if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG)
{
krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu));
nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG;
}
os_disable_console_access();
if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD)
{
rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT);
NV_ASSERT(rmStatus == NV_OK);
nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD;
}
os_enable_console_access();
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
// UNLOCK: unlock all clients for eGPU hot unplug path
if (nv->is_external_gpu)
serverUnlockAllClients(&g_resServ);
}
}

32
src/nvidia/arch/nvalloc/unix/src/rm-gpu-ops.c
View File

@@ -848,6 +848,18 @@ rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *sp,
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_key_rotation_channel_disable(nvidia_stack_t *sp,
gpuChannelHandle channelList[],
NvU32 channeListCount)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsKeyRotationChannelDisable(channelList, channeListCount);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *sp,
struct ccslContext_t **ctx,
gpuChannelHandle channel)
@@ -871,14 +883,14 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *sp,
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_rotate_key(nvidia_stack_t *sp,
UvmCslContext *contextList[],
NvU32 contextListCount)
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_context_update(nvidia_stack_t *sp,
UvmCslContext *contextList[],
NvU32 contextListCount)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsCcslRotateKey(contextList, contextListCount);
rmStatus = nvGpuOpsCcslContextUpdate(contextList, contextListCount);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}
@@ -931,7 +943,6 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *sp,
NvU32 bufferSize,
NvU8 const *inputBuffer,
NvU8 const *decryptIv,
NvU32 keyRotationId,
NvU8 *outputBuffer,
NvU8 const *addAuthData,
NvU32 addAuthDataSize,
@@ -940,7 +951,7 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *sp,
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsCcslDecrypt(ctx, bufferSize, inputBuffer, decryptIv, keyRotationId, outputBuffer,
rmStatus = nvGpuOpsCcslDecrypt(ctx, bufferSize, inputBuffer, decryptIv, outputBuffer,
addAuthData, addAuthDataSize, authTagData);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
@@ -988,15 +999,14 @@ NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_increment_iv(nvidia_stack_t *sp,
return rmStatus;
}
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_log_encryption(nvidia_stack_t *sp,
struct ccslContext_t *ctx,
NvU8 direction,
NvU32 bufferSize)
NV_STATUS NV_API_CALL rm_gpu_ops_ccsl_log_device_encryption(nvidia_stack_t *sp,
struct ccslContext_t *ctx,
NvU32 bufferSize)
{
NV_STATUS rmStatus;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
rmStatus = nvGpuOpsLogEncryption(ctx, direction, bufferSize);
rmStatus = nvGpuOpsLogDeviceEncryption(ctx, bufferSize);
NV_EXIT_RM_RUNTIME(sp,fp);
return rmStatus;
}

139
src/nvidia/arch/nvalloc/unix/src/unix_intr.c
View File

@@ -33,7 +33,6 @@
#include "gpu/mmu/kern_gmmu.h"
#include "gpu/disp/kern_disp.h"
#include <nv_sriov_defines.h>
#include "objtmr.h"
static NvBool osInterruptPending(
OBJGPU *pGpu,
@@ -107,45 +106,31 @@ static NvBool osInterruptPending(
pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
tlsIsrInit(pIsrAllocator);
//
// For SWRL granular locking process the countdown timer interrupt.
// GSP-RM handles SWRL interrupts if GSP-RM is active
//
if ((!IS_GSP_CLIENT(pGpu)) && (pGpu->getProperty(pGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING)))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
gpuInstance = 0;
while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
{
pIntr = GPU_GET_INTR(pGpu);
if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))
{
// If interrupt enable is garbage the GPU is probably in a bad state
if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX)
{
continue;
}
OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
*serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &threadState);
}
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
//
// Service nonstall interrupts before possibly acquiring GPUs lock
// so that we don't unnecesarily hold the lock while servicing them.
//
// Also check if we need to acquire the GPU lock at all and get critical interrupts
// This should not violate (1) from above since we are not servicing the GPUs in SLI,
// only checking their state.
//
// To do so, two steps are required:
// Step 1: Check if we can service nonstall interrupts outside the GPUs lock. This is true
// if the two PDBs are true. Otherwise we have to acquire the GPUs lock to service the nonstall
// interrupts anyway, and we can't get around acquiring the GPUs lock.
//
// Step 2 is inline below
//
NvBool bIsAnyStallIntrPending = NV_TRUE;
NvBool bIsAnyBottomHalfStallPending = NV_FALSE;
NvU32 isDispPendingPerGpu = 0;
NvU32 isTmrPendingPerGpu = 0;
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
bIsAnyStallIntrPending = NV_FALSE;
gpuMask = gpumgrGetGpuMask(pDeviceLockGpu);
gpuInstance = 0;
@@ -162,13 +147,53 @@ static NvBool osInterruptPending(
intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState);
*serviced = NV_TRUE;
}
//
// Step 2: Service nonstall interrupts and check if there are any stall interrupts
// pending on any GPUs. If there are, then we will take the GPU lock.
//
// Bug 4223192: calling intrGetPendingStall_HAL is rather expensive,
// so save off the critical interrupts to be handled in the top half.
//
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
if (!bitVectorTestAllCleared(&intr0Pending))
{
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
{
isDispPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
// We will attempt to handle this separately from the other stall interrupts
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
}
if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
!IS_VIRTUAL(pGpu) && bitVectorTest(&intr0Pending, MC_ENGINE_IDX_TMR))
{
isTmrPendingPerGpu |= NVBIT32(gpuGetInstance(pGpu));
// We will clear this in the top half
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR);
}
if (!bitVectorTestAllCleared(&intr0Pending))
{
//
// If any other stall interrupts are pending,
// they will be handled in the bottom half.
//
bIsAnyBottomHalfStallPending = NV_TRUE;
}
bIsAnyStallIntrPending = NV_TRUE;
}
}
}
threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
}
// LOCK: try to acquire GPUs lock
if (rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK)
if (bIsAnyStallIntrPending &&
(rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK))
{
threadStateInitISRAndDeferredIntHandler(&threadState,
pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR);
@@ -191,17 +216,52 @@ static NvBool osInterruptPending(
if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX)
continue;
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
//
// If lockless interrupt handling was enabled, we have already cached which critical
// interrupts are pending. We only set the appropriate bits in the intr0Pending bitvector
// for the subsequent code to service them in the same manner as the lockless nonstall
// interrupt handling disabled case. But we also clear them from intr0Pending and
// for disp, check if they're still pending afterwards. We already checked whether any
// other bottom half stall interrupts are pending in bIsAnyBottomHalfStallPending above.
//
// After all this, the combination of bIsAnyBottomHalfStallPending and intr0Pending
// contains whether any stall interrupts are still pending, so check both to determine if
// we need a bottom half.
//
if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) &&
pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS))
{
bitVectorClrAll(&intr0Pending);
if ((isDispPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
{
bitVectorSet(&intr0Pending, MC_ENGINE_IDX_DISP);
}
if ((isTmrPendingPerGpu & NVBIT32(gpuGetInstance(pGpu))) != 0)
{
bitVectorSet(&intr0Pending, MC_ENGINE_IDX_TMR);
}
}
else
{
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
}
if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP))
{
if (pKernelDisplay != NULL)
{
MC_ENGINE_BITVECTOR intrDispPending;
kdispServiceVblank_HAL(pGpu, pKernelDisplay, 0,
(VBLANK_STATE_PROCESS_LOW_LATENCY |
VBLANK_STATE_PROCESS_CALLED_FROM_ISR),
&threadState);
*serviced = NV_TRUE;
intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState);
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_DISP);
intrGetPendingDisplayIntr_HAL(pGpu, pIntr, &intrDispPending, &threadState);
bitVectorOr(&intr0Pending, &intr0Pending, &intrDispPending);
}
}
@@ -211,7 +271,7 @@ static NvBool osInterruptPending(
// We have to clear the top level interrupt bit here since otherwise
// the bottom half will attempt to service the interrupt on the CPU
// side before GSP receives the notification and services it
intrClearLeafVector_HAL(pGpu, pIntr, MC_ENGINE_IDX_TMR, &threadState);
intrClearLeafVector_HAL(pGpu, pIntr, intrGetVectorFromEngineId(pGpu, pIntr, MC_ENGINE_IDX_TMR, NV_FALSE), &threadState);
bitVectorClr(&intr0Pending, MC_ENGINE_IDX_TMR);
NV_ASSERT_OK(intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_TMR_HANDLE));
@@ -228,8 +288,13 @@ static NvBool osInterruptPending(
intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog);
}
}
else
{
bitVectorClrAll(&intr1Pending);
}
if (!bitVectorTestAllCleared(&intr0Pending) ||
if (bIsAnyBottomHalfStallPending ||
!bitVectorTestAllCleared(&intr0Pending) ||
!bitVectorTestAllCleared(&intr1Pending))
{
pending = NV_TRUE;