NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-30 04:59:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

525.89.02

Browse Source
This commit is contained in:
Andy Ritger
2023-02-08 10:15:15 -08:00
parent 1dc88ff75e
commit e598191e8e
58 changed files with 1104 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -915,6 +915,7 @@ NV_STATUS NV_API_CALL rm_write_registry_string (nvidia_stack_t *, nv_state_t *
void NV_API_CALL rm_parse_option_string (nvidia_stack_t *, const char *);
char* NV_API_CALL rm_remove_spaces (const char *);
char* NV_API_CALL rm_string_token (char **, const char);
void NV_API_CALL rm_vgpu_vfio_set_driver_vm(nvidia_stack_t *, NvBool);
NV_STATUS NV_API_CALL rm_run_rc_callback (nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_execute_work_item (nvidia_stack_t *, void *);
@@ -988,7 +989,7 @@ void NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32);
NvBool NV_API_CALL rm_is_altstack_in_use(void);
/* vGPU VFIO specific functions */
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32, NvBool *);
NV_STATUS NV_API_CALL nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32);
NV_STATUS NV_API_CALL nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16);
NV_STATUS NV_API_CALL nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool);
NV_STATUS NV_API_CALL nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8);

7
src/nvidia/arch/nvalloc/unix/src/os-hypervisor.c
View File

@@ -447,14 +447,11 @@ NV_STATUS NV_API_CALL nv_vgpu_create_request(
const NvU8 *pMdevUuid,
NvU32 vgpuTypeId,
NvU16 *vgpuId,
NvU32 gpuPciBdf,
NvBool *is_driver_vm
NvU32 gpuPciBdf
)
{
THREAD_STATE_NODE threadState;
OBJSYS *pSys = SYS_GET_INSTANCE();
void *fp = NULL;
OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
NV_STATUS rmStatus = NV_OK;
NV_ENTER_RM_RUNTIME(sp,fp);
@@ -466,8 +463,6 @@ NV_STATUS NV_API_CALL nv_vgpu_create_request(
rmStatus = kvgpumgrCreateRequestVgpu(pNv->gpu_id, pMdevUuid,
vgpuTypeId, vgpuId, gpuPciBdf);
*is_driver_vm = pHypervisor->getProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED);
// UNLOCK: release API lock
rmapiLockRelease();
}

21
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -1114,7 +1114,7 @@ static void RmCheckNvpcfDsmScope(
OBJGPU *pGpu
)
{
NvU32 supportedFuncs;
NvU32 supportedFuncs = 0;
NvU16 dsmDataSize = sizeof(supportedFuncs);
nv_state_t *nv = NV_GET_NV_STATE(pGpu);
ACPI_DSM_FUNCTION acpiDsmFunction = ACPI_DSM_FUNCTION_NVPCF_2X;
@@ -5318,6 +5318,25 @@ void NV_API_CALL rm_dma_buf_put_client_and_device(
// NOTE: Used only on VMWware
//
void NV_API_CALL rm_vgpu_vfio_set_driver_vm(
nvidia_stack_t *sp,
NvBool is_driver_vm
)
{
OBJSYS *pSys;
POBJHYPERVISOR pHypervisor;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
pSys = SYS_GET_INSTANCE();
pHypervisor = SYS_GET_HYPERVISOR(pSys);
pHypervisor->setProperty(pHypervisor, PDB_PROP_HYPERVISOR_DRIVERVM_ENABLED, is_driver_vm);
NV_EXIT_RM_RUNTIME(sp,fp);
}
NvBool NV_API_CALL rm_is_altstack_in_use(void)
{
#if defined(__use_altstack__)

1
src/nvidia/exports_link_command.txt
View File

@@ -16,6 +16,7 @@
--undefined=rm_init_adapter
--undefined=rm_init_private_state
--undefined=rm_init_rm
--undefined=rm_vgpu_vfio_set_driver_vm
--undefined=rm_ioctl
--undefined=rm_is_supported_device
--undefined=rm_is_supported_pci_device

2
src/nvidia/generated/g_client_nvoc.h
View File

@@ -301,6 +301,8 @@ MAKE_LIST(RmClientList, RmClient*);
extern RmClientList g_clientListBehindGpusLock;
MAKE_LIST(UserInfoList, UserInfo*);
extern UserInfoList g_userInfoList;
MAKE_MULTIMAP(OsInfoMap, RmClient*);
extern OsInfoMap g_osInfoList;
//

1
src/nvidia/generated/g_kernel_gsp_nvoc.h
View File

@@ -338,6 +338,7 @@ struct KernelGsp {
NvU64 logElfDataSize;
NvBool bLibosLogsPollingEnabled;
NvBool bInInit;
NvBool bInLockdown;
NvBool bPollingForRpcResponse;
NvBool bXid119Printed;
MEMORY_DESCRIPTOR *pMemDesc_simAccessBuf;

10
src/nvidia/generated/g_nv_name_released.h
View File

@@ -974,7 +974,17 @@ static const CHIPS_RELEASED sChipsReleased[] = {
{ 0x26B1, 0x16a1, 0x17aa, "NVIDIA RTX 6000 Ada Generation" },
{ 0x26B5, 0x169d, 0x10de, "NVIDIA L40" },
{ 0x2704, 0x0000, 0x0000, "NVIDIA GeForce RTX 4080" },
{ 0x2717, 0x0000, 0x0000, "NVIDIA GeForce RTX 4090 Laptop GPU" },
{ 0x2757, 0x0000, 0x0000, "NVIDIA GeForce RTX 4090 Laptop GPU" },
{ 0x2782, 0x0000, 0x0000, "NVIDIA GeForce RTX 4070 Ti" },
{ 0x27A0, 0x0000, 0x0000, "NVIDIA GeForce RTX 4080 Laptop GPU" },
{ 0x27E0, 0x0000, 0x0000, "NVIDIA GeForce RTX 4080 Laptop GPU" },
{ 0x2820, 0x0000, 0x0000, "NVIDIA GeForce RTX 4070 Laptop GPU" },
{ 0x2860, 0x0000, 0x0000, "NVIDIA GeForce RTX 4070 Laptop GPU" },
{ 0x28A0, 0x0000, 0x0000, "NVIDIA GeForce RTX 4060 Laptop GPU" },
{ 0x28A1, 0x0000, 0x0000, "NVIDIA GeForce RTX 4050 Laptop GPU" },
{ 0x28E0, 0x0000, 0x0000, "NVIDIA GeForce RTX 4060 Laptop GPU" },
{ 0x28E1, 0x0000, 0x0000, "NVIDIA GeForce RTX 4050 Laptop GPU" },
{ 0x13BD, 0x11cc, 0x10DE, "GRID M10-0B" },
{ 0x13BD, 0x11cd, 0x10DE, "GRID M10-1B" },
{ 0x13BD, 0x11ce, 0x10DE, "GRID M10-0Q" },

35
src/nvidia/generated/g_rpc-structures.h
View File

@@ -506,6 +506,13 @@ typedef struct rpc_pfm_req_hndlr_state_sync_callback_v21_04
typedef rpc_pfm_req_hndlr_state_sync_callback_v21_04 rpc_pfm_req_hndlr_state_sync_callback_v;
typedef struct rpc_gsp_lockdown_notice_v17_00
{
NvBool bLockdownEngaging;
} rpc_gsp_lockdown_notice_v17_00;
typedef rpc_gsp_lockdown_notice_v17_00 rpc_gsp_lockdown_notice_v;
#endif
@@ -2133,6 +2140,25 @@ static vmiopd_mdesc_t vmiopd_mdesc_t_rpc_pfm_req_hndlr_state_sync_callback_v21_0
};
#endif
#ifndef SKIP_PRINT_rpc_gsp_lockdown_notice_v17_00
static vmiopd_fdesc_t vmiopd_fdesc_t_rpc_gsp_lockdown_notice_v17_00[] = {
{
.vtype = vtype_NvBool,
.offset = NV_OFFSETOF(rpc_gsp_lockdown_notice_v17_00, bLockdownEngaging),
.name = "bLockdownEngaging"
},
{
.vtype = vt_end
}
};
static vmiopd_mdesc_t vmiopd_mdesc_t_rpc_gsp_lockdown_notice_v17_00 = {
.name = "rpc_gsp_lockdown_notice",
.header_length = NV_SIZEOF32(rpc_gsp_lockdown_notice_v17_00),
.fdesc = vmiopd_fdesc_t_rpc_gsp_lockdown_notice_v17_00
};
#endif
#endif
#ifdef RPC_DEBUG_PRINT_FUNCTIONS
@@ -2546,6 +2572,13 @@ vmiopd_mdesc_t *rpcdebugPfmReqHndlrStateSyncCallback_v21_04(void)
}
#endif
#ifndef SKIP_PRINT_rpc_gsp_lockdown_notice_v17_00
vmiopd_mdesc_t *rpcdebugGspLockdownNotice_v17_00(void)
{
return &vmiopd_mdesc_t_rpc_gsp_lockdown_notice_v17_00;
}
#endif
#endif
@@ -2656,6 +2689,8 @@ typedef union rpc_generic_union {
rpc_extdev_intr_service_v extdev_intr_service_v;
rpc_pfm_req_hndlr_state_sync_callback_v21_04 pfm_req_hndlr_state_sync_callback_v21_04;
rpc_pfm_req_hndlr_state_sync_callback_v pfm_req_hndlr_state_sync_callback_v;
rpc_gsp_lockdown_notice_v17_00 gsp_lockdown_notice_v17_00;
rpc_gsp_lockdown_notice_v gsp_lockdown_notice_v;
} rpc_generic_union;
#endif

2
src/nvidia/generated/g_subdevice_nvoc.h
View File

@@ -604,7 +604,7 @@ struct Subdevice {
struct Device *pDevice;
NvBool bMaxGrTickFreqRequested;
NvU64 P2PfbMappedBytes;
NvU32 notifyActions[177];
NvU32 notifyActions[178];
NvHandle hNotifierMemory;
struct Memory *pNotifierMemory;
NvHandle hSemMemory;

1
src/nvidia/generated/g_system_nvoc.c
View File

@@ -90,6 +90,7 @@ void __nvoc_init_dataField_OBJSYS(OBJSYS *pThis) {
pThis->setProperty(pThis, PDB_PROP_SYS_IS_AGGRESSIVE_GC6_ENABLED, (0));
pThis->setProperty(pThis, PDB_PROP_SYS_PRIORITY_BOOST, (0));
pThis->setProperty(pThis, PDB_PROP_SYS_PRIORITY_THROTTLE_DELAY_US, 16 * 1000);
pThis->setProperty(pThis, PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP, ((NvBool)(0 != 0)));
pThis->setProperty(pThis, PDB_PROP_SYS_ROUTE_TO_PHYSICAL_LOCK_BYPASS, ((NvBool)(0 == 0)));
}

3
src/nvidia/generated/g_system_nvoc.h
View File

@@ -395,6 +395,7 @@ struct OBJSYS {
NvBool PDB_PROP_SYS_PRIORITY_BOOST;
NvU32 PDB_PROP_SYS_PRIORITY_THROTTLE_DELAY_US;
NvBool PDB_PROP_SYS_BUGCHECK_ON_TIMEOUT;
NvBool PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP;
NvU32 apiLockMask;
NvU32 apiLockModuleMask;
NvU32 gpuLockModuleMask;
@@ -485,6 +486,8 @@ extern const struct NVOC_CLASS_DEF __nvoc_class_def_OBJSYS;
#define PDB_PROP_SYS_SBIOS_NVIF_POWERMIZER_LIMIT_BASE_NAME PDB_PROP_SYS_SBIOS_NVIF_POWERMIZER_LIMIT
#define PDB_PROP_SYS_IS_UEFI_BASE_CAST
#define PDB_PROP_SYS_IS_UEFI_BASE_NAME PDB_PROP_SYS_IS_UEFI
#define PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP_BASE_CAST
#define PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP_BASE_NAME PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP
#define PDB_PROP_SYS_INTERNAL_EVENT_BUFFER_ALLOC_ALLOWED_BASE_CAST
#define PDB_PROP_SYS_INTERNAL_EVENT_BUFFER_ALLOC_ALLOWED_BASE_NAME PDB_PROP_SYS_INTERNAL_EVENT_BUFFER_ALLOC_ALLOWED
#define PDB_PROP_SYS_IS_GSYNC_ENABLED_BASE_CAST

155
src/nvidia/generated/g_third_party_p2p_nvoc.c
View File

@@ -7,6 +7,161 @@
#include "utils/nvassert.h"
#include "g_third_party_p2p_nvoc.h"
#ifdef DEBUG
char __nvoc_class_id_uniqueness_check_0x3e3a6a = 1;
#endif
extern const struct NVOC_CLASS_DEF __nvoc_class_def_P2PTokenShare;
extern const struct NVOC_CLASS_DEF __nvoc_class_def_Object;
extern const struct NVOC_CLASS_DEF __nvoc_class_def_RsShared;
void __nvoc_init_P2PTokenShare(P2PTokenShare*);
void __nvoc_init_funcTable_P2PTokenShare(P2PTokenShare*);
NV_STATUS __nvoc_ctor_P2PTokenShare(P2PTokenShare*);
void __nvoc_init_dataField_P2PTokenShare(P2PTokenShare*);
void __nvoc_dtor_P2PTokenShare(P2PTokenShare*);
extern const struct NVOC_EXPORT_INFO __nvoc_export_info_P2PTokenShare;
static const struct NVOC_RTTI __nvoc_rtti_P2PTokenShare_P2PTokenShare = {
/*pClassDef=*/ &__nvoc_class_def_P2PTokenShare,
/*dtor=*/ (NVOC_DYNAMIC_DTOR) &__nvoc_dtor_P2PTokenShare,
/*offset=*/ 0,
};
static const struct NVOC_RTTI __nvoc_rtti_P2PTokenShare_Object = {
/*pClassDef=*/ &__nvoc_class_def_Object,
/*dtor=*/ &__nvoc_destructFromBase,
/*offset=*/ NV_OFFSETOF(P2PTokenShare, __nvoc_base_RsShared.__nvoc_base_Object),
};
static const struct NVOC_RTTI __nvoc_rtti_P2PTokenShare_RsShared = {
/*pClassDef=*/ &__nvoc_class_def_RsShared,
/*dtor=*/ &__nvoc_destructFromBase,
/*offset=*/ NV_OFFSETOF(P2PTokenShare, __nvoc_base_RsShared),
};
static const struct NVOC_CASTINFO __nvoc_castinfo_P2PTokenShare = {
/*numRelatives=*/ 3,
/*relatives=*/ {
&__nvoc_rtti_P2PTokenShare_P2PTokenShare,
&__nvoc_rtti_P2PTokenShare_RsShared,
&__nvoc_rtti_P2PTokenShare_Object,
},
};
const struct NVOC_CLASS_DEF __nvoc_class_def_P2PTokenShare =
{
/*classInfo=*/ {
/*size=*/ sizeof(P2PTokenShare),
/*classId=*/ classId(P2PTokenShare),
/*providerId=*/ &__nvoc_rtti_provider,
#if NV_PRINTF_STRINGS_ALLOWED
/*name=*/ "P2PTokenShare",
#endif
},
/*objCreatefn=*/ (NVOC_DYNAMIC_OBJ_CREATE) &__nvoc_objCreateDynamic_P2PTokenShare,
/*pCastInfo=*/ &__nvoc_castinfo_P2PTokenShare,
/*pExportInfo=*/ &__nvoc_export_info_P2PTokenShare
};
const struct NVOC_EXPORT_INFO __nvoc_export_info_P2PTokenShare =
{
/*numEntries=*/ 0,
/*pExportEntries=*/ 0
};
void __nvoc_dtor_RsShared(RsShared*);
void __nvoc_dtor_P2PTokenShare(P2PTokenShare *pThis) {
__nvoc_shrp2pDestruct(pThis);
__nvoc_dtor_RsShared(&pThis->__nvoc_base_RsShared);
PORT_UNREFERENCED_VARIABLE(pThis);
}
void __nvoc_init_dataField_P2PTokenShare(P2PTokenShare *pThis) {
PORT_UNREFERENCED_VARIABLE(pThis);
}
NV_STATUS __nvoc_ctor_RsShared(RsShared* );
NV_STATUS __nvoc_ctor_P2PTokenShare(P2PTokenShare *pThis) {
NV_STATUS status = NV_OK;
status = __nvoc_ctor_RsShared(&pThis->__nvoc_base_RsShared);
if (status != NV_OK) goto __nvoc_ctor_P2PTokenShare_fail_RsShared;
__nvoc_init_dataField_P2PTokenShare(pThis);
status = __nvoc_shrp2pConstruct(pThis);
if (status != NV_OK) goto __nvoc_ctor_P2PTokenShare_fail__init;
goto __nvoc_ctor_P2PTokenShare_exit; // Success
__nvoc_ctor_P2PTokenShare_fail__init:
__nvoc_dtor_RsShared(&pThis->__nvoc_base_RsShared);
__nvoc_ctor_P2PTokenShare_fail_RsShared:
__nvoc_ctor_P2PTokenShare_exit:
return status;
}
static void __nvoc_init_funcTable_P2PTokenShare_1(P2PTokenShare *pThis) {
PORT_UNREFERENCED_VARIABLE(pThis);
}
void __nvoc_init_funcTable_P2PTokenShare(P2PTokenShare *pThis) {
__nvoc_init_funcTable_P2PTokenShare_1(pThis);
}
void __nvoc_init_RsShared(RsShared*);
void __nvoc_init_P2PTokenShare(P2PTokenShare *pThis) {
pThis->__nvoc_pbase_P2PTokenShare = pThis;
pThis->__nvoc_pbase_Object = &pThis->__nvoc_base_RsShared.__nvoc_base_Object;
pThis->__nvoc_pbase_RsShared = &pThis->__nvoc_base_RsShared;
__nvoc_init_RsShared(&pThis->__nvoc_base_RsShared);
__nvoc_init_funcTable_P2PTokenShare(pThis);
}
NV_STATUS __nvoc_objCreate_P2PTokenShare(P2PTokenShare **ppThis, Dynamic *pParent, NvU32 createFlags) {
NV_STATUS status;
Object *pParentObj;
P2PTokenShare *pThis;
pThis = portMemAllocNonPaged(sizeof(P2PTokenShare));
if (pThis == NULL) return NV_ERR_NO_MEMORY;
portMemSet(pThis, 0, sizeof(P2PTokenShare));
__nvoc_initRtti(staticCast(pThis, Dynamic), &__nvoc_class_def_P2PTokenShare);
if (pParent != NULL && !(createFlags & NVOC_OBJ_CREATE_FLAGS_PARENT_HALSPEC_ONLY))
{
pParentObj = dynamicCast(pParent, Object);
objAddChild(pParentObj, &pThis->__nvoc_base_RsShared.__nvoc_base_Object);
}
else
{
pThis->__nvoc_base_RsShared.__nvoc_base_Object.pParent = NULL;
}
__nvoc_init_P2PTokenShare(pThis);
status = __nvoc_ctor_P2PTokenShare(pThis);
if (status != NV_OK) goto __nvoc_objCreate_P2PTokenShare_cleanup;
*ppThis = pThis;
return NV_OK;
__nvoc_objCreate_P2PTokenShare_cleanup:
// do not call destructors here since the constructor already called them
portMemFree(pThis);
return status;
}
NV_STATUS __nvoc_objCreateDynamic_P2PTokenShare(P2PTokenShare **ppThis, Dynamic *pParent, NvU32 createFlags, va_list args) {
NV_STATUS status;
status = __nvoc_objCreate_P2PTokenShare(ppThis, pParent, createFlags);
return status;
}
#ifdef DEBUG
char __nvoc_class_id_uniqueness_check_0x34d08b = 1;
#endif

63
src/nvidia/generated/g_third_party_p2p_nvoc.h
View File

@@ -179,6 +179,68 @@ struct _def_client_third_party_p2p_pid_client_mapping_info
};
typedef struct _def_client_third_party_p2p_pid_client_mapping_info CLI_THIRD_PARTY_P2P_PID_CLIENT_INFO, *PCLI_THIRD_PARTY_P2P_PID_CLIENT_INFO;
struct ThirdPartyP2P;
#ifndef __NVOC_CLASS_ThirdPartyP2P_TYPEDEF__
#define __NVOC_CLASS_ThirdPartyP2P_TYPEDEF__
typedef struct ThirdPartyP2P ThirdPartyP2P;
#endif /* __NVOC_CLASS_ThirdPartyP2P_TYPEDEF__ */
#ifndef __nvoc_class_id_ThirdPartyP2P
#define __nvoc_class_id_ThirdPartyP2P 0x34d08b
#endif /* __nvoc_class_id_ThirdPartyP2P */
#ifdef NVOC_THIRD_PARTY_P2P_H_PRIVATE_ACCESS_ALLOWED
#define PRIVATE_FIELD(x) x
#else
#define PRIVATE_FIELD(x) NVOC_PRIVATE_FIELD(x)
#endif
struct P2PTokenShare {
const struct NVOC_RTTI *__nvoc_rtti;
struct RsShared __nvoc_base_RsShared;
struct Object *__nvoc_pbase_Object;
struct RsShared *__nvoc_pbase_RsShared;
struct P2PTokenShare *__nvoc_pbase_P2PTokenShare;
struct ThirdPartyP2P *pThirdPartyP2P;
};
#ifndef __NVOC_CLASS_P2PTokenShare_TYPEDEF__
#define __NVOC_CLASS_P2PTokenShare_TYPEDEF__
typedef struct P2PTokenShare P2PTokenShare;
#endif /* __NVOC_CLASS_P2PTokenShare_TYPEDEF__ */
#ifndef __nvoc_class_id_P2PTokenShare
#define __nvoc_class_id_P2PTokenShare 0x3e3a6a
#endif /* __nvoc_class_id_P2PTokenShare */
extern const struct NVOC_CLASS_DEF __nvoc_class_def_P2PTokenShare;
#define __staticCast_P2PTokenShare(pThis) \
((pThis)->__nvoc_pbase_P2PTokenShare)
#ifdef __nvoc_third_party_p2p_h_disabled
#define __dynamicCast_P2PTokenShare(pThis) ((P2PTokenShare*)NULL)
#else //__nvoc_third_party_p2p_h_disabled
#define __dynamicCast_P2PTokenShare(pThis) \
((P2PTokenShare*)__nvoc_dynamicCast(staticCast((pThis), Dynamic), classInfo(P2PTokenShare)))
#endif //__nvoc_third_party_p2p_h_disabled
NV_STATUS __nvoc_objCreateDynamic_P2PTokenShare(P2PTokenShare**, Dynamic*, NvU32, va_list);
NV_STATUS __nvoc_objCreate_P2PTokenShare(P2PTokenShare**, Dynamic*, NvU32);
#define __objCreate_P2PTokenShare(ppNewObj, pParent, createFlags) \
__nvoc_objCreate_P2PTokenShare((ppNewObj), staticCast((pParent), Dynamic), (createFlags))
NV_STATUS shrp2pConstruct_IMPL(struct P2PTokenShare *arg_pP2PTokenShare);
#define __nvoc_shrp2pConstruct(arg_pP2PTokenShare) shrp2pConstruct_IMPL(arg_pP2PTokenShare)
void shrp2pDestruct_IMPL(struct P2PTokenShare *pP2PTokenShare);
#define __nvoc_shrp2pDestruct(pP2PTokenShare) shrp2pDestruct_IMPL(pP2PTokenShare)
#undef PRIVATE_FIELD
#ifdef NVOC_THIRD_PARTY_P2P_H_PRIVATE_ACCESS_ALLOWED
#define PRIVATE_FIELD(x) x
@@ -235,6 +297,7 @@ struct ThirdPartyP2P {
THIRD_PARTY_P2P_DESTROY_CALLBACK *pDestroyCallback;
void *pData;
CLI_THIRD_PARTY_P2P_VIDMEM_INFO_MAP vidmemInfoMap;
struct P2PTokenShare *pTokenShare;
PNODE pAddressRangeTree;
};

10
src/nvidia/inc/kernel/gpu/external_device/dac_p2060.h
View File

@@ -50,12 +50,17 @@
#define NV_P2060_START_DELAY_MAX_UNITS 65535
#define NV_P2060_START_DELAY_RESOLUTION 7800
#define NV_P2060_SYNC_INTERVAL_MAX_UNITS 7
#define NV_P2061_V204_SYNC_SKEW_RESOLUTION 7 // For 2061 V2.04+
#define NV_P2061_V204_SYNC_SKEW_MAX_UNITS 0xFFFFFF // For 2061 V2.04+
#define NV_P2061_V204_SYNC_SKEW_INVALID (NV_P2061_V204_SYNC_SKEW_MAX_UNITS + 1)
#define NV_P2060_WATCHDOG_COUNT_DOWN_VALUE 60 // 1 minute, assuming watchdog time interval is 1 second.
#define NV_P2060_FRAME_COUNT_TIMER_INTERVAL 5000000000LL // 5 sec
#define NV_P2060_MAX_GPU_FRAME_COUNT 65535
#define NV_P2060_MAX_GSYNC_FRAME_COUNT 16777215 // 2^24.Gsync frame count is a 24 bit register
#define P2061_FW_REV(pExtDev) ((pExtDev->deviceRev << 8) | (pExtDev->deviceExRev))
/* ------------------------ Types definitions ------------------------------ */
typedef struct EXTDEV_I2C_HANDLES
@@ -112,6 +117,10 @@ struct DACP2060EXTERNALDEVICE
* swap ready line will remain high.(Provided via a regkey)
*/
NvU32 syncSkewResolutionInNs; // resolution in ns
NvU32 syncSkewMax; // max syncSkew setting in raw units
NvU32 lastUserSkewSent; // Remember the last Sync Skew value sent by the user
struct {
NvU32 currentFrameCount; // gpu frame count register value for current user query
NvU32 previousFrameCount; // gpu frame count register value for previous user query
@@ -239,5 +248,6 @@ NV_STATUS gsyncGetStereoLockMode_P2060 (OBJGPU *, PDACEXTERNALDEVICE, NvU3
NV_STATUS gsyncSetStereoLockMode_P2060 (OBJGPU *, PDACEXTERNALDEVICE, NvU32);
NV_STATUS gsyncSetMosaic_P2060 (OBJGPU *, PDACEXTERNALDEVICE, NV30F1_CTRL_GSYNC_SET_LOCAL_SYNC_PARAMS *);
NV_STATUS gsyncConfigFlashGsync_P2060 (OBJGPU *, PDACEXTERNALDEVICE, NvU32);
NvBool gsyncSupportsLargeSyncSkew_P2060 (DACEXTERNALDEVICE *);
#endif

8
src/nvidia/inc/kernel/gpu/external_device/dac_p2061.h
View File

@@ -28,8 +28,10 @@
// P2061 uses P2060's object.
// P2061 hal ifaces
NV_STATUS gsyncGetHouseSyncMode_P2061 (OBJGPU *, PDACEXTERNALDEVICE, NvU8*);
NV_STATUS gsyncSetHouseSyncMode_P2061 (OBJGPU *, PDACEXTERNALDEVICE, NvU8);
NV_STATUS gsyncGetCplStatus_P2061 (OBJGPU *, PDACEXTERNALDEVICE, GSYNCSTATUS, NvU32 *);
NV_STATUS gsyncGetHouseSyncMode_P2061(OBJGPU *, DACEXTERNALDEVICE *, NvU8*);
NV_STATUS gsyncSetHouseSyncMode_P2061(OBJGPU *, DACEXTERNALDEVICE *, NvU8);
NV_STATUS gsyncGetCplStatus_P2061 (OBJGPU *, DACEXTERNALDEVICE *, GSYNCSTATUS, NvU32 *);
NV_STATUS gsyncSetSyncSkew_P2061_V204(OBJGPU *, DACEXTERNALDEVICE *, NvU32);
NV_STATUS gsyncGetSyncSkew_P2061_V204(OBJGPU *, DACEXTERNALDEVICE *, NvU32 *);
#endif // DAC_P2061_H

5
src/nvidia/interface/nvrm_registry.h
View File

@@ -1736,4 +1736,9 @@
#define NV_REG_STR_BUG_3007008_EMULATE_VF_MMU_TLB_INVALIDATE_DISABLE 0x00000000
#define NV_REG_STR_BUG_3007008_EMULATE_VF_MMU_TLB_INVALIDATE_DEFAULT NV_REG_STR_BUG_3007008_EMULATE_VF_MMU_TLB_INVALIDATE_ENABLE
#define NV_REG_STR_RM_CLIENT_HANDLE_LOOKUP "RmClientHandleLookup"
// Type DWORD (Boolean)
// 1 - Store active RM clients in a multimap to speed up lookups (currently only in thirdpartyp2p)
// 0 - (Default) Linear list search for clients
#endif // NVRM_REGISTRY_H

4
src/nvidia/kernel/inc/dev_p2060.h
View File

@@ -209,6 +209,10 @@
#define NV_P2060_SYNC_SKEW_HIGH_VAL 7:0 /* RWIVF */
#define NV_P2060_SYNC_SKEW_HIGH_VAL_0 0x00 /* RWI-V */
#define NV_P2060_SYNC_SKEW_UPPER 0x35 /* RW-1R */
#define NV_P2060_SYNC_SKEW_UPPER_VAL 7:0 /* RWIVF */
#define NV_P2060_SYNC_SKEW_UPPER_VAL_0 0x00 /* RWI-V */
#define NV_P2060_START_DELAY_LOW 0x0A /* RW-1R */
#define NV_P2060_START_DELAY_LOW_VAL 7:0 /* RWIVF */
#define NV_P2060_START_DELAY_LOW_VAL_0 0x00 /* RWI-V */

1
src/nvidia/kernel/inc/vgpu/rpc_global_enums.h
View File

@@ -247,6 +247,7 @@ enum {
E(NVLINK_IS_GPU_DEGRADED) // 0x1019
E(PFM_REQ_HNDLR_STATE_SYNC_CALLBACK) // 0x101a
E(NVLINK_FAULT_UP) // 0x101c
E(GSP_LOCKDOWN_NOTICE) // 0x101d
E(NUM_EVENTS) // END
#ifdef DEFINING_E_IN_RPC_GLOBAL_ENUMS_H
};

7
src/nvidia/src/kernel/core/system.c
View File

@@ -172,6 +172,7 @@ sysDestruct_IMPL(OBJSYS *pSys)
//
listDestroy(&g_clientListBehindGpusLock);
listDestroy(&g_userInfoList);
multimapDestroy(&g_osInfoList);
rmapiShutdown();
osSyncWithRmDestroy();
@@ -318,6 +319,12 @@ _sysRegistryOverrideResourceServer
{
pSys->apiLockModuleMask = RM_LOCK_MODULE_GRP(RM_LOCK_MODULES_CLIENT);
}
if (osReadRegistryDword(pGpu, NV_REG_STR_RM_CLIENT_HANDLE_LOOKUP,
&data32) == NV_OK)
{
pSys->setProperty(pSys, PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP, !!data32);
}
}
static void

124
src/nvidia/src/kernel/gpu/bus/third_party_p2p.c
View File

@@ -66,6 +66,8 @@ thirdpartyp2pConstruct_IMPL
NvU32 pidIndex = 0;
NV_STATUS status = NV_OK;
NvU32 pid = osGetCurrentProcess();
RsShared *pShare;
P2PTokenShare *pP2PTokenShare;
pSubdevice = dynamicCast(pSubdeviceRef->pResource, Subdevice);
if (pSubdevice == NULL)
@@ -153,6 +155,15 @@ thirdpartyp2pConstruct_IMPL
}
}
status = serverAllocShare(&g_resServ, classInfo(P2PTokenShare), &pShare);
if (status != NV_OK)
return status;
pP2PTokenShare = dynamicCast(pShare, P2PTokenShare);
pP2PTokenShare->pThirdPartyP2P = pThirdPartyP2P;
pThirdPartyP2P->pTokenShare = pP2PTokenShare;
NV_ASSERT(status == NV_OK);
return status;
}
@@ -178,6 +189,9 @@ thirdpartyp2pDestruct_IMPL
resGetFreeParams(staticCast(pThirdPartyP2P, RsResource), &pCallContext, &pParams);
if (pThirdPartyP2P->pTokenShare)
serverFreeShare(&g_resServ, staticCast(pThirdPartyP2P->pTokenShare, RsShared));
pParams->status = gpuFullPowerSanityCheck(pGpu, NV_TRUE);
if (pParams->status != NV_OK)
{
@@ -239,29 +253,24 @@ NV_STATUS CliGetThirdPartyP2PInfoFromToken
)
{
ThirdPartyP2P *pThirdPartyP2P;
RmClient **ppClient;
RmClient *pClient;
RS_SHARE_ITERATOR it;
NV_ASSERT_OR_RETURN((ppThirdPartyP2P != NULL), NV_ERR_INVALID_ARGUMENT);
for (ppClient = serverutilGetFirstClientUnderLock();
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
{
RS_ITERATOR it;
RsClient *pRsClient;
pClient = *ppClient;
pRsClient = staticCast(pClient, RsClient);
it = serverutilShareIter(classId(P2PTokenShare));
it = clientRefIter(pRsClient, NULL, classId(ThirdPartyP2P), RS_ITERATE_DESCENDANTS, NV_TRUE);
while (clientRefIterNext(pRsClient, &it))
while(serverutilShareIterNext(&it))
{
RsShared *pShared = it.pShared;
P2PTokenShare *pP2PTokenShare = dynamicCast(pShared, P2PTokenShare);
if (pP2PTokenShare == NULL)
continue;
pThirdPartyP2P = pP2PTokenShare->pThirdPartyP2P;
if (pThirdPartyP2P->p2pToken == p2pToken)
{
pThirdPartyP2P = dynamicCast(it.pResourceRef->pResource, ThirdPartyP2P);
if (pThirdPartyP2P->p2pToken == p2pToken)
{
*ppThirdPartyP2P = pThirdPartyP2P;
return NV_OK;
}
*ppThirdPartyP2P = pThirdPartyP2P;
return NV_OK;
}
}
@@ -314,62 +323,35 @@ NV_STATUS CliNextThirdPartyP2PInfoWithPid
)
{
ThirdPartyP2P *pThirdPartyP2P;
RmClient **ppClient;
RmClient *pClient;
RS_SHARE_ITERATOR it;
for (ppClient = serverutilGetFirstClientUnderLock();
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
it = serverutilShareIter(classId(P2PTokenShare));
while(serverutilShareIterNext(&it))
{
RsClient *pRsClient;
RS_ITERATOR it, devIt, subDevIt;
pClient = *ppClient;
pRsClient = staticCast(pClient, RsClient);
if (pRsClient->type == CLIENT_TYPE_KERNEL)
{
RsShared *pShared = it.pShared;
P2PTokenShare *pP2PTokenShare = dynamicCast(pShared, P2PTokenShare);
if (pP2PTokenShare == NULL)
continue;
}
devIt = clientRefIter(pRsClient, NULL, classId(Device),
RS_ITERATE_CHILDREN, NV_TRUE);
while(clientRefIterNext(pRsClient, &devIt))
pThirdPartyP2P = pP2PTokenShare->pThirdPartyP2P;
if (NULL == *ppThirdPartyP2P)
{
Device *pDevice = dynamicCast(devIt.pResourceRef->pResource, Device);
OBJGPU *pGpuFromDevice = GPU_RES_GET_GPU(pDevice);
if ((pGpu != NULL) && (pGpu != pGpuFromDevice))
if (thirdpartyp2pIsValidClientPid(pThirdPartyP2P, pid, hClient))
{
continue;
}
subDevIt = clientRefIter(pRsClient, devIt.pResourceRef, classId(Subdevice),
RS_ITERATE_CHILDREN, NV_TRUE);
while(clientRefIterNext(pRsClient, &subDevIt))
{
it = clientRefIter(pRsClient, subDevIt.pResourceRef,
classId(ThirdPartyP2P), RS_ITERATE_CHILDREN, NV_TRUE);
while (clientRefIterNext(pRsClient, &it))
{
pThirdPartyP2P = dynamicCast(it.pResourceRef->pResource, ThirdPartyP2P);
if (NULL == *ppThirdPartyP2P)
{
if (thirdpartyp2pIsValidClientPid(pThirdPartyP2P, pid, hClient))
{
*ppClientOut = pClient;
*ppThirdPartyP2P = pThirdPartyP2P;
return NV_OK;
}
}
else if (pThirdPartyP2P->p2pToken ==
(*ppThirdPartyP2P)->p2pToken)
{
*ppClientOut = NULL;
*ppThirdPartyP2P = NULL;
}
}
RsClient *pClient = RES_GET_CLIENT(pThirdPartyP2P);
*ppClientOut = dynamicCast(pClient, RmClient);
*ppThirdPartyP2P = pThirdPartyP2P;
return NV_OK;
}
}
else if (pThirdPartyP2P->p2pToken ==
(*ppThirdPartyP2P)->p2pToken)
{
*ppClientOut = NULL;
*ppThirdPartyP2P = NULL;
}
}
return NV_ERR_OBJECT_NOT_FOUND;
@@ -1174,6 +1156,16 @@ CliUnregisterFromThirdPartyP2P
return status;
}
NV_STATUS
shrp2pConstruct_IMPL(P2PTokenShare *pP2PTokenShare)
{
return NV_OK;
}
void shrp2pDestruct_IMPL(P2PTokenShare *pP2PTokenShare)
{
}
void
CliUnregisterMemoryFromThirdPartyP2P
(

27
src/nvidia/src/kernel/gpu/external_device/arch/kepler/kern_gsync_p2060.c
View File

@@ -79,7 +79,6 @@ static NV_STATUS gsyncFrameCountTimerService_P2060(OBJGPU *, OBJTMR *, void *);
static NV_STATUS gsyncResetFrameCountData_P2060(OBJGPU *, PDACP2060EXTERNALDEVICE);
static NV_STATUS gsyncGpuStereoHeadSync(OBJGPU *, NvU32, PDACEXTERNALDEVICE, NvU32);
static NvBool supportsLargeSyncSkew(PDACEXTERNALDEVICE);
static NvBool needsMasterBarrierWar(PDACEXTERNALDEVICE);
static NvBool isFirmwareRevMismatch(OBJGPU *, DAC_EXTERNAL_DEVICE_REVS);
@@ -1921,7 +1920,7 @@ gsyncSetSyncSkew_P2060
// update p2060 object
pThis->SyncSkew = SyncSkew;
if (supportsLargeSyncSkew(pExtDev))
if (gsyncSupportsLargeSyncSkew_P2060(pExtDev))
{
SyncSkewLow = (NvU8)((SyncSkew ) & DRF_MASK(NV_P2060_SYNC_SKEW_LOW_VAL ));
SyncSkewHigh = (NvU8)((SyncSkew >> 8) & DRF_MASK(NV_P2060_SYNC_SKEW_HIGH_VAL));
@@ -4104,6 +4103,8 @@ gsyncGetRevision_P2060
(deviceId == DAC_EXTERNAL_DEVICE_P2060 ||
deviceId == DAC_EXTERNAL_DEVICE_P2061))
{
DACP2060EXTERNALDEVICE *p2060 = (DACP2060EXTERNALDEVICE *)pExtDev;
pParams->capFlags = NV30F1_CTRL_GSYNC_GET_CAPS_CAP_FLAGS_FREQ_ACCURACY_3DPS;
if (!pSys->getProperty(pSys, PDB_PROP_SYS_IS_QSYNC_FW_REVISION_CHECK_DISABLED))
@@ -4115,19 +4116,11 @@ gsyncGetRevision_P2060
pParams->isFirmwareRevMismatch = NV_FALSE;
}
if (supportsLargeSyncSkew(pExtDev))
{
pParams->maxSyncSkew = NV_P2060_SYNC_SKEW_MAX_UNITS_FULL_SUPPORT;
}
else
{
pParams->maxSyncSkew = NV_P2060_SYNC_SKEW_MAX_UNITS_LIMITED_SUPPORT;
}
pParams->syncSkewResolution = NV_P2060_SYNC_SKEW_RESOLUTION;
pParams->maxStartDelay = NV_P2060_START_DELAY_MAX_UNITS;
pParams->maxSyncSkew = p2060->syncSkewMax;
pParams->syncSkewResolution = p2060->syncSkewResolutionInNs;
pParams->maxStartDelay = NV_P2060_START_DELAY_MAX_UNITS;
pParams->startDelayResolution = NV_P2060_START_DELAY_RESOLUTION;
pParams->maxSyncInterval = NV_P2060_SYNC_INTERVAL_MAX_UNITS;
pParams->maxSyncInterval = NV_P2060_SYNC_INTERVAL_MAX_UNITS;
// let client know which events we support
pParams->capFlags |= NV30F1_CTRL_GSYNC_GET_CAPS_CAP_FLAGS_SYNC_LOCK_EVENT;
@@ -5356,10 +5349,10 @@ gsyncGetNumberOfGpuFrameCountRollbacks_P2060
}
// Return NV_TRUE if the current Qsync revision supports large sync skew
static NvBool
supportsLargeSyncSkew
NvBool
gsyncSupportsLargeSyncSkew_P2060
(
PDACEXTERNALDEVICE pExtdev
DACEXTERNALDEVICE *pExtdev
)
{
if (pExtdev->deviceId == DAC_EXTERNAL_DEVICE_P2061)

115
src/nvidia/src/kernel/gpu/external_device/arch/pascal/kern_gsync_p2061.c
View File

@@ -140,3 +140,118 @@ gsyncGetCplStatus_P2061
return status;
}
// Write to the SyncSkew register
NV_STATUS
gsyncSetSyncSkew_P2061_V204
(
OBJGPU *pGpu,
DACEXTERNALDEVICE *pExtDev,
NvU32 syncSkew
)
{
DACP2060EXTERNALDEVICE *pThis = (DACP2060EXTERNALDEVICE *)pExtDev;
NvU64 temp;
NV_CHECK_OR_RETURN(LEVEL_INFO,
syncSkew <= NV_P2061_V204_SYNC_SKEW_MAX_UNITS,
NV_ERR_INVALID_ARGUMENT);
//
// Cache the unmodified syncSkew value from the user.
// We use this later to improve the stability of returned values in GetSyncSkew.
//
pThis->lastUserSkewSent = syncSkew;
//
// Fix the skew value which goes to HW, because we passed the resolution up
// to NVAPI as 7us. However, the real resolution in hardware is
// (1 / 131.072 MHz) = 7.6293945...
// Fix it by multiplying by the ratio 7 / 7.6293945
//
temp = syncSkew;
temp *= 70000000; // Safe because temp is 64 bits
temp /= 76293945;
syncSkew = (NvU32)temp; // Safe because we multiplied by less than 1.
// update p2060 object
pThis->SyncStartDelay = syncSkew;
NV_ASSERT_OK_OR_RETURN(
writeregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_LOW,
syncSkew & DRF_MASK(NV_P2060_SYNC_SKEW_LOW_VAL)));
NV_ASSERT_OK_OR_RETURN(
writeregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_HIGH,
(syncSkew >> 8) & DRF_MASK(NV_P2060_SYNC_SKEW_HIGH_VAL)));
NV_ASSERT_OK_OR_RETURN(
writeregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_UPPER,
(syncSkew >> 16) & DRF_MASK(NV_P2060_SYNC_SKEW_UPPER_VAL)));
return NV_OK;
}
// Read from the SyncSkew register
NV_STATUS
gsyncGetSyncSkew_P2061_V204
(
OBJGPU *pGpu,
DACEXTERNALDEVICE *pExtDev,
NvU32 *pSyncSkew
)
{
NvU8 data;
NvU32 syncSkew;
NvU64 temp;
DACP2060EXTERNALDEVICE *pThis = (DACP2060EXTERNALDEVICE *)pExtDev;
*pSyncSkew = 0;
NV_ASSERT_OK_OR_RETURN(readregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_LOW, &data));
syncSkew = DRF_VAL(_P2060, _SYNC_SKEW_LOW, _VAL, data);
NV_ASSERT_OK_OR_RETURN(readregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_HIGH, &data));
syncSkew |= DRF_VAL(_P2060, _SYNC_SKEW_HIGH, _VAL, data) << 8;
NV_ASSERT_OK_OR_RETURN(readregu008_extdeviceTargeted(pGpu, pExtDev, (NvU8)NV_P2060_SYNC_SKEW_UPPER, &data));
syncSkew |= DRF_VAL(_P2060, _SYNC_SKEW_UPPER, _VAL, data) << 16;
// update p2060 object
pThis->SyncStartDelay = syncSkew;
//
// Perform the opposite adjustment for returning the value to the user.
// Multiply by the ratio 7.6293945 / 7
//
temp = syncSkew;
temp *= 76293945; // Safe because temp is 64 bits
temp /= 70000000;
syncSkew = (NvU32)temp; // Safe because the HW register is only 24 bits wide to begin with
//
// Could be higher than maximum because we multiplied by greater than 1,
// but this would only happen by someone manually programming the register
//
NV_CHECK_OR_ELSE(LEVEL_INFO,
syncSkew <= NV_P2061_V204_SYNC_SKEW_MAX_UNITS,
syncSkew = NV_P2061_V204_SYNC_SKEW_MAX_UNITS);
//
// Returning this value with stability to the user:
// The programmed value can change slightly due to divisions, so if we read
// from the hardware and find a value that is very close, just return the
// last sent value. This assures stability even through multiple cycles of
// set-get-set-get.
//
if (pThis->lastUserSkewSent != NV_P2061_V204_SYNC_SKEW_INVALID)
{
NvS32 difference;
difference = syncSkew - pThis->lastUserSkewSent;
if ((difference >= -2) && (difference <= 2))
{
syncSkew = pThis->lastUserSkewSent;
}
}
*pSyncSkew = syncSkew;
return NV_OK;
}

22
src/nvidia/src/kernel/gpu/external_device/gsync.c
View File

@@ -192,6 +192,8 @@ static OBJGPU *gsyncGetMasterableGpu(OBJGSYNC *pGsync)
static NV_STATUS
gsyncP2060StartupProvider(OBJGSYNC *pGsync)
{
DACEXTERNALDEVICE *pExtDev = pGsync->pExtDev;
DACP2060EXTERNALDEVICE *p2060 = (DACP2060EXTERNALDEVICE *)pGsync->pExtDev;
// All four GPU connectors are masterable on P2060
pGsync->masterableGpuConnectors = ((1 << NV30F1_GSYNC_CONNECTOR_ONE) |
@@ -235,6 +237,12 @@ gsyncP2060StartupProvider(OBJGSYNC *pGsync)
pGsync->gsyncHal.gsyncSetMosaic = gsyncSetMosaic_P2060;
pGsync->gsyncHal.gsyncConfigFlashGsync = gsyncConfigFlashGsync_P2060;
// Constants to be returned in NV30F1_CTRL_GSYNC_GET_CAPS
p2060->syncSkewResolutionInNs = NV_P2060_SYNC_SKEW_RESOLUTION;
p2060->syncSkewMax = gsyncSupportsLargeSyncSkew_P2060(pExtDev) ?
NV_P2060_SYNC_SKEW_MAX_UNITS_FULL_SUPPORT :
NV_P2060_SYNC_SKEW_MAX_UNITS_LIMITED_SUPPORT;
return NV_OK;
}
@@ -245,6 +253,8 @@ static NV_STATUS
gsyncP2061StartupProvider(OBJGSYNC *pGsync)
{
NV_STATUS status;
DACP2060EXTERNALDEVICE *p2060 = (DACP2060EXTERNALDEVICE *)pGsync->pExtDev;
// Call P2060 startup provider for setting up those HALs that
// are identical to P2060.
status = gsyncP2060StartupProvider(pGsync);
@@ -256,6 +266,18 @@ gsyncP2061StartupProvider(OBJGSYNC *pGsync)
pGsync->gsyncHal.gsyncGetHouseSyncMode = gsyncGetHouseSyncMode_P2061;
pGsync->gsyncHal.gsyncSetHouseSyncMode = gsyncSetHouseSyncMode_P2061;
// SyncSkew is different for FW V2.04+
if (P2061_FW_REV(pGsync->pExtDev) >= 0x204)
{
pGsync->gsyncHal.gsyncGetSyncSkew = gsyncGetSyncSkew_P2061_V204;
pGsync->gsyncHal.gsyncSetSyncSkew = gsyncSetSyncSkew_P2061_V204;
// Constants to be returned in NV30F1_CTRL_GSYNC_GET_CAPS
p2060->syncSkewResolutionInNs = NV_P2061_V204_SYNC_SKEW_RESOLUTION;
p2060->syncSkewMax = NV_P2061_V204_SYNC_SKEW_MAX_UNITS;
p2060->lastUserSkewSent = NV_P2061_V204_SYNC_SKEW_INVALID;
}
return status;
}

6
src/nvidia/src/kernel/gpu/gsp/arch/turing/kernel_gsp_tu102.c
View File

@@ -843,6 +843,12 @@ kgspService_TU102
DRF_DEF(_PFALCON, _FALCON_IRQSCLR, _SWGEN0, _SET));
kgspRpcRecvEvents(pGpu, pKernelGsp);
//
// If lockdown has been engaged (as notified by an RPC event),
// we shouldn't access any more GSP registers.
//
NV_CHECK_OR_RETURN(LEVEL_SILENT, !pKernelGsp->bInLockdown, 0);
}
// Clear any sources that were serviced and get the new status

25
src/nvidia/src/kernel/gpu/gsp/kernel_gsp.c
View File

@@ -803,6 +803,27 @@ _kgspRpcEventPlatformRequestHandlerStateSyncCallback
return NV_OK;
}
static void
_kgspRpcGspLockdownNotice
(
OBJGPU *pGpu,
OBJRPC *pRpc
)
{
KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu);
RPC_PARAMS(gsp_lockdown_notice, _v17_00);
//
// While the GSP is in lockdown, we cannot access some of its registers,
// including interrupt status and control. We shouldn't receive any more
// SWGEN0 interrupts while the core is in lockdown.
//
pKernelGsp->bInLockdown = rpc_params->bLockdownEngaging;
NV_PRINTF(LEVEL_INFO, "GSP lockdown %s\n",
pKernelGsp->bInLockdown ? "engaged" : "disengaged");
}
static
const char *_getRpcName
(
@@ -946,6 +967,10 @@ _kgspProcessRpcEvent
nvStatus = _kgspRpcEventPlatformRequestHandlerStateSyncCallback(pGpu, pRpc);
break;
case NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE:
_kgspRpcGspLockdownNotice(pGpu, pRpc);
break;
case NV_VGPU_MSG_EVENT_GSP_INIT_DONE: // Handled by _kgspRpcRecvPoll.
default:
//

2
src/nvidia/src/kernel/platform/acpi_common.c
View File

@@ -780,6 +780,8 @@ _acpiDsmSupportedFuncCacheInit
// Just checking to make sure this is correct!
NV_ASSERT_OR_RETURN_VOID(0 == ACPI_DSM_FUNCTION_NBSI);
portMemSet(supportFuncs, 0, sizeof(supportFuncs));
//
// loop through all guids. The read will cache the subfunction list (if
// available)

89
src/nvidia/src/kernel/rmapi/client.c
View File

@@ -39,13 +39,16 @@
#include "gpu/bus/third_party_p2p.h"
#include "virtualization/hypervisor/hypervisor.h"
OsInfoMap g_osInfoList;
UserInfoList g_userInfoList;
RmClientList g_clientListBehindGpusLock; // RS-TODO remove this WAR
#define RS_FW_UNIQUE_HANDLE_BASE (0xc9f00000)
NV_STATUS _registerUserInfo(PUID_TOKEN *ppUidToken, UserInfo **ppUserInfo);
NV_STATUS _unregisterUserInfo(UserInfo *pUserInfo);
static NV_STATUS _registerUserInfo(PUID_TOKEN *ppUidToken, UserInfo **ppUserInfo);
static NV_STATUS _unregisterUserInfo(UserInfo *pUserInfo);
static NV_STATUS _registerOSInfo(RmClient *pClient, void *pOSInfo);
static NV_STATUS _unregisterOSInfo(RmClient *pClient, void *pOSInfo);
NV_STATUS
rmclientConstruct_IMPL
@@ -150,6 +153,8 @@ rmclientConstruct_IMPL
bReleaseLock = NV_TRUE;
}
_registerOSInfo(pClient, pClient->pOSInfo);
pClient->bIsClientVirtualMode = (pSecInfo->pProcessToken != NULL);
//
@@ -262,6 +267,8 @@ rmclientDestruct_IMPL
}
}
_unregisterOSInfo(pClient, pClient->pOSInfo);
listRemoveFirstByValue(&g_clientListBehindGpusLock, (void*)&pClient);
if (pClient->pUserInfo != NULL)
@@ -622,7 +629,7 @@ NV_STATUS rmclientUserClientSecurityCheckByHandle(NvHandle hClient, const API_SE
* @param[inout] ppUidToken
* @param[out] ppUserInfo
*/
NV_STATUS
static NV_STATUS
_registerUserInfo
(
PUID_TOKEN *ppUidToken,
@@ -687,7 +694,7 @@ _registerUserInfo
*
* @param[in] pUserInfo
*/
NV_STATUS
static NV_STATUS
_unregisterUserInfo
(
UserInfo *pUserInfo
@@ -889,3 +896,77 @@ NvBool rmclientIsCapableByHandle
return rmclientIsCapable(pClient, capability);
}
/**
*
* Register a client with a user info list
*
* This function must be protected by a lock (currently the GPUs lock.)
*
* @param[in] pClient
* @param[in] pOSInfo
*/
static NV_STATUS
_registerOSInfo
(
RmClient *pClient,
void *pOSInfo
)
{
OsInfoMapSubmap *pSubmap = NULL;
RmClient **pInsert = NULL;
NvU64 key1 = (NvUPtr)pOSInfo;
NvU64 key2 = (NvU64)(staticCast(pClient,RsClient))->hClient;
if (multimapFindItem(&g_osInfoList, key1, key2) != NULL)
return NV_ERR_INSERT_DUPLICATE_NAME;
if (multimapFindSubmap(&g_osInfoList, key1) == NULL)
{
pSubmap = multimapInsertSubmap(&g_osInfoList, key1);
if (pSubmap == NULL)
return NV_ERR_NO_MEMORY;
}
pInsert = multimapInsertItemNew(&g_osInfoList, key1, key2);
if (pInsert == NULL)
return NV_ERR_NO_MEMORY;
*pInsert = pClient;
return NV_OK;
}
/**
*
* Unregister a client from a user info list
*
* This function must be protected by a lock (currently the GPUs lock.)
*
* @param[in] pClient
* @param[in] pOSInfo
*/
static NV_STATUS
_unregisterOSInfo
(
RmClient *pClient,
void *pOSInfo
)
{
NvU64 key1 = (NvUPtr)pOSInfo;
NvU64 key2 = (NvU64)(staticCast(pClient, RsClient))->hClient;
OsInfoMapSubmap *pSubmap = NULL;
RmClient **pFind = NULL;
pFind = multimapFindItem(&g_osInfoList, key1, key2);
if (pFind != NULL)
multimapRemoveItem(&g_osInfoList, pFind);
pSubmap = multimapFindSubmap(&g_osInfoList, key1);
if (pSubmap == NULL || multimapCountSubmapItems(&g_osInfoList, pSubmap) > 0)
return NV_OK;
multimapRemoveSubmap(&g_osInfoList, pSubmap);
return NV_OK;
}

129
src/nvidia/src/kernel/rmapi/rmapi.c
View File

@@ -99,6 +99,7 @@ rmapiInitialize
listInit(&g_clientListBehindGpusLock, g_resServ.pAllocator);
listInit(&g_userInfoList, g_resServ.pAllocator);
multimapInit(&g_osInfoList, g_resServ.pAllocator);
secInfo.privLevel = RS_PRIV_LEVEL_KERNEL;
secInfo.paramLocation = PARAM_LOCATION_KERNEL;
@@ -636,6 +637,8 @@ rmapiDelPendingClients
}
}
extern OsInfoMap g_osInfoList;
NV_STATUS
rmapiGetClientHandlesFromOSInfo
(
@@ -644,6 +647,8 @@ rmapiGetClientHandlesFromOSInfo
NvU32 *pClientHandleListSize
)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
NvHandle *pClientHandleList;
NvU32 clientHandleListSize = 0;
NvU32 k;
@@ -653,53 +658,97 @@ rmapiGetClientHandlesFromOSInfo
RmClient *pClient;
RsClient *pRsClient;
ppFirstClient = NULL;
for (ppClient = serverutilGetFirstClientUnderLock();
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
NvBool clientHandleLookup = pSys->getProperty(pSys, PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP);
if (!clientHandleLookup)
{
pClient = *ppClient;
if (pClient->pOSInfo != pOSInfo)
ppFirstClient = NULL;
for (ppClient = serverutilGetFirstClientUnderLock();
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
{
continue;
pClient = *ppClient;
if (pClient->pOSInfo != pOSInfo)
{
continue;
}
clientHandleListSize++;
if (NULL == ppFirstClient)
ppFirstClient = ppClient;
}
clientHandleListSize++;
if (NULL == ppFirstClient)
ppFirstClient = ppClient;
}
if (clientHandleListSize == 0)
{
*pClientHandleListSize = 0;
*ppClientHandleList = NULL;
return NV_ERR_INVALID_ARGUMENT;
}
pClientHandleList = portMemAllocNonPaged(clientHandleListSize * sizeof(NvU32));
if (pClientHandleList == NULL)
{
return NV_ERR_NO_MEMORY;
}
*pClientHandleListSize = clientHandleListSize;
*ppClientHandleList = pClientHandleList;
k = 0;
for (ppClient = ppFirstClient;
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
{
pClient = *ppClient;
pRsClient = staticCast(pClient, RsClient);
if (pClient->pOSInfo != pOSInfo)
if (clientHandleListSize == 0)
{
continue;
*pClientHandleListSize = 0;
*ppClientHandleList = NULL;
return NV_ERR_INVALID_ARGUMENT;
}
pClientHandleList[k++] = pRsClient->hClient;
if (clientHandleListSize <= k)
break;
pClientHandleList = portMemAllocNonPaged(clientHandleListSize * sizeof(NvU32));
if (pClientHandleList == NULL)
{
return NV_ERR_NO_MEMORY;
}
*pClientHandleListSize = clientHandleListSize;
*ppClientHandleList = pClientHandleList;
k = 0;
for (ppClient = ppFirstClient;
ppClient;
ppClient = serverutilGetNextClientUnderLock(ppClient))
{
pClient = *ppClient;
pRsClient = staticCast(pClient, RsClient);
if (pClient->pOSInfo != pOSInfo)
{
continue;
}
pClientHandleList[k++] = pRsClient->hClient;
if (clientHandleListSize <= k)
break;
}
}
else
{
OsInfoMapSubmap *pSubmap = NULL;
OsInfoMapIter it;
NvU64 key1 = (NvUPtr)pOSInfo;
pSubmap = multimapFindSubmap(&g_osInfoList, key1);
if (pSubmap != NULL)
{
clientHandleListSize = multimapCountSubmapItems(&g_osInfoList, pSubmap);
NV_PRINTF(LEVEL_INFO, "*** Found %d clients for %llx\n", clientHandleListSize, key1);
}
if (clientHandleListSize == 0)
{
*pClientHandleListSize = 0;
*ppClientHandleList = NULL;
return NV_ERR_INVALID_ARGUMENT;
}
pClientHandleList = portMemAllocNonPaged(clientHandleListSize * sizeof(NvU32));
if (pClientHandleList == NULL)
{
return NV_ERR_NO_MEMORY;
}
*pClientHandleListSize = clientHandleListSize;
*ppClientHandleList = pClientHandleList;
k = 0;
it = multimapSubmapIterItems(&g_osInfoList, pSubmap);
while(multimapItemIterNext(&it))
{
pClient = *it.pValue;
pRsClient = staticCast(pClient, RsClient);
NV_CHECK_OR_ELSE_STR(LEVEL_ERROR, pClient->pOSInfo == pOSInfo, "*** OS info mismatch", continue);
pClientHandleList[k++] = pRsClient->hClient;
NV_PRINTF(LEVEL_INFO, "*** Found: %x\n", pRsClient->hClient);
if (clientHandleListSize <= k)
break;
}
}
return NV_OK;

78
src/nvidia/src/libraries/nvport/memory/memory_tracking.c
View File

@@ -34,15 +34,45 @@
#error "DEBUG module must be present for memory tracking"
#endif
#if !PORT_IS_MODULE_SUPPORTED(atomic)
#error "ATOMIC module must be present for memory tracking"
#endif
#if PORT_MEM_TRACK_USE_LIMIT
#include "os/os.h"
#define PORT_MEM_LIMIT_MAX_PIDS 32
#endif
#if NVOS_IS_LIBOS
#define PORT_MEM_THREAD_SAFE_ALLOCATIONS 0
#else
#define PORT_MEM_THREAD_SAFE_ALLOCATIONS 1
#endif
#if PORT_MEM_THREAD_SAFE_ALLOCATIONS && !PORT_IS_MODULE_SUPPORTED(atomic)
#error "ATOMIC module must be present for memory tracking"
#endif
#if PORT_MEM_THREAD_SAFE_ALLOCATIONS
#define PORT_MEM_ATOMIC_ADD_SIZE portAtomicAddSize
#define PORT_MEM_ATOMIC_SUB_SIZE portAtomicSubSize
#define PORT_MEM_ATOMIC_DEC_U32 portAtomicDecrementU32
#define PORT_MEM_ATOMIC_INC_U32 portAtomicIncrementU32
#define PORT_MEM_ATOMIC_SET_U32 portAtomicSetU32
#define PORT_MEM_ATOMIC_CAS_SIZE portAtomicCompareAndSwapSize
#define PORT_MEM_ATOMIC_CAS_U32 portAtomicCompareAndSwapU32
#else
//
// We can just stub out the atomic operations for non-atomic ones and not waste
// waste cycles on synchronization
//
#define PORT_MEM_ATOMIC_ADD_SIZE(pVal, val) (*((NvSPtr *)pVal) += val)
#define PORT_MEM_ATOMIC_SUB_SIZE(pVal, val) (*((NvSPtr *)pVal) -= val)
#define PORT_MEM_ATOMIC_DEC_U32(pVal) (--(*((NvU32 *)pVal)))
#define PORT_MEM_ATOMIC_INC_U32(pVal) (++(*((NvU32 *)pVal)))
#define PORT_MEM_ATOMIC_SET_U32(pVal, val) (*((NvU32 *)pVal) = val)
#define PORT_MEM_ATOMIC_CAS_SIZE(pVal, newVal, oldVal) \
((*pVal == oldVal) ? ((*((NvSPtr *)pVal) = newVal), NV_TRUE) : NV_FALSE)
#define PORT_MEM_ATOMIC_CAS_U32(pVal, newVal, oldVal) \
((*pVal == oldVal) ? ((*((NvU32 *)pVal) = newVal), NV_TRUE) : NV_FALSE)
#endif // !PORT_MEM_THREAD_SAFE_ALLOCATIONS
struct PORT_MEM_ALLOCATOR_IMPL
{
PORT_MEM_ALLOCATOR_TRACKING tracking;
@@ -78,11 +108,11 @@ static NV_STATUS portSyncSpinlockInitialize(PORT_SPINLOCK *pSpinlock)
}
static void portSyncSpinlockAcquire(PORT_SPINLOCK *pSpinlock)
{
while (!portAtomicCompareAndSwapU32(pSpinlock, 1, 0));
while (!PORT_MEM_ATOMIC_CAS_U32(pSpinlock, 1, 0));
}
static void portSyncSpinlockRelease(PORT_SPINLOCK *pSpinlock)
{
portAtomicSetU32(pSpinlock, 0);
PORT_MEM_ATOMIC_SET_U32(pSpinlock, 0);
}
static void portSyncSpinlockDestroy(PORT_SPINLOCK *pSpinlock)
{
@@ -150,13 +180,13 @@ _portMemCounterInc
NvU32 activeAllocs;
NvLength activeSize = 0;
activeAllocs = portAtomicIncrementU32(&pCounter->activeAllocs);
portAtomicIncrementU32(&pCounter->totalAllocs);
activeAllocs = PORT_MEM_ATOMIC_INC_U32(&pCounter->activeAllocs);
PORT_MEM_ATOMIC_INC_U32(&pCounter->totalAllocs);
if (PORT_MEM_TRACK_USE_FENCEPOSTS)
{
activeSize = portAtomicAddSize(&pCounter->activeSize, size);
activeSize = PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->activeSize, size);
}
portAtomicAddSize(&pCounter->totalSize, size);
PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->totalSize, size);
// Atomically compare the peak value with the active, and update if greater.
while (1)
@@ -164,14 +194,14 @@ _portMemCounterInc
NvU32 peakAllocs = pCounter->peakAllocs;
if (activeAllocs <= peakAllocs)
break;
portAtomicCompareAndSwapU32(&pCounter->peakAllocs, activeAllocs, peakAllocs);
PORT_MEM_ATOMIC_CAS_U32(&pCounter->peakAllocs, activeAllocs, peakAllocs);
}
while (1)
{
NvLength peakSize = pCounter->peakSize;
if (activeSize <= peakSize)
break;
portAtomicCompareAndSwapSize(&pCounter->peakSize, activeSize, peakSize);
PORT_MEM_ATOMIC_CAS_SIZE(&pCounter->peakSize, activeSize, peakSize);
}
}
static NV_INLINE void
@@ -181,11 +211,11 @@ _portMemCounterDec
void *pMem
)
{
portAtomicDecrementU32(&pCounter->activeAllocs);
PORT_MEM_ATOMIC_DEC_U32(&pCounter->activeAllocs);
if (PORT_MEM_TRACK_USE_FENCEPOSTS)
{
portAtomicSubSize(&pCounter->activeSize,
((PORT_MEM_FENCE_HEAD *)pMem-1)->blockSize);
PORT_MEM_ATOMIC_SUB_SIZE(&pCounter->activeSize,
((PORT_MEM_FENCE_HEAD *)pMem-1)->blockSize);
}
}
@@ -273,7 +303,7 @@ _portMemListAdd
PORT_MEM_LIST *pList = &pHead->list;
pList->pNext = pList;
pList->pPrev = pList;
if (!portAtomicCompareAndSwapSize(&pTracking->pFirstAlloc, pList, NULL))
if (!PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList, NULL))
{
PORT_LOCKED_LIST_LINK(pTracking->pFirstAlloc, pList, pTracking->listLock);
}
@@ -288,11 +318,11 @@ _portMemListRemove
PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
PORT_MEM_LIST *pList = &pHead->list;
if (!portAtomicCompareAndSwapSize(&pList->pNext, NULL, pList))
if (!PORT_MEM_ATOMIC_CAS_SIZE(&pList->pNext, NULL, pList))
{
PORT_LOCKED_LIST_UNLINK(pTracking->pFirstAlloc, pList, pTracking->listLock);
}
portAtomicCompareAndSwapSize(&pTracking->pFirstAlloc, pList->pNext, pList);
PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList->pNext, pList);
}
static NV_INLINE PORT_MEM_HEADER *
@@ -517,7 +547,7 @@ _portMemLimitInc(NvU32 pid, void *pMem, NvU64 size)
{
NvU32 pidIdx = pid - 1;
pMemHeader->blockSize = size;
portAtomicAddSize(&portMemGlobals.counterPid[pidIdx], size);
PORT_MEM_ATOMIC_ADD_SIZE(&portMemGlobals.counterPid[pidIdx], size);
}
}
}
@@ -541,7 +571,7 @@ _portMemLimitDec(void *pMem)
}
else
{
portAtomicSubSize(&portMemGlobals.counterPid[pidIdx], pMemHeader->blockSize);
PORT_MEM_ATOMIC_SUB_SIZE(&portMemGlobals.counterPid[pidIdx], pMemHeader->blockSize);
}
}
}
@@ -596,7 +626,7 @@ portMemInitialize(void)
#if PORT_MEM_TRACK_USE_CALLERINFO
PORT_MEM_CALLERINFO_TYPE_PARAM = PORT_MEM_CALLERINFO_MAKE;
#endif
if (portAtomicIncrementU32(&portMemGlobals.initCount) != 1)
if (PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.initCount) != 1)
return;
portMemGlobals.mainTracking.pAllocator = NULL;
@@ -649,7 +679,7 @@ void
portMemShutdown(NvBool bForceSilent)
{
PORT_UNREFERENCED_VARIABLE(bForceSilent);
if (portAtomicDecrementU32(&portMemGlobals.initCount) != 0)
if (PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.initCount) != 0)
return;
#if (PORT_MEM_TRACK_PRINT_LEVEL > PORT_MEM_TRACK_PRINT_LEVEL_SILENT)
@@ -850,7 +880,7 @@ portMemInitializeAllocatorTracking
PORT_MEM_COUNTER_INIT(&pTracking->counter);
PORT_MEM_LIST_INIT(pTracking);
PORT_MEM_CALLERINFO_INIT_TRACKING(pTracking);
portAtomicIncrementU32(&portMemGlobals.totalAllocators);
PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.totalAllocators);
}
#if PORT_MEM_TRACK_USE_LIMIT
@@ -1201,7 +1231,7 @@ _portMemTrackingRelease
PORT_LOCKED_LIST_UNLINK(&portMemGlobals.mainTracking, pTracking, portMemGlobals.trackingLock);
PORT_MEM_LIST_DESTROY(pTracking);
portAtomicDecrementU32(&portMemGlobals.totalAllocators);
PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.totalAllocators);
}
static void

56
src/nvidia/src/libraries/tls/tls.c
View File

@@ -130,6 +130,18 @@ NvU32 osGetMaximumCoreCount(void);
#endif
#endif
#if NVOS_IS_LIBOS
//
// On LibOS we have at most one passive thread (task_rm) and one ISR
// (task_interrupt) active at once (on same CPU core). Since these two will
// use different maps, we don't need to protect them with spinlocks.
//
#define TLS_SPINLOCK_ACQUIRE(x)
#define TLS_SPINLOCK_RELEASE(x)
#else
#define TLS_SPINLOCK_ACQUIRE(x) portSyncSpinlockAcquire(x)
#define TLS_SPINLOCK_RELEASE(x) portSyncSpinlockRelease(x)
#endif // NVOS_IS_LIBOS
#if !PORT_IS_FUNC_SUPPORTED(portSyncExSafeToSleep)
#define portSyncExSafeToSleep() NV_TRUE
@@ -426,9 +438,9 @@ _tlsThreadEntryGet(void)
else
{
NvU64 threadId = portThreadGetCurrentThreadId();
portSyncSpinlockAcquire(tlsDatabase.pLock);
pThreadEntry = mapFind(&tlsDatabase.threadEntries, threadId);
portSyncSpinlockRelease(tlsDatabase.pLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pLock);
pThreadEntry = mapFind(&tlsDatabase.threadEntries, threadId);
TLS_SPINLOCK_RELEASE(tlsDatabase.pLock);
}
return pThreadEntry;
}
@@ -448,11 +460,11 @@ _tlsThreadEntryGetOrAlloc(void)
{
pThreadEntry->key.threadId = portThreadGetCurrentThreadId();
mapInitIntrusive(&pThreadEntry->map);
portSyncSpinlockAcquire(tlsDatabase.pLock);
mapInsertExisting(&tlsDatabase.threadEntries,
pThreadEntry->key.threadId,
pThreadEntry);
portSyncSpinlockRelease(tlsDatabase.pLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pLock);
mapInsertExisting(&tlsDatabase.threadEntries,
pThreadEntry->key.threadId,
pThreadEntry);
TLS_SPINLOCK_RELEASE(tlsDatabase.pLock);
}
}
@@ -510,9 +522,9 @@ _tlsEntryRelease
{
NV_ASSERT(portMemExSafeForNonPagedAlloc());
mapDestroy(&pThreadEntry->map);
portSyncSpinlockAcquire(tlsDatabase.pLock);
mapRemove(&tlsDatabase.threadEntries, pThreadEntry);
portSyncSpinlockRelease(tlsDatabase.pLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pLock);
mapRemove(&tlsDatabase.threadEntries, pThreadEntry);
TLS_SPINLOCK_RELEASE(tlsDatabase.pLock);
PORT_FREE(tlsDatabase.pAllocator, pThreadEntry);
}
}
@@ -555,29 +567,29 @@ static void _tlsIsrEntriesDestroy(void)
}
static void _tlsIsrEntriesInsert(ThreadEntry *pThreadEntry)
{
portSyncSpinlockAcquire(tlsDatabase.pIsrLock);
mapInsertExisting(&tlsDatabase.isrEntries, pThreadEntry->key.sp, pThreadEntry);
portSyncSpinlockRelease(tlsDatabase.pIsrLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pIsrLock);
mapInsertExisting(&tlsDatabase.isrEntries, pThreadEntry->key.sp, pThreadEntry);
TLS_SPINLOCK_RELEASE(tlsDatabase.pIsrLock);
}
static ThreadEntry *_tlsIsrEntriesRemove(NvU64 sp)
{
ThreadEntry *pThreadEntry;
portSyncSpinlockAcquire(tlsDatabase.pIsrLock);
pThreadEntry = mapFind(&tlsDatabase.isrEntries, sp);
mapRemove(&tlsDatabase.isrEntries, pThreadEntry);
portSyncSpinlockRelease(tlsDatabase.pIsrLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pIsrLock);
pThreadEntry = mapFind(&tlsDatabase.isrEntries, sp);
mapRemove(&tlsDatabase.isrEntries, pThreadEntry);
TLS_SPINLOCK_RELEASE(tlsDatabase.pIsrLock);
return pThreadEntry;
}
static ThreadEntry *_tlsIsrEntriesFind(NvU64 approxSp)
{
ThreadEntry *pThreadEntry;
portSyncSpinlockAcquire(tlsDatabase.pIsrLock);
TLS_SPINLOCK_ACQUIRE(tlsDatabase.pIsrLock);
#if STACK_GROWS_DOWNWARD
pThreadEntry = mapFindGEQ(&tlsDatabase.isrEntries, approxSp);
pThreadEntry = mapFindGEQ(&tlsDatabase.isrEntries, approxSp);
#else
pThreadEntry = mapFindLEQ(&tlsDatabase.isrEntries, approxSp);
pThreadEntry = mapFindLEQ(&tlsDatabase.isrEntries, approxSp);
#endif
portSyncSpinlockRelease(tlsDatabase.pIsrLock);
TLS_SPINLOCK_RELEASE(tlsDatabase.pIsrLock);
return pThreadEntry;
}