NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-04 23:29:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

525.78.01

Browse Source
This commit is contained in:
Andy Ritger
2023-01-05 10:40:27 -08:00
parent 9594cc0169
commit dac2350c7f
180 changed files with 9465 additions and 4853 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/nvidia/arch/nvalloc/common/inc/dev_ctrl_defines.h
View File

@@ -53,6 +53,8 @@
#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_START(i) ((i)*2)
#define NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(i) (((i)*2) + 1)
#define NV_CTRL_INTR_LEAF_IDX_TO_SUBTREE(i) ((i)/2)
//
// Given a LEAF register index, the below macros give us the range of GPU
// interrupt vectors that correspond to those leafs.

9
src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h
View File

@@ -139,17 +139,22 @@ typedef struct
NvU32 elfCodeSize;
NvU32 elfDataSize;
// Bit 0 is used to check if [VGPU-GSP] mode is active in init partition
NvU8 driverModel;
// Pad structure to exactly 256 bytes. Can replace padding with additional
// fields without incrementing revision. Padding initialized to 0.
NvU32 padding[3];
NvU8 padding[11];
// BL to use for verification (i.e. Booter says OK to boot)
NvU64 verified; // 0x0 -> unverified, 0xa0a0a0a0a0a0a0a0 -> verified
} GspFwWprMeta;
#define GSP_FW_WPR_META_VERIFIED 0xa0a0a0a0a0a0a0a0ULL
#define GSP_FW_WPR_META_REVISION 1
#define GSP_FW_WPR_META_MAGIC 0xdc3aae21371a60b3ULL
// Bit 0 is used to check if [VGPU-GSP] mode is active in init partition
#define DRIVERMODEL_VGPU 0
#endif // GSP_FW_WPR_META_H_

24
src/nvidia/arch/nvalloc/common/inc/inforom/ifrstruct.h
View File

@@ -28,6 +28,7 @@
#include "inforom/ifrecc.h"
#include "inforom/ifrdem.h"
#include "inforom/omsdef.h"
#define INFOROM_OBD_OBJECT_V1_XX_PACKED_SIZE 128
@@ -49,6 +50,29 @@ struct INFOROM_OBD_OBJECT_V1_XX
#define INFOROM_OBD_OBJECT_V1_XX_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d116b"
typedef struct INFOROM_OBD_OBJECT_V1_XX INFOROM_OBD_OBJECT_V1_XX;
#define INFOROM_OBD_OBJECT_V2_XX_PACKED_SIZE 224
struct INFOROM_OBD_OBJECT_V2_XX
{
INFOROM_OBJECT_HEADER_V1_00 header;
inforom_U032 buildDate;
inforom_U008 marketingName[24];
inforom_U008 serialNumber[16];
inforom_U008 memoryManufacturer;
inforom_U008 memoryPartID[20];
inforom_U008 memoryDateCode[6];
inforom_U008 productPartNumber[20];
inforom_U008 boardRevision[3];
inforom_U008 boardType;
inforom_U008 board699PartNumber[20];
inforom_U032 productLength;
inforom_U032 productWidth;
inforom_U032 productHeight;
inforom_U008 reserved[89];
};
#define INFOROM_OBD_OBJECT_V2_XX_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "d111b3d89b"
typedef struct INFOROM_OBD_OBJECT_V2_XX INFOROM_OBD_OBJECT_V2_XX;
//
// OEM 1.0
//

32
src/nvidia/arch/nvalloc/common/inc/inforom/omsdef.h
View File

@@ -35,6 +35,29 @@
#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_NO 0
#define INFOROM_OMS_OBJECT_V1_SETTINGS_ENTRY_DATA_FORCE_DEVICE_DISABLE_YES 1
#define INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY_DATA_RESERVED 23:2
#define INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY_DATA_ENTRY_CHECKSUM 31:24
typedef struct INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY
{
inforom_U032 data;
inforom_U032 clockLimitMin;
inforom_U032 clockLimitMax;
} INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY;
#define INFOROM_OMS_OBJECT_V1G_NUM_SETTINGS_ENTRIES 8
typedef struct INFOROM_OMS_OBJECT_V1G
{
INFOROM_OBJECT_HEADER_V1_00 header;
inforom_U032 lifetimeRefreshCount;
INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY settings[
INFOROM_OMS_OBJECT_V1G_NUM_SETTINGS_ENTRIES];
inforom_U032 reserved;
} INFOROM_OMS_OBJECT_V1G;
#define INFOROM_OMS_OBJECT_V1G_FMT INFOROM_OBJECT_HEADER_V1_00_FMT "26d"
#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_RESERVED 7:2
#define INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY_DATA_ENTRY_CHECKSUM 15:8
@@ -62,15 +85,24 @@ typedef struct INFOROM_OMS_V1S_DATA
INFOROM_OMS_OBJECT_V1S_SETTINGS_ENTRY *pNext;
} INFOROM_OMS_V1S_DATA;
typedef struct INFOROM_OMS_V1G_DATA
{
INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY *pIter;
INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY prev;
INFOROM_OMS_OBJECT_V1G_SETTINGS_ENTRY *pNext;
} INFOROM_OMS_V1G_DATA;
typedef union
{
INFOROM_OBJECT_HEADER_V1_00 header;
INFOROM_OMS_OBJECT_V1S v1s;
INFOROM_OMS_OBJECT_V1G v1g;
} INFOROM_OMS_OBJECT;
typedef union
{
INFOROM_OMS_V1S_DATA v1s;
INFOROM_OMS_V1G_DATA v1g;
} INFOROM_OMS_DATA;
typedef struct

2
src/nvidia/arch/nvalloc/common/inc/nvcst.h
View File

@@ -290,6 +290,7 @@ VENDORNAME vendorName[] =
{PCI_VENDOR_ID_FUJITSU, "Fujitsu"},
{PCI_VENDOR_ID_CADENCE, "Cadence"},
{PCI_VENDOR_ID_ARM, "ARM"},
{PCI_VENDOR_ID_ALIBABA, "Alibaba"},
{0, "Unknown"} // Indicates end of the table
};
@@ -345,6 +346,7 @@ ARMCSALLOWLISTINFO armChipsetAllowListInfo[] =
{PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1}, // Arm Neoverse N1
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX}, // Marvell OCTEON CN96xx
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX}, // Marvell OCTEON CN98xx
{PCI_VENDOR_ID_ALIBABA, 0x8000, CS_ALIBABA_YITIAN}, // Alibaba Yitian
// last element must have chipset CS_UNKNOWN (zero)
{0, 0, CS_UNKNOWN}

2
src/nvidia/arch/nvalloc/common/inc/nvdevid.h
View File

@@ -145,6 +145,7 @@
#define PCI_VENDOR_ID_CADENCE 0x17CD
#define PCI_VENDOR_ID_ARM 0x13B5
#define PCI_VENDOR_ID_HYGON 0x1D94
#define PCI_VENDOR_ID_ALIBABA 0x1DED
#define NV_PCI_DEVID_DEVICE 31:16 /* RW--F */
#define NV_PCI_SUBID_DEVICE 31:16 /* RW--F */
@@ -642,6 +643,7 @@ enum {
, CS_HYGON_C86
, CS_PHYTIUM_S2500
, CS_MELLANOX_BLUEFIELD2
, CS_ALIBABA_YITIAN
, CS_INTEL_1B81
, CS_INTEL_18DC
, CS_INTEL_7A04

26
src/nvidia/arch/nvalloc/common/inc/oob/smbpbi.h
View File

@@ -111,6 +111,7 @@
#define NV_MSGBOX_CMD_ARG1_TEMP_NUM_SENSORS 8
#define NV_MSGBOX_CMD_ARG1_POWER_TOTAL 0x00000000
#define NV_MSGBOX_CMD_ARG1_SMBPBI_POWER 0x00000001
#define NV_MSGBOX_CMD_ARG1_POWER_FB 0x00000002
/* SysId info type encodings for opcode NV_MSGBOX_CMD_OPCODE_GET_SYS_ID_DATA (0x05) */
#define NV_MSGBOX_CMD_ARG1_BOARD_PART_NUM_V1 0x00000000
#define NV_MSGBOX_CMD_ARG1_OEM_INFO_V1 0x00000001
@@ -171,6 +172,8 @@
#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MIN 0x00000001
#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_MAX 0x00000002
#define NV_MSGBOX_CMD_ARG1_GET_CLOCK_FREQ_INFO_PAGE_3 0x00000003
#define NV_MSGBOX_CMD_ARG1_GET_SUPPORTED_CLOCK_THROTTLE_REASONS 0x00000004
#define NV_MSGBOX_CMD_ARG1_GET_CURRENT_CLOCK_THROTTLE_REASONS 0x00000005
#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_RAW_COUNTS 0x00000000
#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_STATE_FLAGS 0x00000001
#define NV_MSGBOX_CMD_ARG1_REMAP_ROWS_HISTOGRAM 0x00000002
@@ -639,6 +642,9 @@
#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS 29:29
#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_0_GET_FABRIC_STATE_FLAGS_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_0_POWER_FB 30:30
#define NV_MSGBOX_DATA_CAP_0_POWER_FB_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_0_POWER_FB_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_1 1
#define NV_MSGBOX_DATA_CAP_1_BOARD_PART_NUM_V1 0:0
@@ -731,6 +737,9 @@
#define NV_MSGBOX_DATA_CAP_1_ECC_V6 30:30
#define NV_MSGBOX_DATA_CAP_1_ECC_V6_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_1_ECC_V6_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON 31:31
#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON_NOT_AVAILABLE 0x00000000
#define NV_MSGBOX_DATA_CAP_1_CLOCK_THROTTLE_REASON_AVAILABLE 0x00000001
#define NV_MSGBOX_DATA_CAP_2 2
#define NV_MSGBOX_DATA_CAP_2_GPU_DRIVER 0:0
@@ -1057,6 +1066,21 @@
#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE 3:0
#define NV_MSGBOX_DATA_GET_CLOCK_FREQ_INFO_PAGE_3_CURRENT_PSTATE_INVALID 0x0000000F
/**
* Response to
* NV_MSGBOX_CMD_ARG1_GET_CLOCK_THROTTLE_REASON
*/
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON 31:0
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_NONE 0x00000000
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_POWER_CAP 0x00000001
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_SLOWDOWN 0x00000002
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SYNC_BOOST 0x00000004
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_THERMAL_SLOWDOWN_TLIMIT 0x00000008
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_THERMAL_SLOWDOWN_TAVG 0x00000010
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_SW_THERMAL_SLOWDOWN_TMEM 0x00000020
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_THERMAL_SLOWDOWN 0x00000040
#define NV_MSGBOX_DATA_CLOCK_THROTTLE_REASON_HW_POWER_BREAK_SLOWDOWN 0x00000080
/*
* Number of Nvlink data outputs (dataOut, extData) for
* NV_MSGBOX_CMD_ARG1_GET_NVLINK_INFO_LINK queries
@@ -2525,7 +2549,7 @@ typedef union {
NV_MSGBOX_CMD(_GPU_PERFORMANCE_MONITORING, 0, 0) | \
DRF_DEF(_MSGBOX, _CMD, _ARG1_GPM_ACTION, type) | \
DRF_NUM(_MSGBOX, _CMD, _ARG1_GPM_METRIC, metric) | \
DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_PARTITION, partition) \
DRF_NUM(_MSGBOX, _CMD, _ARG2_GPM_PARTITION_INDEX, partition) \
)
#define NV_MSGBOX_CMD_GPM_SET_MULTIPLIER(multiplier) \

1
src/nvidia/arch/nvalloc/common/inc/rmifrif.h
View File

@@ -112,7 +112,6 @@ typedef struct
* scanning of the whole of memory (e.g. when something goes wrong).
*/
#define RM_IFR_GC6_CTX_END_OFFSET_MAX 0x1000000 // 16MB
#define RM_IFR_GC6_CTX_END_OFFSET_MAX_WITH_GSP 0x10000000 // 256MB
/*!
* Maximum size of the context data in bytes.

1
src/nvidia/arch/nvalloc/unix/include/nv.h
View File

@@ -958,7 +958,6 @@ NV_STATUS NV_API_CALL rm_log_gpu_crash (nv_stack_t *, nv_state_t *);
void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd);
NvBool NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id);
NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *);
NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults_unlocked(nvidia_stack_t *, nv_state_t *, NvU32 *);
NV_STATUS NV_API_CALL rm_gpu_handle_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *);
NvBool NV_API_CALL rm_gpu_need_4k_page_isolation(nv_state_t *);
NvBool NV_API_CALL rm_is_chipset_io_coherent(nv_stack_t *);

20
src/nvidia/arch/nvalloc/unix/src/os.c
View File

@@ -1221,12 +1221,11 @@ static void postEvent(
NvBool dataValid
)
{
nv_state_t *nv = nv_get_ctl_state();
portSyncSpinlockAcquire(nv->event_spinlock);
if (event->active)
nv_post_event(event, hEvent, notifyIndex,
info32, info16, dataValid);
portSyncSpinlockRelease(nv->event_spinlock);
if (osReferenceObjectCount(event) != NV_OK)
return;
nv_post_event(event, hEvent, notifyIndex,
info32, info16, dataValid);
osDereferenceObjectCount(event);
}
NvU32 osSetEvent
@@ -1445,6 +1444,12 @@ NV_STATUS osReferenceObjectCount(void *pEvent)
nv_event_t *event = pEvent;
portSyncSpinlockAcquire(nv->event_spinlock);
// If event->active is false, don't allow any more reference
if (!event->active)
{
portSyncSpinlockRelease(nv->event_spinlock);
return NV_ERR_INVALID_EVENT;
}
++event->refcount;
portSyncSpinlockRelease(nv->event_spinlock);
return NV_OK;
@@ -1457,11 +1462,10 @@ NV_STATUS osDereferenceObjectCount(void *pOSEvent)
portSyncSpinlockAcquire(nv->event_spinlock);
NV_ASSERT(event->refcount > 0);
--event->refcount;
// If event->refcount == 0 but event->active is true, the client
// has not yet freed the OS event. free_os_event will free its
// memory when they do, or else when the client itself is freed.
if (event->refcount == 0 && !event->active)
if (--event->refcount == 0 && !event->active)
portMemFree(event);
portSyncSpinlockRelease(nv->event_spinlock);

21
src/nvidia/arch/nvalloc/unix/src/osapi.c
View File

@@ -354,9 +354,7 @@ static void free_os_event_under_lock(nv_event_t *event)
// If refcount > 0, event will be freed by osDereferenceObjectCount
// when the last associated RM event is freed.
if (event->refcount == 0)
{
portMemFree(event);
}
}
static void free_os_events(
@@ -2910,23 +2908,21 @@ static NV_STATUS RmRunNanoTimerCallback(
void *pTmrEvent
)
{
OBJSYS *pSys = SYS_GET_INSTANCE();
POBJTMR pTmr = GPU_GET_TIMER(pGpu);
THREAD_STATE_NODE threadState;
NV_STATUS status = NV_OK;
// LOCK: try to acquire GPUs lock
if ((status = rmGpuLocksAcquire(GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_TMR)) != NV_OK)
{
PTMR_EVENT_PVT pEvent = (PTMR_EVENT_PVT) pTmrEvent;
// We failed to acquire the lock; schedule a timer to try again.
return osStartNanoTimer(pGpu->pOsGpuInfo, pEvent->super.pOSTmrCBdata, 1000);
}
TMR_EVENT *pEvent = (TMR_EVENT *)pTmrEvent;
if ((status = osCondAcquireRmSema(pSys->pSema)) != NV_OK)
{
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
return status;
//
// We failed to acquire the lock - depending on what's holding it,
// the lock could be held for a while, so try again soon, but not too
// soon to prevent the owner from making forward progress indefinitely.
//
return osStartNanoTimer(pGpu->pOsGpuInfo, pEvent->pOSTmrCBdata,
osGetTickResolution());
}
threadStateInitISRAndDeferredIntHandler(&threadState, pGpu,
@@ -2939,7 +2935,6 @@ static NV_STATUS RmRunNanoTimerCallback(
threadStateFreeISRAndDeferredIntHandler(&threadState,
pGpu, THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER);
osReleaseRmSema(pSys->pSema, NULL);
// UNLOCK: release GPUs lock
rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, pGpu);

89
src/nvidia/arch/nvalloc/unix/src/unix_intr.c
View File

@@ -608,36 +608,19 @@ done:
// Use this call when MMU faults needs to be copied
// outisde of RM lock.
//
NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults_unlocked(
nvidia_stack_t *sp,
nv_state_t *nv,
NvU32 *faultsCopied
static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked(
OBJGPU *pGpu,
NvU32 *pFaultsCopied,
THREAD_STATE_NODE *pThreadState
)
{
OBJGPU *pGpu;
void *fp;
NV_STATUS status = NV_OK;
NV_ENTER_RM_RUNTIME(sp,fp);
pGpu = NV_GET_NV_PRIV_PGPU(nv);
if (pGpu == NULL || faultsCopied == NULL)
{
status = NV_ERR_OBJECT_NOT_FOUND;
goto done;
}
// Non-replayable faults are copied to the client shadow buffer by GSP-RM.
if (IS_GSP_CLIENT(pGpu))
{
status = NV_ERR_NOT_SUPPORTED;
goto done;
return NV_ERR_NOT_SUPPORTED;
}
done:
NV_EXIT_RM_RUNTIME(sp,fp);
return status;
return NV_OK;
}
//
@@ -650,10 +633,12 @@ NV_STATUS rm_gpu_handle_mmu_faults(
)
{
NvU32 status = NV_OK;
OBJGPU *pGpu;
void *fp;
NV_ENTER_RM_RUNTIME(sp,fp);
*faultsCopied = 0;
OBJGPU *pGpu;
pGpu = NV_GET_NV_PRIV_PGPU(nv);
if (pGpu == NULL)
@@ -661,40 +646,50 @@ NV_STATUS rm_gpu_handle_mmu_faults(
return NV_ERR_OBJECT_NOT_FOUND;
}
if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu))
{
KernelGmmu *pKernelGmmu;
PORT_MEM_ALLOCATOR *pIsrAllocator;
THREAD_STATE_NODE threadState;
NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer
KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
Intr *pIntr = GPU_GET_INTR(pGpu);
NvU32 hw_put = 0;
NvU32 hw_get = 0;
pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator));
tlsIsrInit(pIsrAllocator);
threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
kgmmuReadFaultBufferPutPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER,
&hw_put, &threadState);
pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
kgmmuReadFaultBufferGetPtr_HAL(pGpu, pKernelGmmu, NON_REPLAYABLE_FAULT_BUFFER,
&hw_get, &threadState);
if(hw_get != hw_put)
if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu))
{
// We have to clear the top level interrupt bit here since otherwise
// the bottom half will attempt to service the interrupt on the CPU
// side before GSP recieves the notification and services it
kgmmuClearNonReplayableFaultIntr(pGpu, pKernelGmmu, &threadState);
status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE);
Intr *pIntr = GPU_GET_INTR(pGpu);
if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState))
{
// We have to clear the top level interrupt bit here since otherwise
// the bottom half will attempt to service the interrupt on the CPU
// side before GSP recieves the notification and services it
kgmmuClearNonReplayableFaultIntr_HAL(pGpu, pKernelGmmu, &threadState);
status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE);
}
}
else if (IS_VIRTUAL_WITH_SRIOV(pGpu))
{
if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState))
{
status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState);
}
}
else
{
status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState);
}
threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS);
tlsIsrDestroy(pIsrAllocator);
portMemAllocatorRelease(pIsrAllocator);
}
else
{
status = rm_gpu_copy_mmu_faults_unlocked(sp, nv, faultsCopied);
}
NV_EXIT_RM_RUNTIME(sp,fp);
return status;
}