NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-20 23:13:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

515.43.04

Browse Source
This commit is contained in:
Andy Ritger
2022-05-09 13:18:59 -07:00
commit 1739a20efc
2519 changed files with 1060036 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/nvidia/inc/kernel/core/bin_data.h Normal file
View File

@@ -0,0 +1,106 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _BINDATA_H
#define _BINDATA_H
#include "core/core.h"
/**************************************************************************************************************
*
* File: bindata.h
*
* Description:
* Bindata management APIs
*
**************************************************************************************************************/
//
// Public interface for accessing the acquired binary data
//
//
// Binary data access handler
//
typedef struct BINDATA_RUNTIME_INFO BINDATA_RUNTIME_INFO, *PBINDATA_RUNTIME_INFO;
//
// Public binary storage information
//
struct BINDATA_STORAGE; // currently no public data fields
typedef struct BINDATA_STORAGE BINDATA_STORAGE, *PBINDATA_STORAGE;
//
// Primitives
//
NV_STATUS bindataAcquire(const BINDATA_STORAGE *pBinStorage, PBINDATA_RUNTIME_INFO *ppBinInfo);
NV_STATUS bindataGetNextChunk(PBINDATA_RUNTIME_INFO pBinInfo, NvU8 *pBuffer, NvU32 nBytes);
void bindataRelease(PBINDATA_RUNTIME_INFO pBinInfo);
//
// Utilities
//
NV_STATUS bindataWriteToBuffer(const BINDATA_STORAGE *pBinStorage, NvU8 *pBuffer, NvU32 bufferSize);
NvU32 bindataGetBufferSize(const BINDATA_STORAGE *pBinStorage);
//
// Bindata Archive support
//
typedef struct
{
const char* name; // string of file name or name tag
const PBINDATA_STORAGE pBinStorage; // pointer to the binary storage
} BINDATA_ARCHIVE_ENTRY;
typedef struct
{
NvU32 entryNum;
BINDATA_ARCHIVE_ENTRY entries[];
} BINDATA_ARCHIVE;
// Bindata Archive API - get Bindata storage from a Bindata Archive
const BINDATA_STORAGE * bindataArchiveGetStorage(const BINDATA_ARCHIVE *pBinArchive, const char *bindataName);
//
// Iterate over all BINDATA_STORAGE entries that have not been referenced so far
// Returns the pointer to unreferenced data or NULL if no more are available.
// Example usage:
// const BINDATA_STORAGE *iter = NULL;
// void *datablock;
// NvU32 size;
// while ((datablock = bindataGetNextUnreferencedStorage(&iter, &size))) {
// do_stuff(datablock, size);
// }
//
void* bindataGetNextUnreferencedStorage(const BINDATA_STORAGE **iter, NvU32 *pDataSize);
//
// Marks a given BINDATA_STORAGE as destroyed, making all subsequent attempts
// to access it fail and return NULL/0
//
void bindataDestroyStorage(BINDATA_STORAGE *storage);
#endif // _BINDATA_H

50
src/nvidia/inc/kernel/core/core.h Normal file
View File

@@ -0,0 +1,50 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __CORE_H__
#define __CORE_H__
#include "core/prelude.h"
/**
* @brief Global RM initialization
*
* The single entrypoint into the RM for all platforms. This will initial cross
* platform RM subsystems and call into OS specific init as needed.
*
* Must be called once and only once before any RM internal functions can be
* called
*
* @return NV_OK if successful, error otherwise
*/
NV_STATUS coreInitializeRm(void);
/**
* @brief Global RM shutdown
*
* Must be called once and only once when a driver is shutting down and no more
* RM internal functions will be called.
*
*/
void coreShutdownRm(void);
#endif /* __CORE_H__ */

3
src/nvidia/inc/kernel/core/hal.h Normal file
View File

@@ -0,0 +1,3 @@
#include "g_hal_nvoc.h"

3
src/nvidia/inc/kernel/core/hal_mgr.h Normal file
View File

@@ -0,0 +1,3 @@
#include "g_hal_mgr_nvoc.h"

59
src/nvidia/inc/kernel/core/info_block.h Normal file
View File

@@ -0,0 +1,59 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _INFO_BLOCK_H_
#define _INFO_BLOCK_H_
//
// HAL privata data management.
//
typedef struct ENG_INFO_LINK_NODE *PENG_INFO_LINK_NODE;
typedef struct ENG_INFO_LINK_NODE ENG_INFO_LINK_NODE;
// new style typedef for info block functions, simple typedef.
// Used by hal .def files via INFO_BLOCK_GROUP template in Gpuhal.def
typedef void *EngGetInfoBlockFn(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
typedef void *EngAddInfoBlockFn(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId, NvU32 size);
typedef void EngDeleteInfoBlockFn(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId);
typedef NvBool EngTestInfoBlockFn(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
// old style typedef for info block functions (ptr to fn)
// delete these 4 typedefs once all .def files converted to use OBJECT_INTERFACES
typedef EngGetInfoBlockFn *EngGetInfoBlock;
typedef EngAddInfoBlockFn *EngAddInfoBlock;
typedef EngDeleteInfoBlockFn *EngDeleteInfoBlock;
typedef EngTestInfoBlockFn *EngTestInfoBlock;
struct ENG_INFO_LINK_NODE
{
NvU32 dataId;
void *infoBlock;
PENG_INFO_LINK_NODE next;
};
void* getInfoPtr(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
void* addInfoPtr(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId, NvU32 size);
void deleteInfoPtr(PENG_INFO_LINK_NODE * ppHead, NvU32 dataId);
NvBool testInfoPtr(PENG_INFO_LINK_NODE, NvU32 dataId);
#endif // _INFO_BLOCK_H_

205
src/nvidia/inc/kernel/core/locks.h Normal file
View File

@@ -0,0 +1,205 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef LOCKS_H
#define LOCKS_H
#include "core/core.h"
#include "os/os.h"
// Forward declarations
typedef struct OBJSYS OBJSYS;
typedef enum
{
GPU_LOCK_GRP_SUBDEVICE, // locks will be taken for subdevice only
GPU_LOCK_GRP_DEVICE, // locks will be taken for device only
GPU_LOCK_GRP_MASK, // locks will be taken for devices specified by the mask
GPU_LOCK_GRP_ALL // locks will be taken for all devices
} GPU_LOCK_GRP_ID;
typedef NvU32 GPU_MASK;
//
// This structure is used to trace lock acquire/release activity.
// The calling IP is stored in a circular array.
//
#define MAX_TRACE_LOCK_CALLS 32
typedef enum
{
lockTraceEmpty,
lockTraceAcquire,
lockTraceRelease,
lockTraceAlloc,
lockTraceFree
} LOCK_TRACE_TYPE;
typedef struct
{
LOCK_TRACE_TYPE type;
union {
GPU_MASK gpuMask; // For GPU locks
NvU32 lockModule; // For API lock
NvU32 value;
} data32;
union {
NvU16 gpuInst; // For GPU locks
NvU16 lockFlags; // For API lock
NvU16 value;
} data16;
NvBool bHighIrql;
NvU8 priority;
NvU64 callerRA;
NvU64 threadId;
NvU64 timestamp;
} LOCK_TRACE_ENTRY;
typedef struct
{
LOCK_TRACE_ENTRY entries[MAX_TRACE_LOCK_CALLS];
NvU32 index;
} LOCK_TRACE_INFO;
#define INSERT_LOCK_TRACE(plti, ra, t, d16, d32, ti, irql, pr, ts) \
{ \
(plti)->entries[(plti)->index].callerRA = (NvUPtr)ra; \
(plti)->entries[(plti)->index].type = t; \
(plti)->entries[(plti)->index].data16.value = d16; \
(plti)->entries[(plti)->index].data32.value = d32; \
(plti)->entries[(plti)->index].threadId = ti; \
(plti)->entries[(plti)->index].timestamp = ts; \
(plti)->entries[(plti)->index].bHighIrql = irql; \
(plti)->entries[(plti)->index].priority = pr; \
(plti)->index = ((plti)->index + 1) % MAX_TRACE_LOCK_CALLS; \
}
//
// Callers specify this value when they to lock all possible GPUs.
//
#define GPUS_LOCK_ALL (0xFFFFFFFF)
//
// Flags for rmGpusLock[Acquire,Release] operations.
//
// default no flags
#define GPUS_LOCK_FLAGS_NONE (0x00000000)
// conditional acquire; if lock is already held then return error
#define GPU_LOCK_FLAGS_COND_ACQUIRE NVBIT(0)
// acquire the lock in read (shared) mode, if applicable
#define GPU_LOCK_FLAGS_READ NVBIT(1)
// Attempt acquire even if it potentially violates the locking order
// But do not block in a way that could cause a deadlock
#define GPU_LOCK_FLAGS_SAFE_LOCK_UPGRADE NVBIT(2)
// Old name alias
#define GPUS_LOCK_FLAGS_COND_ACQUIRE GPU_LOCK_FLAGS_COND_ACQUIRE
//
// RM Lock Related Functions
//
NV_STATUS rmLocksAlloc(OBJSYS *);
void rmLocksFree(OBJSYS *);
NV_STATUS rmLocksAcquireAll(NvU32 module);
void rmLocksReleaseAll(void);
NV_STATUS workItemLocksAcquire(NvU32 gpuInstance, NvU32 flags, NvU32 *pReleaseLocks, NvU32 *pGpuMask);
void workItemLocksRelease(NvU32 releaseLocks, NvU32 gpuMask);
//
// Thread priority boosting and throttling:
// Used to temporarily increase the priority of a thread on Windows platforms
// in order to prevent starvation from the scheduler.
//
void threadPriorityStateAlloc(void);
void threadPriorityStateFree(void);
//! Temporarily boost the priority of the current thread
void threadPriorityBoost(NvU64* pBoostPriority, NvU64 *pOriginalPriority);
//! Gradually lower the priority of the current thread if it is boosted and sufficient time has elapsed
void threadPriorityThrottle(void);
//! Restore the original priority of the current thread if it is boosted
void threadPriorityRestore(void);
NV_STATUS rmGpuGroupLockGetMask(NvU32 gpuInst, GPU_LOCK_GRP_ID gpuGrpId, GPU_MASK* pGpuMask);
//
// Defines for rmGpuLockSetOwner operation.
//
#define GPUS_LOCK_OWNER_PENDING_DPC_REFRESH (OS_THREAD_HANDLE)(-1)
NV_STATUS rmGpuLockInfoInit(void);
void rmGpuLockInfoDestroy(void);
NV_STATUS rmGpuLockAlloc(NvU32);
void rmGpuLockFree(NvU32);
NV_STATUS rmGpuLocksAcquire(NvU32, NvU32);
NvU32 rmGpuLocksRelease(NvU32, OBJGPU *);
void rmGpuLocksFreeze(GPU_MASK);
void rmGpuLocksUnfreeze(GPU_MASK);
NV_STATUS rmGpuLockHide(NvU32);
void rmGpuLockShow(NvU32);
NvBool rmGpuLockIsOwner(void);
NvU32 rmGpuLocksGetOwnedMask(void);
NvBool rmGpuLockIsHidden(OBJGPU *);
NV_STATUS rmGpuLockSetOwner(OS_THREAD_HANDLE);
NV_STATUS rmGpuGroupLockAcquire(NvU32, GPU_LOCK_GRP_ID, NvU32, NvU32, GPU_MASK *);
NV_STATUS rmGpuGroupLockRelease(GPU_MASK, NvU32);
NvBool rmGpuGroupLockIsOwner(NvU32, GPU_LOCK_GRP_ID, GPU_MASK*);
NvBool rmDeviceGpuLockIsOwner(NvU32);
NV_STATUS rmDeviceGpuLockSetOwner(OBJGPU *, OS_THREAD_HANDLE);
NV_STATUS rmDeviceGpuLocksAcquire(OBJGPU *, NvU32, NvU32);
NvU32 rmDeviceGpuLocksRelease(OBJGPU *, NvU32, OBJGPU *);
NV_STATUS rmIntrMaskLockAlloc(NvU32 gpuInst);
void rmIntrMaskLockFree(NvU32 gpuInst);
/// @note The return value is always zero, not the actual IRQL
NvU64 rmIntrMaskLockAcquire(OBJGPU *pGpu);
void rmIntrMaskLockRelease(OBJGPU *pGpu, NvU64 oldIrql);
// wrappers for handling lock-related NV_ASSERT_OR_RETURNs
#define LOCK_ASSERT_AND_RETURN(cond) NV_ASSERT_OR_ELSE_STR((cond), #cond, return NV_ERR_INVALID_LOCK_STATE)
#define IRQL_ASSERT_AND_RETURN(cond) NV_ASSERT_OR_ELSE_STR((cond), #cond, return NV_ERR_INVALID_IRQ_LEVEL)
#define LOCK_ASSERT_AND_RETURN_BOOL(cond, bRet) NV_ASSERT_OR_ELSE_STR((cond), #cond, return (bRet))
#define LOCK_METER_OP(f,l,t,d0,d1,d2)
#define LOCK_METER_DATA(t,d0,d1,d2)
#define rmInitLockMetering()
#define rmDestroyLockMetering()
//
// RM API lock definitions are handled by the rmapi module. Providing legacy
// rmApiLockXxx interface for temporary compatibility. CORERM-1370
//
#include "rmapi/rmapi.h"
#define API_LOCK_FLAGS_NONE RMAPI_LOCK_FLAGS_NONE
#define API_LOCK_FLAGS_COND_ACQUIRE RMAPI_LOCK_FLAGS_COND_ACQUIRE
#define rmApiLockAcquire(flags, module) (rmapiLockAcquire(flags, module))
static NV_INLINE NV_STATUS rmApiLockRelease(void) {rmapiLockRelease(); return NV_OK;}
#define rmApiLockIsOwner() (rmapiLockIsOwner())
#endif // LOCKS_H

122
src/nvidia/inc/kernel/core/prelude.h Normal file
View File

@@ -0,0 +1,122 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __PRELUDE_H__
#define __PRELUDE_H__
/* ------------------------ C library --------------------------------------- */
#include <stddef.h> // NULL
/* ------------------------ SDK includes ------------------------------------ */
#include "nvtypes.h"
#include "nvrangetypes.h"
#include "nvstatus.h"
#include "nvmisc.h"
#include "nvlimits.h"
#include "nvos.h"
#include "nvctassert.h"
/* ------------------------ RM library and utils ---------------------------- */
#include "nvport/nvport.h"
#include "nvoc/runtime.h"
#include "nvoc/utility.h"
#include "core/printf.h"
#include "core/strict.h"
#include "utils/nvassert.h"
/* ------------------------ Code-generation --------------------------------- */
#include "rmconfig.h" // RMCONFIG header generated by config/rmconfig.pl
#include "g_rmconfig_private.h" // resman-private hal setup such as: IsGK104(), etc.
#include "g_nvh_state.h" // pass enable/disable state to NVOC headers
#include "g_odb.h"
#include "g_hal.h"
#include "rmcd.h"
/* ------------------------ Common types ------------------------------------ */
typedef NvU64 RmPhysAddr; // A physical address should be 64 bits
typedef struct THREAD_STATE_NODE THREAD_STATE_NODE; // FW declare thread state
/* ------------------------ Utility Macros ---------------------------------- */
//
// Power of 2 alignment.
// (Will give unexpected results if 'gran' is not a power of 2.)
// (v - v + gran) ensures that gran is upcasted to match v before
// the ~ operation, without explicitly having to typecast it.
//
#define RM_ALIGN_DOWN(v, gran) ((v) & ~(((v) - (v) + (gran)) - 1))
#define RM_ALIGN_UP(v, gran) (((v) + ((gran) - 1)) & ~(((v) - (v) + (gran))-1))
#define RM_IS_ALIGNED(v, gran) ((((gran) - 1) & (v)) == 0)
#define RM_ALIGN_PTR_DOWN(p, gran) ((void *) RM_ALIGN_DOWN(((NvUPtr)p), (gran)))
#define RM_ALIGN_PTR_UP(p, gran) ((void *) RM_ALIGN_UP(((NvUPtr)p), (gran)))
#define RM_PAGE_ALIGN_DOWN(value) RM_ALIGN_DOWN((value), RM_PAGE_SIZE)
#define RM_PAGE_ALIGN_UP(value) RM_ALIGN_UP((value), RM_PAGE_SIZE)
#define NV_DELTA(a, b) (NV_MAX((a), (b)) - NV_MIN((a), (b))) // Okay for unsigned or signed
#define NV_ROUNDUP(a,b) ((NV_CEIL(a,b))*(b))
#define NV_ROUND_TO_QUANTA(a, quanta) (((quanta) == 0) ? (a): ((((a) + ((quanta) >> 1)) / (quanta)) * (quanta)))
#define NV_FLOOR_TO_QUANTA(a, quanta) (((a) / (quanta)) * (quanta))
#define NV_SIZEOF32(x) (sizeof(x))
#define NV_ARRAY_ELEMENTS(x) ((sizeof(x)/sizeof((x)[0])))
#define NV_ARRAY_ELEMENTS32(x) ((NV_SIZEOF32(x)/NV_SIZEOF32((x)[0])))
#define NV_BYTESWAP16(a) ((((a) & 0xff00)>>8) | \
(((a) & 0x00ff)<<8))
#define NV_BYTESWAP32(a) ((((a) & 0xff000000)>>24) | \
(((a) & 0x00ff0000)>>8) | \
(((a) & 0x0000ff00)<<8) | \
(((a) & 0x000000ff)<<24))
#define NV_TO_LOWER(c) (((c)>='A'&&(c)<='Z')?(c)+('a'-'A'):(c))
#define NV_TO_UPPER(c) (((c)>='a'&&(c)<='z')?((c)-'a'+'A'):(c))
/*!
* Creates a byte mask for a word at given offset.
* offset = 0 0xffffff00
* offset = 1 0xffff00ff
* offset = 2 0xff00ffff
* offset = 3 0x00ffffff
*
* @param[in] offset Offset for the mask.
*/
#define NV_BYTE_MASK(offset) (~(0xff << ((offset)<<3)))
//
// note: the following trick fails if (z-1) * y > max_int
//
// since the calculation contains (x % z) * y,
// and the maximum value of (x % z) is (z-1).
//
// selecting the smaller of x and y to be y reduces the chances
// of problems, but for big enough z, the problem will return...
//
#define OVERFLOW_CAREFUL_MUL_DIV(x, y, z) \
((x) > (y)) ? (((x) / (z)) * (y) + (((x) % (z)) * (y)) / (z)) : (((y) / (z)) * (x) + (((y) % (z)) * (x)) / (z))
#define MASK_BITS(n) (~(0xFFFFFFFF << (n)))
#endif /* __PRELUDE_H__ */

368
src/nvidia/inc/kernel/core/printf.h Normal file
View File

@@ -0,0 +1,368 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2001-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _PRINTF_H_
#define _PRINTF_H_
/*
* RM PRINTF definitions.
*
* Provides RM internal definitions built on the generic nvprintf utilities
*/
#ifdef __cplusplus
extern "C" {
#endif
#include "nvport/nvport.h"
#include "utils/nvprintf.h"
#include "nvlog/nvlog.h"
#define DBG_FILE_LINE_FUNCTION NV_FILE_STR, __LINE__, NV_FUNCTION_STR
/**
* @todo bug 1583359 - Move to NvPort compiler specifics
*/
#if defined(__GNUC__) || defined(__clang__)
#define NV_RETURN_ADDRESS() __builtin_return_address(0)
#else
#define NV_RETURN_ADDRESS() _ReturnAddress()
#endif
//******************************************************************************
// BREAKPOINTS
//******************************************************************************
// NV_DBG_BREAKPOINT_ALLOWED can be overridden through CFLAGS
#if !defined(NV_DBG_BREAKPOINT_ALLOWED)
#if defined(DEBUG) || defined(ASSERT_BUILD) || defined(NV_MODS) || defined(QA_BUILD) || (defined(NVRM) && NVCPU_IS_RISCV64)
#define NV_DBG_BREAKPOINT_ALLOWED 1
#else
#define NV_DBG_BREAKPOINT_ALLOWED 0
#endif
#endif // !defined(NV_DBG_BREAKPOINT_ALLOWED)
#define NV_DEBUG_BREAK_FLAGS_RC 0:0
#define NV_DEBUG_BREAK_FLAGS_RC_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_RC_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_ASSERT 1:1
#define NV_DEBUG_BREAK_FLAGS_ASSERT_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_ASSERT_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK 2:2
#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT 3:3
#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS 4:4
#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_POWER_ON 5:5
#define NV_DEBUG_BREAK_FLAGS_POWER_ON_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_POWER_ON_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR 6:6
#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR_DISABLE (0x0)
#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR_ENABLE (0x1)
#define NV_DEBUG_BREAK_FLAGS_CRASH 7:7
#define NV_DEBUG_BREAK_FLAGS_CRASH_DISABLE (0x00000000)
#define NV_DEBUG_BREAK_FLAGS_CRASH_ENABLE (0x00000001)
#define NV_DEBUG_BREAK_ATTRIBUTES 7:0
#define NV_DEBUG_BREAK_ATTRIBUTES_NONE (0x00000000)
#define NV_DEBUG_BREAK_ATTRIBUTES_RC (0x00000001)
#define NV_DEBUG_BREAK_ATTRIBUTES_ASSERT (0x00000002)
#define NV_DEBUG_BREAK_ATTRIBUTES_DBG_BREAK (0x00000004)
#define NV_DEBUG_BREAK_ATTRIBUTES_GPU_TIMEOUT (0x00000008)
#define NV_DEBUG_BREAK_ATTRIBUTES_POOL_TAGS (0x00000010)
#define NV_DEBUG_BREAK_ATTRIBUTES_POWER_ON (0x00000020)
#define NV_DEBUG_BREAK_ATTRIBUTES_SMU_ERROR (0x00000040)
#define NV_DEBUG_BREAK_ATTRIBUTES_CRASH (0x00000080)
// Checks RMINFO and OS config to see if triggering a breakpoint is ever allowed
NvBool nvDbgBreakpointEnabled(void);
// Flushes the logs before a breakpoint, so we can see all the prints.
void osFlushLog(void);
#define DBG_ROUTINE() \
do \
{ \
if (nvDbgBreakpointEnabled()) \
PORT_BREAKPOINT_ALWAYS(); \
} while (0)
#define REL_DBG_BREAKPOINT() \
REL_DBG_BREAKPOINT_MSG("NVRM-RC: Nvidia GPU Error Detected\n")
#if NV_DBG_BREAKPOINT_ALLOWED
#if !NVCPU_IS_RISCV64
#define DBG_BREAKPOINT_EX(PGPU, LEVEL) \
do \
{ \
NV_PRINTF(LEVEL_ERROR, "bp @ " NV_FILE_FMT ":%d\n", NV_FILE, __LINE__);\
osFlushLog(); \
DBG_ROUTINE(); \
} while (0)
#else // !NVCPU_IS_RISCV64
#define DBG_BREAKPOINT_EX(PGPU, LEVEL) \
do \
{ \
NV_ASSERT_FAILED("DBG_BREAKPOINT"); \
} while (0)
#endif // !NVCPU_IS_RISCV64
#define DBG_BREAKPOINT() DBG_BREAKPOINT_EX(NULL, 0)
#define DBG_BREAKPOINT_EX_ARGS_IGNORED 1
#define REL_DBG_BREAKPOINT_MSG(msg) \
do \
{ \
PORT_DBG_PRINT_STRING_LITERAL(msg); \
DBG_BREAKPOINT(); \
} while (0)
#else // !NV_DBG_BREAKPOINT_ALLOWED
#define DBG_BREAKPOINT()
#define DBG_BREAKPOINT_EX(PGPU, LEVEL)
#define DBG_BREAKPOINT_EX_ARGS_IGNORED 1
#define REL_DBG_BREAKPOINT_MSG(msg) \
do \
{ \
PORT_DBG_PRINT_STRING_LITERAL(msg); \
DBG_ROUTINE(); \
} while (0)
#endif // NV_DBG_BREAKPOINT_ALLOWED
#define DBG_BREAKPOINT_REASON(reason) DBG_BREAKPOINT()
#define DBG_BREAKPOINT_ERROR_INFO(errorCategory, errorInfo) DBG_BREAKPOINT()
//******************************************************************************
// PRINTS
//******************************************************************************
#include "utils/nvprintf.h"
#define MAX_ERROR_STRING 256
#ifndef NVPORT_CHECK_PRINTF_ARGUMENTS
#define NVPORT_CHECK_PRINTF_ARGUMENTS(x,c)
#endif
//
// Prototypes
//
NvBool nvDbgInit(void);
void nvDbgDestroy(void);
void nvDbg_Printf (const char *file, int line, const char *function, int debuglevel, const char *s, ...) NVPORT_CHECK_PRINTF_ARGUMENTS(5, 6);
//
// Like libc's vsnprintf(), nvDbg_vPrintf() invalidates its va_list argument. The va_list argument
// may not be reused after nvDbg_vPrintf() returns. If the va_list is needed after the
// nvDbg_vPrintf() call, create a copy of the va_list using va_copy().
// The caller controls the lifetime of the va_list argument, and should free it using va_end.
//
void nvDbg_vPrintf (const char *file, int line, const char *function, int debuglevel, const char *s, va_list args);
void nvDbg_PrintBuf(const char *file, int line, const char *function, int debgulevel, NvU8 buffer[], NvU32 bufsize);
int nvDbgVsnprintf(char *dest, NvU32 destSize, const char *fmt, va_list args);
int nvDbgSnprintf (char *dest, NvU32 destSize, const char *fmt, ...);
struct OBJGPU;
void nvDbgInitRmMsg(struct OBJGPU *);
// RmMsgPrefix return value
#define NVRM_MSG_PREFIX_NVRM NVBIT(0)
#define NVRM_MSG_PREFIX_FILE NVBIT(1)
#define NVRM_MSG_PREFIX_FUNCTION NVBIT(2)
#define NVRM_MSG_PREFIX_LINE NVBIT(3)
#define NVRM_MSG_PREFIX_OSTIMESTAMP NVBIT(4)
NvU32 RmMsgPrefix(NvU32 prefix, const char *filename, NvU32 linenumber, const char *function, char *str, NvU32 len);
// nvDbgRmMsgCheck return code
#define NVRM_MSG_NORMAL 0 // Use normal message handling (warnings/errors)
#define NVRM_MSG_HIDE 1 // Skip this message
#define NVRM_MSG_PRINT 2 // Force printing of this message
NvU32 nvDbgRmMsgCheck(const char *filename, NvU32 linenumber, const char *function, NvU32 level, const char *format, NvU32 *pPrefix);
void nvDbgDumpBufferBytes(void *pBuffer, NvU32 length);
#if NV_PRINTF_STRINGS_ALLOWED
#define DBG_STRING(str) str
#define DBG_INIT() nvDbgInit()
#define DBG_DESTROY() nvDbgDestroy()
#define DBG_VSNPRINTF(ptr_size_format_and_stuff) nvDbgVsnprintf ptr_size_format_and_stuff
#define DBG_PRINTBUF(dbglevel, buffer, bufsize) nvDbg_PrintBuf(DBG_FILE_LINE_FUNCTION, dbglevel, buffer, bufsize)
#define DBG_RMMSG_CHECK(level) (nvDbgRmMsgCheck(DBG_FILE_LINE_FUNCTION, level, NULL, NULL) == NVRM_MSG_PRINT)
#else // ! NV_PRINTF_STRINGS_ALLOWED -- debug printf strings not enabled
#define DBG_STRING(str) ""
#define DBG_INIT() (NV_TRUE)
#define DBG_DESTROY()
#define DBG_VSNPRINTF(ptr_size_format_and_stuff)
#define DBG_PRINTBUF(dbglevel, buffer, bufsize)
#define DBG_RMMSG_CHECK(level) (0)
#endif // NV_PRINTF_STRINGS_ALLOWED
//******************************************************************************
// POWER SANITY CHECKS
//******************************************************************************
//
// Make sure the GPU is in full power or resuming from D3 state. Else,
// bailout from the calling function. An exception for systems, which support
// surprise removal feature. See Bugs 440565, 479003, and 499228.DO NOT IGNORE
// OR REMOVE THIS ASSERT. If you have problems with it, please talk to cplummer.
//
// bAllowWithoutSysmemAccess: Allow this RM Control when sysmem access is not available
// from the GPU. SHould be NV_TRUE only for NV2080_CTRL_CMD_BUS_SYSMEM_ACCESS
//
// On systems supporting surprise removal, if the GPU is in D3 cold
// and still attached we would consider it a true D3 cold state
// and return NOT_FULL_POWER. See bug 1679965.
//
//
#define API_GPU_FULL_POWER_SANITY_CHECK(pGpu, bGpuAccess, bAllowWithoutSysmemAccess) \
if ((!gpuIsGpuFullPower(pGpu)) && \
(!(pGpu)->getProperty((pGpu), \
PDB_PROP_GPU_IN_PM_RESUME_CODEPATH))) \
{ \
DBG_BREAKPOINT(); \
if (bGpuAccess || (!gpuIsSurpriseRemovalSupported(pGpu))) \
{ \
return NV_ERR_GPU_NOT_FULL_POWER; \
} \
else if (gpuIsSurpriseRemovalSupported(pGpu) && \
(pGpu)->getProperty((pGpu), PDB_PROP_GPU_IS_CONNECTED)) \
{ \
return NV_ERR_GPU_NOT_FULL_POWER; \
} \
} \
if (!(bAllowWithoutSysmemAccess) && !gpuCheckSysmemAccess(pGpu)) \
{ \
return NV_ERR_GPU_NOT_FULL_POWER; \
}
#define API_GPU_FULL_POWER_SANITY_CHECK_OR_GOTO(pGpu, bGpuAccess, bAllowWithoutSysmemAccess, status, tag) \
if ((!gpuIsGpuFullPower(pGpu)) && \
(!(pGpu)->getProperty((pGpu), \
PDB_PROP_GPU_IN_PM_RESUME_CODEPATH))) \
{ \
DBG_BREAKPOINT(); \
if (bGpuAccess || (!gpuIsSurpriseRemovalSupported(pGpu))) \
{ \
status = NV_ERR_GPU_NOT_FULL_POWER; \
goto tag; \
} \
else if (gpuIsSurpriseRemovalSupported(pGpu) && \
(pGpu)->getProperty((pGpu), PDB_PROP_GPU_IS_CONNECTED)) \
{ \
status = NV_ERR_GPU_NOT_FULL_POWER; \
goto tag; \
} \
} \
if (!(bAllowWithoutSysmemAccess) && !gpuCheckSysmemAccess(pGpu)) \
{ \
return NV_ERR_GPU_NOT_FULL_POWER; \
}
#if defined(PORT_IS_FUNC_SUPPORTED)
#if PORT_IS_FUNC_SUPPORTED(portMemExValidate)
#define DBG_VAL_PTR(p) portMemExValidate(p, NV_TRUE)
#endif
#endif
#ifndef DBG_VAL_PTR
#define DBG_VAL_PTR(p)
#endif
//********************************************************************************
//
// NVRM_TRACE support
// low-overhead runtime state capture
// to enable, define USE_NVRM_TRACE (retail or debug builds)
//
//********************************************************************************
#ifdef USE_NVRM_TRACE
NvU32 NVRM_TRACE_INIT(void);
NvU32 NVRM_TRACE_DISABLE(void);
void NVRM_TRACE_ENABLE(void);
void NVRM_TRACE_DUMP(void);
void NVRM_TRACE(NvU32);
void NVRM_TRACEV(NvU32 *,NvU32);
void NVRM_TRACE1(NvU32);
void NVRM_TRACE2(NvU32, NvU32);
void NVRM_TRACE3(NvU32, NvU32, NvU32);
void NVRM_TRACE4(NvU32, NvU32, NvU32, NvU32);
void NVRM_TRACE5(NvU32, NvU32, NvU32, NvU32, NvU32);
// versions of reg read/write that log to trace buffer
//NvU32 NVRM_TRACE_REG_RD32(OBJGPU *, NvU32);
//void NVRM_TRACE_REG_WR32(OBJGPU *, NvU32, NvU32);
// fifolog format looks like:
// 31:28 = unique file number
// 27:4 = file line number
// 1:0 = fifo state bits (bit1 = puller, bit0 = reassign)
#define FIFOLOG(fn,fa,fb) NVRM_TRACE2('FIFO', ((fn << 28) | (__LINE__ << 4) | \
((fa & 0x1) ? 1 : 0) << 1 | \
((fb & 0x1) ? 1 : 0)) )
#else // ! USE_NVRM_TRACE
#define NVRM_TRACE_INIT()
#define NVRM_TRACE_DISABLE() 0
#define NVRM_TRACE_ENABLE()
#define NVRM_TRACE_DUMP()
#define NVRM_TRACE(c0)
#define NVRM_TRACE1(c0)
#define NVRM_TRACE2(c0, c1)
#define NVRM_TRACE3(c0, c1, c2)
#define NVRM_TRACE4(c0, c1, c2, c3)
#define NVRM_TRACE5(c0, c1, c2, c3, c4)
#define FIFOLOG(a,b,c)
#endif // ! USE_NVRM_TRACE
#define NVRM_TRACE_ERROR(code, status) NVRM_TRACE3('EEEE', (code), (status))
#define NVRM_TRACE_API(code, p0, p1, p2) NVRM_TRACE5('API ', (code), (p0), (p1), (p2))
void nvErrorLog(void *pVoid, NvU32 num, const char *pFormat, va_list arglist);
void nvErrorLog_va(void * pGpu, NvU32 num, const char * pFormat, ...);
// memory allocation tracking data structs and globals
#define MAX_STACK_LEVEL 6
#ifdef __cplusplus
}
#endif
#endif // _PRINTF_H_

99
src/nvidia/inc/kernel/core/strict.h Normal file
View File

@@ -0,0 +1,99 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __STRICT_H__
#define __STRICT_H__
//
// RM_STRICT_SUPPRESS_DEPRECATED_DEFINITIONS_VER_XYZ should be set
// before including any RM internal headers when disabling deprecated
// definitions is desired.
//
// For transition during refactoring, we might introduce new types and
// interfaces and use macros/wrappers to forward the old interface to
// the new one.
//
// Once a callsite is migrated to use the new interface it can use RM
// strict to disable the deprecated definitions to prevent changes from
// reintroducing calls to a deprecated interface within a cleansed
// module.
//
// Controlling disablement of deprecated definitions is versioned. This
// enables us to introduce new deprecated interfaces incrementally.
// Example, ModuleA might scrub to versionX (removal of OBJFB defns) but
// not versionY (removal of legacy CLI types).
//
// Flags to turn off deprecated definitions are intended to be
// temporary, once all modules remove references the deprecated
// definitions and knobs in this header should be deleted.
//
#ifdef RM_STRICT_SUPPRESS_DEPRECATED_DEFINITIONS_VER_JAN_21_2020
#define RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS 0
#define RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS 0
#endif
//
// RM_STRICT_SUPPRESS_PHYSICAL_DEFINITIONS_VER_XYZ should be set before
// including any RM internal headers when disabling "physical" definitions is
// desired.
//
// Physical definitions refers to interfaces/types that are only used by GSP-RM
// and VGPU-host, i.e.: not to be used by VGPU Client or GSP Client
//
#ifdef RM_STRICT_SUPPRESS_PHYSICAL_DEFINITIONS_VER_JAN_21_2020
#define RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS 0
#endif
//
// Default deprecated and "physical engine" definitions on unless specified
//
#ifndef RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS
#define RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS 1
#endif
#ifndef RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS
#define RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS 1
#endif
#ifndef RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS
#define RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS 1
#endif
//
// "Physical engine" definitions not yet included in any version, but available
// for T234X. Should be defined to 0 before including any RM internal headers
// when disabling OBJDISP (and related) definitions is desired.
//
#ifndef RM_STRICT_CONFIG_EMIT_DISP_ENGINE_DEFINITIONS
#define RM_STRICT_CONFIG_EMIT_DISP_ENGINE_DEFINITIONS 1
#endif
//
// Generate OBJGPU engine accessors (GPU_GET_FOO(pGpu)) for disabled engines.
// These will always return NULL, but will allow the code that references them
// to compile.
//
#ifndef RM_STRICT_CONFIG_EMIT_DISABLED_GPU_ENGINE_ACCESSORS
#define RM_STRICT_CONFIG_EMIT_DISABLED_GPU_ENGINE_ACCESSORS 1
#endif
#endif /* __STRICT_H__ */

3
src/nvidia/inc/kernel/core/system.h Normal file
View File

@@ -0,0 +1,3 @@
#include "g_system_nvoc.h"

217
src/nvidia/inc/kernel/core/thread_state.h Normal file
View File

@@ -0,0 +1,217 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef THREAD_STATE_H
#define THREAD_STATE_H
/**************** Resource Manager Defines and Structures ******************\
* *
* Defines and structures used for Thread State management *
* *
\***************************************************************************/
#include "core/core.h"
#include "os/os.h"
#include "tls/tls.h"
#include "containers/map.h"
#include "containers/list.h"
typedef struct OBJGPU OBJGPU;
//
// Thread State Tracking structures and defines
//
typedef struct THREAD_TIMEOUT_STATE
{
NvU64 enterTime;
NvU64 nonComputeTime;
NvU64 computeTime;
NvU64 nextCpuYieldTime;
NvU64 overrideTimeoutMsecs;
} THREAD_TIMEOUT_STATE, *PTHREAD_TIMEOUT_STATE;
typedef struct THREAD_STATE_FREE_CALLBACK
{
void *pCbData;
void (*pCb)(void *pCbData);
} THREAD_STATE_FREE_CALLBACK;
MAKE_LIST(THREAD_STATE_FREE_CB_LIST, THREAD_STATE_FREE_CALLBACK);
typedef struct THREAD_STATE_NODE THREAD_STATE_NODE;
struct THREAD_STATE_NODE
{
OS_THREAD_HANDLE threadId;
/*!
* Thread sequencer id. This is a unique identifier for a given thread
* entry into the RM. This is separate from @ref threadId, as the threadId
* is really the OS's thread handle/pointer. In cases where the same
* physical thread is re-used (e.g. WORK_ITEMs are scheduled from a
* pre-allocated pool of worker threads), different RM threads will have the
* same threadId.
*
* This value is set by @ref threadStateInitXYZ() based off the global @ref
* THREAD_STATE_DB::threadSeqCntr.
*/
NvU32 threadSeqId;
NvBool bValid;
THREAD_TIMEOUT_STATE timeout;
NvU32 cpuNum;
NvU32 flags;
MapNode node;
/*!
* If a callback is installed, threadStateFree() may block on it.
*
* The installed callbacks will be processed in FIFO order only.
*
* Only supported on non-ISR CPU RM paths.
*/
THREAD_STATE_FREE_CB_LIST cbList;
};
MAKE_INTRUSIVE_MAP(ThreadStateNodeMap, THREAD_STATE_NODE, node);
typedef struct THREAD_STATE_DB_TIMEOUT
{
NvU64 nonComputeTimeoutMsecs;
NvU64 computeTimeoutMsecs;
NvU32 computeGpuMask;
NvU32 flags;
} THREAD_STATE_DB_TIMEOUT, *PTHREAD_STATE_DB_TIMEOUT;
#define THREAD_STATE_TRACE_MAX_ENTRIES 8
typedef struct THREAD_STATE_TRACE_ENTRY
{
NvU64 callerRA;
NvU32 flags;
} THREAD_STATE_TRACE_ENTRY;
typedef struct THREAD_STATE_TRACE_INFO
{
NvU32 index;
THREAD_STATE_TRACE_ENTRY entries[THREAD_STATE_TRACE_MAX_ENTRIES];
} THREAD_STATE_TRACE_INFO;
typedef struct THREAD_STATE_ISR_LOCKLESS
{
THREAD_STATE_NODE **ppIsrThreadStateGpu;
} THREAD_STATE_ISR_LOCKLESS, *PTHREAD_STATE_ISR_LOCKLESS, **PPTHREAD_STATE_ISR_LOCKLESS;
typedef struct THREAD_STATE_DB
{
NvU32 setupFlags;
NvU32 maxCPUs;
/*!
* Thread state sequencer id counter. The last allocated thread state
* sequencer id via @ref threadStateInitXYZ().
*/
NvU32 threadSeqCntr;
PORT_SPINLOCK *spinlock;
ThreadStateNodeMap dbRoot;
ThreadStateNodeMap dbRootPreempted;
THREAD_STATE_NODE **ppISRDeferredIntHandlerThreadNode;
PTHREAD_STATE_ISR_LOCKLESS pIsrlocklessThreadNode;
THREAD_STATE_DB_TIMEOUT timeout;
THREAD_STATE_TRACE_INFO traceInfo;
} THREAD_STATE_DB, *PTHREAD_STATE_DB;
//
// This is the same for all OSes. This value was chosen because it is
// the minimum found on any OS at the time of this writing (May, 2008).
//
#define TIMEOUT_DEFAULT_OS_RESCHEDULE_INTERVAL_SECS 2
//
// The normal power transition requirement for Windows is 4 seconds.
// Use longer time to let OS fire timeout and ask recovery.
//
#define TIMEOUT_WDDM_POWER_TRANSITION_INTERVAL_MS 9800
//
// Thread State flags used for threadStateInitSetupFlags
//
#define THREAD_STATE_SETUP_FLAGS_NONE 0
#define THREAD_STATE_SETUP_FLAGS_ENABLED NVBIT(0)
#define THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED NVBIT(1)
#define THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED NVBIT(2)
#define THREAD_STATE_SETUP_FLAGS_CHECK_TIMEOUT_AT_FREE_ENABLED NVBIT(3)
#define THREAD_STATE_SETUP_FLAGS_ASSERT_ON_TIMEOUT_ENABLED NVBIT(4)
#define THREAD_STATE_SETUP_FLAGS_ASSERT_ON_FAILED_LOOKUP_ENABLED NVBIT(5)
#define THREAD_STATE_SETUP_FLAGS_RESET_ON_TIMEOUT_ENABLED NVBIT(6)
#define THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED NVBIT(7)
#define THREAD_STATE_SETUP_FLAGS_PRINT_INFO_ENABLED NVBIT(31)
//
// Thread State flags used for threadState[Init,Free]
//
#define THREAD_STATE_FLAGS_NONE 0
#define THREAD_STATE_FLAGS_IS_ISR NVBIT(0)
#define THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER NVBIT(1)
#define THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER NVBIT(2)
#define THREAD_STATE_FLAGS_IS_ISR_LOCKLESS NVBIT(3)
#define THREAD_STATE_FLAGS_TIMEOUT_INITED NVBIT(5)
#define THREAD_STATE_FLAGS_PLACED_ON_PREEMPT_LIST NVBIT(6)
#define THREAD_STATE_FLAGS_DEVICE_INIT NVBIT(7)
#define THREAD_STATE_FLAGS_STATE_FREE_CB_ENABLED NVBIT(8)
// These Threads run exclusively between a conditional acquire
#define THREAD_STATE_FLAGS_EXCLUSIVE_RUNNING (THREAD_STATE_FLAGS_IS_ISR | \
THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER | \
THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER)
#define THREAD_STATE_FLAGS_DEFERRED_INT_HANDLER_RUNNING (THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER | \
THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER)
NV_STATUS threadStateGlobalAlloc(void);
void threadStateGlobalFree(void);
void threadStateInitRegistryOverrides(OBJGPU *pGpu);
void threadStateInitSetupFlags(NvU32 flags);
NvU32 threadStateGetSetupFlags(void);
void threadStateInitISRLockless(THREAD_STATE_NODE *, OBJGPU*, NvU32);
void threadStateFreeISRLockless(THREAD_STATE_NODE *, OBJGPU*, NvU32);
void threadStateInitISRAndDeferredIntHandler(THREAD_STATE_NODE *, OBJGPU*, NvU32);
void threadStateFreeISRAndDeferredIntHandler(THREAD_STATE_NODE *, OBJGPU*, NvU32);
void threadStateInit(THREAD_STATE_NODE *pThreadNode, NvU32 flags);
void threadStateFree(THREAD_STATE_NODE *pThreadNode, NvU32 flags);
NV_STATUS threadStateGetCurrent(THREAD_STATE_NODE **ppThreadNode, OBJGPU *pGpu);
NV_STATUS threadStateGetCurrentUnchecked(THREAD_STATE_NODE **ppThreadNode, OBJGPU *pGpu);
NV_STATUS threadStateInitTimeout(OBJGPU *pGpu, NvU32 timeoutUs, NvU32 flags);
NV_STATUS threadStateCheckTimeout(OBJGPU *pGpu, NvU64 *pElapsedTimeUs);
NV_STATUS threadStateResetTimeout(OBJGPU *pGpu);
void threadStateLogTimeout(OBJGPU *pGpu, NvU64 funcAddr, NvU32 lineNum);
void threadStateYieldCpuIfNecessary(OBJGPU *pGpu);
void threadStateSetTimeoutOverride(THREAD_STATE_NODE *, NvU64);
NV_STATUS threadStateEnqueueCallbackOnFree(THREAD_STATE_NODE *pThreadNode,
THREAD_STATE_FREE_CALLBACK *pCallback);
void threadStateRemoveCallbackOnFree(THREAD_STATE_NODE *pThreadNode,
THREAD_STATE_FREE_CALLBACK *pCallback);
#endif // THREAD_STATE_H