515.43.04

src/nvidia/inc/kernel/compute/fabric.h

@@ -0,0 +1,3 @@
+
+#include "g_fabric_nvoc.h"
+

src/nvidia/inc/kernel/compute/fm_session_api.h

@@ -0,0 +1,3 @@
+
+#include "g_fm_session_api_nvoc.h"
+

src/nvidia/inc/kernel/compute/mps_api.h

@@ -0,0 +1,3 @@
+
+#include "g_mps_api_nvoc.h"
+

src/nvidia/inc/kernel/core/bin_data.h

@@ -0,0 +1,106 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _BINDATA_H
+#define _BINDATA_H
+
+#include "core/core.h"
+
+/**************************************************************************************************************
+*
+*    File:  bindata.h
+*
+*    Description:
+*        Bindata management APIs
+*
+**************************************************************************************************************/
+
+//
+// Public interface for accessing the acquired binary data
+//
+
+//
+// Binary data access handler
+//
+typedef struct BINDATA_RUNTIME_INFO BINDATA_RUNTIME_INFO, *PBINDATA_RUNTIME_INFO;
+
+//
+// Public binary storage information
+//
+struct BINDATA_STORAGE;         // currently no public data fields
+typedef struct BINDATA_STORAGE BINDATA_STORAGE, *PBINDATA_STORAGE;
+
+
+//
+// Primitives
+//
+NV_STATUS bindataAcquire(const BINDATA_STORAGE *pBinStorage, PBINDATA_RUNTIME_INFO *ppBinInfo);
+NV_STATUS bindataGetNextChunk(PBINDATA_RUNTIME_INFO pBinInfo, NvU8 *pBuffer, NvU32 nBytes);
+void      bindataRelease(PBINDATA_RUNTIME_INFO pBinInfo);
+
+
+//
+// Utilities
+//
+NV_STATUS bindataWriteToBuffer(const BINDATA_STORAGE *pBinStorage, NvU8 *pBuffer, NvU32 bufferSize);
+NvU32     bindataGetBufferSize(const BINDATA_STORAGE *pBinStorage);
+
+
+//
+// Bindata Archive support
+//
+typedef struct
+{
+    const char*              name;                // string of file name or name tag
+    const PBINDATA_STORAGE   pBinStorage;         // pointer to the binary storage
+} BINDATA_ARCHIVE_ENTRY;
+
+typedef struct
+{
+    NvU32 entryNum;
+    BINDATA_ARCHIVE_ENTRY entries[];
+} BINDATA_ARCHIVE;
+
+
+// Bindata Archive API - get Bindata storage from a Bindata Archive
+const BINDATA_STORAGE * bindataArchiveGetStorage(const BINDATA_ARCHIVE *pBinArchive, const char *bindataName);
+
+//
+// Iterate over all BINDATA_STORAGE entries that have not been referenced so far
+// Returns the pointer to unreferenced data or NULL if no more are available.
+// Example usage:
+//    const BINDATA_STORAGE *iter = NULL;
+//    void *datablock;
+//    NvU32 size;
+//    while ((datablock = bindataGetNextUnreferencedStorage(&iter, &size))) {
+//        do_stuff(datablock, size);
+//    }
+//
+void* bindataGetNextUnreferencedStorage(const BINDATA_STORAGE **iter, NvU32 *pDataSize);
+//
+// Marks a given BINDATA_STORAGE as destroyed, making all subsequent attempts
+// to access it fail and return NULL/0
+//
+void bindataDestroyStorage(BINDATA_STORAGE *storage);
+
+#endif // _BINDATA_H

src/nvidia/inc/kernel/core/core.h

@@ -0,0 +1,50 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __CORE_H__
+#define __CORE_H__
+
+#include "core/prelude.h"
+
+/**
+ * @brief Global RM initialization
+ *
+ * The single entrypoint into the RM for all platforms. This will initial cross
+ * platform RM subsystems and call into OS specific init as needed.
+ *
+ * Must be called once and only once before any RM internal functions can be
+ * called
+ *
+ * @return NV_OK if successful, error otherwise
+ */
+NV_STATUS coreInitializeRm(void);
+
+/**
+ * @brief Global RM shutdown
+ *
+ * Must be called once and only once when a driver is shutting down and no more
+ * RM internal functions will be called.
+ *
+ */
+void coreShutdownRm(void);
+
+#endif /* __CORE_H__ */

src/nvidia/inc/kernel/core/hal.h

@@ -0,0 +1,3 @@
+
+#include "g_hal_nvoc.h"
+

src/nvidia/inc/kernel/core/hal_mgr.h

@@ -0,0 +1,3 @@
+
+#include "g_hal_mgr_nvoc.h"
+

src/nvidia/inc/kernel/core/info_block.h

@@ -0,0 +1,59 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _INFO_BLOCK_H_
+#define _INFO_BLOCK_H_
+
+//
+// HAL privata data management.
+//
+typedef struct ENG_INFO_LINK_NODE *PENG_INFO_LINK_NODE;
+typedef struct ENG_INFO_LINK_NODE ENG_INFO_LINK_NODE;
+
+// new style typedef for info block functions, simple typedef.
+// Used by hal .def files via INFO_BLOCK_GROUP template in Gpuhal.def
+typedef void    *EngGetInfoBlockFn(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
+typedef void    *EngAddInfoBlockFn(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId, NvU32 size);
+typedef void     EngDeleteInfoBlockFn(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId);
+typedef NvBool   EngTestInfoBlockFn(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
+
+// old style typedef for info block functions (ptr to fn)
+// delete these 4 typedefs once all .def files converted to use OBJECT_INTERFACES
+typedef EngGetInfoBlockFn    *EngGetInfoBlock;
+typedef EngAddInfoBlockFn    *EngAddInfoBlock;
+typedef EngDeleteInfoBlockFn *EngDeleteInfoBlock;
+typedef EngTestInfoBlockFn   *EngTestInfoBlock;
+
+
+struct ENG_INFO_LINK_NODE
+{
+    NvU32   dataId;
+    void    *infoBlock;
+    PENG_INFO_LINK_NODE next;
+};
+
+void* getInfoPtr(PENG_INFO_LINK_NODE pHead, NvU32 dataId);
+void* addInfoPtr(PENG_INFO_LINK_NODE *ppHead, NvU32 dataId, NvU32 size);
+void  deleteInfoPtr(PENG_INFO_LINK_NODE * ppHead, NvU32 dataId);
+NvBool testInfoPtr(PENG_INFO_LINK_NODE, NvU32 dataId);
+
+#endif // _INFO_BLOCK_H_

src/nvidia/inc/kernel/core/locks.h

@@ -0,0 +1,205 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef LOCKS_H
+#define LOCKS_H
+
+#include "core/core.h"
+#include "os/os.h"
+
+// Forward declarations
+typedef struct OBJSYS OBJSYS;
+
+typedef enum
+{
+    GPU_LOCK_GRP_SUBDEVICE,  // locks will be taken for subdevice only
+    GPU_LOCK_GRP_DEVICE,     // locks will be taken for device only
+    GPU_LOCK_GRP_MASK,       // locks will be taken for devices specified by the mask
+    GPU_LOCK_GRP_ALL         // locks will be taken for all devices
+} GPU_LOCK_GRP_ID;
+typedef NvU32 GPU_MASK;
+
+//
+// This structure is used to trace lock acquire/release activity.
+// The calling IP is stored in a circular array.
+//
+#define MAX_TRACE_LOCK_CALLS           32
+
+typedef enum
+{
+    lockTraceEmpty,
+    lockTraceAcquire,
+    lockTraceRelease,
+    lockTraceAlloc,
+    lockTraceFree
+} LOCK_TRACE_TYPE;
+
+typedef struct
+{
+    LOCK_TRACE_TYPE type;
+    union {
+        GPU_MASK gpuMask; // For GPU locks
+        NvU32 lockModule; // For API lock
+        NvU32 value;
+    } data32;
+    union {
+        NvU16 gpuInst;    // For GPU locks
+        NvU16 lockFlags;  // For API lock
+        NvU16 value;
+    } data16;
+    NvBool bHighIrql;
+    NvU8   priority;
+    NvU64  callerRA;
+    NvU64  threadId;
+    NvU64  timestamp;
+} LOCK_TRACE_ENTRY;
+
+typedef struct
+{
+    LOCK_TRACE_ENTRY entries[MAX_TRACE_LOCK_CALLS];
+    NvU32 index;
+} LOCK_TRACE_INFO;
+
+#define INSERT_LOCK_TRACE(plti, ra, t, d16, d32, ti, irql, pr, ts)      \
+{                                                               \
+    (plti)->entries[(plti)->index].callerRA = (NvUPtr)ra;       \
+    (plti)->entries[(plti)->index].type = t;                    \
+    (plti)->entries[(plti)->index].data16.value = d16;          \
+    (plti)->entries[(plti)->index].data32.value = d32;          \
+    (plti)->entries[(plti)->index].threadId = ti;               \
+    (plti)->entries[(plti)->index].timestamp = ts;              \
+    (plti)->entries[(plti)->index].bHighIrql = irql;            \
+    (plti)->entries[(plti)->index].priority = pr;               \
+    (plti)->index = ((plti)->index + 1) % MAX_TRACE_LOCK_CALLS; \
+}
+
+//
+// Callers specify this value when they to lock all possible GPUs.
+//
+#define GPUS_LOCK_ALL                                   (0xFFFFFFFF)
+
+//
+// Flags for rmGpusLock[Acquire,Release] operations.
+//
+
+// default no flags
+#define GPUS_LOCK_FLAGS_NONE                            (0x00000000)
+// conditional acquire; if lock is already held then return error
+#define GPU_LOCK_FLAGS_COND_ACQUIRE                     NVBIT(0)
+// acquire the lock in read (shared) mode, if applicable
+#define GPU_LOCK_FLAGS_READ                             NVBIT(1)
+// Attempt acquire even if it potentially violates the locking order
+// But do not block in a way that could cause a deadlock
+#define GPU_LOCK_FLAGS_SAFE_LOCK_UPGRADE                NVBIT(2)
+// Old name alias
+#define GPUS_LOCK_FLAGS_COND_ACQUIRE                    GPU_LOCK_FLAGS_COND_ACQUIRE
+
+//
+// RM Lock Related Functions
+//
+NV_STATUS  rmLocksAlloc(OBJSYS *);
+void       rmLocksFree(OBJSYS *);
+
+NV_STATUS  rmLocksAcquireAll(NvU32 module);
+void       rmLocksReleaseAll(void);
+
+NV_STATUS   workItemLocksAcquire(NvU32 gpuInstance, NvU32 flags, NvU32 *pReleaseLocks, NvU32 *pGpuMask);
+void        workItemLocksRelease(NvU32 releaseLocks, NvU32 gpuMask);
+
+//
+// Thread priority boosting and throttling:
+//   Used to temporarily increase the priority of a thread on Windows platforms
+//   in order to prevent starvation from the scheduler.
+//
+void        threadPriorityStateAlloc(void);
+void        threadPriorityStateFree(void);
+
+//! Temporarily boost the priority of the current thread
+void        threadPriorityBoost(NvU64* pBoostPriority, NvU64 *pOriginalPriority);
+
+//! Gradually lower the priority of the current thread if it is boosted and sufficient time has elapsed
+void        threadPriorityThrottle(void);
+
+//! Restore the original priority of the current thread if it is boosted
+void        threadPriorityRestore(void);
+
+NV_STATUS  rmGpuGroupLockGetMask(NvU32 gpuInst, GPU_LOCK_GRP_ID gpuGrpId, GPU_MASK* pGpuMask);
+
+//
+// Defines for rmGpuLockSetOwner operation.
+//
+#define GPUS_LOCK_OWNER_PENDING_DPC_REFRESH             (OS_THREAD_HANDLE)(-1)
+
+NV_STATUS  rmGpuLockInfoInit(void);
+void       rmGpuLockInfoDestroy(void);
+NV_STATUS  rmGpuLockAlloc(NvU32);
+void       rmGpuLockFree(NvU32);
+NV_STATUS  rmGpuLocksAcquire(NvU32, NvU32);
+NvU32      rmGpuLocksRelease(NvU32, OBJGPU *);
+void       rmGpuLocksFreeze(GPU_MASK);
+void       rmGpuLocksUnfreeze(GPU_MASK);
+NV_STATUS  rmGpuLockHide(NvU32);
+void       rmGpuLockShow(NvU32);
+NvBool     rmGpuLockIsOwner(void);
+NvU32      rmGpuLocksGetOwnedMask(void);
+NvBool     rmGpuLockIsHidden(OBJGPU *);
+NV_STATUS  rmGpuLockSetOwner(OS_THREAD_HANDLE);
+NV_STATUS  rmGpuGroupLockAcquire(NvU32, GPU_LOCK_GRP_ID, NvU32, NvU32, GPU_MASK *);
+NV_STATUS  rmGpuGroupLockRelease(GPU_MASK, NvU32);
+NvBool     rmGpuGroupLockIsOwner(NvU32, GPU_LOCK_GRP_ID, GPU_MASK*);
+
+NvBool     rmDeviceGpuLockIsOwner(NvU32);
+NV_STATUS  rmDeviceGpuLockSetOwner(OBJGPU *, OS_THREAD_HANDLE);
+NV_STATUS  rmDeviceGpuLocksAcquire(OBJGPU *, NvU32, NvU32);
+NvU32      rmDeviceGpuLocksRelease(OBJGPU *, NvU32, OBJGPU *);
+
+NV_STATUS  rmIntrMaskLockAlloc(NvU32 gpuInst);
+void       rmIntrMaskLockFree(NvU32 gpuInst);
+/// @note The return value is always zero, not the actual IRQL
+NvU64      rmIntrMaskLockAcquire(OBJGPU *pGpu);
+void       rmIntrMaskLockRelease(OBJGPU *pGpu, NvU64 oldIrql);
+
+// wrappers for handling lock-related NV_ASSERT_OR_RETURNs
+#define LOCK_ASSERT_AND_RETURN(cond)                    NV_ASSERT_OR_ELSE_STR((cond), #cond, return NV_ERR_INVALID_LOCK_STATE)
+#define IRQL_ASSERT_AND_RETURN(cond)                    NV_ASSERT_OR_ELSE_STR((cond), #cond, return NV_ERR_INVALID_IRQ_LEVEL)
+#define LOCK_ASSERT_AND_RETURN_BOOL(cond, bRet)         NV_ASSERT_OR_ELSE_STR((cond), #cond, return (bRet))
+
+#define LOCK_METER_OP(f,l,t,d0,d1,d2)
+#define LOCK_METER_DATA(t,d0,d1,d2)
+
+#define rmInitLockMetering()
+#define rmDestroyLockMetering()
+
+//
+// RM API lock definitions are handled by the rmapi module. Providing legacy
+// rmApiLockXxx interface for temporary compatibility. CORERM-1370
+//
+#include "rmapi/rmapi.h"
+
+#define API_LOCK_FLAGS_NONE                             RMAPI_LOCK_FLAGS_NONE
+#define API_LOCK_FLAGS_COND_ACQUIRE                     RMAPI_LOCK_FLAGS_COND_ACQUIRE
+
+#define rmApiLockAcquire(flags, module)                 (rmapiLockAcquire(flags, module))
+static NV_INLINE NV_STATUS rmApiLockRelease(void)       {rmapiLockRelease(); return NV_OK;}
+#define rmApiLockIsOwner()                              (rmapiLockIsOwner())
+
+#endif // LOCKS_H

src/nvidia/inc/kernel/core/prelude.h

@@ -0,0 +1,122 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __PRELUDE_H__
+#define __PRELUDE_H__
+
+/* ------------------------ C library --------------------------------------- */
+#include <stddef.h>                 // NULL
+
+/* ------------------------ SDK includes ------------------------------------ */
+
+#include "nvtypes.h"
+#include "nvrangetypes.h"
+#include "nvstatus.h"
+#include "nvmisc.h"
+#include "nvlimits.h"
+#include "nvos.h"
+
+#include "nvctassert.h"
+
+/* ------------------------ RM library and utils ---------------------------- */
+#include "nvport/nvport.h"
+#include "nvoc/runtime.h"
+#include "nvoc/utility.h"
+#include "core/printf.h"
+#include "core/strict.h"
+#include "utils/nvassert.h"
+
+/* ------------------------ Code-generation --------------------------------- */
+#include "rmconfig.h"               // RMCONFIG header generated by config/rmconfig.pl
+#include "g_rmconfig_private.h"     // resman-private hal setup such as: IsGK104(), etc.
+#include "g_nvh_state.h"            // pass enable/disable state to NVOC headers
+#include "g_odb.h"
+#include "g_hal.h"
+
+#include "rmcd.h"
+
+/* ------------------------ Common types ------------------------------------ */
+typedef NvU64 RmPhysAddr;           // A physical address should be 64 bits
+
+typedef struct THREAD_STATE_NODE THREAD_STATE_NODE; // FW declare thread state
+
+/* ------------------------ Utility Macros ---------------------------------- */
+
+//
+// Power of 2 alignment.
+//    (Will give unexpected results if 'gran' is not a power of 2.)
+//    (v - v + gran) ensures that gran is upcasted to match v before
+//    the ~ operation, without explicitly having to typecast it.
+//
+#define RM_ALIGN_DOWN(v, gran)          ((v) & ~(((v) - (v) + (gran)) - 1))
+#define RM_ALIGN_UP(v, gran)            (((v) + ((gran) - 1)) & ~(((v) - (v) + (gran))-1))
+#define RM_IS_ALIGNED(v, gran)          ((((gran) - 1) & (v)) == 0)
+
+#define RM_ALIGN_PTR_DOWN(p, gran)      ((void *) RM_ALIGN_DOWN(((NvUPtr)p), (gran)))
+#define RM_ALIGN_PTR_UP(p, gran)        ((void *) RM_ALIGN_UP(((NvUPtr)p), (gran)))
+
+#define RM_PAGE_ALIGN_DOWN(value)       RM_ALIGN_DOWN((value), RM_PAGE_SIZE)
+#define RM_PAGE_ALIGN_UP(value)         RM_ALIGN_UP((value), RM_PAGE_SIZE)
+
+#define NV_DELTA(a, b)                  (NV_MAX((a), (b)) - NV_MIN((a), (b)))       // Okay for unsigned or signed
+
+#define NV_ROUNDUP(a,b)                 ((NV_CEIL(a,b))*(b))
+#define NV_ROUND_TO_QUANTA(a, quanta)   (((quanta) == 0) ? (a): ((((a) + ((quanta) >> 1)) / (quanta)) *  (quanta)))
+#define NV_FLOOR_TO_QUANTA(a, quanta)   (((a) / (quanta)) *  (quanta))
+#define NV_SIZEOF32(x)                  (sizeof(x))
+#define NV_ARRAY_ELEMENTS(x)            ((sizeof(x)/sizeof((x)[0])))
+#define NV_ARRAY_ELEMENTS32(x)          ((NV_SIZEOF32(x)/NV_SIZEOF32((x)[0])))
+#define NV_BYTESWAP16(a)                ((((a) & 0xff00)>>8)      |  \
+                                         (((a) & 0x00ff)<<8))
+#define NV_BYTESWAP32(a)                ((((a) & 0xff000000)>>24) |  \
+                                         (((a) & 0x00ff0000)>>8)  |  \
+                                         (((a) & 0x0000ff00)<<8)  |  \
+                                         (((a) & 0x000000ff)<<24))
+#define NV_TO_LOWER(c)                  (((c)>='A'&&(c)<='Z')?(c)+('a'-'A'):(c))
+#define NV_TO_UPPER(c)                  (((c)>='a'&&(c)<='z')?((c)-'a'+'A'):(c))
+
+/*!
+ *  Creates a byte mask for a word at given offset.
+ *  offset = 0   0xffffff00
+ *  offset = 1   0xffff00ff
+ *  offset = 2   0xff00ffff
+ *  offset = 3   0x00ffffff
+ *
+ *  @param[in]  offset  Offset for the mask.
+ */
+#define NV_BYTE_MASK(offset)            (~(0xff << ((offset)<<3)))
+
+//
+// note: the following trick fails if (z-1) * y > max_int
+//
+// since the calculation contains (x % z) * y,
+// and the maximum value of (x % z) is (z-1).
+//
+// selecting the smaller of x and y to be y reduces the chances
+// of problems, but for big enough z, the problem will return...
+//
+#define OVERFLOW_CAREFUL_MUL_DIV(x, y, z) \
+    ((x) > (y)) ? (((x) / (z)) * (y) + (((x) % (z)) * (y)) / (z)) : (((y) / (z)) * (x) + (((y) % (z)) * (x)) / (z))
+
+#define MASK_BITS(n)                    (~(0xFFFFFFFF << (n)))
+
+#endif /* __PRELUDE_H__ */

src/nvidia/inc/kernel/core/printf.h

@@ -0,0 +1,368 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2001-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PRINTF_H_
+#define _PRINTF_H_
+
+/*
+ * RM PRINTF definitions.
+ *
+ * Provides RM internal definitions built on the generic nvprintf utilities
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "nvport/nvport.h"
+#include "utils/nvprintf.h"
+#include "nvlog/nvlog.h"
+
+#define DBG_FILE_LINE_FUNCTION  NV_FILE_STR, __LINE__, NV_FUNCTION_STR
+
+/**
+ * @todo bug 1583359 - Move to NvPort compiler specifics
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#define NV_RETURN_ADDRESS()    __builtin_return_address(0)
+#else
+#define NV_RETURN_ADDRESS()    _ReturnAddress()
+#endif
+
+
+//******************************************************************************
+//                                  BREAKPOINTS
+//******************************************************************************
+
+// NV_DBG_BREAKPOINT_ALLOWED can be overridden through CFLAGS
+#if !defined(NV_DBG_BREAKPOINT_ALLOWED)
+#if defined(DEBUG) || defined(ASSERT_BUILD) || defined(NV_MODS) || defined(QA_BUILD) || (defined(NVRM) && NVCPU_IS_RISCV64)
+#define NV_DBG_BREAKPOINT_ALLOWED 1
+#else
+#define NV_DBG_BREAKPOINT_ALLOWED 0
+#endif
+#endif // !defined(NV_DBG_BREAKPOINT_ALLOWED)
+
+#define NV_DEBUG_BREAK_FLAGS_RC                        0:0
+#define NV_DEBUG_BREAK_FLAGS_RC_DISABLE                (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_RC_ENABLE                 (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_ASSERT                    1:1
+#define NV_DEBUG_BREAK_FLAGS_ASSERT_DISABLE            (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_ASSERT_ENABLE             (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK                 2:2
+#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK_DISABLE         (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_DBG_BREAK_ENABLE          (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT               3:3
+#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT_DISABLE       (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_GPU_TIMEOUT_ENABLE        (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS                 4:4
+#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS_DISABLE         (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_POOL_TAGS_ENABLE          (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_POWER_ON                  5:5
+#define NV_DEBUG_BREAK_FLAGS_POWER_ON_DISABLE          (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_POWER_ON_ENABLE           (0x00000001)
+#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR                 6:6
+#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR_DISABLE         (0x0)
+#define NV_DEBUG_BREAK_FLAGS_SMU_ERROR_ENABLE          (0x1)
+#define NV_DEBUG_BREAK_FLAGS_CRASH                     7:7
+#define NV_DEBUG_BREAK_FLAGS_CRASH_DISABLE             (0x00000000)
+#define NV_DEBUG_BREAK_FLAGS_CRASH_ENABLE              (0x00000001)
+
+#define NV_DEBUG_BREAK_ATTRIBUTES                      7:0
+#define NV_DEBUG_BREAK_ATTRIBUTES_NONE                 (0x00000000)
+#define NV_DEBUG_BREAK_ATTRIBUTES_RC                   (0x00000001)
+#define NV_DEBUG_BREAK_ATTRIBUTES_ASSERT               (0x00000002)
+#define NV_DEBUG_BREAK_ATTRIBUTES_DBG_BREAK            (0x00000004)
+#define NV_DEBUG_BREAK_ATTRIBUTES_GPU_TIMEOUT          (0x00000008)
+#define NV_DEBUG_BREAK_ATTRIBUTES_POOL_TAGS            (0x00000010)
+#define NV_DEBUG_BREAK_ATTRIBUTES_POWER_ON             (0x00000020)
+#define NV_DEBUG_BREAK_ATTRIBUTES_SMU_ERROR            (0x00000040)
+#define NV_DEBUG_BREAK_ATTRIBUTES_CRASH                (0x00000080)
+
+// Checks RMINFO and OS config to see if triggering a breakpoint is ever allowed
+NvBool nvDbgBreakpointEnabled(void);
+// Flushes the logs before a breakpoint, so we can see all the prints.
+void osFlushLog(void);
+
+#define DBG_ROUTINE()                                                          \
+    do                                                                         \
+    {                                                                          \
+        if (nvDbgBreakpointEnabled())                                          \
+            PORT_BREAKPOINT_ALWAYS();                                          \
+    } while (0)
+
+#define REL_DBG_BREAKPOINT() \
+    REL_DBG_BREAKPOINT_MSG("NVRM-RC: Nvidia GPU Error Detected\n")
+
+#if NV_DBG_BREAKPOINT_ALLOWED
+
+#if !NVCPU_IS_RISCV64
+
+#define DBG_BREAKPOINT_EX(PGPU, LEVEL)                                         \
+    do                                                                         \
+    {                                                                          \
+        NV_PRINTF(LEVEL_ERROR, "bp @ " NV_FILE_FMT ":%d\n", NV_FILE, __LINE__);\
+        osFlushLog();                                                          \
+        DBG_ROUTINE();                                                         \
+    } while (0)
+
+#else // !NVCPU_IS_RISCV64
+
+#define DBG_BREAKPOINT_EX(PGPU, LEVEL)                                         \
+    do                                                                         \
+    {                                                                          \
+        NV_ASSERT_FAILED("DBG_BREAKPOINT");                                    \
+    } while (0)
+
+#endif // !NVCPU_IS_RISCV64
+
+#define DBG_BREAKPOINT() DBG_BREAKPOINT_EX(NULL, 0)
+
+#define DBG_BREAKPOINT_EX_ARGS_IGNORED 1
+#define REL_DBG_BREAKPOINT_MSG(msg)                                            \
+    do                                                                         \
+    {                                                                          \
+        PORT_DBG_PRINT_STRING_LITERAL(msg);                                    \
+        DBG_BREAKPOINT();                                                      \
+    } while (0)
+
+#else // !NV_DBG_BREAKPOINT_ALLOWED
+
+#define DBG_BREAKPOINT()
+#define DBG_BREAKPOINT_EX(PGPU, LEVEL)
+#define DBG_BREAKPOINT_EX_ARGS_IGNORED 1
+
+#define REL_DBG_BREAKPOINT_MSG(msg)                                            \
+    do                                                                         \
+    {                                                                          \
+        PORT_DBG_PRINT_STRING_LITERAL(msg);                                    \
+        DBG_ROUTINE();                                                         \
+    } while (0)
+
+
+#endif // NV_DBG_BREAKPOINT_ALLOWED
+
+#define DBG_BREAKPOINT_REASON(reason) DBG_BREAKPOINT()
+
+#define DBG_BREAKPOINT_ERROR_INFO(errorCategory, errorInfo) DBG_BREAKPOINT()
+
+//******************************************************************************
+//                                  PRINTS
+//******************************************************************************
+
+#include "utils/nvprintf.h"
+
+#define MAX_ERROR_STRING 256
+#ifndef NVPORT_CHECK_PRINTF_ARGUMENTS
+#define NVPORT_CHECK_PRINTF_ARGUMENTS(x,c)
+#endif
+//
+// Prototypes
+//
+NvBool nvDbgInit(void);
+void nvDbgDestroy(void);
+void nvDbg_Printf  (const char *file, int line, const char *function, int debuglevel, const char *s, ...) NVPORT_CHECK_PRINTF_ARGUMENTS(5, 6);
+
+//
+// Like libc's vsnprintf(), nvDbg_vPrintf() invalidates its va_list argument. The va_list argument
+// may not be reused after nvDbg_vPrintf() returns.  If the va_list is needed after the
+// nvDbg_vPrintf() call, create a copy of the va_list using va_copy().
+// The caller controls the lifetime of the va_list argument, and should free it using va_end.
+//
+void nvDbg_vPrintf (const char *file, int line, const char *function, int debuglevel, const char *s, va_list args);
+void nvDbg_PrintBuf(const char *file, int line, const char *function, int debgulevel, NvU8 buffer[], NvU32 bufsize);
+
+int nvDbgVsnprintf(char *dest, NvU32 destSize, const char *fmt, va_list args);
+int nvDbgSnprintf (char *dest, NvU32 destSize, const char *fmt, ...);
+
+struct OBJGPU;
+void nvDbgInitRmMsg(struct OBJGPU *);
+// RmMsgPrefix return value
+#define NVRM_MSG_PREFIX_NVRM            NVBIT(0)
+#define NVRM_MSG_PREFIX_FILE            NVBIT(1)
+#define NVRM_MSG_PREFIX_FUNCTION        NVBIT(2)
+#define NVRM_MSG_PREFIX_LINE            NVBIT(3)
+#define NVRM_MSG_PREFIX_OSTIMESTAMP     NVBIT(4)
+NvU32 RmMsgPrefix(NvU32 prefix, const char *filename, NvU32 linenumber, const char *function, char *str, NvU32 len);
+// nvDbgRmMsgCheck return code
+#define NVRM_MSG_NORMAL 0    // Use normal message handling (warnings/errors)
+#define NVRM_MSG_HIDE   1    // Skip this message
+#define NVRM_MSG_PRINT  2    // Force printing of this message
+NvU32 nvDbgRmMsgCheck(const char *filename, NvU32 linenumber, const char *function, NvU32 level, const char *format, NvU32 *pPrefix);
+void nvDbgDumpBufferBytes(void *pBuffer, NvU32 length);
+
+
+#if NV_PRINTF_STRINGS_ALLOWED
+#define DBG_STRING(str) str
+#define DBG_INIT()                               nvDbgInit()
+#define DBG_DESTROY()                            nvDbgDestroy()
+#define DBG_VSNPRINTF(ptr_size_format_and_stuff) nvDbgVsnprintf ptr_size_format_and_stuff
+#define DBG_PRINTBUF(dbglevel, buffer, bufsize)  nvDbg_PrintBuf(DBG_FILE_LINE_FUNCTION, dbglevel, buffer, bufsize)
+#define DBG_RMMSG_CHECK(level)            (nvDbgRmMsgCheck(DBG_FILE_LINE_FUNCTION, level, NULL, NULL) == NVRM_MSG_PRINT)
+#else  // ! NV_PRINTF_STRINGS_ALLOWED -- debug printf strings not enabled
+#define DBG_STRING(str) ""
+#define DBG_INIT()      (NV_TRUE)
+#define DBG_DESTROY()
+#define DBG_VSNPRINTF(ptr_size_format_and_stuff)
+#define DBG_PRINTBUF(dbglevel, buffer, bufsize)
+#define DBG_RMMSG_CHECK(level)   (0)
+#endif // NV_PRINTF_STRINGS_ALLOWED
+
+
+
+//******************************************************************************
+//                            POWER SANITY CHECKS
+//******************************************************************************
+//
+// Make sure the GPU is in full power or resuming from D3 state. Else,
+// bailout from the calling function. An exception for systems, which support
+// surprise removal feature. See Bugs 440565, 479003, and 499228.DO NOT IGNORE
+// OR REMOVE THIS ASSERT.  If you have problems with it, please talk to cplummer.
+//
+// bAllowWithoutSysmemAccess: Allow this RM Control when sysmem access is not available
+// from the GPU. SHould be NV_TRUE only for NV2080_CTRL_CMD_BUS_SYSMEM_ACCESS
+//
+// On systems supporting surprise removal, if the GPU is in D3 cold
+// and still attached we would consider it a true D3 cold state
+// and return NOT_FULL_POWER. See bug 1679965.
+//
+//
+#define API_GPU_FULL_POWER_SANITY_CHECK(pGpu, bGpuAccess, bAllowWithoutSysmemAccess)  \
+    if ((!gpuIsGpuFullPower(pGpu)) &&                                       \
+              (!(pGpu)->getProperty((pGpu),                                 \
+                             PDB_PROP_GPU_IN_PM_RESUME_CODEPATH)))          \
+    {                                                                       \
+        DBG_BREAKPOINT();                                                   \
+        if (bGpuAccess || (!gpuIsSurpriseRemovalSupported(pGpu)))           \
+        {                                                                   \
+            return NV_ERR_GPU_NOT_FULL_POWER;                               \
+        }                                                                   \
+        else if (gpuIsSurpriseRemovalSupported(pGpu) &&                     \
+                 (pGpu)->getProperty((pGpu), PDB_PROP_GPU_IS_CONNECTED))    \
+        {                                                                   \
+            return NV_ERR_GPU_NOT_FULL_POWER;                               \
+        }                                                                   \
+    }                                                                       \
+    if (!(bAllowWithoutSysmemAccess) && !gpuCheckSysmemAccess(pGpu))        \
+    {                                                                       \
+        return NV_ERR_GPU_NOT_FULL_POWER;                                   \
+    }
+
+#define API_GPU_FULL_POWER_SANITY_CHECK_OR_GOTO(pGpu, bGpuAccess, bAllowWithoutSysmemAccess, status, tag) \
+    if ((!gpuIsGpuFullPower(pGpu)) &&                                       \
+              (!(pGpu)->getProperty((pGpu),                                 \
+                             PDB_PROP_GPU_IN_PM_RESUME_CODEPATH)))          \
+    {                                                                       \
+        DBG_BREAKPOINT();                                                   \
+        if (bGpuAccess || (!gpuIsSurpriseRemovalSupported(pGpu)))           \
+        {                                                                   \
+            status = NV_ERR_GPU_NOT_FULL_POWER;                             \
+            goto tag;                                                       \
+        }                                                                   \
+        else if (gpuIsSurpriseRemovalSupported(pGpu) &&                     \
+                (pGpu)->getProperty((pGpu), PDB_PROP_GPU_IS_CONNECTED))     \
+        {                                                                   \
+            status = NV_ERR_GPU_NOT_FULL_POWER;                             \
+            goto tag;                                                       \
+        }                                                                   \
+    }                                                                       \
+    if (!(bAllowWithoutSysmemAccess) && !gpuCheckSysmemAccess(pGpu))        \
+    {                                                                       \
+        return NV_ERR_GPU_NOT_FULL_POWER;                                   \
+    }
+
+
+#if defined(PORT_IS_FUNC_SUPPORTED)
+#if PORT_IS_FUNC_SUPPORTED(portMemExValidate)
+#define DBG_VAL_PTR(p) portMemExValidate(p, NV_TRUE)
+#endif
+#endif
+#ifndef DBG_VAL_PTR
+#define DBG_VAL_PTR(p)
+#endif
+
+
+//********************************************************************************
+//
+//  NVRM_TRACE support
+//    low-overhead runtime state capture
+//    to enable, define USE_NVRM_TRACE (retail or debug builds)
+//
+//********************************************************************************
+
+#ifdef USE_NVRM_TRACE
+
+NvU32 NVRM_TRACE_INIT(void);
+NvU32 NVRM_TRACE_DISABLE(void);
+void NVRM_TRACE_ENABLE(void);
+void NVRM_TRACE_DUMP(void);
+void NVRM_TRACE(NvU32);
+void NVRM_TRACEV(NvU32 *,NvU32);
+void NVRM_TRACE1(NvU32);
+void NVRM_TRACE2(NvU32, NvU32);
+void NVRM_TRACE3(NvU32, NvU32, NvU32);
+void NVRM_TRACE4(NvU32, NvU32, NvU32, NvU32);
+void NVRM_TRACE5(NvU32, NvU32, NvU32, NvU32, NvU32);
+
+// versions of reg read/write that log to trace buffer
+//NvU32 NVRM_TRACE_REG_RD32(OBJGPU *, NvU32);
+//void NVRM_TRACE_REG_WR32(OBJGPU *, NvU32, NvU32);
+
+// fifolog format looks like:
+//  31:28   = unique file number
+//  27:4    = file line number
+//  1:0     = fifo state bits (bit1 = puller, bit0 = reassign)
+#define FIFOLOG(fn,fa,fb) NVRM_TRACE2('FIFO', ((fn << 28) | (__LINE__ << 4) |       \
+                                                    ((fa & 0x1) ? 1 : 0) << 1 |     \
+                                                    ((fb & 0x1) ? 1 : 0)) )
+
+#else  // ! USE_NVRM_TRACE
+
+#define NVRM_TRACE_INIT()
+#define NVRM_TRACE_DISABLE() 0
+#define NVRM_TRACE_ENABLE()
+#define NVRM_TRACE_DUMP()
+#define NVRM_TRACE(c0)
+#define NVRM_TRACE1(c0)
+#define NVRM_TRACE2(c0, c1)
+#define NVRM_TRACE3(c0, c1, c2)
+#define NVRM_TRACE4(c0, c1, c2, c3)
+#define NVRM_TRACE5(c0, c1, c2, c3, c4)
+#define FIFOLOG(a,b,c)
+
+#endif  // ! USE_NVRM_TRACE
+
+#define NVRM_TRACE_ERROR(code, status)   NVRM_TRACE3('EEEE', (code), (status))
+#define NVRM_TRACE_API(code, p0, p1, p2) NVRM_TRACE5('API ', (code), (p0), (p1), (p2))
+
+void nvErrorLog(void *pVoid, NvU32 num, const char *pFormat, va_list arglist);
+void nvErrorLog_va(void * pGpu, NvU32 num, const char * pFormat, ...);
+
+// memory allocation tracking data structs and globals
+#define MAX_STACK_LEVEL 6
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // _PRINTF_H_

src/nvidia/inc/kernel/core/strict.h

@@ -0,0 +1,99 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __STRICT_H__
+#define __STRICT_H__
+
+//
+// RM_STRICT_SUPPRESS_DEPRECATED_DEFINITIONS_VER_XYZ should be set
+// before including any RM internal headers when disabling deprecated
+// definitions is desired.
+//
+// For transition during refactoring, we might introduce new types and
+// interfaces and use macros/wrappers to forward the old interface to
+// the new one.
+//
+// Once a callsite is migrated to use the new interface it can use RM
+// strict to disable the deprecated definitions to prevent changes from
+// reintroducing calls to a deprecated interface within a cleansed
+// module.
+//
+// Controlling disablement of deprecated definitions is versioned. This
+// enables us to introduce new deprecated interfaces incrementally.
+// Example, ModuleA might scrub to versionX (removal of OBJFB defns) but
+// not versionY (removal of legacy CLI types).
+//
+// Flags to turn off deprecated definitions are intended to be
+// temporary, once all modules remove references the deprecated
+// definitions and knobs in this header should be deleted.
+//
+#ifdef RM_STRICT_SUPPRESS_DEPRECATED_DEFINITIONS_VER_JAN_21_2020
+#define RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS         0
+#define RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS   0
+#endif
+
+//
+// RM_STRICT_SUPPRESS_PHYSICAL_DEFINITIONS_VER_XYZ should be set before
+// including any RM internal headers when disabling "physical" definitions is
+// desired.
+//
+// Physical definitions refers to interfaces/types that are only used by GSP-RM
+// and VGPU-host, i.e.: not to be used by VGPU Client or GSP Client
+//
+#ifdef RM_STRICT_SUPPRESS_PHYSICAL_DEFINITIONS_VER_JAN_21_2020
+#define RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS    0
+#endif
+
+//
+// Default deprecated and "physical engine" definitions on unless specified
+//
+#ifndef RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS
+#define RM_STRICT_CONFIG_EMIT_DEPRECATED_OBJFB_DEFINITIONS   1
+#endif
+
+#ifndef RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS
+#define RM_STRICT_CONFIG_EMIT_DEPRECATED_CONTEXT_DMA_DEFINITIONS    1
+#endif
+
+#ifndef RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS
+#define RM_STRICT_CONFIG_EMIT_MEMORY_SYSTEM_DEFINITIONS     1
+#endif
+
+//
+// "Physical engine" definitions not yet included in any version, but available
+// for T234X. Should be defined to 0 before including any RM internal headers
+// when disabling OBJDISP (and related) definitions is desired.
+//
+#ifndef RM_STRICT_CONFIG_EMIT_DISP_ENGINE_DEFINITIONS
+#define RM_STRICT_CONFIG_EMIT_DISP_ENGINE_DEFINITIONS     1
+#endif
+
+//
+// Generate OBJGPU engine accessors (GPU_GET_FOO(pGpu)) for disabled engines.
+// These will always return NULL, but will allow the code that references them
+// to compile.
+//
+#ifndef RM_STRICT_CONFIG_EMIT_DISABLED_GPU_ENGINE_ACCESSORS
+#define RM_STRICT_CONFIG_EMIT_DISABLED_GPU_ENGINE_ACCESSORS     1
+#endif
+
+#endif /* __STRICT_H__ */

src/nvidia/inc/kernel/core/system.h

@@ -0,0 +1,3 @@
+
+#include "g_system_nvoc.h"
+

src/nvidia/inc/kernel/core/thread_state.h

@@ -0,0 +1,217 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef THREAD_STATE_H
+#define THREAD_STATE_H
+
+/**************** Resource Manager Defines and Structures ******************\
+*                                                                           *
+*       Defines and structures used for Thread State management             *
+*                                                                           *
+\***************************************************************************/
+
+#include "core/core.h"
+#include "os/os.h"
+#include "tls/tls.h"
+#include "containers/map.h"
+#include "containers/list.h"
+
+typedef struct OBJGPU OBJGPU;
+
+//
+// Thread State Tracking structures and defines
+//
+typedef struct THREAD_TIMEOUT_STATE
+{
+    NvU64   enterTime;
+    NvU64   nonComputeTime;
+    NvU64   computeTime;
+    NvU64   nextCpuYieldTime;
+    NvU64   overrideTimeoutMsecs;
+
+} THREAD_TIMEOUT_STATE, *PTHREAD_TIMEOUT_STATE;
+
+typedef struct THREAD_STATE_FREE_CALLBACK
+{
+    void *pCbData;
+    void (*pCb)(void *pCbData);
+} THREAD_STATE_FREE_CALLBACK;
+
+MAKE_LIST(THREAD_STATE_FREE_CB_LIST, THREAD_STATE_FREE_CALLBACK);
+
+typedef struct THREAD_STATE_NODE THREAD_STATE_NODE;
+
+struct THREAD_STATE_NODE
+{
+    OS_THREAD_HANDLE     threadId;
+    /*!
+     * Thread sequencer id.  This is a unique identifier for a given thread
+     * entry into the RM.  This is separate from @ref threadId, as the threadId
+     * is really the OS's thread handle/pointer.  In cases where the same
+     * physical thread is re-used (e.g. WORK_ITEMs are scheduled from a
+     * pre-allocated pool of worker threads), different RM threads will have the
+     * same threadId.
+     *
+     * This value is set by @ref threadStateInitXYZ() based off the global @ref
+     * THREAD_STATE_DB::threadSeqCntr.
+     */
+    NvU32                threadSeqId;
+    NvBool               bValid;
+    THREAD_TIMEOUT_STATE timeout;
+    NvU32                cpuNum;
+    NvU32                flags;
+    MapNode              node;
+
+    /*!
+     * If a callback is installed, threadStateFree() may block on it.
+     *
+     * The installed callbacks will be processed in FIFO order only.
+     *
+     * Only supported on non-ISR CPU RM paths.
+     */
+    THREAD_STATE_FREE_CB_LIST cbList;
+};
+
+MAKE_INTRUSIVE_MAP(ThreadStateNodeMap, THREAD_STATE_NODE, node);
+
+typedef struct THREAD_STATE_DB_TIMEOUT
+{
+    NvU64    nonComputeTimeoutMsecs;
+    NvU64    computeTimeoutMsecs;
+    NvU32    computeGpuMask;
+    NvU32    flags;
+
+} THREAD_STATE_DB_TIMEOUT, *PTHREAD_STATE_DB_TIMEOUT;
+
+#define THREAD_STATE_TRACE_MAX_ENTRIES      8
+
+typedef struct THREAD_STATE_TRACE_ENTRY
+{
+    NvU64 callerRA;
+    NvU32 flags;
+
+} THREAD_STATE_TRACE_ENTRY;
+
+typedef struct THREAD_STATE_TRACE_INFO
+{
+    NvU32 index;
+    THREAD_STATE_TRACE_ENTRY entries[THREAD_STATE_TRACE_MAX_ENTRIES];
+
+} THREAD_STATE_TRACE_INFO;
+
+typedef struct THREAD_STATE_ISR_LOCKLESS
+{
+    THREAD_STATE_NODE **ppIsrThreadStateGpu;
+} THREAD_STATE_ISR_LOCKLESS, *PTHREAD_STATE_ISR_LOCKLESS, **PPTHREAD_STATE_ISR_LOCKLESS;
+
+typedef struct THREAD_STATE_DB
+{
+    NvU32   setupFlags;
+    NvU32   maxCPUs;
+    /*!
+     * Thread state sequencer id counter.  The last allocated thread state
+     * sequencer id via @ref threadStateInitXYZ().
+     */
+    NvU32   threadSeqCntr;
+    PORT_SPINLOCK *spinlock;
+    ThreadStateNodeMap  dbRoot;
+    ThreadStateNodeMap  dbRootPreempted;
+    THREAD_STATE_NODE **ppISRDeferredIntHandlerThreadNode;
+    PTHREAD_STATE_ISR_LOCKLESS pIsrlocklessThreadNode;
+    THREAD_STATE_DB_TIMEOUT timeout;
+    THREAD_STATE_TRACE_INFO traceInfo;
+} THREAD_STATE_DB, *PTHREAD_STATE_DB;
+
+//
+// This is the same for all OSes. This value was chosen because it is
+// the minimum found on any OS at the time of this writing (May, 2008).
+//
+#define TIMEOUT_DEFAULT_OS_RESCHEDULE_INTERVAL_SECS     2
+
+//
+// The normal power transition requirement for Windows is 4 seconds.
+// Use longer time to let OS fire timeout and ask recovery. 
+//
+#define TIMEOUT_WDDM_POWER_TRANSITION_INTERVAL_MS       9800
+
+//
+// Thread State flags used for threadStateInitSetupFlags
+//
+#define THREAD_STATE_SETUP_FLAGS_NONE                               0
+#define THREAD_STATE_SETUP_FLAGS_ENABLED                            NVBIT(0)
+#define THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED                    NVBIT(1)
+#define THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED                  NVBIT(2)
+#define THREAD_STATE_SETUP_FLAGS_CHECK_TIMEOUT_AT_FREE_ENABLED      NVBIT(3)
+#define THREAD_STATE_SETUP_FLAGS_ASSERT_ON_TIMEOUT_ENABLED          NVBIT(4)
+#define THREAD_STATE_SETUP_FLAGS_ASSERT_ON_FAILED_LOOKUP_ENABLED    NVBIT(5)
+#define THREAD_STATE_SETUP_FLAGS_RESET_ON_TIMEOUT_ENABLED           NVBIT(6)
+#define THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED  NVBIT(7)
+#define THREAD_STATE_SETUP_FLAGS_PRINT_INFO_ENABLED                 NVBIT(31)
+
+//
+// Thread State flags used for threadState[Init,Free]
+//
+#define THREAD_STATE_FLAGS_NONE                             0
+#define THREAD_STATE_FLAGS_IS_ISR                       NVBIT(0)
+#define THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER  NVBIT(1)
+#define THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER      NVBIT(2)
+#define THREAD_STATE_FLAGS_IS_ISR_LOCKLESS              NVBIT(3)
+#define THREAD_STATE_FLAGS_TIMEOUT_INITED               NVBIT(5)
+#define THREAD_STATE_FLAGS_PLACED_ON_PREEMPT_LIST       NVBIT(6)
+#define THREAD_STATE_FLAGS_DEVICE_INIT                  NVBIT(7)
+#define THREAD_STATE_FLAGS_STATE_FREE_CB_ENABLED        NVBIT(8)
+
+// These Threads run exclusively between a conditional acquire
+#define THREAD_STATE_FLAGS_EXCLUSIVE_RUNNING   (THREAD_STATE_FLAGS_IS_ISR                       | \
+                                                THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER  | \
+                                                THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER)
+
+#define THREAD_STATE_FLAGS_DEFERRED_INT_HANDLER_RUNNING (THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER | \
+                                       THREAD_STATE_FLAGS_IS_DEFERRED_INT_HANDLER)
+
+NV_STATUS   threadStateGlobalAlloc(void);
+void        threadStateGlobalFree(void);
+void        threadStateInitRegistryOverrides(OBJGPU *pGpu);
+void        threadStateInitSetupFlags(NvU32 flags);
+NvU32       threadStateGetSetupFlags(void);
+
+void        threadStateInitISRLockless(THREAD_STATE_NODE *, OBJGPU*, NvU32);
+void        threadStateFreeISRLockless(THREAD_STATE_NODE *, OBJGPU*, NvU32);
+void        threadStateInitISRAndDeferredIntHandler(THREAD_STATE_NODE *, OBJGPU*, NvU32);
+void        threadStateFreeISRAndDeferredIntHandler(THREAD_STATE_NODE *, OBJGPU*, NvU32);
+void        threadStateInit(THREAD_STATE_NODE *pThreadNode, NvU32 flags);
+void        threadStateFree(THREAD_STATE_NODE *pThreadNode, NvU32 flags);
+
+NV_STATUS   threadStateGetCurrent(THREAD_STATE_NODE **ppThreadNode, OBJGPU *pGpu);
+NV_STATUS   threadStateGetCurrentUnchecked(THREAD_STATE_NODE **ppThreadNode, OBJGPU *pGpu);
+NV_STATUS   threadStateInitTimeout(OBJGPU *pGpu, NvU32 timeoutUs, NvU32 flags);
+NV_STATUS   threadStateCheckTimeout(OBJGPU *pGpu, NvU64 *pElapsedTimeUs);
+NV_STATUS   threadStateResetTimeout(OBJGPU *pGpu);
+void        threadStateLogTimeout(OBJGPU *pGpu, NvU64 funcAddr, NvU32 lineNum);
+void        threadStateYieldCpuIfNecessary(OBJGPU *pGpu);
+void        threadStateSetTimeoutOverride(THREAD_STATE_NODE *, NvU64);
+
+NV_STATUS   threadStateEnqueueCallbackOnFree(THREAD_STATE_NODE *pThreadNode,
+                                             THREAD_STATE_FREE_CALLBACK *pCallback);
+void        threadStateRemoveCallbackOnFree(THREAD_STATE_NODE *pThreadNode,
+                                            THREAD_STATE_FREE_CALLBACK *pCallback);
+#endif // THREAD_STATE_H

src/nvidia/inc/kernel/diagnostics/gpu_acct.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_acct_nvoc.h"
+

src/nvidia/inc/kernel/diagnostics/journal.h

@@ -0,0 +1,3 @@
+
+#include "g_journal_nvoc.h"
+

src/nvidia/inc/kernel/diagnostics/journal_structs.h

@@ -0,0 +1,53 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef JOURNAL_STRUCTS_H
+#define JOURNAL_STRUCTS_H 1
+
+#include "nvcd.h"
+#include "rmcd.h"
+
+// Meta Data to Describe an error block
+typedef struct RMCD_ERROR_BLOCK {
+    NvU8 * pBlock;
+    NvU32 blockSize;
+    struct RMCD_ERROR_BLOCK * pNext;
+} RMCD_ERROR_BLOCK;
+
+typedef struct RMERRORHEADER {
+    struct RMFIFOERRORELEMENT_V3 *pNextError;
+    RMCD_ERROR_BLOCK *pErrorBlock;
+    NvU32 GPUTag;
+    NvU32 ErrorNumber;
+} RMERRORHEADER;
+
+typedef struct {
+    RMERRORHEADER ErrorHeader;
+    RmPrbInfo_RECORD_V2 RmPrbErrorData;
+} RMPRBERRORELEMENT_V2;
+
+typedef struct RMFIFOERRORELEMENT_V3 {
+    RMERRORHEADER ErrorHeader;
+} RMFIFOERRORELEMENT_V3;
+
+#endif /* ifndef JOURNAL_STRUCTS_H */

src/nvidia/inc/kernel/diagnostics/nv_debug_dump.h

@@ -0,0 +1,3 @@
+
+#include "g_nv_debug_dump_nvoc.h"
+

src/nvidia/inc/kernel/diagnostics/profiler.h

@@ -0,0 +1,119 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2012-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _PROFILER_H_
+#define _PROFILER_H_
+
+/*!
+ * @file  profiler.h
+ * @brief Simple API to measure elapsed times in RM for profiling and statistics
+ *
+ * The primary goals of this API are to:
+ *   1. Be lightweight and have little-to-no setup required (built into release drivers)
+ *   2. Defer as much analysis as possible to the user of the data (keep it simple)
+ *   3. Provide sub-millisecond resolution if possible (medium-high granularity)
+ *
+ * This is intended mainly for coarse measurements of time-critical software
+ * sequences, such as GC6. For example, the measurements could be used to catch
+ * major latency regressions in a particular timing module.
+ *
+ * For more sophisticated profiling (e.g. for prospective analysis), use of an
+ * external profiling tool (e.g. xperf with ETW) is recommended instead.
+ */
+
+#include "core/core.h"
+
+/*!
+ * Record containing the statistics of a single time module to be profiled
+ * periodically.
+ *
+ * This tracks the min/max elapsed time over all the measurement
+ * cycles, as well as the total elapsed time and number of cycles.
+ * To calculate the average elapsed time per cycle, divide total_ns by count.
+ *
+ * 64-bit precision integers are used to hold nanosecond resolution
+ * over long periods of time (e.g. greater than 4 seconds).
+ */
+typedef struct
+{
+    NvU64   count;    //<! Number of cycles this time record has measured
+    NvU64   start_ns; //<! Starting time of current cycle in nanoseconds
+    NvU64   total_ns; //<! Total nanoseconds elapsed for this record
+    NvU64   min_ns;   //<! Minimum nanoseconds elapsed in a single cycle
+    NvU64   max_ns;   //<! Maximum nanoseconds elapsed in a single cycle
+} RM_PROF_STATS;
+
+/*!
+ * A profiling group is a chain of continuous profiling modules. The purpose of
+ * chaining the modules together is to reduce the number of timestamps taken
+ * by approximately half and to prevent any time from "escaping" between the
+ * end of one module and the start of another (e.g. due to preemption).
+ */
+typedef struct
+{
+    RM_PROF_STATS *pTotal; //<! Stats for the total time spent in the group.
+    RM_PROF_STATS *pLast;  //<! Stats for the previous module of the group.
+} RM_PROF_GROUP;
+
+/*!
+ * Start measuring time for the specified module stats (begin a new cycle).
+ */
+#define RM_PROF_START(pStats)            rmProfStart(pStats)
+
+/*!
+ * Stop measuring time for the specified module stats (end a cycle).
+ * Must be called after RM_PROF_START - these two APIs are not reentrant.
+ */
+#define RM_PROF_STOP(pStats)             rmProfStop(pStats)
+
+/*!
+ * Lower-level API to record a cycle time manually. The provided time can
+ * be derived from any timer source as long as it is converted to nanoseconds.
+ */
+#define RM_PROF_RECORD(pStats, time_ns)  rmProfRecord(pStats, time_ns)
+
+/*!
+ * Start measuring time for the specified profiling group (begin a new cycle).
+ *   pTotal is the optional stats for the whole group.
+ *   pFirst is the first module of the group.
+ */
+#define RM_PROF_GROUP_START(pGroup, pTotal, pFirst)  rmProfGroupStart(pGroup, pTotal, pFirst)
+
+/*!
+ * Continue profiling the next module of a profiling group.
+ */
+#define RM_PROF_GROUP_NEXT(pGroup, pNext)            rmProfGroupNext(pGroup, pNext)
+
+/*!
+ * Stop profiling a cycle of a profiling group (ends both the last and total modules).
+ */
+#define RM_PROF_GROUP_STOP(pGroup)                   rmProfGroupStop(pGroup)
+
+// Underlying functions - use the wrapper macros instead.
+void rmProfStart (RM_PROF_STATS *pStats);
+void rmProfStop  (RM_PROF_STATS *pStats);
+void rmProfRecord(RM_PROF_STATS *pStats, NvU64 time_ns);
+void rmProfGroupStart(RM_PROF_GROUP *pGroup, RM_PROF_STATS *pTotal, RM_PROF_STATS *pFirst);
+void rmProfGroupNext (RM_PROF_GROUP *pGroup, RM_PROF_STATS *pNext);
+void rmProfGroupStop (RM_PROF_GROUP *pGroup);
+
+#endif /* _PROFILER_H_ */

src/nvidia/inc/kernel/diagnostics/traceable.h

@@ -0,0 +1,3 @@
+
+#include "g_traceable_nvoc.h"
+

src/nvidia/inc/kernel/diagnostics/tracer.h

@@ -0,0 +1,188 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+/*****************************************************************************
+*
+* Description:
+* RM Event Tracer
+*
+******************************************************************************/
+
+
+#ifndef TRACER_H
+#define TRACER_H
+
+
+#define RMTRACE_EVENT_FLAG_ENABLED      NVBIT(0)      // whether the event is enabled
+#define RMTRACE_EVENT_FLAG_FREQ         NVBIT(1)      // an event frequently happens.  In ETW, freq events use an independent Provider 'NVRM_FREQ'
+#define RMTRACE_EVENT_FLAG_FLUSHABLE    NVBIT(2)      // an event may cause a flush.  'Flush' is enabled by regkey 'RMEnableEventTracer'
+
+//
+// Function Progress Value
+//
+// RMTRACE_FUNC_PROG_ENTER and RMTRACE_FUNC_PROG_EXIT are used to indicate enter and leave state of a specific events, so that the time interval
+// of this event can be computed.
+//
+// RMTRACE_FUNC_PROG_STEP can be combined with RMTRACE_FUNC_PROG_ENTER and RMTRACE_FUNC_PROG_EXIT, used in ETW_Event_Generic_Marker. Below is a
+// sample how to use it.
+//
+// void AAA()
+// {
+//      RMTRACE_MARKER_PROBE("AAA Function", pGpu->gpuId, data, RMTRACE_FUNC_PROG_ENTER);
+//      ...
+//      RMTRACE_MARKER_PROBE("About To Enter XXX", pGpu->gpuId, data, RMTRACE_FUNC_PROG_STEP);
+//      XXX();
+//      ...
+//      BBB();
+//      ...
+//      RMTRACE_MARKER_PROBE("AAA Function", pGpu->gpuId, data, RMTRACE_FUNC_PROG_EXIT);
+// }
+//
+//
+// void BBB()
+// {
+//      RMTRACE_MARKER_PROBE("BBB Function", pGpu->gpuId, data, RMTRACE_FUNC_PROG_ENTER);
+//      ...
+//      CCC();
+//      ...
+//      RMTRACE_MARKER_PROBE("BBB Function", pGpu->gpuId, data, RMTRACE_FUNC_PROG_EXIT);
+// }
+//
+// With a tool (like EtwTool), we can generate below message automatically
+//
+// AAA Function (Enter)
+//      (0.1234ms)
+//   About to Enter XXX
+//      (0.0012ms)
+//      BBB Function (Enter)
+//      BBB Function (Leave) - 0.23ms
+// AAA Function (Leave) -- 0.4111ms
+//
+
+#define RMTRACE_FUNC_PROG_ENTER          0x0000
+#define RMTRACE_FUNC_PROG_EXIT           0x00FF
+#define RMTRACE_FUNC_PROG_STEP           0x007F
+#define RMTRACE_UNKNOWN_GPUID            0xFFFFFFFF
+#define RMTRACE_UNUSED_PARAM             0
+#define RMTRACE_MAX_PRINT_BUFFER_SIZE    128
+
+//
+// Empty macros
+//
+
+#define RMTRACE_INIT_NEW()
+#define RMTRACE_DESTROY_NEW()
+#define RMTRACE_SET_PTIMER_LOG(enable)
+#define RMTRACE_IS_PTIMER_LOG_ENABLED()                                             \
+    NV_FALSE
+#define RMTRACE_RMAPI(id, cmd)
+#define RMTRACE_RMLOCK(id)
+#define RMTRACE_DISP1(id, gpuId, param1)
+#define RMTRACE_DISP2(id, gpuId, param1, param2)
+#define RMTRACE_DISP3(id, gpuId, param1, param2, param3)
+#define RMTRACE_DISP4(id, gpuId, param1, param2, param3, param4)
+#define RMTRACE_DISP5(id, gpuId, param1, param2, param3, param4, param5)
+#define RMTRACE_DISP6(id, gpuId, param1, param2, param3, param4, param5, param6)
+#define RMTRACE_DISP_EDID(gpuId, publicId, connectedId, data, size)
+#define RMTRACE_DISP_BRIGHTNESS_ENTRY(dispId, flags, blType, pwmInfoProvider, pwmInfoEntries, SBEnable, lmnProvider, lmnEntryCount, blPwmInfoSize, blPwmInfo)
+#define RMTRACE_DISP_ERROR(id, gpuId, param1, param2, status)
+#define RMTRACE_DISP_EXCEPTION(gpuId, param1, param2, param3, param4, param5)
+#define RMTRACE_GPIO(id, _function, _state, _gpioPin, param)
+#define RMTRACE_GPIO_LIST(id, count, list)
+#define RMTRACE_I2C(id, gpuId, portId, address, indexSize, pIndex, dataSize, pData, status)
+#define RMTRACE_I2C_SET_ACQUIRED(gpuId, portId, acquirer, status, curTime)
+#define RMTRACE_I2C_ENUM_PORTS(gpuId, count, ports)
+#define RMTRACE_GPU(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_RMJOURNAL(id, gpuId, type, group, key, count, firstTime, lastTime)
+#define RMTRACE_POWER(id, gpuId, state, head, forcePerf, fastBootPowerState)
+#define RMTRACE_PERF(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_THERM2(id, gpuId, param1, param2)
+#define RMTRACE_THERM3(id, gpuId, param1, param2, param3)
+#define RMTRACE_THERM6(id, gpuId, param1, param2, param3, param4, param5, param6)
+#define RMTRACE_TIMEOUT(id, gpuId)
+#define RMTRACE_VBIOS(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_VBIOS_ERROR(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_NVLOG(id, pData, dataSize)
+#define RMTRACE_SBIOS(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_USBC0(id, gpuId)
+#define RMTRACE_USBC1(id, gpuId, param1)
+#define RMTRACE_USBC2(id, gpuId, param1, param2)
+#define RMTRACE_USBC7(id, gpuId, param1, param2, param3, param4, param5, param6, param7)
+#define RMTRACE_RMGENERAL(id, param1, param2, param3)
+#define RMTRACE_NVTELEMETRY(id, gpuId, param1, param2, param3)
+#define RMTRACE_NOCAT(id, gpuId, type, group, key, count, timeStamp)
+#define RMTRACE_PRINT
+
+
+#ifndef RMTRACE_FLAG_ENABLED
+#define RMTRACE_FLAG_ENABLED        (0)
+#endif
+
+//
+// Empty macros
+//
+#define RMTRACE_INIT()
+#define RMTRACE_DESTROY()
+#define RMTRACE_ENABLE(eventEventMask)
+#define RMTRACE_PROBE(module, event)
+
+#define RMTRACE_PROBE1(module, event, dataType, data, dataSize)
+
+#define RMTRACE_PROBE2(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2)
+
+#define RMTRACE_PROBE3(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,     \
+                       dataType3, data3, dataSize3)
+
+#define RMTRACE_PROBE4(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,     \
+                       dataType3, data3, dataSize3, dataType4, data4, dataSize4)
+
+#define RMTRACE_PROBE5(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,     \
+                       dataType3, data3, dataSize3, dataType4, data4, dataSize4,                    \
+                       dataType5, data5, dataSize5)
+
+#define RMTRACE_PROBE6(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,     \
+                       dataType3, data3, dataSize3, dataType4, data4, dataSize4,                    \
+                       dataType5, data5, dataSize5, dataType6, data6, dataSize6)
+
+#define RMTRACE_PROBE7(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,     \
+                       dataType3, data3, dataSize3, dataType4, data4, dataSize4,                    \
+                       dataType5, data5, dataSize5, dataType6, data6, dataSize6,                    \
+                       dataType7, data7, dataSize7)
+#define RMTRACE_PROBE10(module, event, dataType1, data1, dataSize1, dataType2, data2, dataSize2,    \
+                       dataType3, data3, dataSize3, dataType4, data4, dataSize4,                    \
+                       dataType5, data5, dataSize5, dataType6, data6, dataSize6,                    \
+                       dataType7, data7, dataSize7, dataType8, data8, dataSize8,                    \
+                       dataType9, data9, dataSize9, dataType10, data10, dataSize10)
+#define RMTRACE_PROBE2_PRIMTYPE(module, event, type0, val0, type1, val1)
+#define RMTRACE_PROBE3_PRIMTYPE(module, event, type0, val0, type1, val1, type2, val2)
+#define RMTRACE_PROBE4_PRIMTYPE(module, event, type0, val0, type1, val1, type2, val2, type3, val3)
+#define RMTRACE_PROBE5_PRIMTYPE(module, event, type0, val0, type1, val1, type2, val2, type3, val3,                  \
+                                type4, val4)
+#define RMTRACE_PROBE7_PRIMTYPE(module, event, type0, val0, type1, val1, type2, val2, type3, val3,                  \
+                                type4, val4, type5, val5, type6, val6)
+#define RMTRACE_PROBE10_PRIMTYPE(module, event, type0, val0, type1, val1, type2, val2, type3, val3,                 \
+                                 type4, val4, type5, val5, type6, val6, type7, val7, type8, val8,                   \
+                                 type9, val9)
+#define RMTRACE_MARKER_PROBE(name, gpuId, payload, id)
+
+
+#endif /* TRACER_H */

src/nvidia/inc/kernel/gpu/audio/hda_codec_api.h

@@ -0,0 +1,3 @@
+
+#include "g_hda_codec_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bif/kernel_bif.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_bif_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/kern_bus.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_bus_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/p2p_api.h

@@ -0,0 +1,3 @@
+
+#include "g_p2p_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/third_party_p2p.h

@@ -0,0 +1,3 @@
+
+#include "g_third_party_p2p_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ce_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce_context.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ce_context_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce_gv100_private.h

@@ -0,0 +1,84 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_GV100_PRIVATE_H
+#define KERNEL_CE_GV100_PRIVATE_H
+
+#define MAX_CE_CNT 15
+
+/*
+ * sysmemLinks
+ *    Represents the number of sysmem links detected
+ *    This affects how many PCEs LCE0(sysmem read CE)
+ *    and LCE1(sysmem write CE) should be mapped to
+ * maxLinksPerPeer
+ *    Represents the maximum number of peer links
+ *    between this GPU and all its peers. This affects
+ *    how many PCEs LCE3(P2P CE) should be mapped to
+ * numPeers
+ *    Represents the number of Peer GPUs discovered so far
+ * bSymmetric
+ *    Represents whether the topology detected so far
+ *    is symmetric i.e. has same number of links to all
+ *    peers connected through nvlink. This affects how
+ *    many PCEs to assign to LCEs3-5 (nvlink P2P CEs)
+ * bSwitchConfig
+ *    Represents whether the config listed is intended
+ *    for use with nvswitch systems
+ * pceLceMap
+ *    Value of NV_CE_PCE2LCE_CONFIG0 register with the
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ * grceConfig
+ *    Value of NV_CE_GRCE_CONFIG register with the
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ * exposeCeMask
+ *    Mask of CEs to expose to clients for the above
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ */
+typedef struct NVLINK_CE_AUTO_CONFIG_TABLE
+{
+    NvU32  sysmemLinks;
+    NvU32  maxLinksPerPeer;
+    NvU32  numPeers;
+    NvBool bSymmetric;
+    NvBool bSwitchConfig;
+    NvU32  pceLceMap[MAX_CE_CNT];
+    NvU32  grceConfig[MAX_CE_CNT];
+    NvU32  exposeCeMask;
+} NVLINK_CE_AUTO_CONFIG_TABLE;
+
+//
+// General convention decided on between HW and SW:
+//  - CE2 is for SYSMEM reads
+//  - CE3 is for SYSMEM writes
+//  - CE4-8 are for P2P
+//
+#define NVLINK_SYSMEM_READ_LCE  2
+#define NVLINK_SYSMEM_WRITE_LCE 3
+#define NVLINK_MIN_P2P_LCE      4
+#define NVLINK_MAX_P2P_LCE      8
+
+#endif

src/nvidia/inc/kernel/gpu/ce/kernel_ce_private.h

@@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_PRIVATE_H
+#define KERNEL_CE_PRIVATE_H
+
+#include "gpu/gpu.h"
+#include "gpu_mgr/gpu_mgr.h"
+#include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
+
+#define CE_GET_LOWEST_AVAILABLE_IDX(x) portUtilCountTrailingZeros32(x)
+
+/*!
+ * @brief Obtain relative CE index.
+ *
+ * @param localEngType NV2080_ENGINE_TYPE_ for this CE, or partition-local engine type.
+ * @param ceIdx CE index in 0..GPU_MAX_CES-1
+ *
+ * @return NV_OK if the conversion is successful.
+ */
+static NV_INLINE
+NV_STATUS ceIndexFromType(OBJGPU *pGpu, NvHandle hClient, NvU32 localEngType, NvU32 *ceIdx)
+{
+    //
+    // If MIG is enabled, client passes a logical engineId w.r.t its own partition
+    // we need to convert this logical Id to a physical engine Id as we use it
+    // to set runlistId
+    //
+    if (IS_MIG_IN_USE(pGpu))
+    {
+        KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+        MIG_INSTANCE_REF ref;
+
+        NV_CHECK_OK_OR_RETURN(
+            LEVEL_ERROR,
+            kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager,
+                                            hClient, &ref));
+
+        NV_CHECK_OK_OR_RETURN(
+            LEVEL_ERROR,
+            kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
+                                              localEngType,
+                                              ceIdx));
+    }
+    else
+    {
+        *ceIdx = localEngType;
+    }
+
+    if (!NV2080_ENGINE_TYPE_IS_COPY(*ceIdx))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    *ceIdx = NV2080_ENGINE_TYPE_COPY_IDX(*ceIdx);
+    return NV_OK;
+}
+
+#endif // KERNEL_CE_PRIVATE_H

src/nvidia/inc/kernel/gpu/ce/kernel_ce_shared.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_SHARED_H
+#define KERNEL_CE_SHARED_H
+
+#include "gpu/gpu.h"
+
+/**
+ * Routines shared between CE and KCE.
+ */
+
+NvBool ceIsCeGrce(OBJGPU *pGpu, NvU32 ceEngineType);
+NvU32 ceCountGrCe(OBJGPU *pGpu);
+
+#endif

src/nvidia/inc/kernel/gpu/dce_client/dce_client.h

@@ -0,0 +1,3 @@
+
+#include "g_dce_client_nvoc.h"
+

src/nvidia/inc/kernel/gpu/deferred_api.h

@@ -0,0 +1,3 @@
+
+#include "g_deferred_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/device/device.h

@@ -0,0 +1,3 @@
+
+#include "g_device_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_capabilities.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_capabilities_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_channel.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_channel_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_objs.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_objs_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_sf_user.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_sf_user_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/head/kernel_head.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_head_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/inst_mem/disp_inst_mem.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_inst_mem_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/kern_disp.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_disp_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/kern_disp_max.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERN_DISP_MAX_H
+#define KERN_DISP_MAX_H
+
+/*!
+ * @brief Defines max values used for the KernelDisplay Engine Object,
+ *        including values shared by OBJDISP code.
+ */
+
+#define OBJ_MAX_HEADS                     4
+#define MAX_RG_LINE_CALLBACKS_PER_HEAD    2
+#define OBJ_MAX_DFPS                     31
+
+#endif // KERN_DISP_MAX_H

src/nvidia/inc/kernel/gpu/disp/kern_disp_type.h

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERN_DISP_TYPE_H
+#define KERN_DISP_TYPE_H
+
+/******************************************************************************
+*
+*  Defines display type enums that can be used in the KernelDisplay object.
+*
+******************************************************************************/
+
+typedef enum
+{
+    dispChnClass_Curs,
+    dispChnClass_Ovim,
+    dispChnClass_Base,
+    dispChnClass_Core,
+    dispChnClass_Ovly,
+    dispChnClass_Winim,
+    dispChnClass_Win,
+    dispChnClass_Supported
+} DISPCHNCLASS;
+
+enum DISPLAY_ICC_BW_CLIENT
+{
+    DISPLAY_ICC_BW_CLIENT_RM,
+    DISPLAY_ICC_BW_CLIENT_EXT,  // DD or MODS
+    NUM_DISPLAY_ICC_BW_CLIENTS
+};
+
+#endif // #ifndef KERN_DISP_TYPE_H

src/nvidia/inc/kernel/gpu/disp/rg_line_callback/rg_line_callback.h

@@ -0,0 +1,3 @@
+
+#include "g_rg_line_callback_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/vblank_callback/vblank.h

@@ -0,0 +1,112 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef VBLANK_H
+#define VBLANK_H
+
+#include "gpu/gpu.h"
+/* ------------------------ Types definitions ------------------------------ */
+/*!
+ * Callback function prototype
+ */
+typedef NV_STATUS (*VBLANKCALLBACKPROC)(OBJGPU*, void *, NvU32, NvU32, NV_STATUS);
+
+typedef struct VBLANKCALLBACK
+{
+    VBLANKCALLBACKPROC Proc;
+    void              *pObject;
+    NvBool             bObjectIsChannelDescendant;
+    NvU32              Param1;
+    NvU32              Param2;
+    NvU32              VBlankCount;
+    NvU32              VBlankOffset;
+    NvU64              TimeStamp;
+    NvU32              MC_CallbackFlag;
+    NvU32              Flags;
+    NV_STATUS          Status;
+    struct VBLANKCALLBACK *Next;
+    NvBool             bImmediateCallback;
+    NvBool             bIsVblankNotifyEnable;
+}VBLANKCALLBACK;
+
+/* ------------------------ Macros & Defines ------------------------------- */
+
+/*!
+ * Callback function registration flags
+ */
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_COUNT        0x00000001
+#define VBLANK_CALLBACK_FLAG_COMPLETE_ON_OBJECT_CLEANUP    0x00000002
+#define VBLANK_CALLBACK_FLAG_PERSISTENT                    0x00000004
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_TIMESTAMP           0x00000010
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_NEXT         0x00000020  // Explicit request for the next vblank.
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_OFFSET       0x00000040  // Explicit request for the vblank offset from the current one
+#define VBLANK_CALLBACK_FLAG_PROMOTE_TO_FRONT              0x00000080  // Promotes to being 'first', while still honoring VBlankCount
+#define VBLANK_CALLBACK_FLAG_RELEASES_SEMAPHORE            0x00000100  // A flag for deadlock detection to check if this callback could release a semaphore
+#define VBLANK_CALLBACK_FLAG_GUARANTEE_SAFETY              0x00000200  // This callback absolutely needs to run during vertical blank, even if it runs late as a consequence.
+#define VBLANK_CALLBACK_FLAG_LOW_LATENCY__ISR_ONLY         0x08000000  // This means always process during ISR (never DPC.) Be careful!
+#define VBLANK_CALLBACK_FLAG_LOW_LATENCY                   0x10000000  // This now means ASAP, which could be ISR or DPC, depending on which happens first
+#define VBLANK_CALLBACK_FLAG_MC_EXECUTE_ONCE               0x40000000  // A special flag for MultiChip configurations to have the callback execute only once
+#define VBLANK_CALLBACK_FLAG_USER                          0x80000000
+
+/*!
+ * A little macro help for the CALLBACK_FLAG_MC_EXECUTE_ONCE flag above
+ */
+#define VBLANK_CALLBACK_EXECUTE_ONCE(x)    (x & VBLANK_CALLBACK_FLAG_MC_EXECUTE_ONCE)
+
+/*!
+ * VBlank Service info gathering keep-alive in seconds. This value is the number of seconds the vblank service will run after a client request vblank info.
+ */
+#define VBLANK_INFO_GATHER_KEEPALIVE_SECONDS    (5)
+
+/*!
+ * VBLANK SERVICE RELATED
+ * VBlank Service callback processing flags
+ * These two flags describe when to process the queues
+ */
+
+#define VBLANK_STATE_PROCESS_NORMAL                  (0x00000000)    // Process the requested queues if associated vblank interrupt is pending
+#define VBLANK_STATE_PROCESS_IMMEDIATE               (0x00000001)    // Process the requested queues now, regardless of any vblank interrupt pending state
+
+/*!
+ * These three flags describe which queues to process
+ */
+#define VBLANK_STATE_PROCESS_LOW_LATENCY             (0x00000002)    // Process the low-latency vblank callback queue
+#define VBLANK_STATE_PROCESS_NORMAL_LATENCY          (0x00000004)    // Process the normal-latency vblank callback queue
+
+#define VBLANK_STATE_PROCESS_ALL_CALLBACKS           (VBLANK_STATE_PROCESS_LOW_LATENCY|VBLANK_STATE_PROCESS_NORMAL_LATENCY) // Process all callback (high and low latency) queues
+
+#define VBLANK_STATE_PROCESS_CALLBACKS_ONLY          (0x00000008)    // Process only the callback queue(s) and nothing else
+
+/*!
+ * set when called from an ISR; if VBlank() is in an ISR and there is
+ * more work to do, then VBlank() will not clear the pending bit
+ */
+#define VBLANK_STATE_PROCESS_CALLED_FROM_ISR         (0x00000010)
+#define VBLANK_STATE_PROCESS_CALLED_FROM_DPC         (0x00000020)
+
+/*! Vblank Interrupt state */
+#define NV_HEAD_VBLANK_INTR_UNAVAILABLE              (0x00000000)
+#define NV_HEAD_VBLANK_INTR_AVAILABLE                (0x00000001)
+#define NV_HEAD_VBLANK_INTR_ENABLED                  (0x00000002)
+
+#endif // VBLANK_H

src/nvidia/inc/kernel/gpu/disp/vblank_callback/vblank_callback.h

@@ -0,0 +1,3 @@
+
+#include "g_vblank_callback_nvoc.h"
+

src/nvidia/inc/kernel/gpu/eng_desc.h

@@ -0,0 +1,3 @@
+
+#include "g_eng_desc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/eng_state.h

@@ -0,0 +1,3 @@
+
+#include "g_eng_state_nvoc.h"
+

src/nvidia/inc/kernel/gpu/falcon/falcon_common.h

@@ -0,0 +1,61 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-202 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef FALCON_COMMON_H
+#define FALCON_COMMON_H
+
+/*!
+ * Alignment/granularity of falcon memory blocks
+ */
+#define FLCN_BLK_ALIGNMENT (256)
+
+/*!
+ * Address/offset alignment required for falcon IMEM accesses
+ */
+#define FLCN_IMEM_ACCESS_ALIGNMENT (4)
+
+/*!
+ * Address/offset alignment required for falcon DMEM accesses
+ */
+#define FLCN_DMEM_ACCESS_ALIGNMENT (4)
+
+/*!
+ * Falcon IMEM block-size (as a power-of-2)
+ */
+#define FALCON_IMEM_BLKSIZE2 (8)
+
+/*!
+ * Falcon DMEM block-size (as a power-of-2)
+ */
+#define FALCON_DMEM_BLKSIZE2 (8)
+
+/*!
+ * Denotes invalid/absent VA for falcon ucode loading
+ */
+#define FLCN_DMEM_VA_INVALID 0xffffffff
+
+/*!
+ * Default Falcon context buffer size
+ */
+#define FLCN_CTX_ENG_BUFFER_SIZE_HW      4096
+
+#endif  // FALCON_COMMON_H

src/nvidia/inc/kernel/gpu/falcon/kernel_falcon.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_falcon_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/channel_descendant.h

@@ -0,0 +1,3 @@
+
+#include "g_channel_descendant_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel_group.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_group_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel_group_api.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_group_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_ctxshare.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ctxshare_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_fifo.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_fifo_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_sched_mgr.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_sched_mgr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/usermode_api.h

@@ -0,0 +1,3 @@
+
+#include "g_usermode_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/uvm_channel_retainer.h

@@ -0,0 +1,3 @@
+
+#include "g_uvm_channel_retainer_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_access.h

@@ -0,0 +1,381 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_ACCESS_H_
+#define _GPU_ACCESS_H_
+
+#include "ioaccess/ioaccess.h"
+#include "gpu/gpu_device_mapping.h"
+
+// Go straight at the memory or hardware.
+#define MEM_RD08(a) (*(const volatile NvU8  *)(a))
+#define MEM_RD16(a) (*(const volatile NvU16 *)(a))
+#define MEM_RD32(a) (*(const volatile NvU32 *)(a))
+#define MEM_WR08(a, d) do { *(volatile NvU8  *)(a) = (d); } while (0)
+#define MEM_WR16(a, d) do { *(volatile NvU16 *)(a) = (d); } while (0)
+#define MEM_WR32(a, d) do { *(volatile NvU32 *)(a) = (d); } while (0)
+#define MEM_WR64(a, d) do { *(volatile NvU64 *)(a) = (d); } while (0)
+
+//
+// Define the signature of the register filter callback function
+//
+// flags can be optionally used for filters to decide whether to actually
+// touch HW or not. flags should be OR'ed every time a new filter is found. (see objgpu.c)
+//
+typedef void (*GpuWriteRegCallback)(OBJGPU *, void *, NvU32 addr, NvU32 val, NvU32 accessSize, NvU32 flags);
+typedef NvU32 (*GpuReadRegCallback)(OBJGPU *, void *, NvU32 addr, NvU32 accessSize, NvU32 flags);
+
+union GPUHWREG
+{
+    volatile NvV8 Reg008[1];
+    volatile NvV16 Reg016[1];
+    volatile NvV32 Reg032[1];
+};
+
+typedef union  GPUHWREG  GPUHWREG;
+
+//
+// Register filter record
+//
+// If REGISTER_FILTER_FLAGS_READ is set, then that means that the base RegRead
+// function will not read the register, so the provided read callback function
+// is expected to read the register and return the value.
+//
+// If REGISTER_FILTER_FLAGS_WRITE is set, then that means that the base RegWrite
+// function will not write the register, so the provided callback write function
+// is expected to write the given value to the register.
+//
+// It is an error to specify REGISTER_FILTER_FLAGS_READ and not provide a
+// read callback function.
+//
+// It is an error to specify REGISTER_FILTER_FLAGS_WRITE and not provide a
+// write callback function.
+//
+#define REGISTER_FILTER_FLAGS_READ         (NVBIT(0))
+#define REGISTER_FILTER_FLAGS_WRITE        (NVBIT(1))
+// filter is in the list but it is invalid and should be removed
+#define REGISTER_FILTER_FLAGS_INVALID      (NVBIT(2))
+
+#define REGISTER_FILTER_FLAGS_VIRTUAL      (0)
+#define REGISTER_FILTER_FLAGS_READ_WRITE   (REGISTER_FILTER_FLAGS_READ | REGISTER_FILTER_FLAGS_WRITE)
+
+typedef struct REGISTER_FILTER REGISTER_FILTER;
+
+struct REGISTER_FILTER
+{
+    REGISTER_FILTER            *pNext;           //!< pointer to next filter
+    NvU32                       flags;           //!< attributes of this filter
+    DEVICE_INDEX                devIndex;        //!< filter device
+    NvU32                       devInstance;     //!< filter device instance
+    NvU32                       rangeStart;      //!< filter range start (can overlap)
+    NvU32                       rangeEnd;        //!< filter range end   (can overlap)
+    GpuWriteRegCallback         pWriteCallback;  //!< callback for write
+    GpuReadRegCallback          pReadCallback;   //!< callback for read
+    void                       *pParam;          //!< pointer to param which gets passed to callbacks
+};
+
+typedef struct {
+    REGISTER_FILTER        *pRegFilterList;         // Active filters
+    REGISTER_FILTER        *pRegFilterRecycleList;  // Inactive filters
+    PORT_SPINLOCK *         pRegFilterLock;         // Thread-safe list management
+    NvU32                   regFilterRefCnt;        // Thread-safe list management
+    NvBool                  bRegFilterNeedRemove;   // Thread-safe list garbage collection
+} DEVICE_REGFILTER_INFO;
+
+typedef struct DEVICE_MAPPING
+{
+    GPUHWREG             *gpuNvAddr;        // CPU Virtual Address
+    RmPhysAddr            gpuNvPAddr;       // Physical Base Address
+    NvU32                 gpuNvLength;      // Length of the Aperture
+    NvU32                 gpuNvSaveLength;
+    NvU32                 gpuDeviceEnum;    // Device ID NV_DEVID_*
+    NvU32                 refCount;         // refCount for the device map.
+    DEVICE_REGFILTER_INFO devRegFilterInfo; // register filter range list
+} DEVICE_MAPPING;
+
+typedef struct
+{
+    IO_DEVICE             parent;
+    OBJGPU               *pGpu;
+    DEVICE_INDEX          deviceIndex;
+    NvU32                 instance;
+    // The following members are initialized in objgpu.c,
+    // but are not used anywhere. gpuApertureReg* functions
+    // fall back to DEVICE_MAPPING instead
+    GPUHWREG             *gpuNvAddr;        // CPU Virtual Address
+    RmPhysAddr            gpuNvPAddr;       // Physical Base Address
+    NvU32                 gpuNvLength;      // Length of Aperture
+    NvU32                 gpuDeviceEnum;    // Device ID NV_DEVID_*
+    NvU32                 refCount;         // refCount for the device map.
+    DEVICE_REGFILTER_INFO devRegFilterInfo; // register filter range list
+} GPU_IO_DEVICE;
+
+typedef struct
+{
+    // Pointer to GPU linked to this RegisterAccess object
+    OBJGPU       *pGpu;
+
+    // HW register access tools
+    GPUHWREG     *gpuFbAddr;
+    GPUHWREG     *gpuInstAddr;
+
+    // Register access profiling
+    NvU32         regReadCount;
+    NvU32         regWriteCount;
+} RegisterAccess;
+
+/*! Init register IO access path */
+NV_STATUS regAccessConstruct(RegisterAccess *, OBJGPU *pGpu);
+
+/*! Shutdown register IO access path */
+void regAccessDestruct(RegisterAccess *);
+
+/*! Writes to 8 bit register */
+void regWrite008(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV8);
+
+/*! Writes to 16 bit register */
+void regWrite016(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV16);
+
+/*! Writes to 32 bit register, with thread state on the stack */
+void regWrite032(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV32, THREAD_STATE_NODE *);
+
+/*! Unicast register access, with thread state on the stack */
+void regWrite032Unicast(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV32, THREAD_STATE_NODE *);
+
+/*! Reads from 8 bit register */
+NvU8 regRead008(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32);
+
+/*! Reads from 16 bit register */
+NvU16 regRead016(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32);
+
+/*! Reads from 32 bit register, with thread state on the stack */
+NvU32 regRead032(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, THREAD_STATE_NODE *);
+
+/*! Reads from 32 bit register and checks bit mask, with thread state on the stack */
+NvU32 regCheckRead032(RegisterAccess *, NvU32, NvU32, THREAD_STATE_NODE *);
+
+/*! Reads 32 bit register and polls bit field for specific value */
+NV_STATUS regRead032_AndPoll(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvU32);
+
+/*! Adds a register filter */
+NV_STATUS regAddRegisterFilter(RegisterAccess *, NvU32, DEVICE_INDEX, NvU32, NvU32, NvU32, GpuWriteRegCallback, GpuReadRegCallback, void *, REGISTER_FILTER **);
+
+/*! Removes register filter */
+void regRemoveRegisterFilter(RegisterAccess *, REGISTER_FILTER *);
+
+/*! Check status of read return value for GPU/bus errors */
+void regCheckAndLogReadFailure(RegisterAccess *, NvU32 addr, NvU32 mask, NvU32 value);
+
+//
+// GPU register I/O macros.
+//
+
+//
+// GPU neutral macros typically used for register I/O.
+//
+#define GPU_DRF_SHIFT(drf)          ((0?drf) % 32)
+#define GPU_DRF_MASK(drf)           (0xFFFFFFFF>>(31-((1?drf) % 32)+((0?drf) % 32)))
+#define GPU_DRF_DEF(d,r,f,c)        ((NV ## d ## r ## f ## c)<<GPU_DRF_SHIFT(NV ## d ## r ## f))
+#define GPU_DRF_NUM(d,r,f,n)        (((n)&GPU_DRF_MASK(NV ## d ## r ## f))<<GPU_DRF_SHIFT(NV ## d ## r ## f))
+#define GPU_DRF_VAL(d,r,f,v)        (((v)>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_DRF_SHIFTMASK(drf)      (GPU_DRF_MASK(drf)<<(GPU_DRF_SHIFT(drf)))
+#define GPU_DRF_WIDTH(drf)          ((1?drf) - (0?drf) + 1)
+
+
+// Device independent macros
+// Multiple device instance macros
+
+#define REG_INST_RD08(g,dev,inst,a)             regRead008(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a)
+#define REG_INST_RD16(g,dev,inst,a)             regRead016(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a)
+#define REG_INST_RD32(g,dev,inst,a)             regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, NULL)
+
+#define REG_INST_WR08(g,dev,inst,a,v)     regWrite008(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v)
+#define REG_INST_WR16(g,dev,inst,a,v)     regWrite016(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v)
+#define REG_INST_WR32(g,dev,inst,a,v)     regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, NULL)
+#define REG_INST_WR32_UC(g,dev,inst,a,v)  regWrite032Unicast(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, NULL)
+
+#define REG_INST_RD32_EX(g,dev,inst,a,t)    regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, t)
+#define REG_INST_WR32_EX(g,dev,inst,a,v,t)  regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, t)
+
+#define REG_INST_DEVIDX_RD32_EX(g,devidx,inst,a,t)    regRead032(GPU_GET_REGISTER_ACCESS(g), devidx, inst, a, t)
+#define REG_INST_DEVIDX_WR32_EX(g,devidx,inst,a,v,t)  regWrite032(GPU_GET_REGISTER_ACCESS(g), devidx, inst, a, v, t)
+
+// GPU macros defined in terms of DEV_ macros
+#define GPU_REG_RD08(g,a) REG_INST_RD08(g,GPU,0,a)
+#define GPU_REG_RD16(g,a) REG_INST_RD16(g,GPU,0,a)
+#define GPU_REG_RD32(g,a) REG_INST_RD32(g,GPU,0,a)
+#define GPU_CHECK_REG_RD32(g,a,m) regCheckRead032(GPU_GET_REGISTER_ACCESS(g),a,m,NULL)
+#define GPU_REG_RD32_AND_POLL(g,r,m,v) regRead032_AndPoll(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_GPU, r, m, v)
+
+#define GPU_REG_WR08(g,a,v)    REG_INST_WR08(g,GPU,0,a,v)
+#define GPU_REG_WR16(g,a,v)    REG_INST_WR16(g,GPU,0,a,v)
+#define GPU_REG_WR32(g,a,v)    REG_INST_WR32(g,GPU,0,a,v)
+#define GPU_REG_WR32_UC(g,a,v) REG_INST_WR32_UC(g,GPU,0,a,v)
+
+// GPU macros for SR-IOV
+#define GPU_VREG_RD32(g, a)                GPU_REG_RD32(g, g->sriovState.virtualRegPhysOffset + a)
+#define GPU_VREG_WR32(g, a, v)             GPU_REG_WR32(g, g->sriovState.virtualRegPhysOffset + a, v)
+#define GPU_VREG_RD32_EX(g,a,t)            REG_INST_RD32_EX(g, GPU, 0, g->sriovState.virtualRegPhysOffset + a, t)
+#define GPU_VREG_WR32_EX(g,a,v,t)          REG_INST_WR32_EX(g, GPU, 0, g->sriovState.virtualRegPhysOffset + a, v, t)
+#define GPU_VREG_FLD_WR_DRF_DEF(g,d,r,f,c) GPU_VREG_WR32(g, NV##d##r,(GPU_VREG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_VREG_RD_DRF(g,d,r,f)        (((GPU_VREG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+#define VREG_INST_RD32(g,dev,inst,a)            regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, g->sriovState.virtualRegPhysOffset + a, NULL)
+#define VREG_INST_WR32(g,dev,inst,a,v)          regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, g->sriovState.virtualRegPhysOffset + a, v, NULL)
+#define GPU_VREG_FLD_WR_DRF_NUM(g,d,r,f,n) VREG_INST_WR32(g,GPU,0,NV##d##r,(VREG_INST_RD32(g,GPU,0,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+
+#define GPU_VREG_FLD_TEST_DRF_DEF(g,d,r,f,c) (GPU_VREG_RD_DRF(g, d, r, f) == NV##d##r##f##c)
+
+#define GPU_GET_VREG_OFFSET(g, a) (g->sriovState.virtualRegPhysOffset + a)
+
+#define GPU_VREG_IDX_RD_DRF(g,d,r,i,f)           (((GPU_VREG_RD32(g, NV ## d ## r(i)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_VREG_FLD_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_VREG_WR32(g, NV##d##r(i),(GPU_VREG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+#define GPU_REG_RD32_EX(g,a,t)    REG_INST_RD32_EX(g,GPU,0,a,t)
+#define GPU_REG_WR32_EX(g,a,v,t)  REG_INST_WR32_EX(g,GPU,0,a,v,t)
+
+// Uncomment this to enable register access dump in gsp client
+// #define GPU_REGISTER_ACCESS_DUMP    RMCFG_FEATURE_GSP_CLIENT_RM
+#ifndef GPU_REGISTER_ACCESS_DUMP
+#define GPU_REGISTER_ACCESS_DUMP    0
+#endif
+
+#if GPU_REGISTER_ACCESS_DUMP
+
+NvU8  gpuRegRd08_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+NvU16 gpuRegRd16_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+NvU32 gpuRegRd32_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+void  gpuRegWr08_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV8 val);
+void  gpuRegWr16_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV16 val);
+void  gpuRegWr32_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV32 val);
+void  gpuRegWr32Uc_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV32 val);
+
+#undef GPU_REG_RD08
+#undef GPU_REG_RD16
+#undef GPU_REG_RD32
+#undef GPU_REG_WR08
+#undef GPU_REG_WR16
+#undef GPU_REG_WR32
+#undef GPU_REG_WR32_UC
+#undef GPU_VREG_RD32
+#undef GPU_VREG_WR32
+
+#define GPU_REG_RD08(g,a)       gpuRegRd08_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_RD16(g,a)       gpuRegRd16_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_RD32(g,a)       gpuRegRd32_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_WR08(g,a,v)     gpuRegWr08_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR16(g,a,v)     gpuRegWr16_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR32(g,a,v)     gpuRegWr32_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR32_UC(g,a,v)  gpuRegWr32Uc_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_VREG_RD32(g, a)     gpuRegRd32_dumpinfo(__FUNCTION__,#a,"(VREG)",g, g->sriovState.virtualRegPhysOffset + a)
+#define GPU_VREG_WR32(g, a, v)  gpuRegWr32_dumpinfo(__FUNCTION__,#a,"(VREG)",g, g->sriovState.virtualRegPhysOffset + a, v)
+
+#endif // GPU_REGISTER_ACCESS_DUMP
+
+//
+// Macros for register I/O
+//
+#define GPU_FLD_WR_DRF_NUM(g,d,r,f,n) REG_INST_WR32(g,GPU,0,NV##d##r,(REG_INST_RD32(g,GPU,0,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_WR_DRF_NUM_UC(g,d,r,f,n) GPU_REG_WR32_UC(g, NV##d##r,(GPU_REG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_WR_DRF_DEF(g,d,r,f,c) GPU_REG_WR32(g, NV##d##r,(GPU_REG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_RD_DRF(g,d,r,f)       (((GPU_REG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_FLD_TEST_DRF_DEF(g,d,r,f,c) (GPU_REG_RD_DRF(g, d, r, f) == NV##d##r##f##c)
+#define GPU_FLD_TEST_DRF_NUM(g,d,r,f,n) (GPU_REG_RD_DRF(g, d, r, f) == n)
+#define GPU_FLD_IDX_TEST_DRF_DEF(g,d,r,f,c,i) (GPU_REG_IDX_RD_DRF(g, d, r, i, f) == NV##d##r##f##c)
+#define GPU_FLD_2IDX_TEST_DRF_DEF(g,d,r,f,c,i,j) (GPU_REG_2IDX_RD_DRF(g, d, r, i, j, f) == NV##d##r##f##c)
+
+#define GPU_REG_RD_DRF_EX(g,d,r,f,t)       (((GPU_REG_RD32_EX(g, NV ## d ## r, t))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+#define GPU_FLD_WR_DRF_NUM_EX(g,d,r,f,n,t) REG_INST_WR32_EX(g,GPU,0,NV##d##r,(REG_INST_RD32_EX(g,GPU,0,NV##d##r,t)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n),t)
+
+// Read/write a field or entire register of which there are several copies each accessed via an index
+#define GPU_REG_IDX_WR_DRF_NUM(g,d,r,i,f,n) GPU_REG_WR32(g, NV ## d ## r(i), GPU_DRF_NUM(d,r,f,n))
+#define GPU_REG_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_REG_WR32(g, NV ## d ## r(i), GPU_DRF_DEF(d,r,f,c))
+#define GPU_FLD_IDX_WR_DRF_NUM(g,d,r,i,f,n) GPU_REG_WR32(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_REG_WR32(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_IDX_WR_DRF_NUM_UC(g,d,r,i,f,n) GPU_REG_WR32_UC(g, NV ## d ## r(i), GPU_DRF_NUM(d,r,f,n))
+#define GPU_REG_IDX_WR_DRF_DEF_UC(g,d,r,i,f,c) GPU_REG_WR32_UC(g, NV ## d ## r(i), GPU_DRF_DEF(d,r,f,c))
+#define GPU_FLD_IDX_WR_DRF_DEF_UC(g,d,r,i,f,c) GPU_REG_WR32_UC(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_IDX_RD_DRF(g,d,r,i,f)              (((GPU_REG_RD32(g, NV ## d ## r(i)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_REG_2IDX_RD_DRF(g,d,r,i,j,f)             (((GPU_REG_RD32(g, NV ## d ## r(i, j)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_REG_RD_DRF_IDX(g,d,r,f,i)              (((GPU_REG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f(i)))&GPU_DRF_MASK(NV ## d ## r ## f(i)))
+#define GPU_REG_IDX_OFFSET_RD_DRF(g,d,r,i,o,f)     (((GPU_REG_RD32(g, NV ## d ## r(i,o)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+//
+// Macros that abstract the use of bif object to access GPU bus config registers
+// This is the preferred set >= NV50
+//
+#define GPU_BUS_CFG_RD32(g,r,d)             gpuReadBusConfigReg_HAL(g, r, d)
+#define GPU_BUS_CFG_WR32(g,r,d)             gpuWriteBusConfigReg_HAL(g, r, d)
+#define GPU_BUS_CFG_FLD_WR_DRF_DEF(g,x,d,r,f,c) GPU_BUS_CFG_WR32(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_BUS_CFG_FLD_WR_DRF_NUM(g,x,d,r,f,n) GPU_BUS_CFG_WR32(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+
+#define GPU_BUS_CFG_RD32_EX(g,r,d,t)        gpuReadBusConfigRegEx_HAL(g, r, d, t)
+
+//
+// Macros that provide access to the config space of functions other than the gpu
+//
+#define PCI_FUNCTION_BUS_CFG_RD32(g,f,r,d)            gpuReadFunctionConfigReg_HAL(g, f, r, d)
+#define PCI_FUNCTION_BUS_CFG_WR32(g,f,r,d)            gpuWriteFunctionConfigReg_HAL(g, f, r, d)
+#define PCI_FUNCTION_BUS_CFG_FLD_WR_DRF_NUM(g,fn,x,d,r,f,n) gpuWriteFunctionConfigReg_HAL(g, fn, NV##d##r, (x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define PCI_FUNCTION_BUS_CFG_FLD_WR_DRF_DEF(g,fn,x,d,r,f,c) gpuWriteFunctionConfigReg_HAL(g, fn, NV##d##r, (x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+#define PCI_FUNCTION_BUS_CFG_WR32_EX(g,f,r,d,t)       gpuWriteFunctionConfigRegEx_HAL(g, f, r, d, t)
+
+#define GPU_BUS_CFG_CYCLE_RD32(g,r,d)                  gpuReadBusConfigCycle(g, r, d)
+#define GPU_BUS_CFG_CYCLE_WR32(g,r,d)                  gpuWriteBusConfigCycle(g, r, d)
+#define GPU_BUS_CFG_CYCLE_FLD_WR_DRF_DEF(g,x,d,r,f,c)  gpuWriteBusConfigCycle(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+//
+// Instance memory structure access definitions.
+//
+// DRF macros (nvmisc.h) should be used when possible instead of these
+// definitions.
+//
+// Key difference is SF variants take structure ## field (2-level nested
+// namespace), DRF take device ## register ## field (3-level nested
+// namespace).
+//
+// SF variants are primarily used for GPU host memory structures. DRF
+// should be used for manipulation of most registers
+//
+#define SF_INDEX(sf)            ((0?sf)/32)
+#define SF_OFFSET(sf)           (((0?sf)/32)<<2)
+#define SF_SHIFT(sf)            ((0?sf)&31)
+#undef  SF_MASK
+#define SF_MASK(sf)             (0xFFFFFFFF>>(31-(1?sf)+(0?sf)))
+#define SF_SHIFTMASK(sf)        (SF_MASK(sf) << SF_SHIFT(sf))
+#define SF_DEF(s,f,c)           ((NV ## s ## f ## c)<<SF_SHIFT(NV ## s ## f))
+#define SF_IDX_DEF(s,f,c,i)     ((NV ## s ## f ## c)<<SF_SHIFT(NV ## s ## f(i)))
+#define SF_NUM(s,f,n)           (((n)&SF_MASK(NV ## s ## f))<<SF_SHIFT(NV ## s ## f))
+#define SF_IDX_NUM(s,f,n,i)     (((n)&SF_MASK(NV ## s ## f(i)))<<SF_SHIFT(NV ## s ## f(i)))
+#define SF_VAL(s,f,v)           (((v)>>SF_SHIFT(NV ## s ## f))&SF_MASK(NV ## s ## f))
+#define SF_WIDTH(sf)            ((1?sf) - (0?sf) + 1)
+// This macro parses multi-word/array defines
+#define SF_ARR32_VAL(s,f,arr) \
+    (((arr)[SF_INDEX(NV ## s ## f)] >> SF_SHIFT(NV ## s ## f)) & SF_MASK(NV ## s ## f))
+#define FLD_SF_DEF(s,f,d,l)     ((l)&~(SF_MASK(NV##s##f) << SF_SHIFT(NV##s##f)))| SF_DEF(s,f,d)
+#define FLD_SF_NUM(s,f,n,l)     ((l)&~(SF_MASK(NV##s##f) << SF_SHIFT(NV##s##f)))| SF_NUM(s,f,n)
+#define FLD_SF_IDX_DEF(s,f,c,i,l) (((l) & ~SF_SHIFTMASK(NV ## s ## f(i))) | SF_IDX_DEF(s,f,c,i))
+#define FLD_SF_IDX_NUM(s,f,n,i,l) (((l) & ~SF_SHIFTMASK(NV ## s ## f(i))) | SF_IDX_NUM(s,f,n,i))
+
+#endif // _GPU_ACCESS_H_

src/nvidia/inc/kernel/gpu/gpu_child_list.h

@@ -0,0 +1,303 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+//
+// No include guards - this file is included multiple times, each time with a
+// different definition for GPU_CHILD_SINGLE_INST and GPU_CHILD_GPU_CHILD_MULTI_INST
+//
+// Callers that will use the same definition for single- and multi- instance
+// can define GPU_CHILD that will be used for both
+//
+#if defined(GPU_CHILD)
+#if !defined(GPU_CHILD_SINGLE_INST) && !defined(GPU_CHILD_MULTI_INST)
+#define GPU_CHILD_SINGLE_INST GPU_CHILD
+#define GPU_CHILD_MULTI_INST GPU_CHILD
+#else
+#error "Must not define GPU_CHILD_{SINGLE,MULTI}_INST and GPU_CHILD at the same time"
+#endif
+#endif
+
+//
+// GPU child list. All objects must inherit from OBJENGSTATE. Objects are
+// constructed in the listed order and destructed in reverse order. Storage in
+// OBJGPU and accessor macros (i.e.: GET_GPU_XXX) are generated from this list.
+//
+
+//
+// Temporarily needed to generate stubs for disabled modules
+// To be removed when the references to these modules are gone
+//
+#if defined(GPU_CHILD_LIST_DISABLED_ONLY)
+#define GPU_CHILD_MODULE(_rmcfgModule) !RMCFG_MODULE_ENABLED(_rmcfgModule)
+#else
+#define GPU_CHILD_MODULE(_rmcfgModule) RMCFG_MODULE_ENABLED(_rmcfgModule)
+#endif
+
+                        /* Class Name            Accessor  Name                       Max Instances     bConstructEarly  bAlwaysCreate  OBJGPU Field */
+#if GPU_CHILD_MODULE(FUSE)
+    GPU_CHILD_SINGLE_INST( OBJFUSE,              GPU_GET_FUSE,                        1,                NV_TRUE,         NV_TRUE,       pFuse            )
+#endif
+#if GPU_CHILD_MODULE(BIF)
+    GPU_CHILD_SINGLE_INST( OBJBIF,               GPU_GET_BIF,                         1,                NV_TRUE,         NV_FALSE,      pBif             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_BIF)
+    GPU_CHILD_SINGLE_INST( KernelBif,            GPU_GET_KERNEL_BIF,                  1,                NV_TRUE,         NV_FALSE,      pKernelBif       )
+#endif
+#if GPU_CHILD_MODULE(NNE)
+    GPU_CHILD_SINGLE_INST( OBJNNE,               GPU_GET_NNE,                         1,                NV_TRUE,         NV_FALSE,      pNne             )
+#endif
+#if GPU_CHILD_MODULE(MC)
+    GPU_CHILD_SINGLE_INST( OBJMC,                GPU_GET_MC,                          1,                NV_FALSE,        NV_FALSE,      pMc              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MC)
+    GPU_CHILD_SINGLE_INST( KernelMc,             GPU_GET_KERNEL_MC,                   1,                NV_FALSE,        NV_FALSE,      pKernelMc        )
+#endif
+#if GPU_CHILD_MODULE(PRIV_RING)
+    GPU_CHILD_SINGLE_INST( PrivRing,             GPU_GET_PRIV_RING,                   1,                NV_FALSE,        NV_FALSE,      pPrivRing        )
+#endif
+#if GPU_CHILD_MODULE(INTR)
+    GPU_CHILD_SINGLE_INST( SwIntr,               GPU_GET_SW_INTR,                     1,                NV_FALSE,        NV_FALSE,      pSwIntr          )
+#endif
+#if GPU_CHILD_MODULE(MEMORY_SYSTEM)
+    GPU_CHILD_SINGLE_INST( MemorySystem,         GPU_GET_MEMORY_SYSTEM,               1,                NV_FALSE,        NV_FALSE,      pMemorySystem    )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MEMORY_SYSTEM)
+    GPU_CHILD_SINGLE_INST( KernelMemorySystem,   GPU_GET_KERNEL_MEMORY_SYSTEM,        1,                NV_FALSE,        NV_FALSE,      pKernelMemorySystem )
+#endif
+#if GPU_CHILD_MODULE(MEMORY_MANAGER)
+    GPU_CHILD_SINGLE_INST( MemoryManager,        GPU_GET_MEMORY_MANAGER,              1,                NV_FALSE,        NV_FALSE,      pMemoryManager   )
+#endif
+#if GPU_CHILD_MODULE(FBFLCN)
+    GPU_CHILD_SINGLE_INST( OBJFBFLCN,            GPU_GET_FBFLCN,                      1,                NV_FALSE,        NV_FALSE,      pFbflcn          )
+#endif
+#if GPU_CHILD_MODULE(HSHUB)
+    GPU_CHILD_MULTI_INST ( OBJHSHUB,             GPU_GET_HSHUB,                       GPU_MAX_HSHUBS,   NV_FALSE,        NV_FALSE,      pHshub           )
+#endif
+#if GPU_CHILD_MODULE(SEQ)
+    GPU_CHILD_SINGLE_INST( OBJSEQ,               GPU_GET_SEQ,                         1,                NV_FALSE,        NV_TRUE,       pSeq             )
+#endif
+#if GPU_CHILD_MODULE(GpuMutexMgr)
+    GPU_CHILD_SINGLE_INST( GpuMutexMgr,          GPU_GET_MUTEX_MGR,                   1,                NV_FALSE,        NV_TRUE,       pMutexMgr        )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_DISPLAY)
+    GPU_CHILD_SINGLE_INST( KernelDisplay,        GPU_GET_KERNEL_DISPLAY,              1,                NV_FALSE,        NV_FALSE,      pKernelDisplay   )
+#endif
+#if GPU_CHILD_MODULE(DISP)
+    GPU_CHILD_SINGLE_INST( OBJDISP,              GPU_GET_DISP,                        1,                NV_FALSE,        NV_FALSE,      pDisp            )
+#endif
+#if GPU_CHILD_MODULE(TMR)
+    GPU_CHILD_SINGLE_INST( OBJTMR,               GPU_GET_TIMER,                       1,                NV_TRUE,         NV_TRUE,       pTmr             )
+#endif
+#if GPU_CHILD_MODULE(BUS)
+    GPU_CHILD_SINGLE_INST( OBJBUS,               GPU_GET_BUS,                         1,                NV_FALSE,        NV_FALSE,      pBus             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_BUS)
+    GPU_CHILD_SINGLE_INST( KernelBus,            GPU_GET_KERNEL_BUS,                  1,                NV_FALSE,        NV_FALSE,      pKernelBus       )
+#endif
+#if GPU_CHILD_MODULE(GMMU)
+    GPU_CHILD_SINGLE_INST( OBJGMMU,              GPU_GET_GMMU,                        1,                NV_FALSE,        NV_FALSE,      pGmmu            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GMMU)
+    GPU_CHILD_SINGLE_INST( KernelGmmu,           GPU_GET_KERNEL_GMMU,                 1,                NV_FALSE,        NV_FALSE,      pKernelGmmu      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_NVDEC)
+    GPU_CHILD_SINGLE_INST( KernelNvdec,          GPU_GET_KERNEL_NVDEC,                1,                NV_FALSE,        NV_FALSE,      pKernelNvdec     )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_SEC2)
+    GPU_CHILD_SINGLE_INST( KernelSec2,           GPU_GET_KERNEL_SEC2,                 1,                NV_FALSE,        NV_FALSE,      pKernelSec2      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GSP)
+    GPU_CHILD_SINGLE_INST( KernelGsp,            GPU_GET_KERNEL_GSP,                  1,                NV_FALSE,        NV_FALSE,      pKernelGsp       )
+#endif
+#if GPU_CHILD_MODULE(DCECLIENTRM)
+    GPU_CHILD_SINGLE_INST( OBJDCECLIENTRM,       GPU_GET_DCECLIENTRM,                 1,                NV_FALSE,        NV_FALSE,      pDceclientrm     )
+#endif
+#if GPU_CHILD_MODULE(VIRT_MEM_ALLOCATOR)
+    GPU_CHILD_SINGLE_INST( VirtMemAllocator,     GPU_GET_DMA,                         1,                NV_FALSE,        NV_FALSE,      pDma             )
+#endif
+#if GPU_CHILD_MODULE(GRMGR)
+    GPU_CHILD_SINGLE_INST( GraphicsManager,      GPU_GET_GRMGR,                       1,                NV_FALSE,        NV_TRUE,       pGrMgr           )
+#endif
+#if GPU_CHILD_MODULE(MIG_MANAGER)
+    GPU_CHILD_SINGLE_INST( MIGManager,           GPU_GET_MIG_MANAGER,                 1,                NV_FALSE,        NV_TRUE,       pMIGManager      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MIG_MANAGER)
+    GPU_CHILD_SINGLE_INST( KernelMIGManager,     GPU_GET_KERNEL_MIG_MANAGER,          1,                NV_FALSE,        NV_TRUE,       pKernelMIGManager )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GRAPHICS_MANAGER)
+    GPU_CHILD_SINGLE_INST( KernelGraphicsManager, GPU_GET_KERNEL_GRAPHICS_MANAGER,    1,                NV_FALSE,        NV_TRUE,       pKernelGraphicsManager )
+#endif
+#if GPU_CHILD_MODULE(GR)
+    GPU_CHILD_MULTI_INST ( Graphics,             GPU_GET_GR_UNSAFE,                   GPU_MAX_GRS,      NV_FALSE,        NV_FALSE,      pGr              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GRAPHICS)
+    GPU_CHILD_MULTI_INST ( KernelGraphics,       GPU_GET_KERNEL_GRAPHICS,             GPU_MAX_GRS,      NV_FALSE,        NV_FALSE,      pKernelGraphics  )
+#endif
+#if GPU_CHILD_MODULE(ClockManager)
+    GPU_CHILD_SINGLE_INST( ClockManager,         GPU_GET_CLK_MGR,                     1,                NV_FALSE,        NV_FALSE,      pClk             )
+#endif
+#if GPU_CHILD_MODULE(FAN)
+    GPU_CHILD_SINGLE_INST( OBJFAN,               GPU_GET_FAN,                         1,                NV_FALSE,        NV_FALSE,      pFan             )
+#endif
+#if GPU_CHILD_MODULE(PERF)
+    GPU_CHILD_SINGLE_INST( Perf,                 GPU_GET_PERF,                        1,                NV_FALSE,        NV_FALSE,      pPerf            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_PERF)
+    GPU_CHILD_SINGLE_INST( KernelPerf,           GPU_GET_KERNEL_PERF,                 1,                NV_FALSE,        NV_FALSE,      pKernelPerf      )
+#endif
+#if GPU_CHILD_MODULE(THERM)
+    GPU_CHILD_SINGLE_INST( Therm,                GPU_GET_THERM,                       1,                NV_FALSE,        NV_FALSE,      pTherm           )
+#endif
+#if GPU_CHILD_MODULE(BSP)
+    GPU_CHILD_MULTI_INST ( OBJBSP,               GPU_GET_BSP,                         GPU_MAX_NVDECS,   NV_FALSE,        NV_FALSE,      pBsp             )
+#endif
+#if GPU_CHILD_MODULE(CIPHER)
+    GPU_CHILD_SINGLE_INST( OBJCIPHER,            GPU_GET_CIPHER,                      1,                NV_FALSE,        NV_FALSE,      pCipher          )
+#endif
+#if GPU_CHILD_MODULE(VBIOS)
+    GPU_CHILD_SINGLE_INST( OBJVBIOS,             GPU_GET_VBIOS,                       1,                NV_FALSE,        NV_TRUE,       pVbios           )
+#endif
+#if GPU_CHILD_MODULE(DCB)
+    GPU_CHILD_SINGLE_INST( OBJDCB,               GPU_GET_DCB,                         1,                NV_FALSE,        NV_TRUE,       pDcb             )
+#endif
+#if GPU_CHILD_MODULE(GPIO)
+    GPU_CHILD_SINGLE_INST( OBJGPIO,              GPU_GET_GPIO,                        1,                NV_FALSE,        NV_TRUE,       pGpio            )
+#endif
+#if GPU_CHILD_MODULE(VOLT)
+    GPU_CHILD_SINGLE_INST( OBJVOLT,              GPU_GET_VOLT,                        1,                NV_FALSE,        NV_FALSE,      pVolt            )
+#endif
+#if GPU_CHILD_MODULE(I2C)
+    GPU_CHILD_SINGLE_INST( OBJI2C,               GPU_GET_I2C,                         1,                NV_FALSE,        NV_TRUE,       pI2c             )
+#endif
+#if GPU_CHILD_MODULE(SPI)
+    GPU_CHILD_SINGLE_INST( Spi,               GPU_GET_SPI,                            1,                NV_FALSE,        NV_TRUE,       pSpi             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_RC)
+    GPU_CHILD_SINGLE_INST( KernelRc,             GPU_GET_KERNEL_RC,                   1,                NV_FALSE,        NV_TRUE,       pKernelRc        )
+#endif
+#if GPU_CHILD_MODULE(RC)
+    GPU_CHILD_SINGLE_INST( OBJRC,                GPU_GET_RC,                          1,                NV_FALSE,        NV_TRUE,       pRC              )
+#endif
+#if GPU_CHILD_MODULE(STEREO)
+    GPU_CHILD_SINGLE_INST( OBJSTEREO,            GPU_GET_STEREO,                      1,                NV_FALSE,        NV_TRUE,       pStereo          )
+#endif
+#if GPU_CHILD_MODULE(INTR)
+    GPU_CHILD_SINGLE_INST( Intr,                 GPU_GET_INTR,                        1,                NV_FALSE,        NV_TRUE,       pIntr            )
+#endif
+#if GPU_CHILD_MODULE(DPAUX)
+    GPU_CHILD_SINGLE_INST( OBJDPAUX,             GPU_GET_DPAUX,                       1,                NV_FALSE,        NV_FALSE,      pDpAux           )
+#endif
+#if GPU_CHILD_MODULE(PMU)
+    GPU_CHILD_SINGLE_INST( Pmu,                  GPU_GET_PMU,                         1,                NV_FALSE,        NV_FALSE,      pPmu             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_PMU)
+    GPU_CHILD_SINGLE_INST( KernelPmu,            GPU_GET_KERNEL_PMU,                  1,                NV_FALSE,        NV_FALSE,      pKernelPmu       )
+#endif
+#if GPU_CHILD_MODULE(CE)
+    GPU_CHILD_MULTI_INST ( OBJCE,                GPU_GET_CE,                          GPU_MAX_CES,      NV_FALSE,        NV_FALSE,      pCe              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_CE)
+    GPU_CHILD_MULTI_INST ( KernelCE,             GPU_GET_KCE,                         GPU_MAX_CES,      NV_FALSE,        NV_FALSE,      pKCe              )
+#endif
+#if GPU_CHILD_MODULE(MSENC)
+    GPU_CHILD_MULTI_INST ( OBJMSENC,             GPU_GET_MSENC,                       GPU_MAX_MSENCS,   NV_FALSE,        NV_FALSE,      pMsenc           )
+#endif
+#if GPU_CHILD_MODULE(HDA)
+    GPU_CHILD_SINGLE_INST( OBJHDA,               GPU_GET_HDA,                         1,                NV_FALSE,        NV_FALSE,      pHda             )
+#endif
+#if GPU_CHILD_MODULE(HDACODEC)
+    GPU_CHILD_SINGLE_INST( OBJHDACODEC,          GPU_GET_HDACODEC,                    1,                NV_FALSE,        NV_FALSE,      pHdacodec        )
+#endif
+#if GPU_CHILD_MODULE(LPWR)
+    GPU_CHILD_SINGLE_INST( Lpwr,              GPU_GET_LPWR,                        1,                NV_FALSE,        NV_FALSE,      pLpwr            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_FIFO)
+    GPU_CHILD_SINGLE_INST( KernelFifo,           GPU_GET_KERNEL_FIFO_UC,              1,                NV_FALSE,        NV_FALSE,      pKernelFifo      )
+#endif
+#if GPU_CHILD_MODULE(FIFO)
+    GPU_CHILD_SINGLE_INST( OBJFIFO,              GPU_GET_FIFO_UC,                     1,                NV_FALSE,        NV_FALSE,      pFifo            )
+#endif
+#if GPU_CHILD_MODULE(INFOROM)
+    GPU_CHILD_SINGLE_INST( OBJINFOROM,           GPU_GET_INFOROM,                     1,                NV_FALSE,        NV_TRUE,       pInforom         )
+#endif
+#if GPU_CHILD_MODULE(PMGR)
+    GPU_CHILD_SINGLE_INST( Pmgr,                 GPU_GET_PMGR,                        1,                NV_FALSE,        NV_FALSE,      pPmgr            )
+#endif
+#if GPU_CHILD_MODULE(UVM)
+    GPU_CHILD_SINGLE_INST( OBJUVM,               GPU_GET_UVM,                         1,                NV_FALSE,        NV_FALSE,      pUvm             )
+#endif
+#if GPU_CHILD_MODULE(NV_DEBUG_DUMP)
+    GPU_CHILD_SINGLE_INST( NvDebugDump,          GPU_GET_NVD,                         1,                NV_FALSE,        NV_TRUE,       pNvd             )
+#endif
+#if GPU_CHILD_MODULE(GRDBG)
+    GPU_CHILD_SINGLE_INST( SMDebugger,           GPU_GET_GRDBG,                       1,                NV_FALSE,        NV_TRUE,       pGrdbg           )
+#endif
+#if GPU_CHILD_MODULE(SEC2)
+    GPU_CHILD_SINGLE_INST( OBJSEC2,              GPU_GET_SEC2,                        1,                NV_FALSE,        NV_FALSE,      pSec2            )
+#endif
+#if GPU_CHILD_MODULE(LSFM)
+    GPU_CHILD_SINGLE_INST( OBJLSFM,              GPU_GET_LSFM,                        1,                NV_FALSE,        NV_FALSE,      pLsfm            )
+#endif
+#if GPU_CHILD_MODULE(ACR)
+    GPU_CHILD_SINGLE_INST( OBJACR,               GPU_GET_ACR,                         1,                NV_FALSE,        NV_FALSE,      pAcr             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_NVLINK)
+    GPU_CHILD_SINGLE_INST( KernelNvlink,         GPU_GET_KERNEL_NVLINK,               1,                NV_FALSE,        NV_FALSE,      pKernelNvlink    )
+#endif
+#if GPU_CHILD_MODULE(NVLINK)
+    GPU_CHILD_SINGLE_INST( Nvlink,               GPU_GET_NVLINK,                      1,                NV_FALSE,        NV_FALSE,      pNvLink          )
+#endif
+#if GPU_CHILD_MODULE(GPULOG)
+    GPU_CHILD_SINGLE_INST( OBJGPULOG,            GPU_GET_GPULOG,                      1,                NV_FALSE,        NV_TRUE,       pGpuLog          )
+#endif
+#if GPU_CHILD_MODULE(GPUMON)
+    GPU_CHILD_SINGLE_INST( OBJGPUMON,            GPU_GET_GPUMON,                      1,                NV_FALSE,        NV_TRUE,       pGpuMon          )
+#endif
+#if GPU_CHILD_MODULE(HWPM)
+    GPU_CHILD_SINGLE_INST( OBJHWPM,              GPU_GET_HWPM,                        1,                NV_FALSE,        NV_FALSE,      pHwpm            )
+#endif
+#if GPU_CHILD_MODULE(GRIDDISPLAYLESS)
+    GPU_CHILD_SINGLE_INST( OBJGRIDDISPLAYLESS,   GPU_GET_GRIDDISPLAYLESS,             1,                NV_FALSE,        NV_FALSE,      pGridDisplayless )
+#endif
+#if GPU_CHILD_MODULE(SWENG)
+    GPU_CHILD_SINGLE_INST( OBJSWENG,             GPU_GET_SWENG,                       1,                NV_FALSE,        NV_FALSE,      pSwEng           )
+#endif
+#if GPU_CHILD_MODULE(VMMU)
+    GPU_CHILD_SINGLE_INST( OBJVMMU,              GPU_GET_VMMU,                        1,                NV_FALSE,        NV_FALSE,      pVmmu            )
+#endif
+#if GPU_CHILD_MODULE(NVJPG)
+    GPU_CHILD_MULTI_INST( OBJNVJPG,              GPU_GET_NVJPG,                       GPU_MAX_NVJPGS,   NV_FALSE,        NV_FALSE,      pNvjpg           )
+#endif
+#if GPU_CHILD_MODULE(GSP)
+    GPU_CHILD_SINGLE_INST( Gsp,                  GPU_GET_GSP,                         1,                NV_FALSE,        NV_FALSE,      pGsp             )
+#endif
+#if GPU_CHILD_MODULE(OFA)
+    GPU_CHILD_SINGLE_INST( OBJOFA,               GPU_GET_OFA,                         1,                NV_FALSE,        NV_FALSE,      pOfa             )
+#endif
+
+// Undefine the entry macros to simplify call sites
+#undef GPU_CHILD
+#undef GPU_CHILD_SINGLE_INST
+#undef GPU_CHILD_MULTI_INST
+#undef GPU_CHILD_MODULE
+#undef GPU_CHILD_LIST_DISABLED_ONLY

src/nvidia/inc/kernel/gpu/gpu_device_mapping.h

@@ -0,0 +1,62 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_DEVICE_MAPPING_H_
+#define _GPU_DEVICE_MAPPING_H_
+
+// Defines the enum type DEVICE_INDEX used for identifying the device type being accessed
+typedef enum
+{
+    DEVICE_INDEX_GPU = 0,
+    DEVICE_INDEX_HOST1X,
+    DEVICE_INDEX_DISPLAY,
+    DEVICE_INDEX_DPAUX,
+    DEVICE_INDEX_MC,
+    DEVICE_INDEX_CLKRST,
+    DEVICE_INDEX_MSS_NVLINK,
+    DEVICE_INDEX_HDACODEC,
+    DEVICE_INDEX_EMC,
+    DEVICE_INDEX_FUSE,
+    DEVICE_INDEX_KFUSE,
+    DEVICE_INDEX_MIPICAL,
+    DEVICE_INDEX_MAX  //Should always be the last entry
+} DEVICE_INDEX;
+
+typedef enum
+{
+    SOC_DEV_MAPPING_DISP = 0,
+    SOC_DEV_MAPPING_DPAUX0,
+    SOC_DEV_MAPPING_DPAUX1,  // Update NV_MAX_SOC_DPAUX_NUM_DEVICES if adding new DPAUX mappings
+    SOC_DEV_MAPPING_HDACODEC,
+    SOC_DEV_MAPPING_MIPICAL,
+    SOC_DEV_MAPPING_MAX       // Keep this as last entry
+} SOC_DEV_MAPPING;
+
+#define GPU_MAX_DEVICE_MAPPINGS                   (60)
+
+typedef struct
+{
+    DEVICE_INDEX deviceIndex;    // DEVICE_INDEX_*
+    NvU32        devId;          // NV_DEVID_*
+} DEVICE_ID_MAPPING;
+
+#endif // _GPU_DEVICE_MAPPING_H_

src/nvidia/inc/kernel/gpu/gpu_halspec.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_halspec_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_resource.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_resource_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_resource_desc.h

@@ -0,0 +1,37 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_RESOURCE_DESC_H_
+#define _GPU_RESOURCE_DESC_H_
+
+#include "gpu/eng_desc.h"
+
+typedef struct GPU_RESOURCE_DESC
+{
+    NvU32          externalClassId;
+    ENGDESCRIPTOR  engDesc;
+} GPU_RESOURCE_DESC;
+
+// CLASSDESCRIPTOR is deprecated, please use GPU_RESOURCE_DESC
+typedef struct GPU_RESOURCE_DESC CLASSDESCRIPTOR, *PCLASSDESCRIPTOR;
+
+#endif // _GPU_RESOURCE_DESC_H_

src/nvidia/inc/kernel/gpu/gpu_timeout.h

@@ -0,0 +1,144 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_TIMEOUT_H_
+#define _GPU_TIMEOUT_H_
+
+/* ------------------------ Includes ---------------------------------------- */
+#include "core/core.h"
+
+
+/* ------------------------ Forward Definitions ----------------------------- */
+struct OBJGPU;
+
+/* ------------------------ Macros ------------------------------------------ */
+/*!
+ * @note    GPU_TIMEOUT_DEFAULT is different per platform and can range anywhere
+ *          from 2 to 30 secs depending on the GPU Mode and Platform.
+ *          By default if GPU_TIMEOUT_DEFAULT is specified, we use the ThreadState
+ *          unless explicitly told not to via GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE.
+ */
+#define GPU_TIMEOUT_DEFAULT                 0
+
+/*!
+ * gpuSetTimeout Flags - saved in pTimeout->flags
+ */
+#define GPU_TIMEOUT_FLAGS_DEFAULT               NVBIT(0)  //!< default timeout mechanism as set by platform
+#define GPU_TIMEOUT_FLAGS_USE_THREAD_STATE      NVBIT(1)  //!< default timeout time used - use the ThreadState
+#define GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE   NVBIT(2)  //!< even if default time was used - skip the ThreadState
+#define GPU_TIMEOUT_FLAGS_OSTIMER               NVBIT(3)  //!< osGetCurrentTime()
+#define GPU_TIMEOUT_FLAGS_OSDELAY               NVBIT(4)  //!< osDelay()
+#define GPU_TIMEOUT_FLAGS_TMR                   NVBIT(5)  //!< tmrGetCurrentTime()
+#define GPU_TIMEOUT_FLAGS_BYPASS_JOURNAL_LOG    NVBIT(6)  //!< bypass timeout logging in the RM journal
+#define GPU_TIMEOUT_FLAGS_TMRDELAY              NVBIT(7)  //!< tmrDelay()
+#define GPU_TIMEOUT_FLAGS_BYPASS_CPU_YIELD      NVBIT(8)  //!< don't explicitly let other threads run first
+/*!
+ * gpuCheckTimeout Flags set in pTimeout->flags upon NV_ERR_TIMEOUT
+ */
+#define GPU_TIMEOUT_FLAGS_STATUS_LOCAL_TIMEOUT          NVBIT(30)
+#define GPU_TIMEOUT_FLAGS_STATUS_THREAD_STATE_TIMEOUT   NVBIT(31)
+
+/* ------------------------ Datatypes --------------------------------------- */
+/*!
+ * Timeout support.
+ */
+typedef struct
+{
+    NvU64   timeout;
+    NvU32   flags;
+    OBJGPU *pTmrGpu;        //!< The GPU whose timer is used in SLI mode
+                            //   Defined only if flags is set to _TMR or _TMRDELAY
+} RMTIMEOUT,
+*PRMTIMEOUT;
+
+/*!
+ * @brief   GPU timeout related data.
+ */
+typedef struct
+{
+    volatile NvBool bDefaultOverridden;
+    volatile NvBool bScaled;
+    volatile NvU32  defaultus;          //!< Default timeout in us
+    volatile NvU32  defaultResetus;     //!< Default timeout reset value in us
+    NvU32           defaultFlags;       //!< Default timeout mode
+    NvU32           scale;              //!< Emulation/Simulation multiplier
+    OBJGPU         *pGpu;
+} TIMEOUT_DATA;
+
+/*!
+ * @brief   A prototype of the condition evaluation function required by the
+ *          @ref gpuTimeoutCondWait_IMPL interface.
+ *
+ * @note    Function is responsible for evaluation of the encapsulated condition
+ *          as well as for triggering of required prerequisites (if any).
+ *          For example if condition depends on a PMU issued message function
+ *          should assure proper servicing of the PMU interrupts.
+ *
+ * @param[in] pGpu    OBJGPU pointer for this conditional function
+ * @param[in] pVoid
+ *     Void parameter pointer which can be used to pass in the
+ *     pCondData from @ref gpuTimeoutCondWait_IMPL().
+ *
+ * @return NV_TRUE
+ *     Waited condition has happened and @ref
+ *     gpuTimeoutCondWait_IMPL() may return to caller.
+ * @return NV_FALSE
+ *     Waited condition has not happened and @ref
+ *     gpuTimeoutCondWait_IMPL() should continue to wait until this
+ *     interface returns NV_TRUE or timeout occurs (whichever occurs
+ *     first).
+ */
+typedef NvBool GpuWaitConditionFunc(OBJGPU *pGpu, void *pVoid);
+
+/* ------------------------ Function Prototypes ----------------------------- */
+
+void timeoutInitializeGpuDefault(TIMEOUT_DATA *pTD, OBJGPU *pGpu);
+
+void timeoutRegistryOverride(TIMEOUT_DATA *pTD, OBJGPU *pGpu);
+
+void timeoutOverride(TIMEOUT_DATA *pTD, NvBool bOverride, NvU32 timeoutMs);
+
+/*! Initialize the RMTIMEOUT structure with the selected timeout scheme. */
+void timeoutSet(TIMEOUT_DATA *, RMTIMEOUT *, NvU32 timeoutUs, NvU32 flags);
+
+/*! Check if the passed in RMTIMEOUT struct has expired. */
+NV_STATUS timeoutCheck(TIMEOUT_DATA *, RMTIMEOUT *, NvU32);
+
+/*! Wait for the condition to become satisfied while checking for the timeout */
+NV_STATUS timeoutCondWait(TIMEOUT_DATA *, RMTIMEOUT *, GpuWaitConditionFunc *, void *pCondData, NvU32);
+
+/*! Scales timeout values depending on the environment we are running in. */
+static NV_INLINE NvU32 timeoutApplyScale(TIMEOUT_DATA *pTD, NvU32 timeout)
+{
+    return timeout * pTD->scale;
+}
+
+
+// Deprecated macros
+#define gpuSetTimeout(g,a,t,c)       timeoutSet(&(g)->timeoutData, t, a, c)
+#define gpuCheckTimeout(g,t)         timeoutCheck(&(g)->timeoutData, t, __LINE__)
+#define gpuScaleTimeout(g,a)         timeoutApplyScale(&(g)->timeoutData, a)
+#define gpuTimeoutCondWait(g,a,b,t)  timeoutCondWait(&(g)->timeoutData, t, a, b, __LINE__)
+
+#define GPU_ENG_RESET_TIMEOUT_VALUE(g, t) (t)
+
+#endif // _GPU_TIMEOUT_H_

src/nvidia/inc/kernel/gpu/gpu_uuid.h

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2018-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPUUUID_H_
+#define _GPUUUID_H_
+
+#include "core/core.h"
+#include "nvCpuUuid.h"
+
+//
+// GPU unique ID sizes.  RM_SHA1_GID_SIZE uses the first 16 bytes of
+// the SHA-1 digest (this is consistent with the way canonical UUIDs are
+// constructed)
+//
+#define RM_SHA1_GID_SIZE         16
+
+// UUID conversion routine:
+NV_STATUS transformGidToUserFriendlyString(const NvU8 *pGidData, NvU32 gidSize, NvU8 **ppGidString,
+                                           NvU32 *pGidStrlen, NvU32 gidFlags);
+
+NV_STATUS nvGenerateGpuUuid(NvU16 chipId, NvU64 pdi, NvUuid *pUuid);
+
+NV_STATUS nvGenerateSmcUuid(NvU16 chipId, NvU64 pdi,
+                            NvU32 swizzId, NvU32 syspipeId, NvUuid *pUuid);
+
+// 'G' 'P' 'U' '-'(x5), '\0x0', extra = 9
+#define NV_UUID_STR_LEN ((NV_UUID_LEN << 1) + 9)
+
+void nvGetSmcUuidString(const NvUuid *pUuid, char *pUuidStr);
+
+void nvGetGpuUuidString(const NvUuid *pUuid, char *pUuidStr);
+
+#endif // _GPUUUID_H_

src/nvidia/inc/kernel/gpu/gr/fecs_event_list.h

@@ -0,0 +1,3 @@
+
+#include "g_fecs_event_list_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_context.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_context_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_manager.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_manager_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_object.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_object_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_sm_debugger_session.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_sm_debugger_session_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GSP_INIT_ARGS_H
+#define GSP_INIT_ARGS_H
+
+#include "core/core.h"
+
+typedef struct {
+    RmPhysAddr cmdQueuePhysAddr;
+} GSP_RMFS_INIT_ARGUMENTS;
+
+typedef struct {
+    RmPhysAddr sharedMemPhysAddr;
+    NvU32 pageTableEntryCount;
+    NvLength cmdQueueOffset;
+    NvLength statQueueOffset;
+} MESSAGE_QUEUE_INIT_ARGUMENTS;
+
+typedef struct {
+    NvU32 oldLevel;
+    NvU32 flags;
+    NvBool bInPMTransition;
+} GSP_SR_INIT_ARGUMENTS;
+
+/*!
+ * (Cached) GSP fw RM initialization arguments.
+ */
+typedef struct
+{
+    MESSAGE_QUEUE_INIT_ARGUMENTS      messageQueueInitArguments;
+    GSP_SR_INIT_ARGUMENTS             srInitArguments;
+} GSP_ARGUMENTS_CACHED;
+
+#endif // GSP_INIT_ARGS_H

src/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h

@@ -0,0 +1,145 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GSP_STATIC_CONFIG_H
+#define GSP_STATIC_CONFIG_H
+
+//
+// This header describes the set of static GPU configuration information
+// that is collected during GSP RM init and made available to the
+// CPU RM (aka GSP client) via the NV_RM_RPC_GET_STATIC_INFO() and
+// NV_RM_RPC_GET_GSP_STATIC_INFO() calls.
+
+#include "ctrl/ctrl0080/ctrl0080gpu.h"
+#include "ctrl/ctrl0080/ctrl0080gr.h"
+#include "ctrl/ctrl2080/ctrl2080bios.h"
+#include "ctrl/ctrl2080/ctrl2080fb.h"
+#include "ctrl/ctrl2080/ctrl2080gpu.h"
+
+#include "gpu/gpu.h" // COMPUTE_BRANDING_TYPE
+#include "vgpu/rpc_headers.h" // MAX_GPC_COUNT
+#include "platform/chipset/chipset.h" // BUSINFO
+
+typedef struct GspSMInfo_t
+{
+    NvU32 version;
+    NvU32 regBankCount;
+    NvU32 regBankRegCount;
+    NvU32 maxWarpsPerSM;
+    NvU32 maxThreadsPerWarp;
+    NvU32 geomGsObufEntries;
+    NvU32 geomXbufEntries;
+    NvU32 maxSPPerSM;
+    NvU32 rtCoreCount;
+} GspSMInfo;
+
+// Fetched from GSP-RM into CPU-RM
+typedef struct GspStaticConfigInfo_t
+{
+    NvU8 grCapsBits[NV0080_CTRL_GR_CAPS_TBL_SIZE];
+    NV2080_CTRL_GPU_GET_GID_INFO_PARAMS gidInfo;
+    NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS gpcInfo;
+    NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS tpcInfo[MAX_GPC_COUNT];
+    NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS zcullInfo[MAX_GPC_COUNT];
+    NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS SKUInfo;
+    NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS fbRegionInfoParams;
+    COMPUTE_BRANDING_TYPE computeBranding;
+
+    NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS sriovCaps;
+    NvU32 sriovMaxGfid;
+
+    NvU64 engineCaps;
+
+    GspSMInfo SM_info;
+
+    NvBool poisonFuseEnabled;
+  
+    NvU64 fb_length;
+    NvU32 fbio_mask;
+    NvU32 fb_bus_width;
+    NvU32 fb_ram_type;
+    NvU32 fbp_mask;
+    NvU32 l2_cache_size;
+
+    NvU32 gfxpBufferSize[NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL];
+    NvU32 gfxpBufferAlignment[NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL];
+
+    NvU8 gpuNameString[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvU8 gpuShortNameString[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvU16 gpuNameString_Unicode[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvBool bGpuInternalSku;
+    NvBool bIsQuadroGeneric;
+    NvBool bIsQuadroAd;
+    NvBool bIsNvidiaNvs;
+    NvBool bIsVgx;
+    NvBool bGeforceSmb;
+    NvBool bIsTitan;
+    NvBool bIsTesla;
+
+    NvU64 bar1PdeBase;
+    NvU64 bar2PdeBase;
+
+    NvBool bVbiosValid;
+    NvU32 vbiosSubVendor;
+    NvU32 vbiosSubDevice;
+
+    NvBool bPageRetirementSupported;
+
+    NvBool bSplitVasBetweenServerClientRm;
+
+    NvBool bClRootportNeedsNosnoopWAR;
+
+    VIRTUAL_DISPLAY_GET_NUM_HEADS_PARAMS displaylessMaxHeads;
+    VIRTUAL_DISPLAY_GET_MAX_RESOLUTION_PARAMS displaylessMaxResolution;
+    NvU64 displaylessMaxPixels;
+
+    // Client handle for internal RMAPI control.
+    NvHandle hInternalClient;
+
+    // Device handle for internal RMAPI control.
+    NvHandle hInternalDevice;
+
+    // Subdevice handle for internal RMAPI control.
+    NvHandle hInternalSubdevice;
+} GspStaticConfigInfo;
+
+// Pushed from CPU-RM to GSP-RM
+typedef struct GspSystemInfo
+{
+    NvU64 gpuPhysAddr;
+    NvU64 gpuPhysFbAddr;
+    NvU64 gpuPhysInstAddr;
+    NvU64 nvDomainBusDeviceFunc;
+    NvU64 simAccessBufPhysAddr;
+    NvU64 pcieAtomicsOpMask;
+    NvU64 consoleMemSize;
+    NvU32 pciConfigMirrorBase;
+    NvU32 pciConfigMirrorSize;
+    NvU8 oorArch;
+    NvU64 clPdbProperties;
+    NvU32 Chipset;
+    BUSINFO FHBBusInfo;
+} GspSystemInfo;
+
+
+#endif /* GSP_STATIC_CONFIG_H */

src/nvidia/inc/kernel/gpu/gsp/kernel_gsp.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_gsp_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gsp/message_queue.h

@@ -0,0 +1,39 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+/*
+ * GSP MESSAGE QUEUE
+ */
+
+#ifndef _MESSAGE_QUEUE_H_
+#define _MESSAGE_QUEUE_H_
+
+typedef struct _message_queue_info MESSAGE_QUEUE_INFO;
+
+// CPU-side calls
+NV_STATUS GspMsgQueueInit(OBJGPU *pGpu, MESSAGE_QUEUE_INFO **ppMQI);
+NV_STATUS GspStatusQueueInit(OBJGPU *pGpu, MESSAGE_QUEUE_INFO **ppMQI);
+void GspMsgQueueCleanup(MESSAGE_QUEUE_INFO **ppMQI);
+NV_STATUS GspMsgQueueSendCommand(MESSAGE_QUEUE_INFO *pMQI, OBJGPU *pGpu);
+NV_STATUS GspMsgQueueReceiveStatus(MESSAGE_QUEUE_INFO *pMQI);
+
+#endif // _MESSAGE_QUEUE_H_

src/nvidia/inc/kernel/gpu/gsp/message_queue_priv.h

@@ -0,0 +1,95 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * GSP MESSAGE QUEUE
+ */
+
+#ifndef _MESSAGE_QUEUE_PRIV_H_
+#define _MESSAGE_QUEUE_PRIV_H_
+
+#include "msgq/msgq.h"
+#include "gpu/mem_mgr/virt_mem_allocator_common.h"
+
+// Shared memory layout.
+//
+// Each of the following are page aligned:
+//   Page table for entire shared memory layout.
+//   Command queue header
+//   Command queue entries
+//   Status queue header
+//   Status queue entries
+
+typedef struct GSP_MSG_QUEUE_ELEMENT
+{
+    NvU32 checkSum;        // Set to value needed to make checksum always zero.
+    NvU32 seqNum;          // Sequence number maintained by the message queue.
+    rpc_message_header_v  rpc;
+} GSP_MSG_QUEUE_ELEMENT;
+
+typedef struct _message_queue_info
+{
+    // Parameters
+    NvLength               pageTableEntryCount;
+    NvLength               pageTableSize;
+    NvLength               commandQueueSize;
+    NvLength               statusQueueSize;
+
+    // Shared memory area.
+    MEMORY_DESCRIPTOR     *pSharedMemDesc;
+    RmPhysAddr             sharedMemPA;   // Page table for all of shared mem.
+    void                  *pCommandQueue;
+    void                  *pStatusQueue;
+    rpc_message_header_v  *pRpcMsgBuf;    // RPC message buffer VA.
+
+    void                  *pInitMsgBuf;   // RPC message buffer VA.
+    RmPhysAddr             initMsgBufPA;  // RPC message buffer PA.
+
+    // Other CPU-side fields
+    void                  *pWorkArea;
+    GSP_MSG_QUEUE_ELEMENT *pCmdQueueElement;    // Working copy of command queue element.
+    void                  *pMetaData;
+    msgqHandle             hQueue;              // Do not allow requests when hQueue is null.
+    NvU32                  txSeqNum;            // Next sequence number for tx.
+    NvU32                  rxSeqNum;            // Next sequence number for rx.
+} MESSAGE_QUEUE_INFO;
+
+//
+// Most of the following defines resolve to compile-time constants.
+//
+#define GSP_MSG_QUEUE_ELEMENT_SIZE_MIN                              RM_PAGE_SIZE
+#define GSP_MSG_QUEUE_ELEMENT_SIZE_MAX     (GSP_MSG_QUEUE_ELEMENT_SIZE_MIN * 16)
+#define GSP_MSG_QUEUE_ELEMENT_HDR_SIZE   NV_OFFSETOF(GSP_MSG_QUEUE_ELEMENT, rpc)
+
+#define GSP_MSG_QUEUE_RPC_SIZE_MAX                                             \
+    (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX - GSP_MSG_QUEUE_ELEMENT_HDR_SIZE)
+
+#define GSP_MSG_QUEUE_BYTES_TO_ELEMENTS(b)                                     \
+        NV_DIV_AND_CEIL(b, GSP_MSG_QUEUE_ELEMENT_SIZE_MIN)
+
+#define GSP_MSG_QUEUE_ALIGN                                        RM_PAGE_SHIFT   // 2 ^ 12 = 4096
+#define GSP_MSG_QUEUE_ELEMENT_ALIGN                                RM_PAGE_SHIFT   // 2 ^ 12 = 4096
+#define GSP_MSG_QUEUE_HEADER_SIZE                                   RM_PAGE_SIZE
+#define GSP_MSG_QUEUE_HEADER_ALIGN                                             4   // 2 ^ 4 = 16
+
+#endif // _MESSAGE_QUEUE_PRIV_H_

src/nvidia/inc/kernel/gpu/host_eng/host_eng.h

@@ -0,0 +1,3 @@
+
+#include "g_host_eng_nvoc.h"
+

src/nvidia/inc/kernel/gpu/hwpm/profiler_v1.h

@@ -0,0 +1,3 @@
+
+#include "g_profiler_v1_nvoc.h"
+

src/nvidia/inc/kernel/gpu/hwpm/profiler_v2.h

@@ -0,0 +1,3 @@
+
+#include "g_profiler_v2_nvoc.h"
+

src/nvidia/inc/kernel/gpu/i2c/i2c_api.h

@@ -0,0 +1,3 @@
+
+#include "g_i2c_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/engine_idx.h

@@ -0,0 +1,164 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ENGINE_IDX_H
+#define ENGINE_IDX_H
+
+#include "utils/nvbitvector.h"
+
+/***************************************************************************\
+*                                                                           *
+* Module: engine_idx.h
+*       List of engines for use by INTR (and MC) modules.
+*                                                                           *
+\***************************************************************************/
+
+//
+// Engine bits for use by various MC HAL routines
+//
+#define MC_ENGINE_IDX_NULL                          0 // This must be 0
+#define MC_ENGINE_IDX_TMR                           1
+#define MC_ENGINE_IDX_DISP                          2
+#define MC_ENGINE_IDX_FB                            3
+#define MC_ENGINE_IDX_FIFO                          4
+#define MC_ENGINE_IDX_VIDEO                         5
+#define MC_ENGINE_IDX_MD                            6
+#define MC_ENGINE_IDX_BUS                           7
+// UNUSED
+#define MC_ENGINE_IDX_PMGR                          9
+#define MC_ENGINE_IDX_VP2                           10
+#define MC_ENGINE_IDX_CIPHER                        11
+#define MC_ENGINE_IDX_BIF                           12
+#define MC_ENGINE_IDX_PPP                           13
+#define MC_ENGINE_IDX_PRIVRING                      14
+#define MC_ENGINE_IDX_PMU                           15
+#define MC_ENGINE_IDX_CE0                           16
+#define MC_ENGINE_IDX_CE1                           17
+#define MC_ENGINE_IDX_CE2                           18
+#define MC_ENGINE_IDX_CE3                           19
+#define MC_ENGINE_IDX_CE4                           20
+#define MC_ENGINE_IDX_CE5                           21
+#define MC_ENGINE_IDX_CE6                           22
+#define MC_ENGINE_IDX_CE7                           23
+#define MC_ENGINE_IDX_CE8                           24
+#define MC_ENGINE_IDX_CE9                           25
+#define MC_ENGINE_IDX_VIC                           26
+#define MC_ENGINE_IDX_ISOHUB                        27
+#define MC_ENGINE_IDX_VGPU                          28
+#define MC_ENGINE_IDX_MSENC                         29
+#define MC_ENGINE_IDX_MSENC1                        30
+#define MC_ENGINE_IDX_MSENC2                        31
+#define MC_ENGINE_IDX_C2C                           32
+// UNUSED
+#define MC_ENGINE_IDX_LTC                           34
+#define MC_ENGINE_IDX_FBHUB                         35
+#define MC_ENGINE_IDX_HDACODEC                      36
+#define MC_ENGINE_IDX_GMMU                          37
+#define MC_ENGINE_IDX_SEC2                          38
+#define MC_ENGINE_IDX_FSP                           39
+#define MC_ENGINE_IDX_NVLINK                        40
+#define MC_ENGINE_IDX_GSP                           41
+#define MC_ENGINE_IDX_NVJPG                         42
+#define MC_ENGINE_IDX_NVJPEG                        MC_ENGINE_IDX_NVJPG
+#define MC_ENGINE_IDX_NVJPEG0                       MC_ENGINE_IDX_NVJPEG
+#define MC_ENGINE_IDX_RESERVED43                    43
+#define MC_ENGINE_IDX_RESERVED44                    44
+#define MC_ENGINE_IDX_RESERVED45                    45
+#define MC_ENGINE_IDX_RESERVED46                    46
+#define MC_ENGINE_IDX_RESERVED47                    47
+#define MC_ENGINE_IDX_RESERVED48                    48
+#define MC_ENGINE_IDX_RESERVED49                    49
+#define MC_ENGINE_IDX_REPLAYABLE_FAULT              50
+#define MC_ENGINE_IDX_ACCESS_CNTR                   51
+#define MC_ENGINE_IDX_NON_REPLAYABLE_FAULT          52
+#define MC_ENGINE_IDX_REPLAYABLE_FAULT_ERROR        53
+#define MC_ENGINE_IDX_NON_REPLAYABLE_FAULT_ERROR    54
+#define MC_ENGINE_IDX_INFO_FAULT                    55
+#define MC_ENGINE_IDX_BSP                           56
+#define MC_ENGINE_IDX_NVDEC                         MC_ENGINE_IDX_BSP
+#define MC_ENGINE_IDX_NVDEC0                        MC_ENGINE_IDX_NVDEC
+#define MC_ENGINE_IDX_NVDEC1                        57
+#define MC_ENGINE_IDX_NVDEC2                        58
+#define MC_ENGINE_IDX_NVDEC3                        59
+#define MC_ENGINE_IDX_NVDEC4                        60
+#define MC_ENGINE_IDX_RESERVED61                    61
+#define MC_ENGINE_IDX_RESERVED62                    62
+#define MC_ENGINE_IDX_RESERVED63                    63
+#define MC_ENGINE_IDX_CPU_DOORBELL                  64
+#define MC_ENGINE_IDX_PRIV_DOORBELL                 65
+#define MC_ENGINE_IDX_MMU_ECC_ERROR                 66
+#define MC_ENGINE_IDX_BLG                           67
+#define MC_ENGINE_IDX_PERFMON                       68
+#define MC_ENGINE_IDX_BUF_RESET                     69
+#define MC_ENGINE_IDX_XBAR                          70
+#define MC_ENGINE_IDX_ZPW                           71
+#define MC_ENGINE_IDX_OFA0                          72
+#define MC_ENGINE_IDX_TEGRA                         73
+#define MC_ENGINE_IDX_GR                            74
+#define MC_ENGINE_IDX_GR0                           MC_ENGINE_IDX_GR
+#define MC_ENGINE_IDX_GR1                           75
+#define MC_ENGINE_IDX_GR2                           76
+#define MC_ENGINE_IDX_GR3                           77
+#define MC_ENGINE_IDX_GR4                           78
+#define MC_ENGINE_IDX_GR5                           79
+#define MC_ENGINE_IDX_GR6                           80
+#define MC_ENGINE_IDX_GR7                           81
+#define MC_ENGINE_IDX_ESCHED                        82
+#define MC_ENGINE_IDX_ESCHED__SIZE                  64
+#define MC_ENGINE_IDX_GR_FECS_LOG                   146
+#define MC_ENGINE_IDX_GR0_FECS_LOG                  MC_ENGINE_IDX_GR_FECS_LOG
+#define MC_ENGINE_IDX_GR1_FECS_LOG                  147
+#define MC_ENGINE_IDX_GR2_FECS_LOG                  148
+#define MC_ENGINE_IDX_GR3_FECS_LOG                  149
+#define MC_ENGINE_IDX_GR4_FECS_LOG                  150
+#define MC_ENGINE_IDX_GR5_FECS_LOG                  151
+#define MC_ENGINE_IDX_GR6_FECS_LOG                  152
+#define MC_ENGINE_IDX_GR7_FECS_LOG                  153
+#define MC_ENGINE_IDX_TMR_SWRL                      154
+#define MC_ENGINE_IDX_MAX                           155 // This must be kept as the max bit if
+                                                        // we need to add more engines
+#define MC_ENGINE_IDX_INVALID                0xFFFFFFFF
+
+// Index GR reference
+#define MC_ENGINE_IDX_GRn(x)            (MC_ENGINE_IDX_GR0 + (x))
+#define MC_ENGINE_IDX_GRn_FECS_LOG(x)   (MC_ENGINE_IDX_GR0_FECS_LOG + (x))
+
+// Index CE reference
+#define MC_ENGINE_IDX_CE(x)             (MC_ENGINE_IDX_CE0 + (x))
+
+// Index MSENC reference
+#define MC_ENGINE_IDX_MSENCn(x)         (MC_ENGINE_IDX_MSENC + (x))
+
+// Index NVDEC reference
+#define MC_ENGINE_IDX_NVDECn(x)         (MC_ENGINE_IDX_NVDEC + (x))
+
+// Index NVJPEG reference
+#define MC_ENGINE_IDX_NVJPEGn(x)        (MC_ENGINE_IDX_NVJPEG + (x))
+
+// Index ESCHED reference
+#define MC_ENGINE_IDX_ESCHEDn(x)        (MC_ENGINE_IDX_ESCHED + (x))
+
+MAKE_BITVECTOR(MC_ENGINE_BITVECTOR, MC_ENGINE_IDX_MAX);
+typedef MC_ENGINE_BITVECTOR *PMC_ENGINE_BITVECTOR;
+
+#endif // ENGINE_IDX_H

src/nvidia/inc/kernel/gpu/intr/intr.h

@@ -0,0 +1,3 @@
+
+#include "g_intr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/intr_service.h

@@ -0,0 +1,3 @@
+
+#include "g_intr_service_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/swintr.h

@@ -0,0 +1,3 @@
+
+#include "g_swintr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intrable/intrable.h

@@ -0,0 +1,3 @@
+
+#include "g_intrable_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mc/kernel_mc.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_mc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/context_dma.h

@@ -0,0 +1,3 @@
+
+#include "g_context_dma_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/fbsr.h

@@ -0,0 +1,3 @@
+
+#include "g_fbsr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/fermi_dma.h

@@ -0,0 +1,65 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FERMI_DMA_H
+#define FERMI_DMA_H
+
+#ifdef NV_MMU_PTE
+#error "dev_mmu.h included before fermi_dma.h"
+#endif
+#ifdef NV_PLTCG
+#error "dev_ltc.h included before fermi_dma.h"
+#endif
+
+#include "gpu/mem_mgr/mem_desc.h"
+#include "mem_mgr/gpu_vaspace.h"
+#include "gpu/mem_mgr/virt_mem_allocator.h"
+
+// TODO -- replace those FERMI_* define usages in RM code
+#define FERMI_SMALL_PAGESIZE    RM_PAGE_SIZE
+#define FERMI_BIG_PAGESIZE_64K  RM_PAGE_SIZE_64K
+#define FERMI_BIG_PAGESIZE_128K RM_PAGE_SIZE_128K
+
+typedef struct DMAHALINFO_FERMI
+{
+    NvU32              vasReverse;
+    NvU32              compTagLineMultiplier;     // Comptaglines increment by this value for VERIF only, see Bug 501651
+} DMAHALINFO_FERMI, *PDMAHALINFO_FERMI;
+
+#define DMA_GET_FERMI_INFOBLK(p)    ((PDMAHALINFO_FERMI) getInfoPtr((p)->infoList, HAL_IMPL_GF100))
+
+//
+// From GF100 dev_mmu.ref:
+//    Each PDE maps a 64MB region of virtual memory when using 64KB big pages, or 128MB
+//    when using 128KB big pages.  To map all 40b of virtual address space, the page
+//    directory consists of 16K entries when using 64KB big pages (64MB * 16K = 2^26 * 2^14 = 2^40),
+//    or 8K entries when using 128KB big pages (128MB * 8K = 2^27 * 2^13 = 2^40).
+//
+#define VASPACE_SIZE_MB_FERMI                   (1 << 20)
+#define VASPACE_SIZE_FERMI                      (((NvU64)VASPACE_SIZE_MB_FERMI) << 20)
+
+// Alignment Defines for page tables
+#define PDB_SHIFT_FERMI                         12   // FERMITODO::DMA Will be there in dev_ram very soon
+#define PDB_ALIGNMENT_FERMI                     (1 << PDB_SHIFT_FERMI)
+
+#endif // FERMI_DMA_H

src/nvidia/inc/kernel/gpu/mem_mgr/heap.h

@@ -0,0 +1,3 @@
+
+#include "g_heap_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/heap_base.h

@@ -0,0 +1,162 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _HEAP_BASE_H_
+#define _HEAP_BASE_H_
+
+#include "nvtypes.h"
+#include "core/prelude.h"
+#include "gpu/mem_mgr/mem_desc.h"
+
+// Contains the minimal set of resources used to compute a PTE kind
+typedef struct _def_fb_alloc_page_format
+{
+    NvU32    attr;
+    NvU32    attr2;
+    NvU32    flags;
+    NvU32    kind;
+    NvU32    type;
+} FB_ALLOC_PAGE_FORMAT;
+
+//
+// FB allocation resources structure
+// Need to be allocated from heap
+//
+typedef struct _def_fb_alloc_info
+{
+    NvU32    owner;
+    NvU32    hwResId;
+    NvU32    height;
+    NvU32    width;
+    NvU32    pitch;
+    NvU64    size;
+    NvU64    align;
+    NvU64    alignPad;
+    NvU64    pad;
+    NvU64    offset;
+    NvU32    internalflags;
+    NvU32    retAttr;
+    NvU32    retAttr2;
+    NvU32    format;
+    NvU32    comprCovg;
+    NvU32    zcullCovg;
+    NvU32    uncompressedKind;
+    NvU32    compPageShift;
+    NvU32    compressedKind;
+    NvU32    compTagLineMin;
+    NvU32    compPageIndexLo;
+    NvU32    compPageIndexHi;
+    NvU32    compTagLineMultiplier;
+    NvU32    startCovg;
+    NvU64    origSize;
+    NvU64    adjustedSize;
+    NvU64    desiredOffset;
+
+    FB_ALLOC_PAGE_FORMAT * pageFormat;
+
+    // Tracking client for VGPU
+    NvHandle hClient;
+    NvHandle hDevice;
+
+    // These are only used in Vista
+    // no need yet for possAttr2
+    NvU32    possAttr;          // AllocHint, BindCompr
+    NvU32    ctagOffset;
+
+    // Special flag for kernel allocations
+    NvBool bIsKernelAlloc;
+
+    //
+    // Number of 4KB pages in the PTE array
+    // For contiguous allocation, this will be set to '1'
+    //
+    NvU64 pageCount4k;
+
+    // denote that underlying physical allocation is contiguous or not
+    NvBool bContig;
+
+    //
+    // Store the PTE Array to be used for allocating comptaglines
+    // If the NVOS32_ATTR_PHYSICALITY_CONTIGUOUS is set, it will only have
+    // one entry, otherwise it will have dynamically allocated memory
+    // This will track the pages in 4KB granularity
+    //
+    RmPhysAddr pteArray[1];
+} FB_ALLOC_INFO;
+
+//
+// Contains information on the various hw resources (compr, etc...) that
+// can be associated with a memory allocation.
+//
+typedef struct HWRESOURCE_INFO
+{
+    NvU32 attr;       // NVOS32_ATTR_*
+    NvU32 attr2;      // NVOS32_ATTR2_*
+    NvU32 comprCovg;  // compression coverage
+    NvU32 ctagOffset; // comptag offset
+    NvU32 hwResId;
+    NvU32 refCount;
+    NvBool isVgpuHostAllocated; // used in vGPU guest RM to indicate if this HW resource is allocated by host RM or not. Used in Windows guest.
+    NvBool isGuestAllocated;    // used in vGPU host RM to indicate if this HW resource is allocated from LIST_OBJECT path on behalf of Linux guest.
+} HWRESOURCE_INFO;
+
+
+typedef struct PMA_ALLOC_INFO
+{
+    NvBool bContig;
+    NvU32  pageCount;
+    NvU32  pageSize;
+    NvU32  refCount;
+    NvU64  allocSize;
+    NvU32  flags;
+    //
+    // If bContig is TRUE, this array consists of one element.
+    // If bContig is FALSE, this array is actually larger and
+    // has one entry for each physical page in the allocation.
+    // As a result, this structure must be allocated from heap.
+    //
+    NvU64  pageArray[1];
+    //!!! Place nothing behind pageArray!!!
+} PMA_ALLOC_INFO;
+
+typedef struct MEMORY_ALLOCATION_REQUEST
+{
+    NV_MEMORY_ALLOCATION_PARAMS *pUserParams;
+    OBJGPU                      *pGpu;
+    NvHandle                     hMemory;            // in: can be NULL (translates to 0)
+    NvU32                        internalflags;      // Extended flags ?! flags seem exhausted.
+    HWRESOURCE_INFO             *pHwResource;        // out: data copied in if non-NULL
+    MEMORY_DESCRIPTOR           *pMemDesc;           // in/out: allocate memdesc if NULL
+    PMA_ALLOC_INFO              *pPmaAllocInfo[NV_MAX_SUBDEVICES];      // out: tracks the pre-allocated memory per GPU.
+    NvU32                        classNum;
+    NvHandle                     hClient;
+    NvHandle                     hParent;
+} MEMORY_ALLOCATION_REQUEST;
+
+typedef struct
+{
+    NvU64 address;
+    NvU32 type;
+} BLACKLIST_ADDRESS;
+
+#endif //_HEAP_BASE_H_

src/nvidia/inc/kernel/gpu/mem_mgr/mem_desc.h

@@ -0,0 +1,3 @@
+
+#include "g_mem_desc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/mem_mgr.h

@@ -0,0 +1,3 @@
+
+#include "g_mem_mgr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h

@@ -0,0 +1,252 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*********************** Memory Scrubber Routines **************************\
+*      Defines and structures used for CE Physical Memory Scrubber          *
+\***************************************************************************/
+
+#ifndef MEM_SCRUB_H
+#define MEM_SCRUB_H
+
+#include "nvport/nvport.h"
+#include "class/cl906f.h"
+#include "class/cl906fsw.h"
+
+#include "nvctassert.h"
+#include "vgpu/vgpu_guest_pma_scrubber.h"
+
+struct OBJGPU;
+struct Heap;
+struct OBJCHANNEL;
+
+#define RM_SUBCHANNEL                             0x0
+#define MEMSET_PATTERN                            0x00000000
+#define SCRUBBER_NUM_PAYLOAD_SEMAPHORES           (2)
+#define SCRUBBER_SEMAPHORE_SIZE_INBYTES           (4)
+#define SCRUBBER_CHANNEL_SEMAPHORE_SIZE           (SCRUBBER_SEMAPHORE_SIZE_INBYTES *\
+                                                  SCRUBBER_NUM_PAYLOAD_SEMAPHORES)
+#define SCRUBBER_CHANNEL_NOTIFIER_SIZE            (sizeof(NvNotification) * NV_CHANNELGPFIFO_NOTIFICATION_TYPE__SIZE_1)
+
+#define SCRUBBER_VASPACE_BUFFER_SIZE              0x20000000ULL  //512MB
+#define SIZE_OF_ONE_MEMSET_BLOCK                  0x60
+#define SCRUB_MAX_BYTES_PER_LINE                  0xffffffffULL
+#define MAX_SCRUB_ITEMS                           4096 // 4K scrub items
+
+#define READ_SCRUBBER_PAYLOAD_SEMA(channel)       MEM_RD32((NvU8*)channel->pbCpuVA +\
+                                                  channel->finishPayloadOffset)
+
+#define READ_SCRUBBER_PB_SEMA(channel)            MEM_RD32((NvU8*)channel->pbCpuVA +\
+                                                  channel->semaOffset)
+
+#define WRITE_SCRUBBER_PB_SEMA(channel, val)      MEM_WR32((NvU8*)channel->pbCpuVA +\
+                                                  channel->semaOffset, val);
+
+#define WRITE_SCRUBBER_PAYLOAD_SEMA(channel,val)  MEM_WR32((NvU8*)channel->pbCpuVA +\
+                                                  channel->finishPayloadOffset, val);
+// Use Incrementing Methods to save the PB Space
+#define _NV_ASSERT_CONTIGUOUS_METHODS(a1, a2)     NV_ASSERT((a2) - (a1) == 4)
+
+#define NV_PUSH_METHOD(OpType, SubCh, Method, Count)                                                      \
+                                                  (DRF_DEF(906F, _DMA, _SEC_OP, OpType)                 |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_ADDRESS,    (Method) >> 2) |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_SUBCHANNEL, (SubCh))       |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_COUNT,      (Count)))
+
+#define NV_PUSH_DATA(Data)                        MEM_WR32(pPtr++, (Data))
+
+#define _NV_PUSH_INC_1U(SubCh, a1,d1, Count)                                                                          \
+                                                  do{                                                               \
+                                                        NV_PUSH_DATA(NV_PUSH_METHOD(_INC_METHOD, SubCh, a1, Count));\
+                                                        NV_PUSH_DATA(d1);                                           \
+                                                    } while(0)
+
+#define NV_PUSH_INC_1U(SubCh, a1,d1)                                                      \
+                                                  do{                                     \
+                                                        _NV_PUSH_INC_1U (SubCh, a1,d1, 1);\
+                                                    } while(0)
+
+#define NV_PUSH_INC_2U(SubCh, a1,d1, a2,d2)                                                    \
+                                                  do{                                          \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1, a2);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 2);      \
+                                                        NV_PUSH_DATA(d2);                      \
+                                                    } while(0)
+
+#define NV_PUSH_INC_3U(SubCh, a1,d1, a2,d2, a3,d3)                                            \
+                                                  do{                                         \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 3);     \
+                                                        NV_PUSH_DATA(d2);                     \
+                                                        NV_PUSH_DATA(d3);                     \
+                                                    } while(0)
+
+#define NV_PUSH_INC_4U(SubCh, a1,d1, a2,d2, a3,d3, a4,d4)                                    \
+                                                  do{                                        \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a3,a4);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 4);    \
+                                                        NV_PUSH_DATA(d2);                    \
+                                                        NV_PUSH_DATA(d3);                    \
+                                                        NV_PUSH_DATA(d4);                    \
+                                                    } while(0)
+
+
+// structure to store the details of a scrubbing work
+typedef struct SCRUB_NODE {
+    // The 64 bit ID assigned to each work
+    NvU64      id;
+    // The base address from which the scrub to start
+    NvU64      base;
+    // The size of a scrub work
+    NvU64      size;
+} SCRUB_NODE, *PSCRUB_NODE;
+
+//
+// OBJMEMSCRUB OBJECT
+// Memory scrubber struct encapsulates the CE Channel object,
+// SCRUB_NODE array of size MAX_SCRUB_ITEMS, index to track
+// the scrub work list. The scrubber data structures are
+// synchronized using the mutex pScrubberMutex.
+//
+
+typedef struct OBJMEMSCRUB {
+    // Mutex for Scrubber Object
+    PORT_MUTEX                        *pScrubberMutex;
+    // Last completed work ID communicated to Client
+    NvU64                              lastSeenIdByClient;
+    // The last ID assigned to a work
+    NvU64                              lastSubmittedWorkId;
+    // Last ID checked with the HW scrubber
+    NvU64                              lastSWSemaphoreDone;
+    // Size of the scrub list
+    NvLength                           scrubListSize;
+    // Pre-allocated Free Scrub List
+    PSCRUB_NODE                        pScrubList;
+    // Scrubber Channel
+    struct OBJCHANNEL *pChannel;
+    struct OBJGPU                     *pGpu;
+    VGPU_GUEST_PMA_SCRUB_BUFFER_RING   vgpuScrubBuffRing;
+    NvBool                             bVgpuScrubberEnabled;
+} OBJMEMSCRUB, *POBJMEMSCRUB;
+
+ct_assert(VGPU_GUEST_PMA_MAX_SCRUB_ITEMS == MAX_SCRUB_ITEMS);
+
+/**
+ * Constructs the memory scrubber object and signals
+ * RM to create CE channels for submitting scrubbing work
+ *
+ * @param[in]     pGpu       OBJGPU pointer
+ * @param[in]     pHeap      Heap pointer
+ *
+ * @returns NV_STATUS on success.
+ *          error, if something fails
+ */
+
+NV_STATUS scrubberConstruct(struct OBJGPU *pGpu, struct Heap *pHeap);
+
+/**
+ * Destructs the scrubber
+ *  1. De-registers the scrubber from the PMA object
+ *  2. Free the scrubber list and scrubber lock
+ *
+ * @param[in]     pGpu       OBJGPU pointer
+ * @param[in]     pHeap      Heap pointer
+ * @param[in]     pScrubber  OBJMEMSCRUB pointer
+ *
+ */
+
+NV_STATUS vgpuAllocGuestPmaScrubberSharedBuffer(struct OBJGPU *pGpu, OBJMEMSCRUB *pScrubber);
+void vgpuFreeGuestPmaScrubberSharedBuffer(struct OBJGPU *pGpu, OBJMEMSCRUB *pScrubber);
+
+void scrubberDestruct(struct OBJGPU *pGpu, struct Heap *pHeap, OBJMEMSCRUB *pMemscrub);
+
+
+/**
+ * This function checks for the completed scrub work items,
+ * and populates the SCRUB_NODE in the array.
+ * @param[in]  pScrubber OBJMEMSCRUB pointer
+ * @param[out]  ppList    SCRUB_NODE double pointer
+ * @param[out]  pSize     NvU64 pointer
+ * @returns NV_OK on success,
+ *          NV_ERR_INSUFFICIENT_RESOURCES when the list allocation fails.
+ */
+
+NV_STATUS scrubCheck(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *size);
+
+
+/**
+ * This function submits work to the memory scrubber.
+ * This function interface is changed to return a list of scrubbed pages to the
+ * client, since the scrubber work list resources are limited, if the submission
+ * page count is more than scrubber list resources the completed scrubbed pages
+ * are saved in the list and the submission progresses.
+ *
+ * @param[in] pScrubber  OBJMEMSCRUB pointer
+ * @param[in] chunkSize   NvU64 size of each page
+ * @param[in] pPages     NvU64 array of base address
+ * @param[in] pageCount  NvU64 number of pages
+ * @param[out] ppList     SCRUB_NODE double pointer to hand off the list
+ * @param[out] pSize      NvU64 pointer to store the size
+ *
+ * @returns NV_OK on success, NV_ERR_GENERIC on HW Failure
+ */
+
+NV_STATUS scrubSubmitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages,
+                           NvU64 pageCount, PSCRUB_NODE *ppList, NvU64 *size);
+
+/**
+ *  This function waits for the memory scrubber to wait for the scrubbing of
+ *  pages within the range [pagesStart, pagesEnd] for the for the array of pages
+ *  of size pageCount
+ *
+ * @param[in]   pScrubber   OBJMEMSCRUB pointer
+ * @param[in]   pageSize    NvU64 size of each page
+ * @param[in]   pPages      NvU64 pointer to store the base address
+ * @param[in]   pageCount   NvU64 number of pages in the array
+ *
+ * @returns NV_OK
+ */
+
+NV_STATUS scrubWaitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages,
+                         NvU32 pageCount);
+
+
+/**
+ *  This function waits for the scrubber to finish scrubbing enough items
+ *  to have numPages fully scrubbed and then saves the work items to the list
+ *  passed to the client.
+ *
+ * @param[in]  pScrubber OBJMEMSCRUB pointer
+ * @param[in]  numPages the number of pages we should wait to be scrubbed
+ * @param[in]  pageSize the page size
+ * @param[out]  ppList    SCRUB_NODE double pointer to return the saved list pointer
+ * @param[out]  pSize     NvU64 pointer to return the size of saved work.
+ *
+ * @returns NV_OK if at least one work is pending in the scrubber list
+ *          NV_ERR_NO_MEMORY when no work is pending in the scrubber list
+ */
+NV_STATUS scrubCheckAndWaitForSize (OBJMEMSCRUB *pScrubber, NvU64 numPages,
+                                    NvU64 pageSize, PSCRUB_NODE *ppList, NvU64 *pSize);
+#endif // MEM_SCRUB_H

src/nvidia/inc/kernel/gpu/mem_mgr/mem_utils.h

@@ -0,0 +1,49 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _MEM_UTILS_H_
+#define _MEM_UTILS_H_
+
+#include "core/prelude.h"
+
+#define CLEAR_HAL_ATTR(a) \
+    a = (a &~(DRF_NUM(OS32, _ATTR, _COMPR, 0x3) | \
+              DRF_NUM(OS32, _ATTR, _TILED, 0x3) | \
+              DRF_NUM(OS32, _ATTR, _ZCULL, 0x3)));
+
+#define CLEAR_HAL_ATTR2(a) \
+    a = (a & ~(DRF_SHIFTMASK(NVOS32_ATTR2_ZBC) | \
+               DRF_SHIFTMASK(NVOS32_ATTR2_GPU_CACHEABLE)));
+
+NvU64 memUtilsLeastCommonAlignment(NvU64 align1, NvU64 align2);
+
+void memUtilsInitFBAllocInfo(NV_MEMORY_ALLOCATION_PARAMS *pAllocParams, FB_ALLOC_INFO *pFbAllocInfo,
+                             NvHandle hClient, NvHandle hDevice);
+
+NV_STATUS memUtilsAllocMemDesc(OBJGPU *pGpu, MEMORY_ALLOCATION_REQUEST *pAllocRequest, FB_ALLOC_INFO *pFbAllocInfo,
+                               MEMORY_DESCRIPTOR **ppMemDesc, Heap *pHeap, NV_ADDRESS_SPACE addrSpace,
+                               NvBool bContig, NvBool *bAllocedMemDesc);
+
+NV_STATUS  memUtilsMemSetNoBAR2(OBJGPU *pGpu, PMEMORY_DESCRIPTOR pMemDesc, NvU8 value);
+
+#endif //_MEM_UTILS_H_

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/addrtree.h

@@ -0,0 +1,227 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ * @brief Implement PMA address tree
+ *
+ */
+
+#ifndef ADDRTREE_H
+#define ADDRTREE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "map_defines.h"
+
+// Declare this before its definition because it refers to itself
+typedef struct addrtree_node ADDRTREE_NODE;
+
+struct addrtree_node
+{
+    NvU32 level;                    // The level this node belongs to
+    NvU32 numChildren;              // The number of children in the children array
+    NvU64 frame;                    // The first frame this node holds
+    NvU64 state[PMA_BITS_PER_PAGE];          // Tracks the actual state for each map
+    NvU64 seeChild[PMA_BITS_PER_PAGE];       // Whether this node is partially allocated
+                                    //   If it is partially allocated, we must go to the children
+                                    //   to find the correct information.
+
+    ADDRTREE_NODE *parent;          // The node's parent
+    ADDRTREE_NODE *children;        // Pointer to an array of children
+};
+
+typedef struct addrtree_level
+{
+    NvU64 nodeCount;                // Count of total number of nodes on this level
+    ADDRTREE_NODE *pNodeList;       // Pointer to the start of the list of nodes on this level
+    NvU32 pageSizeShift;            // Page size this level is tracking
+    NvU32 maxFramesPerNode;         // The max number of this level frames per node
+} ADDRTREE_LEVEL;
+
+typedef struct pma_addrtree
+{
+    NvU64           totalFrames;    // Total number of 64KB frames being tracked
+    NvU32           levelCount;     // Number of levels in this tree
+    ADDRTREE_LEVEL *levels;         // List of levels in the tree
+    ADDRTREE_NODE  *root;           // Start of the node list
+    NvU64           numPaddingFrames; // Number of 64KB frames needed for padding for alignment
+
+    NvU64 frameEvictionsInProcess;  // Count of frame evictions in-process
+    PMA_STATS      *pPmaStats;      // Point back to the public struct in PMA structure
+    NvBool          bProtected;     // The memory segment tracked by this tree is protected (VPR/CPR)
+} PMA_ADDRTREE;
+
+/*!
+ * @brief Initializes the addrtree for PMA uses
+ *
+ * Allocates the address tree structure for all the pages being managed in this tree.
+ * Address Tree implementation will use a default configuration for its own level
+ * structures.
+ *
+ * @param[in] numPages    The number of pages being managed in this tree
+ * @param[in] addrBase    The base address of this region. Required for addrtree alignment
+ * @param[in] pPmaStats   Pointer to the PMA-wide stats structure
+ * @param[in] bProtected  The tree tracks pages in protected memory
+ *
+ * @return PMA_ADDRTREE Pointer to the addrtree if succeeded, NULL otherwise
+ */
+void *pmaAddrtreeInit(NvU64 numFrames, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected);
+
+/*!
+ * @brief Destroys the addrtree and free the memory
+ *
+ * @param[in] pMap  The addrtree to destroy
+ *
+ * @return void
+ */
+void pmaAddrtreeDestroy(void *pMap);
+
+/*!
+ * @brief Get/set number of evicting frames
+ * Used for sanity checking in PMA layer as well as performance optimization
+ * for the map layer to scan faster.
+ */
+NvU64 pmaAddrtreeGetEvictingFrames(void *pMap);
+void pmaAddrtreeSetEvictingFrames(void *pMap, NvU64 frameEvictionsInProcess);
+
+
+/*!
+ * @brief Scans the addrtree for contiguous space that has the certain status.
+ *
+ * @param[in]   pMap            The addrtree to be scanned
+ * @param[in]   addrBase        The base address of this region
+ * @param[in]   rangeStart      The start of the restricted range
+ * @param[in]   rangeEnd        The end of the restricted range
+ * @param[in]   numPages        The number of pages we are scanning for
+ * @param[out]  freeList        A list of free frame numbers -- contains only 1 element
+ * @param[in]   pageSize        Size of one page
+ * @param[in]   alignment       Alignment requested by client
+ * @param[out]  pagesAllocated  Number of pages this call allocated
+ * @param[in]   bSkipEvict      Whether it's ok to skip the scan for evictable pages
+ *
+ * @return NV_OK if succeeded
+ * @return NV_ERR_IN_USE if found pages that can be evicted
+ * @return NV_ERR_NO_MEMORY if no available pages could be found
+ */
+NV_STATUS pmaAddrtreeScanContiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+NV_STATUS pmaAddrtreeScanDiscontiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+void pmaAddrtreePrintTree(void *pMap, const char* str);
+
+
+/*!
+ * @brief Changes the state & attrib bits specified by mask
+ *
+ * Changes the state of the bits given the physical frame number
+ * TODO: all four interfaces need to be merged from PMA level so we can remove them!
+ *
+ * @param[in]   pMap         The addrtree to change
+ * @param[in]   frameNum     The frame number to change
+ * @param[in]   newState     The new state to change to
+ * @param[in]   newStateMask Specific bits to write
+ *
+ * @return void
+ */
+void pmaAddrtreeChangeState(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
+void pmaAddrtreeChangeStateAttrib(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib);
+void pmaAddrtreeChangeStateAttribEx(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState,PMA_PAGESTATUS newStateMask);
+void pmaAddrtreeChangePageStateAttrib(void * pMap, NvU64 startFrame, NvU32 pageSize,
+                                      PMA_PAGESTATUS newState, NvBool writeAttrib);
+
+/*!
+ * @brief Read the page state & attrib bits
+ *
+ * Read the state of the page given the physical frame number
+ *
+ * @param[in]   pMap        The addrtree to read
+ * @param[in]   frameNum    The frame number to read
+ * @param[in]   readAttrib  Read attribute bits as well
+ *
+ * @return PAGESTATUS of the frame
+ */
+PMA_PAGESTATUS pmaAddrtreeRead(void *pMap, NvU64 frameNum, NvBool readAttrib);
+
+
+/*!
+ * @brief Gets the total size of specified PMA managed region.
+ *
+ * Gets the total size of current PMA managed region in the FB.
+ *
+ * @param[in]  pMap         Pointer to the addrtree for the region
+ * @param[in]  pBytesTotal  Pointer that will return total bytes for current region.
+ *
+ */
+void pmaAddrtreeGetSize(void *pMap, NvU64 *pBytesTotal);
+
+
+/*!
+ * @brief Gets the size of the maximum free chunk of memory in specified region.
+ *
+ * Gets the size of the maximum free chunk of memory in the specified PMA managed
+ * region of the FB.
+ *
+ * @param[in]  pMap         Pointer to the addrtree for the region
+ * @param[in]  pLargestFree Pointer that will return largest free in current region.
+ *
+ */
+void pmaAddrtreeGetLargestFree(void *pMap, NvU64 *pLargestFree);
+
+/*!
+ * @brief Returns the address range that is completely available for eviction.
+ *        - Should be ALLOC_UNPIN.
+ * In NUMA, OS manages memory and PMA will only track allocated memory in ALLOC_PIN
+ * and ALLOC_UNPIN state. FREE memory is managed by OS and cannot be tracked by PMA
+ * and hence PMA cannot consider FREE memory for eviction and can only consider frames
+ * in known state to PMA or eviction. ALLOC_PIN cannot be evicted and hence only ALLOC_UNPIN
+ * can be evictable.
+ *
+ *
+ * @param[in]  pMap         Pointer to the regmap for the region
+ * @param[in]  addrBase     Base address of the region
+ * @param[in]  actualSize   Size of the eviction range
+ * @param[in]  pageSize     Pagesize
+ * @param[out] evictStart   Starting address of the eviction range
+ * @param[out] evictEnd     End address of the eviction range.
+ *
+ * Returns:
+ *  -  NV_OK If there is evictable range of given size : actualSize
+ *
+ *  -  NV_ERR_NO_MEMORY if no contiguous range is evictable.
+ */
+NV_STATUS pmaAddrtreeScanContiguousNumaEviction(void *pMap, NvU64 addrBase,
+    NvLength actualSize, NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // ADDRTREE_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/map_defines.h

@@ -0,0 +1,143 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *  @brief Contains common defines between addrtree and regmap
+ */
+
+#ifndef MAP_DEFINES_H
+#define MAP_DEFINES_H
+
+#include "nvtypes.h"
+#include "nvstatus.h"
+#include "nvmisc.h"
+#if !defined(NVWATCH)
+
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+// Default page size 64KB
+#define PMA_GRANULARITY 0x10000
+#define PMA_PAGE_SHIFT 16
+
+//
+// _PMA_1GB will cause overflows with an NvU32. It's bigger than NvU32 can store,
+// but compilation still fails when using a NvU64 instead
+// So just use bitshift.
+// 1 << _TREE_64KB == sizeof(1 frame)
+//
+
+#define _TREE_64KB               16
+#define _TREE_128KB              17
+#define _TREE_2MB                21
+#define _TREE_128MB              27
+#define _TREE_512MB              29
+#define _TREE_32GB               35
+#define _TREE_2TB                40
+
+// Defines shared between pma.c and regmap.c
+#define _PMA_64KB               (64 * 1024)
+#define _PMA_128KB              (128 * 1024)
+#define _PMA_2MB                (2 * 1024 * 1024)
+#define _PMA_512MB              (512 * 1024 * 1024)
+
+// Scanning function return code
+#define EVICTABLE -2
+#define ALL_FREE  -3
+
+typedef NvU32 PMA_PAGESTATUS;
+
+#define MAP_IDX_ALLOC_UNPIN 0
+#define MAP_IDX_ALLOC_PIN   1
+#define MAP_IDX_EVICTING    2
+#define MAP_IDX_SCRUBBING   3
+#define MAP_IDX_PERSISTENT  4
+#define MAP_IDX_NUMA_REUSE  5
+#define MAP_IDX_BLACKLIST   6
+
+#define STATE_FREE      0x00
+#define STATE_UNPIN     NVBIT(MAP_IDX_ALLOC_UNPIN)
+#define STATE_PIN       NVBIT(MAP_IDX_ALLOC_PIN)
+#define STATE_MASK      (STATE_UNPIN | STATE_PIN | STATE_FREE)
+#define STATE_COUNT     3
+
+#define ATTRIB_EVICTING    NVBIT(MAP_IDX_EVICTING)
+#define ATTRIB_SCRUBBING   NVBIT(MAP_IDX_SCRUBBING)
+#define ATTRIB_PERSISTENT  NVBIT(MAP_IDX_PERSISTENT)
+#define ATTRIB_NUMA_REUSE  NVBIT(MAP_IDX_NUMA_REUSE)
+#define ATTRIB_BLACKLIST   NVBIT(MAP_IDX_BLACKLIST)
+#define ATTRIB_MASK        (ATTRIB_EVICTING | ATTRIB_SCRUBBING \
+                            | ATTRIB_PERSISTENT | ATTRIB_NUMA_REUSE \
+                            | ATTRIB_BLACKLIST)
+
+#define MAP_MASK    (STATE_MASK | ATTRIB_MASK)
+
+#define PMA_STATE_BITS_PER_PAGE     2   // Alloc & pinned state
+#define PMA_ATTRIB_BITS_PER_PAGE    5   // Persistence, Scrubbing, Evicting, Reuse & Blacklisting attributes
+#define PMA_BITS_PER_PAGE           (PMA_STATE_BITS_PER_PAGE + PMA_ATTRIB_BITS_PER_PAGE)
+
+//
+// Stores PMA-wide statistics.
+//
+// In NUMA mode "free" means "not allocated by PMA". Since the kernel owns
+// memory, any amount of those pages could be allocated by the kernel but PMA
+// does not have that visibility. The provided counts are thus an upper bound on
+// the number of free pages.
+//
+typedef struct _PMA_STATS
+{
+    NvU64 num2mbPages;               // PMA-wide total number of 2MB pages
+    NvU64 numFreeFrames;             // PMA-wide free 64KB frame count
+    NvU64 numFree2mbPages;           // PMA-wide free 2MB pages count
+#if !defined(NVWATCH)
+#endif // !defined(NVWATCH)
+} PMA_STATS;
+
+// Stores blacklisting information passed in from heap layer
+typedef struct
+{
+    NvU64  physOffset; // base address of blacklisted page
+    NvBool bIsDynamic; // True if page was dynamically blacklisted
+} PMA_BLACKLIST_ADDRESS, *PPMA_BLACKLIST_ADDRESS;
+
+//
+// Store the blacklist chunk information with the physical offset aligned to 64K,
+// and whether the blacklist chunk is managed by RM or Client.
+//
+typedef struct
+{
+    NvU64  physOffset; // physical offset of blacklisted FB address
+    NvBool bIsDynamic;  // True if the page was dynamically blacklisted
+    NvBool bIsValid;   // If the blacklisted address is still managed by RM
+} PMA_BLACKLIST_CHUNK;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MAP_DEFINES_H