515.43.04

src/nvidia/inc/kernel/gpu/audio/hda_codec_api.h

@@ -0,0 +1,3 @@
+
+#include "g_hda_codec_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bif/kernel_bif.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_bif_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/kern_bus.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_bus_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/p2p_api.h

@@ -0,0 +1,3 @@
+
+#include "g_p2p_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/bus/third_party_p2p.h

@@ -0,0 +1,3 @@
+
+#include "g_third_party_p2p_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ce_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce_context.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ce_context_nvoc.h"
+

src/nvidia/inc/kernel/gpu/ce/kernel_ce_gv100_private.h

@@ -0,0 +1,84 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_GV100_PRIVATE_H
+#define KERNEL_CE_GV100_PRIVATE_H
+
+#define MAX_CE_CNT 15
+
+/*
+ * sysmemLinks
+ *    Represents the number of sysmem links detected
+ *    This affects how many PCEs LCE0(sysmem read CE)
+ *    and LCE1(sysmem write CE) should be mapped to
+ * maxLinksPerPeer
+ *    Represents the maximum number of peer links
+ *    between this GPU and all its peers. This affects
+ *    how many PCEs LCE3(P2P CE) should be mapped to
+ * numPeers
+ *    Represents the number of Peer GPUs discovered so far
+ * bSymmetric
+ *    Represents whether the topology detected so far
+ *    is symmetric i.e. has same number of links to all
+ *    peers connected through nvlink. This affects how
+ *    many PCEs to assign to LCEs3-5 (nvlink P2P CEs)
+ * bSwitchConfig
+ *    Represents whether the config listed is intended
+ *    for use with nvswitch systems
+ * pceLceMap
+ *    Value of NV_CE_PCE2LCE_CONFIG0 register with the
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ * grceConfig
+ *    Value of NV_CE_GRCE_CONFIG register with the
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ * exposeCeMask
+ *    Mask of CEs to expose to clients for the above
+ *    above values for sysmemLinks, maxLinksPerPeer,
+ *    numLinks and bSymmetric
+ */
+typedef struct NVLINK_CE_AUTO_CONFIG_TABLE
+{
+    NvU32  sysmemLinks;
+    NvU32  maxLinksPerPeer;
+    NvU32  numPeers;
+    NvBool bSymmetric;
+    NvBool bSwitchConfig;
+    NvU32  pceLceMap[MAX_CE_CNT];
+    NvU32  grceConfig[MAX_CE_CNT];
+    NvU32  exposeCeMask;
+} NVLINK_CE_AUTO_CONFIG_TABLE;
+
+//
+// General convention decided on between HW and SW:
+//  - CE2 is for SYSMEM reads
+//  - CE3 is for SYSMEM writes
+//  - CE4-8 are for P2P
+//
+#define NVLINK_SYSMEM_READ_LCE  2
+#define NVLINK_SYSMEM_WRITE_LCE 3
+#define NVLINK_MIN_P2P_LCE      4
+#define NVLINK_MAX_P2P_LCE      8
+
+#endif

src/nvidia/inc/kernel/gpu/ce/kernel_ce_private.h

@@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_PRIVATE_H
+#define KERNEL_CE_PRIVATE_H
+
+#include "gpu/gpu.h"
+#include "gpu_mgr/gpu_mgr.h"
+#include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
+
+#define CE_GET_LOWEST_AVAILABLE_IDX(x) portUtilCountTrailingZeros32(x)
+
+/*!
+ * @brief Obtain relative CE index.
+ *
+ * @param localEngType NV2080_ENGINE_TYPE_ for this CE, or partition-local engine type.
+ * @param ceIdx CE index in 0..GPU_MAX_CES-1
+ *
+ * @return NV_OK if the conversion is successful.
+ */
+static NV_INLINE
+NV_STATUS ceIndexFromType(OBJGPU *pGpu, NvHandle hClient, NvU32 localEngType, NvU32 *ceIdx)
+{
+    //
+    // If MIG is enabled, client passes a logical engineId w.r.t its own partition
+    // we need to convert this logical Id to a physical engine Id as we use it
+    // to set runlistId
+    //
+    if (IS_MIG_IN_USE(pGpu))
+    {
+        KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
+        MIG_INSTANCE_REF ref;
+
+        NV_CHECK_OK_OR_RETURN(
+            LEVEL_ERROR,
+            kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager,
+                                            hClient, &ref));
+
+        NV_CHECK_OK_OR_RETURN(
+            LEVEL_ERROR,
+            kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
+                                              localEngType,
+                                              ceIdx));
+    }
+    else
+    {
+        *ceIdx = localEngType;
+    }
+
+    if (!NV2080_ENGINE_TYPE_IS_COPY(*ceIdx))
+    {
+        return NV_ERR_INVALID_ARGUMENT;
+    }
+
+    *ceIdx = NV2080_ENGINE_TYPE_COPY_IDX(*ceIdx);
+    return NV_OK;
+}
+
+#endif // KERNEL_CE_PRIVATE_H

src/nvidia/inc/kernel/gpu/ce/kernel_ce_shared.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERNEL_CE_SHARED_H
+#define KERNEL_CE_SHARED_H
+
+#include "gpu/gpu.h"
+
+/**
+ * Routines shared between CE and KCE.
+ */
+
+NvBool ceIsCeGrce(OBJGPU *pGpu, NvU32 ceEngineType);
+NvU32 ceCountGrCe(OBJGPU *pGpu);
+
+#endif

src/nvidia/inc/kernel/gpu/dce_client/dce_client.h

@@ -0,0 +1,3 @@
+
+#include "g_dce_client_nvoc.h"
+

src/nvidia/inc/kernel/gpu/deferred_api.h

@@ -0,0 +1,3 @@
+
+#include "g_deferred_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/device/device.h

@@ -0,0 +1,3 @@
+
+#include "g_device_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_capabilities.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_capabilities_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_channel.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_channel_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_objs.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_objs_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/disp_sf_user.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_sf_user_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/head/kernel_head.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_head_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/inst_mem/disp_inst_mem.h

@@ -0,0 +1,3 @@
+
+#include "g_disp_inst_mem_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/kern_disp.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_disp_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/kern_disp_max.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERN_DISP_MAX_H
+#define KERN_DISP_MAX_H
+
+/*!
+ * @brief Defines max values used for the KernelDisplay Engine Object,
+ *        including values shared by OBJDISP code.
+ */
+
+#define OBJ_MAX_HEADS                     4
+#define MAX_RG_LINE_CALLBACKS_PER_HEAD    2
+#define OBJ_MAX_DFPS                     31
+
+#endif // KERN_DISP_MAX_H

src/nvidia/inc/kernel/gpu/disp/kern_disp_type.h

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef KERN_DISP_TYPE_H
+#define KERN_DISP_TYPE_H
+
+/******************************************************************************
+*
+*  Defines display type enums that can be used in the KernelDisplay object.
+*
+******************************************************************************/
+
+typedef enum
+{
+    dispChnClass_Curs,
+    dispChnClass_Ovim,
+    dispChnClass_Base,
+    dispChnClass_Core,
+    dispChnClass_Ovly,
+    dispChnClass_Winim,
+    dispChnClass_Win,
+    dispChnClass_Supported
+} DISPCHNCLASS;
+
+enum DISPLAY_ICC_BW_CLIENT
+{
+    DISPLAY_ICC_BW_CLIENT_RM,
+    DISPLAY_ICC_BW_CLIENT_EXT,  // DD or MODS
+    NUM_DISPLAY_ICC_BW_CLIENTS
+};
+
+#endif // #ifndef KERN_DISP_TYPE_H

src/nvidia/inc/kernel/gpu/disp/rg_line_callback/rg_line_callback.h

@@ -0,0 +1,3 @@
+
+#include "g_rg_line_callback_nvoc.h"
+

src/nvidia/inc/kernel/gpu/disp/vblank_callback/vblank.h

@@ -0,0 +1,112 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef VBLANK_H
+#define VBLANK_H
+
+#include "gpu/gpu.h"
+/* ------------------------ Types definitions ------------------------------ */
+/*!
+ * Callback function prototype
+ */
+typedef NV_STATUS (*VBLANKCALLBACKPROC)(OBJGPU*, void *, NvU32, NvU32, NV_STATUS);
+
+typedef struct VBLANKCALLBACK
+{
+    VBLANKCALLBACKPROC Proc;
+    void              *pObject;
+    NvBool             bObjectIsChannelDescendant;
+    NvU32              Param1;
+    NvU32              Param2;
+    NvU32              VBlankCount;
+    NvU32              VBlankOffset;
+    NvU64              TimeStamp;
+    NvU32              MC_CallbackFlag;
+    NvU32              Flags;
+    NV_STATUS          Status;
+    struct VBLANKCALLBACK *Next;
+    NvBool             bImmediateCallback;
+    NvBool             bIsVblankNotifyEnable;
+}VBLANKCALLBACK;
+
+/* ------------------------ Macros & Defines ------------------------------- */
+
+/*!
+ * Callback function registration flags
+ */
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_COUNT        0x00000001
+#define VBLANK_CALLBACK_FLAG_COMPLETE_ON_OBJECT_CLEANUP    0x00000002
+#define VBLANK_CALLBACK_FLAG_PERSISTENT                    0x00000004
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_TIMESTAMP           0x00000010
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_NEXT         0x00000020  // Explicit request for the next vblank.
+#define VBLANK_CALLBACK_FLAG_SPECIFIED_VBLANK_OFFSET       0x00000040  // Explicit request for the vblank offset from the current one
+#define VBLANK_CALLBACK_FLAG_PROMOTE_TO_FRONT              0x00000080  // Promotes to being 'first', while still honoring VBlankCount
+#define VBLANK_CALLBACK_FLAG_RELEASES_SEMAPHORE            0x00000100  // A flag for deadlock detection to check if this callback could release a semaphore
+#define VBLANK_CALLBACK_FLAG_GUARANTEE_SAFETY              0x00000200  // This callback absolutely needs to run during vertical blank, even if it runs late as a consequence.
+#define VBLANK_CALLBACK_FLAG_LOW_LATENCY__ISR_ONLY         0x08000000  // This means always process during ISR (never DPC.) Be careful!
+#define VBLANK_CALLBACK_FLAG_LOW_LATENCY                   0x10000000  // This now means ASAP, which could be ISR or DPC, depending on which happens first
+#define VBLANK_CALLBACK_FLAG_MC_EXECUTE_ONCE               0x40000000  // A special flag for MultiChip configurations to have the callback execute only once
+#define VBLANK_CALLBACK_FLAG_USER                          0x80000000
+
+/*!
+ * A little macro help for the CALLBACK_FLAG_MC_EXECUTE_ONCE flag above
+ */
+#define VBLANK_CALLBACK_EXECUTE_ONCE(x)    (x & VBLANK_CALLBACK_FLAG_MC_EXECUTE_ONCE)
+
+/*!
+ * VBlank Service info gathering keep-alive in seconds. This value is the number of seconds the vblank service will run after a client request vblank info.
+ */
+#define VBLANK_INFO_GATHER_KEEPALIVE_SECONDS    (5)
+
+/*!
+ * VBLANK SERVICE RELATED
+ * VBlank Service callback processing flags
+ * These two flags describe when to process the queues
+ */
+
+#define VBLANK_STATE_PROCESS_NORMAL                  (0x00000000)    // Process the requested queues if associated vblank interrupt is pending
+#define VBLANK_STATE_PROCESS_IMMEDIATE               (0x00000001)    // Process the requested queues now, regardless of any vblank interrupt pending state
+
+/*!
+ * These three flags describe which queues to process
+ */
+#define VBLANK_STATE_PROCESS_LOW_LATENCY             (0x00000002)    // Process the low-latency vblank callback queue
+#define VBLANK_STATE_PROCESS_NORMAL_LATENCY          (0x00000004)    // Process the normal-latency vblank callback queue
+
+#define VBLANK_STATE_PROCESS_ALL_CALLBACKS           (VBLANK_STATE_PROCESS_LOW_LATENCY|VBLANK_STATE_PROCESS_NORMAL_LATENCY) // Process all callback (high and low latency) queues
+
+#define VBLANK_STATE_PROCESS_CALLBACKS_ONLY          (0x00000008)    // Process only the callback queue(s) and nothing else
+
+/*!
+ * set when called from an ISR; if VBlank() is in an ISR and there is
+ * more work to do, then VBlank() will not clear the pending bit
+ */
+#define VBLANK_STATE_PROCESS_CALLED_FROM_ISR         (0x00000010)
+#define VBLANK_STATE_PROCESS_CALLED_FROM_DPC         (0x00000020)
+
+/*! Vblank Interrupt state */
+#define NV_HEAD_VBLANK_INTR_UNAVAILABLE              (0x00000000)
+#define NV_HEAD_VBLANK_INTR_AVAILABLE                (0x00000001)
+#define NV_HEAD_VBLANK_INTR_ENABLED                  (0x00000002)
+
+#endif // VBLANK_H

src/nvidia/inc/kernel/gpu/disp/vblank_callback/vblank_callback.h

@@ -0,0 +1,3 @@
+
+#include "g_vblank_callback_nvoc.h"
+

src/nvidia/inc/kernel/gpu/eng_desc.h

@@ -0,0 +1,3 @@
+
+#include "g_eng_desc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/eng_state.h

@@ -0,0 +1,3 @@
+
+#include "g_eng_state_nvoc.h"
+

src/nvidia/inc/kernel/gpu/falcon/falcon_common.h

@@ -0,0 +1,61 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-202 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef FALCON_COMMON_H
+#define FALCON_COMMON_H
+
+/*!
+ * Alignment/granularity of falcon memory blocks
+ */
+#define FLCN_BLK_ALIGNMENT (256)
+
+/*!
+ * Address/offset alignment required for falcon IMEM accesses
+ */
+#define FLCN_IMEM_ACCESS_ALIGNMENT (4)
+
+/*!
+ * Address/offset alignment required for falcon DMEM accesses
+ */
+#define FLCN_DMEM_ACCESS_ALIGNMENT (4)
+
+/*!
+ * Falcon IMEM block-size (as a power-of-2)
+ */
+#define FALCON_IMEM_BLKSIZE2 (8)
+
+/*!
+ * Falcon DMEM block-size (as a power-of-2)
+ */
+#define FALCON_DMEM_BLKSIZE2 (8)
+
+/*!
+ * Denotes invalid/absent VA for falcon ucode loading
+ */
+#define FLCN_DMEM_VA_INVALID 0xffffffff
+
+/*!
+ * Default Falcon context buffer size
+ */
+#define FLCN_CTX_ENG_BUFFER_SIZE_HW      4096
+
+#endif  // FALCON_COMMON_H

src/nvidia/inc/kernel/gpu/falcon/kernel_falcon.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_falcon_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/channel_descendant.h

@@ -0,0 +1,3 @@
+
+#include "g_channel_descendant_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel_group.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_group_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_channel_group_api.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_channel_group_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_ctxshare.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_ctxshare_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_fifo.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_fifo_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/kernel_sched_mgr.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_sched_mgr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/usermode_api.h

@@ -0,0 +1,3 @@
+
+#include "g_usermode_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/fifo/uvm_channel_retainer.h

@@ -0,0 +1,3 @@
+
+#include "g_uvm_channel_retainer_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_access.h

@@ -0,0 +1,381 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_ACCESS_H_
+#define _GPU_ACCESS_H_
+
+#include "ioaccess/ioaccess.h"
+#include "gpu/gpu_device_mapping.h"
+
+// Go straight at the memory or hardware.
+#define MEM_RD08(a) (*(const volatile NvU8  *)(a))
+#define MEM_RD16(a) (*(const volatile NvU16 *)(a))
+#define MEM_RD32(a) (*(const volatile NvU32 *)(a))
+#define MEM_WR08(a, d) do { *(volatile NvU8  *)(a) = (d); } while (0)
+#define MEM_WR16(a, d) do { *(volatile NvU16 *)(a) = (d); } while (0)
+#define MEM_WR32(a, d) do { *(volatile NvU32 *)(a) = (d); } while (0)
+#define MEM_WR64(a, d) do { *(volatile NvU64 *)(a) = (d); } while (0)
+
+//
+// Define the signature of the register filter callback function
+//
+// flags can be optionally used for filters to decide whether to actually
+// touch HW or not. flags should be OR'ed every time a new filter is found. (see objgpu.c)
+//
+typedef void (*GpuWriteRegCallback)(OBJGPU *, void *, NvU32 addr, NvU32 val, NvU32 accessSize, NvU32 flags);
+typedef NvU32 (*GpuReadRegCallback)(OBJGPU *, void *, NvU32 addr, NvU32 accessSize, NvU32 flags);
+
+union GPUHWREG
+{
+    volatile NvV8 Reg008[1];
+    volatile NvV16 Reg016[1];
+    volatile NvV32 Reg032[1];
+};
+
+typedef union  GPUHWREG  GPUHWREG;
+
+//
+// Register filter record
+//
+// If REGISTER_FILTER_FLAGS_READ is set, then that means that the base RegRead
+// function will not read the register, so the provided read callback function
+// is expected to read the register and return the value.
+//
+// If REGISTER_FILTER_FLAGS_WRITE is set, then that means that the base RegWrite
+// function will not write the register, so the provided callback write function
+// is expected to write the given value to the register.
+//
+// It is an error to specify REGISTER_FILTER_FLAGS_READ and not provide a
+// read callback function.
+//
+// It is an error to specify REGISTER_FILTER_FLAGS_WRITE and not provide a
+// write callback function.
+//
+#define REGISTER_FILTER_FLAGS_READ         (NVBIT(0))
+#define REGISTER_FILTER_FLAGS_WRITE        (NVBIT(1))
+// filter is in the list but it is invalid and should be removed
+#define REGISTER_FILTER_FLAGS_INVALID      (NVBIT(2))
+
+#define REGISTER_FILTER_FLAGS_VIRTUAL      (0)
+#define REGISTER_FILTER_FLAGS_READ_WRITE   (REGISTER_FILTER_FLAGS_READ | REGISTER_FILTER_FLAGS_WRITE)
+
+typedef struct REGISTER_FILTER REGISTER_FILTER;
+
+struct REGISTER_FILTER
+{
+    REGISTER_FILTER            *pNext;           //!< pointer to next filter
+    NvU32                       flags;           //!< attributes of this filter
+    DEVICE_INDEX                devIndex;        //!< filter device
+    NvU32                       devInstance;     //!< filter device instance
+    NvU32                       rangeStart;      //!< filter range start (can overlap)
+    NvU32                       rangeEnd;        //!< filter range end   (can overlap)
+    GpuWriteRegCallback         pWriteCallback;  //!< callback for write
+    GpuReadRegCallback          pReadCallback;   //!< callback for read
+    void                       *pParam;          //!< pointer to param which gets passed to callbacks
+};
+
+typedef struct {
+    REGISTER_FILTER        *pRegFilterList;         // Active filters
+    REGISTER_FILTER        *pRegFilterRecycleList;  // Inactive filters
+    PORT_SPINLOCK *         pRegFilterLock;         // Thread-safe list management
+    NvU32                   regFilterRefCnt;        // Thread-safe list management
+    NvBool                  bRegFilterNeedRemove;   // Thread-safe list garbage collection
+} DEVICE_REGFILTER_INFO;
+
+typedef struct DEVICE_MAPPING
+{
+    GPUHWREG             *gpuNvAddr;        // CPU Virtual Address
+    RmPhysAddr            gpuNvPAddr;       // Physical Base Address
+    NvU32                 gpuNvLength;      // Length of the Aperture
+    NvU32                 gpuNvSaveLength;
+    NvU32                 gpuDeviceEnum;    // Device ID NV_DEVID_*
+    NvU32                 refCount;         // refCount for the device map.
+    DEVICE_REGFILTER_INFO devRegFilterInfo; // register filter range list
+} DEVICE_MAPPING;
+
+typedef struct
+{
+    IO_DEVICE             parent;
+    OBJGPU               *pGpu;
+    DEVICE_INDEX          deviceIndex;
+    NvU32                 instance;
+    // The following members are initialized in objgpu.c,
+    // but are not used anywhere. gpuApertureReg* functions
+    // fall back to DEVICE_MAPPING instead
+    GPUHWREG             *gpuNvAddr;        // CPU Virtual Address
+    RmPhysAddr            gpuNvPAddr;       // Physical Base Address
+    NvU32                 gpuNvLength;      // Length of Aperture
+    NvU32                 gpuDeviceEnum;    // Device ID NV_DEVID_*
+    NvU32                 refCount;         // refCount for the device map.
+    DEVICE_REGFILTER_INFO devRegFilterInfo; // register filter range list
+} GPU_IO_DEVICE;
+
+typedef struct
+{
+    // Pointer to GPU linked to this RegisterAccess object
+    OBJGPU       *pGpu;
+
+    // HW register access tools
+    GPUHWREG     *gpuFbAddr;
+    GPUHWREG     *gpuInstAddr;
+
+    // Register access profiling
+    NvU32         regReadCount;
+    NvU32         regWriteCount;
+} RegisterAccess;
+
+/*! Init register IO access path */
+NV_STATUS regAccessConstruct(RegisterAccess *, OBJGPU *pGpu);
+
+/*! Shutdown register IO access path */
+void regAccessDestruct(RegisterAccess *);
+
+/*! Writes to 8 bit register */
+void regWrite008(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV8);
+
+/*! Writes to 16 bit register */
+void regWrite016(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV16);
+
+/*! Writes to 32 bit register, with thread state on the stack */
+void regWrite032(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV32, THREAD_STATE_NODE *);
+
+/*! Unicast register access, with thread state on the stack */
+void regWrite032Unicast(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvV32, THREAD_STATE_NODE *);
+
+/*! Reads from 8 bit register */
+NvU8 regRead008(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32);
+
+/*! Reads from 16 bit register */
+NvU16 regRead016(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32);
+
+/*! Reads from 32 bit register, with thread state on the stack */
+NvU32 regRead032(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, THREAD_STATE_NODE *);
+
+/*! Reads from 32 bit register and checks bit mask, with thread state on the stack */
+NvU32 regCheckRead032(RegisterAccess *, NvU32, NvU32, THREAD_STATE_NODE *);
+
+/*! Reads 32 bit register and polls bit field for specific value */
+NV_STATUS regRead032_AndPoll(RegisterAccess *, DEVICE_INDEX, NvU32, NvU32, NvU32);
+
+/*! Adds a register filter */
+NV_STATUS regAddRegisterFilter(RegisterAccess *, NvU32, DEVICE_INDEX, NvU32, NvU32, NvU32, GpuWriteRegCallback, GpuReadRegCallback, void *, REGISTER_FILTER **);
+
+/*! Removes register filter */
+void regRemoveRegisterFilter(RegisterAccess *, REGISTER_FILTER *);
+
+/*! Check status of read return value for GPU/bus errors */
+void regCheckAndLogReadFailure(RegisterAccess *, NvU32 addr, NvU32 mask, NvU32 value);
+
+//
+// GPU register I/O macros.
+//
+
+//
+// GPU neutral macros typically used for register I/O.
+//
+#define GPU_DRF_SHIFT(drf)          ((0?drf) % 32)
+#define GPU_DRF_MASK(drf)           (0xFFFFFFFF>>(31-((1?drf) % 32)+((0?drf) % 32)))
+#define GPU_DRF_DEF(d,r,f,c)        ((NV ## d ## r ## f ## c)<<GPU_DRF_SHIFT(NV ## d ## r ## f))
+#define GPU_DRF_NUM(d,r,f,n)        (((n)&GPU_DRF_MASK(NV ## d ## r ## f))<<GPU_DRF_SHIFT(NV ## d ## r ## f))
+#define GPU_DRF_VAL(d,r,f,v)        (((v)>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_DRF_SHIFTMASK(drf)      (GPU_DRF_MASK(drf)<<(GPU_DRF_SHIFT(drf)))
+#define GPU_DRF_WIDTH(drf)          ((1?drf) - (0?drf) + 1)
+
+
+// Device independent macros
+// Multiple device instance macros
+
+#define REG_INST_RD08(g,dev,inst,a)             regRead008(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a)
+#define REG_INST_RD16(g,dev,inst,a)             regRead016(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a)
+#define REG_INST_RD32(g,dev,inst,a)             regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, NULL)
+
+#define REG_INST_WR08(g,dev,inst,a,v)     regWrite008(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v)
+#define REG_INST_WR16(g,dev,inst,a,v)     regWrite016(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v)
+#define REG_INST_WR32(g,dev,inst,a,v)     regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, NULL)
+#define REG_INST_WR32_UC(g,dev,inst,a,v)  regWrite032Unicast(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, NULL)
+
+#define REG_INST_RD32_EX(g,dev,inst,a,t)    regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, t)
+#define REG_INST_WR32_EX(g,dev,inst,a,v,t)  regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, a, v, t)
+
+#define REG_INST_DEVIDX_RD32_EX(g,devidx,inst,a,t)    regRead032(GPU_GET_REGISTER_ACCESS(g), devidx, inst, a, t)
+#define REG_INST_DEVIDX_WR32_EX(g,devidx,inst,a,v,t)  regWrite032(GPU_GET_REGISTER_ACCESS(g), devidx, inst, a, v, t)
+
+// GPU macros defined in terms of DEV_ macros
+#define GPU_REG_RD08(g,a) REG_INST_RD08(g,GPU,0,a)
+#define GPU_REG_RD16(g,a) REG_INST_RD16(g,GPU,0,a)
+#define GPU_REG_RD32(g,a) REG_INST_RD32(g,GPU,0,a)
+#define GPU_CHECK_REG_RD32(g,a,m) regCheckRead032(GPU_GET_REGISTER_ACCESS(g),a,m,NULL)
+#define GPU_REG_RD32_AND_POLL(g,r,m,v) regRead032_AndPoll(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_GPU, r, m, v)
+
+#define GPU_REG_WR08(g,a,v)    REG_INST_WR08(g,GPU,0,a,v)
+#define GPU_REG_WR16(g,a,v)    REG_INST_WR16(g,GPU,0,a,v)
+#define GPU_REG_WR32(g,a,v)    REG_INST_WR32(g,GPU,0,a,v)
+#define GPU_REG_WR32_UC(g,a,v) REG_INST_WR32_UC(g,GPU,0,a,v)
+
+// GPU macros for SR-IOV
+#define GPU_VREG_RD32(g, a)                GPU_REG_RD32(g, g->sriovState.virtualRegPhysOffset + a)
+#define GPU_VREG_WR32(g, a, v)             GPU_REG_WR32(g, g->sriovState.virtualRegPhysOffset + a, v)
+#define GPU_VREG_RD32_EX(g,a,t)            REG_INST_RD32_EX(g, GPU, 0, g->sriovState.virtualRegPhysOffset + a, t)
+#define GPU_VREG_WR32_EX(g,a,v,t)          REG_INST_WR32_EX(g, GPU, 0, g->sriovState.virtualRegPhysOffset + a, v, t)
+#define GPU_VREG_FLD_WR_DRF_DEF(g,d,r,f,c) GPU_VREG_WR32(g, NV##d##r,(GPU_VREG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_VREG_RD_DRF(g,d,r,f)        (((GPU_VREG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+#define VREG_INST_RD32(g,dev,inst,a)            regRead032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, g->sriovState.virtualRegPhysOffset + a, NULL)
+#define VREG_INST_WR32(g,dev,inst,a,v)          regWrite032(GPU_GET_REGISTER_ACCESS(g), DEVICE_INDEX_##dev, inst, g->sriovState.virtualRegPhysOffset + a, v, NULL)
+#define GPU_VREG_FLD_WR_DRF_NUM(g,d,r,f,n) VREG_INST_WR32(g,GPU,0,NV##d##r,(VREG_INST_RD32(g,GPU,0,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+
+#define GPU_VREG_FLD_TEST_DRF_DEF(g,d,r,f,c) (GPU_VREG_RD_DRF(g, d, r, f) == NV##d##r##f##c)
+
+#define GPU_GET_VREG_OFFSET(g, a) (g->sriovState.virtualRegPhysOffset + a)
+
+#define GPU_VREG_IDX_RD_DRF(g,d,r,i,f)           (((GPU_VREG_RD32(g, NV ## d ## r(i)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_VREG_FLD_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_VREG_WR32(g, NV##d##r(i),(GPU_VREG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+#define GPU_REG_RD32_EX(g,a,t)    REG_INST_RD32_EX(g,GPU,0,a,t)
+#define GPU_REG_WR32_EX(g,a,v,t)  REG_INST_WR32_EX(g,GPU,0,a,v,t)
+
+// Uncomment this to enable register access dump in gsp client
+// #define GPU_REGISTER_ACCESS_DUMP    RMCFG_FEATURE_GSP_CLIENT_RM
+#ifndef GPU_REGISTER_ACCESS_DUMP
+#define GPU_REGISTER_ACCESS_DUMP    0
+#endif
+
+#if GPU_REGISTER_ACCESS_DUMP
+
+NvU8  gpuRegRd08_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+NvU16 gpuRegRd16_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+NvU32 gpuRegRd32_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr);
+void  gpuRegWr08_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV8 val);
+void  gpuRegWr16_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV16 val);
+void  gpuRegWr32_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV32 val);
+void  gpuRegWr32Uc_dumpinfo(const char *func, const char *addrStr, const char *vreg, OBJGPU *pGpu, NvU32 addr, NvV32 val);
+
+#undef GPU_REG_RD08
+#undef GPU_REG_RD16
+#undef GPU_REG_RD32
+#undef GPU_REG_WR08
+#undef GPU_REG_WR16
+#undef GPU_REG_WR32
+#undef GPU_REG_WR32_UC
+#undef GPU_VREG_RD32
+#undef GPU_VREG_WR32
+
+#define GPU_REG_RD08(g,a)       gpuRegRd08_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_RD16(g,a)       gpuRegRd16_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_RD32(g,a)       gpuRegRd32_dumpinfo(__FUNCTION__,#a,"",g,a)
+#define GPU_REG_WR08(g,a,v)     gpuRegWr08_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR16(g,a,v)     gpuRegWr16_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR32(g,a,v)     gpuRegWr32_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_REG_WR32_UC(g,a,v)  gpuRegWr32Uc_dumpinfo(__FUNCTION__,#a,"",g,a,v)
+#define GPU_VREG_RD32(g, a)     gpuRegRd32_dumpinfo(__FUNCTION__,#a,"(VREG)",g, g->sriovState.virtualRegPhysOffset + a)
+#define GPU_VREG_WR32(g, a, v)  gpuRegWr32_dumpinfo(__FUNCTION__,#a,"(VREG)",g, g->sriovState.virtualRegPhysOffset + a, v)
+
+#endif // GPU_REGISTER_ACCESS_DUMP
+
+//
+// Macros for register I/O
+//
+#define GPU_FLD_WR_DRF_NUM(g,d,r,f,n) REG_INST_WR32(g,GPU,0,NV##d##r,(REG_INST_RD32(g,GPU,0,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_WR_DRF_NUM_UC(g,d,r,f,n) GPU_REG_WR32_UC(g, NV##d##r,(GPU_REG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_WR_DRF_DEF(g,d,r,f,c) GPU_REG_WR32(g, NV##d##r,(GPU_REG_RD32(g,NV##d##r)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_RD_DRF(g,d,r,f)       (((GPU_REG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_FLD_TEST_DRF_DEF(g,d,r,f,c) (GPU_REG_RD_DRF(g, d, r, f) == NV##d##r##f##c)
+#define GPU_FLD_TEST_DRF_NUM(g,d,r,f,n) (GPU_REG_RD_DRF(g, d, r, f) == n)
+#define GPU_FLD_IDX_TEST_DRF_DEF(g,d,r,f,c,i) (GPU_REG_IDX_RD_DRF(g, d, r, i, f) == NV##d##r##f##c)
+#define GPU_FLD_2IDX_TEST_DRF_DEF(g,d,r,f,c,i,j) (GPU_REG_2IDX_RD_DRF(g, d, r, i, j, f) == NV##d##r##f##c)
+
+#define GPU_REG_RD_DRF_EX(g,d,r,f,t)       (((GPU_REG_RD32_EX(g, NV ## d ## r, t))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+#define GPU_FLD_WR_DRF_NUM_EX(g,d,r,f,n,t) REG_INST_WR32_EX(g,GPU,0,NV##d##r,(REG_INST_RD32_EX(g,GPU,0,NV##d##r,t)&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n),t)
+
+// Read/write a field or entire register of which there are several copies each accessed via an index
+#define GPU_REG_IDX_WR_DRF_NUM(g,d,r,i,f,n) GPU_REG_WR32(g, NV ## d ## r(i), GPU_DRF_NUM(d,r,f,n))
+#define GPU_REG_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_REG_WR32(g, NV ## d ## r(i), GPU_DRF_DEF(d,r,f,c))
+#define GPU_FLD_IDX_WR_DRF_NUM(g,d,r,i,f,n) GPU_REG_WR32(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define GPU_FLD_IDX_WR_DRF_DEF(g,d,r,i,f,c) GPU_REG_WR32(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_IDX_WR_DRF_NUM_UC(g,d,r,i,f,n) GPU_REG_WR32_UC(g, NV ## d ## r(i), GPU_DRF_NUM(d,r,f,n))
+#define GPU_REG_IDX_WR_DRF_DEF_UC(g,d,r,i,f,c) GPU_REG_WR32_UC(g, NV ## d ## r(i), GPU_DRF_DEF(d,r,f,c))
+#define GPU_FLD_IDX_WR_DRF_DEF_UC(g,d,r,i,f,c) GPU_REG_WR32_UC(g, NV##d##r(i),(GPU_REG_RD32(g,NV##d##r(i))&~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_REG_IDX_RD_DRF(g,d,r,i,f)              (((GPU_REG_RD32(g, NV ## d ## r(i)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_REG_2IDX_RD_DRF(g,d,r,i,j,f)             (((GPU_REG_RD32(g, NV ## d ## r(i, j)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+#define GPU_REG_RD_DRF_IDX(g,d,r,f,i)              (((GPU_REG_RD32(g, NV ## d ## r))>>GPU_DRF_SHIFT(NV ## d ## r ## f(i)))&GPU_DRF_MASK(NV ## d ## r ## f(i)))
+#define GPU_REG_IDX_OFFSET_RD_DRF(g,d,r,i,o,f)     (((GPU_REG_RD32(g, NV ## d ## r(i,o)))>>GPU_DRF_SHIFT(NV ## d ## r ## f))&GPU_DRF_MASK(NV ## d ## r ## f))
+
+//
+// Macros that abstract the use of bif object to access GPU bus config registers
+// This is the preferred set >= NV50
+//
+#define GPU_BUS_CFG_RD32(g,r,d)             gpuReadBusConfigReg_HAL(g, r, d)
+#define GPU_BUS_CFG_WR32(g,r,d)             gpuWriteBusConfigReg_HAL(g, r, d)
+#define GPU_BUS_CFG_FLD_WR_DRF_DEF(g,x,d,r,f,c) GPU_BUS_CFG_WR32(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+#define GPU_BUS_CFG_FLD_WR_DRF_NUM(g,x,d,r,f,n) GPU_BUS_CFG_WR32(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+
+#define GPU_BUS_CFG_RD32_EX(g,r,d,t)        gpuReadBusConfigRegEx_HAL(g, r, d, t)
+
+//
+// Macros that provide access to the config space of functions other than the gpu
+//
+#define PCI_FUNCTION_BUS_CFG_RD32(g,f,r,d)            gpuReadFunctionConfigReg_HAL(g, f, r, d)
+#define PCI_FUNCTION_BUS_CFG_WR32(g,f,r,d)            gpuWriteFunctionConfigReg_HAL(g, f, r, d)
+#define PCI_FUNCTION_BUS_CFG_FLD_WR_DRF_NUM(g,fn,x,d,r,f,n) gpuWriteFunctionConfigReg_HAL(g, fn, NV##d##r, (x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_NUM(d,r,f,n))
+#define PCI_FUNCTION_BUS_CFG_FLD_WR_DRF_DEF(g,fn,x,d,r,f,c) gpuWriteFunctionConfigReg_HAL(g, fn, NV##d##r, (x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+#define PCI_FUNCTION_BUS_CFG_WR32_EX(g,f,r,d,t)       gpuWriteFunctionConfigRegEx_HAL(g, f, r, d, t)
+
+#define GPU_BUS_CFG_CYCLE_RD32(g,r,d)                  gpuReadBusConfigCycle(g, r, d)
+#define GPU_BUS_CFG_CYCLE_WR32(g,r,d)                  gpuWriteBusConfigCycle(g, r, d)
+#define GPU_BUS_CFG_CYCLE_FLD_WR_DRF_DEF(g,x,d,r,f,c)  gpuWriteBusConfigCycle(g, NV##d##r,(x &~(GPU_DRF_MASK(NV##d##r##f)<<GPU_DRF_SHIFT(NV##d##r##f)))|GPU_DRF_DEF(d,r,f,c))
+
+//
+// Instance memory structure access definitions.
+//
+// DRF macros (nvmisc.h) should be used when possible instead of these
+// definitions.
+//
+// Key difference is SF variants take structure ## field (2-level nested
+// namespace), DRF take device ## register ## field (3-level nested
+// namespace).
+//
+// SF variants are primarily used for GPU host memory structures. DRF
+// should be used for manipulation of most registers
+//
+#define SF_INDEX(sf)            ((0?sf)/32)
+#define SF_OFFSET(sf)           (((0?sf)/32)<<2)
+#define SF_SHIFT(sf)            ((0?sf)&31)
+#undef  SF_MASK
+#define SF_MASK(sf)             (0xFFFFFFFF>>(31-(1?sf)+(0?sf)))
+#define SF_SHIFTMASK(sf)        (SF_MASK(sf) << SF_SHIFT(sf))
+#define SF_DEF(s,f,c)           ((NV ## s ## f ## c)<<SF_SHIFT(NV ## s ## f))
+#define SF_IDX_DEF(s,f,c,i)     ((NV ## s ## f ## c)<<SF_SHIFT(NV ## s ## f(i)))
+#define SF_NUM(s,f,n)           (((n)&SF_MASK(NV ## s ## f))<<SF_SHIFT(NV ## s ## f))
+#define SF_IDX_NUM(s,f,n,i)     (((n)&SF_MASK(NV ## s ## f(i)))<<SF_SHIFT(NV ## s ## f(i)))
+#define SF_VAL(s,f,v)           (((v)>>SF_SHIFT(NV ## s ## f))&SF_MASK(NV ## s ## f))
+#define SF_WIDTH(sf)            ((1?sf) - (0?sf) + 1)
+// This macro parses multi-word/array defines
+#define SF_ARR32_VAL(s,f,arr) \
+    (((arr)[SF_INDEX(NV ## s ## f)] >> SF_SHIFT(NV ## s ## f)) & SF_MASK(NV ## s ## f))
+#define FLD_SF_DEF(s,f,d,l)     ((l)&~(SF_MASK(NV##s##f) << SF_SHIFT(NV##s##f)))| SF_DEF(s,f,d)
+#define FLD_SF_NUM(s,f,n,l)     ((l)&~(SF_MASK(NV##s##f) << SF_SHIFT(NV##s##f)))| SF_NUM(s,f,n)
+#define FLD_SF_IDX_DEF(s,f,c,i,l) (((l) & ~SF_SHIFTMASK(NV ## s ## f(i))) | SF_IDX_DEF(s,f,c,i))
+#define FLD_SF_IDX_NUM(s,f,n,i,l) (((l) & ~SF_SHIFTMASK(NV ## s ## f(i))) | SF_IDX_NUM(s,f,n,i))
+
+#endif // _GPU_ACCESS_H_

src/nvidia/inc/kernel/gpu/gpu_child_list.h

@@ -0,0 +1,303 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+//
+// No include guards - this file is included multiple times, each time with a
+// different definition for GPU_CHILD_SINGLE_INST and GPU_CHILD_GPU_CHILD_MULTI_INST
+//
+// Callers that will use the same definition for single- and multi- instance
+// can define GPU_CHILD that will be used for both
+//
+#if defined(GPU_CHILD)
+#if !defined(GPU_CHILD_SINGLE_INST) && !defined(GPU_CHILD_MULTI_INST)
+#define GPU_CHILD_SINGLE_INST GPU_CHILD
+#define GPU_CHILD_MULTI_INST GPU_CHILD
+#else
+#error "Must not define GPU_CHILD_{SINGLE,MULTI}_INST and GPU_CHILD at the same time"
+#endif
+#endif
+
+//
+// GPU child list. All objects must inherit from OBJENGSTATE. Objects are
+// constructed in the listed order and destructed in reverse order. Storage in
+// OBJGPU and accessor macros (i.e.: GET_GPU_XXX) are generated from this list.
+//
+
+//
+// Temporarily needed to generate stubs for disabled modules
+// To be removed when the references to these modules are gone
+//
+#if defined(GPU_CHILD_LIST_DISABLED_ONLY)
+#define GPU_CHILD_MODULE(_rmcfgModule) !RMCFG_MODULE_ENABLED(_rmcfgModule)
+#else
+#define GPU_CHILD_MODULE(_rmcfgModule) RMCFG_MODULE_ENABLED(_rmcfgModule)
+#endif
+
+                        /* Class Name            Accessor  Name                       Max Instances     bConstructEarly  bAlwaysCreate  OBJGPU Field */
+#if GPU_CHILD_MODULE(FUSE)
+    GPU_CHILD_SINGLE_INST( OBJFUSE,              GPU_GET_FUSE,                        1,                NV_TRUE,         NV_TRUE,       pFuse            )
+#endif
+#if GPU_CHILD_MODULE(BIF)
+    GPU_CHILD_SINGLE_INST( OBJBIF,               GPU_GET_BIF,                         1,                NV_TRUE,         NV_FALSE,      pBif             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_BIF)
+    GPU_CHILD_SINGLE_INST( KernelBif,            GPU_GET_KERNEL_BIF,                  1,                NV_TRUE,         NV_FALSE,      pKernelBif       )
+#endif
+#if GPU_CHILD_MODULE(NNE)
+    GPU_CHILD_SINGLE_INST( OBJNNE,               GPU_GET_NNE,                         1,                NV_TRUE,         NV_FALSE,      pNne             )
+#endif
+#if GPU_CHILD_MODULE(MC)
+    GPU_CHILD_SINGLE_INST( OBJMC,                GPU_GET_MC,                          1,                NV_FALSE,        NV_FALSE,      pMc              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MC)
+    GPU_CHILD_SINGLE_INST( KernelMc,             GPU_GET_KERNEL_MC,                   1,                NV_FALSE,        NV_FALSE,      pKernelMc        )
+#endif
+#if GPU_CHILD_MODULE(PRIV_RING)
+    GPU_CHILD_SINGLE_INST( PrivRing,             GPU_GET_PRIV_RING,                   1,                NV_FALSE,        NV_FALSE,      pPrivRing        )
+#endif
+#if GPU_CHILD_MODULE(INTR)
+    GPU_CHILD_SINGLE_INST( SwIntr,               GPU_GET_SW_INTR,                     1,                NV_FALSE,        NV_FALSE,      pSwIntr          )
+#endif
+#if GPU_CHILD_MODULE(MEMORY_SYSTEM)
+    GPU_CHILD_SINGLE_INST( MemorySystem,         GPU_GET_MEMORY_SYSTEM,               1,                NV_FALSE,        NV_FALSE,      pMemorySystem    )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MEMORY_SYSTEM)
+    GPU_CHILD_SINGLE_INST( KernelMemorySystem,   GPU_GET_KERNEL_MEMORY_SYSTEM,        1,                NV_FALSE,        NV_FALSE,      pKernelMemorySystem )
+#endif
+#if GPU_CHILD_MODULE(MEMORY_MANAGER)
+    GPU_CHILD_SINGLE_INST( MemoryManager,        GPU_GET_MEMORY_MANAGER,              1,                NV_FALSE,        NV_FALSE,      pMemoryManager   )
+#endif
+#if GPU_CHILD_MODULE(FBFLCN)
+    GPU_CHILD_SINGLE_INST( OBJFBFLCN,            GPU_GET_FBFLCN,                      1,                NV_FALSE,        NV_FALSE,      pFbflcn          )
+#endif
+#if GPU_CHILD_MODULE(HSHUB)
+    GPU_CHILD_MULTI_INST ( OBJHSHUB,             GPU_GET_HSHUB,                       GPU_MAX_HSHUBS,   NV_FALSE,        NV_FALSE,      pHshub           )
+#endif
+#if GPU_CHILD_MODULE(SEQ)
+    GPU_CHILD_SINGLE_INST( OBJSEQ,               GPU_GET_SEQ,                         1,                NV_FALSE,        NV_TRUE,       pSeq             )
+#endif
+#if GPU_CHILD_MODULE(GpuMutexMgr)
+    GPU_CHILD_SINGLE_INST( GpuMutexMgr,          GPU_GET_MUTEX_MGR,                   1,                NV_FALSE,        NV_TRUE,       pMutexMgr        )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_DISPLAY)
+    GPU_CHILD_SINGLE_INST( KernelDisplay,        GPU_GET_KERNEL_DISPLAY,              1,                NV_FALSE,        NV_FALSE,      pKernelDisplay   )
+#endif
+#if GPU_CHILD_MODULE(DISP)
+    GPU_CHILD_SINGLE_INST( OBJDISP,              GPU_GET_DISP,                        1,                NV_FALSE,        NV_FALSE,      pDisp            )
+#endif
+#if GPU_CHILD_MODULE(TMR)
+    GPU_CHILD_SINGLE_INST( OBJTMR,               GPU_GET_TIMER,                       1,                NV_TRUE,         NV_TRUE,       pTmr             )
+#endif
+#if GPU_CHILD_MODULE(BUS)
+    GPU_CHILD_SINGLE_INST( OBJBUS,               GPU_GET_BUS,                         1,                NV_FALSE,        NV_FALSE,      pBus             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_BUS)
+    GPU_CHILD_SINGLE_INST( KernelBus,            GPU_GET_KERNEL_BUS,                  1,                NV_FALSE,        NV_FALSE,      pKernelBus       )
+#endif
+#if GPU_CHILD_MODULE(GMMU)
+    GPU_CHILD_SINGLE_INST( OBJGMMU,              GPU_GET_GMMU,                        1,                NV_FALSE,        NV_FALSE,      pGmmu            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GMMU)
+    GPU_CHILD_SINGLE_INST( KernelGmmu,           GPU_GET_KERNEL_GMMU,                 1,                NV_FALSE,        NV_FALSE,      pKernelGmmu      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_NVDEC)
+    GPU_CHILD_SINGLE_INST( KernelNvdec,          GPU_GET_KERNEL_NVDEC,                1,                NV_FALSE,        NV_FALSE,      pKernelNvdec     )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_SEC2)
+    GPU_CHILD_SINGLE_INST( KernelSec2,           GPU_GET_KERNEL_SEC2,                 1,                NV_FALSE,        NV_FALSE,      pKernelSec2      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GSP)
+    GPU_CHILD_SINGLE_INST( KernelGsp,            GPU_GET_KERNEL_GSP,                  1,                NV_FALSE,        NV_FALSE,      pKernelGsp       )
+#endif
+#if GPU_CHILD_MODULE(DCECLIENTRM)
+    GPU_CHILD_SINGLE_INST( OBJDCECLIENTRM,       GPU_GET_DCECLIENTRM,                 1,                NV_FALSE,        NV_FALSE,      pDceclientrm     )
+#endif
+#if GPU_CHILD_MODULE(VIRT_MEM_ALLOCATOR)
+    GPU_CHILD_SINGLE_INST( VirtMemAllocator,     GPU_GET_DMA,                         1,                NV_FALSE,        NV_FALSE,      pDma             )
+#endif
+#if GPU_CHILD_MODULE(GRMGR)
+    GPU_CHILD_SINGLE_INST( GraphicsManager,      GPU_GET_GRMGR,                       1,                NV_FALSE,        NV_TRUE,       pGrMgr           )
+#endif
+#if GPU_CHILD_MODULE(MIG_MANAGER)
+    GPU_CHILD_SINGLE_INST( MIGManager,           GPU_GET_MIG_MANAGER,                 1,                NV_FALSE,        NV_TRUE,       pMIGManager      )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_MIG_MANAGER)
+    GPU_CHILD_SINGLE_INST( KernelMIGManager,     GPU_GET_KERNEL_MIG_MANAGER,          1,                NV_FALSE,        NV_TRUE,       pKernelMIGManager )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GRAPHICS_MANAGER)
+    GPU_CHILD_SINGLE_INST( KernelGraphicsManager, GPU_GET_KERNEL_GRAPHICS_MANAGER,    1,                NV_FALSE,        NV_TRUE,       pKernelGraphicsManager )
+#endif
+#if GPU_CHILD_MODULE(GR)
+    GPU_CHILD_MULTI_INST ( Graphics,             GPU_GET_GR_UNSAFE,                   GPU_MAX_GRS,      NV_FALSE,        NV_FALSE,      pGr              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_GRAPHICS)
+    GPU_CHILD_MULTI_INST ( KernelGraphics,       GPU_GET_KERNEL_GRAPHICS,             GPU_MAX_GRS,      NV_FALSE,        NV_FALSE,      pKernelGraphics  )
+#endif
+#if GPU_CHILD_MODULE(ClockManager)
+    GPU_CHILD_SINGLE_INST( ClockManager,         GPU_GET_CLK_MGR,                     1,                NV_FALSE,        NV_FALSE,      pClk             )
+#endif
+#if GPU_CHILD_MODULE(FAN)
+    GPU_CHILD_SINGLE_INST( OBJFAN,               GPU_GET_FAN,                         1,                NV_FALSE,        NV_FALSE,      pFan             )
+#endif
+#if GPU_CHILD_MODULE(PERF)
+    GPU_CHILD_SINGLE_INST( Perf,                 GPU_GET_PERF,                        1,                NV_FALSE,        NV_FALSE,      pPerf            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_PERF)
+    GPU_CHILD_SINGLE_INST( KernelPerf,           GPU_GET_KERNEL_PERF,                 1,                NV_FALSE,        NV_FALSE,      pKernelPerf      )
+#endif
+#if GPU_CHILD_MODULE(THERM)
+    GPU_CHILD_SINGLE_INST( Therm,                GPU_GET_THERM,                       1,                NV_FALSE,        NV_FALSE,      pTherm           )
+#endif
+#if GPU_CHILD_MODULE(BSP)
+    GPU_CHILD_MULTI_INST ( OBJBSP,               GPU_GET_BSP,                         GPU_MAX_NVDECS,   NV_FALSE,        NV_FALSE,      pBsp             )
+#endif
+#if GPU_CHILD_MODULE(CIPHER)
+    GPU_CHILD_SINGLE_INST( OBJCIPHER,            GPU_GET_CIPHER,                      1,                NV_FALSE,        NV_FALSE,      pCipher          )
+#endif
+#if GPU_CHILD_MODULE(VBIOS)
+    GPU_CHILD_SINGLE_INST( OBJVBIOS,             GPU_GET_VBIOS,                       1,                NV_FALSE,        NV_TRUE,       pVbios           )
+#endif
+#if GPU_CHILD_MODULE(DCB)
+    GPU_CHILD_SINGLE_INST( OBJDCB,               GPU_GET_DCB,                         1,                NV_FALSE,        NV_TRUE,       pDcb             )
+#endif
+#if GPU_CHILD_MODULE(GPIO)
+    GPU_CHILD_SINGLE_INST( OBJGPIO,              GPU_GET_GPIO,                        1,                NV_FALSE,        NV_TRUE,       pGpio            )
+#endif
+#if GPU_CHILD_MODULE(VOLT)
+    GPU_CHILD_SINGLE_INST( OBJVOLT,              GPU_GET_VOLT,                        1,                NV_FALSE,        NV_FALSE,      pVolt            )
+#endif
+#if GPU_CHILD_MODULE(I2C)
+    GPU_CHILD_SINGLE_INST( OBJI2C,               GPU_GET_I2C,                         1,                NV_FALSE,        NV_TRUE,       pI2c             )
+#endif
+#if GPU_CHILD_MODULE(SPI)
+    GPU_CHILD_SINGLE_INST( Spi,               GPU_GET_SPI,                            1,                NV_FALSE,        NV_TRUE,       pSpi             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_RC)
+    GPU_CHILD_SINGLE_INST( KernelRc,             GPU_GET_KERNEL_RC,                   1,                NV_FALSE,        NV_TRUE,       pKernelRc        )
+#endif
+#if GPU_CHILD_MODULE(RC)
+    GPU_CHILD_SINGLE_INST( OBJRC,                GPU_GET_RC,                          1,                NV_FALSE,        NV_TRUE,       pRC              )
+#endif
+#if GPU_CHILD_MODULE(STEREO)
+    GPU_CHILD_SINGLE_INST( OBJSTEREO,            GPU_GET_STEREO,                      1,                NV_FALSE,        NV_TRUE,       pStereo          )
+#endif
+#if GPU_CHILD_MODULE(INTR)
+    GPU_CHILD_SINGLE_INST( Intr,                 GPU_GET_INTR,                        1,                NV_FALSE,        NV_TRUE,       pIntr            )
+#endif
+#if GPU_CHILD_MODULE(DPAUX)
+    GPU_CHILD_SINGLE_INST( OBJDPAUX,             GPU_GET_DPAUX,                       1,                NV_FALSE,        NV_FALSE,      pDpAux           )
+#endif
+#if GPU_CHILD_MODULE(PMU)
+    GPU_CHILD_SINGLE_INST( Pmu,                  GPU_GET_PMU,                         1,                NV_FALSE,        NV_FALSE,      pPmu             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_PMU)
+    GPU_CHILD_SINGLE_INST( KernelPmu,            GPU_GET_KERNEL_PMU,                  1,                NV_FALSE,        NV_FALSE,      pKernelPmu       )
+#endif
+#if GPU_CHILD_MODULE(CE)
+    GPU_CHILD_MULTI_INST ( OBJCE,                GPU_GET_CE,                          GPU_MAX_CES,      NV_FALSE,        NV_FALSE,      pCe              )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_CE)
+    GPU_CHILD_MULTI_INST ( KernelCE,             GPU_GET_KCE,                         GPU_MAX_CES,      NV_FALSE,        NV_FALSE,      pKCe              )
+#endif
+#if GPU_CHILD_MODULE(MSENC)
+    GPU_CHILD_MULTI_INST ( OBJMSENC,             GPU_GET_MSENC,                       GPU_MAX_MSENCS,   NV_FALSE,        NV_FALSE,      pMsenc           )
+#endif
+#if GPU_CHILD_MODULE(HDA)
+    GPU_CHILD_SINGLE_INST( OBJHDA,               GPU_GET_HDA,                         1,                NV_FALSE,        NV_FALSE,      pHda             )
+#endif
+#if GPU_CHILD_MODULE(HDACODEC)
+    GPU_CHILD_SINGLE_INST( OBJHDACODEC,          GPU_GET_HDACODEC,                    1,                NV_FALSE,        NV_FALSE,      pHdacodec        )
+#endif
+#if GPU_CHILD_MODULE(LPWR)
+    GPU_CHILD_SINGLE_INST( Lpwr,              GPU_GET_LPWR,                        1,                NV_FALSE,        NV_FALSE,      pLpwr            )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_FIFO)
+    GPU_CHILD_SINGLE_INST( KernelFifo,           GPU_GET_KERNEL_FIFO_UC,              1,                NV_FALSE,        NV_FALSE,      pKernelFifo      )
+#endif
+#if GPU_CHILD_MODULE(FIFO)
+    GPU_CHILD_SINGLE_INST( OBJFIFO,              GPU_GET_FIFO_UC,                     1,                NV_FALSE,        NV_FALSE,      pFifo            )
+#endif
+#if GPU_CHILD_MODULE(INFOROM)
+    GPU_CHILD_SINGLE_INST( OBJINFOROM,           GPU_GET_INFOROM,                     1,                NV_FALSE,        NV_TRUE,       pInforom         )
+#endif
+#if GPU_CHILD_MODULE(PMGR)
+    GPU_CHILD_SINGLE_INST( Pmgr,                 GPU_GET_PMGR,                        1,                NV_FALSE,        NV_FALSE,      pPmgr            )
+#endif
+#if GPU_CHILD_MODULE(UVM)
+    GPU_CHILD_SINGLE_INST( OBJUVM,               GPU_GET_UVM,                         1,                NV_FALSE,        NV_FALSE,      pUvm             )
+#endif
+#if GPU_CHILD_MODULE(NV_DEBUG_DUMP)
+    GPU_CHILD_SINGLE_INST( NvDebugDump,          GPU_GET_NVD,                         1,                NV_FALSE,        NV_TRUE,       pNvd             )
+#endif
+#if GPU_CHILD_MODULE(GRDBG)
+    GPU_CHILD_SINGLE_INST( SMDebugger,           GPU_GET_GRDBG,                       1,                NV_FALSE,        NV_TRUE,       pGrdbg           )
+#endif
+#if GPU_CHILD_MODULE(SEC2)
+    GPU_CHILD_SINGLE_INST( OBJSEC2,              GPU_GET_SEC2,                        1,                NV_FALSE,        NV_FALSE,      pSec2            )
+#endif
+#if GPU_CHILD_MODULE(LSFM)
+    GPU_CHILD_SINGLE_INST( OBJLSFM,              GPU_GET_LSFM,                        1,                NV_FALSE,        NV_FALSE,      pLsfm            )
+#endif
+#if GPU_CHILD_MODULE(ACR)
+    GPU_CHILD_SINGLE_INST( OBJACR,               GPU_GET_ACR,                         1,                NV_FALSE,        NV_FALSE,      pAcr             )
+#endif
+#if GPU_CHILD_MODULE(KERNEL_NVLINK)
+    GPU_CHILD_SINGLE_INST( KernelNvlink,         GPU_GET_KERNEL_NVLINK,               1,                NV_FALSE,        NV_FALSE,      pKernelNvlink    )
+#endif
+#if GPU_CHILD_MODULE(NVLINK)
+    GPU_CHILD_SINGLE_INST( Nvlink,               GPU_GET_NVLINK,                      1,                NV_FALSE,        NV_FALSE,      pNvLink          )
+#endif
+#if GPU_CHILD_MODULE(GPULOG)
+    GPU_CHILD_SINGLE_INST( OBJGPULOG,            GPU_GET_GPULOG,                      1,                NV_FALSE,        NV_TRUE,       pGpuLog          )
+#endif
+#if GPU_CHILD_MODULE(GPUMON)
+    GPU_CHILD_SINGLE_INST( OBJGPUMON,            GPU_GET_GPUMON,                      1,                NV_FALSE,        NV_TRUE,       pGpuMon          )
+#endif
+#if GPU_CHILD_MODULE(HWPM)
+    GPU_CHILD_SINGLE_INST( OBJHWPM,              GPU_GET_HWPM,                        1,                NV_FALSE,        NV_FALSE,      pHwpm            )
+#endif
+#if GPU_CHILD_MODULE(GRIDDISPLAYLESS)
+    GPU_CHILD_SINGLE_INST( OBJGRIDDISPLAYLESS,   GPU_GET_GRIDDISPLAYLESS,             1,                NV_FALSE,        NV_FALSE,      pGridDisplayless )
+#endif
+#if GPU_CHILD_MODULE(SWENG)
+    GPU_CHILD_SINGLE_INST( OBJSWENG,             GPU_GET_SWENG,                       1,                NV_FALSE,        NV_FALSE,      pSwEng           )
+#endif
+#if GPU_CHILD_MODULE(VMMU)
+    GPU_CHILD_SINGLE_INST( OBJVMMU,              GPU_GET_VMMU,                        1,                NV_FALSE,        NV_FALSE,      pVmmu            )
+#endif
+#if GPU_CHILD_MODULE(NVJPG)
+    GPU_CHILD_MULTI_INST( OBJNVJPG,              GPU_GET_NVJPG,                       GPU_MAX_NVJPGS,   NV_FALSE,        NV_FALSE,      pNvjpg           )
+#endif
+#if GPU_CHILD_MODULE(GSP)
+    GPU_CHILD_SINGLE_INST( Gsp,                  GPU_GET_GSP,                         1,                NV_FALSE,        NV_FALSE,      pGsp             )
+#endif
+#if GPU_CHILD_MODULE(OFA)
+    GPU_CHILD_SINGLE_INST( OBJOFA,               GPU_GET_OFA,                         1,                NV_FALSE,        NV_FALSE,      pOfa             )
+#endif
+
+// Undefine the entry macros to simplify call sites
+#undef GPU_CHILD
+#undef GPU_CHILD_SINGLE_INST
+#undef GPU_CHILD_MULTI_INST
+#undef GPU_CHILD_MODULE
+#undef GPU_CHILD_LIST_DISABLED_ONLY

src/nvidia/inc/kernel/gpu/gpu_device_mapping.h

@@ -0,0 +1,62 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_DEVICE_MAPPING_H_
+#define _GPU_DEVICE_MAPPING_H_
+
+// Defines the enum type DEVICE_INDEX used for identifying the device type being accessed
+typedef enum
+{
+    DEVICE_INDEX_GPU = 0,
+    DEVICE_INDEX_HOST1X,
+    DEVICE_INDEX_DISPLAY,
+    DEVICE_INDEX_DPAUX,
+    DEVICE_INDEX_MC,
+    DEVICE_INDEX_CLKRST,
+    DEVICE_INDEX_MSS_NVLINK,
+    DEVICE_INDEX_HDACODEC,
+    DEVICE_INDEX_EMC,
+    DEVICE_INDEX_FUSE,
+    DEVICE_INDEX_KFUSE,
+    DEVICE_INDEX_MIPICAL,
+    DEVICE_INDEX_MAX  //Should always be the last entry
+} DEVICE_INDEX;
+
+typedef enum
+{
+    SOC_DEV_MAPPING_DISP = 0,
+    SOC_DEV_MAPPING_DPAUX0,
+    SOC_DEV_MAPPING_DPAUX1,  // Update NV_MAX_SOC_DPAUX_NUM_DEVICES if adding new DPAUX mappings
+    SOC_DEV_MAPPING_HDACODEC,
+    SOC_DEV_MAPPING_MIPICAL,
+    SOC_DEV_MAPPING_MAX       // Keep this as last entry
+} SOC_DEV_MAPPING;
+
+#define GPU_MAX_DEVICE_MAPPINGS                   (60)
+
+typedef struct
+{
+    DEVICE_INDEX deviceIndex;    // DEVICE_INDEX_*
+    NvU32        devId;          // NV_DEVID_*
+} DEVICE_ID_MAPPING;
+
+#endif // _GPU_DEVICE_MAPPING_H_

src/nvidia/inc/kernel/gpu/gpu_halspec.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_halspec_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_resource.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_resource_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gpu_resource_desc.h

@@ -0,0 +1,37 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_RESOURCE_DESC_H_
+#define _GPU_RESOURCE_DESC_H_
+
+#include "gpu/eng_desc.h"
+
+typedef struct GPU_RESOURCE_DESC
+{
+    NvU32          externalClassId;
+    ENGDESCRIPTOR  engDesc;
+} GPU_RESOURCE_DESC;
+
+// CLASSDESCRIPTOR is deprecated, please use GPU_RESOURCE_DESC
+typedef struct GPU_RESOURCE_DESC CLASSDESCRIPTOR, *PCLASSDESCRIPTOR;
+
+#endif // _GPU_RESOURCE_DESC_H_

src/nvidia/inc/kernel/gpu/gpu_timeout.h

@@ -0,0 +1,144 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2004-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPU_TIMEOUT_H_
+#define _GPU_TIMEOUT_H_
+
+/* ------------------------ Includes ---------------------------------------- */
+#include "core/core.h"
+
+
+/* ------------------------ Forward Definitions ----------------------------- */
+struct OBJGPU;
+
+/* ------------------------ Macros ------------------------------------------ */
+/*!
+ * @note    GPU_TIMEOUT_DEFAULT is different per platform and can range anywhere
+ *          from 2 to 30 secs depending on the GPU Mode and Platform.
+ *          By default if GPU_TIMEOUT_DEFAULT is specified, we use the ThreadState
+ *          unless explicitly told not to via GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE.
+ */
+#define GPU_TIMEOUT_DEFAULT                 0
+
+/*!
+ * gpuSetTimeout Flags - saved in pTimeout->flags
+ */
+#define GPU_TIMEOUT_FLAGS_DEFAULT               NVBIT(0)  //!< default timeout mechanism as set by platform
+#define GPU_TIMEOUT_FLAGS_USE_THREAD_STATE      NVBIT(1)  //!< default timeout time used - use the ThreadState
+#define GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE   NVBIT(2)  //!< even if default time was used - skip the ThreadState
+#define GPU_TIMEOUT_FLAGS_OSTIMER               NVBIT(3)  //!< osGetCurrentTime()
+#define GPU_TIMEOUT_FLAGS_OSDELAY               NVBIT(4)  //!< osDelay()
+#define GPU_TIMEOUT_FLAGS_TMR                   NVBIT(5)  //!< tmrGetCurrentTime()
+#define GPU_TIMEOUT_FLAGS_BYPASS_JOURNAL_LOG    NVBIT(6)  //!< bypass timeout logging in the RM journal
+#define GPU_TIMEOUT_FLAGS_TMRDELAY              NVBIT(7)  //!< tmrDelay()
+#define GPU_TIMEOUT_FLAGS_BYPASS_CPU_YIELD      NVBIT(8)  //!< don't explicitly let other threads run first
+/*!
+ * gpuCheckTimeout Flags set in pTimeout->flags upon NV_ERR_TIMEOUT
+ */
+#define GPU_TIMEOUT_FLAGS_STATUS_LOCAL_TIMEOUT          NVBIT(30)
+#define GPU_TIMEOUT_FLAGS_STATUS_THREAD_STATE_TIMEOUT   NVBIT(31)
+
+/* ------------------------ Datatypes --------------------------------------- */
+/*!
+ * Timeout support.
+ */
+typedef struct
+{
+    NvU64   timeout;
+    NvU32   flags;
+    OBJGPU *pTmrGpu;        //!< The GPU whose timer is used in SLI mode
+                            //   Defined only if flags is set to _TMR or _TMRDELAY
+} RMTIMEOUT,
+*PRMTIMEOUT;
+
+/*!
+ * @brief   GPU timeout related data.
+ */
+typedef struct
+{
+    volatile NvBool bDefaultOverridden;
+    volatile NvBool bScaled;
+    volatile NvU32  defaultus;          //!< Default timeout in us
+    volatile NvU32  defaultResetus;     //!< Default timeout reset value in us
+    NvU32           defaultFlags;       //!< Default timeout mode
+    NvU32           scale;              //!< Emulation/Simulation multiplier
+    OBJGPU         *pGpu;
+} TIMEOUT_DATA;
+
+/*!
+ * @brief   A prototype of the condition evaluation function required by the
+ *          @ref gpuTimeoutCondWait_IMPL interface.
+ *
+ * @note    Function is responsible for evaluation of the encapsulated condition
+ *          as well as for triggering of required prerequisites (if any).
+ *          For example if condition depends on a PMU issued message function
+ *          should assure proper servicing of the PMU interrupts.
+ *
+ * @param[in] pGpu    OBJGPU pointer for this conditional function
+ * @param[in] pVoid
+ *     Void parameter pointer which can be used to pass in the
+ *     pCondData from @ref gpuTimeoutCondWait_IMPL().
+ *
+ * @return NV_TRUE
+ *     Waited condition has happened and @ref
+ *     gpuTimeoutCondWait_IMPL() may return to caller.
+ * @return NV_FALSE
+ *     Waited condition has not happened and @ref
+ *     gpuTimeoutCondWait_IMPL() should continue to wait until this
+ *     interface returns NV_TRUE or timeout occurs (whichever occurs
+ *     first).
+ */
+typedef NvBool GpuWaitConditionFunc(OBJGPU *pGpu, void *pVoid);
+
+/* ------------------------ Function Prototypes ----------------------------- */
+
+void timeoutInitializeGpuDefault(TIMEOUT_DATA *pTD, OBJGPU *pGpu);
+
+void timeoutRegistryOverride(TIMEOUT_DATA *pTD, OBJGPU *pGpu);
+
+void timeoutOverride(TIMEOUT_DATA *pTD, NvBool bOverride, NvU32 timeoutMs);
+
+/*! Initialize the RMTIMEOUT structure with the selected timeout scheme. */
+void timeoutSet(TIMEOUT_DATA *, RMTIMEOUT *, NvU32 timeoutUs, NvU32 flags);
+
+/*! Check if the passed in RMTIMEOUT struct has expired. */
+NV_STATUS timeoutCheck(TIMEOUT_DATA *, RMTIMEOUT *, NvU32);
+
+/*! Wait for the condition to become satisfied while checking for the timeout */
+NV_STATUS timeoutCondWait(TIMEOUT_DATA *, RMTIMEOUT *, GpuWaitConditionFunc *, void *pCondData, NvU32);
+
+/*! Scales timeout values depending on the environment we are running in. */
+static NV_INLINE NvU32 timeoutApplyScale(TIMEOUT_DATA *pTD, NvU32 timeout)
+{
+    return timeout * pTD->scale;
+}
+
+
+// Deprecated macros
+#define gpuSetTimeout(g,a,t,c)       timeoutSet(&(g)->timeoutData, t, a, c)
+#define gpuCheckTimeout(g,t)         timeoutCheck(&(g)->timeoutData, t, __LINE__)
+#define gpuScaleTimeout(g,a)         timeoutApplyScale(&(g)->timeoutData, a)
+#define gpuTimeoutCondWait(g,a,b,t)  timeoutCondWait(&(g)->timeoutData, t, a, b, __LINE__)
+
+#define GPU_ENG_RESET_TIMEOUT_VALUE(g, t) (t)
+
+#endif // _GPU_TIMEOUT_H_

src/nvidia/inc/kernel/gpu/gpu_uuid.h

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2018-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _GPUUUID_H_
+#define _GPUUUID_H_
+
+#include "core/core.h"
+#include "nvCpuUuid.h"
+
+//
+// GPU unique ID sizes.  RM_SHA1_GID_SIZE uses the first 16 bytes of
+// the SHA-1 digest (this is consistent with the way canonical UUIDs are
+// constructed)
+//
+#define RM_SHA1_GID_SIZE         16
+
+// UUID conversion routine:
+NV_STATUS transformGidToUserFriendlyString(const NvU8 *pGidData, NvU32 gidSize, NvU8 **ppGidString,
+                                           NvU32 *pGidStrlen, NvU32 gidFlags);
+
+NV_STATUS nvGenerateGpuUuid(NvU16 chipId, NvU64 pdi, NvUuid *pUuid);
+
+NV_STATUS nvGenerateSmcUuid(NvU16 chipId, NvU64 pdi,
+                            NvU32 swizzId, NvU32 syspipeId, NvUuid *pUuid);
+
+// 'G' 'P' 'U' '-'(x5), '\0x0', extra = 9
+#define NV_UUID_STR_LEN ((NV_UUID_LEN << 1) + 9)
+
+void nvGetSmcUuidString(const NvUuid *pUuid, char *pUuidStr);
+
+void nvGetGpuUuidString(const NvUuid *pUuid, char *pUuidStr);
+
+#endif // _GPUUUID_H_

src/nvidia/inc/kernel/gpu/gr/fecs_event_list.h

@@ -0,0 +1,3 @@
+
+#include "g_fecs_event_list_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_context.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_context_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_manager.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_manager_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_graphics_object.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_graphics_object_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gr/kernel_sm_debugger_session.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_sm_debugger_session_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GSP_INIT_ARGS_H
+#define GSP_INIT_ARGS_H
+
+#include "core/core.h"
+
+typedef struct {
+    RmPhysAddr cmdQueuePhysAddr;
+} GSP_RMFS_INIT_ARGUMENTS;
+
+typedef struct {
+    RmPhysAddr sharedMemPhysAddr;
+    NvU32 pageTableEntryCount;
+    NvLength cmdQueueOffset;
+    NvLength statQueueOffset;
+} MESSAGE_QUEUE_INIT_ARGUMENTS;
+
+typedef struct {
+    NvU32 oldLevel;
+    NvU32 flags;
+    NvBool bInPMTransition;
+} GSP_SR_INIT_ARGUMENTS;
+
+/*!
+ * (Cached) GSP fw RM initialization arguments.
+ */
+typedef struct
+{
+    MESSAGE_QUEUE_INIT_ARGUMENTS      messageQueueInitArguments;
+    GSP_SR_INIT_ARGUMENTS             srInitArguments;
+} GSP_ARGUMENTS_CACHED;
+
+#endif // GSP_INIT_ARGS_H

src/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h

@@ -0,0 +1,145 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GSP_STATIC_CONFIG_H
+#define GSP_STATIC_CONFIG_H
+
+//
+// This header describes the set of static GPU configuration information
+// that is collected during GSP RM init and made available to the
+// CPU RM (aka GSP client) via the NV_RM_RPC_GET_STATIC_INFO() and
+// NV_RM_RPC_GET_GSP_STATIC_INFO() calls.
+
+#include "ctrl/ctrl0080/ctrl0080gpu.h"
+#include "ctrl/ctrl0080/ctrl0080gr.h"
+#include "ctrl/ctrl2080/ctrl2080bios.h"
+#include "ctrl/ctrl2080/ctrl2080fb.h"
+#include "ctrl/ctrl2080/ctrl2080gpu.h"
+
+#include "gpu/gpu.h" // COMPUTE_BRANDING_TYPE
+#include "vgpu/rpc_headers.h" // MAX_GPC_COUNT
+#include "platform/chipset/chipset.h" // BUSINFO
+
+typedef struct GspSMInfo_t
+{
+    NvU32 version;
+    NvU32 regBankCount;
+    NvU32 regBankRegCount;
+    NvU32 maxWarpsPerSM;
+    NvU32 maxThreadsPerWarp;
+    NvU32 geomGsObufEntries;
+    NvU32 geomXbufEntries;
+    NvU32 maxSPPerSM;
+    NvU32 rtCoreCount;
+} GspSMInfo;
+
+// Fetched from GSP-RM into CPU-RM
+typedef struct GspStaticConfigInfo_t
+{
+    NvU8 grCapsBits[NV0080_CTRL_GR_CAPS_TBL_SIZE];
+    NV2080_CTRL_GPU_GET_GID_INFO_PARAMS gidInfo;
+    NV2080_CTRL_GPU_GET_FERMI_GPC_INFO_PARAMS gpcInfo;
+    NV2080_CTRL_GPU_GET_FERMI_TPC_INFO_PARAMS tpcInfo[MAX_GPC_COUNT];
+    NV2080_CTRL_GPU_GET_FERMI_ZCULL_INFO_PARAMS zcullInfo[MAX_GPC_COUNT];
+    NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS SKUInfo;
+    NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS fbRegionInfoParams;
+    COMPUTE_BRANDING_TYPE computeBranding;
+
+    NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS sriovCaps;
+    NvU32 sriovMaxGfid;
+
+    NvU64 engineCaps;
+
+    GspSMInfo SM_info;
+
+    NvBool poisonFuseEnabled;
+  
+    NvU64 fb_length;
+    NvU32 fbio_mask;
+    NvU32 fb_bus_width;
+    NvU32 fb_ram_type;
+    NvU32 fbp_mask;
+    NvU32 l2_cache_size;
+
+    NvU32 gfxpBufferSize[NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL];
+    NvU32 gfxpBufferAlignment[NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND_BUFFERS_CONTEXT_POOL];
+
+    NvU8 gpuNameString[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvU8 gpuShortNameString[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvU16 gpuNameString_Unicode[NV2080_GPU_MAX_NAME_STRING_LENGTH];
+    NvBool bGpuInternalSku;
+    NvBool bIsQuadroGeneric;
+    NvBool bIsQuadroAd;
+    NvBool bIsNvidiaNvs;
+    NvBool bIsVgx;
+    NvBool bGeforceSmb;
+    NvBool bIsTitan;
+    NvBool bIsTesla;
+
+    NvU64 bar1PdeBase;
+    NvU64 bar2PdeBase;
+
+    NvBool bVbiosValid;
+    NvU32 vbiosSubVendor;
+    NvU32 vbiosSubDevice;
+
+    NvBool bPageRetirementSupported;
+
+    NvBool bSplitVasBetweenServerClientRm;
+
+    NvBool bClRootportNeedsNosnoopWAR;
+
+    VIRTUAL_DISPLAY_GET_NUM_HEADS_PARAMS displaylessMaxHeads;
+    VIRTUAL_DISPLAY_GET_MAX_RESOLUTION_PARAMS displaylessMaxResolution;
+    NvU64 displaylessMaxPixels;
+
+    // Client handle for internal RMAPI control.
+    NvHandle hInternalClient;
+
+    // Device handle for internal RMAPI control.
+    NvHandle hInternalDevice;
+
+    // Subdevice handle for internal RMAPI control.
+    NvHandle hInternalSubdevice;
+} GspStaticConfigInfo;
+
+// Pushed from CPU-RM to GSP-RM
+typedef struct GspSystemInfo
+{
+    NvU64 gpuPhysAddr;
+    NvU64 gpuPhysFbAddr;
+    NvU64 gpuPhysInstAddr;
+    NvU64 nvDomainBusDeviceFunc;
+    NvU64 simAccessBufPhysAddr;
+    NvU64 pcieAtomicsOpMask;
+    NvU64 consoleMemSize;
+    NvU32 pciConfigMirrorBase;
+    NvU32 pciConfigMirrorSize;
+    NvU8 oorArch;
+    NvU64 clPdbProperties;
+    NvU32 Chipset;
+    BUSINFO FHBBusInfo;
+} GspSystemInfo;
+
+
+#endif /* GSP_STATIC_CONFIG_H */

src/nvidia/inc/kernel/gpu/gsp/kernel_gsp.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_gsp_nvoc.h"
+

src/nvidia/inc/kernel/gpu/gsp/message_queue.h

@@ -0,0 +1,39 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+/*
+ * GSP MESSAGE QUEUE
+ */
+
+#ifndef _MESSAGE_QUEUE_H_
+#define _MESSAGE_QUEUE_H_
+
+typedef struct _message_queue_info MESSAGE_QUEUE_INFO;
+
+// CPU-side calls
+NV_STATUS GspMsgQueueInit(OBJGPU *pGpu, MESSAGE_QUEUE_INFO **ppMQI);
+NV_STATUS GspStatusQueueInit(OBJGPU *pGpu, MESSAGE_QUEUE_INFO **ppMQI);
+void GspMsgQueueCleanup(MESSAGE_QUEUE_INFO **ppMQI);
+NV_STATUS GspMsgQueueSendCommand(MESSAGE_QUEUE_INFO *pMQI, OBJGPU *pGpu);
+NV_STATUS GspMsgQueueReceiveStatus(MESSAGE_QUEUE_INFO *pMQI);
+
+#endif // _MESSAGE_QUEUE_H_

src/nvidia/inc/kernel/gpu/gsp/message_queue_priv.h

@@ -0,0 +1,95 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * GSP MESSAGE QUEUE
+ */
+
+#ifndef _MESSAGE_QUEUE_PRIV_H_
+#define _MESSAGE_QUEUE_PRIV_H_
+
+#include "msgq/msgq.h"
+#include "gpu/mem_mgr/virt_mem_allocator_common.h"
+
+// Shared memory layout.
+//
+// Each of the following are page aligned:
+//   Page table for entire shared memory layout.
+//   Command queue header
+//   Command queue entries
+//   Status queue header
+//   Status queue entries
+
+typedef struct GSP_MSG_QUEUE_ELEMENT
+{
+    NvU32 checkSum;        // Set to value needed to make checksum always zero.
+    NvU32 seqNum;          // Sequence number maintained by the message queue.
+    rpc_message_header_v  rpc;
+} GSP_MSG_QUEUE_ELEMENT;
+
+typedef struct _message_queue_info
+{
+    // Parameters
+    NvLength               pageTableEntryCount;
+    NvLength               pageTableSize;
+    NvLength               commandQueueSize;
+    NvLength               statusQueueSize;
+
+    // Shared memory area.
+    MEMORY_DESCRIPTOR     *pSharedMemDesc;
+    RmPhysAddr             sharedMemPA;   // Page table for all of shared mem.
+    void                  *pCommandQueue;
+    void                  *pStatusQueue;
+    rpc_message_header_v  *pRpcMsgBuf;    // RPC message buffer VA.
+
+    void                  *pInitMsgBuf;   // RPC message buffer VA.
+    RmPhysAddr             initMsgBufPA;  // RPC message buffer PA.
+
+    // Other CPU-side fields
+    void                  *pWorkArea;
+    GSP_MSG_QUEUE_ELEMENT *pCmdQueueElement;    // Working copy of command queue element.
+    void                  *pMetaData;
+    msgqHandle             hQueue;              // Do not allow requests when hQueue is null.
+    NvU32                  txSeqNum;            // Next sequence number for tx.
+    NvU32                  rxSeqNum;            // Next sequence number for rx.
+} MESSAGE_QUEUE_INFO;
+
+//
+// Most of the following defines resolve to compile-time constants.
+//
+#define GSP_MSG_QUEUE_ELEMENT_SIZE_MIN                              RM_PAGE_SIZE
+#define GSP_MSG_QUEUE_ELEMENT_SIZE_MAX     (GSP_MSG_QUEUE_ELEMENT_SIZE_MIN * 16)
+#define GSP_MSG_QUEUE_ELEMENT_HDR_SIZE   NV_OFFSETOF(GSP_MSG_QUEUE_ELEMENT, rpc)
+
+#define GSP_MSG_QUEUE_RPC_SIZE_MAX                                             \
+    (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX - GSP_MSG_QUEUE_ELEMENT_HDR_SIZE)
+
+#define GSP_MSG_QUEUE_BYTES_TO_ELEMENTS(b)                                     \
+        NV_DIV_AND_CEIL(b, GSP_MSG_QUEUE_ELEMENT_SIZE_MIN)
+
+#define GSP_MSG_QUEUE_ALIGN                                        RM_PAGE_SHIFT   // 2 ^ 12 = 4096
+#define GSP_MSG_QUEUE_ELEMENT_ALIGN                                RM_PAGE_SHIFT   // 2 ^ 12 = 4096
+#define GSP_MSG_QUEUE_HEADER_SIZE                                   RM_PAGE_SIZE
+#define GSP_MSG_QUEUE_HEADER_ALIGN                                             4   // 2 ^ 4 = 16
+
+#endif // _MESSAGE_QUEUE_PRIV_H_

src/nvidia/inc/kernel/gpu/host_eng/host_eng.h

@@ -0,0 +1,3 @@
+
+#include "g_host_eng_nvoc.h"
+

src/nvidia/inc/kernel/gpu/hwpm/profiler_v1.h

@@ -0,0 +1,3 @@
+
+#include "g_profiler_v1_nvoc.h"
+

src/nvidia/inc/kernel/gpu/hwpm/profiler_v2.h

@@ -0,0 +1,3 @@
+
+#include "g_profiler_v2_nvoc.h"
+

src/nvidia/inc/kernel/gpu/i2c/i2c_api.h

@@ -0,0 +1,3 @@
+
+#include "g_i2c_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/engine_idx.h

@@ -0,0 +1,164 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ENGINE_IDX_H
+#define ENGINE_IDX_H
+
+#include "utils/nvbitvector.h"
+
+/***************************************************************************\
+*                                                                           *
+* Module: engine_idx.h
+*       List of engines for use by INTR (and MC) modules.
+*                                                                           *
+\***************************************************************************/
+
+//
+// Engine bits for use by various MC HAL routines
+//
+#define MC_ENGINE_IDX_NULL                          0 // This must be 0
+#define MC_ENGINE_IDX_TMR                           1
+#define MC_ENGINE_IDX_DISP                          2
+#define MC_ENGINE_IDX_FB                            3
+#define MC_ENGINE_IDX_FIFO                          4
+#define MC_ENGINE_IDX_VIDEO                         5
+#define MC_ENGINE_IDX_MD                            6
+#define MC_ENGINE_IDX_BUS                           7
+// UNUSED
+#define MC_ENGINE_IDX_PMGR                          9
+#define MC_ENGINE_IDX_VP2                           10
+#define MC_ENGINE_IDX_CIPHER                        11
+#define MC_ENGINE_IDX_BIF                           12
+#define MC_ENGINE_IDX_PPP                           13
+#define MC_ENGINE_IDX_PRIVRING                      14
+#define MC_ENGINE_IDX_PMU                           15
+#define MC_ENGINE_IDX_CE0                           16
+#define MC_ENGINE_IDX_CE1                           17
+#define MC_ENGINE_IDX_CE2                           18
+#define MC_ENGINE_IDX_CE3                           19
+#define MC_ENGINE_IDX_CE4                           20
+#define MC_ENGINE_IDX_CE5                           21
+#define MC_ENGINE_IDX_CE6                           22
+#define MC_ENGINE_IDX_CE7                           23
+#define MC_ENGINE_IDX_CE8                           24
+#define MC_ENGINE_IDX_CE9                           25
+#define MC_ENGINE_IDX_VIC                           26
+#define MC_ENGINE_IDX_ISOHUB                        27
+#define MC_ENGINE_IDX_VGPU                          28
+#define MC_ENGINE_IDX_MSENC                         29
+#define MC_ENGINE_IDX_MSENC1                        30
+#define MC_ENGINE_IDX_MSENC2                        31
+#define MC_ENGINE_IDX_C2C                           32
+// UNUSED
+#define MC_ENGINE_IDX_LTC                           34
+#define MC_ENGINE_IDX_FBHUB                         35
+#define MC_ENGINE_IDX_HDACODEC                      36
+#define MC_ENGINE_IDX_GMMU                          37
+#define MC_ENGINE_IDX_SEC2                          38
+#define MC_ENGINE_IDX_FSP                           39
+#define MC_ENGINE_IDX_NVLINK                        40
+#define MC_ENGINE_IDX_GSP                           41
+#define MC_ENGINE_IDX_NVJPG                         42
+#define MC_ENGINE_IDX_NVJPEG                        MC_ENGINE_IDX_NVJPG
+#define MC_ENGINE_IDX_NVJPEG0                       MC_ENGINE_IDX_NVJPEG
+#define MC_ENGINE_IDX_RESERVED43                    43
+#define MC_ENGINE_IDX_RESERVED44                    44
+#define MC_ENGINE_IDX_RESERVED45                    45
+#define MC_ENGINE_IDX_RESERVED46                    46
+#define MC_ENGINE_IDX_RESERVED47                    47
+#define MC_ENGINE_IDX_RESERVED48                    48
+#define MC_ENGINE_IDX_RESERVED49                    49
+#define MC_ENGINE_IDX_REPLAYABLE_FAULT              50
+#define MC_ENGINE_IDX_ACCESS_CNTR                   51
+#define MC_ENGINE_IDX_NON_REPLAYABLE_FAULT          52
+#define MC_ENGINE_IDX_REPLAYABLE_FAULT_ERROR        53
+#define MC_ENGINE_IDX_NON_REPLAYABLE_FAULT_ERROR    54
+#define MC_ENGINE_IDX_INFO_FAULT                    55
+#define MC_ENGINE_IDX_BSP                           56
+#define MC_ENGINE_IDX_NVDEC                         MC_ENGINE_IDX_BSP
+#define MC_ENGINE_IDX_NVDEC0                        MC_ENGINE_IDX_NVDEC
+#define MC_ENGINE_IDX_NVDEC1                        57
+#define MC_ENGINE_IDX_NVDEC2                        58
+#define MC_ENGINE_IDX_NVDEC3                        59
+#define MC_ENGINE_IDX_NVDEC4                        60
+#define MC_ENGINE_IDX_RESERVED61                    61
+#define MC_ENGINE_IDX_RESERVED62                    62
+#define MC_ENGINE_IDX_RESERVED63                    63
+#define MC_ENGINE_IDX_CPU_DOORBELL                  64
+#define MC_ENGINE_IDX_PRIV_DOORBELL                 65
+#define MC_ENGINE_IDX_MMU_ECC_ERROR                 66
+#define MC_ENGINE_IDX_BLG                           67
+#define MC_ENGINE_IDX_PERFMON                       68
+#define MC_ENGINE_IDX_BUF_RESET                     69
+#define MC_ENGINE_IDX_XBAR                          70
+#define MC_ENGINE_IDX_ZPW                           71
+#define MC_ENGINE_IDX_OFA0                          72
+#define MC_ENGINE_IDX_TEGRA                         73
+#define MC_ENGINE_IDX_GR                            74
+#define MC_ENGINE_IDX_GR0                           MC_ENGINE_IDX_GR
+#define MC_ENGINE_IDX_GR1                           75
+#define MC_ENGINE_IDX_GR2                           76
+#define MC_ENGINE_IDX_GR3                           77
+#define MC_ENGINE_IDX_GR4                           78
+#define MC_ENGINE_IDX_GR5                           79
+#define MC_ENGINE_IDX_GR6                           80
+#define MC_ENGINE_IDX_GR7                           81
+#define MC_ENGINE_IDX_ESCHED                        82
+#define MC_ENGINE_IDX_ESCHED__SIZE                  64
+#define MC_ENGINE_IDX_GR_FECS_LOG                   146
+#define MC_ENGINE_IDX_GR0_FECS_LOG                  MC_ENGINE_IDX_GR_FECS_LOG
+#define MC_ENGINE_IDX_GR1_FECS_LOG                  147
+#define MC_ENGINE_IDX_GR2_FECS_LOG                  148
+#define MC_ENGINE_IDX_GR3_FECS_LOG                  149
+#define MC_ENGINE_IDX_GR4_FECS_LOG                  150
+#define MC_ENGINE_IDX_GR5_FECS_LOG                  151
+#define MC_ENGINE_IDX_GR6_FECS_LOG                  152
+#define MC_ENGINE_IDX_GR7_FECS_LOG                  153
+#define MC_ENGINE_IDX_TMR_SWRL                      154
+#define MC_ENGINE_IDX_MAX                           155 // This must be kept as the max bit if
+                                                        // we need to add more engines
+#define MC_ENGINE_IDX_INVALID                0xFFFFFFFF
+
+// Index GR reference
+#define MC_ENGINE_IDX_GRn(x)            (MC_ENGINE_IDX_GR0 + (x))
+#define MC_ENGINE_IDX_GRn_FECS_LOG(x)   (MC_ENGINE_IDX_GR0_FECS_LOG + (x))
+
+// Index CE reference
+#define MC_ENGINE_IDX_CE(x)             (MC_ENGINE_IDX_CE0 + (x))
+
+// Index MSENC reference
+#define MC_ENGINE_IDX_MSENCn(x)         (MC_ENGINE_IDX_MSENC + (x))
+
+// Index NVDEC reference
+#define MC_ENGINE_IDX_NVDECn(x)         (MC_ENGINE_IDX_NVDEC + (x))
+
+// Index NVJPEG reference
+#define MC_ENGINE_IDX_NVJPEGn(x)        (MC_ENGINE_IDX_NVJPEG + (x))
+
+// Index ESCHED reference
+#define MC_ENGINE_IDX_ESCHEDn(x)        (MC_ENGINE_IDX_ESCHED + (x))
+
+MAKE_BITVECTOR(MC_ENGINE_BITVECTOR, MC_ENGINE_IDX_MAX);
+typedef MC_ENGINE_BITVECTOR *PMC_ENGINE_BITVECTOR;
+
+#endif // ENGINE_IDX_H

src/nvidia/inc/kernel/gpu/intr/intr.h

@@ -0,0 +1,3 @@
+
+#include "g_intr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/intr_service.h

@@ -0,0 +1,3 @@
+
+#include "g_intr_service_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intr/swintr.h

@@ -0,0 +1,3 @@
+
+#include "g_swintr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/intrable/intrable.h

@@ -0,0 +1,3 @@
+
+#include "g_intrable_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mc/kernel_mc.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_mc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/context_dma.h

@@ -0,0 +1,3 @@
+
+#include "g_context_dma_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/fbsr.h

@@ -0,0 +1,3 @@
+
+#include "g_fbsr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/fermi_dma.h

@@ -0,0 +1,65 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FERMI_DMA_H
+#define FERMI_DMA_H
+
+#ifdef NV_MMU_PTE
+#error "dev_mmu.h included before fermi_dma.h"
+#endif
+#ifdef NV_PLTCG
+#error "dev_ltc.h included before fermi_dma.h"
+#endif
+
+#include "gpu/mem_mgr/mem_desc.h"
+#include "mem_mgr/gpu_vaspace.h"
+#include "gpu/mem_mgr/virt_mem_allocator.h"
+
+// TODO -- replace those FERMI_* define usages in RM code
+#define FERMI_SMALL_PAGESIZE    RM_PAGE_SIZE
+#define FERMI_BIG_PAGESIZE_64K  RM_PAGE_SIZE_64K
+#define FERMI_BIG_PAGESIZE_128K RM_PAGE_SIZE_128K
+
+typedef struct DMAHALINFO_FERMI
+{
+    NvU32              vasReverse;
+    NvU32              compTagLineMultiplier;     // Comptaglines increment by this value for VERIF only, see Bug 501651
+} DMAHALINFO_FERMI, *PDMAHALINFO_FERMI;
+
+#define DMA_GET_FERMI_INFOBLK(p)    ((PDMAHALINFO_FERMI) getInfoPtr((p)->infoList, HAL_IMPL_GF100))
+
+//
+// From GF100 dev_mmu.ref:
+//    Each PDE maps a 64MB region of virtual memory when using 64KB big pages, or 128MB
+//    when using 128KB big pages.  To map all 40b of virtual address space, the page
+//    directory consists of 16K entries when using 64KB big pages (64MB * 16K = 2^26 * 2^14 = 2^40),
+//    or 8K entries when using 128KB big pages (128MB * 8K = 2^27 * 2^13 = 2^40).
+//
+#define VASPACE_SIZE_MB_FERMI                   (1 << 20)
+#define VASPACE_SIZE_FERMI                      (((NvU64)VASPACE_SIZE_MB_FERMI) << 20)
+
+// Alignment Defines for page tables
+#define PDB_SHIFT_FERMI                         12   // FERMITODO::DMA Will be there in dev_ram very soon
+#define PDB_ALIGNMENT_FERMI                     (1 << PDB_SHIFT_FERMI)
+
+#endif // FERMI_DMA_H

src/nvidia/inc/kernel/gpu/mem_mgr/heap.h

@@ -0,0 +1,3 @@
+
+#include "g_heap_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/heap_base.h

@@ -0,0 +1,162 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _HEAP_BASE_H_
+#define _HEAP_BASE_H_
+
+#include "nvtypes.h"
+#include "core/prelude.h"
+#include "gpu/mem_mgr/mem_desc.h"
+
+// Contains the minimal set of resources used to compute a PTE kind
+typedef struct _def_fb_alloc_page_format
+{
+    NvU32    attr;
+    NvU32    attr2;
+    NvU32    flags;
+    NvU32    kind;
+    NvU32    type;
+} FB_ALLOC_PAGE_FORMAT;
+
+//
+// FB allocation resources structure
+// Need to be allocated from heap
+//
+typedef struct _def_fb_alloc_info
+{
+    NvU32    owner;
+    NvU32    hwResId;
+    NvU32    height;
+    NvU32    width;
+    NvU32    pitch;
+    NvU64    size;
+    NvU64    align;
+    NvU64    alignPad;
+    NvU64    pad;
+    NvU64    offset;
+    NvU32    internalflags;
+    NvU32    retAttr;
+    NvU32    retAttr2;
+    NvU32    format;
+    NvU32    comprCovg;
+    NvU32    zcullCovg;
+    NvU32    uncompressedKind;
+    NvU32    compPageShift;
+    NvU32    compressedKind;
+    NvU32    compTagLineMin;
+    NvU32    compPageIndexLo;
+    NvU32    compPageIndexHi;
+    NvU32    compTagLineMultiplier;
+    NvU32    startCovg;
+    NvU64    origSize;
+    NvU64    adjustedSize;
+    NvU64    desiredOffset;
+
+    FB_ALLOC_PAGE_FORMAT * pageFormat;
+
+    // Tracking client for VGPU
+    NvHandle hClient;
+    NvHandle hDevice;
+
+    // These are only used in Vista
+    // no need yet for possAttr2
+    NvU32    possAttr;          // AllocHint, BindCompr
+    NvU32    ctagOffset;
+
+    // Special flag for kernel allocations
+    NvBool bIsKernelAlloc;
+
+    //
+    // Number of 4KB pages in the PTE array
+    // For contiguous allocation, this will be set to '1'
+    //
+    NvU64 pageCount4k;
+
+    // denote that underlying physical allocation is contiguous or not
+    NvBool bContig;
+
+    //
+    // Store the PTE Array to be used for allocating comptaglines
+    // If the NVOS32_ATTR_PHYSICALITY_CONTIGUOUS is set, it will only have
+    // one entry, otherwise it will have dynamically allocated memory
+    // This will track the pages in 4KB granularity
+    //
+    RmPhysAddr pteArray[1];
+} FB_ALLOC_INFO;
+
+//
+// Contains information on the various hw resources (compr, etc...) that
+// can be associated with a memory allocation.
+//
+typedef struct HWRESOURCE_INFO
+{
+    NvU32 attr;       // NVOS32_ATTR_*
+    NvU32 attr2;      // NVOS32_ATTR2_*
+    NvU32 comprCovg;  // compression coverage
+    NvU32 ctagOffset; // comptag offset
+    NvU32 hwResId;
+    NvU32 refCount;
+    NvBool isVgpuHostAllocated; // used in vGPU guest RM to indicate if this HW resource is allocated by host RM or not. Used in Windows guest.
+    NvBool isGuestAllocated;    // used in vGPU host RM to indicate if this HW resource is allocated from LIST_OBJECT path on behalf of Linux guest.
+} HWRESOURCE_INFO;
+
+
+typedef struct PMA_ALLOC_INFO
+{
+    NvBool bContig;
+    NvU32  pageCount;
+    NvU32  pageSize;
+    NvU32  refCount;
+    NvU64  allocSize;
+    NvU32  flags;
+    //
+    // If bContig is TRUE, this array consists of one element.
+    // If bContig is FALSE, this array is actually larger and
+    // has one entry for each physical page in the allocation.
+    // As a result, this structure must be allocated from heap.
+    //
+    NvU64  pageArray[1];
+    //!!! Place nothing behind pageArray!!!
+} PMA_ALLOC_INFO;
+
+typedef struct MEMORY_ALLOCATION_REQUEST
+{
+    NV_MEMORY_ALLOCATION_PARAMS *pUserParams;
+    OBJGPU                      *pGpu;
+    NvHandle                     hMemory;            // in: can be NULL (translates to 0)
+    NvU32                        internalflags;      // Extended flags ?! flags seem exhausted.
+    HWRESOURCE_INFO             *pHwResource;        // out: data copied in if non-NULL
+    MEMORY_DESCRIPTOR           *pMemDesc;           // in/out: allocate memdesc if NULL
+    PMA_ALLOC_INFO              *pPmaAllocInfo[NV_MAX_SUBDEVICES];      // out: tracks the pre-allocated memory per GPU.
+    NvU32                        classNum;
+    NvHandle                     hClient;
+    NvHandle                     hParent;
+} MEMORY_ALLOCATION_REQUEST;
+
+typedef struct
+{
+    NvU64 address;
+    NvU32 type;
+} BLACKLIST_ADDRESS;
+
+#endif //_HEAP_BASE_H_

src/nvidia/inc/kernel/gpu/mem_mgr/mem_desc.h

@@ -0,0 +1,3 @@
+
+#include "g_mem_desc_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/mem_mgr.h

@@ -0,0 +1,3 @@
+
+#include "g_mem_mgr_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h

@@ -0,0 +1,252 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*********************** Memory Scrubber Routines **************************\
+*      Defines and structures used for CE Physical Memory Scrubber          *
+\***************************************************************************/
+
+#ifndef MEM_SCRUB_H
+#define MEM_SCRUB_H
+
+#include "nvport/nvport.h"
+#include "class/cl906f.h"
+#include "class/cl906fsw.h"
+
+#include "nvctassert.h"
+#include "vgpu/vgpu_guest_pma_scrubber.h"
+
+struct OBJGPU;
+struct Heap;
+struct OBJCHANNEL;
+
+#define RM_SUBCHANNEL                             0x0
+#define MEMSET_PATTERN                            0x00000000
+#define SCRUBBER_NUM_PAYLOAD_SEMAPHORES           (2)
+#define SCRUBBER_SEMAPHORE_SIZE_INBYTES           (4)
+#define SCRUBBER_CHANNEL_SEMAPHORE_SIZE           (SCRUBBER_SEMAPHORE_SIZE_INBYTES *\
+                                                  SCRUBBER_NUM_PAYLOAD_SEMAPHORES)
+#define SCRUBBER_CHANNEL_NOTIFIER_SIZE            (sizeof(NvNotification) * NV_CHANNELGPFIFO_NOTIFICATION_TYPE__SIZE_1)
+
+#define SCRUBBER_VASPACE_BUFFER_SIZE              0x20000000ULL  //512MB
+#define SIZE_OF_ONE_MEMSET_BLOCK                  0x60
+#define SCRUB_MAX_BYTES_PER_LINE                  0xffffffffULL
+#define MAX_SCRUB_ITEMS                           4096 // 4K scrub items
+
+#define READ_SCRUBBER_PAYLOAD_SEMA(channel)       MEM_RD32((NvU8*)channel->pbCpuVA +\
+                                                  channel->finishPayloadOffset)
+
+#define READ_SCRUBBER_PB_SEMA(channel)            MEM_RD32((NvU8*)channel->pbCpuVA +\
+                                                  channel->semaOffset)
+
+#define WRITE_SCRUBBER_PB_SEMA(channel, val)      MEM_WR32((NvU8*)channel->pbCpuVA +\
+                                                  channel->semaOffset, val);
+
+#define WRITE_SCRUBBER_PAYLOAD_SEMA(channel,val)  MEM_WR32((NvU8*)channel->pbCpuVA +\
+                                                  channel->finishPayloadOffset, val);
+// Use Incrementing Methods to save the PB Space
+#define _NV_ASSERT_CONTIGUOUS_METHODS(a1, a2)     NV_ASSERT((a2) - (a1) == 4)
+
+#define NV_PUSH_METHOD(OpType, SubCh, Method, Count)                                                      \
+                                                  (DRF_DEF(906F, _DMA, _SEC_OP, OpType)                 |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_ADDRESS,    (Method) >> 2) |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_SUBCHANNEL, (SubCh))       |\
+                                                  DRF_NUM(906F, _DMA, _METHOD_COUNT,      (Count)))
+
+#define NV_PUSH_DATA(Data)                        MEM_WR32(pPtr++, (Data))
+
+#define _NV_PUSH_INC_1U(SubCh, a1,d1, Count)                                                                          \
+                                                  do{                                                               \
+                                                        NV_PUSH_DATA(NV_PUSH_METHOD(_INC_METHOD, SubCh, a1, Count));\
+                                                        NV_PUSH_DATA(d1);                                           \
+                                                    } while(0)
+
+#define NV_PUSH_INC_1U(SubCh, a1,d1)                                                      \
+                                                  do{                                     \
+                                                        _NV_PUSH_INC_1U (SubCh, a1,d1, 1);\
+                                                    } while(0)
+
+#define NV_PUSH_INC_2U(SubCh, a1,d1, a2,d2)                                                    \
+                                                  do{                                          \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1, a2);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 2);      \
+                                                        NV_PUSH_DATA(d2);                      \
+                                                    } while(0)
+
+#define NV_PUSH_INC_3U(SubCh, a1,d1, a2,d2, a3,d3)                                            \
+                                                  do{                                         \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 3);     \
+                                                        NV_PUSH_DATA(d2);                     \
+                                                        NV_PUSH_DATA(d3);                     \
+                                                    } while(0)
+
+#define NV_PUSH_INC_4U(SubCh, a1,d1, a2,d2, a3,d3, a4,d4)                                    \
+                                                  do{                                        \
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
+                                                        _NV_ASSERT_CONTIGUOUS_METHODS(a3,a4);\
+                                                        _NV_PUSH_INC_1U(SubCh, a1,d1, 4);    \
+                                                        NV_PUSH_DATA(d2);                    \
+                                                        NV_PUSH_DATA(d3);                    \
+                                                        NV_PUSH_DATA(d4);                    \
+                                                    } while(0)
+
+
+// structure to store the details of a scrubbing work
+typedef struct SCRUB_NODE {
+    // The 64 bit ID assigned to each work
+    NvU64      id;
+    // The base address from which the scrub to start
+    NvU64      base;
+    // The size of a scrub work
+    NvU64      size;
+} SCRUB_NODE, *PSCRUB_NODE;
+
+//
+// OBJMEMSCRUB OBJECT
+// Memory scrubber struct encapsulates the CE Channel object,
+// SCRUB_NODE array of size MAX_SCRUB_ITEMS, index to track
+// the scrub work list. The scrubber data structures are
+// synchronized using the mutex pScrubberMutex.
+//
+
+typedef struct OBJMEMSCRUB {
+    // Mutex for Scrubber Object
+    PORT_MUTEX                        *pScrubberMutex;
+    // Last completed work ID communicated to Client
+    NvU64                              lastSeenIdByClient;
+    // The last ID assigned to a work
+    NvU64                              lastSubmittedWorkId;
+    // Last ID checked with the HW scrubber
+    NvU64                              lastSWSemaphoreDone;
+    // Size of the scrub list
+    NvLength                           scrubListSize;
+    // Pre-allocated Free Scrub List
+    PSCRUB_NODE                        pScrubList;
+    // Scrubber Channel
+    struct OBJCHANNEL *pChannel;
+    struct OBJGPU                     *pGpu;
+    VGPU_GUEST_PMA_SCRUB_BUFFER_RING   vgpuScrubBuffRing;
+    NvBool                             bVgpuScrubberEnabled;
+} OBJMEMSCRUB, *POBJMEMSCRUB;
+
+ct_assert(VGPU_GUEST_PMA_MAX_SCRUB_ITEMS == MAX_SCRUB_ITEMS);
+
+/**
+ * Constructs the memory scrubber object and signals
+ * RM to create CE channels for submitting scrubbing work
+ *
+ * @param[in]     pGpu       OBJGPU pointer
+ * @param[in]     pHeap      Heap pointer
+ *
+ * @returns NV_STATUS on success.
+ *          error, if something fails
+ */
+
+NV_STATUS scrubberConstruct(struct OBJGPU *pGpu, struct Heap *pHeap);
+
+/**
+ * Destructs the scrubber
+ *  1. De-registers the scrubber from the PMA object
+ *  2. Free the scrubber list and scrubber lock
+ *
+ * @param[in]     pGpu       OBJGPU pointer
+ * @param[in]     pHeap      Heap pointer
+ * @param[in]     pScrubber  OBJMEMSCRUB pointer
+ *
+ */
+
+NV_STATUS vgpuAllocGuestPmaScrubberSharedBuffer(struct OBJGPU *pGpu, OBJMEMSCRUB *pScrubber);
+void vgpuFreeGuestPmaScrubberSharedBuffer(struct OBJGPU *pGpu, OBJMEMSCRUB *pScrubber);
+
+void scrubberDestruct(struct OBJGPU *pGpu, struct Heap *pHeap, OBJMEMSCRUB *pMemscrub);
+
+
+/**
+ * This function checks for the completed scrub work items,
+ * and populates the SCRUB_NODE in the array.
+ * @param[in]  pScrubber OBJMEMSCRUB pointer
+ * @param[out]  ppList    SCRUB_NODE double pointer
+ * @param[out]  pSize     NvU64 pointer
+ * @returns NV_OK on success,
+ *          NV_ERR_INSUFFICIENT_RESOURCES when the list allocation fails.
+ */
+
+NV_STATUS scrubCheck(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *size);
+
+
+/**
+ * This function submits work to the memory scrubber.
+ * This function interface is changed to return a list of scrubbed pages to the
+ * client, since the scrubber work list resources are limited, if the submission
+ * page count is more than scrubber list resources the completed scrubbed pages
+ * are saved in the list and the submission progresses.
+ *
+ * @param[in] pScrubber  OBJMEMSCRUB pointer
+ * @param[in] chunkSize   NvU64 size of each page
+ * @param[in] pPages     NvU64 array of base address
+ * @param[in] pageCount  NvU64 number of pages
+ * @param[out] ppList     SCRUB_NODE double pointer to hand off the list
+ * @param[out] pSize      NvU64 pointer to store the size
+ *
+ * @returns NV_OK on success, NV_ERR_GENERIC on HW Failure
+ */
+
+NV_STATUS scrubSubmitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages,
+                           NvU64 pageCount, PSCRUB_NODE *ppList, NvU64 *size);
+
+/**
+ *  This function waits for the memory scrubber to wait for the scrubbing of
+ *  pages within the range [pagesStart, pagesEnd] for the for the array of pages
+ *  of size pageCount
+ *
+ * @param[in]   pScrubber   OBJMEMSCRUB pointer
+ * @param[in]   pageSize    NvU64 size of each page
+ * @param[in]   pPages      NvU64 pointer to store the base address
+ * @param[in]   pageCount   NvU64 number of pages in the array
+ *
+ * @returns NV_OK
+ */
+
+NV_STATUS scrubWaitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages,
+                         NvU32 pageCount);
+
+
+/**
+ *  This function waits for the scrubber to finish scrubbing enough items
+ *  to have numPages fully scrubbed and then saves the work items to the list
+ *  passed to the client.
+ *
+ * @param[in]  pScrubber OBJMEMSCRUB pointer
+ * @param[in]  numPages the number of pages we should wait to be scrubbed
+ * @param[in]  pageSize the page size
+ * @param[out]  ppList    SCRUB_NODE double pointer to return the saved list pointer
+ * @param[out]  pSize     NvU64 pointer to return the size of saved work.
+ *
+ * @returns NV_OK if at least one work is pending in the scrubber list
+ *          NV_ERR_NO_MEMORY when no work is pending in the scrubber list
+ */
+NV_STATUS scrubCheckAndWaitForSize (OBJMEMSCRUB *pScrubber, NvU64 numPages,
+                                    NvU64 pageSize, PSCRUB_NODE *ppList, NvU64 *pSize);
+#endif // MEM_SCRUB_H

src/nvidia/inc/kernel/gpu/mem_mgr/mem_utils.h

@@ -0,0 +1,49 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _MEM_UTILS_H_
+#define _MEM_UTILS_H_
+
+#include "core/prelude.h"
+
+#define CLEAR_HAL_ATTR(a) \
+    a = (a &~(DRF_NUM(OS32, _ATTR, _COMPR, 0x3) | \
+              DRF_NUM(OS32, _ATTR, _TILED, 0x3) | \
+              DRF_NUM(OS32, _ATTR, _ZCULL, 0x3)));
+
+#define CLEAR_HAL_ATTR2(a) \
+    a = (a & ~(DRF_SHIFTMASK(NVOS32_ATTR2_ZBC) | \
+               DRF_SHIFTMASK(NVOS32_ATTR2_GPU_CACHEABLE)));
+
+NvU64 memUtilsLeastCommonAlignment(NvU64 align1, NvU64 align2);
+
+void memUtilsInitFBAllocInfo(NV_MEMORY_ALLOCATION_PARAMS *pAllocParams, FB_ALLOC_INFO *pFbAllocInfo,
+                             NvHandle hClient, NvHandle hDevice);
+
+NV_STATUS memUtilsAllocMemDesc(OBJGPU *pGpu, MEMORY_ALLOCATION_REQUEST *pAllocRequest, FB_ALLOC_INFO *pFbAllocInfo,
+                               MEMORY_DESCRIPTOR **ppMemDesc, Heap *pHeap, NV_ADDRESS_SPACE addrSpace,
+                               NvBool bContig, NvBool *bAllocedMemDesc);
+
+NV_STATUS  memUtilsMemSetNoBAR2(OBJGPU *pGpu, PMEMORY_DESCRIPTOR pMemDesc, NvU8 value);
+
+#endif //_MEM_UTILS_H_

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/addrtree.h

@@ -0,0 +1,227 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ * @brief Implement PMA address tree
+ *
+ */
+
+#ifndef ADDRTREE_H
+#define ADDRTREE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "map_defines.h"
+
+// Declare this before its definition because it refers to itself
+typedef struct addrtree_node ADDRTREE_NODE;
+
+struct addrtree_node
+{
+    NvU32 level;                    // The level this node belongs to
+    NvU32 numChildren;              // The number of children in the children array
+    NvU64 frame;                    // The first frame this node holds
+    NvU64 state[PMA_BITS_PER_PAGE];          // Tracks the actual state for each map
+    NvU64 seeChild[PMA_BITS_PER_PAGE];       // Whether this node is partially allocated
+                                    //   If it is partially allocated, we must go to the children
+                                    //   to find the correct information.
+
+    ADDRTREE_NODE *parent;          // The node's parent
+    ADDRTREE_NODE *children;        // Pointer to an array of children
+};
+
+typedef struct addrtree_level
+{
+    NvU64 nodeCount;                // Count of total number of nodes on this level
+    ADDRTREE_NODE *pNodeList;       // Pointer to the start of the list of nodes on this level
+    NvU32 pageSizeShift;            // Page size this level is tracking
+    NvU32 maxFramesPerNode;         // The max number of this level frames per node
+} ADDRTREE_LEVEL;
+
+typedef struct pma_addrtree
+{
+    NvU64           totalFrames;    // Total number of 64KB frames being tracked
+    NvU32           levelCount;     // Number of levels in this tree
+    ADDRTREE_LEVEL *levels;         // List of levels in the tree
+    ADDRTREE_NODE  *root;           // Start of the node list
+    NvU64           numPaddingFrames; // Number of 64KB frames needed for padding for alignment
+
+    NvU64 frameEvictionsInProcess;  // Count of frame evictions in-process
+    PMA_STATS      *pPmaStats;      // Point back to the public struct in PMA structure
+    NvBool          bProtected;     // The memory segment tracked by this tree is protected (VPR/CPR)
+} PMA_ADDRTREE;
+
+/*!
+ * @brief Initializes the addrtree for PMA uses
+ *
+ * Allocates the address tree structure for all the pages being managed in this tree.
+ * Address Tree implementation will use a default configuration for its own level
+ * structures.
+ *
+ * @param[in] numPages    The number of pages being managed in this tree
+ * @param[in] addrBase    The base address of this region. Required for addrtree alignment
+ * @param[in] pPmaStats   Pointer to the PMA-wide stats structure
+ * @param[in] bProtected  The tree tracks pages in protected memory
+ *
+ * @return PMA_ADDRTREE Pointer to the addrtree if succeeded, NULL otherwise
+ */
+void *pmaAddrtreeInit(NvU64 numFrames, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected);
+
+/*!
+ * @brief Destroys the addrtree and free the memory
+ *
+ * @param[in] pMap  The addrtree to destroy
+ *
+ * @return void
+ */
+void pmaAddrtreeDestroy(void *pMap);
+
+/*!
+ * @brief Get/set number of evicting frames
+ * Used for sanity checking in PMA layer as well as performance optimization
+ * for the map layer to scan faster.
+ */
+NvU64 pmaAddrtreeGetEvictingFrames(void *pMap);
+void pmaAddrtreeSetEvictingFrames(void *pMap, NvU64 frameEvictionsInProcess);
+
+
+/*!
+ * @brief Scans the addrtree for contiguous space that has the certain status.
+ *
+ * @param[in]   pMap            The addrtree to be scanned
+ * @param[in]   addrBase        The base address of this region
+ * @param[in]   rangeStart      The start of the restricted range
+ * @param[in]   rangeEnd        The end of the restricted range
+ * @param[in]   numPages        The number of pages we are scanning for
+ * @param[out]  freeList        A list of free frame numbers -- contains only 1 element
+ * @param[in]   pageSize        Size of one page
+ * @param[in]   alignment       Alignment requested by client
+ * @param[out]  pagesAllocated  Number of pages this call allocated
+ * @param[in]   bSkipEvict      Whether it's ok to skip the scan for evictable pages
+ *
+ * @return NV_OK if succeeded
+ * @return NV_ERR_IN_USE if found pages that can be evicted
+ * @return NV_ERR_NO_MEMORY if no available pages could be found
+ */
+NV_STATUS pmaAddrtreeScanContiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+NV_STATUS pmaAddrtreeScanDiscontiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+void pmaAddrtreePrintTree(void *pMap, const char* str);
+
+
+/*!
+ * @brief Changes the state & attrib bits specified by mask
+ *
+ * Changes the state of the bits given the physical frame number
+ * TODO: all four interfaces need to be merged from PMA level so we can remove them!
+ *
+ * @param[in]   pMap         The addrtree to change
+ * @param[in]   frameNum     The frame number to change
+ * @param[in]   newState     The new state to change to
+ * @param[in]   newStateMask Specific bits to write
+ *
+ * @return void
+ */
+void pmaAddrtreeChangeState(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
+void pmaAddrtreeChangeStateAttrib(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib);
+void pmaAddrtreeChangeStateAttribEx(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState,PMA_PAGESTATUS newStateMask);
+void pmaAddrtreeChangePageStateAttrib(void * pMap, NvU64 startFrame, NvU32 pageSize,
+                                      PMA_PAGESTATUS newState, NvBool writeAttrib);
+
+/*!
+ * @brief Read the page state & attrib bits
+ *
+ * Read the state of the page given the physical frame number
+ *
+ * @param[in]   pMap        The addrtree to read
+ * @param[in]   frameNum    The frame number to read
+ * @param[in]   readAttrib  Read attribute bits as well
+ *
+ * @return PAGESTATUS of the frame
+ */
+PMA_PAGESTATUS pmaAddrtreeRead(void *pMap, NvU64 frameNum, NvBool readAttrib);
+
+
+/*!
+ * @brief Gets the total size of specified PMA managed region.
+ *
+ * Gets the total size of current PMA managed region in the FB.
+ *
+ * @param[in]  pMap         Pointer to the addrtree for the region
+ * @param[in]  pBytesTotal  Pointer that will return total bytes for current region.
+ *
+ */
+void pmaAddrtreeGetSize(void *pMap, NvU64 *pBytesTotal);
+
+
+/*!
+ * @brief Gets the size of the maximum free chunk of memory in specified region.
+ *
+ * Gets the size of the maximum free chunk of memory in the specified PMA managed
+ * region of the FB.
+ *
+ * @param[in]  pMap         Pointer to the addrtree for the region
+ * @param[in]  pLargestFree Pointer that will return largest free in current region.
+ *
+ */
+void pmaAddrtreeGetLargestFree(void *pMap, NvU64 *pLargestFree);
+
+/*!
+ * @brief Returns the address range that is completely available for eviction.
+ *        - Should be ALLOC_UNPIN.
+ * In NUMA, OS manages memory and PMA will only track allocated memory in ALLOC_PIN
+ * and ALLOC_UNPIN state. FREE memory is managed by OS and cannot be tracked by PMA
+ * and hence PMA cannot consider FREE memory for eviction and can only consider frames
+ * in known state to PMA or eviction. ALLOC_PIN cannot be evicted and hence only ALLOC_UNPIN
+ * can be evictable.
+ *
+ *
+ * @param[in]  pMap         Pointer to the regmap for the region
+ * @param[in]  addrBase     Base address of the region
+ * @param[in]  actualSize   Size of the eviction range
+ * @param[in]  pageSize     Pagesize
+ * @param[out] evictStart   Starting address of the eviction range
+ * @param[out] evictEnd     End address of the eviction range.
+ *
+ * Returns:
+ *  -  NV_OK If there is evictable range of given size : actualSize
+ *
+ *  -  NV_ERR_NO_MEMORY if no contiguous range is evictable.
+ */
+NV_STATUS pmaAddrtreeScanContiguousNumaEviction(void *pMap, NvU64 addrBase,
+    NvLength actualSize, NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // ADDRTREE_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/map_defines.h

@@ -0,0 +1,143 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *  @brief Contains common defines between addrtree and regmap
+ */
+
+#ifndef MAP_DEFINES_H
+#define MAP_DEFINES_H
+
+#include "nvtypes.h"
+#include "nvstatus.h"
+#include "nvmisc.h"
+#if !defined(NVWATCH)
+
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+// Default page size 64KB
+#define PMA_GRANULARITY 0x10000
+#define PMA_PAGE_SHIFT 16
+
+//
+// _PMA_1GB will cause overflows with an NvU32. It's bigger than NvU32 can store,
+// but compilation still fails when using a NvU64 instead
+// So just use bitshift.
+// 1 << _TREE_64KB == sizeof(1 frame)
+//
+
+#define _TREE_64KB               16
+#define _TREE_128KB              17
+#define _TREE_2MB                21
+#define _TREE_128MB              27
+#define _TREE_512MB              29
+#define _TREE_32GB               35
+#define _TREE_2TB                40
+
+// Defines shared between pma.c and regmap.c
+#define _PMA_64KB               (64 * 1024)
+#define _PMA_128KB              (128 * 1024)
+#define _PMA_2MB                (2 * 1024 * 1024)
+#define _PMA_512MB              (512 * 1024 * 1024)
+
+// Scanning function return code
+#define EVICTABLE -2
+#define ALL_FREE  -3
+
+typedef NvU32 PMA_PAGESTATUS;
+
+#define MAP_IDX_ALLOC_UNPIN 0
+#define MAP_IDX_ALLOC_PIN   1
+#define MAP_IDX_EVICTING    2
+#define MAP_IDX_SCRUBBING   3
+#define MAP_IDX_PERSISTENT  4
+#define MAP_IDX_NUMA_REUSE  5
+#define MAP_IDX_BLACKLIST   6
+
+#define STATE_FREE      0x00
+#define STATE_UNPIN     NVBIT(MAP_IDX_ALLOC_UNPIN)
+#define STATE_PIN       NVBIT(MAP_IDX_ALLOC_PIN)
+#define STATE_MASK      (STATE_UNPIN | STATE_PIN | STATE_FREE)
+#define STATE_COUNT     3
+
+#define ATTRIB_EVICTING    NVBIT(MAP_IDX_EVICTING)
+#define ATTRIB_SCRUBBING   NVBIT(MAP_IDX_SCRUBBING)
+#define ATTRIB_PERSISTENT  NVBIT(MAP_IDX_PERSISTENT)
+#define ATTRIB_NUMA_REUSE  NVBIT(MAP_IDX_NUMA_REUSE)
+#define ATTRIB_BLACKLIST   NVBIT(MAP_IDX_BLACKLIST)
+#define ATTRIB_MASK        (ATTRIB_EVICTING | ATTRIB_SCRUBBING \
+                            | ATTRIB_PERSISTENT | ATTRIB_NUMA_REUSE \
+                            | ATTRIB_BLACKLIST)
+
+#define MAP_MASK    (STATE_MASK | ATTRIB_MASK)
+
+#define PMA_STATE_BITS_PER_PAGE     2   // Alloc & pinned state
+#define PMA_ATTRIB_BITS_PER_PAGE    5   // Persistence, Scrubbing, Evicting, Reuse & Blacklisting attributes
+#define PMA_BITS_PER_PAGE           (PMA_STATE_BITS_PER_PAGE + PMA_ATTRIB_BITS_PER_PAGE)
+
+//
+// Stores PMA-wide statistics.
+//
+// In NUMA mode "free" means "not allocated by PMA". Since the kernel owns
+// memory, any amount of those pages could be allocated by the kernel but PMA
+// does not have that visibility. The provided counts are thus an upper bound on
+// the number of free pages.
+//
+typedef struct _PMA_STATS
+{
+    NvU64 num2mbPages;               // PMA-wide total number of 2MB pages
+    NvU64 numFreeFrames;             // PMA-wide free 64KB frame count
+    NvU64 numFree2mbPages;           // PMA-wide free 2MB pages count
+#if !defined(NVWATCH)
+#endif // !defined(NVWATCH)
+} PMA_STATS;
+
+// Stores blacklisting information passed in from heap layer
+typedef struct
+{
+    NvU64  physOffset; // base address of blacklisted page
+    NvBool bIsDynamic; // True if page was dynamically blacklisted
+} PMA_BLACKLIST_ADDRESS, *PPMA_BLACKLIST_ADDRESS;
+
+//
+// Store the blacklist chunk information with the physical offset aligned to 64K,
+// and whether the blacklist chunk is managed by RM or Client.
+//
+typedef struct
+{
+    NvU64  physOffset; // physical offset of blacklisted FB address
+    NvBool bIsDynamic;  // True if the page was dynamically blacklisted
+    NvBool bIsValid;   // If the blacklisted address is still managed by RM
+} PMA_BLACKLIST_CHUNK;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MAP_DEFINES_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/numa.h

@@ -0,0 +1,77 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ * This file defines the internal interfaces to the NUMA allocator component,
+ * used by parent module PMA.
+ *
+ * All interfaces here should only be used on PowerPC 9 systems with NUMA
+ * mode enabled, where GPU's framebuffer memory is onlined to the Linux kernel.
+ *
+ * Therefore, PMA just needs to sub-allocate from the Linux kernel for most
+ * allocation requests. However, PMA does keep states in order to support
+ * eviction and GPU scrubber. Please see below for more details.
+ */
+
+#ifndef NUMA_H
+#define NUMA_H
+
+#include "nvport/nvport.h"
+#include "phys_mem_allocator.h"
+#include "nvmisc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*!
+ * @brief Handles NUMA allocation by calling kernel APIs
+ *
+ * This function will implement a mixture of pass-through memory allocation
+ * from the Linux kernel as well as eviction from UVM if the Linux kernel
+ * does not have any free memory.
+ *
+ * From a high level, PMA keep a bitmap for all UVM unpinned (evictable)
+ * allocations and PMA will decide to kick off eviction based on the bitmap.
+ *
+ * Please note that GPU scrubber is used for any memory evicted and CPU scrubber
+ * is used for allocations coming from the Linux kernel.
+ * The perf implication is under further study. See bug #1999793.
+ */
+NV_STATUS pmaNumaAllocate(PMA *pPma, NvLength allocationCount, NvU32 pageSize,
+    PMA_ALLOCATION_OPTIONS *allocationOptions, NvU64 *pPages);
+
+/*!
+ * @brief Frees pages on a NUMA node.
+ * This function implements pass-through free calls to the Linux kernel.
+ * For UVM allocations PMA also updates the bitmap used for eviction.
+ */
+void pmaNumaFreeInternal(PMA *pPma, NvU64 *pPages, NvU64 pageCount, NvU64 size, NvU32 flag);
+
+void pmaNumaSetReclaimSkipThreshold(PMA *pPma, NvU32 skipReclaimPercent);
+#ifdef __cplusplus
+}
+#endif
+
+#endif // NUMA_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h

@@ -0,0 +1,856 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *  @brief This file exposes the PMA interfaces.
+ *  The PMA module interacts with the RM and UVM components. UVM will use the
+ *  ops layer to call out to PMA, while RM directly calls into PMA.
+ *  The PMA module takes a global lock to protect its internal structure. It
+ *  uses a bitmap structure called regmap.
+ *
+ *  @bug
+ *  1. status code -- decide if we need to add to global
+ *  2. suspend/resume -- might add one more function to support
+ *
+ *  @TODO
+ *  1. external fragmentation
+ *  2. use new scrubber API and remove the initScrubbing atomic variable
+ */
+
+#ifndef PHYS_MEM_ALLOCATOR_H
+#define PHYS_MEM_ALLOCATOR_H
+
+#include "nvport/nvport.h"
+#include "regmap.h"
+#include "addrtree.h"
+#include "nvmisc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct OBJMEMSCRUB OBJMEMSCRUB;
+typedef struct SCRUB_NODE SCRUB_NODE;
+
+#define PMA_REGION_SIZE 64
+#define PMA_ADDR2FRAME(addr, base)  (((addr) - (base)) >> PMA_PAGE_SHIFT)
+#define PMA_FRAME2ADDR(frame, base) ((base) + ((frame) << PMA_PAGE_SHIFT))
+
+//
+// These flags are used for initialization in order to set global PMA states,
+// in case we need to wait for scrubber to be initialized or wait for a NUMA
+// node being onlined, etc.
+//
+#define PMA_INIT_NONE                   NVBIT(0)
+#define PMA_INIT_SCRUB_ON_FREE          NVBIT(1)
+#define PMA_INIT_NUMA                   NVBIT(2)
+#define PMA_INIT_INTERNAL               NVBIT(3) // Used after heap is removed
+#define PMA_INIT_FORCE_PERSISTENCE      NVBIT(4)
+#define PMA_INIT_ADDRTREE               NVBIT(5)
+
+// These flags are used for querying PMA's config and/or state.
+#define PMA_QUERY_SCRUB_ENABLED         NVBIT(0)
+#define PMA_QUERY_SCRUB_VALID           NVBIT(1)
+#define PMA_QUERY_NUMA_ENABLED          NVBIT(2)
+#define PMA_QUERY_NUMA_ONLINED          NVBIT(3)
+
+//
+// When modifying flags, make sure they are compatible with the mirrored
+// UVM_PMA_* flags in nv_uvm_types.h.
+//
+// Input flags
+#define PMA_ALLOCATE_DONT_EVICT             NVBIT(0)
+#define PMA_ALLOCATE_PINNED                 NVBIT(1)
+#define PMA_ALLOCATE_SPECIFY_MINIMUM_SPEED  NVBIT(2)
+#define PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE  NVBIT(3)
+#define PMA_ALLOCATE_SPECIFY_REGION_ID      NVBIT(4)
+#define PMA_ALLOCATE_PREFER_SLOWEST         NVBIT(5)
+#define PMA_ALLOCATE_CONTIGUOUS             NVBIT(6)
+#define PMA_ALLOCATE_PERSISTENT             NVBIT(7)
+#define PMA_ALLOCATE_PROTECTED_REGION       NVBIT(8)
+#define PMA_ALLOCATE_FORCE_ALIGNMENT        NVBIT(9)
+#define PMA_ALLOCATE_NO_ZERO                NVBIT(10)
+#define PMA_ALLOCATE_TURN_BLACKLIST_OFF     NVBIT(11)
+#define PMA_ALLOCATE_ALLOW_PARTIAL          NVBIT(12)
+
+// Output flags
+#define PMA_ALLOCATE_RESULT_IS_ZERO         NVBIT(0)
+
+// These are flags input to the pmaFreePages call
+#define PMA_FREE_SKIP_SCRUB           NVBIT(0)
+
+// State bits for debugging utilities like nvwatch
+#define PMA_SCRUB_INITIALIZE   0
+#define PMA_SCRUB_IN_PROGRESS  1
+#define PMA_SCRUB_DONE         2
+
+#define PMA_SCRUBBER_VALID     1
+#define PMA_SCRUBBER_INVALID   0
+
+#define PMA_NUMA_NO_NODE      -1
+
+// Maximum blacklist entries possible
+#define PMA_MAX_BLACKLIST_ENTRIES 512
+
+typedef struct
+{
+    NvU32 flags;
+    NvU32 minimumSpeed;         // valid if flags & PMA_ALLOCATE_SPECIFY_MININUM_SPEED
+    NvU64 physBegin, physEnd;   // valid if flags & PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE
+    NvU32 regionId;             // valid if flags & PMA_ALLOCATE_SPECIFY_REGION_ID
+    NvU64 alignment;            // valid if flags & PMA_ALLOCATE_FORCE_ALIGNMENT
+    NvLength numPagesAllocated; // valid if flags & PMA_ALLOCATE_ALLOW_PARTIAL
+
+    NvU32 resultFlags;          // valid if the allocation function returns NV_OK
+} PMA_ALLOCATION_OPTIONS;
+
+//
+// Explanation: This struct will be provided when UVM/RM registers a region with PMA,
+// after which the struct is stored locally in PMA. The internal "filter" function will
+// use the information everytime a request comes in.
+//
+typedef struct
+{
+    NvU64   base;               // Base/start address of the region
+    NvU64   limit;              // Last/end address of region
+    NvU32   performance;        // Relative performance.  Higher is faster
+    NvBool  bSupportCompressed; // Support compressed kinds
+    NvBool  bSupportISO;        // Support ISO (display, cursor, video) surfaces
+    NvBool  bProtected;         // Represents a protected region of memory.
+} PMA_REGION_DESCRIPTOR;
+
+typedef struct _PMA_MAP_INFO PMA_MAP_INFO;
+typedef struct _PMA PMA;
+
+// Range descriptors for managing persistent range lists
+typedef struct _RANGELISTTYPE
+{
+    NvU64           base;
+    NvU64           limit;
+    struct _RANGELISTTYPE  *pNext;
+} RANGELISTTYPE, *PRANGELISTTYPE;
+
+/*!
+ * @brief Callbacks to UVM for eviction
+ */
+typedef NV_STATUS (*pmaEvictPagesCb_t)(void *ctxPtr, NvU32 pageSize, NvU64 *pPages, NvU32 count, NvU64 physBegin, NvU64 physEnd);
+typedef NV_STATUS (*pmaEvictRangeCb_t)(void *ctxPtr, NvU64 physBegin, NvU64 physEnd);
+
+/*!
+ * @brief Pluggable data structure management. Currently we have regmap and address tree.
+ */
+typedef void *(*pmaMapInit_t)(NvU64 numFrames, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected);
+typedef void  (*pmaMapDestroy_t)(void *pMap);
+typedef void  (*pmaMapChangeState_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
+typedef void  (*pmaMapChangeStateAttrib_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib);
+typedef void  (*pmaMapChangeStateAttribEx_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask);
+typedef void  (*pmaMapChangePageStateAttrib_t)(void *pMap, NvU64 startFrame, NvU32 pageSize, PMA_PAGESTATUS newState, NvBool writeAttrib);
+typedef PMA_PAGESTATUS (*pmaMapRead_t)(void *pMap, NvU64 frameNum, NvBool readAttrib);
+typedef NV_STATUS (*pmaMapScanContiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+                                            NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+                                            NvU64 *pagesAllocated, NvBool bSkipEvict);
+typedef NV_STATUS (*pmaMapScanDiscontiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+                                               NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+                                               NvU64 *pagesAllocated, NvBool bSkipEvict);
+typedef void (*pmaMapGetSize_t)(void *pMap, NvU64 *pBytesTotal);
+typedef void (*pmaMapGetLargestFree_t)(void *pMap, NvU64 *pLargestFree);
+typedef NV_STATUS (*pmaMapScanContiguousNumaEviction_t)(void *pMap, NvU64 addrBase, NvLength actualSize,
+                                                        NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd);
+typedef NvU64 (*pmaMapGetEvictingFrames_t)(void *pMap);
+typedef void (*pmaMapSetEvictingFrames_t)(void *pMap, NvU64 frameEvictionsInProcess);
+
+struct _PMA_MAP_INFO
+{
+    NvU32                       mode;
+    pmaMapInit_t                pmaMapInit;
+    pmaMapDestroy_t             pmaMapDestroy;
+    pmaMapChangeState_t         pmaMapChangeState;
+    pmaMapChangeStateAttrib_t   pmaMapChangeStateAttrib;
+    pmaMapChangeStateAttribEx_t pmaMapChangeStateAttribEx;
+    pmaMapChangePageStateAttrib_t pmaMapChangePageStateAttrib;
+    pmaMapRead_t                pmaMapRead;
+    pmaMapScanContiguous_t      pmaMapScanContiguous;
+    pmaMapScanDiscontiguous_t   pmaMapScanDiscontiguous;
+    pmaMapGetSize_t             pmaMapGetSize;
+    pmaMapGetLargestFree_t      pmaMapGetLargestFree;
+    pmaMapScanContiguousNumaEviction_t pmaMapScanContiguousNumaEviction;
+    pmaMapGetEvictingFrames_t  pmaMapGetEvictingFrames;
+    pmaMapSetEvictingFrames_t  pmaMapSetEvictingFrames;
+};
+
+struct _PMA
+{
+    PORT_SPINLOCK           *pPmaLock;                          // PMA-wide lock
+    PORT_MUTEX              *pEvictionCallbacksLock;            // Eviction callback registration lock
+
+    // Only used when free scrub-on-free feature is turned on
+    PORT_RWLOCK             *pScrubberValidLock;                // A reader-writer lock to protect the scrubber valid bit
+    PORT_MUTEX              *pAllocLock;                        // Used to protect page stealing in the allocation path
+
+    // Region related states
+    NvU32                   regSize;                            // Actual size of regions array
+    void *                  pRegions[PMA_REGION_SIZE];          // All the region maps stored as opaque pointers
+    NvU32                   *pSortedFastFirst;                  // Pre-sorted array of region IDs
+    PMA_REGION_DESCRIPTOR   *pRegDescriptors [PMA_REGION_SIZE]; // Stores the descriptions of each region
+    PMA_MAP_INFO            *pMapInfo;                          // The pluggable layer for managing scanning
+
+    // Allocation related states
+    void *                  evictCtxPtr;                        // Opaque context pointer for eviction callback
+    pmaEvictPagesCb_t       evictPagesCb;                       // Discontiguous eviction callback
+    pmaEvictRangeCb_t       evictRangeCb;                       // Contiguous eviction callback
+    NvU64                   frameAllocDemand;                   // Frame count of allocations in-process
+    NvBool                  bForcePersistence;                  // Force all allocations to persist across suspend/resume
+    PMA_STATS               pmaStats;                           // PMA statistics used for client heuristics
+
+    // Scrubber related states
+    NvSPtr                  initScrubbing;                      // If the init scrubber has finished in this PMA
+    NvBool                  bScrubOnFree;                       // If "scrub on free" is enabled for this PMA object
+    NvSPtr                  scrubberValid;                      // If scrubber object is valid, using atomic variable to prevent races
+    OBJMEMSCRUB            *pScrubObj;                          // Object to store the FreeScrub header
+
+    // NUMA states
+    NvBool                  bNuma;                              // If we are allocating for a NUMA system
+    NvBool                  nodeOnlined;                        // If node is onlined
+    NvS32                   numaNodeId;                         // Current Node ID, set at initialization. -1 means invalid
+    NvU64                   coherentCpuFbBase;                  // Used to calculate FB offset from bus address
+    NvU64                   coherentCpuFbSize;                  // Used for error checking only
+    NvU32                   numaReclaimSkipThreshold;           // percent value below which __GFP_RECLAIM will not be used.
+
+    // Blacklist related states
+    PMA_BLACKLIST_CHUNK    *pBlacklistChunks;                   // Tracking for blacklist pages
+    NvU32                   blacklistCount;                     // Number of blacklist pages
+    NvBool                  bClientManagedBlacklist;            // Blacklisted pages in PMA that will be taken over by Client
+};
+
+/*!
+ * @brief This must be called before any other PMA functions. Returns a PMA
+ * object for later use
+ *
+ * @param[in] gpuId         The UVM global GPU ID. Defined in nvCpuUuid.h as a 16
+ *                          byte digest. PMA will only store reference to it.
+ * @param[in] pma           Pointer to PMA object being initialized
+ * @param[in] initFlags     PMA initialization flags for special modes
+ * @return
+ *      NV_ERR_INVALID_ARGUMENT:
+ *          If the combination of initFlags is invalid.
+ *      NV_ERR_NO_MEMORY:
+ *          Internal memory allocation failed.
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code, because it is not very informative.
+ *
+ */
+NV_STATUS pmaInitialize(PMA *pPma, NvU32 initFlags);
+
+
+/*!
+ * @brief Release a PMA object. Also frees memory.
+ *
+ * All eviction handlers must have been unregistered by this point.
+ *
+ * @param[in] pma       Pointer to PMA object being destroyed.
+ */
+void  pmaDestroy(PMA *pPma);
+
+
+/*!
+ * @brief Queries PMA configuration and state.
+ *
+ * Any clients of PMA can query config and state with a valid PMA object.
+ * Querying at different times may return different values when states change.
+ *
+ * @param[in]     pma       Pointer to PMA object being destroyed.
+ * @param[in/out] pConfigs  Configs/states to query. See PMA_QUERY_* above.
+ */
+NV_STATUS pmaQueryConfigs(PMA* pPma, NvU32 *pConfigs);
+
+
+/*!
+ * @brief Attaches a region of physical memory to be managed by the PMA.
+ *
+ * Systems with floorswept memory configurations will have (multi gigabyte)
+ * holes in memory. Each contiguous region should be reported by this
+ * registration function. This is also intended to be used by systems with
+ * two speed memories.
+ *
+ * Note: Some 0FB configurations may choose to skip registering any regions.
+ *       At most 64 regions may be registered with a single PMA.
+ *
+ * @param[in] id
+ *      A unique value in the range [0, 64) that uniquely identifies this
+ *      region. Passed in by RM in region-order. Should be continuous for
+ *      best performance.
+ *
+ * @param[in] bAsyncEccScrub
+ *      RM will set this when it is performing the initial ECC scrub
+ *      asynchronously.  All pages in the pma will be marked 'allocated pinned'.
+ *      RM will call pmaFreeContiguous as memory is scrubbed.
+ *
+ *      Until the scrub is complete (by calling pmaScrubComplete), no
+ *      allocation calls will fail with out of memory. Instead, it will hang
+ *      until scrubbing is complete. One exception is when the client passes
+ *      in the PMA_DONT_EVICT flag, in which case the call will actually fail
+ *      regardless of whether RM is scrubbing memory.
+ *
+ *      CAUTION! RM is responsible for ensuring black-listed pages are not
+ *      marked free during the scrub.
+ *
+ * @param[in] regionState:
+ *      Contains physical information about the region.
+ *
+ * @param[in] pBlacklistPageBase:
+ *      List of base addresses of bad GPU pages.
+ *      Each address is assumed to reference to a page of size
+ *      PMA_GRANULARITY (64kb).
+ *
+ * Implementors note: PMA will simply mark these pages as "allocatedPinned".
+ * This list should be saved so that during pmaDestroy we can verify that only
+ * these pages are still allocated.
+ *
+ * @param[in] blacklistCount:
+ *      Number of pages in above list.
+ *
+ * @return
+ *      NV_ERR_NO_MEMORY:
+ *          Internal memory allocation failed.
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error code,
+ *          because it is not very informative.
+ *
+ */
+NV_STATUS pmaRegisterRegion(PMA *pPma, NvU32 id, NvBool bAsyncEccScrub,
+    PMA_REGION_DESCRIPTOR *pRegionDesc, NvU32 blacklistCount, PPMA_BLACKLIST_ADDRESS pBlacklistPage);
+
+
+/*!
+ * @brief Synchronous API for allocating pages from the PMA.
+ * PMA will decide which pma regions to allocate from based on the provided
+ * flags.  PMA will also initiate UVM evictions to make room for this
+ * allocation unless prohibited by PMA_FLAGS_DONT_EVICT.  UVM callers must pass
+ * this flag to avoid deadlock.  Only UVM may allocate unpinned memory from this
+ * API and note that eviction callbacks need to be registered before that
+ * happens.
+ *
+ * Alignment of the allocated pages is guaranteed to be greater or equal to the
+ * requested page size. For contiguous allocations, a greater alignment can be
+ * specified with the PMA_ALLOCATE_FORCE_ALIGNMENT flag and the alignment
+ * allocation option. For non-contiguous allocations, it's an error to specify
+ * an alignment larger than the page size.
+ *
+ * For broadcast methods, PMA will guarantee the same physical frames are
+ * allocated on multiple GPUs, specified by the PMA objects passed in.
+ *
+ * Implementors note:
+ *      If region registered with asyncEccScrub and pmaScrubComplete
+ *      has not yet been issued then we cannot return NV_ERR_NO_MEMORY.
+ *      We must instead drop the lock and wait until the next call to
+ *      either pmaScrubComplete or pmaFreeContiguous/Pages to retry.
+ *      Exception: PMA_ALLOCATE_DONT_EVICT
+ *
+ * @param[in] pPma
+ *      The input PMA object
+ *
+ * @param[in] pageCount
+ *      Number of pages to allocate.
+ *
+ * @param[in] pageSize
+ *      64kb, 128kb or 2mb.  No other values are permissible.
+ *
+ * @param[in/out] allocationOptions
+ * Input flags:
+ *      PMA_ALLOCATE_DONT_EVICT
+ *          Do not evict in order to service this allocation.
+ *          Do not wait for ECC scrub completion if out of memory.
+ *      PMA_ALLOCATE_PINNED
+ *          The allocation is pinned (RM must pass this)
+ *      PMA_ALLOCATE_SPECIFY_MININUM_SPEED
+ *          Only examines regions whose speed is greater than
+ *          minimumSpeed.
+ *      PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE
+ *          Restrict the allowable physical address range for
+ *          allocation to [physBegin, physEnd).
+ *      PMA_ALLOCATE_SPECIFY_REGION_ID
+ *          Only service allocations out of 'regionId'.  The
+ *          regionId is assigned in pmaRegisterRegion.
+ *      PMA_ALLOCATE_PREFER_SLOWEST
+ *          Prefer slower memory over faster.
+ *      PMA_ALLOCATE_CONTIGUOUS
+ *          If this allocation is a contiguous allocation
+ *      PMA_ALLOCATE_PERSISTENT
+ *          If this allocation should persist across suspend/resume
+ *      PMA_ALLOCATE_FORCE_ALIGNMENT
+ *          Force a specific alignment of the allocation. For non-contiguous
+ *          allocations has to be less or equal to the page size.
+ *
+ * Output flags:
+ *      PMA_ALLOCATE_RESULT_IS_ZERO
+ *          If the allocated pages have been scrubbed.
+ *
+ * @param[out] pPages
+ *      Return array of base addresses of allocated pages.
+ *
+ * @return
+ *      NV_ERR_NO_MEMORY:
+ *          Internal memory allocation failed.
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code,because it is not very informative.
+ *
+ */
+NV_STATUS pmaAllocatePages(PMA *pPma, NvLength pageCount, NvU32 pageSize,
+    PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages);
+
+// allocate on multiple GPU, thus pmaCount
+NV_STATUS pmaAllocatePagesBroadcast(PMA **pPma, NvU32 pmaCount, NvLength allocationCount,
+    NvU32 pageSize, PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages);
+
+
+/*!
+ * @brief Marks previously unpinned pages as pinned.
+ *
+ * It will return an error and rollback any change if any page is not
+ * previously marked "unpinned".
+ *
+ * @param[in] pPages
+ *      Array of base addresses of pages to pin
+ *
+ * @param[in] pageCount
+ *      Number of pages to pin
+ *
+ * @param[in] pageSize
+ *      Page size of each page being pinned
+ *
+ * @return
+ *      NV_OK:
+ *          The pages have been pinned successfully.
+ *
+ *      NV_ERR_IN_USE:
+ *          Some of the pages requested to be pinned are being evicted and thus
+ *          cannot be pinned. None of the pages have been pinned and the caller
+ *          needs to make sure the pages can be successfully evicted.
+ *
+ *      NV_ERR_INVALID_STATE:
+ *          Some of the pages requested to be pinned weren't in the allocated
+ *          unpinned state.
+ *
+ *      TODO some error for rollback
+ *
+ */
+NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize);
+
+
+/*!
+ * @brief Marks previously pinned pages as unpinned.
+ *
+ * It will return an error and rollback any change if any page is not
+ * previously marked "pinned". Behaviour is undefined is any blacklisted
+ * pages are unpinned.
+ *
+ * @param[in] pPages
+ *      Array of base addresses of pages to pin
+ *
+ * @param[in] pageCount
+ *      Number of pages to pin
+ *
+ * @param[in] pageSize
+ *      Page size of each page being unpinned
+ *
+ * @return
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code,because it is not very informative.
+ *      TODO some error for rollback
+ *
+ */
+NV_STATUS pmaUnpinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize);
+
+
+/*!
+ * @brief Marks a list of pages as free.
+ * This operation is also used by RM to mark pages as "scrubbed" for the
+ * initial ECC sweep. This function does not fail.
+ *
+ * @param[in] pPages
+ *      Array of base addresses of pages to free
+ *
+ * @param[in] pageCount
+ *      Number of pages to free
+ *      If the value is 1, this is a contiguous free
+ *
+ * @param[in] size
+ *      When freeing contiguous memory, this is the total size;
+ *      When freeing discontiguous memory, this is page size of each page.
+ *
+ * @param[in] flag
+ *      PMA_FREE_SKIP_SCRUB
+ *          This flag is used to disable scrub on free when PMA frees the page
+ *
+ * @return Void
+ *
+ */
+void pmaFreePages(PMA *pPma, NvU64 *pPages, NvU64 pageCount, NvU64 size, NvU32 flag);
+
+/*!
+ * @brief Clears scrubbing bit on PMA pages within the supplied range.
+ *
+ * @param[in] pma
+ *      PMA object
+ *
+ * @param[in] rangeBase
+ *      The start address
+ *
+ * @param[in] rangeLimit
+ *      The end address
+ *
+ * @return
+ *      void
+ */
+void pmaClearScrubRange(PMA *pPma, NvU64 rangeBase, NvU64 rangeLimit);
+
+
+/*!
+ * @brief Notifies the PMA that the ECC scrub is complete.
+ *
+ * Until this function is called no PMA allocations will fail with
+ * "insufficient memory". They will hang and wait for the scrubbing.
+ *
+ * TODO consider suspend/resume behavior!
+ *
+ * Design Note:
+ *      pmaRegisterRegion(w/ asyncEccScrub) leaves all pages as
+ *      "allocated and pinned".
+ *
+ *      As the ECC scrub progresses, RM will call PMA and "pmaFreeContiguous"
+ *      the regions as they are scrubbed.
+ *
+ * @param[in] pma
+ *      PMA object
+ *
+ * @return
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code,because it is not very informative.
+ *
+ */
+NV_STATUS pmaScrubComplete(PMA *pPma);
+
+
+/*!
+ * Register the eviction callbacks.
+ *
+ * Only one set of callbacks can be registered at a time and they need to be
+ * unregistered with pmaUnregisterEvictionCb() before a new set can be
+ * registered.
+ *
+ * Note that eviction callbacks need to be registered before allocating any unpinned memory.
+ *
+ * See the documentation of the callbacks above for details of the eviction.
+ *
+ * @param[in] pma
+ *      PMA object
+ *
+ * @param[in] evictPageCb
+ *      The callback function for evicting pages at a time
+ *
+ * @param[in] evictRangeCb
+ *      The callback function for evicting a range
+ *
+ * @param[in] ctxPtr
+ *      The callback context pointer to be passed back on callback
+ *
+ * @return
+ *      NV_ERR_INVALID_ARGUMENT:
+ *          One of the callbacks or PMA object was NULL.
+ *
+ *      NV_ERR_INVALID_STATE:
+ *          Callbacks already registered.
+ */
+NV_STATUS pmaRegisterEvictionCb(PMA *pPma, pmaEvictPagesCb_t evictPagesCb, pmaEvictRangeCb_t evictRangeCb, void *ctxPtr);
+
+
+/*!
+ * Unregister the eviction callbacks.
+ *
+ * Guarantees that all pending eviction callbacks complete before returning.
+ *
+ * All unpinned allocations must be freed before the callbacks are unregistered
+ * and the caller needs to guarantee that any pending eviction callbacks won't
+ * block on the thread unregistering the callbacks.
+ *
+ * The call does nothing if the PMA object is NULL.
+ *
+ * @param[in] pma
+ *      PMA object.
+ */
+void pmaUnregisterEvictionCb(PMA *pPma);
+
+/*!
+ * @brief Returns information about the total FB memory.
+ *
+ * @param[in]  pPma           PMA pointer
+ * @param[in]  pBytesTotal    Pointer that will return the total FB memory size.
+ *
+ * @return
+ *      void
+ */
+void pmaGetTotalMemory(PMA *pPma, NvU64 *pBytesTotal);
+
+/*!
+ * @brief Returns information about each region managed by PMA
+ *
+ * @param[in]  pPma           PMA pointer
+ * @param[out] pRegSize       Pointer to size of region descriptor array
+ * @param[out] ppRegionDesc   Pointer to the array of region descriptors
+ *
+ * @return
+ *      NV_STATUS codes based on convention
+ */
+NV_STATUS pmaGetRegionInfo(PMA *pPma, NvU32 *pRegSize, PMA_REGION_DESCRIPTOR **ppRegionDesc);
+
+/*!
+ * @brief Returns information about the total free FB memory.
+ *
+ * @param[in]  pPma           PMA pointer
+ * @param[in]  pBytesFree     Pointer that will return the free FB memory size.
+ *
+ * @return
+ *      void
+ */
+void pmaGetFreeMemory(PMA *pPma, NvU64 *pBytesFree);
+
+/*!
+ * @brief Returns information about the largest free FB memory chunk across all regions.
+ *
+ * @param[in]  pPma           PMA pointer
+ * @param[in]  pLargestFree   Pointer that will return the largest free FB memory size.
+ * @param[in]  pRegionBase    Pointer that will return the region base of largest free FB memory.
+ * @param[in]  pLargestOffset Pointer that will return the offset in region for largest free FB memory.
+ *
+ * @return
+ *      void
+ */
+void pmaGetLargestFree(PMA *pPma, NvU64 *pLargestFree, NvU64 *pRegionBase, NvU64 *pLargestOffset);
+
+/*!
+ * @brief Returns a list of PMA allocated blocks which has ATTRIB_PERSISTENT
+ *        attribute set. It will be used by FBSR module to save/restore
+ *        clients PMA allocations during system suspend/resume.
+ *
+ * @param[in]     pPma              PMA pointer
+ * @param[in/out] ppPersistList     Pointer to list of persistent segments
+ *
+ * @return
+ *      NV_OK                   Success
+ *      NV_ERR_NO_MEMORY        Failure to allocate list
+ */
+NV_STATUS pmaBuildPersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList);
+
+
+/*!
+ * @brief Returns a list of all PMA allocated blocks. For all the PMA
+ *        allocated blocks, either STATE_PIN or STATE_UNPIN attribute will
+ *        be set. It will be used by FBSR module to save/restore clients
+ *        PMA allocations for Unix GC-OFF based power management.
+ *
+ * @param[in]     pPma      PMA pointer
+ * @param[in/out] ppList    Pointer to list of all the PMA allocated blocks.
+ *
+ * @return
+ *      NV_OK                   Success
+ *      NV_ERR_NO_MEMORY        Failure to allocate list
+ */
+NV_STATUS pmaBuildAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList);
+
+
+/*!
+ * @brief Frees previously generated list by function pmaBuildPersistentList().
+ *
+ * @param[in]       pPma            PMA pointer
+ * @param[in/out]   ppPersistList   Pointer to list of persistent segments
+ *
+ * @return
+ *      void
+ */
+void pmaFreePersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList);
+
+
+/*!
+ * @brief Frees previously generated list by function
+ *        pmaBuildAllocatedBlocksList().
+ *
+ * @param[in]     pPma      PMA pointer
+ * @param[in/out] ppList    Pointer to list of all the PMA allocated blocks.
+ *
+ * @return
+ *      void
+ */
+void pmaFreeAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList);
+
+/*!
+ * @brief Registers a memory scrubber to PMA. Currently only used for
+ * the scrub-on-free feature.
+ *
+ * This function will take the PMA lock to protect against races between
+ * the the use of the MemScrub object and any Reg/Unreg calls.
+ *
+ * @param[in]  pPma             PMA pointer
+ * @param[in]  pScrubObj        Pointer to the scrubber
+ *
+ * @return
+ *     NV_OK                    Success
+ *     NV_INVALID_STATE         When PMA is NULL or bMemScrub is NV_FALSE
+ */
+NV_STATUS pmaRegMemScrub(PMA *pPma, OBJMEMSCRUB *pScrubObj);
+
+
+/*!
+ * @brief Unregisters the memory scrubber, when the scrubber is torn
+ * down. If any feature is turned on that would require this scrubber to be
+ * present, after this call, the PMA object will be unavailable to give out
+ * any free pages for use, until pmaRegMemScrub is called.
+ *
+ * This function will take the PMA lock to protect against races between
+ * the use of the MemScrub object and any Reg/Unreg calls.
+ *
+ * @param[in]  pPma             PMA pointer
+ *
+ * @return
+ *     void
+ */
+void pmaUnregMemScrub(PMA *pPma);
+
+
+/*!
+ * @brief Notifies PMA that node onlining is complete and we can make pass-through
+ * calls to OS.
+ *
+ * This function will take the PMA lock to protect against races between
+ * the use of online/offline calls.
+ *
+ * TODO: Under development!
+ * Currently:
+ * - We online at hardcoded offset and size of PMA memory
+ * - Normal PMA allocation will go to the part of PMA that is not onlined
+ *
+ * Future:
+ * - Online all of PMA memory
+ * - PMA requests will suballocate from OS
+ * - Any mapping created over the PMA range will have to use kmap instead of ioremap
+ *
+ * Bug #1973975, 1858365, 1943187
+ *
+ * @param[in]  pPma              PMA pointer
+ * @param[in]  numaNodeId        NUMA node ID that PMA is managing
+ * @param[in]  coherentCpuFbBase The ATS aperture base corresponding to start of FB
+ * @param[in]  coherentCpuFbSize The ATS aperture size. The actual size we
+ *                               onlined. This could be different from the HW
+ *                               ATS aperture size.
+ *
+ * @return
+ *     NV_OK                    Success
+ *     NV_INVALID_STATE         When PMA is NULL or bNuma is NV_FALSE or nodeId is too big
+ */
+NV_STATUS pmaNumaOnlined(PMA *pPma, NvS32 numaNodeId,
+                         NvU64 coherentCpuFbBase, NvU64 coherentCpuFbSize);
+
+
+/*!
+ * @brief Notifies PMA that node offlining has started and PMA should start failing
+ * all allocation calls.
+ *
+ * This function will take the PMA lock to protect against races between
+ * the use of online/offline calls.
+ *
+ * @param[in]  pPma             PMA pointer
+ *
+ * @return
+ *     void
+ */
+void pmaNumaOfflined(PMA *pPma);
+
+/*!
+ * @brief Returns client managed blacklisted pages in the PMA region
+ *
+ * @param[in]  pPma             PMA pointer
+ * @param[in]  pChunks          pointer to blacklist addresses in the PMA region
+ * @param[in]  pPageSize        pointer to Size of each blacklist page addresses
+ * @param[in]  pNumChunks       pointer to valid client managed blacklist pages
+ *
+ * @return
+ *     void
+ */
+void pmaGetClientBlacklistedPages(PMA *pPma, NvU64 *pChunks, NvU32 *pPageSize, NvU32 *pNumChunks);
+
+/*!
+ * @brief Returns the PMA blacklist size in bytes for
+ *        both statically and dynamically blacklisted pages.
+ *        pDynamicBlacklistSize and pStaticBlacklistSize are only copied-out if non-NULL.
+ *
+ * @param[in]  pPma                     PMA pointer
+ * @param[in]  pDynamicBlacklistSize    pointer to dynamic blacklist size (bytes)
+ * @param[in]  pStaticBlacklistSize     pointer to static blacklist size (bytes)
+ *
+ * @return
+ *     void
+ */
+void pmaGetBlacklistSize(PMA *pPma, NvU32 *pDynamicBlacklistSize, NvU32 *pStaticBlacklistSize);
+
+/*!
+ * @brief Clear scrub bit for pages from the list of scrub items
+ *        that fall in the base to base+size range to return these pages to PMA.
+ *
+ * @param[in]  pPma                     PMA pointer
+ * @param[in]  pPmaScrubList            list of scrub items each with a id, base and size
+ * @param[in]  count                    count of scrub items
+ *
+ * @return
+ *     void
+ */
+void pmaClearScrubbedPages(PMA *pPma, SCRUB_NODE *pPmaScrubList, NvU64 count);
+
+/*!
+ * @brief Print states of all regions
+ */
+void pmaPrintMapState(PMA *pPma);
+
+/*!
+ * @brief Track the given physical address as blacklisted page in PMA. This call will blacklist
+ *        the entire PMA page frame of size 64KB which contains the physical address.
+ *
+ * @param[in]  pPma                     PMA pointer
+ * @param[in]  physAddr                 Address of the blacklisted page
+ *
+ * Locking:
+ * - DO NOT call this function with the PMA lock already held.
+ * - This function will internally grab the PMA lock to update the attribute bits.
+ *
+ * @return
+ *     void
+ */
+NV_STATUS pmaAddToBlacklistTracking(PMA *pPma, NvU64 physBase);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PHYS_MEM_ALLOCATOR_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.h

@@ -0,0 +1,228 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef UTIL_H
+#define UTIL_H
+
+#include "phys_mem_allocator.h"
+#include "nvport/nvport.h"
+#include "regmap.h"
+#include "nvmisc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// TODO See if this can be added to NvPort
+#define pmaPortAtomicGet(ptr) portAtomicOrSize((ptr), 0)
+
+NvU32 findRegionID(PMA *pPma, NvU64 address);
+void pmaPrintBlockStatus(PMA_PAGESTATUS blockStatus);
+void pmaRegionPrint(PMA *pPma, PMA_REGION_DESCRIPTOR *pRegion, void *pMap);
+NvBool pmaStateCheck(PMA *pPma);
+
+// Temporary putting these here. TODO refactor them in the next CL.
+NV_STATUS _pmaEvictContiguous(PMA *pPma, void *pMap, NvU64 evictStart, NvU64 evictEnd);
+NV_STATUS _pmaEvictPages(PMA *pPma, void *pMap, NvU64 *evictPages, NvU64 evictPageCount,
+                         NvU64 *allocPages, NvU64 allocPageCount, NvU32 pageSize, NvU64 physBegin,
+                         NvU64 physEnd);
+void      _pmaClearScrubBit(PMA *pPma, SCRUB_NODE *pPmaScrubList, NvU64 count);
+NV_STATUS _pmaCheckScrubbedPages(PMA *pPma, NvU64 chunkSize, NvU64 *pPages, NvU32 pageCount);
+NV_STATUS _pmaPredictOutOfMemory(PMA *pPma, NvLength allocationCount, NvU32 pageSize,
+                                 PMA_ALLOCATION_OPTIONS *allocationOptions);
+NV_STATUS pmaSelector(PMA *pPma, PMA_ALLOCATION_OPTIONS *allocationOptions, NvS32 *regionList);
+void      _pmaReallocBlacklistPages (PMA  *pPma, NvU32 regId, NvU64 rangeBegin, NvU64 rangeSize);
+void      _pmaFreeBlacklistPages (PMA  *pPma, NvU32 regId, NvU64 rangeBegin, NvU64 rangeSize);
+NvBool    _pmaLookupBlacklistFrame (PMA  *pPma, NvU32 regId, NvU64 frameNum);
+
+/*!
+ * @brief Marks a list of pages with the specified state and attributes.
+ * This operation is also used by RM to mark pages as "scrubbed" for the
+ * initial ECC sweep. This function does not fail.
+ *
+ * @param[in] base:
+ *      Start of address range to pin.
+ *
+ * @param[in] size:
+ *      Region size in bytes
+ *      Will try to align to 64KB page. For desired behavior, pass in aligned
+ *      sizes.
+ *
+ * @param[in] pmaState:
+ *      FREE, ALLOC_UNPIN, ALLOC_PIN
+ *      PERSISTENT
+ *      SCRUBBING
+ *      EVICTING
+ *
+ * @param[in] pmaStateWriteMask:
+ *      ALLOC_MASK
+ *      PERSISTENT
+ *      SCRUBBING
+ *      EVICTING
+ *
+ * @return
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code,because it is not very informative.
+ *
+ */
+void pmaSetBlockStateAttribUnderPmaLock(PMA *pPma, NvU64 base, NvU64 size,
+    PMA_PAGESTATUS pmaState, PMA_PAGESTATUS pmaStateWriteMask);
+
+/*!
+ * @brief Marks a list of pages with the specified state and attributes.
+ * This operation is also used by RM to mark pages as "scrubbed" for the
+ * initial ECC sweep. This function does not fail.
+ *
+ * @param[in] base:
+ *      Start of address range to pin.
+ *
+ * @param[in] size:
+ *      Region size in bytes
+ *      Will try to align to 64KB page. For desired behavior, pass in aligned
+ *      sizes.
+ *
+ * @param[in] pmaState:
+ *      FREE, ALLOC_UNPIN, ALLOC_PIN
+ *      PERSISTENT
+ *      SCRUBBING
+ *      EVICTING
+ *
+ * @param[in] pmaStateWriteMask:
+ *      ALLOC_MASK
+ *      PERSISTENT
+ *      SCRUBBING
+ *      EVICTING
+ *
+ * @return
+ *      NV_ERR_GENERIC:
+ *          Unexpected error. We try hard to avoid returning this error
+ *          code,because it is not very informative.
+ *
+ */
+void pmaSetBlockStateAttrib(PMA *pPma, NvU64 base, NvU64 size,
+    PMA_PAGESTATUS pmaState, PMA_PAGESTATUS pmaStateWriteMask);
+
+/*
+ * @brief Update the per region specific frame state statistics. This function helps
+ * keep a running count of number of frames that are STATE_FREE, STATE_UNPIN, STATE_PIN.
+ * It also keeps the 64KB free frame & 2MB pages statistics via pNumFree statistics.
+ *
+ * @param[in/out] pNumFree:
+ *      Pointer to the statistic counter to update
+ * @param[in/out] numPages:
+ *      The number of pages to adjust pNumFree by
+ * @param[in] oldState:
+ *      The state the page was in
+ *
+ * @param[in] newState:
+ *      The state the page will be in
+ */
+void pmaStatsUpdateState(NvU64 *pNumFree, NvU64 numPages, PMA_PAGESTATUS oldState,
+	                     PMA_PAGESTATUS newState);
+
+NvBool pmaIsEvictionPending(PMA *pPma);
+
+void pmaOsSchedule(void);
+
+/*!
+ * @brief Returns a list of PMA-managed blocks with the specified state and
+ *        attributes.
+ *
+ * @param[in]     pPma          PMA pointer
+ * @param[in/out] ppList        Pointer to list of segments having specified
+ *                              state and attributes
+ * @param[in]     pageStatus    PMA page state and attribute
+ *
+ * @return
+ *      NV_OK                   Success
+ *      NV_ERR_NO_MEMORY        Failure to allocate list
+ */
+NV_STATUS pmaBuildList(PMA *pPma, PRANGELISTTYPE *ppList,
+                       PMA_PAGESTATUS pageStatus);
+
+/*!
+ * @brief Frees previously generated list of PMA-managed blocks with
+ *        function pmaBuildList()
+ *
+ * @param[in]     pPma      PMA pointer
+ * @param[in/out] ppList    Pointer to list of PMA segments
+ *
+ * @return
+ *      None
+ */
+void pmaFreeList(PMA *pPma, PRANGELISTTYPE *ppList);
+
+/*!
+ * @brief Registers blacklisting information
+ * Called during pmaRegisterRegion to set the attribute for blacklisted pages
+ *
+ * @param[in] pPma                  PMA pointer
+ * @param[in] physAddrBase          The base address of this address tree
+ * @param[in] pBlacklistPageBase    Structure that contains the blacklisted pages
+ * @param[in] blacklistCount        Number of blacklisted pages
+ *
+ * @return NV_OK
+ *         NV_ERR_NO_MEMORY if memory allocation fails
+ */
+NV_STATUS pmaRegisterBlacklistInfo(PMA *pPma, NvU64 physAddrBase,
+                                      PPMA_BLACKLIST_ADDRESS pBlacklistPageBase, NvU32 blacklistCount);
+
+/*!
+ * @brief Query blacklisting states tracked by PMA
+ *
+ * @param[in] pPma                      PMA pointer
+ * @param[in] pBlacklistCount           Pointer to store count of blacklisted pages
+ * @param[in] pbClientManagedBlacklist  Pointer to store whether client manages blacklisting
+ * @param[in] ppBlacklistChunks         Pointer to store the blacklisted chunks
+ *
+ * @return void
+ */
+void pmaQueryBlacklistInfo(PMA *pPma, NvU32 *pBlacklistCount, NvBool *pbClientManagedBlacklist,
+                              PMA_BLACKLIST_CHUNK **ppBlacklistChunks);
+
+/*!
+ * @brief Update whether PMA has client managed blacklisting or not
+ *
+ * @param[in] pPma                     PMA pointer
+ * @param[in] bClientManagedBlacklist  Whether PMA has client managed blacklisting
+ *                                     When set to TRUE, PMA hands over control of blacklisted
+ *                                     pages to the OS
+ *                                     When set to FALSE, blacklisted pages are managed by PMA
+ *
+ * @return void
+ */
+void pmaSetClientManagedBlacklist(PMA *pPma, NvBool bClientManagedBlacklist);
+
+
+/*!
+ * @brief Checks if the address is already present in the blacklist info
+ *
+ * @param[in] pPma                     PMA pointer
+ * @param[in] physAddress              Physical address that needs to be blacklisted
+
+ * @return NvBool
+ */
+NvBool pmaIsBlacklistingAddrUnique(PMA *pPma, NvU64 physAddress);
+
+#endif // UTIL_H

src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/regmap.h

@@ -0,0 +1,275 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*!
+ *  @brief A bit map to keep track of FB frames
+ *
+ *  @bug Return status needs to conform to NV_STATUS
+ *          -- take a reference as output and always return NV_STATUS
+ *  @bug Is enum a good choice for this?
+ *
+ */
+
+#ifndef REGMAP_H
+#define REGMAP_H
+
+#include "map_defines.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//
+// Store the type here because we might use different algorithms for
+// different types of memory scan
+//
+typedef struct pma_regmap
+{
+    NvU64 totalFrames;                /* Total number of frames */
+    NvU64 mapLength;                  /* Length of the map */
+    NvU64 *map[PMA_BITS_PER_PAGE];    /* The bit map */
+    NvU64 frameEvictionsInProcess;    /* Count of frame evictions in-process */
+    PMA_STATS *pPmaStats;             /* Point back to the public struct in PMA structure */
+    NvBool bProtected;                /* The memory segment tracked by this regmap is protected (VPR/CPR) */
+} PMA_REGMAP;
+
+void pmaRegmapPrint(PMA_REGMAP *pMap);
+
+
+/*!
+ * @brief Initializes the regmap for PMA uses
+ *
+ * Allocates a bit map for all the pages being managed in this map. All bits
+ * are initialized to be FREE.
+ *
+ * @param[in]   numPages    The number of pages being managed in this map
+ * @param[in]   addrBase    The base address of this region. Unused in regmap
+ * @param[in]   pPmaPubStats The PMA stat struct to update
+ * @param[in]   bProtected   The map tracks pages in protected memory
+ *
+ * @return PMA_REGMAP Pointer to the regmap if succeeded, NULL otherwise
+ */
+void *pmaRegmapInit(NvU64 numPages, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected);
+
+
+/*!
+ * @brief Destroys the region map and free the memory
+ *
+ * @param[in]   pMap         The regmap to destroy
+ *
+ * @return void
+ */
+void pmaRegmapDestroy(void *pMap);
+
+/*!
+ * @brief Get/set number of evicting frames
+ * Used for sanity checking in PMA layer as well as performance optimization
+ * for the map layer to scan faster.
+ */
+NvU64 pmaRegmapGetEvictingFrames(void *pMap);
+void pmaRegmapSetEvictingFrames(void *pMap, NvU64 frameEvictionsInProcess);
+
+/*!
+ * @brief Changes the recorded state bits
+ *
+ * Changes the state of the bits given the physical frame number
+ *
+ * @param[in]   pMap         The regmap to change
+ * @param[in]   frameNum     The frame number to change
+ * @param[in]   newState     The new state to change to
+ *
+ * @return void
+ */
+void pmaRegmapChangeState(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
+
+/*!
+ * @brief Changes the recorded state & attrib bits
+ *
+ * Changes the state of the bits given the physical frame number
+ *
+ * @param[in]   pMap         The regmap to change
+ * @param[in]   frameNum     The frame number to change
+ * @param[in]   newState     The new state to change to
+ * @param[in]   writeAttrib  Write attribute bits as well
+ *
+ * @return void
+ */
+void pmaRegmapChangeStateAttrib(void *pMap, NvU64 frameNum,
+                                PMA_PAGESTATUS newState, NvBool writeAttrib);
+
+/*!
+ * @brief Changes the recorded state & attrib bits for an entire page
+ *
+ * Changes the state of the bits for an entire page given the physical
+ * frame number and the page size
+ *
+ * @param[in]   pMap          The regmap to change
+ * @param[in]   frameNumStart The frame number to change
+ * @param[in]   pageSize      The page size of the page to change
+ * @param[in]   newState      The new state to change to
+ * @param[in]   writeAttrib   Write attribute bits as well
+ *
+ * @return void
+ */
+void pmaRegmapChangePageStateAttrib(void * pMap, NvU64 frameNumStart, NvU32 pageSize,
+                                    PMA_PAGESTATUS newState, NvBool writeAttrib);
+
+/*!
+ * @brief Changes the state & attrib bits specified by mask
+ *
+ * Changes the state of the bits given the physical frame number
+ *
+ * @param[in]   pMap         The regmap to change
+ * @param[in]   frameNum     The frame number to change
+ * @param[in]   newState     The new state to change to
+ * @param[in]   newStateMask Specific bits to write
+ *
+ * @return void
+ */
+void pmaRegmapChangeStateAttribEx(void *pMap, NvU64 frameNum,
+                                  PMA_PAGESTATUS newState,
+                                  PMA_PAGESTATUS newStateMask);
+
+/*!
+ * @brief Read the page state & attrib bits
+ *
+ * Read the state of the page given the physical frame number
+ *
+ * @param[in]   pMap        The regmap to read
+ * @param[in]   frameNum    The frame number to read
+ * @param[in]   readAttrib  Read attribute bits as well
+ *
+ * @return PAGESTATUS of the frame
+ */
+PMA_PAGESTATUS pmaRegmapRead(void *pMap, NvU64 frameNum, NvBool readAttrib);
+
+
+/*!
+ * @brief Scan the list for contiguous space
+ *
+ * Scans the regmap for contiguous space that has the certain status. This
+ * function is optimized for performance if PMA_BITS_PER_PAGE is 2. It uses
+ * trailing-zero algorithm to determine which frame(s) has different status.
+ *
+ * @param[in]   pMap            The regmap to be scanned
+ * @param[in]   addrBase        The base address of this region
+ * @param[in]   rangeStart      The start of the restricted range
+ * @param[in]   rangeEnd        The end of the restricted range
+ * @param[in]   numPages        The number of pages we are scanning for
+ * @param[out]  freeList        A list of free frame numbers -- contains only 1 element
+ * @param[in]   pageSize        Size of one page
+ * @param[in]   alignment       Alignment requested by client
+ * @param[out]  pagesAllocated  Number of pages this call allocated
+ * @param[in]   bSkipEvict      Whether it's ok to skip the scan for evictable pages
+ *
+ * @return NV_OK if succeeded
+ * @return NV_ERR_IN_USE if found pages that can be evicted
+ * @return NV_ERR_NO_MEMORY if no available pages could be found
+ */
+NV_STATUS pmaRegmapScanContiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+/*!
+ * @brief Scan the list for any space
+ *
+ * Scans the regmap for numFrames number of frames, possibly scattered across
+ * the regmap. This function will allocate contiguous space whenever possible.
+ * Not optimized for performance yet.
+ *
+ * @param[in]   pMap            The regmap to be scanned
+ * @param[in]   addrBase        The base address of this region
+ * @param[in]   rangeStart      The start of the restricted range
+ * @param[in]   rangeEnd        The end of the restricted range
+ * @param[in]   numFrames       The number of frames we are scanning for
+ * @param[out]  freeList        A list of free frame numbers (allocated by client)
+ * @param[in]   pageSize        Size of one page
+ * @param[in]   alignment       Alignment requested by client. Has to be equal to the pageSize.
+ * @param[out]  pagesAllocated  Number of pages this call allocated
+ * @param[in]   bSkipEvict      Whether it's ok to skip the scan for evictable pages
+ *
+ * @return NV_OK if succeeded
+ * @return NV_ERR_IN_USE if found pages that can be evicted
+ * @return NV_ERR_NO_MEMORY if no available pages could be found
+ */
+NV_STATUS pmaRegmapScanDiscontiguous(
+    void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
+    NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
+    NvU64 *pagesAllocated, NvBool bSkipEvict);
+
+/*!
+ * @brief Gets the total size of specified PMA managed region.
+ *
+ * Gets the total size of current PMA managed region in the FB.
+ *
+ * @param[in]  pMap         Pointer to the regmap for the region
+ * @param[in]  pBytesTotal  Pointer that will return total bytes for current region.
+ *
+ */
+void pmaRegmapGetSize(void *pMap, NvU64 *pBytesTotal);
+
+
+/*!
+ * @brief Gets the size of the maximum free chunk of memory in specified region.
+ *
+ * Gets the size of the maximum free chunk of memory in the specified PMA managed
+ * region of the FB.
+ *
+ * @param[in]  pMap         Pointer to the regmap for the region
+ * @param[in]  pLargestFree Pointer that will return largest free in current region.
+ *
+ */
+void pmaRegmapGetLargestFree(void *pMap, NvU64 *pLargestFree);
+
+
+/*!
+ * @brief Returns the address range that is completely available for eviction.
+ *        - Should be ALLOC_UNPIN.
+ * In NUMA, OS manages memory and PMA will only track allocated memory in ALLOC_PIN
+ * and ALLOC_UNPIN state. FREE memory is managed by OS and cannot be tracked by PMA
+ * and hence PMA cannot consider FREE memory for eviction and can only consider frames
+ * in known state to PMA or eviction. ALLOC_PIN cannot be evicted and hence only ALLOC_UNPIN
+ * can be evictable.
+ *
+ *
+ * @param[in]  pMap         Pointer to the regmap for the region
+ * @param[in]  addrBase     Base address of the region
+ * @param[in]  actualSize   Size of the eviction range
+ * @param[in]  pageSize     Pagesize
+ * @param[out] evictStart   Starting address of the eviction range
+ * @param[out] evictEnd     End address of the eviction range.
+ *
+ * Returns:
+ *  -  NV_OK If there is evictable range of given size : actualSize
+ *
+ *  -  NV_ERR_NO_MEMORY if no contiguous range is evictable.
+ */
+NV_STATUS pmaRegMapScanContiguousNumaEviction(void *pMap, NvU64 addrBase,
+    NvLength actualSize, NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // REGMAP_H

src/nvidia/inc/kernel/gpu/mem_mgr/vaspace_api.h

@@ -0,0 +1,3 @@
+
+#include "g_vaspace_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/virt_mem_allocator.h

@@ -0,0 +1,3 @@
+
+#include "g_virt_mem_allocator_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_mgr/virt_mem_allocator_common.h

@@ -0,0 +1,151 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef VIRT_MEM_ALLOCATOR_COMMON_H
+#define VIRT_MEM_ALLOCATOR_COMMON_H
+
+/********************************* DMA Manager *****************************\
+*                                                                           *
+*       DMA object/engine management.                                       *
+*                                                                           *
+****************************************************************************/
+
+#include "nvtypes.h"
+#include "nvgputypes.h"
+#include "nvstatus.h"
+
+typedef struct OBJGPU OBJGPU;
+typedef struct ChannelDescendant ChannelDescendant;
+typedef struct ContextDma ContextDma;
+typedef struct Memory Memory;
+typedef struct EVENTNOTIFICATION EVENTNOTIFICATION;
+//---------------------------------------------------------------------------
+//
+//  Memory page defines.
+//
+//  These correspond to the granularity understood by the hardware
+//  for address mapping; the system page size can be larger.
+//
+//---------------------------------------------------------------------------
+#define RM_PAGE_SIZE        4096
+#define RM_PAGE_SIZE_64K    (64 * 1024)
+#define RM_PAGE_SIZE_128K   (128 * 1024)
+#define RM_PAGE_MASK        0x0FFF
+#define RM_PAGE_SHIFT       12
+#define RM_PAGE_SHIFT_64K   16
+
+// Huge page size is 2 MB
+#define RM_PAGE_SHIFT_HUGE 21
+#define RM_PAGE_SIZE_HUGE  (1 << RM_PAGE_SHIFT_HUGE)
+#define RM_PAGE_MASK_HUGE  ((1 << RM_PAGE_SHIFT_HUGE) - 1)
+
+// 512MB page size
+#define RM_PAGE_SHIFT_512M 29
+#define RM_PAGE_SIZE_512M  (1 << RM_PAGE_SHIFT_512M)
+#define RM_PAGE_MASK_512M  (RM_PAGE_SIZE_512M - 1)
+
+//---------------------------------------------------------------------------
+//
+//  Memory page attributes.
+//
+//  These attributes are used by software for page size mapping;
+//  Big pages can be of 64/128KB[Fermi/Kepler/Pascal]
+//  Huge page is 2 MB[Pascal+]
+//  512MB page is Ampere+
+//  Default page attribute lets driver decide the optimal page size
+//
+//---------------------------------------------------------------------------
+typedef enum
+{
+    RM_ATTR_PAGE_SIZE_DEFAULT = 0x0,
+    RM_ATTR_PAGE_SIZE_4KB     = 0x1,
+    RM_ATTR_PAGE_SIZE_BIG     = 0x2,
+    RM_ATTR_PAGE_SIZE_HUGE    = 0x3,
+    RM_ATTR_PAGE_SIZE_512MB   = 0x4,
+    RM_ATTR_PAGE_SIZE_INVALID = 0x5
+}
+RM_ATTR_PAGE_SIZE;
+
+//---------------------------------------------------------------------------
+//
+//  Notification buffer structure.
+//
+//---------------------------------------------------------------------------
+typedef union _def_info_status_buffer
+{
+    struct
+    {
+        NvV16 OtherInfo16;
+        NvV16 Status;
+    } Info16Status_16;
+
+    NvU32 Info16Status_32;
+
+} INFO16_STATUS;
+
+typedef struct _def_notification_buffer
+{
+    NvU32 TimeLo;
+    NvU32 TimeHi;
+    NvV32 OtherInfo32;
+    INFO16_STATUS Info16Status;
+} NOTIFICATION, *PNOTIFICATION;
+
+
+//---------------------------------------------------------------------------
+//
+//  Function prototypes.
+//
+//---------------------------------------------------------------------------
+void      notifyMethodComplete(OBJGPU*, ChannelDescendant *, NvU32, NvV32, NvU32, NvU16, NV_STATUS);
+
+NV_STATUS notifyFillNotifier               (OBJGPU*, ContextDma *, NvV32, NvV16, NV_STATUS);
+NV_STATUS notifyFillNotifierOffset         (OBJGPU*, ContextDma *, NvV32, NvV16, NV_STATUS, NvU64);
+NV_STATUS notifyFillNotifierOffsetTimestamp(OBJGPU*, ContextDma *, NvV32, NvV16, NV_STATUS, NvU64, NvU64);
+NV_STATUS notifyFillNotifierArray          (OBJGPU*, ContextDma *, NvV32, NvV16, NV_STATUS, NvU32);
+NV_STATUS notifyFillNotifierArrayTimestamp (OBJGPU*, ContextDma *, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
+void notifyFillNOTIFICATION(OBJGPU       *pGpu,
+                            NOTIFICATION *pNotifyBuffer,
+                            NvV32         Info32,
+                            NvV16         Info16,
+                            NV_STATUS     CompletionStatus,
+                            NvBool        TimeSupplied,
+                            NvU64         Time);
+NV_STATUS notifyFillNotifierGPUVA          (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32);
+NV_STATUS notifyFillNotifierGPUVATimestamp (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
+NV_STATUS notifyFillNotifierMemory         (OBJGPU*, Memory *, NvV32, NvV16, NV_STATUS, NvU32);
+NV_STATUS notifyFillNotifierMemoryTimestamp(OBJGPU*, Memory *, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
+void notifyFillNvNotification(OBJGPU         *pGpu,
+                              NvNotification *pNotification,
+                              NvV32           Info32,
+                              NvV16           Info16,
+                              NV_STATUS       CompletionStatus,
+                              NvBool          TimeSupplied,
+                              NvU64           Time);
+
+NV_STATUS semaphoreFillGPUVA         (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV32, NvBool);
+NV_STATUS semaphoreFillGPUVATimestamp(OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV32, NvBool, NvU64);
+
+RM_ATTR_PAGE_SIZE dmaNvos32ToPageSizeAttr(NvU32 attr, NvU32 attr2);
+
+#endif // VIRT_MEM_ALLOCATOR_COMMON_H

src/nvidia/inc/kernel/gpu/mem_sys/kern_mem_sys.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_mem_sys_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mem_sys/zbc_api.h

@@ -0,0 +1,3 @@
+
+#include "g_zbc_api_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mig_mgr/compute_instance_subscription.h

@@ -0,0 +1,3 @@
+
+#include "g_compute_instance_subscription_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mig_mgr/gpu_instance_subscription.h

@@ -0,0 +1,3 @@
+
+#include "g_gpu_instance_subscription_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mig_mgr/kernel_mig_manager.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_mig_manager_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mig_mgr/mig_config_session.h

@@ -0,0 +1,3 @@
+
+#include "g_mig_config_session_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mig_mgr/mig_monitor_session.h

@@ -0,0 +1,3 @@
+
+#include "g_mig_monitor_session_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mmu/kern_gmmu.h

@@ -0,0 +1,3 @@
+
+#include "g_kern_gmmu_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mmu/mmu_fault_buffer.h

@@ -0,0 +1,3 @@
+
+#include "g_mmu_fault_buffer_nvoc.h"
+

src/nvidia/inc/kernel/gpu/mmu/mmu_trace.h

@@ -0,0 +1,111 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2013-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef MMU_TRACE_H
+#define MMU_TRACE_H
+
+#include "core/core.h"
+#include "mem_mgr/vaspace.h"
+#include "mmu/mmu_fmt.h"
+#include "ctrl/ctrl83de.h"
+
+#define MMU_INVALID_ADDR (0xf) // All base addresses are aligned and 0xf is unaligned
+
+#define MMU_MAX_ENTRY_SIZE_BYTES 16
+
+/* ------------------------ Types definitions ------------------------------ */
+
+typedef union
+{
+  NvU8   v8[MMU_MAX_ENTRY_SIZE_BYTES                ];
+  NvU32 v32[MMU_MAX_ENTRY_SIZE_BYTES / sizeof(NvU32)];
+  NvU64 v64[MMU_MAX_ENTRY_SIZE_BYTES / sizeof(NvU64)];
+} MMU_ENTRY;
+
+typedef NvBool (*MmuTraceCbIsPte)(const void *pFmt, const MMU_FMT_LEVEL *pFmtLevel,
+                                  const MMU_ENTRY *pEntry, NvBool *pValid);
+typedef const void *(*MmuTraceCbGetFmtPde)(const void *pFmt, const MMU_FMT_LEVEL *pFmtLevel,
+                                           NvU32 sublevel);
+typedef const void *(*MmuTraceCbGetFmtPte)(const void *pFmt);
+typedef NvU64 (*MmuTraceCbGetPdePa)(OBJGPU *pGpu, const void *pFmtPde, const MMU_ENTRY *pPde);
+typedef NvU64 (*MmuTraceCbGetPtePa)(OBJGPU *pGpu, const void *pFmtPte, const MMU_ENTRY *pPte);
+typedef void (*MmuTraceCbPrintPdb)(OBJGPU *pGpu, OBJVASPACE *pVAS, NvU64 va,
+                                   NvU64 vaLimit);
+typedef void (*MmuTraceCbPrintPde)(OBJGPU *pGpu, const void *pFmt, const MMU_FMT_LEVEL *pFmtLevel,
+                                   const MMU_ENTRY *pPde);
+typedef void (*MmuTraceCbPrintPt)(OBJGPU *pGpu, const MMU_FMT_LEVEL *pFmtLevel, const void *pFmtPde,
+                                  const MMU_ENTRY *pPde);
+typedef void (*MmuTraceCbPrintPte)(OBJGPU *pGpu, const MMU_FMT_LEVEL *pFmtLevel, const void *pFmtPte,
+                                   const MMU_ENTRY *pPte, NvU32 index);
+typedef NvBool (*MmuTraceCbIsInvalidPdeOk)(OBJGPU *pGpu, const void *pFmt, const void *pFmtPde,
+                                           const MMU_ENTRY *pPde, NvU32 sublevel);
+typedef NvU32 (*MmuTraceCbPdeAddrSpace)(const void *pFmtPde, const MMU_ENTRY *pPde);
+typedef NvU32 (*MmuTraceCbPteAddrSpace)(const void *pFmtPte, const MMU_ENTRY *pPte);
+typedef NvU32 (*MmuTraceCbSwToHwLevel)(const void *pFmt, NvU32 level);
+
+typedef enum
+{
+    MMU_TRACE_MODE_TRACE         = 0,
+    MMU_TRACE_MODE_TRACE_VERBOSE = 1,
+    MMU_TRACE_MODE_TRANSLATE     = 2,
+    MMU_TRACE_MODE_VALIDATE      = 3,
+    MMU_TRACE_MODE_DUMP_RANGE    = 4
+} MMU_TRACE_MODE, *PMMU_TRACE_MODE;
+
+typedef struct
+{
+    NvU64                                      pa;
+    NvU32                                      aperture;
+    NvBool                                     valid;
+    NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS *pMapParams;
+    NvU64                                      validateCount;
+} MMU_TRACE_ARG, *PMMU_TRACE_ARG;
+
+typedef struct
+{
+    MMU_TRACE_MODE                     mode;
+    NvU64                              va;
+    NvU64                              vaLimit;
+    PMMU_TRACE_ARG                     pArg;
+} MMU_TRACE_PARAM, *PMMU_TRACE_PARAM;
+
+typedef struct
+{
+    MmuTraceCbIsPte          isPte;
+    MmuTraceCbGetFmtPde      getFmtPde;
+    MmuTraceCbGetFmtPte      getFmtPte;
+    MmuTraceCbGetPdePa       getPdePa;
+    MmuTraceCbGetPtePa       getPtePa;
+    MmuTraceCbPrintPdb       printPdb;
+    MmuTraceCbPrintPde       printPde;
+    MmuTraceCbPrintPt        printPt;
+    MmuTraceCbPrintPte       printPte;
+    MmuTraceCbIsInvalidPdeOk isInvalidPdeOk;
+    MmuTraceCbPdeAddrSpace   pdeAddrSpace;
+    MmuTraceCbPteAddrSpace   pteAddrSpace;
+    MmuTraceCbSwToHwLevel    swToHwLevel;
+} MMU_TRACE_CALLBACKS;
+
+NV_STATUS mmuTrace(OBJGPU *pGpu, OBJVASPACE *pVAS, MMU_TRACE_PARAM *pParams);
+
+#endif // MMU_TRACE_H

src/nvidia/inc/kernel/gpu/mmu/uvm_sw.h

@@ -0,0 +1,3 @@
+
+#include "g_uvm_sw_nvoc.h"
+

src/nvidia/inc/kernel/gpu/nvdec/kernel_nvdec.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_nvdec_nvoc.h"
+

src/nvidia/inc/kernel/gpu/nvdec/kernel_nvdec_ctx.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_nvdec_ctx_nvoc.h"
+

src/nvidia/inc/kernel/gpu/nvenc/kernel_nvenc_ctx.h

@@ -0,0 +1,3 @@
+
+#include "g_kernel_nvenc_ctx_nvoc.h"
+