570.211.01

(cherry picked from commit 1795a8bb20)

kernel-open/Kbuild

@@ -57,25 +57,32 @@ ifeq ($(NV_UNDEF_BEHAVIOR_SANITIZER),1)
  UBSAN_SANITIZE := y
 endif
 
+#
+# Command to create a symbolic link, explicitly resolving the symlink target
+# to an absolute path to abstract away the difference between Linux < 6.13,
+# where the CWD is the Linux kernel source tree for Kbuild extmod builds, and
+# Linux >= 6.13, where the CWD is the external module source tree.
+#
+# This is used to create the nv*-kernel.o -> nv*-kernel.o_binary symlinks for
+# kernel modules which use precompiled binary object files.
+#
+
+quiet_cmd_symlink = SYMLINK $@
+ cmd_symlink = ln -sf $(abspath $<) $@
+
+
 $(foreach _module, $(NV_KERNEL_MODULES), \
  $(eval include $(src)/$(_module)/$(_module).Kbuild))
 
 
-#
-# Define CFLAGS that apply to all the NVIDIA kernel modules. EXTRA_CFLAGS
-# is deprecated since 2.6.24 in favor of ccflags-y, but we need to support
-# older kernels which do not have ccflags-y. Newer kernels append
-# $(EXTRA_CFLAGS) to ccflags-y for compatibility.
-#
-
-EXTRA_CFLAGS += -I$(src)/common/inc
-EXTRA_CFLAGS += -I$(src)
-EXTRA_CFLAGS += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
-EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
-EXTRA_CFLAGS += -DNV_VERSION_STRING=\"550.127.08\"
+ccflags-y += -I$(src)/common/inc
+ccflags-y += -I$(src)
+ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
+ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
+ccflags-y += -DNV_VERSION_STRING=\"570.211.01\"
 
 ifneq ($(SYSSRCHOST1X),)
- EXTRA_CFLAGS += -I$(SYSSRCHOST1X)
+ ccflags-y += -I$(SYSSRCHOST1X)
 endif
 
 # Some Android kernels prohibit driver use of filesystem functions like
@@ -85,57 +92,57 @@ endif
 PLATFORM_IS_ANDROID ?= 0
 
 ifeq ($(PLATFORM_IS_ANDROID),1)
- EXTRA_CFLAGS += -DNV_FILESYSTEM_ACCESS_AVAILABLE=0
+ ccflags-y += -DNV_FILESYSTEM_ACCESS_AVAILABLE=0
 else
- EXTRA_CFLAGS += -DNV_FILESYSTEM_ACCESS_AVAILABLE=1
+ ccflags-y += -DNV_FILESYSTEM_ACCESS_AVAILABLE=1
 endif
 
-EXTRA_CFLAGS += -Wno-unused-function
+ccflags-y += -Wno-unused-function
 
 ifneq ($(NV_BUILD_TYPE),debug)
- EXTRA_CFLAGS += -Wuninitialized
+ ccflags-y += -Wuninitialized
 endif
 
-EXTRA_CFLAGS += -fno-strict-aliasing
+ccflags-y += -fno-strict-aliasing
 
 ifeq ($(ARCH),arm64)
- EXTRA_CFLAGS += -mstrict-align
+ ccflags-y += -mstrict-align
 endif
 
 ifeq ($(NV_BUILD_TYPE),debug)
- EXTRA_CFLAGS += -g
+ ccflags-y += -g
 endif
 
-EXTRA_CFLAGS += -ffreestanding
+ccflags-y += -ffreestanding
 
 ifeq ($(ARCH),arm64)
- EXTRA_CFLAGS += -mgeneral-regs-only -march=armv8-a
- EXTRA_CFLAGS += $(call cc-option,-mno-outline-atomics,)
+ ccflags-y += -mgeneral-regs-only -march=armv8-a
+ ccflags-y += $(call cc-option,-mno-outline-atomics,)
 endif
 
 ifeq ($(ARCH),x86_64)
- EXTRA_CFLAGS += -mno-red-zone -mcmodel=kernel
+ ccflags-y += -mno-red-zone -mcmodel=kernel
 endif
 
 ifeq ($(ARCH),powerpc)
- EXTRA_CFLAGS += -mlittle-endian -mno-strict-align -mno-altivec
+ ccflags-y += -mlittle-endian -mno-strict-align
 endif
 
-EXTRA_CFLAGS += -DNV_UVM_ENABLE
-EXTRA_CFLAGS += $(call cc-option,-Werror=undef,)
-EXTRA_CFLAGS += -DNV_SPECTRE_V2=$(NV_SPECTRE_V2)
-EXTRA_CFLAGS += -DNV_KERNEL_INTERFACE_LAYER
+ccflags-y += -DNV_UVM_ENABLE
+ccflags-y += $(call cc-option,-Werror=undef,)
+ccflags-y += -DNV_SPECTRE_V2=$(NV_SPECTRE_V2)
+ccflags-y += -DNV_KERNEL_INTERFACE_LAYER
 
 #
 # Detect SGI UV systems and apply system-specific optimizations.
 #
 
 ifneq ($(wildcard /proc/sgi_uv),)
- EXTRA_CFLAGS += -DNV_CONFIG_X86_UV
+ ccflags-y += -DNV_CONFIG_X86_UV
 endif
 
 ifdef VGX_FORCE_VFIO_PCI_CORE
- EXTRA_CFLAGS += -DNV_VGPU_FORCE_VFIO_PCI_CORE
+ ccflags-y += -DNV_VGPU_FORCE_VFIO_PCI_CORE
 endif
 
 WARNINGS_AS_ERRORS ?=
@@ -169,9 +176,12 @@ NV_CONFTEST_CMD := /bin/sh $(NV_CONFTEST_SCRIPT) \
 
 NV_CFLAGS_FROM_CONFTEST := $(shell $(NV_CONFTEST_CMD) build_cflags)
 
-NV_CONFTEST_CFLAGS = $(NV_CFLAGS_FROM_CONFTEST) $(EXTRA_CFLAGS) -fno-pie
+NV_CONFTEST_CFLAGS = $(NV_CFLAGS_FROM_CONFTEST) $(ccflags-y) -fno-pie
+NV_CONFTEST_CFLAGS += $(filter -std=%,$(KBUILD_CFLAGS))
 NV_CONFTEST_CFLAGS += $(call cc-disable-warning,pointer-sign)
 NV_CONFTEST_CFLAGS += $(call cc-option,-fshort-wchar,)
+NV_CONFTEST_CFLAGS += $(call cc-option,-Werror=incompatible-pointer-types,)
+NV_CONFTEST_CFLAGS += $(call cc-option,-fms-extensions,)
 NV_CONFTEST_CFLAGS += -Wno-error
 
 NV_CONFTEST_COMPILE_TEST_HEADERS := $(obj)/conftest/macros.h

kernel-open/Makefile

@@ -52,6 +52,22 @@ else
     endif
   endif
 
+  # If CC hasn't been set explicitly, check the value of CONFIG_CC_VERSION_TEXT.
+  # Look for the compiler specified there, and use it by default, if found.
+  ifeq ($(origin CC),default)
+    cc_version_text=$(firstword $(shell . $(KERNEL_OUTPUT)/.config; \
+                      echo "$$CONFIG_CC_VERSION_TEXT"))
+
+    ifneq ($(cc_version_text),)
+      ifeq ($(shell command -v $(cc_version_text)),)
+          $(warning WARNING: Unable to locate the compiler $(cc_version_text) \
+            from CONFIG_CC_VERSION_TEXT in the kernel configuration.)
+      else
+          CC=$(cc_version_text)
+      endif
+    endif
+  endif
+
   CC ?= cc
   LD ?= ld
   OBJDUMP ?= objdump
@@ -65,6 +81,16 @@ else
     )
   endif
 
+  KERNEL_ARCH = $(ARCH)
+
+  ifneq ($(filter $(ARCH),i386 x86_64),)
+    KERNEL_ARCH = x86
+  else
+    ifeq ($(filter $(ARCH),arm64 powerpc),)
+        $(error Unsupported architecture $(ARCH))
+    endif
+  endif
+
   NV_KERNEL_MODULES ?= $(wildcard nvidia nvidia-uvm nvidia-vgpu-vfio nvidia-modeset nvidia-drm nvidia-peermem)
   NV_KERNEL_MODULES := $(filter-out $(NV_EXCLUDE_KERNEL_MODULES), \
                                     $(NV_KERNEL_MODULES))
@@ -106,8 +132,9 @@ else
   # module symbols on which the Linux kernel's module resolution is dependent
   # and hence must be used whenever present.
 
-  LD_SCRIPT ?= $(KERNEL_SOURCES)/scripts/module-common.lds      \
-               $(KERNEL_SOURCES)/arch/$(ARCH)/kernel/module.lds \
+  LD_SCRIPT ?= $(KERNEL_SOURCES)/scripts/module-common.lds             \
+               $(KERNEL_SOURCES)/arch/$(KERNEL_ARCH)/kernel/module.lds \
+               $(KERNEL_OUTPUT)/arch/$(KERNEL_ARCH)/module.lds         \
                $(KERNEL_OUTPUT)/scripts/module.lds
   NV_MODULE_COMMON_SCRIPTS := $(foreach s, $(wildcard $(LD_SCRIPT)), -T $(s))
 

kernel-open/common/inc/nv-firmware.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -32,7 +32,10 @@
 typedef enum
 {
     NV_FIRMWARE_TYPE_GSP,
-    NV_FIRMWARE_TYPE_GSP_LOG
+    NV_FIRMWARE_TYPE_GSP_LOG,
+#if defined(NV_VMWARE)
+    NV_FIRMWARE_TYPE_BINDATA
+#endif
 } nv_firmware_type_t;
 
 typedef enum
@@ -44,6 +47,8 @@ typedef enum
     NV_FIRMWARE_CHIP_FAMILY_GA10X = 4,
     NV_FIRMWARE_CHIP_FAMILY_AD10X = 5,
     NV_FIRMWARE_CHIP_FAMILY_GH100 = 6,
+    NV_FIRMWARE_CHIP_FAMILY_GB10X = 8,
+    NV_FIRMWARE_CHIP_FAMILY_GB20X = 9,
     NV_FIRMWARE_CHIP_FAMILY_END,
 } nv_firmware_chip_family_t;
 
@@ -52,6 +57,8 @@ static inline const char *nv_firmware_chip_family_to_string(
 )
 {
     switch (fw_chip_family) {
+        case NV_FIRMWARE_CHIP_FAMILY_GB10X: return "gb10x";
+        case NV_FIRMWARE_CHIP_FAMILY_GB20X: return "gb20x";
         case NV_FIRMWARE_CHIP_FAMILY_GH100: return "gh100";
         case NV_FIRMWARE_CHIP_FAMILY_AD10X: return "ad10x";
         case NV_FIRMWARE_CHIP_FAMILY_GA10X: return "ga10x";
@@ -66,13 +73,13 @@ static inline const char *nv_firmware_chip_family_to_string(
     return NULL;
 }
 
-// The includer (presumably nv.c) may optionally define
-// NV_FIRMWARE_PATH_FOR_FILENAME(filename)
-// to return a string "path" given a gsp_*.bin or gsp_log_*.bin filename.
+// The includer may optionally define
+// NV_FIRMWARE_FOR_NAME(name)
+// to return a platform-defined string for a given a gsp_* or gsp_log_* name.
 //
-// The function nv_firmware_path will then be available.
-#if defined(NV_FIRMWARE_PATH_FOR_FILENAME)
-static inline const char *nv_firmware_path(
+// The function nv_firmware_for_chip_family will then be available.
+#if defined(NV_FIRMWARE_FOR_NAME)
+static inline const char *nv_firmware_for_chip_family(
     nv_firmware_type_t fw_type,
     nv_firmware_chip_family_t fw_chip_family
 )
@@ -81,15 +88,17 @@ static inline const char *nv_firmware_path(
     {
         switch (fw_chip_family)
         {
+            case NV_FIRMWARE_CHIP_FAMILY_GB10X:  // fall through
+            case NV_FIRMWARE_CHIP_FAMILY_GB20X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GH100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_AD10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GA10X:
-                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_ga10x.bin");
+                return NV_FIRMWARE_FOR_NAME("gsp_ga10x");
 
             case NV_FIRMWARE_CHIP_FAMILY_GA100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU11X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU10X:
-                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_tu10x.bin");
+                return NV_FIRMWARE_FOR_NAME("gsp_tu10x");
 
             case NV_FIRMWARE_CHIP_FAMILY_END:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_NULL:
@@ -100,33 +109,40 @@ static inline const char *nv_firmware_path(
     {
         switch (fw_chip_family)
         {
+            case NV_FIRMWARE_CHIP_FAMILY_GB10X:  // fall through
+            case NV_FIRMWARE_CHIP_FAMILY_GB20X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GH100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_AD10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GA10X:
-                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_ga10x.bin");
+                return NV_FIRMWARE_FOR_NAME("gsp_log_ga10x");
 
             case NV_FIRMWARE_CHIP_FAMILY_GA100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU11X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU10X:
-                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_tu10x.bin");
+                return NV_FIRMWARE_FOR_NAME("gsp_log_tu10x");
 
             case NV_FIRMWARE_CHIP_FAMILY_END:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_NULL:
                 return "";
         }
     }
-
+#if defined(NV_VMWARE)
+    else if (fw_type == NV_FIRMWARE_TYPE_BINDATA)
+    {
+        return NV_FIRMWARE_FOR_NAME("bindata_image");
+    }
+#endif
     return "";
 }
-#endif  // defined(NV_FIRMWARE_PATH_FOR_FILENAME)
+#endif  // defined(NV_FIRMWARE_FOR_NAME)
 
-// The includer (presumably nv.c) may optionally define
-// NV_FIRMWARE_DECLARE_GSP_FILENAME(filename)
+// The includer may optionally define
+// NV_FIRMWARE_DECLARE_GSP(name)
 // which will then be invoked (at the top-level) for each
-// gsp_*.bin (but not gsp_log_*.bin)
-#if defined(NV_FIRMWARE_DECLARE_GSP_FILENAME)
-NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_ga10x.bin")
-NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_tu10x.bin")
-#endif  // defined(NV_FIRMWARE_DECLARE_GSP_FILENAME)
+// gsp_* (but not gsp_log_*)
+#if defined(NV_FIRMWARE_DECLARE_GSP)
+NV_FIRMWARE_DECLARE_GSP("gsp_ga10x")
+NV_FIRMWARE_DECLARE_GSP("gsp_tu10x")
+#endif  // defined(NV_FIRMWARE_DECLARE_GSP)
 
-#endif  // NV_FIRMWARE_DECLARE_GSP_FILENAME
+#endif  // NV_FIRMWARE_DECLARE_GSP

kernel-open/common/inc/nv-ioctl.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -128,6 +128,9 @@ typedef struct nv_ioctl_register_fd
 
 #define NV_DMABUF_EXPORT_MAX_HANDLES 128
 
+#define NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT        0
+#define NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE     1
+
 typedef struct nv_ioctl_export_to_dma_buf_fd
 {
     int         fd;
@@ -136,6 +139,7 @@ typedef struct nv_ioctl_export_to_dma_buf_fd
     NvU32       numObjects;
     NvU32       index;
     NvU64       totalSize NV_ALIGN_BYTES(8);
+    NvU8        mappingType;
     NvHandle    handles[NV_DMABUF_EXPORT_MAX_HANDLES];
     NvU64       offsets[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8);
     NvU64       sizes[NV_DMABUF_EXPORT_MAX_HANDLES] NV_ALIGN_BYTES(8);

kernel-open/common/inc/nv-kthread-q-os.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2016 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -43,6 +43,8 @@ struct nv_kthread_q
     atomic_t main_loop_should_exit;
 
     struct task_struct *q_kthread;
+
+    bool is_unload_flush_ongoing;
 };
 
 struct nv_kthread_q_item

kernel-open/common/inc/nv-linux.h

@@ -58,14 +58,10 @@
 #include <linux/version.h>
 #include <linux/utsname.h>
 
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
-#error "This driver does not support kernels older than 2.6.32!"
-#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 7, 0)
-#  define KERNEL_2_6
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
-#  define KERNEL_3
-#else
-#error "This driver does not support development kernels!"
+#if LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0)
+// Version 4.4 is allowed, temporarily, although not officially supported.
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
+#error "This driver does not support kernels older than Linux 4.15!"
 #endif
 
 #if defined (CONFIG_SMP) && !defined (__SMP__)
@@ -235,12 +231,6 @@ NV_STATUS nvos_forward_error_to_cray(struct pci_dev *, NvU32,
         const char *, va_list);
 #endif
 
-#if defined(NVCPU_PPC64LE) && defined(CONFIG_EEH)
-#include <asm/eeh.h>
-#define NV_PCI_ERROR_RECOVERY_ENABLED() eeh_enabled()
-#define NV_PCI_ERROR_RECOVERY
-#endif
-
 #if defined(NV_ASM_SET_MEMORY_H_PRESENT)
 #include <asm/set_memory.h>
 #endif
@@ -355,8 +345,6 @@ extern int nv_pat_mode;
 
 #define NV_PAGE_COUNT(page) \
   ((unsigned int)page_count(page))
-#define NV_GET_PAGE_COUNT(page_ptr) \
-  (NV_PAGE_COUNT(NV_GET_PAGE_STRUCT(page_ptr->phys_addr)))
 #define NV_GET_PAGE_FLAGS(page_ptr) \
   (NV_GET_PAGE_STRUCT(page_ptr->phys_addr)->flags)
 
@@ -613,7 +601,7 @@ static NvBool nv_numa_node_has_memory(int node_id)
 
 #define NV_ALLOC_PAGES_NODE(ptr, nid, order, gfp_mask) \
     { \
-        (ptr) = (unsigned long)page_address(alloc_pages_node(nid, gfp_mask, order)); \
+        (ptr) = (unsigned long) alloc_pages_node(nid, gfp_mask, order); \
     }
 
 #define NV_GET_FREE_PAGES(ptr, order, gfp_mask)      \
@@ -728,6 +716,7 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
 #endif
 }
 
+#define NV_GET_OFFSET_IN_PAGE(phys_page) offset_in_page(phys_page)
 #define NV_GET_PAGE_STRUCT(phys_page) virt_to_page(__va(phys_page))
 #define NV_VMA_PGOFF(vma)             ((vma)->vm_pgoff)
 #define NV_VMA_SIZE(vma)              ((vma)->vm_end - (vma)->vm_start)
@@ -844,16 +833,16 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
 #define NV_PRINT_AT(nv_debug_level,at)                                           \
     {                                                                            \
         nv_printf(nv_debug_level,                                                \
-            "NVRM: VM: %s:%d: 0x%p, %d page(s), count = %d, flags = 0x%08x, "    \
+            "NVRM: VM: %s:%d: 0x%p, %d page(s), count = %lld, "                  \
             "page_table = 0x%p\n",  __FUNCTION__, __LINE__, at,                  \
-            at->num_pages, NV_ATOMIC_READ(at->usage_count),                      \
-            at->flags, at->page_table);                                          \
+            at->num_pages, (long long)atomic64_read(&at->usage_count),           \
+            at->page_table);                                                     \
     }
 
 #define NV_PRINT_VMA(nv_debug_level,vma)                                                 \
     {                                                                                    \
         nv_printf(nv_debug_level,                                                        \
-            "NVRM: VM: %s:%d: 0x%lx - 0x%lx, 0x%08x bytes @ 0x%016llx, 0x%p, 0x%p\n",    \
+            "NVRM: VM: %s:%d: 0x%lx - 0x%lx, 0x%08lx bytes @ 0x%016llx, 0x%p, 0x%p\n",    \
             __FUNCTION__, __LINE__, vma->vm_start, vma->vm_end, NV_VMA_SIZE(vma),        \
             NV_VMA_OFFSET(vma), NV_VMA_PRIVATE(vma), NV_VMA_FILE(vma));                  \
     }
@@ -884,16 +873,6 @@ typedef void irqreturn_t;
 #define PCI_CAP_ID_EXP 0x10
 #endif
 
-/*
- * On Linux on PPC64LE enable basic support for Linux PCI error recovery (see
- * Documentation/PCI/pci-error-recovery.txt). Currently RM only supports error
- * notification and data collection, not actual recovery of the device.
- */
-#if defined(NVCPU_PPC64LE) && defined(CONFIG_EEH)
-#include <asm/eeh.h>
-#define NV_PCI_ERROR_RECOVERY
-#endif
-
 /*
  * If the host OS has page sizes larger than 4KB, we may have a security
  * problem. Registers are typically grouped in 4KB pages, but if there are
@@ -955,14 +934,14 @@ static inline int nv_remap_page_range(struct vm_area_struct *vma,
 }
 
 static inline int nv_io_remap_page_range(struct vm_area_struct *vma,
-    NvU64 phys_addr, NvU64 size, NvU32 extra_prot)
+    NvU64 phys_addr, NvU64 size, NvU32 extra_prot, NvU64 start)
 {
     int ret = -1;
 #if !defined(NV_XEN_SUPPORT_FULLY_VIRTUALIZED_KERNEL)
-    ret = nv_remap_page_range(vma, vma->vm_start, phys_addr, size,
+    ret = nv_remap_page_range(vma, start, phys_addr, size,
         nv_adjust_pgprot(vma->vm_page_prot, extra_prot));
 #else
-    ret = io_remap_pfn_range(vma, vma->vm_start, (phys_addr >> PAGE_SHIFT),
+    ret = io_remap_pfn_range(vma, start, (phys_addr >> PAGE_SHIFT),
         size, nv_adjust_pgprot(vma->vm_page_prot, extra_prot));
 #endif
     return ret;
@@ -1086,6 +1065,8 @@ static inline void nv_kmem_ctor_dummy(void *arg)
         kmem_cache_destroy(kmem_cache);     \
     }
 
+#define NV_KMEM_CACHE_ALLOC_ATOMIC(kmem_cache)     \
+    kmem_cache_alloc(kmem_cache, GFP_ATOMIC)
 #define NV_KMEM_CACHE_ALLOC(kmem_cache)     \
     kmem_cache_alloc(kmem_cache, GFP_KERNEL)
 #define NV_KMEM_CACHE_FREE(ptr, kmem_cache) \
@@ -1112,6 +1093,23 @@ static inline void *nv_kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)
 #endif
 }
 
+static inline int nv_kmem_cache_alloc_stack_atomic(nvidia_stack_t **stack)
+{
+    nvidia_stack_t *sp = NULL;
+#if defined(NVCPU_X86_64)
+    if (rm_is_altstack_in_use())
+    {
+        sp = NV_KMEM_CACHE_ALLOC_ATOMIC(nvidia_stack_t_cache);
+        if (sp == NULL)
+            return -ENOMEM;
+        sp->size = sizeof(sp->stack);
+        sp->top = sp->stack + sp->size;
+    }
+#endif
+    *stack = sp;
+    return 0;
+}
+
 static inline int nv_kmem_cache_alloc_stack(nvidia_stack_t **stack)
 {
     nvidia_stack_t *sp = NULL;
@@ -1161,16 +1159,22 @@ typedef struct nvidia_pte_s {
     NvU64           phys_addr;
     unsigned long   virt_addr;
     NvU64           dma_addr;
-#ifdef CONFIG_XEN
-    unsigned int    guest_pfn;
-#endif
-    unsigned int    page_count;
 } nvidia_pte_t;
 
+#if defined(CONFIG_DMA_SHARED_BUFFER)
+/* Standard dma_buf-related information. */
+struct nv_dma_buf
+{
+    struct dma_buf *dma_buf;
+    struct dma_buf_attachment *dma_attach;
+    struct sg_table *sgt;
+};
+#endif // CONFIG_DMA_SHARED_BUFFER
+
 typedef struct nv_alloc_s {
     struct nv_alloc_s *next;
     struct device     *dev;
-    atomic_t       usage_count;
+    atomic64_t       usage_count;
     struct {
         NvBool contig      : 1;
         NvBool guest       : 1;
@@ -1182,6 +1186,7 @@ typedef struct nv_alloc_s {
         NvBool physical    : 1;
         NvBool unencrypted : 1;
         NvBool coherent    : 1;
+        NvBool carveout    : 1;
     } flags;
     unsigned int   cache_type;
     unsigned int   num_pages;
@@ -1392,8 +1397,6 @@ typedef struct nv_dma_map_s {
             0 ? NV_OK : NV_ERR_OPERATING_SYSTEM)
 #endif
 
-typedef struct nv_ibmnpu_info nv_ibmnpu_info_t;
-
 typedef struct nv_work_s {
     struct work_struct task;
     void *data;
@@ -1441,7 +1444,6 @@ struct nv_dma_device {
     } addressable_range;
 
     struct device *dev;
-    NvBool nvlink;
 };
 
 /* Properties of the coherent link */
@@ -1484,15 +1486,13 @@ typedef struct
 typedef struct nv_linux_state_s {
     nv_state_t nv_state;
 
-    atomic_t usage_count;
+    atomic64_t usage_count;
+
     NvU32    suspend_count;
 
     struct device  *dev;
     struct pci_dev *pci_dev;
 
-    /* IBM-NPU info associated with this GPU */
-    nv_ibmnpu_info_t *npu;
-
     /* coherent link information */
      coherent_link_info_t coherent_link_info;
 
@@ -1808,27 +1808,13 @@ static inline int nv_is_control_device(struct inode *inode)
     return (minor((inode)->i_rdev) == NV_MINOR_DEVICE_NUMBER_CONTROL_DEVICE);
 }
 
-#if defined(NV_DOM0_KERNEL_PRESENT) || defined(NV_VGPU_KVM_BUILD)
+#if defined(NV_DOM0_KERNEL_PRESENT) || defined(NV_VGPU_KVM_BUILD) || defined(NV_DEVICE_VM_BUILD)
 #define NV_VGX_HYPER
 #if defined(NV_XEN_IOEMU_INJECT_MSI)
 #include <xen/ioemu.h>
 #endif
 #endif
 
-static inline NvU64 nv_pci_bus_address(struct pci_dev *dev, NvU8 bar_index)
-{
-    NvU64 bus_addr = 0;
-#if defined(NV_PCI_BUS_ADDRESS_PRESENT)
-    bus_addr = pci_bus_address(dev, bar_index);
-#elif defined(CONFIG_PCI)
-    struct pci_bus_region region;
-
-    pcibios_resource_to_bus(dev, &region, &dev->resource[bar_index]);
-    bus_addr = region.start;
-#endif
-    return bus_addr;
-}
-
 /*
  * Decrements the usage count of the allocation, and moves the allocation to
  * the given nvlfp's free list if the usage count drops to zero.
@@ -1839,9 +1825,9 @@ static inline NvBool nv_alloc_release(nv_linux_file_private_t *nvlfp, nv_alloc_t
 {
     NV_PRINT_AT(NV_DBG_MEMINFO, at);
 
-    if (NV_ATOMIC_DEC_AND_TEST(at->usage_count))
+    if (atomic64_dec_and_test(&at->usage_count))
     {
-        NV_ATOMIC_INC(at->usage_count);
+        atomic64_inc(&at->usage_count);
 
         at->next = nvlfp->free_list;
         nvlfp->free_list = at;
@@ -1859,59 +1845,6 @@ static inline NvBool nv_alloc_release(nv_linux_file_private_t *nvlfp, nv_alloc_t
 #define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL)
 #endif
 
-/*
- * Starting on Power9 systems, DMA addresses for NVLink are no longer
- * the same as used over PCIe.
- *
- * Power9 supports a 56-bit Real Address. This address range is compressed
- * when accessed over NVLink to allow the GPU to access all of memory using
- * its 47-bit Physical address.
- *
- * If there is an NPU device present on the system, it implies that NVLink
- * sysmem links are present and we need to apply the required address
- * conversion for NVLink within the driver.
- *
- * See Bug 1920398 for further background and details.
- *
- * Note, a deviation from the documented compression scheme is that the
- * upper address bits (i.e. bit 56-63) instead of being set to zero are
- * preserved during NVLink address compression so the orignal PCIe DMA
- * address can be reconstructed on expansion. These bits can be safely
- * ignored on NVLink since they are truncated by the GPU.
- *
- * Bug 1968345: As a performance enhancement it is the responsibility of
- * the caller on PowerPC platforms to check for presence of an NPU device
- * before the address transformation is applied.
- */
-static inline NvU64 nv_compress_nvlink_addr(NvU64 addr)
-{
-    NvU64 addr47 = addr;
-
-#if defined(NVCPU_PPC64LE)
-    addr47 = addr & ((1ULL << 43) - 1);
-    addr47 |= (addr & (0x3ULL << 45)) >> 2;
-    WARN_ON(addr47 & (1ULL << 44));
-    addr47 |= (addr & (0x3ULL << 49)) >> 4;
-    addr47 |= addr & ~((1ULL << 56) - 1);
-#endif
-
-    return addr47;
-}
-
-static inline NvU64 nv_expand_nvlink_addr(NvU64 addr47)
-{
-    NvU64 addr = addr47;
-
-#if defined(NVCPU_PPC64LE)
-    addr = addr47 & ((1ULL << 43) - 1);
-    addr |= (addr47 & (3ULL << 43)) << 2;
-    addr |= (addr47 & (3ULL << 45)) << 4;
-    addr |= addr47 & ~((1ULL << 56) - 1);
-#endif
-
-    return addr;
-}
-
 // Default flags for ISRs
 static inline NvU32 nv_default_irq_flags(nv_state_t *nv)
 {

kernel-open/common/inc/nv-lock.h

@@ -35,18 +35,6 @@
 #include <linux/sched/signal.h>     /* signal_pending for kernels >= 4.11 */
 #endif
 
-#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_PREEMPT_RT_FULL)
-typedef raw_spinlock_t            nv_spinlock_t;
-#define NV_DEFINE_SPINLOCK(lock)  DEFINE_RAW_SPINLOCK(lock)
-#define NV_SPIN_LOCK_INIT(lock)   raw_spin_lock_init(lock)
-#define NV_SPIN_LOCK_IRQ(lock)    raw_spin_lock_irq(lock)
-#define NV_SPIN_UNLOCK_IRQ(lock)  raw_spin_unlock_irq(lock)
-#define NV_SPIN_LOCK_IRQSAVE(lock,flags) raw_spin_lock_irqsave(lock,flags)
-#define NV_SPIN_UNLOCK_IRQRESTORE(lock,flags) raw_spin_unlock_irqrestore(lock,flags)
-#define NV_SPIN_LOCK(lock)        raw_spin_lock(lock)
-#define NV_SPIN_UNLOCK(lock)      raw_spin_unlock(lock)
-#define NV_SPIN_UNLOCK_WAIT(lock) raw_spin_unlock_wait(lock)
-#else
 typedef spinlock_t                nv_spinlock_t;
 #define NV_DEFINE_SPINLOCK(lock)  DEFINE_SPINLOCK(lock)
 #define NV_SPIN_LOCK_INIT(lock)   spin_lock_init(lock)
@@ -57,7 +45,6 @@ typedef spinlock_t                nv_spinlock_t;
 #define NV_SPIN_LOCK(lock)        spin_lock(lock)
 #define NV_SPIN_UNLOCK(lock)      spin_unlock(lock)
 #define NV_SPIN_UNLOCK_WAIT(lock) spin_unlock_wait(lock)
-#endif
 
 #define NV_INIT_MUTEX(mutex) sema_init(mutex, 1)
 

kernel-open/common/inc/nv-mm.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,17 +29,17 @@
 typedef int vm_fault_t;
 #endif
 
-/* pin_user_pages
+/*
+ * pin_user_pages()
+ *
  * Presence of pin_user_pages() also implies the presence of unpin-user_page().
- * Both were added in the v5.6-rc1
+ * Both were added in the v5.6.
  *
- * pin_user_pages() was added by commit eddb1c228f7951d399240
- * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6-rc1 (2020-01-30)
- *
- * Removed vmas parameter from pin_user_pages() by commit 40896a02751
- * ("mm/gup: remove vmas parameter from pin_user_pages()")
- * in linux-next, expected in v6.5-rc1 (2023-05-17)
+ * pin_user_pages() was added by commit eddb1c228f79
+ * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6.
  *
+ * Removed vmas parameter from pin_user_pages() by commit 4c630f307455
+ * ("mm/gup: remove vmas parameter from pin_user_pages()") in v6.5.
  */
 
 #include <linux/mm.h>
@@ -63,25 +63,28 @@ typedef int vm_fault_t;
     #define NV_UNPIN_USER_PAGE put_page
 #endif // NV_PIN_USER_PAGES_PRESENT
 
-/* get_user_pages
+/*
+ * get_user_pages()
  *
- * The 8-argument version of get_user_pages was deprecated by commit
- * (2016 Feb 12: cde70140fed8429acf7a14e2e2cbd3e329036653)for the non-remote case
+ * The 8-argument version of get_user_pages() was deprecated by commit
+ * cde70140fed8 ("mm/gup: Overload get_user_pages() functions") in v4.6-rc1.
  * (calling get_user_pages with current and current->mm).
  *
- * Completely moved to the 6 argument version of get_user_pages -
- * 2016 Apr 4: c12d2da56d0e07d230968ee2305aaa86b93a6832
+ * Completely moved to the 6 argument version of get_user_pages() by
+ * commit c12d2da56d0e ("mm/gup: Remove the macro overload API migration
+ * helpers from the get_user*() APIs") in v4.6-rc4.
  *
- * write and force parameters were replaced with gup_flags by -
- * 2016 Oct 12: 768ae309a96103ed02eb1e111e838c87854d8b51
+ * write and force parameters were replaced with gup_flags by
+ * commit 768ae309a961 ("mm: replace get_user_pages() write/force parameters
+ * with gup_flags") in v4.9.
  *
  * A 7-argument version of get_user_pages was introduced into linux-4.4.y by
- * commit 8e50b8b07f462ab4b91bc1491b1c91bd75e4ad40 which cherry-picked the
- * replacement of the write and force parameters with gup_flags
+ * commit 8e50b8b07f462 ("mm: replace get_user_pages() write/force parameters
+ * with gup_flags") which cherry-picked the replacement of the write and
+ * force parameters with gup_flags.
  *
- * Removed vmas parameter from get_user_pages() by commit 7bbf9c8c99
- * ("mm/gup: remove unused vmas parameter from get_user_pages()")
- * in linux-next, expected in v6.5-rc1 (2023-05-17)
+ * Removed vmas parameter from get_user_pages() by commit 54d020692b34
+ * ("mm/gup: remove unused vmas parameter from get_user_pages()") in v6.5.
  *
  */
 
@@ -112,18 +115,19 @@ typedef int vm_fault_t;
     }
 #endif // NV_GET_USER_PAGES_HAS_ARGS_FLAGS
 
-/* pin_user_pages_remote
+/*
+ * pin_user_pages_remote()
  *
- * pin_user_pages_remote() was added by commit eddb1c228f7951d399240
- * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6 (2020-01-30)
+ * pin_user_pages_remote() was added by commit eddb1c228f79
+ * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6.
  *
  * pin_user_pages_remote() removed 'tsk' parameter by commit
- * 64019a2e467a ("mm/gup: remove task_struct pointer for  all gup code")
- * in v5.9-rc1 (2020-08-11). *
+ * 64019a2e467a ("mm/gup: remove task_struct pointer for all gup code")
+ * in v5.9.
  *
  * Removed unused vmas parameter from pin_user_pages_remote() by commit
- * 83bcc2e132("mm/gup: remove unused vmas parameter from pin_user_pages_remote()")
- * in linux-next, expected in v6.5-rc1 (2023-05-14)
+ * 0b295316b3a9 ("mm/gup: remove unused vmas parameter from
+ * pin_user_pages_remote()") in v6.5.
  *
  */
 
@@ -143,7 +147,7 @@ typedef int vm_fault_t;
 
 /*
  * get_user_pages_remote() was added by commit 1e9877902dc7
- * ("mm/gup: Introduce get_user_pages_remote()") in v4.6 (2016-02-12).
+ * ("mm/gup: Introduce get_user_pages_remote()") in v4.6.
  *
  * Note that get_user_pages_remote() requires the caller to hold a reference on
  * the task_struct (if non-NULL and if this API has tsk argument) and the mm_struct.
@@ -153,19 +157,17 @@ typedef int vm_fault_t;
  *
  * get_user_pages_remote() write/force parameters were replaced
  * with gup_flags by commit 9beae1ea8930 ("mm: replace get_user_pages_remote()
- * write/force parameters with gup_flags") in v4.9 (2016-10-13).
+ * write/force parameters with gup_flags") in v4.9.
  *
  * get_user_pages_remote() added 'locked' parameter by commit 5b56d49fc31d
- * ("mm: add locked parameter to get_user_pages_remote()") in
- * v4.10 (2016-12-14).
+ * ("mm: add locked parameter to get_user_pages_remote()") in v4.10.
  *
  * get_user_pages_remote() removed 'tsk' parameter by
  * commit 64019a2e467a ("mm/gup: remove task_struct pointer for
- * all gup code") in v5.9-rc1 (2020-08-11).
+ * all gup code") in v5.9.
  *
- * Removed vmas parameter from get_user_pages_remote() by commit a4bde14d549 
- * ("mm/gup: remove vmas parameter from get_user_pages_remote()")
- * in linux-next, expected in v6.5-rc1 (2023-05-14)
+ * Removed vmas parameter from get_user_pages_remote() by commit ca5e863233e8
+ * ("mm/gup: remove vmas parameter from get_user_pages_remote()") in v6.5.
  *
  */
 
@@ -295,9 +297,21 @@ static inline struct rw_semaphore *nv_mmap_get_lock(struct mm_struct *mm)
 #endif
 }
 
+#define NV_CAN_CALL_VMA_START_WRITE 1
+
+#if !NV_CAN_CALL_VMA_START_WRITE
+/*
+ * Commit 45ad9f5290dc updated vma_start_write() to call __vma_start_write().
+ */
+void nv_vma_start_write(struct vm_area_struct *);
+#endif
+
 static inline void nv_vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags)
 {
-#if defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
+#if !NV_CAN_CALL_VMA_START_WRITE
+    nv_vma_start_write(vma);
+    ACCESS_PRIVATE(vma, __vm_flags) |= flags;
+#elif defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
     vm_flags_set(vma, flags);
 #else
     vma->vm_flags |= flags;
@@ -306,7 +320,10 @@ static inline void nv_vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags)
 
 static inline void nv_vm_flags_clear(struct vm_area_struct *vma, vm_flags_t flags)
 {
-#if defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
+#if !NV_CAN_CALL_VMA_START_WRITE
+    nv_vma_start_write(vma);
+    ACCESS_PRIVATE(vma, __vm_flags) &= ~flags;
+#elif defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
     vm_flags_clear(vma, flags);
 #else
     vma->vm_flags &= ~flags;

kernel-open/common/inc/nv-pci.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -36,5 +36,6 @@ int nv_pci_count_devices(void);
 NvU8 nv_find_pci_capability(struct pci_dev *, NvU8);
 int nvidia_dev_get_pci_info(const NvU8 *, struct pci_dev **, NvU64 *, NvU64 *);
 nv_linux_state_t * find_pci(NvU32, NvU8, NvU8, NvU8);
+NvBool nv_pci_is_valid_topology_for_direct_pci(nv_state_t *, struct device *);
 
 #endif

kernel-open/common/inc/nv-proto.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -59,6 +59,8 @@ NV_STATUS   nv_uvm_resume               (void);
 void        nv_uvm_notify_start_device  (const NvU8 *uuid);
 void        nv_uvm_notify_stop_device   (const NvU8 *uuid);
 NV_STATUS   nv_uvm_event_interrupt      (const NvU8 *uuid);
+NV_STATUS   nv_uvm_drain_P2P            (const NvU8 *uuid);
+NV_STATUS   nv_uvm_resume_P2P           (const NvU8 *uuid);
 
 /* Move these to nv.h once implemented by other UNIX platforms */
 NvBool      nvidia_get_gpuid_list       (NvU32 *gpu_ids, NvU32 *gpu_count);

kernel-open/common/inc/nv-timer.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2017-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -63,4 +63,13 @@ static inline void nv_timer_setup(struct nv_timer *nv_timer,
 #endif
 }
 
+static inline void nv_timer_delete_sync(struct timer_list *timer)
+{
+#if !defined(NV_BSD) && NV_IS_EXPORT_SYMBOL_PRESENT_timer_delete_sync
+    timer_delete_sync(timer);
+#else
+    del_timer_sync(timer);
+#endif
+}
+
 #endif // __NV_TIMER_H__

kernel-open/common/inc/nv.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -44,6 +44,7 @@
 #include <nv-ioctl.h>
 #include <nv-ioctl-numa.h>
 #include <nvmisc.h>
+#include <os/nv_memory_area.h>
 
 extern nv_cap_t *nvidia_caps_root;
 
@@ -110,15 +111,15 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTPN,
     TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTA,
     TEGRASOC_WHICH_CLK_SPPLL0_VCO,
-    TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN,
     TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTA,
     TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTB,
+    TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN,
+    TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN,
+    TEGRASOC_WHICH_CLK_SPPLL0_DIV27,
+    TEGRASOC_WHICH_CLK_SPPLL1_DIV27,
     TEGRASOC_WHICH_CLK_SPPLL0_DIV10,
     TEGRASOC_WHICH_CLK_SPPLL0_DIV25,
-    TEGRASOC_WHICH_CLK_SPPLL0_DIV27,
     TEGRASOC_WHICH_CLK_SPPLL1_VCO,
-    TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN,
-    TEGRASOC_WHICH_CLK_SPPLL1_DIV27,
     TEGRASOC_WHICH_CLK_VPLL0_REF,
     TEGRASOC_WHICH_CLK_VPLL0,
     TEGRASOC_WHICH_CLK_VPLL1,
@@ -132,7 +133,7 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_DSI_PIXEL,
     TEGRASOC_WHICH_CLK_PRE_SOR0,
     TEGRASOC_WHICH_CLK_PRE_SOR1,
-    TEGRASOC_WHICH_CLK_DP_LINK_REF,
+    TEGRASOC_WHICH_CLK_DP_LINKA_REF,
     TEGRASOC_WHICH_CLK_SOR_LINKA_INPUT,
     TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO,
     TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO_M,
@@ -143,7 +144,7 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_PLLHUB,
     TEGRASOC_WHICH_CLK_SOR0,
     TEGRASOC_WHICH_CLK_SOR1,
-    TEGRASOC_WHICH_CLK_SOR_PAD_INPUT,
+    TEGRASOC_WHICH_CLK_SOR_PADA_INPUT,
     TEGRASOC_WHICH_CLK_PRE_SF0,
     TEGRASOC_WHICH_CLK_SF0,
     TEGRASOC_WHICH_CLK_SF1,
@@ -279,8 +280,7 @@ typedef struct nv_usermap_access_params_s
     NvU64    offset;
     NvU64   *page_array;
     NvU64    num_pages;
-    NvU64    mmap_start;
-    NvU64    mmap_size;
+    MemoryArea memArea;
     NvU64    access_start;
     NvU64    access_size;
     NvU64    remap_prot_extra;
@@ -296,8 +296,7 @@ typedef struct nv_alloc_mapping_context_s {
     NvU64  page_index;
     NvU64 *page_array;
     NvU64  num_pages;
-    NvU64  mmap_start;
-    NvU64  mmap_size;
+    MemoryArea memArea;
     NvU64  access_start;
     NvU64  access_size;
     NvU64  remap_prot_extra;
@@ -330,9 +329,11 @@ typedef struct nv_soc_irq_info_s {
     NvS32 ref_count;
 } nv_soc_irq_info_t;
 
-#define NV_MAX_SOC_IRQS              6
+#define NV_MAX_SOC_IRQS              10
 #define NV_MAX_DPAUX_NUM_DEVICES     4
-#define NV_MAX_SOC_DPAUX_NUM_DEVICES 2 // From SOC_DEV_MAPPING
+
+#define NV_MAX_SOC_DPAUX_NUM_DEVICES 2
+
 
 #define NV_IGPU_LEGACY_STALL_IRQ     70
 #define NV_IGPU_MAX_STALL_IRQS       3
@@ -367,6 +368,8 @@ typedef struct nv_state_t
     {
         NvBool         valid;
         NvU8           uuid[GPU_UUID_LEN];
+        NvBool         pci_uuid_read_attempted;
+        NV_STATUS      pci_uuid_status;
     } nv_uuid_cache;
     void *handle;
 
@@ -478,6 +481,8 @@ typedef struct nv_state_t
     /* Bool to check if the GPU has a coherent sysmem link */
     NvBool coherent;
 
+    /* OS detected GPU has ATS capability */
+    NvBool ats_support;
     /*
      * NUMA node ID of the CPU to which the GPU is attached.
      * Holds NUMA_NO_NODE on platforms that don't support NUMA configuration.
@@ -493,13 +498,10 @@ typedef struct nv_state_t
         NvU32 dispIsoStreamId;
         NvU32 dispNisoStreamId;
     } iommus;
-} nv_state_t;
 
-// These define need to be in sync with defines in system.h
-#define OS_TYPE_LINUX   0x1
-#define OS_TYPE_FREEBSD 0x2
-#define OS_TYPE_SUNOS   0x3
-#define OS_TYPE_VMWARE  0x4
+    /* Console is managed by drm drivers or NVKMS */
+    NvBool client_managed_console;
+} nv_state_t;
 
 #define NVFP_TYPE_NONE       0x0
 #define NVFP_TYPE_REFCOUNTED 0x1
@@ -539,16 +541,18 @@ typedef struct UvmGpuAddressSpaceInfo_tag           *nvgpuAddressSpaceInfo_t;
 typedef struct UvmGpuAllocInfo_tag                  *nvgpuAllocInfo_t;
 typedef struct UvmGpuP2PCapsParams_tag              *nvgpuP2PCapsParams_t;
 typedef struct UvmGpuFbInfo_tag                     *nvgpuFbInfo_t;
+typedef struct UvmGpuNvlinkInfo_tag                 *nvgpuNvlinkInfo_t;
 typedef struct UvmGpuEccInfo_tag                    *nvgpuEccInfo_t;
 typedef struct UvmGpuFaultInfo_tag                  *nvgpuFaultInfo_t;
 typedef struct UvmGpuAccessCntrInfo_tag             *nvgpuAccessCntrInfo_t;
-typedef struct UvmGpuAccessCntrConfig_tag           *nvgpuAccessCntrConfig_t;
-typedef struct UvmGpuInfo_tag                       nvgpuInfo_t;
-typedef struct UvmGpuClientInfo_tag                 nvgpuClientInfo_t;
+typedef struct UvmGpuAccessCntrConfig_tag            nvgpuAccessCntrConfig_t;
+typedef struct UvmGpuInfo_tag                        nvgpuInfo_t;
+typedef struct UvmGpuClientInfo_tag                  nvgpuClientInfo_t;
 typedef struct UvmPmaAllocationOptions_tag          *nvgpuPmaAllocationOptions_t;
 typedef struct UvmPmaStatistics_tag                 *nvgpuPmaStatistics_t;
 typedef struct UvmGpuMemoryInfo_tag                 *nvgpuMemoryInfo_t;
 typedef struct UvmGpuExternalMappingInfo_tag        *nvgpuExternalMappingInfo_t;
+typedef struct UvmGpuExternalPhysAddrInfo_tag       *nvgpuExternalPhysAddrInfo_t;
 typedef struct UvmGpuChannelResourceInfo_tag        *nvgpuChannelResourceInfo_t;
 typedef struct UvmGpuChannelInstanceInfo_tag        *nvgpuChannelInstanceInfo_t;
 typedef struct UvmGpuChannelResourceBindParams_tag  *nvgpuChannelResourceBindParams_t;
@@ -573,7 +577,8 @@ typedef NV_STATUS (*nvPmaEvictRangeCallback)(void *, NvU64, NvU64, nvgpuGpuMemor
 #define NV_FLAG_PASSTHRU               0x0080
 #define NV_FLAG_SUSPENDED              0x0100
 #define NV_FLAG_SOC_IGPU               0x0200
-// Unused                              0x0400
+/* To be set when an FLR needs to be triggered after device shut down. */
+#define NV_FLAG_TRIGGER_FLR            0x0400
 #define NV_FLAG_PERSISTENT_SW_STATE    0x0800
 #define NV_FLAG_IN_RECOVERY            0x1000
 // Unused                              0x2000
@@ -609,6 +614,16 @@ typedef enum
     NV_POWER_STATE_RUNNING
 } nv_power_state_t;
 
+typedef struct
+{
+    const char *vidmem_power_status;
+    const char *dynamic_power_status;
+    const char *gc6_support;
+    const char *gcoff_support;
+    const char *s0ix_status;
+    const char *db_support;
+} nv_power_info_t;
+
 #define NV_PRIMARY_VGA(nv)      ((nv)->primary_vga)
 
 #define NV_IS_CTL_DEVICE(nv)    ((nv)->flags & NV_FLAG_CONTROL)
@@ -752,6 +767,7 @@ static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
 #define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1))
 #endif
 
+
 /*
  * driver internal interfaces
  */
@@ -778,7 +794,7 @@ nv_state_t*  NV_API_CALL  nv_get_ctl_state       (void);
 
 void   NV_API_CALL  nv_set_dma_address_size      (nv_state_t *, NvU32 );
 
-NV_STATUS  NV_API_CALL  nv_alias_pages           (nv_state_t *, NvU32, NvU32, NvU32, NvU64, NvU64 *, void **);
+NV_STATUS  NV_API_CALL  nv_alias_pages           (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, NvBool, void **);
 NV_STATUS  NV_API_CALL  nv_alloc_pages           (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **);
 NV_STATUS  NV_API_CALL  nv_free_pages            (nv_state_t *, NvU32, NvBool, NvU32, void *);
 
@@ -807,7 +823,6 @@ NV_STATUS  NV_API_CALL  nv_dma_map_mmio          (nv_dma_device_t *, NvU64, NvU6
 void       NV_API_CALL  nv_dma_unmap_mmio        (nv_dma_device_t *, NvU64, NvU64);
 
 void       NV_API_CALL  nv_dma_cache_invalidate  (nv_dma_device_t *, void *);
-void       NV_API_CALL  nv_dma_enable_nvlink     (nv_dma_device_t *);
 
 NvS32  NV_API_CALL  nv_start_rc_timer            (nv_state_t *);
 NvS32  NV_API_CALL  nv_stop_rc_timer             (nv_state_t *);
@@ -822,6 +837,7 @@ void   NV_API_CALL  nv_acpi_methods_init         (NvU32 *);
 void   NV_API_CALL  nv_acpi_methods_uninit       (void);
 
 NV_STATUS  NV_API_CALL  nv_acpi_method           (NvU32, NvU32, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
+NV_STATUS  NV_API_CALL  nv_acpi_d3cold_dsm_for_upstream_port (nv_state_t *, NvU8 *, NvU32, NvU32, NvU32 *);
 NV_STATUS  NV_API_CALL  nv_acpi_dsm_method       (nv_state_t *, NvU8 *, NvU32, NvBool, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
 NV_STATUS  NV_API_CALL  nv_acpi_ddc_method       (nv_state_t *, void *, NvU32 *, NvBool);
 NV_STATUS  NV_API_CALL  nv_acpi_dod_method       (nv_state_t *, NvU32 *, NvU32 *);
@@ -833,9 +849,7 @@ NV_STATUS  NV_API_CALL  nv_acpi_mux_method       (nv_state_t *, NvU32 *, NvU32,
 
 NV_STATUS  NV_API_CALL  nv_log_error             (nv_state_t *, NvU32, const char *, va_list);
 
-NvU64      NV_API_CALL  nv_get_dma_start_address (nv_state_t *);
 NV_STATUS  NV_API_CALL  nv_set_primary_vga_status(nv_state_t *);
-NV_STATUS  NV_API_CALL  nv_pci_trigger_recovery  (nv_state_t *);
 NvBool     NV_API_CALL  nv_requires_dma_remap    (nv_state_t *);
 
 NvBool     NV_API_CALL  nv_is_rm_firmware_active(nv_state_t *);
@@ -848,19 +862,8 @@ void               NV_API_CALL nv_put_file_private(void *);
 NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
 NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
 
-NV_STATUS NV_API_CALL nv_get_ibmnpu_genreg_info(nv_state_t *, NvU64 *, NvU64 *, void**);
-NV_STATUS NV_API_CALL nv_get_ibmnpu_relaxed_ordering_mode(nv_state_t *nv, NvBool *mode);
-
-void      NV_API_CALL nv_wait_for_ibmnpu_rsync(nv_state_t *nv);
-
-void      NV_API_CALL nv_ibmnpu_cache_flush_range(nv_state_t *nv, NvU64, NvU64);
-
 void      NV_API_CALL nv_p2p_free_platform_data(void *data);
 
-#if defined(NVCPU_PPC64LE)
-NV_STATUS NV_API_CALL nv_get_nvlink_line_rate    (nv_state_t *, NvU32 *);
-#endif
-
 NV_STATUS NV_API_CALL nv_revoke_gpu_mappings     (nv_state_t *);
 void      NV_API_CALL nv_acquire_mmap_lock       (nv_state_t *);
 void      NV_API_CALL nv_release_mmap_lock       (nv_state_t *);
@@ -883,8 +886,6 @@ void      NV_API_CALL nv_cap_drv_exit(void);
 NvBool    NV_API_CALL nv_is_gpu_accessible(nv_state_t *);
 NvBool    NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *);
 
-NvU32     NV_API_CALL nv_get_os_type(void);
-
 void      NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end);
 void      NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *);
 
@@ -900,6 +901,9 @@ void      NV_API_CALL nv_dma_release_dma_buf     (nv_dma_buf_t *);
 
 void      NV_API_CALL nv_schedule_uvm_isr        (nv_state_t *);
 
+NV_STATUS NV_API_CALL nv_schedule_uvm_drain_p2p  (NvU8 *);
+void      NV_API_CALL nv_schedule_uvm_resume_p2p (NvU8 *);
+
 NvBool    NV_API_CALL nv_platform_supports_s0ix  (void);
 NvBool    NV_API_CALL nv_s2idle_pm_configured    (void);
 
@@ -990,18 +994,24 @@ NV_STATUS  NV_API_CALL  rm_p2p_init_mapping       (nvidia_stack_t *, NvU64, NvU6
 NV_STATUS  NV_API_CALL  rm_p2p_destroy_mapping    (nvidia_stack_t *, NvU64);
 NV_STATUS  NV_API_CALL  rm_p2p_get_pages          (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *);
 NV_STATUS  NV_API_CALL  rm_p2p_get_gpu_info       (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **);
-NV_STATUS  NV_API_CALL  rm_p2p_get_pages_persistent (nvidia_stack_t *,  NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *);
+NV_STATUS  NV_API_CALL  rm_p2p_get_pages_persistent (nvidia_stack_t *,  NvU64, NvU64, void **, NvU64 *, NvU32 *, NvBool, void *, void *, void **);
 NV_STATUS  NV_API_CALL  rm_p2p_register_callback  (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
 NV_STATUS  NV_API_CALL  rm_p2p_put_pages          (nvidia_stack_t *, NvU64, NvU32, NvU64, void *);
-NV_STATUS  NV_API_CALL  rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *);
+NV_STATUS  NV_API_CALL  rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *, void *);
 NV_STATUS  NV_API_CALL  rm_p2p_dma_map_pages      (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **);
 NV_STATUS  NV_API_CALL  rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *, void **);
 void       NV_API_CALL  rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle);
-NV_STATUS  NV_API_CALL  rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, NvU64, void *, nv_phys_addr_range_t **, NvU32 *);
-void       NV_API_CALL  rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvU64, nv_phys_addr_range_t **, NvU32);
-NV_STATUS  NV_API_CALL  rm_dma_buf_get_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle *, NvHandle *, NvHandle *, void **, NvBool *);
+NV_STATUS  NV_API_CALL  rm_dma_buf_map_mem_handle (nvidia_stack_t *, nv_state_t *,
+                                                   NvHandle, NvHandle, MemoryRange,
+                                                   NvU8, void *, NvBool, MemoryArea *);
+void       NV_API_CALL  rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t *,
+                                                    NvHandle, NvHandle, NvU8, void *,
+                                                    NvBool, MemoryArea);
+NV_STATUS  NV_API_CALL  rm_dma_buf_get_client_and_device(nvidia_stack_t *,
+                                                nv_state_t *, NvHandle, NvHandle,
+                                                NvU8, NvHandle *, NvHandle *,
+                                                NvHandle *, void **, NvBool *);
 void       NV_API_CALL  rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *);
-NV_STATUS  NV_API_CALL  rm_log_gpu_crash          (nv_stack_t *, nv_state_t *);
 
 void       NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd);
 NvBool     NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id);
@@ -1018,7 +1028,6 @@ NvBool     NV_API_CALL rm_is_device_sequestered(nvidia_stack_t *, nv_state_t *);
 void       NV_API_CALL rm_check_for_gpu_surprise_removal(nvidia_stack_t *, nv_state_t *);
 NV_STATUS  NV_API_CALL rm_set_external_kernel_client_count(nvidia_stack_t *, nv_state_t *, NvBool);
 NV_STATUS  NV_API_CALL rm_schedule_gpu_wakeup(nvidia_stack_t *, nv_state_t *);
-NvBool     NV_API_CALL rm_is_iommu_needed_for_sriov(nvidia_stack_t *, nv_state_t *);
 NvBool     NV_API_CALL rm_disable_iomap_wc(void);
 
 void       NV_API_CALL rm_init_dynamic_power_management(nvidia_stack_t *, nv_state_t *, NvBool);
@@ -1027,9 +1036,7 @@ void       NV_API_CALL rm_enable_dynamic_power_management(nvidia_stack_t *, nv_s
 NV_STATUS  NV_API_CALL rm_ref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
 void       NV_API_CALL rm_unref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
 NV_STATUS  NV_API_CALL rm_transition_dynamic_power(nvidia_stack_t *, nv_state_t *, NvBool, NvBool *);
-const char* NV_API_CALL rm_get_vidmem_power_status(nvidia_stack_t *, nv_state_t *);
-const char* NV_API_CALL rm_get_dynamic_power_management_status(nvidia_stack_t *, nv_state_t *);
-const char* NV_API_CALL rm_get_gpu_gcx_support(nvidia_stack_t *, nv_state_t *, NvBool);
+void       NV_API_CALL rm_get_power_info(nvidia_stack_t *, nv_state_t *, nv_power_info_t *);
 
 void       NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32);
 void       NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);
@@ -1037,12 +1044,14 @@ void       NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);
 NvBool     NV_API_CALL rm_is_altstack_in_use(void);
 
 /* vGPU VFIO specific functions */
-NV_STATUS  NV_API_CALL  nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *, NvU32);
+NV_STATUS  NV_API_CALL  nv_vgpu_create_request(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU16 *,
+                                               NvU32 *, NvU32 *, NvU32);
 NV_STATUS  NV_API_CALL  nv_vgpu_delete(nvidia_stack_t *, const NvU8 *, NvU16);
 NV_STATUS  NV_API_CALL  nv_vgpu_get_type_ids(nvidia_stack_t *, nv_state_t *, NvU32 *, NvU32 *, NvBool, NvU8, NvBool);
 NV_STATUS  NV_API_CALL  nv_vgpu_get_type_info(nvidia_stack_t *, nv_state_t *, NvU32, char *, int, NvU8);
 NV_STATUS  NV_API_CALL  nv_vgpu_get_bar_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *,
                                              NvU64 *, NvU64 *, NvU32 *, NvBool *, NvU8 *);
+NV_STATUS  NV_API_CALL  nv_vgpu_update_sysfs_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU32, NvU32);
 NV_STATUS  NV_API_CALL  nv_vgpu_get_hbm_info(nvidia_stack_t *, nv_state_t *, const NvU8 *, NvU64 *, NvU64 *);
 NV_STATUS  NV_API_CALL  nv_vgpu_process_vf_info(nvidia_stack_t *, nv_state_t *, NvU8, NvU32, NvU8, NvU8, NvU8, NvBool, void *);
 NV_STATUS  NV_API_CALL  nv_gpu_bind_event(nvidia_stack_t *, NvU32, NvBool *);
@@ -1077,6 +1086,9 @@ NV_STATUS   NV_API_CALL rm_run_nano_timer_callback(nvidia_stack_t *, nv_state_t
 void        NV_API_CALL nv_cancel_nano_timer(nv_state_t *, nv_nano_timer_t *);
 void        NV_API_CALL nv_destroy_nano_timer(nv_state_t *nv, nv_nano_timer_t *);
 
+// Host1x specific functions.
+NV_STATUS NV_API_CALL nv_get_syncpoint_aperture(NvU32, NvU64 *, NvU64 *, NvU32 *);
+
 #if defined(NVCPU_X86_64)
 
 static inline NvU64 nv_rdtsc(void)

kernel-open/common/inc/nv_uvm_interface.h

@@ -597,8 +597,9 @@ void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel);
       NV_ERR_NOT_READY
       NV_ERR_INVALID_LOCK_STATE
       NV_ERR_INVALID_STATE
-      NV_ERR_NVSWITCH_FABRIC_NOT_READY
-      NV_ERR_NVSWITCH_FABRIC_FAILURE
+      NV_ERR_NVLINK_FABRIC_NOT_READY
+      NV_ERR_NVLINK_FABRIC_FAILURE
+      NV_ERR_GPU_MEMORY_ONLINING_FAILURE
 */
 NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuDeviceHandle device,
                                   UvmGpuCaps *caps);
@@ -1055,7 +1056,7 @@ NV_STATUS nvUvmInterfaceDestroyAccessCntrInfo(uvmGpuDeviceHandle device,
 */
 NV_STATUS nvUvmInterfaceEnableAccessCntr(uvmGpuDeviceHandle device,
                                          UvmGpuAccessCntrInfo *pAccessCntrInfo,
-                                         UvmGpuAccessCntrConfig *pAccessCntrConfig);
+                                         const UvmGpuAccessCntrConfig *pAccessCntrConfig);
 
 /*******************************************************************************
     nvUvmInterfaceDisableAccessCntr
@@ -1092,6 +1093,22 @@ NV_STATUS nvUvmInterfaceRegisterUvmCallbacks(struct UvmOpsUvmEvents *importedUvm
 //
 void nvUvmInterfaceDeRegisterUvmOps(void);
 
+/*******************************************************************************
+    nvUvmInterfaceGetNvlinkInfo
+
+    Gets NVLINK information from RM.
+
+    Arguments:
+        device[IN]        - GPU device handle
+        nvlinkInfo [OUT]     - Pointer to NvlinkInfo structure
+
+    Error codes:
+      NV_ERROR
+      NV_ERR_INVALID_ARGUMENT
+*/
+NV_STATUS nvUvmInterfaceGetNvlinkInfo(uvmGpuDeviceHandle device,
+                                      UvmGpuNvlinkInfo *nvlinkInfo);
+
 /*******************************************************************************
     nvUvmInterfaceP2pObjectCreate
 
@@ -1168,6 +1185,48 @@ NV_STATUS nvUvmInterfaceGetExternalAllocPtes(uvmGpuAddressSpaceHandle vaSpace,
                                              NvU64 size,
                                              UvmGpuExternalMappingInfo *gpuExternalMappingInfo);
 
+/*******************************************************************************
+    nvUvmInterfaceGetExternalAllocPhysAddrs
+
+    The interface builds the RM physical addrs using the provided input parameters.
+
+    Arguments:
+        vaSpace[IN]                     -  vaSpace handle.
+        hMemory[IN]                     -  Memory handle.
+        offset [IN]                     -  Offset from the beginning of the allocation
+                                           where PTE mappings should begin.
+                                           Should be aligned with mappingPagesize
+                                           in gpuExternalMappingInfo associated
+                                           with the allocation.
+        size [IN]                       -  Length of the allocation for which PhysAddrs
+                                           should be built.
+                                           Should be aligned with mappingPagesize
+                                           in gpuExternalMappingInfo associated
+                                           with the allocation.
+                                           size = 0 will be interpreted as the total size
+                                           of the allocation.
+        gpuExternalMappingInfo[IN/OUT]  -  See nv_uvm_types.h for more information.
+
+   Error codes:
+        NV_ERR_INVALID_ARGUMENT         - Invalid parameter/s is passed.
+        NV_ERR_INVALID_OBJECT_HANDLE    - Invalid memory handle is passed.
+        NV_ERR_NOT_SUPPORTED            - Functionality is not supported (see comments in nv_gpu_ops.c)
+        NV_ERR_INVALID_BASE             - offset is beyond the allocation size
+        NV_ERR_INVALID_LIMIT            - (offset + size) is beyond the allocation size.
+        NV_ERR_BUFFER_TOO_SMALL         - gpuExternalMappingInfo.physAddrBufferSize is insufficient to
+                                          store single physAddr.
+        NV_ERR_NOT_READY                - Returned when querying the physAddrs requires a deferred setup
+                                          which has not yet completed. It is expected that the caller
+                                          will reattempt the call until a different code is returned.
+                                          As an example, multi-node systems which require querying
+                                          physAddrs from the Fabric Manager may return this code.
+*/
+NV_STATUS nvUvmInterfaceGetExternalAllocPhysAddrs(uvmGpuAddressSpaceHandle vaSpace,
+                                                  NvHandle hMemory,
+                                                  NvU64 offset,
+                                                  NvU64 size,
+                                                  UvmGpuExternalPhysAddrInfo *gpuExternalPhysAddrsInfo);
+
 /*******************************************************************************
     nvUvmInterfaceRetainChannel
 
@@ -1469,6 +1528,16 @@ NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channe
                                                 char *methodStream,
                                                 NvU32 methodStreamSize);
 
+/*******************************************************************************
+    nvUvmInterfaceReportFatalError
+
+    Reports a global fatal error so RM can inform the clients that a node reboot
+    is necessary to recover from this error. This function can be called from
+    any lock environment, bottom half or non-interrupt context.
+
+*/
+void nvUvmInterfaceReportFatalError(NV_STATUS error);
+
 /*******************************************************************************
     Cryptography Services Library (CSL) Interface
 */

kernel-open/common/inc/nv_uvm_types.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -39,12 +39,13 @@
 // are multiple BIG page sizes in RM. These defines are used as flags to "0"
 // should be OK when user is not sure which pagesize allocation it wants
 //
-#define UVM_PAGE_SIZE_DEFAULT    0x0
-#define UVM_PAGE_SIZE_4K         0x1000
-#define UVM_PAGE_SIZE_64K        0x10000
-#define UVM_PAGE_SIZE_128K       0x20000
-#define UVM_PAGE_SIZE_2M         0x200000
-#define UVM_PAGE_SIZE_512M       0x20000000
+#define UVM_PAGE_SIZE_DEFAULT    0x0ULL
+#define UVM_PAGE_SIZE_4K         0x1000ULL
+#define UVM_PAGE_SIZE_64K        0x10000ULL
+#define UVM_PAGE_SIZE_128K       0x20000ULL
+#define UVM_PAGE_SIZE_2M         0x200000ULL
+#define UVM_PAGE_SIZE_512M       0x20000000ULL
+#define UVM_PAGE_SIZE_256G       0x4000000000ULL
 
 //
 // When modifying flags, make sure they are compatible with the mirrored
@@ -395,6 +396,7 @@ typedef enum
     UVM_LINK_TYPE_NVLINK_2,
     UVM_LINK_TYPE_NVLINK_3,
     UVM_LINK_TYPE_NVLINK_4,
+    UVM_LINK_TYPE_NVLINK_5,
     UVM_LINK_TYPE_C2C,
 } UVM_LINK_TYPE;
 
@@ -541,6 +543,36 @@ typedef struct UvmGpuExternalMappingInfo_tag
     NvU32 pteSize;
 } UvmGpuExternalMappingInfo;
 
+typedef struct UvmGpuExternalPhysAddrInfo_tag
+{
+    // In: Virtual permissions. Returns
+    // NV_ERR_INVALID_ACCESS_TYPE if input is
+    // inaccurate
+    UvmRmGpuMappingType mappingType;
+
+    // In: Size of the buffer to store PhysAddrs (in bytes).
+    NvU64 physAddrBufferSize;
+
+    // In: Page size for mapping
+    //     If this field is passed as 0, the page size
+    //     of the allocation is used for mapping.
+    //     nvUvmInterfaceGetExternalAllocPtes must pass
+    //     this field as zero.
+    NvU64 mappingPageSize;
+
+    // In: Pointer to a buffer to store PhysAddrs.
+    // Out: The interface will fill the buffer with PhysAddrs
+    NvU64 *physAddrBuffer;
+
+    // Out: Number of PhysAddrs filled in to the buffer.
+    NvU64 numWrittenPhysAddrs;
+
+    // Out: Number of PhysAddrs remaining to be filled
+    //      if the buffer is not sufficient to accommodate
+    //      requested PhysAddrs.
+    NvU64 numRemainingPhysAddrs;
+} UvmGpuExternalPhysAddrInfo;
+
 typedef struct UvmGpuP2PCapsParams_tag
 {
     // Out: peerId[i] contains gpu[i]'s peer id of gpu[1 - i]. Only defined if
@@ -566,11 +598,6 @@ typedef struct UvmGpuP2PCapsParams_tag
     // second, not taking into account the protocols overhead. The reported
     // bandwidth for indirect peers is zero.
     NvU32 totalLinkLineRateMBps;
-
-    // Out: True if the peers have a indirect link to communicate. On P9
-    // systems, this is true if peers are connected to different NPUs that
-    // forward the requests between them.
-    NvU32 indirectAccess      : 1;
 } UvmGpuP2PCapsParams;
 
 // Platform-wide information
@@ -593,17 +620,12 @@ typedef struct UvmGpuClientInfo_tag
     NvHandle hSmcPartRef;
 } UvmGpuClientInfo;
 
-typedef enum
-{
-    UVM_GPU_CONF_COMPUTE_MODE_NONE = 0,
-    UVM_GPU_CONF_COMPUTE_MODE_HCC = 2
-} UvmGpuConfComputeMode;
-
 typedef struct UvmGpuConfComputeCaps_tag
 {
-    // Out: GPU's confidential compute mode
-    UvmGpuConfComputeMode mode;
-    // Is key rotation enabled for UVM keys
+    // Out: true if Confidential Computing is enabled on the GPU
+    NvBool bConfComputingEnabled;
+
+    // Out: true if key rotation is enabled (for UVM keys) on the GPU
     NvBool bKeyRotationEnabled;
 } UvmGpuConfComputeCaps;
 
@@ -661,6 +683,9 @@ typedef struct UvmGpuInfo_tag
     // Maximum number of TPCs per GPC
     NvU32 maxTpcPerGpcCount;
 
+    // Number of access counter buffers.
+    NvU32 accessCntrBufferCount;
+
     // NV_TRUE if SMC is enabled on this GPU.
     NvBool smcEnabled;
 
@@ -707,6 +732,15 @@ typedef struct UvmGpuInfo_tag
 
     // EGM base address to offset in the GMMU PTE entry for EGM mappings
     NvU64    egmBaseAddr;
+
+    // If connectedToSwitch is NV_TRUE,
+    // nvswitchEgmMemoryWindowStart tells the base address for the GPU's EGM memory in the
+    // NVSwitch address space. It is used when creating PTEs of GPU memory mappings
+    // to NVSwitch peers.
+    NvU64 nvswitchEgmMemoryWindowStart;
+
+    // GPU supports ATS capability
+    NvBool atsSupport;
 } UvmGpuInfo;
 
 typedef struct UvmGpuFbInfo_tag
@@ -715,10 +749,15 @@ typedef struct UvmGpuFbInfo_tag
     // RM regions that are not registered with PMA either.
     NvU64 maxAllocatableAddress;
 
-    NvU32 heapSize;          // RAM in KB available for user allocations
-    NvU32 reservedHeapSize;  // RAM in KB reserved for internal RM allocation
-    NvBool bZeroFb;          // Zero FB mode enabled.
-    NvU64 maxVidmemPageSize; // Largest GPU page size to access vidmem.
+    NvU32  heapSize;           // RAM in KB available for user allocations
+    NvU32  reservedHeapSize;   // RAM in KB reserved for internal RM allocation
+    NvBool bZeroFb;            // Zero FB mode enabled.
+    NvU64  maxVidmemPageSize;  // Largest GPU page size to access vidmem.
+    NvBool bStaticBar1Enabled; // Static BAR1 mode is enabled
+    NvU64  staticBar1StartOffset;  // The start offset of the the static mapping
+    NvU64  staticBar1Size;     // The size of the static mapping
+    NvU32  heapStart;          // The start offset of heap in KB, helpful for MIG
+                               // systems
 } UvmGpuFbInfo;
 
 typedef struct UvmGpuEccInfo_tag
@@ -730,6 +769,15 @@ typedef struct UvmGpuEccInfo_tag
     NvBool   bEccEnabled;
 } UvmGpuEccInfo;
 
+typedef struct UvmGpuNvlinkInfo_tag
+{
+    unsigned nvlinkMask;
+    unsigned nvlinkOffset;
+    void    *nvlinkReadLocation;
+    NvBool  *nvlinkErrorNotifier;
+    NvBool   bNvlinkRecoveryEnabled;
+} UvmGpuNvlinkInfo;
+
 typedef struct UvmPmaAllocationOptions_tag
 {
     NvU32 flags;
@@ -846,6 +894,41 @@ typedef NV_STATUS (*uvmEventIsrTopHalf_t) (const NvProcessorUuid *pGpuUuidStruct
 typedef void (*uvmEventIsrTopHalf_t) (void);
 #endif
 
+/*******************************************************************************
+    uvmEventDrainP2P
+    This function will be called by the GPU driver to signal to UVM that the
+    GPU has encountered an uncontained error, and all peer work must be drained
+    to recover.  When it is called, the following assumptions/guarantees are
+    valid/made:
+
+      * Impacted user channels have been preempted and disabled
+      * UVM channels are still running normally and will continue to do
+        so unless an unrecoverable error is hit on said channels
+      * UVM must not return from this function until all enqueued work on
+      * peer channels has drained
+      * In the context of this function call, RM will still service faults
+      * UVM must prevent new peer work from being enqueued until the
+        uvmEventResumeP2P callback is issued
+
+    Returns:
+        NV_OK if UVM has idled peer work and will prevent new peer workloads.
+        NV_ERR_TIMEOUT if peer work was unable to be drained within a timeout
+        XXX NV_ERR_* for any other failure (TBD)
+
+*/
+typedef NV_STATUS (*uvmEventDrainP2P_t) (const NvProcessorUuid *pGpuUuidStruct);
+
+/*******************************************************************************
+    uvmEventResumeP2P
+    This function will be called by the GPU driver to signal to UVM that the
+    GPU has recovered from the previously reported uncontained NVLINK error.
+    When it is called, the following assumptions/guarantees are valid/made:
+
+      * UVM is again allowed to enqueue peer work
+      * UVM channels are still running normally
+*/
+typedef NV_STATUS (*uvmEventResumeP2P_t) (const NvProcessorUuid *pGpuUuidStruct);
+
 struct UvmOpsUvmEvents
 {
     uvmEventSuspend_t     suspend;
@@ -858,6 +941,8 @@ struct UvmOpsUvmEvents
     uvmEventWddmRestartAfterTimeout_t wddmRestartAfterTimeout;
     uvmEventServiceInterrupt_t serviceInterrupt;
 #endif
+    uvmEventDrainP2P_t drainP2P;
+    uvmEventResumeP2P_t resumeP2P;
 };
 
 #define UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES 32
@@ -1018,24 +1103,9 @@ typedef enum
     UVM_ACCESS_COUNTER_GRANULARITY_16G = 4,
 } UVM_ACCESS_COUNTER_GRANULARITY;
 
-typedef enum
-{
-    UVM_ACCESS_COUNTER_USE_LIMIT_NONE = 1,
-    UVM_ACCESS_COUNTER_USE_LIMIT_QTR  = 2,
-    UVM_ACCESS_COUNTER_USE_LIMIT_HALF = 3,
-    UVM_ACCESS_COUNTER_USE_LIMIT_FULL = 4,
-} UVM_ACCESS_COUNTER_USE_LIMIT;
-
 typedef struct UvmGpuAccessCntrConfig_tag
 {
-    NvU32 mimcGranularity;
-
-    NvU32 momcGranularity;
-
-    NvU32 mimcUseLimit;
-
-    NvU32 momcUseLimit;
-
+    NvU32 granularity;
     NvU32 threshold;
 } UvmGpuAccessCntrConfig;
 
@@ -1065,11 +1135,13 @@ typedef UvmGpuAccessCntrConfig gpuAccessCntrConfig;
 typedef UvmGpuFaultInfo gpuFaultInfo;
 typedef UvmGpuMemoryInfo gpuMemoryInfo;
 typedef UvmGpuExternalMappingInfo gpuExternalMappingInfo;
+typedef UvmGpuExternalPhysAddrInfo gpuExternalPhysAddrInfo;
 typedef UvmGpuChannelResourceInfo gpuChannelResourceInfo;
 typedef UvmGpuChannelInstanceInfo gpuChannelInstanceInfo;
 typedef UvmGpuChannelResourceBindParams gpuChannelResourceBindParams;
 typedef UvmGpuFbInfo gpuFbInfo;
 typedef UvmGpuEccInfo gpuEccInfo;
+typedef UvmGpuNvlinkInfo gpuNvlinkInfo;
 typedef UvmGpuPagingChannel *gpuPagingChannelHandle;
 typedef UvmGpuPagingChannelInfo gpuPagingChannelInfo;
 typedef UvmGpuPagingChannelAllocParams gpuPagingChannelAllocParams;

kernel-open/common/inc/nvkms-api-types.h

@@ -50,6 +50,8 @@
 #define NVKMS_LOG2_LUT_ARRAY_SIZE             10
 #define NVKMS_LUT_ARRAY_SIZE                  (1 << NVKMS_LOG2_LUT_ARRAY_SIZE)
 
+#define NVKMS_OLUT_FP_NORM_SCALE_DEFAULT      0xffffffff
+
 typedef NvU32 NvKmsDeviceHandle;
 typedef NvU32 NvKmsDispHandle;
 typedef NvU32 NvKmsConnectorHandle;
@@ -245,6 +247,80 @@ struct NvKmsLutRamps {
     NvU16 blue[NVKMS_LUT_ARRAY_SIZE];  /*! in */
 };
 
+/* Datatypes for LUT capabilities */
+enum NvKmsLUTFormat {
+    /*
+     * Normalized fixed-point format mapping [0, 1] to [0x0, 0xFFFF].
+     */
+    NVKMS_LUT_FORMAT_UNORM16,
+
+    /*
+     * Half-precision floating point.
+     */
+    NVKMS_LUT_FORMAT_FP16,
+
+    /*
+     * 14-bit fixed-point format required to work around hardware bug 813188.
+     *
+     * To convert from UNORM16 to UNORM14_WAR_813188:
+     * unorm14_war_813188 = ((unorm16 >> 2) & ~7) + 0x6000
+     */
+    NVKMS_LUT_FORMAT_UNORM14_WAR_813188
+};
+
+enum NvKmsLUTVssSupport {
+    NVKMS_LUT_VSS_NOT_SUPPORTED,
+    NVKMS_LUT_VSS_SUPPORTED,
+    NVKMS_LUT_VSS_REQUIRED,
+};
+
+enum NvKmsLUTVssType {
+    NVKMS_LUT_VSS_TYPE_NONE,
+    NVKMS_LUT_VSS_TYPE_LINEAR,
+    NVKMS_LUT_VSS_TYPE_LOGARITHMIC,
+};
+
+struct NvKmsLUTCaps {
+    /*! Whether this layer or head on this device supports this LUT stage. */
+    NvBool supported;
+
+    /*! Whether this LUT supports VSS. */
+    enum NvKmsLUTVssSupport vssSupport;
+
+    /*!
+     * The type of VSS segmenting this LUT uses.
+     */
+    enum NvKmsLUTVssType vssType;
+
+    /*!
+     * Expected number of VSS segments.
+     */
+    NvU32 vssSegments;
+
+    /*!
+     * Expected number of LUT entries.
+     */
+    NvU32 lutEntries;
+
+    /*!
+     * Format for each of the LUT entries.
+     */
+    enum NvKmsLUTFormat entryFormat;
+};
+
+/* each LUT entry uses this many bytes */
+#define NVKMS_LUT_CAPS_LUT_ENTRY_SIZE (4 * sizeof(NvU16))
+
+/* if the LUT surface uses VSS, size of the VSS header */
+#define NVKMS_LUT_VSS_HEADER_SIZE (4 * NVKMS_LUT_CAPS_LUT_ENTRY_SIZE)
+
+struct NvKmsLUTSurfaceParams {
+    NvKmsSurfaceHandle surfaceHandle;
+    NvU64 offset NV_ALIGN_BYTES(8);
+    NvU32 vssSegments;
+    NvU32 lutEntries;
+};
+
 /*
  * A 3x4 row-major colorspace conversion matrix.
  *
@@ -440,9 +516,9 @@ struct NvKmsLayerCapabilities {
     NvBool supportsWindowMode              :1;
 
     /*!
-     * Whether layer supports HDR pipe.
+     * Whether layer supports ICtCp pipe.
      */
-    NvBool supportsHDR                     :1;
+    NvBool supportsICtCp                   :1;
 
 
     /*!
@@ -463,6 +539,10 @@ struct NvKmsLayerCapabilities {
      * still expected to honor the NvKmsUsageBounds for each head.
      */
     NvU64 supportedSurfaceMemoryFormats NV_ALIGN_BYTES(8);
+
+    /* Capabilities for each LUT stage in the EVO3 precomp pipeline. */
+    struct NvKmsLUTCaps ilut;
+    struct NvKmsLUTCaps tmo;
 };
 
 /*!
@@ -683,4 +763,20 @@ struct NvKmsSuperframeInfo {
     } view[NVKMS_MAX_SUPERFRAME_VIEWS];
 };
 
+/* Fields within NvKmsVblankSemControlDataOneHead::flags */
+#define NVKMS_VBLANK_SEM_CONTROL_SWAP_INTERVAL          15:0
+
+struct NvKmsVblankSemControlDataOneHead {
+    NvU32 requestCounterAccel;
+    NvU32 requestCounter;
+    NvU32 flags;
+
+    NvU32 semaphore;
+    NvU64 vblankCount NV_ALIGN_BYTES(8);
+};
+
+struct NvKmsVblankSemControlData {
+    struct NvKmsVblankSemControlDataOneHead head[NV_MAX_HEADS];
+};
+
 #endif /* NVKMS_API_TYPES_H */

kernel-open/common/inc/nvkms-kapi.h

@@ -124,6 +124,14 @@ struct NvKmsKapiDisplayMode {
 #define NVKMS_KAPI_LAYER_INVALID_IDX           0xff
 #define NVKMS_KAPI_LAYER_PRIMARY_IDX              0
 
+struct NvKmsKapiLutCaps {
+    struct {
+        struct NvKmsLUTCaps ilut;
+        struct NvKmsLUTCaps tmo;
+    } layer[NVKMS_KAPI_LAYER_MAX];
+    struct NvKmsLUTCaps olut;
+};
+
 struct NvKmsKapiDeviceResourcesInfo {
 
     NvU32 numHeads;
@@ -158,13 +166,19 @@ struct NvKmsKapiDeviceResourcesInfo {
 
         NvU32 hasVideoMemory;
 
+        NvU32 numDisplaySemaphores;
+
         NvU8  genericPageKind;
 
         NvBool  supportsSyncpts;
+
+        NvBool requiresVrrSemaphores;
     } caps;
 
     NvU64 supportedSurfaceMemoryFormats[NVKMS_KAPI_LAYER_MAX];
-    NvBool supportsHDR[NVKMS_KAPI_LAYER_MAX];
+    NvBool supportsICtCp[NVKMS_KAPI_LAYER_MAX];
+
+    struct NvKmsKapiLutCaps lutCaps;
 };
 
 #define NVKMS_KAPI_LAYER_MASK(layerType) (1 << (layerType))
@@ -210,18 +224,26 @@ struct NvKmsKapiStaticDisplayInfo {
     NvU32 headMask;
 };
 
-struct NvKmsKapiSyncpt {
+struct NvKmsKapiSyncParams {
+    union {
+        struct {
+            /*!
+             * Possible syncpt use case in kapi.
+             * For pre-syncpt, use only id and value
+             * and for post-syncpt, use only fd.
+             */
+            NvU32   preSyncptId;
+            NvU32   preSyncptValue;
+        } syncpt;
 
-    /*!
-     * Possible syncpt use case in kapi.
-     * For pre-syncpt, use only id and value
-     * and for post-syncpt, use only fd.
-     */
-    NvBool  preSyncptSpecified;
-    NvU32   preSyncptId;
-    NvU32   preSyncptValue;
+        struct {
+            NvU32 index;
+        } semaphore;
+    } u;
 
-    NvBool  postSyncptRequested;
+    NvBool preSyncptSpecified;
+    NvBool postSyncptRequested;
+    NvBool semaphoreSpecified;
 };
 
 struct NvKmsKapiLayerConfig {
@@ -231,7 +253,7 @@ struct NvKmsKapiLayerConfig {
         NvU8 surfaceAlpha;
     } compParams;
     struct NvKmsRRParams rrParams;
-    struct NvKmsKapiSyncpt syncptParams;
+    struct NvKmsKapiSyncParams syncParams;
 
     struct {
         struct NvKmsHDRStaticMetadata val;
@@ -250,21 +272,54 @@ struct NvKmsKapiLayerConfig {
     NvU16 dstWidth, dstHeight;
 
     enum NvKmsInputColorSpace inputColorSpace;
+
+    struct {
+        NvBool enabled;
+        struct NvKmsKapiSurface *lutSurface;
+        NvU64 offset;
+        NvU32 vssSegments;
+        NvU32 lutEntries;
+    } ilut;
+
+    struct {
+        NvBool enabled;
+        struct NvKmsKapiSurface *lutSurface;
+        NvU64 offset;
+        NvU32 vssSegments;
+        NvU32 lutEntries;
+    } tmo;
+
     struct NvKmsCscMatrix csc;
     NvBool cscUseMain;
+
+    struct {
+        struct NvKmsCscMatrix lmsCtm;
+        struct NvKmsCscMatrix lmsToItpCtm;
+        struct NvKmsCscMatrix itpToLmsCtm;
+        struct NvKmsCscMatrix blendCtm;
+        struct {
+            NvBool lmsCtm      : 1;
+            NvBool lmsToItpCtm : 1;
+            NvBool itpToLmsCtm : 1;
+            NvBool blendCtm    : 1;
+        } enabled;
+    } matrixOverrides;
 };
 
 struct NvKmsKapiLayerRequestedConfig {
     struct NvKmsKapiLayerConfig config;
     struct {
-        NvBool surfaceChanged     : 1;
-        NvBool srcXYChanged       : 1;
-        NvBool srcWHChanged       : 1;
-        NvBool dstXYChanged       : 1;
-        NvBool dstWHChanged       : 1;
-        NvBool cscChanged         : 1;
-        NvBool tfChanged          : 1;
-        NvBool hdrMetadataChanged : 1;
+        NvBool surfaceChanged          : 1;
+        NvBool srcXYChanged            : 1;
+        NvBool srcWHChanged            : 1;
+        NvBool dstXYChanged            : 1;
+        NvBool dstWHChanged            : 1;
+        NvBool cscChanged              : 1;
+        NvBool tfChanged               : 1;
+        NvBool hdrMetadataChanged      : 1;
+        NvBool matrixOverridesChanged  : 1;
+        NvBool ilutChanged             : 1;
+        NvBool tmoChanged              : 1;
     } flags;
 };
 
@@ -319,7 +374,6 @@ struct NvKmsKapiHeadModeSetConfig {
 
     struct {
         struct {
-            NvBool specified;
             NvU32 depth;
             NvU32 start;
             NvU32 end;
@@ -327,22 +381,34 @@ struct NvKmsKapiHeadModeSetConfig {
         } input;
 
         struct {
-            NvBool specified;
             NvBool enabled;
             struct NvKmsLutRamps *pRamps;
         } output;
     } lut;
+
+    struct {
+        NvBool enabled;
+        struct NvKmsKapiSurface *lutSurface;
+        NvU64 offset;
+        NvU32 vssSegments;
+        NvU32 lutEntries;
+    } olut;
+
+    NvU32 olutFpNormScale;
 };
 
 struct NvKmsKapiHeadRequestedConfig {
     struct NvKmsKapiHeadModeSetConfig modeSetConfig;
     struct {
-        NvBool activeChanged       : 1;
-        NvBool displaysChanged     : 1;
-        NvBool modeChanged         : 1;
-        NvBool hdrInfoFrameChanged : 1;
-        NvBool colorimetryChanged  : 1;
-        NvBool lutChanged      : 1;
+        NvBool activeChanged          : 1;
+        NvBool displaysChanged        : 1;
+        NvBool modeChanged            : 1;
+        NvBool hdrInfoFrameChanged    : 1;
+        NvBool colorimetryChanged     : 1;
+        NvBool legacyIlutChanged      : 1;
+        NvBool legacyOlutChanged      : 1;
+        NvBool olutChanged            : 1;
+        NvBool olutFpNormScaleChanged : 1;
     } flags;
 
     struct NvKmsKapiCursorRequestedConfig cursorRequestedConfig;
@@ -368,6 +434,8 @@ struct NvKmsKapiHeadReplyConfig {
 
 struct NvKmsKapiModeSetReplyConfig {
     enum NvKmsFlipResult flipResult;
+    NvBool vrrFlip;
+    NvS32 vrrSemaphoreIndex;
     struct NvKmsKapiHeadReplyConfig
         headReplyConfig[NVKMS_KAPI_MAX_HEADS];
 };
@@ -476,6 +544,9 @@ struct NvKmsKapiCreateSurfaceParams {
      *      explicit_layout is NV_TRUE and layout is
      *      NvKmsSurfaceMemoryLayoutBlockLinear */
     NvU8 log2GobsPerBlockY;
+
+    /* [IN] Whether a surface can be updated directly on the screen */
+    NvBool noDisplayCaching;
 };
 
 enum NvKmsKapiAllocationType {
@@ -943,6 +1014,17 @@ struct NvKmsKapiFunctionsTable {
         const void *pLinearAddress
     );
 
+    /*!
+     * Check if memory object allocated is video memory.
+     *
+     * \param [in]  memory           Memory allocated using allocateMemory()
+     *
+     * \return NV_TRUE if memory is vidmem, NV_FALSE otherwise.
+     */
+    NvBool (*isVidmem)(
+        const struct NvKmsKapiMemory *memory
+    );
+
     /*!
      * Create a formatted surface from an NvKmsKapiMemory object.
      *
@@ -1159,21 +1241,6 @@ struct NvKmsKapiFunctionsTable {
         NvU64 *pPages
     );
 
-     /*!
-     * Check if this memory object can be scanned out for display.
-     *
-     * \param [in]  device  A device allocated using allocateDevice().
-     *
-     * \param [in]  memory  The memory object to check for display support.
-     *
-     * \return NV_TRUE if this memory can be displayed, NV_FALSE if not.
-     */
-    NvBool (*isMemoryValidForDisplay)
-    (
-        const struct NvKmsKapiDevice *device,
-        const struct NvKmsKapiMemory *memory
-    );
-
     /*
      * Import SGT as a memory handle.
      *
@@ -1410,6 +1477,117 @@ struct NvKmsKapiFunctionsTable {
     (
         NvKmsKapiSuspendResumeCallbackFunc *function
     );
+
+    /*!
+     * Immediately initialize the specified display semaphore to the pending state.
+     *
+     * Must be called prior to applying a mode set that utilizes the specified
+     * display semaphore for synchronization.
+     *
+     * \param [in] device         The device which will utilize the semaphore.
+     *
+     * \param [in] semaphoreIndex Index of the desired semaphore within the
+     *                            NVKMS semaphore pool. Must be less than
+     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
+     *                            for the specified device.
+     */
+    NvBool
+    (*tryInitDisplaySemaphore)
+    (
+        struct NvKmsKapiDevice *device,
+        NvU32 semaphoreIndex
+    );
+
+    /*!
+     * Immediately set the specified display semaphore to the displayable state.
+     *
+     * Must be called after \ref tryInitDisplaySemaphore to indicate a mode
+     * configuration change that utilizes the specified display semaphore for
+     * synchronization may proceed.
+     *
+     * \param [in] device         The device which will utilize the semaphore.
+     *
+     * \param [in] semaphoreIndex Index of the desired semaphore within the
+     *                            NVKMS semaphore pool. Must be less than
+     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
+     *                            for the specified device.
+     */
+    void
+    (*signalDisplaySemaphore)
+    (
+        struct NvKmsKapiDevice *device,
+        NvU32 semaphoreIndex
+    );
+
+    /*!
+     * Immediately cancel use of a display semaphore by resetting its value to
+     * its initial state.
+     *
+     * This can be used by clients to restore a semaphore to a consistent state
+     * when they have prepared it for use by previously calling
+     * \ref tryInitDisplaySemaphore() on it, but are then prevented from
+     * submitting the associated hardware operations to consume it due to the
+     * subsequent failure of some software or hardware operation.
+     *
+     * \param [in] device         The device which will utilize the semaphore.
+     *
+     * \param [in] semaphoreIndex Index of the desired semaphore within the
+     *                            NVKMS semaphore pool. Must be less than
+     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
+     *                            for the specified device.
+     */
+    void
+    (*cancelDisplaySemaphore)
+    (
+        struct NvKmsKapiDevice *device,
+        NvU32 semaphoreIndex
+    );
+
+    /*!
+     * Signal the VRR semaphore at the specified index from the CPU.
+     * If device does not support VRR semaphores, this is a no-op.
+     * Returns true if signal is success or no-op, otherwise returns false.
+     *
+     * \param [in]  device  A device allocated using allocateDevice().
+     *
+     * \param [in]  index   The VRR semaphore index to be signalled.
+     */
+    NvBool
+    (*signalVrrSemaphore)
+    (
+        struct NvKmsKapiDevice *device,
+        NvS32 index
+    );
+
+    /*!
+     * Check or wait on a head's LUT notifier.
+     *
+     * \param [in]  device              A device allocated using allocateDevice().
+     *
+     * \param [in]  head                The head to check for LUT completion.
+     *
+     * \param [in]  waitForCompletion   If true, wait for the notifier in NvKms
+     *                                  before returning.
+     *
+     * \param [out] complete            Returns whether the notifier has completed.
+     */
+    NvBool
+    (*checkLutNotifier)
+    (
+        struct NvKmsKapiDevice *device,
+        NvU32 head,
+        NvBool waitForCompletion
+    );
+
+    /*
+     * Notify NVKMS that the system's framebuffer console has been disabled and
+     * the reserved allocation for the old framebuffer console can be unmapped.
+     */
+    void
+    (*framebufferConsoleDisabled)
+    (
+        struct NvKmsKapiDevice *device
+    );
 };
 
 /** @} */
@@ -1424,6 +1602,20 @@ NvBool nvKmsKapiGetFunctionsTable
     struct NvKmsKapiFunctionsTable *funcsTable
 );
 
+NvU32 nvKmsKapiF16ToF32(NvU16 a);
+
+NvU16 nvKmsKapiF32ToF16(NvU32 a);
+
+NvU32 nvKmsKapiF32Mul(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32Div(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32Add(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32ToUI32RMinMag(NvU32 a, NvBool exact);
+
+NvU32 nvKmsKapiUI32ToF32(NvU32 a);
+
 /** @} */
 
 #endif /* defined(__NVKMS_KAPI_H__) */

kernel-open/common/inc/nvlimits.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2017-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -34,19 +34,25 @@
 /*
  * This is the maximum number of GPUs supported in a single system.
  */
-#define NV_MAX_DEVICES          32
+#define NV_MAX_DEVICES                32
 
 /*
  * This is the maximum number of subdevices within a single device.
  */
-#define NV_MAX_SUBDEVICES       8
+#define NV_MAX_SUBDEVICES             8
 
 /*
  * This is the maximum length of the process name string.
  */
-#define NV_PROC_NAME_MAX_LENGTH 100U
+#define NV_PROC_NAME_MAX_LENGTH       100U
 
 /*
  * This is the maximum number of heads per GPU.
  */
-#define NV_MAX_HEADS            4
+#define NV_MAX_HEADS                  4
+
+/*
+ * Maximum length of a MIG device UUID. It is a 36-byte UUID string plus a
+ * 4-byte prefix and NUL terminator: 'M' 'I' 'G' '-' UUID '\0x0'
+ */
+#define NV_MIG_DEVICE_UUID_STR_LENGTH 41U

kernel-open/common/inc/nvmisc.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 1993-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -33,43 +33,26 @@ extern "C" {
 
 #include "nvtypes.h"
 
-#if !defined(NVIDIA_UNDEF_LEGACY_BIT_MACROS)
-//
-// Miscellaneous macros useful for bit field manipulations
-//
-// STUPID HACK FOR CL 19434692.  Will revert when fix CL is delivered bfm -> chips_a.
-#ifndef BIT
-#define BIT(b)                  (1U<<(b))
-#endif
-#ifndef BIT32
-#define BIT32(b)                ((NvU32)1U<<(b))
-#endif
-#ifndef BIT64
-#define BIT64(b)                ((NvU64)1U<<(b))
-#endif
-
-#endif
-
-//
-// It is recommended to use the following bit macros to avoid macro name
-// collisions with other src code bases.
-//
+// Miscellaneous macros useful for bit field manipulations.
 #ifndef NVBIT
-#define NVBIT(b)                  (1U<<(b))
+#define NVBIT(b)         (1U<<(b))
 #endif
 #ifndef NVBIT_TYPE
-#define NVBIT_TYPE(b, t)          (((t)1U)<<(b))
+#define NVBIT_TYPE(b, t) (((t)1U)<<(b))
 #endif
 #ifndef NVBIT32
-#define NVBIT32(b)                NVBIT_TYPE(b, NvU32)
+#define NVBIT32(b)       NVBIT_TYPE(b, NvU32)
 #endif
 #ifndef NVBIT64
-#define NVBIT64(b)                NVBIT_TYPE(b, NvU64)
+#define NVBIT64(b)       NVBIT_TYPE(b, NvU64)
 #endif
 
+//Concatenate 2 32bit values to a 64bit value
+#define NV_CONCAT_32_TO_64(hi, lo) ((((NvU64)hi) << 32) | ((NvU64)lo))
+
 // Helper macro's for 32 bit bitmasks
 #define NV_BITMASK32_ELEMENT_SIZE            (sizeof(NvU32) << 3)
-#define NV_BITMASK32_IDX(chId)               (((chId) & ~(0x1F)) >> 5)  
+#define NV_BITMASK32_IDX(chId)               (((chId) & ~(0x1F)) >> 5)
 #define NV_BITMASK32_OFFSET(chId)            ((chId) & (0x1F))
 #define NV_BITMASK32_SET(pChannelMask, chId) \
         (pChannelMask)[NV_BITMASK32_IDX(chId)] |= NVBIT(NV_BITMASK32_OFFSET(chId))
@@ -494,6 +477,23 @@ do                                                      \
 //
 #define NV_TWO_N_MINUS_ONE(n) (((1ULL<<(n/2))<<((n+1)/2))-1)
 
+//
+// Create a 64b bitmask with n bits set
+// This is the same as ((1ULL<<n) - 1), but it doesn't overflow for n=64
+//
+// ...
+// n=-1, 0x0000000000000000
+// n=0,  0x0000000000000000
+// n=1,  0x0000000000000001
+// ...
+// n=63, 0x7FFFFFFFFFFFFFFF
+// n=64, 0xFFFFFFFFFFFFFFFF
+// n=65, 0xFFFFFFFFFFFFFFFF
+// n=66, 0xFFFFFFFFFFFFFFFF
+// ...
+//
+#define NV_BITMASK64(n) ((n<1) ? 0ULL : (NV_U64_MAX>>((n>64) ? 0 : (64-n))))
+
 #define DRF_READ_1WORD_BS(d,r,f,v) \
     ((DRF_EXTENT_MW(NV##d##r##f)<8)?DRF_READ_1BYTE_BS(NV##d##r##f,(v)): \
     ((DRF_EXTENT_MW(NV##d##r##f)<16)?DRF_READ_2BYTE_BS(NV##d##r##f,(v)): \
@@ -574,6 +574,13 @@ nvMaskPos32(const NvU32 mask, const NvU32 bitIdx)
     n32 = BIT_IDX_32(LOWESTBIT(n32));\
 }
 
+// Destructive operation on n64
+#define LOWESTBITIDX_64(n64)         \
+{                                    \
+    n64 = BIT_IDX_64(LOWESTBIT(n64));\
+}
+
+
 // Destructive operation on n32
 #define HIGHESTBITIDX_32(n32)   \
 {                               \
@@ -694,6 +701,42 @@ nvPrevPow2_U64(const NvU64 x )
     }                                                       \
 }
 
+//
+// Bug 4851259: Newly added functions must be hidden from certain HS-signed
+// ucode compilers to avoid signature mismatch.
+//
+#ifndef NVDEC_1_0
+/*!
+ * Returns the position of nth set bit in the given mask.
+ *
+ * Returns -1 if mask has fewer than n bits set.
+ *
+ * n is 0 indexed and has valid values 0..31 inclusive, so "zeroth" set bit is
+ * the first set LSB.
+ *
+ * Example, if mask = 0x000000F0u and n = 1, the return value will be 5.
+ * Example, if mask = 0x000000F0u and n = 4, the return value will be -1.
+ */
+static NV_FORCEINLINE NvS32
+nvGetNthSetBitIndex32(NvU32 mask, NvU32 n)
+{
+    NvU32 seenSetBitsCount = 0;
+    NvS32 index;
+    FOR_EACH_INDEX_IN_MASK(32, index, mask)
+    {
+        if (seenSetBitsCount == n)
+        {
+            return index;
+        }
+        ++seenSetBitsCount;
+    }
+    FOR_EACH_INDEX_IN_MASK_END;
+
+    return -1;
+}
+
+#endif // NVDEC_1_0
+
 //
 // Size to use when declaring variable-sized arrays
 //
@@ -918,10 +961,31 @@ static NV_FORCEINLINE void *NV_NVUPTR_TO_PTR(NvUPtr address)
 // Use (lo) if (b) is less than 64, and (hi) if >= 64.
 //
 #define NV_BIT_SET_128(b, lo, hi)              { nvAssert( (b) < 128 ); if ( (b) < 64 ) (lo) |= NVBIT64(b); else (hi) |= NVBIT64( b & 0x3F ); }
+//
+// Clear the bit at pos (b) for U64 which is < 128.
+// Use (lo) if (b) is less than 64, and (hi) if >= 64.
+//
+#define NV_BIT_CLEAR_128(b, lo, hi)            { nvAssert( (b) < 128 ); if ( (b) < 64 ) (lo) &= ~NVBIT64(b); else (hi) &= ~NVBIT64( b & 0x3F ); }
 
 // Get the number of elements the specified fixed-size array
 #define NV_ARRAY_ELEMENTS(x)                   ((sizeof(x)/sizeof((x)[0])))
 
+#if !defined(NVIDIA_UNDEF_LEGACY_BIT_MACROS)
+//
+// Deprecated macros whose definition can be removed once the code base no longer references them.
+// Use the NVBIT* macros instead of these macros.
+//
+#ifndef BIT
+#define BIT(b)   (1U<<(b))
+#endif
+#ifndef BIT32
+#define BIT32(b) ((NvU32)1U<<(b))
+#endif
+#ifndef BIT64
+#define BIT64(b) ((NvU64)1U<<(b))
+#endif
+#endif
+
 #ifdef __cplusplus
 }
 #endif //__cplusplus

kernel-open/common/inc/nvstatuscodes.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -152,8 +152,14 @@ NV_STATUS_CODE(NV_ERR_FABRIC_MANAGER_NOT_PRESENT,      0x0000007A, "Fabric Manag
 NV_STATUS_CODE(NV_ERR_ALREADY_SIGNALLED,               0x0000007B, "Semaphore Surface value already >= requested wait value")
 NV_STATUS_CODE(NV_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE,   0x0000007C, "PMU RPC error due to no queue slot available for this event")
 NV_STATUS_CODE(NV_ERR_KEY_ROTATION_IN_PROGRESS,        0x0000007D, "Operation not allowed as key rotation is in progress")
-NV_STATUS_CODE(NV_ERR_NVSWITCH_FABRIC_NOT_READY,       0x00000081, "Nvswitch Fabric Status or Fabric Probe is not yet complete, caller needs to retry")
-NV_STATUS_CODE(NV_ERR_NVSWITCH_FABRIC_FAILURE,         0x00000082, "Nvswitch Fabric Probe failed")
+NV_STATUS_CODE(NV_ERR_TEST_ONLY_CODE_NOT_ENABLED,      0x0000007E, "Test-only code path not enabled")
+NV_STATUS_CODE(NV_ERR_SECURE_BOOT_FAILED,              0x0000007F, "GFW secure boot failed")
+NV_STATUS_CODE(NV_ERR_INSUFFICIENT_ZBC_ENTRY,          0x00000080, "No more ZBC entry for the client")
+NV_STATUS_CODE(NV_ERR_NVLINK_FABRIC_NOT_READY,         0x00000081, "Nvlink Fabric Status or Fabric Probe is not yet complete, caller needs to retry")
+NV_STATUS_CODE(NV_ERR_NVLINK_FABRIC_FAILURE,           0x00000082, "Nvlink Fabric Probe failed")
+NV_STATUS_CODE(NV_ERR_GPU_MEMORY_ONLINING_FAILURE,     0x00000083, "GPU Memory Onlining failed")
+NV_STATUS_CODE(NV_ERR_REDUCTION_MANAGER_NOT_AVAILABLE, 0x00000084, "Reduction Manager is not available")
+NV_STATUS_CODE(NV_ERR_RESOURCE_RETIREMENT_ERROR,       0x00000086, "An error occurred while trying to retire a resource")
 
 // Warnings:
 NV_STATUS_CODE(NV_WARN_HOT_SWITCH,                     0x00010001, "WARNING Hot switch")

kernel-open/common/inc/nvtypes.h

@@ -152,6 +152,12 @@ typedef   signed short     NvS16; /* -32768 to 32767                         */
      (((NvU32)(c) & 0xff) << 8)  | \
      (((NvU32)(d) & 0xff))))
 
+// Macro to build an NvU64 from two DWORDS, listed from msb to lsb
+#define NvU64_BUILD(a, b) \
+    ((NvU64)( \
+     (((NvU64)(a) & ~0U) << 32) | \
+     (((NvU64)(b) & ~0U))))
+
 #if NVTYPES_USE_STDINT
 typedef uint32_t           NvV32; /* "void": enumerated or multiple fields   */
 typedef uint32_t           NvU32; /* 0 to 4294967295                         */

kernel-open/common/inc/os-interface.h

@@ -40,8 +40,11 @@
 #include "nv_stdarg.h"
 #include <nv-kernel-interface-api.h>
 #include <os/nv_memory_type.h>
+#include <os/nv_memory_area.h>
 #include <nv-caps.h>
 
+#include "rs_access.h"
+
 
 
 typedef struct
@@ -59,6 +62,11 @@ struct os_work_queue;
 /* Each OS defines its own version of this opaque type */
 typedef struct os_wait_queue os_wait_queue;
 
+/* Flags needed by os_get_current_proccess_flags */
+#define OS_CURRENT_PROCESS_FLAG_NONE            0x0
+#define OS_CURRENT_PROCESS_FLAG_KERNEL_THREAD   0x1
+#define OS_CURRENT_PROCESS_FLAG_EXITING         0x2
+
 /*
  * ---------------------------------------------------------------------------
  *
@@ -102,8 +110,10 @@ NvBool      NV_API_CALL  os_pci_remove_supported     (void);
 void        NV_API_CALL  os_pci_remove               (void *);
 void*       NV_API_CALL  os_map_kernel_space         (NvU64, NvU64, NvU32);
 void        NV_API_CALL  os_unmap_kernel_space       (void *, NvU64);
-void*       NV_API_CALL  os_map_user_space           (NvU64, NvU64, NvU32, NvU32, void **);
+#if defined(NV_VMWARE)
+void*       NV_API_CALL  os_map_user_space           (MemoryArea *, NvU32, NvU32, void **);
 void        NV_API_CALL  os_unmap_user_space         (void *, NvU64, void *);
+#endif
 NV_STATUS   NV_API_CALL  os_flush_cpu_cache_all      (void);
 NV_STATUS   NV_API_CALL  os_flush_user_cache         (void);
 void        NV_API_CALL  os_flush_cpu_write_combine_buffer(void);
@@ -114,7 +124,7 @@ void        NV_API_CALL  os_io_write_byte            (NvU32, NvU8);
 void        NV_API_CALL  os_io_write_word            (NvU32, NvU16);
 void        NV_API_CALL  os_io_write_dword           (NvU32, NvU32);
 NvBool      NV_API_CALL  os_is_administrator         (void);
-NvBool      NV_API_CALL  os_allow_priority_override  (void);
+NvBool      NV_API_CALL  os_check_access             (RsAccessRight accessRight);
 void        NV_API_CALL  os_dbg_init                 (void);
 void        NV_API_CALL  os_dbg_breakpoint           (void);
 void        NV_API_CALL  os_dbg_set_level            (NvU32);
@@ -130,7 +140,8 @@ void        NV_API_CALL  os_free_spinlock            (void *);
 NvU64       NV_API_CALL  os_acquire_spinlock         (void *);
 void        NV_API_CALL  os_release_spinlock         (void *, NvU64);
 NV_STATUS   NV_API_CALL  os_queue_work_item          (struct os_work_queue *, void *);
-NV_STATUS   NV_API_CALL  os_flush_work_queue         (struct os_work_queue *);
+NV_STATUS   NV_API_CALL  os_flush_work_queue         (struct os_work_queue *, NvBool);
+NvBool      NV_API_CALL  os_is_queue_flush_ongoing   (struct os_work_queue *);
 NV_STATUS   NV_API_CALL  os_alloc_mutex              (void **);
 void        NV_API_CALL  os_free_mutex               (void *);
 NV_STATUS   NV_API_CALL  os_acquire_mutex            (void *);
@@ -151,6 +162,7 @@ void        NV_API_CALL  os_release_rwlock_read      (void *);
 void        NV_API_CALL  os_release_rwlock_write     (void *);
 NvBool      NV_API_CALL  os_semaphore_may_sleep      (void);
 NV_STATUS   NV_API_CALL  os_get_version_info         (os_version_info*);
+NV_STATUS   NV_API_CALL  os_get_is_openrm            (NvBool *);
 NvBool      NV_API_CALL  os_is_isr                   (void);
 NvBool      NV_API_CALL  os_pat_supported            (void);
 void        NV_API_CALL  os_dump_stack               (void);
@@ -163,7 +175,7 @@ NvU32       NV_API_CALL  os_get_grid_csp_support     (void);
 void        NV_API_CALL  os_bug_check                (NvU32, const char *);
 NV_STATUS   NV_API_CALL  os_lock_user_pages          (void *, NvU64, void **, NvU32);
 NV_STATUS   NV_API_CALL  os_lookup_user_io_memory    (void *, NvU64, NvU64 **);
-NV_STATUS   NV_API_CALL  os_unlock_user_pages        (NvU64, void *);
+NV_STATUS   NV_API_CALL  os_unlock_user_pages        (NvU64, void *, NvU32);
 NV_STATUS   NV_API_CALL  os_match_mmap_offset        (void *, NvU64, NvU64 *);
 NV_STATUS   NV_API_CALL  os_get_euid                 (NvU32 *);
 NV_STATUS   NV_API_CALL  os_get_smbios_header        (NvU64 *pSmbsAddr);
@@ -171,6 +183,7 @@ NV_STATUS   NV_API_CALL  os_get_acpi_rsdp_from_uefi  (NvU32 *);
 void        NV_API_CALL  os_add_record_for_crashLog  (void *, NvU32);
 void        NV_API_CALL  os_delete_record_for_crashLog (void *);
 NV_STATUS   NV_API_CALL  os_call_vgpu_vfio           (void *, NvU32);
+NV_STATUS   NV_API_CALL  os_device_vm_present        (void);
 NV_STATUS   NV_API_CALL  os_numa_memblock_size       (NvU64 *);
 NV_STATUS   NV_API_CALL  os_alloc_pages_node         (NvS32, NvU32, NvU32, NvU64 *);
 NV_STATUS   NV_API_CALL  os_get_page                 (NvU64 address);
@@ -186,6 +199,7 @@ NV_STATUS   NV_API_CALL  os_open_readonly_file       (const char *, void **);
 NV_STATUS   NV_API_CALL  os_open_and_read_file       (const char *, NvU8 *, NvU64);
 NvBool      NV_API_CALL  os_is_nvswitch_present      (void);
 NV_STATUS   NV_API_CALL  os_get_random_bytes         (NvU8 *, NvU16);
+NvU32       NV_API_CALL  os_get_current_process_flags(void);
 NV_STATUS   NV_API_CALL  os_alloc_wait_queue         (os_wait_queue **);
 void        NV_API_CALL  os_free_wait_queue          (os_wait_queue *);
 void        NV_API_CALL  os_wait_uninterruptible     (os_wait_queue *);
@@ -206,6 +220,7 @@ enum os_pci_req_atomics_type {
     OS_INTF_PCIE_REQ_ATOMICS_128BIT
 };
 NV_STATUS   NV_API_CALL  os_enable_pci_req_atomics   (void *, enum os_pci_req_atomics_type);
+void        NV_API_CALL  os_pci_trigger_flr(void *handle);
 NV_STATUS   NV_API_CALL  os_get_numa_node_memory_usage (NvS32, NvU64 *, NvU64 *);
 NV_STATUS   NV_API_CALL  os_numa_add_gpu_memory      (void *, NvU64, NvU64, NvU32 *);
 NV_STATUS   NV_API_CALL  os_numa_remove_gpu_memory   (void *, NvU64, NvU64, NvU32); 
@@ -213,6 +228,7 @@ NV_STATUS   NV_API_CALL  os_offline_page_at_address(NvU64 address);
 void*       NV_API_CALL  os_get_pid_info(void);
 void        NV_API_CALL  os_put_pid_info(void *pid_info);
 NV_STATUS   NV_API_CALL  os_find_ns_pid(void *pid_info, NvU32 *ns_pid);
+NvBool      NV_API_CALL  os_is_init_ns(void);
 
 extern NvU32 os_page_size;
 extern NvU64 os_page_mask;

kernel-open/common/inc/os/nv_memory_area.h

@@ -0,0 +1,104 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NV_MEMORY_AREA_H
+#define NV_MEMORY_AREA_H
+
+typedef struct MemoryRange
+{
+    NvU64 start;
+    NvU64 size;
+} MemoryRange;
+
+typedef struct MemoryArea
+{
+    MemoryRange *pRanges;
+    NvU64 numRanges;
+} MemoryArea;
+
+static inline NvU64 memareaSize(MemoryArea memArea)
+{
+    NvU64 size = 0;
+    NvU64 idx = 0;
+    for (idx = 0; idx < memArea.numRanges; idx++)
+    {
+        size += memArea.pRanges[idx].size;
+    }
+    return size;
+}
+
+static inline MemoryRange
+mrangeMake
+(
+    NvU64 start,
+    NvU64 size
+)
+{
+    MemoryRange range;
+    range.start = start;
+    range.size = size;
+    return range;
+}
+
+static inline NvU64
+mrangeLimit
+(
+    MemoryRange a
+)
+{
+    return a.start + a.size;
+}
+
+static inline NvBool
+mrangeIntersects
+(
+    MemoryRange a,
+    MemoryRange b
+)
+{
+    return ((a.start >= b.start) && (a.start < mrangeLimit(b))) ||
+        ((b.start >= a.start) && (b.start < mrangeLimit(a)));
+}
+
+static inline NvBool
+mrangeContains
+(
+    MemoryRange outer,
+    MemoryRange inner
+)
+{
+    return (inner.start >= outer.start) && (mrangeLimit(inner) <= mrangeLimit(outer));
+}
+
+static inline MemoryRange
+mrangeOffset
+(
+    MemoryRange range,
+    NvU64 amt
+)
+{
+    range.start += amt;
+    return range;
+}
+
+#endif /* NV_MEMORY_AREA_H */

kernel-open/common/inc/rm-gpu-ops.h

@@ -81,13 +81,15 @@ NV_STATUS  NV_API_CALL rm_gpu_ops_has_pending_non_replayable_faults(nvidia_stack
 NV_STATUS  NV_API_CALL rm_gpu_ops_init_access_cntr_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t, NvU32);
 NV_STATUS  NV_API_CALL rm_gpu_ops_destroy_access_cntr_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_own_access_cntr_intr(nvidia_stack_t *, nvgpuSessionHandle_t, nvgpuAccessCntrInfo_t, NvBool);
-NV_STATUS  NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t, nvgpuAccessCntrConfig_t);
+NV_STATUS  NV_API_CALL rm_gpu_ops_enable_access_cntr(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t, const nvgpuAccessCntrConfig_t *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_disable_access_cntr(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuAccessCntrInfo_t);
 NV_STATUS  NV_API_CALL  rm_gpu_ops_set_page_directory (nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64, unsigned, NvBool, NvU32);
 NV_STATUS  NV_API_CALL  rm_gpu_ops_unset_page_directory (nvidia_stack_t *, nvgpuAddressSpaceHandle_t);
+NV_STATUS  NV_API_CALL rm_gpu_ops_get_nvlink_info(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuNvlinkInfo_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_p2p_object_create(nvidia_stack_t *, nvgpuDeviceHandle_t, nvgpuDeviceHandle_t, NvHandle *);
 void       NV_API_CALL rm_gpu_ops_p2p_object_destroy(nvidia_stack_t *, nvgpuSessionHandle_t, NvHandle);
 NV_STATUS  NV_API_CALL rm_gpu_ops_get_external_alloc_ptes(nvidia_stack_t*, nvgpuAddressSpaceHandle_t, NvHandle, NvU64, NvU64, nvgpuExternalMappingInfo_t);
+NV_STATUS  NV_API_CALL rm_gpu_ops_get_external_alloc_phys_addrs(nvidia_stack_t*, nvgpuAddressSpaceHandle_t, NvHandle, NvU64, NvU64, nvgpuExternalPhysAddrInfo_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_retain_channel(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvHandle, NvHandle, void **, nvgpuChannelInstanceInfo_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_bind_channel_resources(nvidia_stack_t *, void *, nvgpuChannelResourceBindParams_t);
 void       NV_API_CALL rm_gpu_ops_release_channel(nvidia_stack_t *, void *);
@@ -100,6 +102,7 @@ void       NV_API_CALL rm_gpu_ops_paging_channel_destroy(nvidia_stack_t *, nvgpu
 NV_STATUS  NV_API_CALL rm_gpu_ops_paging_channels_map(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64, nvgpuDeviceHandle_t, NvU64 *);
 void       NV_API_CALL rm_gpu_ops_paging_channels_unmap(nvidia_stack_t *, nvgpuAddressSpaceHandle_t, NvU64, nvgpuDeviceHandle_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *, nvgpuPagingChannelHandle_t, char *, NvU32);
+void       NV_API_CALL rm_gpu_ops_report_fatal_error(nvidia_stack_t *, NV_STATUS error);
 
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *, struct ccslContext_t **, nvgpuChannelHandle_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *, struct ccslContext_t *);

kernel-open/common/inc/rs_access.h

@@ -0,0 +1,276 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#pragma once
+
+#include <nvtypes.h>
+#if defined(_MSC_VER)
+#pragma warning(disable:4324)
+#endif
+
+//
+// This file was generated with FINN, an NVIDIA coding tool.
+// Source file:      rs_access.finn
+//
+
+
+
+
+#include "nvtypes.h"
+#include "nvmisc.h"
+
+
+/****************************************************************************/
+/*                       Access right definitions                           */
+/****************************************************************************/
+
+//
+// The meaning of each access right is documented in
+//   resman/docs/rmapi/resource_server/rm_capabilities.adoc
+//
+// RS_ACCESS_COUNT is the number of access rights that have been defined
+// and are in use. All integers in the range [0, RS_ACCESS_COUNT) should
+// represent valid access rights.
+//
+// When adding a new access right, don't forget to update
+//   1) The descriptions in the resman/docs/rmapi/resource_server/rm_capabilities.adoc
+//   2) RS_ACCESS_COUNT, defined below
+//   3) The declaration of g_rsAccessMetadata in rs_access_rights.c
+//   4) The list of access rights in drivers/common/chip-config/Chipcontrols.pm
+//   5) Any relevant access right callbacks
+//
+
+#define RS_ACCESS_DUP_OBJECT 0U
+#define RS_ACCESS_NICE       1U
+#define RS_ACCESS_DEBUG      2U
+#define RS_ACCESS_PERFMON    3U
+#define RS_ACCESS_COUNT      4U
+
+
+/****************************************************************************/
+/*                     Access right data structures                         */
+/****************************************************************************/
+
+/*!
+ * @brief A type that can be used to represent any access right.
+ */
+typedef NvU16 RsAccessRight;
+
+/*!
+ * @brief An internal type used to represent one limb in an access right mask.
+ */
+typedef NvU32 RsAccessLimb;
+#define SDK_RS_ACCESS_LIMB_BITS 32
+
+/*!
+ * @brief The number of limbs in the RS_ACCESS_MASK struct.
+ */
+#define SDK_RS_ACCESS_MAX_LIMBS 1
+
+/*!
+ * @brief The maximum number of possible access rights supported by the
+ *        current data structure definition.
+ *
+ * You probably want RS_ACCESS_COUNT instead, which is the number of actual
+ * access rights defined.
+ */
+#define SDK_RS_ACCESS_MAX_COUNT (0x20) /* finn: Evaluated from "(SDK_RS_ACCESS_LIMB_BITS * SDK_RS_ACCESS_MAX_LIMBS)" */
+
+/**
+ * @brief A struct representing a set of access rights.
+ *
+ * Note that the values of bit positions larger than RS_ACCESS_COUNT is
+ * undefined, and should not be assumed to be 0 (see RS_ACCESS_MASK_FILL).
+ */
+typedef struct RS_ACCESS_MASK {
+    RsAccessLimb limbs[SDK_RS_ACCESS_MAX_LIMBS];
+} RS_ACCESS_MASK;
+
+/**
+ * @brief A struct representing auxiliary information about each access right.
+ */
+typedef struct RS_ACCESS_INFO {
+    NvU32 flags;
+} RS_ACCESS_INFO;
+
+
+/****************************************************************************/
+/*                           Access right macros                            */
+/****************************************************************************/
+
+#define SDK_RS_ACCESS_LIMB_INDEX(index) ((index) / SDK_RS_ACCESS_LIMB_BITS)
+#define SDK_RS_ACCESS_LIMB_POS(index)   ((index) % SDK_RS_ACCESS_LIMB_BITS)
+
+#define SDK_RS_ACCESS_LIMB_ELT(pAccessMask, index) \
+    ((pAccessMask)->limbs[SDK_RS_ACCESS_LIMB_INDEX(index)])
+#define SDK_RS_ACCESS_OFFSET_MASK(index) \
+    NVBIT_TYPE(SDK_RS_ACCESS_LIMB_POS(index), RsAccessLimb)
+
+/*!
+ * @brief Checks that accessRight represents a valid access right.
+ *
+ * The valid range of access rights is [0, RS_ACCESS_COUNT).
+ *
+ * @param[in] accessRight The access right value to check
+ *
+ * @return true if accessRight is valid
+ * @return false otherwise
+ */
+#define RS_ACCESS_BOUNDS_CHECK(accessRight) \
+    (accessRight < RS_ACCESS_COUNT)
+
+/*!
+ * @brief Test whether an access right is present in a set
+ *
+ * @param[in] pAccessMask The set of access rights to read
+ * @param[in] index The access right to examine
+ *
+ * @return NV_TRUE if the access right specified by index was present in the set,
+ *         and NV_FALSE otherwise
+ */
+#define RS_ACCESS_MASK_TEST(pAccessMask, index) \
+    (RS_ACCESS_BOUNDS_CHECK(index) && \
+        (SDK_RS_ACCESS_LIMB_ELT(pAccessMask, index) & SDK_RS_ACCESS_OFFSET_MASK(index)) != 0)
+
+/*!
+ * @brief Add an access right to a mask
+ *
+ * @param[in] pAccessMask The set of access rights to modify
+ * @param[in] index The access right to set
+ */
+#define RS_ACCESS_MASK_ADD(pAccessMask, index) \
+    do \
+    { \
+        if (RS_ACCESS_BOUNDS_CHECK(index)) { \
+            SDK_RS_ACCESS_LIMB_ELT(pAccessMask, index) |= SDK_RS_ACCESS_OFFSET_MASK(index); \
+        } \
+    } while (NV_FALSE)
+
+/*!
+ * @brief Remove an access right from a mask
+ *
+ * @param[in] pAccessMask The set of access rights to modify
+ * @param[in] index The access right to unset
+ */
+#define RS_ACCESS_MASK_REMOVE(pAccessMask, index) \
+    do \
+    { \
+        if (RS_ACCESS_BOUNDS_CHECK(index)) { \
+            SDK_RS_ACCESS_LIMB_ELT(pAccessMask, index) &= ~SDK_RS_ACCESS_OFFSET_MASK(index); \
+        } \
+    } while (NV_FALSE)
+
+/*!
+ * @brief Performs an in-place union between two access right masks
+ *
+ * @param[in,out] pMaskOut The access rights mask to be updated
+ * @param[in] pMaskIn The set of access rights to be added to pMaskOut
+ */
+#define RS_ACCESS_MASK_UNION(pMaskOut, pMaskIn) \
+    do \
+    { \
+        NvLength limb; \
+        for (limb = 0; limb < SDK_RS_ACCESS_MAX_LIMBS; limb++) \
+        { \
+            SDK_RS_ACCESS_LIMB_ELT(pMaskOut, limb) |= SDK_RS_ACCESS_LIMB_ELT(pMaskIn, limb); \
+        } \
+    } while (NV_FALSE)
+
+/*!
+ * @brief Performs an in-place subtract of one mask's rights from another
+ *
+ * @param[in,out] pMaskOut The access rights mask to be updated
+ * @param[in] pMaskIn The set of access rights to be removed from pMaskOut
+ */
+#define RS_ACCESS_MASK_SUBTRACT(pMaskOut, pMaskIn) \
+    do \
+    { \
+        NvLength limb; \
+        for (limb = 0; limb < SDK_RS_ACCESS_MAX_LIMBS; limb++) \
+        { \
+            SDK_RS_ACCESS_LIMB_ELT(pMaskOut, limb) &= ~SDK_RS_ACCESS_LIMB_ELT(pMaskIn, limb); \
+        } \
+    } while (NV_FALSE)
+
+/*!
+ * @brief Removes all rights from an access rights mask
+ *
+ * @param[in,out] pAccessMask The access rights mask to be updated
+ */
+#define RS_ACCESS_MASK_CLEAR(pAccessMask) \
+    do \
+    { \
+        portMemSet(pAccessMask, 0, sizeof(*pAccessMask)); \
+    } while (NV_FALSE)
+
+/*!
+ * @brief Adds all rights to an access rights mask
+ *
+ * @param[in,out] pAccessMask The access rights mask to be updated
+ */
+#define RS_ACCESS_MASK_FILL(pAccessMask) \
+    do \
+    { \
+        portMemSet(pAccessMask, 0xff, sizeof(*pAccessMask)); \
+    } while (NV_FALSE)
+
+
+/****************************************************************************/
+/*                           Share definitions                              */
+/****************************************************************************/
+
+//
+// The usage of Share Policy and the meaning of each share type is documented in
+//   resman/docs/rmapi/resource_server/rm_capabilities.adoc
+//
+#define RS_SHARE_TYPE_NONE              (0U)
+#define RS_SHARE_TYPE_ALL               (1U)
+#define RS_SHARE_TYPE_OS_SECURITY_TOKEN (2U)
+#define RS_SHARE_TYPE_CLIENT            (3U)
+#define RS_SHARE_TYPE_PID               (4U)
+#define RS_SHARE_TYPE_SMC_PARTITION     (5U)
+#define RS_SHARE_TYPE_GPU               (6U)
+#define RS_SHARE_TYPE_FM_CLIENT         (7U)
+// Must be last. Update when a new SHARE_TYPE is added
+#define RS_SHARE_TYPE_MAX               (8U)
+
+
+//
+// Use Revoke to remove an existing policy from the list.
+// Allow is based on OR logic, Require is based on AND logic.
+// To share a right, at least one Allow (non-Require) must match, and all Require must pass.
+// If Compose is specified, policies will be added to the list. Otherwise, they will replace the list.
+//
+#define RS_SHARE_ACTION_FLAG_REVOKE      NVBIT(0)
+#define RS_SHARE_ACTION_FLAG_REQUIRE     NVBIT(1)
+#define RS_SHARE_ACTION_FLAG_COMPOSE     NVBIT(2)
+
+/****************************************************************************/
+/*                       Share flag data structures                         */
+/****************************************************************************/
+
+typedef struct RS_SHARE_POLICY {
+    NvU32          target;
+    RS_ACCESS_MASK accessMask;
+    NvU16          type;                         ///< RS_SHARE_TYPE_
+    NvU8           action;                        ///< RS_SHARE_ACTION_
+} RS_SHARE_POLICY;

kernel-open/header-presence-tests.mk

@@ -3,6 +3,7 @@
 NV_HEADER_PRESENCE_TESTS = \
   asm/system.h \
   drm/drmP.h \
+  drm/drm_print.h \
   drm/drm_aperture.h \
   drm/drm_auth.h \
   drm/drm_gem.h \
@@ -16,6 +17,7 @@ NV_HEADER_PRESENCE_TESTS = \
   drm/drm_drv.h \
   drm/drm_fbdev_generic.h \
   drm/drm_fbdev_ttm.h \
+  drm/drm_client_setup.h \
   drm/drm_framebuffer.h \
   drm/drm_connector.h \
   drm/drm_probe_helper.h \
@@ -29,10 +31,15 @@ NV_HEADER_PRESENCE_TESTS = \
   drm/drm_device.h \
   drm/drm_mode_config.h \
   drm/drm_modeset_lock.h \
+  drm/drm_property.h \
+  drm/clients/drm_client_setup.h \
   dt-bindings/interconnect/tegra_icc_id.h \
   generated/autoconf.h \
   generated/compile.h \
   generated/utsrelease.h \
+  linux/pfn_t.h \
+  linux/aperture.h \
+  linux/dma-direct.h \
   linux/efi.h \
   linux/kconfig.h \
   linux/platform/tegra/mc_utils.h \
@@ -53,6 +60,7 @@ NV_HEADER_PRESENCE_TESTS = \
   linux/dma-resv.h \
   soc/tegra/chip-id.h \
   soc/tegra/fuse.h \
+  soc/tegra/fuse-helper.h \
   soc/tegra/tegra_bpmp.h \
   video/nv_internal.h \
   linux/platform/tegra/dce/dce-client-ipc.h \
@@ -99,5 +107,7 @@ NV_HEADER_PRESENCE_TESTS = \
   linux/cc_platform.h \
   asm/cpufeature.h \
   linux/mpi.h \
-  asm/mshyperv.h
+  asm/mshyperv.h \
+  crypto/sig.h \
+  linux/pfn_t.h
 

kernel-open/nvidia-drm/nv_common_utils.h

@@ -0,0 +1,120 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2015 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __NV_COMMON_UTILS_H__
+#define __NV_COMMON_UTILS_H__
+
+#include "nvtypes.h"
+#include "nvmisc.h"
+
+#if !defined(TRUE)
+#define TRUE NV_TRUE
+#endif
+
+#if !defined(FALSE)
+#define FALSE NV_FALSE
+#endif
+
+#define NV_IS_UNSIGNED(x) ((__typeof__(x))-1 > 0)
+
+/* Get the length of a statically-sized array. */
+#define ARRAY_LEN(_arr) (sizeof(_arr) / sizeof(_arr[0]))
+
+#define NV_INVALID_HEAD         0xFFFFFFFF
+
+#define NV_INVALID_CONNECTOR_PHYSICAL_INFORMATION (~0)
+
+#if !defined(NV_MIN)
+# define NV_MIN(a,b) (((a)<(b))?(a):(b))
+#endif
+
+#define NV_MIN3(a,b,c) NV_MIN(NV_MIN(a, b), c)
+#define NV_MIN4(a,b,c,d) NV_MIN3(NV_MIN(a,b),c,d)
+
+#if !defined(NV_MAX)
+# define NV_MAX(a,b) (((a)>(b))?(a):(b))
+#endif
+
+#define NV_MAX3(a,b,c) NV_MAX(NV_MAX(a, b), c)
+#define NV_MAX4(a,b,c,d) NV_MAX3(NV_MAX(a,b),c,d)
+
+static inline int NV_LIMIT_VAL_TO_MIN_MAX(int val, int min, int max)
+{
+    if (val < min) {
+        return min;
+    }
+    if (val > max) {
+        return max;
+    }
+    return val;
+}
+
+#define NV_ROUNDUP_DIV(x,y) ((x) / (y) + (((x) % (y)) ? 1 : 0))
+
+/*
+ * Macros used for computing palette entries:
+ *
+ * NV_UNDER_REPLICATE(val, source_size, result_size) expands a value
+ * of source_size bits into a value of target_size bits by shifting
+ * the source value into the high bits and replicating the high bits
+ * of the value into the low bits of the result.
+ *
+ * PALETTE_DEPTH_SHIFT(val, w) maps a colormap entry for a component
+ * that has w bits to an appropriate entry in a LUT of 256 entries.
+ */
+static inline unsigned int NV_UNDER_REPLICATE(unsigned short val,
+                                              int source_size,
+                                              int result_size)
+{
+    return (val << (result_size - source_size)) |
+        (val >> ((source_size << 1) - result_size));
+}
+
+
+static inline unsigned short PALETTE_DEPTH_SHIFT(unsigned short val, int depth)
+{
+    return NV_UNDER_REPLICATE(val, depth, 8);
+}
+
+/*
+ *  Use __builtin_ffs where it is supported, or provide an equivalent
+ *  implementation for platforms like riscv where it is not.
+ */
+#if defined(__GNUC__) && !NVCPU_IS_RISCV64
+static inline int nv_ffs(int x)
+{
+    return __builtin_ffs(x);
+}
+#else
+static inline int nv_ffs(int x)
+{
+    if (x == 0)
+        return 0;
+
+    LOWESTBITIDX_32(x);
+
+    return 1 + x;
+}
+#endif
+
+#endif /* __NV_COMMON_UTILS_H__ */

kernel-open/nvidia-drm/nvidia-drm-conftest.h

@@ -62,6 +62,31 @@
 #undef NV_DRM_FENCE_AVAILABLE
 #endif
 
+#if defined(NV_DRM_FBDEV_GENERIC_SETUP_PRESENT) &&                             \
+    defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+#define NV_DRM_FBDEV_AVAILABLE
+#define NV_DRM_FBDEV_GENERIC_AVAILABLE
+#endif
+
+#if defined(NV_DRM_FBDEV_TTM_SETUP_PRESENT) &&                                 \
+    defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+#if IS_ENABLED(CONFIG_DRM_TTM_HELPER)
+#define NV_DRM_FBDEV_AVAILABLE
+#define NV_DRM_FBDEV_TTM_AVAILABLE
+#endif
+#endif
+
+#if defined(NV_DRM_CLIENT_SETUP_PRESENT) &&                                    \
+    (defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT) ||   \
+     defined(NV_APERTURE_REMOVE_CONFLICTING_PCI_DEVICES_PRESENT))
+// XXX remove dependency on DRM_TTM_HELPER by implementing nvidia-drm's own
+// .fbdev_probe callback that uses NVKMS kapi
+#if IS_ENABLED(CONFIG_DRM_TTM_HELPER)
+#define NV_DRM_FBDEV_AVAILABLE
+#define NV_DRM_CLIENT_AVAILABLE
+#endif
+#endif
+
 /*
  * We can support color management if either drm_helper_crtc_enable_color_mgmt()
  * or drm_crtc_enable_color_mgmt() exist.
@@ -85,7 +110,11 @@
 
 /* For nv_drm_gem_prime_force_fence_signal */
 #ifndef spin_is_locked
+#if ((__FreeBSD_version >= 1500000) && (__FreeBSD_version < 1500018)) || (__FreeBSD_version < 1401501)
 #define spin_is_locked(lock) mtx_owned(lock.m)
+#else
+#define spin_is_locked(lock) mtx_owned(lock)
+#endif
 #endif
 
 #ifndef rwsem_is_locked

kernel-open/nvidia-drm/nvidia-drm-connector.c

@@ -314,7 +314,11 @@ static int nv_drm_connector_get_modes(struct drm_connector *connector)
 }
 
 static int nv_drm_connector_mode_valid(struct drm_connector    *connector,
+#if defined(NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG)
+                                       const struct drm_display_mode *mode)
+#else
                                        struct drm_display_mode *mode)
+#endif
 {
     struct drm_device *dev = connector->dev;
     struct nv_drm_device *nv_dev = to_nv_device(dev);

kernel-open/nvidia-drm/nvidia-drm-crtc.h

@@ -38,6 +38,13 @@
 #include "nvtypes.h"
 #include "nvkms-kapi.h"
 
+enum nv_drm_transfer_function {
+    NV_DRM_TRANSFER_FUNCTION_DEFAULT,
+    NV_DRM_TRANSFER_FUNCTION_LINEAR,
+    NV_DRM_TRANSFER_FUNCTION_PQ,
+    NV_DRM_TRANSFER_FUNCTION_MAX,
+};
+
 struct nv_drm_crtc {
     NvU32 head;
 
@@ -63,6 +70,8 @@ struct nv_drm_crtc {
      */
     struct drm_file *modeset_permission_filep;
 
+    struct NvKmsLUTCaps olut_caps;
+
     struct drm_crtc base;
 };
 
@@ -129,9 +138,6 @@ struct nv_drm_crtc_state {
      */
     struct NvKmsKapiHeadRequestedConfig req_config;
 
-    struct NvKmsLutRamps *ilut_ramps;
-    struct NvKmsLutRamps *olut_ramps;
-
     /**
      * @nv_flip:
      *
@@ -145,6 +151,12 @@ struct nv_drm_crtc_state {
      * nv_drm_atomic_crtc_destroy_state().
      */
     struct nv_drm_flip *nv_flip;
+
+    enum nv_drm_transfer_function regamma_tf;
+    struct drm_property_blob *regamma_lut;
+    uint64_t regamma_divisor;
+    struct nv_drm_lut_surface *regamma_drm_lut_surface;
+    NvBool regamma_changed;
 };
 
 static inline struct nv_drm_crtc_state *to_nv_crtc_state(struct drm_crtc_state *state)
@@ -152,6 +164,11 @@ static inline struct nv_drm_crtc_state *to_nv_crtc_state(struct drm_crtc_state *
     return container_of(state, struct nv_drm_crtc_state, base);
 }
 
+static inline const struct nv_drm_crtc_state *to_nv_crtc_state_const(const struct drm_crtc_state *state)
+{
+    return container_of(state, struct nv_drm_crtc_state, base);
+}
+
 struct nv_drm_plane {
     /**
      * @base:
@@ -173,6 +190,9 @@ struct nv_drm_plane {
      * Index of this plane in the per head array of layers.
      */
     uint32_t layer_idx;
+
+    struct NvKmsLUTCaps ilut_caps;
+    struct NvKmsLUTCaps tmo_caps;
 };
 
 static inline struct nv_drm_plane *to_nv_plane(struct drm_plane *plane)
@@ -183,6 +203,22 @@ static inline struct nv_drm_plane *to_nv_plane(struct drm_plane *plane)
     return container_of(plane, struct nv_drm_plane, base);
 }
 
+struct nv_drm_lut_surface {
+    struct NvKmsKapiDevice *pDevice;
+    struct NvKmsKapiMemory *nvkms_memory;
+    struct NvKmsKapiSurface *nvkms_surface;
+    struct {
+        NvU32 vssSegments;
+        enum NvKmsLUTVssType vssType;
+
+        NvU32 lutEntries;
+        enum NvKmsLUTFormat entryFormat;
+
+    } properties;
+    void *buffer;
+    struct kref refcount;
+};
+
 struct nv_drm_plane_state {
     struct drm_plane_state base;
     s32 __user *fd_user_ptr;
@@ -190,6 +226,20 @@ struct nv_drm_plane_state {
 #if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
     struct drm_property_blob *hdr_output_metadata;
 #endif
+    struct drm_property_blob *lms_ctm;
+    struct drm_property_blob *lms_to_itp_ctm;
+    struct drm_property_blob *itp_to_lms_ctm;
+    struct drm_property_blob *blend_ctm;
+
+    enum nv_drm_transfer_function degamma_tf;
+    struct drm_property_blob *degamma_lut;
+    uint64_t degamma_multiplier; /* S31.32 Sign-Magnitude Format */
+    struct nv_drm_lut_surface *degamma_drm_lut_surface;
+    NvBool degamma_changed;
+
+    struct drm_property_blob *tmo_lut;
+    struct nv_drm_lut_surface *tmo_drm_lut_surface;
+    NvBool tmo_changed;
 };
 
 static inline struct nv_drm_plane_state *to_nv_drm_plane_state(struct drm_plane_state *state)

kernel-open/nvidia-drm/nvidia-drm-drv.c

@@ -64,11 +64,24 @@
 #include <drm/drm_ioctl.h>
 #endif
 
-#if defined(NV_DRM_FBDEV_AVAILABLE)
+#if defined(NV_LINUX_APERTURE_H_PRESENT)
+#include <linux/aperture.h>
+#endif
+
+#if defined(NV_DRM_DRM_APERTURE_H_PRESENT)
 #include <drm/drm_aperture.h>
+#endif
+
+#if defined(NV_DRM_FBDEV_AVAILABLE)
 #include <drm/drm_fb_helper.h>
 #endif
 
+#if defined(NV_DRM_DRM_CLIENT_SETUP_H_PRESENT)
+#include <drm/drm_client_setup.h>
+#elif defined(NV_DRM_CLIENTS_DRM_CLIENT_SETUP_H_PRESENT)
+#include <drm/clients/drm_client_setup.h>
+#endif
+
 #if defined(NV_DRM_DRM_FBDEV_TTM_H_PRESENT)
 #include <drm/drm_fbdev_ttm.h>
 #elif defined(NV_DRM_DRM_FBDEV_GENERIC_H_PRESENT)
@@ -107,16 +120,16 @@ static int nv_drm_revoke_sub_ownership(struct drm_device *dev);
 
 static struct nv_drm_device *dev_list = NULL;
 
-static const char* nv_get_input_colorspace_name(
+static char* nv_get_input_colorspace_name(
     enum NvKmsInputColorSpace colorSpace)
 {
     switch (colorSpace) {
         case NVKMS_INPUT_COLORSPACE_NONE:
             return "None";
         case NVKMS_INPUT_COLORSPACE_SCRGB_LINEAR:
-            return "IEC 61966-2-2 linear FP";
+            return "scRGB Linear FP16";
         case NVKMS_INPUT_COLORSPACE_BT2100_PQ:
-            return "ITU-R BT.2100-PQ YCbCr";
+            return "BT.2100 PQ";
         default:
             /* We shoudn't hit this */
             WARN_ON("Unsupported input colorspace");
@@ -124,6 +137,27 @@ static const char* nv_get_input_colorspace_name(
     }
 };
 
+static char* nv_get_transfer_function_name(
+    enum nv_drm_transfer_function tf)
+{
+    switch (tf) {
+        case NV_DRM_TRANSFER_FUNCTION_LINEAR:
+            return "Linear";
+        case NV_DRM_TRANSFER_FUNCTION_PQ:
+            return "PQ (Perceptual Quantizer)";
+        default:
+            /* We shoudn't hit this */
+            WARN_ON("Unsupported transfer function");
+#if defined(fallthrough)
+            fallthrough;
+#else
+            /* Fallthrough */
+#endif
+        case NV_DRM_TRANSFER_FUNCTION_DEFAULT:
+            return "Default";
+    }
+};
+
 #if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)
 
 #if defined(NV_DRM_OUTPUT_POLL_CHANGED_PRESENT)
@@ -175,11 +209,14 @@ static void nv_drm_output_poll_changed(struct drm_device *dev)
 static struct drm_framebuffer *nv_drm_framebuffer_create(
     struct drm_device *dev,
     struct drm_file *file,
-    #if defined(NV_DRM_HELPER_MODE_FILL_FB_STRUCT_HAS_CONST_MODE_CMD_ARG)
+#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO)
+    const struct drm_format_info *info,
+#endif
+#if defined(NV_DRM_HELPER_MODE_FILL_FB_STRUCT_HAS_CONST_MODE_CMD_ARG)
     const struct drm_mode_fb_cmd2 *cmd
-    #else
+#else
     struct drm_mode_fb_cmd2 *cmd
-    #endif
+#endif
 )
 {
     struct drm_mode_fb_cmd2 local_cmd;
@@ -190,11 +227,14 @@ static struct drm_framebuffer *nv_drm_framebuffer_create(
     fb = nv_drm_internal_framebuffer_create(
             dev,
             file,
+#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO)
+            info,
+#endif
             &local_cmd);
 
-    #if !defined(NV_DRM_HELPER_MODE_FILL_FB_STRUCT_HAS_CONST_MODE_CMD_ARG)
+#if !defined(NV_DRM_HELPER_MODE_FILL_FB_STRUCT_HAS_CONST_MODE_CMD_ARG)
     *cmd = local_cmd;
-    #endif
+#endif
 
     return fb;
 }
@@ -370,31 +410,33 @@ static void nv_drm_enumerate_encoders_and_connectors
  */
 static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
 {
-    struct drm_prop_enum_list enum_list[3] = { };
+    struct drm_prop_enum_list colorspace_enum_list[3] = { };
+    struct drm_prop_enum_list tf_enum_list[NV_DRM_TRANSFER_FUNCTION_MAX] = { };
     int i, len = 0;
 
     for (i = 0; i < 3; i++) {
-        enum_list[len].type = i;
-        enum_list[len].name = nv_get_input_colorspace_name(i);
+        colorspace_enum_list[len].type = i;
+        colorspace_enum_list[len].name = nv_get_input_colorspace_name(i);
         len++;
     }
 
-#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
-    if (!nv_dev->supportsSyncpts) {
-        return 0;
+    for (i = 0; i < NV_DRM_TRANSFER_FUNCTION_MAX; i++) {
+        tf_enum_list[i].type = i;
+        tf_enum_list[i].name = nv_get_transfer_function_name(i);
     }
 
-    nv_dev->nv_out_fence_property =
-        drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_ATOMIC,
-            "NV_DRM_OUT_FENCE_PTR", 0, U64_MAX);
-    if (nv_dev->nv_out_fence_property == NULL) {
-        return -ENOMEM;
+    if (nv_dev->supportsSyncpts) {
+        nv_dev->nv_out_fence_property =
+            drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_ATOMIC,
+                    "NV_DRM_OUT_FENCE_PTR", 0, U64_MAX);
+        if (nv_dev->nv_out_fence_property == NULL) {
+            return -ENOMEM;
+        }
     }
-#endif
 
     nv_dev->nv_input_colorspace_property =
         drm_property_create_enum(nv_dev->dev, 0, "NV_INPUT_COLORSPACE",
-                                 enum_list, len);
+                                 colorspace_enum_list, len);
     if (nv_dev->nv_input_colorspace_property == NULL) {
         NV_DRM_LOG_ERR("Failed to create NV_INPUT_COLORSPACE property");
         return -ENOMEM;
@@ -409,6 +451,109 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
     }
 #endif
 
+    nv_dev->nv_plane_lms_ctm_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_LMS_CTM", 0);
+    if (nv_dev->nv_plane_lms_ctm_property == NULL) {
+        return -ENOMEM;
+    }
+
+    nv_dev->nv_plane_lms_to_itp_ctm_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_LMS_TO_ITP_CTM", 0);
+    if (nv_dev->nv_plane_lms_to_itp_ctm_property == NULL) {
+        return -ENOMEM;
+    }
+
+    nv_dev->nv_plane_itp_to_lms_ctm_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_ITP_TO_LMS_CTM", 0);
+    if (nv_dev->nv_plane_itp_to_lms_ctm_property == NULL) {
+        return -ENOMEM;
+    }
+
+    nv_dev->nv_plane_blend_ctm_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_BLEND_CTM", 0);
+    if (nv_dev->nv_plane_blend_ctm_property == NULL) {
+        return -ENOMEM;
+    }
+
+    // Degamma TF + LUT + LUT Size + Multiplier
+
+    nv_dev->nv_plane_degamma_tf_property =
+        drm_property_create_enum(nv_dev->dev, 0,
+            "NV_PLANE_DEGAMMA_TF", tf_enum_list,
+            NV_DRM_TRANSFER_FUNCTION_MAX);
+    if (nv_dev->nv_plane_degamma_tf_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_plane_degamma_lut_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_DEGAMMA_LUT", 0);
+    if (nv_dev->nv_plane_degamma_lut_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_plane_degamma_lut_size_property =
+        drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_IMMUTABLE,
+            "NV_PLANE_DEGAMMA_LUT_SIZE", 0, UINT_MAX);
+    if (nv_dev->nv_plane_degamma_lut_size_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_plane_degamma_multiplier_property =
+        drm_property_create_range(nv_dev->dev, 0,
+            "NV_PLANE_DEGAMMA_MULTIPLIER", 0,
+            U64_MAX & ~(((NvU64) 1) << 63)); // No negative values
+    if (nv_dev->nv_plane_degamma_multiplier_property == NULL) {
+        return -ENOMEM;
+    }
+
+    // TMO LUT + LUT Size
+
+    nv_dev->nv_plane_tmo_lut_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_PLANE_TMO_LUT", 0);
+    if (nv_dev->nv_plane_tmo_lut_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_plane_tmo_lut_size_property =
+        drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_IMMUTABLE,
+            "NV_PLANE_TMO_LUT_SIZE", 0, UINT_MAX);
+    if (nv_dev->nv_plane_tmo_lut_size_property == NULL) {
+        return -ENOMEM;
+    }
+
+    // REGAMMA TF + LUT + LUT Size + Divisor
+
+    nv_dev->nv_crtc_regamma_tf_property =
+        drm_property_create_enum(nv_dev->dev, 0,
+            "NV_CRTC_REGAMMA_TF", tf_enum_list,
+            NV_DRM_TRANSFER_FUNCTION_MAX);
+    if (nv_dev->nv_crtc_regamma_tf_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_crtc_regamma_lut_property =
+        drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
+            "NV_CRTC_REGAMMA_LUT", 0);
+    if (nv_dev->nv_crtc_regamma_lut_property == NULL) {
+        return -ENOMEM;
+    }
+    nv_dev->nv_crtc_regamma_lut_size_property =
+        drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_IMMUTABLE,
+            "NV_CRTC_REGAMMA_LUT_SIZE", 0, UINT_MAX);
+    if (nv_dev->nv_crtc_regamma_lut_size_property == NULL) {
+        return -ENOMEM;
+    }
+    // S31.32
+    nv_dev->nv_crtc_regamma_divisor_property =
+        drm_property_create_range(nv_dev->dev, 0,
+            "NV_CRTC_REGAMMA_DIVISOR",
+            (((NvU64) 1) << 32), // No values between 0 and 1
+            U64_MAX & ~(((NvU64) 1) << 63)); // No negative values
+    if (nv_dev->nv_crtc_regamma_divisor_property == NULL) {
+        return -ENOMEM;
+    }
+
     return 0;
 }
 
@@ -440,7 +585,7 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
 
     struct NvKmsKapiAllocateDeviceParams allocateDeviceParams;
     struct NvKmsKapiDeviceResourcesInfo resInfo;
-#endif
+#endif /* defined(NV_DRM_ATOMIC_MODESET_AVAILABLE) */
 #if defined(NV_DRM_FORMAT_MODIFIERS_PRESENT)
     NvU64 kind;
     NvU64 gen;
@@ -527,6 +672,12 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
     nv_dev->semsurf_max_submitted_offset =
         resInfo.caps.semsurf.maxSubmittedOffset;
 
+    nv_dev->display_semaphores.count =
+        resInfo.caps.numDisplaySemaphores;
+    nv_dev->display_semaphores.next_index = 0;
+
+    nv_dev->requiresVrrSemaphores = resInfo.caps.requiresVrrSemaphores;
+
 #if defined(NV_DRM_FORMAT_MODIFIERS_PRESENT)
     gen = nv_dev->pageKindGeneration;
     kind = nv_dev->genericPageKind;
@@ -552,6 +703,13 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
 
     ret = nv_drm_create_properties(nv_dev);
     if (ret < 0) {
+        drm_mode_config_cleanup(dev);
+#if defined(NV_DRM_FBDEV_AVAILABLE)
+        if (nv_dev->hasFramebufferConsole) {
+            nvKms->releaseOwnership(nv_dev->pDevice);
+        }
+#endif
+        nvKms->freeDevice(nv_dev->pDevice);
         return -ENODEV;
     }
 
@@ -683,7 +841,6 @@ static int __nv_drm_master_set(struct drm_device *dev,
         !nvKms->grabOwnership(nv_dev->pDevice)) {
         return -EINVAL;
     }
-    nv_dev->drmMasterChangedSinceLastAtomicCommit = NV_TRUE;
 
     return 0;
 }
@@ -715,36 +872,37 @@ void nv_drm_master_drop(struct drm_device *dev, struct drm_file *file_priv)
 #endif
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
-    int err;
 
     nv_drm_revoke_modeset_permission(dev, file_priv, 0);
     nv_drm_revoke_sub_ownership(dev);
 
-    /*
-     * After dropping nvkms modeset onwership, it is not guaranteed that
-     * drm and nvkms modeset state will remain in sync.  Therefore, disable
-     * all outputs and crtcs before dropping nvkms modeset ownership.
-     *
-     * First disable all active outputs atomically and then disable each crtc one
-     * by one, there is not helper function available to disable all crtcs
-     * atomically.
-     */
-
-    drm_modeset_lock_all(dev);
-
-    if ((err = nv_drm_atomic_helper_disable_all(
-            dev,
-            dev->mode_config.acquire_ctx)) != 0) {
-
-        NV_DRM_DEV_LOG_ERR(
-            nv_dev,
-            "nv_drm_atomic_helper_disable_all failed with error code %d !",
-            err);
-    }
-
-    drm_modeset_unlock_all(dev);
-
     if (!nv_dev->hasFramebufferConsole) {
+        int err;
+
+        /*
+         * After dropping nvkms modeset onwership, it is not guaranteed that drm
+         * and nvkms modeset state will remain in sync.  Therefore, disable all
+         * outputs and crtcs before dropping nvkms modeset ownership.
+         *
+         * First disable all active outputs atomically and then disable each
+         * crtc one by one, there is not helper function available to disable
+         * all crtcs atomically.
+         */
+
+        drm_modeset_lock_all(dev);
+
+        if ((err = nv_drm_atomic_helper_disable_all(
+                dev,
+                dev->mode_config.acquire_ctx)) != 0) {
+
+            NV_DRM_DEV_LOG_ERR(
+                nv_dev,
+                "nv_drm_atomic_helper_disable_all failed with error code %d !",
+                err);
+        }
+
+        drm_modeset_unlock_all(dev);
+
         nvKms->releaseOwnership(nv_dev->pDevice);
     }
 }
@@ -812,6 +970,14 @@ static int nv_drm_get_dev_info_ioctl(struct drm_device *dev,
     return 0;
 }
 
+static int nv_drm_get_drm_file_unique_id_ioctl(struct drm_device *dev,
+                                               void *data, struct drm_file *filep)
+{
+    struct drm_nvidia_get_drm_file_unique_id_params *params = data;
+    params->id = (u64)(filep->driver_priv);
+    return 0;
+}
+
 static int nv_drm_dmabuf_supported_ioctl(struct drm_device *dev,
                                          void *data, struct drm_file *filep)
 {
@@ -1328,6 +1494,17 @@ static void nv_drm_postclose(struct drm_device *dev, struct drm_file *filep)
 }
 #endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */
 
+static int nv_drm_open(struct drm_device *dev, struct drm_file *filep)
+{
+    _Static_assert(sizeof(filep->driver_priv) >= sizeof(u64),
+                   "filep->driver_priv can not hold an u64");
+    static atomic64_t id = ATOMIC_INIT(0);
+
+    filep->driver_priv = (void *)atomic64_inc_return(&id);
+
+    return 0;
+}
+
 #if defined(NV_DRM_MASTER_HAS_LEASES)
 static struct drm_master *nv_drm_find_lessee(struct drm_master *master,
                                              int lessee_id)
@@ -1553,6 +1730,10 @@ static const struct file_operations nv_drm_fops = {
     .read           = drm_read,
 
     .llseek         = noop_llseek,
+
+#if defined(NV_FILE_OPERATIONS_FOP_UNSIGNED_OFFSET_PRESENT)
+    .fop_flags   = FOP_UNSIGNED_OFFSET,
+#endif
 };
 
 static const struct drm_ioctl_desc nv_drm_ioctls[] = {
@@ -1571,6 +1752,9 @@ static const struct drm_ioctl_desc nv_drm_ioctls[] = {
     DRM_IOCTL_DEF_DRV(NVIDIA_GET_DEV_INFO,
                       nv_drm_get_dev_info_ioctl,
                       DRM_RENDER_ALLOW|DRM_UNLOCKED),
+    DRM_IOCTL_DEF_DRV(NVIDIA_GET_DRM_FILE_UNIQUE_ID,
+                      nv_drm_get_drm_file_unique_id_ioctl,
+                      DRM_RENDER_ALLOW|DRM_UNLOCKED),
 
 #if defined(NV_DRM_FENCE_AVAILABLE)
     DRM_IOCTL_DEF_DRV(NVIDIA_FENCE_SUPPORTED,
@@ -1653,6 +1837,9 @@ static struct drm_driver nv_drm_driver = {
     .driver_features        =
 #if defined(NV_DRM_DRIVER_PRIME_FLAG_PRESENT)
                                DRIVER_PRIME |
+#endif
+#if defined(NV_DRM_SYNCOBJ_FEATURES_PRESENT)
+                               DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE |
 #endif
                                DRIVER_GEM  | DRIVER_RENDER,
 
@@ -1664,14 +1851,19 @@ static struct drm_driver nv_drm_driver = {
     .num_ioctls             = ARRAY_SIZE(nv_drm_ioctls),
 
 /*
- * linux-next commit 71a7974ac701 ("drm/prime: Unexport helpers for fd/handle
- * conversion") unexports drm_gem_prime_handle_to_fd() and
- * drm_gem_prime_fd_to_handle().
+ * Linux kernel v6.6 commit 6b85aa68d9d5 ("drm: Enable PRIME import/export for
+ * all drivers") made drm_gem_prime_handle_to_fd() /
+ * drm_gem_prime_fd_to_handle() the default when .prime_handle_to_fd /
+ * .prime_fd_to_handle are unspecified, respectively.
  *
- * Prior linux-next commit 6b85aa68d9d5 ("drm: Enable PRIME import/export for
- * all drivers") made these helpers the default when .prime_handle_to_fd /
- * .prime_fd_to_handle are unspecified, so it's fine to just skip specifying
- * them if the helpers aren't present.
+ * Linux kernel v6.6 commit 71a7974ac701 ("drm/prime: Unexport helpers for
+ * fd/handle conversion") unexports drm_gem_prime_handle_to_fd() and
+ * drm_gem_prime_fd_to_handle(). However, because of the aforementioned commit,
+ * it's fine to just skip specifying them in this case.
+ *
+ * Linux kernel v6.7 commit 0514f63cfff3 ("Revert "drm/prime: Unexport helpers
+ * for fd/handle conversion"") exported the helpers again, but left the default
+ * behavior intact. Nonetheless, it does not hurt to specify them.
  */
 #if NV_IS_EXPORT_SYMBOL_PRESENT_drm_gem_prime_handle_to_fd
     .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
@@ -1683,6 +1875,21 @@ static struct drm_driver nv_drm_driver = {
     .gem_prime_import       = nv_drm_gem_prime_import,
     .gem_prime_import_sg_table = nv_drm_gem_prime_import_sg_table,
 
+/*
+ * Linux kernel v5.0 commit 7698799f95 ("drm/prime: Add drm_gem_prime_mmap()")
+ * added drm_gem_prime_mmap().
+ *
+ * Linux kernel v6.6 commit 0adec22702d4 ("drm: Remove struct
+ * drm_driver.gem_prime_mmap") removed .gem_prime_mmap, but replaced it with a
+ * direct call to drm_gem_prime_mmap().
+ *
+ * TODO: Support .gem_prime_mmap on Linux < v5.0 using internal implementation.
+ */
+#if defined(NV_DRM_GEM_PRIME_MMAP_PRESENT) && \
+    defined(NV_DRM_DRIVER_HAS_GEM_PRIME_MMAP)
+    .gem_prime_mmap         = drm_gem_prime_mmap,
+#endif
+
 #if defined(NV_DRM_DRIVER_HAS_GEM_PRIME_CALLBACKS)
     .gem_prime_export       = drm_gem_prime_export,
     .gem_prime_get_sg_table = nv_drm_gem_prime_get_sg_table,
@@ -1705,6 +1912,7 @@ static struct drm_driver nv_drm_driver = {
 #if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)
     .postclose              = nv_drm_postclose,
 #endif
+    .open                   = nv_drm_open,
 
     .fops                   = &nv_drm_fops,
 
@@ -1715,13 +1923,20 @@ static struct drm_driver nv_drm_driver = {
     .name                   = "nvidia-drm",
 
     .desc                   = "NVIDIA DRM driver",
+
+#if defined(NV_DRM_DRIVER_HAS_DATE)
     .date                   = "20160202",
+#endif
 
 #if defined(NV_DRM_DRIVER_HAS_DEVICE_LIST)
     .device_list            = LIST_HEAD_INIT(nv_drm_driver.device_list),
 #elif defined(NV_DRM_DRIVER_HAS_LEGACY_DEV_LIST)
     .legacy_dev_list        = LIST_HEAD_INIT(nv_drm_driver.legacy_dev_list),
 #endif
+// XXX implement nvidia-drm's own .fbdev_probe callback that uses NVKMS kapi directly
+#if defined(NV_DRM_FBDEV_AVAILABLE) && defined(DRM_FBDEV_TTM_DRIVER_OPS)
+    DRM_FBDEV_TTM_DRIVER_OPS,
+#endif
 };
 
 
@@ -1763,6 +1978,7 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
     struct nv_drm_device *nv_dev = NULL;
     struct drm_device *dev = NULL;
     struct device *device = gpu_info->os_device_ptr;
+    bool bus_is_pci;
 
     DRM_DEBUG(
         "Registering device for NVIDIA GPU ID 0x08%x",
@@ -1796,7 +2012,7 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
     dev->dev_private = nv_dev;
     nv_dev->dev = dev;
 
-    bool bus_is_pci =
+    bus_is_pci =
 #if defined(NV_LINUX)
         device->bus == &pci_bus_type;
 #elif defined(NV_BSD)
@@ -1823,17 +2039,26 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
         if (bus_is_pci) {
             struct pci_dev *pdev = to_pci_dev(device);
 
+#if defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+
 #if defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_HAS_DRIVER_ARG)
             drm_aperture_remove_conflicting_pci_framebuffers(pdev, &nv_drm_driver);
 #else
             drm_aperture_remove_conflicting_pci_framebuffers(pdev, nv_drm_driver.name);
 #endif
+
+#elif defined(NV_APERTURE_REMOVE_CONFLICTING_PCI_DEVICES_PRESENT)
+            aperture_remove_conflicting_pci_devices(pdev, nv_drm_driver.name);
+#endif
+            nvKms->framebufferConsoleDisabled(nv_dev->pDevice);
         }
-        #if defined(NV_DRM_FBDEV_TTM_AVAILABLE)
+#if defined(NV_DRM_CLIENT_AVAILABLE)
+	    drm_client_setup(dev, NULL);
+#elif defined(NV_DRM_FBDEV_TTM_AVAILABLE)
         drm_fbdev_ttm_setup(dev, 32);
-        #elif defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#elif defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
         drm_fbdev_generic_setup(dev, 32);
-        #endif
+#endif
     }
 #endif /* defined(NV_DRM_FBDEV_AVAILABLE) */
 

kernel-open/nvidia-drm/nvidia-drm-fb.c

@@ -36,12 +36,15 @@
 
 static void __nv_drm_framebuffer_free(struct nv_drm_framebuffer *nv_fb)
 {
+    struct drm_framebuffer *fb = &nv_fb->base;
     uint32_t i;
 
     /* Unreference gem object */
-    for (i = 0; i < ARRAY_SIZE(nv_fb->nv_gem); i++) {
-        if (nv_fb->nv_gem[i] != NULL) {
-            nv_drm_gem_object_unreference_unlocked(nv_fb->nv_gem[i]);
+    for (i = 0; i < NVKMS_MAX_PLANES_PER_SURFACE; i++) {
+        struct drm_gem_object *gem = nv_fb_get_gem_obj(fb, i);
+        if (gem != NULL) {
+            struct nv_drm_gem_object *nv_gem = to_nv_gem_object(gem);
+            nv_drm_gem_object_unreference_unlocked(nv_gem);
         }
     }
 
@@ -69,10 +72,8 @@ static int
 nv_drm_framebuffer_create_handle(struct drm_framebuffer *fb,
                                  struct drm_file *file, unsigned int *handle)
 {
-    struct nv_drm_framebuffer *nv_fb = to_nv_framebuffer(fb);
-
     return nv_drm_gem_handle_create(file,
-                                    nv_fb->nv_gem[0],
+                                    to_nv_gem_object(nv_fb_get_gem_obj(fb, 0)),
                                     handle);
 }
 
@@ -88,6 +89,7 @@ static struct nv_drm_framebuffer *nv_drm_framebuffer_alloc(
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
     struct nv_drm_framebuffer *nv_fb;
+    struct nv_drm_gem_object *nv_gem;
     const int num_planes = nv_drm_format_num_planes(cmd->pixel_format);
     uint32_t i;
 
@@ -101,21 +103,22 @@ static struct nv_drm_framebuffer *nv_drm_framebuffer_alloc(
         return ERR_PTR(-ENOMEM);
     }
 
-    if (num_planes > ARRAY_SIZE(nv_fb->nv_gem)) {
+    if (num_planes > NVKMS_MAX_PLANES_PER_SURFACE) {
         NV_DRM_DEV_DEBUG_DRIVER(nv_dev, "Unsupported number of planes");
         goto failed;
     }
 
     for (i = 0; i < num_planes; i++) {
-        if ((nv_fb->nv_gem[i] = nv_drm_gem_object_lookup(
-                        dev,
-                        file,
-                        cmd->handles[i])) == NULL) {
+        nv_gem = nv_drm_gem_object_lookup(dev, file, cmd->handles[i]);
+
+        if (nv_gem == NULL) {
             NV_DRM_DEV_DEBUG_DRIVER(
                 nv_dev,
                 "Failed to find gem object of type nvkms memory");
             goto failed;
         }
+
+        nv_fb_set_gem_obj(&nv_fb->base, i, &nv_gem->base);
     }
 
      return nv_fb;
@@ -135,12 +138,14 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
     struct NvKmsKapiCreateSurfaceParams params = { };
+    struct nv_drm_gem_object *nv_gem;
+    struct drm_framebuffer *fb = &nv_fb->base;
     uint32_t i;
     int ret;
 
     /* Initialize the base framebuffer object and add it to drm subsystem */
 
-    ret = drm_framebuffer_init(dev, &nv_fb->base, &nv_framebuffer_funcs);
+    ret = drm_framebuffer_init(dev, fb, &nv_framebuffer_funcs);
     if (ret != 0) {
         NV_DRM_DEV_DEBUG_DRIVER(
             nv_dev,
@@ -148,23 +153,32 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
         return ret;
     }
 
-    for (i = 0; i < ARRAY_SIZE(nv_fb->nv_gem); i++) {
-        if (nv_fb->nv_gem[i] != NULL) {
-            if (!nvKms->isMemoryValidForDisplay(nv_dev->pDevice,
-                                                nv_fb->nv_gem[i]->pMemory)) {
-                NV_DRM_DEV_LOG_INFO(
-                        nv_dev,
-                        "Framebuffer memory not appropriate for scanout");
-                goto fail;
-            }
+    for (i = 0; i < NVKMS_MAX_PLANES_PER_SURFACE; i++) {
+        struct drm_gem_object *gem = nv_fb_get_gem_obj(fb, i);
+        if (gem != NULL) {
+            nv_gem = to_nv_gem_object(gem);
 
-            params.planes[i].memory = nv_fb->nv_gem[i]->pMemory;
-            params.planes[i].offset = nv_fb->base.offsets[i];
-            params.planes[i].pitch = nv_fb->base.pitches[i];
+            params.planes[i].memory = nv_gem->pMemory;
+            params.planes[i].offset = fb->offsets[i];
+            params.planes[i].pitch = fb->pitches[i];
+
+            /*
+             * XXX Use drm_framebuffer_funcs.dirty and
+             * drm_fb_helper_funcs.fb_dirty instead
+             *
+             * Currently using noDisplayCaching when registering surfaces with
+             * NVKMS that are using memory allocated through the DRM
+             * Dumb-Buffers API. This prevents Display Idle Frame Rate from
+             * kicking in and preventing CPU updates to the surface memory from
+             * not being reflected on the display. Ideally, DIFR would be
+             * dynamically disabled whenever a user of the memory blits to the
+             * frontbuffer. DRM provides the needed callbacks to achieve this.
+             */
+            params.noDisplayCaching |= !!nv_gem->is_drm_dumb;
         }
     }
-    params.height = nv_fb->base.height;
-    params.width = nv_fb->base.width;
+    params.height = fb->height;
+    params.width = fb->width;
     params.format = format;
 
     if (have_modifier) {
@@ -199,13 +213,16 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
     return 0;
 
 fail:
-    drm_framebuffer_cleanup(&nv_fb->base);
+    drm_framebuffer_cleanup(fb);
     return -EINVAL;
 }
 
 struct drm_framebuffer *nv_drm_internal_framebuffer_create(
     struct drm_device *dev,
     struct drm_file *file,
+#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO)
+    const struct drm_format_info *info,
+#endif
     struct drm_mode_fb_cmd2 *cmd)
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
@@ -259,6 +276,9 @@ struct drm_framebuffer *nv_drm_internal_framebuffer_create(
         dev,
         #endif
         &nv_fb->base,
+        #if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO)
+        info,
+        #endif 
         cmd);
 
     /*

kernel-open/nvidia-drm/nvidia-drm-fb.h

@@ -41,8 +41,10 @@
 struct nv_drm_framebuffer {
     struct NvKmsKapiSurface *pSurface;
 
-    struct nv_drm_gem_object*
-        nv_gem[NVKMS_MAX_PLANES_PER_SURFACE];
+#if !defined(NV_DRM_FRAMEBUFFER_OBJ_PRESENT)
+    struct drm_gem_object*
+        obj[NVKMS_MAX_PLANES_PER_SURFACE];
+#endif
 
     struct drm_framebuffer base;
 };
@@ -56,9 +58,35 @@ static inline struct nv_drm_framebuffer *to_nv_framebuffer(
     return container_of(fb, struct nv_drm_framebuffer, base);
 }
 
+static inline struct drm_gem_object *nv_fb_get_gem_obj(
+    struct drm_framebuffer *fb,
+    uint32_t plane)
+{
+#if defined(NV_DRM_FRAMEBUFFER_OBJ_PRESENT)
+    return fb->obj[plane];
+#else
+    return to_nv_framebuffer(fb)->obj[plane];
+#endif
+}
+
+static inline void nv_fb_set_gem_obj(
+    struct drm_framebuffer *fb,
+    uint32_t plane,
+    struct drm_gem_object *obj)
+{
+#if defined(NV_DRM_FRAMEBUFFER_OBJ_PRESENT)
+    fb->obj[plane] = obj;
+#else
+    to_nv_framebuffer(fb)->obj[plane] = obj;
+#endif
+}
+
 struct drm_framebuffer *nv_drm_internal_framebuffer_create(
     struct drm_device *dev,
     struct drm_file *file,
+#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO)
+    const struct drm_format_info *info,
+#endif
     struct drm_mode_fb_cmd2 *cmd);
 
 #endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */

kernel-open/nvidia-drm/nvidia-drm-fence.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2016-2025, NVIDIA CORPORATION. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -293,14 +293,12 @@ __nv_drm_prime_fence_context_new(
      * to check a return value.
      */
 
-    *nv_prime_fence_context = (struct nv_drm_prime_fence_context) {
-        .base.ops = &nv_drm_prime_fence_context_ops,
-        .base.nv_dev = nv_dev,
-        .base.context = nv_dma_fence_context_alloc(1),
-        .base.fenceSemIndex = p->index,
-        .pSemSurface = pSemSurface,
-        .pLinearAddress = pLinearAddress,
-    };
+    nv_prime_fence_context->base.ops = &nv_drm_prime_fence_context_ops;
+    nv_prime_fence_context->base.nv_dev = nv_dev;
+    nv_prime_fence_context->base.context = nv_dma_fence_context_alloc(1);
+    nv_prime_fence_context->base.fenceSemIndex = p->index;
+    nv_prime_fence_context->pSemSurface = pSemSurface;
+    nv_prime_fence_context->pLinearAddress = pLinearAddress;
 
     INIT_LIST_HEAD(&nv_prime_fence_context->pending);
 
@@ -1134,7 +1132,7 @@ static void __nv_drm_semsurf_fence_ctx_destroy(
      */
     nv_drm_workthread_shutdown(&ctx->worker);
 
-    nv_drm_del_timer_sync(&ctx->timer);
+    nv_timer_delete_sync(&ctx->timer.kernel_timer);
 
     /*
      * The semaphore surface could still be sending callbacks, so it is still
@@ -1271,18 +1269,16 @@ __nv_drm_semsurf_fence_ctx_new(
      * to check a return value.
      */
 
-    *ctx = (struct nv_drm_semsurf_fence_ctx) {
-        .base.ops = &nv_drm_semsurf_fence_ctx_ops,
-        .base.nv_dev = nv_dev,
-        .base.context = nv_dma_fence_context_alloc(1),
-        .base.fenceSemIndex = p->index,
-        .pSemSurface = pSemSurface,
-        .pSemMapping.pVoid = semMapping,
-        .pMaxSubmittedMapping = (volatile NvU64 *)maxSubmittedMapping,
-        .callback.local = NULL,
-        .callback.nvKms = NULL,
-        .current_wait_value = 0,
-    };
+    ctx->base.ops = &nv_drm_semsurf_fence_ctx_ops;
+    ctx->base.nv_dev = nv_dev;
+    ctx->base.context = nv_dma_fence_context_alloc(1);
+    ctx->base.fenceSemIndex = p->index;
+    ctx->pSemSurface = pSemSurface;
+    ctx->pSemMapping.pVoid = semMapping;
+    ctx->pMaxSubmittedMapping = (volatile NvU64 *)maxSubmittedMapping;
+    ctx->callback.local = NULL;
+    ctx->callback.nvKms = NULL;
+    ctx->current_wait_value = 0;
 
     spin_lock_init(&ctx->lock);
     INIT_LIST_HEAD(&ctx->pending_fences);

kernel-open/nvidia-drm/nvidia-drm-gem-nvkms-memory.c

@@ -71,9 +71,20 @@ static void __nv_drm_gem_nvkms_memory_free(struct nv_drm_gem_object *nv_gem)
     nv_drm_free(nv_nvkms_memory);
 }
 
+static int __nv_drm_gem_nvkms_map(
+    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory);
+
 static int __nv_drm_gem_nvkms_mmap(struct nv_drm_gem_object *nv_gem,
                                    struct vm_area_struct *vma)
 {
+    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
+        to_nv_nvkms_memory(nv_gem);
+
+    int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
+    if (ret) {
+       return ret;
+    }
+
     return drm_gem_mmap_obj(&nv_gem->base,
                 drm_vma_node_size(&nv_gem->base.vma_node) << PAGE_SHIFT, vma);
 }
@@ -146,11 +157,18 @@ static struct drm_gem_object *__nv_drm_gem_nvkms_prime_dup(
 static int __nv_drm_gem_nvkms_map(
     struct nv_drm_gem_nvkms_memory *nv_nvkms_memory)
 {
+    int ret = 0;
     struct nv_drm_device *nv_dev = nv_nvkms_memory->base.nv_dev;
     struct NvKmsKapiMemory *pMemory = nv_nvkms_memory->base.pMemory;
 
-    if (!nv_dev->hasVideoMemory) {
-        return 0;
+    mutex_lock(&nv_nvkms_memory->map_lock);
+
+    if (nv_nvkms_memory->physically_mapped) {
+        goto done;
+    }
+
+    if (!nvKms->isVidmem(pMemory)) {
+        goto done;
     }
 
     if (!nvKms->mapMemory(nv_dev->pDevice,
@@ -161,7 +179,8 @@ static int __nv_drm_gem_nvkms_map(
             nv_dev,
             "Failed to map NvKmsKapiMemory 0x%p",
             pMemory);
-        return -ENOMEM;
+        ret = -ENOMEM;
+        goto done;
     }
 
     nv_nvkms_memory->pWriteCombinedIORemapAddress = ioremap_wc(
@@ -177,7 +196,9 @@ static int __nv_drm_gem_nvkms_map(
 
     nv_nvkms_memory->physically_mapped = true;
 
-    return 0;
+done:
+    mutex_unlock(&nv_nvkms_memory->map_lock);
+    return ret;
 }
 
 static void *__nv_drm_gem_nvkms_prime_vmap(
@@ -186,14 +207,40 @@ static void *__nv_drm_gem_nvkms_prime_vmap(
     struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
         to_nv_nvkms_memory(nv_gem);
 
-    if (!nv_nvkms_memory->physically_mapped) {
-        int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
-        if (ret) {
-           return ERR_PTR(ret);
-        }
+    int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
+    if (ret) {
+       return ERR_PTR(ret);
     }
 
-    return nv_nvkms_memory->pWriteCombinedIORemapAddress;
+    if (nv_nvkms_memory->physically_mapped) {
+        return nv_nvkms_memory->pWriteCombinedIORemapAddress;
+    }
+
+    /*
+     * If this buffer isn't physically mapped, it might be backed by struct
+     * pages. Use vmap in that case. Do a noncached mapping for system memory
+     * as display is non io-coherent device in case of Tegra.
+     */
+    if (nv_nvkms_memory->pages_count > 0) {
+         return nv_drm_vmap(nv_nvkms_memory->pages,
+                            nv_nvkms_memory->pages_count,
+                            false);
+    }
+
+    return ERR_PTR(-ENOMEM);
+}
+
+static void __nv_drm_gem_nvkms_prime_vunmap(
+    struct nv_drm_gem_object *nv_gem,
+    void *address)
+{
+    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
+        to_nv_nvkms_memory(nv_gem);
+
+    if (!nv_nvkms_memory->physically_mapped &&
+        nv_nvkms_memory->pages_count > 0) {
+        nv_drm_vunmap(address);
+    }
 }
 
 static int __nv_drm_gem_map_nvkms_memory_offset(
@@ -201,17 +248,7 @@ static int __nv_drm_gem_map_nvkms_memory_offset(
     struct nv_drm_gem_object *nv_gem,
     uint64_t *offset)
 {
-    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
-        to_nv_nvkms_memory(nv_gem);
-
-    if (!nv_nvkms_memory->physically_mapped) {
-        int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
-        if (ret) {
-           return ret;
-        }
-    }
-
-    return nv_drm_gem_create_mmap_offset(&nv_nvkms_memory->base, offset);
+    return nv_drm_gem_create_mmap_offset(nv_gem, offset);
 }
 
 static struct sg_table *__nv_drm_gem_nvkms_memory_prime_get_sg_table(
@@ -223,7 +260,7 @@ static struct sg_table *__nv_drm_gem_nvkms_memory_prime_get_sg_table(
     struct sg_table *sg_table;
 
     if (nv_nvkms_memory->pages_count == 0) {
-        NV_DRM_DEV_LOG_ERR(
+        NV_DRM_DEV_DEBUG_DRIVER(
                 nv_dev,
                 "Cannot create sg_table for NvKmsKapiMemory 0x%p",
                 nv_gem->pMemory);
@@ -241,6 +278,7 @@ const struct nv_drm_gem_object_funcs nv_gem_nvkms_memory_ops = {
     .free = __nv_drm_gem_nvkms_memory_free,
     .prime_dup = __nv_drm_gem_nvkms_prime_dup,
     .prime_vmap = __nv_drm_gem_nvkms_prime_vmap,
+    .prime_vunmap = __nv_drm_gem_nvkms_prime_vunmap,
     .mmap = __nv_drm_gem_nvkms_mmap,
     .handle_vma_fault = __nv_drm_gem_nvkms_handle_vma_fault,
     .create_mmap_offset = __nv_drm_gem_map_nvkms_memory_offset,
@@ -265,6 +303,7 @@ static int __nv_drm_nvkms_gem_obj_init(
         return -EINVAL;
     }
 
+    mutex_init(&nv_nvkms_memory->map_lock);
     nv_nvkms_memory->pPhysicalAddress = NULL;
     nv_nvkms_memory->pWriteCombinedIORemapAddress = NULL;
     nv_nvkms_memory->physically_mapped = false;
@@ -273,7 +312,7 @@ static int __nv_drm_nvkms_gem_obj_init(
                                pMemory,
                                &pages,
                                &numPages) &&
-        !nv_dev->hasVideoMemory) {
+        !nvKms->isVidmem(pMemory)) {
         /* GetMemoryPages may fail for vidmem allocations,
          * but it should not fail for sysmem allocations. */
         NV_DRM_DEV_LOG_ERR(nv_dev,
@@ -346,6 +385,8 @@ int nv_drm_dumb_create(
         goto nvkms_gem_obj_init_failed;
     }
 
+    nv_nvkms_memory->base.is_drm_dumb = true;
+
     /* Always map dumb buffer memory up front.  Clients are only expected
      * to use dumb buffers for software rendering, so they're not much use
      * without a CPU mapping.
@@ -551,14 +592,12 @@ static struct drm_gem_object *__nv_drm_gem_nvkms_prime_dup(
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
     const struct nv_drm_device *nv_dev_src;
-    const struct nv_drm_gem_nvkms_memory *nv_nvkms_memory_src;
     struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
     struct NvKmsKapiMemory *pMemory;
 
     BUG_ON(nv_gem_src == NULL || nv_gem_src->ops != &nv_gem_nvkms_memory_ops);
 
     nv_dev_src = to_nv_device(nv_gem_src->base.dev);
-    nv_nvkms_memory_src = to_nv_nvkms_memory_const(nv_gem_src);
 
     if ((nv_nvkms_memory =
             nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {

kernel-open/nvidia-drm/nvidia-drm-gem-nvkms-memory.h

@@ -32,8 +32,15 @@
 struct nv_drm_gem_nvkms_memory {
     struct nv_drm_gem_object base;
 
+    /*
+     * Lock to protect concurrent writes to physically_mapped, pPhysicalAddress,
+     * and pWriteCombinedIORemapAddress.
+     *
+     * __nv_drm_gem_nvkms_map(), the sole writer, is structured such that
+     * readers are not required to hold the lock.
+     */
+    struct mutex map_lock;
     bool physically_mapped;
-
     void *pPhysicalAddress;
     void *pWriteCombinedIORemapAddress;
 

kernel-open/nvidia-drm/nvidia-drm-gem-user-memory.c

@@ -36,6 +36,10 @@
 #include "linux/mm.h"
 #include "nv-mm.h"
 
+#if defined(NV_LINUX_PFN_T_H_PRESENT)
+#include "linux/pfn_t.h"
+#endif
+
 #if defined(NV_BSD)
 #include <vm/vm_pageout.h>
 #endif
@@ -68,7 +72,8 @@ static void *__nv_drm_gem_user_memory_prime_vmap(
     struct nv_drm_gem_user_memory *nv_user_memory = to_nv_user_memory(nv_gem);
 
     return nv_drm_vmap(nv_user_memory->pages,
-                           nv_user_memory->pages_count);
+                           nv_user_memory->pages_count,
+                           true);
 }
 
 static void __nv_drm_gem_user_memory_prime_vunmap(
@@ -103,6 +108,37 @@ static int __nv_drm_gem_user_memory_mmap(struct nv_drm_gem_object *nv_gem,
     return 0;
 }
 
+#if defined(NV_LINUX) && !defined(NV_VMF_INSERT_MIXED_PRESENT)
+static vm_fault_t __nv_vm_insert_mixed_helper(
+    struct vm_area_struct *vma,
+    unsigned long address,
+    unsigned long pfn)
+{
+    int ret;
+
+#if defined(NV_PFN_TO_PFN_T_PRESENT)
+    ret = vm_insert_mixed(vma, address, pfn_to_pfn_t(pfn));
+#else
+    ret = vm_insert_mixed(vma, address, pfn);
+#endif
+
+    switch (ret) {
+        case 0:
+        case -EBUSY:
+            /*
+             * EBUSY indicates that another thread already handled
+             * the faulted range.
+             */
+            return VM_FAULT_NOPAGE;
+        case -ENOMEM:
+            return VM_FAULT_OOM;
+        default:
+            WARN_ONCE(1, "Unhandled error in %s: %d\n", __FUNCTION__, ret);
+            return VM_FAULT_SIGBUS;
+    }
+}
+#endif
+
 static vm_fault_t __nv_drm_gem_user_memory_handle_vma_fault(
     struct nv_drm_gem_object *nv_gem,
     struct vm_area_struct *vma,
@@ -112,36 +148,25 @@ static vm_fault_t __nv_drm_gem_user_memory_handle_vma_fault(
     unsigned long address = nv_page_fault_va(vmf);
     struct drm_gem_object *gem = vma->vm_private_data;
     unsigned long page_offset;
-    vm_fault_t ret;
+    unsigned long pfn;
 
     page_offset = vmf->pgoff - drm_vma_node_start(&gem->vma_node);
-
     BUG_ON(page_offset >= nv_user_memory->pages_count);
+    pfn = page_to_pfn(nv_user_memory->pages[page_offset]);
 
 #if !defined(NV_LINUX)
-    ret = vmf_insert_pfn(vma, address, page_to_pfn(nv_user_memory->pages[page_offset]));
-#else /* !defined(NV_LINUX) */
-    ret = vm_insert_page(vma, address, nv_user_memory->pages[page_offset]);
-    switch (ret) {
-        case 0:
-        case -EBUSY:
-            /*
-             * EBUSY indicates that another thread already handled
-             * the faulted range.
-             */
-            ret = VM_FAULT_NOPAGE;
-            break;
-        case -ENOMEM:
-            ret = VM_FAULT_OOM;
-            break;
-        default:
-            WARN_ONCE(1, "Unhandled error in %s: %d\n", __FUNCTION__, ret);
-            ret = VM_FAULT_SIGBUS;
-            break;
-    }
-#endif /* !defined(NV_LINUX) */
+    return vmf_insert_pfn(vma, address, pfn);
+#elif defined(NV_VMF_INSERT_MIXED_PRESENT)
 
-    return ret;
+#if defined(NV_LINUX_PFN_T_H_PRESENT)
+    return vmf_insert_mixed(vma, address, pfn_to_pfn_t(pfn));
+#else
+    return vmf_insert_mixed(vma, address, pfn);
+#endif
+
+#else
+    return __nv_vm_insert_mixed_helper(vma, address, pfn);
+#endif
 }
 
 static int __nv_drm_gem_user_create_mmap_offset(

kernel-open/nvidia-drm/nvidia-drm-gem.c

@@ -144,6 +144,12 @@ void nv_drm_gem_object_init(struct nv_drm_device *nv_dev,
 #endif
 
     drm_gem_private_object_init(dev, &nv_gem->base, size);
+
+    /* Create mmap offset early for drm_gem_prime_mmap(), if possible. */
+    if (nv_gem->ops->create_mmap_offset) {
+        uint64_t offset;
+        nv_gem->ops->create_mmap_offset(nv_dev, nv_gem, &offset);
+    }
 }
 
 struct drm_gem_object *nv_drm_gem_prime_import(struct drm_device *dev,
@@ -166,8 +172,11 @@ struct drm_gem_object *nv_drm_gem_prime_import(struct drm_device *dev,
              */
             gem_dst = nv_gem_src->ops->prime_dup(dev, nv_gem_src);
 
-            if (gem_dst)
-                return gem_dst;
+            if (gem_dst == NULL) {
+                return ERR_PTR(-ENOTSUPP);
+            }
+
+            return gem_dst;
         }
     }
 #endif /* NV_DMA_BUF_OWNER_PRESENT */
@@ -232,6 +241,7 @@ int nv_drm_gem_map_offset_ioctl(struct drm_device *dev,
         return -EINVAL;
     }
 
+    /* mmap offset creation is idempotent, fetch it by creating it again. */
     if (nv_gem->ops->create_mmap_offset) {
         ret = nv_gem->ops->create_mmap_offset(nv_dev, nv_gem, &params->offset);
     } else {

kernel-open/nvidia-drm/nvidia-drm-gem.h

@@ -73,6 +73,8 @@ struct nv_drm_gem_object {
 
     struct NvKmsKapiMemory *pMemory;
 
+    bool is_drm_dumb;
+
 #if defined(NV_DRM_FENCE_AVAILABLE) && !defined(NV_DRM_GEM_OBJECT_HAS_RESV)
     nv_dma_resv_t  resv;
 #endif

kernel-open/nvidia-drm/nvidia-drm-helper.c

@@ -45,8 +45,7 @@
 
 /*
  * The inclusion of drm_framebuffer.h was removed from drm_crtc.h by commit
- * 720cf96d8fecde29b72e1101f8a567a0ce99594f ("drm: Drop drm_framebuffer.h from
- * drm_crtc.h") in linux-next, expected in v5.19-rc7.
+ * 720cf96d8fec ("drm: Drop drm_framebuffer.h from drm_crtc.h") in v6.0.
  *
  * We only need drm_framebuffer.h for drm_framebuffer_put(), and it is always
  * present (v4.9+) when drm_framebuffer_{put,get}() is present (v4.12+), so it

kernel-open/nvidia-drm/nvidia-drm-helper.h

@@ -40,8 +40,13 @@
 #include <drm/drm_blend.h>
 #endif
 
-#if defined(NV_DRM_ROTATION_AVAILABLE)
-/* For DRM_MODE_ROTATE_* and DRM_MODE_REFLECT_* */
+#if defined(NV_DRM_ROTATION_AVAILABLE) || \
+    defined(NV_DRM_COLOR_CTM_3X4_PRESENT) || \
+    defined(NV_DRM_COLOR_LUT_PRESENT)
+/*
+ * For DRM_MODE_ROTATE_*, DRM_MODE_REFLECT_*, struct drm_color_ctm_3x4, and
+ * struct drm_color_lut.
+ */
 #include <uapi/drm/drm_mode.h>
 #endif
 
@@ -358,6 +363,24 @@ static inline void nv_drm_connector_put(struct drm_connector *connector)
 #endif
 }
 
+static inline void nv_drm_property_blob_put(struct drm_property_blob *blob)
+{
+#if defined(NV_DRM_PROPERTY_BLOB_PUT_PRESENT)
+    drm_property_blob_put(blob);
+#else
+    drm_property_unreference_blob(blob);
+#endif
+}
+
+static inline void nv_drm_property_blob_get(struct drm_property_blob *blob)
+{
+#if defined(NV_DRM_PROPERTY_BLOB_PUT_PRESENT)
+    drm_property_blob_get(blob);
+#else
+    drm_property_reference_blob(blob);
+#endif
+}
+
 static inline struct drm_crtc *
 nv_drm_crtc_find(struct drm_device *dev, struct drm_file *filep, uint32_t id)
 {
@@ -613,8 +636,8 @@ static inline int nv_drm_format_num_planes(uint32_t format)
 #endif /* defined(NV_DRM_FORMAT_MODIFIERS_PRESENT) */
 
 /*
- * DRM_UNLOCKED was removed with linux-next commit 2798ffcc1d6a ("drm: Remove
- * locking for legacy ioctls and DRM_UNLOCKED"), but it was previously made
+ * DRM_UNLOCKED was removed with commit 2798ffcc1d6a ("drm: Remove locking for
+ * legacy ioctls and DRM_UNLOCKED") in v6.8, but it was previously made
  * implicit for all non-legacy DRM driver IOCTLs since Linux v4.10 commit
  * fa5386459f06 "drm: Used DRM_LEGACY for all legacy functions" (Linux v4.4
  * commit ea487835e887 "drm: Enforce unlocked ioctl operation for kms driver
@@ -625,6 +648,31 @@ static inline int nv_drm_format_num_planes(uint32_t format)
 #define DRM_UNLOCKED 0
 #endif
 
+/*
+ * struct drm_color_ctm_3x4 was added by commit 6872a189be50 ("drm/amd/display:
+ * Add 3x4 CTM support for plane CTM") in v6.8. For backwards compatibility,
+ * define it when not present.
+ */
+#if !defined(NV_DRM_COLOR_CTM_3X4_PRESENT)
+struct drm_color_ctm_3x4 {
+    __u64 matrix[12];
+};
+#endif
+
+/*
+ * struct drm_color_lut was added by commit 5488dc16fde7 ("drm: introduce pipe
+ * color correction properties") in v4.6. For backwards compatibility, define it
+ * when not present.
+ */
+#if !defined(NV_DRM_COLOR_LUT_PRESENT)
+struct drm_color_lut {
+    __u16 red;
+    __u16 green;
+    __u16 blue;
+    __u16 reserved;
+};
+#endif
+
 /*
  * drm_vma_offset_exact_lookup_locked() were added
  * by kernel commit 2225cfe46bcc which was Signed-off-by:

kernel-open/nvidia-drm/nvidia-drm-ioctl.h

@@ -52,6 +52,7 @@
 #define DRM_NVIDIA_SEMSURF_FENCE_CREATE             0x15
 #define DRM_NVIDIA_SEMSURF_FENCE_WAIT               0x16
 #define DRM_NVIDIA_SEMSURF_FENCE_ATTACH             0x17
+#define DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID           0x18
 
 #define DRM_IOCTL_NVIDIA_GEM_IMPORT_NVKMS_MEMORY                           \
     DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GEM_IMPORT_NVKMS_MEMORY),      \
@@ -157,6 +158,11 @@
               DRM_NVIDIA_SEMSURF_FENCE_ATTACH),                         \
               struct drm_nvidia_semsurf_fence_attach_params)
 
+#define DRM_IOCTL_NVIDIA_GET_DRM_FILE_UNIQUE_ID                         \
+    DRM_IOWR((DRM_COMMAND_BASE +                                        \
+              DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID),                       \
+              struct drm_nvidia_get_drm_file_unique_id_params)
+
 struct drm_nvidia_gem_import_nvkms_memory_params {
     uint64_t mem_size;           /* IN */
 
@@ -385,4 +391,8 @@ struct drm_nvidia_semsurf_fence_attach_params {
     uint64_t wait_value;            /* IN Semaphore value to reach before signal */
 };
 
+struct drm_nvidia_get_drm_file_unique_id_params {
+    uint64_t id;                    /* OUT Unique ID of the DRM file */
+};
+
 #endif /* _UAPI_NVIDIA_DRM_IOCTL_H_ */

kernel-open/nvidia-drm/nvidia-drm-linux.c

@@ -37,7 +37,7 @@ module_param_named(modeset, nv_drm_modeset_module_param, bool, 0400);
 #if defined(NV_DRM_FBDEV_AVAILABLE)
 MODULE_PARM_DESC(
     fbdev,
-    "Create a framebuffer device (1 = enable, 0 = disable (default)) (EXPERIMENTAL)");
+    "Create a framebuffer device (1 = enable (default), 0 = disable)");
 module_param_named(fbdev, nv_drm_fbdev_module_param, bool, 0400);
 #endif
 

kernel-open/nvidia-drm/nvidia-drm-modeset.c

@@ -42,6 +42,16 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+#include <linux/nvhost.h>
+#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)            
+#include <linux/host1x-next.h>
+#endif
+
+#if defined(NV_DRM_FENCE_AVAILABLE)
+#include "nvidia-dma-fence-helper.h"
+#endif
+
 struct nv_drm_atomic_state {
     struct NvKmsKapiRequestedModeSetConfig config;
     struct drm_atomic_state base;
@@ -146,6 +156,165 @@ static int __nv_drm_put_back_post_fence_fd(
     return ret;
 }
 
+#if defined(NV_DRM_FENCE_AVAILABLE)
+struct nv_drm_plane_fence_cb_data {
+    nv_dma_fence_cb_t dma_fence_cb;
+    struct nv_drm_device *nv_dev;
+    NvU32 semaphore_index;
+};
+
+static void
+__nv_drm_plane_fence_cb(
+    nv_dma_fence_t *fence,
+    nv_dma_fence_cb_t *cb_data
+)
+{
+    struct nv_drm_plane_fence_cb_data *fence_data =
+        container_of(cb_data, typeof(*fence_data), dma_fence_cb);
+    struct nv_drm_device *nv_dev = fence_data->nv_dev;
+
+    nv_dma_fence_put(fence);
+    nvKms->signalDisplaySemaphore(nv_dev->pDevice, fence_data->semaphore_index);
+    nv_drm_free(fence_data);
+}
+
+static int __nv_drm_convert_in_fences(
+    struct nv_drm_device *nv_dev,
+    struct drm_atomic_state *state,
+    struct drm_crtc *crtc,
+    struct drm_crtc_state *crtc_state)
+{
+    struct drm_plane *plane = NULL;
+    struct drm_plane_state *plane_state = NULL;
+    struct nv_drm_plane *nv_plane = NULL;
+    struct NvKmsKapiLayerRequestedConfig *plane_req_config = NULL;
+    struct NvKmsKapiHeadRequestedConfig *head_req_config =
+        &to_nv_crtc_state(crtc_state)->req_config;
+    struct nv_drm_plane_fence_cb_data *fence_data;
+    uint32_t semaphore_index;
+    uint32_t idx_count;
+    int ret, i;
+
+    if (!crtc_state->active) {
+        return 0;
+    }
+
+    nv_drm_for_each_new_plane_in_state(state, plane, plane_state, i) {
+        if ((plane->type == DRM_PLANE_TYPE_CURSOR) ||
+            (plane_state->crtc != crtc) ||
+            (plane_state->fence == NULL)) {
+            continue;
+        }
+
+        nv_plane = to_nv_plane(plane);
+        plane_req_config =
+            &head_req_config->layerRequestedConfig[nv_plane->layer_idx];
+
+        if (nv_dev->supportsSyncpts) {
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+#if defined(NV_NVHOST_DMA_FENCE_UNPACK_PRESENT)
+            int ret =
+                nvhost_dma_fence_unpack(
+                    plane_state->fence,
+                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
+                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
+            if (ret == 0) {
+                plane_req_config->config.syncParams.preSyncptSpecified = true;
+                continue;
+            }
+#endif
+#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)
+            int ret =
+                host1x_fence_extract(
+                    plane_state->fence,
+                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
+                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
+            if (ret == 0) {
+                plane_req_config->config.syncParams.preSyncptSpecified = true;
+                continue;
+            }
+#endif
+        }
+
+        /*
+         * Syncpt extraction failed, or syncpts are not supported.
+         * Use general DRM fence support with semaphores instead.
+         */
+        if (plane_req_config->config.syncParams.postSyncptRequested) {
+            // Can't mix Syncpts and semaphores in a given request.
+            return -EINVAL;
+        }
+
+        for (idx_count = 0; idx_count < nv_dev->display_semaphores.count; idx_count++) {
+            semaphore_index = nv_drm_next_display_semaphore(nv_dev);
+            if (nvKms->tryInitDisplaySemaphore(nv_dev->pDevice, semaphore_index)) {
+                break;
+            }
+        }
+
+        if (idx_count == nv_dev->display_semaphores.count) {
+            NV_DRM_DEV_LOG_ERR(
+                nv_dev,
+                "Failed to initialize semaphore for plane fence");
+            /*
+             * This should only happen if the semaphore pool was somehow
+             * exhausted. Waiting a bit and retrying may help in that case.
+             */
+            return -EAGAIN;
+        }
+
+        plane_req_config->config.syncParams.semaphoreSpecified = true;
+        plane_req_config->config.syncParams.u.semaphore.index = semaphore_index;
+
+        fence_data = nv_drm_calloc(1, sizeof(*fence_data));
+
+        if (!fence_data) {
+            NV_DRM_DEV_LOG_ERR(
+                nv_dev,
+                "Failed to allocate callback data for plane fence");
+            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
+            return -ENOMEM;
+        }
+
+        fence_data->nv_dev = nv_dev;
+        fence_data->semaphore_index = semaphore_index;
+
+        ret = nv_dma_fence_add_callback(plane_state->fence,
+                                        &fence_data->dma_fence_cb,
+                                        __nv_drm_plane_fence_cb);
+
+        switch (ret) {
+        case -ENOENT:
+            /* The fence is already signaled */
+            __nv_drm_plane_fence_cb(plane_state->fence,
+                                    &fence_data->dma_fence_cb);
+#if defined(fallthrough)
+            fallthrough;
+#else
+            /* Fallthrough */
+#endif
+        case 0:
+            /*
+             * The plane state's fence reference has either been consumed or
+             * belongs to the outstanding callback now.
+             */
+            plane_state->fence = NULL;
+            break;
+        default:
+            NV_DRM_DEV_LOG_ERR(
+                nv_dev,
+                "Failed plane fence callback registration");
+            /* Fence callback registration failed */
+            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
+            nv_drm_free(fence_data);
+            return ret;
+        }
+    }
+
+    return 0;
+}
+#endif /* defined(NV_DRM_FENCE_AVAILABLE) */
+
 static int __nv_drm_get_syncpt_data(
     struct nv_drm_device *nv_dev,
     struct drm_crtc *crtc,
@@ -245,6 +414,31 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
         return -EINVAL;
     }
 
+#if defined(NV_DRM_FRAMEBUFFER_OBJ_PRESENT)
+    if (commit) {
+        /*
+         * This function does what is necessary to prepare the framebuffers
+         * attached to each new plane in the state for scan out, mostly by
+         * calling back into driver callbacks the NVIDIA driver does not
+         * provide. The end result is that all it does on the NVIDIA driver
+         * is populate the plane state's dma fence pointers with any implicit
+         * sync fences attached to the GEM objects associated with those planes
+         * in the new state, prefering explicit sync fences when appropriate.
+         * This must be done prior to converting the per-plane fences to
+         * semaphore waits below.
+         *
+         * Note this only works when the drm_framebuffer:obj[] field is present
+         * and populated, so skip calling this function on kernels where that
+         * field is not present.
+         */
+        ret = drm_atomic_helper_prepare_planes(dev, state);
+
+        if (ret) {
+            return ret;
+        }
+    }
+#endif /* defined(NV_DRM_FRAMEBUFFER_OBJ_PRESENT) */
+
     memset(requested_config, 0, sizeof(*requested_config));
 
     /* Loop over affected crtcs and construct NvKmsKapiRequestedModeSetConfig */
@@ -258,11 +452,6 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
                                commit ? crtc->state : crtc_state;
         struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
 
-        requested_config->headRequestedConfig[nv_crtc->head] =
-            to_nv_crtc_state(new_crtc_state)->req_config;
-
-        requested_config->headsMask |= 1 << nv_crtc->head;
-
         if (commit) {
             struct drm_crtc_state *old_crtc_state = crtc_state;
             struct nv_drm_crtc_state *nv_new_crtc_state =
@@ -282,7 +471,27 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
 
                 nv_new_crtc_state->nv_flip = NULL;
             }
+
+#if defined(NV_DRM_FENCE_AVAILABLE)
+            ret = __nv_drm_convert_in_fences(nv_dev,
+                                             state,
+                                             crtc,
+                                             new_crtc_state);
+
+            if (ret != 0) {
+                return ret;
+            }
+#endif /* defined(NV_DRM_FENCE_AVAILABLE) */
         }
+
+        /*
+         * Do this deep copy after calling __nv_drm_convert_in_fences,
+         * which modifies the new CRTC state's req_config member
+         */
+        requested_config->headRequestedConfig[nv_crtc->head] =
+            to_nv_crtc_state(new_crtc_state)->req_config;
+
+        requested_config->headsMask |= 1 << nv_crtc->head;
     }
 
     if (commit && nvKms->systemInfo.bAllowWriteCombining) {
@@ -313,6 +522,10 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
         }
     }
 
+    if (commit && nv_dev->requiresVrrSemaphores && reply_config.vrrFlip) {
+        nvKms->signalVrrSemaphore(nv_dev->pDevice, reply_config.vrrSemaphoreIndex);
+    }
+
     return 0;
 }
 
@@ -464,6 +677,33 @@ int nv_drm_atomic_commit(struct drm_device *dev,
                     "Flip event timeout on head %u", nv_crtc->head);
             }
         }
+
+#if defined(NV_DRM_COLOR_MGMT_AVAILABLE)
+        /*
+         * If the legacy LUT needs to be updated, ensure that the previous LUT
+         * update is complete first.
+         */
+        if (crtc_state->color_mgmt_changed) {
+            NvBool complete = nvKms->checkLutNotifier(nv_dev->pDevice,
+                                                      nv_crtc->head,
+                                                      !nonblock /* waitForCompletion */);
+
+            /* If checking the LUT notifier failed, assume no LUT notifier is set. */
+            if (!complete) {
+                if (nonblock) {
+                    return -EBUSY;
+                } else {
+                    /*
+                     * checkLutNotifier should wait on the notifier in this
+                     * case, so we should only get here if the wait timed out.
+                     */
+                    NV_DRM_DEV_LOG_ERR(
+                        nv_dev,
+                        "LUT notifier timeout on head %u", nv_crtc->head);
+                }
+            }
+        }
+#endif
     }
 
 #if defined(NV_DRM_ATOMIC_HELPER_SWAP_STATE_HAS_STALL_ARG)
@@ -490,6 +730,13 @@ int nv_drm_atomic_commit(struct drm_device *dev,
 #else
     drm_atomic_helper_swap_state(dev, state);
 #endif
+    /*
+     * Used to update legacy modeset state pointers to support UAPIs not updated
+     * by the core atomic modeset infrastructure.
+     *
+     * Example: /sys/class/drm/<card connector>/enabled
+     */
+    drm_atomic_helper_update_legacy_modeset_state(dev, state);
 
     /*
      * nv_drm_atomic_commit_internal() must not return failure after
@@ -506,7 +753,6 @@ int nv_drm_atomic_commit(struct drm_device *dev,
 
         goto done;
     }
-    nv_dev->drmMasterChangedSinceLastAtomicCommit = NV_FALSE;
 
     nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
         struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
@@ -587,7 +833,23 @@ int nv_drm_atomic_commit(struct drm_device *dev,
                 NV_DRM_DEV_LOG_ERR(
                     nv_dev,
                     "Flip event timeout on head %u", nv_crtc->head);
+                while (!list_empty(&nv_crtc->flip_list)) {
+                    __nv_drm_handle_flip_event(nv_crtc);
+                }
             }
+
+#if defined(NV_DRM_COLOR_MGMT_AVAILABLE)
+            if (crtc_state->color_mgmt_changed) {
+                NvBool complete = nvKms->checkLutNotifier(nv_dev->pDevice,
+                                                          nv_crtc->head,
+                                                          true /* waitForCompletion */);
+                if (!complete) {
+                    NV_DRM_DEV_LOG_ERR(
+                        nv_dev,
+                        "LUT notifier timeout on head %u", nv_crtc->head);
+                }
+            }
+#endif
         }
     }
 

kernel-open/nvidia-drm/nvidia-drm-os-interface.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -42,7 +42,7 @@
 #endif
 
 bool nv_drm_modeset_module_param = false;
-bool nv_drm_fbdev_module_param = false;
+bool nv_drm_fbdev_module_param = true;
 
 void *nv_drm_calloc(size_t nmemb, size_t size)
 {
@@ -156,9 +156,15 @@ void nv_drm_unlock_user_pages(unsigned long  pages_count, struct page **pages)
 #define VM_USERMAP 0
 #endif
 
-void *nv_drm_vmap(struct page **pages, unsigned long pages_count)
+void *nv_drm_vmap(struct page **pages, unsigned long pages_count, bool cached)
 {
-    return vmap(pages, pages_count, VM_USERMAP, PAGE_KERNEL);
+    pgprot_t prot = PAGE_KERNEL;
+
+    if (!cached) {
+        prot = pgprot_noncached(PAGE_KERNEL);
+    }
+
+    return vmap(pages, pages_count, VM_USERMAP, prot);
 }
 
 void nv_drm_vunmap(void *address)
@@ -230,15 +236,6 @@ unsigned long nv_drm_timeout_from_ms(NvU64 relative_timeout_ms)
     return jiffies + msecs_to_jiffies(relative_timeout_ms);
 }
 
-bool nv_drm_del_timer_sync(nv_drm_timer *timer)
-{
-    if (del_timer_sync(&timer->kernel_timer)) {
-        return true;
-    } else {
-        return false;
-    }
-}
-
 #if defined(NV_DRM_FENCE_AVAILABLE)
 int nv_drm_create_sync_file(nv_dma_fence_t *fence)
 {

kernel-open/nvidia-drm/nvidia-drm-os-interface.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -58,16 +58,6 @@ typedef struct nv_timer nv_drm_timer;
 #error "Need to define kernel timer callback primitives for this OS"
 #endif
 
-#if defined(NV_DRM_FBDEV_GENERIC_SETUP_PRESENT) && defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
-#define NV_DRM_FBDEV_AVAILABLE
-#define NV_DRM_FBDEV_GENERIC_AVAILABLE
-#endif
-
-#if defined(NV_DRM_FBDEV_TTM_SETUP_PRESENT) && defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
-#define NV_DRM_FBDEV_AVAILABLE
-#define NV_DRM_FBDEV_TTM_AVAILABLE
-#endif
-
 struct page;
 
 /* Set to true when the atomic modeset feature is enabled. */
@@ -90,7 +80,7 @@ int nv_drm_lock_user_pages(unsigned long address,
 
 void nv_drm_unlock_user_pages(unsigned long  pages_count, struct page **pages);
 
-void *nv_drm_vmap(struct page **pages, unsigned long pages_count);
+void *nv_drm_vmap(struct page **pages, unsigned long pages_count, bool cached);
 
 void nv_drm_vunmap(void *address);
 
@@ -111,8 +101,6 @@ void nv_drm_timer_setup(nv_drm_timer *timer,
 
 void nv_drm_mod_timer(nv_drm_timer *timer, unsigned long relative_timeout_ms);
 
-bool nv_drm_del_timer_sync(nv_drm_timer *timer);
-
 unsigned long nv_drm_timer_now(void);
 
 unsigned long nv_drm_timeout_from_ms(NvU64 relative_timeout_ms);

kernel-open/nvidia-drm/nvidia-drm-priv.h

@@ -31,6 +31,10 @@
 #include <drm/drmP.h>
 #endif
 
+#if defined(NV_DRM_PRINT_H_PRESENT)
+#include <drm/drm_print.h>
+#endif
+
 #if defined(NV_DRM_DRM_DEVICE_H_PRESENT)
 #include <drm/drm_device.h>
 #endif
@@ -147,29 +151,56 @@ struct nv_drm_device {
     NvBool hasVideoMemory;
 
     NvBool supportsSyncpts;
+    NvBool requiresVrrSemaphores;
     NvBool subOwnershipGranted;
     NvBool hasFramebufferConsole;
 
-    /**
-     * @drmMasterChangedSinceLastAtomicCommit:
-     *
-     * This flag is set in nv_drm_master_set and reset after a completed atomic
-     * commit. It is used to restore or recommit state that is lost by the
-     * NvKms modeset owner change, such as the CRTC color management
-     * properties.
-     */
-    NvBool drmMasterChangedSinceLastAtomicCommit;
-
     struct drm_property *nv_out_fence_property;
     struct drm_property *nv_input_colorspace_property;
 
+    struct {
+        NvU32 count;
+        NvU32 next_index;
+    } display_semaphores;
+
 #if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
     struct drm_property *nv_hdr_output_metadata_property;
 #endif
 
+    struct drm_property *nv_plane_lms_ctm_property;
+    struct drm_property *nv_plane_lms_to_itp_ctm_property;
+    struct drm_property *nv_plane_itp_to_lms_ctm_property;
+    struct drm_property *nv_plane_blend_ctm_property;
+
+    struct drm_property *nv_plane_degamma_tf_property;
+    struct drm_property *nv_plane_degamma_lut_property;
+    struct drm_property *nv_plane_degamma_lut_size_property;
+    struct drm_property *nv_plane_degamma_multiplier_property;
+
+    struct drm_property *nv_plane_tmo_lut_property;
+    struct drm_property *nv_plane_tmo_lut_size_property;
+
+    struct drm_property *nv_crtc_regamma_tf_property;
+    struct drm_property *nv_crtc_regamma_lut_property;
+    struct drm_property *nv_crtc_regamma_lut_size_property;
+    struct drm_property *nv_crtc_regamma_divisor_property;
+
     struct nv_drm_device *next;
 };
 
+static inline NvU32 nv_drm_next_display_semaphore(
+    struct nv_drm_device *nv_dev)
+{
+    NvU32 current_index = nv_dev->display_semaphores.next_index++;
+
+    if (nv_dev->display_semaphores.next_index >=
+        nv_dev->display_semaphores.count) {
+        nv_dev->display_semaphores.next_index = 0;
+    }
+
+    return current_index;
+}
+
 static inline struct nv_drm_device *to_nv_device(
     struct drm_device *dev)
 {

kernel-open/nvidia-drm/nvidia-drm-sources.mk

@@ -37,6 +37,8 @@ NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_gpl_refcount_dec_and_test
 NV_CONFTEST_GENERIC_COMPILE_TESTS += drm_alpha_blending_available
 NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_present_drm_gem_prime_fd_to_handle
 NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_present_drm_gem_prime_handle_to_fd
+NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_present_timer_delete_sync
+NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_gpl___vma_start_write
 
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_dev_unref
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_reinit_primary_mode_group
@@ -66,12 +68,18 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_fence_set_error
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += fence_set_error
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += sync_file_get_fence
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_aperture_remove_conflicting_pci_framebuffers
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += aperture_remove_conflicting_devices
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += aperture_remove_conflicting_pci_devices
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_fbdev_generic_setup
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_fbdev_ttm_setup
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_client_setup
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_connector_attach_hdr_output_metadata_property
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_helper_crtc_enable_color_mgmt
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_crtc_enable_color_mgmt
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_atomic_helper_legacy_gamma_set
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += vmf_insert_mixed
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += pfn_to_pfn_t
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_gem_prime_mmap
 
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_bus_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_bus_has_bus_type
@@ -129,5 +137,15 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_dumb_destroy
 NV_CONFTEST_TYPE_COMPILE_TESTS += fence_ops_use_64bit_seqno
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_aperture_remove_conflicting_pci_framebuffers_has_driver_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_mode_create_dp_colorspace_property_has_supported_colorspaces_arg
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_syncobj_features_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_unlocked_ioctl_flag_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_framebuffer_obj_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_color_ctm_3x4_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_color_lut
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_property_blob_put
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_gem_prime_mmap
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_output_poll_changed
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_date
+NV_CONFTEST_TYPE_COMPILE_TESTS += file_operations_fop_unsigned_offset_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_helper_funcs_mode_valid_has_const_mode_arg
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_fb_create_takes_format_info

kernel-open/nvidia-modeset/nvidia-modeset-linux.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2015-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -53,6 +53,7 @@
 #include "nv-procfs.h"
 #include "nv-kthread-q.h"
 #include "nv-time.h"
+#include "nv-timer.h"
 #include "nv-lock.h"
 #include "nv-chardev-numbers.h"
 
@@ -77,15 +78,21 @@ module_param_named(disable_hdmi_frl, disable_hdmi_frl, bool, 0400);
 static bool disable_vrr_memclk_switch = false;
 module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0400);
 
-static bool hdmi_deepcolor = false;
+static bool hdmi_deepcolor = true;
 module_param_named(hdmi_deepcolor, hdmi_deepcolor, bool, 0400);
 
-static bool vblank_sem_control = false;
+static bool vblank_sem_control = true;
 module_param_named(vblank_sem_control, vblank_sem_control, bool, 0400);
 
 static bool opportunistic_display_sync = true;
 module_param_named(opportunistic_display_sync, opportunistic_display_sync, bool, 0400);
 
+static enum NvKmsDebugForceColorSpace debug_force_color_space = NVKMS_DEBUG_FORCE_COLOR_SPACE_NONE;
+module_param_named(debug_force_color_space, debug_force_color_space, uint, 0400);
+
+static bool enable_overlay_layers = true;
+module_param_named(enable_overlay_layers, enable_overlay_layers, bool, 0400);
+
 /* These parameters are used for fault injection tests.  Normally the defaults
  * should be used. */
 MODULE_PARM_DESC(fail_malloc, "Fail the Nth call to nvkms_alloc");
@@ -96,19 +103,40 @@ MODULE_PARM_DESC(malloc_verbose, "Report information about malloc calls on modul
 static bool malloc_verbose = false;
 module_param_named(malloc_verbose, malloc_verbose, bool, 0400);
 
+/* Fail allocating the RM core channel for NVKMS using the i-th method (see
+ * FailAllocCoreChannelMethod). Failures not using the i-th method are ignored. */
+MODULE_PARM_DESC(fail_alloc_core_channel, "Control testing for hardware core channel allocation failure");
+static int fail_alloc_core_channel_method = -1;
+module_param_named(fail_alloc_core_channel, fail_alloc_core_channel_method, int, 0400);
+
 #if NVKMS_CONFIG_FILE_SUPPORTED
 /* This parameter is used to find the dpy override conf file */
 #define NVKMS_CONF_FILE_SPECIFIED (nvkms_conf != NULL)
 
 MODULE_PARM_DESC(config_file,
-                 "Path to the nvidia-modeset configuration file "
-                 "(default: disabled)");
+                 "Path to the nvidia-modeset configuration file (default: disabled)");
 static char *nvkms_conf = NULL;
 module_param_named(config_file, nvkms_conf, charp, 0400);
 #endif
 
 static atomic_t nvkms_alloc_called_count;
 
+NvBool nvkms_test_fail_alloc_core_channel(
+    enum FailAllocCoreChannelMethod method
+)
+{
+    if (method != fail_alloc_core_channel_method) {
+        // don't fail if it's not the currently specified method
+        return NV_FALSE;
+    } 
+
+    printk(KERN_INFO NVKMS_LOG_PREFIX 
+        "Failing core channel allocation using method %d", 
+        fail_alloc_core_channel_method);    
+
+    return NV_TRUE;
+}
+    
 NvBool nvkms_output_rounding_fix(void)
 {
     return output_rounding_fix;
@@ -139,6 +167,33 @@ NvBool nvkms_opportunistic_display_sync(void)
     return opportunistic_display_sync;
 }
 
+enum NvKmsDebugForceColorSpace nvkms_debug_force_color_space(void)
+{
+    if (debug_force_color_space >= NVKMS_DEBUG_FORCE_COLOR_SPACE_MAX) {
+        return NVKMS_DEBUG_FORCE_COLOR_SPACE_NONE;
+    }
+    return debug_force_color_space;
+}
+
+NvBool nvkms_enable_overlay_layers(void)
+{
+    return enable_overlay_layers;
+}
+
+NvBool nvkms_kernel_supports_syncpts(void)
+{
+/*
+ * Note this only checks that the kernel has the prerequisite
+ * support for syncpts; callers must also check that the hardware
+ * supports syncpts.
+ */
+#if (defined(CONFIG_TEGRA_GRHOST) || defined(NV_LINUX_HOST1X_NEXT_H_PRESENT))
+    return NV_TRUE;
+#else
+    return NV_FALSE;
+#endif
+}
+
 #define NVKMS_SYNCPT_STUBS_NEEDED
 
 /*************************************************************************
@@ -684,7 +739,7 @@ static void nvkms_kthread_q_callback(void *arg)
      * pending timers and than waiting for workqueue callbacks.
      */
     if (timer->kernel_timer_created) {
-        del_timer_sync(&timer->kernel_timer);
+        nv_timer_delete_sync(&timer->kernel_timer);
     }
 
     /*
@@ -996,6 +1051,11 @@ nvkms_register_backlight(NvU32 gpu_id, NvU32 display_id, void *drv_priv,
 
 #if defined(NV_ACPI_VIDEO_BACKLIGHT_USE_NATIVE)
     if (!acpi_video_backlight_use_native()) {
+#if defined(NV_ACPI_VIDEO_REGISTER_BACKLIGHT)
+        nvkms_log(NVKMS_LOG_LEVEL_INFO, NVKMS_LOG_PREFIX,
+                  "ACPI reported no NVIDIA native backlight available; attempting to use ACPI backlight.");
+        acpi_video_register_backlight();
+#endif
         return NULL;
     }
 #endif
@@ -1234,6 +1294,26 @@ void nvkms_close_from_kapi(struct nvkms_per_open *popen)
     nvkms_close_pm_unlocked(popen);
 }
 
+NvBool nvkms_ioctl_from_kapi_try_pmlock
+(
+    struct nvkms_per_open *popen,
+    NvU32 cmd, void *params_address, const size_t param_size
+)
+{
+    NvBool ret;
+
+    if (nvkms_read_trylock_pm_lock()) {
+        return NV_FALSE;
+    }
+
+    ret = nvkms_ioctl_common(popen,
+                             cmd,
+                             (NvU64)(NvUPtr)params_address, param_size) == 0;
+    nvkms_read_unlock_pm_lock();
+
+    return ret;
+}
+
 NvBool nvkms_ioctl_from_kapi
 (
     struct nvkms_per_open *popen,
@@ -1418,6 +1498,8 @@ static size_t nvkms_config_file_open
     loff_t pos = 0;
 #endif
 
+    *buff = NULL;
+    
     if (!nvkms_fs_mounted()) {
         printk(KERN_ERR NVKMS_LOG_PREFIX "ERROR: Filesystems not mounted\n");
         return 0;
@@ -1441,6 +1523,11 @@ static size_t nvkms_config_file_open
         goto done;
     }
 
+    // Do not alloc a 0 sized buffer
+    if (file_size == 0) {
+        goto done;
+    }
+
     *buff = nvkms_alloc(file_size, NV_FALSE);
     if (*buff == NULL) {
         printk(KERN_WARNING NVKMS_LOG_PREFIX "WARNING: Out of memory\n");
@@ -1524,6 +1611,48 @@ NvBool nvKmsKapiGetFunctionsTable
 }
 EXPORT_SYMBOL(nvKmsKapiGetFunctionsTable);
 
+NvU32 nvKmsKapiF16ToF32(NvU16 a)
+{
+    return nvKmsKapiF16ToF32Internal(a);
+}
+EXPORT_SYMBOL(nvKmsKapiF16ToF32);
+
+NvU16 nvKmsKapiF32ToF16(NvU32 a)
+{
+    return nvKmsKapiF32ToF16Internal(a);
+}
+EXPORT_SYMBOL(nvKmsKapiF32ToF16);
+
+NvU32 nvKmsKapiF32Mul(NvU32 a, NvU32 b)
+{
+    return nvKmsKapiF32MulInternal(a, b);
+}
+EXPORT_SYMBOL(nvKmsKapiF32Mul);
+
+NvU32 nvKmsKapiF32Div(NvU32 a, NvU32 b)
+{
+    return nvKmsKapiF32DivInternal(a, b);
+}
+EXPORT_SYMBOL(nvKmsKapiF32Div);
+
+NvU32 nvKmsKapiF32Add(NvU32 a, NvU32 b)
+{
+    return nvKmsKapiF32AddInternal(a, b);
+}
+EXPORT_SYMBOL(nvKmsKapiF32Add);
+
+NvU32 nvKmsKapiF32ToUI32RMinMag(NvU32 a, NvBool exact)
+{
+    return nvKmsKapiF32ToUI32RMinMagInternal(a, exact);
+}
+EXPORT_SYMBOL(nvKmsKapiF32ToUI32RMinMag);
+
+NvU32 nvKmsKapiUI32ToF32(NvU32 a)
+{
+    return nvKmsKapiUI32ToF32Internal(a);
+}
+EXPORT_SYMBOL(nvKmsKapiUI32ToF32);
+
 /*************************************************************************
  * File operation callback functions.
  *************************************************************************/
@@ -1794,7 +1923,11 @@ restart:
              * completion, and we wait for queue completion with
              * nv_kthread_q_stop below.
              */
+#if !defined(NV_BSD) && NV_IS_EXPORT_SYMBOL_PRESENT_timer_delete_sync
+            if (timer_delete_sync(&timer->kernel_timer) == 1) {
+#else
             if (del_timer_sync(&timer->kernel_timer) == 1) {
+#endif
                 /*  We've deactivated timer so we need to clean after it */
                 list_del(&timer->timers_list);
 

kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h

@@ -67,6 +67,14 @@ enum NvKmsSyncPtOp {
     NVKMS_SYNCPT_OP_READ_MINVAL,
 };
 
+enum NvKmsDebugForceColorSpace {
+    NVKMS_DEBUG_FORCE_COLOR_SPACE_NONE,
+    NVKMS_DEBUG_FORCE_COLOR_SPACE_RGB,
+    NVKMS_DEBUG_FORCE_COLOR_SPACE_YUV444,
+    NVKMS_DEBUG_FORCE_COLOR_SPACE_YUV422,
+    NVKMS_DEBUG_FORCE_COLOR_SPACE_MAX,
+};
+
 typedef struct {
 
     struct {
@@ -96,12 +104,20 @@ typedef struct {
     } read_minval;
 } NvKmsSyncPtOpParams;
 
+enum FailAllocCoreChannelMethod {
+    FAIL_ALLOC_CORE_CHANNEL_RM_SETUP_CORE_CHANNEL = 0,
+    FAIL_ALLOC_CORE_CHANNEL_RESTORE_CONSOLE = 1,
+};
+
+NvBool nvkms_test_fail_alloc_core_channel(enum FailAllocCoreChannelMethod method);
 NvBool nvkms_output_rounding_fix(void);
 NvBool nvkms_disable_hdmi_frl(void);
 NvBool nvkms_disable_vrr_memclk_switch(void);
 NvBool nvkms_hdmi_deepcolor(void);
 NvBool nvkms_vblank_sem_control(void);
 NvBool nvkms_opportunistic_display_sync(void);
+enum NvKmsDebugForceColorSpace nvkms_debug_force_color_space(void);
+NvBool nvkms_enable_overlay_layers(void);
 
 void   nvkms_call_rm    (void *ops);
 void*  nvkms_alloc      (size_t size,
@@ -304,6 +320,11 @@ NvU32 nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info);
 
 NvBool nvkms_allow_write_combining(void);
 
+/*!
+ * Check if OS supports syncpoints.
+ */
+NvBool nvkms_kernel_supports_syncpts(void);
+
 /*!
  * Checks whether the fd is associated with an nvidia character device.
  */
@@ -328,6 +349,16 @@ NvBool nvkms_ioctl_from_kapi
     NvU32 cmd, void *params_address, const size_t params_size
 );
 
+/*!
+ * Like nvkms_ioctl_from_kapi, but return NV_FALSE instead of waiting if the
+ * power management read lock cannot be acquired.
+ */
+NvBool nvkms_ioctl_from_kapi_try_pmlock
+(
+    struct nvkms_per_open *popen,
+    NvU32 cmd, void *params_address, const size_t params_size
+);
+
 /*!
  * APIs for locking.
  */

kernel-open/nvidia-modeset/nvidia-modeset.Kbuild

@@ -40,9 +40,6 @@ NV_KERNEL_MODULE_TARGETS += $(NVIDIA_MODESET_KO)
 NVIDIA_MODESET_BINARY_OBJECT := $(src)/nvidia-modeset/nv-modeset-kernel.o_binary
 NVIDIA_MODESET_BINARY_OBJECT_O := nvidia-modeset/nv-modeset-kernel.o
 
-quiet_cmd_symlink = SYMLINK $@
-cmd_symlink = ln -sf $< $@
-
 targets += $(NVIDIA_MODESET_BINARY_OBJECT_O)
 
 $(obj)/$(NVIDIA_MODESET_BINARY_OBJECT_O): $(NVIDIA_MODESET_BINARY_OBJECT) FORCE
@@ -105,3 +102,6 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += list_is_first
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_real_ts64
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_raw_ts64
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += acpi_video_backlight_use_native
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += acpi_video_register_backlight
+NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_timer_delete_sync
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += kernel_read_has_pointer_pos_arg

kernel-open/nvidia-modeset/nvkms.h

@@ -110,4 +110,18 @@ NvBool nvKmsSetBacklight(NvU32 display_id, void *drv_priv, NvU32 brightness);
 
 NvBool nvKmsOpenDevHasSubOwnerPermissionOrBetter(const struct NvKmsPerOpenDev *pOpenDev);
 
+NvU32 nvKmsKapiF16ToF32Internal(NvU16 a);
+
+NvU16 nvKmsKapiF32ToF16Internal(NvU32 a);
+
+NvU32 nvKmsKapiF32MulInternal(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32DivInternal(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32AddInternal(NvU32 a, NvU32 b);
+
+NvU32 nvKmsKapiF32ToUI32RMinMagInternal(NvU32 a, NvBool exact);
+
+NvU32 nvKmsKapiUI32ToF32Internal(NvU32 a);
+
 #endif /* __NV_KMS_H__ */

kernel-open/nvidia-peermem/nv-p2p.h

@@ -189,6 +189,12 @@ int nvidia_p2p_get_pages( uint64_t p2p_token, uint32_t va_space,
         struct nvidia_p2p_page_table **page_table,
         void (*free_callback)(void *data), void *data);
 
+/*
+ * Flags to be used with persistent APIs
+ */
+#define NVIDIA_P2P_FLAGS_DEFAULT            0
+#define NVIDIA_P2P_FLAGS_FORCE_BAR1_MAPPING 1
+
 /*
  * @brief
  *   Pin and make the pages underlying a range of GPU virtual memory
@@ -212,7 +218,11 @@ int nvidia_p2p_get_pages( uint64_t p2p_token, uint32_t va_space,
  * @param[out]    page_table
  *   A pointer to an array of structures with P2P PTEs.
  * @param[in]     flags
- *   Must be set to zero for now.
+ *   NVIDIA_P2P_FLAGS_DEFAULT:
+ *     Default value to be used if no specific behavior is expected.
+ *   NVIDIA_P2P_FLAGS_FORCE_BAR1_MAPPING:
+ *     Force BAR1 mappings on certain coherent platforms,
+ *     subject to capability and supported topology.
  *
  * @return
  *    0           upon successful completion.

kernel-open/nvidia-peermem/nvidia-peermem.c

@@ -60,6 +60,13 @@ static int peerdirect_support = NV_MEM_PEERDIRECT_SUPPORT_DEFAULT;
 module_param(peerdirect_support, int, S_IRUGO);
 MODULE_PARM_DESC(peerdirect_support, "Set level of support for Peer-direct, 0 [default] or 1 [legacy, for example MLNX_OFED 4.9 LTS]");
 
+enum {
+        NV_MEM_PERSISTENT_API_SUPPORT_LEGACY = 0,
+        NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT = 1,
+};
+static int persistent_api_support = NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT;
+module_param(persistent_api_support, int, S_IRUGO);
+MODULE_PARM_DESC(persistent_api_support, "Set level of support for persistent APIs, 0 [legacy] or 1 [default]");
 
 #define peer_err(FMT, ARGS...) printk(KERN_ERR "nvidia-peermem" " %s:%d ERROR " FMT, __FUNCTION__, __LINE__, ## ARGS)
 #ifdef NV_MEM_DEBUG
@@ -479,32 +486,8 @@ static struct peer_memory_client nv_mem_client_nc = {
     .release        = nv_mem_release,
 };
 
-#endif /* NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT */
-
-static int nv_mem_param_conf_check(void)
+static int nv_mem_legacy_client_init(void)
 {
-    int rc = 0;
-    switch (peerdirect_support) {
-    case NV_MEM_PEERDIRECT_SUPPORT_DEFAULT:
-    case NV_MEM_PEERDIRECT_SUPPORT_LEGACY:
-        break;
-    default:
-        peer_err("invalid peerdirect_support param value %d\n", peerdirect_support);
-        rc = -EINVAL;
-        break;
-    }
-    return rc;
-}
-
-static int __init nv_mem_client_init(void)
-{
-    int rc;
-    rc = nv_mem_param_conf_check();
-    if (rc) {
-        return rc;
-    }
-
-#if defined (NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT)
     // off by one, to leave space for the trailing '1' which is flagging
     // the new client type
     BUG_ON(strlen(DRV_NAME) > IB_PEER_MEMORY_NAME_MAX-1);
@@ -533,19 +516,96 @@ static int __init nv_mem_client_init(void)
                          &mem_invalidate_callback);
     if (!reg_handle) {
         peer_err("nv_mem_client_init -- error while registering traditional client\n");
-        rc = -EINVAL;
-        goto out;
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static int nv_mem_nc_client_init(void)
+{
+    // The nc client enables support for persistent pages.
+    if (persistent_api_support == NV_MEM_PERSISTENT_API_SUPPORT_LEGACY)
+    {
+        //
+        // If legacy behavior is forced via module param,
+        // both legacy and persistent clients are registered and are named
+        // "nv_mem"(legacy) and "nv_mem_nc"(persistent).
+        //
+        strcpy(nv_mem_client_nc.name, DRV_NAME "_nc");
+    }
+    else
+    {
+        //
+        // With default persistent behavior, the client name shall be "nv_mem"
+        // so that libraries can use the persistent client under the same name.
+        //
+        strcpy(nv_mem_client_nc.name, DRV_NAME);
     }
 
-    // The nc client enables support for persistent pages.
-    strcpy(nv_mem_client_nc.name, DRV_NAME "_nc");
     strcpy(nv_mem_client_nc.version, DRV_VERSION);
     reg_handle_nc = ib_register_peer_memory_client(&nv_mem_client_nc, NULL);
     if (!reg_handle_nc) {
         peer_err("nv_mem_client_init -- error while registering nc client\n");
-        rc = -EINVAL;
-        goto out;
+        return -EINVAL;
     }
+    return 0;
+}
+
+#endif /* NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT */
+
+static int nv_mem_param_peerdirect_conf_check(void)
+{
+    int rc = 0;
+    switch (peerdirect_support) {
+    case NV_MEM_PEERDIRECT_SUPPORT_DEFAULT:
+    case NV_MEM_PEERDIRECT_SUPPORT_LEGACY:
+        break;
+    default:
+        peer_err("invalid peerdirect_support param value %d\n", peerdirect_support);
+        rc = -EINVAL;
+        break;
+    }
+    return rc;
+}
+
+static int nv_mem_param_persistent_api_conf_check(void)
+{
+    int rc = 0;
+    switch (persistent_api_support) {
+    case NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT:
+    case NV_MEM_PERSISTENT_API_SUPPORT_LEGACY:
+        break;
+    default:
+        peer_err("invalid persistent_api_support param value %d\n", persistent_api_support);
+        rc = -EINVAL;
+        break;
+    }
+    return rc;
+}
+
+static int __init nv_mem_client_init(void)
+{
+#if defined (NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT)
+    int rc;
+    rc = nv_mem_param_peerdirect_conf_check();
+    if (rc) {
+        return rc;
+    }
+
+    rc = nv_mem_param_persistent_api_conf_check();
+    if (rc) {
+        return rc;
+    }
+
+    if (persistent_api_support == NV_MEM_PERSISTENT_API_SUPPORT_LEGACY) {
+        rc = nv_mem_legacy_client_init();
+        if (rc)
+            goto out;
+    }
+
+    rc = nv_mem_nc_client_init();
+    if (rc)
+        goto out;
 
 out:
     if (rc) {

kernel-open/nvidia-uvm/clc365.h

@@ -1,30 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2023 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-// AUTO GENERATED -- DO NOT EDIT - this file automatically generated by refhdr2class.pl
-// Command: ../../../bin/manuals/refhdr2class.pl clc365.h c365 ACCESS_COUNTER_NOTIFY_BUFFER --search_str=NV_ACCESS_COUNTER --input_file=nv_ref_dev_access_counter.h
-
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #ifndef _clc365_h_
 #define _clc365_h_

kernel-open/nvidia-uvm/clc369.h

@@ -1,30 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2023 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-// AUTO GENERATED -- DO NOT EDIT - this file automatically generated by refhdr2class.pl
-// Command: ../../../bin/manuals/refhdr2class.pl clc369.h c369 MMU_FAULT_BUFFER --search_str=NV_MMU_FAULT --input_file=nv_ref_dev_mmu_fault.h
-
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #ifndef _clc369_h_
 #define _clc369_h_

kernel-open/nvidia-uvm/clc36f.h

@@ -1,26 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2012-2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #ifndef _clc36f_h_
 #define _clc36f_h_
@@ -257,7 +256,6 @@ typedef volatile struct Nvc36fControl_struct {
 #define NVC36F_CLEAR_FAULTED_TYPE                                        31:31
 #define NVC36F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED                     0x00000000
 #define NVC36F_CLEAR_FAULTED_TYPE_ENG_FAULTED                       0x00000001
-#define NVC36F_QUADRO_VERIFY                                       (0x000000a0)
 
 
 /* GPFIFO entry format */

kernel-open/nvidia-uvm/clc46f.h

@@ -1,26 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2012-2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #ifndef _clc46f_h_
 #define _clc46f_h_
@@ -259,7 +258,6 @@ typedef volatile struct Nvc46fControl_struct {
 #define NVC46F_CLEAR_FAULTED_TYPE                                        31:31
 #define NVC46F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED                     0x00000000
 #define NVC46F_CLEAR_FAULTED_TYPE_ENG_FAULTED                       0x00000001
-#define NVC46F_QUADRO_VERIFY                                       (0x000000a0)
 
 
 /* GPFIFO entry format */

kernel-open/nvidia-uvm/clc56f.h

@@ -1,26 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2012-2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #ifndef _clc56f_h_
 #define _clc56f_h_
@@ -261,7 +260,6 @@ typedef volatile struct Nvc56fControl_struct {
 #define NVC56F_CLEAR_FAULTED_TYPE                                        31:31
 #define NVC56F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED                     0x00000000
 #define NVC56F_CLEAR_FAULTED_TYPE_ENG_FAULTED                       0x00000001
-#define NVC56F_QUADRO_VERIFY                                       (0x000000a0)
 
 
 /* GPFIFO entry format */

kernel-open/nvidia-uvm/clc5b5.h

@@ -1,19 +1,19 @@
 /*******************************************************************************
-    Copyright (c) 1993-2004 NVIDIA Corporation
+    Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
 
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
 
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
@@ -21,8 +21,6 @@
 
 *******************************************************************************/
 
-
-
 #include "nvtypes.h"
 
 #ifndef _clc5b5_h_
@@ -34,64 +32,6 @@ extern "C" {
 
 #define TURING_DMA_COPY_A                                                            (0x0000C5B5)
 
-typedef volatile struct _clc5b5_tag0 {
-    NvV32 Reserved00[0x40];
-    NvV32 Nop;                                                                  // 0x00000100 - 0x00000103
-    NvV32 Reserved01[0xF];
-    NvV32 PmTrigger;                                                            // 0x00000140 - 0x00000143
-    NvV32 Reserved02[0x3F];
-    NvV32 SetSemaphoreA;                                                        // 0x00000240 - 0x00000243
-    NvV32 SetSemaphoreB;                                                        // 0x00000244 - 0x00000247
-    NvV32 SetSemaphorePayload;                                                  // 0x00000248 - 0x0000024B
-    NvV32 Reserved03[0x2];
-    NvV32 SetRenderEnableA;                                                     // 0x00000254 - 0x00000257
-    NvV32 SetRenderEnableB;                                                     // 0x00000258 - 0x0000025B
-    NvV32 SetRenderEnableC;                                                     // 0x0000025C - 0x0000025F
-    NvV32 SetSrcPhysMode;                                                       // 0x00000260 - 0x00000263
-    NvV32 SetDstPhysMode;                                                       // 0x00000264 - 0x00000267
-    NvV32 Reserved04[0x6];
-    NvV32 SetGlobalCounterUpper;                                                // 0x00000280 - 0x00000283
-    NvV32 SetGlobalCounterLower;                                                // 0x00000284 - 0x00000287
-    NvV32 SetPageoutStartPAUpper;                                               // 0x00000288 - 0x0000028B
-    NvV32 SetPageoutStartPALower;                                               // 0x0000028C - 0x0000028F
-    NvV32 Reserved05[0x1C];
-    NvV32 LaunchDma;                                                            // 0x00000300 - 0x00000303
-    NvV32 Reserved06[0x3F];
-    NvV32 OffsetInUpper;                                                        // 0x00000400 - 0x00000403
-    NvV32 OffsetInLower;                                                        // 0x00000404 - 0x00000407
-    NvV32 OffsetOutUpper;                                                       // 0x00000408 - 0x0000040B
-    NvV32 OffsetOutLower;                                                       // 0x0000040C - 0x0000040F
-    NvV32 PitchIn;                                                              // 0x00000410 - 0x00000413
-    NvV32 PitchOut;                                                             // 0x00000414 - 0x00000417
-    NvV32 LineLengthIn;                                                         // 0x00000418 - 0x0000041B
-    NvV32 LineCount;                                                            // 0x0000041C - 0x0000041F
-    NvV32 Reserved07[0xB8];
-    NvV32 SetRemapConstA;                                                       // 0x00000700 - 0x00000703
-    NvV32 SetRemapConstB;                                                       // 0x00000704 - 0x00000707
-    NvV32 SetRemapComponents;                                                   // 0x00000708 - 0x0000070B
-    NvV32 SetDstBlockSize;                                                      // 0x0000070C - 0x0000070F
-    NvV32 SetDstWidth;                                                          // 0x00000710 - 0x00000713
-    NvV32 SetDstHeight;                                                         // 0x00000714 - 0x00000717
-    NvV32 SetDstDepth;                                                          // 0x00000718 - 0x0000071B
-    NvV32 SetDstLayer;                                                          // 0x0000071C - 0x0000071F
-    NvV32 SetDstOrigin;                                                         // 0x00000720 - 0x00000723
-    NvV32 Reserved08[0x1];
-    NvV32 SetSrcBlockSize;                                                      // 0x00000728 - 0x0000072B
-    NvV32 SetSrcWidth;                                                          // 0x0000072C - 0x0000072F
-    NvV32 SetSrcHeight;                                                         // 0x00000730 - 0x00000733
-    NvV32 SetSrcDepth;                                                          // 0x00000734 - 0x00000737
-    NvV32 SetSrcLayer;                                                          // 0x00000738 - 0x0000073B
-    NvV32 SetSrcOrigin;                                                         // 0x0000073C - 0x0000073F
-    NvV32 Reserved09[0x1];
-    NvV32 SrcOriginX;                                                           // 0x00000744 - 0x00000747
-    NvV32 SrcOriginY;                                                           // 0x00000748 - 0x0000074B
-    NvV32 DstOriginX;                                                           // 0x0000074C - 0x0000074F
-    NvV32 DstOriginY;                                                           // 0x00000750 - 0x00000753
-    NvV32 Reserved10[0x270];
-    NvV32 PmTriggerEnd;                                                         // 0x00001114 - 0x00001117
-    NvV32 Reserved11[0x3BA];
-} turing_dma_copy_aControlPio;
-
 #define NVC5B5_NOP                                                              (0x00000100)
 #define NVC5B5_NOP_PARAMETER                                                    31:0
 #define NVC5B5_PM_TRIGGER                                                       (0x00000140)
@@ -125,14 +65,6 @@ typedef volatile struct _clc5b5_tag0 {
 #define NVC5B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM                         (0x00000001)
 #define NVC5B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM                      (0x00000002)
 #define NVC5B5_SET_DST_PHYS_MODE_BASIC_KIND                                     5:2
-#define NVC5B5_SET_GLOBAL_COUNTER_UPPER                                         (0x00000280)
-#define NVC5B5_SET_GLOBAL_COUNTER_UPPER_V                                       31:0
-#define NVC5B5_SET_GLOBAL_COUNTER_LOWER                                         (0x00000284)
-#define NVC5B5_SET_GLOBAL_COUNTER_LOWER_V                                       31:0
-#define NVC5B5_SET_PAGEOUT_START_PAUPPER                                        (0x00000288)
-#define NVC5B5_SET_PAGEOUT_START_PAUPPER_V                                      4:0
-#define NVC5B5_SET_PAGEOUT_START_PALOWER                                        (0x0000028C)
-#define NVC5B5_SET_PAGEOUT_START_PALOWER_V                                      31:0
 #define NVC5B5_LAUNCH_DMA                                                       (0x00000300)
 #define NVC5B5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                    1:0
 #define NVC5B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                               (0x00000000)
@@ -199,8 +131,6 @@ typedef volatile struct _clc5b5_tag0 {
 #define NVC5B5_LAUNCH_DMA_VPRMODE                                               23:22
 #define NVC5B5_LAUNCH_DMA_VPRMODE_VPR_NONE                                      (0x00000000)
 #define NVC5B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID                                   (0x00000001)
-#define NVC5B5_LAUNCH_DMA_VPRMODE_VPR_VID2SYS                                   (0x00000002)
-#define NVC5B5_LAUNCH_DMA_VPRMODE_VPR_SYS2VID                                   (0x00000003)
 #define NVC5B5_LAUNCH_DMA_RESERVED_START_OF_COPY                                24:24
 #define NVC5B5_LAUNCH_DMA_DISABLE_PLC                                           26:26
 #define NVC5B5_LAUNCH_DMA_DISABLE_PLC_FALSE                                     (0x00000000)

kernel-open/nvidia-uvm/clc6b5.h

@@ -1,19 +1,19 @@
 /*******************************************************************************
-    Copyright (c) 1993-2004 NVIDIA Corporation
+    Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
 
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
 
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
@@ -21,8 +21,6 @@
 
 *******************************************************************************/
 
-
-
 #include "nvtypes.h"
 
 #ifndef _clc6b5_h_
@@ -34,64 +32,6 @@ extern "C" {
 
 #define AMPERE_DMA_COPY_A                                                            (0x0000C6B5)
 
-typedef volatile struct _clc6b5_tag0 {
-    NvV32 Reserved00[0x40];
-    NvV32 Nop;                                                                  // 0x00000100 - 0x00000103
-    NvV32 Reserved01[0xF];
-    NvV32 PmTrigger;                                                            // 0x00000140 - 0x00000143
-    NvV32 Reserved02[0x3F];
-    NvV32 SetSemaphoreA;                                                        // 0x00000240 - 0x00000243
-    NvV32 SetSemaphoreB;                                                        // 0x00000244 - 0x00000247
-    NvV32 SetSemaphorePayload;                                                  // 0x00000248 - 0x0000024B
-    NvV32 Reserved03[0x2];
-    NvV32 SetRenderEnableA;                                                     // 0x00000254 - 0x00000257
-    NvV32 SetRenderEnableB;                                                     // 0x00000258 - 0x0000025B
-    NvV32 SetRenderEnableC;                                                     // 0x0000025C - 0x0000025F
-    NvV32 SetSrcPhysMode;                                                       // 0x00000260 - 0x00000263
-    NvV32 SetDstPhysMode;                                                       // 0x00000264 - 0x00000267
-    NvV32 Reserved04[0x6];
-    NvV32 SetGlobalCounterUpper;                                                // 0x00000280 - 0x00000283
-    NvV32 SetGlobalCounterLower;                                                // 0x00000284 - 0x00000287
-    NvV32 SetPageoutStartPAUpper;                                               // 0x00000288 - 0x0000028B
-    NvV32 SetPageoutStartPALower;                                               // 0x0000028C - 0x0000028F
-    NvV32 Reserved05[0x1C];
-    NvV32 LaunchDma;                                                            // 0x00000300 - 0x00000303
-    NvV32 Reserved06[0x3F];
-    NvV32 OffsetInUpper;                                                        // 0x00000400 - 0x00000403
-    NvV32 OffsetInLower;                                                        // 0x00000404 - 0x00000407
-    NvV32 OffsetOutUpper;                                                       // 0x00000408 - 0x0000040B
-    NvV32 OffsetOutLower;                                                       // 0x0000040C - 0x0000040F
-    NvV32 PitchIn;                                                              // 0x00000410 - 0x00000413
-    NvV32 PitchOut;                                                             // 0x00000414 - 0x00000417
-    NvV32 LineLengthIn;                                                         // 0x00000418 - 0x0000041B
-    NvV32 LineCount;                                                            // 0x0000041C - 0x0000041F
-    NvV32 Reserved07[0xB8];
-    NvV32 SetRemapConstA;                                                       // 0x00000700 - 0x00000703
-    NvV32 SetRemapConstB;                                                       // 0x00000704 - 0x00000707
-    NvV32 SetRemapComponents;                                                   // 0x00000708 - 0x0000070B
-    NvV32 SetDstBlockSize;                                                      // 0x0000070C - 0x0000070F
-    NvV32 SetDstWidth;                                                          // 0x00000710 - 0x00000713
-    NvV32 SetDstHeight;                                                         // 0x00000714 - 0x00000717
-    NvV32 SetDstDepth;                                                          // 0x00000718 - 0x0000071B
-    NvV32 SetDstLayer;                                                          // 0x0000071C - 0x0000071F
-    NvV32 SetDstOrigin;                                                         // 0x00000720 - 0x00000723
-    NvV32 Reserved08[0x1];
-    NvV32 SetSrcBlockSize;                                                      // 0x00000728 - 0x0000072B
-    NvV32 SetSrcWidth;                                                          // 0x0000072C - 0x0000072F
-    NvV32 SetSrcHeight;                                                         // 0x00000730 - 0x00000733
-    NvV32 SetSrcDepth;                                                          // 0x00000734 - 0x00000737
-    NvV32 SetSrcLayer;                                                          // 0x00000738 - 0x0000073B
-    NvV32 SetSrcOrigin;                                                         // 0x0000073C - 0x0000073F
-    NvV32 Reserved09[0x1];
-    NvV32 SrcOriginX;                                                           // 0x00000744 - 0x00000747
-    NvV32 SrcOriginY;                                                           // 0x00000748 - 0x0000074B
-    NvV32 DstOriginX;                                                           // 0x0000074C - 0x0000074F
-    NvV32 DstOriginY;                                                           // 0x00000750 - 0x00000753
-    NvV32 Reserved10[0x270];
-    NvV32 PmTriggerEnd;                                                         // 0x00001114 - 0x00001117
-    NvV32 Reserved11[0x3BA];
-} ampere_dma_copy_aControlPio;
-
 #define NVC6B5_NOP                                                              (0x00000100)
 #define NVC6B5_NOP_PARAMETER                                                    31:0
 #define NVC6B5_PM_TRIGGER                                                       (0x00000140)
@@ -131,14 +71,6 @@ typedef volatile struct _clc6b5_tag0 {
 #define NVC6B5_SET_DST_PHYS_MODE_BASIC_KIND                                     5:2
 #define NVC6B5_SET_DST_PHYS_MODE_PEER_ID                                        8:6
 #define NVC6B5_SET_DST_PHYS_MODE_FLA                                            9:9
-#define NVC6B5_SET_GLOBAL_COUNTER_UPPER                                         (0x00000280)
-#define NVC6B5_SET_GLOBAL_COUNTER_UPPER_V                                       31:0
-#define NVC6B5_SET_GLOBAL_COUNTER_LOWER                                         (0x00000284)
-#define NVC6B5_SET_GLOBAL_COUNTER_LOWER_V                                       31:0
-#define NVC6B5_SET_PAGEOUT_START_PAUPPER                                        (0x00000288)
-#define NVC6B5_SET_PAGEOUT_START_PAUPPER_V                                      4:0
-#define NVC6B5_SET_PAGEOUT_START_PALOWER                                        (0x0000028C)
-#define NVC6B5_SET_PAGEOUT_START_PALOWER_V                                      31:0
 #define NVC6B5_LAUNCH_DMA                                                       (0x00000300)
 #define NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                    1:0
 #define NVC6B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                               (0x00000000)
@@ -199,8 +131,6 @@ typedef volatile struct _clc6b5_tag0 {
 #define NVC6B5_LAUNCH_DMA_VPRMODE                                               23:22
 #define NVC6B5_LAUNCH_DMA_VPRMODE_VPR_NONE                                      (0x00000000)
 #define NVC6B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID                                   (0x00000001)
-#define NVC6B5_LAUNCH_DMA_VPRMODE_VPR_VID2SYS                                   (0x00000002)
-#define NVC6B5_LAUNCH_DMA_VPRMODE_VPR_SYS2VID                                   (0x00000003)
 #define NVC6B5_LAUNCH_DMA_RESERVED_START_OF_COPY                                24:24
 #define NVC6B5_LAUNCH_DMA_DISABLE_PLC                                           26:26
 #define NVC6B5_LAUNCH_DMA_DISABLE_PLC_FALSE                                     (0x00000000)

kernel-open/nvidia-uvm/clc7b5.h

@@ -1,19 +1,19 @@
 /*******************************************************************************
-    Copyright (c) 1993-2004 NVIDIA Corporation
+    Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
 
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
 
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
@@ -21,8 +21,6 @@
 
 *******************************************************************************/
 
-
-
 #include "nvtypes.h"
 
 #ifndef _clc7b5_h_
@@ -34,69 +32,6 @@ extern "C" {
 
 #define AMPERE_DMA_COPY_B                                                            (0x0000C7B5)
 
-typedef volatile struct _clc7b5_tag0 {
-    NvV32 Reserved00[0x40];
-    NvV32 Nop;                                                                  // 0x00000100 - 0x00000103
-    NvV32 Reserved01[0xF];
-    NvV32 PmTrigger;                                                            // 0x00000140 - 0x00000143
-    NvV32 Reserved02[0x36];
-    NvV32 SetMonitoredFenceType;                                                // 0x0000021C - 0x0000021F
-    NvV32 SetMonitoredFenceSignalAddrBaseUpper;                                 // 0x00000220 - 0x00000223
-    NvV32 SetMonitoredFenceSignalAddrBaseLower;                                 // 0x00000224 - 0x00000227
-    NvV32 Reserved03[0x6];
-    NvV32 SetSemaphoreA;                                                        // 0x00000240 - 0x00000243
-    NvV32 SetSemaphoreB;                                                        // 0x00000244 - 0x00000247
-    NvV32 SetSemaphorePayload;                                                  // 0x00000248 - 0x0000024B
-    NvV32 SetSemaphorePayloadUpper;                                             // 0x0000024C - 0x0000024F
-    NvV32 Reserved04[0x1];
-    NvV32 SetRenderEnableA;                                                     // 0x00000254 - 0x00000257
-    NvV32 SetRenderEnableB;                                                     // 0x00000258 - 0x0000025B
-    NvV32 SetRenderEnableC;                                                     // 0x0000025C - 0x0000025F
-    NvV32 SetSrcPhysMode;                                                       // 0x00000260 - 0x00000263
-    NvV32 SetDstPhysMode;                                                       // 0x00000264 - 0x00000267
-    NvV32 Reserved05[0x6];
-    NvV32 SetGlobalCounterUpper;                                                // 0x00000280 - 0x00000283
-    NvV32 SetGlobalCounterLower;                                                // 0x00000284 - 0x00000287
-    NvV32 SetPageoutStartPAUpper;                                               // 0x00000288 - 0x0000028B
-    NvV32 SetPageoutStartPALower;                                               // 0x0000028C - 0x0000028F
-    NvV32 Reserved06[0x1C];
-    NvV32 LaunchDma;                                                            // 0x00000300 - 0x00000303
-    NvV32 Reserved07[0x3F];
-    NvV32 OffsetInUpper;                                                        // 0x00000400 - 0x00000403
-    NvV32 OffsetInLower;                                                        // 0x00000404 - 0x00000407
-    NvV32 OffsetOutUpper;                                                       // 0x00000408 - 0x0000040B
-    NvV32 OffsetOutLower;                                                       // 0x0000040C - 0x0000040F
-    NvV32 PitchIn;                                                              // 0x00000410 - 0x00000413
-    NvV32 PitchOut;                                                             // 0x00000414 - 0x00000417
-    NvV32 LineLengthIn;                                                         // 0x00000418 - 0x0000041B
-    NvV32 LineCount;                                                            // 0x0000041C - 0x0000041F
-    NvV32 Reserved08[0xB8];
-    NvV32 SetRemapConstA;                                                       // 0x00000700 - 0x00000703
-    NvV32 SetRemapConstB;                                                       // 0x00000704 - 0x00000707
-    NvV32 SetRemapComponents;                                                   // 0x00000708 - 0x0000070B
-    NvV32 SetDstBlockSize;                                                      // 0x0000070C - 0x0000070F
-    NvV32 SetDstWidth;                                                          // 0x00000710 - 0x00000713
-    NvV32 SetDstHeight;                                                         // 0x00000714 - 0x00000717
-    NvV32 SetDstDepth;                                                          // 0x00000718 - 0x0000071B
-    NvV32 SetDstLayer;                                                          // 0x0000071C - 0x0000071F
-    NvV32 SetDstOrigin;                                                         // 0x00000720 - 0x00000723
-    NvV32 Reserved09[0x1];
-    NvV32 SetSrcBlockSize;                                                      // 0x00000728 - 0x0000072B
-    NvV32 SetSrcWidth;                                                          // 0x0000072C - 0x0000072F
-    NvV32 SetSrcHeight;                                                         // 0x00000730 - 0x00000733
-    NvV32 SetSrcDepth;                                                          // 0x00000734 - 0x00000737
-    NvV32 SetSrcLayer;                                                          // 0x00000738 - 0x0000073B
-    NvV32 SetSrcOrigin;                                                         // 0x0000073C - 0x0000073F
-    NvV32 Reserved10[0x1];
-    NvV32 SrcOriginX;                                                           // 0x00000744 - 0x00000747
-    NvV32 SrcOriginY;                                                           // 0x00000748 - 0x0000074B
-    NvV32 DstOriginX;                                                           // 0x0000074C - 0x0000074F
-    NvV32 DstOriginY;                                                           // 0x00000750 - 0x00000753
-    NvV32 Reserved11[0x270];
-    NvV32 PmTriggerEnd;                                                         // 0x00001114 - 0x00001117
-    NvV32 Reserved12[0x3BA];
-} ampere_dma_copy_bControlPio;
-
 #define NVC7B5_NOP                                                              (0x00000100)
 #define NVC7B5_NOP_PARAMETER                                                    31:0
 #define NVC7B5_PM_TRIGGER                                                       (0x00000140)
@@ -146,14 +81,6 @@ typedef volatile struct _clc7b5_tag0 {
 #define NVC7B5_SET_DST_PHYS_MODE_BASIC_KIND                                     5:2
 #define NVC7B5_SET_DST_PHYS_MODE_PEER_ID                                        8:6
 #define NVC7B5_SET_DST_PHYS_MODE_FLA                                            9:9
-#define NVC7B5_SET_GLOBAL_COUNTER_UPPER                                         (0x00000280)
-#define NVC7B5_SET_GLOBAL_COUNTER_UPPER_V                                       31:0
-#define NVC7B5_SET_GLOBAL_COUNTER_LOWER                                         (0x00000284)
-#define NVC7B5_SET_GLOBAL_COUNTER_LOWER_V                                       31:0
-#define NVC7B5_SET_PAGEOUT_START_PAUPPER                                        (0x00000288)
-#define NVC7B5_SET_PAGEOUT_START_PAUPPER_V                                      4:0
-#define NVC7B5_SET_PAGEOUT_START_PALOWER                                        (0x0000028C)
-#define NVC7B5_SET_PAGEOUT_START_PALOWER_V                                      31:0
 #define NVC7B5_LAUNCH_DMA                                                       (0x00000300)
 #define NVC7B5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                    1:0
 #define NVC7B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                               (0x00000000)
@@ -223,8 +150,6 @@ typedef volatile struct _clc7b5_tag0 {
 #define NVC7B5_LAUNCH_DMA_VPRMODE                                               23:22
 #define NVC7B5_LAUNCH_DMA_VPRMODE_VPR_NONE                                      (0x00000000)
 #define NVC7B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID                                   (0x00000001)
-#define NVC7B5_LAUNCH_DMA_VPRMODE_VPR_VID2SYS                                   (0x00000002)
-#define NVC7B5_LAUNCH_DMA_VPRMODE_VPR_SYS2VID                                   (0x00000003)
 #define NVC7B5_LAUNCH_DMA_RESERVED_START_OF_COPY                                24:24
 #define NVC7B5_LAUNCH_DMA_DISABLE_PLC                                           26:26
 #define NVC7B5_LAUNCH_DMA_DISABLE_PLC_FALSE                                     (0x00000000)

kernel-open/nvidia-uvm/clc96f.h

@@ -0,0 +1,329 @@
+/*******************************************************************************
+    Copyright (c) 2012-2015 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+
+#ifndef _clc96f_h_
+#define _clc96f_h_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "nvtypes.h"
+
+/* class BLACKWELL_CHANNEL_GPFIFO  */
+/*
+ * Documentation for BLACKWELL_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
+ * chapter "User Control Registers". It is documented as device NV_UDMA.
+ * The GPFIFO format itself is also documented in dev_pbdma.ref,
+ * NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
+ * chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
+ *
+ * Note there is no .mfs file for this class.
+ */
+#define  BLACKWELL_CHANNEL_GPFIFO_A                           (0x0000C96F)
+
+#define NVC96F_TYPEDEF                             BLACKWELL_CHANNELChannelGPFifoA
+
+/* dma flow control data structure */
+typedef volatile struct Nvc96fControl_struct {
+ NvU32 Ignored00[0x23];        /*                                  0000-008b*/
+ NvU32 GPPut;                   /* GP FIFO put offset               008c-008f*/
+ NvU32 Ignored01[0x5c];
+} Nvc96fControl, BlackwellAControlGPFifo;
+
+/* fields and values */
+#define NVC96F_NUMBER_OF_SUBCHANNELS                               (8)
+#define NVC96F_SET_OBJECT                                          (0x00000000)
+#define NVC96F_SET_OBJECT_NVCLASS                                         15:0
+#define NVC96F_SET_OBJECT_ENGINE                                         20:16
+#define NVC96F_SET_OBJECT_ENGINE_SW                                 0x0000001f
+#define NVC96F_NOP                                                 (0x00000008)
+#define NVC96F_NOP_HANDLE                                                 31:0
+#define NVC96F_NON_STALL_INTERRUPT                                 (0x00000020)
+#define NVC96F_NON_STALL_INTERRUPT_HANDLE                                 31:0
+#define NVC96F_FB_FLUSH                                            (0x00000024) // Deprecated - use MEMBAR TYPE SYS_MEMBAR
+#define NVC96F_FB_FLUSH_HANDLE                                            31:0
+// NOTE - MEM_OP_A and MEM_OP_B have been replaced in gp100 with methods for
+// specifying the page address for a targeted TLB invalidate and the uTLB for
+// a targeted REPLAY_CANCEL for UVM.
+// The previous MEM_OP_A/B functionality is in MEM_OP_C/D, with slightly
+// rearranged fields.
+#define NVC96F_MEM_OP_A                                            (0x00000028)
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID        5:0  // only relevant for REPLAY_CANCEL_TARGETED
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE                   5:0  // Used to specify size of invalidate, used for invalidates which are not of the REPLAY_CANCEL_TARGETED type
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID               10:6  // only relevant for REPLAY_CANCEL_TARGETED
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE                         7:6  // only relevant for invalidates with NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE for invalidating  link TLB only, or non-link TLB only or all TLBs
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_ALL_TLBS                  0
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_LINK_TLBS                 1
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_NON_LINK_TLBS             2
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_RSVRVD                    3
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_MMU_ENGINE_ID                8:0  // only relevant for REPLAY_CANCEL_VA_GLOBAL
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR                         11:11
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN                 0x00000001
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS                0x00000000
+#define NVC96F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO                    31:12
+#define NVC96F_MEM_OP_B                                            (0x0000002c)
+#define NVC96F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI                     31:0
+#define NVC96F_MEM_OP_C                                            (0x00000030)
+#define NVC96F_MEM_OP_C_MEMBAR_TYPE                                        2:0
+#define NVC96F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR                      0x00000000
+#define NVC96F_MEM_OP_C_MEMBAR_TYPE_MEMBAR                          0x00000001
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB                                 0:0
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE                      0x00000000
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL                      0x00000001  // Probably nonsensical for MMU_TLB_INVALIDATE_TARGETED
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC                                 1:1
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE                   0x00000000
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE                  0x00000001
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY                              4:2  // only relevant if GPC ENABLE
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE                  0x00000000
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START                 0x00000001
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL         0x00000002
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED       0x00000003
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL         0x00000004
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_VA_GLOBAL      0x00000005
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE                            6:5  // only relevant if GPC ENABLE
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE                0x00000000
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY            0x00000001
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE           0x00000002
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE                         9:7 //only relevant for REPLAY_CANCEL_VA_GLOBAL
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_READ                 0
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE                1
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_STRONG        2
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_RSVRVD               3
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_WEAK          4
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_ALL           5
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE_AND_ATOMIC     6
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ALL                  7
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL                    9:7  // Invalidate affects this level and all below
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL         0x00000000  // Invalidate tlb caches at all levels of the page table
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY    0x00000001
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0  0x00000002
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1  0x00000003
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2  0x00000004
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3  0x00000005
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4  0x00000006
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5  0x00000007
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE                          11:10  // only relevant if PDB_ONE
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM             0x00000000
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT    0x00000002
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT 0x00000003
+#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO                       31:12  // only relevant if PDB_ONE
+#define NVC96F_MEM_OP_C_ACCESS_COUNTER_CLR_TARGETED_NOTIFY_TAG            19:0
+// MEM_OP_D MUST be preceded by MEM_OPs A-C.
+#define NVC96F_MEM_OP_D                                            (0x00000034)
+#define NVC96F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI                        26:0  // only relevant if PDB_ONE
+#define NVC96F_MEM_OP_D_OPERATION                                        31:27
+#define NVC96F_MEM_OP_D_OPERATION_MEMBAR                            0x00000005
+#define NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE                0x00000009
+#define NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED       0x0000000a
+#define NVC96F_MEM_OP_D_OPERATION_MMU_OPERATION                     0x0000000b
+#define NVC96F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE             0x0000000d
+#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE              0x0000000e
+// CLEAN_LINES is an alias for Tegra/GPU IP usage
+#define NVC96F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES         0x0000000e
+#define NVC96F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS                 0x0000000f
+#define NVC96F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY                    0x00000010
+#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_NCOH_INVALIDATE         0x00000011
+#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_COH_INVALIDATE          0x00000012
+#define NVC96F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS     0x00000015
+#define NVC96F_MEM_OP_D_OPERATION_ACCESS_COUNTER_CLR                0x00000016
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE                            1:0
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MIMC                0x00000000
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MOMC                0x00000001
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_ALL                 0x00000002
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_TARGETED            0x00000003
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE                   2:2
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MIMC       0x00000000
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MOMC       0x00000001
+#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_BANK                   6:3
+#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE                               23:20
+#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE_RESERVED                 0x00000000
+#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE_VIDMEM_ACCESS_BIT_DUMP   0x00000001
+#define NVC96F_SEM_ADDR_LO                                         (0x0000005c)
+#define NVC96F_SEM_ADDR_LO_OFFSET                                         31:2
+#define NVC96F_SEM_ADDR_HI                                         (0x00000060)
+#define NVC96F_SEM_ADDR_HI_OFFSET                                         24:0
+#define NVC96F_SEM_PAYLOAD_LO                                      (0x00000064)
+#define NVC96F_SEM_PAYLOAD_LO_PAYLOAD                                     31:0
+#define NVC96F_SEM_PAYLOAD_HI                                      (0x00000068)
+#define NVC96F_SEM_PAYLOAD_HI_PAYLOAD                                     31:0
+#define NVC96F_SEM_EXECUTE                                         (0x0000006c)
+#define NVC96F_SEM_EXECUTE_OPERATION                                       2:0
+#define NVC96F_SEM_EXECUTE_OPERATION_ACQUIRE                        0x00000000
+#define NVC96F_SEM_EXECUTE_OPERATION_RELEASE                        0x00000001
+#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_STRICT_GEQ                 0x00000002
+#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_CIRC_GEQ                   0x00000003
+#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_AND                        0x00000004
+#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_NOR                        0x00000005
+#define NVC96F_SEM_EXECUTE_OPERATION_REDUCTION                      0x00000006
+#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG                            12:12
+#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_DIS                   0x00000000
+#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_EN                    0x00000001
+#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK                               18:18
+#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK_DIS                      0x00000000
+#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK_EN                       0x00000001
+#define NVC96F_SEM_EXECUTE_RELEASE_WFI                                   20:20
+#define NVC96F_SEM_EXECUTE_RELEASE_WFI_DIS                          0x00000000
+#define NVC96F_SEM_EXECUTE_RELEASE_WFI_EN                           0x00000001
+#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE                                  24:24
+#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT                       0x00000000
+#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE_64BIT                       0x00000001
+#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP                             25:25
+#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP_DIS                    0x00000000
+#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP_EN                     0x00000001
+#define NVC96F_SEM_EXECUTE_REDUCTION                                     30:27
+#define NVC96F_SEM_EXECUTE_REDUCTION_IMIN                           0x00000000
+#define NVC96F_SEM_EXECUTE_REDUCTION_IMAX                           0x00000001
+#define NVC96F_SEM_EXECUTE_REDUCTION_IXOR                           0x00000002
+#define NVC96F_SEM_EXECUTE_REDUCTION_IAND                           0x00000003
+#define NVC96F_SEM_EXECUTE_REDUCTION_IOR                            0x00000004
+#define NVC96F_SEM_EXECUTE_REDUCTION_IADD                           0x00000005
+#define NVC96F_SEM_EXECUTE_REDUCTION_INC                            0x00000006
+#define NVC96F_SEM_EXECUTE_REDUCTION_DEC                            0x00000007
+#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT                              31:31
+#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT_SIGNED                  0x00000000
+#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT_UNSIGNED                0x00000001
+#define NVC96F_WFI                                                 (0x00000078)
+#define NVC96F_WFI_SCOPE                                                   0:0
+#define NVC96F_WFI_SCOPE_CURRENT_SCG_TYPE                           0x00000000
+#define NVC96F_WFI_SCOPE_CURRENT_VEID                               0x00000000
+#define NVC96F_WFI_SCOPE_ALL                                        0x00000001
+#define NVC96F_YIELD                                               (0x00000080)
+#define NVC96F_YIELD_OP                                                    1:0
+#define NVC96F_YIELD_OP_NOP                                         0x00000000
+#define NVC96F_YIELD_OP_TSG                                         0x00000003
+#define NVC96F_CLEAR_FAULTED                                       (0x00000084)
+// Note: RM provides the HANDLE as an opaque value; the internal detail fields
+// are intentionally not exposed to the driver through these defines.
+#define NVC96F_CLEAR_FAULTED_HANDLE                                       30:0
+#define NVC96F_CLEAR_FAULTED_TYPE                                        31:31
+#define NVC96F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED                     0x00000000
+#define NVC96F_CLEAR_FAULTED_TYPE_ENG_FAULTED                       0x00000001
+
+
+/* GPFIFO entry format */
+#define NVC96F_GP_ENTRY__SIZE                                          8
+#define NVC96F_GP_ENTRY0_FETCH                                       0:0
+#define NVC96F_GP_ENTRY0_FETCH_UNCONDITIONAL                  0x00000000
+#define NVC96F_GP_ENTRY0_FETCH_CONDITIONAL                    0x00000001
+#define NVC96F_GP_ENTRY0_GET                                        31:2
+#define NVC96F_GP_ENTRY0_OPERAND                                    31:0
+#define NVC96F_GP_ENTRY0_PB_EXTENDED_BASE_OPERAND                   24:8
+#define NVC96F_GP_ENTRY1_GET_HI                                      7:0
+#define NVC96F_GP_ENTRY1_LEVEL                                       9:9
+#define NVC96F_GP_ENTRY1_LEVEL_MAIN                           0x00000000
+#define NVC96F_GP_ENTRY1_LEVEL_SUBROUTINE                     0x00000001
+#define NVC96F_GP_ENTRY1_LENGTH                                    30:10
+#define NVC96F_GP_ENTRY1_SYNC                                      31:31
+#define NVC96F_GP_ENTRY1_SYNC_PROCEED                         0x00000000
+#define NVC96F_GP_ENTRY1_SYNC_WAIT                            0x00000001
+#define NVC96F_GP_ENTRY1_OPCODE                                      7:0
+#define NVC96F_GP_ENTRY1_OPCODE_NOP                           0x00000000
+#define NVC96F_GP_ENTRY1_OPCODE_ILLEGAL                       0x00000001
+#define NVC96F_GP_ENTRY1_OPCODE_GP_CRC                        0x00000002
+#define NVC96F_GP_ENTRY1_OPCODE_PB_CRC                        0x00000003
+#define NVC96F_GP_ENTRY1_OPCODE_SET_PB_SEGMENT_EXTENDED_BASE  0x00000004
+
+/* dma method formats */
+#define NVC96F_DMA_METHOD_ADDRESS_OLD                              12:2
+#define NVC96F_DMA_METHOD_ADDRESS                                  11:0
+#define NVC96F_DMA_SUBDEVICE_MASK                                  15:4
+#define NVC96F_DMA_METHOD_SUBCHANNEL                               15:13
+#define NVC96F_DMA_TERT_OP                                         17:16
+#define NVC96F_DMA_TERT_OP_GRP0_INC_METHOD                         (0x00000000)
+#define NVC96F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK                   (0x00000001)
+#define NVC96F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK                 (0x00000002)
+#define NVC96F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK                   (0x00000003)
+#define NVC96F_DMA_TERT_OP_GRP2_NON_INC_METHOD                     (0x00000000)
+#define NVC96F_DMA_METHOD_COUNT_OLD                                28:18
+#define NVC96F_DMA_METHOD_COUNT                                    28:16
+#define NVC96F_DMA_IMMD_DATA                                       28:16
+#define NVC96F_DMA_SEC_OP                                          31:29
+#define NVC96F_DMA_SEC_OP_GRP0_USE_TERT                            (0x00000000)
+#define NVC96F_DMA_SEC_OP_INC_METHOD                               (0x00000001)
+#define NVC96F_DMA_SEC_OP_GRP2_USE_TERT                            (0x00000002)
+#define NVC96F_DMA_SEC_OP_NON_INC_METHOD                           (0x00000003)
+#define NVC96F_DMA_SEC_OP_IMMD_DATA_METHOD                         (0x00000004)
+#define NVC96F_DMA_SEC_OP_ONE_INC                                  (0x00000005)
+#define NVC96F_DMA_SEC_OP_RESERVED6                                (0x00000006)
+#define NVC96F_DMA_SEC_OP_END_PB_SEGMENT                           (0x00000007)
+/* dma incrementing method format */
+#define NVC96F_DMA_INCR_ADDRESS                                    11:0
+#define NVC96F_DMA_INCR_SUBCHANNEL                                 15:13
+#define NVC96F_DMA_INCR_COUNT                                      28:16
+#define NVC96F_DMA_INCR_OPCODE                                     31:29
+#define NVC96F_DMA_INCR_OPCODE_VALUE                               (0x00000001)
+#define NVC96F_DMA_INCR_DATA                                       31:0
+/* dma non-incrementing method format */
+#define NVC96F_DMA_NONINCR_ADDRESS                                 11:0
+#define NVC96F_DMA_NONINCR_SUBCHANNEL                              15:13
+#define NVC96F_DMA_NONINCR_COUNT                                   28:16
+#define NVC96F_DMA_NONINCR_OPCODE                                  31:29
+#define NVC96F_DMA_NONINCR_OPCODE_VALUE                            (0x00000003)
+#define NVC96F_DMA_NONINCR_DATA                                    31:0
+/* dma increment-once method format */
+#define NVC96F_DMA_ONEINCR_ADDRESS                                 11:0
+#define NVC96F_DMA_ONEINCR_SUBCHANNEL                              15:13
+#define NVC96F_DMA_ONEINCR_COUNT                                   28:16
+#define NVC96F_DMA_ONEINCR_OPCODE                                  31:29
+#define NVC96F_DMA_ONEINCR_OPCODE_VALUE                            (0x00000005)
+#define NVC96F_DMA_ONEINCR_DATA                                    31:0
+/* dma no-operation format */
+#define NVC96F_DMA_NOP                                             (0x00000000)
+/* dma immediate-data format */
+#define NVC96F_DMA_IMMD_ADDRESS                                    11:0
+#define NVC96F_DMA_IMMD_SUBCHANNEL                                 15:13
+#define NVC96F_DMA_IMMD_DATA                                       28:16
+#define NVC96F_DMA_IMMD_OPCODE                                     31:29
+#define NVC96F_DMA_IMMD_OPCODE_VALUE                               (0x00000004)
+/* dma set sub-device mask format */
+#define NVC96F_DMA_SET_SUBDEVICE_MASK_VALUE                        15:4
+#define NVC96F_DMA_SET_SUBDEVICE_MASK_OPCODE                       31:16
+#define NVC96F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE                 (0x00000001)
+/* dma store sub-device mask format */
+#define NVC96F_DMA_STORE_SUBDEVICE_MASK_VALUE                      15:4
+#define NVC96F_DMA_STORE_SUBDEVICE_MASK_OPCODE                     31:16
+#define NVC96F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE               (0x00000002)
+/* dma use sub-device mask format */
+#define NVC96F_DMA_USE_SUBDEVICE_MASK_OPCODE                       31:16
+#define NVC96F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE                 (0x00000003)
+/* dma end-segment format */
+#define NVC96F_DMA_ENDSEG_OPCODE                                   31:29
+#define NVC96F_DMA_ENDSEG_OPCODE_VALUE                             (0x00000007)
+/* dma legacy incrementing/non-incrementing formats */
+#define NVC96F_DMA_ADDRESS                                         12:2
+#define NVC96F_DMA_SUBCH                                           15:13
+#define NVC96F_DMA_OPCODE3                                         17:16
+#define NVC96F_DMA_OPCODE3_NONE                                    (0x00000000)
+#define NVC96F_DMA_COUNT                                           28:18
+#define NVC96F_DMA_OPCODE                                          31:29
+#define NVC96F_DMA_OPCODE_METHOD                                   (0x00000000)
+#define NVC96F_DMA_OPCODE_NONINC_METHOD                            (0x00000002)
+#define NVC96F_DMA_DATA                                            31:0
+
+#ifdef __cplusplus
+};     /* extern "C" */
+#endif
+
+#endif /* _clc96f_h_ */

kernel-open/nvidia-uvm/clc9b5.h

@@ -0,0 +1,460 @@
+/*******************************************************************************
+    Copyright (c) 1993-2004 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+
+
+#include "nvtypes.h"
+
+#ifndef _clc9b5_h_
+#define _clc9b5_h_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define BLACKWELL_DMA_COPY_A                                                            (0x0000C9B5)
+
+typedef volatile struct _clc9b5_tag0 {
+    NvV32 Reserved00[0x40];
+    NvV32 Nop;                                                                  // 0x00000100 - 0x00000103
+    NvV32 Reserved01[0xF];
+    NvV32 PmTrigger;                                                            // 0x00000140 - 0x00000143
+    NvV32 Reserved02[0x36];
+    NvV32 SetMonitoredFenceType;                                                // 0x0000021C - 0x0000021F
+    NvV32 SetMonitoredFenceSignalAddrBaseUpper;                                 // 0x00000220 - 0x00000223
+    NvV32 SetMonitoredFenceSignalAddrBaseLower;                                 // 0x00000224 - 0x00000227
+    NvV32 Reserved03[0x6];
+    NvV32 SetSemaphoreA;                                                        // 0x00000240 - 0x00000243
+    NvV32 SetSemaphoreB;                                                        // 0x00000244 - 0x00000247
+    NvV32 SetSemaphorePayload;                                                  // 0x00000248 - 0x0000024B
+    NvV32 SetSemaphorePayloadUpper;                                             // 0x0000024C - 0x0000024F
+    NvV32 Reserved04[0x1];
+    NvV32 SetRenderEnableA;                                                     // 0x00000254 - 0x00000257
+    NvV32 SetRenderEnableB;                                                     // 0x00000258 - 0x0000025B
+    NvV32 SetRenderEnableC;                                                     // 0x0000025C - 0x0000025F
+    NvV32 SetSrcPhysMode;                                                       // 0x00000260 - 0x00000263
+    NvV32 SetDstPhysMode;                                                       // 0x00000264 - 0x00000267
+    NvV32 Reserved05[0x26];
+    NvV32 LaunchDma;                                                            // 0x00000300 - 0x00000303
+    NvV32 Reserved06[0x3F];
+    NvV32 OffsetInUpper;                                                        // 0x00000400 - 0x00000403
+    NvV32 OffsetInLower;                                                        // 0x00000404 - 0x00000407
+    NvV32 OffsetOutUpper;                                                       // 0x00000408 - 0x0000040B
+    NvV32 OffsetOutLower;                                                       // 0x0000040C - 0x0000040F
+    NvV32 PitchIn;                                                              // 0x00000410 - 0x00000413
+    NvV32 PitchOut;                                                             // 0x00000414 - 0x00000417
+    NvV32 LineLengthIn;                                                         // 0x00000418 - 0x0000041B
+    NvV32 LineCount;                                                            // 0x0000041C - 0x0000041F
+    NvV32 Reserved07[0x38];
+    NvV32 SetSecureCopyMode;                                                    // 0x00000500 - 0x00000503
+    NvV32 SetDecryptIv0;                                                        // 0x00000504 - 0x00000507
+    NvV32 SetDecryptIv1;                                                        // 0x00000508 - 0x0000050B
+    NvV32 SetDecryptIv2;                                                        // 0x0000050C - 0x0000050F
+    NvV32 Reserved_SetAESCounter;                                               // 0x00000510 - 0x00000513
+    NvV32 SetDecryptAuthTagCompareAddrUpper;                                    // 0x00000514 - 0x00000517
+    NvV32 SetDecryptAuthTagCompareAddrLower;                                    // 0x00000518 - 0x0000051B
+    NvV32 Reserved08[0x5];
+    NvV32 SetEncryptAuthTagAddrUpper;                                           // 0x00000530 - 0x00000533
+    NvV32 SetEncryptAuthTagAddrLower;                                           // 0x00000534 - 0x00000537
+    NvV32 SetEncryptIvAddrUpper;                                                // 0x00000538 - 0x0000053B
+    NvV32 SetEncryptIvAddrLower;                                                // 0x0000053C - 0x0000053F
+    NvV32 Reserved09[0x10];
+    NvV32 SetCompressionParameters;                                             // 0x00000580 - 0x00000583
+    NvV32 SetDecompressOutLength;                                               // 0x00000584 - 0x00000587
+    NvV32 SetDecompressOutLengthAddrUpper;                                      // 0x00000588 - 0x0000058B
+    NvV32 SetDecompressOutLengthAddrLower;                                      // 0x0000058C - 0x0000058F
+    NvV32 SetDecompressChecksum;                                                // 0x00000590 - 0x00000593
+    NvV32 Reserved10[0x5A];
+    NvV32 SetMemoryScrubParameters;                                             // 0x000006FC - 0x000006FF
+    NvV32 SetRemapConstA;                                                       // 0x00000700 - 0x00000703
+    NvV32 SetRemapConstB;                                                       // 0x00000704 - 0x00000707
+    NvV32 SetRemapComponents;                                                   // 0x00000708 - 0x0000070B
+    NvV32 SetDstBlockSize;                                                      // 0x0000070C - 0x0000070F
+    NvV32 SetDstWidth;                                                          // 0x00000710 - 0x00000713
+    NvV32 SetDstHeight;                                                         // 0x00000714 - 0x00000717
+    NvV32 SetDstDepth;                                                          // 0x00000718 - 0x0000071B
+    NvV32 SetDstLayer;                                                          // 0x0000071C - 0x0000071F
+    NvV32 SetDstOrigin;                                                         // 0x00000720 - 0x00000723
+    NvV32 Reserved11[0x1];
+    NvV32 SetSrcBlockSize;                                                      // 0x00000728 - 0x0000072B
+    NvV32 SetSrcWidth;                                                          // 0x0000072C - 0x0000072F
+    NvV32 SetSrcHeight;                                                         // 0x00000730 - 0x00000733
+    NvV32 SetSrcDepth;                                                          // 0x00000734 - 0x00000737
+    NvV32 SetSrcLayer;                                                          // 0x00000738 - 0x0000073B
+    NvV32 SetSrcOrigin;                                                         // 0x0000073C - 0x0000073F
+    NvV32 Reserved12[0x1];
+    NvV32 SrcOriginX;                                                           // 0x00000744 - 0x00000747
+    NvV32 SrcOriginY;                                                           // 0x00000748 - 0x0000074B
+    NvV32 DstOriginX;                                                           // 0x0000074C - 0x0000074F
+    NvV32 DstOriginY;                                                           // 0x00000750 - 0x00000753
+    NvV32 Reserved13[0x270];
+    NvV32 PmTriggerEnd;                                                         // 0x00001114 - 0x00001117
+    NvV32 Reserved14[0x3BA];
+} blackwell_dma_copy_aControlPio;
+
+#define NVC9B5_NOP                                                              (0x00000100)
+#define NVC9B5_NOP_PARAMETER                                                    31:0
+#define NVC9B5_PM_TRIGGER                                                       (0x00000140)
+#define NVC9B5_PM_TRIGGER_V                                                     31:0
+#define NVC9B5_SET_MONITORED_FENCE_TYPE                                         (0x0000021C)
+#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE                                    0:0
+#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE_MONITORED_FENCE                    (0x00000000)
+#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE_MONITORED_FENCE_EXT                (0x00000001)
+#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_UPPER                       (0x00000220)
+#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_UPPER_UPPER                 24:0
+#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_LOWER                       (0x00000224)
+#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_LOWER_LOWER                 31:0
+#define NVC9B5_SET_SEMAPHORE_A                                                  (0x00000240)
+#define NVC9B5_SET_SEMAPHORE_A_UPPER                                            24:0
+#define NVC9B5_SET_SEMAPHORE_B                                                  (0x00000244)
+#define NVC9B5_SET_SEMAPHORE_B_LOWER                                            31:0
+#define NVC9B5_SET_SEMAPHORE_PAYLOAD                                            (0x00000248)
+#define NVC9B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD                                    31:0
+#define NVC9B5_SET_SEMAPHORE_PAYLOAD_UPPER                                      (0x0000024C)
+#define NVC9B5_SET_SEMAPHORE_PAYLOAD_UPPER_PAYLOAD                              31:0
+#define NVC9B5_SET_RENDER_ENABLE_A                                              (0x00000254)
+#define NVC9B5_SET_RENDER_ENABLE_A_UPPER                                        24:0
+#define NVC9B5_SET_RENDER_ENABLE_B                                              (0x00000258)
+#define NVC9B5_SET_RENDER_ENABLE_B_LOWER                                        31:0
+#define NVC9B5_SET_RENDER_ENABLE_C                                              (0x0000025C)
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE                                         2:0
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE_FALSE                                   (0x00000000)
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE_TRUE                                    (0x00000001)
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL                             (0x00000002)
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL                         (0x00000003)
+#define NVC9B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL                     (0x00000004)
+#define NVC9B5_SET_SRC_PHYS_MODE                                                (0x00000260)
+#define NVC9B5_SET_SRC_PHYS_MODE_TARGET                                         1:0
+#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB                                (0x00000000)
+#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM                         (0x00000001)
+#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM                      (0x00000002)
+#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_PEERMEM                                 (0x00000003)
+#define NVC9B5_SET_SRC_PHYS_MODE_BASIC_KIND                                     5:2
+#define NVC9B5_SET_SRC_PHYS_MODE_PEER_ID                                        8:6
+#define NVC9B5_SET_SRC_PHYS_MODE_FLA                                            9:9
+#define NVC9B5_SET_DST_PHYS_MODE                                                (0x00000264)
+#define NVC9B5_SET_DST_PHYS_MODE_TARGET                                         1:0
+#define NVC9B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB                                (0x00000000)
+#define NVC9B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM                         (0x00000001)
+#define NVC9B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM                      (0x00000002)
+#define NVC9B5_SET_DST_PHYS_MODE_TARGET_PEERMEM                                 (0x00000003)
+#define NVC9B5_SET_DST_PHYS_MODE_BASIC_KIND                                     5:2
+#define NVC9B5_SET_DST_PHYS_MODE_PEER_ID                                        8:6
+#define NVC9B5_SET_DST_PHYS_MODE_FLA                                            9:9
+#define NVC9B5_LAUNCH_DMA                                                       (0x00000300)
+#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                    1:0
+#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                               (0x00000000)
+#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED                          (0x00000001)
+#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED                      (0x00000002)
+#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE                                          2:2
+#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE                                    (0x00000000)
+#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE                                     (0x00000001)
+#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE                                            25:25
+#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE_SYS                                        (0x00000000)
+#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE_GL                                         (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE                                        4:3
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE                                   (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_SEMAPHORE_NO_TIMESTAMP         (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_SEMAPHORE_WITH_TIMESTAMP       (0x00000002)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE             (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE            (0x00000002)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_CONDITIONAL_INTR_SEMAPHORE     (0x00000003)
+#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE                                        6:5
+#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE                                   (0x00000000)
+#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING                               (0x00000001)
+#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING                           (0x00000002)
+#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT                                     7:7
+#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR                         (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH                               (0x00000001)
+#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT                                     8:8
+#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR                         (0x00000000)
+#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH                               (0x00000001)
+#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE                                     9:9
+#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE                               (0x00000000)
+#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE                                (0x00000001)
+#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE                                          10:10
+#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE_FALSE                                    (0x00000000)
+#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE_TRUE                                     (0x00000001)
+#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE                                    11:11
+#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE_FALSE                              (0x00000000)
+#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE_TRUE                               (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SRC_TYPE                                              12:12
+#define NVC9B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL                                      (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL                                     (0x00000001)
+#define NVC9B5_LAUNCH_DMA_DST_TYPE                                              13:13
+#define NVC9B5_LAUNCH_DMA_DST_TYPE_VIRTUAL                                      (0x00000000)
+#define NVC9B5_LAUNCH_DMA_DST_TYPE_PHYSICAL                                     (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION                                   17:14
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN                              (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX                              (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR                              (0x00000002)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND                              (0x00000003)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR                               (0x00000004)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD                              (0x00000005)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC                               (0x00000006)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC                               (0x00000007)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDA                          (0x00000008)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDB                          (0x00000009)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD                              (0x0000000A)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FMIN                              (0x0000000B)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FMAX                              (0x0000000C)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDC                          (0x0000000D)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDD                          (0x0000000E)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDE                          (0x0000000F)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN                              18:18
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED                       (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED                     (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE                            19:19
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE                      (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE                       (0x00000001)
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE                                             21:20
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE_PROT2PROT                                   (0x00000000)
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE_DEFAULT                                     (0x00000000)
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE_SECURE                                      (0x00000001)
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE_NONPROT2NONPROT                             (0x00000002)
+#define NVC9B5_LAUNCH_DMA_COPY_TYPE_RESERVED                                    (0x00000003)
+#define NVC9B5_LAUNCH_DMA_VPRMODE                                               22:22
+#define NVC9B5_LAUNCH_DMA_VPRMODE_VPR_NONE                                      (0x00000000)
+#define NVC9B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID                                   (0x00000001)
+#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE                                   23:23
+#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE_FALSE                             (0x00000000)
+#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE_TRUE                              (0x00000001)
+#define NVC9B5_LAUNCH_DMA_RESERVED_START_OF_COPY                                24:24
+#define NVC9B5_LAUNCH_DMA_DISABLE_PLC                                           26:26
+#define NVC9B5_LAUNCH_DMA_DISABLE_PLC_FALSE                                     (0x00000000)
+#define NVC9B5_LAUNCH_DMA_DISABLE_PLC_TRUE                                      (0x00000001)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE                                27:27
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE_ONE_WORD                       (0x00000000)
+#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE_TWO_WORD                       (0x00000001)
+#define NVC9B5_LAUNCH_DMA_RESERVED_ERR_CODE                                     31:28
+#define NVC9B5_OFFSET_IN_UPPER                                                  (0x00000400)
+#define NVC9B5_OFFSET_IN_UPPER_UPPER                                            24:0
+#define NVC9B5_OFFSET_IN_LOWER                                                  (0x00000404)
+#define NVC9B5_OFFSET_IN_LOWER_VALUE                                            31:0
+#define NVC9B5_OFFSET_OUT_UPPER                                                 (0x00000408)
+#define NVC9B5_OFFSET_OUT_UPPER_UPPER                                           24:0
+#define NVC9B5_OFFSET_OUT_LOWER                                                 (0x0000040C)
+#define NVC9B5_OFFSET_OUT_LOWER_VALUE                                           31:0
+#define NVC9B5_PITCH_IN                                                         (0x00000410)
+#define NVC9B5_PITCH_IN_VALUE                                                   31:0
+#define NVC9B5_PITCH_OUT                                                        (0x00000414)
+#define NVC9B5_PITCH_OUT_VALUE                                                  31:0
+#define NVC9B5_LINE_LENGTH_IN                                                   (0x00000418)
+#define NVC9B5_LINE_LENGTH_IN_VALUE                                             31:0
+#define NVC9B5_LINE_COUNT                                                       (0x0000041C)
+#define NVC9B5_LINE_COUNT_VALUE                                                 31:0
+#define NVC9B5_SET_SECURE_COPY_MODE                                             (0x00000500)
+#define NVC9B5_SET_SECURE_COPY_MODE_MODE                                        0:0
+#define NVC9B5_SET_SECURE_COPY_MODE_MODE_ENCRYPT                                (0x00000000)
+#define NVC9B5_SET_SECURE_COPY_MODE_MODE_DECRYPT                                (0x00000001)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET                         20:19
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_LOCAL_FB                (0x00000000)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_COHERENT_SYSMEM         (0x00000001)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_NONCOHERENT_SYSMEM      (0x00000002)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_PEERMEM                 (0x00000003)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_PEER_ID                        23:21
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_FLA                            24:24
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET                         26:25
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_LOCAL_FB                (0x00000000)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_COHERENT_SYSMEM         (0x00000001)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_NONCOHERENT_SYSMEM      (0x00000002)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_PEERMEM                 (0x00000003)
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_PEER_ID                        29:27
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_FLA                            30:30
+#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_END_OF_COPY                        31:31
+#define NVC9B5_SET_DECRYPT_IV0                                                  (0x00000504)
+#define NVC9B5_SET_DECRYPT_IV0_VALUE                                            31:0
+#define NVC9B5_SET_DECRYPT_IV1                                                  (0x00000508)
+#define NVC9B5_SET_DECRYPT_IV1_VALUE                                            31:0
+#define NVC9B5_SET_DECRYPT_IV2                                                  (0x0000050C)
+#define NVC9B5_SET_DECRYPT_IV2_VALUE                                            31:0
+#define NVC9B5_RESERVED_SET_AESCOUNTER                                          (0x00000510)
+#define NVC9B5_RESERVED_SET_AESCOUNTER_VALUE                                    31:0
+#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER                          (0x00000514)
+#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER_UPPER                    24:0
+#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER                          (0x00000518)
+#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER_LOWER                    31:0
+#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER                                  (0x00000530)
+#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER_UPPER                            24:0
+#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER                                  (0x00000534)
+#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER_LOWER                            31:0
+#define NVC9B5_SET_ENCRYPT_IV_ADDR_UPPER                                        (0x00000538)
+#define NVC9B5_SET_ENCRYPT_IV_ADDR_UPPER_UPPER                                  24:0
+#define NVC9B5_SET_ENCRYPT_IV_ADDR_LOWER                                        (0x0000053C)
+#define NVC9B5_SET_ENCRYPT_IV_ADDR_LOWER_LOWER                                  31:0
+#define NVC9B5_SET_COMPRESSION_PARAMETERS                                       (0x00000580)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION                             0:0
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION_DECOMPRESS                  (0x00000000)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION_COMPRESS                    (0x00000001)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO                                  3:1
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_SNAPPY                           (0x00000000)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_DATA_ONLY                    (0x00000001)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_BLOCK                        (0x00000002)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_BLOCK_CHECKSUM               (0x00000003)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_DEFLATE                          (0x00000004)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_SNAPPY_WITH_LONG_FETCH           (0x00000005)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM                             29:28
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_NONE                        (0x00000000)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_ADLER32                     (0x00000001)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_CRC32                       (0x00000002)
+#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_SNAPPY_CRC                  (0x00000003)
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH                                        (0x00000584)
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_V                                      31:0
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_UPPER                             (0x00000588)
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_UPPER_UPPER                       24:0
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_LOWER                             (0x0000058C)
+#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_LOWER_LOWER                       31:0
+#define NVC9B5_SET_DECOMPRESS_CHECKSUM                                          (0x00000590)
+#define NVC9B5_SET_DECOMPRESS_CHECKSUM_V                                        31:0
+#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS                                      (0x000006FC)
+#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE                          0:0
+#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE_FALSE                    (0x00000000)
+#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE_TRUE                     (0x00000001)
+#define NVC9B5_SET_REMAP_CONST_A                                                (0x00000700)
+#define NVC9B5_SET_REMAP_CONST_A_V                                              31:0
+#define NVC9B5_SET_REMAP_CONST_B                                                (0x00000704)
+#define NVC9B5_SET_REMAP_CONST_B_V                                              31:0
+#define NVC9B5_SET_REMAP_COMPONENTS                                             (0x00000708)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X                                       2:0
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_X                                 (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y                                 (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z                                 (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_W                                 (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_CONST_A                               (0x00000004)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_CONST_B                               (0x00000005)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE                              (0x00000006)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y                                       6:4
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X                                 (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y                                 (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z                                 (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W                                 (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A                               (0x00000004)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B                               (0x00000005)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE                              (0x00000006)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z                                       10:8
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X                                 (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y                                 (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z                                 (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W                                 (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A                               (0x00000004)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B                               (0x00000005)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE                              (0x00000006)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W                                       14:12
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_X                                 (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y                                 (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z                                 (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_W                                 (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_CONST_A                               (0x00000004)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_CONST_B                               (0x00000005)
+#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE                              (0x00000006)
+#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE                              17:16
+#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE                          (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO                          (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE                        (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR                         (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS                          21:20
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE                      (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO                      (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE                    (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR                     (0x00000003)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS                          25:24
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE                      (0x00000000)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO                      (0x00000001)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE                    (0x00000002)
+#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR                     (0x00000003)
+#define NVC9B5_SET_DST_BLOCK_SIZE                                               (0x0000070C)
+#define NVC9B5_SET_DST_BLOCK_SIZE_WIDTH                                         3:0
+#define NVC9B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB                                 (0x00000000)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT                                        7:4
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB                                (0x00000000)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS                               (0x00000001)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS                              (0x00000002)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS                             (0x00000003)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS                           (0x00000004)
+#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS                         (0x00000005)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH                                         11:8
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB                                 (0x00000000)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS                                (0x00000001)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS                               (0x00000002)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS                              (0x00000003)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS                            (0x00000004)
+#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS                          (0x00000005)
+#define NVC9B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT                                    15:12
+#define NVC9B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8                 (0x00000001)
+#define NVC9B5_SET_DST_WIDTH                                                    (0x00000710)
+#define NVC9B5_SET_DST_WIDTH_V                                                  31:0
+#define NVC9B5_SET_DST_HEIGHT                                                   (0x00000714)
+#define NVC9B5_SET_DST_HEIGHT_V                                                 31:0
+#define NVC9B5_SET_DST_DEPTH                                                    (0x00000718)
+#define NVC9B5_SET_DST_DEPTH_V                                                  31:0
+#define NVC9B5_SET_DST_LAYER                                                    (0x0000071C)
+#define NVC9B5_SET_DST_LAYER_V                                                  31:0
+#define NVC9B5_SET_DST_ORIGIN                                                   (0x00000720)
+#define NVC9B5_SET_DST_ORIGIN_X                                                 15:0
+#define NVC9B5_SET_DST_ORIGIN_Y                                                 31:16
+#define NVC9B5_SET_SRC_BLOCK_SIZE                                               (0x00000728)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_WIDTH                                         3:0
+#define NVC9B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB                                 (0x00000000)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT                                        7:4
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB                                (0x00000000)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS                               (0x00000001)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS                              (0x00000002)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS                             (0x00000003)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS                           (0x00000004)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS                         (0x00000005)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH                                         11:8
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB                                 (0x00000000)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS                                (0x00000001)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS                               (0x00000002)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS                              (0x00000003)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS                            (0x00000004)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS                          (0x00000005)
+#define NVC9B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT                                    15:12
+#define NVC9B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8                 (0x00000001)
+#define NVC9B5_SET_SRC_WIDTH                                                    (0x0000072C)
+#define NVC9B5_SET_SRC_WIDTH_V                                                  31:0
+#define NVC9B5_SET_SRC_HEIGHT                                                   (0x00000730)
+#define NVC9B5_SET_SRC_HEIGHT_V                                                 31:0
+#define NVC9B5_SET_SRC_DEPTH                                                    (0x00000734)
+#define NVC9B5_SET_SRC_DEPTH_V                                                  31:0
+#define NVC9B5_SET_SRC_LAYER                                                    (0x00000738)
+#define NVC9B5_SET_SRC_LAYER_V                                                  31:0
+#define NVC9B5_SET_SRC_ORIGIN                                                   (0x0000073C)
+#define NVC9B5_SET_SRC_ORIGIN_X                                                 15:0
+#define NVC9B5_SET_SRC_ORIGIN_Y                                                 31:16
+#define NVC9B5_SRC_ORIGIN_X                                                     (0x00000744)
+#define NVC9B5_SRC_ORIGIN_X_VALUE                                               31:0
+#define NVC9B5_SRC_ORIGIN_Y                                                     (0x00000748)
+#define NVC9B5_SRC_ORIGIN_Y_VALUE                                               31:0
+#define NVC9B5_DST_ORIGIN_X                                                     (0x0000074C)
+#define NVC9B5_DST_ORIGIN_X_VALUE                                               31:0
+#define NVC9B5_DST_ORIGIN_Y                                                     (0x00000750)
+#define NVC9B5_DST_ORIGIN_Y_VALUE                                               31:0
+#define NVC9B5_PM_TRIGGER_END                                                   (0x00001114)
+#define NVC9B5_PM_TRIGGER_END_V                                                 31:0
+
+#ifdef __cplusplus
+};     /* extern "C" */
+#endif
+#endif // _clc9b5_h
+

kernel-open/nvidia-uvm/clca6f.h

@@ -0,0 +1,74 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2003-2023 NVIDIA CORPORATION & AFFILIATES
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __gb202_clca6f_h__
+#define __gb202_clca6f_h__
+
+typedef volatile struct Nvca6fControl_struct {
+ NvU32 Ignored00[0x23];        /*                                  0000-008b*/
+ NvU32 GPPut;                   /* GP FIFO put offset               008c-008f*/
+ NvU32 Ignored01[0x5c];
+} Nvca6fControl, BlackwellBControlGPFifo;
+
+#define  BLACKWELL_CHANNEL_GPFIFO_B                           (0x0000CA6F)
+
+#define NVCA6F_SET_OBJECT                                          (0x00000000)
+#define NVCA6F_SEM_ADDR_LO                                         (0x0000005c)
+#define NVCA6F_SEM_ADDR_LO_OFFSET                                         31:2
+#define NVCA6F_SEM_ADDR_HI                                         (0x00000060)
+#define NVCA6F_SEM_ADDR_HI_OFFSET                                         24:0
+#define NVCA6F_SEM_PAYLOAD_LO                                      (0x00000064)
+#define NVCA6F_SEM_PAYLOAD_HI                                      (0x00000068)
+#define NVCA6F_SEM_EXECUTE                                         (0x0000006c)
+#define NVCA6F_SEM_EXECUTE_OPERATION                                       2:0
+#define NVCA6F_SEM_EXECUTE_OPERATION_ACQUIRE                        0x00000000
+#define NVCA6F_SEM_EXECUTE_OPERATION_RELEASE                        0x00000001
+#define NVCA6F_SEM_EXECUTE_RELEASE_WFI                                   20:20
+#define NVCA6F_SEM_EXECUTE_RELEASE_WFI_DIS                          0x00000000
+#define NVCA6F_SEM_EXECUTE_PAYLOAD_SIZE                                  24:24
+#define NVCA6F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT                       0x00000000
+
+/* GPFIFO entry format */
+#define NVCA6F_GP_ENTRY__SIZE                                          8
+#define NVCA6F_GP_ENTRY0_FETCH                                       0:0
+#define NVCA6F_GP_ENTRY0_FETCH_UNCONDITIONAL                  0x00000000
+#define NVCA6F_GP_ENTRY0_FETCH_CONDITIONAL                    0x00000001
+#define NVCA6F_GP_ENTRY0_GET                                        31:2
+#define NVCA6F_GP_ENTRY0_OPERAND                                    31:0
+#define NVCA6F_GP_ENTRY0_PB_EXTENDED_BASE_OPERAND                   24:8
+#define NVCA6F_GP_ENTRY1_GET_HI                                      7:0
+#define NVCA6F_GP_ENTRY1_LEVEL                                       9:9
+#define NVCA6F_GP_ENTRY1_LEVEL_MAIN                           0x00000000
+#define NVCA6F_GP_ENTRY1_LEVEL_SUBROUTINE                     0x00000001
+#define NVCA6F_GP_ENTRY1_LENGTH                                    30:10
+#define NVCA6F_GP_ENTRY1_SYNC                                      31:31
+#define NVCA6F_GP_ENTRY1_SYNC_PROCEED                         0x00000000
+#define NVCA6F_GP_ENTRY1_SYNC_WAIT                            0x00000001
+#define NVCA6F_GP_ENTRY1_OPCODE                                      7:0
+#define NVCA6F_GP_ENTRY1_OPCODE_NOP                           0x00000000
+#define NVCA6F_GP_ENTRY1_OPCODE_ILLEGAL                       0x00000001
+#define NVCA6F_GP_ENTRY1_OPCODE_GP_CRC                        0x00000002
+#define NVCA6F_GP_ENTRY1_OPCODE_PB_CRC                        0x00000003
+#define NVCA6F_GP_ENTRY1_OPCODE_SET_PB_SEGMENT_EXTENDED_BASE  0x00000004
+
+#endif // __gb202_clca6f_h__

kernel-open/nvidia-uvm/clcab5.h

@@ -0,0 +1,44 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _clcab5_h_
+#define _clcab5_h_
+
+#define BLACKWELL_DMA_COPY_B                                                            (0x0000CAB5)
+
+#define NVCAB5_LAUNCH_DMA                                                               (0x00000300)
+#define NVCAB5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                            1:0
+#define NVCAB5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                                       (0x00000000)
+#define NVCAB5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED                                  (0x00000001)
+#define NVCAB5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED                              (0x00000002)
+#define NVCAB5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PREFETCH                                   (0x00000003)
+
+#define NVCAB5_REQ_ATTR                                                                 (0x00000754)
+#define NVCAB5_REQ_ATTR_PREFETCH_L2_CLASS                                               1:0
+#define NVCAB5_REQ_ATTR_PREFETCH_L2_CLASS_EVICT_FIRST                                   (0x00000000)
+#define NVCAB5_REQ_ATTR_PREFETCH_L2_CLASS_EVICT_NORMAL                                  (0x00000001)
+#define NVCAB5_REQ_ATTR_PREFETCH_L2_CLASS_EVICT_LAST                                    (0x00000002)
+#define NVCAB5_REQ_ATTR_PREFETCH_L2_CLASS_EVICT_DEMOTE                                  (0x00000003)
+
+#endif /* _clcab5_h_ */
+

kernel-open/nvidia-uvm/clcba2.h

@@ -1,25 +1,25 @@
-/*******************************************************************************
-    Copyright (c) 2021-2022 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
 #include "nvtypes.h"
 
@@ -32,6 +32,28 @@ extern "C" {
 
 #define HOPPER_SEC2_WORK_LAUNCH_A                                               (0x0000CBA2)
 
+typedef volatile struct _clcba2_tag0 {
+    NvV32 Reserved00[0x100];
+    NvV32 DecryptCopySrcAddrHi;                                                 // 0x00000400 - 0x00000403
+    NvV32 DecryptCopySrcAddrLo;                                                 // 0x00000404 - 0x00000407
+    NvV32 DecryptCopyDstAddrHi;                                                 // 0x00000408 - 0x0000040B
+    NvV32 DecryptCopyDstAddrLo;                                                 // 0x0000040c - 0x0000040F
+    NvU32 DecryptCopySize;                                                      // 0x00000410 - 0x00000413
+    NvU32 DecryptCopyAuthTagAddrHi;                                             // 0x00000414 - 0x00000417
+    NvU32 DecryptCopyAuthTagAddrLo;                                             // 0x00000418 - 0x0000041B
+    NvV32 DigestAddrHi;                                                         // 0x0000041C - 0x0000041F
+    NvV32 DigestAddrLo;                                                         // 0x00000420 - 0x00000423
+    NvV32 Reserved01[0x7];
+    NvV32 SemaphoreA;                                                           // 0x00000440 - 0x00000443
+    NvV32 SemaphoreB;                                                           // 0x00000444 - 0x00000447
+    NvV32 SemaphoreSetPayloadLower;                                             // 0x00000448 - 0x0000044B
+    NvV32 SemaphoreSetPayloadUppper;                                            // 0x0000044C - 0x0000044F
+    NvV32 SemaphoreD;                                                           // 0x00000450 - 0x00000453
+    NvU32 Reserved02[0x7];
+    NvV32 Execute;                                                              // 0x00000470 - 0x00000473
+    NvV32 Reserved03[0x23];
+} NVCBA2_HOPPER_SEC2_WORK_LAUNCH_AControlPio;
+
 #define NVCBA2_DECRYPT_COPY_SRC_ADDR_HI                                         (0x00000400)
 #define NVCBA2_DECRYPT_COPY_SRC_ADDR_HI_DATA                                    24:0
 #define NVCBA2_DECRYPT_COPY_SRC_ADDR_LO                                         (0x00000404)
@@ -90,6 +112,45 @@ extern "C" {
 #define NVCBA2_EXECUTE_TIMESTAMP                                                5:5
 #define NVCBA2_EXECUTE_TIMESTAMP_DISABLE                                        (0x00000000)
 #define NVCBA2_EXECUTE_TIMESTAMP_ENABLE                                         (0x00000001)
+#define NVCBA2_EXECUTE_PHYSICAL_SCRUBBER                                        6:6
+#define NVCBA2_EXECUTE_PHYSICAL_SCRUBBER_DISABLE                                (0x00000000)
+#define NVCBA2_EXECUTE_PHYSICAL_SCRUBBER_ENABLE                                 (0x00000001)
+
+// Class definitions
+#define NVCBA2_DECRYPT_COPY_SIZE_MAX_BYTES                                      (2*1024*1024)
+#define NVCBA2_DECRYPT_SCRUB_SIZE_MAX_BYTES                                     (1024*1024*1024)
+
+// Errors
+#define NVCBA2_ERROR_NONE                                                       (0x00000000)
+#define NVCBA2_ERROR_DECRYPT_COPY_SRC_ADDR_MISALIGNED_POINTER                   (0x00000001)
+#define NVCBA2_ERROR_DECRYPT_COPY_DEST_ADDR_MISALIGNED_POINTER                  (0x00000002)
+#define NVCBA2_ERROR_DECRYPT_COPY_AUTH_TAG_ADDR_MISALIGNED_POINTER              (0x00000003)
+#define NVCBA2_ERROR_DECRYPT_COPY_DMA_NACK                                      (0x00000004)
+#define NVCBA2_ERROR_DECRYPT_COPY_AUTH_TAG_MISMATCH                             (0x00000005)
+#define NVCBA2_ERROR_METHOD_STREAM_AUTH_TAG_ADDR_MISALIGNED_POINTER             (0x00000006)
+#define NVCBA2_ERROR_METHOD_STREAM_AUTH_TAG_ADDR_DMA_NACK                       (0x00000007)
+#define NVCBA2_ERROR_METHOD_STREAM_AUTH_TAG_CHECK_FAILURE                       (0x00000008)
+#define NVCBA2_ERROR_MISALIGNED_SIZE                                            (0x00000009)
+#define NVCBA2_ERROR_MISSING_METHODS                                            (0x0000000A)
+#define NVCBA2_ERROR_SEMAPHORE_RELEASE_DMA_NACK                                 (0x0000000B)
+#define NVCBA2_ERROR_DECRYPT_SIZE_MAX_EXCEEDED                                  (0x0000000C)
+#define NVCBA2_ERROR_OS_APPLICATION                                             (0x0000000D)
+#define NVCBA2_ERROR_INVALID_CTXSW_REQUEST                                      (0x0000000E)
+#define NVCBA2_ERROR_BUFFER_OVERFLOW                                            (0x0000000F)
+#define NVCBA2_ERROR_IV_OVERFLOW                                                (0x00000010)
+#define NVCBA2_ERROR_INTERNAL_SETUP_FAILURE                                     (0x00000011)
+#define NVCBA2_ERROR_DECRYPT_COPY_INTERNAL_DMA_FAILURE                          (0x00000012)
+#define NVCBA2_ERROR_METHOD_STREAM_AUTH_TAG_ADDR_INTERNAL_DMA_FAILURE           (0x00000013)
+#define NVCBA2_ERROR_METHOD_STREAM_AUTH_TAG_HMAC_CALC_FAILURE                   (0x00000014)
+#define NVCBA2_ERROR_NONCE_OVERFLOW                                             (0x00000015)
+#define NVCBA2_ERROR_AES_GCM_DECRYPTION_FAILURE                                 (0x00000016)
+#define NVCBA2_ERROR_SEMAPHORE_RELEASE_INTERNAL_DMA_FAILURE                     (0x00000017)
+#define NVCBA2_ERROR_KEY_DERIVATION_FAILURE                                     (0x00000018)
+#define NVCBA2_ERROR_SCRUBBER_FAILURE                                           (0x00000019)
+#define NVCBA2_ERROR_SCRUBBER_INVALD_ADDRESS                                    (0x0000001a)
+#define NVCBA2_ERROR_SCRUBBER_INSUFFICIENT_PERMISSIONS                          (0x0000001b)
+#define NVCBA2_ERROR_SCRUBBER_MUTEX_ACQUIRE_FAILURE                             (0x0000001c)
+#define NVCBA2_ERROR_SCRUB_SIZE_MAX_EXCEEDED                                    (0x0000001d)
 
 #ifdef __cplusplus
 };     /* extern "C" */

kernel-open/nvidia-uvm/ctrl2080mc.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2013-2023 NVIDIA Corporation
+    Copyright (c) 2013-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -34,6 +34,8 @@
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100                (0x00000170)
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100                (0x00000180)
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100                (0x00000190)
+#define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100                (0x000001A0)
+#define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB200                (0x000001B0)
 
 /* valid ARCHITECTURE_GP10x implementation values */
 #define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GP100              (0x00000000)

kernel-open/nvidia-uvm/hwref/blackwell/gb100/dev_fault.h

@@ -0,0 +1,547 @@
+/*******************************************************************************
+    Copyright (c) 2003-2016 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+
+#ifndef __gb100_dev_fault_h__
+#define __gb100_dev_fault_h__
+/* This file is autogenerated.  Do not edit */
+#define NV_PFAULT                                              /* ----G */
+#define NV_PFAULT_MMU_ENG_ID_GRAPHICS          384 /*       */
+#define NV_PFAULT_MMU_ENG_ID_DISPLAY           1 /*       */
+#define NV_PFAULT_MMU_ENG_ID_GSP               2 /*       */
+#define NV_PFAULT_MMU_ENG_ID_IFB               55 /*       */
+#define NV_PFAULT_MMU_ENG_ID_FLA               4 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1              256 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2              320 /*       */
+#define NV_PFAULT_MMU_ENG_ID_SEC               6 /*       */
+#define NV_PFAULT_MMU_ENG_ID_FSP               7 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF              10 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF0             10 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF1             11 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF2             12 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF3             13 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF4             14 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF5             15 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF6             16 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF7             17 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF8             18 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PERF9             19 /*       */
+#define NV_PFAULT_MMU_ENG_ID_GSPLITE          20 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC             28 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC0            28 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC1            29 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC2            30 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC3            31 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC4            32 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC5            33 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC6            34 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVDEC7            35 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG0            36 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG1            37 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG2            38 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG3            39 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG4            40 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG5            41 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG6            42 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVJPG7            43 /*       */
+#define NV_PFAULT_MMU_ENG_ID_GRCOPY            65 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE0               65 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE1               66 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE2               67 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE3               68 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE4               69 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE5               70 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE6               71 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE7               72 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE8               73 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE9               74 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE10               75 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE11               76 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE12               77 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE13               78 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE14               79 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE15               80 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE16               81 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE17               82 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE18               83 /*       */
+#define NV_PFAULT_MMU_ENG_ID_CE19               84 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PWR_PMU           5 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PTP               3 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVENC0            44 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVENC1            45 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVENC2            46 /*       */
+#define NV_PFAULT_MMU_ENG_ID_NVENC3            47 /*       */
+#define NV_PFAULT_MMU_ENG_ID_OFA0              48 /*       */
+#define NV_PFAULT_MMU_ENG_ID_PHYSICAL          56 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST0             85 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST1             86 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST2             87 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST3             88 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST4             89 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST5             90 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST6             91 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST7             92 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST8             93 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST9             94 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST10            95 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST11            96 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST12            97 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST13            98 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST14            99 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST15            100 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST16            101 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST17            102 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST18            103 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST19            104 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST20            105 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST21            106 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST22            107 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST23            108 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST24            109 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST25            110 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST26            111 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST27            112 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST28            113 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST29            114 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST30            115 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST31            116 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST32            117 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST33            118 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST34            119 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST35            120 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST36            121 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST37            122 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST38            123 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST39            124 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST40            125 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST41            126 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST42            127 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST43            128 /*       */
+#define NV_PFAULT_MMU_ENG_ID_HOST44            129 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN0          256  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN1          257  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN2          258  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN3          259  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN4          260  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN5          261  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN6          262  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN7          263  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN8          264  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN9          265  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN10         266 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN11         267 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN12         268 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN13         269 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN14         270 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN15         271 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN16         272 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN17         273 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN18         274 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN19         275 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN20         276 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN21         277 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN22         278 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN23         279 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN24         280 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN25         281 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN26         282 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN27         283 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN28         284 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN29         285 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN30         286 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN31         287 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN32         288 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN33         289 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN34         290 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN35         291 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN36         292 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN37         293 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN38         294 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN39         295 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN40         296 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN41         297 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN42         298 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN43         299 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN44         300 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN45         301 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN46         302 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN47         303 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN48         304 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN49         305 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN50         306 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN51         307 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN52         308 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN53         309 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN54         310 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN55         311 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN56         312 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN57         313 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN58         314 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN59         315 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN60         316 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN61         317 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN62         318 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR1_FN63         319 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN0          320  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN1          321  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN2          322  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN3          323  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN4          324  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN5          325  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN6          326  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN7          327  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN8          328  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN9          329  /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN10         330 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN11         331 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN12         332 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN13         333 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN14         334 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN15         335 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN16         336 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN17         337 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN18         338 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN19         339 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN20         340 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN21         341 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN22         342 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN23         343 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN24         344 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN25         345 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN26         346 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN27         347 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN28         348 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN29         349 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN30         350 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN31         351 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN32         352 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN33         353 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN34         354 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN35         355 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN36         356 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN37         357 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN38         358 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN39         359 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN40         360 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN41         361 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN42         362 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN43         363 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN44         364 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN45         365 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN46         366 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN47         367 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN48         368 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN49         369 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN50         370 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN51         371 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN52         372 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN53         373 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN54         374 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN55         375 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN56         376 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN57         377 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN58         378 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN59         379 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN60         380 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN61         381 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN62         382 /*       */
+#define NV_PFAULT_MMU_ENG_ID_BAR2_FN63         383 /*       */
+#define NV_PFAULT_FAULT_TYPE                             4:0 /*       */
+#define NV_PFAULT_FAULT_TYPE_PDE                  0x00000000 /*       */
+#define NV_PFAULT_FAULT_TYPE_PDE_SIZE             0x00000001 /*       */
+#define NV_PFAULT_FAULT_TYPE_PTE                  0x00000002 /*       */
+#define NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION   0x00000003 /*       */
+#define NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK   0x00000004 /*       */
+#define NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION       0x00000005 /*       */
+#define NV_PFAULT_FAULT_TYPE_RO_VIOLATION         0x00000006 /*       */
+#define NV_PFAULT_FAULT_TYPE_WO_VIOLATION         0x00000007 /*       */
+#define NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION 0x00000008 /*       */
+#define NV_PFAULT_FAULT_TYPE_WORK_CREATION        0x00000009 /*       */
+#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE 0x0000000a /*       */
+#define NV_PFAULT_FAULT_TYPE_CC_VIOLATION         0x0000000b /*       */
+#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND     0x0000000c /*       */
+#define NV_PFAULT_FAULT_TYPE_REGION_VIOLATION     0x0000000d /*       */
+#define NV_PFAULT_FAULT_TYPE_POISONED             0x0000000e /*       */
+#define NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION     0x0000000f /*       */
+#define NV_PFAULT_CLIENT                       14:8 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_0        0x00000000 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_1        0x00000001 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_2        0x00000002 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_3        0x00000003 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_4        0x00000004 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_5        0x00000005 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_6        0x00000006 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_7        0x00000007 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_0        0x00000008 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_1        0x00000009 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_2        0x0000000A /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_3        0x0000000B /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_4        0x0000000C /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_5        0x0000000D /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_6        0x0000000E /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_7        0x0000000F /*       */
+#define NV_PFAULT_CLIENT_GPC_RAST        0x00000010 /*       */
+#define NV_PFAULT_CLIENT_GPC_GCC         0x00000011 /*       */
+#define NV_PFAULT_CLIENT_GPC_GPCCS       0x00000012 /*       */
+#define NV_PFAULT_CLIENT_GPC_PROP_0      0x00000013 /*       */
+#define NV_PFAULT_CLIENT_GPC_PROP_1      0x00000014 /*       */
+#define NV_PFAULT_CLIENT_GPC_PROP_2      0x00000015 /*       */
+#define NV_PFAULT_CLIENT_GPC_PROP_3      0x00000016 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_8        0x00000021 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_9        0x00000022 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_10       0x00000023 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_11       0x00000024 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_12       0x00000025 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_13       0x00000026 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_14       0x00000027 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_15       0x00000028 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_0     0x00000029 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_1     0x0000002A /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_2     0x0000002B /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_3     0x0000002C /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_4     0x0000002D /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_5     0x0000002E /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_6     0x0000002F /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_7     0x00000030 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_8        0x00000031 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_9        0x00000032 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_8     0x00000033 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_9     0x00000034 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_16       0x00000035 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_17       0x00000036 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_18       0x00000037 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_19       0x00000038 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_10       0x00000039 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_11       0x0000003A /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_10    0x0000003B /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_11    0x0000003C /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_20       0x0000003D /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_21       0x0000003E /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_22       0x0000003F /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_23       0x00000040 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_12       0x00000041 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_13       0x00000042 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_12    0x00000043 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_13    0x00000044 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_24       0x00000045 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_25       0x00000046 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_26       0x00000047 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_27       0x00000048 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_14       0x00000049 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_15       0x0000004A /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_14    0x0000004B /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_15    0x0000004C /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_28       0x0000004D /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_29       0x0000004E /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_30       0x0000004F /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_31       0x00000050 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_16       0x00000051 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_17       0x00000052 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_16    0x00000053 /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_17    0x00000054 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_32       0x00000055 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_33       0x00000056 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_34       0x00000057 /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_35       0x00000058 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_18       0x00000059 /*       */
+#define NV_PFAULT_CLIENT_GPC_PE_19       0x0000005A /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_18    0x0000005B /*       */
+#define NV_PFAULT_CLIENT_GPC_TPCCS_19    0x0000005C /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_36       0x0000005D /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_37       0x0000005E /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_38       0x0000005F /*       */
+#define NV_PFAULT_CLIENT_GPC_T1_39       0x00000060 /*       */
+#define NV_PFAULT_CLIENT_GPC_ROP_0       0x00000070 /*       */
+#define NV_PFAULT_CLIENT_GPC_ROP_1       0x00000071 /*       */
+#define NV_PFAULT_CLIENT_GPC_ROP_2       0x00000072 /*       */
+#define NV_PFAULT_CLIENT_GPC_ROP_3       0x00000073 /*       */
+#define NV_PFAULT_CLIENT_GPC_GPM          0x00000017 /*       */
+#define NV_PFAULT_CLIENT_HUB_VIP         0x00000000 /*       */
+#define NV_PFAULT_CLIENT_HUB_CE0         0x00000001 /*       */
+#define NV_PFAULT_CLIENT_HUB_CE1         0x00000002 /*       */
+#define NV_PFAULT_CLIENT_HUB_DNISO       0x00000003 /*       */
+#define NV_PFAULT_CLIENT_HUB_DISPNISO    0x00000003 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE0         0x00000004 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE          0x00000004 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS0       0x00000005 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS        0x00000005 /*       */
+#define NV_PFAULT_CLIENT_HUB_HOST        0x00000006 /*       */
+#define NV_PFAULT_CLIENT_HUB_HOST_CPU    0x00000007 /*       */
+#define NV_PFAULT_CLIENT_HUB_HOST_CPU_NB 0x00000008 /*       */
+#define NV_PFAULT_CLIENT_HUB_ISO         0x00000009 /*       */
+#define NV_PFAULT_CLIENT_HUB_MMU         0x0000000A /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC0      0x0000000B /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC       0x0000000B /*       */
+#define NV_PFAULT_CLIENT_HUB_CE3         0x0000000C /*       */
+#define NV_PFAULT_CLIENT_HUB_NVENC1      0x0000000D /*       */
+#define NV_PFAULT_CLIENT_HUB_NISO        0x0000000E /*       */
+#define NV_PFAULT_CLIENT_HUB_ACTRS       0x0000000E /*       */
+#define NV_PFAULT_CLIENT_HUB_P2P         0x0000000F /*       */
+#define NV_PFAULT_CLIENT_HUB_PD          0x00000010 /*       */
+#define NV_PFAULT_CLIENT_HUB_PD0         0x00000010 /*       */
+#define NV_PFAULT_CLIENT_HUB_PERF0       0x00000011 /*       */
+#define NV_PFAULT_CLIENT_HUB_PERF        0x00000011 /*       */
+#define NV_PFAULT_CLIENT_HUB_PMU         0x00000012 /*       */
+#define NV_PFAULT_CLIENT_HUB_RASTERTWOD  0x00000013 /*       */
+#define NV_PFAULT_CLIENT_HUB_RASTERTWOD0 0x00000013 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC         0x00000014 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC0        0x00000014 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC_NB      0x00000015 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC_NB0     0x00000015 /*       */
+#define NV_PFAULT_CLIENT_HUB_SEC         0x00000016 /*       */
+#define NV_PFAULT_CLIENT_HUB_SSYNC       0x00000017 /*       */
+#define NV_PFAULT_CLIENT_HUB_SSYNC0      0x00000017 /*       */
+#define NV_PFAULT_CLIENT_HUB_GRCOPY      0x00000018 /*       */
+#define NV_PFAULT_CLIENT_HUB_CE2         0x00000018 /*       */
+#define NV_PFAULT_CLIENT_HUB_XV          0x00000019 /*       */
+#define NV_PFAULT_CLIENT_HUB_MMU_NB      0x0000001A /*       */
+#define NV_PFAULT_CLIENT_HUB_NVENC0      0x0000001B /*       */
+#define NV_PFAULT_CLIENT_HUB_NVENC       0x0000001B /*       */
+#define NV_PFAULT_CLIENT_HUB_DFALCON     0x0000001C /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED0       0x0000001D /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED        0x0000001D /*       */
+#define NV_PFAULT_CLIENT_HUB_PD1         0x0000001E /*       */
+#define NV_PFAULT_CLIENT_HUB_DONT_CARE   0x0000001F /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE0       0x00000020 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE1       0x00000021 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE2       0x00000022 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE3       0x00000023 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE4       0x00000024 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE5       0x00000025 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE6       0x00000026 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSCE7       0x00000027 /*       */
+#define NV_PFAULT_CLIENT_HUB_SSYNC1      0x00000028 /*       */
+#define NV_PFAULT_CLIENT_HUB_SSYNC2      0x00000029 /*       */
+#define NV_PFAULT_CLIENT_HUB_HSHUB       0x0000002A /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X0      0x0000002B /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X1      0x0000002C /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X2      0x0000002D /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X3      0x0000002E /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X4      0x0000002F /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X5      0x00000030 /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X6      0x00000031 /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X7      0x00000032 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVENC2      0x00000033 /*       */
+#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 0x00000034 /*       */
+#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 0x00000035 /*       */
+#define NV_PFAULT_CLIENT_HUB_SSYNC3      0x00000036 /*       */
+#define NV_PFAULT_CLIENT_HUB_FBFALCON    0x00000037 /*       */
+#define NV_PFAULT_CLIENT_HUB_CE_SHIM     0x00000038 /*       */
+#define NV_PFAULT_CLIENT_HUB_CE_SHIM0    0x00000038 /*       */
+#define NV_PFAULT_CLIENT_HUB_GSP         0x00000039 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC1      0x0000003A /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC2      0x0000003B /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG0      0x0000003C /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC3      0x0000003D /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC4      0x0000003E /*       */
+#define NV_PFAULT_CLIENT_HUB_OFA0        0x0000003F /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC1        0x00000040 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC_NB1     0x00000041 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC2        0x00000042 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC_NB2     0x00000043 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC3        0x00000044 /*       */
+#define NV_PFAULT_CLIENT_HUB_SCC_NB3     0x00000045 /*       */
+#define NV_PFAULT_CLIENT_HUB_RASTERTWOD1 0x00000046 /*       */
+#define NV_PFAULT_CLIENT_HUB_PTP_X8      0x00000046 /*       */
+#define NV_PFAULT_CLIENT_HUB_RASTERTWOD2 0x00000047 /*       */
+#define NV_PFAULT_CLIENT_HUB_RASTERTWOD3 0x00000048 /*       */
+#define NV_PFAULT_CLIENT_HUB_GSPLITE1    0x00000049 /*       */
+#define NV_PFAULT_CLIENT_HUB_GSPLITE2    0x0000004A /*       */
+#define NV_PFAULT_CLIENT_HUB_GSPLITE3    0x0000004B /*       */
+#define NV_PFAULT_CLIENT_HUB_PD2         0x0000004C /*       */
+#define NV_PFAULT_CLIENT_HUB_PD3         0x0000004D /*       */
+#define NV_PFAULT_CLIENT_HUB_FE1         0x0000004E /*       */
+#define NV_PFAULT_CLIENT_HUB_FE2         0x0000004F /*       */
+#define NV_PFAULT_CLIENT_HUB_FE3         0x00000050 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE4         0x00000051 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE5         0x00000052 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE6         0x00000053 /*       */
+#define NV_PFAULT_CLIENT_HUB_FE7         0x00000054 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS1       0x00000055 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS2       0x00000056 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS3       0x00000057 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS4       0x00000058 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS5       0x00000059 /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS6       0x0000005A /*       */
+#define NV_PFAULT_CLIENT_HUB_FECS7       0x0000005B /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED1       0x0000005C /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED2       0x0000005D /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED3       0x0000005E /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED4       0x0000005F /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED5       0x00000060 /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED6       0x00000061 /*       */
+#define NV_PFAULT_CLIENT_HUB_SKED7       0x00000062 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC          0x00000063 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC0         0x00000063 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC1         0x00000064 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC2         0x00000065 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC3         0x00000066 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC4         0x00000067 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC5         0x00000068 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC6         0x00000069 /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC7         0x0000006a /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC8         0x0000006b /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC9         0x0000006c /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC10        0x0000006d /*       */
+#define NV_PFAULT_CLIENT_HUB_ESC11        0x0000006e /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC5      0x0000006F /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC6      0x00000070 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVDEC7      0x00000071 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG1      0x00000072 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG2      0x00000073 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG3      0x00000074 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG4      0x00000075 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG5      0x00000076 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG6      0x00000077 /*       */
+#define NV_PFAULT_CLIENT_HUB_NVJPG7      0x00000078 /*       */
+#define NV_PFAULT_CLIENT_HUB_FSP         0x00000079 /*       */
+#define NV_PFAULT_CLIENT_HUB_BSI         0x0000007A /*       */
+#define NV_PFAULT_CLIENT_HUB_GSPLITE     0x0000007B /*       */
+#define NV_PFAULT_CLIENT_HUB_GSPLITE0    0x0000007B /*       */
+#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER2 0x0000007C /*       */
+#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER3 0x0000007D /*       */
+#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER4 0x0000007E /*       */
+#define NV_PFAULT_CLIENT_HUB_NVENC3      0x0000007F /*       */
+#define NV_PFAULT_ACCESS_TYPE                 19:16 /*       */
+#define NV_PFAULT_ACCESS_TYPE_READ       0x00000000 /*       */
+#define NV_PFAULT_ACCESS_TYPE_WRITE      0x00000001 /*       */
+#define NV_PFAULT_ACCESS_TYPE_ATOMIC     0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PREFETCH   0x00000003 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_READ          0x00000000 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_WRITE         0x00000001 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC        0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG 0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH      0x00000003 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK   0x00000004 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_READ          0x00000008 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_WRITE         0x00000009 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC        0x0000000a /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH      0x0000000b /*       */
+#define NV_PFAULT_MMU_CLIENT_TYPE             20:20 /*       */
+#define NV_PFAULT_MMU_CLIENT_TYPE_GPC    0x00000000 /*       */
+#define NV_PFAULT_MMU_CLIENT_TYPE_HUB    0x00000001 /*       */
+#define NV_PFAULT_GPC_ID                      28:24 /*       */
+#define NV_PFAULT_PROTECTED_MODE              29:29 /*       */
+#define NV_PFAULT_REPLAYABLE_FAULT_EN         30:30 /*       */
+#define NV_PFAULT_VALID                       31:31 /*       */
+#endif // __gb100_dev_fault_h__

kernel-open/nvidia-uvm/nvidia-uvm-sources.Kbuild

@@ -1,14 +1,8 @@
 NVIDIA_UVM_SOURCES ?=
 NVIDIA_UVM_SOURCES_CXX ?=
 
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ats_sva.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_conf_computing.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_sec2_test.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_sec2.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_sec2.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_common.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_linux.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_debug_optimized.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nvstatus.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nvCpuUuid.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nv-kthread-q.c
@@ -32,6 +26,7 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_range_tree.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_rb_tree.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_range_allocator.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_range.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_range_device_p2p.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_policy.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_block.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_range_group.c
@@ -50,6 +45,7 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_tracker.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_ce.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_sec2.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_mmu.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_fault_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_access_counter_buffer.c
@@ -63,7 +59,6 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_mmu.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_fault_buffer.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_access_counter_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_access_counter_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_fault_buffer.c
@@ -72,13 +67,19 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_ce.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_host.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_fault_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_mmu.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_fault_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_ce.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_host.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_sec2.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_mmu.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ada.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_fault_buffer.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_mmu.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_policy.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_perf_utils.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_kvmalloc.c
@@ -96,6 +97,8 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_perf_prefetch.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ats.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ats_ibm.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ats_faults.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ats_sva.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_conf_computing.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_test.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_test_rng.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_range_tree_test.c
@@ -123,3 +126,4 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_block_test.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_range_group_tree_test.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_thread_context_test.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_rb_tree_test.c
+NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_sec2_test.c

kernel-open/nvidia-uvm/nvidia-uvm.Kbuild

@@ -13,19 +13,6 @@ NVIDIA_UVM_OBJECTS =
 include $(src)/nvidia-uvm/nvidia-uvm-sources.Kbuild
 NVIDIA_UVM_OBJECTS += $(patsubst %.c,%.o,$(NVIDIA_UVM_SOURCES))
 
-# Some linux kernel functions rely on being built with optimizations on and
-# to work around this we put wrappers for them in a separate file that's built
-# with optimizations on in debug builds and skipped in other builds.
-# Notably gcc 4.4 supports per function optimization attributes that would be
-# easier to use, but is too recent to rely on for now.
-NVIDIA_UVM_DEBUG_OPTIMIZED_SOURCE := nvidia-uvm/uvm_debug_optimized.c
-NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT := $(patsubst %.c,%.o,$(NVIDIA_UVM_DEBUG_OPTIMIZED_SOURCE))
-
-ifneq ($(UVM_BUILD_TYPE),debug)
-  # Only build the wrappers on debug builds
-  NVIDIA_UVM_OBJECTS := $(filter-out $(NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT), $(NVIDIA_UVM_OBJECTS))
-endif
-
 obj-m += nvidia-uvm.o
 nvidia-uvm-y := $(NVIDIA_UVM_OBJECTS)
 
@@ -36,15 +23,14 @@ NVIDIA_UVM_KO = nvidia-uvm/nvidia-uvm.ko
 #
 
 ifeq ($(UVM_BUILD_TYPE),debug)
-  NVIDIA_UVM_CFLAGS += -DDEBUG -O1 -g
-else
-  ifeq ($(UVM_BUILD_TYPE),develop)
-    # -DDEBUG is required, in order to allow pr_devel() print statements to
-    # work:
-    NVIDIA_UVM_CFLAGS += -DDEBUG
-    NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_DEVELOP
-  endif
-  NVIDIA_UVM_CFLAGS += -O2
+  NVIDIA_UVM_CFLAGS += -DDEBUG -g
+endif
+
+ifeq ($(UVM_BUILD_TYPE),develop)
+  # -DDEBUG is required, in order to allow pr_devel() print statements to
+  # work:
+  NVIDIA_UVM_CFLAGS += -DDEBUG
+  NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_DEVELOP
 endif
 
 NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_ENABLED
@@ -56,30 +42,17 @@ NVIDIA_UVM_CFLAGS += -I$(src)/nvidia-uvm
 
 $(call ASSIGN_PER_OBJ_CFLAGS, $(NVIDIA_UVM_OBJECTS), $(NVIDIA_UVM_CFLAGS))
 
-ifeq ($(UVM_BUILD_TYPE),debug)
-  # Force optimizations on for the wrappers
-  $(call ASSIGN_PER_OBJ_CFLAGS, $(NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT), $(NVIDIA_UVM_CFLAGS) -O2)
-endif
-
 #
 # Register the conftests needed by nvidia-uvm.ko
 #
 
 NV_OBJECTS_DEPEND_ON_CONFTEST += $(NVIDIA_UVM_OBJECTS)
 
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += wait_on_bit_lock_argument_count
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += pde_data
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += radix_tree_empty
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += radix_tree_replace_slot
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += pnv_npu2_init_context
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += vmf_insert_pfn
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += cpumask_of_node
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += list_is_first
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += timer_setup
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_bus_address
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += set_memory_uc
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += set_pages_uc
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_raw_ts64
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += ioasid_get
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += mm_pasid_drop
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += mmget_not_zero
@@ -88,32 +61,26 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_fault_to_errno
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += find_next_bit_wrap
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_is_dma_domain
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += for_each_sgtable_dma_page
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += folio_test_swapcache
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += page_pgmap
 
-NV_CONFTEST_TYPE_COMPILE_TESTS += backing_dev_info
-NV_CONFTEST_TYPE_COMPILE_TESTS += mm_context_t
-NV_CONFTEST_TYPE_COMPILE_TESTS += get_user_pages_remote
-NV_CONFTEST_TYPE_COMPILE_TESTS += get_user_pages
-NV_CONFTEST_TYPE_COMPILE_TESTS += pin_user_pages_remote
-NV_CONFTEST_TYPE_COMPILE_TESTS += pin_user_pages
-NV_CONFTEST_TYPE_COMPILE_TESTS += vm_fault_has_address
 NV_CONFTEST_TYPE_COMPILE_TESTS += vm_ops_fault_removed_vma_arg
-NV_CONFTEST_TYPE_COMPILE_TESTS += kmem_cache_has_kobj_remove_work
-NV_CONFTEST_TYPE_COMPILE_TESTS += sysfs_slab_unlink
-NV_CONFTEST_TYPE_COMPILE_TESTS += vm_fault_t
 NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_notifier_ops_invalidate_range
 NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_notifier_ops_arch_invalidate_secondary_tlbs
-NV_CONFTEST_TYPE_COMPILE_TESTS += proc_ops
-NV_CONFTEST_TYPE_COMPILE_TESTS += timespec64
-NV_CONFTEST_TYPE_COMPILE_TESTS += mm_has_mmap_lock
 NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_vma_added_flags
 NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_device_range
-NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
 NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_mm_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_pt_regs_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
 NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
 NV_CONFTEST_TYPE_COMPILE_TESTS += mpol_preferred_many_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_interval_notifier
+NV_CONFTEST_TYPE_COMPILE_TESTS += fault_flag_remote_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += sg_dma_page_iter
+NV_CONFTEST_TYPE_COMPILE_TESTS += struct_page_has_zone_device_data
 
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_migrate_vma_setup
+NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present___iowrite64_lo_hi
+NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_make_device_exclusive

kernel-open/nvidia-uvm/uvm.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2023 NVIDIA Corporation
+    Copyright (c) 2015-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -127,9 +127,9 @@ static NV_STATUS uvm_api_mm_initialize(UVM_MM_INITIALIZE_PARAMS *params, struct
         goto err;
     }
 
-    old_fd_type = nv_atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
-                                         UVM_FD_UNINITIALIZED,
-                                         UVM_FD_INITIALIZING);
+    old_fd_type = atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
+                                      UVM_FD_UNINITIALIZED,
+                                      UVM_FD_INITIALIZING);
     old_fd_type &= UVM_FD_TYPE_MASK;
     if (old_fd_type != UVM_FD_UNINITIALIZED) {
         status = NV_ERR_IN_USE;
@@ -222,10 +222,6 @@ static int uvm_open(struct inode *inode, struct file *filp)
     // assigning f_mapping.
     mapping->a_ops = inode->i_mapping->a_ops;
 
-#if defined(NV_ADDRESS_SPACE_HAS_BACKING_DEV_INFO)
-    mapping->backing_dev_info = inode->i_mapping->backing_dev_info;
-#endif
-
     filp->private_data = NULL;
     filp->f_mapping = mapping;
 
@@ -244,7 +240,7 @@ static void uvm_release_deferred(void *data)
     // Since this function is only scheduled to run when uvm_release() fails
     // to trylock-acquire the pm.lock, the following acquisition attempt
     // is expected to block this thread, and cause it to remain blocked until
-    // uvm_resume() releases the lock.  As a result, the deferred release
+    // uvm_resume() releases the lock. As a result, the deferred release
     // kthread queue may stall for long periods of time.
     uvm_down_read(&g_uvm_global.pm.lock);
 
@@ -296,14 +292,14 @@ static int uvm_release(struct inode *inode, struct file *filp)
 
     // Because the kernel discards the status code returned from this release
     // callback, early exit in case of a pm.lock acquisition failure is not
-    // an option.  Instead, the teardown work normally performed synchronously
+    // an option. Instead, the teardown work normally performed synchronously
     // needs to be scheduled to run after uvm_resume() releases the lock.
     if (uvm_down_read_trylock(&g_uvm_global.pm.lock)) {
         uvm_va_space_destroy(va_space);
         uvm_up_read(&g_uvm_global.pm.lock);
     }
     else {
-        // Remove references to this inode from the address_space.  This isn't
+        // Remove references to this inode from the address_space. This isn't
         // strictly necessary, as any CPU mappings of this file have already
         // been destroyed, and va_space->mapping won't be used again. Still,
         // the va_space survives the inode if its destruction is deferred, in
@@ -325,21 +321,21 @@ static int uvm_release_entry(struct inode *inode, struct file *filp)
 
 static void uvm_destroy_vma_managed(struct vm_area_struct *vma, bool make_zombie)
 {
-    uvm_va_range_t *va_range, *va_range_next;
+    uvm_va_range_managed_t *managed_range, *managed_range_next;
     NvU64 size = 0;
 
     uvm_assert_rwsem_locked_write(&uvm_va_space_get(vma->vm_file)->lock);
-    uvm_for_each_va_range_in_vma_safe(va_range, va_range_next, vma) {
+    uvm_for_each_va_range_managed_in_vma_safe(managed_range, managed_range_next, vma) {
         // On exit_mmap (process teardown), current->mm is cleared so
         // uvm_va_range_vma_current would return NULL.
-        UVM_ASSERT(uvm_va_range_vma(va_range) == vma);
-        UVM_ASSERT(va_range->node.start >= vma->vm_start);
-        UVM_ASSERT(va_range->node.end   <  vma->vm_end);
-        size += uvm_va_range_size(va_range);
+        UVM_ASSERT(uvm_va_range_vma(managed_range) == vma);
+        UVM_ASSERT(managed_range->va_range.node.start >= vma->vm_start);
+        UVM_ASSERT(managed_range->va_range.node.end   <  vma->vm_end);
+        size += uvm_va_range_size(&managed_range->va_range);
         if (make_zombie)
-            uvm_va_range_zombify(va_range);
+            uvm_va_range_zombify(managed_range);
         else
-            uvm_va_range_destroy(va_range, NULL);
+            uvm_va_range_destroy(&managed_range->va_range, NULL);
     }
 
     if (vma->vm_private_data) {
@@ -351,18 +347,17 @@ static void uvm_destroy_vma_managed(struct vm_area_struct *vma, bool make_zombie
 
 static void uvm_destroy_vma_semaphore_pool(struct vm_area_struct *vma)
 {
+    uvm_va_range_semaphore_pool_t *semaphore_pool_range;
     uvm_va_space_t *va_space;
-    uvm_va_range_t *va_range;
 
     va_space = uvm_va_space_get(vma->vm_file);
     uvm_assert_rwsem_locked(&va_space->lock);
-    va_range = uvm_va_range_find(va_space, vma->vm_start);
-    UVM_ASSERT(va_range &&
-               va_range->node.start   == vma->vm_start &&
-               va_range->node.end + 1 == vma->vm_end &&
-               va_range->type == UVM_VA_RANGE_TYPE_SEMAPHORE_POOL);
+    semaphore_pool_range = uvm_va_range_semaphore_pool_find(va_space, vma->vm_start);
+    UVM_ASSERT(semaphore_pool_range &&
+               semaphore_pool_range->va_range.node.start   == vma->vm_start &&
+               semaphore_pool_range->va_range.node.end + 1 == vma->vm_end);
 
-    uvm_mem_unmap_cpu_user(va_range->semaphore_pool.mem);
+    uvm_mem_unmap_cpu_user(semaphore_pool_range->mem);
 }
 
 // If a fault handler is not set, paths like handle_pte_fault in older kernels
@@ -478,7 +473,7 @@ static void uvm_vm_open_failure(struct vm_area_struct *original,
 static void uvm_vm_open_managed(struct vm_area_struct *vma)
 {
     uvm_va_space_t *va_space = uvm_va_space_get(vma->vm_file);
-    uvm_va_range_t *va_range;
+    uvm_va_range_managed_t *managed_range;
     struct vm_area_struct *original;
     NV_STATUS status;
     NvU64 new_end;
@@ -534,13 +529,13 @@ static void uvm_vm_open_managed(struct vm_area_struct *vma)
         goto out;
     }
 
-    // There can be multiple va_ranges under the vma already. Check if one spans
+    // There can be multiple ranges under the vma already. Check if one spans
     // the new split boundary. If so, split it.
-    va_range = uvm_va_range_find(va_space, new_end);
-    UVM_ASSERT(va_range);
-    UVM_ASSERT(uvm_va_range_vma_current(va_range) == original);
-    if (va_range->node.end != new_end) {
-        status = uvm_va_range_split(va_range, new_end, NULL);
+    managed_range = uvm_va_range_managed_find(va_space, new_end);
+    UVM_ASSERT(managed_range);
+    UVM_ASSERT(uvm_va_range_vma_current(managed_range) == original);
+    if (managed_range->va_range.node.end != new_end) {
+        status = uvm_va_range_split(managed_range, new_end, NULL);
         if (status != NV_OK) {
             UVM_DBG_PRINT("Failed to split VA range, destroying both: %s. "
                           "original vma [0x%lx, 0x%lx) new vma [0x%lx, 0x%lx)\n",
@@ -552,10 +547,10 @@ static void uvm_vm_open_managed(struct vm_area_struct *vma)
         }
     }
 
-    // Point va_ranges to the new vma
-    uvm_for_each_va_range_in_vma(va_range, vma) {
-        UVM_ASSERT(uvm_va_range_vma_current(va_range) == original);
-        va_range->managed.vma_wrapper = vma->vm_private_data;
+    // Point managed_ranges to the new vma
+    uvm_for_each_va_range_managed_in_vma(managed_range, vma) {
+        UVM_ASSERT(uvm_va_range_vma_current(managed_range) == original);
+        managed_range->vma_wrapper = vma->vm_private_data;
     }
 
 out:
@@ -657,12 +652,12 @@ static struct vm_operations_struct uvm_vm_ops_managed =
 };
 
 // vm operations on semaphore pool allocations only control CPU mappings. Unmapping GPUs,
-// freeing the allocation, and destroying the va_range are handled by UVM_FREE.
+// freeing the allocation, and destroying the range are handled by UVM_FREE.
 static void uvm_vm_open_semaphore_pool(struct vm_area_struct *vma)
 {
     struct vm_area_struct *origin_vma = (struct vm_area_struct *)vma->vm_private_data;
     uvm_va_space_t *va_space = uvm_va_space_get(origin_vma->vm_file);
-    uvm_va_range_t *va_range;
+    uvm_va_range_semaphore_pool_t *semaphore_pool_range;
     bool is_fork = (vma->vm_mm != origin_vma->vm_mm);
     NV_STATUS status;
 
@@ -670,18 +665,24 @@ static void uvm_vm_open_semaphore_pool(struct vm_area_struct *vma)
 
     uvm_va_space_down_write(va_space);
 
-    va_range = uvm_va_range_find(va_space, origin_vma->vm_start);
-    UVM_ASSERT(va_range);
-    UVM_ASSERT_MSG(va_range->type == UVM_VA_RANGE_TYPE_SEMAPHORE_POOL &&
-                   va_range->node.start == origin_vma->vm_start &&
-                   va_range->node.end + 1 == origin_vma->vm_end,
+    semaphore_pool_range = uvm_va_range_semaphore_pool_find(va_space, origin_vma->vm_start);
+    UVM_ASSERT(semaphore_pool_range);
+    UVM_ASSERT_MSG(semaphore_pool_range &&
+                   semaphore_pool_range->va_range.node.start == origin_vma->vm_start &&
+                   semaphore_pool_range->va_range.node.end + 1 == origin_vma->vm_end,
                    "origin vma [0x%llx, 0x%llx); va_range [0x%llx, 0x%llx) type %d\n",
-                   (NvU64)origin_vma->vm_start, (NvU64)origin_vma->vm_end, va_range->node.start,
-                   va_range->node.end + 1, va_range->type);
+                   (NvU64)origin_vma->vm_start,
+                   (NvU64)origin_vma->vm_end,
+                   semaphore_pool_range->va_range.node.start,
+                   semaphore_pool_range->va_range.node.end + 1,
+                   semaphore_pool_range->va_range.type);
 
     // Semaphore pool vmas do not have vma wrappers, but some functions will
     // assume vm_private_data is a wrapper.
     vma->vm_private_data = NULL;
+#if defined(VM_WIPEONFORK)
+    nv_vm_flags_set(vma, VM_WIPEONFORK);
+#endif
 
     if (is_fork) {
         // If we forked, leave the parent vma alone.
@@ -689,9 +690,9 @@ static void uvm_vm_open_semaphore_pool(struct vm_area_struct *vma)
 
         // uvm_disable_vma unmaps in the parent as well; clear the uvm_mem CPU
         // user mapping metadata and then remap.
-        uvm_mem_unmap_cpu_user(va_range->semaphore_pool.mem);
+        uvm_mem_unmap_cpu_user(semaphore_pool_range->mem);
 
-        status = uvm_mem_map_cpu_user(va_range->semaphore_pool.mem, va_range->va_space, origin_vma);
+        status = uvm_mem_map_cpu_user(semaphore_pool_range->mem, semaphore_pool_range->va_range.va_space, origin_vma);
         if (status != NV_OK) {
             UVM_DBG_PRINT("Failed to remap semaphore pool to CPU for parent after fork; status = %d (%s)",
                     status, nvstatusToString(status));
@@ -702,7 +703,7 @@ static void uvm_vm_open_semaphore_pool(struct vm_area_struct *vma)
         origin_vma->vm_private_data = NULL;
         origin_vma->vm_ops = &uvm_vm_ops_disabled;
         vma->vm_ops = &uvm_vm_ops_disabled;
-        uvm_mem_unmap_cpu_user(va_range->semaphore_pool.mem);
+        uvm_mem_unmap_cpu_user(semaphore_pool_range->mem);
     }
 
     uvm_va_space_up_write(va_space);
@@ -751,10 +752,84 @@ static struct vm_operations_struct uvm_vm_ops_semaphore_pool =
 #endif
 };
 
+static void uvm_vm_open_device_p2p(struct vm_area_struct *vma)
+{
+    struct vm_area_struct *origin_vma = (struct vm_area_struct *)vma->vm_private_data;
+    uvm_va_space_t *va_space = uvm_va_space_get(origin_vma->vm_file);
+    uvm_va_range_t *va_range;
+    bool is_fork = (vma->vm_mm != origin_vma->vm_mm);
+
+    uvm_record_lock_mmap_lock_write(current->mm);
+
+    uvm_va_space_down_write(va_space);
+
+    va_range = uvm_va_range_find(va_space, origin_vma->vm_start);
+    UVM_ASSERT(va_range);
+    UVM_ASSERT_MSG(va_range->type == UVM_VA_RANGE_TYPE_DEVICE_P2P &&
+                   va_range->node.start == origin_vma->vm_start &&
+                   va_range->node.end + 1 == origin_vma->vm_end,
+                   "origin vma [0x%llx, 0x%llx); va_range [0x%llx, 0x%llx) type %d\n",
+                   (NvU64)origin_vma->vm_start, (NvU64)origin_vma->vm_end, va_range->node.start,
+                   va_range->node.end + 1, va_range->type);
+
+    // Device P2P vmas do not have vma wrappers, but some functions will
+    // assume vm_private_data is a wrapper.
+    vma->vm_private_data = NULL;
+#if defined(VM_WIPEONFORK)
+    nv_vm_flags_set(vma, VM_WIPEONFORK);
+#endif
+
+    if (is_fork) {
+        // If we forked, leave the parent vma alone.
+        uvm_disable_vma(vma);
+
+        // uvm_disable_vma unmaps in the parent as well so remap the parent
+        uvm_va_range_device_p2p_map_cpu(va_range->va_space, origin_vma, uvm_va_range_to_device_p2p(va_range));
+    }
+    else {
+        // mremap will free the backing pages via unmap so we can't support it.
+        origin_vma->vm_private_data = NULL;
+        origin_vma->vm_ops = &uvm_vm_ops_disabled;
+        vma->vm_ops = &uvm_vm_ops_disabled;
+        unmap_mapping_range(va_space->mapping, va_range->node.start, va_range->node.end - va_range->node.start + 1, 1);
+    }
+
+    uvm_va_space_up_write(va_space);
+
+    uvm_record_unlock_mmap_lock_write(current->mm);
+}
+
+static void uvm_vm_open_device_p2p_entry(struct vm_area_struct *vma)
+{
+    UVM_ENTRY_VOID(uvm_vm_open_device_p2p(vma));
+}
+
+// Device P2P pages are only mapped on the CPU. Pages are allocated externally
+// to UVM but destroying the range must unpin the RM object.
+static void uvm_vm_close_device_p2p(struct vm_area_struct *vma)
+{
+}
+
+static void uvm_vm_close_device_p2p_entry(struct vm_area_struct *vma)
+{
+    UVM_ENTRY_VOID(uvm_vm_close_device_p2p(vma));
+}
+
+static struct vm_operations_struct uvm_vm_ops_device_p2p =
+{
+    .open         = uvm_vm_open_device_p2p_entry,
+    .close        = uvm_vm_close_device_p2p_entry,
+
+#if defined(NV_VM_OPS_FAULT_REMOVED_VMA_ARG)
+    .fault        = uvm_vm_fault_sigbus_wrapper_entry,
+#else
+    .fault        = uvm_vm_fault_sigbus_entry,
+#endif
+};
+
 static int uvm_mmap(struct file *filp, struct vm_area_struct *vma)
 {
     uvm_va_space_t *va_space;
-    uvm_va_range_t *va_range;
     NV_STATUS status = uvm_global_get_status();
     int ret = 0;
     bool vma_wrapper_allocated = false;
@@ -792,8 +867,8 @@ static int uvm_mmap(struct file *filp, struct vm_area_struct *vma)
     }
 
     // If the PM lock cannot be acquired, disable the VMA and report success
-    // to the caller.  The caller is expected to determine whether the
-    // map operation succeeded via an ioctl() call.  This is necessary to
+    // to the caller. The caller is expected to determine whether the
+    // map operation succeeded via an ioctl() call. This is necessary to
     // safely handle MAP_FIXED, which needs to complete atomically to prevent
     // the loss of the virtual address range.
     if (!uvm_down_read_trylock(&g_uvm_global.pm.lock)) {
@@ -845,18 +920,28 @@ static int uvm_mmap(struct file *filp, struct vm_area_struct *vma)
     status = uvm_va_range_create_mmap(va_space, current->mm, vma->vm_private_data, NULL);
 
     if (status == NV_ERR_UVM_ADDRESS_IN_USE) {
+        uvm_va_range_semaphore_pool_t *semaphore_pool_range;
+        uvm_va_range_device_p2p_t *device_p2p_range;
         // If the mmap is for a semaphore pool, the VA range will have been
         // allocated by a previous ioctl, and the mmap just creates the CPU
         // mapping.
-        va_range = uvm_va_range_find(va_space, vma->vm_start);
-        if (va_range && va_range->node.start == vma->vm_start &&
-                va_range->node.end + 1 == vma->vm_end &&
-                va_range->type == UVM_VA_RANGE_TYPE_SEMAPHORE_POOL) {
+        semaphore_pool_range = uvm_va_range_semaphore_pool_find(va_space, vma->vm_start);
+        device_p2p_range = uvm_va_range_device_p2p_find(va_space, vma->vm_start);
+        if (semaphore_pool_range && semaphore_pool_range->va_range.node.start == vma->vm_start &&
+                semaphore_pool_range->va_range.node.end + 1 == vma->vm_end) {
             uvm_vma_wrapper_destroy(vma->vm_private_data);
             vma_wrapper_allocated = false;
             vma->vm_private_data = vma;
             vma->vm_ops = &uvm_vm_ops_semaphore_pool;
-            status = uvm_mem_map_cpu_user(va_range->semaphore_pool.mem, va_range->va_space, vma);
+            status = uvm_mem_map_cpu_user(semaphore_pool_range->mem, semaphore_pool_range->va_range.va_space, vma);
+        }
+        else if (device_p2p_range && device_p2p_range->va_range.node.start == vma->vm_start &&
+                 device_p2p_range->va_range.node.end + 1 == vma->vm_end) {
+            uvm_vma_wrapper_destroy(vma->vm_private_data);
+            vma_wrapper_allocated = false;
+            vma->vm_private_data = vma;
+            vma->vm_ops = &uvm_vm_ops_device_p2p;
+            status = uvm_va_range_device_p2p_map_cpu(va_space, vma, device_p2p_range);
         }
     }
 
@@ -914,8 +999,9 @@ static NV_STATUS uvm_api_initialize(UVM_INITIALIZE_PARAMS *params, struct file *
     // attempt to be made. This is safe because other threads will have only had
     // a chance to observe UVM_FD_INITIALIZING and not UVM_FD_VA_SPACE in this
     // case.
-    old_fd_type = nv_atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
-                                         UVM_FD_UNINITIALIZED, UVM_FD_INITIALIZING);
+    old_fd_type = atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
+                                      UVM_FD_UNINITIALIZED,
+                                      UVM_FD_INITIALIZING);
     old_fd_type &= UVM_FD_TYPE_MASK;
     if (old_fd_type == UVM_FD_UNINITIALIZED) {
         status = uvm_va_space_create(filp->f_mapping, &va_space, params->flags);
@@ -1001,6 +1087,9 @@ static long uvm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
         UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_CLEAN_UP_ZOMBIE_RESOURCES,      uvm_api_clean_up_zombie_resources);
         UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_POPULATE_PAGEABLE,              uvm_api_populate_pageable);
         UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_VALIDATE_VA_RANGE,              uvm_api_validate_va_range);
+        UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_V2,uvm_api_tools_get_processor_uuid_table_v2);
+        UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_ALLOC_DEVICE_P2P,               uvm_api_alloc_device_p2p);
+        UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_CLEAR_ALL_ACCESS_COUNTERS,      uvm_api_clear_all_access_counters);
     }
 
     // Try the test ioctls if none of the above matched
@@ -1144,19 +1233,8 @@ static int uvm_init(void)
         goto error;
     }
 
-    pr_info("Loaded the UVM driver, major device number %d.\n", MAJOR(g_uvm_base_dev));
-
     if (uvm_enable_builtin_tests)
-        pr_info("Built-in UVM tests are enabled. This is a security risk.\n");
-
-    // After Open RM is released, both the enclosing "#if" and this comment
-    // block should be removed, because the uvm_hmm_is_enabled_system_wide()
-    // check is both necessary and sufficient for reporting functionality.
-    // Until that time, however, we need to avoid advertisting UVM's ability to
-    // enable HMM functionality.
-
-    if (uvm_hmm_is_enabled_system_wide())
-        UVM_INFO_PRINT("HMM (Heterogeneous Memory Management) is enabled in the UVM driver.\n");
+        UVM_INFO_PRINT("Built-in UVM tests are enabled. This is a security risk.\n");
 
     return 0;
 
@@ -1185,8 +1263,6 @@ static void uvm_exit(void)
     uvm_global_exit();
 
     uvm_test_unload_state_exit();
-
-    pr_info("Unloaded the UVM driver.\n");
 }
 
 static void __exit uvm_exit_entry(void)

kernel-open/nvidia-uvm/uvm.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2013-2023 NVIDIA Corporation
+    Copyright (c) 2013-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -45,20 +45,20 @@
 //     #endif
 // 3) Do the same thing for the function definition, and for any structs that
 //    are taken as arguments to these functions.
-// 4) Let this change propagate over to cuda_a and dev_a, so that the CUDA and
-//    nvidia-cfg libraries can start using the new API by bumping up the API
+// 4) Let this change propagate over to cuda_a and bugfix_main, so that the CUDA
+//    and nvidia-cfg libraries can start using the new API by bumping up the API
 //    version number it's using.
 //    Places where UVM_API_REVISION is defined are:
 //      drivers/gpgpu/cuda/cuda.nvmk (cuda_a)
-//      drivers/setup/linux/nvidia-cfg/makefile.nvmk (dev_a)
-// 5) Once the dev_a and cuda_a changes have made it back into chips_a,
+//      drivers/setup/linux/nvidia-cfg/makefile.nvmk (bugfix_main)
+// 5) Once the bugfix_main and cuda_a changes have made it back into chips_a,
 //    remove the old API declaration, definition, and any old structs that were
 //    in use.
 
 #ifndef _UVM_H_
 #define _UVM_H_
 
-#define UVM_API_LATEST_REVISION 11
+#define UVM_API_LATEST_REVISION 13
 
 #if !defined(UVM_API_REVISION)
 #error "please define UVM_API_REVISION macro to a desired version number or UVM_API_LATEST_REVISION macro"
@@ -167,7 +167,7 @@ NV_STATUS UvmSetDriverVersion(NvU32 major, NvU32 changelist);
 //
 // Error codes:
 //     NV_ERR_NOT_SUPPORTED:
-//         The Linux kernel is not able to support UVM. This could be because
+//         The kernel is not able to support UVM. This could be because
 //         the kernel is too old, or because it lacks a feature that UVM
 //         requires. The kernel log will have details.
 //
@@ -379,11 +379,11 @@ NV_STATUS UvmIsPageableMemoryAccessSupportedOnGpu(const NvProcessorUuid *gpuUuid
 //         OS state required to register the GPU is malformed, or the partition
 //         identified by the user handles or its configuration changed.
 //
-//     NV_ERR_NVSWITCH_FABRIC_NOT_READY:
+//     NV_ERR_NVLINK_FABRIC_NOT_READY:
 //         (On NvSwitch-connected system) Indicates that the fabric has not been
 //         configured yet. Caller must retry GPU registration.
 //
-//     NV_ERR_NVSWITCH_FABRIC_FAILURE:
+//     NV_ERR_NVLINK_FABRIC_FAILURE:
 //         (On NvSwitch-connected systems) Indicates that the NvLink fabric
 //         failed to be configured.
 //
@@ -395,36 +395,8 @@ NV_STATUS UvmIsPageableMemoryAccessSupportedOnGpu(const NvProcessorUuid *gpuUuid
 //         because it is not very informative.
 //
 //------------------------------------------------------------------------------
-#if UVM_API_REV_IS_AT_MOST(8)
-NV_STATUS UvmRegisterGpu(const NvProcessorUuid *gpuUuid);
-#else
 NV_STATUS UvmRegisterGpu(const NvProcessorUuid *gpuUuid,
                          const UvmGpuPlatformParams *platformParams);
-#endif
-
-#if UVM_API_REV_IS_AT_MOST(8)
-//------------------------------------------------------------------------------
-// UvmRegisterGpuSmc
-//
-// The same as UvmRegisterGpu, but takes additional parameters to specify the
-// GPU partition being registered if SMC is enabled.
-//
-// Arguments:
-//     gpuUuid: (INPUT)
-//         UUID of the physical GPU of the SMC partition to register.
-//
-//     platformParams: (INPUT)
-//         User handles identifying the partition to register.
-//
-// Error codes (see UvmRegisterGpu also):
-//
-//     NV_ERR_INVALID_STATE:
-//         SMC was not enabled, or the partition identified by the user
-//         handles or its configuration changed.
-//
-NV_STATUS UvmRegisterGpuSmc(const NvProcessorUuid *gpuUuid,
-                            const UvmGpuPlatformParams *platformParams);
-#endif
 
 //------------------------------------------------------------------------------
 // UvmUnregisterGpu
@@ -1356,9 +1328,8 @@ NV_STATUS UvmCleanUpZombieResources(void);
 //
 //     NV_ERR_INVALID_ARGUMENT:
 //         perGpuAttribs is NULL but gpuAttribsCount is non-zero or vice-versa,
-//         or caching is requested on more than one GPU.
-//         The Confidential Computing feature is enabled and the perGpuAttribs
-//         list is empty.
+//         or caching is requested on more than one GPU, or (in Confidential
+//         Computing only) the perGpuAttribs list is empty.
 //
 //     NV_ERR_NOT_SUPPORTED:
 //         The current process is not the one which called UvmInitialize, and
@@ -1375,6 +1346,86 @@ NV_STATUS UvmAllocSemaphorePool(void                          *base,
                                 const UvmGpuMappingAttributes *perGpuAttribs,
                                 NvLength                       gpuAttribsCount);
 
+//------------------------------------------------------------------------------
+// UvmAllocDeviceP2P
+//
+// Create a VA range within the process's address space reserved for use by
+// other devices to directly access GPU memory. The memory associated with the
+// RM handle is mapped into the user address space associated with the range for
+// direct access from the CPU.
+//
+// The VA range must not overlap with an existing VA range, irrespective of
+// whether the existing range corresponds to a UVM allocation or an external
+// allocation.
+//
+// Multiple VA ranges may be created mapping the same physical memory associated
+// with the RM handle. The associated GPU memory will not be freed until all VA
+// ranges have been destroyed either explicitly or implicitly and all non-UVM
+// users (eg. third party device drivers) have stopped using the associated
+// GPU memory.
+//
+// The VA range can be unmapped and freed by calling UvmFree.
+//
+// Destroying the final range mapping the RM handle may block until all third
+// party device drivers and other kernel users have stopped using the memory.
+//
+// These VA ranges are only associated with a single GPU.
+//
+// Arguments:
+//     gpuUuid: (INPUT)
+//         UUID of the physical GPU if the GPU is not SMC capable or SMC
+//         enabled, or the GPU instance UUID of the partition containing the
+//         memory to be mapped on the CPU.
+//
+//     base: (INPUT)
+//         Base address of the virtual address range.
+//
+//     length: (INPUT)
+//         Length, in bytes, of the range.
+//
+//     offset: (INPUT)
+//         Offset, in bytes, from the start of the externally allocated memory
+//         to map from.
+//
+//     platformParams: (INPUT)
+//         Platform specific parameters that identify the allocation.
+//         On Linux: RM ctrl fd, hClient and the handle (hMemory) of the
+//         externally allocated memory to map.
+//
+// Errors:
+//
+//     NV_ERR_INVALID_ADDRESS:
+//         base is NULL or length is zero or at least one of base and length is
+//         not aligned to 4K.
+//
+//     NV_ERR_INVALID_DEVICE:
+//         The gpuUuid was either not registered or has no GPU VA space
+//         registered for it.
+//
+//     NV_ERR_INVALID_ARGUMENT:
+//         base + offset + length exceeeds the end of the externally allocated
+//         memory handle or the externally allocated handle is not valid.
+//
+//     NV_ERR_UVM_ADDRESS_IN_USE:
+//         The requested virtual address range overlaps with an existing
+//         allocation.
+//
+//     NV_ERR_NO_MEMORY:
+//         Internal memory allocation failed.
+//
+//     NV_ERR_NOT_SUPPORTED:
+//         The device peer-to-peer feature is not supported by the current
+//         system configuration. This may be because the GPU doesn't support
+//         the peer-to-peer feature or the kernel was not built with the correct
+//         configuration options.
+//
+//------------------------------------------------------------------------------
+NV_STATUS UvmAllocDeviceP2P(NvProcessorUuid gpuUuid,
+                            void     *base,
+                            NvLength length,
+                            NvLength offset,
+                            const UvmDeviceP2PPlatformParams *platformParams);
+
 //------------------------------------------------------------------------------
 // UvmMigrate
 //
@@ -1428,7 +1479,9 @@ NV_STATUS UvmAllocSemaphorePool(void                          *base,
 // If read duplication is enabled on any pages in the VA range, then those pages
 // are read duplicated at the destination processor, leaving the source copy, if
 // present, intact with only its mapping changed to read-only if it wasn't
-// already mapped that way.
+// already mapped that way. The exception to this behavior is migrating pages
+// between different NUMA nodes, in which case the pages are migrated to the
+// destination node and a read-only mapping is created to the migrated pages.
 //
 // Pages in the VA range are migrated even if their preferred location is set to
 // a processor other than the destination processor.
@@ -2171,7 +2224,9 @@ NV_STATUS UvmMapDynamicParallelismRegion(void                  *base,
 //
 // If UvmMigrate, UvmMigrateAsync or UvmMigrateRangeGroup is called on any pages
 // in this VA range, then those pages will also be read duplicated on the
-// destination processor for the migration.
+// destination processor for the migration unless the migration is between CPU
+// NUMA nodes, in which case the pages are migrated to the destination NUMA
+// node and a read-only mapping to the migrated pages is created.
 //
 // Enabling read duplication on a VA range requires the CPU and all GPUs with
 // registered VA spaces to be fault-capable. Otherwise, the migration and
@@ -3287,7 +3342,7 @@ NV_STATUS UvmEventGetGpuUuidTable(NvProcessorUuid *gpuUuidTable,
 //------------------------------------------------------------------------------
 NV_STATUS UvmEventFetch(UvmDebugSession      sessionHandle,
                         UvmEventQueueHandle  queueHandle,
-                        UvmEventEntry_V1    *pBuffer,
+                        UvmEventEntry       *pBuffer,
                         NvU64               *nEntries);
 
 //------------------------------------------------------------------------------
@@ -3483,35 +3538,21 @@ NV_STATUS UvmToolsDestroySession(UvmToolsSessionHandle session);
 // 4. Destroy event Queue using UvmToolsDestroyEventQueue
 //
 
-#if UVM_API_REV_IS_AT_MOST(10)
-// This is deprecated and replaced by sizeof(UvmToolsEventControlData_V1) or
-// sizeof(UvmToolsEventControlData_V2).
-NvLength UvmToolsGetEventControlSize(void);
-
-// This is deprecated and replaced by sizeof(UvmEventEntry_V1) or
-// sizeof(UvmEventEntry_V2).
-NvLength UvmToolsGetEventEntrySize(void);
-#endif
-
 NvLength UvmToolsGetNumberOfCounters(void);
 
 //------------------------------------------------------------------------------
 // UvmToolsCreateEventQueue
 //
-// This call creates an event queue that can hold the given number of events.
-// All events are disabled by default. Event queue data persists lifetime of the
-// target process.
+// This function is deprecated. See UvmToolsCreateEventQueue_V2.
+//
+// This call creates an event queue that can hold the given number of
+// UvmEventEntry events. All events are disabled by default. Event queue data
+// persists lifetime of the target process.
 //
 // Arguments:
 //     session: (INPUT)
 //         Handle to the tools session.
 //
-//     version: (INPUT)
-//         Requested version for events or counters.
-//         See UvmEventEntry_V1 and UvmEventEntry_V2.
-//         UvmToolsEventControlData_V2::version records the entry version that
-//         will be generated.
-//
 //     event_buffer: (INPUT)
 //         User allocated buffer. Must be page-aligned. Must be large enough to
 //         hold at least event_buffer_size events. Gets pinned until queue is
@@ -3523,8 +3564,7 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //
 //     event_control (INPUT)
 //         User allocated buffer. Must be page-aligned. Must be large enough to
-//         hold UvmToolsEventControlData_V1 if version is UvmEventEntry_V1 or
-//         UvmToolsEventControlData_V2 (although single page-size allocation
+//         hold UvmToolsEventControlData (although single page-size allocation
 //         should be more than enough). Gets pinned until queue is destroyed.
 //
 //     queue: (OUTPUT)
@@ -3535,7 +3575,6 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //         Session handle does not refer to a valid session
 //
 //     NV_ERR_INVALID_ARGUMENT:
-//         The version is not UvmEventEntry_V1 or UvmEventEntry_V2.
 //         One of the parameters: event_buffer, event_buffer_size, event_control
 //         is not valid
 //
@@ -3546,21 +3585,66 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //         pinned (e.g. because of OS limitation of pinnable memory). Also it
 //         could not have been possible to create UvmToolsEventQueueDescriptor.
 //
-//------------------------------------------------------------------------------
-#if UVM_API_REV_IS_AT_MOST(10)
-NV_STATUS UvmToolsCreateEventQueue(UvmToolsSessionHandle     session,
-                                   void                     *event_buffer,
-                                   NvLength                  event_buffer_size,
-                                   void                     *event_control,
-                                   UvmToolsEventQueueHandle *queue);
-#else
 NV_STATUS UvmToolsCreateEventQueue(UvmToolsSessionHandle        session,
-                                   UvmToolsEventQueueVersion    version,
                                    void                        *event_buffer,
                                    NvLength                     event_buffer_size,
                                    void                        *event_control,
                                    UvmToolsEventQueueHandle    *queue);
-#endif
+
+//------------------------------------------------------------------------------
+// UvmToolsCreateEventQueue_V2
+//
+// This call creates an event queue that can hold the given number of
+// UvmEventEntry_V2 events. All events are disabled by default. Event queue data
+// persists beyond the lifetime of the target process.
+//
+// Arguments:
+//     session: (INPUT)
+//         Handle to the tools session.
+//
+//     event_buffer: (INPUT)
+//         User allocated buffer. Must be page-aligned. Must be large enough to
+//         hold at least event_buffer_size events. Gets pinned until queue is
+//         destroyed.
+//
+//     event_buffer_size: (INPUT)
+//         Size of the event queue buffer in units of UvmEventEntry_V2's. Must
+//         be a power of two, and greater than 1.
+//
+//     event_control (INPUT)
+//         User allocated buffer. Must be page-aligned. Must be large enough to
+//         hold UvmToolsEventControlData (although single page-size allocation
+//         should be more than enough). Gets pinned until queue is destroyed.
+//
+//     queue: (OUTPUT)
+//         Handle to the created queue.
+//
+// Error codes:
+//     NV_ERR_INSUFFICIENT_PERMISSIONS:
+//         Session handle does not refer to a valid session
+//
+//     NV_ERR_INVALID_ARGUMENT:
+//         One of the parameters: event_buffer, event_buffer_size, event_control
+//         is not valid
+//
+//     NV_ERR_NOT_SUPPORTED:
+//         The requested version queue could not be created
+//         (i.e., the UVM kernel driver is older and doesn't support
+//         UvmToolsEventQueueVersion_V2).
+//
+//     NV_ERR_INSUFFICIENT_RESOURCES:
+//         There could be multiple reasons for this error. One would be that
+//         it's not possible to allocate a queue of requested size. Another
+//         would be either event_buffer or event_control memory couldn't be
+//         pinned (e.g. because of OS limitation of pinnable memory). Also it
+//         could not have been possible to create UvmToolsEventQueueDescriptor.
+//
+//------------------------------------------------------------------------------
+NV_STATUS UvmToolsCreateEventQueue_V2(UvmToolsSessionHandle        session,
+                                      void                        *event_buffer,
+                                      NvLength                     event_buffer_size,
+                                      void                        *event_control,
+                                      UvmToolsEventQueueHandle    *queue);
 
 UvmToolsEventQueueDescriptor UvmToolsGetEventQueueDescriptor(UvmToolsEventQueueHandle queue);
 
@@ -3875,9 +3959,7 @@ NV_STATUS UvmToolsDisableCounters(UvmToolsCountersHandle counters,
 // In-process scenario when targetVa address + size overlaps with buffer + size.
 //
 // This is essentially a UVM version of RM ctrl call
-// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY. For implementation constraints (and more
-// information), please refer to the documentation:
-// //sw/docs/resman/components/compute/UVM/subsystems/UVM_8_Tools_API_Design.docx
+// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY.
 //
 // Arguments:
 //     session: (INPUT)
@@ -3931,9 +4013,7 @@ NV_STATUS UvmToolsReadProcessMemory(UvmToolsSessionHandle  session,
 // buffer + size.
 //
 // This is essentially a UVM version of RM ctrl call
-// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY. For implementation constraints (and more
-// information), please refer to the documentation:
-// //sw/docs/resman/components/compute/UVM/subsystems/UVM_8_Tools_API_Design.docx
+// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY.
 //
 // Arguments:
 //     session: (INPUT)
@@ -3976,6 +4056,8 @@ NV_STATUS UvmToolsWriteProcessMemory(UvmToolsSessionHandle  session,
 //------------------------------------------------------------------------------
 // UvmToolsGetProcessorUuidTable
 //
+// This function is deprecated. See UvmToolsGetProcessorUuidTable_V2.
+//
 // Populate a table with the UUIDs of all the currently registered processors
 // in the target process. When a GPU is registered, it is added to the table.
 // When a GPU is unregistered, it is removed. As long as a GPU remains
@@ -3988,61 +4070,63 @@ NV_STATUS UvmToolsWriteProcessMemory(UvmToolsSessionHandle  session,
 //     session: (INPUT)
 //         Handle to the tools session.
 //
-//     version: (INPUT)
-//         Requested version for the UUID table returned. The version must
-//         match the requested version of the event queue created with
-//         UvmToolsCreateEventQueue().
-//         See UvmEventEntry_V1 and UvmEventEntry_V2.
-//
 //     table: (OUTPUT)
 //         Array of processor UUIDs, including the CPU's UUID which is always
-//         at index zero.  The srcIndex and dstIndex fields of the
-//         UvmEventMigrationInfo struct index this array.  Unused indices will
-//         have a UUID of zero. Version UvmEventEntry_V1 only uses GPU UUIDs
-//         for the UUID of the physical GPU and only supports a single SMC
-//         partition registered per process. Version UvmEventEntry_V2 supports
-//         multiple SMC partitions registered per process and uses physical GPU
-//         UUIDs if the GPU is not SMC capable or SMC enabled and GPU instance
-//         UUIDs for SMC partitions.
-//         The table pointer can be NULL in which case, the size of the table
-//         needed to hold all the UUIDs is returned in 'count'.
-//
-//     table_size: (INPUT)
-//         The size of the table in number of array elements. This can be
-//         zero if the table pointer is NULL.
-//
-//     count: (OUTPUT)
-//         On output, it is set by UVM to the number of UUIDs needed to hold
-//         all the UUIDs, including any gaps in the table due to unregistered
-//         GPUs.
+//         at index zero. The number of elements in the array must be greater
+//         or equal to UVM_MAX_PROCESSORS_V1.
+//         The srcIndex and dstIndex fields of the UvmEventMigrationInfo struct
+//         index this array. Unused indices will have a UUID of zero.
+//         The reported UUID will be that of the corresponding physical GPU,
+//         even if multiple SMC partitions are registered under that physical
+//         GPU.
 //
 // Error codes:
 //     NV_ERR_INVALID_ADDRESS:
-//         writing to table failed or the count pointer was invalid.
-//
-//     NV_ERR_INVALID_ARGUMENT:
-//         The version is not UvmEventEntry_V1 or UvmEventEntry_V2.
-//         The count pointer is NULL.
-//         See UvmToolsEventQueueVersion.
-//
-//     NV_WARN_MISMATCHED_TARGET:
-//         The kernel returned a table suitable for UvmEventEntry_V1 events.
-//         (i.e., the kernel is older and doesn't support UvmEventEntry_V2).
+//         writing to table failed.
 //
 //     NV_ERR_NO_MEMORY:
 //         Internal memory allocation failed.
 //------------------------------------------------------------------------------
-#if UVM_API_REV_IS_AT_MOST(10)
-NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle  session,
-                                        NvProcessorUuid       *table,
-                                        NvLength              *count);
-#else
-NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle      session,
-                                        UvmToolsEventQueueVersion  version,
-                                        NvProcessorUuid           *table,
-                                        NvLength                   table_size,
-                                        NvLength                  *count);
-#endif
+NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle     session,
+                                        NvProcessorUuid          *table);
+
+//------------------------------------------------------------------------------
+// UvmToolsGetProcessorUuidTable_V2
+//
+// Populate a table with the UUIDs of all the currently registered processors
+// in the target process. When a GPU is registered, it is added to the table.
+// When a GPU is unregistered, it is removed. As long as a GPU remains
+// registered, its index in the table does not change.
+// Note that the index in the table corresponds to the processor ID reported
+// in UvmEventEntry event records and that the table is not contiguously packed
+// with non-zero UUIDs even with no GPU unregistrations.
+//
+// Arguments:
+//     session: (INPUT)
+//         Handle to the tools session.
+//
+//     table: (OUTPUT)
+//         Array of processor UUIDs, including the CPU's UUID which is always
+//         at index zero. The number of elements in the array must be greater
+//         or equal to UVM_MAX_PROCESSORS.
+//         The srcIndex and dstIndex fields of the UvmEventMigrationInfo struct
+//         index this array. Unused indices will have a UUID of zero.
+//         The reported UUID will be the GPU instance UUID if SMC is enabled,
+//         otherwise it will be the UUID of the physical GPU.
+//
+// Error codes:
+//     NV_ERR_INVALID_ADDRESS:
+//         writing to table failed.
+//
+//     NV_ERR_NOT_SUPPORTED:
+//         The UVM kernel driver is older and doesn't support
+//         UvmToolsGetProcessorUuidTable_V2.
+//
+//     NV_ERR_NO_MEMORY:
+//         Internal memory allocation failed.
+//------------------------------------------------------------------------------
+NV_STATUS UvmToolsGetProcessorUuidTable_V2(UvmToolsSessionHandle     session,
+                                           NvProcessorUuid          *table);
 
 //------------------------------------------------------------------------------
 // UvmToolsFlushEvents

kernel-open/nvidia-uvm/uvm_ada.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2021-2023 NVIDIA Corporation
+    Copyright (c) 2021-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -38,12 +38,10 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
 
     parent_gpu->utlb_per_gpc_count = uvm_ada_get_utlbs_per_gpc(parent_gpu);
 
-    parent_gpu->fault_buffer_info.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount *
-                                                          parent_gpu->utlb_per_gpc_count;
+    parent_gpu->fault_buffer.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount * parent_gpu->utlb_per_gpc_count;
     {
         uvm_fault_buffer_entry_t *dummy;
-        UVM_ASSERT(parent_gpu->fault_buffer_info.replayable.utlb_count <= (1 << (sizeof(dummy->fault_source.utlb_id) *
-                                                                                 8)));
+        UVM_ASSERT(parent_gpu->fault_buffer.replayable.utlb_count <= (1 << (sizeof(dummy->fault_source.utlb_id) * 8)));
     }
 
     // A single top level PDE on Ada covers 128 TB and that's the minimum size
@@ -51,6 +49,9 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
     parent_gpu->rm_va_base = 0;
     parent_gpu->rm_va_size = 128 * UVM_SIZE_1TB;
 
+    parent_gpu->peer_va_base = parent_gpu->rm_va_base + parent_gpu->rm_va_size;
+    parent_gpu->peer_va_size = NV_MAX_DEVICES * UVM_PEER_IDENTITY_VA_SIZE;
+
     parent_gpu->uvm_mem_va_base = 384 * UVM_SIZE_1TB;
     parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
 
@@ -79,8 +80,6 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
 
     parent_gpu->access_counters_supported = true;
 
-    parent_gpu->access_counters_can_use_physical_addresses = false;
-
     parent_gpu->fault_cancel_va_supported = true;
 
     parent_gpu->scoped_atomics_supported = true;
@@ -98,4 +97,6 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
     parent_gpu->plc_supported = true;
 
     parent_gpu->no_ats_range_required = false;
+
+    parent_gpu->conf_computing.per_channel_key_rotation = false;
 }

kernel-open/nvidia-uvm/uvm_ampere.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2023 NVIDIA Corporation
+    Copyright (c) 2018-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -38,12 +38,10 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
 
     parent_gpu->utlb_per_gpc_count = uvm_ampere_get_utlbs_per_gpc(parent_gpu);
 
-    parent_gpu->fault_buffer_info.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount *
-                                                          parent_gpu->utlb_per_gpc_count;
+    parent_gpu->fault_buffer.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount * parent_gpu->utlb_per_gpc_count;
     {
         uvm_fault_buffer_entry_t *dummy;
-        UVM_ASSERT(parent_gpu->fault_buffer_info.replayable.utlb_count <= (1 <<
-                                                                           (sizeof(dummy->fault_source.utlb_id) * 8)));
+        UVM_ASSERT(parent_gpu->fault_buffer.replayable.utlb_count <= (1 << (sizeof(dummy->fault_source.utlb_id) * 8)));
     }
 
     // A single top level PDE on Ampere covers 128 TB and that's the minimum
@@ -51,6 +49,9 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
     parent_gpu->rm_va_base = 0;
     parent_gpu->rm_va_size = 128 * UVM_SIZE_1TB;
 
+    parent_gpu->peer_va_base = parent_gpu->rm_va_base + parent_gpu->rm_va_size;
+    parent_gpu->peer_va_size = NV_MAX_DEVICES * UVM_PEER_IDENTITY_VA_SIZE;
+
     parent_gpu->uvm_mem_va_base = 384 * UVM_SIZE_1TB;
     parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
 
@@ -83,8 +84,6 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
 
     parent_gpu->access_counters_supported = true;
 
-    parent_gpu->access_counters_can_use_physical_addresses = false;
-
     parent_gpu->fault_cancel_va_supported = true;
 
     parent_gpu->scoped_atomics_supported = true;
@@ -107,4 +106,6 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
     parent_gpu->plc_supported = true;
 
     parent_gpu->no_ats_range_required = false;
+
+    parent_gpu->conf_computing.per_channel_key_rotation = false;
 }

kernel-open/nvidia-uvm/uvm_ampere_ce.c

@@ -29,6 +29,8 @@
 
 bool uvm_hal_ampere_ce_method_is_valid_c6b5(uvm_push_t *push, NvU32 method_address, NvU32 method_data)
 {
+    UVM_ASSERT(push->channel);
+
     if (!uvm_channel_is_proxy(push->channel))
         return true;
 
@@ -116,6 +118,16 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
 {
     NvU64 push_begin_gpu_va;
     uvm_gpu_t *gpu = uvm_push_get_gpu(push);
+    const bool peer_copy = uvm_gpu_address_is_peer(gpu, dst) || uvm_gpu_address_is_peer(gpu, src);
+
+    UVM_ASSERT(push->channel);
+
+    if (peer_copy && !uvm_channel_is_p2p(push->channel)) {
+        UVM_ERR_PRINT("Peer copy from address (0x%llx) to address (0x%llx) should use designated p2p channels!",
+                      src.address,
+                      dst.address);
+        return false;
+    }
 
     if (!uvm_parent_gpu_is_virt_mode_sriov_heavy(gpu->parent))
         return true;
@@ -182,6 +194,8 @@ void uvm_hal_ampere_ce_memcopy_patch_src_c6b5(uvm_push_t *push, uvm_gpu_address_
 {
     uvm_pushbuffer_t *pushbuffer;
 
+    UVM_ASSERT(push->channel);
+
     if (!uvm_channel_is_proxy(push->channel))
         return;
 

kernel-open/nvidia-uvm/uvm_ampere_fault_buffer.c

@@ -0,0 +1,75 @@
+/*******************************************************************************
+    Copyright (c) 2024 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+#include "uvm_linux.h"
+#include "uvm_global.h"
+#include "uvm_gpu.h"
+#include "uvm_hal.h"
+#include "hwref/ampere/ga100/dev_fault.h"
+
+static bool client_id_ce(NvU16 client_id)
+{
+    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE0 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE9)
+        return true;
+
+    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE10 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE15)
+        return true;
+
+    switch (client_id) {
+        case NV_PFAULT_CLIENT_HUB_CE0:
+        case NV_PFAULT_CLIENT_HUB_CE1:
+        case NV_PFAULT_CLIENT_HUB_CE2:
+            return true;
+    }
+
+    return false;
+}
+
+uvm_mmu_engine_type_t uvm_hal_ampere_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
+                                                                      uvm_fault_client_type_t client_type,
+                                                                      NvU16 client_id)
+{
+    // Servicing CE and Host (HUB clients) faults.
+    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB) {
+        if (client_id_ce(client_id)) {
+            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9);
+
+            return UVM_MMU_ENGINE_TYPE_CE;
+        }
+
+        if (client_id == NV_PFAULT_CLIENT_HUB_HOST || client_id == NV_PFAULT_CLIENT_HUB_ESC) {
+            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST31);
+
+            return UVM_MMU_ENGINE_TYPE_HOST;
+        }
+    }
+
+    // We shouldn't be servicing faults from any other engines other than GR.
+    UVM_ASSERT_MSG(client_id <= NV_PFAULT_CLIENT_GPC_ROP_3, "Unexpected client ID: 0x%x\n", client_id);
+    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS && mmu_engine_id < NV_PFAULT_MMU_ENG_ID_BAR1,
+                   "Unexpected engine ID: 0x%x\n",
+                   mmu_engine_id);
+    UVM_ASSERT(client_type == UVM_FAULT_CLIENT_TYPE_GPC);
+
+    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
+}

kernel-open/nvidia-uvm/uvm_ampere_host.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2023 NVIDIA Corporation
+    Copyright (c) 2018-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -36,6 +36,8 @@ bool uvm_hal_ampere_host_method_is_valid(uvm_push_t *push, NvU32 method_address,
     if (!uvm_parent_gpu_is_virt_mode_sriov_heavy(gpu->parent))
         return true;
 
+    UVM_ASSERT(push->channel);
+
     if (uvm_channel_is_privileged(push->channel)) {
         switch (method_address) {
             case NVC56F_SET_OBJECT:
@@ -84,6 +86,8 @@ bool uvm_hal_ampere_host_method_is_valid(uvm_push_t *push, NvU32 method_address,
 
 bool uvm_hal_ampere_host_sw_method_is_valid(uvm_push_t *push, NvU32 method_address, NvU32 method_data)
 {
+    UVM_ASSERT(push->channel);
+
     if (!uvm_channel_is_proxy(push->channel))
         return true;
 
@@ -205,17 +209,18 @@ void uvm_hal_ampere_host_clear_faulted_channel_sw_method(uvm_push_t *push,
                      CLEAR_FAULTED_B, HWVALUE(C076, CLEAR_FAULTED_B, INST_HI, instance_ptr_hi));
 }
 
-// Copy from Pascal, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
-                                            uvm_gpu_phys_address_t pdb,
-                                            NvU32 depth,
-                                            uvm_membar_t membar)
+                                           uvm_gpu_phys_address_t pdb,
+                                           NvU32 depth,
+                                           uvm_membar_t membar)
 {
     NvU32 aperture_value;
     NvU32 page_table_level;
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
 
@@ -230,8 +235,8 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
     pdb_lo = pdb.address & HWMASK(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
-    // PDE3 is the highest level on Pascal, see the comment in uvm_pascal_mmu.c
-    // for details.
+    // PDE3 is the highest level on Pascal-Ampere, see the comment in
+    // uvm_pascal_mmu.c for details.
     UVM_ASSERT_MSG(depth < NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
     page_table_level = NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
 
@@ -242,7 +247,12 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
+    if (membar == UVM_MEMBAR_SYS)
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
+    NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
                                HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                      MEM_OP_B, 0,
                      MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
@@ -255,16 +265,18 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C56F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
                                HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    uvm_hal_tlb_invalidate_membar(push, membar);
+    // GPU membar still requires an explicit membar method.
+    if (membar == UVM_MEMBAR_GPU)
+        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
 }
 
-// Copy from Volta, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
                                            uvm_gpu_phys_address_t pdb,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU32 page_size,
+                                           NvU64 page_size,
                                            uvm_membar_t membar)
 {
     NvU32 aperture_value;
@@ -272,6 +284,7 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
     NvU32 va_lo;
     NvU32 va_hi;
     NvU64 end;
@@ -281,9 +294,9 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 log2_invalidation_size;
     uvm_gpu_t *gpu = uvm_push_get_gpu(push);
 
-    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%x\n", page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%x\n", base, page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%x\n", size, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
     UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
 
     // The invalidation size must be a power-of-two number of pages containing
@@ -325,7 +338,7 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     pdb_lo = pdb.address & HWMASK(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
-    // PDE3 is the highest level on Pascal-Ampere , see the comment in
+    // PDE3 is the highest level on Pascal-Ampere, see the comment in
     // uvm_pascal_mmu.c for details.
     UVM_ASSERT_MSG(depth < NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
     page_table_level = NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
@@ -337,10 +350,15 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
+    if (membar == UVM_MEMBAR_SYS)
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
     NV_PUSH_4U(C56F, MEM_OP_A, HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_INVALIDATION_SIZE, log2_invalidation_size) |
-                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
-                               HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo) |
-                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
+                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
+                               sysmembar_value |
+                               HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
                      MEM_OP_B, HWVALUE(C56F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
                      MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
                                HWVALUE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
@@ -352,21 +370,23 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C56F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
                                HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    uvm_hal_tlb_invalidate_membar(push, membar);
+    // GPU membar still requires an explicit membar method.
+    if (membar == UVM_MEMBAR_GPU)
+        gpu->parent->host_hal->membar_gpu(push);
 }
 
-// Copy from Pascal, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                              uvm_gpu_phys_address_t pdb,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params)
 {
     NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
     NvU32 invalidate_gpc_value = 0;
     NvU32 aperture_value = 0;
     NvU32 pdb_lo = 0;
     NvU32 pdb_hi = 0;
     NvU32 page_table_level = 0;
-    uvm_membar_t membar;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
     if (pdb.aperture == UVM_APERTURE_VID)
@@ -381,7 +401,7 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
     if (params->page_table_level != UvmInvalidatePageTableLevelAll) {
-        // PDE3 is the highest level on Pascal, see the comment in
+        // PDE3 is the highest level on Pascal-Ampere, see the comment in
         // uvm_pascal_mmu.c for details.
         page_table_level = min((NvU32)UvmInvalidatePageTableLevelPde3, params->page_table_level) - 1;
     }
@@ -393,6 +413,11 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
+    if (params->membar == UvmInvalidateTlbMemBarSys)
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
     if (params->disable_gpc_invalidate)
         invalidate_gpc_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
     else
@@ -403,9 +428,9 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
 
         NvU32 va_lo = va & HWMASK(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
         NvU32 va_hi = va >> HWSIZE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-        NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
-                                   HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo) |
-                                   HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
+        NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
+                                   HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
+                                   HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
                          MEM_OP_B, HWVALUE(C56F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
                          MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
                                    HWVALUE(C56F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
@@ -418,7 +443,7 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
     else {
-        NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
+        NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
                                    HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                          MEM_OP_B, 0,
                          MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
@@ -432,12 +457,7 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
 
-    if (params->membar == UvmInvalidateTlbMemBarSys)
-        membar = UVM_MEMBAR_SYS;
-    else if (params->membar == UvmInvalidateTlbMemBarLocal)
-        membar = UVM_MEMBAR_GPU;
-    else
-        membar = UVM_MEMBAR_NONE;
-
-    uvm_hal_tlb_invalidate_membar(push, membar);
+    // GPU membar still requires an explicit membar method.
+    if (params->membar == UvmInvalidateTlbMemBarLocal)
+        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
 }

kernel-open/nvidia-uvm/uvm_ampere_mmu.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2020 NVIDIA Corporation
+    Copyright (c) 2018-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -36,22 +36,7 @@
 #include "uvm_ampere_fault_buffer.h"
 #include "hwref/ampere/ga100/dev_fault.h"
 
-uvm_mmu_engine_type_t uvm_hal_ampere_mmu_engine_id_to_type(NvU16 mmu_engine_id)
-{
-    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST31)
-        return UVM_MMU_ENGINE_TYPE_HOST;
-
-    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9)
-        return UVM_MMU_ENGINE_TYPE_CE;
-
-    // We shouldn't be servicing faults from any other engines
-    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS && mmu_engine_id < NV_PFAULT_MMU_ENG_ID_BAR1,
-                   "Unexpected engine ID: 0x%x\n", mmu_engine_id);
-
-    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
-}
-
-static NvU32 page_table_depth_ampere(NvU32 page_size)
+static NvU32 page_table_depth_ampere(NvU64 page_size)
 {
     // The common-case is page_size == UVM_PAGE_SIZE_2M, hence the first check
     if (page_size == UVM_PAGE_SIZE_2M)
@@ -62,14 +47,14 @@ static NvU32 page_table_depth_ampere(NvU32 page_size)
         return 4;
 }
 
-static NvU32 page_sizes_ampere(void)
+static NvU64 page_sizes_ampere(void)
 {
     return UVM_PAGE_SIZE_512M | UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
 }
 
 static uvm_mmu_mode_hal_t ampere_mmu_mode_hal;
 
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU32 big_page_size)
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU64 big_page_size)
 {
     static bool initialized = false;
 

kernel-open/nvidia-uvm/uvm_api.h

@@ -47,7 +47,7 @@
     {                                                                               \
         params_type params;                                                         \
         BUILD_BUG_ON(sizeof(params) > UVM_MAX_IOCTL_PARAM_STACK_SIZE);              \
-        if (nv_copy_from_user(&params, (void __user*)arg, sizeof(params)))          \
+        if (copy_from_user(&params, (void __user*)arg, sizeof(params)))             \
             return -EFAULT;                                                         \
                                                                                     \
         params.rmStatus = uvm_global_get_status();                                  \
@@ -60,7 +60,7 @@
                 params.rmStatus = function_name(&params, filp);                     \
         }                                                                           \
                                                                                     \
-        if (nv_copy_to_user((void __user*)arg, &params, sizeof(params)))            \
+        if (copy_to_user((void __user*)arg, &params, sizeof(params)))               \
             return -EFAULT;                                                         \
                                                                                     \
         return 0;                                                                   \
@@ -84,7 +84,7 @@
         if (!params)                                                                    \
             return -ENOMEM;                                                             \
         BUILD_BUG_ON(sizeof(*params) <= UVM_MAX_IOCTL_PARAM_STACK_SIZE);                \
-        if (nv_copy_from_user(params, (void __user*)arg, sizeof(*params))) {            \
+        if (copy_from_user(params, (void __user*)arg, sizeof(*params))) {               \
             uvm_kvfree(params);                                                         \
             return -EFAULT;                                                             \
         }                                                                               \
@@ -99,7 +99,7 @@
                 params->rmStatus = function_name(params, filp);                         \
         }                                                                               \
                                                                                         \
-        if (nv_copy_to_user((void __user*)arg, params, sizeof(*params)))                \
+        if (copy_to_user((void __user*)arg, params, sizeof(*params)))                   \
             ret = -EFAULT;                                                              \
                                                                                         \
         uvm_kvfree(params);                                                             \
@@ -244,6 +244,7 @@ NV_STATUS uvm_api_migrate(UVM_MIGRATE_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_enable_system_wide_atomics(UVM_ENABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_disable_system_wide_atomics(UVM_DISABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_tools_init_event_tracker(UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS *params, struct file *filp);
+NV_STATUS uvm_api_tools_init_event_tracker_v2(UVM_TOOLS_INIT_EVENT_TRACKER_V2_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_tools_set_notification_threshold(UVM_TOOLS_SET_NOTIFICATION_THRESHOLD_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_tools_event_queue_enable_events(UVM_TOOLS_EVENT_QUEUE_ENABLE_EVENTS_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_tools_event_queue_disable_events(UVM_TOOLS_EVENT_QUEUE_DISABLE_EVENTS_PARAMS *params, struct file *filp);
@@ -256,5 +257,7 @@ NV_STATUS uvm_api_unmap_external(UVM_UNMAP_EXTERNAL_PARAMS *params, struct file
 NV_STATUS uvm_api_migrate_range_group(UVM_MIGRATE_RANGE_GROUP_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_alloc_semaphore_pool(UVM_ALLOC_SEMAPHORE_POOL_PARAMS *params, struct file *filp);
 NV_STATUS uvm_api_populate_pageable(const UVM_POPULATE_PAGEABLE_PARAMS *params, struct file *filp);
+NV_STATUS uvm_api_alloc_device_p2p(UVM_ALLOC_DEVICE_P2P_PARAMS *params, struct file *filp);
+NV_STATUS uvm_api_clear_all_access_counters(UVM_CLEAR_ALL_ACCESS_COUNTERS_PARAMS *params, struct file *filp);
 
 #endif // __UVM_API_H__

kernel-open/nvidia-uvm/uvm_ats.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2021 NVIDIA Corporation
+    Copyright (c) 2018-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to

kernel-open/nvidia-uvm/uvm_ats.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2021 NVIDIA Corporation
+    Copyright (c) 2018-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -29,10 +29,9 @@
 #include "uvm_ats_ibm.h"
 #include "nv_uvm_types.h"
 #include "uvm_lock.h"
+#include "uvm_ats_sva.h"
 
-    #include "uvm_ats_sva.h"
-
-    #define UVM_ATS_SUPPORTED() (UVM_ATS_IBM_SUPPORTED() || UVM_ATS_SVA_SUPPORTED())
+#define UVM_ATS_SUPPORTED() (UVM_ATS_IBM_SUPPORTED() || UVM_ATS_SVA_SUPPORTED())
 
 typedef struct
 {

kernel-open/nvidia-uvm/uvm_ats_faults.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2023 NVIDIA Corporation
+    Copyright (c) 2024-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -29,6 +29,7 @@
 #include <linux/nodemask.h>
 #include <linux/mempolicy.h>
 #include <linux/mmu_notifier.h>
+#include <linux/topology.h>
 
 #if UVM_HMM_RANGE_FAULT_SUPPORTED()
 #include <linux/hmm.h>
@@ -55,8 +56,9 @@ static NV_STATUS service_ats_requests(uvm_gpu_va_space_t *gpu_va_space,
     NvU64 user_space_start;
     NvU64 user_space_length;
     bool write = (access_type >= UVM_FAULT_ACCESS_TYPE_WRITE);
-    bool fault_service_type = (service_type == UVM_ATS_SERVICE_TYPE_FAULTS);
-    uvm_populate_permissions_t populate_permissions = fault_service_type ?
+    bool is_fault_service_type = (service_type == UVM_ATS_SERVICE_TYPE_FAULTS);
+    bool is_prefetch_faults = (is_fault_service_type && (access_type == UVM_FAULT_ACCESS_TYPE_PREFETCH));
+    uvm_populate_permissions_t populate_permissions = is_fault_service_type ?
                                             (write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY) :
                                             UVM_POPULATE_PERMISSIONS_INHERIT;
 
@@ -90,18 +92,27 @@ static NV_STATUS service_ats_requests(uvm_gpu_va_space_t *gpu_va_space,
 
     uvm_migrate_args_t uvm_migrate_args =
     {
-        .va_space                       = va_space,
-        .mm                             = mm,
-        .dst_id                         = ats_context->residency_id,
-        .dst_node_id                    = ats_context->residency_node,
-        .start                          = start,
-        .length                         = length,
-        .populate_permissions           = populate_permissions,
-        .touch                          = fault_service_type,
-        .skip_mapped                    = fault_service_type,
-        .populate_on_cpu_alloc_failures = fault_service_type,
-        .user_space_start               = &user_space_start,
-        .user_space_length              = &user_space_length,
+        .va_space                           = va_space,
+        .mm                                 = mm,
+        .dst_id                             = ats_context->residency_id,
+        .dst_node_id                        = ats_context->residency_node,
+        .start                              = start,
+        .length                             = length,
+        .populate_permissions               = populate_permissions,
+        .touch                              = is_fault_service_type,
+        .skip_mapped                        = is_fault_service_type,
+        .populate_on_cpu_alloc_failures     = is_fault_service_type,
+        .populate_on_migrate_vma_failures   = is_fault_service_type,
+        .user_space_start                   = &user_space_start,
+        .user_space_length                  = &user_space_length,
+
+        // Potential STO NVLINK errors cannot be resolved in fault or access
+        // counter handlers. If there are GPUs to check for STO, it's either
+        // a) a false positive, and the migration went through ok, or
+        // b) a true positive, and the destination is all zeros, and the
+        //    application will be terminated soon.
+        .gpus_to_check_for_nvlink_errors    = NULL,
+        .fail_on_unresolved_sto_errors      = !is_fault_service_type || is_prefetch_faults,
     };
 
     UVM_ASSERT(uvm_ats_can_service_faults(gpu_va_space, mm));
@@ -112,7 +123,7 @@ static NV_STATUS service_ats_requests(uvm_gpu_va_space_t *gpu_va_space,
     // set skip_mapped to true. For pages already mapped, this will only handle
     // PTE upgrades if needed.
     status = uvm_migrate_pageable(&uvm_migrate_args);
-    if (status == NV_WARN_NOTHING_TO_DO)
+    if (is_fault_service_type && (status == NV_WARN_NOTHING_TO_DO))
         status = NV_OK;
 
     UVM_ASSERT(status != NV_ERR_MORE_PROCESSING_REQUIRED);
@@ -128,9 +139,9 @@ static void flush_tlb_va_region(uvm_gpu_va_space_t *gpu_va_space,
     uvm_ats_fault_invalidate_t *ats_invalidate;
 
     if (client_type == UVM_FAULT_CLIENT_TYPE_GPC)
-        ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer_info.replayable.ats_invalidate;
+        ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer.replayable.ats_invalidate;
     else
-        ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer_info.non_replayable.ats_invalidate;
+        ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer.non_replayable.ats_invalidate;
 
     if (!ats_invalidate->tlb_batch_pending) {
         uvm_tlb_batch_begin(&gpu_va_space->page_tables, &ats_invalidate->tlb_batch);
@@ -145,7 +156,10 @@ static void ats_batch_select_residency(uvm_gpu_va_space_t *gpu_va_space,
                                        uvm_ats_fault_context_t *ats_context)
 {
     uvm_gpu_t *gpu = gpu_va_space->gpu;
-    int residency = uvm_gpu_numa_node(gpu);
+    int residency;
+
+    UVM_ASSERT(gpu->mem_info.numa.enabled);
+    residency = uvm_gpu_numa_node(gpu);
 
 #if defined(NV_MEMPOLICY_HAS_UNIFIED_NODES)
     struct mempolicy *vma_policy = vma_policy(vma);
@@ -278,6 +292,27 @@ static const struct mmu_interval_notifier_ops uvm_ats_notifier_ops =
 
 #endif
 
+static bool resident_policy_match(struct vm_area_struct *vma, int dst_nid, int src_nid)
+{
+#if defined(NV_MEMPOLICY_HAS_UNIFIED_NODES)
+    struct mempolicy *vma_policy = vma_policy(vma);
+
+    // TODO: Bug 4981209: When migrations between CPU numa nodes are supported,
+    // add (dst_nid != closest_cpu_numa_node) to allow migrations between CPU
+    // NUMA nodes when destination is the closest_cpu_numa_node.
+    if (vma_policy &&
+        node_isset(src_nid, vma_policy->nodes) &&
+        node_isset(dst_nid, vma_policy->nodes) &&
+        !cpumask_empty(cpumask_of_node(src_nid)) &&
+        !cpumask_empty(cpumask_of_node(dst_nid))) {
+
+        return true;
+    }
+#endif
+
+    return false;
+}
+
 static NV_STATUS ats_compute_residency_mask(uvm_gpu_va_space_t *gpu_va_space,
                                             struct vm_area_struct *vma,
                                             NvU64 base,
@@ -357,9 +392,23 @@ static NV_STATUS ats_compute_residency_mask(uvm_gpu_va_space_t *gpu_va_space,
 
             if (pfn & HMM_PFN_VALID) {
                 struct page *page = hmm_pfn_to_page(pfn);
+                int resident_node = page_to_nid(page);
 
-                if (page_to_nid(page) == ats_context->residency_node)
+                // Set the residency_mask if:
+                // - The page is already resident at the intended destination.
+                //   or
+                // - If both the source and destination nodes are CPU nodes and
+                //   source node is already in the list of preferred nodes for
+                //   the vma. On multi-CPU NUMA node architectures, this avoids
+                //   unnecessary migrations between CPU nodes. Since the
+                //   specific ats_context->residency_node selected by
+                //   ats_batch_select_residency() is just a guess among the list
+                //   of preferred nodes, paying the cost of migration across the
+                //   CPU preferred nodes in this case can't be justified.
+                if ((resident_node == ats_context->residency_node) ||
+                    resident_policy_match(vma, ats_context->residency_node, resident_node)) {
                     uvm_page_mask_set(residency_mask, page_index);
+                }
 
                 ats_context->prefetch_state.first_touch = false;
             }
@@ -379,14 +428,20 @@ static NV_STATUS ats_compute_residency_mask(uvm_gpu_va_space_t *gpu_va_space,
 
 static void ats_compute_prefetch_mask(uvm_gpu_va_space_t *gpu_va_space,
                                       struct vm_area_struct *vma,
+                                      uvm_ats_service_type_t service_type,
                                       uvm_ats_fault_context_t *ats_context,
                                       uvm_va_block_region_t max_prefetch_region)
 {
-    uvm_page_mask_t *accessed_mask = &ats_context->accessed_mask;
+    uvm_page_mask_t *accessed_mask;
     uvm_page_mask_t *residency_mask = &ats_context->prefetch_state.residency_mask;
     uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask;
     uvm_perf_prefetch_bitmap_tree_t *bitmap_tree = &ats_context->prefetch_state.bitmap_tree;
 
+    if (service_type == UVM_ATS_SERVICE_TYPE_FAULTS)
+        accessed_mask = &ats_context->faults.accessed_mask;
+    else
+        accessed_mask = &ats_context->access_counters.accessed_mask;
+
     if (uvm_page_mask_empty(accessed_mask))
         return;
 
@@ -406,7 +461,7 @@ static NV_STATUS ats_compute_prefetch(uvm_gpu_va_space_t *gpu_va_space,
                                       uvm_ats_fault_context_t *ats_context)
 {
     NV_STATUS status;
-    uvm_page_mask_t *accessed_mask = &ats_context->accessed_mask;
+    uvm_page_mask_t *accessed_mask;
     uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask;
     uvm_va_block_region_t max_prefetch_region = uvm_ats_region_from_vma(vma, base);
 
@@ -420,6 +475,11 @@ static NV_STATUS ats_compute_prefetch(uvm_gpu_va_space_t *gpu_va_space,
     if (!uvm_perf_prefetch_enabled(gpu_va_space->va_space))
         return status;
 
+    if (service_type == UVM_ATS_SERVICE_TYPE_FAULTS)
+        accessed_mask = &ats_context->faults.accessed_mask;
+    else
+        accessed_mask = &ats_context->access_counters.accessed_mask;
+
     if (uvm_page_mask_empty(accessed_mask))
         return status;
 
@@ -432,12 +492,12 @@ static NV_STATUS ats_compute_prefetch(uvm_gpu_va_space_t *gpu_va_space,
         uvm_page_mask_init_from_region(prefetch_mask, max_prefetch_region, NULL);
     }
     else {
-        ats_compute_prefetch_mask(gpu_va_space, vma, ats_context, max_prefetch_region);
+        ats_compute_prefetch_mask(gpu_va_space, vma, service_type, ats_context, max_prefetch_region);
     }
 
     if (service_type == UVM_ATS_SERVICE_TYPE_FAULTS) {
-        uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask;
-        uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask;
+        uvm_page_mask_t *read_fault_mask = &ats_context->faults.read_fault_mask;
+        uvm_page_mask_t *write_fault_mask = &ats_context->faults.write_fault_mask;
 
         uvm_page_mask_or(read_fault_mask, read_fault_mask, prefetch_mask);
 
@@ -451,6 +511,65 @@ static NV_STATUS ats_compute_prefetch(uvm_gpu_va_space_t *gpu_va_space,
     return status;
 }
 
+static NV_STATUS uvm_ats_service_faults_region(uvm_gpu_va_space_t *gpu_va_space,
+                                               struct vm_area_struct *vma,
+                                               NvU64 base,
+                                               uvm_va_block_region_t region,
+                                               uvm_fault_access_type_t access_type,
+                                               uvm_ats_fault_context_t *ats_context,
+                                               uvm_page_mask_t *faults_serviced_mask)
+{
+    NvU64 start = base + (region.first * PAGE_SIZE);
+    size_t length = uvm_va_block_region_size(region);
+    NV_STATUS status;
+
+    UVM_ASSERT(start >= vma->vm_start);
+    UVM_ASSERT((start + length) <= vma->vm_end);
+
+    status = service_ats_requests(gpu_va_space,
+                                  vma,
+                                  start,
+                                  length,
+                                  access_type,
+                                  UVM_ATS_SERVICE_TYPE_FAULTS,
+                                  ats_context);
+    if (status != NV_OK)
+        return status;
+
+    uvm_page_mask_region_fill(faults_serviced_mask, region);
+
+    // WAR for older kernel versions missing an SMMU invalidate on RO -> RW
+    // transition. The SMMU and GPU could have the stale RO copy cached in their
+    // TLBs, which could have caused this write fault. This operation
+    // invalidates the SMMU TLBs but not the GPU TLBs. That will happen below as
+    // necessary.
+    if (access_type == UVM_FAULT_ACCESS_TYPE_WRITE)
+        uvm_ats_smmu_invalidate_tlbs(gpu_va_space, start, length);
+
+    // The Linux kernel does not invalidate TLB entries on an invalid to valid
+    // PTE transition. The GPU might have the invalid PTE cached in its TLB.
+    // The GPU will re-fetch an entry on access if the PTE is invalid and the
+    // page size is not 4K, but if the page size is 4K no re-fetch will happen
+    // and the GPU will fault despite the CPU PTE being valid. We don't know
+    // whether these faults happened due to stale entries after a transition,
+    // so use the hammer of always invalidating the GPU's TLB on each fault.
+    //
+    // The second case is similar and handles missing ATS invalidations on RO ->
+    // RW transitions for all page sizes. See the uvm_ats_smmu_invalidate_tlbs()
+    // call above.
+    if (PAGE_SIZE == UVM_PAGE_SIZE_4K || (UVM_ATS_SMMU_WAR_REQUIRED() && access_type == UVM_FAULT_ACCESS_TYPE_WRITE)) {
+        flush_tlb_va_region(gpu_va_space, start, length, ats_context->client_type);
+    }
+    else {
+        // ARM requires TLB invalidations on RO -> RW, but not all architectures
+        // do. If we implement ATS support on other architectures, we might need
+        // to issue GPU invalidates.
+        UVM_ASSERT(NVCPU_IS_AARCH64);
+    }
+
+    return NV_OK;
+}
+
 NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
                                  struct vm_area_struct *vma,
                                  NvU64 base,
@@ -459,12 +578,11 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
     NV_STATUS status = NV_OK;
     uvm_va_block_region_t subregion;
     uvm_va_block_region_t region = uvm_va_block_region(0, PAGES_PER_UVM_VA_BLOCK);
-    uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask;
-    uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask;
-    uvm_page_mask_t *faults_serviced_mask = &ats_context->faults_serviced_mask;
-    uvm_page_mask_t *reads_serviced_mask = &ats_context->reads_serviced_mask;
-    uvm_fault_client_type_t client_type = ats_context->client_type;
-    uvm_ats_service_type_t service_type = UVM_ATS_SERVICE_TYPE_FAULTS;
+    uvm_page_mask_t *prefetch_only_fault_mask = &ats_context->faults.prefetch_only_fault_mask;
+    uvm_page_mask_t *read_fault_mask = &ats_context->faults.read_fault_mask;
+    uvm_page_mask_t *write_fault_mask = &ats_context->faults.write_fault_mask;
+    uvm_page_mask_t *faults_serviced_mask = &ats_context->faults.faults_serviced_mask;
+    uvm_page_mask_t *reads_serviced_mask = &ats_context->faults.reads_serviced_mask;
 
     UVM_ASSERT(vma);
     UVM_ASSERT(IS_ALIGNED(base, UVM_VA_BLOCK_SIZE));
@@ -480,7 +598,7 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
     uvm_page_mask_zero(reads_serviced_mask);
 
     if (!(vma->vm_flags & VM_READ))
-        return status;
+        return NV_OK;
 
     if (!(vma->vm_flags & VM_WRITE)) {
         // If VMA doesn't have write permissions, all write faults are fatal.
@@ -496,72 +614,65 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
 
         // There are no pending faults beyond write faults to RO region.
         if (uvm_page_mask_empty(read_fault_mask))
-            return status;
+            return NV_OK;
     }
 
     ats_batch_select_residency(gpu_va_space, vma, ats_context);
 
-    ats_compute_prefetch(gpu_va_space, vma, base, service_type, ats_context);
+    ats_compute_prefetch(gpu_va_space, vma, base, UVM_ATS_SERVICE_TYPE_FAULTS, ats_context);
 
     for_each_va_block_subregion_in_mask(subregion, write_fault_mask, region) {
-        NvU64 start = base + (subregion.first * PAGE_SIZE);
-        size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
-        uvm_fault_access_type_t access_type = (vma->vm_flags & VM_WRITE) ?
-                                                                          UVM_FAULT_ACCESS_TYPE_WRITE :
-                                                                          UVM_FAULT_ACCESS_TYPE_READ;
-
-        UVM_ASSERT(start >= vma->vm_start);
-        UVM_ASSERT((start + length) <= vma->vm_end);
-
-        status = service_ats_requests(gpu_va_space, vma, start, length, access_type, service_type, ats_context);
-        if (status != NV_OK)
-            return status;
+        uvm_fault_access_type_t access_type;
+        uvm_page_mask_t *serviced_mask;
 
         if (vma->vm_flags & VM_WRITE) {
-            uvm_page_mask_region_fill(faults_serviced_mask, subregion);
-            uvm_ats_smmu_invalidate_tlbs(gpu_va_space, start, length);
-
-            // The Linux kernel never invalidates TLB entries on mapping
-            // permission upgrade. This is a problem if the GPU has cached
-            // entries with the old permission. The GPU will re-fetch the entry
-            // if the PTE is invalid and page size is not 4K (this is the case
-            // on P9). However, if a page gets upgraded from R/O to R/W and GPU
-            // has the PTEs cached with R/O permissions we will enter an
-            // infinite loop because we just forward the fault to the Linux
-            // kernel and it will see that the permissions in the page table are
-            // correct. Therefore, we flush TLB entries on ATS write faults.
-            flush_tlb_va_region(gpu_va_space, start, length, client_type);
+            access_type = UVM_FAULT_ACCESS_TYPE_WRITE;
+            serviced_mask = faults_serviced_mask;
         }
         else {
-            uvm_page_mask_region_fill(reads_serviced_mask, subregion);
+            // write_fault_mask contains just the addresses with both read and
+            // fatal write faults, so we need to service the read component.
+            access_type = UVM_FAULT_ACCESS_TYPE_READ;
+            serviced_mask = reads_serviced_mask;
         }
+
+        status = uvm_ats_service_faults_region(gpu_va_space,
+                                               vma,
+                                               base,
+                                               subregion,
+                                               access_type,
+                                               ats_context,
+                                               serviced_mask);
+        if (status != NV_OK)
+            return status;
     }
 
-    // Remove write faults from read_fault_mask
+    // Remove write faults from read_fault_mask to avoid double-service
     uvm_page_mask_andnot(read_fault_mask, read_fault_mask, write_fault_mask);
 
     for_each_va_block_subregion_in_mask(subregion, read_fault_mask, region) {
-        NvU64 start = base + (subregion.first * PAGE_SIZE);
-        size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
-        uvm_fault_access_type_t access_type = UVM_FAULT_ACCESS_TYPE_READ;
-
-        UVM_ASSERT(start >= vma->vm_start);
-        UVM_ASSERT((start + length) <= vma->vm_end);
-
-        status = service_ats_requests(gpu_va_space, vma, start, length, access_type, service_type, ats_context);
+        status = uvm_ats_service_faults_region(gpu_va_space,
+                                               vma,
+                                               base,
+                                               subregion,
+                                               UVM_FAULT_ACCESS_TYPE_READ,
+                                               ats_context,
+                                               faults_serviced_mask);
         if (status != NV_OK)
             return status;
+    }
 
-        uvm_page_mask_region_fill(faults_serviced_mask, subregion);
-
-        // Similarly to permission upgrade scenario, discussed above, GPU
-        // will not re-fetch the entry if the PTE is invalid and page size
-        // is 4K. To avoid infinite faulting loop, invalidate TLB for every
-        // new translation written explicitly like in the case of permission
-        // upgrade.
-        if (PAGE_SIZE == UVM_PAGE_SIZE_4K)
-            flush_tlb_va_region(gpu_va_space, start, length, client_type);
-
+    // Handle HW prefetch only faults
+    for_each_va_block_subregion_in_mask(subregion, prefetch_only_fault_mask, region) {
+        status = uvm_ats_service_faults_region(gpu_va_space,
+                                               vma,
+                                               base,
+                                               subregion,
+                                               UVM_FAULT_ACCESS_TYPE_PREFETCH,
+                                               ats_context,
+                                               faults_serviced_mask);
+        if (status != NV_OK)
+            return status;
     }
 
     return status;
@@ -637,6 +748,8 @@ NV_STATUS uvm_ats_service_access_counters(uvm_gpu_va_space_t *gpu_va_space,
     UVM_ASSERT(gpu_va_space->ats.enabled);
     UVM_ASSERT(uvm_gpu_va_space_state(gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE);
 
+    uvm_page_mask_zero(&ats_context->access_counters.migrated_mask);
+
     uvm_assert_mmap_lock_locked(vma->vm_mm);
     uvm_assert_rwsem_locked(&gpu_va_space->va_space->lock);
 
@@ -650,21 +763,41 @@ NV_STATUS uvm_ats_service_access_counters(uvm_gpu_va_space_t *gpu_va_space,
 
     // Remove pages which are already resident at the intended destination from
     // the accessed_mask.
-    uvm_page_mask_andnot(&ats_context->accessed_mask,
-                         &ats_context->accessed_mask,
+    uvm_page_mask_andnot(&ats_context->access_counters.accessed_mask,
+                         &ats_context->access_counters.accessed_mask,
                          &ats_context->prefetch_state.residency_mask);
 
-    for_each_va_block_subregion_in_mask(subregion, &ats_context->accessed_mask, region) {
+    // Pretend that pages that are already resident at the destination GPU were
+    // migrated now. This makes sure that the access counter is cleared even if
+    // the accessed pages, were already resident on the target.
+    // TODO: Bug 5296998: [uvm][ats] Not clearing stale access counter
+    //                     notifications can lead to missed migrations
+    // The same problem of stale notification exists for migration to other
+    // locations than local vidmem. However, stale notifications to data
+    // migrated to another remote location are identical to those triggered
+    // by accessing memory that cannot or should not be migrated.
+    if (uvm_id_equal(ats_context->residency_id, gpu_va_space->gpu->id)) {
+        uvm_page_mask_copy(&ats_context->access_counters.migrated_mask,
+                           &ats_context->prefetch_state.residency_mask);
+    }
+
+    for_each_va_block_subregion_in_mask(subregion, &ats_context->access_counters.accessed_mask, region) {
         NV_STATUS status;
         NvU64 start = base + (subregion.first * PAGE_SIZE);
         size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
         uvm_fault_access_type_t access_type = UVM_FAULT_ACCESS_TYPE_COUNT;
+        uvm_page_mask_t *migrated_mask = &ats_context->access_counters.migrated_mask;
 
         UVM_ASSERT(start >= vma->vm_start);
         UVM_ASSERT((start + length) <= vma->vm_end);
 
         status = service_ats_requests(gpu_va_space, vma, start, length, access_type, service_type, ats_context);
-        if (status != NV_OK)
+
+        // Clear access counters if pages were migrated or migration needs to
+        // be retried
+        if (status == NV_OK || status == NV_ERR_BUSY_RETRY)
+            uvm_page_mask_region_fill(migrated_mask, subregion);
+        else if (status != NV_WARN_NOTHING_TO_DO)
             return status;
     }
 

kernel-open/nvidia-uvm/uvm_ats_faults.h

@@ -29,18 +29,19 @@
 
 // Service ATS faults in the range (base, base + UVM_VA_BLOCK_SIZE) with service
 // type for individual pages in the range requested by page masks set in
-// ats_context->read_fault_mask/write_fault_mask. base must be aligned to
-// UVM_VA_BLOCK_SIZE. The caller is responsible for ensuring that faulting
-// addresses fall completely within the VMA. The caller is also responsible for
-// ensuring that the faulting addresses don't overlap a GMMU region. (See
-// uvm_ats_check_in_gmmu_region). The caller is also responsible for handling
-// any errors returned by this function (fault cancellations etc.).
+// ats_context->fault.read_fault_mask/write_fault_mask/prefetch_only_mask.
+// base must be aligned to UVM_VA_BLOCK_SIZE. The caller is responsible for
+// ensuring that faulting addresses fall completely within the VMA. The caller
+// is also responsible for ensuring that the faulting addresses don't overlap
+// a GMMU region. (See uvm_ats_check_in_gmmu_region). The caller is also
+// responsible for handling any errors returned by this function (fault
+// cancellations etc.).
 //
-// Returns the fault service status in ats_context->faults_serviced_mask. In
-// addition, ats_context->reads_serviced_mask returns whether read servicing
-// worked on write faults iff the read service was also requested in the
-// corresponding bit in read_fault_mask. These returned masks are only valid if
-// the return status is NV_OK. Status other than NV_OK indicate system global
+// Returns the fault service status in ats_context->fault.faults_serviced_mask.
+// In addition, ats_context->fault.reads_serviced_mask returns whether read
+// servicing worked on write faults iff the read service was also requested in
+// the corresponding bit in read_fault_mask. These returned masks are only valid
+// if the return status is NV_OK. Status other than NV_OK indicate system global
 // fault servicing failures.
 //
 // LOCKING: The caller must retain and hold the mmap_lock and hold the va_space
@@ -52,9 +53,9 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
 
 // Service access counter notifications on ATS regions in the range (base, base
 // + UVM_VA_BLOCK_SIZE) for individual pages in the range requested by page_mask
-// set in ats_context->accessed_mask. base must be aligned to UVM_VA_BLOCK_SIZE.
-// The caller is responsible for ensuring that the addresses in the
-// accessed_mask is completely covered by the VMA. The caller is also
+// set in ats_context->access_counters.accessed_mask. base must be aligned to
+// UVM_VA_BLOCK_SIZE. The caller is responsible for ensuring that the addresses
+// in the accessed_mask is completely covered by the VMA. The caller is also
 // responsible for handling any errors returned by this function.
 //
 // Returns NV_OK if servicing was successful. Any other error indicates an error

kernel-open/nvidia-uvm/uvm_ats_sva.c

@@ -127,19 +127,23 @@ static NvU32 smmu_vintf_read32(void __iomem *smmu_cmdqv_base, int reg)
 
 // We always use VCMDQ127 for the WAR
 #define VCMDQ 127
-void smmu_vcmdq_write32(void __iomem *smmu_cmdqv_base, int reg, NvU32 val)
+static void smmu_vcmdq_write32(void __iomem *smmu_cmdqv_base, int reg, NvU32 val)
 {
     iowrite32(val, SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
 }
 
-NvU32 smmu_vcmdq_read32(void __iomem *smmu_cmdqv_base, int reg)
+static NvU32 smmu_vcmdq_read32(void __iomem *smmu_cmdqv_base, int reg)
 {
     return ioread32(SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
 }
 
 static void smmu_vcmdq_write64(void __iomem *smmu_cmdqv_base, int reg, NvU64 val)
 {
+#if NV_IS_EXPORT_SYMBOL_PRESENT___iowrite64_lo_hi
+    __iowrite64_lo_hi(val, SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
+#else
     iowrite64(val, SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
+#endif
 }
 
 // Fix for Bug 4130089: [GH180][r535] WAR for kernel not issuing SMMU
@@ -304,12 +308,13 @@ void uvm_ats_smmu_invalidate_tlbs(uvm_gpu_va_space_t *gpu_va_space, NvU64 addr,
 
 NV_STATUS uvm_ats_sva_add_gpu(uvm_parent_gpu_t *parent_gpu)
 {
+#if NV_IS_EXPORT_SYMBOL_GPL_iommu_dev_enable_feature
     int ret;
 
     ret = iommu_dev_enable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA);
     if (ret)
         return errno_to_nv_status(ret);
-
+#endif
     if (UVM_ATS_SMMU_WAR_REQUIRED())
         return uvm_ats_smmu_war_init(parent_gpu);
     else
@@ -321,7 +326,9 @@ void uvm_ats_sva_remove_gpu(uvm_parent_gpu_t *parent_gpu)
     if (UVM_ATS_SMMU_WAR_REQUIRED())
         uvm_ats_smmu_war_deinit(parent_gpu);
 
+#if NV_IS_EXPORT_SYMBOL_GPL_iommu_dev_disable_feature
     iommu_dev_disable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA);
+#endif
 }
 
 NV_STATUS uvm_ats_sva_bind_gpu(uvm_gpu_va_space_t *gpu_va_space)

kernel-open/nvidia-uvm/uvm_blackwell.c

@@ -0,0 +1,105 @@
+/*******************************************************************************
+    Copyright (c) 2022-2024 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+#include "uvm_global.h"
+#include "uvm_hal.h"
+#include "uvm_gpu.h"
+#include "uvm_mem.h"
+#include "uvm_blackwell_fault_buffer.h"
+
+void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
+{
+    parent_gpu->tlb_batch.va_invalidate_supported = true;
+
+    parent_gpu->tlb_batch.va_range_invalidate_supported = true;
+
+    // TODO: Bug 1767241: Run benchmarks to figure out a good number
+    parent_gpu->tlb_batch.max_ranges = 8;
+
+    parent_gpu->utlb_per_gpc_count = uvm_blackwell_get_utlbs_per_gpc(parent_gpu);
+
+    parent_gpu->fault_buffer.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount * parent_gpu->utlb_per_gpc_count;
+    {
+        uvm_fault_buffer_entry_t *dummy;
+        UVM_ASSERT(parent_gpu->fault_buffer.replayable.utlb_count <= (1 << (sizeof(dummy->fault_source.utlb_id) * 8)));
+    }
+
+    // A single top level PDE on Blackwell covers 64 PB and that's the minimum
+    // size that can be used.
+    parent_gpu->rm_va_base = 0;
+    parent_gpu->rm_va_size = 64 * UVM_SIZE_1PB;
+
+    parent_gpu->peer_va_base = parent_gpu->rm_va_base + parent_gpu->rm_va_size;
+    parent_gpu->peer_va_size = NV_MAX_DEVICES * UVM_PEER_IDENTITY_VA_SIZE;
+
+    parent_gpu->uvm_mem_va_base = parent_gpu->rm_va_size + 384 * UVM_SIZE_1TB;
+    parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
+
+    // See uvm_mmu.h for mapping placement
+    parent_gpu->flat_vidmem_va_base = (64 * UVM_SIZE_1PB) + (32 * UVM_SIZE_1TB);
+
+    parent_gpu->ce_phys_vidmem_write_supported = true;
+
+    parent_gpu->peer_copy_mode = g_uvm_global.peer_copy_mode;
+
+    // All GR context buffers may be mapped to 57b wide VAs. All "compute" units
+    // accessing GR context buffers support the 57-bit VA range.
+    parent_gpu->max_channel_va = 1ull << 57;
+
+    parent_gpu->max_host_va = 1ull << 57;
+
+    // Blackwell can map sysmem with any page size
+    parent_gpu->can_map_sysmem_with_large_pages = true;
+
+    // Prefetch instructions will generate faults
+    parent_gpu->prefetch_fault_supported = true;
+
+    // Blackwell can place GPFIFO in vidmem
+    parent_gpu->gpfifo_in_vidmem_supported = true;
+
+    parent_gpu->replayable_faults_supported = true;
+
+    parent_gpu->non_replayable_faults_supported = true;
+
+    parent_gpu->access_counters_supported = true;
+
+    parent_gpu->fault_cancel_va_supported = true;
+
+    parent_gpu->scoped_atomics_supported = true;
+
+    parent_gpu->has_clear_faulted_channel_sw_method = true;
+
+    parent_gpu->has_clear_faulted_channel_method = false;
+
+    parent_gpu->smc.supported = true;
+
+    parent_gpu->sparse_mappings_supported = true;
+
+    parent_gpu->map_remap_larger_page_promotion = false;
+
+    parent_gpu->plc_supported = true;
+
+    parent_gpu->no_ats_range_required = true;
+
+    parent_gpu->conf_computing.per_channel_key_rotation = true;
+}

kernel-open/nvidia-uvm/uvm_blackwell_fault_buffer.c

@@ -0,0 +1,122 @@
+/*******************************************************************************
+    Copyright (c) 2023-2024 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+#include "uvm_linux.h"
+#include "uvm_global.h"
+#include "uvm_gpu.h"
+#include "uvm_hal.h"
+#include "uvm_hal_types.h"
+#include "hwref/blackwell/gb100/dev_fault.h"
+#include "clc369.h"
+
+// NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE fault type is deprecated on
+// Blackwell.
+uvm_fault_type_t uvm_hal_blackwell_fault_buffer_get_fault_type(const NvU32 *fault_entry)
+{
+    NvU32 hw_fault_type_value = READ_HWVALUE_MW(fault_entry, C369, BUF_ENTRY, FAULT_TYPE);
+
+    switch (hw_fault_type_value) {
+        case NV_PFAULT_FAULT_TYPE_PDE:
+            return UVM_FAULT_TYPE_INVALID_PDE;
+        case NV_PFAULT_FAULT_TYPE_PTE:
+            return UVM_FAULT_TYPE_INVALID_PTE;
+        case NV_PFAULT_FAULT_TYPE_RO_VIOLATION:
+            return UVM_FAULT_TYPE_WRITE;
+        case NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION:
+            return UVM_FAULT_TYPE_ATOMIC;
+        case NV_PFAULT_FAULT_TYPE_WO_VIOLATION:
+            return UVM_FAULT_TYPE_READ;
+
+        case NV_PFAULT_FAULT_TYPE_PDE_SIZE:
+            return UVM_FAULT_TYPE_PDE_SIZE;
+        case NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION:
+            return UVM_FAULT_TYPE_VA_LIMIT_VIOLATION;
+        case NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK:
+            return UVM_FAULT_TYPE_UNBOUND_INST_BLOCK;
+        case NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION:
+            return UVM_FAULT_TYPE_PRIV_VIOLATION;
+        case NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION:
+            return UVM_FAULT_TYPE_PITCH_MASK_VIOLATION;
+        case NV_PFAULT_FAULT_TYPE_WORK_CREATION:
+            return UVM_FAULT_TYPE_WORK_CREATION;
+        case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE:
+            return UVM_FAULT_TYPE_UNSUPPORTED_APERTURE;
+        case NV_PFAULT_FAULT_TYPE_CC_VIOLATION:
+            return UVM_FAULT_TYPE_CC_VIOLATION;
+        case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND:
+            return UVM_FAULT_TYPE_UNSUPPORTED_KIND;
+        case NV_PFAULT_FAULT_TYPE_REGION_VIOLATION:
+            return UVM_FAULT_TYPE_REGION_VIOLATION;
+        case NV_PFAULT_FAULT_TYPE_POISONED:
+            return UVM_FAULT_TYPE_POISONED;
+    }
+
+    UVM_ASSERT_MSG(false, "Invalid fault type value: %d\n", hw_fault_type_value);
+
+    return UVM_FAULT_TYPE_COUNT;
+}
+
+static bool client_id_ce(NvU16 client_id)
+{
+    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE0 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE7)
+        return true;
+
+    switch (client_id) {
+        case NV_PFAULT_CLIENT_HUB_CE0:
+        case NV_PFAULT_CLIENT_HUB_CE1:
+        case NV_PFAULT_CLIENT_HUB_CE2:
+        case NV_PFAULT_CLIENT_HUB_CE3:
+            return true;
+    }
+
+    return false;
+}
+
+uvm_mmu_engine_type_t uvm_hal_blackwell_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
+                                                                         uvm_fault_client_type_t client_type,
+                                                                         NvU16 client_id)
+{
+    // Servicing CE and Host (HUB clients) faults.
+    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB) {
+        if (client_id_ce(client_id)) {
+            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE19);
+
+            return UVM_MMU_ENGINE_TYPE_CE;
+        }
+
+        if (client_id == NV_PFAULT_CLIENT_HUB_HOST ||
+            (client_id >= NV_PFAULT_CLIENT_HUB_ESC0 && client_id <= NV_PFAULT_CLIENT_HUB_ESC11)) {
+            UVM_ASSERT((mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST44) ||
+                       (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS));
+
+            return UVM_MMU_ENGINE_TYPE_HOST;
+        }
+    }
+
+    // We shouldn't be servicing faults from any other engines other than GR.
+    UVM_ASSERT_MSG(client_id <= NV_PFAULT_CLIENT_GPC_ROP_3, "Unexpected client ID: 0x%x\n", client_id);
+    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS, "Unexpected engine ID: 0x%x\n", mmu_engine_id);
+    UVM_ASSERT(client_type == UVM_FAULT_CLIENT_TYPE_GPC);
+
+    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
+}

kernel-open/nvidia-uvm/uvm_blackwell_fault_buffer.h

@@ -0,0 +1,92 @@
+/*******************************************************************************
+    Copyright (c) 2022 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+#ifndef __UVM_HAL_BLACKWELL_FAULT_BUFFER_H__
+#define __UVM_HAL_BLACKWELL_FAULT_BUFFER_H__
+
+#include "nvtypes.h"
+#include "uvm_common.h"
+#include "uvm_gpu.h"
+
+// There are up to 10 TPCs per GPC in Blackwell, and there are 2 LTP uTLBs per
+// TPC. Besides, there is one active RGG uTLB per GPC. Each TPC has a number of
+// clients that can make requests to its uTLBs: 1xTPCCS, 1xPE, 2xT1. Requests
+// from these units are routed as follows to the 2 LTP uTLBs:
+//
+// --------                    ---------
+// | T1_0 | -----------------> | uTLB0 |
+// --------                    ---------
+//
+// --------                    ---------
+// | T1_1 | -----------------> | uTLB1 |
+// --------          --------> ---------
+//                   |             ^
+// -------           |             |
+// | PE  | -----------             |
+// -------                         |
+//                                 |
+// ---------                       |
+// | TPCCS | -----------------------
+// ---------
+//
+//
+// The client ids are local to their GPC and the id mapping is linear across
+// TPCs: TPC_n has TPCCS_n, PE_n, T1_p, and T1_q, where p=2*n and q=p+1.
+//
+// NV_PFAULT_CLIENT_GPC_LTP_UTLB_n and NV_PFAULT_CLIENT_GPC_RGG_UTLB enums can
+// be ignored. These will never be reported in a fault message, and should
+// never be used in an invalidate. Therefore, we define our own values.
+typedef enum {
+    UVM_BLACKWELL_GPC_UTLB_ID_RGG = 0,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP0 = 1,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP1 = 2,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP2 = 3,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP3 = 4,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP4 = 5,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP5 = 6,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP6 = 7,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP7 = 8,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP8 = 9,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP9 = 10,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP10 = 11,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP11 = 12,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP12 = 13,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP13 = 14,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP14 = 15,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP15 = 16,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP16 = 17,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP17 = 18,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP18 = 19,
+    UVM_BLACKWELL_GPC_UTLB_ID_LTP19 = 20,
+
+    UVM_BLACKWELL_GPC_UTLB_COUNT,
+} uvm_blackwell_gpc_utlb_id_t;
+
+static NvU32 uvm_blackwell_get_utlbs_per_gpc(uvm_parent_gpu_t *parent_gpu)
+{
+    NvU32 utlbs = parent_gpu->rm_info.maxTpcPerGpcCount * 2 + 1;
+    UVM_ASSERT(utlbs <= UVM_BLACKWELL_GPC_UTLB_COUNT);
+    return utlbs;
+}
+
+#endif

kernel-open/nvidia-uvm/uvm_blackwell_host.c

@@ -0,0 +1,284 @@
+/*******************************************************************************
+    Copyright (c) 2024 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+#include "uvm_hal.h"
+#include "uvm_push.h"
+#include "uvm_push_macros.h"
+#include "clc96f.h"
+
+// TODO: Bug 3210931: Rename HOST references and files to ESCHED.
+
+void uvm_hal_blackwell_host_tlb_invalidate_all(uvm_push_t *push,
+                                               uvm_gpu_phys_address_t pdb,
+                                               NvU32 depth,
+                                               uvm_membar_t membar)
+{
+    NvU32 aperture_value;
+    NvU32 page_table_level;
+    NvU32 pdb_lo;
+    NvU32 pdb_hi;
+    NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
+
+    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
+
+    if (pdb.aperture == UVM_APERTURE_VID)
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
+    else
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
+
+    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
+    pdb.address >>= 12;
+
+    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+
+    // PDE4 is the highest level on Blackwell, see the comment in
+    // uvm_blackwell_mmu.c for details.
+    UVM_ASSERT_MSG(depth < NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4, "depth %u", depth);
+    page_table_level = NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 - depth;
+
+    if (membar != UVM_MEMBAR_NONE)
+        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
+
+    if (membar == UVM_MEMBAR_SYS)
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
+    NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
+                               HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
+                     MEM_OP_B, 0,
+                     MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
+                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
+                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
+                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
+                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
+                               aperture_value |
+                               ack_value,
+                     MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
+                               HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
+}
+
+void uvm_hal_blackwell_host_tlb_invalidate_va(uvm_push_t *push,
+                                              uvm_gpu_phys_address_t pdb,
+                                              NvU32 depth,
+                                              NvU64 base,
+                                              NvU64 size,
+                                              NvU64 page_size,
+                                              uvm_membar_t membar)
+{
+    NvU32 aperture_value;
+    NvU32 page_table_level;
+    NvU32 pdb_lo;
+    NvU32 pdb_hi;
+    NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
+    NvU32 va_lo;
+    NvU32 va_hi;
+    NvU64 end;
+    NvU64 actual_base;
+    NvU64 actual_size;
+    NvU64 actual_end;
+    NvU32 log2_invalidation_size;
+    uvm_gpu_t *gpu = uvm_push_get_gpu(push);
+
+    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
+    UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
+
+    // The invalidation size must be a power-of-two number of pages containing
+    // the passed interval
+    end = base + size - 1;
+    log2_invalidation_size = __fls((unsigned long)(end ^ base)) + 1;
+
+    if (log2_invalidation_size == 64) {
+        // Invalidate everything
+        gpu->parent->host_hal->tlb_invalidate_all(push, pdb, depth, membar);
+        return;
+    }
+
+    // The hardware aligns the target address down to the invalidation size.
+    actual_size = 1ULL << log2_invalidation_size;
+    actual_base = UVM_ALIGN_DOWN(base, actual_size);
+    actual_end = actual_base + actual_size - 1;
+    UVM_ASSERT(actual_end >= end);
+
+    // The invalidation size field expects log2(invalidation size in 4K), not
+    // log2(invalidation size in bytes)
+    log2_invalidation_size -= 12;
+
+    // Address to invalidate, as a multiple of 4K.
+    base >>= 12;
+    va_lo = base & HWMASK(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
+    va_hi = base >> HWSIZE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
+
+    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
+
+    if (pdb.aperture == UVM_APERTURE_VID)
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
+    else
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
+
+    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
+    pdb.address >>= 12;
+
+    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+
+    // PDE4 is the highest level on Blackwell, see the comment in
+    // uvm_blackwell_mmu.c for details.
+    UVM_ASSERT_MSG(depth < NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4, "depth %u", depth);
+    page_table_level = NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 - depth;
+
+    if (membar != UVM_MEMBAR_NONE)
+        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
+
+    if (membar == UVM_MEMBAR_SYS)
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
+    NV_PUSH_4U(C96F, MEM_OP_A, HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_INVALIDATION_SIZE, log2_invalidation_size) |
+                               sysmembar_value |
+                               HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
+                               HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
+                     MEM_OP_B, HWVALUE(C96F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
+                     MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
+                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
+                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
+                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
+                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
+                               aperture_value |
+                               ack_value,
+                     MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
+                               HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
+}
+
+void uvm_hal_blackwell_host_tlb_invalidate_test(uvm_push_t *push,
+                                                uvm_gpu_phys_address_t pdb,
+                                                UVM_TEST_INVALIDATE_TLB_PARAMS *params)
+{
+    NvU32 ack_value = 0;
+    NvU32 sysmembar_value = 0;
+    NvU32 invalidate_gpc_value = 0;
+    NvU32 aperture_value = 0;
+    NvU32 pdb_lo = 0;
+    NvU32 pdb_hi = 0;
+    NvU32 page_table_level = 0;
+
+    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
+    if (pdb.aperture == UVM_APERTURE_VID)
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
+    else
+        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
+
+    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
+    pdb.address >>= 12;
+
+    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
+
+    if (params->page_table_level != UvmInvalidatePageTableLevelAll) {
+        // PDE4 is the highest level on Blackwell, see the comment in
+        // uvm_blackwell_mmu.c for details.
+        page_table_level = min((NvU32)UvmInvalidatePageTableLevelPde4, params->page_table_level) - 1;
+    }
+
+    if (params->membar != UvmInvalidateTlbMemBarNone)
+        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
+
+    if (params->membar == UvmInvalidateTlbMemBarSys)
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
+    else
+        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
+
+    if (params->disable_gpc_invalidate)
+        invalidate_gpc_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
+    else
+        invalidate_gpc_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE);
+
+    if (params->target_va_mode == UvmTargetVaModeTargeted) {
+        NvU64 va = params->va >> 12;
+
+        NvU32 va_lo = va & HWMASK(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
+        NvU32 va_hi = va >> HWSIZE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
+
+        NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
+                                   HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
+                                   HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
+                         MEM_OP_B, HWVALUE(C96F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
+                         MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
+                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
+                                   HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
+                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
+                                   invalidate_gpc_value |
+                                   aperture_value |
+                                   ack_value,
+                         MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
+                                   HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
+    }
+    else {
+        NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
+                                   HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
+                         MEM_OP_B, 0,
+                         MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
+                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
+                                   HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
+                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
+                                   invalidate_gpc_value |
+                                   aperture_value |
+                                   ack_value,
+                         MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
+                                   HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
+    }
+}
+
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb100(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries)
+{
+    if (parent_gpu->rm_info.accessCntrBufferCount > 1) {
+        NvU32 i;
+
+        for (i = 0; i < num_entries; i++) {
+            const uvm_access_counter_buffer_entry_t *entry = buffer_entries[i];
+
+            // The LSb identifies the die ID.
+            if ((entry->tag & 0x1) == 1)
+                return UVM_ACCESS_COUNTER_CLEAR_OP_ALL;
+        }
+    }
+
+    return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
+}
+
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb20x(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries)
+{
+    return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
+}

kernel-open/nvidia-uvm/uvm_blackwell_mmu.c

@@ -0,0 +1,164 @@
+/*******************************************************************************
+    Copyright (c) 2022-2024 NVIDIA Corporation
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to
+    deal in the Software without restriction, including without limitation the
+    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be
+        included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+
+*******************************************************************************/
+
+// On Blackwell, the UVM page tree 'depth' maps to hardware as follows:
+//
+// UVM depth   HW level                            VA bits
+// 0           PDE4                                56:56
+// 1           PDE3                                55:47
+// 2           PDE2 (or 256G PTE)                  46:38
+// 3           PDE1 (or 512M PTE)                  37:29
+// 4           PDE0 (dual 64K/4K PDE, or 2M PTE)   28:21
+// 5           PTE_64K / PTE_4K                    20:16 / 20:12
+
+#include "uvm_types.h"
+#include "uvm_global.h"
+#include "uvm_hal.h"
+#include "uvm_hal_types.h"
+#include "uvm_blackwell_fault_buffer.h"
+#include "hwref/blackwell/gb100/dev_fault.h"
+
+static uvm_mmu_mode_hal_t blackwell_mmu_mode_hal;
+
+static NvU32 page_table_depth_blackwell(NvU64 page_size)
+{
+    switch (page_size) {
+        case UVM_PAGE_SIZE_2M:
+            return 4;
+        case UVM_PAGE_SIZE_512M:
+            return 3;
+        case UVM_PAGE_SIZE_256G:
+            return 2;
+        default:
+            return 5;
+    }
+}
+
+static NvU64 page_sizes_blackwell(void)
+{
+    return UVM_PAGE_SIZE_256G | UVM_PAGE_SIZE_512M | UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
+}
+
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size)
+{
+    static bool initialized = false;
+
+    UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
+
+    // TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
+    // 128K big page size for Pascal+ GPUs
+    if (big_page_size == UVM_PAGE_SIZE_128K)
+        return NULL;
+
+    if (!initialized) {
+        uvm_mmu_mode_hal_t *hopper_mmu_mode_hal = uvm_hal_mmu_mode_hopper(big_page_size);
+        UVM_ASSERT(hopper_mmu_mode_hal);
+
+        // The assumption made is that arch_hal->mmu_mode_hal() will be called
+        // under the global lock the first time, so check it here.
+        uvm_assert_mutex_locked(&g_uvm_global.global_lock);
+
+        blackwell_mmu_mode_hal = *hopper_mmu_mode_hal;
+        blackwell_mmu_mode_hal.page_table_depth = page_table_depth_blackwell;
+        blackwell_mmu_mode_hal.page_sizes = page_sizes_blackwell;
+
+        initialized = true;
+    }
+
+    return &blackwell_mmu_mode_hal;
+}
+
+NvU16 uvm_hal_blackwell_mmu_client_id_to_utlb_id(NvU16 client_id)
+{
+    switch (client_id) {
+        case NV_PFAULT_CLIENT_GPC_RAST:
+        case NV_PFAULT_CLIENT_GPC_GCC:
+        case NV_PFAULT_CLIENT_GPC_GPCCS:
+            return UVM_BLACKWELL_GPC_UTLB_ID_RGG;
+        case NV_PFAULT_CLIENT_GPC_T1_0:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP0;
+        case NV_PFAULT_CLIENT_GPC_T1_1:
+        case NV_PFAULT_CLIENT_GPC_PE_0:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_0:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP1;
+        case NV_PFAULT_CLIENT_GPC_T1_2:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP2;
+        case NV_PFAULT_CLIENT_GPC_T1_3:
+        case NV_PFAULT_CLIENT_GPC_PE_1:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_1:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP3;
+        case NV_PFAULT_CLIENT_GPC_T1_4:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP4;
+        case NV_PFAULT_CLIENT_GPC_T1_5:
+        case NV_PFAULT_CLIENT_GPC_PE_2:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_2:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP5;
+        case NV_PFAULT_CLIENT_GPC_T1_6:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP6;
+        case NV_PFAULT_CLIENT_GPC_T1_7:
+        case NV_PFAULT_CLIENT_GPC_PE_3:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_3:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP7;
+        case NV_PFAULT_CLIENT_GPC_T1_8:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP8;
+        case NV_PFAULT_CLIENT_GPC_T1_9:
+        case NV_PFAULT_CLIENT_GPC_PE_4:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_4:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP9;
+        case NV_PFAULT_CLIENT_GPC_T1_10:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP10;
+        case NV_PFAULT_CLIENT_GPC_T1_11:
+        case NV_PFAULT_CLIENT_GPC_PE_5:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_5:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP11;
+        case NV_PFAULT_CLIENT_GPC_T1_12:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP12;
+        case NV_PFAULT_CLIENT_GPC_T1_13:
+        case NV_PFAULT_CLIENT_GPC_PE_6:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_6:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP13;
+        case NV_PFAULT_CLIENT_GPC_T1_14:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP14;
+        case NV_PFAULT_CLIENT_GPC_T1_15:
+        case NV_PFAULT_CLIENT_GPC_PE_7:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_7:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP15;
+        case NV_PFAULT_CLIENT_GPC_T1_16:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP16;
+        case NV_PFAULT_CLIENT_GPC_T1_17:
+        case NV_PFAULT_CLIENT_GPC_PE_8:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_8:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP17;
+        case NV_PFAULT_CLIENT_GPC_T1_18:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP18;
+        case NV_PFAULT_CLIENT_GPC_T1_19:
+        case NV_PFAULT_CLIENT_GPC_PE_9:
+        case NV_PFAULT_CLIENT_GPC_TPCCS_9:
+            return UVM_BLACKWELL_GPC_UTLB_ID_LTP19;
+
+        default:
+            UVM_ASSERT_MSG(false, "Invalid client value: 0x%x\n", client_id);
+    }
+
+    return 0;
+}

kernel-open/nvidia-uvm/uvm_channel.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2023 NVIDIA Corporation
+    Copyright (c) 2015-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -38,6 +38,7 @@
 #include "clb06f.h"
 #include "uvm_conf_computing.h"
 
+
 // WLC push is decrypted by SEC2 or CE (in WLC schedule).
 // In sysmem it's followed by auth tag.
 #define WLC_PUSHBUFFER_ALIGNMENT max3(UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT, \
@@ -97,6 +98,37 @@ typedef enum
     UVM_CHANNEL_UPDATE_MODE_FORCE_ALL
 } uvm_channel_update_mode_t;
 
+typedef enum
+{
+    // Reserve an entry that is expected to use p2p operations.
+    UVM_CHANNEL_RESERVE_WITH_P2P,
+
+    // Reserve an entry that is not expected to use p2p operations.
+    UVM_CHANNEL_RESERVE_NO_P2P,
+} uvm_channel_reserve_type_t;
+
+bool uvm_channel_pool_is_p2p(uvm_channel_pool_t *pool)
+{
+    uvm_channel_manager_t *manager = pool->manager;
+    uvm_gpu_t *gpu = manager->gpu;
+    uvm_gpu_id_t id;
+
+    if (manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_GPU] == pool)
+        return true;
+
+    uvm_spin_lock(&gpu->peer_info.peer_gpu_lock);
+    for_each_gpu_id_in_mask(id, &gpu->peer_info.peer_gpu_mask) {
+        if (manager->pool_to_use.gpu_to_gpu[uvm_id_gpu_index(id)] == pool) {
+            uvm_spin_unlock(&gpu->peer_info.peer_gpu_lock);
+            return true;
+        }
+    }
+
+    uvm_spin_unlock(&gpu->peer_info.peer_gpu_lock);
+
+    return false;
+}
+
 bool uvm_channel_pool_uses_mutex(uvm_channel_pool_t *pool)
 {
     // Work submission to proxy channels in SR-IOV heavy entails calling RM API
@@ -119,10 +151,12 @@ bool uvm_channel_pool_uses_mutex(uvm_channel_pool_t *pool)
 
 static void channel_pool_lock_init(uvm_channel_pool_t *pool)
 {
-    uvm_lock_order_t order = UVM_LOCK_ORDER_CHANNEL;
+    uvm_lock_order_t order;
 
     if (g_uvm_global.conf_computing_enabled && uvm_channel_pool_is_wlc(pool))
         order = UVM_LOCK_ORDER_WLC_CHANNEL;
+    else
+        order = UVM_LOCK_ORDER_CHANNEL;
 
     if (uvm_channel_pool_uses_mutex(pool))
         uvm_mutex_init(&pool->mutex, order);
@@ -274,7 +308,9 @@ NvU32 uvm_channel_get_available_gpfifo_entries(uvm_channel_t *channel)
     return available;
 }
 
-static bool try_claim_channel_locked(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
+static bool try_claim_channel_locked(uvm_channel_t *channel,
+                                     NvU32 num_gpfifo_entries,
+                                     uvm_channel_reserve_type_t reserve_type)
 {
     bool claimed = false;
 
@@ -283,6 +319,9 @@ static bool try_claim_channel_locked(uvm_channel_t *channel, NvU32 num_gpfifo_en
 
     uvm_channel_pool_assert_locked(channel->pool);
 
+    if (reserve_type == UVM_CHANNEL_RESERVE_WITH_P2P && channel->suspended_p2p)
+        return false;
+
     if (channel_get_available_gpfifo_entries(channel) >= num_gpfifo_entries) {
         channel->current_gpfifo_count += num_gpfifo_entries;
         claimed = true;
@@ -291,12 +330,14 @@ static bool try_claim_channel_locked(uvm_channel_t *channel, NvU32 num_gpfifo_en
     return claimed;
 }
 
-static bool try_claim_channel(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
+static bool try_claim_channel(uvm_channel_t *channel,
+                              NvU32 num_gpfifo_entries,
+                              uvm_channel_reserve_type_t reserve_type)
 {
     bool claimed;
 
     channel_pool_lock(channel->pool);
-    claimed = try_claim_channel_locked(channel, num_gpfifo_entries);
+    claimed = try_claim_channel_locked(channel, num_gpfifo_entries, reserve_type);
     channel_pool_unlock(channel->pool);
 
     return claimed;
@@ -349,7 +390,8 @@ static bool test_claim_and_lock_channel(uvm_channel_t *channel, NvU32 num_gpfifo
     if (uvm_channel_is_locked_for_push(channel))
         return false;
 
-    if (try_claim_channel_locked(channel, num_gpfifo_entries)) {
+    // Confidential compute is not using p2p ops, reserve without p2p
+    if (try_claim_channel_locked(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P)) {
         lock_channel_for_push(channel);
         return true;
     }
@@ -490,7 +532,9 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, uvm_
 
     for_each_clear_bit(index, pool->conf_computing.push_locks, pool->num_channels) {
         channel = &pool->channels[index];
-        if (try_claim_channel_locked(channel, 1)) {
+
+        // Confidential compute is not using p2p ops, reserve without p2p
+        if (try_claim_channel_locked(channel, 1, UVM_CHANNEL_RESERVE_NO_P2P)) {
             lock_channel_for_push(channel);
             goto done;
         }
@@ -529,7 +573,9 @@ done:
 }
 
 // Reserve a channel in the specified pool
-static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, uvm_channel_t **channel_out)
+static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool,
+                                         uvm_channel_reserve_type_t reserve_type,
+                                         uvm_channel_t **channel_out)
 {
     uvm_channel_t *channel;
     uvm_spin_loop_t spin;
@@ -541,7 +587,7 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, uvm_channel_t
 
     uvm_for_each_channel_in_pool(channel, pool) {
         // TODO: Bug 1764953: Prefer idle/less busy channels
-        if (try_claim_channel(channel, 1)) {
+        if (try_claim_channel(channel, 1, reserve_type)) {
             *channel_out = channel;
             return NV_OK;
         }
@@ -554,7 +600,7 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, uvm_channel_t
 
             uvm_channel_update_progress(channel);
 
-            if (try_claim_channel(channel, 1)) {
+            if (try_claim_channel(channel, 1, reserve_type)) {
                 *channel_out = channel;
 
                 return NV_OK;
@@ -564,6 +610,9 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, uvm_channel_t
             if (status != NV_OK)
                 return status;
 
+            if (reserve_type == UVM_CHANNEL_RESERVE_WITH_P2P && channel->suspended_p2p)
+                return NV_ERR_BUSY_RETRY;
+
             UVM_SPIN_LOOP(&spin);
         }
     }
@@ -575,12 +624,18 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool, uvm_channel_t
 
 NV_STATUS uvm_channel_reserve_type(uvm_channel_manager_t *manager, uvm_channel_type_t type, uvm_channel_t **channel_out)
 {
+    uvm_channel_reserve_type_t reserve_type;
     uvm_channel_pool_t *pool = manager->pool_to_use.default_for_type[type];
 
     UVM_ASSERT(pool != NULL);
     UVM_ASSERT(type < UVM_CHANNEL_TYPE_COUNT);
 
-    return channel_reserve_in_pool(pool, channel_out);
+    if (type == UVM_CHANNEL_TYPE_GPU_TO_GPU)
+        reserve_type = UVM_CHANNEL_RESERVE_WITH_P2P;
+    else
+        reserve_type = UVM_CHANNEL_RESERVE_NO_P2P;
+
+    return channel_reserve_in_pool(pool, reserve_type, channel_out);
 }
 
 NV_STATUS uvm_channel_reserve_gpu_to_gpu(uvm_channel_manager_t *manager,
@@ -596,7 +651,7 @@ NV_STATUS uvm_channel_reserve_gpu_to_gpu(uvm_channel_manager_t *manager,
 
     UVM_ASSERT(pool->pool_type == UVM_CHANNEL_POOL_TYPE_CE);
 
-    return channel_reserve_in_pool(pool, channel_out);
+    return channel_reserve_in_pool(pool, UVM_CHANNEL_RESERVE_WITH_P2P, channel_out);
 }
 
 NV_STATUS uvm_channel_manager_wait(uvm_channel_manager_t *manager)
@@ -693,6 +748,40 @@ static void push_reserve_csl_sign_buf(uvm_push_t *push)
     UVM_ASSERT((buf - UVM_METHOD_SIZE / sizeof(*buf)) == push->begin);
 }
 
+// Reserve space for a single authentication tag and return addresses to unprotected sysmem
+static void *push_reserve_auth_tag(uvm_push_t *push, uvm_gpu_address_t *gpu_address_out)
+{
+    void *cpu_address;
+    uvm_gpu_address_t gpu_address;
+
+    UVM_ASSERT(push->channel);
+
+    cpu_address = uvm_push_get_single_inline_buffer(push,
+                                                    UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
+                                                    UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
+                                                    &gpu_address);
+
+    // SEC2 channel uses unprotected sysmem for pushes
+    if (uvm_channel_is_sec2(push->channel)) {
+        *gpu_address_out = gpu_address;
+        return cpu_address;
+    }
+    else {
+        uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(push->channel);
+        NvU64 unprotected_gpu_va_for_push = uvm_pushbuffer_get_unprotected_gpu_va_for_push(pushbuffer, push);
+        char *unprotected_cpu_va_for_push = uvm_pushbuffer_get_unprotected_cpu_va_for_push(pushbuffer, push);
+        const size_t offset = (char*)cpu_address - (char*)push->begin;
+
+        UVM_ASSERT(offset == gpu_address.address - uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push));
+        UVM_ASSERT(!uvm_channel_is_wlc(push->channel));
+        UVM_ASSERT(!uvm_channel_is_lcic(push->channel));
+        UVM_ASSERT(uvm_channel_is_ce(push->channel));
+
+        *gpu_address_out = uvm_gpu_address_virtual_unprotected(unprotected_gpu_va_for_push + offset);
+        return unprotected_cpu_va_for_push + offset;
+    }
+}
+
 static uvm_channel_pool_t *get_paired_pool(uvm_channel_pool_t *pool)
 {
     uvm_channel_type_t paired_channel_type;
@@ -798,44 +887,6 @@ char* uvm_channel_get_static_pb_unprotected_sysmem_cpu(uvm_channel_t *channel)
     return get_channel_unprotected_sysmem_cpu(channel) + WLC_SYSMEM_PUSHBUFFER_OFFSET;
 }
 
-char *uvm_channel_get_push_crypto_bundle_auth_tags_cpu_va(uvm_channel_t *channel, unsigned tag_index)
-{
-    char *pool_sysmem_base;
-    unsigned index;
-
-    UVM_ASSERT(channel);
-    UVM_ASSERT(!uvm_channel_is_wlc(channel));
-    UVM_ASSERT(!uvm_channel_is_lcic(channel));
-    UVM_ASSERT(uvm_channel_is_ce(channel));
-    UVM_ASSERT(channel->num_gpfifo_entries == channel_pool_num_gpfifo_entries(channel->pool));
-    UVM_ASSERT(tag_index < channel->num_gpfifo_entries);
-
-    index = uvm_channel_index_in_pool(channel) * channel->num_gpfifo_entries + tag_index;
-    pool_sysmem_base = uvm_rm_mem_get_cpu_va(channel->pool->conf_computing.pool_sysmem);
-
-    return pool_sysmem_base + index * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
-}
-
-static NvU64 get_push_crypto_bundle_auth_tags_gpu_va(uvm_channel_t *channel, unsigned tag_index)
-{
-    unsigned index;
-    NvU64 pool_sysmem_base;
-
-    UVM_ASSERT(channel);
-    UVM_ASSERT(!uvm_channel_is_wlc(channel));
-    UVM_ASSERT(!uvm_channel_is_lcic(channel));
-    UVM_ASSERT(uvm_channel_is_ce(channel));
-    UVM_ASSERT(channel->num_gpfifo_entries == channel_pool_num_gpfifo_entries(channel->pool));
-    UVM_ASSERT(tag_index < channel->num_gpfifo_entries);
-
-    index = uvm_channel_index_in_pool(channel) * channel->num_gpfifo_entries + tag_index;
-    pool_sysmem_base = uvm_rm_mem_get_gpu_uvm_va(channel->pool->conf_computing.pool_sysmem,
-                                                 uvm_channel_get_gpu(channel));
-
-
-    return pool_sysmem_base + index * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
-}
-
 static NV_STATUS channel_rotate_and_reserve_launch_channel(uvm_channel_t *channel, uvm_channel_t **launch_channel)
 {
     uvm_channel_manager_t *manager = channel->pool->manager;
@@ -1195,10 +1246,7 @@ static void update_gpput_via_sec2(uvm_push_t *sec2_push, uvm_channel_t *channel,
                                                       UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT,
                                                       UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT,
                                                       &gpput_enc_gpu);
-    gpput_auth_tag_cpu = uvm_push_get_single_inline_buffer(sec2_push,
-                                                           UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                           UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                           &gpput_auth_tag_gpu);
+    gpput_auth_tag_cpu = push_reserve_auth_tag(sec2_push, &gpput_auth_tag_gpu);
 
     // Update GPPUT. The update needs 4B write to specific offset,
     // however we can only do 16B aligned decrypt writes.
@@ -1249,10 +1297,7 @@ static void set_gpfifo_via_sec2(uvm_push_t *sec2_push, uvm_channel_t *channel, N
                                                        sizeof(gpfifo_scratchpad),
                                                        UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT,
                                                        &gpfifo_enc_gpu);
-    gpfifo_auth_tag_cpu = uvm_push_get_single_inline_buffer(sec2_push,
-                                                            UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                            UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                            &gpfifo_auth_tag_gpu);
+    gpfifo_auth_tag_cpu = push_reserve_auth_tag(sec2_push, &gpfifo_auth_tag_gpu);
 
     if (IS_ALIGNED(gpfifo_gpu, UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT)) {
         gpfifo_scratchpad[0] = value;
@@ -1339,10 +1384,7 @@ static NV_STATUS internal_channel_submit_work_indirect_sec2(uvm_push_t *push, Nv
 
 
     // Move over the pushbuffer data
-    push_auth_tag_cpu = uvm_push_get_single_inline_buffer(&indirect_push,
-                                                          UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                          UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                          &push_auth_tag_gpu);
+    push_auth_tag_cpu = push_reserve_auth_tag(&indirect_push, &push_auth_tag_gpu);
 
     uvm_conf_computing_cpu_encrypt(indirect_push.channel,
                                    push_enc_cpu,
@@ -1393,7 +1435,7 @@ static void encrypt_push(uvm_push_t *push)
 {
     NvU64 push_protected_gpu_va;
     NvU64 push_unprotected_gpu_va;
-    NvU64 auth_tag_gpu_va;
+    uvm_gpu_address_t auth_tag_gpu_va;
     uvm_channel_t *channel = push->channel;
     uvm_push_crypto_bundle_t *crypto_bundle;
     uvm_gpu_t *gpu = uvm_push_get_gpu(push);
@@ -1418,9 +1460,14 @@ static void encrypt_push(uvm_push_t *push)
     UVM_ASSERT(channel->conf_computing.push_crypto_bundles != NULL);
 
     crypto_bundle = channel->conf_computing.push_crypto_bundles + push->push_info_index;
-    auth_tag_gpu_va = get_push_crypto_bundle_auth_tags_gpu_va(channel, push->push_info_index);
 
+    // Auth tag is reserved after 'push_size' was queried above so it won't be
+    // overwritten during the encryption below. It will be overwritten by the
+    // launch encryption though. This is OK as it doesn't store any useful
+    // value at launch time.
+    crypto_bundle->auth_tag = push_reserve_auth_tag(push, &auth_tag_gpu_va);
     crypto_bundle->push_size = push_size;
+
     push_protected_gpu_va = uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
     push_unprotected_gpu_va = uvm_pushbuffer_get_unprotected_gpu_va_for_push(pushbuffer, push);
 
@@ -1431,7 +1478,7 @@ static void encrypt_push(uvm_push_t *push)
                                  uvm_gpu_address_virtual_unprotected(push_unprotected_gpu_va),
                                  uvm_gpu_address_virtual(push_protected_gpu_va),
                                  push_size,
-                                 uvm_gpu_address_virtual_unprotected(auth_tag_gpu_va));
+                                 auth_tag_gpu_va);
 }
 
 void uvm_channel_end_push(uvm_push_t *push)
@@ -1449,7 +1496,6 @@ void uvm_channel_end_push(uvm_push_t *push)
     bool needs_sec2_work_submit = false;
 
     channel_pool_lock(channel->pool);
-
     encrypt_push(push);
 
     new_tracking_value = ++channel->tracking_sem.queued_value;
@@ -1531,6 +1577,7 @@ void uvm_channel_end_push(uvm_push_t *push)
     // push must be updated before that. Notably uvm_pushbuffer_end_push() has
     // to be called first.
     unlock_channel_for_push(channel);
+
     channel_pool_unlock(channel->pool);
 
     // This memory barrier is borrowed from CUDA, as it supposedly fixes perf
@@ -1813,13 +1860,14 @@ NV_STATUS uvm_channel_reserve(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
     if (g_uvm_global.conf_computing_enabled)
         return channel_reserve_and_lock(channel, num_gpfifo_entries);
 
-    if (try_claim_channel(channel, num_gpfifo_entries))
+    // Direct channel reservations don't use p2p
+    if (try_claim_channel(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P))
         return NV_OK;
 
     uvm_channel_update_progress(channel);
 
     uvm_spin_loop_init(&spin);
-    while (!try_claim_channel(channel, num_gpfifo_entries) && status == NV_OK) {
+    while (!try_claim_channel(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P) && status == NV_OK) {
         UVM_SPIN_LOOP(&spin);
         status = uvm_channel_check_errors(channel);
         uvm_channel_update_progress(channel);
@@ -1833,9 +1881,11 @@ void uvm_channel_release(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
     channel_pool_lock(channel->pool);
 
     UVM_ASSERT(uvm_channel_is_locked_for_push(channel));
+
     unlock_channel_for_push(channel);
 
     UVM_ASSERT(channel->current_gpfifo_count >= num_gpfifo_entries);
+
     channel->current_gpfifo_count -= num_gpfifo_entries;
     channel_pool_unlock(channel->pool);
 }
@@ -1860,6 +1910,140 @@ static uvm_gpfifo_entry_t *uvm_channel_get_first_pending_entry(uvm_channel_t *ch
     return entry;
 }
 
+static NV_STATUS channel_suspend_p2p(uvm_channel_t *channel)
+{
+    NV_STATUS status = NV_OK;
+
+    UVM_ASSERT(channel);
+    UVM_ASSERT(!channel->suspended_p2p);
+
+    // Reserve all entries to block traffic.
+    // Each channel needs 1 entry as sentinel.
+    status = uvm_channel_reserve(channel, channel->num_gpfifo_entries - 1);
+
+    // Prevent p2p traffic from reserving entries
+    if (status == NV_OK)
+        channel->suspended_p2p = true;
+
+    // Release the entries reserved above to allow non-p2p traffic
+    uvm_channel_release(channel, channel->num_gpfifo_entries - 1);
+
+    return status;
+}
+
+static void channel_resume_p2p(uvm_channel_t *channel)
+{
+    UVM_ASSERT(channel);
+    UVM_ASSERT(channel->suspended_p2p);
+
+    channel->suspended_p2p = false;
+}
+
+static NV_STATUS channel_pool_suspend_p2p(uvm_channel_pool_t *pool)
+{
+    NV_STATUS status = NV_OK;
+    NvU32 i;
+
+    UVM_ASSERT(pool);
+    UVM_ASSERT(!uvm_channel_pool_is_wlc(pool));
+    UVM_ASSERT(!uvm_channel_pool_is_lcic(pool));
+
+    for (i = 0; i < pool->num_channels; ++i) {
+        status = channel_suspend_p2p(pool->channels + i);
+        if (status != NV_OK)
+            break;
+    }
+
+    // Resume suspended channels in case of error
+    while ((status != NV_OK) && (i-- > 0))
+        channel_resume_p2p(pool->channels + i);
+
+    for (i = 0; i < pool->num_channels; ++i)
+        UVM_ASSERT(pool->channels[i].suspended_p2p == (status == NV_OK));
+
+    return status;
+}
+
+static void channel_pool_resume_p2p(uvm_channel_pool_t *pool)
+{
+    NvU32 i;
+
+    UVM_ASSERT(pool);
+    UVM_ASSERT(!uvm_channel_pool_is_wlc(pool));
+    UVM_ASSERT(!uvm_channel_pool_is_lcic(pool));
+
+    for (i = 0; i < pool->num_channels; ++i)
+        channel_resume_p2p(pool->channels + i);
+}
+
+NV_STATUS uvm_channel_manager_suspend_p2p(uvm_channel_manager_t *channel_manager)
+{
+    uvm_channel_pool_t *pool;
+    NV_STATUS status = NV_OK;
+    uvm_gpu_t *gpu = channel_manager->gpu;
+    uvm_gpu_id_t gpu_id;
+    DECLARE_BITMAP(suspended_pools, UVM_COPY_ENGINE_COUNT_MAX);
+
+    // Pools can be assigned to multiple 'pool_to_use' locations
+    // Use bitmap to track which were suspended.
+    bitmap_zero(suspended_pools, channel_manager->num_channel_pools);
+
+    uvm_assert_mutex_locked(&g_uvm_global.global_lock);
+
+    for_each_gpu_id_in_mask(gpu_id, &gpu->peer_info.peer_gpu_mask) {
+        pool = channel_manager->pool_to_use.gpu_to_gpu[uvm_id_gpu_index(gpu_id)];
+        if (pool && !test_bit(uvm_channel_pool_index_in_channel_manager(pool), suspended_pools)) {
+            status = channel_pool_suspend_p2p(pool);
+            if (status != NV_OK)
+                break;
+
+            __set_bit(uvm_channel_pool_index_in_channel_manager(pool), suspended_pools);
+        }
+    }
+
+    pool = channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_GPU];
+    if (status == NV_OK && !test_bit(uvm_channel_pool_index_in_channel_manager(pool), suspended_pools)) {
+        status = channel_pool_suspend_p2p(pool);
+
+        // Do not set the suspended_pools bit here. If status is NV_OK it's not
+        // needed, otherwise it should not be set anyway.
+    }
+
+    // Resume suspended pools in case of error
+    if (status != NV_OK) {
+        unsigned i;
+
+        for_each_set_bit(i, suspended_pools, channel_manager->num_channel_pools)
+            channel_pool_resume_p2p(channel_manager->channel_pools + i);
+    }
+
+    return status;
+}
+
+void uvm_channel_manager_resume_p2p(uvm_channel_manager_t *channel_manager)
+{
+    uvm_channel_pool_t *pool;
+    uvm_gpu_t *gpu = channel_manager->gpu;
+    uvm_gpu_id_t gpu_id;
+    DECLARE_BITMAP(resumed_pools, UVM_COPY_ENGINE_COUNT_MAX);
+
+    // Pools can be assigned to multiple 'pool_to_use' locations
+    // Use bitmap to track which were suspended.
+    bitmap_zero(resumed_pools, channel_manager->num_channel_pools);
+
+    uvm_assert_mutex_locked(&g_uvm_global.global_lock);
+
+    for_each_gpu_id_in_mask(gpu_id, &gpu->peer_info.peer_gpu_mask) {
+        pool = channel_manager->pool_to_use.gpu_to_gpu[uvm_id_gpu_index(gpu_id)];
+        if (pool && !test_and_set_bit(uvm_channel_pool_index_in_channel_manager(pool), resumed_pools))
+            channel_pool_resume_p2p(pool);
+    }
+
+    pool = channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_GPU];
+    if (!test_and_set_bit(uvm_channel_pool_index_in_channel_manager(pool), resumed_pools))
+        channel_pool_resume_p2p(pool);
+}
+
 NV_STATUS uvm_channel_get_status(uvm_channel_t *channel)
 {
     uvm_gpu_t *gpu;
@@ -1899,7 +2083,7 @@ NV_STATUS uvm_channel_check_errors(uvm_channel_t *channel)
     NV_STATUS status = uvm_channel_get_status(channel);
 
     if (status == NV_OK)
-        return NV_OK;
+        return status;
 
     UVM_ERR_PRINT("Detected a channel error, channel %s GPU %s\n",
                   channel->name,
@@ -2142,6 +2326,7 @@ static uvmGpuTsgHandle channel_get_tsg(uvm_channel_t *channel)
 
         tsg_index = uvm_channel_index_in_pool(channel);
     }
+
     UVM_ASSERT(tsg_index < pool->num_tsgs);
 
     return pool->tsg_handles[tsg_index];
@@ -2259,6 +2444,7 @@ static NV_STATUS channel_create(uvm_channel_pool_t *pool, uvm_channel_t *channel
      if (status != NV_OK)
          goto error;
 
+    channel->suspended_p2p = false;
     channel->num_gpfifo_entries = channel_pool_num_gpfifo_entries(pool);
     channel->gpfifo_entries = uvm_kvmalloc_zero(sizeof(*channel->gpfifo_entries) * channel->num_gpfifo_entries);
     if (channel->gpfifo_entries == NULL) {
@@ -2452,6 +2638,7 @@ static UVM_GPU_CHANNEL_ENGINE_TYPE pool_type_to_engine_type(uvm_channel_pool_typ
 {
     if (pool_type ==  UVM_CHANNEL_POOL_TYPE_SEC2)
         return UVM_GPU_CHANNEL_ENGINE_TYPE_SEC2;
+
     return UVM_GPU_CHANNEL_ENGINE_TYPE_CE;
 }
 
@@ -2633,20 +2820,6 @@ static NV_STATUS channel_pool_alloc_conf_computing_buffers(uvm_channel_pool_t *p
         if (status != NV_OK)
             return status;
     }
-    else if (uvm_channel_pool_is_ce(pool)) {
-
-        // General CE channels need to provide bi-directional communication
-        // using the pushbuffer. Encrypting an updated push from vidmem
-        // to sysmem still needs a place for auth tag in sysmem.
-        status = uvm_rm_mem_alloc_and_map_cpu(gpu,
-                                              UVM_RM_MEM_TYPE_SYS,
-                                              UVM_CONF_COMPUTING_AUTH_TAG_SIZE * num_channels *
-                                                  channel_pool_num_gpfifo_entries(pool),
-                                              UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                              &pool->conf_computing.pool_sysmem);
-        if (status != NV_OK)
-            return status;
-    }
 
     status = channel_pool_alloc_key_rotation_data(pool, num_channels);
     if (status != NV_OK)
@@ -2884,10 +3057,6 @@ static void pick_ces_for_channel_types(uvm_channel_manager_t *manager,
 {
     unsigned i;
 
-    // In Confidential Computing, do not mark all usable CEs, only the preferred
-    // ones, because non-preferred CE channels are guaranteed to not be used.
-    bool mark_all_usable_ces = !g_uvm_global.conf_computing_enabled;
-
     for (i = 0; i < num_channel_types; ++i) {
         unsigned ce;
         unsigned best_ce = UVM_COPY_ENGINE_COUNT_MAX;
@@ -2897,7 +3066,10 @@ static void pick_ces_for_channel_types(uvm_channel_manager_t *manager,
             if (!ce_is_usable(ce_caps + ce))
                 continue;
 
-            if (mark_all_usable_ces)
+            // In Confidential Computing, do not mark all usable CEs, only the
+            // preferred ones, because non-preferred CE channels are guaranteed
+            // to not be used.
+            if (!g_uvm_global.conf_computing_enabled)
                 __set_bit(ce, manager->ce_mask);
 
             if (best_ce == UVM_COPY_ENGINE_COUNT_MAX) {
@@ -2941,6 +3113,7 @@ static void pick_ces_conf_computing(uvm_channel_manager_t *manager,
                                     unsigned *preferred_ce)
 {
     unsigned best_wlc_ce;
+    uvm_gpu_t *gpu = manager->gpu;
 
     // The WLC type must go last so an unused CE is chosen, if available
     uvm_channel_type_t types[] = {UVM_CHANNEL_TYPE_CPU_TO_GPU,
@@ -2959,9 +3132,10 @@ static void pick_ces_conf_computing(uvm_channel_manager_t *manager,
 
     best_wlc_ce = preferred_ce[UVM_CHANNEL_TYPE_WLC];
 
-    // TODO: Bug 4576908: in HCC, the WLC type should not share a CE with any
-    // channel type other than LCIC. The assertion should be a check instead.
-    UVM_ASSERT(ce_usage_count(best_wlc_ce, preferred_ce) == 0);
+    // The implementation of engine-wide key rotation depends on using a
+    // dedicated CE for the WLC and LCIC pools.
+    if (uvm_conf_computing_is_key_rotation_enabled(gpu) && !gpu->parent->conf_computing.per_channel_key_rotation)
+        UVM_ASSERT(ce_usage_count(best_wlc_ce, preferred_ce) == 0);
 }
 
 static NV_STATUS channel_manager_pick_ces(uvm_channel_manager_t *manager, unsigned *preferred_ce)
@@ -2989,6 +3163,7 @@ static NV_STATUS channel_manager_pick_ces(uvm_channel_manager_t *manager, unsign
         pick_ces_conf_computing(manager, ces_caps->copyEngineCaps, preferred_ce);
     else
         pick_ces(manager, ces_caps->copyEngineCaps, preferred_ce);
+
 out:
     uvm_kvfree(ces_caps);
 
@@ -3080,14 +3255,14 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
         manager->conf.num_gpfifo_entries = UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT;
 
     if (manager->conf.num_gpfifo_entries != uvm_channel_num_gpfifo_entries) {
-        pr_info("Invalid value for uvm_channel_num_gpfifo_entries = %u, using %u instead\n",
-                uvm_channel_num_gpfifo_entries,
-                manager->conf.num_gpfifo_entries);
+        UVM_INFO_PRINT("Invalid value for uvm_channel_num_gpfifo_entries = %u, using %u instead\n",
+                       uvm_channel_num_gpfifo_entries,
+                       manager->conf.num_gpfifo_entries);
     }
 
     // 2- Allocation locations
 
-    if (uvm_conf_computing_mode_is_hcc(gpu)) {
+    if (g_uvm_global.conf_computing_enabled) {
         UVM_ASSERT(gpu->mem_info.size > 0);
 
         // When the Confidential Computing feature is enabled, the GPU is
@@ -3122,9 +3297,9 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
     pushbuffer_loc_value = uvm_channel_pushbuffer_loc;
     if (!is_string_valid_location(pushbuffer_loc_value)) {
         pushbuffer_loc_value = UVM_CHANNEL_PUSHBUFFER_LOC_DEFAULT;
-        pr_info("Invalid value for uvm_channel_pushbuffer_loc = %s, using %s instead\n",
-                uvm_channel_pushbuffer_loc,
-                pushbuffer_loc_value);
+        UVM_INFO_PRINT("Invalid value for uvm_channel_pushbuffer_loc = %s, using %s instead\n",
+                       uvm_channel_pushbuffer_loc,
+                       pushbuffer_loc_value);
     }
 
     // Override the default value if requested by the user
@@ -3134,8 +3309,8 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
         // so force the location to sys for now.
         // TODO: Bug 2904133: Remove the following "if" after the bug is fixed.
         if (NVCPU_IS_AARCH64) {
-            pr_info("uvm_channel_pushbuffer_loc = %s is not supported on AARCH64, using sys instead\n",
-                    pushbuffer_loc_value);
+            UVM_INFO_PRINT("uvm_channel_pushbuffer_loc = %s is not supported on AARCH64, using sys instead\n",
+                           pushbuffer_loc_value);
             manager->conf.pushbuffer_loc = UVM_BUFFER_LOCATION_SYS;
         }
         else {
@@ -3147,8 +3322,9 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
     // Only support the knobs for GPFIFO/GPPut on Volta+
     if (!gpu->parent->gpfifo_in_vidmem_supported) {
         if (manager->conf.gpput_loc == UVM_BUFFER_LOCATION_SYS) {
-            pr_info("CAUTION: allocating GPPut in sysmem is NOT supported and may crash the system, using %s instead\n",
-                    buffer_location_to_string(UVM_BUFFER_LOCATION_DEFAULT));
+            UVM_INFO_PRINT("CAUTION: allocating GPPut in sysmem is NOT supported and may crash the system, using %s "
+                           "instead\n",
+                           buffer_location_to_string(UVM_BUFFER_LOCATION_DEFAULT));
         }
 
         manager->conf.gpfifo_loc = UVM_BUFFER_LOCATION_DEFAULT;
@@ -3160,17 +3336,17 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
     gpfifo_loc_value = uvm_channel_gpfifo_loc;
     if (!is_string_valid_location(gpfifo_loc_value)) {
         gpfifo_loc_value = UVM_CHANNEL_GPFIFO_LOC_DEFAULT;
-        pr_info("Invalid value for uvm_channel_gpfifo_loc = %s, using %s instead\n",
-                uvm_channel_gpfifo_loc,
-                gpfifo_loc_value);
+        UVM_INFO_PRINT("Invalid value for uvm_channel_gpfifo_loc = %s, using %s instead\n",
+                       uvm_channel_gpfifo_loc,
+                       gpfifo_loc_value);
     }
 
     gpput_loc_value = uvm_channel_gpput_loc;
     if (!is_string_valid_location(gpput_loc_value)) {
         gpput_loc_value = UVM_CHANNEL_GPPUT_LOC_DEFAULT;
-        pr_info("Invalid value for uvm_channel_gpput_loc = %s, using %s instead\n",
-                uvm_channel_gpput_loc,
-                gpput_loc_value);
+        UVM_INFO_PRINT("Invalid value for uvm_channel_gpput_loc = %s, using %s instead\n",
+                       uvm_channel_gpput_loc,
+                       gpput_loc_value);
     }
 
     // On coherent platforms where the GPU does not cache sysmem but the CPU
@@ -3296,7 +3472,7 @@ static NV_STATUS setup_wlc_schedule(uvm_channel_t *wlc)
     void *decrypt_push_unprotected_cpu = (char*)gpfifo_unprotected_cpu + gpfifo_size;
 
     // Tags for upload via SEC2
-    void *decrypt_push_auth_tag, *gpfifo_auth_tag;
+    void *decrypt_push_auth_tag_cpu, *gpfifo_auth_tag_cpu;
     uvm_gpu_address_t decrypt_push_auth_tag_gpu, gpfifo_auth_tag_gpu;
 
     BUILD_BUG_ON(sizeof(*wlc_gpfifo_entries) != sizeof(*wlc->channel_info.gpFifoEntries));
@@ -3314,7 +3490,7 @@ static NV_STATUS setup_wlc_schedule(uvm_channel_t *wlc)
     // WLC can only process one job at a time.
     // Prune any initialization entries and block all but one (+1 for sentinel)
     uvm_channel_update_progress(wlc);
-    if (!try_claim_channel(wlc, wlc->num_gpfifo_entries - 2)) {
+    if (!try_claim_channel(wlc, wlc->num_gpfifo_entries - 2, UVM_CHANNEL_RESERVE_NO_P2P)) {
         status = NV_ERR_INVALID_STATE;
         goto free_gpfifo_entries;
     }
@@ -3368,15 +3544,8 @@ static NV_STATUS setup_wlc_schedule(uvm_channel_t *wlc)
     if (status != NV_OK)
         goto end_wlc_dec_push;
 
-    decrypt_push_auth_tag = uvm_push_get_single_inline_buffer(&sec2_push,
-                                                              UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                              UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                              &decrypt_push_auth_tag_gpu);
-
-    gpfifo_auth_tag = uvm_push_get_single_inline_buffer(&sec2_push,
-                                                        UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                        UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                        &gpfifo_auth_tag_gpu);
+    decrypt_push_auth_tag_cpu = push_reserve_auth_tag(&sec2_push, &decrypt_push_auth_tag_gpu);
+    gpfifo_auth_tag_cpu = push_reserve_auth_tag(&sec2_push, &gpfifo_auth_tag_gpu);
 
     // Upload WLC pushbuffer
     uvm_conf_computing_cpu_encrypt(sec2_push.channel,
@@ -3384,7 +3553,7 @@ static NV_STATUS setup_wlc_schedule(uvm_channel_t *wlc)
                                    wlc_decrypt_push.begin,
                                    NULL,
                                    decrypt_push_size,
-                                   decrypt_push_auth_tag);
+                                   decrypt_push_auth_tag_cpu);
     gpu->parent->sec2_hal->decrypt(&sec2_push,
                                    decrypt_push_protected_gpu_va,
                                    decrypt_push_unprotected_gpu_va,
@@ -3397,7 +3566,7 @@ static NV_STATUS setup_wlc_schedule(uvm_channel_t *wlc)
                                    wlc_gpfifo_entries,
                                    NULL,
                                    gpfifo_size,
-                                   gpfifo_auth_tag);
+                                   gpfifo_auth_tag_cpu);
     gpu->parent->sec2_hal->decrypt(&sec2_push,
                                    wlc->channel_info.gpFifoGpuVa,
                                    gpfifo_unprotected_gpu_va,
@@ -3452,7 +3621,7 @@ static NV_STATUS setup_lcic_schedule(uvm_channel_t *paired_wlc, uvm_channel_t *l
     NvU64 lcic_push_unprotected_gpu = gpfifo_unprotected_gpu + gpfifo_size;
     NvU64 lcic_push_protected_gpu = lcic_pb_base;
 
-    char *lcic_push_enc_tag, *gpfifo_enc_tag;
+    char *lcic_push_enc_tag_cpu, *gpfifo_enc_tag_cpu;
     uvm_gpu_address_t lcic_push_enc_tag_gpu, gpfifo_enc_tag_gpu;
 
     BUILD_BUG_ON(sizeof(*lcic_gpfifo_entries) != sizeof(*lcic->channel_info.gpFifoEntries));
@@ -3468,7 +3637,7 @@ static NV_STATUS setup_lcic_schedule(uvm_channel_t *paired_wlc, uvm_channel_t *l
     // Prune any initialization entries and
     // block all gpfifo entries (-1 for sentinel)
     uvm_channel_update_progress(lcic);
-    if (!try_claim_channel(lcic, lcic->num_gpfifo_entries - 1)) {
+    if (!try_claim_channel(lcic, lcic->num_gpfifo_entries - 1, UVM_CHANNEL_RESERVE_NO_P2P)) {
         status = NV_ERR_INVALID_STATE;
         goto free_gpfifo_entries;
     }
@@ -3522,15 +3691,8 @@ static NV_STATUS setup_lcic_schedule(uvm_channel_t *paired_wlc, uvm_channel_t *l
     if (status != NV_OK)
         goto end_lcic_push;
 
-    lcic_push_enc_tag = uvm_push_get_single_inline_buffer(&sec2_push,
-                                                          UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                          UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                          &lcic_push_enc_tag_gpu);
-
-    gpfifo_enc_tag = uvm_push_get_single_inline_buffer(&sec2_push,
-                                                       UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
-                                                       UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT,
-                                                       &gpfifo_enc_tag_gpu);
+    lcic_push_enc_tag_cpu = push_reserve_auth_tag(&sec2_push, &lcic_push_enc_tag_gpu);
+    gpfifo_enc_tag_cpu = push_reserve_auth_tag(&sec2_push, &gpfifo_enc_tag_gpu);
 
     // Upload LCIC pushbuffer
     uvm_conf_computing_cpu_encrypt(sec2_push.channel,
@@ -3538,7 +3700,7 @@ static NV_STATUS setup_lcic_schedule(uvm_channel_t *paired_wlc, uvm_channel_t *l
                                    lcic_push.begin,
                                    NULL,
                                    lcic_push_size,
-                                   lcic_push_enc_tag);
+                                   lcic_push_enc_tag_cpu);
     gpu->parent->sec2_hal->decrypt(&sec2_push,
                                    lcic_push_protected_gpu,
                                    lcic_push_unprotected_gpu,
@@ -3551,7 +3713,7 @@ static NV_STATUS setup_lcic_schedule(uvm_channel_t *paired_wlc, uvm_channel_t *l
                                    lcic_gpfifo_entries,
                                    NULL,
                                    gpfifo_size,
-                                   gpfifo_enc_tag);
+                                   gpfifo_enc_tag_cpu);
     gpu->parent->sec2_hal->decrypt(&sec2_push,
                                    lcic->channel_info.gpFifoGpuVa,
                                    gpfifo_unprotected_gpu,
@@ -3738,6 +3900,7 @@ static void channel_manager_destroy_pools(uvm_channel_manager_t *manager)
 {
     uvm_rm_mem_free(manager->gpu->conf_computing.iv_rm_mem);
     manager->gpu->conf_computing.iv_rm_mem = NULL;
+
     while (manager->num_channel_pools > 0)
         channel_pool_destroy(manager->channel_pools + manager->num_channel_pools - 1);
 
@@ -3894,6 +4057,7 @@ const char *uvm_channel_type_to_string(uvm_channel_type_t channel_type)
 
 const char *uvm_channel_pool_type_to_string(uvm_channel_pool_type_t channel_pool_type)
 {
+
     BUILD_BUG_ON(UVM_CHANNEL_POOL_TYPE_COUNT != 5);
 
     switch (channel_pool_type) {
@@ -3908,17 +4072,21 @@ const char *uvm_channel_pool_type_to_string(uvm_channel_pool_type_t channel_pool
 
 static const char *get_gpfifo_location_string(uvm_channel_t *channel)
 {
+
     // SEC2 channels override the channel manager location for GPFIFO.
     if (uvm_channel_is_sec2(channel))
         return buffer_location_to_string(UVM_BUFFER_LOCATION_SYS);
+
     return buffer_location_to_string(channel->pool->manager->conf.gpfifo_loc);
 }
 
 static const char *get_gpput_location_string(uvm_channel_t *channel)
 {
+
     // SEC2 channels override the channel manager location for GPPUT.
     if (uvm_channel_is_sec2(channel))
         return buffer_location_to_string(UVM_BUFFER_LOCATION_SYS);
+
     return buffer_location_to_string(channel->pool->manager->conf.gpput_loc);
 }
 

kernel-open/nvidia-uvm/uvm_channel.h

@@ -200,6 +200,7 @@ typedef struct
     // num_tsgs is 1. Pre-Volta GPUs also have a single TSG object, but since HW
     // does not support TSG for CE engines, a HW TSG is not created, but a TSG
     // object is required to allocate channels.
+    //
     // When Confidential Computing mode is enabled, the WLC and LCIC channel
     // types require one TSG for each WLC/LCIC pair of channels. In this case,
     // we do not use a TSG per channel pool, but instead a TSG per WLC/LCIC
@@ -416,6 +417,8 @@ struct uvm_channel_struct
         struct list_head channel_list_node;
         NvU32 pending_event_count;
     } tools;
+
+    bool suspended_p2p;
 };
 
 struct uvm_channel_manager_struct
@@ -478,6 +481,12 @@ struct uvm_channel_manager_struct
     } conf_computing;
 };
 
+// Index of a channel pool within the manager
+static unsigned uvm_channel_pool_index_in_channel_manager(const uvm_channel_pool_t *pool)
+{
+    return pool - pool->manager->channel_pools;
+}
+
 // Create a channel manager for the GPU
 NV_STATUS uvm_channel_manager_create(uvm_gpu_t *gpu, uvm_channel_manager_t **manager_out);
 
@@ -532,7 +541,7 @@ NvU64 uvm_channel_get_static_pb_unprotected_sysmem_gpu_va(uvm_channel_t *channel
 
 char* uvm_channel_get_static_pb_unprotected_sysmem_cpu(uvm_channel_t *channel);
 
-char *uvm_channel_get_push_crypto_bundle_auth_tags_cpu_va(uvm_channel_t *channel, unsigned tag_index);
+bool uvm_channel_pool_is_p2p(uvm_channel_pool_t *pool);
 
 static bool uvm_channel_pool_is_proxy(uvm_channel_pool_t *pool)
 {
@@ -551,6 +560,11 @@ static bool uvm_channel_pool_is_ce(uvm_channel_pool_t *pool)
     return !uvm_channel_pool_is_sec2(pool);
 }
 
+static bool uvm_channel_is_p2p(uvm_channel_t *channel)
+{
+    return uvm_channel_pool_is_p2p(channel->pool);
+}
+
 static bool uvm_channel_is_ce(uvm_channel_t *channel)
 {
     return uvm_channel_pool_is_ce(channel->pool);
@@ -586,14 +600,25 @@ bool uvm_channel_is_privileged(uvm_channel_t *channel);
 // Destroy the channel manager
 void uvm_channel_manager_destroy(uvm_channel_manager_t *channel_manager);
 
+// Suspend p2p traffic on channels used for p2p operations.
+// This is used in STO recovery sequence to quiet nvlink traffic before the
+// links can be restored.
+NV_STATUS uvm_channel_manager_suspend_p2p(uvm_channel_manager_t *channel_manager);
+
+// Resume p2p traffic on channels used for p2p operations.
+// This is used at the end of the STO recovery sequence to resume suspended p2p
+// traffic on p2p channels.
+void uvm_channel_manager_resume_p2p(uvm_channel_manager_t *channel_manager);
+
 // Get the current status of the channel
-// Returns NV_OK if the channel is in a good state and NV_ERR_RC_ERROR
-// otherwise. Notably this never sets the global fatal error.
+// Returns NV_OK if the channel is in a good state,
+//         NV_ERR_RC_ERROR otherwise.
+// Notably this never sets the global fatal error.
 NV_STATUS uvm_channel_get_status(uvm_channel_t *channel);
 
 // Check for channel errors
-// Checks for channel errors by calling uvm_channel_get_status(). If an error
-// occurred, sets the global fatal error and prints errors.
+// Checks for channel errors by calling uvm_channel_get_status().
+// If a fatal error occurred, sets the global fatal error and prints errors.
 NV_STATUS uvm_channel_check_errors(uvm_channel_t *channel);
 
 // Check errors on all channels in the channel manager
@@ -627,6 +652,7 @@ static bool uvm_channel_manager_is_wlc_ready(uvm_channel_manager_t *manager)
 {
     return manager->conf_computing.wlc_ready;
 }
+
 // Get the GPU VA of semaphore_channel's tracking semaphore within the VA space
 // associated with access_channel.
 //

kernel-open/nvidia-uvm/uvm_channel_test.c

@@ -206,9 +206,10 @@ static NV_STATUS uvm_test_rc_for_gpu(uvm_gpu_t *gpu)
     uvm_for_each_pool(pool, manager) {
         uvm_channel_t *channel;
 
-            // Skip LCIC channels as those can't accept any pushes
-            if (uvm_channel_pool_is_lcic(pool))
-                continue;
+        // Skip LCIC channels as those can't accept any pushes
+        if (uvm_channel_pool_is_lcic(pool))
+            continue;
+
         uvm_for_each_channel_in_pool(channel, pool) {
             NvU32 i;
             for (i = 0; i < 512; ++i) {
@@ -793,7 +794,7 @@ done:
 // This test verifies that concurrent pushes using the same channel pool
 // select different channels, when the Confidential Computing feature is
 // enabled.
-NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
+static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
 {
     NV_STATUS status = NV_OK;
     uvm_push_t *pushes = NULL;
@@ -866,7 +867,7 @@ error:
     return status;
 }
 
-NV_STATUS test_channel_iv_rotation(uvm_va_space_t *va_space)
+static NV_STATUS test_channel_iv_rotation(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -1273,7 +1274,7 @@ static NV_STATUS test_channel_key_rotation(uvm_va_space_t *va_space)
     return NV_OK;
 }
 
-NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
+static NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -1292,6 +1293,7 @@ NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
             // after their schedule is set up
             if (uvm_channel_pool_is_wlc(pool))
                 continue;
+
             uvm_for_each_channel_in_pool(channel, pool) {
                 NvU32 i;
 
@@ -1312,7 +1314,7 @@ NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
     return NV_OK;
 }
 
-NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
+static NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -1331,6 +1333,7 @@ NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
             // after their schedule is set up
             if (uvm_channel_pool_is_wlc(pool))
                 continue;
+
             uvm_for_each_channel_in_pool(channel, pool) {
                 NvU32 i;
                 uvm_push_t push;
@@ -1360,7 +1363,7 @@ NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
     return NV_OK;
 }
 
-NV_STATUS test_write_ctrl_gpfifo_tight(uvm_va_space_t *va_space)
+static NV_STATUS test_write_ctrl_gpfifo_tight(uvm_va_space_t *va_space)
 {
     NV_STATUS status = NV_OK;
     uvm_gpu_t *gpu;
@@ -1419,7 +1422,7 @@ NV_STATUS test_write_ctrl_gpfifo_tight(uvm_va_space_t *va_space)
         TEST_NV_CHECK_GOTO(uvm_channel_write_ctrl_gpfifo(channel, entry), error);
 
         // Release the semaphore.
-        UVM_WRITE_ONCE(*cpu_ptr, 1);
+        WRITE_ONCE(*cpu_ptr, 1);
 
         TEST_NV_CHECK_GOTO(uvm_push_wait(&push), error);
 
@@ -1473,6 +1476,7 @@ static NV_STATUS test_channel_pushbuffer_extension_base(uvm_va_space_t *va_space
             // Skip LCIC channels as those can't accept any pushes
             if (uvm_channel_pool_is_lcic(pool))
                 continue;
+
             uvm_for_each_channel_in_pool(channel, pool) {
                 NvU32 i;
                 uvm_push_t push;

kernel-open/nvidia-uvm/uvm_common.c

@@ -281,29 +281,6 @@ NV_STATUS uvm_spin_loop(uvm_spin_loop_t *spin)
     return NV_OK;
 }
 
-// This formats a GPU UUID, in a UVM-friendly way. That is, nearly the same as
-// what nvidia-smi reports.  It will always prefix the UUID with UVM-GPU so
-// that we know that we have a real, binary formatted UUID that will work in
-// the UVM APIs.
-//
-// It comes out like this:
-//
-//     UVM-GPU-d802726c-df8d-a3c3-ec53-48bdec201c27
-//
-//  This routine will always null-terminate the string for you. This is true
-//  even if the buffer was too small!
-//
-//  Return value is the number of non-null characters written.
-//
-// Note that if you were to let the NV2080_CTRL_CMD_GPU_GET_GID_INFO command
-// return it's default format, which is ascii, not binary, then you would get
-// this back:
-//
-//     GPU-d802726c-df8d-a3c3-ec53-48bdec201c27
-//
-//  ...which is actually a character string, and won't work for UVM API calls.
-//  So it's very important to be able to see the difference.
-//
 static char uvm_digit_to_hex(unsigned value)
 {
     if (value >= 10)
@@ -312,27 +289,19 @@ static char uvm_digit_to_hex(unsigned value)
         return value + '0';
 }
 
-int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessorUuid *pUuidStruct)
+void uvm_uuid_string(char *buffer, const NvProcessorUuid *pUuidStruct)
 {
-    char *str = buffer+8;
+    char *str = buffer;
     unsigned i;
     unsigned dashMask = 1 << 4 | 1 << 6 | 1 << 8 | 1 << 10;
 
-    if (bufferLength < (8 /*prefix*/+ 16 * 2 /*digits*/ + 4 * 1 /*dashes*/ + 1 /*null*/))
-        return *buffer = 0;
-
-    memcpy(buffer, "UVM-GPU-", 8);
-
     for (i = 0; i < 16; i++) {
         *str++ = uvm_digit_to_hex(pUuidStruct->uuid[i] >> 4);
         *str++ = uvm_digit_to_hex(pUuidStruct->uuid[i] & 0xF);
 
-        if (dashMask & (1 << (i+1)))
+        if (dashMask & (1 << (i + 1)))
             *str++ = '-';
     }
 
     *str = 0;
-
-    return (int)(str-buffer);
 }
-

kernel-open/nvidia-uvm/uvm_common.h

@@ -50,10 +50,14 @@ enum {
     NVIDIA_UVM_NUM_MINOR_DEVICES
 };
 
-#define UVM_GPU_UUID_TEXT_BUFFER_LENGTH (8+16*2+4+1)
+// UUID has the format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+#define UVM_UUID_STRING_LENGTH ((8 + 1) + 3 * (4 + 1) + 12 + 1)
 
-int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessorUuid *pGpuUuid);
+// Writes UVM_UUID_STRING_LENGTH characters into buffer, including a terminating
+// NULL.
+void uvm_uuid_string(char *buffer, const NvProcessorUuid *uuid);
 
+// Long prefix - typically for debugging and tests.
 #define UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ...) \
     func(prefix "%s:%u %s[pid:%d]" fmt,               \
          kbasename(__FILE__),                         \
@@ -62,10 +66,15 @@ int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessor
          current->pid,                                \
          ##__VA_ARGS__)
 
+// Short prefix - typically for information.
+#define UVM_PRINT_FUNC_SHORT_PREFIX(func, prefix, fmt, ...) \
+    func(prefix fmt, ##__VA_ARGS__)
+
+// No prefix - used by kernel panic messages.
 #define UVM_PRINT_FUNC(func, fmt, ...)  \
     UVM_PRINT_FUNC_PREFIX(func, "", fmt, ##__VA_ARGS__)
 
-// Check whether UVM_{ERR,DBG,INFO)_PRINT* should be enabled
+// Check whether UVM_{ERR,DBG)_PRINT* should be enabled.
 bool uvm_debug_prints_enabled(void);
 
 // A printing helper like UVM_PRINT_FUNC_PREFIX that only prints if
@@ -77,10 +86,10 @@ bool uvm_debug_prints_enabled(void);
         }                                                               \
     } while (0)
 
-#define UVM_ASSERT_PRINT(fmt, ...) \
+#define UVM_ERR_PRINT_ALWAYS(fmt, ...) \
     UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
 
-#define UVM_ASSERT_PRINT_RL(fmt, ...) \
+#define UVM_ERR_PRINT_ALWAYS_RL(fmt, ...) \
     UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
 
 #define UVM_ERR_PRINT(fmt, ...) \
@@ -92,33 +101,18 @@ bool uvm_debug_prints_enabled(void);
 #define UVM_DBG_PRINT(fmt, ...) \
     UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
 
-#define UVM_DBG_PRINT_RL(fmt, ...)                              \
+#define UVM_DBG_PRINT_RL(fmt, ...) \
     UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
 
+// UVM_INFO_PRINT prints in all modes (including in the release mode.) It is
+// used for relaying driver-level information, rather than detailed debugging
+// information; therefore, it does not add the "pretty long prefix".
 #define UVM_INFO_PRINT(fmt, ...) \
-    UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_INFO NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
+    UVM_PRINT_FUNC_SHORT_PREFIX(printk, KERN_INFO NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
 
-//
-// Please see the documentation of format_uuid_to_buffer, for details on what
-// this routine prints for you.
-//
-#define UVM_DBG_PRINT_UUID(msg, uuidPtr)                                \
-    do {                                                                \
-        char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH];               \
-        format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \
-        UVM_DBG_PRINT("%s: %s\n", msg, uuidBuffer);                     \
-    } while (0)
-
-#define UVM_ERR_PRINT_NV_STATUS(msg, rmStatus, ...)                        \
+#define UVM_ERR_PRINT_NV_STATUS(msg, rmStatus, ...) \
     UVM_ERR_PRINT("ERROR: %s : " msg "\n", nvstatusToString(rmStatus), ##__VA_ARGS__)
 
-#define UVM_ERR_PRINT_UUID(msg, uuidPtr, ...)                              \
-    do {                                                                   \
-        char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH];                  \
-        format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr);    \
-        UVM_ERR_PRINT("ERROR: %s : " msg "\n", uuidBuffer, ##__VA_ARGS__); \
-    } while (0)
-
 #define UVM_PANIC()             UVM_PRINT_FUNC(panic, "\n")
 #define UVM_PANIC_MSG(fmt, ...) UVM_PRINT_FUNC(panic, ": " fmt, ##__VA_ARGS__)
 
@@ -149,13 +143,13 @@ void on_uvm_test_fail(void);
 // Unlike on_uvm_test_fail it provides 'panic' coverity semantics
 void on_uvm_assert(void);
 
-#define _UVM_ASSERT_MSG(expr, cond, fmt, ...)                                                   \
-    do {                                                                                        \
-        if (unlikely(!(expr))) {                                                                \
-            UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__);  \
-            dump_stack();                                                                       \
-            on_uvm_assert();                                                                    \
-        }                                                                                       \
+#define _UVM_ASSERT_MSG(expr, cond, fmt, ...)                                                       \
+    do {                                                                                            \
+        if (unlikely(!(expr))) {                                                                    \
+            UVM_ERR_PRINT_ALWAYS("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__);  \
+            dump_stack();                                                                           \
+            on_uvm_assert();                                                                        \
+        }                                                                                           \
     } while (0)
 
 // Prevent function calls in expr and the print argument list from being
@@ -166,7 +160,8 @@ void on_uvm_assert(void);
         UVM_NO_PRINT(fmt, ##__VA_ARGS__);       \
     } while (0)
 
-// UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds
+// UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity
+// builds.
 #if UVM_IS_DEBUG() || defined __COVERITY__
     #define UVM_ASSERT_MSG(expr, fmt, ...)  _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__)
     #define UVM_ASSERT(expr)                _UVM_ASSERT_MSG(expr, #expr, "\n")
@@ -189,16 +184,16 @@ extern bool uvm_release_asserts_set_global_error_for_tests;
 // Given these are enabled for release builds, we need to be more cautious than
 // in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module
 // param is enabled.
-#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...)                                                   \
-    do {                                                                                                \
-        if (uvm_release_asserts && unlikely(!(expr))) {                                                 \
-            UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__);       \
-            if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \
-                uvm_global_set_fatal_error(NV_ERR_INVALID_STATE);                                       \
-            if (uvm_release_asserts_dump_stack)                                                         \
-                dump_stack();                                                                           \
-            on_uvm_assert();                                                                            \
-        }                                                                                               \
+#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...)                                                       \
+    do {                                                                                                    \
+        if (uvm_release_asserts && unlikely(!(expr))) {                                                     \
+            UVM_ERR_PRINT_ALWAYS_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__);       \
+            if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests)     \
+                uvm_global_set_fatal_error(NV_ERR_INVALID_STATE);                                           \
+            if (uvm_release_asserts_dump_stack)                                                             \
+                dump_stack();                                                                               \
+            on_uvm_assert();                                                                                \
+        }                                                                                                   \
     } while (0)
 
 #define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...)  _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
@@ -395,7 +390,7 @@ static inline void uvm_touch_page(struct page *page)
     UVM_ASSERT(page);
 
     mapping = (char *) kmap(page);
-    (void)UVM_READ_ONCE(*mapping);
+    (void)READ_ONCE(*mapping);
     kunmap(page);
 }
 

kernel-open/nvidia-uvm/uvm_conf_computing.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2021-2023 NVIDIA Corporation
+    Copyright (c) 2021-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -65,32 +65,13 @@ static ulong uvm_conf_computing_channel_iv_rotation_limit = UVM_CONF_COMPUTING_I
 
 module_param(uvm_conf_computing_channel_iv_rotation_limit, ulong, S_IRUGO);
 
-static UvmGpuConfComputeMode uvm_conf_computing_get_mode(const uvm_parent_gpu_t *parent)
-{
-    return parent->rm_info.gpuConfComputeCaps.mode;
-}
-
-bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu)
-{
-    return uvm_conf_computing_get_mode(gpu->parent) == UVM_GPU_CONF_COMPUTE_MODE_HCC;
-}
-
 void uvm_conf_computing_check_parent_gpu(const uvm_parent_gpu_t *parent)
 {
-    uvm_parent_gpu_t *other_parent;
-    UvmGpuConfComputeMode parent_mode = uvm_conf_computing_get_mode(parent);
-
     uvm_assert_mutex_locked(&g_uvm_global.global_lock);
 
-    // The Confidential Computing state of the GPU should match that of the
-    // system.
-    UVM_ASSERT((parent_mode != UVM_GPU_CONF_COMPUTE_MODE_NONE) == g_uvm_global.conf_computing_enabled);
-
-    // All GPUs derive Confidential Computing status from their parent. By
-    // current policy all parent GPUs have identical Confidential Computing
-    // status.
-    for_each_parent_gpu(other_parent)
-        UVM_ASSERT(parent_mode == uvm_conf_computing_get_mode(other_parent));
+    // Confidential Computing enablement on the system should match enablement
+    // on the GPU.
+    UVM_ASSERT(parent->rm_info.gpuConfComputeCaps.bConfComputingEnabled == g_uvm_global.conf_computing_enabled);
 }
 
 static void dma_buffer_destroy_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
@@ -343,9 +324,6 @@ static NV_STATUS dummy_iv_mem_init(uvm_gpu_t *gpu)
 {
     NV_STATUS status;
 
-    if (!uvm_conf_computing_mode_is_hcc(gpu))
-        return NV_OK;
-
     status = uvm_mem_alloc_sysmem_dma(sizeof(UvmCslIv), gpu, NULL, &gpu->conf_computing.iv_mem);
     if (status != NV_OK)
         return status;
@@ -554,7 +532,7 @@ NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu,
 {
     NV_STATUS status;
     NvU32 fault_entry_size = parent_gpu->fault_buffer_hal->entry_size(parent_gpu);
-    UvmCslContext *csl_context = &parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx;
+    UvmCslContext *csl_context = &parent_gpu->fault_buffer.rm_info.replayable.cslCtx;
 
     // There is no dedicated lock for the CSL context associated with replayable
     // faults. The mutual exclusion required by the RM CSL API is enforced by
@@ -593,7 +571,7 @@ void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu)
 {
     NV_STATUS status;
     NvU32 fault_entry_size = parent_gpu->fault_buffer_hal->entry_size(parent_gpu);
-    UvmCslContext *csl_context = &parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx;
+    UvmCslContext *csl_context = &parent_gpu->fault_buffer.rm_info.replayable.cslCtx;
 
     // See comment in uvm_conf_computing_fault_decrypt
     UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
@@ -730,7 +708,12 @@ void uvm_conf_computing_disable_key_rotation(uvm_gpu_t *gpu)
 
 bool uvm_conf_computing_is_key_rotation_enabled(uvm_gpu_t *gpu)
 {
-    return gpu->channel_manager->conf_computing.key_rotation_enabled;
+    UVM_ASSERT(gpu);
+
+    // If the channel_manager is not set, we're in channel manager destroy
+    // path after the pointer was NULL-ed. Chances are that other key rotation
+    // infrastructure is not available either. Disallow the key rotation.
+    return gpu->channel_manager && gpu->channel_manager->conf_computing.key_rotation_enabled;
 }
 
 bool uvm_conf_computing_is_key_rotation_enabled_in_pool(uvm_channel_pool_t *pool)

kernel-open/nvidia-uvm/uvm_conf_computing.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2021-2023 NVIDIA Corporation
+    Copyright (c) 2021-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -62,8 +62,6 @@
 
 void uvm_conf_computing_check_parent_gpu(const uvm_parent_gpu_t *parent);
 
-bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu);
-
 typedef struct
 {
     // List of free DMA buffers (uvm_conf_computing_dma_buffer_t).

kernel-open/nvidia-uvm/uvm_debug_optimized.c

@@ -1,53 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-// This file provides simple wrappers that are always built with optimizations
-// turned on to WAR issues with functions that don't build correctly otherwise.
-
-#include "uvm_linux.h"
-
-int nv_atomic_xchg(atomic_t *val, int new)
-{
-    return atomic_xchg(val, new);
-}
-
-int nv_atomic_cmpxchg(atomic_t *val, int old, int new)
-{
-    return atomic_cmpxchg(val, old, new);
-}
-
-long nv_atomic_long_cmpxchg(atomic_long_t *val, long old, long new)
-{
-    return atomic_long_cmpxchg(val, old, new);
-}
-
-unsigned long nv_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-    return copy_from_user(to, from, n);
-}
-
-unsigned long nv_copy_to_user(void __user *to, const void *from, unsigned long n)
-{
-    return copy_to_user(to, from, n);
-}
-