550.163.01

README.md

@@ -1,7 +1,7 @@
 # NVIDIA Linux Open GPU Kernel Module Source
 
 This is the source release of the NVIDIA Linux open GPU kernel modules,
-version 560.31.02.
+version 550.163.01.
 
 
 ## How to Build
@@ -17,7 +17,7 @@ as root:
 
 Note that the kernel modules built here must be used with GSP
 firmware and user-space NVIDIA GPU driver components from a corresponding
-560.31.02 driver release.  This can be achieved by installing
+550.163.01 driver release.  This can be achieved by installing
 the NVIDIA GPU driver from the .run file using the `--no-kernel-modules`
 option.  E.g.,
 
@@ -74,7 +74,7 @@ kernel.
 
 The NVIDIA open kernel modules support the same range of Linux kernel
 versions that are supported with the proprietary NVIDIA kernel modules.
-This is currently Linux kernel 4.15 or newer.
+This is currently Linux kernel 3.10 or newer.
 
 
 ## How to Contribute
@@ -179,13 +179,16 @@ software applications.
 
 ## Compatible GPUs
 
-The NVIDIA open kernel modules can be used on any Turing or later GPU (see the
-table below).
+The NVIDIA open kernel modules can be used on any Turing or later GPU
+(see the table below). However, in the __DRIVER_VERION__ release, GeForce and
+Workstation support is considered to be Beta quality. The open kernel modules
+are suitable for broad usage, and NVIDIA requests feedback on any issues
+encountered specific to them.
 
 For details on feature support and limitations, see the NVIDIA GPU driver
 end user README here:
 
-https://us.download.nvidia.com/XFree86/Linux-x86_64/560.31.02/README/kernel_open.html
+https://us.download.nvidia.com/XFree86/Linux-x86_64/550.163.01/README/kernel_open.html
 
 For vGPU support, please refer to the README.vgpu packaged in the vGPU Host
 Package for more details.
@@ -751,6 +754,7 @@ Subsystem Device ID.
 | NVIDIA H800                                     | 2324 10DE 17A8 |
 | NVIDIA H20                                      | 2329 10DE 198B |
 | NVIDIA H20                                      | 2329 10DE 198C |
+| NVIDIA H20-3e                                   | 232C 10DE 2063 |
 | NVIDIA H100 80GB HBM3                           | 2330 10DE 16C0 |
 | NVIDIA H100 80GB HBM3                           | 2330 10DE 16C1 |
 | NVIDIA H100 PCIe                                | 2331 10DE 1626 |
@@ -831,12 +835,10 @@ Subsystem Device ID.
 | NVIDIA GeForce RTX 2050                         | 25AD           |
 | NVIDIA RTX A1000                                | 25B0 1028 1878 |
 | NVIDIA RTX A1000                                | 25B0 103C 1878 |
-| NVIDIA RTX A1000                                | 25B0 103C 8D96 |
 | NVIDIA RTX A1000                                | 25B0 10DE 1878 |
 | NVIDIA RTX A1000                                | 25B0 17AA 1878 |
 | NVIDIA RTX A400                                 | 25B2 1028 1879 |
 | NVIDIA RTX A400                                 | 25B2 103C 1879 |
-| NVIDIA RTX A400                                 | 25B2 103C 8D95 |
 | NVIDIA RTX A400                                 | 25B2 10DE 1879 |
 | NVIDIA RTX A400                                 | 25B2 17AA 1879 |
 | NVIDIA A16                                      | 25B6 10DE 14A9 |
@@ -857,7 +859,6 @@ Subsystem Device ID.
 | NVIDIA RTX A500 Embedded GPU                    | 25FB           |
 | NVIDIA GeForce RTX 4090                         | 2684           |
 | NVIDIA GeForce RTX 4090 D                       | 2685           |
-| NVIDIA GeForce RTX 4070 Ti SUPER                | 2689           |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 1028 16A1 |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 103C 16A1 |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 10DE 16A1 |
@@ -918,6 +919,7 @@ Subsystem Device ID.
 | NVIDIA GeForce RTX 4060                         | 2882           |
 | NVIDIA GeForce RTX 4060 Laptop GPU              | 28A0           |
 | NVIDIA GeForce RTX 4050 Laptop GPU              | 28A1           |
+| NVIDIA GeForce RTX 3050 A Laptop GPU            | 28A3           |
 | NVIDIA RTX 2000 Ada Generation                  | 28B0 1028 1870 |
 | NVIDIA RTX 2000 Ada Generation                  | 28B0 103C 1870 |
 | NVIDIA RTX 2000E Ada Generation                 | 28B0 103C 1871 |
@@ -931,4 +933,5 @@ Subsystem Device ID.
 | NVIDIA RTX 500 Ada Generation Laptop GPU        | 28BB           |
 | NVIDIA GeForce RTX 4060 Laptop GPU              | 28E0           |
 | NVIDIA GeForce RTX 4050 Laptop GPU              | 28E1           |
+| NVIDIA GeForce RTX 3050 A Laptop GPU            | 28E3           |
 | NVIDIA RTX 2000 Ada Generation Embedded GPU     | 28F8           |

kernel-open/Kbuild

@@ -57,6 +57,20 @@ ifeq ($(NV_UNDEF_BEHAVIOR_SANITIZER),1)
  UBSAN_SANITIZE := y
 endif
 
+#
+# Command to create a symbolic link, explicitly resolving the symlink target
+# to an absolute path to abstract away the difference between Linux < 6.13,
+# where the CWD is the Linux kernel source tree for Kbuild extmod builds, and
+# Linux >= 6.13, where the CWD is the external module source tree.
+#
+# This is used to create the nv*-kernel.o -> nv*-kernel.o_binary symlinks for
+# kernel modules which use precompiled binary object files.
+#
+
+quiet_cmd_symlink = SYMLINK $@
+ cmd_symlink = ln -sf $(abspath $<) $@
+
+
 $(foreach _module, $(NV_KERNEL_MODULES), \
  $(eval include $(src)/$(_module)/$(_module).Kbuild))
 
@@ -72,7 +86,7 @@ EXTRA_CFLAGS += -I$(src)/common/inc
 EXTRA_CFLAGS += -I$(src)
 EXTRA_CFLAGS += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
 EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
-EXTRA_CFLAGS += -DNV_VERSION_STRING=\"560.31.02\"
+EXTRA_CFLAGS += -DNV_VERSION_STRING=\"550.163.01\"
 
 ifneq ($(SYSSRCHOST1X),)
  EXTRA_CFLAGS += -I$(SYSSRCHOST1X)
@@ -118,7 +132,7 @@ ifeq ($(ARCH),x86_64)
 endif
 
 ifeq ($(ARCH),powerpc)
- EXTRA_CFLAGS += -mlittle-endian -mno-strict-align
+ EXTRA_CFLAGS += -mlittle-endian -mno-strict-align -mno-altivec
 endif
 
 EXTRA_CFLAGS += -DNV_UVM_ENABLE

kernel-open/Makefile

@@ -52,6 +52,22 @@ else
     endif
   endif
 
+  # If CC hasn't been set explicitly, check the value of CONFIG_CC_VERSION_TEXT.
+  # Look for the compiler specified there, and use it by default, if found.
+  ifeq ($(origin CC),default)
+    cc_version_text=$(firstword $(shell . $(KERNEL_OUTPUT)/.config; \
+                      echo "$$CONFIG_CC_VERSION_TEXT"))
+
+    ifneq ($(cc_version_text),)
+      ifeq ($(shell command -v $(cc_version_text)),)
+          $(warning WARNING: Unable to locate the compiler $(cc_version_text) \
+            from CONFIG_CC_VERSION_TEXT in the kernel configuration.)
+      else
+          CC=$(cc_version_text)
+      endif
+    endif
+  endif
+
   CC ?= cc
   LD ?= ld
   OBJDUMP ?= objdump
@@ -65,6 +81,16 @@ else
     )
   endif
 
+  KERNEL_ARCH = $(ARCH)
+
+  ifneq ($(filter $(ARCH),i386 x86_64),)
+    KERNEL_ARCH = x86
+  else
+    ifeq ($(filter $(ARCH),arm64 powerpc),)
+        $(error Unsupported architecture $(ARCH))
+    endif
+  endif
+
   NV_KERNEL_MODULES ?= $(wildcard nvidia nvidia-uvm nvidia-vgpu-vfio nvidia-modeset nvidia-drm nvidia-peermem)
   NV_KERNEL_MODULES := $(filter-out $(NV_EXCLUDE_KERNEL_MODULES), \
                                     $(NV_KERNEL_MODULES))
@@ -106,8 +132,9 @@ else
   # module symbols on which the Linux kernel's module resolution is dependent
   # and hence must be used whenever present.
 
-  LD_SCRIPT ?= $(KERNEL_SOURCES)/scripts/module-common.lds      \
-               $(KERNEL_SOURCES)/arch/$(ARCH)/kernel/module.lds \
+  LD_SCRIPT ?= $(KERNEL_SOURCES)/scripts/module-common.lds             \
+               $(KERNEL_SOURCES)/arch/$(KERNEL_ARCH)/kernel/module.lds \
+               $(KERNEL_OUTPUT)/arch/$(KERNEL_ARCH)/module.lds         \
                $(KERNEL_OUTPUT)/scripts/module.lds
   NV_MODULE_COMMON_SCRIPTS := $(foreach s, $(wildcard $(LD_SCRIPT)), -T $(s))
 

kernel-open/common/inc/nv-firmware.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -44,7 +44,6 @@ typedef enum
     NV_FIRMWARE_CHIP_FAMILY_GA10X = 4,
     NV_FIRMWARE_CHIP_FAMILY_AD10X = 5,
     NV_FIRMWARE_CHIP_FAMILY_GH100 = 6,
-    NV_FIRMWARE_CHIP_FAMILY_GB10X = 8,
     NV_FIRMWARE_CHIP_FAMILY_END,
 } nv_firmware_chip_family_t;
 
@@ -53,7 +52,6 @@ static inline const char *nv_firmware_chip_family_to_string(
 )
 {
     switch (fw_chip_family) {
-        case NV_FIRMWARE_CHIP_FAMILY_GB10X: return "gb10x";
         case NV_FIRMWARE_CHIP_FAMILY_GH100: return "gh100";
         case NV_FIRMWARE_CHIP_FAMILY_AD10X: return "ad10x";
         case NV_FIRMWARE_CHIP_FAMILY_GA10X: return "ga10x";
@@ -68,13 +66,13 @@ static inline const char *nv_firmware_chip_family_to_string(
     return NULL;
 }
 
-// The includer may optionally define
-// NV_FIRMWARE_FOR_NAME(name)
-// to return a platform-defined string for a given a gsp_* or gsp_log_* name.
+// The includer (presumably nv.c) may optionally define
+// NV_FIRMWARE_PATH_FOR_FILENAME(filename)
+// to return a string "path" given a gsp_*.bin or gsp_log_*.bin filename.
 //
-// The function nv_firmware_for_chip_family will then be available.
-#if defined(NV_FIRMWARE_FOR_NAME)
-static inline const char *nv_firmware_for_chip_family(
+// The function nv_firmware_path will then be available.
+#if defined(NV_FIRMWARE_PATH_FOR_FILENAME)
+static inline const char *nv_firmware_path(
     nv_firmware_type_t fw_type,
     nv_firmware_chip_family_t fw_chip_family
 )
@@ -83,16 +81,15 @@ static inline const char *nv_firmware_for_chip_family(
     {
         switch (fw_chip_family)
         {
-            case NV_FIRMWARE_CHIP_FAMILY_GB10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GH100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_AD10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GA10X:
-                return NV_FIRMWARE_FOR_NAME("gsp_ga10x");
+                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_ga10x.bin");
 
             case NV_FIRMWARE_CHIP_FAMILY_GA100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU11X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU10X:
-                return NV_FIRMWARE_FOR_NAME("gsp_tu10x");
+                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_tu10x.bin");
 
             case NV_FIRMWARE_CHIP_FAMILY_END:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_NULL:
@@ -103,16 +100,15 @@ static inline const char *nv_firmware_for_chip_family(
     {
         switch (fw_chip_family)
         {
-            case NV_FIRMWARE_CHIP_FAMILY_GB10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GH100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_AD10X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_GA10X:
-                return NV_FIRMWARE_FOR_NAME("gsp_log_ga10x");
+                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_ga10x.bin");
 
             case NV_FIRMWARE_CHIP_FAMILY_GA100:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU11X:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_TU10X:
-                return NV_FIRMWARE_FOR_NAME("gsp_log_tu10x");
+                return NV_FIRMWARE_PATH_FOR_FILENAME("gsp_log_tu10x.bin");
 
             case NV_FIRMWARE_CHIP_FAMILY_END:  // fall through
             case NV_FIRMWARE_CHIP_FAMILY_NULL:
@@ -122,15 +118,15 @@ static inline const char *nv_firmware_for_chip_family(
 
     return "";
 }
-#endif  // defined(NV_FIRMWARE_FOR_NAME)
+#endif  // defined(NV_FIRMWARE_PATH_FOR_FILENAME)
 
-// The includer may optionally define
-// NV_FIRMWARE_DECLARE_GSP(name)
+// The includer (presumably nv.c) may optionally define
+// NV_FIRMWARE_DECLARE_GSP_FILENAME(filename)
 // which will then be invoked (at the top-level) for each
-// gsp_* (but not gsp_log_*)
-#if defined(NV_FIRMWARE_DECLARE_GSP)
-NV_FIRMWARE_DECLARE_GSP("gsp_ga10x")
-NV_FIRMWARE_DECLARE_GSP("gsp_tu10x")
-#endif  // defined(NV_FIRMWARE_DECLARE_GSP)
+// gsp_*.bin (but not gsp_log_*.bin)
+#if defined(NV_FIRMWARE_DECLARE_GSP_FILENAME)
+NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_ga10x.bin")
+NV_FIRMWARE_DECLARE_GSP_FILENAME("gsp_tu10x.bin")
+#endif  // defined(NV_FIRMWARE_DECLARE_GSP_FILENAME)
 
-#endif  // NV_FIRMWARE_DECLARE_GSP
+#endif  // NV_FIRMWARE_DECLARE_GSP_FILENAME

kernel-open/common/inc/nv-linux.h

@@ -58,10 +58,14 @@
 #include <linux/version.h>
 #include <linux/utsname.h>
 
-#if LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0)
-// Version 4.4 is allowed, temporarily, although not officially supported.
-#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
-#error "This driver does not support kernels older than Linux 4.15!"
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
+#error "This driver does not support kernels older than 2.6.32!"
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 7, 0)
+#  define KERNEL_2_6
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
+#  define KERNEL_3
+#else
+#error "This driver does not support development kernels!"
 #endif
 
 #if defined (CONFIG_SMP) && !defined (__SMP__)
@@ -840,16 +844,16 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
 #define NV_PRINT_AT(nv_debug_level,at)                                           \
     {                                                                            \
         nv_printf(nv_debug_level,                                                \
-            "NVRM: VM: %s:%d: 0x%p, %d page(s), count = %d, "                    \
+            "NVRM: VM: %s:%d: 0x%p, %d page(s), count = %d, flags = 0x%08x, "    \
             "page_table = 0x%p\n",  __FUNCTION__, __LINE__, at,                  \
             at->num_pages, NV_ATOMIC_READ(at->usage_count),                      \
-            at->page_table);                                                     \
+            at->flags, at->page_table);                                          \
     }
 
 #define NV_PRINT_VMA(nv_debug_level,vma)                                                 \
     {                                                                                    \
         nv_printf(nv_debug_level,                                                        \
-            "NVRM: VM: %s:%d: 0x%lx - 0x%lx, 0x%08lx bytes @ 0x%016llx, 0x%p, 0x%p\n",    \
+            "NVRM: VM: %s:%d: 0x%lx - 0x%lx, 0x%08x bytes @ 0x%016llx, 0x%p, 0x%p\n",    \
             __FUNCTION__, __LINE__, vma->vm_start, vma->vm_end, NV_VMA_SIZE(vma),        \
             NV_VMA_OFFSET(vma), NV_VMA_PRIVATE(vma), NV_VMA_FILE(vma));                  \
     }
@@ -1082,8 +1086,6 @@ static inline void nv_kmem_ctor_dummy(void *arg)
         kmem_cache_destroy(kmem_cache);     \
     }
 
-#define NV_KMEM_CACHE_ALLOC_ATOMIC(kmem_cache)     \
-    kmem_cache_alloc(kmem_cache, GFP_ATOMIC)
 #define NV_KMEM_CACHE_ALLOC(kmem_cache)     \
     kmem_cache_alloc(kmem_cache, GFP_KERNEL)
 #define NV_KMEM_CACHE_FREE(ptr, kmem_cache) \
@@ -1110,23 +1112,6 @@ static inline void *nv_kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)
 #endif
 }
 
-static inline int nv_kmem_cache_alloc_stack_atomic(nvidia_stack_t **stack)
-{
-    nvidia_stack_t *sp = NULL;
-#if defined(NVCPU_X86_64)
-    if (rm_is_altstack_in_use())
-    {
-        sp = NV_KMEM_CACHE_ALLOC_ATOMIC(nvidia_stack_t_cache);
-        if (sp == NULL)
-            return -ENOMEM;
-        sp->size = sizeof(sp->stack);
-        sp->top = sp->stack + sp->size;
-    }
-#endif
-    *stack = sp;
-    return 0;
-}
-
 static inline int nv_kmem_cache_alloc_stack(nvidia_stack_t **stack)
 {
     nvidia_stack_t *sp = NULL;
@@ -1182,16 +1167,6 @@ typedef struct nvidia_pte_s {
     unsigned int    page_count;
 } nvidia_pte_t;
 
-#if defined(CONFIG_DMA_SHARED_BUFFER)
-/* Standard dma_buf-related information. */
-struct nv_dma_buf
-{
-    struct dma_buf *dma_buf;
-    struct dma_buf_attachment *dma_attach;
-    struct sg_table *sgt;
-};
-#endif // CONFIG_DMA_SHARED_BUFFER
-
 typedef struct nv_alloc_s {
     struct nv_alloc_s *next;
     struct device     *dev;

kernel-open/common/inc/nv-mm.h

@@ -29,17 +29,17 @@
 typedef int vm_fault_t;
 #endif
 
-/*
- * pin_user_pages()
- *
+/* pin_user_pages
  * Presence of pin_user_pages() also implies the presence of unpin-user_page().
- * Both were added in the v5.6.
+ * Both were added in the v5.6-rc1
  *
- * pin_user_pages() was added by commit eddb1c228f79
- * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6.
+ * pin_user_pages() was added by commit eddb1c228f7951d399240
+ * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6-rc1 (2020-01-30)
+ *
+ * Removed vmas parameter from pin_user_pages() by commit 40896a02751
+ * ("mm/gup: remove vmas parameter from pin_user_pages()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-17)
  *
- * Removed vmas parameter from pin_user_pages() by commit 4c630f307455
- * ("mm/gup: remove vmas parameter from pin_user_pages()") in v6.5.
  */
 
 #include <linux/mm.h>
@@ -63,28 +63,25 @@ typedef int vm_fault_t;
     #define NV_UNPIN_USER_PAGE put_page
 #endif // NV_PIN_USER_PAGES_PRESENT
 
-/*
- * get_user_pages()
+/* get_user_pages
  *
- * The 8-argument version of get_user_pages() was deprecated by commit
- * cde70140fed8 ("mm/gup: Overload get_user_pages() functions") in v4.6-rc1.
+ * The 8-argument version of get_user_pages was deprecated by commit
+ * (2016 Feb 12: cde70140fed8429acf7a14e2e2cbd3e329036653)for the non-remote case
  * (calling get_user_pages with current and current->mm).
  *
- * Completely moved to the 6 argument version of get_user_pages() by
- * commit c12d2da56d0e ("mm/gup: Remove the macro overload API migration
- * helpers from the get_user*() APIs") in v4.6-rc4.
+ * Completely moved to the 6 argument version of get_user_pages -
+ * 2016 Apr 4: c12d2da56d0e07d230968ee2305aaa86b93a6832
  *
- * write and force parameters were replaced with gup_flags by
- * commit 768ae309a961 ("mm: replace get_user_pages() write/force parameters
- * with gup_flags") in v4.9.
+ * write and force parameters were replaced with gup_flags by -
+ * 2016 Oct 12: 768ae309a96103ed02eb1e111e838c87854d8b51
  *
  * A 7-argument version of get_user_pages was introduced into linux-4.4.y by
- * commit 8e50b8b07f462 ("mm: replace get_user_pages() write/force parameters
- * with gup_flags") which cherry-picked the replacement of the write and
- * force parameters with gup_flags.
+ * commit 8e50b8b07f462ab4b91bc1491b1c91bd75e4ad40 which cherry-picked the
+ * replacement of the write and force parameters with gup_flags
  *
- * Removed vmas parameter from get_user_pages() by commit 54d020692b34
- * ("mm/gup: remove unused vmas parameter from get_user_pages()") in v6.5.
+ * Removed vmas parameter from get_user_pages() by commit 7bbf9c8c99
+ * ("mm/gup: remove unused vmas parameter from get_user_pages()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-17)
  *
  */
 
@@ -115,19 +112,18 @@ typedef int vm_fault_t;
     }
 #endif // NV_GET_USER_PAGES_HAS_ARGS_FLAGS
 
-/*
- * pin_user_pages_remote()
+/* pin_user_pages_remote
  *
- * pin_user_pages_remote() was added by commit eddb1c228f79
- * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6.
+ * pin_user_pages_remote() was added by commit eddb1c228f7951d399240
+ * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6 (2020-01-30)
  *
  * pin_user_pages_remote() removed 'tsk' parameter by commit
- * 64019a2e467a ("mm/gup: remove task_struct pointer for all gup code")
- * in v5.9.
+ * 64019a2e467a ("mm/gup: remove task_struct pointer for  all gup code")
+ * in v5.9-rc1 (2020-08-11). *
  *
  * Removed unused vmas parameter from pin_user_pages_remote() by commit
- * 0b295316b3a9 ("mm/gup: remove unused vmas parameter from
- * pin_user_pages_remote()") in v6.5.
+ * 83bcc2e132("mm/gup: remove unused vmas parameter from pin_user_pages_remote()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-14)
  *
  */
 
@@ -147,7 +143,7 @@ typedef int vm_fault_t;
 
 /*
  * get_user_pages_remote() was added by commit 1e9877902dc7
- * ("mm/gup: Introduce get_user_pages_remote()") in v4.6.
+ * ("mm/gup: Introduce get_user_pages_remote()") in v4.6 (2016-02-12).
  *
  * Note that get_user_pages_remote() requires the caller to hold a reference on
  * the task_struct (if non-NULL and if this API has tsk argument) and the mm_struct.
@@ -157,17 +153,19 @@ typedef int vm_fault_t;
  *
  * get_user_pages_remote() write/force parameters were replaced
  * with gup_flags by commit 9beae1ea8930 ("mm: replace get_user_pages_remote()
- * write/force parameters with gup_flags") in v4.9.
+ * write/force parameters with gup_flags") in v4.9 (2016-10-13).
  *
  * get_user_pages_remote() added 'locked' parameter by commit 5b56d49fc31d
- * ("mm: add locked parameter to get_user_pages_remote()") in v4.10.
+ * ("mm: add locked parameter to get_user_pages_remote()") in
+ * v4.10 (2016-12-14).
  *
  * get_user_pages_remote() removed 'tsk' parameter by
  * commit 64019a2e467a ("mm/gup: remove task_struct pointer for
- * all gup code") in v5.9.
+ * all gup code") in v5.9-rc1 (2020-08-11).
  *
- * Removed vmas parameter from get_user_pages_remote() by commit ca5e863233e8
- * ("mm/gup: remove vmas parameter from get_user_pages_remote()") in v6.5.
+ * Removed vmas parameter from get_user_pages_remote() by commit a4bde14d549 
+ * ("mm/gup: remove vmas parameter from get_user_pages_remote()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-14)
  *
  */
 

kernel-open/common/inc/nv.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -110,15 +110,15 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTPN,
     TEGRASOC_WHICH_CLK_DSIPLL_CLKOUTA,
     TEGRASOC_WHICH_CLK_SPPLL0_VCO,
+    TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN,
     TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTA,
     TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTB,
-    TEGRASOC_WHICH_CLK_SPPLL0_CLKOUTPN,
-    TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN,
-    TEGRASOC_WHICH_CLK_SPPLL0_DIV27,
-    TEGRASOC_WHICH_CLK_SPPLL1_DIV27,
     TEGRASOC_WHICH_CLK_SPPLL0_DIV10,
     TEGRASOC_WHICH_CLK_SPPLL0_DIV25,
+    TEGRASOC_WHICH_CLK_SPPLL0_DIV27,
     TEGRASOC_WHICH_CLK_SPPLL1_VCO,
+    TEGRASOC_WHICH_CLK_SPPLL1_CLKOUTPN,
+    TEGRASOC_WHICH_CLK_SPPLL1_DIV27,
     TEGRASOC_WHICH_CLK_VPLL0_REF,
     TEGRASOC_WHICH_CLK_VPLL0,
     TEGRASOC_WHICH_CLK_VPLL1,
@@ -132,7 +132,7 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_DSI_PIXEL,
     TEGRASOC_WHICH_CLK_PRE_SOR0,
     TEGRASOC_WHICH_CLK_PRE_SOR1,
-    TEGRASOC_WHICH_CLK_DP_LINKA_REF,
+    TEGRASOC_WHICH_CLK_DP_LINK_REF,
     TEGRASOC_WHICH_CLK_SOR_LINKA_INPUT,
     TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO,
     TEGRASOC_WHICH_CLK_SOR_LINKA_AFIFO_M,
@@ -143,7 +143,7 @@ typedef enum _TEGRASOC_WHICH_CLK
     TEGRASOC_WHICH_CLK_PLLHUB,
     TEGRASOC_WHICH_CLK_SOR0,
     TEGRASOC_WHICH_CLK_SOR1,
-    TEGRASOC_WHICH_CLK_SOR_PADA_INPUT,
+    TEGRASOC_WHICH_CLK_SOR_PAD_INPUT,
     TEGRASOC_WHICH_CLK_PRE_SF0,
     TEGRASOC_WHICH_CLK_SF0,
     TEGRASOC_WHICH_CLK_SF1,
@@ -332,9 +332,7 @@ typedef struct nv_soc_irq_info_s {
 
 #define NV_MAX_SOC_IRQS              6
 #define NV_MAX_DPAUX_NUM_DEVICES     4
-
-#define NV_MAX_SOC_DPAUX_NUM_DEVICES 2
-
+#define NV_MAX_SOC_DPAUX_NUM_DEVICES 2 // From SOC_DEV_MAPPING
 
 #define NV_IGPU_LEGACY_STALL_IRQ     70
 #define NV_IGPU_MAX_STALL_IRQS       3
@@ -497,6 +495,12 @@ typedef struct nv_state_t
     } iommus;
 } nv_state_t;
 
+// These define need to be in sync with defines in system.h
+#define OS_TYPE_LINUX   0x1
+#define OS_TYPE_FREEBSD 0x2
+#define OS_TYPE_SUNOS   0x3
+#define OS_TYPE_VMWARE  0x4
+
 #define NVFP_TYPE_NONE       0x0
 #define NVFP_TYPE_REFCOUNTED 0x1
 #define NVFP_TYPE_REGISTERED 0x2
@@ -605,15 +609,6 @@ typedef enum
     NV_POWER_STATE_RUNNING
 } nv_power_state_t;
 
-typedef struct
-{
-    const char *vidmem_power_status;
-    const char *dynamic_power_status;
-    const char *gc6_support;
-    const char *gcoff_support;
-    const char *s0ix_status;
-} nv_power_info_t;
-
 #define NV_PRIMARY_VGA(nv)      ((nv)->primary_vga)
 
 #define NV_IS_CTL_DEVICE(nv)    ((nv)->flags & NV_FLAG_CONTROL)
@@ -783,7 +778,7 @@ nv_state_t*  NV_API_CALL  nv_get_ctl_state       (void);
 
 void   NV_API_CALL  nv_set_dma_address_size      (nv_state_t *, NvU32 );
 
-NV_STATUS  NV_API_CALL  nv_alias_pages           (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, void **);
+NV_STATUS  NV_API_CALL  nv_alias_pages           (nv_state_t *, NvU32, NvU32, NvU32, NvU64, NvU64 *, void **);
 NV_STATUS  NV_API_CALL  nv_alloc_pages           (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **);
 NV_STATUS  NV_API_CALL  nv_free_pages            (nv_state_t *, NvU32, NvBool, NvU32, void *);
 
@@ -827,7 +822,6 @@ void   NV_API_CALL  nv_acpi_methods_init         (NvU32 *);
 void   NV_API_CALL  nv_acpi_methods_uninit       (void);
 
 NV_STATUS  NV_API_CALL  nv_acpi_method           (NvU32, NvU32, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
-NV_STATUS  NV_API_CALL  nv_acpi_d3cold_dsm_for_upstream_port (nv_state_t *, NvU8 *, NvU32, NvU32, NvU32 *);
 NV_STATUS  NV_API_CALL  nv_acpi_dsm_method       (nv_state_t *, NvU8 *, NvU32, NvBool, NvU32, void *, NvU16, NvU32 *, void *, NvU16 *);
 NV_STATUS  NV_API_CALL  nv_acpi_ddc_method       (nv_state_t *, void *, NvU32 *, NvBool);
 NV_STATUS  NV_API_CALL  nv_acpi_dod_method       (nv_state_t *, NvU32 *, NvU32 *);
@@ -889,6 +883,8 @@ void      NV_API_CALL nv_cap_drv_exit(void);
 NvBool    NV_API_CALL nv_is_gpu_accessible(nv_state_t *);
 NvBool    NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *);
 
+NvU32     NV_API_CALL nv_get_os_type(void);
+
 void      NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end);
 void      NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *);
 
@@ -994,10 +990,10 @@ NV_STATUS  NV_API_CALL  rm_p2p_init_mapping       (nvidia_stack_t *, NvU64, NvU6
 NV_STATUS  NV_API_CALL  rm_p2p_destroy_mapping    (nvidia_stack_t *, NvU64);
 NV_STATUS  NV_API_CALL  rm_p2p_get_pages          (nvidia_stack_t *, NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU8 **, void *);
 NV_STATUS  NV_API_CALL  rm_p2p_get_gpu_info       (nvidia_stack_t *, NvU64, NvU64, NvU8 **, void **);
-NV_STATUS  NV_API_CALL  rm_p2p_get_pages_persistent (nvidia_stack_t *,  NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *, void **);
+NV_STATUS  NV_API_CALL  rm_p2p_get_pages_persistent (nvidia_stack_t *,  NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *);
 NV_STATUS  NV_API_CALL  rm_p2p_register_callback  (nvidia_stack_t *, NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
 NV_STATUS  NV_API_CALL  rm_p2p_put_pages          (nvidia_stack_t *, NvU64, NvU32, NvU64, void *);
-NV_STATUS  NV_API_CALL  rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *, void *);
+NV_STATUS  NV_API_CALL  rm_p2p_put_pages_persistent(nvidia_stack_t *, void *, void *);
 NV_STATUS  NV_API_CALL  rm_p2p_dma_map_pages      (nvidia_stack_t *, nv_dma_device_t *, NvU8 *, NvU64, NvU32, NvU64 *, void **);
 NV_STATUS  NV_API_CALL  rm_dma_buf_dup_mem_handle (nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, NvHandle, void *, NvHandle, NvU64, NvU64, NvHandle *, void **);
 void       NV_API_CALL  rm_dma_buf_undup_mem_handle(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle);
@@ -1031,7 +1027,9 @@ void       NV_API_CALL rm_enable_dynamic_power_management(nvidia_stack_t *, nv_s
 NV_STATUS  NV_API_CALL rm_ref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
 void       NV_API_CALL rm_unref_dynamic_power(nvidia_stack_t *, nv_state_t *, nv_dynamic_power_mode_t);
 NV_STATUS  NV_API_CALL rm_transition_dynamic_power(nvidia_stack_t *, nv_state_t *, NvBool, NvBool *);
-void       NV_API_CALL rm_get_power_info(nvidia_stack_t *, nv_state_t *, nv_power_info_t *);
+const char* NV_API_CALL rm_get_vidmem_power_status(nvidia_stack_t *, nv_state_t *);
+const char* NV_API_CALL rm_get_dynamic_power_management_status(nvidia_stack_t *, nv_state_t *);
+const char* NV_API_CALL rm_get_gpu_gcx_support(nvidia_stack_t *, nv_state_t *, NvBool);
 
 void       NV_API_CALL rm_acpi_notify(nvidia_stack_t *, nv_state_t *, NvU32);
 void       NV_API_CALL rm_acpi_nvpcf_notify(nvidia_stack_t *);
@@ -1079,9 +1077,6 @@ NV_STATUS   NV_API_CALL rm_run_nano_timer_callback(nvidia_stack_t *, nv_state_t
 void        NV_API_CALL nv_cancel_nano_timer(nv_state_t *, nv_nano_timer_t *);
 void        NV_API_CALL nv_destroy_nano_timer(nv_state_t *nv, nv_nano_timer_t *);
 
-// Host1x specific functions.
-NV_STATUS NV_API_CALL nv_get_syncpoint_aperture(NvU32, NvU64 *, NvU64 *, NvU32 *);
-
 #if defined(NVCPU_X86_64)
 
 static inline NvU64 nv_rdtsc(void)

kernel-open/common/inc/nv_uvm_interface.h

@@ -592,6 +592,13 @@ void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel);
     Error codes:
       NV_ERR_GENERIC
       NV_ERR_NO_MEMORY
+      NV_ERR_INVALID_STATE
+      NV_ERR_NOT_SUPPORTED
+      NV_ERR_NOT_READY
+      NV_ERR_INVALID_LOCK_STATE
+      NV_ERR_INVALID_STATE
+      NV_ERR_NVSWITCH_FABRIC_NOT_READY
+      NV_ERR_NVSWITCH_FABRIC_FAILURE
 */
 NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuDeviceHandle device,
                                   UvmGpuCaps *caps);

kernel-open/common/inc/nv_uvm_types.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -39,13 +39,12 @@
 // are multiple BIG page sizes in RM. These defines are used as flags to "0"
 // should be OK when user is not sure which pagesize allocation it wants
 //
-#define UVM_PAGE_SIZE_DEFAULT    0x0ULL
-#define UVM_PAGE_SIZE_4K         0x1000ULL
-#define UVM_PAGE_SIZE_64K        0x10000ULL
-#define UVM_PAGE_SIZE_128K       0x20000ULL
-#define UVM_PAGE_SIZE_2M         0x200000ULL
-#define UVM_PAGE_SIZE_512M       0x20000000ULL
-#define UVM_PAGE_SIZE_256G       0x4000000000ULL
+#define UVM_PAGE_SIZE_DEFAULT    0x0
+#define UVM_PAGE_SIZE_4K         0x1000
+#define UVM_PAGE_SIZE_64K        0x10000
+#define UVM_PAGE_SIZE_128K       0x20000
+#define UVM_PAGE_SIZE_2M         0x200000
+#define UVM_PAGE_SIZE_512M       0x20000000
 
 //
 // When modifying flags, make sure they are compatible with the mirrored
@@ -396,7 +395,6 @@ typedef enum
     UVM_LINK_TYPE_NVLINK_2,
     UVM_LINK_TYPE_NVLINK_3,
     UVM_LINK_TYPE_NVLINK_4,
-    UVM_LINK_TYPE_NVLINK_5,
     UVM_LINK_TYPE_C2C,
 } UVM_LINK_TYPE;
 
@@ -568,6 +566,11 @@ typedef struct UvmGpuP2PCapsParams_tag
     // second, not taking into account the protocols overhead. The reported
     // bandwidth for indirect peers is zero.
     NvU32 totalLinkLineRateMBps;
+
+    // Out: True if the peers have a indirect link to communicate. On P9
+    // systems, this is true if peers are connected to different NPUs that
+    // forward the requests between them.
+    NvU32 indirectAccess      : 1;
 } UvmGpuP2PCapsParams;
 
 // Platform-wide information
@@ -592,10 +595,8 @@ typedef struct UvmGpuClientInfo_tag
 
 typedef enum
 {
-    UVM_GPU_CONF_COMPUTE_MODE_NONE,
-    UVM_GPU_CONF_COMPUTE_MODE_APM,
-    UVM_GPU_CONF_COMPUTE_MODE_HCC,
-    UVM_GPU_CONF_COMPUTE_MODE_COUNT
+    UVM_GPU_CONF_COMPUTE_MODE_NONE = 0,
+    UVM_GPU_CONF_COMPUTE_MODE_HCC = 2
 } UvmGpuConfComputeMode;
 
 typedef struct UvmGpuConfComputeCaps_tag
@@ -706,13 +707,6 @@ typedef struct UvmGpuInfo_tag
 
     // EGM base address to offset in the GMMU PTE entry for EGM mappings
     NvU64    egmBaseAddr;
-
-    // If connectedToSwitch is NV_TRUE,
-    // nvswitchEgmMemoryWindowStart tells the base address for the GPU's EGM memory in the
-    // NVSwitch address space. It is used when creating PTEs of GPU memory mappings
-    // to NVSwitch peers.
-    NvU64 nvswitchEgmMemoryWindowStart;
-
 } UvmGpuInfo;
 
 typedef struct UvmGpuFbInfo_tag

kernel-open/common/inc/nvkms-api-types.h

@@ -440,9 +440,9 @@ struct NvKmsLayerCapabilities {
     NvBool supportsWindowMode              :1;
 
     /*!
-     * Whether layer supports ICtCp pipe.
+     * Whether layer supports HDR pipe.
      */
-    NvBool supportsICtCp                   :1;
+    NvBool supportsHDR                     :1;
 
 
     /*!

kernel-open/common/inc/nvkms-kapi.h

@@ -158,17 +158,13 @@ struct NvKmsKapiDeviceResourcesInfo {
 
         NvU32 hasVideoMemory;
 
-        NvU32 numDisplaySemaphores;
-
         NvU8  genericPageKind;
 
         NvBool  supportsSyncpts;
-
-        NvBool requiresVrrSemaphores;
     } caps;
 
     NvU64 supportedSurfaceMemoryFormats[NVKMS_KAPI_LAYER_MAX];
-    NvBool supportsICtCp[NVKMS_KAPI_LAYER_MAX];
+    NvBool supportsHDR[NVKMS_KAPI_LAYER_MAX];
 };
 
 #define NVKMS_KAPI_LAYER_MASK(layerType) (1 << (layerType))
@@ -214,26 +210,18 @@ struct NvKmsKapiStaticDisplayInfo {
     NvU32 headMask;
 };
 
-struct NvKmsKapiSyncParams {
-    union {
-        struct {
-            /*!
-             * Possible syncpt use case in kapi.
-             * For pre-syncpt, use only id and value
-             * and for post-syncpt, use only fd.
-             */
-            NvU32   preSyncptId;
-            NvU32   preSyncptValue;
-        } syncpt;
+struct NvKmsKapiSyncpt {
 
-        struct {
-            NvU32 index;
-        } semaphore;
-    } u;
+    /*!
+     * Possible syncpt use case in kapi.
+     * For pre-syncpt, use only id and value
+     * and for post-syncpt, use only fd.
+     */
+    NvBool  preSyncptSpecified;
+    NvU32   preSyncptId;
+    NvU32   preSyncptValue;
 
-    NvBool preSyncptSpecified;
-    NvBool postSyncptRequested;
-    NvBool semaphoreSpecified;
+    NvBool  postSyncptRequested;
 };
 
 struct NvKmsKapiLayerConfig {
@@ -243,7 +231,7 @@ struct NvKmsKapiLayerConfig {
         NvU8 surfaceAlpha;
     } compParams;
     struct NvKmsRRParams rrParams;
-    struct NvKmsKapiSyncParams syncParams;
+    struct NvKmsKapiSyncpt syncptParams;
 
     struct {
         struct NvKmsHDRStaticMetadata val;
@@ -331,6 +319,7 @@ struct NvKmsKapiHeadModeSetConfig {
 
     struct {
         struct {
+            NvBool specified;
             NvU32 depth;
             NvU32 start;
             NvU32 end;
@@ -338,6 +327,7 @@ struct NvKmsKapiHeadModeSetConfig {
         } input;
 
         struct {
+            NvBool specified;
             NvBool enabled;
             struct NvKmsLutRamps *pRamps;
         } output;
@@ -352,8 +342,7 @@ struct NvKmsKapiHeadRequestedConfig {
         NvBool modeChanged         : 1;
         NvBool hdrInfoFrameChanged : 1;
         NvBool colorimetryChanged  : 1;
-        NvBool ilutChanged         : 1;
-        NvBool olutChanged         : 1;
+        NvBool lutChanged      : 1;
     } flags;
 
     struct NvKmsKapiCursorRequestedConfig cursorRequestedConfig;
@@ -379,8 +368,6 @@ struct NvKmsKapiHeadReplyConfig {
 
 struct NvKmsKapiModeSetReplyConfig {
     enum NvKmsFlipResult flipResult;
-    NvBool vrrFlip;
-    NvS32 vrrSemaphoreIndex;
     struct NvKmsKapiHeadReplyConfig
         headReplyConfig[NVKMS_KAPI_MAX_HEADS];
 };
@@ -1423,87 +1410,6 @@ struct NvKmsKapiFunctionsTable {
     (
         NvKmsKapiSuspendResumeCallbackFunc *function
     );
-
-    /*!
-     * Immediately reset the specified display semaphore to the pending state.
-     *
-     * Must be called prior to applying a mode set that utilizes the specified
-     * display semaphore for synchronization.
-     *
-     * \param [in] device         The device which will utilize the semaphore.
-     *
-     * \param [in] semaphoreIndex Index of the desired semaphore within the
-     *                            NVKMS semaphore pool. Must be less than
-     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
-     *                            for the specified device.
-     */
-    NvBool
-    (*resetDisplaySemaphore)
-    (
-        struct NvKmsKapiDevice *device,
-        NvU32 semaphoreIndex
-    );
-
-    /*!
-     * Immediately set the specified display semaphore to the displayable state.
-     *
-     * Must be called after \ref resetDisplaySemaphore to indicate a mode
-     * configuration change that utilizes the specified display semaphore for
-     * synchronization may proceed.
-     *
-     * \param [in] device         The device which will utilize the semaphore.
-     *
-     * \param [in] semaphoreIndex Index of the desired semaphore within the
-     *                            NVKMS semaphore pool. Must be less than
-     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
-     *                            for the specified device.
-     */
-    void
-    (*signalDisplaySemaphore)
-    (
-        struct NvKmsKapiDevice *device,
-        NvU32 semaphoreIndex
-    );
-
-    /*!
-     * Immediately cancel use of a display semaphore by resetting its value to
-     * its initial state.
-     *
-     * This can be used by clients to restore a semaphore to a consistent state
-     * when they have prepared it for use by previously calling
-     * \ref resetDisplaySemaphore() on it, but are then prevented from
-     * submitting the associated hardware operations to consume it due to the
-     * subsequent failure of some software or hardware operation.
-     *
-     * \param [in] device         The device which will utilize the semaphore.
-     *
-     * \param [in] semaphoreIndex Index of the desired semaphore within the
-     *                            NVKMS semaphore pool. Must be less than
-     *                            NvKmsKapiDeviceResourcesInfo::caps::numDisplaySemaphores
-     *                            for the specified device.
-     */
-    void
-    (*cancelDisplaySemaphore)
-    (
-        struct NvKmsKapiDevice *device,
-        NvU32 semaphoreIndex
-    );
-
-    /*!
-     * Signal the VRR semaphore at the specified index from the CPU.
-     * If device does not support VRR semaphores, this is a no-op.
-     * Returns true if signal is success or no-op, otherwise returns false.
-     *
-     * \param [in]  device  A device allocated using allocateDevice().
-     *
-     * \param [in]  index   The VRR semaphore index to be signalled.
-     */
-    NvBool
-    (*signalVrrSemaphore)
-    (
-        struct NvKmsKapiDevice *device,
-        NvS32 index
-    );
 };
 
 /** @} */

kernel-open/common/inc/nvmisc.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -67,9 +67,6 @@ extern "C" {
 #define NVBIT64(b)                NVBIT_TYPE(b, NvU64)
 #endif
 
-//Concatenate 2 32bit values to a 64bit value
-#define NV_CONCAT_32_TO_64(hi, lo) ((((NvU64)hi) << 32) | ((NvU64)lo))
-
 // Helper macro's for 32 bit bitmasks
 #define NV_BITMASK32_ELEMENT_SIZE            (sizeof(NvU32) << 3)
 #define NV_BITMASK32_IDX(chId)               (((chId) & ~(0x1F)) >> 5)  
@@ -497,23 +494,6 @@ do                                                      \
 //
 #define NV_TWO_N_MINUS_ONE(n) (((1ULL<<(n/2))<<((n+1)/2))-1)
 
-//
-// Create a 64b bitmask with n bits set
-// This is the same as ((1ULL<<n) - 1), but it doesn't overflow for n=64
-//
-// ...
-// n=-1, 0x0000000000000000
-// n=0,  0x0000000000000000
-// n=1,  0x0000000000000001
-// ...
-// n=63, 0x7FFFFFFFFFFFFFFF
-// n=64, 0xFFFFFFFFFFFFFFFF
-// n=65, 0xFFFFFFFFFFFFFFFF
-// n=66, 0xFFFFFFFFFFFFFFFF
-// ...
-//
-#define NV_BITMASK64(n) ((n<1) ? 0ULL : (NV_U64_MAX>>((n>64) ? 0 : (64-n))))
-
 #define DRF_READ_1WORD_BS(d,r,f,v) \
     ((DRF_EXTENT_MW(NV##d##r##f)<8)?DRF_READ_1BYTE_BS(NV##d##r##f,(v)): \
     ((DRF_EXTENT_MW(NV##d##r##f)<16)?DRF_READ_2BYTE_BS(NV##d##r##f,(v)): \
@@ -594,13 +574,6 @@ nvMaskPos32(const NvU32 mask, const NvU32 bitIdx)
     n32 = BIT_IDX_32(LOWESTBIT(n32));\
 }
 
-// Destructive operation on n64
-#define LOWESTBITIDX_64(n64)         \
-{                                    \
-    n64 = BIT_IDX_64(LOWESTBIT(n64));\
-}
-
-
 // Destructive operation on n32
 #define HIGHESTBITIDX_32(n32)   \
 {                               \
@@ -721,6 +694,42 @@ nvPrevPow2_U64(const NvU64 x )
     }                                                       \
 }
 
+//
+// Bug 4851259: Newly added functions must be hidden from certain HS-signed
+// ucode compilers to avoid signature mismatch.
+//
+#ifndef NVDEC_1_0
+/*!
+ * Returns the position of nth set bit in the given mask.
+ *
+ * Returns -1 if mask has fewer than n bits set.
+ *
+ * n is 0 indexed and has valid values 0..31 inclusive, so "zeroth" set bit is
+ * the first set LSB.
+ *
+ * Example, if mask = 0x000000F0u and n = 1, the return value will be 5.
+ * Example, if mask = 0x000000F0u and n = 4, the return value will be -1.
+ */
+static NV_FORCEINLINE NvS32
+nvGetNthSetBitIndex32(NvU32 mask, NvU32 n)
+{
+    NvU32 seenSetBitsCount = 0;
+    NvS32 index;
+    FOR_EACH_INDEX_IN_MASK(32, index, mask)
+    {
+        if (seenSetBitsCount == n)
+        {
+            return index;
+        }
+        ++seenSetBitsCount;
+    }
+    FOR_EACH_INDEX_IN_MASK_END;
+
+    return -1;
+}
+
+#endif // NVDEC_1_0
+
 //
 // Size to use when declaring variable-sized arrays
 //
@@ -945,11 +954,6 @@ static NV_FORCEINLINE void *NV_NVUPTR_TO_PTR(NvUPtr address)
 // Use (lo) if (b) is less than 64, and (hi) if >= 64.
 //
 #define NV_BIT_SET_128(b, lo, hi)              { nvAssert( (b) < 128 ); if ( (b) < 64 ) (lo) |= NVBIT64(b); else (hi) |= NVBIT64( b & 0x3F ); }
-//
-// Clear the bit at pos (b) for U64 which is < 128.
-// Use (lo) if (b) is less than 64, and (hi) if >= 64.
-//
-#define NV_BIT_CLEAR_128(b, lo, hi)            { nvAssert( (b) < 128 ); if ( (b) < 64 ) (lo) &= ~NVBIT64(b); else (hi) &= ~NVBIT64( b & 0x3F ); }
 
 // Get the number of elements the specified fixed-size array
 #define NV_ARRAY_ELEMENTS(x)                   ((sizeof(x)/sizeof((x)[0])))

kernel-open/common/inc/nvstatuscodes.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -152,9 +152,8 @@ NV_STATUS_CODE(NV_ERR_FABRIC_MANAGER_NOT_PRESENT,      0x0000007A, "Fabric Manag
 NV_STATUS_CODE(NV_ERR_ALREADY_SIGNALLED,               0x0000007B, "Semaphore Surface value already >= requested wait value")
 NV_STATUS_CODE(NV_ERR_QUEUE_TASK_SLOT_NOT_AVAILABLE,   0x0000007C, "PMU RPC error due to no queue slot available for this event")
 NV_STATUS_CODE(NV_ERR_KEY_ROTATION_IN_PROGRESS,        0x0000007D, "Operation not allowed as key rotation is in progress")
-NV_STATUS_CODE(NV_ERR_TEST_ONLY_CODE_NOT_ENABLED,      0x0000007E, "Test-only code path not enabled")
-NV_STATUS_CODE(NV_ERR_SECURE_BOOT_FAILED,              0x0000007F, "GFW secure boot failed")
-NV_STATUS_CODE(NV_ERR_INSUFFICIENT_ZBC_ENTRY,          0x00000080, "No more ZBC entry for the client")
+NV_STATUS_CODE(NV_ERR_NVSWITCH_FABRIC_NOT_READY,       0x00000081, "Nvswitch Fabric Status or Fabric Probe is not yet complete, caller needs to retry")
+NV_STATUS_CODE(NV_ERR_NVSWITCH_FABRIC_FAILURE,         0x00000082, "Nvswitch Fabric Probe failed")
 
 // Warnings:
 NV_STATUS_CODE(NV_WARN_HOT_SWITCH,                     0x00010001, "WARNING Hot switch")

kernel-open/common/inc/nvtypes.h

@@ -152,12 +152,6 @@ typedef   signed short     NvS16; /* -32768 to 32767                         */
      (((NvU32)(c) & 0xff) << 8)  | \
      (((NvU32)(d) & 0xff))))
 
-// Macro to build an NvU64 from two DWORDS, listed from msb to lsb
-#define NvU64_BUILD(a, b) \
-    ((NvU64)( \
-     (((NvU64)(a) & ~0U) << 32) | \
-     (((NvU64)(b) & ~0U))))
-
 #if NVTYPES_USE_STDINT
 typedef uint32_t           NvV32; /* "void": enumerated or multiple fields   */
 typedef uint32_t           NvU32; /* 0 to 4294967295                         */

kernel-open/common/inc/os-interface.h

@@ -151,7 +151,6 @@ void        NV_API_CALL  os_release_rwlock_read      (void *);
 void        NV_API_CALL  os_release_rwlock_write     (void *);
 NvBool      NV_API_CALL  os_semaphore_may_sleep      (void);
 NV_STATUS   NV_API_CALL  os_get_version_info         (os_version_info*);
-NV_STATUS   NV_API_CALL  os_get_is_openrm            (NvBool *);
 NvBool      NV_API_CALL  os_is_isr                   (void);
 NvBool      NV_API_CALL  os_pat_supported            (void);
 void        NV_API_CALL  os_dump_stack               (void);
@@ -219,6 +218,8 @@ extern NvU32 os_page_size;
 extern NvU64 os_page_mask;
 extern NvU8  os_page_shift;
 extern NvBool os_cc_enabled;
+extern NvBool os_cc_sev_snp_enabled;
+extern NvBool os_cc_snp_vtom_enabled;
 extern NvBool os_cc_tdx_enabled;
 extern NvBool os_dma_buf_enabled;
 extern NvBool os_imex_channel_is_supported;

kernel-open/conftest.sh

@@ -71,7 +71,7 @@ test_header_presence() {
     TEST_CFLAGS="-E -M $CFLAGS"
 
     file="$1"
-    file_define=NV_`echo $file | tr '/.-' '___' | tr 'a-z' 'A-Z'`_PRESENT
+    file_define=NV_`echo $file | tr '/.\-a-z' '___A-Z'`_PRESENT
 
     CODE="#include <$file>"
 
@@ -2523,6 +2523,22 @@ compile_test() {
             fi
         ;;
 
+        file_operations_fop_unsigned_offset_present)
+            #
+            # Determine if the FOP_UNSIGNED_OFFSET define is present.
+            #
+            # Added by commit 641bb4394f40 ("fs: move FMODE_UNSIGNED_OFFSET to
+            # fop_flags") in v6.12.
+            #
+            CODE="
+            #include <linux/fs.h>
+            int conftest_file_operations_fop_unsigned_offset_present(void) {
+                return FOP_UNSIGNED_OFFSET;
+            }"
+
+            compile_check_conftest "$CODE" "NV_FILE_OPERATIONS_FOP_UNSIGNED_OFFSET_PRESENT" "" "types"
+        ;;
+
         mm_context_t)
             #
             # Determine if the 'mm_context_t' data type is present
@@ -5102,6 +5118,42 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_CC_PLATFORM_PRESENT" "" "functions"
         ;;
 
+        cc_attr_guest_sev_snp)
+            #
+            # Determine if 'CC_ATTR_GUEST_SEV_SNP' is present.
+            #
+            # Added by commit aa5a461171f9 ("x86/mm: Extend cc_attr to
+            # include AMD SEV-SNP") in v5.19.
+            #
+            CODE="
+            #if defined(NV_LINUX_CC_PLATFORM_H_PRESENT)
+            #include <linux/cc_platform.h>
+            #endif
+
+            enum cc_attr cc_attributes = CC_ATTR_GUEST_SEV_SNP;
+            "
+
+            compile_check_conftest "$CODE" "NV_CC_ATTR_SEV_SNP" "" "types"
+        ;;
+
+        hv_get_isolation_type)
+            #
+            # Determine if 'hv_get_isolation_type()' is present.
+            # Added by commit faff44069ff5 ("x86/hyperv: Add Write/Read MSR
+            # registers via ghcb page") in v5.16.
+            #
+            CODE="
+            #if defined(NV_ASM_MSHYPERV_H_PRESENT)
+            #include <asm/mshyperv.h>
+            #endif
+            void conftest_hv_get_isolation_type(void) {
+                int i;
+                hv_get_isolation_type(i);
+            }"
+
+            compile_check_conftest "$CODE" "NV_HV_GET_ISOLATION_TYPE" "" "functions"
+        ;;
+
         drm_prime_pages_to_sg_has_drm_device_arg)
             #
             # Determine if drm_prime_pages_to_sg() has 'dev' argument.
@@ -5269,6 +5321,45 @@ compile_test() {
 
             compile_check_conftest "$CODE" "NV_FOLLOW_PFN_PRESENT" "" "functions"
         ;;
+
+        follow_pte_arg_vma)
+            #
+            # Determine if the first argument of follow_pte is
+            # mm_struct or vm_area_struct.
+            #
+            # The first argument was changed from mm_struct to vm_area_struct by
+            # commit 29ae7d96d166 ("mm: pass VMA instead of MM to follow_pte()")
+            #
+            CODE="
+            #include <linux/mm.h>
+
+            typeof(follow_pte) conftest_follow_pte_has_vma_arg;
+            int conftest_follow_pte_has_vma_arg(struct vm_area_struct *vma,
+                                                unsigned long address,
+                                                pte_t **ptep,
+                                                spinlock_t **ptl) {
+                return 0;
+            }"
+
+            compile_check_conftest "$CODE" "NV_FOLLOW_PTE_ARG1_VMA" "" "types"
+        ;;
+
+        ptep_get)
+            #
+            # Determine if ptep_get() is present.
+            #
+            # ptep_get() was added by commit 481e980a7c19
+            # ("mm: Allow arches to provide ptep_get()")
+            #
+            CODE="
+            #include <linux/mm.h>
+            void conftest_ptep_get(void) {
+                ptep_get();
+            }"
+
+            compile_check_conftest "$CODE" "NV_PTEP_GET_PRESENT" "" "functions"
+        ;;
+
         drm_plane_atomic_check_has_atomic_state_arg)
             #
             # Determine if drm_plane_helper_funcs::atomic_check takes 'state'
@@ -5531,6 +5622,23 @@ compile_test() {
             else
                 echo "#undef NV_OF_PROPERTY_READ_VARIABLE_U32_ARRAY_PRESENT" | append_conftest "functions"
             fi
+            ;;
+
+        module_import_ns_takes_string_literal)
+            #
+            # Determine if the MODULE_IMPORT_NS macro takes a string literal
+            # or constant.
+            #
+            # Commit cdd30ebb1b9f ("module: Convert symbol namespace to
+            # string literal") changed MODULE_IMPORT_NS to take a string
+            # literal in Linux kernel v6.13.
+            #
+            CODE="
+            #include <linux/module.h>
+
+            MODULE_IMPORT_NS(DMA_BUF);"
+
+            compile_check_conftest "$CODE" "NV_MODULE_IMPORT_NS_TAKES_STRING_LITERAL" "" "functions"
         ;;
 
         devm_of_platform_populate)
@@ -5554,8 +5662,7 @@ compile_test() {
 
         of_dma_configure)
             #
-            # Determine if of_dma_configure() function is present, and how
-            # many arguments it takes.
+            # Determine if of_dma_configure() function is present
             #
             # Added by commit 591c1ee465ce ("of: configure the platform
             # device dma parameters") in v3.16.  However, it was a static,
@@ -5565,69 +5672,17 @@ compile_test() {
             # commit 1f5c69aa51f9 ("of: Move of_dma_configure() to device.c
             # to help re-use") in v4.1.
             #
-            # It subsequently began taking a third parameter with commit
-            # 3d6ce86ee794 ("drivers: remove force dma flag from buses")
-            # in v4.18.
-            #
-
-            echo "$CONFTEST_PREAMBLE
+            CODE="
             #if defined(NV_LINUX_OF_DEVICE_H_PRESENT)
             #include <linux/of_device.h>
             #endif
-
             void conftest_of_dma_configure(void)
             {
                 of_dma_configure();
             }
-            " > conftest$$.c
+            "
 
-            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
-            rm -f conftest$$.c
-
-            if [ -f conftest$$.o ]; then
-                rm -f conftest$$.o
-
-                echo "#undef NV_OF_DMA_CONFIGURE_PRESENT" | append_conftest "functions"
-                echo "#undef NV_OF_DMA_CONFIGURE_ARGUMENT_COUNT" | append_conftest "functions"
-            else
-                echo "#define NV_OF_DMA_CONFIGURE_PRESENT" | append_conftest "functions"
-
-                echo "$CONFTEST_PREAMBLE
-                #if defined(NV_LINUX_OF_DEVICE_H_PRESENT)
-                #include <linux/of_device.h>
-                #endif
-
-                void conftest_of_dma_configure(void) {
-                    of_dma_configure(NULL, NULL, false);
-                }" > conftest$$.c
-
-                $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
-                rm -f conftest$$.c
-
-                if [ -f conftest$$.o ]; then
-                    rm -f conftest$$.o
-                    echo "#define NV_OF_DMA_CONFIGURE_ARGUMENT_COUNT 3" | append_conftest "functions"
-                    return
-                fi
-
-                echo "$CONFTEST_PREAMBLE
-                #if defined(NV_LINUX_OF_DEVICE_H_PRESENT)
-                #include <linux/of_device.h>
-                #endif
-
-                void conftest_of_dma_configure(void) {
-                    of_dma_configure(NULL, NULL);
-                }" > conftest$$.c
-
-                $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
-                rm -f conftest$$.c
-
-                if [ -f conftest$$.o ]; then
-                    rm -f conftest$$.o
-                    echo "#define NV_OF_DMA_CONFIGURE_ARGUMENT_COUNT 2" | append_conftest "functions"
-                    return
-                fi
-            fi
+            compile_check_conftest "$CODE" "NV_OF_DMA_CONFIGURE_PRESENT" "" "functions"
         ;;
 
         icc_get)
@@ -6253,6 +6308,32 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_NUM_REGISTERED_FB_PRESENT" "" "types"
         ;;
 
+        acpi_video_register_backlight)
+            #
+            # Determine if acpi_video_register_backlight() function is present
+            #
+            # acpi_video_register_backlight was added by commit 3dbc80a3e4c55c
+            # (ACPI: video: Make backlight class device registration a separate
+            # step (v2)) for v6.0 (2022-09-02).
+            # Note: the include directive for <linux/types> in this conftest is
+            # necessary in order to support kernels between commit 0b9f7d93ca61
+            # ("ACPI / i915: ignore firmware requests backlight change") for
+            # v3.16 (2014-07-07) and commit 3bd6bce369f5 ("ACPI / video: Port
+            # to new backlight interface selection API") for v4.2 (2015-07-16).
+            # Kernels within this range use the 'bool' type and the related
+            # 'false' value in <acpi/video.h> without first including the
+            # definitions of that type and value.
+            #
+            CODE="
+            #include <linux/types.h>
+            #include <acpi/video.h>
+            void conftest_acpi_video_register_backlight(void) {
+                acpi_video_register_backlight(0);
+            }"
+
+            compile_check_conftest "$CODE" "NV_ACPI_VIDEO_REGISTER_BACKLIGHT" "" "functions"
+        ;;
+
         acpi_video_backlight_use_native)
             #
             # Determine if acpi_video_backlight_use_native() function is present
@@ -6596,7 +6677,9 @@ compile_test() {
             # Determine whether drm_fbdev_generic_setup is present.
             #
             # Added by commit 9060d7f49376 ("drm/fb-helper: Finish the
-            # generic fbdev emulation") in v4.19.
+            # generic fbdev emulation") in v4.19. Removed by commit
+            # aae4682e5d66 ("drm/fbdev-generic: Convert to fbdev-ttm")
+            # in v6.11.
             #
             CODE="
             #include <drm/drm_fb_helper.h>
@@ -6608,6 +6691,105 @@ compile_test() {
             }"
 
             compile_check_conftest "$CODE" "NV_DRM_FBDEV_GENERIC_SETUP_PRESENT" "" "functions"
+            ;;
+
+        drm_fbdev_ttm_setup)
+            #
+            # Determine whether drm_fbdev_ttm_setup is present.
+            #
+            # Added by commit aae4682e5d66 ("drm/fbdev-generic:
+            # Convert to fbdev-ttm") in v6.11. Removed by commit
+            # 1000634477d8 ("drm/fbdev-ttm:Convert to client-setup") in v6.13.
+            #
+            CODE="
+            #include <drm/drm_fb_helper.h>
+            #if defined(NV_DRM_DRM_FBDEV_TTM_H_PRESENT)
+            #include <drm/drm_fbdev_ttm.h>
+            #endif
+            void conftest_drm_fbdev_ttm_setup(void) {
+                drm_fbdev_ttm_setup();
+            }"
+
+            compile_check_conftest "$CODE" "NV_DRM_FBDEV_TTM_SETUP_PRESENT" "" "functions"
+        ;;
+
+        drm_client_setup)
+            #
+            # Determine whether drm_client_setup is present.
+            #
+            # Added by commit d07fdf922592 ("drm/fbdev-ttm: Convert to
+            # client-setup") in v6.13 in drm/drm_client_setup.h, but then moved
+            # to drm/clients/drm_client_setup.h by commit b86711c6d6e2
+            # ("drm/client: Move public client header to clients/ subdirectory")
+            # in linux-next b86711c6d6e2.
+            #
+            CODE="
+            #include <drm/drm_fb_helper.h>
+            #if defined(NV_DRM_DRM_CLIENT_SETUP_H_PRESENT)
+            #include <drm/drm_client_setup.h>
+            #elif defined(NV_DRM_CLIENTS_DRM_CLIENT_SETUP_H_PRESENT)
+            #include <drm/clients/drm_client_setup.h>
+            #endif
+            void conftest_drm_client_setup(void) {
+                drm_client_setup();
+            }"
+
+            compile_check_conftest "$CODE" "NV_DRM_CLIENT_SETUP_PRESENT" "" "functions"
+        ;;
+
+        drm_output_poll_changed)
+            #
+            # Determine whether drm_mode_config_funcs.output_poll_changed
+            # callback is present
+            #
+            # Removed by commit 446d0f4849b1 ("drm: Remove struct
+            # drm_mode_config_funcs.output_poll_changed") in v6.12. Hotplug
+            # event support is handled through the fbdev emulation interface
+            # going forward.
+            #
+            CODE="
+            #if defined(NV_DRM_DRM_MODE_CONFIG_H_PRESENT)
+            #include <drm/drm_mode_config.h>
+            #else
+            #include <drm/drm_crtc.h>
+            #endif
+            int conftest_drm_output_poll_changed_available(void) {
+                return offsetof(struct drm_mode_config_funcs, output_poll_changed);
+            }"
+
+            compile_check_conftest "$CODE" "NV_DRM_OUTPUT_POLL_CHANGED_PRESENT" "" "types"
+        ;;
+
+        aperture_remove_conflicting_devices)
+            #
+            # Determine whether aperture_remove_conflicting_devices is present.
+            # 
+            # Added by commit 7283f862bd991 ("drm: Implement DRM aperture
+            # helpers under video/") in v6.0
+            CODE="
+            #if defined(NV_LINUX_APERTURE_H_PRESENT)
+            #include <linux/aperture.h>
+            #endif
+            void conftest_aperture_remove_conflicting_devices(void) {
+                aperture_remove_conflicting_devices();
+            }"
+            compile_check_conftest "$CODE" "NV_APERTURE_REMOVE_CONFLICTING_DEVICES_PRESENT" "" "functions"
+        ;;
+
+        aperture_remove_conflicting_pci_devices)
+            #
+            # Determine whether aperture_remove_conflicting_pci_devices is present.
+            #
+            # Added by commit 7283f862bd991 ("drm: Implement DRM aperture
+            # helpers under video/") in v6.0
+            CODE="
+            #if defined(NV_LINUX_APERTURE_H_PRESENT)
+            #include <linux/aperture.h>
+            #endif
+            void conftest_aperture_remove_conflicting_pci_devices(void) {
+                aperture_remove_conflicting_pci_devices();
+            }"
+            compile_check_conftest "$CODE" "NV_APERTURE_REMOVE_CONFLICTING_PCI_DEVICES_PRESENT" "" "functions"
         ;;
 
         drm_aperture_remove_conflicting_pci_framebuffers)
@@ -6703,17 +6885,17 @@ compile_test() {
             # This test is not complete and may return false positive.
             #
             CODE="
-	    #include <crypto/akcipher.h>
-	    #include <crypto/algapi.h>
-	    #include <crypto/ecc_curve.h>
-	    #include <crypto/ecdh.h>
-	    #include <crypto/hash.h>
-	    #include <crypto/internal/ecc.h>
-	    #include <crypto/kpp.h>
-	    #include <crypto/public_key.h>
-	    #include <crypto/sm3.h>
-	    #include <keys/asymmetric-type.h>
-	    #include <linux/crypto.h>
+            #include <crypto/akcipher.h>
+            #include <crypto/algapi.h>
+            #include <crypto/ecc_curve.h>
+            #include <crypto/ecdh.h>
+            #include <crypto/hash.h>
+            #include <crypto/internal/ecc.h>
+            #include <crypto/kpp.h>
+            #include <crypto/public_key.h>
+            #include <crypto/sm3.h>
+            #include <keys/asymmetric-type.h>
+            #include <linux/crypto.h>
             void conftest_crypto(void) {
                 struct shash_desc sd;
                 struct crypto_shash cs;
@@ -6723,6 +6905,47 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_CRYPTO_PRESENT" "" "symbols"
         ;;
 
+        crypto_akcipher_verify)
+            #
+            # Determine whether the crypto_akcipher_verify API is still present.
+            # It was removed by commit 6b34562 ('crypto: akcipher - Drop sign/verify operations')
+            # in v6.13-rc1 (2024-10-04).
+            #
+            # This test is dependent on the crypto conftest to determine whether crypto should be
+            # enabled at all. That means that if the kernel is old enough such that crypto_akcipher_verify
+            #
+            # The test merely checks for the presence of the API, as it assumes that if the API
+            # is no longer present, the new API to replace it (crypto_sig_verify) must be present.
+            # If the kernel version is too old to have crypto_akcipher_verify, it will fail the crypto
+            # conftest above and all crypto code will be compiled out.
+            #
+            CODE="
+            #include <crypto/akcipher.h>
+            #include <linux/crypto.h>
+            void conftest_crypto_akcipher_verify(void) {
+                (void)crypto_akcipher_verify;
+            }"
+
+            compile_check_conftest "$CODE" "NV_CRYPTO_AKCIPHER_VERIFY_PRESENT" "" "symbols"
+            ;;
+
+        ecc_digits_from_bytes)
+            #
+            # Determine whether ecc_digits_from_bytes is present.
+            # It was added in commit c6ab5c915da4 ('crypto: ecc - Prevent ecc_digits_from_bytes from
+            # reading too many bytes') in v6.10.
+            #
+            # This functionality is needed when crypto_akcipher_verify is not present.
+            #
+            CODE="
+            #include <crypto/internal/ecc.h>
+            void conftest_ecc_digits_from_bytes(void) {
+                (void)ecc_digits_from_bytes;
+            }"
+
+            compile_check_conftest "$CODE" "NV_ECC_DIGITS_FROM_BYTES_PRESENT" "" "symbols"
+        ;;
+
         mempolicy_has_unified_nodes)
             #
             # Determine if the 'mempolicy' structure has
@@ -6848,45 +7071,12 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_DRM_MODE_CREATE_DP_COLORSPACE_PROPERTY_HAS_SUPPORTED_COLORSPACES_ARG" "" "types"
         ;;
 
-        drm_syncobj_features_present)
-            # Determine if DRIVER_SYNCOBJ and DRIVER_SYNCOBJ_TIMELINE DRM
-            # driver features are present. Timeline DRM synchronization objects
-            # may only be used if both of these are supported by the driver.
-            #
-            # DRIVER_SYNCOBJ_TIMELINE Added by commit 060cebb20cdb ("drm:
-            # introduce a capability flag for syncobj timeline support") in
-            # v5.2
-            #
-            # DRIVER_SYNCOBJ Added by commit e9083420bbac ("drm: introduce
-            # sync objects (v4)") in v4.12
-            CODE="
-            #if defined(NV_DRM_DRM_DRV_H_PRESENT)
-            #include <drm/drm_drv.h>
-            #endif
-            int features = DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE;"
-
-            compile_check_conftest "$CODE" "NV_DRM_SYNCOBJ_FEATURES_PRESENT" "" "types"
-        ;;
-
-        stack_trace)
-            # Determine if functions stack_trace_{save,print} are present.
-            # Added by commit e9b98e162 ("stacktrace: Provide helpers for
-            # common stack trace operations") in v5.2.
-            CODE="
-            #include <linux/stacktrace.h>
-            void conftest_stack_trace(void) {
-                stack_trace_save();
-                stack_trace_print();
-            }"
-
-            compile_check_conftest "$CODE" "NV_STACK_TRACE_PRESENT" "" "functions"
-        ;;
-
         drm_unlocked_ioctl_flag_present)
             # Determine if DRM_UNLOCKED IOCTL flag is present.
             #
             # DRM_UNLOCKED was removed by commit 2798ffcc1d6a ("drm: Remove
-            # locking for legacy ioctls and DRM_UNLOCKED") in v6.8.
+            # locking for legacy ioctls and DRM_UNLOCKED") in Linux
+            # next-20231208.
             #
             # DRM_UNLOCKED definition was moved from drmP.h to drm_ioctl.h by
             # commit 2640981f3600 ("drm: document drm_ioctl.[hc]") in v4.12.
@@ -6902,92 +7092,72 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_DRM_UNLOCKED_IOCTL_FLAG_PRESENT" "" "types"
         ;;
 
-        fault_flag_remote_present)
-            # Determine if FAULT_FLAG_REMOTE is present in the kernel, either
-            # as a define or an enum
+    folio_test_swapcache)
             #
-            # FAULT_FLAG_REMOTE define added by Kernel commit 1b2ee1266ea6
-            # ("mm/core: Do not enforce PKEY permissions on remote mm access")
-            # in v4.6
-            # FAULT_FLAG_REMOTE changed from define to enum by Kernel commit
-            # da2f5eb3d344 ("mm/doc: turn fault flags into an enum") in v5.13
-            # FAULT_FLAG_REMOTE moved from `mm.h` to `mm_types.h` by Kernel
-            # commit 36090def7bad ("mm: move tlb_flush_pending inline helpers
-            # to mm_inline.h") in v5.17
+            # Determine if the folio_test_swapcache() function is present.
+            #
+            # folio_test_swapcache() was exported by commit d389a4a811551 ("mm:
+            # Add folio flag manipulation functions") in v5.16.
             #
             CODE="
-            #include <linux/mm.h>
-            int fault_flag_remote = FAULT_FLAG_REMOTE;
-            "
+            #include <linux/page-flags.h>
+            void conftest_folio_test_swapcache(void) {
+                folio_test_swapcache();
+            }"
 
-            compile_check_conftest "$CODE" "NV_MM_HAS_FAULT_FLAG_REMOTE" "" "types"
+            compile_check_conftest "$CODE" "NV_FOLIO_TEST_SWAPCACHE_PRESENT" "" "functions"
         ;;
 
-        drm_framebuffer_obj_present)
+
+        drm_driver_has_date)
             #
-            # Determine if the drm_framebuffer struct has an obj member.
+            # Determine if the 'drm_driver' structure has a 'date' field.
             #
-            # Added by commit 4c3dbb2c312c ("drm: Add GEM backed framebuffer
-            # library") in v4.14.
+            # Removed by commit cb2e1c2136f7 ("drm: remove driver date from
+            # struct drm_driver and all drivers") in linux-next, expected in
+            # v6.14.
             #
             CODE="
             #if defined(NV_DRM_DRMP_H_PRESENT)
             #include <drm/drmP.h>
             #endif
 
-            #if defined(NV_DRM_DRM_FRAMEBUFFER_H_PRESENT)
-            #include <drm/drm_framebuffer.h>
+            #if defined(NV_DRM_DRM_DRV_H_PRESENT)
+            #include <drm/drm_drv.h>
             #endif
 
-            int conftest_drm_framebuffer_obj_present(void) {
-                return offsetof(struct drm_framebuffer, obj);
+            int conftest_drm_driver_has_date(void) {
+                return offsetof(struct drm_driver, date);
             }"
 
-            compile_check_conftest "$CODE" "NV_DRM_FRAMEBUFFER_OBJ_PRESENT" "" "types"
+            compile_check_conftest "$CODE" "NV_DRM_DRIVER_HAS_DATE" "" "types"
         ;;
 
-        drm_color_ctm_3x4_present)
-            # Determine if struct drm_color_ctm_3x4 is present.
+        drm_connector_helper_funcs_mode_valid_has_const_mode_arg)
+            #
+            # Determine if the 'mode' pointer argument is const in
+            # drm_connector_helper_funcs::mode_valid.
+            #
+            # The 'mode' pointer argument in
+            # drm_connector_helper_funcs::mode_valid was made const by commit
+            # 26d6fd81916e ("drm/connector: make mode_valid take a const struct
+            # drm_display_mode") in linux-next, expected in v6.15.
             #
-            # struct drm_color_ctm_3x4 was added by commit 6872a189be50
-            # ("drm/amd/display: Add 3x4 CTM support for plane CTM") in v6.8.
             CODE="
-            #include <uapi/drm/drm_mode.h>
-            struct drm_color_ctm_3x4 ctm;"
-
-            compile_check_conftest "$CODE" "NV_DRM_COLOR_CTM_3X4_PRESENT" "" "types"
-        ;;
-
-        drm_color_lut)
-            # Determine if struct drm_color_lut is present.
-            #
-            # struct drm_color_lut was added by commit 5488dc16fde7
-            # ("drm: introduce pipe color correction properties") in v4.6.
-            CODE="
-            #include <uapi/drm/drm_mode.h>
-            struct drm_color_lut lut;"
-
-            compile_check_conftest "$CODE" "NV_DRM_COLOR_LUT_PRESENT" "" "types"
-        ;;
-
-        drm_property_blob_put)
-            #
-            # Determine if function drm_property_blob_put() is present.
-            #
-            # Added by commit 6472e5090be7 ("drm: Introduce
-            # drm_property_blob_{get,put}()") v4.12, when it replaced
-            # drm_property_unreference_blob().
-            #
-
-            CODE="
-            #if defined(NV_DRM_DRM_PROPERTY_H_PRESENT)
-            #include <drm/drm_property.h>
+            #if defined(NV_DRM_DRM_ATOMIC_HELPER_H_PRESENT)
+            #include <drm/drm_atomic_helper.h>
             #endif
-            void conftest_drm_property_blob_put(void) {
-                drm_property_blob_put();
-            }"
 
-            compile_check_conftest "$CODE" "NV_DRM_PROPERTY_BLOB_PUT_PRESENT" "" "functions"
+            static int conftest_drm_connector_mode_valid(struct drm_connector *connector,
+                                                         const struct drm_display_mode *mode) {
+                return 0;
+            }
+
+            const struct drm_connector_helper_funcs conftest_drm_connector_helper_funcs = {
+                .mode_valid = conftest_drm_connector_mode_valid,
+            };"
+
+            compile_check_conftest "$CODE" "NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG" "" "types"
         ;;
 
         # When adding a new conftest entry, please use the correct format for

kernel-open/header-presence-tests.mk

@@ -15,6 +15,8 @@ NV_HEADER_PRESENCE_TESTS = \
   drm/drm_atomic_uapi.h \
   drm/drm_drv.h \
   drm/drm_fbdev_generic.h \
+  drm/drm_fbdev_ttm.h \
+  drm/drm_client_setup.h \
   drm/drm_framebuffer.h \
   drm/drm_connector.h \
   drm/drm_probe_helper.h \
@@ -28,11 +30,11 @@ NV_HEADER_PRESENCE_TESTS = \
   drm/drm_device.h \
   drm/drm_mode_config.h \
   drm/drm_modeset_lock.h \
-  drm/drm_property.h \
   dt-bindings/interconnect/tegra_icc_id.h \
   generated/autoconf.h \
   generated/compile.h \
   generated/utsrelease.h \
+  linux/aperture.h \
   linux/efi.h \
   linux/kconfig.h \
   linux/platform/tegra/mc_utils.h \
@@ -53,7 +55,6 @@ NV_HEADER_PRESENCE_TESTS = \
   linux/dma-resv.h \
   soc/tegra/chip-id.h \
   soc/tegra/fuse.h \
-  soc/tegra/fuse-helper.h \
   soc/tegra/tegra_bpmp.h \
   video/nv_internal.h \
   linux/platform/tegra/dce/dce-client-ipc.h \
@@ -99,5 +100,7 @@ NV_HEADER_PRESENCE_TESTS = \
   linux/sync_file.h \
   linux/cc_platform.h \
   asm/cpufeature.h \
-  linux/mpi.h
+  linux/mpi.h \
+  asm/mshyperv.h \
+  crypto/sig.h
 

kernel-open/nvidia-drm/nv-kthread-q.c

@@ -201,7 +201,7 @@ static struct task_struct *thread_create_on_node(int (*threadfn)(void *data),
 
         // Ran out of attempts - return thread even if its stack may not be
         // allocated on the preferred node
-        if ((i == (attempts - 1)))
+        if (i == (attempts - 1))
             break;
 
         // Get the NUMA node where the first page of the stack is resident. If

kernel-open/nvidia-drm/nvidia-drm-conftest.h

@@ -62,6 +62,17 @@
 #undef NV_DRM_FENCE_AVAILABLE
 #endif
 
+#if defined(NV_DRM_CLIENT_SETUP_PRESENT) &&                                    \
+    (defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT) ||   \
+     defined(NV_APERTURE_REMOVE_CONFLICTING_PCI_DEVICES_PRESENT))
+// XXX remove dependency on DRM_TTM_HELPER by implementing nvidia-drm's own
+// .fbdev_probe callback that uses NVKMS kapi
+#if IS_ENABLED(CONFIG_DRM_TTM_HELPER)
+#define NV_DRM_FBDEV_AVAILABLE
+#define NV_DRM_CLIENT_AVAILABLE
+#endif
+#endif
+
 /*
  * We can support color management if either drm_helper_crtc_enable_color_mgmt()
  * or drm_crtc_enable_color_mgmt() exist.

kernel-open/nvidia-drm/nvidia-drm-connector.c

@@ -314,7 +314,11 @@ static int nv_drm_connector_get_modes(struct drm_connector *connector)
 }
 
 static int nv_drm_connector_mode_valid(struct drm_connector    *connector,
+#if defined(NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG)
+                                       const struct drm_display_mode *mode)
+#else
                                        struct drm_display_mode *mode)
+#endif
 {
     struct drm_device *dev = connector->dev;
     struct nv_drm_device *nv_dev = to_nv_device(dev);

kernel-open/nvidia-drm/nvidia-drm-crtc.c

@@ -42,6 +42,12 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+#include <linux/nvhost.h>
+#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)            
+#include <linux/host1x-next.h>
+#endif
+
 #if defined(NV_DRM_DRM_COLOR_MGMT_H_PRESENT)
 #include <drm/drm_color_mgmt.h>
 #endif
@@ -170,10 +176,12 @@ cursor_plane_req_config_update(struct drm_plane *plane,
         return;
     }
 
-    memset(req_config, 0, sizeof(*req_config));
-    req_config->surface = to_nv_framebuffer(plane_state->fb)->pSurface;
-    req_config->dstX = plane_state->crtc_x;
-    req_config->dstY = plane_state->crtc_y;
+    *req_config = (struct NvKmsKapiCursorRequestedConfig) {
+        .surface = to_nv_framebuffer(plane_state->fb)->pSurface,
+
+        .dstX = plane_state->crtc_x,
+        .dstY = plane_state->crtc_y,
+    };
 
 #if defined(NV_DRM_ALPHA_BLENDING_AVAILABLE)
     if (plane->blend_mode_property != NULL && plane->alpha_property != NULL) {
@@ -258,6 +266,7 @@ plane_req_config_update(struct drm_plane *plane,
 {
     struct nv_drm_plane *nv_plane = to_nv_plane(plane);
     struct NvKmsKapiLayerConfig old_config = req_config->config;
+    struct nv_drm_device *nv_dev = to_nv_device(plane->dev);
     struct nv_drm_plane_state *nv_drm_plane_state =
         to_nv_drm_plane_state(plane_state);
 
@@ -266,22 +275,24 @@ plane_req_config_update(struct drm_plane *plane,
         return 0;
     }
 
-    memset(req_config, 0, sizeof(*req_config));
+    *req_config = (struct NvKmsKapiLayerRequestedConfig) {
+        .config = {
+            .surface = to_nv_framebuffer(plane_state->fb)->pSurface,
 
-    req_config->config.surface = to_nv_framebuffer(plane_state->fb)->pSurface;
+            /* Source values are 16.16 fixed point */
+            .srcX = plane_state->src_x >> 16,
+            .srcY = plane_state->src_y >> 16,
+            .srcWidth  = plane_state->src_w >> 16,
+            .srcHeight = plane_state->src_h >> 16,
 
-    /* Source values are 16.16 fixed point */
-    req_config->config.srcX = plane_state->src_x >> 16;
-    req_config->config.srcY = plane_state->src_y >> 16;
-    req_config->config.srcWidth  = plane_state->src_w >> 16;
-    req_config->config.srcHeight = plane_state->src_h >> 16;
+            .dstX = plane_state->crtc_x,
+            .dstY = plane_state->crtc_y,
+            .dstWidth  = plane_state->crtc_w,
+            .dstHeight = plane_state->crtc_h,
 
-    req_config->config.dstX = plane_state->crtc_x;
-    req_config->config.dstY = plane_state->crtc_y;
-    req_config->config.dstWidth  = plane_state->crtc_w;
-    req_config->config.dstHeight = plane_state->crtc_h;
-
-    req_config->config.csc = old_config.csc;
+            .csc = old_config.csc
+        },
+    };
 
 #if defined(NV_DRM_ROTATION_AVAILABLE)
     /*
@@ -385,16 +396,49 @@ plane_req_config_update(struct drm_plane *plane,
     req_config->config.inputColorSpace =
         nv_drm_plane_state->input_colorspace;
 
-    req_config->config.syncParams.preSyncptSpecified = false;
-    req_config->config.syncParams.postSyncptRequested = false;
-    req_config->config.syncParams.semaphoreSpecified = false;
+    req_config->config.syncptParams.preSyncptSpecified = false;
+    req_config->config.syncptParams.postSyncptRequested = false;
 
-    if (nv_drm_plane_state->fd_user_ptr) {
-        if (to_nv_device(plane->dev)->supportsSyncpts) {
-            req_config->config.syncParams.postSyncptRequested = true;
-        } else {
+    if (plane_state->fence != NULL || nv_drm_plane_state->fd_user_ptr) {
+        if (!nv_dev->supportsSyncpts) {
             return -1;
         }
+
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+#if defined(NV_NVHOST_DMA_FENCE_UNPACK_PRESENT)
+        if (plane_state->fence != NULL) {
+            int ret = nvhost_dma_fence_unpack(
+                          plane_state->fence,
+                          &req_config->config.syncptParams.preSyncptId,
+                          &req_config->config.syncptParams.preSyncptValue);
+            if (ret != 0) {
+                return ret;
+            }
+            req_config->config.syncptParams.preSyncptSpecified = true;
+        }
+#endif
+
+        if (nv_drm_plane_state->fd_user_ptr) {
+            req_config->config.syncptParams.postSyncptRequested = true;
+        }           
+#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)            
+        if (plane_state->fence != NULL) {            
+            int ret = host1x_fence_extract(            
+                      plane_state->fence,            
+                      &req_config->config.syncptParams.preSyncptId,            
+                      &req_config->config.syncptParams.preSyncptValue);            
+            if (ret != 0) {            
+                return ret;            
+            }            
+            req_config->config.syncptParams.preSyncptSpecified = true;            
+        }            
+
+        if (nv_drm_plane_state->fd_user_ptr) {            
+            req_config->config.syncptParams.postSyncptRequested = true;            
+        }
+#else
+        return -1;
+#endif
     }
 
 #if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
@@ -644,7 +688,9 @@ static int nv_drm_plane_atomic_set_property(
         to_nv_drm_plane_state(state);
 
     if (property == nv_dev->nv_out_fence_property) {
-        nv_drm_plane_state->fd_user_ptr = (void __user *)(uintptr_t)(val);
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+        nv_drm_plane_state->fd_user_ptr = u64_to_user_ptr(val);
+#endif
         return 0;
     } else if (property == nv_dev->nv_input_colorspace_property) {
         nv_drm_plane_state->input_colorspace = val;
@@ -817,7 +863,7 @@ __nv_drm_atomic_helper_crtc_destroy_state(struct drm_crtc *crtc,
 #endif
 }
 
-static inline bool nv_drm_crtc_duplicate_req_head_modeset_config(
+static inline void nv_drm_crtc_duplicate_req_head_modeset_config(
     const struct NvKmsKapiHeadRequestedConfig *old,
     struct NvKmsKapiHeadRequestedConfig *new)
 {
@@ -829,41 +875,15 @@ static inline bool nv_drm_crtc_duplicate_req_head_modeset_config(
      * there is no change in new configuration yet with respect
      * to older one!
      */
-    memset(new, 0, sizeof(*new));
-    new->modeSetConfig = old->modeSetConfig;
+    *new = (struct NvKmsKapiHeadRequestedConfig) {
+        .modeSetConfig = old->modeSetConfig,
+    };
 
     for (i = 0; i < ARRAY_SIZE(old->layerRequestedConfig); i++) {
-        new->layerRequestedConfig[i].config =
-            old->layerRequestedConfig[i].config;
+        new->layerRequestedConfig[i] = (struct NvKmsKapiLayerRequestedConfig) {
+            .config = old->layerRequestedConfig[i].config,
+        };
     }
-
-    if (old->modeSetConfig.lut.input.pRamps) {
-        new->modeSetConfig.lut.input.pRamps =
-            nv_drm_calloc(1, sizeof(*new->modeSetConfig.lut.input.pRamps));
-
-        if (!new->modeSetConfig.lut.input.pRamps) {
-            return false;
-        }
-        *new->modeSetConfig.lut.input.pRamps =
-            *old->modeSetConfig.lut.input.pRamps;
-    }
-    if (old->modeSetConfig.lut.output.pRamps) {
-        new->modeSetConfig.lut.output.pRamps =
-            nv_drm_calloc(1, sizeof(*new->modeSetConfig.lut.output.pRamps));
-
-        if (!new->modeSetConfig.lut.output.pRamps) {
-            /*
-             * new->modeSetConfig.lut.input.pRamps is either NULL or it was
-             * just allocated
-             */
-            nv_drm_free(new->modeSetConfig.lut.input.pRamps);
-            new->modeSetConfig.lut.input.pRamps = NULL;
-            return false;
-        }
-        *new->modeSetConfig.lut.output.pRamps =
-            *old->modeSetConfig.lut.output.pRamps;
-    }
-    return true;
 }
 
 static inline struct nv_drm_crtc_state *nv_drm_crtc_state_alloc(void)
@@ -935,24 +955,17 @@ nv_drm_atomic_crtc_duplicate_state(struct drm_crtc *crtc)
         return NULL;
     }
 
+    __drm_atomic_helper_crtc_duplicate_state(crtc, &nv_state->base);
+
     INIT_LIST_HEAD(&nv_state->nv_flip->list_entry);
     INIT_LIST_HEAD(&nv_state->nv_flip->deferred_flip_list);
 
-    /*
-     * nv_drm_crtc_duplicate_req_head_modeset_config potentially allocates
-     * nv_state->req_config.modeSetConfig.lut.{in,out}put.pRamps, so they should
-     * be freed in any following failure paths.
-     */
-    if (!nv_drm_crtc_duplicate_req_head_modeset_config(
-             &(to_nv_crtc_state(crtc->state)->req_config),
-             &nv_state->req_config)) {
+    nv_drm_crtc_duplicate_req_head_modeset_config(
+        &(to_nv_crtc_state(crtc->state)->req_config),
+        &nv_state->req_config);
 
-        nv_drm_free(nv_state->nv_flip);
-        nv_drm_free(nv_state);
-        return NULL;
-    }
-
-    __drm_atomic_helper_crtc_duplicate_state(crtc, &nv_state->base);
+    nv_state->ilut_ramps = NULL;
+    nv_state->olut_ramps = NULL;
 
     return &nv_state->base;
 }
@@ -977,8 +990,8 @@ static void nv_drm_atomic_crtc_destroy_state(struct drm_crtc *crtc,
 
     __nv_drm_atomic_helper_crtc_destroy_state(crtc, &nv_state->base);
 
-    nv_drm_free(nv_state->req_config.modeSetConfig.lut.input.pRamps);
-    nv_drm_free(nv_state->req_config.modeSetConfig.lut.output.pRamps);
+    nv_drm_free(nv_state->ilut_ramps);
+    nv_drm_free(nv_state->olut_ramps);
 
     nv_drm_free(nv_state);
 }
@@ -1061,82 +1074,94 @@ static int color_mgmt_config_set_luts(struct nv_drm_crtc_state *nv_crtc_state,
      * According to the comment in the Linux kernel's
      * drivers/gpu/drm/drm_color_mgmt.c, if either property is NULL, that LUT
      * needs to be changed to a linear LUT
-     *
-     * On failure, any LUT ramps allocated in this function are freed when the
-     * subsequent atomic state cleanup calls nv_drm_atomic_crtc_destroy_state.
      */
 
+    req_config->flags.lutChanged = NV_TRUE;
     if (crtc_state->degamma_lut) {
         struct drm_color_lut *degamma_lut = NULL;
         uint64_t degamma_len = 0;
 
-        if (!modeset_config->lut.input.pRamps) {
-            modeset_config->lut.input.pRamps =
-                nv_drm_calloc(1, sizeof(*modeset_config->lut.input.pRamps));
-            if (!modeset_config->lut.input.pRamps) {
-                return -ENOMEM;
-            }
+        nv_crtc_state->ilut_ramps = nv_drm_calloc(1, sizeof(*nv_crtc_state->ilut_ramps));
+        if (!nv_crtc_state->ilut_ramps) {
+            ret = -ENOMEM;
+            goto fail;
         }
 
         degamma_lut = (struct drm_color_lut *)crtc_state->degamma_lut->data;
         degamma_len = crtc_state->degamma_lut->length /
                       sizeof(struct drm_color_lut);
 
-        if ((ret = color_mgmt_config_copy_lut(modeset_config->lut.input.pRamps,
+        if ((ret = color_mgmt_config_copy_lut(nv_crtc_state->ilut_ramps,
                                               degamma_lut,
                                               degamma_len)) != 0) {
-            return ret;
+            goto fail;
         }
 
+        modeset_config->lut.input.specified = NV_TRUE;
         modeset_config->lut.input.depth     = 30; /* specify the full LUT */
         modeset_config->lut.input.start     = 0;
         modeset_config->lut.input.end       = degamma_len - 1;
+        modeset_config->lut.input.pRamps    = nv_crtc_state->ilut_ramps;
     } else {
         /* setting input.end to 0 is equivalent to disabling the LUT, which
          * should be equivalent to a linear LUT */
+        modeset_config->lut.input.specified = NV_TRUE;
         modeset_config->lut.input.depth     = 30; /* specify the full LUT */
         modeset_config->lut.input.start     = 0;
         modeset_config->lut.input.end       = 0;
-
-        nv_drm_free(modeset_config->lut.input.pRamps);
         modeset_config->lut.input.pRamps    = NULL;
+
     }
-    req_config->flags.ilutChanged = NV_TRUE;
 
     if (crtc_state->gamma_lut) {
         struct drm_color_lut *gamma_lut = NULL;
         uint64_t gamma_len = 0;
 
-        if (!modeset_config->lut.output.pRamps) {
-            modeset_config->lut.output.pRamps =
-                nv_drm_calloc(1, sizeof(*modeset_config->lut.output.pRamps));
-            if (!modeset_config->lut.output.pRamps) {
-                return -ENOMEM;
-            }
+        nv_crtc_state->olut_ramps = nv_drm_calloc(1, sizeof(*nv_crtc_state->olut_ramps));
+        if (!nv_crtc_state->olut_ramps) {
+            ret = -ENOMEM;
+            goto fail;
         }
 
         gamma_lut = (struct drm_color_lut *)crtc_state->gamma_lut->data;
         gamma_len = crtc_state->gamma_lut->length /
                     sizeof(struct drm_color_lut);
 
-        if ((ret = color_mgmt_config_copy_lut(modeset_config->lut.output.pRamps,
+        if ((ret = color_mgmt_config_copy_lut(nv_crtc_state->olut_ramps,
                                               gamma_lut,
                                               gamma_len)) != 0) {
-            return ret;
+            goto fail;
         }
 
+        modeset_config->lut.output.specified = NV_TRUE;
         modeset_config->lut.output.enabled   = NV_TRUE;
+        modeset_config->lut.output.pRamps    = nv_crtc_state->olut_ramps;
     } else {
         /* disabling the output LUT should be equivalent to setting a linear
          * LUT */
+        modeset_config->lut.output.specified = NV_TRUE;
         modeset_config->lut.output.enabled   = NV_FALSE;
-
-        nv_drm_free(modeset_config->lut.output.pRamps);
         modeset_config->lut.output.pRamps    = NULL;
     }
-    req_config->flags.olutChanged = NV_TRUE;
 
     return 0;
+
+fail:
+    /* free allocated state */
+    nv_drm_free(nv_crtc_state->ilut_ramps);
+    nv_drm_free(nv_crtc_state->olut_ramps);
+
+    /* remove dangling pointers */
+    nv_crtc_state->ilut_ramps = NULL;
+    nv_crtc_state->olut_ramps = NULL;
+    modeset_config->lut.input.pRamps = NULL;
+    modeset_config->lut.output.pRamps = NULL;
+
+    /* prevent attempts at reading NULLs */
+    modeset_config->lut.input.specified = NV_FALSE;
+    modeset_config->lut.output.specified = NV_FALSE;
+
+    return ret;
 }
 #endif /* NV_DRM_COLOR_MGMT_AVAILABLE */
 
@@ -1161,6 +1186,9 @@ static int nv_drm_crtc_atomic_check(struct drm_crtc *crtc,
     struct NvKmsKapiHeadRequestedConfig *req_config =
         &nv_crtc_state->req_config;
     int ret = 0;
+#if defined(NV_DRM_COLOR_MGMT_AVAILABLE)
+    struct nv_drm_device *nv_dev = to_nv_device(crtc_state->crtc->dev);
+#endif
 
     if (crtc_state->mode_changed) {
         drm_mode_to_nvkms_display_mode(&crtc_state->mode,
@@ -1204,6 +1232,13 @@ static int nv_drm_crtc_atomic_check(struct drm_crtc *crtc,
 #endif
 
 #if defined(NV_DRM_COLOR_MGMT_AVAILABLE)
+    if (nv_dev->drmMasterChangedSinceLastAtomicCommit &&
+        (crtc_state->degamma_lut ||
+         crtc_state->ctm ||
+         crtc_state->gamma_lut)) {
+
+        crtc_state->color_mgmt_changed = NV_TRUE;
+    }
     if (crtc_state->color_mgmt_changed) {
         if ((ret = color_mgmt_config_set_luts(nv_crtc_state, req_config)) != 0) {
             return ret;
@@ -1229,7 +1264,7 @@ static const struct drm_crtc_helper_funcs nv_crtc_helper_funcs = {
 
 static void nv_drm_plane_install_properties(
     struct drm_plane *plane,
-    NvBool supportsICtCp)
+    NvBool supportsHDR)
 {
     struct nv_drm_device *nv_dev = to_nv_device(plane->dev);
 
@@ -1245,7 +1280,7 @@ static void nv_drm_plane_install_properties(
     }
 
 #if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
-    if (supportsICtCp && nv_dev->nv_hdr_output_metadata_property) {
+    if (supportsHDR && nv_dev->nv_hdr_output_metadata_property) {
         drm_object_attach_property(
             &plane->base, nv_dev->nv_hdr_output_metadata_property, 0);
     }
@@ -1431,7 +1466,7 @@ nv_drm_plane_create(struct drm_device *dev,
     if (plane_type != DRM_PLANE_TYPE_CURSOR) {
         nv_drm_plane_install_properties(
                 plane,
-                pResInfo->supportsICtCp[layer_idx]);
+                pResInfo->supportsHDR[layer_idx]);
     }
 
     __nv_drm_plane_create_alpha_blending_properties(

kernel-open/nvidia-drm/nvidia-drm-crtc.h

@@ -129,6 +129,9 @@ struct nv_drm_crtc_state {
      */
     struct NvKmsKapiHeadRequestedConfig req_config;
 
+    struct NvKmsLutRamps *ilut_ramps;
+    struct NvKmsLutRamps *olut_ramps;
+
     /**
      * @nv_flip:
      *

kernel-open/nvidia-drm/nvidia-drm-drv.c

@@ -64,12 +64,25 @@
 #include <drm/drm_ioctl.h>
 #endif
 
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_LINUX_APERTURE_H_PRESENT)
+#include <linux/aperture.h>
+#endif
+
+#if defined(NV_DRM_DRM_APERTURE_H_PRESENT)
 #include <drm/drm_aperture.h>
+#endif
+
+#if defined(NV_DRM_FBDEV_AVAILABLE)
 #include <drm/drm_fb_helper.h>
 #endif
 
-#if defined(NV_DRM_DRM_FBDEV_GENERIC_H_PRESENT)
+#if defined(NV_DRM_DRM_CLIENT_SETUP_H_PRESENT)
+#include <drm/drm_client_setup.h>
+#endif
+
+#if defined(NV_DRM_DRM_FBDEV_TTM_H_PRESENT)
+#include <drm/drm_fbdev_ttm.h>
+#elif defined(NV_DRM_DRM_FBDEV_GENERIC_H_PRESENT)
 #include <drm/drm_fbdev_generic.h>
 #endif
 
@@ -124,6 +137,7 @@ static const char* nv_get_input_colorspace_name(
 
 #if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)
 
+#if defined(NV_DRM_OUTPUT_POLL_CHANGED_PRESENT)
 static void nv_drm_output_poll_changed(struct drm_device *dev)
 {
     struct drm_connector *connector = NULL;
@@ -167,6 +181,7 @@ static void nv_drm_output_poll_changed(struct drm_device *dev)
     nv_drm_connector_list_iter_end(&conn_iter);
 #endif
 }
+#endif /* NV_DRM_OUTPUT_POLL_CHANGED_PRESENT */
 
 static struct drm_framebuffer *nv_drm_framebuffer_create(
     struct drm_device *dev,
@@ -204,7 +219,9 @@ static const struct drm_mode_config_funcs nv_mode_config_funcs = {
     .atomic_check  = nv_drm_atomic_check,
     .atomic_commit = nv_drm_atomic_commit,
 
+    #if defined(NV_DRM_OUTPUT_POLL_CHANGED_PRESENT)
     .output_poll_changed = nv_drm_output_poll_changed,
+    #endif
 };
 
 static void nv_drm_event_callback(const struct NvKmsKapiEvent *event)
@@ -373,15 +390,19 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
         len++;
     }
 
-    if (nv_dev->supportsSyncpts) {
-        nv_dev->nv_out_fence_property =
-            drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_ATOMIC,
-                    "NV_DRM_OUT_FENCE_PTR", 0, U64_MAX);
-        if (nv_dev->nv_out_fence_property == NULL) {
-            return -ENOMEM;
-        }
+#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
+    if (!nv_dev->supportsSyncpts) {
+        return 0;
     }
 
+    nv_dev->nv_out_fence_property =
+        drm_property_create_range(nv_dev->dev, DRM_MODE_PROP_ATOMIC,
+            "NV_DRM_OUT_FENCE_PTR", 0, U64_MAX);
+    if (nv_dev->nv_out_fence_property == NULL) {
+        return -ENOMEM;
+    }
+#endif
+
     nv_dev->nv_input_colorspace_property =
         drm_property_create_enum(nv_dev->dev, 0, "NV_INPUT_COLORSPACE",
                                  enum_list, len);
@@ -430,7 +451,7 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
 
     struct NvKmsKapiAllocateDeviceParams allocateDeviceParams;
     struct NvKmsKapiDeviceResourcesInfo resInfo;
-#endif /* defined(NV_DRM_ATOMIC_MODESET_AVAILABLE) */
+#endif
 #if defined(NV_DRM_FORMAT_MODIFIERS_PRESENT)
     NvU64 kind;
     NvU64 gen;
@@ -476,7 +497,7 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
         return -ENODEV;
     }
 
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
     /*
      * If fbdev is enabled, take modeset ownership now before other DRM clients
      * can take master (and thus NVKMS ownership).
@@ -517,12 +538,6 @@ static int nv_drm_load(struct drm_device *dev, unsigned long flags)
     nv_dev->semsurf_max_submitted_offset =
         resInfo.caps.semsurf.maxSubmittedOffset;
 
-    nv_dev->display_semaphores.count =
-        resInfo.caps.numDisplaySemaphores;
-    nv_dev->display_semaphores.next_index = 0;
-
-    nv_dev->requiresVrrSemaphores = resInfo.caps.requiresVrrSemaphores;
-
 #if defined(NV_DRM_FORMAT_MODIFIERS_PRESENT)
     gen = nv_dev->pageKindGeneration;
     kind = nv_dev->genericPageKind;
@@ -610,7 +625,7 @@ static void __nv_drm_unload(struct drm_device *dev)
 
     /* Release modeset ownership if fbdev is enabled */
 
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
     if (nv_dev->hasFramebufferConsole) {
         drm_atomic_helper_shutdown(dev);
         nvKms->releaseOwnership(nv_dev->pDevice);
@@ -679,6 +694,7 @@ static int __nv_drm_master_set(struct drm_device *dev,
         !nvKms->grabOwnership(nv_dev->pDevice)) {
         return -EINVAL;
     }
+    nv_dev->drmMasterChangedSinceLastAtomicCommit = NV_TRUE;
 
     return 0;
 }
@@ -807,14 +823,6 @@ static int nv_drm_get_dev_info_ioctl(struct drm_device *dev,
     return 0;
 }
 
-static int nv_drm_get_drm_file_unique_id_ioctl(struct drm_device *dev,
-                                               void *data, struct drm_file *filep)
-{
-    struct drm_nvidia_get_drm_file_unique_id_params *params = data;
-    params->id = (u64)(filep->driver_priv);
-    return 0;
-}
-
 static int nv_drm_dmabuf_supported_ioctl(struct drm_device *dev,
                                          void *data, struct drm_file *filep)
 {
@@ -1331,17 +1339,6 @@ static void nv_drm_postclose(struct drm_device *dev, struct drm_file *filep)
 }
 #endif /* NV_DRM_ATOMIC_MODESET_AVAILABLE */
 
-static int nv_drm_open(struct drm_device *dev, struct drm_file *filep)
-{
-    _Static_assert(sizeof(filep->driver_priv) >= sizeof(u64),
-                   "filep->driver_priv can not hold an u64");
-    static atomic64_t id = ATOMIC_INIT(0);
-
-    filep->driver_priv = (void *)atomic64_inc_return(&id);
-
-    return 0;
-}
-
 #if defined(NV_DRM_MASTER_HAS_LEASES)
 static struct drm_master *nv_drm_find_lessee(struct drm_master *master,
                                              int lessee_id)
@@ -1567,6 +1564,10 @@ static const struct file_operations nv_drm_fops = {
     .read           = drm_read,
 
     .llseek         = noop_llseek,
+
+#if defined(NV_FILE_OPERATIONS_FOP_UNSIGNED_OFFSET_PRESENT)
+    .fop_flags   = FOP_UNSIGNED_OFFSET,
+#endif
 };
 
 static const struct drm_ioctl_desc nv_drm_ioctls[] = {
@@ -1585,9 +1586,6 @@ static const struct drm_ioctl_desc nv_drm_ioctls[] = {
     DRM_IOCTL_DEF_DRV(NVIDIA_GET_DEV_INFO,
                       nv_drm_get_dev_info_ioctl,
                       DRM_RENDER_ALLOW|DRM_UNLOCKED),
-    DRM_IOCTL_DEF_DRV(NVIDIA_GET_DRM_FILE_UNIQUE_ID,
-                      nv_drm_get_drm_file_unique_id_ioctl,
-                      DRM_RENDER_ALLOW|DRM_UNLOCKED),
 
 #if defined(NV_DRM_FENCE_AVAILABLE)
     DRM_IOCTL_DEF_DRV(NVIDIA_FENCE_SUPPORTED,
@@ -1670,9 +1668,6 @@ static struct drm_driver nv_drm_driver = {
     .driver_features        =
 #if defined(NV_DRM_DRIVER_PRIME_FLAG_PRESENT)
                                DRIVER_PRIME |
-#endif
-#if defined(NV_DRM_SYNCOBJ_FEATURES_PRESENT)
-                               DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE |
 #endif
                                DRIVER_GEM  | DRIVER_RENDER,
 
@@ -1684,14 +1679,14 @@ static struct drm_driver nv_drm_driver = {
     .num_ioctls             = ARRAY_SIZE(nv_drm_ioctls),
 
 /*
- * Linux kernel v6.6 commit 71a7974ac701 ("drm/prime: Unexport helpers
- * for fd/handle conversion") unexports drm_gem_prime_handle_to_fd() and
+ * linux-next commit 71a7974ac701 ("drm/prime: Unexport helpers for fd/handle
+ * conversion") unexports drm_gem_prime_handle_to_fd() and
  * drm_gem_prime_fd_to_handle().
  *
- * Prior Linux kernel v6.6 commit 6b85aa68d9d5 ("drm: Enable PRIME
- * import/export for all drivers") made these helpers the default when
- * .prime_handle_to_fd / .prime_fd_to_handle are unspecified, so it's fine
- * to just skip specifying them if the helpers aren't present.
+ * Prior linux-next commit 6b85aa68d9d5 ("drm: Enable PRIME import/export for
+ * all drivers") made these helpers the default when .prime_handle_to_fd /
+ * .prime_fd_to_handle are unspecified, so it's fine to just skip specifying
+ * them if the helpers aren't present.
  */
 #if NV_IS_EXPORT_SYMBOL_PRESENT_drm_gem_prime_handle_to_fd
     .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
@@ -1725,7 +1720,6 @@ static struct drm_driver nv_drm_driver = {
 #if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)
     .postclose              = nv_drm_postclose,
 #endif
-    .open                   = nv_drm_open,
 
     .fops                   = &nv_drm_fops,
 
@@ -1736,13 +1730,20 @@ static struct drm_driver nv_drm_driver = {
     .name                   = "nvidia-drm",
 
     .desc                   = "NVIDIA DRM driver",
+
+#if defined(NV_DRM_DRIVER_HAS_DATE)
     .date                   = "20160202",
+#endif
 
 #if defined(NV_DRM_DRIVER_HAS_DEVICE_LIST)
     .device_list            = LIST_HEAD_INIT(nv_drm_driver.device_list),
 #elif defined(NV_DRM_DRIVER_HAS_LEGACY_DEV_LIST)
     .legacy_dev_list        = LIST_HEAD_INIT(nv_drm_driver.legacy_dev_list),
 #endif
+// XXX implement nvidia-drm's own .fbdev_probe callback that uses NVKMS kapi directly
+#if defined(NV_DRM_FBDEV_AVAILABLE) && defined(DRM_FBDEV_TTM_DRIVER_OPS)
+    DRM_FBDEV_TTM_DRIVER_OPS,
+#endif
 };
 
 
@@ -1784,7 +1785,6 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
     struct nv_drm_device *nv_dev = NULL;
     struct drm_device *dev = NULL;
     struct device *device = gpu_info->os_device_ptr;
-    bool bus_is_pci;
 
     DRM_DEBUG(
         "Registering device for NVIDIA GPU ID 0x08%x",
@@ -1818,7 +1818,7 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
     dev->dev_private = nv_dev;
     nv_dev->dev = dev;
 
-    bus_is_pci =
+    bool bus_is_pci =
 #if defined(NV_LINUX)
         device->bus == &pci_bus_type;
 #elif defined(NV_BSD)
@@ -1838,22 +1838,34 @@ void nv_drm_register_drm_device(const nv_gpu_info_t *gpu_info)
         goto failed_drm_register;
     }
 
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
     if (nv_drm_fbdev_module_param &&
         drm_core_check_feature(dev, DRIVER_MODESET)) {
 
         if (bus_is_pci) {
             struct pci_dev *pdev = to_pci_dev(device);
 
+#if defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+
 #if defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_HAS_DRIVER_ARG)
             drm_aperture_remove_conflicting_pci_framebuffers(pdev, &nv_drm_driver);
 #else
             drm_aperture_remove_conflicting_pci_framebuffers(pdev, nv_drm_driver.name);
 #endif
+
+#elif defined(NV_APERTURE_REMOVE_CONFLICTING_PCI_DEVICES_PRESENT)
+            aperture_remove_conflicting_pci_devices(pdev, nv_drm_driver.name);
+#endif
         }
+        #if defined(NV_DRM_CLIENT_AVAILABLE)
+	    drm_client_setup(dev, NULL);
+        #elif defined(NV_DRM_FBDEV_TTM_AVAILABLE)
+        drm_fbdev_ttm_setup(dev, 32);
+        #elif defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
         drm_fbdev_generic_setup(dev, 32);
+        #endif
     }
-#endif /* defined(NV_DRM_FBDEV_GENERIC_AVAILABLE) */
+#endif /* defined(NV_DRM_FBDEV_AVAILABLE) */
 
     /* Add NVIDIA-DRM device into list */
 
@@ -1995,12 +2007,12 @@ void nv_drm_suspend_resume(NvBool suspend)
 
         if (suspend) {
             drm_kms_helper_poll_disable(dev);
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
             drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 1);
 #endif
             drm_mode_config_reset(dev);
         } else {
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
             drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 0);
 #endif
             drm_kms_helper_poll_enable(dev);

kernel-open/nvidia-drm/nvidia-drm-fence.c

@@ -293,12 +293,14 @@ __nv_drm_prime_fence_context_new(
      * to check a return value.
      */
 
-    nv_prime_fence_context->base.ops = &nv_drm_prime_fence_context_ops;
-    nv_prime_fence_context->base.nv_dev = nv_dev;
-    nv_prime_fence_context->base.context = nv_dma_fence_context_alloc(1);
-    nv_prime_fence_context->base.fenceSemIndex = p->index;
-    nv_prime_fence_context->pSemSurface = pSemSurface;
-    nv_prime_fence_context->pLinearAddress = pLinearAddress;
+    *nv_prime_fence_context = (struct nv_drm_prime_fence_context) {
+        .base.ops = &nv_drm_prime_fence_context_ops,
+        .base.nv_dev = nv_dev,
+        .base.context = nv_dma_fence_context_alloc(1),
+        .base.fenceSemIndex = p->index,
+        .pSemSurface = pSemSurface,
+        .pLinearAddress = pLinearAddress,
+    };
 
     INIT_LIST_HEAD(&nv_prime_fence_context->pending);
 
@@ -1269,16 +1271,18 @@ __nv_drm_semsurf_fence_ctx_new(
      * to check a return value.
      */
 
-    ctx->base.ops = &nv_drm_semsurf_fence_ctx_ops;
-    ctx->base.nv_dev = nv_dev;
-    ctx->base.context = nv_dma_fence_context_alloc(1);
-    ctx->base.fenceSemIndex = p->index;
-    ctx->pSemSurface = pSemSurface;
-    ctx->pSemMapping.pVoid = semMapping;
-    ctx->pMaxSubmittedMapping = (volatile NvU64 *)maxSubmittedMapping;
-    ctx->callback.local = NULL;
-    ctx->callback.nvKms = NULL;
-    ctx->current_wait_value = 0;
+    *ctx = (struct nv_drm_semsurf_fence_ctx) {
+        .base.ops = &nv_drm_semsurf_fence_ctx_ops,
+        .base.nv_dev = nv_dev,
+        .base.context = nv_dma_fence_context_alloc(1),
+        .base.fenceSemIndex = p->index,
+        .pSemSurface = pSemSurface,
+        .pSemMapping.pVoid = semMapping,
+        .pMaxSubmittedMapping = (volatile NvU64 *)maxSubmittedMapping,
+        .callback.local = NULL,
+        .callback.nvKms = NULL,
+        .current_wait_value = 0,
+    };
 
     spin_lock_init(&ctx->lock);
     INIT_LIST_HEAD(&ctx->pending_fences);

kernel-open/nvidia-drm/nvidia-drm-gem-nvkms-memory.c

@@ -551,12 +551,14 @@ static struct drm_gem_object *__nv_drm_gem_nvkms_prime_dup(
 {
     struct nv_drm_device *nv_dev = to_nv_device(dev);
     const struct nv_drm_device *nv_dev_src;
+    const struct nv_drm_gem_nvkms_memory *nv_nvkms_memory_src;
     struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
     struct NvKmsKapiMemory *pMemory;
 
     BUG_ON(nv_gem_src == NULL || nv_gem_src->ops != &nv_gem_nvkms_memory_ops);
 
     nv_dev_src = to_nv_device(nv_gem_src->base.dev);
+    nv_nvkms_memory_src = to_nv_nvkms_memory_const(nv_gem_src);
 
     if ((nv_nvkms_memory =
             nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {

kernel-open/nvidia-drm/nvidia-drm-helper.c

@@ -45,7 +45,8 @@
 
 /*
  * The inclusion of drm_framebuffer.h was removed from drm_crtc.h by commit
- * 720cf96d8fec ("drm: Drop drm_framebuffer.h from drm_crtc.h") in v6.0.
+ * 720cf96d8fecde29b72e1101f8a567a0ce99594f ("drm: Drop drm_framebuffer.h from
+ * drm_crtc.h") in linux-next, expected in v5.19-rc7.
  *
  * We only need drm_framebuffer.h for drm_framebuffer_put(), and it is always
  * present (v4.9+) when drm_framebuffer_{put,get}() is present (v4.12+), so it

kernel-open/nvidia-drm/nvidia-drm-helper.h

@@ -613,8 +613,8 @@ static inline int nv_drm_format_num_planes(uint32_t format)
 #endif /* defined(NV_DRM_FORMAT_MODIFIERS_PRESENT) */
 
 /*
- * DRM_UNLOCKED was removed with commit 2798ffcc1d6a ("drm: Remove locking for
- * legacy ioctls and DRM_UNLOCKED") in v6.8, but it was previously made
+ * DRM_UNLOCKED was removed with linux-next commit 2798ffcc1d6a ("drm: Remove
+ * locking for legacy ioctls and DRM_UNLOCKED"), but it was previously made
  * implicit for all non-legacy DRM driver IOCTLs since Linux v4.10 commit
  * fa5386459f06 "drm: Used DRM_LEGACY for all legacy functions" (Linux v4.4
  * commit ea487835e887 "drm: Enforce unlocked ioctl operation for kms driver

kernel-open/nvidia-drm/nvidia-drm-ioctl.h

@@ -52,7 +52,6 @@
 #define DRM_NVIDIA_SEMSURF_FENCE_CREATE             0x15
 #define DRM_NVIDIA_SEMSURF_FENCE_WAIT               0x16
 #define DRM_NVIDIA_SEMSURF_FENCE_ATTACH             0x17
-#define DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID           0x18
 
 #define DRM_IOCTL_NVIDIA_GEM_IMPORT_NVKMS_MEMORY                           \
     DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GEM_IMPORT_NVKMS_MEMORY),      \
@@ -158,11 +157,6 @@
               DRM_NVIDIA_SEMSURF_FENCE_ATTACH),                         \
               struct drm_nvidia_semsurf_fence_attach_params)
 
-#define DRM_IOCTL_NVIDIA_GET_DRM_FILE_UNIQUE_ID                         \
-    DRM_IOWR((DRM_COMMAND_BASE +                                        \
-              DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID),                       \
-              struct drm_nvidia_get_drm_file_unique_id_params)
-
 struct drm_nvidia_gem_import_nvkms_memory_params {
     uint64_t mem_size;           /* IN */
 
@@ -391,8 +385,4 @@ struct drm_nvidia_semsurf_fence_attach_params {
     uint64_t wait_value;            /* IN Semaphore value to reach before signal */
 };
 
-struct drm_nvidia_get_drm_file_unique_id_params {
-    uint64_t id;                    /* OUT Unique ID of the DRM file */
-};
-
 #endif /* _UAPI_NVIDIA_DRM_IOCTL_H_ */

kernel-open/nvidia-drm/nvidia-drm-linux.c

@@ -34,7 +34,7 @@ MODULE_PARM_DESC(
     "Enable atomic kernel modesetting (1 = enable, 0 = disable (default))");
 module_param_named(modeset, nv_drm_modeset_module_param, bool, 0400);
 
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
 MODULE_PARM_DESC(
     fbdev,
     "Create a framebuffer device (1 = enable, 0 = disable (default)) (EXPERIMENTAL)");

kernel-open/nvidia-drm/nvidia-drm-modeset.c

@@ -42,16 +42,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 
-#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
-#include <linux/nvhost.h>
-#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)            
-#include <linux/host1x-next.h>
-#endif
-
-#if defined(NV_DRM_FENCE_AVAILABLE)
-#include "nvidia-dma-fence-helper.h"
-#endif
-
 struct nv_drm_atomic_state {
     struct NvKmsKapiRequestedModeSetConfig config;
     struct drm_atomic_state base;
@@ -156,159 +146,6 @@ static int __nv_drm_put_back_post_fence_fd(
     return ret;
 }
 
-#if defined(NV_DRM_FENCE_AVAILABLE)
-struct nv_drm_plane_fence_cb_data {
-    nv_dma_fence_cb_t dma_fence_cb;
-    struct nv_drm_device *nv_dev;
-    NvU32 semaphore_index;
-};
-
-static void
-__nv_drm_plane_fence_cb(
-    nv_dma_fence_t *fence,
-    nv_dma_fence_cb_t *cb_data
-)
-{
-    struct nv_drm_plane_fence_cb_data *fence_data =
-        container_of(cb_data, typeof(*fence_data), dma_fence_cb);
-    struct nv_drm_device *nv_dev = fence_data->nv_dev;
-
-    nv_dma_fence_put(fence);
-    nvKms->signalDisplaySemaphore(nv_dev->pDevice, fence_data->semaphore_index);
-    nv_drm_free(fence_data);
-}
-
-static int __nv_drm_convert_in_fences(
-    struct nv_drm_device *nv_dev,
-    struct drm_atomic_state *state,
-    struct drm_crtc *crtc,
-    struct drm_crtc_state *crtc_state)
-{
-    struct drm_plane *plane = NULL;
-    struct drm_plane_state *plane_state = NULL;
-    struct nv_drm_plane *nv_plane = NULL;
-    struct NvKmsKapiLayerRequestedConfig *plane_req_config = NULL;
-    struct NvKmsKapiHeadRequestedConfig *head_req_config =
-        &to_nv_crtc_state(crtc_state)->req_config;
-    struct nv_drm_plane_fence_cb_data *fence_data;
-    uint32_t semaphore_index;
-    int ret, i;
-
-    if (!crtc_state->active) {
-        return 0;
-    }
-
-    nv_drm_for_each_new_plane_in_state(state, plane, plane_state, i) {
-        if ((plane->type == DRM_PLANE_TYPE_CURSOR) ||
-            (plane_state->crtc != crtc) ||
-            (plane_state->fence == NULL)) {
-            continue;
-        }
-
-        nv_plane = to_nv_plane(plane);
-        plane_req_config =
-            &head_req_config->layerRequestedConfig[nv_plane->layer_idx];
-
-        if (nv_dev->supportsSyncpts) {
-#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
-#if defined(NV_NVHOST_DMA_FENCE_UNPACK_PRESENT)
-            int ret =
-                nvhost_dma_fence_unpack(
-                    plane_state->fence,
-                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
-                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
-            if (ret == 0) {
-                plane_req_config->config.syncParams.preSyncptSpecified = true;
-                continue;
-            }
-#endif
-#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)
-            int ret =
-                host1x_fence_extract(
-                    plane_state->fence,
-                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
-                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
-            if (ret == 0) {
-                plane_req_config->config.syncParams.preSyncptSpecified = true;
-                continue;
-            }
-#endif
-        }
-
-        /*
-         * Syncpt extraction failed, or syncpts are not supported.
-         * Use general DRM fence support with semaphores instead.
-         */
-        if (plane_req_config->config.syncParams.postSyncptRequested) {
-            // Can't mix Syncpts and semaphores in a given request.
-            return -EINVAL;
-        }
-
-        semaphore_index = nv_drm_next_display_semaphore(nv_dev);
-
-        if (!nvKms->resetDisplaySemaphore(nv_dev->pDevice, semaphore_index)) {
-            NV_DRM_DEV_LOG_ERR(
-                nv_dev,
-                "Failed to initialize semaphore for plane fence");
-            /*
-             * This should only happen if the semaphore pool was somehow
-             * exhausted. Waiting a bit and retrying may help in that case.
-             */
-            return -EAGAIN;
-        }
-
-        plane_req_config->config.syncParams.semaphoreSpecified = true;
-        plane_req_config->config.syncParams.u.semaphore.index = semaphore_index;
-
-        fence_data = nv_drm_calloc(1, sizeof(*fence_data));
-
-        if (!fence_data) {
-            NV_DRM_DEV_LOG_ERR(
-                nv_dev,
-                "Failed to allocate callback data for plane fence");
-            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
-            return -ENOMEM;
-        }
-
-        fence_data->nv_dev = nv_dev;
-        fence_data->semaphore_index = semaphore_index;
-
-        ret = nv_dma_fence_add_callback(plane_state->fence,
-                                        &fence_data->dma_fence_cb,
-                                        __nv_drm_plane_fence_cb);
-
-        switch (ret) {
-        case -ENOENT:
-            /* The fence is already signaled */
-            __nv_drm_plane_fence_cb(plane_state->fence,
-                                    &fence_data->dma_fence_cb);
-#if defined(fallthrough)
-            fallthrough;
-#else
-            /* Fallthrough */
-#endif
-        case 0:
-            /*
-             * The plane state's fence reference has either been consumed or
-             * belongs to the outstanding callback now.
-             */
-            plane_state->fence = NULL;
-            break;
-        default:
-            NV_DRM_DEV_LOG_ERR(
-                nv_dev,
-                "Failed plane fence callback registration");
-            /* Fence callback registration failed */
-            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
-            nv_drm_free(fence_data);
-            return ret;
-        }
-    }
-
-    return 0;
-}
-#endif /* defined(NV_DRM_FENCE_AVAILABLE) */
-
 static int __nv_drm_get_syncpt_data(
     struct nv_drm_device *nv_dev,
     struct drm_crtc *crtc,
@@ -421,6 +258,11 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
                                commit ? crtc->state : crtc_state;
         struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
 
+        requested_config->headRequestedConfig[nv_crtc->head] =
+            to_nv_crtc_state(new_crtc_state)->req_config;
+
+        requested_config->headsMask |= 1 << nv_crtc->head;
+
         if (commit) {
             struct drm_crtc_state *old_crtc_state = crtc_state;
             struct nv_drm_crtc_state *nv_new_crtc_state =
@@ -440,27 +282,7 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
 
                 nv_new_crtc_state->nv_flip = NULL;
             }
-
-#if defined(NV_DRM_FENCE_AVAILABLE)
-            ret = __nv_drm_convert_in_fences(nv_dev,
-                                             state,
-                                             crtc,
-                                             new_crtc_state);
-
-            if (ret != 0) {
-                return ret;
-            }
-#endif /* defined(NV_DRM_FENCE_AVAILABLE) */
         }
-
-        /*
-         * Do this deep copy after calling __nv_drm_convert_in_fences,
-         * which modifies the new CRTC state's req_config member
-         */
-        requested_config->headRequestedConfig[nv_crtc->head] =
-            to_nv_crtc_state(new_crtc_state)->req_config;
-
-        requested_config->headsMask |= 1 << nv_crtc->head;
     }
 
     if (commit && nvKms->systemInfo.bAllowWriteCombining) {
@@ -491,10 +313,6 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
         }
     }
 
-    if (commit && nv_dev->requiresVrrSemaphores && reply_config.vrrFlip) {
-        nvKms->signalVrrSemaphore(nv_dev->pDevice, reply_config.vrrSemaphoreIndex);
-    }
-
     return 0;
 }
 
@@ -688,6 +506,7 @@ int nv_drm_atomic_commit(struct drm_device *dev,
 
         goto done;
     }
+    nv_dev->drmMasterChangedSinceLastAtomicCommit = NV_FALSE;
 
     nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
         struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
@@ -768,9 +587,6 @@ int nv_drm_atomic_commit(struct drm_device *dev,
                 NV_DRM_DEV_LOG_ERR(
                     nv_dev,
                     "Flip event timeout on head %u", nv_crtc->head);
-                while (!list_empty(&nv_crtc->flip_list)) {
-                    __nv_drm_handle_flip_event(nv_crtc);
-                }
             }
         }
     }

kernel-open/nvidia-drm/nvidia-drm-os-interface.h

@@ -59,14 +59,20 @@ typedef struct nv_timer nv_drm_timer;
 #endif
 
 #if defined(NV_DRM_FBDEV_GENERIC_SETUP_PRESENT) && defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+#define NV_DRM_FBDEV_AVAILABLE
 #define NV_DRM_FBDEV_GENERIC_AVAILABLE
 #endif
 
+#if defined(NV_DRM_FBDEV_TTM_SETUP_PRESENT) && defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
+#define NV_DRM_FBDEV_AVAILABLE
+#define NV_DRM_FBDEV_TTM_AVAILABLE
+#endif
+
 struct page;
 
 /* Set to true when the atomic modeset feature is enabled. */
 extern bool nv_drm_modeset_module_param;
-#if defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
+#if defined(NV_DRM_FBDEV_AVAILABLE)
 /* Set to true when the nvidia-drm driver should install a framebuffer device */
 extern bool nv_drm_fbdev_module_param;
 #endif

kernel-open/nvidia-drm/nvidia-drm-priv.h

@@ -147,18 +147,22 @@ struct nv_drm_device {
     NvBool hasVideoMemory;
 
     NvBool supportsSyncpts;
-    NvBool requiresVrrSemaphores;
     NvBool subOwnershipGranted;
     NvBool hasFramebufferConsole;
 
+    /**
+     * @drmMasterChangedSinceLastAtomicCommit:
+     *
+     * This flag is set in nv_drm_master_set and reset after a completed atomic
+     * commit. It is used to restore or recommit state that is lost by the
+     * NvKms modeset owner change, such as the CRTC color management
+     * properties.
+     */
+    NvBool drmMasterChangedSinceLastAtomicCommit;
+
     struct drm_property *nv_out_fence_property;
     struct drm_property *nv_input_colorspace_property;
 
-    struct {
-        NvU32 count;
-        NvU32 next_index;
-    } display_semaphores;
-
 #if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
     struct drm_property *nv_hdr_output_metadata_property;
 #endif
@@ -166,19 +170,6 @@ struct nv_drm_device {
     struct nv_drm_device *next;
 };
 
-static inline NvU32 nv_drm_next_display_semaphore(
-    struct nv_drm_device *nv_dev)
-{
-    NvU32 current_index = nv_dev->display_semaphores.next_index++;
-
-    if (nv_dev->display_semaphores.next_index >=
-        nv_dev->display_semaphores.count) {
-        nv_dev->display_semaphores.next_index = 0;
-    }
-
-    return current_index;
-}
-
 static inline struct nv_drm_device *to_nv_device(
     struct drm_device *dev)
 {

kernel-open/nvidia-drm/nvidia-drm-sources.mk

@@ -66,7 +66,11 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_fence_set_error
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += fence_set_error
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += sync_file_get_fence
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_aperture_remove_conflicting_pci_framebuffers
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += aperture_remove_conflicting_devices
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += aperture_remove_conflicting_pci_devices
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_fbdev_generic_setup
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_fbdev_ttm_setup
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_client_setup
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_connector_attach_hdr_output_metadata_property
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_helper_crtc_enable_color_mgmt
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_crtc_enable_color_mgmt
@@ -128,5 +132,8 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_dumb_destroy
 NV_CONFTEST_TYPE_COMPILE_TESTS += fence_ops_use_64bit_seqno
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_aperture_remove_conflicting_pci_framebuffers_has_driver_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_mode_create_dp_colorspace_property_has_supported_colorspaces_arg
-NV_CONFTEST_TYPE_COMPILE_TESTS += drm_syncobj_features_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_unlocked_ioctl_flag_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_output_poll_changed
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_date
+NV_CONFTEST_TYPE_COMPILE_TESTS += file_operations_fop_unsigned_offset_present
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_helper_funcs_mode_valid_has_const_mode_arg

kernel-open/nvidia-modeset/nv-kthread-q.c

@@ -201,7 +201,7 @@ static struct task_struct *thread_create_on_node(int (*threadfn)(void *data),
 
         // Ran out of attempts - return thread even if its stack may not be
         // allocated on the preferred node
-        if ((i == (attempts - 1)))
+        if (i == (attempts - 1))
             break;
 
         // Get the NUMA node where the first page of the stack is resident. If

kernel-open/nvidia-modeset/nvidia-modeset-linux.c

@@ -77,10 +77,10 @@ module_param_named(disable_hdmi_frl, disable_hdmi_frl, bool, 0400);
 static bool disable_vrr_memclk_switch = false;
 module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0400);
 
-static bool hdmi_deepcolor = true;
+static bool hdmi_deepcolor = false;
 module_param_named(hdmi_deepcolor, hdmi_deepcolor, bool, 0400);
 
-static bool vblank_sem_control = true;
+static bool vblank_sem_control = false;
 module_param_named(vblank_sem_control, vblank_sem_control, bool, 0400);
 
 static bool opportunistic_display_sync = true;
@@ -139,20 +139,6 @@ NvBool nvkms_opportunistic_display_sync(void)
     return opportunistic_display_sync;
 }
 
-NvBool nvkms_kernel_supports_syncpts(void)
-{
-/*
- * Note this only checks that the kernel has the prerequisite
- * support for syncpts; callers must also check that the hardware
- * supports syncpts.
- */
-#if (defined(CONFIG_TEGRA_GRHOST) || defined(NV_LINUX_HOST1X_NEXT_H_PRESENT))
-    return NV_TRUE;
-#else
-    return NV_FALSE;
-#endif
-}
-
 #define NVKMS_SYNCPT_STUBS_NEEDED
 
 /*************************************************************************
@@ -1010,6 +996,11 @@ nvkms_register_backlight(NvU32 gpu_id, NvU32 display_id, void *drv_priv,
 
 #if defined(NV_ACPI_VIDEO_BACKLIGHT_USE_NATIVE)
     if (!acpi_video_backlight_use_native()) {
+#if defined(NV_ACPI_VIDEO_REGISTER_BACKLIGHT)
+        nvkms_log(NVKMS_LOG_LEVEL_INFO, NVKMS_LOG_PREFIX,
+                  "ACPI reported no NVIDIA native backlight available; attempting to use ACPI backlight.");
+        acpi_video_register_backlight();
+#endif
         return NULL;
     }
 #endif
@@ -1084,7 +1075,7 @@ static void nvkms_kapi_event_kthread_q_callback(void *arg)
     nvKmsKapiHandleEventQueueChange(device);
 }
 
-struct nvkms_per_open *nvkms_open_common(enum NvKmsClientType type,
+static struct nvkms_per_open *nvkms_open_common(enum NvKmsClientType type,
                                          struct NvKmsKapiDevice *device,
                                          int *status)
 {
@@ -1136,7 +1127,7 @@ failed:
     return NULL;
 }
 
-void nvkms_close_pm_locked(struct nvkms_per_open *popen)
+static void nvkms_close_pm_locked(struct nvkms_per_open *popen)
 {
     /*
      * Don't use down_interruptible(): we need to free resources
@@ -1199,7 +1190,7 @@ static void nvkms_close_popen(struct nvkms_per_open *popen)
     }
 }
 
-int nvkms_ioctl_common
+static int nvkms_ioctl_common
 (
     struct nvkms_per_open *popen,
     NvU32 cmd, NvU64 address, const size_t size
@@ -1248,26 +1239,6 @@ void nvkms_close_from_kapi(struct nvkms_per_open *popen)
     nvkms_close_pm_unlocked(popen);
 }
 
-NvBool nvkms_ioctl_from_kapi_try_pmlock
-(
-    struct nvkms_per_open *popen,
-    NvU32 cmd, void *params_address, const size_t param_size
-)
-{
-    NvBool ret;
-
-    if (nvkms_read_trylock_pm_lock()) {
-        return NV_FALSE;
-    }
-
-    ret = nvkms_ioctl_common(popen,
-                             cmd,
-                             (NvU64)(NvUPtr)params_address, param_size) == 0;
-    nvkms_read_unlock_pm_lock();
-
-    return ret;
-}
-
 NvBool nvkms_ioctl_from_kapi
 (
     struct nvkms_per_open *popen,

kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h

@@ -304,11 +304,6 @@ NvU32 nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info);
 
 NvBool nvkms_allow_write_combining(void);
 
-/*!
- * Check if OS supports syncpoints.
- */
-NvBool nvkms_kernel_supports_syncpts(void);
-
 /*!
  * Checks whether the fd is associated with an nvidia character device.
  */
@@ -333,16 +328,6 @@ NvBool nvkms_ioctl_from_kapi
     NvU32 cmd, void *params_address, const size_t params_size
 );
 
-/*!
- * Like nvkms_ioctl_from_kapi, but return NV_FALSE instead of waiting if the
- * power management read lock cannot be acquired.
- */
-NvBool nvkms_ioctl_from_kapi_try_pmlock
-(
-    struct nvkms_per_open *popen,
-    NvU32 cmd, void *params_address, const size_t params_size
-);
-
 /*!
  * APIs for locking.
  */

kernel-open/nvidia-modeset/nvidia-modeset.Kbuild

@@ -40,9 +40,6 @@ NV_KERNEL_MODULE_TARGETS += $(NVIDIA_MODESET_KO)
 NVIDIA_MODESET_BINARY_OBJECT := $(src)/nvidia-modeset/nv-modeset-kernel.o_binary
 NVIDIA_MODESET_BINARY_OBJECT_O := nvidia-modeset/nv-modeset-kernel.o
 
-quiet_cmd_symlink = SYMLINK $@
-cmd_symlink = ln -sf $< $@
-
 targets += $(NVIDIA_MODESET_BINARY_OBJECT_O)
 
 $(obj)/$(NVIDIA_MODESET_BINARY_OBJECT_O): $(NVIDIA_MODESET_BINARY_OBJECT) FORCE
@@ -105,4 +102,4 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += list_is_first
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_real_ts64
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_raw_ts64
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += acpi_video_backlight_use_native
-NV_CONFTEST_FUNCTION_COMPILE_TESTS += kernel_read_has_pointer_pos_arg
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += acpi_video_register_backlight

kernel-open/nvidia-peermem/nvidia-peermem.c

@@ -60,13 +60,6 @@ static int peerdirect_support = NV_MEM_PEERDIRECT_SUPPORT_DEFAULT;
 module_param(peerdirect_support, int, S_IRUGO);
 MODULE_PARM_DESC(peerdirect_support, "Set level of support for Peer-direct, 0 [default] or 1 [legacy, for example MLNX_OFED 4.9 LTS]");
 
-enum {
-        NV_MEM_PERSISTENT_API_SUPPORT_LEGACY = 0,
-        NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT = 1,
-};
-static int persistent_api_support = NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT;
-module_param(persistent_api_support, int, S_IRUGO);
-MODULE_PARM_DESC(persistent_api_support, "Set level of support for persistent APIs, 0 [legacy] or 1 [default]");
 
 #define peer_err(FMT, ARGS...) printk(KERN_ERR "nvidia-peermem" " %s:%d ERROR " FMT, __FUNCTION__, __LINE__, ## ARGS)
 #ifdef NV_MEM_DEBUG
@@ -486,8 +479,32 @@ static struct peer_memory_client nv_mem_client_nc = {
     .release        = nv_mem_release,
 };
 
-static int nv_mem_legacy_client_init(void)
+#endif /* NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT */
+
+static int nv_mem_param_conf_check(void)
 {
+    int rc = 0;
+    switch (peerdirect_support) {
+    case NV_MEM_PEERDIRECT_SUPPORT_DEFAULT:
+    case NV_MEM_PEERDIRECT_SUPPORT_LEGACY:
+        break;
+    default:
+        peer_err("invalid peerdirect_support param value %d\n", peerdirect_support);
+        rc = -EINVAL;
+        break;
+    }
+    return rc;
+}
+
+static int __init nv_mem_client_init(void)
+{
+    int rc;
+    rc = nv_mem_param_conf_check();
+    if (rc) {
+        return rc;
+    }
+
+#if defined (NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT)
     // off by one, to leave space for the trailing '1' which is flagging
     // the new client type
     BUG_ON(strlen(DRV_NAME) > IB_PEER_MEMORY_NAME_MAX-1);
@@ -516,96 +533,19 @@ static int nv_mem_legacy_client_init(void)
                          &mem_invalidate_callback);
     if (!reg_handle) {
         peer_err("nv_mem_client_init -- error while registering traditional client\n");
-        return -EINVAL;
+        rc = -EINVAL;
+        goto out;
     }
-    return 0;
-}
 
-static int nv_mem_nc_client_init(void)
-{
     // The nc client enables support for persistent pages.
-    if (persistent_api_support == NV_MEM_PERSISTENT_API_SUPPORT_LEGACY)
-    {
-        //
-        // If legacy behavior is forced via module param,
-        // both legacy and persistent clients are registered and are named
-        // "nv_mem"(legacy) and "nv_mem_nc"(persistent).
-        //
-        strcpy(nv_mem_client_nc.name, DRV_NAME "_nc");
-    }
-    else
-    {
-        //
-        // With default persistent behavior, the client name shall be "nv_mem"
-        // so that libraries can use the persistent client under the same name.
-        //
-        strcpy(nv_mem_client_nc.name, DRV_NAME);
-    }
-
+    strcpy(nv_mem_client_nc.name, DRV_NAME "_nc");
     strcpy(nv_mem_client_nc.version, DRV_VERSION);
     reg_handle_nc = ib_register_peer_memory_client(&nv_mem_client_nc, NULL);
     if (!reg_handle_nc) {
         peer_err("nv_mem_client_init -- error while registering nc client\n");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-#endif /* NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT */
-
-static int nv_mem_param_peerdirect_conf_check(void)
-{
-    int rc = 0;
-    switch (peerdirect_support) {
-    case NV_MEM_PEERDIRECT_SUPPORT_DEFAULT:
-    case NV_MEM_PEERDIRECT_SUPPORT_LEGACY:
-        break;
-    default:
-        peer_err("invalid peerdirect_support param value %d\n", peerdirect_support);
         rc = -EINVAL;
-        break;
-    }
-    return rc;
-}
-
-static int nv_mem_param_persistent_api_conf_check(void)
-{
-    int rc = 0;
-    switch (persistent_api_support) {
-    case NV_MEM_PERSISTENT_API_SUPPORT_DEFAULT:
-    case NV_MEM_PERSISTENT_API_SUPPORT_LEGACY:
-        break;
-    default:
-        peer_err("invalid persistent_api_support param value %d\n", persistent_api_support);
-        rc = -EINVAL;
-        break;
-    }
-    return rc;
-}
-
-static int __init nv_mem_client_init(void)
-{
-#if defined (NV_MLNX_IB_PEER_MEM_SYMBOLS_PRESENT)
-    int rc;
-    rc = nv_mem_param_peerdirect_conf_check();
-    if (rc) {
-        return rc;
-    }
-
-    rc = nv_mem_param_persistent_api_conf_check();
-    if (rc) {
-        return rc;
-    }
-
-    if (persistent_api_support == NV_MEM_PERSISTENT_API_SUPPORT_LEGACY) {
-        rc = nv_mem_legacy_client_init();
-        if (rc)
-            goto out;
-    }
-
-    rc = nv_mem_nc_client_init();
-    if (rc)
         goto out;
+    }
 
 out:
     if (rc) {

kernel-open/nvidia-uvm/clc96f.h

@@ -1,329 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2012-2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-#ifndef _clc96f_h_
-#define _clc96f_h_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "nvtypes.h"
-
-/* class BLACKWELL_CHANNEL_GPFIFO  */
-/*
- * Documentation for BLACKWELL_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
- * chapter "User Control Registers". It is documented as device NV_UDMA.
- * The GPFIFO format itself is also documented in dev_pbdma.ref,
- * NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
- * chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
- *
- * Note there is no .mfs file for this class.
- */
-#define  BLACKWELL_CHANNEL_GPFIFO_A                           (0x0000C96F)
-
-#define NVC96F_TYPEDEF                             BLACKWELL_CHANNELChannelGPFifoA
-
-/* dma flow control data structure */
-typedef volatile struct Nvc96fControl_struct {
- NvU32 Ignored00[0x23];        /*                                  0000-008b*/
- NvU32 GPPut;                   /* GP FIFO put offset               008c-008f*/
- NvU32 Ignored01[0x5c];
-} Nvc96fControl, BlackwellAControlGPFifo;
-
-/* fields and values */
-#define NVC96F_NUMBER_OF_SUBCHANNELS                               (8)
-#define NVC96F_SET_OBJECT                                          (0x00000000)
-#define NVC96F_SET_OBJECT_NVCLASS                                         15:0
-#define NVC96F_SET_OBJECT_ENGINE                                         20:16
-#define NVC96F_SET_OBJECT_ENGINE_SW                                 0x0000001f
-#define NVC96F_NOP                                                 (0x00000008)
-#define NVC96F_NOP_HANDLE                                                 31:0
-#define NVC96F_NON_STALL_INTERRUPT                                 (0x00000020)
-#define NVC96F_NON_STALL_INTERRUPT_HANDLE                                 31:0
-#define NVC96F_FB_FLUSH                                            (0x00000024) // Deprecated - use MEMBAR TYPE SYS_MEMBAR
-#define NVC96F_FB_FLUSH_HANDLE                                            31:0
-// NOTE - MEM_OP_A and MEM_OP_B have been replaced in gp100 with methods for
-// specifying the page address for a targeted TLB invalidate and the uTLB for
-// a targeted REPLAY_CANCEL for UVM.
-// The previous MEM_OP_A/B functionality is in MEM_OP_C/D, with slightly
-// rearranged fields.
-#define NVC96F_MEM_OP_A                                            (0x00000028)
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID        5:0  // only relevant for REPLAY_CANCEL_TARGETED
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE                   5:0  // Used to specify size of invalidate, used for invalidates which are not of the REPLAY_CANCEL_TARGETED type
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID               10:6  // only relevant for REPLAY_CANCEL_TARGETED
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE                         7:6  // only relevant for invalidates with NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE for invalidating  link TLB only, or non-link TLB only or all TLBs
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_ALL_TLBS                  0
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_LINK_TLBS                 1
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_NON_LINK_TLBS             2
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_INVAL_SCOPE_RSVRVD                    3
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_CANCEL_MMU_ENGINE_ID                8:0  // only relevant for REPLAY_CANCEL_VA_GLOBAL
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR                         11:11
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN                 0x00000001
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS                0x00000000
-#define NVC96F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO                    31:12
-#define NVC96F_MEM_OP_B                                            (0x0000002c)
-#define NVC96F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI                     31:0
-#define NVC96F_MEM_OP_C                                            (0x00000030)
-#define NVC96F_MEM_OP_C_MEMBAR_TYPE                                        2:0
-#define NVC96F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR                      0x00000000
-#define NVC96F_MEM_OP_C_MEMBAR_TYPE_MEMBAR                          0x00000001
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB                                 0:0
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE                      0x00000000
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL                      0x00000001  // Probably nonsensical for MMU_TLB_INVALIDATE_TARGETED
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC                                 1:1
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE                   0x00000000
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE                  0x00000001
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY                              4:2  // only relevant if GPC ENABLE
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE                  0x00000000
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START                 0x00000001
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL         0x00000002
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED       0x00000003
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL         0x00000004
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_VA_GLOBAL      0x00000005
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE                            6:5  // only relevant if GPC ENABLE
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE                0x00000000
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY            0x00000001
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE           0x00000002
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE                         9:7 //only relevant for REPLAY_CANCEL_VA_GLOBAL
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_READ                 0
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE                1
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_STRONG        2
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_RSVRVD               3
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_WEAK          4
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_ALL           5
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE_AND_ATOMIC     6
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ALL                  7
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL                    9:7  // Invalidate affects this level and all below
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL         0x00000000  // Invalidate tlb caches at all levels of the page table
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY    0x00000001
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0  0x00000002
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1  0x00000003
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2  0x00000004
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3  0x00000005
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4  0x00000006
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5  0x00000007
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE                          11:10  // only relevant if PDB_ONE
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM             0x00000000
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT    0x00000002
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT 0x00000003
-#define NVC96F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO                       31:12  // only relevant if PDB_ONE
-#define NVC96F_MEM_OP_C_ACCESS_COUNTER_CLR_TARGETED_NOTIFY_TAG            19:0
-// MEM_OP_D MUST be preceded by MEM_OPs A-C.
-#define NVC96F_MEM_OP_D                                            (0x00000034)
-#define NVC96F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI                        26:0  // only relevant if PDB_ONE
-#define NVC96F_MEM_OP_D_OPERATION                                        31:27
-#define NVC96F_MEM_OP_D_OPERATION_MEMBAR                            0x00000005
-#define NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE                0x00000009
-#define NVC96F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED       0x0000000a
-#define NVC96F_MEM_OP_D_OPERATION_MMU_OPERATION                     0x0000000b
-#define NVC96F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE             0x0000000d
-#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE              0x0000000e
-// CLEAN_LINES is an alias for Tegra/GPU IP usage
-#define NVC96F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES         0x0000000e
-#define NVC96F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS                 0x0000000f
-#define NVC96F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY                    0x00000010
-#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_NCOH_INVALIDATE         0x00000011
-#define NVC96F_MEM_OP_D_OPERATION_L2_SYSMEM_COH_INVALIDATE          0x00000012
-#define NVC96F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS     0x00000015
-#define NVC96F_MEM_OP_D_OPERATION_ACCESS_COUNTER_CLR                0x00000016
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE                            1:0
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MIMC                0x00000000
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MOMC                0x00000001
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_ALL                 0x00000002
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_TARGETED            0x00000003
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE                   2:2
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MIMC       0x00000000
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MOMC       0x00000001
-#define NVC96F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_BANK                   6:3
-#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE                               23:20
-#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE_RESERVED                 0x00000000
-#define NVC96F_MEM_OP_D_MMU_OPERATION_TYPE_VIDMEM_ACCESS_BIT_DUMP   0x00000001
-#define NVC96F_SEM_ADDR_LO                                         (0x0000005c)
-#define NVC96F_SEM_ADDR_LO_OFFSET                                         31:2
-#define NVC96F_SEM_ADDR_HI                                         (0x00000060)
-#define NVC96F_SEM_ADDR_HI_OFFSET                                         24:0
-#define NVC96F_SEM_PAYLOAD_LO                                      (0x00000064)
-#define NVC96F_SEM_PAYLOAD_LO_PAYLOAD                                     31:0
-#define NVC96F_SEM_PAYLOAD_HI                                      (0x00000068)
-#define NVC96F_SEM_PAYLOAD_HI_PAYLOAD                                     31:0
-#define NVC96F_SEM_EXECUTE                                         (0x0000006c)
-#define NVC96F_SEM_EXECUTE_OPERATION                                       2:0
-#define NVC96F_SEM_EXECUTE_OPERATION_ACQUIRE                        0x00000000
-#define NVC96F_SEM_EXECUTE_OPERATION_RELEASE                        0x00000001
-#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_STRICT_GEQ                 0x00000002
-#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_CIRC_GEQ                   0x00000003
-#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_AND                        0x00000004
-#define NVC96F_SEM_EXECUTE_OPERATION_ACQ_NOR                        0x00000005
-#define NVC96F_SEM_EXECUTE_OPERATION_REDUCTION                      0x00000006
-#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG                            12:12
-#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_DIS                   0x00000000
-#define NVC96F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_EN                    0x00000001
-#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK                               18:18
-#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK_DIS                      0x00000000
-#define NVC96F_SEM_EXECUTE_ACQUIRE_RECHECK_EN                       0x00000001
-#define NVC96F_SEM_EXECUTE_RELEASE_WFI                                   20:20
-#define NVC96F_SEM_EXECUTE_RELEASE_WFI_DIS                          0x00000000
-#define NVC96F_SEM_EXECUTE_RELEASE_WFI_EN                           0x00000001
-#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE                                  24:24
-#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT                       0x00000000
-#define NVC96F_SEM_EXECUTE_PAYLOAD_SIZE_64BIT                       0x00000001
-#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP                             25:25
-#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP_DIS                    0x00000000
-#define NVC96F_SEM_EXECUTE_RELEASE_TIMESTAMP_EN                     0x00000001
-#define NVC96F_SEM_EXECUTE_REDUCTION                                     30:27
-#define NVC96F_SEM_EXECUTE_REDUCTION_IMIN                           0x00000000
-#define NVC96F_SEM_EXECUTE_REDUCTION_IMAX                           0x00000001
-#define NVC96F_SEM_EXECUTE_REDUCTION_IXOR                           0x00000002
-#define NVC96F_SEM_EXECUTE_REDUCTION_IAND                           0x00000003
-#define NVC96F_SEM_EXECUTE_REDUCTION_IOR                            0x00000004
-#define NVC96F_SEM_EXECUTE_REDUCTION_IADD                           0x00000005
-#define NVC96F_SEM_EXECUTE_REDUCTION_INC                            0x00000006
-#define NVC96F_SEM_EXECUTE_REDUCTION_DEC                            0x00000007
-#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT                              31:31
-#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT_SIGNED                  0x00000000
-#define NVC96F_SEM_EXECUTE_REDUCTION_FORMAT_UNSIGNED                0x00000001
-#define NVC96F_WFI                                                 (0x00000078)
-#define NVC96F_WFI_SCOPE                                                   0:0
-#define NVC96F_WFI_SCOPE_CURRENT_SCG_TYPE                           0x00000000
-#define NVC96F_WFI_SCOPE_CURRENT_VEID                               0x00000000
-#define NVC96F_WFI_SCOPE_ALL                                        0x00000001
-#define NVC96F_YIELD                                               (0x00000080)
-#define NVC96F_YIELD_OP                                                    1:0
-#define NVC96F_YIELD_OP_NOP                                         0x00000000
-#define NVC96F_YIELD_OP_TSG                                         0x00000003
-#define NVC96F_CLEAR_FAULTED                                       (0x00000084)
-// Note: RM provides the HANDLE as an opaque value; the internal detail fields
-// are intentionally not exposed to the driver through these defines.
-#define NVC96F_CLEAR_FAULTED_HANDLE                                       30:0
-#define NVC96F_CLEAR_FAULTED_TYPE                                        31:31
-#define NVC96F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED                     0x00000000
-#define NVC96F_CLEAR_FAULTED_TYPE_ENG_FAULTED                       0x00000001
-
-
-/* GPFIFO entry format */
-#define NVC96F_GP_ENTRY__SIZE                                          8
-#define NVC96F_GP_ENTRY0_FETCH                                       0:0
-#define NVC96F_GP_ENTRY0_FETCH_UNCONDITIONAL                  0x00000000
-#define NVC96F_GP_ENTRY0_FETCH_CONDITIONAL                    0x00000001
-#define NVC96F_GP_ENTRY0_GET                                        31:2
-#define NVC96F_GP_ENTRY0_OPERAND                                    31:0
-#define NVC96F_GP_ENTRY0_PB_EXTENDED_BASE_OPERAND                   24:8
-#define NVC96F_GP_ENTRY1_GET_HI                                      7:0
-#define NVC96F_GP_ENTRY1_LEVEL                                       9:9
-#define NVC96F_GP_ENTRY1_LEVEL_MAIN                           0x00000000
-#define NVC96F_GP_ENTRY1_LEVEL_SUBROUTINE                     0x00000001
-#define NVC96F_GP_ENTRY1_LENGTH                                    30:10
-#define NVC96F_GP_ENTRY1_SYNC                                      31:31
-#define NVC96F_GP_ENTRY1_SYNC_PROCEED                         0x00000000
-#define NVC96F_GP_ENTRY1_SYNC_WAIT                            0x00000001
-#define NVC96F_GP_ENTRY1_OPCODE                                      7:0
-#define NVC96F_GP_ENTRY1_OPCODE_NOP                           0x00000000
-#define NVC96F_GP_ENTRY1_OPCODE_ILLEGAL                       0x00000001
-#define NVC96F_GP_ENTRY1_OPCODE_GP_CRC                        0x00000002
-#define NVC96F_GP_ENTRY1_OPCODE_PB_CRC                        0x00000003
-#define NVC96F_GP_ENTRY1_OPCODE_SET_PB_SEGMENT_EXTENDED_BASE  0x00000004
-
-/* dma method formats */
-#define NVC96F_DMA_METHOD_ADDRESS_OLD                              12:2
-#define NVC96F_DMA_METHOD_ADDRESS                                  11:0
-#define NVC96F_DMA_SUBDEVICE_MASK                                  15:4
-#define NVC96F_DMA_METHOD_SUBCHANNEL                               15:13
-#define NVC96F_DMA_TERT_OP                                         17:16
-#define NVC96F_DMA_TERT_OP_GRP0_INC_METHOD                         (0x00000000)
-#define NVC96F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK                   (0x00000001)
-#define NVC96F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK                 (0x00000002)
-#define NVC96F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK                   (0x00000003)
-#define NVC96F_DMA_TERT_OP_GRP2_NON_INC_METHOD                     (0x00000000)
-#define NVC96F_DMA_METHOD_COUNT_OLD                                28:18
-#define NVC96F_DMA_METHOD_COUNT                                    28:16
-#define NVC96F_DMA_IMMD_DATA                                       28:16
-#define NVC96F_DMA_SEC_OP                                          31:29
-#define NVC96F_DMA_SEC_OP_GRP0_USE_TERT                            (0x00000000)
-#define NVC96F_DMA_SEC_OP_INC_METHOD                               (0x00000001)
-#define NVC96F_DMA_SEC_OP_GRP2_USE_TERT                            (0x00000002)
-#define NVC96F_DMA_SEC_OP_NON_INC_METHOD                           (0x00000003)
-#define NVC96F_DMA_SEC_OP_IMMD_DATA_METHOD                         (0x00000004)
-#define NVC96F_DMA_SEC_OP_ONE_INC                                  (0x00000005)
-#define NVC96F_DMA_SEC_OP_RESERVED6                                (0x00000006)
-#define NVC96F_DMA_SEC_OP_END_PB_SEGMENT                           (0x00000007)
-/* dma incrementing method format */
-#define NVC96F_DMA_INCR_ADDRESS                                    11:0
-#define NVC96F_DMA_INCR_SUBCHANNEL                                 15:13
-#define NVC96F_DMA_INCR_COUNT                                      28:16
-#define NVC96F_DMA_INCR_OPCODE                                     31:29
-#define NVC96F_DMA_INCR_OPCODE_VALUE                               (0x00000001)
-#define NVC96F_DMA_INCR_DATA                                       31:0
-/* dma non-incrementing method format */
-#define NVC96F_DMA_NONINCR_ADDRESS                                 11:0
-#define NVC96F_DMA_NONINCR_SUBCHANNEL                              15:13
-#define NVC96F_DMA_NONINCR_COUNT                                   28:16
-#define NVC96F_DMA_NONINCR_OPCODE                                  31:29
-#define NVC96F_DMA_NONINCR_OPCODE_VALUE                            (0x00000003)
-#define NVC96F_DMA_NONINCR_DATA                                    31:0
-/* dma increment-once method format */
-#define NVC96F_DMA_ONEINCR_ADDRESS                                 11:0
-#define NVC96F_DMA_ONEINCR_SUBCHANNEL                              15:13
-#define NVC96F_DMA_ONEINCR_COUNT                                   28:16
-#define NVC96F_DMA_ONEINCR_OPCODE                                  31:29
-#define NVC96F_DMA_ONEINCR_OPCODE_VALUE                            (0x00000005)
-#define NVC96F_DMA_ONEINCR_DATA                                    31:0
-/* dma no-operation format */
-#define NVC96F_DMA_NOP                                             (0x00000000)
-/* dma immediate-data format */
-#define NVC96F_DMA_IMMD_ADDRESS                                    11:0
-#define NVC96F_DMA_IMMD_SUBCHANNEL                                 15:13
-#define NVC96F_DMA_IMMD_DATA                                       28:16
-#define NVC96F_DMA_IMMD_OPCODE                                     31:29
-#define NVC96F_DMA_IMMD_OPCODE_VALUE                               (0x00000004)
-/* dma set sub-device mask format */
-#define NVC96F_DMA_SET_SUBDEVICE_MASK_VALUE                        15:4
-#define NVC96F_DMA_SET_SUBDEVICE_MASK_OPCODE                       31:16
-#define NVC96F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE                 (0x00000001)
-/* dma store sub-device mask format */
-#define NVC96F_DMA_STORE_SUBDEVICE_MASK_VALUE                      15:4
-#define NVC96F_DMA_STORE_SUBDEVICE_MASK_OPCODE                     31:16
-#define NVC96F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE               (0x00000002)
-/* dma use sub-device mask format */
-#define NVC96F_DMA_USE_SUBDEVICE_MASK_OPCODE                       31:16
-#define NVC96F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE                 (0x00000003)
-/* dma end-segment format */
-#define NVC96F_DMA_ENDSEG_OPCODE                                   31:29
-#define NVC96F_DMA_ENDSEG_OPCODE_VALUE                             (0x00000007)
-/* dma legacy incrementing/non-incrementing formats */
-#define NVC96F_DMA_ADDRESS                                         12:2
-#define NVC96F_DMA_SUBCH                                           15:13
-#define NVC96F_DMA_OPCODE3                                         17:16
-#define NVC96F_DMA_OPCODE3_NONE                                    (0x00000000)
-#define NVC96F_DMA_COUNT                                           28:18
-#define NVC96F_DMA_OPCODE                                          31:29
-#define NVC96F_DMA_OPCODE_METHOD                                   (0x00000000)
-#define NVC96F_DMA_OPCODE_NONINC_METHOD                            (0x00000002)
-#define NVC96F_DMA_DATA                                            31:0
-
-#ifdef __cplusplus
-};     /* extern "C" */
-#endif
-
-#endif /* _clc96f_h_ */

kernel-open/nvidia-uvm/clc9b5.h

@@ -1,460 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 1993-2004 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-
-#include "nvtypes.h"
-
-#ifndef _clc9b5_h_
-#define _clc9b5_h_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define BLACKWELL_DMA_COPY_A                                                            (0x0000C9B5)
-
-typedef volatile struct _clc9b5_tag0 {
-    NvV32 Reserved00[0x40];
-    NvV32 Nop;                                                                  // 0x00000100 - 0x00000103
-    NvV32 Reserved01[0xF];
-    NvV32 PmTrigger;                                                            // 0x00000140 - 0x00000143
-    NvV32 Reserved02[0x36];
-    NvV32 SetMonitoredFenceType;                                                // 0x0000021C - 0x0000021F
-    NvV32 SetMonitoredFenceSignalAddrBaseUpper;                                 // 0x00000220 - 0x00000223
-    NvV32 SetMonitoredFenceSignalAddrBaseLower;                                 // 0x00000224 - 0x00000227
-    NvV32 Reserved03[0x6];
-    NvV32 SetSemaphoreA;                                                        // 0x00000240 - 0x00000243
-    NvV32 SetSemaphoreB;                                                        // 0x00000244 - 0x00000247
-    NvV32 SetSemaphorePayload;                                                  // 0x00000248 - 0x0000024B
-    NvV32 SetSemaphorePayloadUpper;                                             // 0x0000024C - 0x0000024F
-    NvV32 Reserved04[0x1];
-    NvV32 SetRenderEnableA;                                                     // 0x00000254 - 0x00000257
-    NvV32 SetRenderEnableB;                                                     // 0x00000258 - 0x0000025B
-    NvV32 SetRenderEnableC;                                                     // 0x0000025C - 0x0000025F
-    NvV32 SetSrcPhysMode;                                                       // 0x00000260 - 0x00000263
-    NvV32 SetDstPhysMode;                                                       // 0x00000264 - 0x00000267
-    NvV32 Reserved05[0x26];
-    NvV32 LaunchDma;                                                            // 0x00000300 - 0x00000303
-    NvV32 Reserved06[0x3F];
-    NvV32 OffsetInUpper;                                                        // 0x00000400 - 0x00000403
-    NvV32 OffsetInLower;                                                        // 0x00000404 - 0x00000407
-    NvV32 OffsetOutUpper;                                                       // 0x00000408 - 0x0000040B
-    NvV32 OffsetOutLower;                                                       // 0x0000040C - 0x0000040F
-    NvV32 PitchIn;                                                              // 0x00000410 - 0x00000413
-    NvV32 PitchOut;                                                             // 0x00000414 - 0x00000417
-    NvV32 LineLengthIn;                                                         // 0x00000418 - 0x0000041B
-    NvV32 LineCount;                                                            // 0x0000041C - 0x0000041F
-    NvV32 Reserved07[0x38];
-    NvV32 SetSecureCopyMode;                                                    // 0x00000500 - 0x00000503
-    NvV32 SetDecryptIv0;                                                        // 0x00000504 - 0x00000507
-    NvV32 SetDecryptIv1;                                                        // 0x00000508 - 0x0000050B
-    NvV32 SetDecryptIv2;                                                        // 0x0000050C - 0x0000050F
-    NvV32 Reserved_SetAESCounter;                                               // 0x00000510 - 0x00000513
-    NvV32 SetDecryptAuthTagCompareAddrUpper;                                    // 0x00000514 - 0x00000517
-    NvV32 SetDecryptAuthTagCompareAddrLower;                                    // 0x00000518 - 0x0000051B
-    NvV32 Reserved08[0x5];
-    NvV32 SetEncryptAuthTagAddrUpper;                                           // 0x00000530 - 0x00000533
-    NvV32 SetEncryptAuthTagAddrLower;                                           // 0x00000534 - 0x00000537
-    NvV32 SetEncryptIvAddrUpper;                                                // 0x00000538 - 0x0000053B
-    NvV32 SetEncryptIvAddrLower;                                                // 0x0000053C - 0x0000053F
-    NvV32 Reserved09[0x10];
-    NvV32 SetCompressionParameters;                                             // 0x00000580 - 0x00000583
-    NvV32 SetDecompressOutLength;                                               // 0x00000584 - 0x00000587
-    NvV32 SetDecompressOutLengthAddrUpper;                                      // 0x00000588 - 0x0000058B
-    NvV32 SetDecompressOutLengthAddrLower;                                      // 0x0000058C - 0x0000058F
-    NvV32 SetDecompressChecksum;                                                // 0x00000590 - 0x00000593
-    NvV32 Reserved10[0x5A];
-    NvV32 SetMemoryScrubParameters;                                             // 0x000006FC - 0x000006FF
-    NvV32 SetRemapConstA;                                                       // 0x00000700 - 0x00000703
-    NvV32 SetRemapConstB;                                                       // 0x00000704 - 0x00000707
-    NvV32 SetRemapComponents;                                                   // 0x00000708 - 0x0000070B
-    NvV32 SetDstBlockSize;                                                      // 0x0000070C - 0x0000070F
-    NvV32 SetDstWidth;                                                          // 0x00000710 - 0x00000713
-    NvV32 SetDstHeight;                                                         // 0x00000714 - 0x00000717
-    NvV32 SetDstDepth;                                                          // 0x00000718 - 0x0000071B
-    NvV32 SetDstLayer;                                                          // 0x0000071C - 0x0000071F
-    NvV32 SetDstOrigin;                                                         // 0x00000720 - 0x00000723
-    NvV32 Reserved11[0x1];
-    NvV32 SetSrcBlockSize;                                                      // 0x00000728 - 0x0000072B
-    NvV32 SetSrcWidth;                                                          // 0x0000072C - 0x0000072F
-    NvV32 SetSrcHeight;                                                         // 0x00000730 - 0x00000733
-    NvV32 SetSrcDepth;                                                          // 0x00000734 - 0x00000737
-    NvV32 SetSrcLayer;                                                          // 0x00000738 - 0x0000073B
-    NvV32 SetSrcOrigin;                                                         // 0x0000073C - 0x0000073F
-    NvV32 Reserved12[0x1];
-    NvV32 SrcOriginX;                                                           // 0x00000744 - 0x00000747
-    NvV32 SrcOriginY;                                                           // 0x00000748 - 0x0000074B
-    NvV32 DstOriginX;                                                           // 0x0000074C - 0x0000074F
-    NvV32 DstOriginY;                                                           // 0x00000750 - 0x00000753
-    NvV32 Reserved13[0x270];
-    NvV32 PmTriggerEnd;                                                         // 0x00001114 - 0x00001117
-    NvV32 Reserved14[0x3BA];
-} blackwell_dma_copy_aControlPio;
-
-#define NVC9B5_NOP                                                              (0x00000100)
-#define NVC9B5_NOP_PARAMETER                                                    31:0
-#define NVC9B5_PM_TRIGGER                                                       (0x00000140)
-#define NVC9B5_PM_TRIGGER_V                                                     31:0
-#define NVC9B5_SET_MONITORED_FENCE_TYPE                                         (0x0000021C)
-#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE                                    0:0
-#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE_MONITORED_FENCE                    (0x00000000)
-#define NVC9B5_SET_MONITORED_FENCE_TYPE_TYPE_MONITORED_FENCE_EXT                (0x00000001)
-#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_UPPER                       (0x00000220)
-#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_UPPER_UPPER                 24:0
-#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_LOWER                       (0x00000224)
-#define NVC9B5_SET_MONITORED_FENCE_SIGNAL_ADDR_BASE_LOWER_LOWER                 31:0
-#define NVC9B5_SET_SEMAPHORE_A                                                  (0x00000240)
-#define NVC9B5_SET_SEMAPHORE_A_UPPER                                            24:0
-#define NVC9B5_SET_SEMAPHORE_B                                                  (0x00000244)
-#define NVC9B5_SET_SEMAPHORE_B_LOWER                                            31:0
-#define NVC9B5_SET_SEMAPHORE_PAYLOAD                                            (0x00000248)
-#define NVC9B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD                                    31:0
-#define NVC9B5_SET_SEMAPHORE_PAYLOAD_UPPER                                      (0x0000024C)
-#define NVC9B5_SET_SEMAPHORE_PAYLOAD_UPPER_PAYLOAD                              31:0
-#define NVC9B5_SET_RENDER_ENABLE_A                                              (0x00000254)
-#define NVC9B5_SET_RENDER_ENABLE_A_UPPER                                        24:0
-#define NVC9B5_SET_RENDER_ENABLE_B                                              (0x00000258)
-#define NVC9B5_SET_RENDER_ENABLE_B_LOWER                                        31:0
-#define NVC9B5_SET_RENDER_ENABLE_C                                              (0x0000025C)
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE                                         2:0
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE_FALSE                                   (0x00000000)
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE_TRUE                                    (0x00000001)
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL                             (0x00000002)
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL                         (0x00000003)
-#define NVC9B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL                     (0x00000004)
-#define NVC9B5_SET_SRC_PHYS_MODE                                                (0x00000260)
-#define NVC9B5_SET_SRC_PHYS_MODE_TARGET                                         1:0
-#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB                                (0x00000000)
-#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM                         (0x00000001)
-#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM                      (0x00000002)
-#define NVC9B5_SET_SRC_PHYS_MODE_TARGET_PEERMEM                                 (0x00000003)
-#define NVC9B5_SET_SRC_PHYS_MODE_BASIC_KIND                                     5:2
-#define NVC9B5_SET_SRC_PHYS_MODE_PEER_ID                                        8:6
-#define NVC9B5_SET_SRC_PHYS_MODE_FLA                                            9:9
-#define NVC9B5_SET_DST_PHYS_MODE                                                (0x00000264)
-#define NVC9B5_SET_DST_PHYS_MODE_TARGET                                         1:0
-#define NVC9B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB                                (0x00000000)
-#define NVC9B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM                         (0x00000001)
-#define NVC9B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM                      (0x00000002)
-#define NVC9B5_SET_DST_PHYS_MODE_TARGET_PEERMEM                                 (0x00000003)
-#define NVC9B5_SET_DST_PHYS_MODE_BASIC_KIND                                     5:2
-#define NVC9B5_SET_DST_PHYS_MODE_PEER_ID                                        8:6
-#define NVC9B5_SET_DST_PHYS_MODE_FLA                                            9:9
-#define NVC9B5_LAUNCH_DMA                                                       (0x00000300)
-#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE                                    1:0
-#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE                               (0x00000000)
-#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED                          (0x00000001)
-#define NVC9B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED                      (0x00000002)
-#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE                                          2:2
-#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE                                    (0x00000000)
-#define NVC9B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE                                     (0x00000001)
-#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE                                            25:25
-#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE_SYS                                        (0x00000000)
-#define NVC9B5_LAUNCH_DMA_FLUSH_TYPE_GL                                         (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE                                        4:3
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE                                   (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_SEMAPHORE_NO_TIMESTAMP         (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_SEMAPHORE_WITH_TIMESTAMP       (0x00000002)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE             (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE            (0x00000002)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_CONDITIONAL_INTR_SEMAPHORE     (0x00000003)
-#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE                                        6:5
-#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE                                   (0x00000000)
-#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING                               (0x00000001)
-#define NVC9B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING                           (0x00000002)
-#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT                                     7:7
-#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR                         (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH                               (0x00000001)
-#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT                                     8:8
-#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR                         (0x00000000)
-#define NVC9B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH                               (0x00000001)
-#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE                                     9:9
-#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE                               (0x00000000)
-#define NVC9B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE                                (0x00000001)
-#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE                                          10:10
-#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE_FALSE                                    (0x00000000)
-#define NVC9B5_LAUNCH_DMA_REMAP_ENABLE_TRUE                                     (0x00000001)
-#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE                                    11:11
-#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE_FALSE                              (0x00000000)
-#define NVC9B5_LAUNCH_DMA_COMPRESSION_ENABLE_TRUE                               (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SRC_TYPE                                              12:12
-#define NVC9B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL                                      (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL                                     (0x00000001)
-#define NVC9B5_LAUNCH_DMA_DST_TYPE                                              13:13
-#define NVC9B5_LAUNCH_DMA_DST_TYPE_VIRTUAL                                      (0x00000000)
-#define NVC9B5_LAUNCH_DMA_DST_TYPE_PHYSICAL                                     (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION                                   17:14
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN                              (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX                              (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR                              (0x00000002)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND                              (0x00000003)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR                               (0x00000004)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD                              (0x00000005)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC                               (0x00000006)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC                               (0x00000007)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDA                          (0x00000008)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDB                          (0x00000009)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD                              (0x0000000A)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FMIN                              (0x0000000B)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FMAX                              (0x0000000C)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDC                          (0x0000000D)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDD                          (0x0000000E)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INVALIDE                          (0x0000000F)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN                              18:18
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED                       (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED                     (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE                            19:19
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE                      (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE                       (0x00000001)
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE                                             21:20
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE_PROT2PROT                                   (0x00000000)
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE_DEFAULT                                     (0x00000000)
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE_SECURE                                      (0x00000001)
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE_NONPROT2NONPROT                             (0x00000002)
-#define NVC9B5_LAUNCH_DMA_COPY_TYPE_RESERVED                                    (0x00000003)
-#define NVC9B5_LAUNCH_DMA_VPRMODE                                               22:22
-#define NVC9B5_LAUNCH_DMA_VPRMODE_VPR_NONE                                      (0x00000000)
-#define NVC9B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID                                   (0x00000001)
-#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE                                   23:23
-#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE_FALSE                             (0x00000000)
-#define NVC9B5_LAUNCH_DMA_MEMORY_SCRUB_ENABLE_TRUE                              (0x00000001)
-#define NVC9B5_LAUNCH_DMA_RESERVED_START_OF_COPY                                24:24
-#define NVC9B5_LAUNCH_DMA_DISABLE_PLC                                           26:26
-#define NVC9B5_LAUNCH_DMA_DISABLE_PLC_FALSE                                     (0x00000000)
-#define NVC9B5_LAUNCH_DMA_DISABLE_PLC_TRUE                                      (0x00000001)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE                                27:27
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE_ONE_WORD                       (0x00000000)
-#define NVC9B5_LAUNCH_DMA_SEMAPHORE_PAYLOAD_SIZE_TWO_WORD                       (0x00000001)
-#define NVC9B5_LAUNCH_DMA_RESERVED_ERR_CODE                                     31:28
-#define NVC9B5_OFFSET_IN_UPPER                                                  (0x00000400)
-#define NVC9B5_OFFSET_IN_UPPER_UPPER                                            24:0
-#define NVC9B5_OFFSET_IN_LOWER                                                  (0x00000404)
-#define NVC9B5_OFFSET_IN_LOWER_VALUE                                            31:0
-#define NVC9B5_OFFSET_OUT_UPPER                                                 (0x00000408)
-#define NVC9B5_OFFSET_OUT_UPPER_UPPER                                           24:0
-#define NVC9B5_OFFSET_OUT_LOWER                                                 (0x0000040C)
-#define NVC9B5_OFFSET_OUT_LOWER_VALUE                                           31:0
-#define NVC9B5_PITCH_IN                                                         (0x00000410)
-#define NVC9B5_PITCH_IN_VALUE                                                   31:0
-#define NVC9B5_PITCH_OUT                                                        (0x00000414)
-#define NVC9B5_PITCH_OUT_VALUE                                                  31:0
-#define NVC9B5_LINE_LENGTH_IN                                                   (0x00000418)
-#define NVC9B5_LINE_LENGTH_IN_VALUE                                             31:0
-#define NVC9B5_LINE_COUNT                                                       (0x0000041C)
-#define NVC9B5_LINE_COUNT_VALUE                                                 31:0
-#define NVC9B5_SET_SECURE_COPY_MODE                                             (0x00000500)
-#define NVC9B5_SET_SECURE_COPY_MODE_MODE                                        0:0
-#define NVC9B5_SET_SECURE_COPY_MODE_MODE_ENCRYPT                                (0x00000000)
-#define NVC9B5_SET_SECURE_COPY_MODE_MODE_DECRYPT                                (0x00000001)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET                         20:19
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_LOCAL_FB                (0x00000000)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_COHERENT_SYSMEM         (0x00000001)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_NONCOHERENT_SYSMEM      (0x00000002)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_TARGET_PEERMEM                 (0x00000003)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_PEER_ID                        23:21
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_SRC_FLA                            24:24
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET                         26:25
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_LOCAL_FB                (0x00000000)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_COHERENT_SYSMEM         (0x00000001)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_NONCOHERENT_SYSMEM      (0x00000002)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_TARGET_PEERMEM                 (0x00000003)
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_PEER_ID                        29:27
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_DST_FLA                            30:30
-#define NVC9B5_SET_SECURE_COPY_MODE_RESERVED_END_OF_COPY                        31:31
-#define NVC9B5_SET_DECRYPT_IV0                                                  (0x00000504)
-#define NVC9B5_SET_DECRYPT_IV0_VALUE                                            31:0
-#define NVC9B5_SET_DECRYPT_IV1                                                  (0x00000508)
-#define NVC9B5_SET_DECRYPT_IV1_VALUE                                            31:0
-#define NVC9B5_SET_DECRYPT_IV2                                                  (0x0000050C)
-#define NVC9B5_SET_DECRYPT_IV2_VALUE                                            31:0
-#define NVC9B5_RESERVED_SET_AESCOUNTER                                          (0x00000510)
-#define NVC9B5_RESERVED_SET_AESCOUNTER_VALUE                                    31:0
-#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER                          (0x00000514)
-#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_UPPER_UPPER                    24:0
-#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER                          (0x00000518)
-#define NVC9B5_SET_DECRYPT_AUTH_TAG_COMPARE_ADDR_LOWER_LOWER                    31:0
-#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER                                  (0x00000530)
-#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_UPPER_UPPER                            24:0
-#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER                                  (0x00000534)
-#define NVC9B5_SET_ENCRYPT_AUTH_TAG_ADDR_LOWER_LOWER                            31:0
-#define NVC9B5_SET_ENCRYPT_IV_ADDR_UPPER                                        (0x00000538)
-#define NVC9B5_SET_ENCRYPT_IV_ADDR_UPPER_UPPER                                  24:0
-#define NVC9B5_SET_ENCRYPT_IV_ADDR_LOWER                                        (0x0000053C)
-#define NVC9B5_SET_ENCRYPT_IV_ADDR_LOWER_LOWER                                  31:0
-#define NVC9B5_SET_COMPRESSION_PARAMETERS                                       (0x00000580)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION                             0:0
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION_DECOMPRESS                  (0x00000000)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_OPERATION_COMPRESS                    (0x00000001)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO                                  3:1
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_SNAPPY                           (0x00000000)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_DATA_ONLY                    (0x00000001)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_BLOCK                        (0x00000002)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_LZ4_BLOCK_CHECKSUM               (0x00000003)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_DEFLATE                          (0x00000004)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_ALGO_SNAPPY_WITH_LONG_FETCH           (0x00000005)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM                             29:28
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_NONE                        (0x00000000)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_ADLER32                     (0x00000001)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_CRC32                       (0x00000002)
-#define NVC9B5_SET_COMPRESSION_PARAMETERS_CHECK_SUM_SNAPPY_CRC                  (0x00000003)
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH                                        (0x00000584)
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_V                                      31:0
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_UPPER                             (0x00000588)
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_UPPER_UPPER                       24:0
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_LOWER                             (0x0000058C)
-#define NVC9B5_SET_DECOMPRESS_OUT_LENGTH_ADDR_LOWER_LOWER                       31:0
-#define NVC9B5_SET_DECOMPRESS_CHECKSUM                                          (0x00000590)
-#define NVC9B5_SET_DECOMPRESS_CHECKSUM_V                                        31:0
-#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS                                      (0x000006FC)
-#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE                          0:0
-#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE_FALSE                    (0x00000000)
-#define NVC9B5_SET_MEMORY_SCRUB_PARAMETERS_DISCARDABLE_TRUE                     (0x00000001)
-#define NVC9B5_SET_REMAP_CONST_A                                                (0x00000700)
-#define NVC9B5_SET_REMAP_CONST_A_V                                              31:0
-#define NVC9B5_SET_REMAP_CONST_B                                                (0x00000704)
-#define NVC9B5_SET_REMAP_CONST_B_V                                              31:0
-#define NVC9B5_SET_REMAP_COMPONENTS                                             (0x00000708)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X                                       2:0
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_X                                 (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y                                 (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z                                 (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_SRC_W                                 (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_CONST_A                               (0x00000004)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_CONST_B                               (0x00000005)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE                              (0x00000006)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y                                       6:4
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X                                 (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y                                 (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z                                 (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W                                 (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A                               (0x00000004)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B                               (0x00000005)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE                              (0x00000006)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z                                       10:8
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X                                 (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y                                 (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z                                 (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W                                 (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A                               (0x00000004)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B                               (0x00000005)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE                              (0x00000006)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W                                       14:12
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_X                                 (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y                                 (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z                                 (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_SRC_W                                 (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_CONST_A                               (0x00000004)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_CONST_B                               (0x00000005)
-#define NVC9B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE                              (0x00000006)
-#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE                              17:16
-#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE                          (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO                          (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE                        (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR                         (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS                          21:20
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE                      (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO                      (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE                    (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR                     (0x00000003)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS                          25:24
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE                      (0x00000000)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO                      (0x00000001)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE                    (0x00000002)
-#define NVC9B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR                     (0x00000003)
-#define NVC9B5_SET_DST_BLOCK_SIZE                                               (0x0000070C)
-#define NVC9B5_SET_DST_BLOCK_SIZE_WIDTH                                         3:0
-#define NVC9B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB                                 (0x00000000)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT                                        7:4
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB                                (0x00000000)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS                               (0x00000001)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS                              (0x00000002)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS                             (0x00000003)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS                           (0x00000004)
-#define NVC9B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS                         (0x00000005)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH                                         11:8
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB                                 (0x00000000)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS                                (0x00000001)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS                               (0x00000002)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS                              (0x00000003)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS                            (0x00000004)
-#define NVC9B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS                          (0x00000005)
-#define NVC9B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT                                    15:12
-#define NVC9B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8                 (0x00000001)
-#define NVC9B5_SET_DST_WIDTH                                                    (0x00000710)
-#define NVC9B5_SET_DST_WIDTH_V                                                  31:0
-#define NVC9B5_SET_DST_HEIGHT                                                   (0x00000714)
-#define NVC9B5_SET_DST_HEIGHT_V                                                 31:0
-#define NVC9B5_SET_DST_DEPTH                                                    (0x00000718)
-#define NVC9B5_SET_DST_DEPTH_V                                                  31:0
-#define NVC9B5_SET_DST_LAYER                                                    (0x0000071C)
-#define NVC9B5_SET_DST_LAYER_V                                                  31:0
-#define NVC9B5_SET_DST_ORIGIN                                                   (0x00000720)
-#define NVC9B5_SET_DST_ORIGIN_X                                                 15:0
-#define NVC9B5_SET_DST_ORIGIN_Y                                                 31:16
-#define NVC9B5_SET_SRC_BLOCK_SIZE                                               (0x00000728)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_WIDTH                                         3:0
-#define NVC9B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB                                 (0x00000000)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT                                        7:4
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB                                (0x00000000)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS                               (0x00000001)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS                              (0x00000002)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS                             (0x00000003)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS                           (0x00000004)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS                         (0x00000005)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH                                         11:8
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB                                 (0x00000000)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS                                (0x00000001)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS                               (0x00000002)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS                              (0x00000003)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS                            (0x00000004)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS                          (0x00000005)
-#define NVC9B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT                                    15:12
-#define NVC9B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8                 (0x00000001)
-#define NVC9B5_SET_SRC_WIDTH                                                    (0x0000072C)
-#define NVC9B5_SET_SRC_WIDTH_V                                                  31:0
-#define NVC9B5_SET_SRC_HEIGHT                                                   (0x00000730)
-#define NVC9B5_SET_SRC_HEIGHT_V                                                 31:0
-#define NVC9B5_SET_SRC_DEPTH                                                    (0x00000734)
-#define NVC9B5_SET_SRC_DEPTH_V                                                  31:0
-#define NVC9B5_SET_SRC_LAYER                                                    (0x00000738)
-#define NVC9B5_SET_SRC_LAYER_V                                                  31:0
-#define NVC9B5_SET_SRC_ORIGIN                                                   (0x0000073C)
-#define NVC9B5_SET_SRC_ORIGIN_X                                                 15:0
-#define NVC9B5_SET_SRC_ORIGIN_Y                                                 31:16
-#define NVC9B5_SRC_ORIGIN_X                                                     (0x00000744)
-#define NVC9B5_SRC_ORIGIN_X_VALUE                                               31:0
-#define NVC9B5_SRC_ORIGIN_Y                                                     (0x00000748)
-#define NVC9B5_SRC_ORIGIN_Y_VALUE                                               31:0
-#define NVC9B5_DST_ORIGIN_X                                                     (0x0000074C)
-#define NVC9B5_DST_ORIGIN_X_VALUE                                               31:0
-#define NVC9B5_DST_ORIGIN_Y                                                     (0x00000750)
-#define NVC9B5_DST_ORIGIN_Y_VALUE                                               31:0
-#define NVC9B5_PM_TRIGGER_END                                                   (0x00001114)
-#define NVC9B5_PM_TRIGGER_END_V                                                 31:0
-
-#ifdef __cplusplus
-};     /* extern "C" */
-#endif
-#endif // _clc9b5_h
-

kernel-open/nvidia-uvm/ctrl2080mc.h

@@ -34,7 +34,6 @@
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100                (0x00000170)
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100                (0x00000180)
 #define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100                (0x00000190)
-#define NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100                (0x000001A0)
 
 /* valid ARCHITECTURE_GP10x implementation values */
 #define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GP100              (0x00000000)

kernel-open/nvidia-uvm/hwref/blackwell/gb100/dev_fault.h

@@ -1,546 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2003-2016 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-#ifndef __gb100_dev_fault_h__
-#define __gb100_dev_fault_h__
-/* This file is autogenerated.  Do not edit */
-#define NV_PFAULT                                              /* ----G */
-#define NV_PFAULT_MMU_ENG_ID_GRAPHICS          384 /*       */
-#define NV_PFAULT_MMU_ENG_ID_DISPLAY           1 /*       */
-#define NV_PFAULT_MMU_ENG_ID_GSP               2 /*       */
-#define NV_PFAULT_MMU_ENG_ID_IFB               55 /*       */
-#define NV_PFAULT_MMU_ENG_ID_FLA               4 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1              256 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2              320 /*       */
-#define NV_PFAULT_MMU_ENG_ID_SEC               6 /*       */
-#define NV_PFAULT_MMU_ENG_ID_FSP               7 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF              10 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF0             10 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF1             11 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF2             12 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF3             13 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF4             14 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF5             15 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF6             16 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF7             17 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF8             18 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PERF9             19 /*       */
-#define NV_PFAULT_MMU_ENG_ID_GSPLITE          20 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC             28 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC0            28 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC1            29 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC2            30 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC3            31 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC4            32 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC5            33 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC6            34 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVDEC7            35 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG0            36 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG1            37 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG2            38 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG3            39 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG4            40 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG5            41 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG6            42 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVJPG7            43 /*       */
-#define NV_PFAULT_MMU_ENG_ID_GRCOPY            65 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE0               65 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE1               66 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE2               67 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE3               68 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE4               69 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE5               70 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE6               71 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE7               72 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE8               73 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE9               74 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE10               75 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE11               76 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE12               77 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE13               78 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE14               79 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE15               80 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE16               81 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE17               82 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE18               83 /*       */
-#define NV_PFAULT_MMU_ENG_ID_CE19               84 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PWR_PMU           5 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PTP               3 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVENC0            44 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVENC1            45 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVENC2            46 /*       */
-#define NV_PFAULT_MMU_ENG_ID_NVENC3            47 /*       */
-#define NV_PFAULT_MMU_ENG_ID_OFA0              48 /*       */
-#define NV_PFAULT_MMU_ENG_ID_PHYSICAL          56 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST0             85 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST1             86 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST2             87 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST3             88 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST4             89 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST5             90 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST6             91 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST7             92 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST8             93 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST9             94 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST10            95 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST11            96 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST12            97 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST13            98 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST14            99 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST15            100 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST16            101 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST17            102 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST18            103 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST19            104 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST20            105 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST21            106 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST22            107 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST23            108 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST24            109 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST25            110 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST26            111 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST27            112 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST28            113 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST29            114 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST30            115 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST31            116 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST32            117 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST33            118 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST34            119 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST35            120 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST36            121 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST37            122 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST38            123 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST39            124 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST40            125 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST41            126 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST42            127 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST43            128 /*       */
-#define NV_PFAULT_MMU_ENG_ID_HOST44            129 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN0          256  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN1          257  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN2          258  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN3          259  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN4          260  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN5          261  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN6          262  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN7          263  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN8          264  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN9          265  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN10         266 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN11         267 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN12         268 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN13         269 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN14         270 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN15         271 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN16         272 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN17         273 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN18         274 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN19         275 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN20         276 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN21         277 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN22         278 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN23         279 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN24         280 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN25         281 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN26         282 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN27         283 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN28         284 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN29         285 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN30         286 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN31         287 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN32         288 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN33         289 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN34         290 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN35         291 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN36         292 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN37         293 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN38         294 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN39         295 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN40         296 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN41         297 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN42         298 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN43         299 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN44         300 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN45         301 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN46         302 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN47         303 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN48         304 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN49         305 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN50         306 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN51         307 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN52         308 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN53         309 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN54         310 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN55         311 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN56         312 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN57         313 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN58         314 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN59         315 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN60         316 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN61         317 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN62         318 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR1_FN63         319 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN0          320  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN1          321  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN2          322  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN3          323  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN4          324  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN5          325  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN6          326  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN7          327  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN8          328  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN9          329  /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN10         330 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN11         331 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN12         332 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN13         333 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN14         334 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN15         335 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN16         336 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN17         337 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN18         338 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN19         339 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN20         340 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN21         341 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN22         342 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN23         343 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN24         344 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN25         345 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN26         346 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN27         347 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN28         348 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN29         349 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN30         350 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN31         351 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN32         352 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN33         353 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN34         354 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN35         355 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN36         356 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN37         357 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN38         358 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN39         359 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN40         360 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN41         361 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN42         362 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN43         363 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN44         364 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN45         365 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN46         366 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN47         367 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN48         368 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN49         369 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN50         370 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN51         371 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN52         372 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN53         373 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN54         374 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN55         375 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN56         376 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN57         377 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN58         378 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN59         379 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN60         380 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN61         381 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN62         382 /*       */
-#define NV_PFAULT_MMU_ENG_ID_BAR2_FN63         383 /*       */
-#define NV_PFAULT_FAULT_TYPE                             4:0 /*       */
-#define NV_PFAULT_FAULT_TYPE_PDE                  0x00000000 /*       */
-#define NV_PFAULT_FAULT_TYPE_PDE_SIZE             0x00000001 /*       */
-#define NV_PFAULT_FAULT_TYPE_PTE                  0x00000002 /*       */
-#define NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION   0x00000003 /*       */
-#define NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK   0x00000004 /*       */
-#define NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION       0x00000005 /*       */
-#define NV_PFAULT_FAULT_TYPE_RO_VIOLATION         0x00000006 /*       */
-#define NV_PFAULT_FAULT_TYPE_WO_VIOLATION         0x00000007 /*       */
-#define NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION 0x00000008 /*       */
-#define NV_PFAULT_FAULT_TYPE_WORK_CREATION        0x00000009 /*       */
-#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE 0x0000000a /*       */
-#define NV_PFAULT_FAULT_TYPE_CC_VIOLATION         0x0000000b /*       */
-#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND     0x0000000c /*       */
-#define NV_PFAULT_FAULT_TYPE_REGION_VIOLATION     0x0000000d /*       */
-#define NV_PFAULT_FAULT_TYPE_POISONED             0x0000000e /*       */
-#define NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION     0x0000000f /*       */
-#define NV_PFAULT_CLIENT                       14:8 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_0        0x00000000 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_1        0x00000001 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_2        0x00000002 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_3        0x00000003 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_4        0x00000004 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_5        0x00000005 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_6        0x00000006 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_7        0x00000007 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_0        0x00000008 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_1        0x00000009 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_2        0x0000000A /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_3        0x0000000B /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_4        0x0000000C /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_5        0x0000000D /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_6        0x0000000E /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_7        0x0000000F /*       */
-#define NV_PFAULT_CLIENT_GPC_RAST        0x00000010 /*       */
-#define NV_PFAULT_CLIENT_GPC_GCC         0x00000011 /*       */
-#define NV_PFAULT_CLIENT_GPC_GPCCS       0x00000012 /*       */
-#define NV_PFAULT_CLIENT_GPC_PROP_0      0x00000013 /*       */
-#define NV_PFAULT_CLIENT_GPC_PROP_1      0x00000014 /*       */
-#define NV_PFAULT_CLIENT_GPC_PROP_2      0x00000015 /*       */
-#define NV_PFAULT_CLIENT_GPC_PROP_3      0x00000016 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_8        0x00000021 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_9        0x00000022 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_10       0x00000023 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_11       0x00000024 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_12       0x00000025 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_13       0x00000026 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_14       0x00000027 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_15       0x00000028 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_0     0x00000029 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_1     0x0000002A /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_2     0x0000002B /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_3     0x0000002C /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_4     0x0000002D /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_5     0x0000002E /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_6     0x0000002F /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_7     0x00000030 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_8        0x00000031 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_9        0x00000032 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_8     0x00000033 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_9     0x00000034 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_16       0x00000035 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_17       0x00000036 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_18       0x00000037 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_19       0x00000038 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_10       0x00000039 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_11       0x0000003A /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_10    0x0000003B /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_11    0x0000003C /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_20       0x0000003D /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_21       0x0000003E /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_22       0x0000003F /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_23       0x00000040 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_12       0x00000041 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_13       0x00000042 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_12    0x00000043 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_13    0x00000044 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_24       0x00000045 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_25       0x00000046 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_26       0x00000047 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_27       0x00000048 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_14       0x00000049 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_15       0x0000004A /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_14    0x0000004B /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_15    0x0000004C /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_28       0x0000004D /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_29       0x0000004E /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_30       0x0000004F /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_31       0x00000050 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_16       0x00000051 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_17       0x00000052 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_16    0x00000053 /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_17    0x00000054 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_32       0x00000055 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_33       0x00000056 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_34       0x00000057 /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_35       0x00000058 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_18       0x00000059 /*       */
-#define NV_PFAULT_CLIENT_GPC_PE_19       0x0000005A /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_18    0x0000005B /*       */
-#define NV_PFAULT_CLIENT_GPC_TPCCS_19    0x0000005C /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_36       0x0000005D /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_37       0x0000005E /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_38       0x0000005F /*       */
-#define NV_PFAULT_CLIENT_GPC_T1_39       0x00000060 /*       */
-#define NV_PFAULT_CLIENT_GPC_ROP_0       0x00000070 /*       */
-#define NV_PFAULT_CLIENT_GPC_ROP_1       0x00000071 /*       */
-#define NV_PFAULT_CLIENT_GPC_ROP_2       0x00000072 /*       */
-#define NV_PFAULT_CLIENT_GPC_ROP_3       0x00000073 /*       */
-#define NV_PFAULT_CLIENT_GPC_GPM          0x00000017 /*       */
-#define NV_PFAULT_CLIENT_HUB_VIP         0x00000000 /*       */
-#define NV_PFAULT_CLIENT_HUB_CE0         0x00000001 /*       */
-#define NV_PFAULT_CLIENT_HUB_CE1         0x00000002 /*       */
-#define NV_PFAULT_CLIENT_HUB_DNISO       0x00000003 /*       */
-#define NV_PFAULT_CLIENT_HUB_DISPNISO    0x00000003 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE0         0x00000004 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE          0x00000004 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS0       0x00000005 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS        0x00000005 /*       */
-#define NV_PFAULT_CLIENT_HUB_HOST        0x00000006 /*       */
-#define NV_PFAULT_CLIENT_HUB_HOST_CPU    0x00000007 /*       */
-#define NV_PFAULT_CLIENT_HUB_HOST_CPU_NB 0x00000008 /*       */
-#define NV_PFAULT_CLIENT_HUB_ISO         0x00000009 /*       */
-#define NV_PFAULT_CLIENT_HUB_MMU         0x0000000A /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC0      0x0000000B /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC       0x0000000B /*       */
-#define NV_PFAULT_CLIENT_HUB_CE3         0x0000000C /*       */
-#define NV_PFAULT_CLIENT_HUB_NVENC1      0x0000000D /*       */
-#define NV_PFAULT_CLIENT_HUB_NISO        0x0000000E /*       */
-#define NV_PFAULT_CLIENT_HUB_ACTRS       0x0000000E /*       */
-#define NV_PFAULT_CLIENT_HUB_P2P         0x0000000F /*       */
-#define NV_PFAULT_CLIENT_HUB_PD          0x00000010 /*       */
-#define NV_PFAULT_CLIENT_HUB_PD0         0x00000010 /*       */
-#define NV_PFAULT_CLIENT_HUB_PERF0       0x00000011 /*       */
-#define NV_PFAULT_CLIENT_HUB_PERF        0x00000011 /*       */
-#define NV_PFAULT_CLIENT_HUB_PMU         0x00000012 /*       */
-#define NV_PFAULT_CLIENT_HUB_RASTERTWOD  0x00000013 /*       */
-#define NV_PFAULT_CLIENT_HUB_RASTERTWOD0 0x00000013 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC         0x00000014 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC0        0x00000014 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC_NB      0x00000015 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC_NB0     0x00000015 /*       */
-#define NV_PFAULT_CLIENT_HUB_SEC         0x00000016 /*       */
-#define NV_PFAULT_CLIENT_HUB_SSYNC       0x00000017 /*       */
-#define NV_PFAULT_CLIENT_HUB_SSYNC0      0x00000017 /*       */
-#define NV_PFAULT_CLIENT_HUB_GRCOPY      0x00000018 /*       */
-#define NV_PFAULT_CLIENT_HUB_CE2         0x00000018 /*       */
-#define NV_PFAULT_CLIENT_HUB_XV          0x00000019 /*       */
-#define NV_PFAULT_CLIENT_HUB_MMU_NB      0x0000001A /*       */
-#define NV_PFAULT_CLIENT_HUB_NVENC0      0x0000001B /*       */
-#define NV_PFAULT_CLIENT_HUB_NVENC       0x0000001B /*       */
-#define NV_PFAULT_CLIENT_HUB_DFALCON     0x0000001C /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED0       0x0000001D /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED        0x0000001D /*       */
-#define NV_PFAULT_CLIENT_HUB_PD1         0x0000001E /*       */
-#define NV_PFAULT_CLIENT_HUB_DONT_CARE   0x0000001F /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE0       0x00000020 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE1       0x00000021 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE2       0x00000022 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE3       0x00000023 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE4       0x00000024 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE5       0x00000025 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE6       0x00000026 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSCE7       0x00000027 /*       */
-#define NV_PFAULT_CLIENT_HUB_SSYNC1      0x00000028 /*       */
-#define NV_PFAULT_CLIENT_HUB_SSYNC2      0x00000029 /*       */
-#define NV_PFAULT_CLIENT_HUB_HSHUB       0x0000002A /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X0      0x0000002B /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X1      0x0000002C /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X2      0x0000002D /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X3      0x0000002E /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X4      0x0000002F /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X5      0x00000030 /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X6      0x00000031 /*       */
-#define NV_PFAULT_CLIENT_HUB_PTP_X7      0x00000032 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVENC2      0x00000033 /*       */
-#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 0x00000034 /*       */
-#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 0x00000035 /*       */
-#define NV_PFAULT_CLIENT_HUB_SSYNC3      0x00000036 /*       */
-#define NV_PFAULT_CLIENT_HUB_FBFALCON    0x00000037 /*       */
-#define NV_PFAULT_CLIENT_HUB_CE_SHIM     0x00000038 /*       */
-#define NV_PFAULT_CLIENT_HUB_CE_SHIM0    0x00000038 /*       */
-#define NV_PFAULT_CLIENT_HUB_GSP         0x00000039 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC1      0x0000003A /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC2      0x0000003B /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG0      0x0000003C /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC3      0x0000003D /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC4      0x0000003E /*       */
-#define NV_PFAULT_CLIENT_HUB_OFA0        0x0000003F /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC1        0x00000040 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC_NB1     0x00000041 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC2        0x00000042 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC_NB2     0x00000043 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC3        0x00000044 /*       */
-#define NV_PFAULT_CLIENT_HUB_SCC_NB3     0x00000045 /*       */
-#define NV_PFAULT_CLIENT_HUB_RASTERTWOD1 0x00000046 /*       */
-#define NV_PFAULT_CLIENT_HUB_RASTERTWOD2 0x00000047 /*       */
-#define NV_PFAULT_CLIENT_HUB_RASTERTWOD3 0x00000048 /*       */
-#define NV_PFAULT_CLIENT_HUB_GSPLITE1    0x00000049 /*       */
-#define NV_PFAULT_CLIENT_HUB_GSPLITE2    0x0000004A /*       */
-#define NV_PFAULT_CLIENT_HUB_GSPLITE3    0x0000004B /*       */
-#define NV_PFAULT_CLIENT_HUB_PD2         0x0000004C /*       */
-#define NV_PFAULT_CLIENT_HUB_PD3         0x0000004D /*       */
-#define NV_PFAULT_CLIENT_HUB_FE1         0x0000004E /*       */
-#define NV_PFAULT_CLIENT_HUB_FE2         0x0000004F /*       */
-#define NV_PFAULT_CLIENT_HUB_FE3         0x00000050 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE4         0x00000051 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE5         0x00000052 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE6         0x00000053 /*       */
-#define NV_PFAULT_CLIENT_HUB_FE7         0x00000054 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS1       0x00000055 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS2       0x00000056 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS3       0x00000057 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS4       0x00000058 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS5       0x00000059 /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS6       0x0000005A /*       */
-#define NV_PFAULT_CLIENT_HUB_FECS7       0x0000005B /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED1       0x0000005C /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED2       0x0000005D /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED3       0x0000005E /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED4       0x0000005F /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED5       0x00000060 /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED6       0x00000061 /*       */
-#define NV_PFAULT_CLIENT_HUB_SKED7       0x00000062 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC          0x00000063 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC0         0x00000063 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC1         0x00000064 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC2         0x00000065 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC3         0x00000066 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC4         0x00000067 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC5         0x00000068 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC6         0x00000069 /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC7         0x0000006a /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC8         0x0000006b /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC9         0x0000006c /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC10        0x0000006d /*       */
-#define NV_PFAULT_CLIENT_HUB_ESC11        0x0000006e /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC5      0x0000006F /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC6      0x00000070 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVDEC7      0x00000071 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG1      0x00000072 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG2      0x00000073 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG3      0x00000074 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG4      0x00000075 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG5      0x00000076 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG6      0x00000077 /*       */
-#define NV_PFAULT_CLIENT_HUB_NVJPG7      0x00000078 /*       */
-#define NV_PFAULT_CLIENT_HUB_FSP         0x00000079 /*       */
-#define NV_PFAULT_CLIENT_HUB_BSI         0x0000007A /*       */
-#define NV_PFAULT_CLIENT_HUB_GSPLITE     0x0000007B /*       */
-#define NV_PFAULT_CLIENT_HUB_GSPLITE0    0x0000007B /*       */
-#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER2 0x0000007C /*       */
-#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER3 0x0000007D /*       */
-#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER4 0x0000007E /*       */
-#define NV_PFAULT_CLIENT_HUB_NVENC3      0x0000007F /*       */
-#define NV_PFAULT_ACCESS_TYPE                 19:16 /*       */
-#define NV_PFAULT_ACCESS_TYPE_READ       0x00000000 /*       */
-#define NV_PFAULT_ACCESS_TYPE_WRITE      0x00000001 /*       */
-#define NV_PFAULT_ACCESS_TYPE_ATOMIC     0x00000002 /*       */
-#define NV_PFAULT_ACCESS_TYPE_PREFETCH   0x00000003 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_READ          0x00000000 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_WRITE         0x00000001 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC        0x00000002 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG 0x00000002 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH      0x00000003 /*       */
-#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK   0x00000004 /*       */
-#define NV_PFAULT_ACCESS_TYPE_PHYS_READ          0x00000008 /*       */
-#define NV_PFAULT_ACCESS_TYPE_PHYS_WRITE         0x00000009 /*       */
-#define NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC        0x0000000a /*       */
-#define NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH      0x0000000b /*       */
-#define NV_PFAULT_MMU_CLIENT_TYPE             20:20 /*       */
-#define NV_PFAULT_MMU_CLIENT_TYPE_GPC    0x00000000 /*       */
-#define NV_PFAULT_MMU_CLIENT_TYPE_HUB    0x00000001 /*       */
-#define NV_PFAULT_GPC_ID                      28:24 /*       */
-#define NV_PFAULT_PROTECTED_MODE              29:29 /*       */
-#define NV_PFAULT_REPLAYABLE_FAULT_EN         30:30 /*       */
-#define NV_PFAULT_VALID                       31:31 /*       */
-#endif // __gb100_dev_fault_h__

kernel-open/nvidia-uvm/hwref/blackwell/gb100/dev_mmu.h

@@ -1,560 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2003-2016 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-
-#ifndef __gb100_dev_mmu_h__
-#define __gb100_dev_mmu_h__
-/* This file is autogenerated.  Do not edit */
-#define NV_MMU_PDE                                                      /* ----G */
-#define NV_MMU_PDE_APERTURE_BIG                       (0*32+1):(0*32+0) /* RWXVF */
-#define NV_MMU_PDE_APERTURE_BIG_INVALID                      0x00000000 /* RW--V */
-#define NV_MMU_PDE_APERTURE_BIG_VIDEO_MEMORY                 0x00000001 /* RW--V */
-#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY       0x00000002 /* RW--V */
-#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY   0x00000003 /* RW--V */
-#define NV_MMU_PDE_SIZE                               (0*32+3):(0*32+2) /* RWXVF */
-#define NV_MMU_PDE_SIZE_FULL                                 0x00000000 /* RW--V */
-#define NV_MMU_PDE_SIZE_HALF                                 0x00000001 /* RW--V */
-#define NV_MMU_PDE_SIZE_QUARTER                              0x00000002 /* RW--V */
-#define NV_MMU_PDE_SIZE_EIGHTH                               0x00000003 /* RW--V */
-#define NV_MMU_PDE_ADDRESS_BIG_SYS                   (0*32+31):(0*32+4) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_BIG_VID                   (0*32+31-3):(0*32+4) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER             (0*32+31):(0*32+32-3) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER_0                    0x00000000 /* RW--V */
-#define NV_MMU_PDE_APERTURE_SMALL                     (1*32+1):(1*32+0) /* RWXVF */
-#define NV_MMU_PDE_APERTURE_SMALL_INVALID                    0x00000000 /* RW--V */
-#define NV_MMU_PDE_APERTURE_SMALL_VIDEO_MEMORY               0x00000001 /* RW--V */
-#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY     0x00000002 /* RW--V */
-#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
-#define NV_MMU_PDE_VOL_SMALL                          (1*32+2):(1*32+2) /* RWXVF */
-#define NV_MMU_PDE_VOL_SMALL_TRUE                            0x00000001 /* RW--V */
-#define NV_MMU_PDE_VOL_SMALL_FALSE                           0x00000000 /* RW--V */
-#define NV_MMU_PDE_VOL_BIG                            (1*32+3):(1*32+3) /* RWXVF */
-#define NV_MMU_PDE_VOL_BIG_TRUE                              0x00000001 /* RW--V */
-#define NV_MMU_PDE_VOL_BIG_FALSE                             0x00000000 /* RW--V */
-#define NV_MMU_PDE_ADDRESS_SMALL_SYS                 (1*32+31):(1*32+4) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_SMALL_VID                 (1*32+31-3):(1*32+4) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER           (1*32+31):(1*32+32-3) /* RWXVF */
-#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER_0                  0x00000000 /* RW--V */
-#define NV_MMU_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_PDE__SIZE                                              8
-#define NV_MMU_PTE                                                      /* ----G */
-#define NV_MMU_PTE_VALID                              (0*32+0):(0*32+0) /* RWXVF */
-#define NV_MMU_PTE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_PTE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_PTE_PRIVILEGE                          (0*32+1):(0*32+1) /* RWXVF */
-#define NV_MMU_PTE_PRIVILEGE_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_PTE_PRIVILEGE_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_PTE_READ_ONLY                          (0*32+2):(0*32+2) /* RWXVF */
-#define NV_MMU_PTE_READ_ONLY_TRUE                                  0x1  /* RW--V */
-#define NV_MMU_PTE_READ_ONLY_FALSE                                 0x0  /* RW--V */
-#define NV_MMU_PTE_ENCRYPTED                          (0*32+3):(0*32+3) /* RWXVF */
-#define NV_MMU_PTE_ENCRYPTED_TRUE                            0x00000001 /* R---V */
-#define NV_MMU_PTE_ENCRYPTED_FALSE                           0x00000000 /* R---V */
-#define NV_MMU_PTE_ADDRESS_SYS                      (0*32+31):(0*32+4) /* RWXVF */
-#define NV_MMU_PTE_ADDRESS_VID                      (0*32+31-3):(0*32+4) /* RWXVF */
-#define NV_MMU_PTE_ADDRESS_VID_PEER                (0*32+31):(0*32+32-3) /* RWXVF */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_0                       0x00000000 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_1                       0x00000001 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_2                       0x00000002 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_3                       0x00000003 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_4                       0x00000004 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_5                       0x00000005 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_6                       0x00000006 /* RW--V */
-#define NV_MMU_PTE_ADDRESS_VID_PEER_7                       0x00000007 /* RW--V */
-#define NV_MMU_PTE_VOL                                (1*32+0):(1*32+0) /* RWXVF */
-#define NV_MMU_PTE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_PTE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_PTE_APERTURE                           (1*32+2):(1*32+1) /* RWXVF */
-#define NV_MMU_PTE_APERTURE_VIDEO_MEMORY                     0x00000000 /* RW--V */
-#define NV_MMU_PTE_APERTURE_PEER_MEMORY                      0x00000001 /* RW--V */
-#define NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_PTE_LOCK                               (1*32+3):(1*32+3) /* RWXVF */
-#define NV_MMU_PTE_LOCK_TRUE                                        0x1 /* RW--V */
-#define NV_MMU_PTE_LOCK_FALSE                                       0x0 /* RW--V */
-#define NV_MMU_PTE_ATOMIC_DISABLE                     (1*32+3):(1*32+3) /* RWXVF */
-#define NV_MMU_PTE_ATOMIC_DISABLE_TRUE                              0x1 /* RW--V */
-#define NV_MMU_PTE_ATOMIC_DISABLE_FALSE                             0x0 /* RW--V */
-#define NV_MMU_PTE_COMPTAGLINE                      (1*32+20+11):(1*32+12) /* RWXVF */
-#define NV_MMU_PTE_READ_DISABLE                     (1*32+30):(1*32+30) /* RWXVF */
-#define NV_MMU_PTE_READ_DISABLE_TRUE                               0x1  /* RW--V */
-#define NV_MMU_PTE_READ_DISABLE_FALSE                              0x0  /* RW--V */
-#define NV_MMU_PTE_WRITE_DISABLE                    (1*32+31):(1*32+31) /* RWXVF */
-#define NV_MMU_PTE_WRITE_DISABLE_TRUE                              0x1  /* RW--V */
-#define NV_MMU_PTE_WRITE_DISABLE_FALSE                             0x0  /* RW--V */
-#define NV_MMU_PTE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_PTE__SIZE                                             8
-#define NV_MMU_PTE_COMPTAGS_NONE                                    0x0 /*       */
-#define NV_MMU_PTE_COMPTAGS_1                                       0x1 /*       */
-#define NV_MMU_PTE_COMPTAGS_2                                       0x2 /*       */
-#define NV_MMU_PTE_KIND                              (1*32+7):(1*32+4) /* RWXVF */
-#define NV_MMU_PTE_KIND_INVALID                       0x07 /* R---V */
-#define NV_MMU_PTE_KIND_PITCH                         0x00 /* R---V */
-#define NV_MMU_PTE_KIND_GENERIC_MEMORY                                                  0x6 /* R---V */
-#define NV_MMU_PTE_KIND_Z16                                                             0x1 /* R---V */
-#define NV_MMU_PTE_KIND_S8                                                              0x2 /* R---V */
-#define NV_MMU_PTE_KIND_S8Z24                                                           0x3 /* R---V */
-#define NV_MMU_PTE_KIND_ZF32_X24S8                                                      0x4 /* R---V */
-#define NV_MMU_PTE_KIND_Z24S8                                                           0x5 /* R---V */
-#define NV_MMU_PTE_KIND_GENERIC_MEMORY_COMPRESSIBLE                                     0x8 /* R---V */
-#define NV_MMU_PTE_KIND_GENERIC_MEMORY_COMPRESSIBLE_DISABLE_PLC                         0x9 /* R---V */
-#define NV_MMU_PTE_KIND_S8_COMPRESSIBLE_DISABLE_PLC                                     0xA /* R---V */
-#define NV_MMU_PTE_KIND_Z16_COMPRESSIBLE_DISABLE_PLC                                    0xB /* R---V */
-#define NV_MMU_PTE_KIND_S8Z24_COMPRESSIBLE_DISABLE_PLC                                  0xC /* R---V */
-#define NV_MMU_PTE_KIND_ZF32_X24S8_COMPRESSIBLE_DISABLE_PLC                             0xD /* R---V */
-#define NV_MMU_PTE_KIND_Z24S8_COMPRESSIBLE_DISABLE_PLC                                  0xE /* R---V */
-#define NV_MMU_PTE_KIND_SMSKED_MESSAGE                                                  0xF /* R---V */
-#define NV_MMU_VER1_PDE                                                      /* ----G */
-#define NV_MMU_VER1_PDE_APERTURE_BIG                       (0*32+1):(0*32+0) /* RWXVF */
-#define NV_MMU_VER1_PDE_APERTURE_BIG_INVALID                      0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_BIG_VIDEO_MEMORY                 0x00000001 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY       0x00000002 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY   0x00000003 /* RW--V */
-#define NV_MMU_VER1_PDE_SIZE                               (0*32+3):(0*32+2) /* RWXVF */
-#define NV_MMU_VER1_PDE_SIZE_FULL                                 0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_SIZE_HALF                                 0x00000001 /* RW--V */
-#define NV_MMU_VER1_PDE_SIZE_QUARTER                              0x00000002 /* RW--V */
-#define NV_MMU_VER1_PDE_SIZE_EIGHTH                               0x00000003 /* RW--V */
-#define NV_MMU_VER1_PDE_ADDRESS_BIG_SYS                   (0*32+31):(0*32+4) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID                   (0*32+31-3):(0*32+4) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER             (0*32+31):(0*32+32-3) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER_0                    0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_SMALL                     (1*32+1):(1*32+0) /* RWXVF */
-#define NV_MMU_VER1_PDE_APERTURE_SMALL_INVALID                    0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_SMALL_VIDEO_MEMORY               0x00000001 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY     0x00000002 /* RW--V */
-#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
-#define NV_MMU_VER1_PDE_VOL_SMALL                          (1*32+2):(1*32+2) /* RWXVF */
-#define NV_MMU_VER1_PDE_VOL_SMALL_TRUE                            0x00000001 /* RW--V */
-#define NV_MMU_VER1_PDE_VOL_SMALL_FALSE                           0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_VOL_BIG                            (1*32+3):(1*32+3) /* RWXVF */
-#define NV_MMU_VER1_PDE_VOL_BIG_TRUE                              0x00000001 /* RW--V */
-#define NV_MMU_VER1_PDE_VOL_BIG_FALSE                             0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_ADDRESS_SMALL_SYS                 (1*32+31):(1*32+4) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID                 (1*32+31-3):(1*32+4) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER           (1*32+31):(1*32+32-3) /* RWXVF */
-#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER_0                  0x00000000 /* RW--V */
-#define NV_MMU_VER1_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER1_PDE__SIZE                                              8
-#define NV_MMU_VER1_PTE                                                      /* ----G */
-#define NV_MMU_VER1_PTE_VALID                              (0*32+0):(0*32+0) /* RWXVF */
-#define NV_MMU_VER1_PTE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER1_PTE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER1_PTE_PRIVILEGE                          (0*32+1):(0*32+1) /* RWXVF */
-#define NV_MMU_VER1_PTE_PRIVILEGE_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_VER1_PTE_PRIVILEGE_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_VER1_PTE_READ_ONLY                          (0*32+2):(0*32+2) /* RWXVF */
-#define NV_MMU_VER1_PTE_READ_ONLY_TRUE                                  0x1  /* RW--V */
-#define NV_MMU_VER1_PTE_READ_ONLY_FALSE                                 0x0  /* RW--V */
-#define NV_MMU_VER1_PTE_ENCRYPTED                          (0*32+3):(0*32+3) /* RWXVF */
-#define NV_MMU_VER1_PTE_ENCRYPTED_TRUE                            0x00000001 /* R---V */
-#define NV_MMU_VER1_PTE_ENCRYPTED_FALSE                           0x00000000 /* R---V */
-#define NV_MMU_VER1_PTE_ADDRESS_SYS                      (0*32+31):(0*32+4) /* RWXVF */
-#define NV_MMU_VER1_PTE_ADDRESS_VID                      (0*32+31-3):(0*32+4) /* RWXVF */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER                (0*32+31):(0*32+32-3) /* RWXVF */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_0                       0x00000000 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_1                       0x00000001 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_2                       0x00000002 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_3                       0x00000003 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_4                       0x00000004 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_5                       0x00000005 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_6                       0x00000006 /* RW--V */
-#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_7                       0x00000007 /* RW--V */
-#define NV_MMU_VER1_PTE_VOL                                (1*32+0):(1*32+0) /* RWXVF */
-#define NV_MMU_VER1_PTE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_VER1_PTE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_VER1_PTE_APERTURE                           (1*32+2):(1*32+1) /* RWXVF */
-#define NV_MMU_VER1_PTE_APERTURE_VIDEO_MEMORY                     0x00000000 /* RW--V */
-#define NV_MMU_VER1_PTE_APERTURE_PEER_MEMORY                      0x00000001 /* RW--V */
-#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_VER1_PTE_ATOMIC_DISABLE                     (1*32+3):(1*32+3) /* RWXVF */
-#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_TRUE                              0x1 /* RW--V */
-#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_FALSE                             0x0 /* RW--V */
-#define NV_MMU_VER1_PTE_COMPTAGLINE                      (1*32+20+11):(1*32+12) /* RWXVF */
-#define NV_MMU_VER1_PTE_KIND                              (1*32+11):(1*32+4) /* RWXVF */
-#define NV_MMU_VER1_PTE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER1_PTE__SIZE                                             8
-#define NV_MMU_VER1_PTE_COMPTAGS_NONE                                    0x0 /*       */
-#define NV_MMU_VER1_PTE_COMPTAGS_1                                       0x1 /*       */
-#define NV_MMU_VER1_PTE_COMPTAGS_2                                       0x2 /*       */
-#define NV_MMU_NEW_PDE                                                      /* ----G */
-#define NV_MMU_NEW_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_NEW_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_NEW_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_NEW_PDE_IS_PDE                                           0:0 /* RWXVF */
-#define NV_MMU_NEW_PDE_IS_PDE_TRUE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_PDE_IS_PDE_FALSE                                     0x1 /* RW--V */
-#define NV_MMU_NEW_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_NEW_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_NEW_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_PDE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_NEW_PDE_APERTURE_INVALID                          0x00000000 /* RW--V */
-#define NV_MMU_NEW_PDE_APERTURE_VIDEO_MEMORY                     0x00000001 /* RW--V */
-#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_NEW_PDE_VOL                                              3:3 /* RWXVF */
-#define NV_MMU_NEW_PDE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_NEW_PDE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_NEW_PDE_NO_ATS                                            5:5 /* RWXVF */
-#define NV_MMU_NEW_PDE_NO_ATS_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_NEW_PDE_NO_ATS_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_PDE_ADDRESS_SYS                                     53:8 /* RWXVF */
-#define NV_MMU_NEW_PDE_ADDRESS_VID             (35-3):8 /* RWXVF */
-#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER       35:(36-3) /* RWXVF */
-#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER_0                        0x00000000 /* RW--V */
-#define NV_MMU_NEW_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_NEW_PDE__SIZE                                              8
-#define NV_MMU_NEW_DUAL_PDE                                                      /* ----G */
-#define NV_MMU_NEW_DUAL_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_IS_PDE                                           0:0 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_IS_PDE_TRUE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_IS_PDE_FALSE                                     0x1 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG                                     2:1 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_INVALID                      0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY                 0x00000001 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY       0x00000002 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY   0x00000003 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VOL_BIG                                          3:3 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_TRUE                              0x00000001 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_FALSE                             0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_NO_ATS                                       5:5 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_NO_ATS_TRUE                                  0x1 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_NO_ATS_FALSE                                 0x0 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SYS                                 53:(8-4) /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID         (35-3):(8-4) /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER   35:(36-3) /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER_0                    0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL                                 66:65 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_INVALID                    0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY               0x00000001 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY     0x00000002 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL                                      67:67 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_TRUE                            0x00000001 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_FALSE                           0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_SYS                             117:72 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID      (99-3):72 /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0                  0x00000000 /* RW--V */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /*       */
-#define NV_MMU_NEW_DUAL_PDE__SIZE                                             16
-#define NV_MMU_NEW_PTE                                                      /* ----G */
-#define NV_MMU_NEW_PTE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_NEW_PTE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_NEW_PTE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_NEW_PTE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_NEW_PTE_APERTURE_VIDEO_MEMORY                     0x00000000 /* RW--V */
-#define NV_MMU_NEW_PTE_APERTURE_PEER_MEMORY                      0x00000001 /* RW--V */
-#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_NEW_PTE_VOL                                              3:3 /* RWXVF */
-#define NV_MMU_NEW_PTE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_NEW_PTE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_NEW_PTE_ENCRYPTED                                        4:4 /* RWXVF */
-#define NV_MMU_NEW_PTE_ENCRYPTED_TRUE                            0x00000001 /* R---V */
-#define NV_MMU_NEW_PTE_ENCRYPTED_FALSE                           0x00000000 /* R---V */
-#define NV_MMU_NEW_PTE_PRIVILEGE                                        5:5 /* RWXVF */
-#define NV_MMU_NEW_PTE_PRIVILEGE_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_NEW_PTE_PRIVILEGE_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_NEW_PTE_READ_ONLY                                        6:6 /* RWXVF */
-#define NV_MMU_NEW_PTE_READ_ONLY_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_NEW_PTE_READ_ONLY_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_NEW_PTE_ATOMIC_DISABLE                                   7:7 /* RWXVF */
-#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_TRUE                              0x1 /* RW--V */
-#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_FALSE                             0x0 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_SYS                                     53:8 /* RWXVF */
-#define NV_MMU_NEW_PTE_ADDRESS_VID             (35-3):8 /* RWXVF */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER       35:(36-3) /* RWXVF */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_0                        0x00000000 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_1                        0x00000001 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_2                        0x00000002 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_3                        0x00000003 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_4                        0x00000004 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_5                        0x00000005 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_6                        0x00000006 /* RW--V */
-#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_7                        0x00000007 /* RW--V */
-#define NV_MMU_NEW_PTE_COMPTAGLINE   (20+35):36 /* RWXVF */
-#define NV_MMU_NEW_PTE_KIND                                           63:56 /* RWXVF */
-#define NV_MMU_NEW_PTE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_NEW_PTE__SIZE                                              8
-#define NV_MMU_VER2_PDE                                                      /* ----G */
-#define NV_MMU_VER2_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_VER2_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_VER2_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_VER2_PDE_IS_PDE                                           0:0 /* RWXVF */
-#define NV_MMU_VER2_PDE_IS_PDE_TRUE                                      0x0 /* RW--V */
-#define NV_MMU_VER2_PDE_IS_PDE_FALSE                                     0x1 /* RW--V */
-#define NV_MMU_VER2_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER2_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER2_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER2_PDE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_VER2_PDE_APERTURE_INVALID                          0x00000000 /* RW--V */
-#define NV_MMU_VER2_PDE_APERTURE_VIDEO_MEMORY                     0x00000001 /* RW--V */
-#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_VER2_PDE_VOL                                              3:3 /* RWXVF */
-#define NV_MMU_VER2_PDE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_VER2_PDE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_VER2_PDE_NO_ATS                                           5:5 /* RWXVF */
-#define NV_MMU_VER2_PDE_NO_ATS_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_VER2_PDE_NO_ATS_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_VER2_PDE_ADDRESS_SYS                                     53:8 /* RWXVF */
-#define NV_MMU_VER2_PDE_ADDRESS_VID             (35-3):8 /* RWXVF */
-#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER       35:(36-3) /* RWXVF */
-#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER_0                        0x00000000 /* RW--V */
-#define NV_MMU_VER2_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER2_PDE__SIZE                                              8
-#define NV_MMU_VER2_DUAL_PDE                                                      /* ----G */
-#define NV_MMU_VER2_DUAL_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_IS_PDE                                           0:0 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_IS_PDE_TRUE                                      0x0 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_IS_PDE_FALSE                                     0x1 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG                                     2:1 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_INVALID                      0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY                 0x00000001 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY       0x00000002 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY   0x00000003 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VOL_BIG                                          3:3 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_TRUE                              0x00000001 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_FALSE                             0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_NO_ATS                                      5:5 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_NO_ATS_TRUE                                 0x1 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_NO_ATS_FALSE                                0x0 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SYS                                 53:(8-4) /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID         (35-3):(8-4) /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER   35:(36-3) /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER_0                    0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL                                 66:65 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_INVALID                    0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY               0x00000001 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY     0x00000002 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL                                      67:67 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_TRUE                            0x00000001 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_FALSE                           0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_SYS                             117:72 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID      (99-3):72 /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0                  0x00000000 /* RW--V */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /*       */
-#define NV_MMU_VER2_DUAL_PDE__SIZE                                             16
-#define NV_MMU_VER2_PTE                                                      /* ----G */
-#define NV_MMU_VER2_PTE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER2_PTE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER2_PTE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER2_PTE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY                     0x00000000 /* RW--V */
-#define NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY                      0x00000001 /* RW--V */
-#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_VER2_PTE_VOL                                              3:3 /* RWXVF */
-#define NV_MMU_VER2_PTE_VOL_TRUE                                  0x00000001 /* RW--V */
-#define NV_MMU_VER2_PTE_VOL_FALSE                                 0x00000000 /* RW--V */
-#define NV_MMU_VER2_PTE_ENCRYPTED                                        4:4 /* RWXVF */
-#define NV_MMU_VER2_PTE_ENCRYPTED_TRUE                            0x00000001 /* R---V */
-#define NV_MMU_VER2_PTE_ENCRYPTED_FALSE                           0x00000000 /* R---V */
-#define NV_MMU_VER2_PTE_PRIVILEGE                                        5:5 /* RWXVF */
-#define NV_MMU_VER2_PTE_PRIVILEGE_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_VER2_PTE_PRIVILEGE_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_VER2_PTE_READ_ONLY                                        6:6 /* RWXVF */
-#define NV_MMU_VER2_PTE_READ_ONLY_TRUE                                   0x1 /* RW--V */
-#define NV_MMU_VER2_PTE_READ_ONLY_FALSE                                  0x0 /* RW--V */
-#define NV_MMU_VER2_PTE_ATOMIC_DISABLE                                   7:7 /* RWXVF */
-#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_TRUE                              0x1 /* RW--V */
-#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_FALSE                             0x0 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_SYS                                     53:8 /* RWXVF */
-#define NV_MMU_VER2_PTE_ADDRESS_VID             (35-3):8 /* RWXVF */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER       35:(36-3) /* RWXVF */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_0                        0x00000000 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_1                        0x00000001 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_2                        0x00000002 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_3                        0x00000003 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_4                        0x00000004 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_5                        0x00000005 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_6                        0x00000006 /* RW--V */
-#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_7                        0x00000007 /* RW--V */
-#define NV_MMU_VER2_PTE_COMPTAGLINE   (20+35):36 /* RWXVF */
-#define NV_MMU_VER2_PTE_KIND                                           63:56 /* RWXVF */
-#define NV_MMU_VER2_PTE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER2_PTE__SIZE                                              8
-#define NV_MMU_VER3_PDE                                                      /* ----G */
-#define NV_MMU_VER3_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_VER3_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_VER3_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_VER3_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER3_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER3_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER3_PDE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_VER3_PDE_APERTURE_INVALID                          0x00000000 /* RW--V */
-#define NV_MMU_VER3_PDE_APERTURE_VIDEO_MEMORY                     0x00000001 /* RW--V */
-#define NV_MMU_VER3_PDE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_VER3_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF                                                                        5:3 /* RWXVF */
-#define NV_MMU_VER3_PDE_PCF_VALID_CACHED_ATS_ALLOWED__OR__INVALID_ATS_ALLOWED               0x00000000 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_CACHED_ATS_ALLOWED                                        0x00000000 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_INVALID_ATS_ALLOWED                                             0x00000000 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_UNCACHED_ATS_ALLOWED__OR__SPARSE_ATS_ALLOWED              0x00000001 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_UNCACHED_ATS_ALLOWED                                      0x00000001 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_SPARSE_ATS_ALLOWED                                              0x00000001 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_CACHED_ATS_NOT_ALLOWED__OR__INVALID_ATS_NOT_ALLOWED       0x00000002 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_CACHED_ATS_NOT_ALLOWED                                    0x00000002 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_INVALID_ATS_NOT_ALLOWED                                         0x00000002 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_UNCACHED_ATS_NOT_ALLOWED__OR__SPARSE_ATS_NOT_ALLOWED      0x00000003 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_VALID_UNCACHED_ATS_NOT_ALLOWED                                  0x00000003 /* RW--V */
-#define NV_MMU_VER3_PDE_PCF_SPARSE_ATS_NOT_ALLOWED                                          0x00000003 /* RW--V */
-#define NV_MMU_VER3_PDE_ADDRESS                                             51:12 /* RWXVF */
-#define NV_MMU_VER3_PDE_ADDRESS_SHIFT                                  0x0000000c /*       */
-#define NV_MMU_VER3_PDE__SIZE                                              8
-#define NV_MMU_VER3_DUAL_PDE                                                      /* ----G */
-#define NV_MMU_VER3_DUAL_PDE_IS_PTE                                           0:0 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_IS_PTE_TRUE                                      0x1 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_IS_PTE_FALSE                                     0x0 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_BIG                                     2:1 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_BIG_INVALID                      0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY                 0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY       0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY   0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG                                                                        5:3 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_CACHED_ATS_ALLOWED__OR__INVALID_ATS_ALLOWED               0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_CACHED_ATS_ALLOWED                                        0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_INVALID_ATS_ALLOWED                                             0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_UNCACHED_ATS_ALLOWED__OR__SPARSE_ATS_ALLOWED              0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_UNCACHED_ATS_ALLOWED                                      0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_SPARSE_ATS_ALLOWED                                              0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_CACHED_ATS_NOT_ALLOWED__OR__INVALID_ATS_NOT_ALLOWED       0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_CACHED_ATS_NOT_ALLOWED                                    0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_INVALID_ATS_NOT_ALLOWED                                         0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_UNCACHED_ATS_NOT_ALLOWED__OR__SPARSE_ATS_NOT_ALLOWED      0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_VALID_UNCACHED_ATS_NOT_ALLOWED                                  0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_BIG_SPARSE_ATS_NOT_ALLOWED                                          0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_ADDRESS_BIG                                     51:8 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL                                 66:65 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL_INVALID                    0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY               0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY     0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL                                                                      69:67 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_CACHED_ATS_ALLOWED__OR__INVALID_ATS_ALLOWED               0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_CACHED_ATS_ALLOWED                                        0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_INVALID_ATS_ALLOWED                                             0x00000000 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_UNCACHED_ATS_ALLOWED__OR__SPARSE_ATS_ALLOWED              0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_UNCACHED_ATS_ALLOWED                                      0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_SPARSE_ATS_ALLOWED                                              0x00000001 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_CACHED_ATS_NOT_ALLOWED__OR__INVALID_ATS_NOT_ALLOWED       0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_CACHED_ATS_NOT_ALLOWED                                    0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_INVALID_ATS_NOT_ALLOWED                                         0x00000002 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_UNCACHED_ATS_NOT_ALLOWED__OR__SPARSE_ATS_NOT_ALLOWED      0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_VALID_UNCACHED_ATS_NOT_ALLOWED                                  0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_PCF_SMALL_SPARSE_ATS_NOT_ALLOWED                                          0x00000003 /* RW--V */
-#define NV_MMU_VER3_DUAL_PDE_ADDRESS_SMALL                                 115:76 /* RWXVF */
-#define NV_MMU_VER3_DUAL_PDE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER3_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /*       */
-#define NV_MMU_VER3_DUAL_PDE__SIZE                                             16
-#define NV_MMU_VER3_PTE                                                      /* ----G */
-#define NV_MMU_VER3_PTE_VALID                                            0:0 /* RWXVF */
-#define NV_MMU_VER3_PTE_VALID_TRUE                                       0x1 /* RW--V */
-#define NV_MMU_VER3_PTE_VALID_FALSE                                      0x0 /* RW--V */
-#define NV_MMU_VER3_PTE_APERTURE                                         2:1 /* RWXVF */
-#define NV_MMU_VER3_PTE_APERTURE_VIDEO_MEMORY                     0x00000000 /* RW--V */
-#define NV_MMU_VER3_PTE_APERTURE_PEER_MEMORY                      0x00000001 /* RW--V */
-#define NV_MMU_VER3_PTE_APERTURE_SYSTEM_COHERENT_MEMORY           0x00000002 /* RW--V */
-#define NV_MMU_VER3_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY       0x00000003 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF                                                                        7:3 /* RWXVF */
-#define NV_MMU_VER3_PTE_PCF_INVALID                                                         0x00000000 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_SPARSE                                                          0x00000001 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_MAPPING_NOWHERE                                                 0x00000002 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_NO_VALID_4KB_PAGE                                               0x00000003 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACE                                    0x00000000 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACE                                  0x00000001 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_CACHED_ACE                                  0x00000002 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_UNCACHED_ACE                                0x00000003 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_CACHED_ACE                                    0x00000004 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_UNCACHED_ACE                                   0x00000005 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_CACHED_ACE                                  0x00000006 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_UNCACHED_ACE                                0x00000007 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_CACHED_ACE                                 0x00000008 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_UNCACHED_ACE                               0x00000009 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_NO_ATOMIC_CACHED_ACE                               0x0000000A /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_NO_ATOMIC_UNCACHED_ACE                             0x0000000B /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_CACHED_ACE                                 0x0000000C /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_UNCACHED_ACE                               0x0000000D /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_NO_ATOMIC_CACHED_ACE                               0x0000000E /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_NO_ATOMIC_UNCACHED_ACE                             0x0000000F /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACD                                    0x00000010 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACD                                  0x00000011 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_CACHED_ACD                                  0x00000012 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_UNCACHED_ACD                                0x00000013 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_CACHED_ACD                                    0x00000014 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_UNCACHED_ACD                                  0x00000015 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_CACHED_ACD                                  0x00000016 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_UNCACHED_ACD                                0x00000017 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_CACHED_ACD                                 0x00000018 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_UNCACHED_ACD                               0x00000019 /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_NO_ATOMIC_CACHED_ACD                               0x0000001A /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_NO_ATOMIC_UNCACHED_ACD                             0x0000001B /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_CACHED_ACD                                 0x0000001C /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_UNCACHED_ACD                               0x0000001D /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_NO_ATOMIC_CACHED_ACD                               0x0000001E /* RW--V */
-#define NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_NO_ATOMIC_UNCACHED_ACD                             0x0000001F /* RW--V */
-#define NV_MMU_VER3_PTE_KIND                                           11:8 /* RWXVF */
-#define NV_MMU_VER3_PTE_ADDRESS                                         51:12 /* RWXVF */
-#define NV_MMU_VER3_PTE_ADDRESS_SYS                                     51:12 /* RWXVF */
-#define NV_MMU_VER3_PTE_ADDRESS_PEER                                    51:12 /* RWXVF */
-#define NV_MMU_VER3_PTE_ADDRESS_VID                                     39:12 /* RWXVF */
-#define NV_MMU_VER3_PTE_PEER_ID                63:(64-3) /* RWXVF */
-#define NV_MMU_VER3_PTE_PEER_ID_0                                 0x00000000 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_1                                 0x00000001 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_2                                 0x00000002 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_3                                 0x00000003 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_4                                 0x00000004 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_5                                 0x00000005 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_6                                 0x00000006 /* RW--V */
-#define NV_MMU_VER3_PTE_PEER_ID_7                                 0x00000007 /* RW--V */
-#define NV_MMU_VER3_PTE_ADDRESS_SHIFT                             0x0000000c /*       */
-#define NV_MMU_VER3_PTE__SIZE                                              8
-#define NV_MMU_CLIENT                                             /* ----G */
-#define NV_MMU_CLIENT_KIND                                    2:0 /* RWXVF */
-#define NV_MMU_CLIENT_KIND_Z16                                0x1 /* R---V */
-#define NV_MMU_CLIENT_KIND_S8                                 0x2 /* R---V */
-#define NV_MMU_CLIENT_KIND_S8Z24                              0x3 /* R---V */
-#define NV_MMU_CLIENT_KIND_ZF32_X24S8                         0x4 /* R---V */
-#define NV_MMU_CLIENT_KIND_Z24S8                              0x5 /* R---V */
-#define NV_MMU_CLIENT_KIND_GENERIC_MEMORY                     0x6 /* R---V */
-#define NV_MMU_CLIENT_KIND_INVALID                            0x7 /* R---V */
-#endif // __gb100_dev_mmu_h__

kernel-open/nvidia-uvm/nv-kthread-q-selftest.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016 NVIDIA Corporation
+    Copyright (c) 2016-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -81,7 +81,7 @@
 #define NUM_Q_ITEMS_IN_MULTITHREAD_TEST (NUM_TEST_Q_ITEMS * NUM_TEST_KTHREADS)
 
 // This exists in order to have a function to place a breakpoint on:
-void on_nvq_assert(void)
+static void on_nvq_assert(void)
 {
     (void)NULL;
 }

kernel-open/nvidia-uvm/nv-kthread-q.c

@@ -201,7 +201,7 @@ static struct task_struct *thread_create_on_node(int (*threadfn)(void *data),
 
         // Ran out of attempts - return thread even if its stack may not be
         // allocated on the preferred node
-        if ((i == (attempts - 1)))
+        if (i == (attempts - 1))
             break;
 
         // Get the NUMA node where the first page of the stack is resident. If

kernel-open/nvidia-uvm/nvidia-uvm-sources.Kbuild

@@ -6,13 +6,8 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_conf_computing.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_sec2_test.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_sec2.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_sec2.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_fault_buffer.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_mmu.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_blackwell_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_common.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_linux.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_debug_optimized.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nvstatus.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nvCpuUuid.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/nv-kthread-q.c
@@ -76,7 +71,6 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_host.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_ce.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_host.c
-NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_fault_buffer.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_mmu.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper.c
 NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hopper_fault_buffer.c

kernel-open/nvidia-uvm/nvidia-uvm.Kbuild

@@ -13,19 +13,6 @@ NVIDIA_UVM_OBJECTS =
 include $(src)/nvidia-uvm/nvidia-uvm-sources.Kbuild
 NVIDIA_UVM_OBJECTS += $(patsubst %.c,%.o,$(NVIDIA_UVM_SOURCES))
 
-# Some linux kernel functions rely on being built with optimizations on and
-# to work around this we put wrappers for them in a separate file that's built
-# with optimizations on in debug builds and skipped in other builds.
-# Notably gcc 4.4 supports per function optimization attributes that would be
-# easier to use, but is too recent to rely on for now.
-NVIDIA_UVM_DEBUG_OPTIMIZED_SOURCE := nvidia-uvm/uvm_debug_optimized.c
-NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT := $(patsubst %.c,%.o,$(NVIDIA_UVM_DEBUG_OPTIMIZED_SOURCE))
-
-ifneq ($(UVM_BUILD_TYPE),debug)
-  # Only build the wrappers on debug builds
-  NVIDIA_UVM_OBJECTS := $(filter-out $(NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT), $(NVIDIA_UVM_OBJECTS))
-endif
-
 obj-m += nvidia-uvm.o
 nvidia-uvm-y := $(NVIDIA_UVM_OBJECTS)
 
@@ -36,15 +23,14 @@ NVIDIA_UVM_KO = nvidia-uvm/nvidia-uvm.ko
 #
 
 ifeq ($(UVM_BUILD_TYPE),debug)
-  NVIDIA_UVM_CFLAGS += -DDEBUG -O1 -g
-else
-  ifeq ($(UVM_BUILD_TYPE),develop)
-    # -DDEBUG is required, in order to allow pr_devel() print statements to
-    # work:
-    NVIDIA_UVM_CFLAGS += -DDEBUG
-    NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_DEVELOP
-  endif
-  NVIDIA_UVM_CFLAGS += -O2
+  NVIDIA_UVM_CFLAGS += -DDEBUG -g
+endif
+
+ifeq ($(UVM_BUILD_TYPE),develop)
+  # -DDEBUG is required, in order to allow pr_devel() print statements to
+  # work:
+  NVIDIA_UVM_CFLAGS += -DDEBUG
+  NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_DEVELOP
 endif
 
 NVIDIA_UVM_CFLAGS += -DNVIDIA_UVM_ENABLED
@@ -56,11 +42,6 @@ NVIDIA_UVM_CFLAGS += -I$(src)/nvidia-uvm
 
 $(call ASSIGN_PER_OBJ_CFLAGS, $(NVIDIA_UVM_OBJECTS), $(NVIDIA_UVM_CFLAGS))
 
-ifeq ($(UVM_BUILD_TYPE),debug)
-  # Force optimizations on for the wrappers
-  $(call ASSIGN_PER_OBJ_CFLAGS, $(NVIDIA_UVM_DEBUG_OPTIMIZED_OBJECT), $(NVIDIA_UVM_CFLAGS) -O2)
-endif
-
 #
 # Register the conftests needed by nvidia-uvm.ko
 #
@@ -88,6 +69,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_fault_to_errno
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += find_next_bit_wrap
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_is_dma_domain
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += folio_test_swapcache
 
 NV_CONFTEST_TYPE_COMPILE_TESTS += backing_dev_info
 NV_CONFTEST_TYPE_COMPILE_TESTS += mm_context_t
@@ -114,7 +96,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
 NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
 NV_CONFTEST_TYPE_COMPILE_TESTS += mpol_preferred_many_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_interval_notifier
-NV_CONFTEST_TYPE_COMPILE_TESTS += fault_flag_remote_present
 
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_migrate_vma_setup

kernel-open/nvidia-uvm/uvm.c

@@ -127,9 +127,9 @@ static NV_STATUS uvm_api_mm_initialize(UVM_MM_INITIALIZE_PARAMS *params, struct
         goto err;
     }
 
-    old_fd_type = nv_atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
-                                         UVM_FD_UNINITIALIZED,
-                                         UVM_FD_INITIALIZING);
+    old_fd_type = atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
+                                      UVM_FD_UNINITIALIZED,
+                                      UVM_FD_INITIALIZING);
     old_fd_type &= UVM_FD_TYPE_MASK;
     if (old_fd_type != UVM_FD_UNINITIALIZED) {
         status = NV_ERR_IN_USE;
@@ -682,6 +682,9 @@ static void uvm_vm_open_semaphore_pool(struct vm_area_struct *vma)
     // Semaphore pool vmas do not have vma wrappers, but some functions will
     // assume vm_private_data is a wrapper.
     vma->vm_private_data = NULL;
+#if defined(VM_WIPEONFORK)
+    nv_vm_flags_set(vma, VM_WIPEONFORK);
+#endif
 
     if (is_fork) {
         // If we forked, leave the parent vma alone.
@@ -914,8 +917,9 @@ static NV_STATUS uvm_api_initialize(UVM_INITIALIZE_PARAMS *params, struct file *
     // attempt to be made. This is safe because other threads will have only had
     // a chance to observe UVM_FD_INITIALIZING and not UVM_FD_VA_SPACE in this
     // case.
-    old_fd_type = nv_atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
-                                         UVM_FD_UNINITIALIZED, UVM_FD_INITIALIZING);
+    old_fd_type = atomic_long_cmpxchg((atomic_long_t *)&filp->private_data,
+                                      UVM_FD_UNINITIALIZED,
+                                      UVM_FD_INITIALIZING);
     old_fd_type &= UVM_FD_TYPE_MASK;
     if (old_fd_type == UVM_FD_UNINITIALIZED) {
         status = uvm_va_space_create(filp->f_mapping, &va_space, params->flags);

kernel-open/nvidia-uvm/uvm.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2013-2024 NVIDIA Corporation
+    Copyright (c) 2013-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -58,7 +58,7 @@
 #ifndef _UVM_H_
 #define _UVM_H_
 
-#define UVM_API_LATEST_REVISION 12
+#define UVM_API_LATEST_REVISION 11
 
 #if !defined(UVM_API_REVISION)
 #error "please define UVM_API_REVISION macro to a desired version number or UVM_API_LATEST_REVISION macro"
@@ -167,7 +167,7 @@ NV_STATUS UvmSetDriverVersion(NvU32 major, NvU32 changelist);
 //
 // Error codes:
 //     NV_ERR_NOT_SUPPORTED:
-//         The kernel is not able to support UVM. This could be because
+//         The Linux kernel is not able to support UVM. This could be because
 //         the kernel is too old, or because it lacks a feature that UVM
 //         requires. The kernel log will have details.
 //
@@ -379,6 +379,17 @@ NV_STATUS UvmIsPageableMemoryAccessSupportedOnGpu(const NvProcessorUuid *gpuUuid
 //         OS state required to register the GPU is malformed, or the partition
 //         identified by the user handles or its configuration changed.
 //
+//     NV_ERR_NVSWITCH_FABRIC_NOT_READY:
+//         (On NvSwitch-connected system) Indicates that the fabric has not been
+//         configured yet. Caller must retry GPU registration.
+//
+//     NV_ERR_NVSWITCH_FABRIC_FAILURE:
+//         (On NvSwitch-connected systems) Indicates that the NvLink fabric
+//         failed to be configured.
+//
+//     NV_ERR_GPU_MEMORY_ONLINING_FAULURE:
+//         (On coherent systems) The GPU's memory onlining failed.
+//
 //     NV_ERR_GENERIC:
 //         Unexpected error. We try hard to avoid returning this error code,
 //         because it is not very informative.
@@ -3473,7 +3484,8 @@ NV_STATUS UvmToolsDestroySession(UvmToolsSessionHandle session);
 //
 
 #if UVM_API_REV_IS_AT_MOST(10)
-// This is deprecated and replaced by sizeof(UvmToolsEventControlData).
+// This is deprecated and replaced by sizeof(UvmToolsEventControlData_V1) or
+// sizeof(UvmToolsEventControlData_V2).
 NvLength UvmToolsGetEventControlSize(void);
 
 // This is deprecated and replaced by sizeof(UvmEventEntry_V1) or
@@ -3496,7 +3508,9 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //
 //     version: (INPUT)
 //         Requested version for events or counters.
-//         See UvmToolsEventQueueVersion.
+//         See UvmEventEntry_V1 and UvmEventEntry_V2.
+//         UvmToolsEventControlData_V2::version records the entry version that
+//         will be generated.
 //
 //     event_buffer: (INPUT)
 //         User allocated buffer. Must be page-aligned. Must be large enough to
@@ -3509,7 +3523,8 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //
 //     event_control (INPUT)
 //         User allocated buffer. Must be page-aligned. Must be large enough to
-//         hold UvmToolsEventControlData (although single page-size allocation
+//         hold UvmToolsEventControlData_V1 if version is UvmEventEntry_V1 or
+//         UvmToolsEventControlData_V2 (although single page-size allocation
 //         should be more than enough). Gets pinned until queue is destroyed.
 //
 //     queue: (OUTPUT)
@@ -3520,16 +3535,10 @@ NvLength UvmToolsGetNumberOfCounters(void);
 //         Session handle does not refer to a valid session
 //
 //     NV_ERR_INVALID_ARGUMENT:
-//         The version is not UvmToolsEventQueueVersion_V1 or
-//         UvmToolsEventQueueVersion_V2.
+//         The version is not UvmEventEntry_V1 or UvmEventEntry_V2.
 //         One of the parameters: event_buffer, event_buffer_size, event_control
 //         is not valid
 //
-//     NV_ERR_NOT_SUPPORTED:
-//         The requested version queue could not be created
-//         (i.e., the UVM kernel driver is older and doesn't support
-//         UvmToolsEventQueueVersion_V2).
-//
 //     NV_ERR_INSUFFICIENT_RESOURCES:
 //         There could be multiple reasons for this error. One would be that
 //         it's not possible to allocate a queue of requested size. Another
@@ -3982,51 +3991,57 @@ NV_STATUS UvmToolsWriteProcessMemory(UvmToolsSessionHandle  session,
 //     version: (INPUT)
 //         Requested version for the UUID table returned. The version must
 //         match the requested version of the event queue created with
-//         UvmToolsCreateEventQueue(). See UvmToolsEventQueueVersion.
-//         If the version of the event queue does not match the version of the
-//         UUID table, the behavior is undefined.
+//         UvmToolsCreateEventQueue().
+//         See UvmEventEntry_V1 and UvmEventEntry_V2.
 //
 //     table: (OUTPUT)
 //         Array of processor UUIDs, including the CPU's UUID which is always
-//         at index zero. The number of elements in the array must be greater
-//         or equal to UVM_MAX_PROCESSORS_V1 if the version is
-//         UvmToolsEventQueueVersion_V1 and UVM_MAX_PROCESSORS if the version is
-//         UvmToolsEventQueueVersion_V2.
-//         The srcIndex and dstIndex fields of the UvmEventMigrationInfo struct
-//         index this array. Unused indices will have a UUID of zero.
-//         If version is UvmToolsEventQueueVersion_V1 then the reported UUID
-//         will be that of the corresponding physical GPU, even if multiple SMC
-//         partitions are registered under that physical GPU. If version is
-//         UvmToolsEventQueueVersion_V2 then the reported UUID will be the GPU
-//         instance UUID if SMC is enabled, otherwise it will be the UUID of
-//         the physical GPU.
+//         at index zero.  The srcIndex and dstIndex fields of the
+//         UvmEventMigrationInfo struct index this array.  Unused indices will
+//         have a UUID of zero. Version UvmEventEntry_V1 only uses GPU UUIDs
+//         for the UUID of the physical GPU and only supports a single SMC
+//         partition registered per process. Version UvmEventEntry_V2 supports
+//         multiple SMC partitions registered per process and uses physical GPU
+//         UUIDs if the GPU is not SMC capable or SMC enabled and GPU instance
+//         UUIDs for SMC partitions.
+//         The table pointer can be NULL in which case, the size of the table
+//         needed to hold all the UUIDs is returned in 'count'.
+//
+//     table_size: (INPUT)
+//         The size of the table in number of array elements. This can be
+//         zero if the table pointer is NULL.
+//
+//     count: (OUTPUT)
+//         On output, it is set by UVM to the number of UUIDs needed to hold
+//         all the UUIDs, including any gaps in the table due to unregistered
+//         GPUs.
 //
 // Error codes:
 //     NV_ERR_INVALID_ADDRESS:
-//         writing to table failed.
+//         writing to table failed or the count pointer was invalid.
 //
 //     NV_ERR_INVALID_ARGUMENT:
-//         The version is not UvmToolsEventQueueVersion_V1 or
-//         UvmToolsEventQueueVersion_V2.
+//         The version is not UvmEventEntry_V1 or UvmEventEntry_V2.
+//         The count pointer is NULL.
+//         See UvmToolsEventQueueVersion.
 //
-//     NV_ERR_NOT_SUPPORTED:
-//         The kernel is not able to support the requested version
-//         (i.e., the UVM kernel driver is older and doesn't support
-//         UvmToolsEventQueueVersion_V2).
+//     NV_WARN_MISMATCHED_TARGET:
+//         The kernel returned a table suitable for UvmEventEntry_V1 events.
+//         (i.e., the kernel is older and doesn't support UvmEventEntry_V2).
 //
 //     NV_ERR_NO_MEMORY:
 //         Internal memory allocation failed.
 //------------------------------------------------------------------------------
-#if UVM_API_REV_IS_AT_MOST(11)
+#if UVM_API_REV_IS_AT_MOST(10)
+NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle  session,
+                                        NvProcessorUuid       *table,
+                                        NvLength              *count);
+#else
 NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle      session,
                                         UvmToolsEventQueueVersion  version,
                                         NvProcessorUuid           *table,
                                         NvLength                   table_size,
                                         NvLength                  *count);
-#else
-NV_STATUS UvmToolsGetProcessorUuidTable(UvmToolsSessionHandle     session,
-                                        UvmToolsEventQueueVersion version,
-                                        NvProcessorUuid          *table);
 #endif
 
 //------------------------------------------------------------------------------

kernel-open/nvidia-uvm/uvm_ampere_fault_buffer.c

@@ -1,75 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2024 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-#include "uvm_linux.h"
-#include "uvm_global.h"
-#include "uvm_gpu.h"
-#include "uvm_hal.h"
-#include "hwref/ampere/ga100/dev_fault.h"
-
-static bool client_id_ce(NvU16 client_id)
-{
-    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE0 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE9)
-        return true;
-
-    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE10 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE15)
-        return true;
-
-    switch (client_id) {
-        case NV_PFAULT_CLIENT_HUB_CE0:
-        case NV_PFAULT_CLIENT_HUB_CE1:
-        case NV_PFAULT_CLIENT_HUB_CE2:
-            return true;
-    }
-
-    return false;
-}
-
-uvm_mmu_engine_type_t uvm_hal_ampere_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                      uvm_fault_client_type_t client_type,
-                                                                      NvU16 client_id)
-{
-    // Servicing CE and Host (HUB clients) faults.
-    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB) {
-        if (client_id_ce(client_id)) {
-            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9);
-
-            return UVM_MMU_ENGINE_TYPE_CE;
-        }
-
-        if (client_id == NV_PFAULT_CLIENT_HUB_HOST || client_id == NV_PFAULT_CLIENT_HUB_ESC) {
-            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST31);
-
-            return UVM_MMU_ENGINE_TYPE_HOST;
-        }
-    }
-
-    // We shouldn't be servicing faults from any other engines other than GR.
-    UVM_ASSERT_MSG(client_id <= NV_PFAULT_CLIENT_GPC_ROP_3, "Unexpected client ID: 0x%x\n", client_id);
-    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS && mmu_engine_id < NV_PFAULT_MMU_ENG_ID_BAR1,
-                   "Unexpected engine ID: 0x%x\n",
-                   mmu_engine_id);
-    UVM_ASSERT(client_type == UVM_FAULT_CLIENT_TYPE_GPC);
-
-    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
-}

kernel-open/nvidia-uvm/uvm_ampere_host.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2024 NVIDIA Corporation
+    Copyright (c) 2018-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -205,18 +205,17 @@ void uvm_hal_ampere_host_clear_faulted_channel_sw_method(uvm_push_t *push,
                      CLEAR_FAULTED_B, HWVALUE(C076, CLEAR_FAULTED_B, INST_HI, instance_ptr_hi));
 }
 
-// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Pascal, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
-                                           uvm_gpu_phys_address_t pdb,
-                                           NvU32 depth,
-                                           uvm_membar_t membar)
+                                            uvm_gpu_phys_address_t pdb,
+                                            NvU32 depth,
+                                            uvm_membar_t membar)
 {
     NvU32 aperture_value;
     NvU32 page_table_level;
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
 
@@ -231,8 +230,8 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
     pdb_lo = pdb.address & HWMASK(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
-    // PDE3 is the highest level on Pascal-Ampere, see the comment in
-    // uvm_pascal_mmu.c for details.
+    // PDE3 is the highest level on Pascal, see the comment in uvm_pascal_mmu.c
+    // for details.
     UVM_ASSERT_MSG(depth < NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
     page_table_level = NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
 
@@ -243,12 +242,7 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
-    NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
+    NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                      MEM_OP_B, 0,
                      MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
@@ -261,18 +255,16 @@ void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C56F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
                                HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    // GPU membar still requires an explicit membar method.
-    if (membar == UVM_MEMBAR_GPU)
-        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }
 
-// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Volta, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
                                            uvm_gpu_phys_address_t pdb,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU64 page_size,
+                                           NvU32 page_size,
                                            uvm_membar_t membar)
 {
     NvU32 aperture_value;
@@ -280,7 +272,6 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
     NvU32 va_lo;
     NvU32 va_hi;
     NvU64 end;
@@ -290,9 +281,9 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 log2_invalidation_size;
     uvm_gpu_t *gpu = uvm_push_get_gpu(push);
 
-    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%x\n", page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%x\n", base, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%x\n", size, page_size);
     UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
 
     // The invalidation size must be a power-of-two number of pages containing
@@ -334,7 +325,7 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
     pdb_lo = pdb.address & HWMASK(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
-    // PDE3 is the highest level on Pascal-Ampere, see the comment in
+    // PDE3 is the highest level on Pascal-Ampere , see the comment in
     // uvm_pascal_mmu.c for details.
     UVM_ASSERT_MSG(depth < NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
     page_table_level = NVC56F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
@@ -346,15 +337,10 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
     NV_PUSH_4U(C56F, MEM_OP_A, HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_INVALIDATION_SIZE, log2_invalidation_size) |
-                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
-                               sysmembar_value |
-                               HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
+                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
+                               HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo) |
+                               HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                      MEM_OP_B, HWVALUE(C56F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
                      MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
                                HWVALUE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
@@ -366,23 +352,21 @@ void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C56F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
                                HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    // GPU membar still requires an explicit membar method.
-    if (membar == UVM_MEMBAR_GPU)
-        gpu->parent->host_hal->membar_gpu(push);
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }
 
-// Copy from Turing, this version sets TLB_INVALIDATE_INVAL_SCOPE.
+// Copy from Pascal, this version sets TLB_INVALIDATE_INVAL_SCOPE.
 void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                              uvm_gpu_phys_address_t pdb,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params)
 {
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
     NvU32 invalidate_gpc_value = 0;
     NvU32 aperture_value = 0;
     NvU32 pdb_lo = 0;
     NvU32 pdb_hi = 0;
     NvU32 page_table_level = 0;
+    uvm_membar_t membar;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
     if (pdb.aperture == UVM_APERTURE_VID)
@@ -397,7 +381,7 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
     pdb_hi = pdb.address >> HWSIZE(C56F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
 
     if (params->page_table_level != UvmInvalidatePageTableLevelAll) {
-        // PDE3 is the highest level on Pascal-Ampere, see the comment in
+        // PDE3 is the highest level on Pascal, see the comment in
         // uvm_pascal_mmu.c for details.
         page_table_level = min((NvU32)UvmInvalidatePageTableLevelPde3, params->page_table_level) - 1;
     }
@@ -409,11 +393,6 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
         ack_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (params->membar == UvmInvalidateTlbMemBarSys)
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
     if (params->disable_gpc_invalidate)
         invalidate_gpc_value = HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
     else
@@ -424,9 +403,9 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
 
         NvU32 va_lo = va & HWMASK(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
         NvU32 va_hi = va >> HWSIZE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-        NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
-                                   HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
-                                   HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
+        NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
+                                   HWVALUE(C56F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo) |
+                                   HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                          MEM_OP_B, HWVALUE(C56F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
                          MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
                                    HWVALUE(C56F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
@@ -439,7 +418,7 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
     else {
-        NV_PUSH_4U(C56F, MEM_OP_A, sysmembar_value |
+        NV_PUSH_4U(C56F, MEM_OP_A, HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                    HWCONST(C56F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                          MEM_OP_B, 0,
                          MEM_OP_C, HWCONST(C56F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
@@ -453,7 +432,12 @@ void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C56F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
 
-    // GPU membar still requires an explicit membar method.
-    if (params->membar == UvmInvalidateTlbMemBarLocal)
-        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
+    if (params->membar == UvmInvalidateTlbMemBarSys)
+        membar = UVM_MEMBAR_SYS;
+    else if (params->membar == UvmInvalidateTlbMemBarLocal)
+        membar = UVM_MEMBAR_GPU;
+    else
+        membar = UVM_MEMBAR_NONE;
+
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }

kernel-open/nvidia-uvm/uvm_ampere_mmu.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2024 NVIDIA Corporation
+    Copyright (c) 2018-2020 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -36,7 +36,22 @@
 #include "uvm_ampere_fault_buffer.h"
 #include "hwref/ampere/ga100/dev_fault.h"
 
-static NvU32 page_table_depth_ampere(NvU64 page_size)
+uvm_mmu_engine_type_t uvm_hal_ampere_mmu_engine_id_to_type(NvU16 mmu_engine_id)
+{
+    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST31)
+        return UVM_MMU_ENGINE_TYPE_HOST;
+
+    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9)
+        return UVM_MMU_ENGINE_TYPE_CE;
+
+    // We shouldn't be servicing faults from any other engines
+    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS && mmu_engine_id < NV_PFAULT_MMU_ENG_ID_BAR1,
+                   "Unexpected engine ID: 0x%x\n", mmu_engine_id);
+
+    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
+}
+
+static NvU32 page_table_depth_ampere(NvU32 page_size)
 {
     // The common-case is page_size == UVM_PAGE_SIZE_2M, hence the first check
     if (page_size == UVM_PAGE_SIZE_2M)
@@ -47,14 +62,14 @@ static NvU32 page_table_depth_ampere(NvU64 page_size)
         return 4;
 }
 
-static NvU64 page_sizes_ampere(void)
+static NvU32 page_sizes_ampere(void)
 {
     return UVM_PAGE_SIZE_512M | UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
 }
 
 static uvm_mmu_mode_hal_t ampere_mmu_mode_hal;
 
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU64 big_page_size)
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU32 big_page_size)
 {
     static bool initialized = false;
 

kernel-open/nvidia-uvm/uvm_api.h

@@ -47,7 +47,7 @@
     {                                                                               \
         params_type params;                                                         \
         BUILD_BUG_ON(sizeof(params) > UVM_MAX_IOCTL_PARAM_STACK_SIZE);              \
-        if (nv_copy_from_user(&params, (void __user*)arg, sizeof(params)))          \
+        if (copy_from_user(&params, (void __user*)arg, sizeof(params)))             \
             return -EFAULT;                                                         \
                                                                                     \
         params.rmStatus = uvm_global_get_status();                                  \
@@ -60,7 +60,7 @@
                 params.rmStatus = function_name(&params, filp);                     \
         }                                                                           \
                                                                                     \
-        if (nv_copy_to_user((void __user*)arg, &params, sizeof(params)))            \
+        if (copy_to_user((void __user*)arg, &params, sizeof(params)))               \
             return -EFAULT;                                                         \
                                                                                     \
         return 0;                                                                   \
@@ -84,7 +84,7 @@
         if (!params)                                                                    \
             return -ENOMEM;                                                             \
         BUILD_BUG_ON(sizeof(*params) <= UVM_MAX_IOCTL_PARAM_STACK_SIZE);                \
-        if (nv_copy_from_user(params, (void __user*)arg, sizeof(*params))) {            \
+        if (copy_from_user(params, (void __user*)arg, sizeof(*params))) {               \
             uvm_kvfree(params);                                                         \
             return -EFAULT;                                                             \
         }                                                                               \
@@ -99,7 +99,7 @@
                 params->rmStatus = function_name(params, filp);                         \
         }                                                                               \
                                                                                         \
-        if (nv_copy_to_user((void __user*)arg, params, sizeof(*params)))                \
+        if (copy_to_user((void __user*)arg, params, sizeof(*params)))                   \
             ret = -EFAULT;                                                              \
                                                                                         \
         uvm_kvfree(params);                                                             \

kernel-open/nvidia-uvm/uvm_ats.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2024 NVIDIA Corporation
+    Copyright (c) 2018-2021 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to

kernel-open/nvidia-uvm/uvm_ats.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2018-2024 NVIDIA Corporation
+    Copyright (c) 2018-2021 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -29,9 +29,10 @@
 #include "uvm_ats_ibm.h"
 #include "nv_uvm_types.h"
 #include "uvm_lock.h"
-#include "uvm_ats_sva.h"
 
-#define UVM_ATS_SUPPORTED() (UVM_ATS_IBM_SUPPORTED() || UVM_ATS_SVA_SUPPORTED())
+    #include "uvm_ats_sva.h"
+
+    #define UVM_ATS_SUPPORTED() (UVM_ATS_IBM_SUPPORTED() || UVM_ATS_SVA_SUPPORTED())
 
 typedef struct
 {

kernel-open/nvidia-uvm/uvm_blackwell.c

@@ -1,105 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2022-2023 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-#include "uvm_global.h"
-#include "uvm_hal.h"
-#include "uvm_gpu.h"
-#include "uvm_mem.h"
-#include "uvm_blackwell_fault_buffer.h"
-
-void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
-{
-    parent_gpu->tlb_batch.va_invalidate_supported = true;
-
-    parent_gpu->tlb_batch.va_range_invalidate_supported = true;
-
-    // TODO: Bug 1767241: Run benchmarks to figure out a good number
-    parent_gpu->tlb_batch.max_ranges = 8;
-
-    parent_gpu->utlb_per_gpc_count = uvm_blackwell_get_utlbs_per_gpc(parent_gpu);
-
-    parent_gpu->fault_buffer_info.replayable.utlb_count = parent_gpu->rm_info.maxGpcCount *
-                                                          parent_gpu->utlb_per_gpc_count;
-    {
-        uvm_fault_buffer_entry_t *dummy;
-        UVM_ASSERT(parent_gpu->fault_buffer_info.replayable.utlb_count <= (1 <<
-                                                                           (sizeof(dummy->fault_source.utlb_id) * 8)));
-    }
-
-    // A single top level PDE on Blackwell covers 64 PB and that's the minimum
-    // size that can be used.
-    parent_gpu->rm_va_base = 0;
-    parent_gpu->rm_va_size = 64 * UVM_SIZE_1PB;
-
-    parent_gpu->uvm_mem_va_base = parent_gpu->rm_va_size + 384 * UVM_SIZE_1TB;
-    parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
-
-    // See uvm_mmu.h for mapping placement
-    parent_gpu->flat_vidmem_va_base = (64 * UVM_SIZE_1PB) + (32 * UVM_SIZE_1TB);
-
-    // TODO: Bug 3953852: Set this to true pending Blackwell changes
-    parent_gpu->ce_phys_vidmem_write_supported = !uvm_parent_gpu_is_coherent(parent_gpu);
-
-    parent_gpu->peer_copy_mode = g_uvm_global.peer_copy_mode;
-
-    // All GR context buffers may be mapped to 57b wide VAs. All "compute" units
-    // accessing GR context buffers support the 57-bit VA range.
-    parent_gpu->max_channel_va = 1ull << 57;
-
-    parent_gpu->max_host_va = 1ull << 57;
-
-    // Blackwell can map sysmem with any page size
-    parent_gpu->can_map_sysmem_with_large_pages = true;
-
-    // Prefetch instructions will generate faults
-    parent_gpu->prefetch_fault_supported = true;
-
-    // Blackwell can place GPFIFO in vidmem
-    parent_gpu->gpfifo_in_vidmem_supported = true;
-
-    parent_gpu->replayable_faults_supported = true;
-
-    parent_gpu->non_replayable_faults_supported = true;
-
-    parent_gpu->access_counters_supported = true;
-
-    parent_gpu->access_counters_can_use_physical_addresses = false;
-
-    parent_gpu->fault_cancel_va_supported = true;
-
-    parent_gpu->scoped_atomics_supported = true;
-
-    parent_gpu->has_clear_faulted_channel_sw_method = true;
-
-    parent_gpu->has_clear_faulted_channel_method = false;
-
-    parent_gpu->smc.supported = true;
-
-    parent_gpu->sparse_mappings_supported = true;
-
-    parent_gpu->map_remap_larger_page_promotion = false;
-
-    parent_gpu->plc_supported = true;
-
-    parent_gpu->no_ats_range_required = true;
-}

kernel-open/nvidia-uvm/uvm_blackwell_fault_buffer.c

@@ -1,122 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2023-2024 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-#include "uvm_linux.h"
-#include "uvm_global.h"
-#include "uvm_gpu.h"
-#include "uvm_hal.h"
-#include "uvm_hal_types.h"
-#include "hwref/blackwell/gb100/dev_fault.h"
-#include "clc369.h"
-
-// NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE fault type is deprecated on
-// Blackwell.
-uvm_fault_type_t uvm_hal_blackwell_fault_buffer_get_fault_type(const NvU32 *fault_entry)
-{
-    NvU32 hw_fault_type_value = READ_HWVALUE_MW(fault_entry, C369, BUF_ENTRY, FAULT_TYPE);
-
-    switch (hw_fault_type_value) {
-        case NV_PFAULT_FAULT_TYPE_PDE:
-            return UVM_FAULT_TYPE_INVALID_PDE;
-        case NV_PFAULT_FAULT_TYPE_PTE:
-            return UVM_FAULT_TYPE_INVALID_PTE;
-        case NV_PFAULT_FAULT_TYPE_RO_VIOLATION:
-            return UVM_FAULT_TYPE_WRITE;
-        case NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION:
-            return UVM_FAULT_TYPE_ATOMIC;
-        case NV_PFAULT_FAULT_TYPE_WO_VIOLATION:
-            return UVM_FAULT_TYPE_READ;
-
-        case NV_PFAULT_FAULT_TYPE_PDE_SIZE:
-            return UVM_FAULT_TYPE_PDE_SIZE;
-        case NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION:
-            return UVM_FAULT_TYPE_VA_LIMIT_VIOLATION;
-        case NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK:
-            return UVM_FAULT_TYPE_UNBOUND_INST_BLOCK;
-        case NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION:
-            return UVM_FAULT_TYPE_PRIV_VIOLATION;
-        case NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION:
-            return UVM_FAULT_TYPE_PITCH_MASK_VIOLATION;
-        case NV_PFAULT_FAULT_TYPE_WORK_CREATION:
-            return UVM_FAULT_TYPE_WORK_CREATION;
-        case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE:
-            return UVM_FAULT_TYPE_UNSUPPORTED_APERTURE;
-        case NV_PFAULT_FAULT_TYPE_CC_VIOLATION:
-            return UVM_FAULT_TYPE_CC_VIOLATION;
-        case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND:
-            return UVM_FAULT_TYPE_UNSUPPORTED_KIND;
-        case NV_PFAULT_FAULT_TYPE_REGION_VIOLATION:
-            return UVM_FAULT_TYPE_REGION_VIOLATION;
-        case NV_PFAULT_FAULT_TYPE_POISONED:
-            return UVM_FAULT_TYPE_POISONED;
-    }
-
-    UVM_ASSERT_MSG(false, "Invalid fault type value: %d\n", hw_fault_type_value);
-
-    return UVM_FAULT_TYPE_COUNT;
-}
-
-static bool client_id_ce(NvU16 client_id)
-{
-    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE0 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE7)
-        return true;
-
-    switch (client_id) {
-        case NV_PFAULT_CLIENT_HUB_CE0:
-        case NV_PFAULT_CLIENT_HUB_CE1:
-        case NV_PFAULT_CLIENT_HUB_CE2:
-        case NV_PFAULT_CLIENT_HUB_CE3:
-            return true;
-    }
-
-    return false;
-}
-
-uvm_mmu_engine_type_t uvm_hal_blackwell_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                         uvm_fault_client_type_t client_type,
-                                                                         NvU16 client_id)
-{
-    // Servicing CE and Host (HUB clients) faults.
-    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB) {
-        if (client_id_ce(client_id)) {
-            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE19);
-
-            return UVM_MMU_ENGINE_TYPE_CE;
-        }
-
-        if (client_id == NV_PFAULT_CLIENT_HUB_HOST ||
-            (client_id >= NV_PFAULT_CLIENT_HUB_ESC0 && client_id <= NV_PFAULT_CLIENT_HUB_ESC11)) {
-            UVM_ASSERT((mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST44) ||
-                       (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS));
-
-            return UVM_MMU_ENGINE_TYPE_HOST;
-        }
-    }
-
-    // We shouldn't be servicing faults from any other engines other than GR.
-    UVM_ASSERT_MSG(client_id <= NV_PFAULT_CLIENT_GPC_ROP_3, "Unexpected client ID: 0x%x\n", client_id);
-    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS, "Unexpected engine ID: 0x%x\n", mmu_engine_id);
-    UVM_ASSERT(client_type == UVM_FAULT_CLIENT_TYPE_GPC);
-
-    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
-}

kernel-open/nvidia-uvm/uvm_blackwell_fault_buffer.h

@@ -1,92 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2022 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-#ifndef __UVM_HAL_BLACKWELL_FAULT_BUFFER_H__
-#define __UVM_HAL_BLACKWELL_FAULT_BUFFER_H__
-
-#include "nvtypes.h"
-#include "uvm_common.h"
-#include "uvm_gpu.h"
-
-// There are up to 10 TPCs per GPC in Blackwell, and there are 2 LTP uTLBs per
-// TPC. Besides, there is one active RGG uTLB per GPC. Each TPC has a number of
-// clients that can make requests to its uTLBs: 1xTPCCS, 1xPE, 2xT1. Requests
-// from these units are routed as follows to the 2 LTP uTLBs:
-//
-// --------                    ---------
-// | T1_0 | -----------------> | uTLB0 |
-// --------                    ---------
-//
-// --------                    ---------
-// | T1_1 | -----------------> | uTLB1 |
-// --------          --------> ---------
-//                   |             ^
-// -------           |             |
-// | PE  | -----------             |
-// -------                         |
-//                                 |
-// ---------                       |
-// | TPCCS | -----------------------
-// ---------
-//
-//
-// The client ids are local to their GPC and the id mapping is linear across
-// TPCs: TPC_n has TPCCS_n, PE_n, T1_p, and T1_q, where p=2*n and q=p+1.
-//
-// NV_PFAULT_CLIENT_GPC_LTP_UTLB_n and NV_PFAULT_CLIENT_GPC_RGG_UTLB enums can
-// be ignored. These will never be reported in a fault message, and should
-// never be used in an invalidate. Therefore, we define our own values.
-typedef enum {
-    UVM_BLACKWELL_GPC_UTLB_ID_RGG = 0,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP0 = 1,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP1 = 2,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP2 = 3,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP3 = 4,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP4 = 5,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP5 = 6,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP6 = 7,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP7 = 8,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP8 = 9,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP9 = 10,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP10 = 11,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP11 = 12,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP12 = 13,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP13 = 14,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP14 = 15,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP15 = 16,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP16 = 17,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP17 = 18,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP18 = 19,
-    UVM_BLACKWELL_GPC_UTLB_ID_LTP19 = 20,
-
-    UVM_BLACKWELL_GPC_UTLB_COUNT,
-} uvm_blackwell_gpc_utlb_id_t;
-
-static NvU32 uvm_blackwell_get_utlbs_per_gpc(uvm_parent_gpu_t *parent_gpu)
-{
-    NvU32 utlbs = parent_gpu->rm_info.maxTpcPerGpcCount * 2 + 1;
-    UVM_ASSERT(utlbs <= UVM_BLACKWELL_GPC_UTLB_COUNT);
-    return utlbs;
-}
-
-#endif

kernel-open/nvidia-uvm/uvm_blackwell_host.c

@@ -1,256 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2024 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-#include "uvm_hal.h"
-#include "uvm_push.h"
-#include "uvm_push_macros.h"
-#include "clc96f.h"
-
-// TODO: Bug 3210931: Rename HOST references and files to ESCHED.
-
-void uvm_hal_blackwell_host_tlb_invalidate_all(uvm_push_t *push,
-                                               uvm_gpu_phys_address_t pdb,
-                                               NvU32 depth,
-                                               uvm_membar_t membar)
-{
-    NvU32 aperture_value;
-    NvU32 page_table_level;
-    NvU32 pdb_lo;
-    NvU32 pdb_hi;
-    NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
-
-    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
-
-    if (pdb.aperture == UVM_APERTURE_VID)
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
-    else
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
-
-    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
-    pdb.address >>= 12;
-
-    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-
-    // PDE4 is the highest level on Blackwell, see the comment in
-    // uvm_blackwell_mmu.c for details.
-    UVM_ASSERT_MSG(depth < NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4, "depth %u", depth);
-    page_table_level = NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 - depth;
-
-    if (membar != UVM_MEMBAR_NONE)
-        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
-
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
-    NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
-                               HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
-                     MEM_OP_B, 0,
-                     MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
-                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
-                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
-                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
-                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
-                               aperture_value |
-                               ack_value,
-                     MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
-                               HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
-}
-
-void uvm_hal_blackwell_host_tlb_invalidate_va(uvm_push_t *push,
-                                              uvm_gpu_phys_address_t pdb,
-                                              NvU32 depth,
-                                              NvU64 base,
-                                              NvU64 size,
-                                              NvU64 page_size,
-                                              uvm_membar_t membar)
-{
-    NvU32 aperture_value;
-    NvU32 page_table_level;
-    NvU32 pdb_lo;
-    NvU32 pdb_hi;
-    NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
-    NvU32 va_lo;
-    NvU32 va_hi;
-    NvU64 end;
-    NvU64 actual_base;
-    NvU64 actual_size;
-    NvU64 actual_end;
-    NvU32 log2_invalidation_size;
-    uvm_gpu_t *gpu = uvm_push_get_gpu(push);
-
-    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
-    UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
-
-    // The invalidation size must be a power-of-two number of pages containing
-    // the passed interval
-    end = base + size - 1;
-    log2_invalidation_size = __fls((unsigned long)(end ^ base)) + 1;
-
-    if (log2_invalidation_size == 64) {
-        // Invalidate everything
-        gpu->parent->host_hal->tlb_invalidate_all(push, pdb, depth, membar);
-        return;
-    }
-
-    // The hardware aligns the target address down to the invalidation size.
-    actual_size = 1ULL << log2_invalidation_size;
-    actual_base = UVM_ALIGN_DOWN(base, actual_size);
-    actual_end = actual_base + actual_size - 1;
-    UVM_ASSERT(actual_end >= end);
-
-    // The invalidation size field expects log2(invalidation size in 4K), not
-    // log2(invalidation size in bytes)
-    log2_invalidation_size -= 12;
-
-    // Address to invalidate, as a multiple of 4K.
-    base >>= 12;
-    va_lo = base & HWMASK(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-    va_hi = base >> HWSIZE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-
-    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
-
-    if (pdb.aperture == UVM_APERTURE_VID)
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
-    else
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
-
-    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
-    pdb.address >>= 12;
-
-    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-
-    // PDE4 is the highest level on Blackwell, see the comment in
-    // uvm_blackwell_mmu.c for details.
-    UVM_ASSERT_MSG(depth < NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4, "depth %u", depth);
-    page_table_level = NVC96F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 - depth;
-
-    if (membar != UVM_MEMBAR_NONE)
-        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
-
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
-    NV_PUSH_4U(C96F, MEM_OP_A, HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_INVALIDATION_SIZE, log2_invalidation_size) |
-                               sysmembar_value |
-                               HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
-                               HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
-                     MEM_OP_B, HWVALUE(C96F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
-                     MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
-                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
-                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
-                               HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
-                               HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
-                               aperture_value |
-                               ack_value,
-                     MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
-                               HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
-}
-
-void uvm_hal_blackwell_host_tlb_invalidate_test(uvm_push_t *push,
-                                                uvm_gpu_phys_address_t pdb,
-                                                UVM_TEST_INVALIDATE_TLB_PARAMS *params)
-{
-    NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
-    NvU32 invalidate_gpc_value = 0;
-    NvU32 aperture_value = 0;
-    NvU32 pdb_lo = 0;
-    NvU32 pdb_hi = 0;
-    NvU32 page_table_level = 0;
-
-    UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
-    if (pdb.aperture == UVM_APERTURE_VID)
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
-    else
-        aperture_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
-
-    UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
-    pdb.address >>= 12;
-
-    pdb_lo = pdb.address & HWMASK(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-    pdb_hi = pdb.address >> HWSIZE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
-
-    if (params->page_table_level != UvmInvalidatePageTableLevelAll) {
-        // PDE4 is the highest level on Blackwell, see the comment in
-        // uvm_blackwell_mmu.c for details.
-        page_table_level = min((NvU32)UvmInvalidatePageTableLevelPde4, params->page_table_level) - 1;
-    }
-
-    if (params->membar != UvmInvalidateTlbMemBarNone)
-        ack_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
-
-    if (params->membar == UvmInvalidateTlbMemBarSys)
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
-    if (params->disable_gpc_invalidate)
-        invalidate_gpc_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
-    else
-        invalidate_gpc_value = HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE);
-
-    if (params->target_va_mode == UvmTargetVaModeTargeted) {
-        NvU64 va = params->va >> 12;
-
-        NvU32 va_lo = va & HWMASK(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-        NvU32 va_hi = va >> HWSIZE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
-
-        NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
-                                   HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
-                                   HWVALUE(C96F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
-                         MEM_OP_B, HWVALUE(C96F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
-                         MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
-                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
-                                   HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
-                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
-                                   invalidate_gpc_value |
-                                   aperture_value |
-                                   ack_value,
-                         MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
-                                   HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
-    }
-    else {
-        NV_PUSH_4U(C96F, MEM_OP_A, sysmembar_value |
-                                   HWCONST(C96F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
-                         MEM_OP_B, 0,
-                         MEM_OP_C, HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
-                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
-                                   HWCONST(C96F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
-                                   HWVALUE(C96F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
-                                   invalidate_gpc_value |
-                                   aperture_value |
-                                   ack_value,
-                         MEM_OP_D, HWCONST(C96F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
-                                   HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
-    }
-}

kernel-open/nvidia-uvm/uvm_blackwell_mmu.c

@@ -1,165 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2022-2024 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-// On Blackwell, the UVM page tree 'depth' maps to hardware as follows:
-//
-// UVM depth   HW level                            VA bits
-// 0           PDE4                                56:56
-// 1           PDE3                                55:47
-// 2           PDE2 (or 256G PTE)                  46:38
-// 3           PDE1 (or 512M PTE)                  37:29
-// 4           PDE0 (dual 64K/4K PDE, or 2M PTE)   28:21
-// 5           PTE_64K / PTE_4K                    20:16 / 20:12
-
-#include "uvm_types.h"
-#include "uvm_global.h"
-#include "uvm_hal.h"
-#include "uvm_hal_types.h"
-#include "uvm_blackwell_fault_buffer.h"
-#include "hwref/blackwell/gb100/dev_fault.h"
-#include "hwref/blackwell/gb100/dev_mmu.h"
-
-static uvm_mmu_mode_hal_t blackwell_mmu_mode_hal;
-
-static NvU32 page_table_depth_blackwell(NvU64 page_size)
-{
-    switch (page_size) {
-        case UVM_PAGE_SIZE_2M:
-            return 4;
-        case UVM_PAGE_SIZE_512M:
-            return 3;
-        case UVM_PAGE_SIZE_256G:
-            return 2;
-        default:
-            return 5;
-    }
-}
-
-static NvU64 page_sizes_blackwell(void)
-{
-    return UVM_PAGE_SIZE_256G | UVM_PAGE_SIZE_512M | UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
-}
-
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size)
-{
-    static bool initialized = false;
-
-    UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
-
-    // TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
-    // 128K big page size for Pascal+ GPUs
-    if (big_page_size == UVM_PAGE_SIZE_128K)
-        return NULL;
-
-    if (!initialized) {
-        uvm_mmu_mode_hal_t *hopper_mmu_mode_hal = uvm_hal_mmu_mode_hopper(big_page_size);
-        UVM_ASSERT(hopper_mmu_mode_hal);
-
-        // The assumption made is that arch_hal->mmu_mode_hal() will be called
-        // under the global lock the first time, so check it here.
-        uvm_assert_mutex_locked(&g_uvm_global.global_lock);
-
-        blackwell_mmu_mode_hal = *hopper_mmu_mode_hal;
-        blackwell_mmu_mode_hal.page_table_depth = page_table_depth_blackwell;
-        blackwell_mmu_mode_hal.page_sizes = page_sizes_blackwell;
-
-        initialized = true;
-    }
-
-    return &blackwell_mmu_mode_hal;
-}
-
-NvU16 uvm_hal_blackwell_mmu_client_id_to_utlb_id(NvU16 client_id)
-{
-    switch (client_id) {
-        case NV_PFAULT_CLIENT_GPC_RAST:
-        case NV_PFAULT_CLIENT_GPC_GCC:
-        case NV_PFAULT_CLIENT_GPC_GPCCS:
-            return UVM_BLACKWELL_GPC_UTLB_ID_RGG;
-        case NV_PFAULT_CLIENT_GPC_T1_0:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP0;
-        case NV_PFAULT_CLIENT_GPC_T1_1:
-        case NV_PFAULT_CLIENT_GPC_PE_0:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_0:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP1;
-        case NV_PFAULT_CLIENT_GPC_T1_2:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP2;
-        case NV_PFAULT_CLIENT_GPC_T1_3:
-        case NV_PFAULT_CLIENT_GPC_PE_1:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_1:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP3;
-        case NV_PFAULT_CLIENT_GPC_T1_4:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP4;
-        case NV_PFAULT_CLIENT_GPC_T1_5:
-        case NV_PFAULT_CLIENT_GPC_PE_2:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_2:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP5;
-        case NV_PFAULT_CLIENT_GPC_T1_6:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP6;
-        case NV_PFAULT_CLIENT_GPC_T1_7:
-        case NV_PFAULT_CLIENT_GPC_PE_3:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_3:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP7;
-        case NV_PFAULT_CLIENT_GPC_T1_8:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP8;
-        case NV_PFAULT_CLIENT_GPC_T1_9:
-        case NV_PFAULT_CLIENT_GPC_PE_4:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_4:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP9;
-        case NV_PFAULT_CLIENT_GPC_T1_10:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP10;
-        case NV_PFAULT_CLIENT_GPC_T1_11:
-        case NV_PFAULT_CLIENT_GPC_PE_5:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_5:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP11;
-        case NV_PFAULT_CLIENT_GPC_T1_12:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP12;
-        case NV_PFAULT_CLIENT_GPC_T1_13:
-        case NV_PFAULT_CLIENT_GPC_PE_6:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_6:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP13;
-        case NV_PFAULT_CLIENT_GPC_T1_14:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP14;
-        case NV_PFAULT_CLIENT_GPC_T1_15:
-        case NV_PFAULT_CLIENT_GPC_PE_7:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_7:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP15;
-        case NV_PFAULT_CLIENT_GPC_T1_16:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP16;
-        case NV_PFAULT_CLIENT_GPC_T1_17:
-        case NV_PFAULT_CLIENT_GPC_PE_8:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_8:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP17;
-        case NV_PFAULT_CLIENT_GPC_T1_18:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP18;
-        case NV_PFAULT_CLIENT_GPC_T1_19:
-        case NV_PFAULT_CLIENT_GPC_PE_9:
-        case NV_PFAULT_CLIENT_GPC_TPCCS_9:
-            return UVM_BLACKWELL_GPC_UTLB_ID_LTP19;
-
-        default:
-            UVM_ASSERT_MSG(false, "Invalid client value: 0x%x\n", client_id);
-    }
-
-    return 0;
-}

kernel-open/nvidia-uvm/uvm_ce_test.c

@@ -855,6 +855,7 @@ static NV_STATUS cpu_decrypt_in_order(uvm_channel_t *channel,
                                       uvm_mem_t *dst_mem,
                                       uvm_mem_t *src_mem,
                                       const UvmCslIv *decrypt_iv,
+                                      NvU32 key_version,
                                       uvm_mem_t *auth_tag_mem,
                                       size_t size,
                                       NvU32 copy_size)
@@ -869,6 +870,7 @@ static NV_STATUS cpu_decrypt_in_order(uvm_channel_t *channel,
                                                          dst_plain + i * copy_size,
                                                          src_cipher + i * copy_size,
                                                          decrypt_iv + i,
+                                                         key_version,
                                                          copy_size,
                                                          auth_tag_buffer + i * UVM_CONF_COMPUTING_AUTH_TAG_SIZE));
     }
@@ -879,6 +881,7 @@ static NV_STATUS cpu_decrypt_out_of_order(uvm_channel_t *channel,
                                           uvm_mem_t *dst_mem,
                                           uvm_mem_t *src_mem,
                                           const UvmCslIv *decrypt_iv,
+                                          NvU32 key_version,
                                           uvm_mem_t *auth_tag_mem,
                                           size_t size,
                                           NvU32 copy_size)
@@ -896,6 +899,7 @@ static NV_STATUS cpu_decrypt_out_of_order(uvm_channel_t *channel,
                                                          dst_plain + i * copy_size,
                                                          src_cipher + i * copy_size,
                                                          decrypt_iv + i,
+                                                         key_version,
                                                          copy_size,
                                                          auth_tag_buffer + i * UVM_CONF_COMPUTING_AUTH_TAG_SIZE));
     }
@@ -959,7 +963,7 @@ static void gpu_encrypt(uvm_push_t *push,
                                                           i * UVM_CONF_COMPUTING_AUTH_TAG_SIZE,
                                                           dst_cipher);
 
-        uvm_conf_computing_log_gpu_encryption(push->channel, decrypt_iv);
+        uvm_conf_computing_log_gpu_encryption(push->channel, copy_size, decrypt_iv);
 
         if (i > 0)
             uvm_push_set_flag(push, UVM_PUSH_FLAG_CE_NEXT_PIPELINED);
@@ -1020,6 +1024,7 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu,
     size_t auth_tag_buffer_size = (size / copy_size) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
     UvmCslIv *decrypt_iv = NULL;
     UvmCslIv *encrypt_iv = NULL;
+    NvU32 key_version;
     uvm_tracker_t tracker;
     size_t src_plain_size;
 
@@ -1089,6 +1094,11 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu,
 
     gpu_encrypt(&push, dst_cipher, dst_plain_gpu, auth_tag_mem, decrypt_iv, size, copy_size);
 
+    // There shouldn't be any key rotation between the end of the push and the
+    // CPU decryption(s), but it is more robust against test changes to force
+    // decryption to use the saved key.
+    key_version = uvm_channel_pool_key_version(push.channel->pool);
+
     TEST_NV_CHECK_GOTO(uvm_push_end_and_wait(&push), out);
 
     TEST_CHECK_GOTO(!mem_match(src_plain, src_cipher, size), out);
@@ -1101,6 +1111,7 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu,
                                                 dst_plain,
                                                 dst_cipher,
                                                 decrypt_iv,
+                                                key_version,
                                                 auth_tag_mem,
                                                 size,
                                                 copy_size),
@@ -1111,6 +1122,7 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu,
                                                     dst_plain,
                                                     dst_cipher,
                                                     decrypt_iv,
+                                                    key_version,
                                                     auth_tag_mem,
                                                     size,
                                                     copy_size),

kernel-open/nvidia-uvm/uvm_channel.h

@@ -228,21 +228,65 @@ typedef struct
     // variant is required when the thread holding the pool lock must sleep
     // (ex: acquire another mutex) deeper in the call stack, either in UVM or
     // RM.
-    union {
+    union
+    {
         uvm_spinlock_t spinlock;
         uvm_mutex_t mutex;
     };
 
-    // Secure operations require that uvm_push_begin order matches
-    // uvm_push_end order, because the engine's state is used in its internal
-    // operation and each push may modify this state. push_locks is protected by
-    // the channel pool lock.
-    DECLARE_BITMAP(push_locks, UVM_CHANNEL_MAX_NUM_CHANNELS_PER_POOL);
+    struct
+    {
+        // Secure operations require that uvm_push_begin order matches
+        // uvm_push_end order, because the engine's state is used in its
+        // internal operation and each push may modify this state.
+        // push_locks is protected by the channel pool lock.
+        DECLARE_BITMAP(push_locks, UVM_CHANNEL_MAX_NUM_CHANNELS_PER_POOL);
 
-    // Counting semaphore for available and unlocked channels, it must be
-    // acquired before submitting work to a channel when the Confidential
-    // Computing feature is enabled.
-    uvm_semaphore_t push_sem;
+        // Counting semaphore for available and unlocked channels, it must be
+        // acquired before submitting work to a channel when the Confidential
+        // Computing feature is enabled.
+        uvm_semaphore_t push_sem;
+
+        // Per channel buffers in unprotected sysmem.
+        uvm_rm_mem_t *pool_sysmem;
+
+        // Per channel buffers in protected vidmem.
+        uvm_rm_mem_t *pool_vidmem;
+
+       struct
+       {
+            // Current encryption key version, incremented upon key rotation.
+            // While there are separate keys for encryption and decryption, the
+            // two keys are rotated at once, so the versioning applies to both.
+            NvU32 version;
+
+            // Lock used to ensure mutual exclusion during key rotation.
+            uvm_mutex_t mutex;
+
+            // CSL contexts passed to RM for key rotation. This is usually an
+            // array containing the CSL contexts associated with the channels in
+            // the pool. In the case of the WLC pool, the array also includes
+            // CSL contexts associated with LCIC channels.
+            UvmCslContext **csl_contexts;
+
+            // Number of elements in the CSL context array.
+            unsigned num_csl_contexts;
+
+            // Number of bytes encrypted, or decrypted, on the engine associated
+            // with the pool since the last key rotation. Only used during
+            // testing, to force key rotations after a certain encryption size,
+            // see UVM_CONF_COMPUTING_KEY_ROTATION_LOWER_THRESHOLD.
+            //
+            // Encryptions on a LCIC pool are accounted for in the paired WLC
+            // pool.
+            //
+            // TODO: Bug 4612912: these accounting variables can be removed once
+            // RM exposes an API to set the key rotation lower threshold.
+            atomic64_t encrypted;
+            atomic64_t decrypted;
+        } key_rotation;
+
+    } conf_computing;
 } uvm_channel_pool_t;
 
 struct uvm_channel_struct
@@ -322,43 +366,14 @@ struct uvm_channel_struct
         // work launches to match the order of push end-s that triggered them.
         volatile NvU32 gpu_put;
 
-        // Static pushbuffer for channels with static schedule (WLC/LCIC)
-        uvm_rm_mem_t *static_pb_protected_vidmem;
-
-        // Static pushbuffer staging buffer for WLC
-        uvm_rm_mem_t *static_pb_unprotected_sysmem;
-        void *static_pb_unprotected_sysmem_cpu;
-        void *static_pb_unprotected_sysmem_auth_tag_cpu;
-
-        // The above static locations are required by the WLC (and LCIC)
-        // schedule. Protected sysmem location completes WLC's independence
-        // from the pushbuffer allocator.
+        // Protected sysmem location makes WLC independent from the pushbuffer
+        // allocator. Unprotected sysmem and protected vidmem counterparts
+        // are allocated from the channel pool (sysmem, vidmem).
         void *static_pb_protected_sysmem;
 
-        // Static tracking semaphore notifier values
-        // Because of LCIC's fixed schedule, the secure semaphore release
-        // mechanism uses two additional static locations for incrementing the
-        // notifier values. See:
-        // . channel_semaphore_secure_release()
-        // . setup_lcic_schedule()
-        // . internal_channel_submit_work_wlc()
-        uvm_rm_mem_t *static_notifier_unprotected_sysmem;
-        NvU32 *static_notifier_entry_unprotected_sysmem_cpu;
-        NvU32 *static_notifier_exit_unprotected_sysmem_cpu;
-        uvm_gpu_address_t static_notifier_entry_unprotected_sysmem_gpu_va;
-        uvm_gpu_address_t static_notifier_exit_unprotected_sysmem_gpu_va;
-
-        // Explicit location for push launch tag used by WLC.
-        // Encryption auth tags have to be located in unprotected sysmem.
-        void *launch_auth_tag_cpu;
-        NvU64 launch_auth_tag_gpu_va;
-
         // Used to decrypt the push back to protected sysmem.
         // This happens when profilers register callbacks for migration data.
         uvm_push_crypto_bundle_t *push_crypto_bundles;
-
-        // Accompanying authentication tags for the crypto bundles
-        uvm_rm_mem_t *push_crypto_bundle_auth_tags;
     } conf_computing;
 
     // RM channel information
@@ -451,6 +466,16 @@ struct uvm_channel_manager_struct
         UVM_BUFFER_LOCATION gpput_loc;
         UVM_BUFFER_LOCATION pushbuffer_loc;
     } conf;
+
+    struct
+    {
+        // Flag indicating that the WLC/LCIC mechanism is ready/setup; should
+        // only be false during (de)initialization.
+        bool wlc_ready;
+
+        // True indicates that key rotation is enabled (UVM-wise).
+        bool key_rotation_enabled;
+    } conf_computing;
 };
 
 // Create a channel manager for the GPU
@@ -501,6 +526,14 @@ uvm_channel_t *uvm_channel_lcic_get_paired_wlc(uvm_channel_t *lcic_channel);
 
 uvm_channel_t *uvm_channel_wlc_get_paired_lcic(uvm_channel_t *wlc_channel);
 
+NvU64 uvm_channel_get_static_pb_protected_vidmem_gpu_va(uvm_channel_t *channel);
+
+NvU64 uvm_channel_get_static_pb_unprotected_sysmem_gpu_va(uvm_channel_t *channel);
+
+char* uvm_channel_get_static_pb_unprotected_sysmem_cpu(uvm_channel_t *channel);
+
+char *uvm_channel_get_push_crypto_bundle_auth_tags_cpu_va(uvm_channel_t *channel, unsigned tag_index);
+
 static bool uvm_channel_pool_is_proxy(uvm_channel_pool_t *pool)
 {
     UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));
@@ -532,6 +565,17 @@ static uvm_channel_type_t uvm_channel_proxy_channel_type(void)
     return UVM_CHANNEL_TYPE_MEMOPS;
 }
 
+// Force key rotation in the engine associated with the given channel pool.
+// Rotation may still not happen if RM cannot acquire the necessary locks (in
+// which case the function returns NV_ERR_STATE_IN_USE).
+//
+// This function should be only invoked in pools in which key rotation is
+// enabled.
+NV_STATUS uvm_channel_pool_rotate_key(uvm_channel_pool_t *pool);
+
+// Retrieve the current encryption key version associated with the channel pool.
+NvU32 uvm_channel_pool_key_version(uvm_channel_pool_t *pool);
+
 // Privileged channels support all the Host and engine methods, while
 // non-privileged channels don't support privileged methods.
 //
@@ -579,12 +623,9 @@ NvU32 uvm_channel_manager_update_progress(uvm_channel_manager_t *channel_manager
 // beginning.
 NV_STATUS uvm_channel_manager_wait(uvm_channel_manager_t *manager);
 
-// Check if WLC/LCIC mechanism is ready/setup
-// Should only return false during initialization
 static bool uvm_channel_manager_is_wlc_ready(uvm_channel_manager_t *manager)
 {
-    return (manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_WLC] != NULL) &&
-           (manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_LCIC] != NULL);
+    return manager->conf_computing.wlc_ready;
 }
 // Get the GPU VA of semaphore_channel's tracking semaphore within the VA space
 // associated with access_channel.

kernel-open/nvidia-uvm/uvm_channel_test.c

@@ -793,14 +793,11 @@ done:
 // This test verifies that concurrent pushes using the same channel pool
 // select different channels, when the Confidential Computing feature is
 // enabled.
-static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
+NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
 {
     NV_STATUS status = NV_OK;
-    uvm_channel_pool_t *pool;
-    uvm_push_t *pushes;
-    uvm_gpu_t *gpu;
-    NvU32 i;
-    NvU32 num_pushes;
+    uvm_push_t *pushes = NULL;
+    uvm_gpu_t *gpu = NULL;
 
     if (!g_uvm_global.conf_computing_enabled)
         return NV_OK;
@@ -810,9 +807,19 @@ static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
     for_each_va_space_gpu(gpu, va_space) {
         uvm_channel_type_t channel_type;
 
+        // Key rotation is disabled because this test relies on nested pushes,
+        // which is illegal. If any push other than the first one triggers key
+        // rotation, the test won't complete. This is because key rotation
+        // depends on waiting for ongoing pushes to end, which doesn't happen
+        // if those pushes are ended after the current one begins.
+        uvm_conf_computing_disable_key_rotation(gpu);
+
         for (channel_type = 0; channel_type < UVM_CHANNEL_TYPE_COUNT; channel_type++) {
-            pool = gpu->channel_manager->pool_to_use.default_for_type[channel_type];
-            TEST_CHECK_RET(pool != NULL);
+            NvU32 i;
+            NvU32 num_pushes;
+            uvm_channel_pool_t *pool = gpu->channel_manager->pool_to_use.default_for_type[channel_type];
+
+            TEST_CHECK_GOTO(pool != NULL, error);
 
             // Skip LCIC channels as those can't accept any pushes
             if (uvm_channel_pool_is_lcic(pool))
@@ -824,7 +831,7 @@ static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
             num_pushes = min(pool->num_channels, (NvU32)UVM_PUSH_MAX_CONCURRENT_PUSHES);
 
             pushes = uvm_kvmalloc_zero(sizeof(*pushes) * num_pushes);
-            TEST_CHECK_RET(pushes != NULL);
+            TEST_CHECK_GOTO(pushes != NULL, error);
 
             for (i = 0; i < num_pushes; i++) {
                 uvm_push_t *push = &pushes[i];
@@ -841,19 +848,25 @@ static NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
 
             uvm_kvfree(pushes);
         }
+
+        uvm_conf_computing_enable_key_rotation(gpu);
     }
 
     uvm_thread_context_lock_enable_tracking();
 
     return status;
+
 error:
+    if (gpu != NULL)
+        uvm_conf_computing_enable_key_rotation(gpu);
+
     uvm_thread_context_lock_enable_tracking();
     uvm_kvfree(pushes);
 
     return status;
 }
 
-static NV_STATUS test_channel_iv_rotation(uvm_va_space_t *va_space)
+NV_STATUS test_channel_iv_rotation(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -948,7 +961,319 @@ release:
     return NV_OK;
 }
 
-static NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
+static NV_STATUS force_key_rotations(uvm_channel_pool_t *pool, unsigned num_rotations)
+{
+    unsigned num_tries;
+    unsigned max_num_tries = 20;
+    unsigned num_rotations_completed = 0;
+
+    if (num_rotations == 0)
+        return NV_OK;
+
+    // The number of accepted rotations is kept low, so failed rotation
+    // invocations due to RM not acquiring the necessary locks (which imply a
+    // sleep in the test) do not balloon the test execution time.
+    UVM_ASSERT(num_rotations <= 10);
+
+    for (num_tries = 0; (num_tries < max_num_tries) && (num_rotations_completed < num_rotations); num_tries++) {
+        // Force key rotation, irrespective of encryption usage.
+        NV_STATUS status = uvm_channel_pool_rotate_key(pool);
+
+        // Key rotation may not be able to complete due to RM failing to acquire
+        // the necessary locks. Detect the situation, sleep for a bit, and then
+        // try again
+        //
+        // The maximum time spent sleeping in a single rotation call is
+        // (max_num_tries * max_sleep_us)
+        if (status == NV_ERR_STATE_IN_USE) {
+            NvU32 min_sleep_us = 1000;
+            NvU32 max_sleep_us = 10000;
+
+            usleep_range(min_sleep_us, max_sleep_us);
+            continue;
+        }
+
+        TEST_NV_CHECK_RET(status);
+
+        num_rotations_completed++;
+    }
+
+    // If not a single key rotation occurred, the dependent tests still pass,
+    // but there is no much value to them. Instead, return an error so the
+    // maximum number of tries, or the maximum sleep time, are adjusted to
+    // ensure that at least one rotation completes.
+    if (num_rotations_completed > 0)
+        return NV_OK;
+    else
+        return NV_ERR_STATE_IN_USE;
+}
+
+static NV_STATUS force_key_rotation(uvm_channel_pool_t *pool)
+{
+    return force_key_rotations(pool, 1);
+}
+
+// Test key rotation in all pools. This is useful because key rotation may not
+// happen otherwise on certain engines during UVM test execution. For example,
+// if the MEMOPS channel type is mapped to a CE not shared with any other
+// channel type, then the only encryption taking place in the engine is due to
+// semaphore releases (4 bytes each). This small encryption size makes it
+// unlikely to exceed even small rotation thresholds.
+static NV_STATUS test_channel_key_rotation_basic(uvm_gpu_t *gpu)
+{
+    uvm_channel_pool_t *pool;
+
+    uvm_for_each_pool(pool, gpu->channel_manager) {
+        if (!uvm_conf_computing_is_key_rotation_enabled_in_pool(pool))
+            continue;
+
+        TEST_NV_CHECK_RET(force_key_rotation(pool));
+    }
+
+    return NV_OK;
+}
+
+// Interleave GPU encryptions and decryptions, and their CPU counterparts, with
+// key rotations.
+static NV_STATUS test_channel_key_rotation_interleave(uvm_gpu_t *gpu)
+{
+    int i;
+    uvm_channel_pool_t *gpu_to_cpu_pool;
+    uvm_channel_pool_t *cpu_to_gpu_pool;
+    NV_STATUS status = NV_OK;
+    size_t size = UVM_CONF_COMPUTING_DMA_BUFFER_SIZE;
+    void *initial_plain_cpu = NULL;
+    void *final_plain_cpu = NULL;
+    uvm_mem_t *plain_gpu = NULL;
+    uvm_gpu_address_t plain_gpu_address;
+
+    cpu_to_gpu_pool = gpu->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_CPU_TO_GPU];
+    TEST_CHECK_RET(uvm_conf_computing_is_key_rotation_enabled_in_pool(cpu_to_gpu_pool));
+
+    gpu_to_cpu_pool = gpu->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_CPU];
+    TEST_CHECK_RET(uvm_conf_computing_is_key_rotation_enabled_in_pool(gpu_to_cpu_pool));
+
+    initial_plain_cpu = uvm_kvmalloc_zero(size);
+    if (initial_plain_cpu == NULL) {
+        status = NV_ERR_NO_MEMORY;
+        goto out;
+    }
+
+    final_plain_cpu = uvm_kvmalloc_zero(size);
+    if (final_plain_cpu == NULL) {
+        status = NV_ERR_NO_MEMORY;
+        goto out;
+    }
+
+    TEST_NV_CHECK_GOTO(uvm_mem_alloc_vidmem(size, gpu, &plain_gpu), out);
+    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(plain_gpu, gpu), out);
+    plain_gpu_address = uvm_mem_gpu_address_virtual_kernel(plain_gpu, gpu);
+
+    memset(initial_plain_cpu, 1, size);
+
+    for (i = 0; i < 5; i++) {
+        TEST_NV_CHECK_GOTO(force_key_rotation(gpu_to_cpu_pool), out);
+        TEST_NV_CHECK_GOTO(force_key_rotation(cpu_to_gpu_pool), out);
+
+        TEST_NV_CHECK_GOTO(uvm_conf_computing_util_memcopy_cpu_to_gpu(gpu,
+                                                                      plain_gpu_address,
+                                                                      initial_plain_cpu,
+                                                                      size,
+                                                                      NULL,
+                                                                      "CPU > GPU"),
+                           out);
+
+        TEST_NV_CHECK_GOTO(force_key_rotation(gpu_to_cpu_pool), out);
+        TEST_NV_CHECK_GOTO(force_key_rotation(cpu_to_gpu_pool), out);
+
+        TEST_NV_CHECK_GOTO(uvm_conf_computing_util_memcopy_gpu_to_cpu(gpu,
+                                                                      final_plain_cpu,
+                                                                      plain_gpu_address,
+                                                                      size,
+                                                                      NULL,
+                                                                      "GPU > CPU"),
+                           out);
+
+        TEST_CHECK_GOTO(!memcmp(initial_plain_cpu, final_plain_cpu, size), out);
+
+        memset(final_plain_cpu, 0, size);
+    }
+
+out:
+    uvm_mem_free(plain_gpu);
+    uvm_kvfree(final_plain_cpu);
+    uvm_kvfree(initial_plain_cpu);
+
+    return status;
+}
+
+static NV_STATUS memset_vidmem(uvm_mem_t *mem, NvU8 val)
+{
+    uvm_push_t push;
+    uvm_gpu_address_t gpu_address;
+    uvm_gpu_t *gpu = mem->backing_gpu;
+
+    UVM_ASSERT(uvm_mem_is_vidmem(mem));
+
+    TEST_NV_CHECK_RET(uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, &push, "zero vidmem"));
+
+    gpu_address = uvm_mem_gpu_address_virtual_kernel(mem, gpu);
+    gpu->parent->ce_hal->memset_1(&push, gpu_address, val, mem->size);
+
+    TEST_NV_CHECK_RET(uvm_push_end_and_wait(&push));
+
+    return NV_OK;
+}
+
+// Custom version of uvm_conf_computing_util_memcopy_gpu_to_cpu that allows
+// testing to insert key rotations in between the push end, and the CPU
+// decryption
+static NV_STATUS encrypted_memcopy_gpu_to_cpu(uvm_gpu_t *gpu,
+                                              void *dst_plain,
+                                              uvm_gpu_address_t src_gpu_address,
+                                              size_t size,
+                                              unsigned num_rotations_to_insert)
+{
+    NV_STATUS status;
+    uvm_push_t push;
+    uvm_conf_computing_dma_buffer_t *dma_buffer;
+    uvm_gpu_address_t dst_gpu_address, auth_tag_gpu_address;
+    void *src_cipher, *auth_tag;
+    uvm_channel_t *channel;
+
+    UVM_ASSERT(g_uvm_global.conf_computing_enabled);
+    UVM_ASSERT(size <= UVM_CONF_COMPUTING_DMA_BUFFER_SIZE);
+
+    status = uvm_conf_computing_dma_buffer_alloc(&gpu->conf_computing.dma_buffer_pool, &dma_buffer, NULL);
+    if (status != NV_OK)
+        return status;
+
+    status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_TO_CPU, &push, "Small GPU > CPU encryption");
+    if (status != NV_OK)
+        goto out;
+
+    channel = push.channel;
+    uvm_conf_computing_log_gpu_encryption(channel, size, dma_buffer->decrypt_iv);
+    dma_buffer->key_version[0] = uvm_channel_pool_key_version(channel->pool);
+
+    dst_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->alloc, gpu);
+    auth_tag_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->auth_tag, gpu);
+    gpu->parent->ce_hal->encrypt(&push, dst_gpu_address, src_gpu_address, size, auth_tag_gpu_address);
+
+    status = uvm_push_end_and_wait(&push);
+    if (status != NV_OK)
+        goto out;
+
+    TEST_NV_CHECK_GOTO(force_key_rotations(channel->pool, num_rotations_to_insert), out);
+
+    // If num_rotations_to_insert is not zero, the current encryption key will
+    // be different from the one used during CE encryption.
+
+    src_cipher = uvm_mem_get_cpu_addr_kernel(dma_buffer->alloc);
+    auth_tag = uvm_mem_get_cpu_addr_kernel(dma_buffer->auth_tag);
+    status = uvm_conf_computing_cpu_decrypt(channel,
+                                            dst_plain,
+                                            src_cipher,
+                                            dma_buffer->decrypt_iv,
+                                            dma_buffer->key_version[0],
+                                            size,
+                                            auth_tag);
+
+ out:
+    uvm_conf_computing_dma_buffer_free(&gpu->conf_computing.dma_buffer_pool, dma_buffer, NULL);
+    return status;
+}
+
+static NV_STATUS test_channel_key_rotation_cpu_decryption(uvm_gpu_t *gpu,
+                                                          unsigned num_repetitions,
+                                                          unsigned num_rotations_to_insert)
+{
+    unsigned i;
+    uvm_channel_pool_t *gpu_to_cpu_pool;
+    NV_STATUS status = NV_OK;
+    size_t size = UVM_CONF_COMPUTING_DMA_BUFFER_SIZE;
+    NvU8 *plain_cpu = NULL;
+    uvm_mem_t *plain_gpu = NULL;
+    uvm_gpu_address_t plain_gpu_address;
+
+    if (!uvm_conf_computing_is_key_rotation_enabled(gpu))
+        return NV_OK;
+
+    gpu_to_cpu_pool = gpu->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_CPU];
+    TEST_CHECK_RET(uvm_conf_computing_is_key_rotation_enabled_in_pool(gpu_to_cpu_pool));
+
+    plain_cpu = (NvU8 *) uvm_kvmalloc_zero(size);
+    if (plain_cpu == NULL) {
+        status = NV_ERR_NO_MEMORY;
+        goto out;
+    }
+
+    TEST_NV_CHECK_GOTO(uvm_mem_alloc_vidmem(size, gpu, &plain_gpu), out);
+    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(plain_gpu, gpu), out);
+    TEST_NV_CHECK_GOTO(memset_vidmem(plain_gpu, 1), out);
+
+    plain_gpu_address = uvm_mem_gpu_address_virtual_kernel(plain_gpu, gpu);
+
+    for (i = 0; i < num_repetitions; i++) {
+        unsigned j;
+
+        TEST_NV_CHECK_GOTO(encrypted_memcopy_gpu_to_cpu(gpu,
+                                                        plain_cpu,
+                                                        plain_gpu_address,
+                                                        size,
+                                                        num_rotations_to_insert),
+                          out);
+
+        for (j = 0; j < size; j++)
+            TEST_CHECK_GOTO(plain_cpu[j] == 1, out);
+
+        memset(plain_cpu, 0, size);
+
+    }
+out:
+    uvm_mem_free(plain_gpu);
+    uvm_kvfree(plain_cpu);
+
+    return status;
+}
+
+// Test that CPU decryptions can use old keys i.e. previous versions of the keys
+// that are no longer the current key, due to key rotation. Given that SEC2
+// does not expose encryption capabilities, the "decrypt-after-rotation" problem
+// is exclusive of CE encryptions.
+static NV_STATUS test_channel_key_rotation_decrypt_after_key_rotation(uvm_gpu_t *gpu)
+{
+    // Instruct encrypted_memcopy_gpu_to_cpu to insert several key rotations
+    // between the GPU encryption, and the associated CPU decryption.
+    unsigned num_rotations_to_insert = 8;
+
+    TEST_NV_CHECK_RET(test_channel_key_rotation_cpu_decryption(gpu, 1, num_rotations_to_insert));
+
+    return NV_OK;
+}
+
+static NV_STATUS test_channel_key_rotation(uvm_va_space_t *va_space)
+{
+    uvm_gpu_t *gpu;
+
+    if (!g_uvm_global.conf_computing_enabled)
+        return NV_OK;
+
+    for_each_va_space_gpu(gpu, va_space) {
+        if (!uvm_conf_computing_is_key_rotation_enabled(gpu))
+            break;
+
+        TEST_NV_CHECK_RET(test_channel_key_rotation_basic(gpu));
+
+        TEST_NV_CHECK_RET(test_channel_key_rotation_interleave(gpu));
+
+        TEST_NV_CHECK_RET(test_channel_key_rotation_decrypt_after_key_rotation(gpu));
+    }
+
+    return NV_OK;
+}
+
+NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -987,7 +1312,7 @@ static NV_STATUS test_write_ctrl_gpfifo_noop(uvm_va_space_t *va_space)
     return NV_OK;
 }
 
-static NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
+NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
 
@@ -1035,7 +1360,7 @@ static NV_STATUS test_write_ctrl_gpfifo_and_pushes(uvm_va_space_t *va_space)
     return NV_OK;
 }
 
-static NV_STATUS test_write_ctrl_gpfifo_tight(uvm_va_space_t *va_space)
+NV_STATUS test_write_ctrl_gpfifo_tight(uvm_va_space_t *va_space)
 {
     NV_STATUS status = NV_OK;
     uvm_gpu_t *gpu;
@@ -1203,6 +1528,10 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct
     if (status != NV_OK)
         goto done;
 
+    status = test_channel_key_rotation(va_space);
+    if (status != NV_OK)
+        goto done;
+
     // The following tests have side effects, they reset the GPU's
     // channel_manager.
     status = test_channel_pushbuffer_extension_base(va_space);
@@ -1338,6 +1667,126 @@ done:
     return status;
 }
 
+static NV_STATUS channel_stress_key_rotation_cpu_encryption(uvm_gpu_t *gpu, UVM_TEST_CHANNEL_STRESS_PARAMS *params)
+{
+    int i;
+    uvm_channel_pool_t *cpu_to_gpu_pool;
+    NV_STATUS status = NV_OK;
+    size_t size = UVM_CONF_COMPUTING_DMA_BUFFER_SIZE;
+    void *initial_plain_cpu = NULL;
+    uvm_mem_t *plain_gpu = NULL;
+    uvm_gpu_address_t plain_gpu_address;
+
+    UVM_ASSERT(params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_CPU_TO_GPU);
+
+    cpu_to_gpu_pool = gpu->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_CPU_TO_GPU];
+    TEST_CHECK_RET(uvm_conf_computing_is_key_rotation_enabled_in_pool(cpu_to_gpu_pool));
+
+    initial_plain_cpu = uvm_kvmalloc_zero(size);
+    if (initial_plain_cpu == NULL) {
+        status = NV_ERR_NO_MEMORY;
+        goto out;
+    }
+
+    TEST_NV_CHECK_GOTO(uvm_mem_alloc_vidmem(size, gpu, &plain_gpu), out);
+    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(plain_gpu, gpu), out);
+    plain_gpu_address = uvm_mem_gpu_address_virtual_kernel(plain_gpu, gpu);
+
+    memset(initial_plain_cpu, 1, size);
+
+    for (i = 0; i < params->iterations; i++) {
+        TEST_NV_CHECK_GOTO(uvm_conf_computing_util_memcopy_cpu_to_gpu(gpu,
+                                                                      plain_gpu_address,
+                                                                      initial_plain_cpu,
+                                                                      size,
+                                                                      NULL,
+                                                                      "CPU > GPU"),
+                           out);
+    }
+
+out:
+    uvm_mem_free(plain_gpu);
+    uvm_kvfree(initial_plain_cpu);
+
+    return status;
+}
+
+static NV_STATUS channel_stress_key_rotation_cpu_decryption(uvm_gpu_t *gpu, UVM_TEST_CHANNEL_STRESS_PARAMS *params)
+{
+    unsigned num_rotations_to_insert = 0;
+
+    UVM_ASSERT(params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_GPU_TO_CPU);
+
+    return test_channel_key_rotation_cpu_decryption(gpu, params->iterations, num_rotations_to_insert);
+}
+
+static NV_STATUS channel_stress_key_rotation_rotate(uvm_gpu_t *gpu, UVM_TEST_CHANNEL_STRESS_PARAMS *params)
+{
+    NvU32 i;
+
+    UVM_ASSERT(params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_ROTATE);
+
+    for (i = 0; i < params->iterations; ++i) {
+        NV_STATUS status;
+        uvm_channel_pool_t *pool;
+        uvm_channel_type_t type;
+
+        if ((i % 3) == 0)
+            type = UVM_CHANNEL_TYPE_CPU_TO_GPU;
+        else if ((i % 3) == 1)
+            type = UVM_CHANNEL_TYPE_GPU_TO_CPU;
+        else
+            type = UVM_CHANNEL_TYPE_WLC;
+
+        pool = gpu->channel_manager->pool_to_use.default_for_type[type];
+
+        if (!uvm_conf_computing_is_key_rotation_enabled_in_pool(pool))
+            return NV_ERR_INVALID_STATE;
+
+        status = force_key_rotation(pool);
+        if (status != NV_OK)
+            return status;
+    }
+
+    return NV_OK;
+}
+
+// The objective of this test is documented in the user-level function
+static NV_STATUS uvm_test_channel_stress_key_rotation(uvm_va_space_t *va_space, UVM_TEST_CHANNEL_STRESS_PARAMS *params)
+{
+    uvm_test_rng_t rng;
+    uvm_gpu_t *gpu;
+    NV_STATUS status = NV_OK;
+
+    if (!g_uvm_global.conf_computing_enabled)
+        return NV_OK;
+
+    uvm_test_rng_init(&rng, params->seed);
+
+    uvm_va_space_down_read(va_space);
+
+    // Key rotation should be enabled, or disabled, in all GPUs. Pick a random
+    // one.
+    gpu = random_va_space_gpu(&rng, va_space);
+
+    if (!uvm_conf_computing_is_key_rotation_enabled(gpu))
+        goto out;
+
+    if (params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_CPU_TO_GPU)
+        status = channel_stress_key_rotation_cpu_encryption(gpu, params);
+    else if (params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_GPU_TO_CPU)
+        status = channel_stress_key_rotation_cpu_decryption(gpu, params);
+    else if (params->key_rotation_operation == UVM_TEST_CHANNEL_STRESS_KEY_ROTATION_OPERATION_ROTATE)
+        status = channel_stress_key_rotation_rotate(gpu, params);
+    else
+        status = NV_ERR_INVALID_PARAMETER;
+
+out:
+    uvm_va_space_up_read(va_space);
+
+    return status;
+}
+
 NV_STATUS uvm_test_channel_stress(UVM_TEST_CHANNEL_STRESS_PARAMS *params, struct file *filp)
 {
     uvm_va_space_t *va_space = uvm_va_space_get(filp);
@@ -1349,6 +1798,8 @@ NV_STATUS uvm_test_channel_stress(UVM_TEST_CHANNEL_STRESS_PARAMS *params, struct
             return uvm_test_channel_stress_update_channels(va_space, params);
         case UVM_TEST_CHANNEL_STRESS_MODE_NOOP_PUSH:
             return uvm_test_channel_noop_push(va_space, params);
+        case UVM_TEST_CHANNEL_STRESS_MODE_KEY_ROTATION:
+            return uvm_test_channel_stress_key_rotation(va_space, params);
         default:
             return NV_ERR_INVALID_PARAMETER;
     }

kernel-open/nvidia-uvm/uvm_common.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2013-2023 NVIDIA Corporation
+    Copyright (c) 2013-2024 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -423,7 +423,9 @@ static void uvm_get_unaddressable_range(NvU32 num_va_bits, NvU64 *first, NvU64 *
     UVM_ASSERT(first);
     UVM_ASSERT(outer);
 
-    if (uvm_platform_uses_canonical_form_address()) {
+    // Maxwell GPUs (num_va_bits == 40b) do not support canonical form address
+    // even when plugged into platforms using it.
+    if (uvm_platform_uses_canonical_form_address() && num_va_bits > 40) {
         *first = 1ULL << (num_va_bits - 1);
         *outer = (NvU64)((NvS64)(1ULL << 63) >> (64 - num_va_bits));
     }

kernel-open/nvidia-uvm/uvm_conf_computing.c

@@ -33,6 +33,15 @@
 #include "nv_uvm_interface.h"
 #include "uvm_va_block.h"
 
+// Amount of encrypted data on a given engine that triggers key rotation. This
+// is a UVM internal threshold, different from that of RM, and used only during
+// testing.
+//
+// Key rotation is triggered when the total encryption size, or the total
+// decryption size (whatever comes first) reaches this lower threshold on the
+// engine.
+#define UVM_CONF_COMPUTING_KEY_ROTATION_LOWER_THRESHOLD (UVM_SIZE_1MB * 8)
+
 // The maximum number of secure operations per push is:
 // UVM_MAX_PUSH_SIZE / min(CE encryption size, CE decryption size)
 // + 1 (tracking semaphore) =  128 * 1024 / 56 + 1 = 2342
@@ -352,6 +361,19 @@ error:
     return status;
 }
 
+// The production key rotation defaults are such that key rotations rarely
+// happen. During UVM testing more frequent rotations are triggering by relying
+// on internal encryption usage accounting. When key rotations are triggered by
+// UVM, the driver does not rely on channel key rotation notifiers.
+//
+// TODO: Bug 4612912: UVM should be able to programmatically set the rotation
+// lower threshold. This function, and all the metadata associated with it
+// (per-pool encryption accounting, for example) can be removed at that point.
+static bool key_rotation_is_notifier_driven(void)
+{
+    return !uvm_enable_builtin_tests;
+}
+
 NV_STATUS uvm_conf_computing_gpu_init(uvm_gpu_t *gpu)
 {
     NV_STATUS status;
@@ -394,17 +416,35 @@ void uvm_conf_computing_gpu_deinit(uvm_gpu_t *gpu)
     conf_computing_dma_buffer_pool_deinit(&gpu->conf_computing.dma_buffer_pool);
 }
 
-void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv)
+void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, size_t size, UvmCslIv *iv)
 {
     NV_STATUS status;
+    uvm_channel_pool_t *pool;
+
+    if (uvm_channel_is_lcic(channel))
+        pool = uvm_channel_lcic_get_paired_wlc(channel)->pool;
+    else
+        pool = channel->pool;
 
     uvm_mutex_lock(&channel->csl.ctx_lock);
+
+    if (uvm_conf_computing_is_key_rotation_enabled_in_pool(pool)) {
+        status = nvUvmInterfaceCslLogEncryption(&channel->csl.ctx, UVM_CSL_OPERATION_DECRYPT, size);
+
+        // Informing RM of an encryption/decryption should not fail
+        UVM_ASSERT(status == NV_OK);
+
+        if (!key_rotation_is_notifier_driven())
+            atomic64_add(size, &pool->conf_computing.key_rotation.encrypted);
+    }
+
     status = nvUvmInterfaceCslIncrementIv(&channel->csl.ctx, UVM_CSL_OPERATION_DECRYPT, 1, iv);
-    uvm_mutex_unlock(&channel->csl.ctx_lock);
 
     // IV rotation is done preemptively as needed, so the above
     // call cannot return failure.
     UVM_ASSERT(status == NV_OK);
+
+    uvm_mutex_unlock(&channel->csl.ctx_lock);
 }
 
 void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *iv)
@@ -428,27 +468,46 @@ void uvm_conf_computing_cpu_encrypt(uvm_channel_t *channel,
                                     void *auth_tag_buffer)
 {
     NV_STATUS status;
+    uvm_channel_pool_t *pool;
 
     UVM_ASSERT(size);
 
+    if (uvm_channel_is_lcic(channel))
+        pool = uvm_channel_lcic_get_paired_wlc(channel)->pool;
+    else
+        pool = channel->pool;
+
     uvm_mutex_lock(&channel->csl.ctx_lock);
+
     status = nvUvmInterfaceCslEncrypt(&channel->csl.ctx,
                                       size,
                                       (NvU8 const *) src_plain,
                                       encrypt_iv,
                                       (NvU8 *) dst_cipher,
                                       (NvU8 *) auth_tag_buffer);
-    uvm_mutex_unlock(&channel->csl.ctx_lock);
 
     // IV rotation is done preemptively as needed, so the above
     // call cannot return failure.
     UVM_ASSERT(status == NV_OK);
+
+    if (uvm_conf_computing_is_key_rotation_enabled_in_pool(pool)) {
+        status = nvUvmInterfaceCslLogEncryption(&channel->csl.ctx, UVM_CSL_OPERATION_ENCRYPT, size);
+
+        // Informing RM of an encryption/decryption should not fail
+        UVM_ASSERT(status == NV_OK);
+
+        if (!key_rotation_is_notifier_driven())
+            atomic64_add(size, &pool->conf_computing.key_rotation.decrypted);
+    }
+
+    uvm_mutex_unlock(&channel->csl.ctx_lock);
 }
 
 NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
                                          void *dst_plain,
                                          const void *src_cipher,
                                          const UvmCslIv *src_iv,
+                                         NvU32 key_version,
                                          size_t size,
                                          const void *auth_tag_buffer)
 {
@@ -469,11 +528,19 @@ NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
                                       size,
                                       (const NvU8 *) src_cipher,
                                       src_iv,
-                                      NV_U32_MAX,
+                                      key_version,
                                       (NvU8 *) dst_plain,
                                       NULL,
                                       0,
                                       (const NvU8 *) auth_tag_buffer);
+
+    if (status != NV_OK) {
+        UVM_ERR_PRINT("nvUvmInterfaceCslDecrypt() failed: %s, channel %s, GPU %s\n",
+                      nvstatusToString(status),
+                      channel->name,
+                      uvm_gpu_name(uvm_channel_get_gpu(channel)));
+    }
+
     uvm_mutex_unlock(&channel->csl.ctx_lock);
 
     return status;
@@ -640,3 +707,236 @@ NV_STATUS uvm_conf_computing_maybe_rotate_channel_ivs_retry_busy(uvm_channel_t *
 {
     return uvm_conf_computing_rotate_channel_ivs_below_limit(channel, uvm_conf_computing_channel_iv_rotation_limit, true);
 }
+
+void uvm_conf_computing_enable_key_rotation(uvm_gpu_t *gpu)
+{
+    if (!g_uvm_global.conf_computing_enabled)
+        return;
+
+    // Key rotation cannot be enabled on UVM if it is disabled on RM
+    if (!gpu->parent->rm_info.gpuConfComputeCaps.bKeyRotationEnabled)
+        return;
+
+    gpu->channel_manager->conf_computing.key_rotation_enabled = true;
+}
+
+void uvm_conf_computing_disable_key_rotation(uvm_gpu_t *gpu)
+{
+    if (!g_uvm_global.conf_computing_enabled)
+        return;
+
+    gpu->channel_manager->conf_computing.key_rotation_enabled = false;
+}
+
+bool uvm_conf_computing_is_key_rotation_enabled(uvm_gpu_t *gpu)
+{
+    UVM_ASSERT(gpu);
+
+    // If the channel_manager is not set, we're in channel manager destroy
+    // path after the pointer was NULL-ed. Chances are that other key rotation
+    // infrastructure is not available either. Disallow the key rotation.
+    return gpu->channel_manager && gpu->channel_manager->conf_computing.key_rotation_enabled;
+}
+
+bool uvm_conf_computing_is_key_rotation_enabled_in_pool(uvm_channel_pool_t *pool)
+{
+    if (!uvm_conf_computing_is_key_rotation_enabled(pool->manager->gpu))
+        return false;
+
+    // TODO: Bug 4586447: key rotation must be disabled in the SEC2 engine,
+    // because currently the encryption key is shared between UVM and RM, but
+    // UVM is not able to idle SEC2 channels owned by RM.
+    if (uvm_channel_pool_is_sec2(pool))
+        return false;
+
+    // Key rotation happens as part of channel reservation, and LCIC channels
+    // are never reserved directly. Rotation of keys in LCIC channels happens
+    // as the result of key rotation in WLC channels.
+    //
+    // Return false even if there is nothing fundamental prohibiting direct key
+    // rotation on LCIC pools
+    if (uvm_channel_pool_is_lcic(pool))
+        return false;
+
+    return true;
+}
+
+static bool conf_computing_is_key_rotation_pending_use_stats(uvm_channel_pool_t *pool)
+{
+    NvU64 decrypted, encrypted;
+
+    UVM_ASSERT(!key_rotation_is_notifier_driven());
+
+    decrypted = atomic64_read(&pool->conf_computing.key_rotation.decrypted);
+
+    if (decrypted > UVM_CONF_COMPUTING_KEY_ROTATION_LOWER_THRESHOLD)
+        return true;
+
+    encrypted = atomic64_read(&pool->conf_computing.key_rotation.encrypted);
+
+    if (encrypted > UVM_CONF_COMPUTING_KEY_ROTATION_LOWER_THRESHOLD)
+        return true;
+
+    return false;
+}
+
+static bool conf_computing_is_key_rotation_pending_use_notifier(uvm_channel_pool_t *pool)
+{
+    // If key rotation is pending for the pool's engine, then the key rotation
+    // notifier in any of the engine channels can be used by UVM to detect the
+    // situation. Note that RM doesn't update all the notifiers in a single
+    // atomic operation, so it is possible that the channel read by UVM (the
+    // first one in the pool) indicates that a key rotation is pending, but
+    // another channel in the pool (temporarily) indicates the opposite, or vice
+    // versa.
+    uvm_channel_t *first_channel = pool->channels;
+
+    UVM_ASSERT(key_rotation_is_notifier_driven());
+    UVM_ASSERT(first_channel != NULL);
+
+    return first_channel->channel_info.keyRotationNotifier->status == UVM_KEY_ROTATION_STATUS_PENDING;
+}
+
+bool uvm_conf_computing_is_key_rotation_pending_in_pool(uvm_channel_pool_t *pool)
+{
+    if (!uvm_conf_computing_is_key_rotation_enabled_in_pool(pool))
+        return false;
+
+    if (key_rotation_is_notifier_driven())
+        return conf_computing_is_key_rotation_pending_use_notifier(pool);
+    else
+        return conf_computing_is_key_rotation_pending_use_stats(pool);
+}
+
+NV_STATUS uvm_conf_computing_rotate_pool_key(uvm_channel_pool_t *pool)
+{
+    NV_STATUS status;
+
+    UVM_ASSERT(uvm_conf_computing_is_key_rotation_enabled_in_pool(pool));
+    UVM_ASSERT(pool->conf_computing.key_rotation.csl_contexts != NULL);
+    UVM_ASSERT(pool->conf_computing.key_rotation.num_csl_contexts > 0);
+
+    // NV_ERR_STATE_IN_USE indicates that RM was not able to acquire the
+    // required locks at this time. This status is not interpreted as an error,
+    // but as a sign for UVM to try again later. This is the same "protocol"
+    // used in IV rotation.
+    status = nvUvmInterfaceCslRotateKey(pool->conf_computing.key_rotation.csl_contexts,
+                                        pool->conf_computing.key_rotation.num_csl_contexts);
+
+    if (status == NV_OK) {
+        pool->conf_computing.key_rotation.version++;
+
+        if (!key_rotation_is_notifier_driven()) {
+            atomic64_set(&pool->conf_computing.key_rotation.decrypted, 0);
+            atomic64_set(&pool->conf_computing.key_rotation.encrypted, 0);
+        }
+    }
+    else if (status != NV_ERR_STATE_IN_USE) {
+        UVM_DBG_PRINT("nvUvmInterfaceCslRotateKey() failed in engine %u: %s\n",
+                      pool->engine_index,
+                      nvstatusToString(status));
+    }
+
+    return status;
+}
+
+__attribute__ ((format(printf, 6, 7)))
+NV_STATUS uvm_conf_computing_util_memcopy_cpu_to_gpu(uvm_gpu_t *gpu,
+                                                     uvm_gpu_address_t dst_gpu_address,
+                                                     void *src_plain,
+                                                     size_t size,
+                                                     uvm_tracker_t *tracker,
+                                                     const char *format,
+                                                     ...)
+{
+    NV_STATUS status;
+    uvm_push_t push;
+    uvm_conf_computing_dma_buffer_t *dma_buffer;
+    uvm_gpu_address_t src_gpu_address, auth_tag_gpu_address;
+    void *dst_cipher, *auth_tag;
+    va_list args;
+
+    UVM_ASSERT(g_uvm_global.conf_computing_enabled);
+    UVM_ASSERT(size <= UVM_CONF_COMPUTING_DMA_BUFFER_SIZE);
+
+    status = uvm_conf_computing_dma_buffer_alloc(&gpu->conf_computing.dma_buffer_pool, &dma_buffer, NULL);
+    if (status != NV_OK)
+        return status;
+
+    va_start(args, format);
+    status = uvm_push_begin_acquire(gpu->channel_manager, UVM_CHANNEL_TYPE_CPU_TO_GPU, tracker, &push, format, args);
+    va_end(args);
+
+    if (status != NV_OK)
+        goto out;
+
+    dst_cipher = uvm_mem_get_cpu_addr_kernel(dma_buffer->alloc);
+    auth_tag = uvm_mem_get_cpu_addr_kernel(dma_buffer->auth_tag);
+    uvm_conf_computing_cpu_encrypt(push.channel, dst_cipher, src_plain, NULL, size, auth_tag);
+
+    src_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->alloc, gpu);
+    auth_tag_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->auth_tag, gpu);
+    gpu->parent->ce_hal->decrypt(&push, dst_gpu_address, src_gpu_address, size, auth_tag_gpu_address);
+
+    status = uvm_push_end_and_wait(&push);
+
+out:
+    uvm_conf_computing_dma_buffer_free(&gpu->conf_computing.dma_buffer_pool, dma_buffer, NULL);
+    return status;
+}
+
+__attribute__ ((format(printf, 6, 7)))
+NV_STATUS uvm_conf_computing_util_memcopy_gpu_to_cpu(uvm_gpu_t *gpu,
+                                                     void *dst_plain,
+                                                     uvm_gpu_address_t src_gpu_address,
+                                                     size_t size,
+                                                     uvm_tracker_t *tracker,
+                                                     const char *format,
+                                                     ...)
+{
+    NV_STATUS status;
+    uvm_push_t push;
+    uvm_conf_computing_dma_buffer_t *dma_buffer;
+    uvm_gpu_address_t dst_gpu_address, auth_tag_gpu_address;
+    void *src_cipher, *auth_tag;
+    va_list args;
+
+    UVM_ASSERT(g_uvm_global.conf_computing_enabled);
+    UVM_ASSERT(size <= UVM_CONF_COMPUTING_DMA_BUFFER_SIZE);
+
+    status = uvm_conf_computing_dma_buffer_alloc(&gpu->conf_computing.dma_buffer_pool, &dma_buffer, NULL);
+    if (status != NV_OK)
+        return status;
+
+    va_start(args, format);
+    status = uvm_push_begin_acquire(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_TO_CPU, tracker, &push, format, args);
+    va_end(args);
+
+    if (status != NV_OK)
+        goto out;
+
+    uvm_conf_computing_log_gpu_encryption(push.channel, size, dma_buffer->decrypt_iv);
+    dma_buffer->key_version[0] = uvm_channel_pool_key_version(push.channel->pool);
+
+    dst_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->alloc, gpu);
+    auth_tag_gpu_address = uvm_mem_gpu_address_virtual_kernel(dma_buffer->auth_tag, gpu);
+    gpu->parent->ce_hal->encrypt(&push, dst_gpu_address, src_gpu_address, size, auth_tag_gpu_address);
+
+    status = uvm_push_end_and_wait(&push);
+    if (status != NV_OK)
+        goto out;
+
+    src_cipher = uvm_mem_get_cpu_addr_kernel(dma_buffer->alloc);
+    auth_tag = uvm_mem_get_cpu_addr_kernel(dma_buffer->auth_tag);
+    status = uvm_conf_computing_cpu_decrypt(push.channel,
+                                            dst_plain,
+                                            src_cipher,
+                                            dma_buffer->decrypt_iv,
+                                            dma_buffer->key_version[0],
+                                            size,
+                                            auth_tag);
+
+ out:
+    uvm_conf_computing_dma_buffer_free(&gpu->conf_computing.dma_buffer_pool, dma_buffer, NULL);
+    return status;
+}

kernel-open/nvidia-uvm/uvm_conf_computing.h

@@ -87,9 +87,9 @@ typedef struct
     // a free buffer.
     uvm_tracker_t tracker;
 
-    // When the DMA buffer is used as the destination of a GPU encryption, SEC2
-    // writes the authentication tag here. Later when the buffer is decrypted
-    // on the CPU the authentication tag is used again (read) for CSL to verify
+    // When the DMA buffer is used as the destination of a GPU encryption, the
+    // engine (CE or SEC2) writes the authentication tag here. When the buffer
+    // is decrypted on the CPU the authentication tag is used by CSL to verify
     // the authenticity. The allocation is big enough for one authentication
     // tag per PAGE_SIZE page in the alloc buffer.
     uvm_mem_t *auth_tag;
@@ -98,7 +98,12 @@ typedef struct
     // to the authentication tag. The allocation is big enough for one IV per
     // PAGE_SIZE page in the alloc buffer. The granularity between the decrypt
     // IV and authentication tag must match.
-    UvmCslIv decrypt_iv[(UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE)];
+    UvmCslIv decrypt_iv[UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE];
+
+    // When the DMA buffer is used as the destination of a GPU encryption, the
+    // key version used during GPU encryption of each PAGE_SIZE page can be
+    // saved here, so CPU decryption uses the correct decryption key.
+    NvU32 key_version[UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE];
 
     // Bitmap of the encrypted pages in the backing allocation
     uvm_page_mask_t encrypted_page_mask;
@@ -147,7 +152,7 @@ NV_STATUS uvm_conf_computing_gpu_init(uvm_gpu_t *gpu);
 void uvm_conf_computing_gpu_deinit(uvm_gpu_t *gpu);
 
 // Logs encryption information from the GPU and returns the IV.
-void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv);
+void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, size_t size, UvmCslIv *iv);
 
 // Acquires next CPU encryption IV and returns it.
 void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *iv);
@@ -167,10 +172,14 @@ void uvm_conf_computing_cpu_encrypt(uvm_channel_t *channel,
 // CPU side decryption helper. Decrypts data from src_cipher and writes the
 // plain text in dst_plain. src_cipher and dst_plain can't overlap. IV obtained
 // from uvm_conf_computing_log_gpu_encryption() needs to be be passed to src_iv.
+//
+// The caller must indicate which key to use for decryption by passing the
+// appropiate key version number.
 NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
                                          void *dst_plain,
                                          const void *src_cipher,
                                          const UvmCslIv *src_iv,
+                                         NvU32 key_version,
                                          size_t size,
                                          const void *auth_tag_buffer);
 
@@ -214,4 +223,71 @@ NV_STATUS uvm_conf_computing_maybe_rotate_channel_ivs_retry_busy(uvm_channel_t *
 // Check if there are fewer than 'limit' messages available in either direction
 // and rotate if not.
 NV_STATUS uvm_conf_computing_rotate_channel_ivs_below_limit(uvm_channel_t *channel, NvU64 limit, bool retry_if_busy);
+
+// Rotate the engine key associated with the given channel pool.
+NV_STATUS uvm_conf_computing_rotate_pool_key(uvm_channel_pool_t *pool);
+
+// Returns true if key rotation is allowed in the channel pool.
+bool uvm_conf_computing_is_key_rotation_enabled_in_pool(uvm_channel_pool_t *pool);
+
+// Returns true if key rotation is pending in the channel pool.
+bool uvm_conf_computing_is_key_rotation_pending_in_pool(uvm_channel_pool_t *pool);
+
+// Enable/disable key rotation in the passed GPU. Note that UVM enablement is
+// dependent on RM enablement: key rotation may still be disabled upon calling
+// this function, if it is disabled in RM. On the other hand, key rotation can
+// be disabled in UVM, even if it is enabled in RM.
+//
+// Enablement/Disablement affects only kernel key rotation in keys owned by UVM.
+// It doesn't affect user key rotation (CUDA, Video...), nor it affects RM
+// kernel key rotation.
+void uvm_conf_computing_enable_key_rotation(uvm_gpu_t *gpu);
+void uvm_conf_computing_disable_key_rotation(uvm_gpu_t *gpu);
+
+// Returns true if key rotation is enabled on UVM in the given GPU. Key rotation
+// can be enabled on the GPU but disabled on some of GPU engines (LCEs or SEC2),
+// see uvm_conf_computing_is_key_rotation_enabled_in_pool.
+bool uvm_conf_computing_is_key_rotation_enabled(uvm_gpu_t *gpu);
+
+// Launch a synchronous, encrypted copy between CPU and GPU.
+//
+// The maximum copy size allowed is UVM_CONF_COMPUTING_DMA_BUFFER_SIZE.
+//
+// The source CPU buffer pointed by src_plain contains the unencrypted (plain
+// text) contents; the function internally performs a CPU-side encryption step
+// before launching the GPU-side CE decryption. The source buffer can be in
+// protected or unprotected sysmem, while the destination buffer must be in
+// protected vidmem.
+//
+// The input tracker, if not NULL, is internally acquired by the push
+// responsible for the encrypted copy.
+__attribute__ ((format(printf, 6, 7)))
+NV_STATUS uvm_conf_computing_util_memcopy_cpu_to_gpu(uvm_gpu_t *gpu,
+                                                     uvm_gpu_address_t dst_gpu_address,
+                                                     void *src_plain,
+                                                     size_t size,
+                                                     uvm_tracker_t *tracker,
+                                                     const char *format,
+                                                     ...);
+
+// Launch a synchronous, encrypted copy between CPU and GPU.
+//
+// The maximum copy size allowed is UVM_CONF_COMPUTING_DMA_BUFFER_SIZE.
+//
+// The source CPU buffer pointed by src_plain contains the unencrypted (plain
+// text) contents; the function internally performs a CPU-side encryption step
+// before launching the GPU-side CE decryption. The source buffer can be in
+// protected or unprotected sysmem, while the destination buffer must be in
+// protected vidmem.
+//
+// The input tracker, if not NULL, is internally acquired by the push
+// responsible for the encrypted copy.
+__attribute__ ((format(printf, 6, 7)))
+NV_STATUS uvm_conf_computing_util_memcopy_gpu_to_cpu(uvm_gpu_t *gpu,
+                                                     void *dst_plain,
+                                                     uvm_gpu_address_t src_gpu_address,
+                                                     size_t size,
+                                                     uvm_tracker_t *tracker,
+                                                     const char *format,
+                                                     ...);
 #endif // __UVM_CONF_COMPUTING_H__

kernel-open/nvidia-uvm/uvm_debug_optimized.c

@@ -1,53 +0,0 @@
-/*******************************************************************************
-    Copyright (c) 2015 NVIDIA Corporation
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to
-    deal in the Software without restriction, including without limitation the
-    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-    sell copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-        The above copyright notice and this permission notice shall be
-        included in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-    DEALINGS IN THE SOFTWARE.
-
-*******************************************************************************/
-
-// This file provides simple wrappers that are always built with optimizations
-// turned on to WAR issues with functions that don't build correctly otherwise.
-
-#include "uvm_linux.h"
-
-int nv_atomic_xchg(atomic_t *val, int new)
-{
-    return atomic_xchg(val, new);
-}
-
-int nv_atomic_cmpxchg(atomic_t *val, int old, int new)
-{
-    return atomic_cmpxchg(val, old, new);
-}
-
-long nv_atomic_long_cmpxchg(atomic_long_t *val, long old, long new)
-{
-    return atomic_long_cmpxchg(val, old, new);
-}
-
-unsigned long nv_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-    return copy_from_user(to, from, n);
-}
-
-unsigned long nv_copy_to_user(void __user *to, const void *from, unsigned long n)
-{
-    return copy_to_user(to, from, n);
-}
-

kernel-open/nvidia-uvm/uvm_get_rm_ptes_test.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -119,6 +119,10 @@ static NV_STATUS verify_mapping_info(uvm_va_space_t *va_space,
     if (memory_owning_gpu == NULL)
         return NV_ERR_INVALID_DEVICE;
 
+    // TODO: Bug 1903234: Once RM supports indirect peer mappings, we'll need to
+    //       update this test since the aperture will be SYS. Depending on how
+    //       RM implements things, we might not be able to compare the physical
+    //       addresses either.
     aperture = get_aperture(va_space, memory_owning_gpu, memory_mapping_gpu, memory_info, sli_supported);
 
     if (is_cacheable(ext_mapping_info, aperture))

kernel-open/nvidia-uvm/uvm_global.c

@@ -412,7 +412,7 @@ void uvm_global_set_fatal_error_impl(NV_STATUS error)
 
     UVM_ASSERT(error != NV_OK);
 
-    previous_error = nv_atomic_cmpxchg(&g_uvm_global.fatal_error, NV_OK, error);
+    previous_error = atomic_cmpxchg(&g_uvm_global.fatal_error, NV_OK, error);
 
     if (previous_error == NV_OK) {
         UVM_ERR_PRINT("Encountered a global fatal error: %s\n", nvstatusToString(error));
@@ -430,7 +430,7 @@ NV_STATUS uvm_global_reset_fatal_error(void)
         return NV_ERR_INVALID_STATE;
     }
 
-    return nv_atomic_xchg(&g_uvm_global.fatal_error, NV_OK);
+    return atomic_xchg(&g_uvm_global.fatal_error, NV_OK);
 }
 
 void uvm_global_gpu_retain(const uvm_processor_mask_t *mask)

kernel-open/nvidia-uvm/uvm_gpu.c

@@ -81,8 +81,6 @@ static uvm_gpu_link_type_t get_gpu_link_type(UVM_LINK_TYPE link_type)
             return UVM_GPU_LINK_NVLINK_3;
         case UVM_LINK_TYPE_NVLINK_4:
             return UVM_GPU_LINK_NVLINK_4;
-        case UVM_LINK_TYPE_NVLINK_5:
-            return UVM_GPU_LINK_NVLINK_5;
         case UVM_LINK_TYPE_C2C:
             return UVM_GPU_LINK_C2C;
         default:
@@ -140,6 +138,7 @@ static NV_STATUS get_gpu_caps(uvm_gpu_t *gpu)
 
     if (gpu_caps.numaEnabled) {
         UVM_ASSERT(uvm_parent_gpu_is_coherent(gpu->parent));
+
         gpu->mem_info.numa.enabled = true;
         gpu->mem_info.numa.node_id = gpu_caps.numaNodeId;
     }
@@ -220,9 +219,8 @@ static NV_STATUS alloc_and_init_address_space(uvm_gpu_t *gpu)
     if (status != NV_OK)
         return status;
 
-    UVM_ASSERT(gpu_address_space_info.bigPageSize <= NV_U32_MAX);
-
     gpu->big_page.internal_size = gpu_address_space_info.bigPageSize;
+
     gpu->time.time0_register = gpu_address_space_info.time0Offset;
     gpu->time.time1_register = gpu_address_space_info.time1Offset;
 
@@ -461,8 +459,7 @@ static const char *uvm_gpu_virt_type_string(UVM_VIRT_MODE virtMode)
 
 static const char *uvm_gpu_link_type_string(uvm_gpu_link_type_t link_type)
 {
-
-    BUILD_BUG_ON(UVM_GPU_LINK_MAX != 8);
+    BUILD_BUG_ON(UVM_GPU_LINK_MAX != 7);
 
     switch (link_type) {
         UVM_ENUM_STRING_CASE(UVM_GPU_LINK_INVALID);
@@ -471,7 +468,6 @@ static const char *uvm_gpu_link_type_string(uvm_gpu_link_type_t link_type)
         UVM_ENUM_STRING_CASE(UVM_GPU_LINK_NVLINK_2);
         UVM_ENUM_STRING_CASE(UVM_GPU_LINK_NVLINK_3);
         UVM_ENUM_STRING_CASE(UVM_GPU_LINK_NVLINK_4);
-        UVM_ENUM_STRING_CASE(UVM_GPU_LINK_NVLINK_5);
         UVM_ENUM_STRING_CASE(UVM_GPU_LINK_C2C);
         UVM_ENUM_STRING_DEFAULT();
     }
@@ -1087,6 +1083,9 @@ static NV_STATUS configure_address_space(uvm_gpu_t *gpu)
                    gpu->parent->rm_va_size,
                    va_per_entry);
 
+    UVM_ASSERT(uvm_mmu_page_size_supported(&gpu->address_space_tree, gpu->big_page.internal_size));
+    UVM_ASSERT(uvm_mmu_page_size_supported(&gpu->address_space_tree, gpu->mem_info.max_vidmem_page_size));
+
     tree_alloc = uvm_page_tree_pdb(&gpu->address_space_tree);
     status = uvm_rm_locked_call(nvUvmInterfaceSetPageDirectory(gpu->rm_address_space,
                                                                tree_alloc->addr.address,
@@ -1282,7 +1281,8 @@ static NV_STATUS init_gpu(uvm_gpu_t *gpu, const UvmGpuInfo *gpu_info)
 
     status = get_gpu_caps(gpu);
     if (status != NV_OK) {
-        UVM_ERR_PRINT("Failed to get GPU caps: %s, GPU %s\n", nvstatusToString(status), uvm_gpu_name(gpu));
+        if (status != NV_ERR_NVSWITCH_FABRIC_NOT_READY)
+            UVM_ERR_PRINT("Failed to get GPU caps: %s, GPU %s\n", nvstatusToString(status), uvm_gpu_name(gpu));
         return status;
     }
 
@@ -1682,9 +1682,12 @@ static void remove_gpu(uvm_gpu_t *gpu)
     // TODO: Bug 2008200: Add and remove the GPU in a more reasonable spot.
     uvm_conf_computing_gpu_deinit(gpu);
 
-    // If the parent is not being freed, the following gpu_table_lock is only
-    // needed to protect concurrent uvm_parent_gpu_find_first_valid_gpu() in BH
-    // from the __clear_bit here.
+    // TODO: Bug 2844714: If the parent is not being freed, the following
+    // gpu_table_lock is only needed to protect concurrent
+    // find_first_valid_gpu() in BH from the __clear_bit here. After
+    // find_first_valid_gpu() is removed, gpu_table_lock should only be acquired
+    // and released in the free_parent case.
+    //
     // In the free_parent case, gpu_table_lock protects the top half from the
     // uvm_global_remove_parent_gpu()
     uvm_spin_lock_irqsave(&g_uvm_global.gpu_table_lock);
@@ -2255,13 +2258,28 @@ static void set_optimal_p2p_write_ces(const UvmGpuP2PCapsParams *p2p_caps_params
     bool sorted;
     NvU32 ce0, ce1;
 
-    if (peer_caps->link_type < UVM_GPU_LINK_NVLINK_1)
+    UVM_ASSERT(peer_caps->ref_count);
+    UVM_ASSERT(gpu0->parent->peer_copy_mode == gpu1->parent->peer_copy_mode);
+
+    if (gpu0->parent->peer_copy_mode == UVM_GPU_PEER_COPY_MODE_UNSUPPORTED)
         return;
 
     sorted = uvm_id_value(gpu0->id) < uvm_id_value(gpu1->id);
     ce0 = p2p_caps_params->optimalNvlinkWriteCEs[sorted ? 0 : 1];
     ce1 = p2p_caps_params->optimalNvlinkWriteCEs[sorted ? 1 : 0];
 
+    // Indirect peers communicate through the CPU, so the optimal CE
+    // should match the one selected for writing to system memory
+    if (peer_caps->is_indirect_peer) {
+        uvm_channel_pool_t *pool;
+
+        pool = gpu0->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_CPU];
+        UVM_ASSERT(ce0 == pool->engine_index);
+
+        pool = gpu1->channel_manager->pool_to_use.default_for_type[UVM_CHANNEL_TYPE_GPU_TO_CPU];
+        UVM_ASSERT(ce1 == pool->engine_index);
+    }
+
     uvm_channel_manager_set_p2p_ce(gpu0->channel_manager, gpu1, ce0);
     uvm_channel_manager_set_p2p_ce(gpu1->channel_manager, gpu0, ce1);
 }
@@ -2269,7 +2287,7 @@ static void set_optimal_p2p_write_ces(const UvmGpuP2PCapsParams *p2p_caps_params
 static int nv_procfs_read_gpu_peer_caps(struct seq_file *s, void *v)
 {
     if (!uvm_down_read_trylock(&g_uvm_global.pm.lock))
-            return -EAGAIN;
+        return -EAGAIN;
 
     gpu_peer_caps_print((uvm_gpu_t **)s->private, s);
 
@@ -2351,51 +2369,74 @@ static NV_STATUS init_peer_access(uvm_gpu_t *gpu0,
 
     // check for peer-to-peer compatibility (PCI-E or NvLink).
     peer_caps->link_type = get_gpu_link_type(p2p_caps_params->p2pLink);
-    if (peer_caps->link_type == UVM_GPU_LINK_INVALID || peer_caps->link_type == UVM_GPU_LINK_C2C)
+    if (peer_caps->link_type == UVM_GPU_LINK_INVALID
+        || peer_caps->link_type == UVM_GPU_LINK_C2C
+        )
         return NV_ERR_NOT_SUPPORTED;
 
     peer_caps->total_link_line_rate_mbyte_per_s = p2p_caps_params->totalLinkLineRateMBps;
 
     // Initialize peer ids and establish peer mappings
-    // Peer id from min(gpu_id0, gpu_id1) -> max(gpu_id0, gpu_id1)
-    peer_caps->peer_ids[0] = p2p_caps_params->peerIds[0];
+    peer_caps->is_indirect_peer = (p2p_caps_params->indirectAccess == NV_TRUE);
 
-    // Peer id from max(gpu_id0, gpu_id1) -> min(gpu_id0, gpu_id1)
-    peer_caps->peer_ids[1] = p2p_caps_params->peerIds[1];
+    if (peer_caps->is_indirect_peer) {
+        UVM_ASSERT(gpu0->mem_info.numa.enabled);
+        UVM_ASSERT(gpu1->mem_info.numa.enabled);
 
-    // Establish peer mappings from each GPU to the other.
-    status = uvm_mmu_create_peer_identity_mappings(gpu0, gpu1);
-    if (status != NV_OK)
-        return status;
+        status = uvm_pmm_gpu_indirect_peer_init(&gpu0->pmm, gpu1);
+        if (status != NV_OK)
+            return status;
 
-    status = uvm_mmu_create_peer_identity_mappings(gpu1, gpu0);
-    if (status != NV_OK)
-        return status;
+        status = uvm_pmm_gpu_indirect_peer_init(&gpu1->pmm, gpu0);
+        if (status != NV_OK)
+            return status;
 
-    set_optimal_p2p_write_ces(p2p_caps_params, peer_caps, gpu0, gpu1);
+        set_optimal_p2p_write_ces(p2p_caps_params, peer_caps, gpu0, gpu1);
+        UVM_ASSERT(peer_caps->total_link_line_rate_mbyte_per_s == 0);
+    }
+    else {
+        // Peer id from min(gpu_id0, gpu_id1) -> max(gpu_id0, gpu_id1)
+        peer_caps->peer_ids[0] = p2p_caps_params->peerIds[0];
 
-    UVM_ASSERT(uvm_gpu_get(gpu0->id) == gpu0);
-    UVM_ASSERT(uvm_gpu_get(gpu1->id) == gpu1);
+        // Peer id from max(gpu_id0, gpu_id1) -> min(gpu_id0, gpu_id1)
+        peer_caps->peer_ids[1] = p2p_caps_params->peerIds[1];
 
-    // In the case of NVLINK peers, this initialization will happen during
-    // add_gpu. As soon as the peer info table is assigned below, the access
-    // counter bottom half could start operating on the GPU being newly
-    // added and inspecting the peer caps, so all of the appropriate
-    // initialization must happen before this point.
-    uvm_spin_lock(&gpu0->peer_info.peer_gpus_lock);
+        // Establish peer mappings from each GPU to the other. Indirect peers
+        // do not require identity mappings since they use sysmem aperture to
+        // communicate.
+        status = uvm_mmu_create_peer_identity_mappings(gpu0, gpu1);
+        if (status != NV_OK)
+            return status;
 
-    uvm_processor_mask_set(&gpu0->peer_info.peer_gpu_mask, gpu1->id);
-    UVM_ASSERT(gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] == NULL);
-    gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] = gpu1;
+        status = uvm_mmu_create_peer_identity_mappings(gpu1, gpu0);
+        if (status != NV_OK)
+            return status;
 
-    uvm_spin_unlock(&gpu0->peer_info.peer_gpus_lock);
-    uvm_spin_lock(&gpu1->peer_info.peer_gpus_lock);
+        set_optimal_p2p_write_ces(p2p_caps_params, peer_caps, gpu0, gpu1);
 
-    uvm_processor_mask_set(&gpu1->peer_info.peer_gpu_mask, gpu0->id);
-    UVM_ASSERT(gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] == NULL);
-    gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] = gpu0;
+        UVM_ASSERT(uvm_gpu_get(gpu0->id) == gpu0);
+        UVM_ASSERT(uvm_gpu_get(gpu1->id) == gpu1);
 
-    uvm_spin_unlock(&gpu1->peer_info.peer_gpus_lock);
+        // In the case of NVLINK peers, this initialization will happen during
+        // add_gpu. As soon as the peer info table is assigned below, the access
+        // counter bottom half could start operating on the GPU being newly
+        // added and inspecting the peer caps, so all of the appropriate
+        // initialization must happen before this point.
+        uvm_spin_lock(&gpu0->peer_info.peer_gpus_lock);
+
+        uvm_processor_mask_set(&gpu0->peer_info.peer_gpu_mask, gpu1->id);
+        UVM_ASSERT(gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] == NULL);
+        gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] = gpu1;
+
+        uvm_spin_unlock(&gpu0->peer_info.peer_gpus_lock);
+        uvm_spin_lock(&gpu1->peer_info.peer_gpus_lock);
+
+        uvm_processor_mask_set(&gpu1->peer_info.peer_gpu_mask, gpu0->id);
+        UVM_ASSERT(gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] == NULL);
+        gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] = gpu0;
+
+        uvm_spin_unlock(&gpu1->peer_info.peer_gpus_lock);
+    }
 
     return init_procfs_peer_files(gpu0, gpu1);
 }
@@ -2463,6 +2504,7 @@ static NV_STATUS enable_pcie_peer_access(uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
         goto cleanup;
 
     // Sanity checks
+    UVM_ASSERT(p2p_caps_params.indirectAccess == NV_FALSE);
     UVM_ASSERT(p2p_caps_params.p2pLink == UVM_LINK_TYPE_PCIE);
 
     status = init_peer_access(gpu0, gpu1, &p2p_caps_params, peer_caps);
@@ -2492,26 +2534,29 @@ static NV_STATUS enable_nvlink_peer_access(uvm_gpu_t *gpu0,
     UVM_ASSERT(peer_caps->ref_count == 0);
     peer_caps->ref_count = 1;
 
-    // Create P2P object for direct NVLink peers
-    status = create_p2p_object(gpu0, gpu1, &p2p_handle);
-    if (status != NV_OK) {
-        UVM_ERR_PRINT("failed to create a P2P object with error: %s, for GPU1:%s and GPU2:%s \n",
-                       nvstatusToString(status),
-                       uvm_gpu_name(gpu0),
-                       uvm_gpu_name(gpu1));
-        return status;
+    if (!p2p_caps_params->indirectAccess) {
+        // Create P2P object for direct NVLink peers
+        status = create_p2p_object(gpu0, gpu1, &p2p_handle);
+        if (status != NV_OK) {
+            UVM_ERR_PRINT("failed to create a P2P object with error: %s, for GPU1:%s and GPU2:%s \n",
+                           nvstatusToString(status),
+                           uvm_gpu_name(gpu0),
+                           uvm_gpu_name(gpu1));
+            return status;
+        }
+
+        UVM_ASSERT(p2p_handle != 0);
+
+        // Store the handle in the global table.
+        peer_caps->p2p_handle = p2p_handle;
+
+        // Update p2p caps after p2p object creation as it generates the peer
+        // ids
+        status = get_p2p_caps(gpu0, gpu1, p2p_caps_params);
+        if (status != NV_OK)
+            goto cleanup;
     }
 
-    UVM_ASSERT(p2p_handle != 0);
-
-    // Store the handle in the global table.
-    peer_caps->p2p_handle = p2p_handle;
-
-    // Update p2p caps after p2p object creation as it generates the peer ids.
-    status = get_p2p_caps(gpu0, gpu1, p2p_caps_params);
-    if (status != NV_OK)
-        goto cleanup;
-
     status = init_peer_access(gpu0, gpu1, p2p_caps_params, peer_caps);
     if (status != NV_OK)
         goto cleanup;
@@ -2546,6 +2591,11 @@ static NV_STATUS discover_nvlink_peers(uvm_gpu_t *gpu)
         if (p2p_caps_params.p2pLink == UVM_LINK_TYPE_NONE || p2p_caps_params.p2pLink == UVM_LINK_TYPE_PCIE)
             continue;
 
+        // Indirect peers are only supported when onlined as NUMA nodes, because
+        // we want to use vm_insert_page and dma_map_page.
+        if (p2p_caps_params.indirectAccess && (!gpu->mem_info.numa.enabled || !other_gpu->mem_info.numa.enabled))
+            continue;
+
         status = enable_nvlink_peer_access(gpu, other_gpu, &p2p_caps_params);
         if (status != NV_OK)
             goto cleanup;
@@ -2634,25 +2684,32 @@ static void disable_peer_access(uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
         deinit_procfs_peer_cap_files(peer_caps);
 
     p2p_handle = peer_caps->p2p_handle;
-    UVM_ASSERT(p2p_handle);
 
-    uvm_mmu_destroy_peer_identity_mappings(gpu0, gpu1);
-    uvm_mmu_destroy_peer_identity_mappings(gpu1, gpu0);
+    if (peer_caps->is_indirect_peer) {
+        uvm_pmm_gpu_indirect_peer_destroy(&gpu0->pmm, gpu1);
+        uvm_pmm_gpu_indirect_peer_destroy(&gpu1->pmm, gpu0);
+    }
+    else {
+        UVM_ASSERT(p2p_handle);
 
-    uvm_rm_locked_call_void(nvUvmInterfaceP2pObjectDestroy(uvm_global_session_handle(), p2p_handle));
+        uvm_mmu_destroy_peer_identity_mappings(gpu0, gpu1);
+        uvm_mmu_destroy_peer_identity_mappings(gpu1, gpu0);
 
-    UVM_ASSERT(uvm_gpu_get(gpu0->id) == gpu0);
-    UVM_ASSERT(uvm_gpu_get(gpu1->id) == gpu1);
+        uvm_rm_locked_call_void(nvUvmInterfaceP2pObjectDestroy(uvm_global_session_handle(), p2p_handle));
 
-    uvm_spin_lock(&gpu0->peer_info.peer_gpus_lock);
-    uvm_processor_mask_clear(&gpu0->peer_info.peer_gpu_mask, gpu1->id);
-    gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] = NULL;
-    uvm_spin_unlock(&gpu0->peer_info.peer_gpus_lock);
+        UVM_ASSERT(uvm_gpu_get(gpu0->id) == gpu0);
+        UVM_ASSERT(uvm_gpu_get(gpu1->id) == gpu1);
 
-    uvm_spin_lock(&gpu1->peer_info.peer_gpus_lock);
-    uvm_processor_mask_clear(&gpu1->peer_info.peer_gpu_mask, gpu0->id);
-    gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] = NULL;
-    uvm_spin_unlock(&gpu1->peer_info.peer_gpus_lock);
+        uvm_spin_lock(&gpu0->peer_info.peer_gpus_lock);
+        uvm_processor_mask_clear(&gpu0->peer_info.peer_gpu_mask, gpu1->id);
+        gpu0->peer_info.peer_gpus[uvm_id_gpu_index(gpu1->id)] = NULL;
+        uvm_spin_unlock(&gpu0->peer_info.peer_gpus_lock);
+
+        uvm_spin_lock(&gpu1->peer_info.peer_gpus_lock);
+        uvm_processor_mask_clear(&gpu1->peer_info.peer_gpu_mask, gpu0->id);
+        gpu1->peer_info.peer_gpus[uvm_id_gpu_index(gpu0->id)] = NULL;
+        uvm_spin_unlock(&gpu1->peer_info.peer_gpus_lock);
+    }
 
     // Flush the access counter buffer to avoid getting stale notifications for
     // accesses to GPUs to which peer access is being disabled. This is also
@@ -2692,6 +2749,10 @@ static uvm_aperture_t uvm_gpu_peer_caps_aperture(uvm_gpu_peer_t *peer_caps, uvm_
 {
     size_t peer_index;
 
+    // Indirect peers are accessed as sysmem addresses
+    if (peer_caps->is_indirect_peer)
+        return UVM_APERTURE_SYS;
+
     // MIG instances in the same physical GPU have vidmem addresses
     if (local_gpu->parent == remote_gpu->parent)
         return UVM_APERTURE_VID;
@@ -2742,7 +2803,6 @@ uvm_processor_id_t uvm_gpu_get_processor_id_by_address(uvm_gpu_t *gpu, uvm_gpu_p
     for_each_gpu_id_in_mask(id, &gpu->peer_info.peer_gpu_mask) {
         uvm_gpu_t *other_gpu = gpu->peer_info.peer_gpus[uvm_id_gpu_index(id)];
         UVM_ASSERT(other_gpu);
-        UVM_ASSERT(!uvm_gpus_are_smc_peers(gpu, other_gpu));
 
         if (uvm_gpus_are_nvswitch_connected(gpu, other_gpu)) {
             // NVSWITCH connected systems use an extended physical address to
@@ -2779,7 +2839,7 @@ static NvU64 instance_ptr_to_key(uvm_gpu_phys_address_t instance_ptr)
 
     // Instance pointers must be 4k aligned and they must have either VID or SYS
     // apertures. Compress them as much as we can both to guarantee that the key
-    // fits within 64 bits, and to make the key space as small as possible.
+    // fits within 64 bits, and to make the table as shallow as possible.
     UVM_ASSERT(IS_ALIGNED(instance_ptr.address, UVM_PAGE_SIZE_4K));
     UVM_ASSERT(instance_ptr.aperture == UVM_APERTURE_VID || instance_ptr.aperture == UVM_APERTURE_SYS);
 
@@ -2796,7 +2856,7 @@ static NV_STATUS parent_gpu_add_user_channel_subctx_info(uvm_parent_gpu_t *paren
     uvm_rb_tree_node_t *channel_tree_node;
     uvm_user_channel_subctx_info_t *channel_subctx_info;
     uvm_user_channel_subctx_info_t *new_channel_subctx_info = NULL;
-    uvm_gpu_va_space_t *gpu_va_space = user_channel->gpu_va_space;
+    uvm_va_space_t *va_space = user_channel->gpu_va_space->va_space;
 
     if (!user_channel->in_subctx)
         return NV_OK;
@@ -2840,21 +2900,21 @@ static NV_STATUS parent_gpu_add_user_channel_subctx_info(uvm_parent_gpu_t *paren
 
     user_channel->subctx_info = channel_subctx_info;
 
-    // Register the GPU VA space of the channel subcontext info descriptor, or
+    // Register the VA space of the channel subcontext info descriptor, or
     // check that the existing one matches the channel's
     if (channel_subctx_info->subctxs[user_channel->subctx_id].refcount++ > 0) {
-        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space == gpu_va_space,
-                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: expected GPU VA space 0x%llx but got 0x%llx instead\n",
+        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].va_space == va_space,
+                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: expected VA space 0x%llx but got 0x%llx instead\n",
                        user_channel->hw_runlist_id,
                        user_channel->hw_channel_id,
                        instance_ptr.address,
                        uvm_aperture_string(instance_ptr.aperture),
                        user_channel->subctx_id,
                        user_channel->tsg.id,
-                       (NvU64)gpu_va_space,
-                       (NvU64)channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space);
-        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space != NULL,
-                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: GPU VA space is NULL\n",
+                       (NvU64)va_space,
+                       (NvU64)channel_subctx_info->subctxs[user_channel->subctx_id].va_space);
+        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].va_space != NULL,
+                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: VA space is NULL\n",
                        user_channel->hw_runlist_id,
                        user_channel->hw_channel_id,
                        instance_ptr.address,
@@ -2871,17 +2931,17 @@ static NV_STATUS parent_gpu_add_user_channel_subctx_info(uvm_parent_gpu_t *paren
                        user_channel->tsg.id);
     }
     else {
-        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space == NULL,
-                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: expected GPU VA space NULL but got 0x%llx instead\n",
+        UVM_ASSERT_MSG(channel_subctx_info->subctxs[user_channel->subctx_id].va_space == NULL,
+                       "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: expected VA space NULL but got 0x%llx instead\n",
                        user_channel->hw_runlist_id,
                        user_channel->hw_channel_id,
                        instance_ptr.address,
                        uvm_aperture_string(instance_ptr.aperture),
                        user_channel->subctx_id,
                        user_channel->tsg.id,
-                       (NvU64)channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space);
+                       (NvU64)channel_subctx_info->subctxs[user_channel->subctx_id].va_space);
 
-        channel_subctx_info->subctxs[user_channel->subctx_id].gpu_va_space = gpu_va_space;
+        channel_subctx_info->subctxs[user_channel->subctx_id].va_space = va_space;
     }
 
     ++channel_subctx_info->total_refcount;
@@ -2905,7 +2965,7 @@ static void parent_gpu_remove_user_channel_subctx_info_locked(uvm_parent_gpu_t *
                                                               uvm_user_channel_t *user_channel)
 {
     uvm_gpu_phys_address_t instance_ptr = user_channel->instance_ptr.addr;
-    uvm_gpu_va_space_t *gpu_va_space = user_channel->gpu_va_space;
+    uvm_va_space_t *va_space = user_channel->gpu_va_space->va_space;
 
     uvm_assert_spinlock_locked(&parent_gpu->instance_ptr_table_lock);
 
@@ -2934,17 +2994,16 @@ static void parent_gpu_remove_user_channel_subctx_info_locked(uvm_parent_gpu_t *
                    user_channel->subctx_id,
                    user_channel->tsg.id);
 
-    UVM_ASSERT_MSG(user_channel->subctx_info->subctxs[user_channel->subctx_id].gpu_va_space == gpu_va_space,
-                   "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: "
-                   "expected GPU VA space 0x%llx but got 0x%llx instead\n",
+    UVM_ASSERT_MSG(user_channel->subctx_info->subctxs[user_channel->subctx_id].va_space == va_space,
+                   "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: expected VA space 0x%llx but got 0x%llx instead\n",
                    user_channel->hw_runlist_id,
                    user_channel->hw_channel_id,
                    instance_ptr.address,
                    uvm_aperture_string(instance_ptr.aperture),
                    user_channel->subctx_id,
                    user_channel->tsg.id,
-                   (NvU64)gpu_va_space,
-                   (NvU64)user_channel->subctx_info->subctxs[user_channel->subctx_id].gpu_va_space);
+                   (NvU64)va_space,
+                   (NvU64)user_channel->subctx_info->subctxs[user_channel->subctx_id].va_space);
 
     UVM_ASSERT_MSG(user_channel->subctx_info->total_refcount > 0,
                    "CH %u:%u instance_ptr {0x%llx:%s} SubCTX %u in TSG %u: TSG refcount is 0\n",
@@ -2957,7 +3016,7 @@ static void parent_gpu_remove_user_channel_subctx_info_locked(uvm_parent_gpu_t *
 
     // Decrement VA space refcount. If it gets to zero, unregister the pointer
     if (--user_channel->subctx_info->subctxs[user_channel->subctx_id].refcount == 0)
-        user_channel->subctx_info->subctxs[user_channel->subctx_id].gpu_va_space = NULL;
+        user_channel->subctx_info->subctxs[user_channel->subctx_id].va_space = NULL;
 
     if (--user_channel->subctx_info->total_refcount == 0) {
         uvm_rb_tree_remove(&parent_gpu->tsg_table, &user_channel->subctx_info->node);
@@ -3040,7 +3099,7 @@ static uvm_user_channel_t *instance_ptr_to_user_channel(uvm_parent_gpu_t *parent
     return get_user_channel(instance_node);
 }
 
-static uvm_gpu_va_space_t *user_channel_and_subctx_to_gpu_va_space(uvm_user_channel_t *user_channel, NvU32 subctx_id)
+static uvm_va_space_t *user_channel_and_subctx_to_va_space(uvm_user_channel_t *user_channel, NvU32 subctx_id)
 {
     uvm_user_channel_subctx_info_t *channel_subctx_info;
 
@@ -3068,31 +3127,28 @@ static uvm_gpu_va_space_t *user_channel_and_subctx_to_gpu_va_space(uvm_user_chan
     // uncleanly and work from that subcontext continues running with work from
     // other subcontexts.
     if (channel_subctx_info->subctxs[subctx_id].refcount == 0) {
-        UVM_ASSERT(channel_subctx_info->subctxs[subctx_id].gpu_va_space == NULL);
+        UVM_ASSERT(channel_subctx_info->subctxs[subctx_id].va_space == NULL);
     }
     else {
-        UVM_ASSERT_MSG(channel_subctx_info->subctxs[subctx_id].gpu_va_space,
-                       "instance_ptr {0x%llx:%s} in TSG %u: no GPU VA space for SubCTX %u\n",
+        UVM_ASSERT_MSG(channel_subctx_info->subctxs[subctx_id].va_space,
+                       "instance_ptr {0x%llx:%s} in TSG %u: no VA space for SubCTX %u\n",
                        user_channel->instance_ptr.addr.address,
                        uvm_aperture_string(user_channel->instance_ptr.addr.aperture),
                        user_channel->tsg.id,
                        subctx_id);
     }
 
-    return channel_subctx_info->subctxs[subctx_id].gpu_va_space;
+    return channel_subctx_info->subctxs[subctx_id].va_space;
 }
 
 NV_STATUS uvm_parent_gpu_fault_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
-                                                 const uvm_fault_buffer_entry_t *fault,
-                                                 uvm_va_space_t **out_va_space,
-                                                 uvm_gpu_t **out_gpu)
+                                                 uvm_fault_buffer_entry_t *fault,
+                                                 uvm_va_space_t **out_va_space)
 {
     uvm_user_channel_t *user_channel;
-    uvm_gpu_va_space_t *gpu_va_space;
     NV_STATUS status = NV_OK;
 
     *out_va_space = NULL;
-    *out_gpu = NULL;
 
     uvm_spin_lock(&parent_gpu->instance_ptr_table_lock);
 
@@ -3113,10 +3169,8 @@ NV_STATUS uvm_parent_gpu_fault_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
         // We can safely access user_channel->gpu_va_space under the
         // instance_ptr_table_lock since gpu_va_space is set to NULL after this
         // function is called in uvm_user_channel_detach
-        gpu_va_space = user_channel->gpu_va_space;
-        UVM_ASSERT(uvm_gpu_va_space_state(gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE);
-        *out_va_space = gpu_va_space->va_space;
-        *out_gpu = gpu_va_space->gpu;
+        UVM_ASSERT(uvm_gpu_va_space_state(user_channel->gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE);
+        *out_va_space = user_channel->gpu_va_space->va_space;
     }
     else {
         NvU32 ve_id = fault->fault_source.ve_id;
@@ -3126,17 +3180,12 @@ NV_STATUS uvm_parent_gpu_fault_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
 
         ve_id -= user_channel->smc_engine_ve_id_offset;
 
-        gpu_va_space = user_channel_and_subctx_to_gpu_va_space(user_channel, ve_id);
+        *out_va_space = user_channel_and_subctx_to_va_space(user_channel, ve_id);
 
         // Instance pointer is valid but the fault targets a non-existent
         // subcontext.
-        if (gpu_va_space) {
-            *out_va_space = gpu_va_space->va_space;
-            *out_gpu = gpu_va_space->gpu;
-        }
-        else {
+        if (!*out_va_space)
             status = NV_ERR_PAGE_TABLE_NOT_AVAIL;
-        }
     }
 
 exit_unlock:
@@ -3146,16 +3195,13 @@ exit_unlock:
 }
 
 NV_STATUS uvm_parent_gpu_access_counter_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
-                                                          const uvm_access_counter_buffer_entry_t *entry,
-                                                          uvm_va_space_t **out_va_space,
-                                                          uvm_gpu_t **out_gpu)
+                                                          uvm_access_counter_buffer_entry_t *entry,
+                                                          uvm_va_space_t **out_va_space)
 {
     uvm_user_channel_t *user_channel;
-    uvm_gpu_va_space_t *gpu_va_space;
     NV_STATUS status = NV_OK;
 
     *out_va_space = NULL;
-    *out_gpu = NULL;
     UVM_ASSERT(entry->address.is_virtual);
 
     uvm_spin_lock(&parent_gpu->instance_ptr_table_lock);
@@ -3171,20 +3217,13 @@ NV_STATUS uvm_parent_gpu_access_counter_entry_to_va_space(uvm_parent_gpu_t *pare
                        "Access counter packet contains SubCTX %u for channel not in subctx\n",
                        entry->virtual_info.ve_id);
 
-        gpu_va_space = user_channel->gpu_va_space;
-        UVM_ASSERT(uvm_gpu_va_space_state(gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE);
-        *out_va_space = gpu_va_space->va_space;
-        *out_gpu = gpu_va_space->gpu;
+        UVM_ASSERT(uvm_gpu_va_space_state(user_channel->gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE);
+        *out_va_space = user_channel->gpu_va_space->va_space;
     }
     else {
-        gpu_va_space = user_channel_and_subctx_to_gpu_va_space(user_channel, entry->virtual_info.ve_id);
-        if (gpu_va_space) {
-            *out_va_space = gpu_va_space->va_space;
-            *out_gpu = gpu_va_space->gpu;
-        }
-        else {
+        *out_va_space = user_channel_and_subctx_to_va_space(user_channel, entry->virtual_info.ve_id);
+        if (!*out_va_space)
             status = NV_ERR_PAGE_TABLE_NOT_AVAIL;
-        }
     }
 
 exit_unlock:
@@ -3262,10 +3301,7 @@ void uvm_parent_gpu_dma_free_page(uvm_parent_gpu_t *parent_gpu, void *va, NvU64
     atomic64_sub(PAGE_SIZE, &parent_gpu->mapped_cpu_pages_size);
 }
 
-NV_STATUS uvm_parent_gpu_map_cpu_pages(uvm_parent_gpu_t *parent_gpu,
-                                       struct page *page,
-                                       size_t size,
-                                       NvU64 *dma_address_out)
+NV_STATUS uvm_parent_gpu_map_cpu_pages(uvm_parent_gpu_t *parent_gpu, struct page *page, size_t size, NvU64 *dma_address_out)
 {
     NvU64 dma_addr;
 

kernel-open/nvidia-uvm/uvm_gpu.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2024 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -279,10 +279,6 @@ struct uvm_fault_service_batch_context_struct
     // pick one to be the target of the cancel sequence.
     uvm_va_space_t *fatal_va_space;
 
-    // TODO: Bug 3900733: refactor service_fault_batch_for_cancel() to handle
-    // iterating over multiple GPU VA spaces and remove fatal_gpu.
-    uvm_gpu_t *fatal_gpu;
-
     bool has_throttled_faults;
 
     NvU32 num_invalid_prefetch_faults;
@@ -597,7 +593,6 @@ typedef enum
     UVM_GPU_LINK_NVLINK_2,
     UVM_GPU_LINK_NVLINK_3,
     UVM_GPU_LINK_NVLINK_4,
-    UVM_GPU_LINK_NVLINK_5,
     UVM_GPU_LINK_C2C,
     UVM_GPU_LINK_MAX
 } uvm_gpu_link_type_t;
@@ -967,6 +962,8 @@ struct uvm_parent_gpu_struct
     // Whether CE supports physical addressing mode for writes to vidmem
     bool ce_phys_vidmem_write_supported;
 
+    // Addressing mode(s) supported for CE transfers between this GPU and its
+    // peers: none, physical only, physical and virtual, etc.
     uvm_gpu_peer_copy_mode_t peer_copy_mode;
 
     // Virtualization mode of the GPU.
@@ -1270,6 +1267,11 @@ struct uvm_gpu_peer_struct
     // peer_id[1] from max(gpu_id_1, gpu_id_2) -> min(gpu_id_1, gpu_id_2)
     NvU8 peer_ids[2];
 
+    // Indirect peers are GPUs which can coherently access each others' memory
+    // over NVLINK, but are routed through the CPU using the SYS aperture rather
+    // than a PEER aperture
+    NvU8 is_indirect_peer : 1;
+
     // The link type between the peer GPUs, currently either PCIe or NVLINK.
     // This field is used to determine the when this peer struct has been
     // initialized (link_type != UVM_GPU_LINK_INVALID). NVLink peers are
@@ -1278,8 +1280,8 @@ struct uvm_gpu_peer_struct
     uvm_gpu_link_type_t link_type;
 
     // Maximum unidirectional bandwidth between the peers in megabytes per
-    // second, not taking into account the protocols' overhead.
-    // See UvmGpuP2PCapsParams.
+    // second, not taking into account the protocols' overhead. The reported
+    // bandwidth for indirect peers is zero. See UvmGpuP2PCapsParams.
     NvU32 total_link_line_rate_mbyte_per_s;
 
     // For PCIe, the number of times that this has been retained by a VA space.
@@ -1423,9 +1425,19 @@ static bool uvm_gpus_are_nvswitch_connected(const uvm_gpu_t *gpu0, const uvm_gpu
     return false;
 }
 
-static bool uvm_gpus_are_smc_peers(uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
+static bool uvm_gpus_are_indirect_peers(uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
 {
-    return gpu0->parent == gpu1->parent;
+    uvm_gpu_peer_t *peer_caps = uvm_gpu_peer_caps(gpu0, gpu1);
+
+    if (peer_caps->link_type != UVM_GPU_LINK_INVALID && peer_caps->is_indirect_peer) {
+        UVM_ASSERT(gpu0->mem_info.numa.enabled);
+        UVM_ASSERT(gpu1->mem_info.numa.enabled);
+        UVM_ASSERT(peer_caps->link_type != UVM_GPU_LINK_PCIE);
+        UVM_ASSERT(!uvm_gpus_are_nvswitch_connected(gpu0, gpu1));
+        return true;
+    }
+
+    return false;
 }
 
 // Retrieve the virtual address corresponding to the given vidmem physical
@@ -1610,25 +1622,16 @@ void uvm_parent_gpu_remove_user_channel(uvm_parent_gpu_t *parent_gpu, uvm_user_c
 //  NV_ERR_PAGE_TABLE_NOT_AVAIL  Entry's instance pointer is valid but the entry
 //                               targets an invalid subcontext
 //
-// out_va_space is valid if NV_OK is returned, otherwise it's NULL.
-// out_gpu is valid if NV_OK is returned, otherwise it's NULL.
-// The caller is responsible for ensuring that the returned va_space and gpu
-// can't be destroyed, so this function should only be called from the bottom
-// half.
+// out_va_space is valid if NV_OK is returned, otherwise it's NULL. The caller
+// is responsibile for ensuring that the returned va_space can't be destroyed,
+// so these functions should only be called from the bottom half.
 NV_STATUS uvm_parent_gpu_fault_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
-                                                 const uvm_fault_buffer_entry_t *fault,
-                                                 uvm_va_space_t **out_va_space,
-                                                 uvm_gpu_t **out_gpu);
+                                                 uvm_fault_buffer_entry_t *fault,
+                                                 uvm_va_space_t **out_va_space);
 
-// Return the GPU VA space for the given instance pointer and ve_id in the
-// access counter entry. This function can only be used for virtual address
-// entries.
-// The return values are the same as uvm_parent_gpu_fault_entry_to_va_space()
-// but for virtual access counter entries.
 NV_STATUS uvm_parent_gpu_access_counter_entry_to_va_space(uvm_parent_gpu_t *parent_gpu,
-                                                          const uvm_access_counter_buffer_entry_t *entry,
-                                                          uvm_va_space_t **out_va_space,
-                                                          uvm_gpu_t **out_gpu);
+                                                          uvm_access_counter_buffer_entry_t *entry,
+                                                          uvm_va_space_t **out_va_space);
 
 typedef enum
 {

kernel-open/nvidia-uvm/uvm_gpu_access_counters.c

@@ -684,7 +684,10 @@ static void access_counter_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu,
 
     while (get != put) {
         // Wait until valid bit is set
-        UVM_SPIN_WHILE(!parent_gpu->access_counter_buffer_hal->entry_is_valid(parent_gpu, get), &spin);
+        UVM_SPIN_WHILE(!parent_gpu->access_counter_buffer_hal->entry_is_valid(parent_gpu, get), &spin) {
+            if (uvm_global_get_status() != NV_OK)
+                goto done;
+        }
 
         parent_gpu->access_counter_buffer_hal->entry_clear_valid(parent_gpu, get);
         ++get;
@@ -692,6 +695,7 @@ static void access_counter_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu,
             get = 0;
     }
 
+done:
     write_get(parent_gpu, get);
 }
 
@@ -734,18 +738,9 @@ static int cmp_sort_virt_notifications_by_instance_ptr(const void *_a, const voi
     return cmp_access_counter_instance_ptr(a, b);
 }
 
-// Compare two GPUs
-static inline int cmp_gpu(const uvm_gpu_t *a, const uvm_gpu_t *b)
-{
-    NvU32 id_a = a ? uvm_id_value(a->id) : 0;
-    NvU32 id_b = b ? uvm_id_value(b->id) : 0;
-
-    return UVM_CMP_DEFAULT(id_a, id_b);
-}
-
 // Sort comparator for pointers to GVA access counter notification buffer
-// entries that sorts by va_space, GPU ID, and fault address.
-static int cmp_sort_virt_notifications_by_va_space_gpu_address(const void *_a, const void *_b)
+// entries that sorts by va_space, and fault address.
+static int cmp_sort_virt_notifications_by_va_space_address(const void *_a, const void *_b)
 {
     const uvm_access_counter_buffer_entry_t **a = (const uvm_access_counter_buffer_entry_t **)_a;
     const uvm_access_counter_buffer_entry_t **b = (const uvm_access_counter_buffer_entry_t **)_b;
@@ -756,10 +751,6 @@ static int cmp_sort_virt_notifications_by_va_space_gpu_address(const void *_a, c
     if (result != 0)
         return result;
 
-    result = cmp_gpu((*a)->gpu, (*b)->gpu);
-    if (result != 0)
-        return result;
-
     return UVM_CMP_DEFAULT((*a)->address.address, (*b)->address.address);
 }
 
@@ -787,7 +778,7 @@ typedef enum
     NOTIFICATION_FETCH_MODE_ALL,
 } notification_fetch_mode_t;
 
-static NvU32 fetch_access_counter_buffer_entries(uvm_parent_gpu_t *parent_gpu,
+static NvU32 fetch_access_counter_buffer_entries(uvm_gpu_t *gpu,
                                                  uvm_access_counter_service_batch_context_t *batch_context,
                                                  notification_fetch_mode_t fetch_mode)
 {
@@ -796,12 +787,12 @@ static NvU32 fetch_access_counter_buffer_entries(uvm_parent_gpu_t *parent_gpu,
     NvU32 notification_index;
     uvm_access_counter_buffer_entry_t *notification_cache;
     uvm_spin_loop_t spin;
-    uvm_access_counter_buffer_info_t *access_counters = &parent_gpu->access_counter_buffer_info;
+    uvm_access_counter_buffer_info_t *access_counters = &gpu->parent->access_counter_buffer_info;
     NvU32 last_instance_ptr_idx = 0;
     uvm_aperture_t last_aperture = UVM_APERTURE_PEER_MAX;
 
-    UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.access_counters.service_lock));
-    UVM_ASSERT(parent_gpu->access_counters_supported);
+    UVM_ASSERT(uvm_sem_is_locked(&gpu->parent->isr.access_counters.service_lock));
+    UVM_ASSERT(gpu->parent->access_counters_supported);
 
     notification_cache = batch_context->notification_cache;
 
@@ -830,19 +821,25 @@ static NvU32 fetch_access_counter_buffer_entries(uvm_parent_gpu_t *parent_gpu,
            (fetch_mode == NOTIFICATION_FETCH_MODE_ALL || notification_index < access_counters->max_batch_size)) {
         uvm_access_counter_buffer_entry_t *current_entry = &notification_cache[notification_index];
 
-        // We cannot just wait for the last entry (the one pointed by put) to become valid, we have to do it
-        // individually since entries can be written out of order
-        UVM_SPIN_WHILE(!parent_gpu->access_counter_buffer_hal->entry_is_valid(parent_gpu, get), &spin) {
+        // We cannot just wait for the last entry (the one pointed by put) to
+        // become valid, we have to do it individually since entries can be
+        // written out of order
+        UVM_SPIN_WHILE(!gpu->parent->access_counter_buffer_hal->entry_is_valid(gpu->parent, get), &spin) {
             // We have some entry to work on. Let's do the rest later.
             if (fetch_mode != NOTIFICATION_FETCH_MODE_ALL && notification_index > 0)
                 goto done;
+
+            // There's no entry to work on and something has gone wrong. Ignore
+            // the rest.
+            if (uvm_global_get_status() != NV_OK)
+               goto done;
         }
 
         // Prevent later accesses being moved above the read of the valid bit
         smp_mb__after_atomic();
 
         // Got valid bit set. Let's cache.
-        parent_gpu->access_counter_buffer_hal->parse_entry(parent_gpu, get, current_entry);
+        gpu->parent->access_counter_buffer_hal->parse_entry(gpu->parent, get, current_entry);
 
         if (current_entry->address.is_virtual) {
             batch_context->virt.notifications[batch_context->virt.num_notifications++] = current_entry;
@@ -858,38 +855,26 @@ static NvU32 fetch_access_counter_buffer_entries(uvm_parent_gpu_t *parent_gpu,
             }
         }
         else {
-            NvU64 translation_size;
-            uvm_gpu_t *gpu;
-
-            translation_size = get_config_for_type(access_counters,
-                                                   current_entry->counter_type)->translation_size;
+            const NvU64 translation_size = get_config_for_type(access_counters, current_entry->counter_type)->translation_size;
             current_entry->address.address = UVM_ALIGN_DOWN(current_entry->address.address, translation_size);
 
             batch_context->phys.notifications[batch_context->phys.num_notifications++] = current_entry;
 
-            gpu = uvm_parent_gpu_find_first_valid_gpu(parent_gpu);
-            if (!gpu) {
-                current_entry->physical_info.resident_id = UVM_ID_INVALID;
-                current_entry->gpu = NULL;
-            }
-            else {
-                current_entry->gpu = gpu;
-                current_entry->physical_info.resident_id =
-                    uvm_gpu_get_processor_id_by_address(gpu, uvm_gpu_phys_address(current_entry->address.aperture,
-                                                                                  current_entry->address.address));
+            current_entry->physical_info.resident_id =
+                uvm_gpu_get_processor_id_by_address(gpu, uvm_gpu_phys_address(current_entry->address.aperture,
+                                                                              current_entry->address.address));
 
-                if (batch_context->phys.is_single_aperture) {
-                    if (batch_context->phys.num_notifications == 1)
-                        last_aperture = current_entry->address.aperture;
-                    else if (current_entry->address.aperture != last_aperture)
-                        batch_context->phys.is_single_aperture = false;
-                }
-
-                if (current_entry->counter_type == UVM_ACCESS_COUNTER_TYPE_MOMC)
-                    UVM_ASSERT(uvm_id_equal(current_entry->physical_info.resident_id, gpu->id));
-                else
-                    UVM_ASSERT(!uvm_id_equal(current_entry->physical_info.resident_id, gpu->id));
+            if (batch_context->phys.is_single_aperture) {
+                if (batch_context->phys.num_notifications == 1)
+                    last_aperture = current_entry->address.aperture;
+                else if (current_entry->address.aperture != last_aperture)
+                    batch_context->phys.is_single_aperture = false;
             }
+
+            if (current_entry->counter_type == UVM_ACCESS_COUNTER_TYPE_MOMC)
+                UVM_ASSERT(uvm_id_equal(current_entry->physical_info.resident_id, gpu->id));
+            else
+                UVM_ASSERT(!uvm_id_equal(current_entry->physical_info.resident_id, gpu->id));
         }
 
         ++notification_index;
@@ -899,7 +884,7 @@ static NvU32 fetch_access_counter_buffer_entries(uvm_parent_gpu_t *parent_gpu,
     }
 
 done:
-    write_get(parent_gpu, get);
+    write_get(gpu->parent, get);
 
     return notification_index;
 }
@@ -920,16 +905,12 @@ static void translate_virt_notifications_instance_ptrs(uvm_parent_gpu_t *parent_
             // simply be ignored in subsequent processing.
             status = uvm_parent_gpu_access_counter_entry_to_va_space(parent_gpu,
                                                                      current_entry,
-                                                                     &current_entry->virtual_info.va_space,
-                                                                     &current_entry->gpu);
-            if (status != NV_OK) {
+                                                                     &current_entry->virtual_info.va_space);
+            if (status != NV_OK)
                 UVM_ASSERT(current_entry->virtual_info.va_space == NULL);
-                UVM_ASSERT(current_entry->gpu == NULL);
-            }
         }
         else {
             current_entry->virtual_info.va_space = batch_context->virt.notifications[i - 1]->virtual_info.va_space;
-            current_entry->gpu = batch_context->virt.notifications[i - 1]->gpu;
         }
     }
 }
@@ -953,7 +934,7 @@ static void preprocess_virt_notifications(uvm_parent_gpu_t *parent_gpu,
     sort(batch_context->virt.notifications,
          batch_context->virt.num_notifications,
          sizeof(*batch_context->virt.notifications),
-         cmp_sort_virt_notifications_by_va_space_gpu_address,
+         cmp_sort_virt_notifications_by_va_space_address,
          NULL);
 }
 
@@ -971,17 +952,13 @@ static void preprocess_phys_notifications(uvm_access_counter_service_batch_conte
     }
 }
 
-static NV_STATUS notify_tools_broadcast_and_process_flags(uvm_parent_gpu_t *parent_gpu,
-                                                          uvm_access_counter_buffer_entry_t **notification_start,
-                                                          NvU32 num_entries,
-                                                          NvU32 flags)
+static NV_STATUS notify_tools_and_process_flags(uvm_gpu_t *gpu,
+                                                uvm_access_counter_buffer_entry_t **notification_start,
+                                                NvU32 num_entries,
+                                                NvU32 flags)
 {
-    uvm_gpu_t *gpu = uvm_parent_gpu_find_first_valid_gpu(parent_gpu);
     NV_STATUS status = NV_OK;
 
-    if (!gpu)
-        return NV_OK;
-
     if (uvm_enable_builtin_tests) {
         // TODO: Bug 4310744: [UVM][TOOLS] Attribute access counter tools events
         //                    to va_space instead of broadcasting.
@@ -997,31 +974,6 @@ static NV_STATUS notify_tools_broadcast_and_process_flags(uvm_parent_gpu_t *pare
     return status;
 }
 
-static NV_STATUS notify_tools_and_process_flags(uvm_va_space_t *va_space,
-                                                uvm_gpu_t *gpu,
-                                                uvm_access_counter_buffer_entry_t **notification_start,
-                                                NvU32 num_entries,
-                                                NvU32 flags)
-{
-    NV_STATUS status = NV_OK;
-
-    if (uvm_enable_builtin_tests) {
-        NvU32 i;
-
-        for (i = 0; i < num_entries; i++) {
-            uvm_tools_record_access_counter(va_space,
-                                            gpu->id,
-                                            notification_start[i],
-                                            flags & UVM_ACCESS_COUNTER_PHYS_ON_MANAGED);
-        }
-    }
-
-    if (flags & UVM_ACCESS_COUNTER_ACTION_CLEAR)
-        status = access_counter_clear_notifications(gpu, notification_start, num_entries);
-
-    return status;
-}
-
 static NV_STATUS service_va_block_locked(uvm_processor_id_t processor,
                                          uvm_va_block_t *va_block,
                                          uvm_va_block_retry_t *va_block_retry,
@@ -1227,13 +1179,13 @@ static void reverse_mappings_to_va_block_page_mask(uvm_va_block_t *va_block,
     }
 }
 
-static NV_STATUS service_phys_single_va_block(uvm_access_counter_service_batch_context_t *batch_context,
+static NV_STATUS service_phys_single_va_block(uvm_gpu_t *gpu,
+                                              uvm_access_counter_service_batch_context_t *batch_context,
                                               const uvm_access_counter_buffer_entry_t *current_entry,
                                               const uvm_reverse_map_t *reverse_mappings,
                                               size_t num_reverse_mappings,
                                               NvU32 *out_flags)
 {
-    uvm_gpu_t *gpu = current_entry->gpu;
     size_t index;
     uvm_va_block_t *va_block = reverse_mappings[0].va_block;
     uvm_va_space_t *va_space = NULL;
@@ -1320,7 +1272,8 @@ done:
     return status;
 }
 
-static NV_STATUS service_phys_va_blocks(uvm_access_counter_service_batch_context_t *batch_context,
+static NV_STATUS service_phys_va_blocks(uvm_gpu_t *gpu,
+                                        uvm_access_counter_service_batch_context_t *batch_context,
                                         const uvm_access_counter_buffer_entry_t *current_entry,
                                         const uvm_reverse_map_t *reverse_mappings,
                                         size_t num_reverse_mappings,
@@ -1333,7 +1286,8 @@ static NV_STATUS service_phys_va_blocks(uvm_access_counter_service_batch_context
 
     for (index = 0; index < num_reverse_mappings; ++index) {
         NvU32 out_flags_local = 0;
-        status = service_phys_single_va_block(batch_context,
+        status = service_phys_single_va_block(gpu,
+                                              batch_context,
                                               current_entry,
                                               reverse_mappings + index,
                                               1,
@@ -1382,7 +1336,8 @@ static bool are_reverse_mappings_on_single_block(const uvm_reverse_map_t *revers
 // Service the given translation range. It will return the count of the reverse
 // mappings found during servicing in num_reverse_mappings, even if the function
 // doesn't return NV_OK.
-static NV_STATUS service_phys_notification_translation(uvm_gpu_t *resident_gpu,
+static NV_STATUS service_phys_notification_translation(uvm_gpu_t *gpu,
+                                                       uvm_gpu_t *resident_gpu,
                                                        uvm_access_counter_service_batch_context_t *batch_context,
                                                        const uvm_gpu_access_counter_type_config_t *config,
                                                        const uvm_access_counter_buffer_entry_t *current_entry,
@@ -1391,7 +1346,6 @@ static NV_STATUS service_phys_notification_translation(uvm_gpu_t *resident_gpu,
                                                        size_t *num_reverse_mappings,
                                                        NvU32 *out_flags)
 {
-    uvm_gpu_t *gpu = current_entry->gpu;
     NV_STATUS status;
     NvU32 region_start, region_end;
 
@@ -1429,14 +1383,16 @@ static NV_STATUS service_phys_notification_translation(uvm_gpu_t *resident_gpu,
 
     // Service all the translations
     if (are_reverse_mappings_on_single_block(batch_context->phys.translations, *num_reverse_mappings)) {
-        status = service_phys_single_va_block(batch_context,
+        status = service_phys_single_va_block(gpu,
+                                              batch_context,
                                               current_entry,
                                               batch_context->phys.translations,
                                               *num_reverse_mappings,
                                               out_flags);
     }
     else {
-        status = service_phys_va_blocks(batch_context,
+        status = service_phys_va_blocks(gpu,
+                                        batch_context,
                                         current_entry,
                                         batch_context->phys.translations,
                                         *num_reverse_mappings,
@@ -1446,14 +1402,14 @@ static NV_STATUS service_phys_notification_translation(uvm_gpu_t *resident_gpu,
     return status;
 }
 
-static NV_STATUS service_phys_notification(uvm_access_counter_service_batch_context_t *batch_context,
-                                           uvm_access_counter_buffer_entry_t *current_entry)
+static NV_STATUS service_phys_notification(uvm_gpu_t *gpu,
+                                           uvm_access_counter_service_batch_context_t *batch_context,
+                                           const uvm_access_counter_buffer_entry_t *current_entry,
+                                           NvU32 *out_flags)
 {
     NvU64 address;
     NvU64 translation_index;
-    uvm_gpu_t *gpu = current_entry->gpu;
-    uvm_parent_gpu_t *parent_gpu = gpu->parent;
-    uvm_access_counter_buffer_info_t *access_counters = &parent_gpu->access_counter_buffer_info;
+    uvm_access_counter_buffer_info_t *access_counters = &gpu->parent->access_counter_buffer_info;
     uvm_access_counter_type_t counter_type = current_entry->counter_type;
     const uvm_gpu_access_counter_type_config_t *config = get_config_for_type(access_counters, counter_type);
     unsigned long sub_granularity;
@@ -1483,13 +1439,14 @@ static NV_STATUS service_phys_notification(uvm_access_counter_service_batch_cont
         // fall outside of the allocatable address range. We just drop
         // them.
         if (address >= resident_gpu->mem_info.max_allocatable_address)
-            goto out;
+            return NV_OK;
     }
 
     for (translation_index = 0; translation_index < config->translations_per_counter; ++translation_index) {
         size_t num_reverse_mappings;
         NvU32 out_flags_local = 0;
-        status = service_phys_notification_translation(resident_gpu,
+        status = service_phys_notification_translation(gpu,
+                                                       resident_gpu,
                                                        batch_context,
                                                        config,
                                                        current_entry,
@@ -1510,32 +1467,37 @@ static NV_STATUS service_phys_notification(uvm_access_counter_service_batch_cont
     }
 
     if (uvm_enable_builtin_tests)
-        flags |= ((total_reverse_mappings != 0) ? UVM_ACCESS_COUNTER_PHYS_ON_MANAGED : 0);
+        *out_flags |= ((total_reverse_mappings != 0) ? UVM_ACCESS_COUNTER_PHYS_ON_MANAGED : 0);
+
+    if (status == NV_OK && (flags & UVM_ACCESS_COUNTER_ACTION_CLEAR))
+        *out_flags |= UVM_ACCESS_COUNTER_ACTION_CLEAR;
 
-out:
-    notify_tools_broadcast_and_process_flags(parent_gpu, &current_entry, 1, flags);
     return status;
 }
 
 // TODO: Bug 2018899: Add statistics for dropped access counter notifications
-static NV_STATUS service_phys_notifications(uvm_parent_gpu_t *parent_gpu,
+static NV_STATUS service_phys_notifications(uvm_gpu_t *gpu,
                                             uvm_access_counter_service_batch_context_t *batch_context)
 {
     NvU32 i;
     uvm_access_counter_buffer_entry_t **notifications = batch_context->phys.notifications;
 
-    UVM_ASSERT(parent_gpu->access_counters_can_use_physical_addresses);
+    UVM_ASSERT(gpu->parent->access_counters_can_use_physical_addresses);
 
     preprocess_phys_notifications(batch_context);
 
     for (i = 0; i < batch_context->phys.num_notifications; ++i) {
         NV_STATUS status;
         uvm_access_counter_buffer_entry_t *current_entry = notifications[i];
+        NvU32 flags = 0;
 
         if (!UVM_ID_IS_VALID(current_entry->physical_info.resident_id))
             continue;
 
-        status = service_phys_notification(batch_context, current_entry);
+        status = service_phys_notification(gpu, batch_context, current_entry, &flags);
+
+        notify_tools_and_process_flags(gpu, &notifications[i], 1, flags);
+
         if (status != NV_OK)
             return status;
     }
@@ -1672,14 +1634,16 @@ static NV_STATUS service_virt_notifications_in_block(uvm_gpu_va_space_t *gpu_va_
         uvm_access_counter_buffer_entry_t *current_entry = notifications[i];
         NvU64 address = current_entry->address.address;
 
-        if (current_entry->virtual_info.va_space != va_space || current_entry->gpu != gpu || address > va_block->end)
+        if ((current_entry->virtual_info.va_space == va_space) && (address <= va_block->end)) {
+            expand_notification_block(gpu_va_space,
+                                      va_block,
+                                      batch_context->block_service_context.block_context,
+                                      accessed_pages,
+                                      current_entry);
+        }
+        else {
             break;
-
-        expand_notification_block(gpu_va_space,
-                                  va_block,
-                                  batch_context->block_service_context.block_context,
-                                  accessed_pages,
-                                  current_entry);
+        }
     }
 
     *out_index = i;
@@ -1694,7 +1658,7 @@ static NV_STATUS service_virt_notifications_in_block(uvm_gpu_va_space_t *gpu_va_
     if (status == NV_OK)
         flags |= UVM_ACCESS_COUNTER_ACTION_CLEAR;
 
-    flags_status = notify_tools_and_process_flags(va_space, gpu, &notifications[index], *out_index - index, flags);
+    flags_status = notify_tools_and_process_flags(gpu, &notifications[index], *out_index - index, flags);
 
     if ((status == NV_OK) && (flags_status != NV_OK))
         status = flags_status;
@@ -1733,7 +1697,7 @@ static NV_STATUS service_virt_notification_ats(uvm_gpu_va_space_t *gpu_va_space,
     if (!vma) {
         // Clear the notification entry to continue receiving access counter
         // notifications when a new VMA is allocated in this range.
-        status = notify_tools_and_process_flags(va_space, gpu, &notifications[index], 1, flags);
+        status = notify_tools_and_process_flags(gpu, &notifications[index], 1, flags);
         *out_index = index + 1;
         return status;
     }
@@ -1747,10 +1711,10 @@ static NV_STATUS service_virt_notification_ats(uvm_gpu_va_space_t *gpu_va_space,
         uvm_access_counter_buffer_entry_t *current_entry = notifications[i];
         address = current_entry->address.address;
 
-        if (current_entry->virtual_info.va_space != va_space || current_entry->gpu != gpu || address >= end)
+        if ((current_entry->virtual_info.va_space == va_space) && (address < end))
+            uvm_page_mask_set(&ats_context->accessed_mask, (address - base) / PAGE_SIZE);
+        else
             break;
-
-        uvm_page_mask_set(&ats_context->accessed_mask, (address - base) / PAGE_SIZE);
     }
 
     *out_index = i;
@@ -1765,7 +1729,7 @@ static NV_STATUS service_virt_notification_ats(uvm_gpu_va_space_t *gpu_va_space,
     if (status != NV_OK)
         flags &= ~UVM_ACCESS_COUNTER_ACTION_CLEAR;
 
-    flags_status = notify_tools_and_process_flags(va_space, gpu, &notifications[index], *out_index - index, flags);
+    flags_status = notify_tools_and_process_flags(gpu, &notifications[index], *out_index - index, flags);
     if ((status == NV_OK) && (flags_status != NV_OK))
         status = flags_status;
 
@@ -1817,7 +1781,7 @@ static NV_STATUS service_virt_notifications_batch(uvm_gpu_va_space_t *gpu_va_spa
             status = service_virt_notifications_in_block(gpu_va_space, mm, va_block, batch_context, index, out_index);
         }
         else {
-            status = notify_tools_and_process_flags(va_space, gpu_va_space->gpu, batch_context->virt.notifications, 1, flags);
+            status = notify_tools_and_process_flags(gpu_va_space->gpu, batch_context->virt.notifications, 1, flags);
             *out_index = index + 1;
         }
     }
@@ -1847,11 +1811,7 @@ static NV_STATUS service_virt_notifications_batch(uvm_gpu_va_space_t *gpu_va_spa
 
         // Clobber status to continue processing the rest of the notifications
         // in the batch.
-        status = notify_tools_and_process_flags(va_space,
-                                                gpu_va_space->gpu,
-                                                batch_context->virt.notifications,
-                                                1,
-                                                flags);
+        status = notify_tools_and_process_flags(gpu_va_space->gpu, batch_context->virt.notifications, 1, flags);
 
         *out_index = index + 1;
     }
@@ -1859,7 +1819,7 @@ static NV_STATUS service_virt_notifications_batch(uvm_gpu_va_space_t *gpu_va_spa
     return status;
 }
 
-static NV_STATUS service_virt_notifications(uvm_parent_gpu_t *parent_gpu,
+static NV_STATUS service_virt_notifications(uvm_gpu_t *gpu,
                                             uvm_access_counter_service_batch_context_t *batch_context)
 {
     NvU32 i = 0;
@@ -1867,19 +1827,18 @@ static NV_STATUS service_virt_notifications(uvm_parent_gpu_t *parent_gpu,
     struct mm_struct *mm = NULL;
     uvm_va_space_t *va_space = NULL;
     uvm_va_space_t *prev_va_space = NULL;
-    uvm_gpu_t *prev_gpu = NULL;
     uvm_gpu_va_space_t *gpu_va_space = NULL;
 
     // TODO: Bug 4299018 : Add support for virtual access counter migrations on
     //                     4K page sizes.
     if (PAGE_SIZE == UVM_PAGE_SIZE_4K) {
-        return notify_tools_broadcast_and_process_flags(parent_gpu,
-                                                        batch_context->virt.notifications,
-                                                        batch_context->virt.num_notifications,
-                                                        0);
+        return notify_tools_and_process_flags(gpu,
+                                              batch_context->virt.notifications,
+                                              batch_context->virt.num_notifications,
+                                              0);
     }
 
-    preprocess_virt_notifications(parent_gpu, batch_context);
+    preprocess_virt_notifications(gpu->parent, batch_context);
 
     while (i < batch_context->virt.num_notifications) {
         uvm_access_counter_buffer_entry_t *current_entry = batch_context->virt.notifications[i];
@@ -1893,38 +1852,25 @@ static NV_STATUS service_virt_notifications(uvm_parent_gpu_t *parent_gpu,
                 uvm_va_space_mm_release_unlock(prev_va_space, mm);
 
                 mm = NULL;
-                prev_gpu = NULL;
+                gpu_va_space = NULL;
             }
 
             // Acquire locks for the new va_space.
             if (va_space) {
                 mm = uvm_va_space_mm_retain_lock(va_space);
                 uvm_va_space_down_read(va_space);
+
+                gpu_va_space = uvm_gpu_va_space_get_by_parent_gpu(va_space, gpu->parent);
             }
 
             prev_va_space = va_space;
         }
 
-        if (va_space) {
-            if (prev_gpu != current_entry->gpu) {
-                prev_gpu = current_entry->gpu;
-                gpu_va_space = uvm_gpu_va_space_get(va_space, current_entry->gpu);
-            }
-
-            if (gpu_va_space && uvm_va_space_has_access_counter_migrations(va_space)) {
-                status = service_virt_notifications_batch(gpu_va_space, mm, batch_context, i, &i);
-            }
-            else {
-                status = notify_tools_and_process_flags(va_space,
-                                                        current_entry->gpu,
-                                                        &batch_context->virt.notifications[i],
-                                                        1,
-                                                        0);
-                i++;
-            }
+        if (va_space && gpu_va_space && uvm_va_space_has_access_counter_migrations(va_space)) {
+            status = service_virt_notifications_batch(gpu_va_space, mm, batch_context, i, &i);
         }
         else {
-            status = notify_tools_broadcast_and_process_flags(parent_gpu, &batch_context->virt.notifications[i], 1, 0);
+            status = notify_tools_and_process_flags(gpu, &batch_context->virt.notifications[i], 1, 0);
             i++;
         }
 
@@ -1940,18 +1886,19 @@ static NV_STATUS service_virt_notifications(uvm_parent_gpu_t *parent_gpu,
     return status;
 }
 
-void uvm_parent_gpu_service_access_counters(uvm_parent_gpu_t *parent_gpu)
+
+void uvm_gpu_service_access_counters(uvm_gpu_t *gpu)
 {
     NV_STATUS status = NV_OK;
-    uvm_access_counter_service_batch_context_t *batch_context = &parent_gpu->access_counter_buffer_info.batch_service_context;
+    uvm_access_counter_service_batch_context_t *batch_context = &gpu->parent->access_counter_buffer_info.batch_service_context;
 
-    UVM_ASSERT(parent_gpu->access_counters_supported);
+    UVM_ASSERT(gpu->parent->access_counters_supported);
 
-    if (parent_gpu->access_counter_buffer_info.notifications_ignored_count > 0)
+    if (gpu->parent->access_counter_buffer_info.notifications_ignored_count > 0)
         return;
 
     while (1) {
-        batch_context->num_cached_notifications = fetch_access_counter_buffer_entries(parent_gpu,
+        batch_context->num_cached_notifications = fetch_access_counter_buffer_entries(gpu,
                                                                                       batch_context,
                                                                                       NOTIFICATION_FETCH_MODE_BATCH_READY);
         if (batch_context->num_cached_notifications == 0)
@@ -1960,13 +1907,13 @@ void uvm_parent_gpu_service_access_counters(uvm_parent_gpu_t *parent_gpu)
         ++batch_context->batch_id;
 
         if (batch_context->virt.num_notifications) {
-            status = service_virt_notifications(parent_gpu, batch_context);
+            status = service_virt_notifications(gpu, batch_context);
             if (status != NV_OK)
                 break;
         }
 
         if (batch_context->phys.num_notifications) {
-            status = service_phys_notifications(parent_gpu, batch_context);
+            status = service_phys_notifications(gpu, batch_context);
             if (status != NV_OK)
                 break;
         }
@@ -1975,7 +1922,7 @@ void uvm_parent_gpu_service_access_counters(uvm_parent_gpu_t *parent_gpu)
     if (status != NV_OK) {
         UVM_DBG_PRINT("Error %s servicing access counter notifications on GPU: %s\n",
                       nvstatusToString(status),
-                      uvm_parent_gpu_name(parent_gpu));
+                      uvm_gpu_name(gpu));
     }
 }
 

kernel-open/nvidia-uvm/uvm_gpu_access_counters.h

@@ -31,7 +31,7 @@ NV_STATUS uvm_parent_gpu_init_access_counters(uvm_parent_gpu_t *parent_gpu);
 void uvm_parent_gpu_deinit_access_counters(uvm_parent_gpu_t *parent_gpu);
 bool uvm_parent_gpu_access_counters_pending(uvm_parent_gpu_t *parent_gpu);
 
-void uvm_parent_gpu_service_access_counters(uvm_parent_gpu_t *parent_gpu);
+void uvm_gpu_service_access_counters(uvm_gpu_t *gpu);
 
 void uvm_parent_gpu_access_counter_buffer_flush(uvm_parent_gpu_t *parent_gpu);
 

kernel-open/nvidia-uvm/uvm_gpu_isr.c

@@ -479,14 +479,17 @@ void uvm_parent_gpu_deinit_isr(uvm_parent_gpu_t *parent_gpu)
     uvm_kvfree(parent_gpu->isr.access_counters.stats.cpu_exec_count);
 }
 
-uvm_gpu_t *uvm_parent_gpu_find_first_valid_gpu(uvm_parent_gpu_t *parent_gpu)
+static uvm_gpu_t *find_first_valid_gpu(uvm_parent_gpu_t *parent_gpu)
 {
     uvm_gpu_t *gpu;
 
     // When SMC is enabled, there's no longer a 1:1 relationship between the
-    // parent and the partitions. It's sufficient to return any valid uvm_gpu_t
-    // since the purpose is to have a channel and push buffer for operations
-    // that affect the whole parent GPU.
+    // parent and the partitions.  But because all relevant interrupt paths
+    // are shared, as is the fault reporting logic, it's sufficient here
+    // to proceed with any valid uvm_gpu_t, even if the corresponding partition
+    // didn't cause all, or even any of the interrupts.
+    // The bottom half handlers will later find the appropriate partitions by
+    // attributing the notifications to VA spaces as necessary.
     if (parent_gpu->smc.enabled) {
         NvU32 sub_processor_index;
 
@@ -515,8 +518,13 @@ uvm_gpu_t *uvm_parent_gpu_find_first_valid_gpu(uvm_parent_gpu_t *parent_gpu)
 static void replayable_faults_isr_bottom_half(void *args)
 {
     uvm_parent_gpu_t *parent_gpu = (uvm_parent_gpu_t *)args;
+    uvm_gpu_t *gpu;
     unsigned int cpu;
 
+    gpu = find_first_valid_gpu(parent_gpu);
+    if (gpu == NULL)
+        goto put_kref;
+
     UVM_ASSERT(parent_gpu->replayable_faults_supported);
 
     // Record the lock ownership
@@ -537,10 +545,11 @@ static void replayable_faults_isr_bottom_half(void *args)
     ++parent_gpu->isr.replayable_faults.stats.cpu_exec_count[cpu];
     put_cpu();
 
-    uvm_parent_gpu_service_replayable_faults(parent_gpu);
+    uvm_gpu_service_replayable_faults(gpu);
 
     uvm_parent_gpu_replayable_faults_isr_unlock(parent_gpu);
 
+put_kref:
     // It is OK to drop a reference on the parent GPU if a bottom half has
     // been retriggered within uvm_parent_gpu_replayable_faults_isr_unlock,
     // because the rescheduling added an additional reference.
@@ -555,8 +564,13 @@ static void replayable_faults_isr_bottom_half_entry(void *args)
 static void non_replayable_faults_isr_bottom_half(void *args)
 {
     uvm_parent_gpu_t *parent_gpu = (uvm_parent_gpu_t *)args;
+    uvm_gpu_t *gpu;
     unsigned int cpu;
 
+    gpu = find_first_valid_gpu(parent_gpu);
+    if (gpu == NULL)
+        goto put_kref;
+
     UVM_ASSERT(parent_gpu->non_replayable_faults_supported);
 
     uvm_parent_gpu_non_replayable_faults_isr_lock(parent_gpu);
@@ -570,10 +584,11 @@ static void non_replayable_faults_isr_bottom_half(void *args)
     ++parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count[cpu];
     put_cpu();
 
-    uvm_parent_gpu_service_non_replayable_fault_buffer(parent_gpu);
+    uvm_gpu_service_non_replayable_fault_buffer(gpu);
 
     uvm_parent_gpu_non_replayable_faults_isr_unlock(parent_gpu);
 
+put_kref:
     uvm_parent_gpu_kref_put(parent_gpu);
 }
 
@@ -585,8 +600,13 @@ static void non_replayable_faults_isr_bottom_half_entry(void *args)
 static void access_counters_isr_bottom_half(void *args)
 {
     uvm_parent_gpu_t *parent_gpu = (uvm_parent_gpu_t *)args;
+    uvm_gpu_t *gpu;
     unsigned int cpu;
 
+    gpu = find_first_valid_gpu(parent_gpu);
+    if (gpu == NULL)
+        goto put_kref;
+
     UVM_ASSERT(parent_gpu->access_counters_supported);
 
     uvm_record_lock(&parent_gpu->isr.access_counters.service_lock, UVM_LOCK_FLAGS_MODE_SHARED);
@@ -600,10 +620,11 @@ static void access_counters_isr_bottom_half(void *args)
     ++parent_gpu->isr.access_counters.stats.cpu_exec_count[cpu];
     put_cpu();
 
-    uvm_parent_gpu_service_access_counters(parent_gpu);
+    uvm_gpu_service_access_counters(gpu);
 
     uvm_parent_gpu_access_counters_isr_unlock(parent_gpu);
 
+put_kref:
     uvm_parent_gpu_kref_put(parent_gpu);
 }
 

kernel-open/nvidia-uvm/uvm_gpu_isr.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -193,10 +193,4 @@ void uvm_parent_gpu_access_counters_intr_disable(uvm_parent_gpu_t *parent_gpu);
 // parent_gpu->isr.interrupts_lock must be held to call this function.
 void uvm_parent_gpu_access_counters_intr_enable(uvm_parent_gpu_t *parent_gpu);
 
-// Return the first valid GPU given the parent GPU or NULL if no MIG instances
-// are registered. This should only be called from bottom halves or if the
-// g_uvm_global.global_lock is held so that the returned pointer remains valid.
-//
-uvm_gpu_t *uvm_parent_gpu_find_first_valid_gpu(uvm_parent_gpu_t *parent_gpu);
-
 #endif // __UVM_GPU_ISR_H__

kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c

@@ -212,7 +212,6 @@ static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *par
 
         // Make sure that all fields in the entry are properly initialized
         fault_entry->va_space = NULL;
-        fault_entry->gpu = NULL;
         fault_entry->is_fatal = (fault_entry->fault_type >= UVM_FAULT_TYPE_FATAL);
         fault_entry->filtered = false;
 
@@ -236,7 +235,7 @@ static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *par
     return NV_OK;
 }
 
-static bool use_clear_faulted_channel_sw_method(uvm_parent_gpu_t *parent_gpu)
+static bool use_clear_faulted_channel_sw_method(uvm_gpu_t *gpu)
 {
     // If true, UVM uses a SW method to request RM to do the clearing on its
     // behalf.
@@ -244,7 +243,7 @@ static bool use_clear_faulted_channel_sw_method(uvm_parent_gpu_t *parent_gpu)
 
     // In SRIOV, the UVM (guest) driver does not have access to the privileged
     // registers used to clear the faulted bit.
-    if (uvm_parent_gpu_is_virt_mode_sriov(parent_gpu))
+    if (uvm_parent_gpu_is_virt_mode_sriov(gpu->parent))
         use_sw_method = true;
 
     // In Confidential Computing access to the privileged registers is blocked,
@@ -254,17 +253,17 @@ static bool use_clear_faulted_channel_sw_method(uvm_parent_gpu_t *parent_gpu)
         use_sw_method = true;
 
     if (use_sw_method)
-        UVM_ASSERT(parent_gpu->has_clear_faulted_channel_sw_method);
+        UVM_ASSERT(gpu->parent->has_clear_faulted_channel_sw_method);
 
     return use_sw_method;
 }
 
-static NV_STATUS clear_faulted_method_on_gpu(uvm_user_channel_t *user_channel,
+static NV_STATUS clear_faulted_method_on_gpu(uvm_gpu_t *gpu,
+                                             uvm_user_channel_t *user_channel,
                                              const uvm_fault_buffer_entry_t *fault_entry,
                                              NvU32 batch_id,
                                              uvm_tracker_t *tracker)
 {
-    uvm_gpu_t *gpu = user_channel->gpu;
     NV_STATUS status;
     uvm_push_t push;
     uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
@@ -284,7 +283,7 @@ static NV_STATUS clear_faulted_method_on_gpu(uvm_user_channel_t *user_channel,
         return status;
     }
 
-    if (use_clear_faulted_channel_sw_method(gpu->parent))
+    if (use_clear_faulted_channel_sw_method(gpu))
         gpu->parent->host_hal->clear_faulted_channel_sw_method(&push, user_channel, fault_entry);
     else
         gpu->parent->host_hal->clear_faulted_channel_method(&push, user_channel, fault_entry);
@@ -306,12 +305,12 @@ static NV_STATUS clear_faulted_method_on_gpu(uvm_user_channel_t *user_channel,
     return status;
 }
 
-static NV_STATUS clear_faulted_register_on_gpu(uvm_user_channel_t *user_channel,
+static NV_STATUS clear_faulted_register_on_gpu(uvm_gpu_t *gpu,
+                                               uvm_user_channel_t *user_channel,
                                                const uvm_fault_buffer_entry_t *fault_entry,
                                                NvU32 batch_id,
                                                uvm_tracker_t *tracker)
 {
-    uvm_gpu_t *gpu = user_channel->gpu;
     NV_STATUS status;
 
     UVM_ASSERT(!gpu->parent->has_clear_faulted_channel_method);
@@ -329,26 +328,25 @@ static NV_STATUS clear_faulted_register_on_gpu(uvm_user_channel_t *user_channel,
     return NV_OK;
 }
 
-static NV_STATUS clear_faulted_on_gpu(uvm_user_channel_t *user_channel,
+static NV_STATUS clear_faulted_on_gpu(uvm_gpu_t *gpu,
+                                      uvm_user_channel_t *user_channel,
                                       const uvm_fault_buffer_entry_t *fault_entry,
                                       NvU32 batch_id,
                                       uvm_tracker_t *tracker)
 {
-    uvm_gpu_t *gpu = user_channel->gpu;
+    if (gpu->parent->has_clear_faulted_channel_method || use_clear_faulted_channel_sw_method(gpu))
+        return clear_faulted_method_on_gpu(gpu, user_channel, fault_entry, batch_id, tracker);
 
-    if (gpu->parent->has_clear_faulted_channel_method || use_clear_faulted_channel_sw_method(gpu->parent))
-        return clear_faulted_method_on_gpu(user_channel, fault_entry, batch_id, tracker);
-
-    return clear_faulted_register_on_gpu(user_channel, fault_entry, batch_id, tracker);
+    return clear_faulted_register_on_gpu(gpu, user_channel, fault_entry, batch_id, tracker);
 }
 
-static NV_STATUS service_managed_fault_in_block_locked(uvm_va_block_t *va_block,
+static NV_STATUS service_managed_fault_in_block_locked(uvm_gpu_t *gpu,
+                                                       uvm_va_block_t *va_block,
                                                        uvm_va_block_retry_t *va_block_retry,
                                                        uvm_fault_buffer_entry_t *fault_entry,
                                                        uvm_service_block_context_t *service_context,
                                                        const bool hmm_migratable)
 {
-    uvm_gpu_t *gpu = fault_entry->gpu;
     NV_STATUS status = NV_OK;
     uvm_page_index_t page_index;
     uvm_perf_thrashing_hint_t thrashing_hint;
@@ -443,13 +441,13 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_va_block_t *va_block,
     return status;
 }
 
-static NV_STATUS service_managed_fault_in_block(uvm_va_block_t *va_block,
+static NV_STATUS service_managed_fault_in_block(uvm_gpu_t *gpu,
+                                                uvm_va_block_t *va_block,
                                                 uvm_fault_buffer_entry_t *fault_entry,
                                                 const bool hmm_migratable)
 {
     NV_STATUS status, tracker_status;
     uvm_va_block_retry_t va_block_retry;
-    uvm_gpu_t *gpu = fault_entry->gpu;
     uvm_service_block_context_t *service_context = &gpu->parent->fault_buffer_info.non_replayable.block_service_context;
 
     service_context->operation = UVM_SERVICE_OPERATION_NON_REPLAYABLE_FAULTS;
@@ -461,7 +459,8 @@ static NV_STATUS service_managed_fault_in_block(uvm_va_block_t *va_block,
     uvm_mutex_lock(&va_block->lock);
 
     status = UVM_VA_BLOCK_RETRY_LOCKED(va_block, &va_block_retry,
-                                       service_managed_fault_in_block_locked(va_block,
+                                       service_managed_fault_in_block_locked(gpu,
+                                                                             va_block,
                                                                              &va_block_retry,
                                                                              fault_entry,
                                                                              service_context,
@@ -503,14 +502,16 @@ static void kill_channel_delayed_entry(void *user_channel)
     UVM_ENTRY_VOID(kill_channel_delayed(user_channel));
 }
 
-static void schedule_kill_channel(uvm_fault_buffer_entry_t *fault_entry, uvm_user_channel_t *user_channel)
+static void schedule_kill_channel(uvm_gpu_t *gpu,
+                                  uvm_fault_buffer_entry_t *fault_entry,
+                                  uvm_user_channel_t *user_channel)
 {
     uvm_va_space_t *va_space = fault_entry->va_space;
-    uvm_parent_gpu_t *parent_gpu = fault_entry->gpu->parent;
-    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &parent_gpu->fault_buffer_info.non_replayable;
+    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
     void *packet = (char *)non_replayable_faults->shadow_buffer_copy +
-                   (fault_entry->non_replayable.buffer_index * parent_gpu->fault_buffer_hal->entry_size(parent_gpu));
+                   (fault_entry->non_replayable.buffer_index * gpu->parent->fault_buffer_hal->entry_size(gpu->parent));
 
+    UVM_ASSERT(gpu);
     UVM_ASSERT(va_space);
     UVM_ASSERT(user_channel);
 
@@ -521,7 +522,7 @@ static void schedule_kill_channel(uvm_fault_buffer_entry_t *fault_entry, uvm_use
     user_channel->kill_channel.va_space = va_space;
 
     // Save the packet to be handled by RM in the channel structure
-    memcpy(user_channel->kill_channel.fault_packet, packet, parent_gpu->fault_buffer_hal->entry_size(parent_gpu));
+    memcpy(user_channel->kill_channel.fault_packet, packet, gpu->parent->fault_buffer_hal->entry_size(gpu->parent));
 
     // Retain the channel here so it is not prematurely destroyed. It will be
     // released after forwarding the fault to RM in kill_channel_delayed.
@@ -532,7 +533,7 @@ static void schedule_kill_channel(uvm_fault_buffer_entry_t *fault_entry, uvm_use
                            kill_channel_delayed_entry,
                            user_channel);
 
-    nv_kthread_q_schedule_q_item(&parent_gpu->isr.kill_channel_q,
+    nv_kthread_q_schedule_q_item(&gpu->parent->isr.kill_channel_q,
                                  &user_channel->kill_channel.kill_channel_q_item);
 }
 
@@ -549,7 +550,6 @@ static NV_STATUS service_non_managed_fault(uvm_gpu_va_space_t *gpu_va_space,
                                            uvm_fault_buffer_entry_t *fault_entry,
                                            NV_STATUS lookup_status)
 {
-    uvm_va_space_t *va_space = gpu_va_space->va_space;
     uvm_gpu_t *gpu = gpu_va_space->gpu;
     uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
     uvm_ats_fault_invalidate_t *ats_invalidate = &non_replayable_faults->ats_invalidate;
@@ -557,11 +557,9 @@ static NV_STATUS service_non_managed_fault(uvm_gpu_va_space_t *gpu_va_space,
     NV_STATUS fatal_fault_status = NV_ERR_INVALID_ADDRESS;
 
     UVM_ASSERT(!fault_entry->is_fatal);
-    UVM_ASSERT(fault_entry->va_space == va_space);
-    UVM_ASSERT(fault_entry->gpu == gpu);
 
     // Avoid dropping fault events when the VA block is not found or cannot be created
-    uvm_perf_event_notify_gpu_fault(&va_space->perf_events,
+    uvm_perf_event_notify_gpu_fault(&fault_entry->va_space->perf_events,
                                     NULL,
                                     gpu->id,
                                     UVM_ID_INVALID,
@@ -586,11 +584,11 @@ static NV_STATUS service_non_managed_fault(uvm_gpu_va_space_t *gpu_va_space,
 
         ats_invalidate->tlb_batch_pending = false;
 
-        va_range_next = uvm_va_space_iter_first(va_space, fault_entry->fault_address, ~0ULL);
+        va_range_next = uvm_va_space_iter_first(gpu_va_space->va_space, fault_entry->fault_address, ~0ULL);
 
         // The VA isn't managed. See if ATS knows about it.
         vma = find_vma_intersection(mm, fault_address, fault_address + 1);
-        if (!vma || uvm_ats_check_in_gmmu_region(va_space, fault_address, va_range_next)) {
+        if (!vma || uvm_ats_check_in_gmmu_region(gpu_va_space->va_space, fault_address, va_range_next)) {
 
             // Do not return error due to logical errors in the application
             status = NV_OK;
@@ -633,24 +631,19 @@ static NV_STATUS service_non_managed_fault(uvm_gpu_va_space_t *gpu_va_space,
     return status;
 }
 
-static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
-                                    uvm_fault_buffer_entry_t *fault_entry,
-                                    const bool hmm_migratable)
+static NV_STATUS service_fault_once(uvm_gpu_t *gpu, uvm_fault_buffer_entry_t *fault_entry, const bool hmm_migratable)
 {
     NV_STATUS status;
     uvm_user_channel_t *user_channel;
     uvm_va_block_t *va_block;
-    uvm_va_space_t *va_space;
+    uvm_va_space_t *va_space = NULL;
     struct mm_struct *mm;
     uvm_gpu_va_space_t *gpu_va_space;
-    uvm_gpu_t *gpu;
-    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &parent_gpu->fault_buffer_info.non_replayable;
-    uvm_va_block_context_t *va_block_context = non_replayable_faults->block_service_context.block_context;
+    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
+    uvm_va_block_context_t *va_block_context =
+        gpu->parent->fault_buffer_info.non_replayable.block_service_context.block_context;
 
-    status = uvm_parent_gpu_fault_entry_to_va_space(parent_gpu,
-                                                    fault_entry,
-                                                    &va_space,
-                                                    &gpu);
+    status = uvm_parent_gpu_fault_entry_to_va_space(gpu->parent, fault_entry, &va_space);
     if (status != NV_OK) {
         // The VA space lookup will fail if we're running concurrently with
         // removal of the channel from the VA space (channel unregister, GPU VA
@@ -664,12 +657,10 @@ static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
         // replayable faults only use the address space of their channel.
         UVM_ASSERT(status == NV_ERR_INVALID_CHANNEL);
         UVM_ASSERT(!va_space);
-        UVM_ASSERT(!gpu);
         return NV_OK;
     }
 
     UVM_ASSERT(va_space);
-    UVM_ASSERT(gpu);
 
     // If an mm is registered with the VA space, we have to retain it
     // in order to lock it before locking the VA space. It is guaranteed
@@ -680,7 +671,8 @@ static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
 
     uvm_va_space_down_read(va_space);
 
-    gpu_va_space = uvm_gpu_va_space_get(va_space, gpu);
+    gpu_va_space = uvm_gpu_va_space_get_by_parent_gpu(va_space, gpu->parent);
+
     if (!gpu_va_space) {
         // The va_space might have gone away. See the comment above.
         status = NV_OK;
@@ -688,7 +680,6 @@ static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
     }
 
     fault_entry->va_space = va_space;
-    fault_entry->gpu = gpu;
 
     user_channel = uvm_gpu_va_space_get_user_channel(gpu_va_space, fault_entry->instance_ptr);
     if (!user_channel) {
@@ -701,25 +692,26 @@ static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
 
     if (!fault_entry->is_fatal) {
         if (mm) {
-            status = uvm_va_block_find_create(va_space,
+            status = uvm_va_block_find_create(fault_entry->va_space,
                                               fault_entry->fault_address,
                                               &va_block_context->hmm.vma,
                                               &va_block);
         }
         else {
-            status = uvm_va_block_find_create_managed(va_space,
+            status = uvm_va_block_find_create_managed(fault_entry->va_space,
                                                       fault_entry->fault_address,
                                                       &va_block);
         }
         if (status == NV_OK)
-            status = service_managed_fault_in_block(va_block, fault_entry, hmm_migratable);
+            status = service_managed_fault_in_block(gpu_va_space->gpu, va_block, fault_entry, hmm_migratable);
         else
             status = service_non_managed_fault(gpu_va_space, mm, fault_entry, status);
 
         // We are done, we clear the faulted bit on the channel, so it can be
         // re-scheduled again
         if (status == NV_OK && !fault_entry->is_fatal) {
-            status = clear_faulted_on_gpu(user_channel,
+            status = clear_faulted_on_gpu(gpu,
+                                          user_channel,
                                           fault_entry,
                                           non_replayable_faults->batch_id,
                                           &non_replayable_faults->fault_service_tracker);
@@ -728,13 +720,13 @@ static NV_STATUS service_fault_once(uvm_parent_gpu_t *parent_gpu,
     }
 
     if (fault_entry->is_fatal)
-        uvm_tools_record_gpu_fatal_fault(gpu->id, va_space, fault_entry, fault_entry->fatal_reason);
+        uvm_tools_record_gpu_fatal_fault(gpu->id, fault_entry->va_space, fault_entry, fault_entry->fatal_reason);
 
     if (fault_entry->is_fatal ||
         (status != NV_OK &&
          status != NV_WARN_MORE_PROCESSING_REQUIRED &&
          status != NV_WARN_MISMATCHED_TARGET))
-        schedule_kill_channel(fault_entry, user_channel);
+        schedule_kill_channel(gpu, fault_entry, user_channel);
 
 exit_no_channel:
     uvm_va_space_up_read(va_space);
@@ -743,23 +735,22 @@ exit_no_channel:
     if (status != NV_OK &&
         status != NV_WARN_MORE_PROCESSING_REQUIRED &&
         status != NV_WARN_MISMATCHED_TARGET)
-        UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n",
-                      uvm_parent_gpu_name(parent_gpu));
+        UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n", uvm_gpu_name(gpu));
 
     return status;
 }
 
-static NV_STATUS service_fault(uvm_parent_gpu_t *parent_gpu, uvm_fault_buffer_entry_t *fault_entry)
+static NV_STATUS service_fault(uvm_gpu_t *gpu, uvm_fault_buffer_entry_t *fault_entry)
 {
     uvm_service_block_context_t *service_context =
-        &parent_gpu->fault_buffer_info.non_replayable.block_service_context;
+        &gpu->parent->fault_buffer_info.non_replayable.block_service_context;
     NV_STATUS status;
     bool hmm_migratable = true;
 
     service_context->num_retries = 0;
 
     do {
-        status = service_fault_once(parent_gpu, fault_entry, hmm_migratable);
+        status = service_fault_once(gpu, fault_entry, hmm_migratable);
         if (status == NV_WARN_MISMATCHED_TARGET) {
             hmm_migratable = false;
             status = NV_WARN_MORE_PROCESSING_REQUIRED;
@@ -769,7 +760,7 @@ static NV_STATUS service_fault(uvm_parent_gpu_t *parent_gpu, uvm_fault_buffer_en
     return status;
 }
 
-void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent_gpu)
+void uvm_gpu_service_non_replayable_fault_buffer(uvm_gpu_t *gpu)
 {
     NvU32 cached_faults;
 
@@ -781,7 +772,7 @@ void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent
         NV_STATUS status;
         NvU32 i;
 
-        status = fetch_non_replayable_fault_buffer_entries(parent_gpu, &cached_faults);
+        status = fetch_non_replayable_fault_buffer_entries(gpu->parent, &cached_faults);
         if (status != NV_OK)
             return;
 
@@ -789,7 +780,7 @@ void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent
         // non-replayable faults since getting multiple faults on the same
         // memory region is not very likely
         for (i = 0; i < cached_faults; ++i) {
-            status = service_fault(parent_gpu, &parent_gpu->fault_buffer_info.non_replayable.fault_cache[i]);
+            status = service_fault(gpu, &gpu->parent->fault_buffer_info.non_replayable.fault_cache[i]);
             if (status != NV_OK)
                 return;
         }

kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.h

@@ -28,7 +28,7 @@
 
 bool uvm_parent_gpu_non_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu);
 
-void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent_gpu);
+void uvm_gpu_service_non_replayable_fault_buffer(uvm_gpu_t *gpu);
 
 NV_STATUS uvm_parent_gpu_fault_buffer_init_non_replayable_faults(uvm_parent_gpu_t *parent_gpu);
 

kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.h

@@ -73,5 +73,5 @@ void uvm_parent_gpu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
 
 // Service pending replayable faults on the given GPU. This function must be
 // only called from the ISR bottom half
-void uvm_parent_gpu_service_replayable_faults(uvm_parent_gpu_t *parent_gpu);
+void uvm_gpu_service_replayable_faults(uvm_gpu_t *gpu);
 #endif // __UVM_GPU_PAGE_FAULT_H__

kernel-open/nvidia-uvm/uvm_gpu_semaphore.c

@@ -507,11 +507,12 @@ uvm_gpu_address_t uvm_gpu_semaphore_get_encrypted_payload_gpu_va(uvm_gpu_semapho
     return uvm_gpu_address_virtual_unprotected(encrypted_base_va + semaphore->index * UVM_SEMAPHORE_SIZE);
 }
 
-NvU32 *uvm_gpu_semaphore_get_notifier_cpu_va(uvm_gpu_semaphore_t *semaphore)
+uvm_gpu_semaphore_notifier_t *uvm_gpu_semaphore_get_notifier_cpu_va(uvm_gpu_semaphore_t *semaphore)
 {
-    char *notifier_base_va = uvm_rm_mem_get_cpu_va(semaphore->page->conf_computing.notifier_memory);
+    uvm_gpu_semaphore_notifier_t *notifier_base_va =
+        uvm_rm_mem_get_cpu_va(semaphore->page->conf_computing.notifier_memory);
 
-    return (NvU32*)(notifier_base_va + semaphore->index * sizeof(NvU32));
+    return notifier_base_va + semaphore->index;
 }
 
 uvm_gpu_address_t uvm_gpu_semaphore_get_notifier_gpu_va(uvm_gpu_semaphore_t *semaphore)
@@ -519,7 +520,8 @@ uvm_gpu_address_t uvm_gpu_semaphore_get_notifier_gpu_va(uvm_gpu_semaphore_t *sem
     NvU64 notifier_base_va = uvm_rm_mem_get_gpu_uvm_va(semaphore->page->conf_computing.notifier_memory,
                                                        semaphore->page->pool->gpu);
 
-    return uvm_gpu_address_virtual_unprotected(notifier_base_va + semaphore->index * sizeof(NvU32));
+    return uvm_gpu_address_virtual_unprotected(notifier_base_va +
+                                               semaphore->index * sizeof(uvm_gpu_semaphore_notifier_t));
 }
 
 void *uvm_gpu_semaphore_get_auth_tag_cpu_va(uvm_gpu_semaphore_t *semaphore)
@@ -583,7 +585,7 @@ static bool tracking_semaphore_check_gpu(uvm_gpu_tracking_semaphore_t *tracking_
     return true;
 }
 
-static bool tracking_semaphore_uses_mutex(uvm_gpu_tracking_semaphore_t *tracking_semaphore)
+bool tracking_semaphore_uses_mutex(uvm_gpu_tracking_semaphore_t *tracking_semaphore)
 {
     UVM_ASSERT(tracking_semaphore_check_gpu(tracking_semaphore));
 
@@ -622,22 +624,11 @@ void uvm_gpu_tracking_semaphore_free(uvm_gpu_tracking_semaphore_t *tracking_sem)
     uvm_gpu_semaphore_free(&tracking_sem->semaphore);
 }
 
-static bool should_skip_secure_semaphore_update(NvU32 last_observed_notifier, NvU32 gpu_notifier)
+static void gpu_semaphore_encrypted_payload_update(uvm_channel_t *channel, uvm_gpu_semaphore_t *semaphore)
 {
-    // No new value, or the GPU is currently writing the new encrypted material
-    // and no change in value would still result in corrupted data.
-    return (last_observed_notifier == gpu_notifier) || (gpu_notifier % 2);
-}
-
-static void uvm_gpu_semaphore_encrypted_payload_update(uvm_channel_t *channel, uvm_gpu_semaphore_t *semaphore)
-{
-    UvmCslIv local_iv;
     NvU32 local_payload;
-    NvU32 new_sem_value;
-    NvU32 gpu_notifier;
-    NvU32 last_observed_notifier;
-    NvU32 new_gpu_notifier = 0;
-    NvU32 iv_index = 0;
+    uvm_gpu_semaphore_notifier_t gpu_notifier;
+    uvm_gpu_semaphore_notifier_t new_gpu_notifier = 0;
 
     // A channel can have multiple entries pending and the tracking semaphore
     // update of each entry can race with this function. Since the semaphore
@@ -646,62 +637,72 @@ static void uvm_gpu_semaphore_encrypted_payload_update(uvm_channel_t *channel, u
     unsigned tries_left = channel->num_gpfifo_entries;
     NV_STATUS status = NV_OK;
     NvU8 local_auth_tag[UVM_CONF_COMPUTING_AUTH_TAG_SIZE];
-    UvmCslIv *ivs_cpu_addr = semaphore->conf_computing.ivs;
-    NvU32 *gpu_notifier_cpu_addr = uvm_gpu_semaphore_get_notifier_cpu_va(semaphore);
+    uvm_gpu_semaphore_notifier_t *semaphore_notifier_cpu_addr = uvm_gpu_semaphore_get_notifier_cpu_va(semaphore);
 
     UVM_ASSERT(g_uvm_global.conf_computing_enabled);
     UVM_ASSERT(uvm_channel_is_ce(channel));
 
-    last_observed_notifier = semaphore->conf_computing.last_observed_notifier;
-    gpu_notifier = UVM_READ_ONCE(*gpu_notifier_cpu_addr);
-    UVM_ASSERT(last_observed_notifier <= gpu_notifier);
-
-    if (should_skip_secure_semaphore_update(last_observed_notifier, gpu_notifier))
-        return;
-
     do {
-        gpu_notifier = UVM_READ_ONCE(*gpu_notifier_cpu_addr);
+        gpu_notifier = UVM_READ_ONCE(*semaphore_notifier_cpu_addr);
+
+        UVM_ASSERT(gpu_notifier >= semaphore->conf_computing.last_observed_notifier);
 
         // Odd notifier value means there's an update in progress.
         if (gpu_notifier % 2)
             continue;
 
+        // There's no change since last time
+        if (gpu_notifier == semaphore->conf_computing.last_observed_notifier)
+            return;
+
         // Make sure no memory accesses happen before we read the notifier
         smp_mb__after_atomic();
 
-        iv_index = (gpu_notifier / 2) % channel->num_gpfifo_entries;
         memcpy(local_auth_tag, uvm_gpu_semaphore_get_auth_tag_cpu_va(semaphore), sizeof(local_auth_tag));
         local_payload = UVM_READ_ONCE(*uvm_gpu_semaphore_get_encrypted_payload_cpu_va(semaphore));
-        memcpy(&local_iv, &ivs_cpu_addr[iv_index], sizeof(local_iv));
 
         // Make sure the second read of notifier happens after
         // all memory accesses.
         smp_mb__before_atomic();
-        new_gpu_notifier = UVM_READ_ONCE(*gpu_notifier_cpu_addr);
+        new_gpu_notifier = UVM_READ_ONCE(*semaphore_notifier_cpu_addr);
         tries_left--;
     } while ((tries_left > 0) && ((gpu_notifier != new_gpu_notifier) || (gpu_notifier % 2)));
 
     if (!tries_left) {
         status = NV_ERR_INVALID_STATE;
-        goto error;
     }
+    else {
+        NvU32 key_version;
+        const NvU32 iv_index = (gpu_notifier / 2) % channel->num_gpfifo_entries;
+        NvU32 new_semaphore_value;
+
+        UVM_ASSERT(gpu_notifier == new_gpu_notifier);
+        UVM_ASSERT(gpu_notifier % 2 == 0);
+
+        // CPU decryption is guaranteed to use the same key version as the
+        // associated GPU encryption, because if there was any key rotation in
+        // between, then key rotation waited for all channels to complete before
+        // proceeding. The wait implies that the semaphore value matches the
+        // last one encrypted on the GPU, so this CPU decryption should happen
+        // before the key is rotated.
+        key_version = uvm_channel_pool_key_version(channel->pool);
 
-    if (gpu_notifier == new_gpu_notifier) {
         status = uvm_conf_computing_cpu_decrypt(channel,
-                                                &new_sem_value,
+                                                &new_semaphore_value,
                                                 &local_payload,
-                                                &local_iv,
-                                                sizeof(new_sem_value),
+                                                &semaphore->conf_computing.ivs[iv_index],
+                                                key_version,
+                                                sizeof(new_semaphore_value),
                                                 &local_auth_tag);
 
         if (status != NV_OK)
             goto error;
 
-        uvm_gpu_semaphore_set_payload(semaphore, new_sem_value);
+        uvm_gpu_semaphore_set_payload(semaphore, new_semaphore_value);
         UVM_WRITE_ONCE(semaphore->conf_computing.last_observed_notifier, new_gpu_notifier);
-    }
 
-    return;
+        return;
+    }
 
 error:
     // Decryption failure is a fatal error as well as running out of try left.
@@ -728,7 +729,7 @@ static NvU64 update_completed_value_locked(uvm_gpu_tracking_semaphore_t *trackin
         // TODO: Bug 4008734: [UVM][HCC] Extend secure tracking semaphore
         //                     mechanism to all semaphore
         uvm_channel_t *channel = container_of(tracking_semaphore, uvm_channel_t, tracking_sem);
-        uvm_gpu_semaphore_encrypted_payload_update(channel, &tracking_semaphore->semaphore);
+        gpu_semaphore_encrypted_payload_update(channel, &tracking_semaphore->semaphore);
     }
 
     new_sem_value = uvm_gpu_semaphore_get_payload(&tracking_semaphore->semaphore);

kernel-open/nvidia-uvm/uvm_gpu_semaphore.h

@@ -29,6 +29,8 @@
 #include "uvm_rm_mem.h"
 #include "uvm_linux.h"
 
+typedef NvU32 uvm_gpu_semaphore_notifier_t;
+
 // A GPU semaphore is a memory location accessible by the GPUs and the CPU
 // that's used for synchronization among them.
 // The GPU has primitives to acquire (wait for) and release (set) 4-byte memory
@@ -52,8 +54,8 @@ struct uvm_gpu_semaphore_struct
         UvmCslIv *ivs;
         NvU32 cached_payload;
 
-        NvU32 last_pushed_notifier;
-        NvU32 last_observed_notifier;
+        uvm_gpu_semaphore_notifier_t last_pushed_notifier;
+        uvm_gpu_semaphore_notifier_t last_observed_notifier;
     } conf_computing;
 };
 
@@ -154,7 +156,7 @@ NvU32 *uvm_gpu_semaphore_get_cpu_va(uvm_gpu_semaphore_t *semaphore);
 NvU32 *uvm_gpu_semaphore_get_encrypted_payload_cpu_va(uvm_gpu_semaphore_t *semaphore);
 uvm_gpu_address_t uvm_gpu_semaphore_get_encrypted_payload_gpu_va(uvm_gpu_semaphore_t *semaphore);
 
-NvU32 *uvm_gpu_semaphore_get_notifier_cpu_va(uvm_gpu_semaphore_t *semaphore);
+uvm_gpu_semaphore_notifier_t *uvm_gpu_semaphore_get_notifier_cpu_va(uvm_gpu_semaphore_t *semaphore);
 uvm_gpu_address_t uvm_gpu_semaphore_get_notifier_gpu_va(uvm_gpu_semaphore_t *semaphore);
 
 void *uvm_gpu_semaphore_get_auth_tag_cpu_va(uvm_gpu_semaphore_t *semaphore);

kernel-open/nvidia-uvm/uvm_hal.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2024 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -44,8 +44,6 @@
 #include "clc7b5.h"
 #include "clc86f.h"
 #include "clc8b5.h"
-#include "clc96f.h"
-#include "clc9b5.h"
 
 static int uvm_downgrade_force_membar_sys = 1;
 module_param(uvm_downgrade_force_membar_sys, uint, 0644);
@@ -166,11 +164,6 @@ static uvm_hal_class_ops_t ce_table[] =
             .decrypt = uvm_hal_hopper_ce_decrypt,
         },
     },
-    {
-        .id = BLACKWELL_DMA_COPY_A,
-        .parent_id = HOPPER_DMA_COPY_A,
-        .u.ce_ops = {},
-    },
 };
 
 // Table for GPFIFO functions.  Same idea as the copy engine table.
@@ -258,9 +251,6 @@ static uvm_hal_class_ops_t host_table[] =
             .semaphore_release = uvm_hal_turing_host_semaphore_release,
             .clear_faulted_channel_method = uvm_hal_turing_host_clear_faulted_channel_method,
             .set_gpfifo_entry = uvm_hal_turing_host_set_gpfifo_entry,
-            .tlb_invalidate_all = uvm_hal_turing_host_tlb_invalidate_all,
-            .tlb_invalidate_va = uvm_hal_turing_host_tlb_invalidate_va,
-            .tlb_invalidate_test = uvm_hal_turing_host_tlb_invalidate_test,
         }
     },
     {
@@ -293,15 +283,6 @@ static uvm_hal_class_ops_t host_table[] =
             .set_gpfifo_pushbuffer_segment_base = uvm_hal_hopper_host_set_gpfifo_pushbuffer_segment_base,
         }
     },
-    {
-        .id = BLACKWELL_CHANNEL_GPFIFO_A,
-        .parent_id = HOPPER_CHANNEL_GPFIFO_A,
-        .u.host_ops = {
-            .tlb_invalidate_all = uvm_hal_blackwell_host_tlb_invalidate_all,
-            .tlb_invalidate_va = uvm_hal_blackwell_host_tlb_invalidate_va,
-            .tlb_invalidate_test = uvm_hal_blackwell_host_tlb_invalidate_test,
-        }
-    },
 };
 
 static uvm_hal_class_ops_t arch_table[] =
@@ -313,6 +294,7 @@ static uvm_hal_class_ops_t arch_table[] =
             .mmu_mode_hal = uvm_hal_mmu_mode_maxwell,
             .enable_prefetch_faults = uvm_hal_maxwell_mmu_enable_prefetch_faults_unsupported,
             .disable_prefetch_faults = uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported,
+            .mmu_engine_id_to_type = uvm_hal_maxwell_mmu_engine_id_to_type_unsupported,
             .mmu_client_id_to_utlb_id = uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported,
         }
     },
@@ -338,6 +320,7 @@ static uvm_hal_class_ops_t arch_table[] =
         .u.arch_ops = {
             .init_properties = uvm_hal_volta_arch_init_properties,
             .mmu_mode_hal = uvm_hal_mmu_mode_volta,
+            .mmu_engine_id_to_type = uvm_hal_volta_mmu_engine_id_to_type,
             .mmu_client_id_to_utlb_id = uvm_hal_volta_mmu_client_id_to_utlb_id,
         },
     },
@@ -347,6 +330,7 @@ static uvm_hal_class_ops_t arch_table[] =
         .u.arch_ops = {
             .init_properties = uvm_hal_turing_arch_init_properties,
             .mmu_mode_hal = uvm_hal_mmu_mode_turing,
+            .mmu_engine_id_to_type = uvm_hal_turing_mmu_engine_id_to_type,
         },
     },
     {
@@ -355,6 +339,7 @@ static uvm_hal_class_ops_t arch_table[] =
         .u.arch_ops = {
             .init_properties = uvm_hal_ampere_arch_init_properties,
             .mmu_mode_hal = uvm_hal_mmu_mode_ampere,
+            .mmu_engine_id_to_type = uvm_hal_ampere_mmu_engine_id_to_type,
             .mmu_client_id_to_utlb_id = uvm_hal_ampere_mmu_client_id_to_utlb_id,
         },
     },
@@ -371,18 +356,10 @@ static uvm_hal_class_ops_t arch_table[] =
         .u.arch_ops = {
             .init_properties = uvm_hal_hopper_arch_init_properties,
             .mmu_mode_hal = uvm_hal_mmu_mode_hopper,
+            .mmu_engine_id_to_type = uvm_hal_hopper_mmu_engine_id_to_type,
             .mmu_client_id_to_utlb_id = uvm_hal_hopper_mmu_client_id_to_utlb_id,
         },
     },
-    {
-        .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100,
-        .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100,
-        .u.arch_ops = {
-            .init_properties = uvm_hal_blackwell_arch_init_properties,
-            .mmu_mode_hal = uvm_hal_mmu_mode_blackwell,
-            .mmu_client_id_to_utlb_id = uvm_hal_blackwell_mmu_client_id_to_utlb_id,
-        }
-    },
 };
 
 static uvm_hal_class_ops_t fault_buffer_table[] =
@@ -397,7 +374,6 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
             .read_get = uvm_hal_maxwell_fault_buffer_read_get_unsupported,
             .write_get = uvm_hal_maxwell_fault_buffer_write_get_unsupported,
             .get_ve_id = uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported,
-            .get_mmu_engine_type = uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported,
             .parse_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported,
             .entry_is_valid = uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported,
             .entry_clear_valid = uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported,
@@ -436,7 +412,6 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
             .read_get = uvm_hal_volta_fault_buffer_read_get,
             .write_get = uvm_hal_volta_fault_buffer_write_get,
             .get_ve_id = uvm_hal_volta_fault_buffer_get_ve_id,
-            .get_mmu_engine_type = uvm_hal_volta_fault_buffer_get_mmu_engine_type,
             .parse_replayable_entry = uvm_hal_volta_fault_buffer_parse_replayable_entry,
             .parse_non_replayable_entry = uvm_hal_volta_fault_buffer_parse_non_replayable_entry,
             .get_fault_type = uvm_hal_volta_fault_buffer_get_fault_type,
@@ -448,15 +423,12 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
         .u.fault_buffer_ops = {
             .disable_replayable_faults = uvm_hal_turing_disable_replayable_faults,
             .clear_replayable_faults = uvm_hal_turing_clear_replayable_faults,
-            .get_mmu_engine_type = uvm_hal_turing_fault_buffer_get_mmu_engine_type,
         }
     },
     {
         .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100,
         .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
-        .u.fault_buffer_ops = {
-            .get_mmu_engine_type = uvm_hal_ampere_fault_buffer_get_mmu_engine_type,
-        }
+        .u.fault_buffer_ops = {}
     },
     {
         .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100,
@@ -468,15 +440,6 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
         .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100,
         .u.fault_buffer_ops = {
             .get_ve_id = uvm_hal_hopper_fault_buffer_get_ve_id,
-            .get_mmu_engine_type = uvm_hal_hopper_fault_buffer_get_mmu_engine_type,
-        }
-    },
-    {
-        .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100,
-        .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100,
-        .u.fault_buffer_ops = {
-            .get_fault_type = uvm_hal_blackwell_fault_buffer_get_fault_type,
-            .get_mmu_engine_type = uvm_hal_blackwell_fault_buffer_get_mmu_engine_type,
         }
     },
 };
@@ -541,11 +504,6 @@ static uvm_hal_class_ops_t access_counter_buffer_table[] =
         .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_AD100,
         .u.access_counter_buffer_ops = {}
     },
-    {
-        .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100,
-        .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100,
-        .u.access_counter_buffer_ops = {}
-    },
 };
 
 static uvm_hal_class_ops_t sec2_table[] =
@@ -599,11 +557,6 @@ static uvm_hal_class_ops_t sec2_table[] =
             .decrypt = uvm_hal_hopper_sec2_decrypt,
         }
     },
-    {
-        .id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100,
-        .parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GH100,
-        .u.sec2_ops = {}
-    },
 };
 
 static inline uvm_hal_class_ops_t *ops_find_by_id(uvm_hal_class_ops_t *table, NvU32 row_count, NvU32 id)
@@ -679,19 +632,13 @@ NV_STATUS uvm_hal_init_table(void)
         return status;
     }
 
-    status = ops_init_from_parent(host_table,
-                                  ARRAY_SIZE(host_table),
-                                  HOST_OP_COUNT,
-                                  offsetof(uvm_hal_class_ops_t, u.host_ops));
+    status = ops_init_from_parent(host_table, ARRAY_SIZE(host_table), HOST_OP_COUNT, offsetof(uvm_hal_class_ops_t, u.host_ops));
     if (status != NV_OK) {
         UVM_ERR_PRINT("ops_init_from_parent(host_table) failed: %s\n", nvstatusToString(status));
         return status;
     }
 
-    status = ops_init_from_parent(arch_table,
-                                  ARRAY_SIZE(arch_table),
-                                  ARCH_OP_COUNT,
-                                  offsetof(uvm_hal_class_ops_t, u.arch_ops));
+    status = ops_init_from_parent(arch_table, ARRAY_SIZE(arch_table), ARCH_OP_COUNT, offsetof(uvm_hal_class_ops_t, u.arch_ops));
     if (status != NV_OK) {
         UVM_ERR_PRINT("ops_init_from_parent(arch_table) failed: %s\n", nvstatusToString(status));
         return status;
@@ -831,9 +778,6 @@ void uvm_hal_tlb_invalidate_membar(uvm_push_t *push, uvm_membar_t membar)
 
     gpu = uvm_push_get_gpu(push);
 
-    // TLB invalidate on Blackwell+ GPUs should not use a standalone membar.
-    UVM_ASSERT(gpu->parent->rm_info.gpuArch < NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100);
-
     for (i = 0; i < gpu->parent->num_hshub_tlb_invalidate_membars; i++)
         gpu->parent->host_hal->membar_gpu(push);
 
@@ -939,7 +883,7 @@ const char *uvm_fault_access_type_string(uvm_fault_access_type_t fault_access_ty
 
 const char *uvm_fault_type_string(uvm_fault_type_t fault_type)
 {
-    BUILD_BUG_ON(UVM_FAULT_TYPE_COUNT != 17);
+    BUILD_BUG_ON(UVM_FAULT_TYPE_COUNT != 16);
 
     switch (fault_type) {
         UVM_ENUM_STRING_CASE(UVM_FAULT_TYPE_INVALID_PDE);
@@ -958,7 +902,6 @@ const char *uvm_fault_type_string(uvm_fault_type_t fault_type)
         UVM_ENUM_STRING_CASE(UVM_FAULT_TYPE_UNSUPPORTED_KIND);
         UVM_ENUM_STRING_CASE(UVM_FAULT_TYPE_REGION_VIOLATION);
         UVM_ENUM_STRING_CASE(UVM_FAULT_TYPE_POISONED);
-        UVM_ENUM_STRING_CASE(UVM_FAULT_TYPE_CC_VIOLATION);
         UVM_ENUM_STRING_DEFAULT();
     }
 }
@@ -989,16 +932,14 @@ const char *uvm_mmu_engine_type_string(uvm_mmu_engine_type_t mmu_engine_type)
 void uvm_hal_print_fault_entry(const uvm_fault_buffer_entry_t *entry)
 {
     UVM_DBG_PRINT("fault_address:                    0x%llx\n", entry->fault_address);
-    UVM_DBG_PRINT("    fault_instance_ptr:           {0x%llx:%s}\n",
-                  entry->instance_ptr.address,
-                  uvm_aperture_string(entry->instance_ptr.aperture));
+    UVM_DBG_PRINT("    fault_instance_ptr:           {0x%llx:%s}\n", entry->instance_ptr.address,
+                                                                     uvm_aperture_string(entry->instance_ptr.aperture));
     UVM_DBG_PRINT("    fault_type:                   %s\n", uvm_fault_type_string(entry->fault_type));
     UVM_DBG_PRINT("    fault_access_type:            %s\n", uvm_fault_access_type_string(entry->fault_access_type));
     UVM_DBG_PRINT("    is_replayable:                %s\n", entry->is_replayable? "true": "false");
     UVM_DBG_PRINT("    is_virtual:                   %s\n", entry->is_virtual? "true": "false");
     UVM_DBG_PRINT("    in_protected_mode:            %s\n", entry->in_protected_mode? "true": "false");
-    UVM_DBG_PRINT("    fault_source.client_type:     %s\n",
-                  uvm_fault_client_type_string(entry->fault_source.client_type));
+    UVM_DBG_PRINT("    fault_source.client_type:     %s\n", uvm_fault_client_type_string(entry->fault_source.client_type));
     UVM_DBG_PRINT("    fault_source.client_id:       %d\n", entry->fault_source.client_id);
     UVM_DBG_PRINT("    fault_source.gpc_id:          %d\n", entry->fault_source.gpc_id);
     UVM_DBG_PRINT("    fault_source.mmu_engine_id:   %d\n", entry->fault_source.mmu_engine_id);
@@ -1021,15 +962,13 @@ const char *uvm_access_counter_type_string(uvm_access_counter_type_t access_coun
 void uvm_hal_print_access_counter_buffer_entry(const uvm_access_counter_buffer_entry_t *entry)
 {
     if (!entry->address.is_virtual) {
-        UVM_DBG_PRINT("physical address: {0x%llx:%s}\n",
-                      entry->address.address,
-                      uvm_aperture_string(entry->address.aperture));
+        UVM_DBG_PRINT("physical address: {0x%llx:%s}\n", entry->address.address,
+                                                         uvm_aperture_string(entry->address.aperture));
     }
     else {
         UVM_DBG_PRINT("virtual address: 0x%llx\n", entry->address.address);
-        UVM_DBG_PRINT("    instance_ptr    {0x%llx:%s}\n",
-                      entry->virtual_info.instance_ptr.address,
-                      uvm_aperture_string(entry->virtual_info.instance_ptr.aperture));
+        UVM_DBG_PRINT("    instance_ptr    {0x%llx:%s}\n", entry->virtual_info.instance_ptr.address,
+                                                    uvm_aperture_string(entry->virtual_info.instance_ptr.aperture));
         UVM_DBG_PRINT("    mmu_engine_type %s\n", uvm_mmu_engine_type_string(entry->virtual_info.mmu_engine_type));
         UVM_DBG_PRINT("    mmu_engine_id   %u\n", entry->virtual_info.mmu_engine_id);
         UVM_DBG_PRINT("    ve_id           %u\n", entry->virtual_info.ve_id);

kernel-open/nvidia-uvm/uvm_hal.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2024 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -112,10 +112,6 @@ void uvm_hal_pascal_host_tlb_invalidate_all(uvm_push_t *push,
                                             uvm_gpu_phys_address_t pdb,
                                             NvU32 depth,
                                             uvm_membar_t membar);
-void uvm_hal_turing_host_tlb_invalidate_all(uvm_push_t *push,
-                                            uvm_gpu_phys_address_t pdb,
-                                            NvU32 depth,
-                                            uvm_membar_t membar);
 void uvm_hal_ampere_host_tlb_invalidate_all(uvm_push_t *push,
                                             uvm_gpu_phys_address_t pdb,
                                             NvU32 depth,
@@ -124,10 +120,6 @@ void uvm_hal_hopper_host_tlb_invalidate_all(uvm_push_t *push,
                                             uvm_gpu_phys_address_t pdb,
                                             NvU32 depth,
                                             uvm_membar_t membar);
-void uvm_hal_blackwell_host_tlb_invalidate_all(uvm_push_t *push,
-                                               uvm_gpu_phys_address_t pdb,
-                                               NvU32 depth,
-                                               uvm_membar_t membar);
 
 // Issue a TLB invalidate applying to the specified VA range in a PDB.
 //
@@ -157,57 +149,43 @@ typedef void (*uvm_hal_host_tlb_invalidate_va_t)(uvm_push_t *push,
                                                  NvU32 depth,
                                                  NvU64 base,
                                                  NvU64 size,
-                                                 NvU64 page_size,
+                                                 NvU32 page_size,
                                                  uvm_membar_t membar);
 void uvm_hal_maxwell_host_tlb_invalidate_va(uvm_push_t *push,
                                             uvm_gpu_phys_address_t pdb,
                                             NvU32 depth,
                                             NvU64 base,
                                             NvU64 size,
-                                            NvU64 page_size,
+                                            NvU32 page_size,
                                             uvm_membar_t membar);
 void uvm_hal_pascal_host_tlb_invalidate_va(uvm_push_t *push,
                                            uvm_gpu_phys_address_t pdb,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU64 page_size,
+                                           NvU32 page_size,
                                            uvm_membar_t membar);
 void uvm_hal_volta_host_tlb_invalidate_va(uvm_push_t *push,
                                           uvm_gpu_phys_address_t pdb,
                                           NvU32 depth,
                                           NvU64 base,
                                           NvU64 size,
-                                          NvU64 page_size,
+                                          NvU32 page_size,
                                           uvm_membar_t membar);
-void uvm_hal_turing_host_tlb_invalidate_va(uvm_push_t *push,
-                                           uvm_gpu_phys_address_t pdb,
-                                           NvU32 depth,
-                                           NvU64 base,
-                                           NvU64 size,
-                                           NvU64 page_size,
-                                           uvm_membar_t membar);
 void uvm_hal_ampere_host_tlb_invalidate_va(uvm_push_t *push,
                                            uvm_gpu_phys_address_t pdb,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU64 page_size,
+                                           NvU32 page_size,
                                            uvm_membar_t membar);
 void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
                                            uvm_gpu_phys_address_t pdb,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU64 page_size,
+                                           NvU32 page_size,
                                            uvm_membar_t membar);
-void uvm_hal_blackwell_host_tlb_invalidate_va(uvm_push_t *push,
-                                              uvm_gpu_phys_address_t pdb,
-                                              NvU32 depth,
-                                              NvU64 base,
-                                              NvU64 size,
-                                              NvU64 page_size,
-                                              uvm_membar_t membar);
 
 typedef void (*uvm_hal_host_tlb_invalidate_test_t)(uvm_push_t *push,
                                                    uvm_gpu_phys_address_t pdb,
@@ -218,18 +196,12 @@ void uvm_hal_maxwell_host_tlb_invalidate_test(uvm_push_t *push,
 void uvm_hal_pascal_host_tlb_invalidate_test(uvm_push_t *push,
                                              uvm_gpu_phys_address_t pdb,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params);
-void uvm_hal_turing_host_tlb_invalidate_test(uvm_push_t *push,
-                                             uvm_gpu_phys_address_t pdb,
-                                             UVM_TEST_INVALIDATE_TLB_PARAMS *params);
 void uvm_hal_ampere_host_tlb_invalidate_test(uvm_push_t *push,
                                              uvm_gpu_phys_address_t pdb,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params);
 void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
                                              uvm_gpu_phys_address_t pdb,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params);
-void uvm_hal_blackwell_host_tlb_invalidate_test(uvm_push_t *push,
-                                                uvm_gpu_phys_address_t pdb,
-                                                UVM_TEST_INVALIDATE_TLB_PARAMS *params);
 
 // By default all semaphore release operations include a membar sys before the
 // operation. This can be affected by using UVM_PUSH_FLAG_NEXT_* flags with
@@ -471,31 +443,38 @@ void uvm_hal_turing_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
-void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
 
 // Retrieve the page-tree HAL for a given big page size
-typedef uvm_mmu_mode_hal_t *(*uvm_hal_lookup_mode_hal_t)(NvU64 big_page_size);
+typedef uvm_mmu_mode_hal_t *(*uvm_hal_lookup_mode_hal_t)(NvU32 big_page_size);
 typedef void (*uvm_hal_mmu_enable_prefetch_faults_t)(uvm_parent_gpu_t *parent_gpu);
 typedef void (*uvm_hal_mmu_disable_prefetch_faults_t)(uvm_parent_gpu_t *parent_gpu);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_pascal(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_volta(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_turing(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU64 big_page_size);
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU32 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_pascal(NvU32 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_volta(NvU32 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_turing(NvU32 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU32 big_page_size);
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU32 big_page_size);
 void uvm_hal_maxwell_mmu_enable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_pascal_mmu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_pascal_mmu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
 
+// Convert a faulted MMU engine ID to a UVM engine type. Only engines which have
+// faults serviced by UVM are handled. On Pascal the only such engine is
+// GRAPHICS, so no translation is provided.
+typedef uvm_mmu_engine_type_t (*uvm_hal_mmu_engine_id_to_type_t)(NvU16 mmu_engine_id);
+uvm_mmu_engine_type_t uvm_hal_maxwell_mmu_engine_id_to_type_unsupported(NvU16 mmu_engine_id);
+uvm_mmu_engine_type_t uvm_hal_volta_mmu_engine_id_to_type(NvU16 mmu_engine_id);
+uvm_mmu_engine_type_t uvm_hal_turing_mmu_engine_id_to_type(NvU16 mmu_engine_id);
+uvm_mmu_engine_type_t uvm_hal_ampere_mmu_engine_id_to_type(NvU16 mmu_engine_id);
+uvm_mmu_engine_type_t uvm_hal_hopper_mmu_engine_id_to_type(NvU16 mmu_engine_id);
+
 typedef NvU16 (*uvm_hal_mmu_client_id_to_utlb_id_t)(NvU16 client_id);
 NvU16 uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported(NvU16 client_id);
 NvU16 uvm_hal_pascal_mmu_client_id_to_utlb_id(NvU16 client_id);
 NvU16 uvm_hal_volta_mmu_client_id_to_utlb_id(NvU16 client_id);
 NvU16 uvm_hal_ampere_mmu_client_id_to_utlb_id(NvU16 client_id);
 NvU16 uvm_hal_hopper_mmu_client_id_to_utlb_id(NvU16 client_id);
-NvU16 uvm_hal_blackwell_mmu_client_id_to_utlb_id(NvU16 client_id);
 
 // Replayable faults
 typedef void (*uvm_hal_enable_replayable_faults_t)(uvm_parent_gpu_t *parent_gpu);
@@ -505,9 +484,6 @@ typedef NvU32 (*uvm_hal_fault_buffer_read_put_t)(uvm_parent_gpu_t *parent_gpu);
 typedef NvU32 (*uvm_hal_fault_buffer_read_get_t)(uvm_parent_gpu_t *parent_gpu);
 typedef void (*uvm_hal_fault_buffer_write_get_t)(uvm_parent_gpu_t *parent_gpu, NvU32 get);
 typedef NvU8 (*uvm_hal_fault_buffer_get_ve_id_t)(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-typedef uvm_mmu_engine_type_t (*uvm_hal_fault_buffer_get_mmu_engine_type_t)(NvU16 mmu_engine_id,
-                                                                            uvm_fault_client_type_t client_type,
-                                                                            NvU16 client_id);
 
 // Parse the replayable entry at the given buffer index. This also clears the
 // valid bit of the entry in the buffer.
@@ -545,9 +521,6 @@ NvU32 uvm_hal_maxwell_fault_buffer_read_put_unsupported(uvm_parent_gpu_t *parent
 NvU32 uvm_hal_maxwell_fault_buffer_read_get_unsupported(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_maxwell_fault_buffer_write_get_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-uvm_mmu_engine_type_t uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported(NvU16 mmu_engine_id,
-                                                                                   uvm_fault_client_type_t client_type,
-                                                                                   NvU16 client_id);
 uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry);
 
 void uvm_hal_pascal_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
@@ -563,31 +536,12 @@ NvU32 uvm_hal_volta_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
 NvU32 uvm_hal_volta_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_volta_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 NvU8 uvm_hal_volta_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-uvm_mmu_engine_type_t uvm_hal_volta_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                     uvm_fault_client_type_t client_type,
-                                                                     NvU16 client_id);
 
 uvm_fault_type_t uvm_hal_volta_fault_buffer_get_fault_type(const NvU32 *fault_entry);
 
 void uvm_hal_turing_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_turing_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32 get);
-uvm_mmu_engine_type_t uvm_hal_turing_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                      uvm_fault_client_type_t client_type,
-                                                                      NvU16 client_id);
-
-uvm_mmu_engine_type_t uvm_hal_ampere_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                      uvm_fault_client_type_t client_type,
-                                                                      NvU16 client_id);
-
 NvU8 uvm_hal_hopper_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-uvm_mmu_engine_type_t uvm_hal_hopper_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                      uvm_fault_client_type_t client_type,
-                                                                      NvU16 client_id);
-
-uvm_mmu_engine_type_t uvm_hal_blackwell_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                         uvm_fault_client_type_t client_type,
-                                                                         NvU16 client_id);
-uvm_fault_type_t uvm_hal_blackwell_fault_buffer_get_fault_type(const NvU32 *fault_entry);
 
 bool uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 void uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
@@ -811,6 +765,7 @@ struct uvm_arch_hal_struct
     uvm_hal_lookup_mode_hal_t mmu_mode_hal;
     uvm_hal_mmu_enable_prefetch_faults_t enable_prefetch_faults;
     uvm_hal_mmu_disable_prefetch_faults_t disable_prefetch_faults;
+    uvm_hal_mmu_engine_id_to_type_t mmu_engine_id_to_type;
     uvm_hal_mmu_client_id_to_utlb_id_t mmu_client_id_to_utlb_id;
 };
 
@@ -823,7 +778,6 @@ struct uvm_fault_buffer_hal_struct
     uvm_hal_fault_buffer_read_get_t read_get;
     uvm_hal_fault_buffer_write_get_t write_get;
     uvm_hal_fault_buffer_get_ve_id_t get_ve_id;
-    uvm_hal_fault_buffer_get_mmu_engine_type_t get_mmu_engine_type;
     uvm_hal_fault_buffer_parse_replayable_entry_t parse_replayable_entry;
     uvm_hal_fault_buffer_entry_is_valid_t entry_is_valid;
     uvm_hal_fault_buffer_entry_clear_valid_t entry_clear_valid;

kernel-open/nvidia-uvm/uvm_hal_types.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -300,7 +300,6 @@ typedef enum
     UVM_FAULT_TYPE_UNSUPPORTED_KIND,
     UVM_FAULT_TYPE_REGION_VIOLATION,
     UVM_FAULT_TYPE_POISONED,
-    UVM_FAULT_TYPE_CC_VIOLATION,
 
     UVM_FAULT_TYPE_COUNT
 } uvm_fault_type_t;
@@ -400,7 +399,6 @@ struct uvm_fault_buffer_entry_struct
     //
 
     uvm_va_space_t                           *va_space;
-    uvm_gpu_t                                     *gpu;
 
     // This is set to true when some fault could not be serviced and a
     // cancel command needs to be issued
@@ -492,9 +490,9 @@ struct uvm_access_counter_buffer_entry_struct
     // Address of the region for which a notification was sent
     uvm_gpu_address_t address;
 
+    // These fields are only valid if address.is_virtual is true
     union
     {
-        // These fields are only valid if address.is_virtual is true
         struct
         {
             // Instance pointer of one of the channels in the TSG that triggered
@@ -524,14 +522,9 @@ struct uvm_access_counter_buffer_entry_struct
             // a regular processor id because P2P is not allowed between
             // partitioned GPUs.
             uvm_processor_id_t resident_id;
-
         } physical_info;
     };
 
-    // This is the GPU that triggered the notification. Note that physical
-    // address based notifications are only supported on non-MIG-capable GPUs.
-    uvm_gpu_t *gpu;
-
     // Number of times the tracked region was accessed since the last time it
     // was cleared. Counter values saturate at the maximum value supported by
     // the GPU (2^16 - 1 in Volta)

kernel-open/nvidia-uvm/uvm_hmm.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -73,6 +73,24 @@ module_param(uvm_disable_hmm, bool, 0444);
 #include "uvm_va_policy.h"
 #include "uvm_tools.h"
 
+// The function nv_PageSwapCache() wraps the check for page swap cache flag in
+// order to support a wide variety of kernel versions.
+// The function PageSwapCache() is removed after 32f51ead3d77 ("mm: remove
+// PageSwapCache") in v6.12-rc1.
+// The function folio_test_swapcache() was added in Linux 5.16 (d389a4a811551
+// "mm: Add folio flag manipulation functions")
+// Systems with HMM patches backported to 5.14 are possible, but those systems
+// do not include folio_test_swapcache()
+// TODO: Bug 4050579: Remove this when migration of swap cached pages is updated
+static __always_inline bool nv_PageSwapCache(struct page *page)
+{
+#if defined(NV_FOLIO_TEST_SWAPCACHE_PRESENT)
+    return folio_test_swapcache(page_folio(page));
+#else
+    return PageSwapCache(page);
+#endif
+}
+
 static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block,
                                uvm_page_index_t page_index,
                                struct page *page);
@@ -145,7 +163,7 @@ static uvm_va_block_t *hmm_va_block_from_node(uvm_range_tree_node_t *node)
 // Copies the contents of the source device-private page to the
 // destination CPU page. This will invalidate mappings, so cannot be
 // called while holding any va_block locks.
-static void hmm_copy_devmem_page(struct page *dst_page, struct page *src_page)
+static NV_STATUS hmm_copy_devmem_page(struct page *dst_page, struct page *src_page)
 {
     uvm_tracker_t tracker = UVM_TRACKER_INIT();
     uvm_gpu_phys_address_t src_addr;
@@ -166,7 +184,7 @@ static void hmm_copy_devmem_page(struct page *dst_page, struct page *src_page)
     gpu = uvm_gpu_chunk_get_gpu(gpu_chunk);
     status = uvm_mmu_chunk_map(gpu_chunk);
     if (status != NV_OK)
-        goto out_zero;
+        goto out;
 
     status = uvm_parent_gpu_map_cpu_pages(gpu->parent, dst_page, PAGE_SIZE, &dma_addr);
     if (status != NV_OK)
@@ -189,7 +207,7 @@ static void hmm_copy_devmem_page(struct page *dst_page, struct page *src_page)
     uvm_push_end(&push);
     status = uvm_tracker_add_push_safe(&tracker, &push);
     if (status == NV_OK)
-        uvm_tracker_wait_deinit(&tracker);
+        status = uvm_tracker_wait_deinit(&tracker);
 
 out_unmap_cpu:
     uvm_parent_gpu_unmap_cpu_pages(gpu->parent, dma_addr, PAGE_SIZE);
@@ -197,13 +215,8 @@ out_unmap_cpu:
 out_unmap_gpu:
     uvm_mmu_chunk_unmap(gpu_chunk, NULL);
 
-out_zero:
-    // We can't fail eviction because we need to free the device-private pages
-    // so the GPU can be unregistered. So the best we can do is warn on any
-    // failures and zero the uninitialised page. This could result in data loss
-    // in the application but failures are not expected.
-    if (WARN_ON(status != NV_OK))
-        memzero_page(dst_page, 0, PAGE_SIZE);
+out:
+    return status;
 }
 
 static NV_STATUS uvm_hmm_pmm_gpu_evict_pfn(unsigned long pfn)
@@ -227,7 +240,13 @@ static NV_STATUS uvm_hmm_pmm_gpu_evict_pfn(unsigned long pfn)
         }
 
         lock_page(dst_page);
-        hmm_copy_devmem_page(dst_page, migrate_pfn_to_page(src_pfn));
+
+        // We can't fail eviction because we need to free the device-private
+        // pages so the GPU can be unregistered. So the best we can do is warn
+        // on any failures and zero the uninitialized page. This could result
+        // in data loss in the application but failures are not expected.
+        if (hmm_copy_devmem_page(dst_page, migrate_pfn_to_page(src_pfn)) != NV_OK)
+            memzero_page(dst_page, 0, PAGE_SIZE);
         dst_pfn = migrate_pfn(page_to_pfn(dst_page));
         migrate_device_pages(&src_pfn, &dst_pfn, 1);
     }
@@ -1601,7 +1620,7 @@ static void hmm_va_block_cpu_unpopulate_chunk(uvm_va_block_t *va_block,
     UVM_ASSERT(uvm_cpu_chunk_get_size(chunk) == PAGE_SIZE);
 
     uvm_cpu_chunk_remove_from_block(va_block, chunk_nid, page_index);
-    uvm_va_block_unmap_cpu_chunk_on_gpus(va_block, chunk);
+    uvm_va_block_unmap_cpu_chunk_on_gpus(va_block, chunk, page_index);
     uvm_cpu_chunk_free(chunk);
 }
 
@@ -1706,6 +1725,8 @@ static void gpu_chunk_remove(uvm_va_block_t *va_block,
         return;
     }
 
+    // TODO: Bug 3898467: unmap indirect peers when freeing GPU chunks
+
     uvm_mmu_chunk_unmap(gpu_chunk, &va_block->tracker);
     gpu_state->chunks[page_index] = NULL;
 }
@@ -1754,6 +1775,8 @@ static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block,
     if (status != NV_OK)
         return status;
 
+    // TODO: Bug 3898467: map indirect peers.
+
     uvm_processor_mask_set(&va_block->resident, id);
     uvm_page_mask_set(&gpu_state->resident, page_index);
 
@@ -2274,7 +2297,7 @@ static NV_STATUS populate_region(uvm_va_block_t *va_block,
             // uvm_hmm_invalidate() should handle that if the underlying page
             // is invalidated.
             // Also note there can be an allocated page due to GPU-to-GPU
-            // migration between non-peer GPUs.
+            // migration between non-peer or indirect peer GPUs.
             continue;
         }
 
@@ -2694,7 +2717,7 @@ static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block,
                 continue;
             }
 
-            if (PageSwapCache(src_page)) {
+            if (nv_PageSwapCache(src_page)) {
                 // TODO: Bug 4050579: Remove this when swap cached pages can be
                 // migrated.
                 status = NV_WARN_MISMATCHED_TARGET;
@@ -3464,12 +3487,17 @@ NV_STATUS uvm_hmm_remote_cpu_fault(struct vm_fault *vmf)
         lock_page(dst_page);
         dst_pfn = migrate_pfn(page_to_pfn(dst_page));
 
-        hmm_copy_devmem_page(dst_page, src_page);
+        status = hmm_copy_devmem_page(dst_page, src_page);
+        if (status != NV_OK) {
+            unlock_page(dst_page);
+            __free_page(dst_page);
+            dst_pfn = 0;
+        }
     }
 
-    migrate_vma_pages(&args);
-
 out:
+    if (status == NV_OK)
+        migrate_vma_pages(&args);
     migrate_vma_finalize(&args);
 
     return status;

kernel-open/nvidia-uvm/uvm_hopper_fault_buffer.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2020-2024 NVIDIA Corporation
+    Copyright (c) 2020 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -21,7 +21,6 @@
 
 *******************************************************************************/
 
-#include "uvm_hal.h"
 #include "uvm_hal_types.h"
 #include "hwref/hopper/gh100/dev_fault.h"
 
@@ -41,49 +40,3 @@ NvU8 uvm_hal_hopper_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_t
         return 0;
     }
 }
-
-static bool client_id_ce(NvU16 client_id)
-{
-    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE0 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE9)
-        return true;
-
-    if (client_id >= NV_PFAULT_CLIENT_HUB_HSCE10 && client_id <= NV_PFAULT_CLIENT_HUB_HSCE15)
-        return true;
-
-    switch (client_id) {
-        case NV_PFAULT_CLIENT_HUB_CE0:
-        case NV_PFAULT_CLIENT_HUB_CE1:
-        case NV_PFAULT_CLIENT_HUB_CE2:
-        case NV_PFAULT_CLIENT_HUB_CE3:
-            return true;
-    }
-
-    return false;
-}
-
-uvm_mmu_engine_type_t uvm_hal_hopper_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
-                                                                      uvm_fault_client_type_t client_type,
-                                                                      NvU16 client_id)
-{
-    // Servicing CE and Host (HUB clients) faults.
-    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB) {
-        if (client_id_ce(client_id)) {
-            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9);
-
-            return UVM_MMU_ENGINE_TYPE_CE;
-        }
-
-        if (client_id == NV_PFAULT_CLIENT_HUB_HOST || client_id == NV_PFAULT_CLIENT_HUB_ESC) {
-            UVM_ASSERT(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST44);
-
-            return UVM_MMU_ENGINE_TYPE_HOST;
-        }
-    }
-
-    // We shouldn't be servicing faults from any other engines other than GR.
-    UVM_ASSERT_MSG(client_id <= NV_PFAULT_CLIENT_GPC_ROP_3, "Unexpected client ID: 0x%x\n", client_id);
-    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS, "Unexpected engine ID: 0x%x\n", mmu_engine_id);
-    UVM_ASSERT(client_type == UVM_FAULT_CLIENT_TYPE_GPC);
-
-    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
-}

kernel-open/nvidia-uvm/uvm_hopper_host.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2020-2024 NVIDIA Corporation
+    Copyright (c) 2020-2022 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -157,7 +157,6 @@ void uvm_hal_hopper_host_tlb_invalidate_all(uvm_push_t *push,
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
 
@@ -184,12 +183,7 @@ void uvm_hal_hopper_host_tlb_invalidate_all(uvm_push_t *push,
         ack_value = HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
-    NV_PUSH_4U(C86F, MEM_OP_A, sysmembar_value |
+    NV_PUSH_4U(C86F, MEM_OP_A, HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                      MEM_OP_B, 0,
                      MEM_OP_C, HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
@@ -202,9 +196,7 @@ void uvm_hal_hopper_host_tlb_invalidate_all(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C86F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
                                HWVALUE(C86F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    // GPU membar still requires an explicit membar method.
-    if (membar == UVM_MEMBAR_GPU)
-        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }
 
 void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
@@ -212,7 +204,7 @@ void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
                                            NvU32 depth,
                                            NvU64 base,
                                            NvU64 size,
-                                           NvU64 page_size,
+                                           NvU32 page_size,
                                            uvm_membar_t membar)
 {
     NvU32 aperture_value;
@@ -220,7 +212,6 @@ void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 pdb_lo;
     NvU32 pdb_hi;
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
     NvU32 va_lo;
     NvU32 va_hi;
     NvU64 end;
@@ -230,9 +221,9 @@ void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
     NvU32 log2_invalidation_size;
     uvm_gpu_t *gpu = uvm_push_get_gpu(push);
 
-    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
-    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%x\n", page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%x\n", base, page_size);
+    UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%x\n", size, page_size);
     UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
 
     // The invalidation size must be a power-of-two number of pages containing
@@ -286,13 +277,8 @@ void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
         ack_value = HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (membar == UVM_MEMBAR_SYS)
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
     NV_PUSH_4U(C86F, MEM_OP_A, HWVALUE(C86F, MEM_OP_A, TLB_INVALIDATE_INVALIDATION_SIZE, log2_invalidation_size) |
-                               sysmembar_value |
+                               HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
                                HWVALUE(C86F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
                      MEM_OP_B, HWVALUE(C86F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
@@ -306,9 +292,7 @@ void uvm_hal_hopper_host_tlb_invalidate_va(uvm_push_t *push,
                      MEM_OP_D, HWCONST(C86F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
                                HWVALUE(C86F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
 
-    // GPU membar still requires an explicit membar method.
-    if (membar == UVM_MEMBAR_GPU)
-        gpu->parent->host_hal->membar_gpu(push);
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }
 
 void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
@@ -316,12 +300,12 @@ void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
                                              UVM_TEST_INVALIDATE_TLB_PARAMS *params)
 {
     NvU32 ack_value = 0;
-    NvU32 sysmembar_value = 0;
     NvU32 invalidate_gpc_value = 0;
     NvU32 aperture_value = 0;
     NvU32 pdb_lo = 0;
     NvU32 pdb_hi = 0;
     NvU32 page_table_level = 0;
+    uvm_membar_t membar;
 
     UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
     if (pdb.aperture == UVM_APERTURE_VID)
@@ -348,11 +332,6 @@ void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
         ack_value = HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
     }
 
-    if (params->membar == UvmInvalidateTlbMemBarSys)
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, EN);
-    else
-        sysmembar_value = HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS);
-
     if (params->disable_gpc_invalidate)
         invalidate_gpc_value = HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
     else
@@ -364,7 +343,7 @@ void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
         NvU32 va_lo = va & HWMASK(C86F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
         NvU32 va_hi = va >> HWSIZE(C86F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
 
-        NV_PUSH_4U(C86F, MEM_OP_A, sysmembar_value |
+        NV_PUSH_4U(C86F, MEM_OP_A, HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                    HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS) |
                                    HWVALUE(C86F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
                          MEM_OP_B, HWVALUE(C86F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
@@ -379,7 +358,7 @@ void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C86F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
     else {
-        NV_PUSH_4U(C86F, MEM_OP_A, sysmembar_value |
+        NV_PUSH_4U(C86F, MEM_OP_A, HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
                                    HWCONST(C86F, MEM_OP_A, TLB_INVALIDATE_INVAL_SCOPE, NON_LINK_TLBS),
                          MEM_OP_B, 0,
                          MEM_OP_C, HWCONST(C86F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
@@ -393,9 +372,14 @@ void uvm_hal_hopper_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C86F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
 
-    // GPU membar still requires an explicit membar method.
-    if (params->membar == UvmInvalidateTlbMemBarLocal)
-        uvm_push_get_gpu(push)->parent->host_hal->membar_gpu(push);
+    if (params->membar == UvmInvalidateTlbMemBarSys)
+        membar = UVM_MEMBAR_SYS;
+    else if (params->membar == UvmInvalidateTlbMemBarLocal)
+        membar = UVM_MEMBAR_GPU;
+    else
+        membar = UVM_MEMBAR_NONE;
+
+    uvm_hal_tlb_invalidate_membar(push, membar);
 }
 
 void uvm_hal_hopper_host_set_gpfifo_pushbuffer_segment_base(NvU64 *fifo_entry, NvU64 pushbuffer_va)

kernel-open/nvidia-uvm/uvm_hopper_mmu.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2020-2024 NVIDIA Corporation
+    Copyright (c) 2020-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -47,7 +47,21 @@
 #define ATS_ALLOWED 0
 #define ATS_NOT_ALLOWED 1
 
-static NvU32 page_table_depth_hopper(NvU64 page_size)
+uvm_mmu_engine_type_t uvm_hal_hopper_mmu_engine_id_to_type(NvU16 mmu_engine_id)
+{
+    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST44)
+        return UVM_MMU_ENGINE_TYPE_HOST;
+
+    if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE9)
+        return UVM_MMU_ENGINE_TYPE_CE;
+
+    // We shouldn't be servicing faults from any other engines
+    UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS, "Unexpected engine ID: 0x%x\n", mmu_engine_id);
+
+    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
+}
+
+static NvU32 page_table_depth_hopper(NvU32 page_size)
 {
     // The common-case is page_size == UVM_PAGE_SIZE_2M, hence the first check
     if (page_size == UVM_PAGE_SIZE_2M)
@@ -65,7 +79,7 @@ static NvU32 entries_per_index_hopper(NvU32 depth)
     return 1;
 }
 
-static NvLength entry_offset_hopper(NvU32 depth, NvU64 page_size)
+static NvLength entry_offset_hopper(NvU32 depth, NvU32 page_size)
 {
     UVM_ASSERT(depth < 6);
     if ((page_size == UVM_PAGE_SIZE_4K) && (depth == 4))
@@ -78,7 +92,7 @@ static NvLength entry_size_hopper(NvU32 depth)
     return entries_per_index_hopper(depth) * 8;
 }
 
-static NvU32 index_bits_hopper(NvU32 depth, NvU64 page_size)
+static NvU32 index_bits_hopper(NvU32 depth, NvU32 page_size)
 {
     static const NvU32 bit_widths[] = {1, 9, 9, 9, 8};
 
@@ -106,7 +120,7 @@ static NvU32 num_va_bits_hopper(void)
     return 57;
 }
 
-static NvLength allocation_size_hopper(NvU32 depth, NvU64 page_size)
+static NvLength allocation_size_hopper(NvU32 depth, NvU32 page_size)
 {
     UVM_ASSERT(depth < 6);
     if (depth == 5 && page_size == UVM_PAGE_SIZE_64K)
@@ -219,7 +233,7 @@ static NvU64 make_sparse_pte_hopper(void)
            HWCONST64(_MMU_VER3, PTE, PCF, SPARSE);
 }
 
-static NvU64 unmapped_pte_hopper(NvU64 page_size)
+static NvU64 unmapped_pte_hopper(NvU32 page_size)
 {
     // Setting PCF to NO_VALID_4KB_PAGE on an otherwise-zeroed big PTE causes
     // the corresponding 4k PTEs to be ignored. This allows the invalidation of
@@ -476,7 +490,7 @@ static void make_pde_hopper(void *entry,
 
 static uvm_mmu_mode_hal_t hopper_mmu_mode_hal;
 
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU64 big_page_size)
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU32 big_page_size)
 {
     static bool initialized = false;
 

kernel-open/nvidia-uvm/uvm_ioctl.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2013-2024 NVidia Corporation
+    Copyright (c) 2013-2023 NVidia Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -494,7 +494,7 @@ typedef struct
     NvU64                   base                            NV_ALIGN_BYTES(8); // IN
     NvU64                   length                          NV_ALIGN_BYTES(8); // IN
     NvU64                   offset                          NV_ALIGN_BYTES(8); // IN
-    UvmGpuMappingAttributes perGpuAttributes[UVM_MAX_GPUS];                    // IN
+    UvmGpuMappingAttributes perGpuAttributes[UVM_MAX_GPUS_V2];                 // IN
     NvU64                   gpuAttributesCount              NV_ALIGN_BYTES(8); // IN
     NvS32                   rmCtrlFd;                                          // IN
     NvU32                   hClient;                                           // IN
@@ -837,6 +837,12 @@ typedef struct
 // Initialize any tracker object such as a queue or counter
 // UvmToolsCreateEventQueue, UvmToolsCreateProcessAggregateCounters,
 // UvmToolsCreateProcessorCounters.
+// Note that the order of structure elements has the version as the last field.
+// This is used to tell whether the kernel supports V2 events or not because
+// the V1 UVM_TOOLS_INIT_EVENT_TRACKER ioctl would not read or update that
+// field but V2 will. This is needed because it is possible to create an event
+// queue before CUDA is initialized which means UvmSetDriverVersion() hasn't
+// been called yet and the kernel version is unknown.
 //
 #define UVM_TOOLS_INIT_EVENT_TRACKER                                  UVM_IOCTL_BASE(56)
 typedef struct
@@ -847,8 +853,9 @@ typedef struct
     NvProcessorUuid processor;                            // IN
     NvU32           allProcessors;                        // IN
     NvU32           uvmFd;                                // IN
-    NvU32           version;                              // IN (UvmToolsEventQueueVersion)
     NV_STATUS       rmStatus;                             // OUT
+    NvU32           requestedVersion;                     // IN
+    NvU32           grantedVersion;                       // OUT
 } UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS;
 
 //
@@ -929,15 +936,23 @@ typedef struct
 
 //
 // UvmToolsGetProcessorUuidTable
+// Note that tablePtr != 0 and count == 0 means that tablePtr is assumed to be
+// an array of size UVM_MAX_PROCESSORS_V1 and that only UvmEventEntry_V1
+// processor IDs (physical GPU UUIDs) will be reported.
+// tablePtr == 0 and count == 0 can be used to query how many processors are
+// present in order to dynamically allocate the correct size array since the
+// total number of processors is returned in 'count'.
 //
 #define UVM_TOOLS_GET_PROCESSOR_UUID_TABLE                            UVM_IOCTL_BASE(64)
 typedef struct
 {
     NvU64     tablePtr                 NV_ALIGN_BYTES(8); // IN
-    NvU32     version;                                    // IN (UvmToolsEventQueueVersion)
+    NvU32     count;                                      // IN/OUT
     NV_STATUS rmStatus;                                   // OUT
+    NvU32     version;                                    // OUT
 } UVM_TOOLS_GET_PROCESSOR_UUID_TABLE_PARAMS;
 
+
 //
 // UvmMapDynamicParallelismRegion
 //
@@ -980,7 +995,7 @@ typedef struct
 {
     NvU64                   base                            NV_ALIGN_BYTES(8); // IN
     NvU64                   length                          NV_ALIGN_BYTES(8); // IN
-    UvmGpuMappingAttributes perGpuAttributes[UVM_MAX_GPUS];                    // IN
+    UvmGpuMappingAttributes perGpuAttributes[UVM_MAX_GPUS_V2];                 // IN
     NvU64                   gpuAttributesCount              NV_ALIGN_BYTES(8); // IN
     NV_STATUS               rmStatus;                                          // OUT
 } UVM_ALLOC_SEMAPHORE_POOL_PARAMS;

kernel-open/nvidia-uvm/uvm_linux.h

@@ -226,48 +226,7 @@ static inline const struct cpumask *uvm_cpumask_of_node(int node)
 #define __GFP_NORETRY 0
 #endif
 
-#define NV_UVM_GFP_FLAGS (GFP_KERNEL)
-
-// Develop builds define DEBUG but enable optimization
-#if defined(DEBUG) && !defined(NVIDIA_UVM_DEVELOP)
-  // Wrappers for functions not building correctly without optimizations on,
-  // implemented in uvm_debug_optimized.c. Notably the file is only built for
-  // debug builds, not develop or release builds.
-
-  // Unoptimized builds of atomic_xchg() hit a BUILD_BUG() on arm64 as it relies
-  // on __xchg being completely inlined:
-  //   /usr/src/linux-3.12.19/arch/arm64/include/asm/cmpxchg.h:67:3: note: in expansion of macro 'BUILD_BUG'
-  //
-  // Powerppc hits a similar issue, but ends up with an undefined symbol:
-  //   WARNING: "__xchg_called_with_bad_pointer" [...] undefined!
-  int nv_atomic_xchg(atomic_t *val, int new);
-
-  // Same problem as atomic_xchg() on powerppc:
-  //   WARNING: "__cmpxchg_called_with_bad_pointer" [...] undefined!
-  int nv_atomic_cmpxchg(atomic_t *val, int old, int new);
-
-  // Same problem as atomic_xchg() on powerppc:
-  //   WARNING: "__cmpxchg_called_with_bad_pointer" [...] undefined!
-  long nv_atomic_long_cmpxchg(atomic_long_t *val, long old, long new);
-
-  // This Linux kernel commit:
-  // 2016-08-30  0d025d271e55f3de21f0aaaf54b42d20404d2b23
-  // leads to build failures on x86_64, when compiling without optimization. Avoid
-  // that problem, by providing our own builds of copy_from_user / copy_to_user,
-  // for debug (non-optimized) UVM builds. Those are accessed via these
-  // nv_copy_to/from_user wrapper functions.
-  //
-  // Bug 1849583 has further details.
-  unsigned long nv_copy_from_user(void *to, const void __user *from, unsigned long n);
-  unsigned long nv_copy_to_user(void __user *to, const void *from, unsigned long n);
-
-#else
-  #define nv_atomic_xchg            atomic_xchg
-  #define nv_atomic_cmpxchg         atomic_cmpxchg
-  #define nv_atomic_long_cmpxchg    atomic_long_cmpxchg
-  #define nv_copy_to_user           copy_to_user
-  #define nv_copy_from_user         copy_from_user
-#endif
+#define NV_UVM_GFP_FLAGS (GFP_KERNEL | __GFP_NOMEMALLOC)
 
 #ifndef NV_ALIGN_DOWN
 #define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1))

kernel-open/nvidia-uvm/uvm_lock.c

@@ -27,7 +27,7 @@
 
 const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
 {
-    BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 34);
+    BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 36);
 
     switch (lock_order) {
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_INVALID);
@@ -48,7 +48,9 @@ const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CHUNK_MAPPING);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_PAGE_TREE);
+        UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_KEY_ROTATION);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CSL_PUSH);
+        UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_KEY_ROTATION_WLC);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CSL_WLC_PUSH);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CSL_SEC2_PUSH);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_PUSH);

kernel-open/nvidia-uvm/uvm_lock.h

@@ -322,6 +322,15 @@
 //      Operations not allowed while holding this lock
 //      - GPU memory allocation which can evict
 //
+// - Channel pool key rotation lock
+//      Order: UVM_LOCK_ORDER_KEY_ROTATION
+//      Condition: Confidential Computing is enabled
+//      Mutex per channel pool
+//
+//      The lock ensures mutual exclusion during key rotation affecting all the
+//      channels in the associated pool. Key rotation in WLC pools is handled
+//      using a separate lock order, see UVM_LOCK_ORDER_KEY_ROTATION_WLC below.
+//
 // - CE channel CSL channel pool semaphore
 //      Order: UVM_LOCK_ORDER_CSL_PUSH
 //      Condition: The Confidential Computing feature is enabled
@@ -338,6 +347,15 @@
 //      Operations allowed while holding this lock
 //      - Pushing work to CE channels (except for WLC channels)
 //
+// - WLC channel pool key rotation lock
+//      Order: UVM_LOCK_ORDER_KEY_ROTATION_WLC
+//      Condition: Confidential Computing is enabled
+//      Mutex of WLC channel pool
+//
+//      The lock has the same purpose as the regular channel pool key rotation
+//      lock. Using a different order lock for WLC channels allows key rotation
+//      on those channels during indirect work submission.
+//
 // - WLC CSL channel pool semaphore
 //      Order: UVM_LOCK_ORDER_CSL_WLC_PUSH
 //      Condition: The Confidential Computing feature is enabled
@@ -484,7 +502,9 @@ typedef enum
     UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL,
     UVM_LOCK_ORDER_CHUNK_MAPPING,
     UVM_LOCK_ORDER_PAGE_TREE,
+    UVM_LOCK_ORDER_KEY_ROTATION,
     UVM_LOCK_ORDER_CSL_PUSH,
+    UVM_LOCK_ORDER_KEY_ROTATION_WLC,
     UVM_LOCK_ORDER_CSL_WLC_PUSH,
     UVM_LOCK_ORDER_CSL_SEC2_PUSH,
     UVM_LOCK_ORDER_PUSH,

kernel-open/nvidia-uvm/uvm_map_external.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -61,7 +61,7 @@ typedef struct
     size_t buffer_size;
 
     // Page size in bytes
-    NvU64 page_size;
+    NvU32 page_size;
 
     // Size of a single PTE in bytes
     NvU32 pte_size;
@@ -91,7 +91,7 @@ static NV_STATUS uvm_pte_buffer_init(uvm_va_range_t *va_range,
                                      uvm_gpu_t *gpu,
                                      const uvm_map_rm_params_t *map_rm_params,
                                      NvU64 length,
-                                     NvU64 page_size,
+                                     NvU32 page_size,
                                      uvm_pte_buffer_t *pte_buffer)
 {
     uvm_gpu_va_space_t *gpu_va_space = uvm_gpu_va_space_get(va_range->va_space, gpu);
@@ -650,7 +650,9 @@ static NV_STATUS set_ext_gpu_map_location(uvm_ext_gpu_map_t *ext_gpu_map,
         return NV_OK;
     }
     // This is a local or peer allocation, so the owning GPU must have been
-    // registered. This also checks for if EGM owning GPU is registered.
+    // registered.
+    // This also checks for if EGM owning GPU is registered.
+
     owning_gpu = uvm_va_space_get_gpu_by_uuid(va_space, &mem_info->uuid);
     if (!owning_gpu)
         return NV_ERR_INVALID_DEVICE;
@@ -663,6 +665,7 @@ static NV_STATUS set_ext_gpu_map_location(uvm_ext_gpu_map_t *ext_gpu_map,
     // semantics of sysmem allocations.
 
     // Check if peer access for peer memory is enabled.
+    // This path also handles EGM allocations.
     if (owning_gpu != mapping_gpu && (!mem_info->sysmem || mem_info->egm)) {
         // TODO: Bug 1757136: In SLI, the returned UUID may be different but a
         //       local mapping must be used. We need to query SLI groups to know
@@ -853,10 +856,9 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_t *va_range,
     uvm_ext_gpu_range_tree_t *range_tree = uvm_ext_gpu_range_tree(va_range, mapping_gpu);
     UvmGpuMemoryInfo mem_info;
     uvm_gpu_va_space_t *gpu_va_space = uvm_gpu_va_space_get(va_space, mapping_gpu);
-    NvU64 mapping_page_size;
-    NvU64 biggest_mapping_page_size;
+    NvU32 mapping_page_size;
     NvU64 alignments;
-    NvU64 smallest_alignment;
+    NvU32 smallest_alignment;
     NV_STATUS status;
 
     uvm_assert_rwsem_locked_read(&va_space->lock);
@@ -945,11 +947,9 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_t *va_range,
 
     // Check for the maximum page size for the mapping of vidmem allocations,
     // the vMMU segment size may limit the range of page sizes.
-    biggest_mapping_page_size = uvm_mmu_biggest_page_size_up_to(&gpu_va_space->page_tables,
-                                                                mapping_gpu->mem_info.max_vidmem_page_size);
     if (!ext_gpu_map->is_sysmem && (ext_gpu_map->gpu == ext_gpu_map->owning_gpu) &&
-        (mapping_page_size > biggest_mapping_page_size))
-        mapping_page_size = biggest_mapping_page_size;
+        (mapping_page_size > mapping_gpu->mem_info.max_vidmem_page_size))
+        mapping_page_size = mapping_gpu->mem_info.max_vidmem_page_size;
 
     mem_info.pageSize = mapping_page_size;
 
@@ -986,7 +986,7 @@ static NV_STATUS uvm_map_external_allocation(uvm_va_space_t *va_space, UVM_MAP_E
     if (uvm_api_range_invalid_4k(params->base, params->length))
         return NV_ERR_INVALID_ADDRESS;
 
-    if (params->gpuAttributesCount == 0 || params->gpuAttributesCount > UVM_MAX_GPUS)
+    if (params->gpuAttributesCount == 0 || params->gpuAttributesCount > UVM_MAX_GPUS_V2)
         return NV_ERR_INVALID_ARGUMENT;
 
     mapped_gpus = uvm_processor_mask_cache_alloc();

kernel-open/nvidia-uvm/uvm_maxwell_fault_buffer.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2021-2024 NVIDIA Corporation
+    Copyright (c) 2021-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -74,14 +74,6 @@ NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm
     return 0;
 }
 
-uvm_mmu_engine_type_t uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported(NvU16 mmu_engine_id,
-                                                                                   uvm_fault_client_type_t client_type,
-                                                                                   NvU16 client_id)
-{
-    UVM_ASSERT_MSG(false, "fault_buffer_get_mmu_engine_type is not supported on Maxwell GPUs.\n");
-    return UVM_MMU_ENGINE_TYPE_GRAPHICS;
-}
-
 uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry)
 {
     UVM_ASSERT_MSG(false, "fault_buffer_get_fault_type is not supported.\n");

kernel-open/nvidia-uvm/uvm_maxwell_host.c

@@ -108,7 +108,7 @@ void uvm_hal_maxwell_host_tlb_invalidate_va(uvm_push_t *push,
                                             NvU32 depth,
                                             NvU64 base,
                                             NvU64 size,
-                                            NvU64 page_size,
+                                            NvU32 page_size,
                                             uvm_membar_t membar)
 {
     // No per VA invalidate on Maxwell, redirect to invalidate all.

kernel-open/nvidia-uvm/uvm_maxwell_mmu.c

@@ -38,7 +38,6 @@
 #include "uvm_forward_decl.h"
 #include "uvm_gpu.h"
 #include "uvm_mmu.h"
-#include "uvm_hal.h"
 #include "uvm_push_macros.h"
 #include "hwref/maxwell/gm107/dev_mmu.h"
 
@@ -53,7 +52,7 @@ static NvU32 entries_per_index_maxwell(NvU32 depth)
     return 1;
 }
 
-static NvLength entry_offset_maxwell(NvU32 depth, NvU64 page_size)
+static NvLength entry_offset_maxwell(NvU32 depth, NvU32 page_size)
 {
     UVM_ASSERT(depth < 2);
     if (page_size == UVM_PAGE_SIZE_4K && depth == 0)
@@ -129,7 +128,7 @@ static NvLength entry_size_maxwell(NvU32 depth)
     return 8;
 }
 
-static NvU32 index_bits_maxwell_64(NvU32 depth, NvU64 page_size)
+static NvU32 index_bits_maxwell_64(NvU32 depth, NvU32 page_size)
 {
     UVM_ASSERT(depth < 2);
     UVM_ASSERT(page_size == UVM_PAGE_SIZE_4K ||
@@ -147,7 +146,7 @@ static NvU32 index_bits_maxwell_64(NvU32 depth, NvU64 page_size)
     }
 }
 
-static NvU32 index_bits_maxwell_128(NvU32 depth, NvU64 page_size)
+static NvU32 index_bits_maxwell_128(NvU32 depth, NvU32 page_size)
 {
     UVM_ASSERT(depth < 2);
     UVM_ASSERT(page_size == UVM_PAGE_SIZE_4K ||
@@ -170,32 +169,32 @@ static NvU32 num_va_bits_maxwell(void)
     return 40;
 }
 
-static NvLength allocation_size_maxwell_64(NvU32 depth, NvU64 page_size)
+static NvLength allocation_size_maxwell_64(NvU32 depth, NvU32 page_size)
 {
     return entry_size_maxwell(depth) << index_bits_maxwell_64(depth, page_size);
 }
 
-static NvLength allocation_size_maxwell_128(NvU32 depth, NvU64 page_size)
+static NvLength allocation_size_maxwell_128(NvU32 depth, NvU32 page_size)
 {
     return entry_size_maxwell(depth) << index_bits_maxwell_128(depth, page_size);
 }
 
-static NvU32 page_table_depth_maxwell(NvU64 page_size)
+static NvU32 page_table_depth_maxwell(NvU32 page_size)
 {
     return 1;
 }
 
-static NvU64 page_sizes_maxwell_128(void)
+static NvU32 page_sizes_maxwell_128(void)
 {
     return UVM_PAGE_SIZE_128K | UVM_PAGE_SIZE_4K;
 }
 
-static NvU64 page_sizes_maxwell_64(void)
+static NvU32 page_sizes_maxwell_64(void)
 {
     return UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
 }
 
-static NvU64 unmapped_pte_maxwell(NvU64 page_size)
+static NvU64 unmapped_pte_maxwell(NvU32 page_size)
 {
     // Setting the privilege bit on an otherwise-zeroed big PTE causes the
     // corresponding 4k PTEs to be ignored. This allows the invalidation of a
@@ -357,7 +356,7 @@ static uvm_mmu_mode_hal_t maxwell_128_mmu_mode_hal =
     .page_sizes = page_sizes_maxwell_128
 };
 
-uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU64 big_page_size)
+uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU32 big_page_size)
 {
     UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
     if (big_page_size == UVM_PAGE_SIZE_64K)
@@ -376,6 +375,12 @@ void uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported(uvm_parent_gpu_t *p
     UVM_ASSERT_MSG(false, "mmu disable_prefetch_faults called on Maxwell GPU\n");
 }
 
+uvm_mmu_engine_type_t uvm_hal_maxwell_mmu_engine_id_to_type_unsupported(NvU16 mmu_engine_id)
+{
+    UVM_ASSERT(0);
+    return UVM_MMU_ENGINE_TYPE_COUNT;
+}
+
 NvU16 uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported(NvU16 client_id)
 {
     UVM_ASSERT(0);

kernel-open/nvidia-uvm/uvm_mem.c

@@ -290,15 +290,15 @@ uvm_chunk_sizes_mask_t uvm_mem_kernel_chunk_sizes(uvm_gpu_t *gpu)
     // Get the mmu mode hal directly as the internal address space tree has not
     // been created yet.
     uvm_mmu_mode_hal_t *hal = gpu->parent->arch_hal->mmu_mode_hal(gpu->big_page.internal_size);
-    NvU64 page_sizes = hal->page_sizes();
+    NvU32 page_sizes = hal->page_sizes();
 
     return (uvm_chunk_sizes_mask_t)(page_sizes & UVM_CHUNK_SIZES_MASK);
 }
 
-static NvU64 mem_pick_chunk_size(uvm_mem_t *mem)
+static NvU32 mem_pick_chunk_size(uvm_mem_t *mem)
 {
-    NvU64 biggest_page_size;
-    NvU64 chunk_size;
+    NvU32 biggest_page_size;
+    NvU32 chunk_size;
 
     if (uvm_mem_is_sysmem(mem))
         return PAGE_SIZE;
@@ -315,12 +315,12 @@ static NvU64 mem_pick_chunk_size(uvm_mem_t *mem)
     // When UVM_PAGE_SIZE_DEFAULT is used on NUMA-enabled GPUs, we force
     // chunk_size to be PAGE_SIZE at least, to allow CPU mappings.
     if (mem->backing_gpu->mem_info.numa.enabled)
-        chunk_size = max(chunk_size, (NvU64)PAGE_SIZE);
+        chunk_size = max(chunk_size, (NvU32)PAGE_SIZE);
 
     return chunk_size;
 }
 
-static NvU64 mem_pick_gpu_page_size(uvm_mem_t *mem, uvm_gpu_t *gpu, uvm_page_tree_t *gpu_page_tree)
+static NvU32 mem_pick_gpu_page_size(uvm_mem_t *mem, uvm_gpu_t *gpu, uvm_page_tree_t *gpu_page_tree)
 {
     if (uvm_mem_is_vidmem(mem)) {
         // For vidmem allocations the chunk size is picked out of the supported
@@ -467,7 +467,7 @@ static NV_STATUS mem_alloc_sysmem_dma_chunks(uvm_mem_t *mem, gfp_t gfp_flags)
     NvU64 *dma_addrs;
 
     UVM_ASSERT_MSG(mem->chunk_size == PAGE_SIZE,
-                   "mem->chunk_size is 0x%llx. PAGE_SIZE is only supported.",
+                   "mem->chunk_size is 0x%x. PAGE_SIZE is only supported.",
                    mem->chunk_size);
     UVM_ASSERT(uvm_mem_is_sysmem_dma(mem));
 
@@ -528,9 +528,10 @@ static NV_STATUS mem_alloc_sysmem_chunks(uvm_mem_t *mem, gfp_t gfp_flags)
 
 // In case of failure, the caller is required to handle cleanup by calling
 // uvm_mem_free
-static NV_STATUS mem_alloc_vidmem_chunks(uvm_mem_t *mem, bool zero)
+static NV_STATUS mem_alloc_vidmem_chunks(uvm_mem_t *mem, bool zero, bool is_unprotected)
 {
     NV_STATUS status;
+    uvm_pmm_gpu_memory_type_t mem_type;
 
     UVM_ASSERT(uvm_mem_is_vidmem(mem));
 
@@ -547,15 +548,23 @@ static NV_STATUS mem_alloc_vidmem_chunks(uvm_mem_t *mem, bool zero)
     if (!mem->vidmem.chunks)
         return NV_ERR_NO_MEMORY;
 
-    status = uvm_pmm_gpu_alloc_kernel(&mem->backing_gpu->pmm,
-                                      mem->chunks_count,
-                                      mem->chunk_size,
-                                      UVM_PMM_ALLOC_FLAGS_NONE,
-                                      mem->vidmem.chunks,
-                                      NULL);
+    // When CC is disabled the behavior is identical to that of PMM, and the
+    // protection flag is ignored (squashed by PMM internally).
+    if (is_unprotected)
+        mem_type = UVM_PMM_GPU_MEMORY_TYPE_KERNEL_UNPROTECTED;
+    else
+        mem_type = UVM_PMM_GPU_MEMORY_TYPE_KERNEL_PROTECTED;
+
+    status = uvm_pmm_gpu_alloc(&mem->backing_gpu->pmm,
+                               mem->chunks_count,
+                               mem->chunk_size,
+                               mem_type,
+                               UVM_PMM_ALLOC_FLAGS_NONE,
+                               mem->vidmem.chunks,
+                               NULL);
 
     if (status != NV_OK) {
-        UVM_ERR_PRINT("uvm_pmm_gpu_alloc_kernel (count=%zd, size=0x%llx) failed: %s\n",
+        UVM_ERR_PRINT("uvm_pmm_gpu_alloc (count=%zd, size=0x%x) failed: %s\n",
                       mem->chunks_count,
                       mem->chunk_size,
                       nvstatusToString(status));
@@ -565,7 +574,7 @@ static NV_STATUS mem_alloc_vidmem_chunks(uvm_mem_t *mem, bool zero)
     return NV_OK;
 }
 
-static NV_STATUS mem_alloc_chunks(uvm_mem_t *mem, struct mm_struct *mm, bool zero)
+static NV_STATUS mem_alloc_chunks(uvm_mem_t *mem, struct mm_struct *mm, bool zero, bool is_unprotected)
 {
     if (uvm_mem_is_sysmem(mem)) {
         gfp_t gfp_flags;
@@ -587,7 +596,7 @@ static NV_STATUS mem_alloc_chunks(uvm_mem_t *mem, struct mm_struct *mm, bool zer
         return status;
     }
 
-    return mem_alloc_vidmem_chunks(mem, zero);
+    return mem_alloc_vidmem_chunks(mem, zero, is_unprotected);
 }
 
 NV_STATUS uvm_mem_map_kernel(uvm_mem_t *mem, const uvm_processor_mask_t *mask)
@@ -617,6 +626,7 @@ NV_STATUS uvm_mem_alloc(const uvm_mem_alloc_params_t *params, uvm_mem_t **mem_ou
     NV_STATUS status;
     NvU64 physical_size;
     uvm_mem_t *mem = NULL;
+    bool is_unprotected = false;
 
     UVM_ASSERT(params->size > 0);
 
@@ -638,7 +648,12 @@ NV_STATUS uvm_mem_alloc(const uvm_mem_alloc_params_t *params, uvm_mem_t **mem_ou
     physical_size = UVM_ALIGN_UP(mem->size, mem->chunk_size);
     mem->chunks_count = physical_size / mem->chunk_size;
 
-    status = mem_alloc_chunks(mem, params->mm, params->zero);
+    if (params->is_unprotected)
+        UVM_ASSERT(uvm_mem_is_vidmem(mem));
+
+    is_unprotected = params->is_unprotected;
+
+    status = mem_alloc_chunks(mem, params->mm, params->zero, is_unprotected);
     if (status != NV_OK)
         goto error;
 
@@ -1035,7 +1050,7 @@ static NV_STATUS mem_map_gpu(uvm_mem_t *mem,
                              uvm_page_table_range_vec_t **range_vec)
 {
     NV_STATUS status;
-    NvU64 page_size;
+    NvU32 page_size;
     uvm_pmm_alloc_flags_t pmm_flags = UVM_PMM_ALLOC_FLAGS_EVICT;
 
     uvm_mem_pte_maker_data_t pte_maker_data = {
@@ -1044,7 +1059,7 @@ static NV_STATUS mem_map_gpu(uvm_mem_t *mem,
         };
 
     page_size = mem_pick_gpu_page_size(mem, gpu, tree);
-    UVM_ASSERT_MSG(uvm_mmu_page_size_supported(tree, page_size), "page_size 0x%llx\n", page_size);
+    UVM_ASSERT_MSG(uvm_mmu_page_size_supported(tree, page_size), "page_size 0x%x\n", page_size);
 
     // When the Confidential Computing feature is enabled, DMA allocations are
     // majoritarily allocated and managed by a per-GPU DMA buffer pool

kernel-open/nvidia-uvm/uvm_mem.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -126,7 +126,12 @@ typedef struct
     //
     // CPU mappings will always use PAGE_SIZE, so the physical allocation chunk
     // has to be aligned to PAGE_SIZE.
-    NvU64 page_size;
+    NvU32 page_size;
+
+    // The protection flag is only observed for vidmem allocations when CC is
+    // enabled. If set to true, the allocation returns unprotected vidmem;
+    // otherwise, the allocation returns protected vidmem.
+    bool is_unprotected;
 
     // If true, the allocation is zeroed (scrubbed).
     bool zero;
@@ -194,7 +199,7 @@ struct uvm_mem_struct
     size_t chunks_count;
 
     // Size of each physical chunk (vidmem) or CPU page (sysmem)
-    NvU64 chunk_size;
+    NvU32 chunk_size;
 
     // Size of the allocation
     NvU64 size;
@@ -324,7 +329,8 @@ uvm_gpu_phys_address_t uvm_mem_gpu_physical(uvm_mem_t *mem, uvm_gpu_t *gpu, NvU6
 uvm_gpu_address_t uvm_mem_gpu_address_physical(uvm_mem_t *mem, uvm_gpu_t *gpu, NvU64 offset, NvU64 size);
 
 // Helper to get an address suitable for accessing_gpu (which may be the backing
-// GPU) to access with CE.
+// GPU) to access with CE. Note that mappings for indirect peers are not
+// created automatically.
 uvm_gpu_address_t uvm_mem_gpu_address_copy(uvm_mem_t *mem, uvm_gpu_t *accessing_gpu, NvU64 offset, NvU64 size);
 
 static bool uvm_mem_is_sysmem(uvm_mem_t *mem)

kernel-open/nvidia-uvm/uvm_mem_test.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2024 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -33,7 +33,7 @@
 
 static const size_t sysmem_alloc_sizes[] = { 1, PAGE_SIZE - 1, PAGE_SIZE, 7 * PAGE_SIZE };
 
-static NvU64 first_page_size(NvU64 page_sizes)
+static NvU32 first_page_size(NvU32 page_sizes)
 {
     return page_sizes & ~(page_sizes - 1);
 }
@@ -43,7 +43,7 @@ static NvU64 first_page_size(NvU64 page_sizes)
          page_size;                                                                 \
          page_size = first_page_size((page_sizes) & ~(page_size | (page_size - 1))))
 
-static inline NV_STATUS mem_alloc_sysmem_and_map_cpu_kernel(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **sys_mem)
+static inline NV_STATUS __alloc_map_sysmem(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **sys_mem)
 {
     if (g_uvm_global.conf_computing_enabled)
         return uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(size, gpu, current->mm, sys_mem);
@@ -67,7 +67,7 @@ static NV_STATUS check_accessible_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem)
     UVM_ASSERT(uvm_mem_physical_size(mem) >= verif_size);
     UVM_ASSERT(verif_size >= sizeof(*sys_verif));
 
-    TEST_NV_CHECK_GOTO(mem_alloc_sysmem_and_map_cpu_kernel(verif_size, gpu, &sys_mem), done);
+    TEST_NV_CHECK_GOTO(__alloc_map_sysmem(verif_size, gpu, &sys_mem), done);
     TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(sys_mem, gpu), done);
 
     sys_verif = (NvU64*)uvm_mem_get_cpu_addr_kernel(sys_mem);
@@ -100,9 +100,9 @@ static NV_STATUS check_accessible_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem)
                                  "Memcopy %zd bytes from virtual sys_mem 0x%llx to %s mem 0x%llx [mem loc: %s, page size: %u]",
                                  size_this_time,
                                  sys_mem_gpu_address.address,
-                                 mem_gpu_address.is_virtual ? "virtual" : "physical",
+                                 mem_gpu_address.is_virtual? "virtual" : "physical",
                                  mem_gpu_address.address,
-                                 uvm_mem_is_sysmem(mem) ? "sys" : "vid",
+                                 uvm_mem_is_sysmem(mem)? "sys" : "vid",
                                  mem->chunk_size);
 
         gpu->parent->ce_hal->memcopy(&push, mem_gpu_address, sys_mem_gpu_address, size_this_time);
@@ -140,7 +140,7 @@ static NV_STATUS check_accessible_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem)
                                  "Memcopy %zd bytes from virtual mem 0x%llx to %s sys_mem 0x%llx",
                                  size_this_time,
                                  mem_gpu_address.address,
-                                 sys_mem_gpu_address.is_virtual ? "virtual" : "physical",
+                                 sys_mem_gpu_address.is_virtual? "virtual" : "physical",
                                  sys_mem_gpu_address.address);
 
         gpu->parent->ce_hal->memcopy(&push, sys_mem_gpu_address, mem_gpu_address, size_this_time);
@@ -153,7 +153,7 @@ static NV_STATUS check_accessible_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem)
 
     for (i = 0; i < verif_size / sizeof(*sys_verif); ++i) {
         if (sys_verif[i] != mem->size + i) {
-            UVM_TEST_PRINT("Verif failed for %zd = 0x%llx instead of 0x%llx, verif_size=0x%llx mem(size=0x%llx, page_size=%llu, processor=%u)\n",
+            UVM_TEST_PRINT("Verif failed for %zd = 0x%llx instead of 0x%llx, verif_size=0x%llx mem(size=0x%llx, page_size=%u, processor=%u)\n",
                            i,
                            sys_verif[i],
                            (NvU64)(verif_size + i),
@@ -241,7 +241,7 @@ static NV_STATUS test_map_cpu(uvm_mem_t *mem)
     return NV_OK;
 }
 
-static NV_STATUS test_alloc_sysmem(uvm_va_space_t *va_space, NvU64 page_size, size_t size, uvm_mem_t **mem_out)
+static NV_STATUS test_alloc_sysmem(uvm_va_space_t *va_space, NvU32 page_size, size_t size, uvm_mem_t **mem_out)
 {
     NV_STATUS status;
     uvm_mem_t *mem;
@@ -252,9 +252,10 @@ static NV_STATUS test_alloc_sysmem(uvm_va_space_t *va_space, NvU64 page_size, si
     params.page_size = page_size;
     params.mm = current->mm;
 
-    TEST_NV_CHECK_GOTO(uvm_mem_alloc(&params, &mem), error);
+    status = uvm_mem_alloc(&params, &mem);
+    TEST_CHECK_GOTO(status == NV_OK, error);
 
-    TEST_NV_CHECK_GOTO(test_map_cpu(mem), error);
+    TEST_CHECK_GOTO(test_map_cpu(mem) == NV_OK, error);
 
     for_each_va_space_gpu(gpu, va_space)
         TEST_NV_CHECK_GOTO(test_map_gpu(mem, gpu), error);
@@ -265,7 +266,6 @@ static NV_STATUS test_alloc_sysmem(uvm_va_space_t *va_space, NvU64 page_size, si
 
 error:
     uvm_mem_free(mem);
-
     return status;
 }
 
@@ -299,7 +299,7 @@ error:
     return status;
 }
 
-static NV_STATUS test_alloc_vidmem(uvm_gpu_t *gpu, NvU64 page_size, size_t size, uvm_mem_t **mem_out)
+static NV_STATUS test_alloc_vidmem(uvm_gpu_t *gpu, NvU32 page_size, size_t size, uvm_mem_t **mem_out)
 {
     NV_STATUS status;
     uvm_mem_t *mem;
@@ -334,7 +334,7 @@ error:
     return status;
 }
 
-static bool should_test_page_size(size_t alloc_size, NvU64 page_size)
+static bool should_test_page_size(size_t alloc_size, NvU32 page_size)
 {
     if (g_uvm_global.num_simulated_devices == 0)
         return true;
@@ -352,22 +352,21 @@ static NV_STATUS test_all(uvm_va_space_t *va_space)
     NvU32 current_alloc = 0;
 
     // Create allocations of these sizes
-    static const size_t sizes[] = { 1, 4, 16, 1024, 4096, 1024 * 1024, 7 * 1024 * 1024 + 17 };
+    static const size_t sizes[] = {1, 4, 16, 1024, 4096, 1024 * 1024, 7 * 1024 * 1024 + 17 };
 
     // Pascal+ can map sysmem with 4K, 64K and 2M PTEs, other GPUs can only use
     // 4K. Test all of the sizes supported by Pascal+ and 128K to match big page
     // size on pre-Pascal GPUs with 128K big page size.
     // Ampere+ also supports 512M PTEs, but since UVM's maximum chunk size is
     // 2M, we don't test for this page size.
-    // Blackwell+ also supports 256G PTEs and the above holds for this case too.
-
-    static const NvU64 cpu_chunk_sizes = PAGE_SIZE | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_128K | UVM_PAGE_SIZE_2M;
+    static const NvU32 cpu_chunk_sizes = PAGE_SIZE | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_128K | UVM_PAGE_SIZE_2M;
 
     // All supported page sizes will be tested, CPU has the most with 4 and +1
     // for the default.
     static const int max_supported_page_sizes = 4 + 1;
     int i;
 
+
     // TODO: Bug 3839176: the test is waived on Confidential Computing because
     // it assumes that GPU can access system memory without using encryption.
     if (g_uvm_global.conf_computing_enabled)
@@ -387,13 +386,13 @@ static NV_STATUS test_all(uvm_va_space_t *va_space)
         return NV_ERR_NO_MEMORY;
 
     for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
-        NvU64 page_size = 0;
+        NvU32 page_size = 0;
         uvm_mem_t *mem;
 
         if (should_test_page_size(sizes[i], UVM_PAGE_SIZE_DEFAULT)) {
             status = test_alloc_sysmem(va_space, UVM_PAGE_SIZE_DEFAULT, sizes[i], &mem);
             if (status != NV_OK) {
-                UVM_TEST_PRINT("Failed to alloc sysmem size %zd, page_size default\n", sizes[i]);
+                UVM_TEST_PRINT("Failed to alloc sysmem size %zd, page_size default\n", sizes[i], page_size);
                 goto cleanup;
             }
             all_mem[current_alloc++] = mem;
@@ -405,14 +404,14 @@ static NV_STATUS test_all(uvm_va_space_t *va_space)
 
             status = test_alloc_sysmem(va_space, page_size, sizes[i], &mem);
             if (status != NV_OK) {
-                UVM_TEST_PRINT("Failed to alloc sysmem size %zd, page_size %llu\n", sizes[i], page_size);
+                UVM_TEST_PRINT("Failed to alloc sysmem size %zd, page_size %u\n", sizes[i], page_size);
                 goto cleanup;
             }
             all_mem[current_alloc++] = mem;
         }
 
         for_each_va_space_gpu(gpu, va_space) {
-            NvU64 page_sizes = gpu->address_space_tree.hal->page_sizes();
+            NvU32 page_sizes = gpu->address_space_tree.hal->page_sizes();
 
             UVM_ASSERT(max_supported_page_sizes >= hweight_long(page_sizes));
 
@@ -429,7 +428,7 @@ static NV_STATUS test_all(uvm_va_space_t *va_space)
             for_each_page_size(page_size, page_sizes) {
                 status = test_alloc_vidmem(gpu, page_size, sizes[i], &mem);
                 if (status != NV_OK) {
-                    UVM_TEST_PRINT("Test alloc vidmem failed, page_size %llu size %zd GPU %s\n",
+                    UVM_TEST_PRINT("Test alloc vidmem failed, page_size %u size %zd GPU %s\n",
                                    page_size,
                                    sizes[i],
                                    uvm_gpu_name(gpu));
@@ -462,17 +461,17 @@ cleanup:
 static NV_STATUS test_basic_vidmem(uvm_gpu_t *gpu)
 {
     NV_STATUS status = NV_OK;
-    NvU64 page_size;
-    NvU64 page_sizes = gpu->address_space_tree.hal->page_sizes();
-    NvU64 biggest_page_size = uvm_mmu_biggest_page_size_up_to(&gpu->address_space_tree, UVM_CHUNK_SIZE_MAX);
-    NvU64 smallest_page_size = page_sizes & ~(page_sizes - 1);
+    NvU32 page_size;
+    NvU32 page_sizes = gpu->address_space_tree.hal->page_sizes();
+    NvU32 biggest_page_size = uvm_mmu_biggest_page_size_up_to(&gpu->address_space_tree, UVM_CHUNK_SIZE_MAX);
+    NvU32 smallest_page_size = page_sizes & ~(page_sizes - 1);
     uvm_mem_t *mem = NULL;
 
     page_sizes &= UVM_CHUNK_SIZES_MASK;
     for_each_page_size(page_size, page_sizes) {
         TEST_CHECK_GOTO(uvm_mem_alloc_vidmem(page_size - 1, gpu, &mem) == NV_OK, done);
         if (gpu->mem_info.numa.enabled)
-            TEST_CHECK_GOTO(mem->chunk_size >= PAGE_SIZE && mem->chunk_size <= max(page_size, (NvU64)PAGE_SIZE), done);
+            TEST_CHECK_GOTO(mem->chunk_size >= PAGE_SIZE && mem->chunk_size <= max(page_size, (NvU32)PAGE_SIZE), done);
         else
             TEST_CHECK_GOTO(mem->chunk_size < page_size || page_size == smallest_page_size, done);
         uvm_mem_free(mem);
@@ -480,14 +479,14 @@ static NV_STATUS test_basic_vidmem(uvm_gpu_t *gpu)
 
         TEST_CHECK_GOTO(uvm_mem_alloc_vidmem(page_size, gpu, &mem) == NV_OK, done);
         if (gpu->mem_info.numa.enabled)
-            TEST_CHECK_GOTO(mem->chunk_size == max(page_size, (NvU64)PAGE_SIZE), done);
+            TEST_CHECK_GOTO(mem->chunk_size == max(page_size, (NvU32)PAGE_SIZE), done);
         else
             TEST_CHECK_GOTO(mem->chunk_size == page_size, done);
         uvm_mem_free(mem);
         mem = NULL;
     }
 
-    TEST_CHECK_GOTO(uvm_mem_alloc_vidmem(5 * biggest_page_size - 1, gpu, &mem) == NV_OK, done);
+    TEST_CHECK_GOTO(uvm_mem_alloc_vidmem(5 * ((NvU64)biggest_page_size) - 1, gpu, &mem) == NV_OK, done);
     TEST_CHECK_GOTO(mem->chunk_size == biggest_page_size, done);
 
 done:
@@ -495,6 +494,41 @@ done:
     return status;
 }
 
+static NV_STATUS test_basic_vidmem_unprotected(uvm_gpu_t *gpu)
+{
+    NV_STATUS status = NV_OK;
+    uvm_mem_t *mem = NULL;
+
+    uvm_mem_alloc_params_t params = { 0 };
+    params.size = UVM_PAGE_SIZE_4K;
+    params.backing_gpu = gpu;
+    params.page_size = UVM_PAGE_SIZE_4K;
+
+    // If CC is enabled, the protection flag is observed. Because currently all
+    // vidmem is in the protected region, the allocation should succeed.
+    //
+    // If CC is disabled, the protection flag is ignored.
+    params.is_unprotected = false;
+    TEST_NV_CHECK_RET(uvm_mem_alloc(&params, &mem));
+
+    uvm_mem_free(mem);
+    mem = NULL;
+
+    // If CC is enabled, the allocation should fail because currently the
+    // unprotected region is empty.
+    //
+    // If CC is disabled, the behavior should be identical to that of a
+    // protected allocation.
+    params.is_unprotected = true;
+    if (g_uvm_global.conf_computing_enabled)
+        TEST_CHECK_RET(uvm_mem_alloc(&params, &mem) == NV_ERR_NO_MEMORY);
+    else
+        TEST_NV_CHECK_RET(uvm_mem_alloc(&params, &mem));
+
+    uvm_mem_free(mem);
+    return status;
+}
+
 static NV_STATUS test_basic_sysmem(void)
 {
     NV_STATUS status = NV_OK;
@@ -570,135 +604,6 @@ done:
     return status;
 }
 
-static NV_STATUS check_huge_page_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem, NvU64 offset)
-{
-    NV_STATUS status = NV_OK;
-    uvm_mem_t *sys_mem = NULL;
-    uvm_push_t push;
-    NvU64 *sys_verif;
-    NvU64 *expected_value;
-    NvU64 verif_size = mem->size;
-    uvm_gpu_address_t mem_gpu_address, sys_mem_gpu_address;
-
-    UVM_ASSERT(uvm_mem_physical_size(mem) >= verif_size);
-
-    TEST_NV_CHECK_GOTO(mem_alloc_sysmem_and_map_cpu_kernel(verif_size, gpu, &sys_mem), done);
-    sys_verif = uvm_mem_get_cpu_addr_kernel(sys_mem);
-    memset(sys_verif, 0x0, mem->size);
-
-    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(sys_mem, gpu), done);
-
-    mem_gpu_address = uvm_gpu_address_virtual(offset);
-    sys_mem_gpu_address = uvm_mem_gpu_address_virtual_kernel(sys_mem, gpu);
-
-    TEST_NV_CHECK_GOTO(uvm_push_begin(gpu->channel_manager,
-                                      UVM_CHANNEL_TYPE_GPU_TO_CPU,
-                                      &push,
-                                      "Memcopy %llu bytes from virtual mem 0x%llx to virtual sys_mem 0x%llx",
-                                      verif_size,
-                                      mem_gpu_address.address,
-                                      sys_mem_gpu_address.address),
-                       done);
-
-    gpu->parent->ce_hal->memcopy(&push, sys_mem_gpu_address, mem_gpu_address, verif_size);
-    TEST_NV_CHECK_GOTO(uvm_push_end_and_wait(&push), done);
-
-    expected_value = uvm_mem_get_cpu_addr_kernel(mem);
-    TEST_CHECK_GOTO(memcmp(sys_verif, expected_value, verif_size) == 0, done);
-
-done:
-    uvm_mem_free(sys_mem);
-
-    return status;
-}
-
-static NvU64 test_pte_maker(uvm_page_table_range_vec_t *range_vec, NvU64 offset, void *phys_addr)
-{
-    uvm_page_tree_t *tree = range_vec->tree;
-    uvm_gpu_phys_address_t phys = uvm_gpu_phys_address(UVM_APERTURE_SYS, (NvU64)phys_addr);
-
-    return tree->hal->make_pte(phys.aperture, phys.address, UVM_PROT_READ_ONLY, UVM_MMU_PTE_FLAGS_NONE);
-}
-
-static NV_STATUS test_huge_page_size(uvm_va_space_t *va_space, uvm_gpu_t *gpu, NvU64 page_size)
-{
-    NV_STATUS status = NV_OK;
-    uvm_mem_t *mem = NULL;
-    size_t size = PAGE_SIZE;
-    NvU64 *cpu_addr;
-    NvU64 huge_gpu_va;
-    NvU64 gpu_phys_addr;
-    uvm_page_table_range_vec_t *range_vec;
-    NvU8 value = 0xA5;
-
-    // TODO: Bug 3839176: the test is waived on Confidential Computing because
-    // it assumes that GPU can access system memory without using encryption.
-    if (g_uvm_global.conf_computing_enabled)
-        return NV_OK;
-
-    TEST_NV_CHECK_GOTO(mem_alloc_sysmem_and_map_cpu_kernel(size, gpu, &mem), cleanup);
-    cpu_addr = uvm_mem_get_cpu_addr_kernel(mem);
-    memset(cpu_addr, value, mem->size);
-
-    // Map it on the GPU (uvm_mem base area), it creates GPU physical address
-    // for the sysmem mapping.
-    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_phys(mem, gpu), cleanup);
-
-    huge_gpu_va = UVM_ALIGN_UP(gpu->parent->uvm_mem_va_base + gpu->parent->uvm_mem_va_size, page_size);
-    TEST_CHECK_GOTO(IS_ALIGNED(huge_gpu_va, page_size), cleanup);
-    TEST_CHECK_GOTO((huge_gpu_va + page_size) < (1ull << gpu->address_space_tree.hal->num_va_bits()), cleanup);
-
-    // Manually mapping huge_gpu_va because page_size is larger than the largest
-    // uvm_mem_t chunk/page size, so we don't use uvm_mem_gpu_kernel() helper.
-    TEST_NV_CHECK_GOTO(uvm_page_table_range_vec_create(&gpu->address_space_tree,
-                                                       huge_gpu_va,
-                                                       page_size,
-                                                       page_size,
-                                                       UVM_PMM_ALLOC_FLAGS_NONE,
-                                                       &range_vec), cleanup);
-
-    gpu_phys_addr = uvm_mem_gpu_physical(mem, gpu, 0, size).address;
-
-    TEST_NV_CHECK_GOTO(uvm_page_table_range_vec_write_ptes(range_vec,
-                                                           UVM_MEMBAR_NONE,
-                                                           test_pte_maker,
-                                                           (void *)gpu_phys_addr), cleanup_range);
-
-    // Despite the huge page_size mapping, only PAGE_SIZE is backed by an
-    // allocation "own" by the test. We compute the offset within the huge page
-    // to verify only this segment.
-    TEST_NV_CHECK_GOTO(check_huge_page_from_gpu(gpu, mem, huge_gpu_va + (gpu_phys_addr % page_size)),
-                       cleanup_range);
-
-cleanup_range:
-    uvm_page_table_range_vec_destroy(range_vec);
-    range_vec = NULL;
-
-cleanup:
-    uvm_mem_free(mem);
-
-    return status;
-}
-
-// Check the GPU access to memory from a 512MB+ page size mapping.
-// The test allocates a PAGE_SIZE sysmem page, but uses the GMMU to map a huge
-// page size area. It maps the allocated page to this area, and uses the CE to
-// access it, thus, exercising a memory access using a huge page.
-static NV_STATUS test_huge_pages(uvm_va_space_t *va_space, uvm_gpu_t *gpu)
-{
-    NvU64 page_sizes = gpu->address_space_tree.hal->page_sizes();
-    NvU64 page_size = 0;
-
-    for_each_page_size(page_size, page_sizes) {
-        if (page_size < UVM_PAGE_SIZE_512M)
-            continue;
-
-        TEST_NV_CHECK_RET(test_huge_page_size(va_space, gpu, page_size));
-    }
-
-    return NV_OK;
-}
-
 static NV_STATUS test_basic(uvm_va_space_t *va_space)
 {
     uvm_gpu_t *gpu;
@@ -708,8 +613,8 @@ static NV_STATUS test_basic(uvm_va_space_t *va_space)
     for_each_va_space_gpu(gpu, va_space) {
         TEST_NV_CHECK_RET(test_basic_vidmem(gpu));
         TEST_NV_CHECK_RET(test_basic_sysmem_dma(gpu));
+        TEST_NV_CHECK_RET(test_basic_vidmem_unprotected(gpu));
         TEST_NV_CHECK_RET(test_basic_dma_pool(gpu));
-        TEST_NV_CHECK_RET(test_huge_pages(va_space, gpu));
     }
 
     return NV_OK;