535.43.23

535.43.22
535.43.20
2026-01-27 11:39:46 +00:00 · 2024-01-24 13:46:26 -08:00 · 2023-12-19 16:03:49 -08:00 · 2023-12-08 11:25:50 -08:00 · 2023-12-01 13:41:12 -08:00 · 2023-11-06 16:08:36 -08:00
457 changed files with 132282 additions and 165616 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,30 @@

 ## Release 535 Entries

+### [535.54.03] 2023-06-14
+
+### [535.43.23] 2024-01-24
+
+### [535.43.22] 2023-12-19
+
+### [535.43.20] 2023-12-08
+
+### [535.43.19] 2023-12-01
+
+### [535.43.16] 2023-11-06
+
+### [535.43.15] 2023-10-24
+
+### [535.43.13] 2023-10-11
+
+### [535.43.11] 2023-10-05
+
+### [535.43.10] 2023-09-27
+
+### [535.43.09] 2023-09-01
+
+### [535.43.08] 2023-08-17
+
 ### [535.43.02] 2023-05-30

 #### Fixed
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # NVIDIA Linux Open GPU Kernel Module Source

 This is the source release of the NVIDIA Linux open GPU kernel modules,
-version 535.43.02.
+version 535.43.23.


 ## How to Build
@@ -17,7 +17,7 @@ as root:

 Note that the kernel modules built here must be used with GSP
 firmware and user-space NVIDIA GPU driver components from a corresponding
-535.43.02 driver release.  This can be achieved by installing
+535.43.23 driver release.  This can be achieved by installing
 the NVIDIA GPU driver from the .run file using the `--no-kernel-modules`
 option.  E.g.,

@@ -180,7 +180,7 @@ software applications.
 ## Compatible GPUs

 The open-gpu-kernel-modules can be used on any Turing or later GPU
-(see the table below). However, in the 535.43.02 release,
+(see the table below). However, in the 535.43.23 release,
 GeForce and Workstation support is still considered alpha-quality.

 To enable use of the open kernel modules on GeForce and Workstation GPUs,
@@ -188,7 +188,7 @@ set the "NVreg_OpenRmEnableUnsupportedGpus" nvidia.ko kernel module
 parameter to 1. For more details, see the NVIDIA GPU driver end user
 README here:

-https://us.download.nvidia.com/XFree86/Linux-x86_64/535.43.02/README/kernel_open.html
+https://us.download.nvidia.com/XFree86/Linux-x86_64/535.43.23/README/kernel_open.html

 In the below table, if three IDs are listed, the first is the PCI Device 
 ID, the second is the PCI Subsystem Vendor ID, and the third is the PCI
@@ -658,12 +658,16 @@ Subsystem Device ID.
 | NVIDIA A100-SXM4-80GB                           | 20B2 10DE 147F |
 | NVIDIA A100-SXM4-80GB                           | 20B2 10DE 1622 |
 | NVIDIA A100-SXM4-80GB                           | 20B2 10DE 1623 |
+| NVIDIA PG509-210                                | 20B2 10DE 1625 |
 | NVIDIA A100-SXM-64GB                            | 20B3 10DE 14A7 |
 | NVIDIA A100-SXM-64GB                            | 20B3 10DE 14A8 |
 | NVIDIA A100 80GB PCIe                           | 20B5 10DE 1533 |
 | NVIDIA A100 80GB PCIe                           | 20B5 10DE 1642 |
 | NVIDIA PG506-232                                | 20B6 10DE 1492 |
 | NVIDIA A30                                      | 20B7 10DE 1532 |
+| NVIDIA A30                                      | 20B7 10DE 1804 |
+| NVIDIA A30                                      | 20B7 10DE 1852 |
+| NVIDIA A800-SXM4-40GB                           | 20BD 10DE 17F4 |
 | NVIDIA A100-PCIE-40GB                           | 20F1 10DE 145F |
 | NVIDIA A800-SXM4-80GB                           | 20F3 10DE 179B |
 | NVIDIA A800-SXM4-80GB                           | 20F3 10DE 179C |
@@ -675,6 +679,11 @@ Subsystem Device ID.
 | NVIDIA A800-SXM4-80GB                           | 20F3 10DE 17A2 |
 | NVIDIA A800 80GB PCIe                           | 20F5 10DE 1799 |
 | NVIDIA A800 80GB PCIe LC                        | 20F5 10DE 179A |
+| NVIDIA A800 40GB Active                         | 20F6 1028 180A |
+| NVIDIA A800 40GB Active                         | 20F6 103C 180A |
+| NVIDIA A800 40GB Active                         | 20F6 10DE 180A |
+| NVIDIA A800 40GB Active                         | 20F6 17AA 180A |
+| NVIDIA AX800                                    | 20FD 10DE 17F8 |
 | NVIDIA GeForce GTX 1660 Ti                      | 2182           |
 | NVIDIA GeForce GTX 1660                         | 2184           |
 | NVIDIA GeForce GTX 1650 SUPER                   | 2187           |
@@ -733,6 +742,7 @@ Subsystem Device ID.
 | NVIDIA A10                                      | 2236 10DE 1482 |
 | NVIDIA A10G                                     | 2237 10DE 152F |
 | NVIDIA A10M                                     | 2238 10DE 1677 |
+| NVIDIA H100 NVL                                 | 2321 10DE 1839 |
 | NVIDIA H800 PCIe                                | 2322 10DE 17A4 |
 | NVIDIA H800                                     | 2324 10DE 17A6 |
 | NVIDIA H800                                     | 2324 10DE 17A8 |
@@ -740,6 +750,9 @@ Subsystem Device ID.
 | NVIDIA H100 80GB HBM3                           | 2330 10DE 16C1 |
 | NVIDIA H100 PCIe                                | 2331 10DE 1626 |
 | NVIDIA H100                                     | 2339 10DE 17FC |
+| NVIDIA H800 NVL                                 | 233A 10DE 183A |
+| NVIDIA GH200 120GB                              | 2342 10DE 16EB |
+| NVIDIA GH200 480GB                              | 2342 10DE 1809 |
 | NVIDIA GeForce RTX 3060 Ti                      | 2414           |
 | NVIDIA GeForce RTX 3080 Ti Laptop GPU           | 2420           |
 | NVIDIA RTX A5500 Laptop GPU                     | 2438           |
@@ -824,16 +837,26 @@ Subsystem Device ID.
 | NVIDIA RTX A2000 Embedded GPU                   | 25FA           |
 | NVIDIA RTX A500 Embedded GPU                    | 25FB           |
 | NVIDIA GeForce RTX 4090                         | 2684           |
+| NVIDIA GeForce RTX 4090 D                       | 2685           |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 1028 16A1 |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 103C 16A1 |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 10DE 16A1 |
 | NVIDIA RTX 6000 Ada Generation                  | 26B1 17AA 16A1 |
+| NVIDIA RTX 5000 Ada Generation                  | 26B2 1028 17FA |
+| NVIDIA RTX 5000 Ada Generation                  | 26B2 103C 17FA |
+| NVIDIA RTX 5000 Ada Generation                  | 26B2 10DE 17FA |
+| NVIDIA RTX 5000 Ada Generation                  | 26B2 17AA 17FA |
+| NVIDIA RTX 5880 Ada Generation                  | 26B3 10DE 1934 |
 | NVIDIA L40                                      | 26B5 10DE 169D |
 | NVIDIA L40                                      | 26B5 10DE 17DA |
+| NVIDIA L40S                                     | 26B9 10DE 1851 |
+| NVIDIA L40S                                     | 26B9 10DE 18CF |
+| NVIDIA L20                                      | 26BA 10DE 1957 |
 | NVIDIA GeForce RTX 4080                         | 2704           |
 | NVIDIA GeForce RTX 4090 Laptop GPU              | 2717           |
 | NVIDIA RTX 5000 Ada Generation Laptop GPU       | 2730           |
 | NVIDIA GeForce RTX 4090 Laptop GPU              | 2757           |
+| NVIDIA RTX 5000 Ada Generation Embedded GPU     | 2770           |
 | NVIDIA GeForce RTX 4070 Ti                      | 2782           |
 | NVIDIA GeForce RTX 4070                         | 2786           |
 | NVIDIA GeForce RTX 4080 Laptop GPU              | 27A0           |
@@ -841,17 +864,30 @@ Subsystem Device ID.
 | NVIDIA RTX 4000 SFF Ada Generation              | 27B0 103C 16FA |
 | NVIDIA RTX 4000 SFF Ada Generation              | 27B0 10DE 16FA |
 | NVIDIA RTX 4000 SFF Ada Generation              | 27B0 17AA 16FA |
+| NVIDIA RTX 4500 Ada Generation                  | 27B1 1028 180C |
+| NVIDIA RTX 4500 Ada Generation                  | 27B1 103C 180C |
+| NVIDIA RTX 4500 Ada Generation                  | 27B1 10DE 180C |
+| NVIDIA RTX 4500 Ada Generation                  | 27B1 17AA 180C |
+| NVIDIA RTX 4000 Ada Generation                  | 27B2 1028 181B |
+| NVIDIA RTX 4000 Ada Generation                  | 27B2 103C 181B |
+| NVIDIA RTX 4000 Ada Generation                  | 27B2 10DE 181B |
+| NVIDIA RTX 4000 Ada Generation                  | 27B2 17AA 181B |
+| NVIDIA L2                                       | 27B6 10DE 1933 |
 | NVIDIA L4                                       | 27B8 10DE 16CA |
 | NVIDIA L4                                       | 27B8 10DE 16EE |
 | NVIDIA RTX 4000 Ada Generation Laptop GPU       | 27BA           |
 | NVIDIA RTX 3500 Ada Generation Laptop GPU       | 27BB           |
 | NVIDIA GeForce RTX 4080 Laptop GPU              | 27E0           |
+| NVIDIA RTX 3500 Ada Generation Embedded GPU     | 27FB           |
 | NVIDIA GeForce RTX 4060 Ti                      | 2803           |
+| NVIDIA GeForce RTX 4060 Ti                      | 2805           |
 | NVIDIA GeForce RTX 4070 Laptop GPU              | 2820           |
 | NVIDIA RTX 3000 Ada Generation Laptop GPU       | 2838           |
 | NVIDIA GeForce RTX 4070 Laptop GPU              | 2860           |
+| NVIDIA GeForce RTX 4060                         | 2882           |
 | NVIDIA GeForce RTX 4060 Laptop GPU              | 28A0           |
 | NVIDIA GeForce RTX 4050 Laptop GPU              | 28A1           |
 | NVIDIA RTX 2000 Ada Generation Laptop GPU       | 28B8           |
 | NVIDIA GeForce RTX 4060 Laptop GPU              | 28E0           |
 | NVIDIA GeForce RTX 4050 Laptop GPU              | 28E1           |
+| NVIDIA RTX 2000 Ada Generation Embedded GPU     | 28F8           |
--- a/kernel-open/Kbuild
+++ b/kernel-open/Kbuild
@@ -72,7 +72,7 @@ EXTRA_CFLAGS += -I$(src)/common/inc
 EXTRA_CFLAGS += -I$(src)
 EXTRA_CFLAGS += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-error -Wno-format-extra-args
 EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
-EXTRA_CFLAGS += -DNV_VERSION_STRING=\"535.43.02\"
+EXTRA_CFLAGS += -DNV_VERSION_STRING=\"535.43.23\"

 ifneq ($(SYSSRCHOST1X),)
 EXTRA_CFLAGS += -I$(SYSSRCHOST1X)
@@ -275,6 +275,7 @@ NV_HEADER_PRESENCE_TESTS = \
 asm/opal-api.h \
 sound/hdaudio.h \
 asm/pgtable_types.h \
+ asm/page.h \
 linux/stringhash.h \
 linux/dma-map-ops.h \
 rdma/peer_mem.h \
@@ -300,7 +301,9 @@ NV_HEADER_PRESENCE_TESTS = \
 linux/vfio_pci_core.h \
 linux/mdev.h \
 soc/tegra/bpmp-abi.h \
- soc/tegra/bpmp.h
+ soc/tegra/bpmp.h \
+ linux/cc_platform.h \
+ asm/cpufeature.h

 # Filename to store the define for the header in $(1); this is only consumed by
 # the rule below that concatenates all of these together.
--- a/kernel-open/common/inc/nv-linux.h
+++ b/kernel-open/common/inc/nv-linux.h
@@ -211,6 +211,7 @@
 #include <linux/highmem.h>

 #include <linux/nodemask.h>
+#include <linux/memory.h>

 #include <linux/workqueue.h>        /* workqueue                        */
 #include "nv-kthread-q.h"           /* kthread based queue              */
@@ -510,7 +511,11 @@ static inline void nv_vfree(void *ptr, NvU64 size)

 static inline void *nv_ioremap(NvU64 phys, NvU64 size)
 {
+#if IS_ENABLED(CONFIG_INTEL_TDX_GUEST) && defined(NV_IOREMAP_DRIVER_HARDENED_PRESENT)
+    void *ptr = ioremap_driver_hardened(phys, size);
+#else
    void *ptr = ioremap(phys, size);
+#endif
    if (ptr)
        NV_MEMDBG_ADD(ptr, size);
    return ptr;
@@ -523,11 +528,11 @@ static inline void *nv_ioremap_nocache(NvU64 phys, NvU64 size)

 static inline void *nv_ioremap_cache(NvU64 phys, NvU64 size)
 {
-#if defined(NV_IOREMAP_CACHE_PRESENT)
-    void *ptr = ioremap_cache(phys, size);
-    if (ptr)
-        NV_MEMDBG_ADD(ptr, size);
-    return ptr;
+    void *ptr = NULL;
+#if IS_ENABLED(CONFIG_INTEL_TDX_GUEST) && defined(NV_IOREMAP_CACHE_SHARED_PRESENT)
+    ptr = ioremap_cache_shared(phys, size);
+#elif defined(NV_IOREMAP_CACHE_PRESENT)
+    ptr = ioremap_cache(phys, size);
 #elif defined(NVCPU_PPC64LE)
    //
    // ioremap_cache() has been only implemented correctly for ppc64le with
@@ -542,25 +547,32 @@ static inline void *nv_ioremap_cache(NvU64 phys, NvU64 size)
    // (commit 40f1ce7fb7e8, kernel 3.0+) and that covers all kernels we
    // support on power.
    //
-    void *ptr = ioremap_prot(phys, size, pgprot_val(PAGE_KERNEL));
-    if (ptr)
-        NV_MEMDBG_ADD(ptr, size);
-    return ptr;
+    ptr = ioremap_prot(phys, size, pgprot_val(PAGE_KERNEL));
 #else
    return nv_ioremap(phys, size);
 #endif
+
+    if (ptr)
+        NV_MEMDBG_ADD(ptr, size);
+
+    return ptr;
 }

 static inline void *nv_ioremap_wc(NvU64 phys, NvU64 size)
 {
-#if defined(NV_IOREMAP_WC_PRESENT)
-    void *ptr = ioremap_wc(phys, size);
-    if (ptr)
-        NV_MEMDBG_ADD(ptr, size);
-    return ptr;
+    void *ptr = NULL;
+#if IS_ENABLED(CONFIG_INTEL_TDX_GUEST) && defined(NV_IOREMAP_DRIVER_HARDENED_WC_PRESENT)
+    ptr = ioremap_driver_hardened_wc(phys, size);
+#elif defined(NV_IOREMAP_WC_PRESENT)
+    ptr = ioremap_wc(phys, size);
 #else
    return nv_ioremap_nocache(phys, size);
 #endif
+
+    if (ptr)
+        NV_MEMDBG_ADD(ptr, size);
+
+    return ptr;
 }

 static inline void nv_iounmap(void *ptr, NvU64 size)
@@ -633,37 +645,24 @@ static NvBool nv_numa_node_has_memory(int node_id)
        free_pages(ptr, order);                      \
    }

-extern NvU64 nv_shared_gpa_boundary;
+static inline pgprot_t nv_sme_clr(pgprot_t prot)
+{
+#if defined(__sme_clr)
+    return __pgprot(__sme_clr(pgprot_val(prot)));
+#else
+    return prot;
+#endif // __sme_clr
+}

 static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot, NvU32 extra)
 {
    pgprot_t prot = __pgprot(pgprot_val(vm_prot) | extra);
-#if defined(CONFIG_AMD_MEM_ENCRYPT) && defined(NV_PGPROT_DECRYPTED_PRESENT)
-    /*
-     * When AMD memory encryption is enabled, device memory mappings with the
-     * C-bit set read as 0xFF, so ensure the bit is cleared for user mappings.
-     *
-     * If cc_mkdec() is present, then pgprot_decrypted() can't be used.
-     */
-#if defined(NV_CC_MKDEC_PRESENT)
-    if (nv_shared_gpa_boundary != 0)
-    {
-        /*
-         * By design, a VM using vTOM doesn't see the SEV setting and
-         * for AMD with vTOM, *set* means decrypted.
-         */
-        prot =  __pgprot(nv_shared_gpa_boundary | (pgprot_val(vm_prot)));
-    }
-    else
-    {
-        prot =  __pgprot(__sme_clr(pgprot_val(vm_prot)));
-    }
-#else
-    prot = pgprot_decrypted(prot);
-#endif
-#endif

-    return prot;
+#if defined(pgprot_decrypted)
+    return pgprot_decrypted(prot);
+#else
+    return nv_sme_clr(prot);
+#endif // pgprot_decrypted
 }

 #if defined(PAGE_KERNEL_NOENC)
@@ -1323,7 +1322,7 @@ nv_dma_maps_swiotlb(struct device *dev)
     * SEV memory encryption") forces SWIOTLB to be enabled when AMD SEV 
     * is active in all cases.
     */
-    if (os_sev_enabled)
+    if (os_cc_enabled)
        swiotlb_in_use = NV_TRUE;
 #endif

--- a/kernel-open/common/inc/nv-mm.h
+++ b/kernel-open/common/inc/nv-mm.h
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2016-2017 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
@@ -36,12 +36,21 @@ typedef int vm_fault_t;
 * pin_user_pages() was added by commit eddb1c228f7951d399240
 * ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in v5.6-rc1 (2020-01-30)
 *
+ * Removed vmas parameter from pin_user_pages() by commit 40896a02751
+ * ("mm/gup: remove vmas parameter from pin_user_pages()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-17)
+ *
 */

 #include <linux/mm.h>
 #include <linux/sched.h>
 #if defined(NV_PIN_USER_PAGES_PRESENT)
-    #define NV_PIN_USER_PAGES pin_user_pages
+    #if defined(NV_PIN_USER_PAGES_HAS_ARGS_VMAS)
+        #define NV_PIN_USER_PAGES pin_user_pages
+    #else
+        #define NV_PIN_USER_PAGES(start, nr_pages, gup_flags, pages, vmas) \
+            pin_user_pages(start, nr_pages, gup_flags, pages)
+    #endif // NV_PIN_USER_PAGES_HAS_ARGS_VMAS
    #define NV_UNPIN_USER_PAGE unpin_user_page
 #else
    #define NV_PIN_USER_PAGES NV_GET_USER_PAGES
@@ -64,11 +73,18 @@ typedef int vm_fault_t;
 * commit 8e50b8b07f462ab4b91bc1491b1c91bd75e4ad40 which cherry-picked the
 * replacement of the write and force parameters with gup_flags
 *
+ * Removed vmas parameter from get_user_pages() by commit 7bbf9c8c99
+ * ("mm/gup: remove unused vmas parameter from get_user_pages()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-17)
+ *
 */

 #if defined(NV_GET_USER_PAGES_HAS_ARGS_FLAGS)
+    #define NV_GET_USER_PAGES(start, nr_pages, flags, pages, vmas) \
+        get_user_pages(start, nr_pages, flags, pages)
+#elif defined(NV_GET_USER_PAGES_HAS_ARGS_FLAGS_VMAS)
    #define NV_GET_USER_PAGES get_user_pages
-#elif defined(NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS)
+#elif defined(NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS_VMAS)
    #define NV_GET_USER_PAGES(start, nr_pages, flags, pages, vmas) \
        get_user_pages(current, current->mm, start, nr_pages, flags, pages, vmas)
 #else
@@ -81,13 +97,13 @@ typedef int vm_fault_t;
        int write = flags & FOLL_WRITE;
        int force = flags & FOLL_FORCE;

-    #if defined(NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE)
+    #if defined(NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS)
        return get_user_pages(start, nr_pages, write, force, pages, vmas);
    #else
-        // NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE
+        // NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS
        return get_user_pages(current, current->mm, start, nr_pages, write,
                              force, pages, vmas);
-    #endif // NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE
+    #endif // NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS
    }
 #endif // NV_GET_USER_PAGES_HAS_ARGS_FLAGS

@@ -100,15 +116,22 @@ typedef int vm_fault_t;
 * 64019a2e467a ("mm/gup: remove task_struct pointer for  all gup code")
 * in v5.9-rc1 (2020-08-11). *
 *
+ * Removed unused vmas parameter from pin_user_pages_remote() by commit
+ * 83bcc2e132("mm/gup: remove unused vmas parameter from pin_user_pages_remote()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-14)
+ *
 */

 #if defined(NV_PIN_USER_PAGES_REMOTE_PRESENT)
-    #if defined (NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK)
+    #if defined(NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS)
        #define NV_PIN_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
            pin_user_pages_remote(NULL, mm, start, nr_pages, flags, pages, vmas, locked)
-    #else
+    #elif defined(NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_VMAS)
        #define NV_PIN_USER_PAGES_REMOTE pin_user_pages_remote
-    #endif // NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK
+    #else
+        #define NV_PIN_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
+            pin_user_pages_remote(mm, start, nr_pages, flags, pages, locked)
+    #endif // NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS
 #else
    #define NV_PIN_USER_PAGES_REMOTE NV_GET_USER_PAGES_REMOTE
 #endif // NV_PIN_USER_PAGES_REMOTE_PRESENT
@@ -135,22 +158,30 @@ typedef int vm_fault_t;
 * commit 64019a2e467a ("mm/gup: remove task_struct pointer for
 * all gup code") in v5.9-rc1 (2020-08-11).
 *
+ * Removed vmas parameter from get_user_pages_remote() by commit a4bde14d549 
+ * ("mm/gup: remove vmas parameter from get_user_pages_remote()")
+ * in linux-next, expected in v6.5-rc1 (2023-05-14)
+ *
 */

 #if defined(NV_GET_USER_PAGES_REMOTE_PRESENT)
    #if defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED)
+        #define NV_GET_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
+            get_user_pages_remote(mm, start, nr_pages, flags, pages, locked)
+
+    #elif defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED_VMAS)
        #define NV_GET_USER_PAGES_REMOTE get_user_pages_remote

-    #elif defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED)
+    #elif defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED_VMAS)
        #define NV_GET_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
            get_user_pages_remote(NULL, mm, start, nr_pages, flags, pages, vmas, locked)

-    #elif defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS)
+    #elif defined(NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_VMAS)
        #define NV_GET_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
            get_user_pages_remote(NULL, mm, start, nr_pages, flags, pages, vmas)

    #else
-        // NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE
+        // NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE_VMAS
        static inline long NV_GET_USER_PAGES_REMOTE(struct mm_struct *mm,
                                                    unsigned long start,
                                                    unsigned long nr_pages,
@@ -167,7 +198,7 @@ typedef int vm_fault_t;
        }
    #endif // NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED
 #else
-    #if defined(NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE)
+    #if defined(NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS)
        static inline long NV_GET_USER_PAGES_REMOTE(struct mm_struct *mm,
                                                    unsigned long start,
                                                    unsigned long nr_pages,
@@ -185,7 +216,7 @@ typedef int vm_fault_t;
    #else
        #define NV_GET_USER_PAGES_REMOTE(mm, start, nr_pages, flags, pages, vmas, locked) \
            get_user_pages(NULL, mm, start, nr_pages, flags, pages, vmas)
-    #endif // NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE
+    #endif // NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS
 #endif // NV_GET_USER_PAGES_REMOTE_PRESENT

 /*
--- a/kernel-open/common/inc/nv.h
+++ b/kernel-open/common/inc/nv.h
@@ -510,6 +510,12 @@ struct nv_file_private_t
    nv_file_private_t *ctl_nvfp;
    void *ctl_nvfp_priv;
    NvU32 register_or_refcount;
+
+    //
+    // True if a client or an event was ever allocated on this fd.
+    // If false, RMAPI cleanup is skipped.
+    //
+    NvBool bCleanupRmapi;
 };

 // Forward define the gpu ops structures
@@ -959,6 +965,8 @@ NV_STATUS  NV_API_CALL  rm_perform_version_check  (nvidia_stack_t *, void *, NvU

 void       NV_API_CALL  rm_power_source_change_event        (nvidia_stack_t *, NvU32);

+void       NV_API_CALL  rm_request_dnotifier_state          (nvidia_stack_t *, nv_state_t *);
+
 void       NV_API_CALL  rm_disable_gpu_state_persistence    (nvidia_stack_t *sp, nv_state_t *);
 NV_STATUS  NV_API_CALL  rm_p2p_init_mapping       (nvidia_stack_t *, NvU64, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU64, NvU64, NvU64, NvU64, void (*)(void *), void *);
 NV_STATUS  NV_API_CALL  rm_p2p_destroy_mapping    (nvidia_stack_t *, NvU64);
--- a/kernel-open/common/inc/nv_uvm_interface.h
+++ b/kernel-open/common/inc/nv_uvm_interface.h
@@ -1455,12 +1455,12 @@ NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channe
    concurrently with the same UvmCslContext parameter in different threads. The caller must
    guarantee this exclusion.

-    * nvUvmInterfaceCslLogDeviceEncryption
    * nvUvmInterfaceCslRotateIv
    * nvUvmInterfaceCslEncrypt
    * nvUvmInterfaceCslDecrypt
    * nvUvmInterfaceCslSign
    * nvUvmInterfaceCslQueryMessagePool
+    * nvUvmInterfaceCslIncrementIv
 */

 /*******************************************************************************
@@ -1495,62 +1495,17 @@ NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext,
 */
 void nvUvmInterfaceDeinitCslContext(UvmCslContext *uvmCslContext);

-
-/*******************************************************************************
-    nvUvmInterfaceCslLogDeviceEncryption
-
-    Returns an IV that can be later used in the nvUvmInterfaceCslEncrypt
-    method. The IV contains a "freshness bit" which value is set by this method
-    and subsequently dirtied by nvUvmInterfaceCslEncrypt to prevent
-    non-malicious reuse of the IV.
-
-    See "CSL Interface and Locking" for locking requirements.
-    This function does not perform dynamic memory allocation.
-
-    Arguments:
-        uvmCslContext[IN/OUT] - The CSL context.
-        encryptIv[OUT]        - Parameter that is stored before a successful
-                                device encryption. It is used as an input to
-                                nvUvmInterfaceCslEncrypt.
-
-    Error codes:
-      NV_ERR_INSUFFICIENT_RESOURCES - New IV would cause a counter to overflow.
-*/
-NV_STATUS nvUvmInterfaceCslAcquireEncryptionIv(UvmCslContext *uvmCslContext,
-                                               UvmCslIv *encryptIv);
-
-/*******************************************************************************
-    nvUvmInterfaceCslLogDeviceEncryption
-
-    Logs and checks information about device encryption.
-
-    See "CSL Interface and Locking" for locking requirements.
-    This function does not perform dynamic memory allocation.
-
-    Arguments:
-        uvmCslContext[IN/OUT] - The CSL context.
-        decryptIv[OUT]        - Parameter that is stored before a successful
-                                device encryption. It is used as an input to
-                                nvUvmInterfaceCslDecrypt.
-
-    Error codes:
-      NV_ERR_INSUFFICIENT_RESOURCES - The device encryption would cause a counter
-                                      to overflow.
-*/
-NV_STATUS nvUvmInterfaceCslLogDeviceEncryption(UvmCslContext *uvmCslContext,
-                                               UvmCslIv *decryptIv);
-
 /*******************************************************************************
    nvUvmInterfaceCslRotateIv

-    Rotates the IV for a given channel and direction.
+    Rotates the IV for a given channel and operation.

    This function will rotate the IV on both the CPU and the GPU.
    Outstanding messages that have been encrypted by the GPU should first be
-    decrypted before calling this function with direction equal to
-    UVM_CSL_DIR_GPU_TO_CPU. Similiarly, outstanding messages that have been
+    decrypted before calling this function with operation equal to
+    UVM_CSL_OPERATION_DECRYPT. Similarly, outstanding messages that have been
    encrypted by the CPU should first be decrypted before calling this function
-    with direction equal to UVM_CSL_DIR_CPU_TO_GPU. For a given direction
+    with operation equal to UVM_CSL_OPERATION_ENCRYPT. For a given operation
    the channel must be idle before calling this function. This function can be
    called regardless of the value of the IV's message counter.

@@ -1559,17 +1514,17 @@ NV_STATUS nvUvmInterfaceCslLogDeviceEncryption(UvmCslContext *uvmCslContext,

 Arguments:
        uvmCslContext[IN/OUT] - The CSL context.
-        direction[IN]         - Either
-                                - UVM_CSL_DIR_CPU_TO_GPU
-                                - UVM_CSL_DIR_GPU_TO_CPU
+        operation[IN]         - Either
+                                - UVM_CSL_OPERATION_ENCRYPT
+                                - UVM_CSL_OPERATION_DECRYPT

    Error codes:
      NV_ERR_INSUFFICIENT_RESOURCES - The rotate operation would cause a counter
                                      to overflow.
-      NV_ERR_INVALID_ARGUMENT       - Invalid value for direction.
+      NV_ERR_INVALID_ARGUMENT       - Invalid value for operation.
 */
 NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext,
-                                    UvmCslDirection direction);
+                                    UvmCslOperation operation);

 /*******************************************************************************
    nvUvmInterfaceCslEncrypt
@@ -1580,7 +1535,7 @@ NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext,
    this function produces undefined behavior. Performance is typically
    maximized when the input and output buffers are 16-byte aligned. This is
    natural alignment for AES block.
-    The encryptIV can be obtained from nvUvmInterfaceCslAcquireEncryptionIv.
+    The encryptIV can be obtained from nvUvmInterfaceCslIncrementIv.
    However, it is optional. If it is NULL, the next IV in line will be used.

    See "CSL Interface and Locking" for locking requirements.
@@ -1623,12 +1578,18 @@ NV_STATUS nvUvmInterfaceCslEncrypt(UvmCslContext *uvmCslContext,

    Arguments:
        uvmCslContext[IN/OUT] - The CSL context.
-        bufferSize[IN]        - Size of the input and output buffers in
-                                units of bytes. Value can range from 1 byte
-                                to (2^32) - 1 bytes.
-        decryptIv[IN]         - Parameter given by nvUvmInterfaceCslLogDeviceEncryption.
+        bufferSize[IN]        - Size of the input and output buffers in units of bytes.
+                                Value can range from 1 byte to (2^32) - 1 bytes.
+        decryptIv[IN]         - IV used to decrypt the ciphertext. Its value can either be given by
+                                nvUvmInterfaceCslIncrementIv, or, if NULL, the CSL context's
+                                internal counter is used.
        inputBuffer[IN]       - Address of ciphertext input buffer.
        outputBuffer[OUT]     - Address of plaintext output buffer.
+        addAuthData[IN]       - Address of the plaintext additional authenticated data used to
+                                calculate the authentication tag. Can be NULL.
+        addAuthDataSize[IN]   - Size of the additional authenticated data in units of bytes.
+                                Value can range from 1 byte to (2^32) - 1 bytes.
+                                This parameter is ignored if addAuthData is NULL.
        authTagBuffer[IN]     - Address of authentication tag buffer.
                                Its size is UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES.

@@ -1643,6 +1604,8 @@ NV_STATUS nvUvmInterfaceCslDecrypt(UvmCslContext *uvmCslContext,
                                   NvU8 const *inputBuffer,
                                   UvmCslIv const *decryptIv,
                                   NvU8 *outputBuffer,
+                                   NvU8 const *addAuthData,
+                                   NvU32 addAuthDataSize,
                                   NvU8 const *authTagBuffer);

 /*******************************************************************************
@@ -1673,7 +1636,6 @@ NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext,
                                NvU8 const *inputBuffer,
                                NvU8 *authTagBuffer);

-
 /*******************************************************************************
    nvUvmInterfaceCslQueryMessagePool

@@ -1684,14 +1646,45 @@ NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext,

    Arguments:
        uvmCslContext[IN/OUT] - The CSL context.
-        direction[IN]         - Either UVM_CSL_DIR_CPU_TO_GPU or UVM_CSL_DIR_GPU_TO_CPU.
+        operation[IN]         - Either UVM_CSL_OPERATION_ENCRYPT or UVM_CSL_OPERATION_DECRYPT.
        messageNum[OUT]       - Number of messages left before overflow.

    Error codes:
-      NV_ERR_INVALID_ARGUMENT - The value of the direction parameter is illegal.
+      NV_ERR_INVALID_ARGUMENT - The value of the operation parameter is illegal.
 */
 NV_STATUS nvUvmInterfaceCslQueryMessagePool(UvmCslContext *uvmCslContext,
-                                            UvmCslDirection direction,
+                                            UvmCslOperation operation,
                                            NvU64 *messageNum);

+/*******************************************************************************
+    nvUvmInterfaceCslIncrementIv
+
+    Increments the message counter by the specified amount.
+
+    If iv is non-NULL then the incremented value is returned.
+    If operation is UVM_CSL_OPERATION_ENCRYPT then the returned IV's "freshness" bit is set and
+    can be used in nvUvmInterfaceCslEncrypt. If operation is UVM_CSL_OPERATION_DECRYPT then
+    the returned IV can be used in nvUvmInterfaceCslDecrypt.
+
+    See "CSL Interface and Locking" for locking requirements.
+    This function does not perform dynamic memory allocation.
+
+Arguments:
+        uvmCslContext[IN/OUT] - The CSL context.
+        operation[IN]         - Either
+                                - UVM_CSL_OPERATION_ENCRYPT
+                                - UVM_CSL_OPERATION_DECRYPT
+        increment[IN]         - The amount by which the IV is incremented. Can be 0.
+        iv[out]               - If non-NULL, a buffer to store the incremented IV.
+
+    Error codes:
+      NV_ERR_INVALID_ARGUMENT       - The value of the operation parameter is illegal.
+      NV_ERR_INSUFFICIENT_RESOURCES - Incrementing the message counter would result
+                                      in an overflow.
+*/
+NV_STATUS nvUvmInterfaceCslIncrementIv(UvmCslContext *uvmCslContext,
+                                       UvmCslOperation operation,
+                                       NvU64 increment,
+                                       UvmCslIv *iv);
+
 #endif // _NV_UVM_INTERFACE_H_
--- a/kernel-open/common/inc/nv_uvm_types.h
+++ b/kernel-open/common/inc/nv_uvm_types.h
@@ -286,6 +286,7 @@ typedef struct UvmGpuChannelInfo_tag
    // so a channel can be controlled via another channel (SEC2 or WLC/LCIC)
    NvU64             gpFifoGpuVa;
    NvU64             gpPutGpuVa;
+    NvU64             gpGetGpuVa;
    // GPU VA of work submission offset is needed in Confidential Computing
    // so CE channels can ring doorbell of other channels as required for
    // WLC/LCIC work submission
@@ -320,10 +321,6 @@ typedef struct UvmGpuChannelAllocParams_tag
    // The next two fields store UVM_BUFFER_LOCATION values
    NvU32 gpFifoLoc;
    NvU32 gpPutLoc;
-
-    // Allocate the channel as secure. This flag should only be set when
-    // Confidential Compute is enabled.
-    NvBool secure;
 } UvmGpuChannelAllocParams;

 typedef struct UvmGpuPagingChannelAllocParams_tag
@@ -367,9 +364,6 @@ typedef struct
    // True if the CE can be used for P2P transactions
    NvBool p2p:1;

-    // True if the CE supports encryption
-    NvBool secure:1;
-
    // Mask of physical CEs assigned to this LCE
    //
    // The value returned by RM for this field may change when a GPU is
@@ -572,8 +566,11 @@ typedef struct UvmPlatformInfo_tag
    // Out: ATS (Address Translation Services) is supported
    NvBool atsSupported;

-    // Out: AMD SEV (Secure Encrypted Virtualization) is enabled
-    NvBool sevEnabled;
+    // Out: True if HW trusted execution, such as AMD's SEV-SNP or Intel's TDX,
+    // is enabled in the VM, indicating that Confidential Computing must be
+    // also enabled in the GPU(s); these two security features are either both
+    // enabled, or both disabled.
+    NvBool confComputingEnabled;
 } UvmPlatformInfo;

 typedef struct UvmGpuClientInfo_tag
@@ -851,6 +848,14 @@ typedef union UvmFaultMetadataPacket_tag
    NvU8 _padding[32];
 } UvmFaultMetadataPacket;

+// This struct shall not be accessed nor modified directly by UVM as it is
+// entirely managed by the RM layer
+typedef struct UvmCslContext_tag
+{
+    struct ccslContext_t *ctx;
+    void *nvidia_stack;
+} UvmCslContext;
+
 typedef struct UvmGpuFaultInfo_tag
 {
    struct
@@ -908,6 +913,10 @@ typedef struct UvmGpuFaultInfo_tag
        // Confidential Computing is disabled.
        UvmFaultMetadataPacket *bufferMetadata;

+        // CSL context used for performing decryption of replayable faults when
+        // Confidential Computing is enabled.
+        UvmCslContext cslCtx;
+
        // Indicates whether UVM owns the replayable fault buffer.
        // The value of this field is always NV_TRUE When Confidential Computing
        // is disabled.
@@ -1046,24 +1055,16 @@ typedef UvmGpuPagingChannelInfo gpuPagingChannelInfo;
 typedef UvmGpuPagingChannelAllocParams gpuPagingChannelAllocParams;
 typedef UvmPmaAllocationOptions gpuPmaAllocationOptions;

-// This struct shall not be accessed nor modified directly by UVM as it is
-// entirely managed by the RM layer
-typedef struct UvmCslContext_tag
-{
-    struct ccslContext_t *ctx;
-    void *nvidia_stack;
-} UvmCslContext;
-
 typedef struct UvmCslIv
 {
    NvU8 iv[12];
    NvU8 fresh;
 } UvmCslIv;

-typedef enum UvmCslDirection
+typedef enum UvmCslOperation
 {
-    UVM_CSL_DIR_CPU_TO_GPU,
-    UVM_CSL_DIR_GPU_TO_CPU
-} UvmCslDirection;
+    UVM_CSL_OPERATION_ENCRYPT,
+    UVM_CSL_OPERATION_DECRYPT
+} UvmCslOperation;

 #endif // _NV_UVM_TYPES_H_
--- a/kernel-open/common/inc/os-interface.h
+++ b/kernel-open/common/inc/os-interface.h
@@ -214,8 +214,8 @@ NV_STATUS   NV_API_CALL  os_offline_page_at_address(NvU64 address);
 extern NvU32 os_page_size;
 extern NvU64 os_page_mask;
 extern NvU8  os_page_shift;
-extern NvU32 os_sev_status;
-extern NvBool os_sev_enabled;
+extern NvBool os_cc_enabled;
+extern NvBool os_cc_tdx_enabled;
 extern NvBool os_dma_buf_enabled;

 /*
--- a/kernel-open/common/inc/rm-gpu-ops.h
+++ b/kernel-open/common/inc/rm-gpu-ops.h
@@ -103,13 +103,12 @@ NV_STATUS  NV_API_CALL rm_gpu_ops_paging_channel_push_stream(nvidia_stack_t *, n

 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_context_init(nvidia_stack_t *, struct ccslContext_t **, nvgpuChannelHandle_t);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_context_clear(nvidia_stack_t *, struct ccslContext_t *);
-NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_log_device_encryption(nvidia_stack_t *, struct ccslContext_t *, NvU8 *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_rotate_iv(nvidia_stack_t *, struct ccslContext_t *, NvU8);
-NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_acquire_encryption_iv(nvidia_stack_t *, struct ccslContext_t *, NvU8 *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_encrypt(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 *, NvU8 *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_encrypt_with_iv(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8*, NvU8 *, NvU8 *);
-NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 const *, NvU8 *, NvU8 const *);
+NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_decrypt(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 const *, NvU8 *, NvU8 const *, NvU32, NvU8 const *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_sign(nvidia_stack_t *, struct ccslContext_t *, NvU32, NvU8 const *, NvU8 *);
 NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_query_message_pool(nvidia_stack_t *, struct ccslContext_t *, NvU8, NvU64 *);
+NV_STATUS  NV_API_CALL rm_gpu_ops_ccsl_increment_iv(nvidia_stack_t *, struct ccslContext_t *, NvU8, NvU64, NvU8 *);

 #endif
--- a/kernel-open/conftest.sh
+++ b/kernel-open/conftest.sh
@@ -445,6 +445,9 @@ compile_test() {
            #if defined(NV_ASM_PGTABLE_TYPES_H_PRESENT)
            #include <asm/pgtable_types.h>
            #endif
+            #if defined(NV_ASM_PAGE_H_PRESENT)
+            #include <asm/page.h>
+            #endif
            #include <asm/set_memory.h>
            #else
            #include <asm/cacheflush.h>
@@ -467,6 +470,9 @@ compile_test() {
            #if defined(NV_ASM_PGTABLE_TYPES_H_PRESENT)
            #include <asm/pgtable_types.h>
            #endif
+            #if defined(NV_ASM_PAGE_H_PRESENT)
+            #include <asm/page.h>
+            #endif
            #include <asm/set_memory.h>
            #else
            #include <asm/cacheflush.h>
@@ -524,6 +530,9 @@ compile_test() {
            #if defined(NV_ASM_PGTABLE_TYPES_H_PRESENT)
            #include <asm/pgtable_types.h>
            #endif
+            #if defined(NV_ASM_PAGE_H_PRESENT)
+            #include <asm/page.h>
+            #endif
            #include <asm/set_memory.h>
            #else
            #include <asm/cacheflush.h>
@@ -551,6 +560,9 @@ compile_test() {
            #if defined(NV_ASM_PGTABLE_TYPES_H_PRESENT)
            #include <asm/pgtable_types.h>
            #endif
+            #if defined(NV_ASM_PAGE_H_PRESENT)
+            #include <asm/page.h>
+            #endif
            #include <asm/set_memory.h>
            #else
            #include <asm/cacheflush.h>
@@ -695,6 +707,50 @@ compile_test() {
            compile_check_conftest "$CODE" "NV_IOREMAP_WC_PRESENT" "" "functions"
        ;;

+        ioremap_driver_hardened)
+            #
+            # Determine if the ioremap_driver_hardened() function is present.
+            # It does not exist on all architectures.
+            # TODO: Update the commit ID once the API is upstreamed.
+            #
+            CODE="
+            #include <asm/io.h>
+            void conftest_ioremap_driver_hardened(void) {
+                ioremap_driver_hardened();
+            }"
+
+            compile_check_conftest "$CODE" "NV_IOREMAP_DRIVER_HARDENED_PRESENT" "" "functions"
+        ;;
+
+        ioremap_driver_hardened_wc)
+            #
+            # Determine if the ioremap_driver_hardened_wc() function is present.
+            # It does not exist on all architectures.
+            # TODO: Update the commit ID once the API is upstreamed.
+            #
+            CODE="
+            #include <asm/io.h>
+            void conftest_ioremap_driver_hardened_wc(void) {
+                ioremap_driver_hardened_wc();
+            }"
+
+            compile_check_conftest "$CODE" "NV_IOREMAP_DRIVER_HARDENED_WC_PRESENT" "" "functions"
+        ;;
+
+        ioremap_cache_shared)
+            #
+            # Determine if the ioremap_cache_shared() function is present.
+            # It does not exist on all architectures.
+            # TODO: Update the commit ID once the API is upstreamed.
+            #
+            CODE="
+            #include <asm/io.h>
+            void conftest_ioremap_cache_shared(void) {
+                ioremap_cache_shared();
+            }"
+
+            compile_check_conftest "$CODE" "NV_IOREMAP_CACHE_SHARED_PRESENT" "" "functions"
+        ;;
        dom0_kernel_present)
            # Add config parameter if running on DOM0.
            if [ -n "$VGX_BUILD" ]; then
@@ -919,6 +975,21 @@ compile_test() {
            compile_check_conftest "$CODE" "NV_VFIO_MIGRATION_OPS_PRESENT" "" "types"
        ;;

+        vfio_precopy_info)
+            #
+            # Determine if vfio_precopy_info struct is present or not
+            #
+            # Added by commit 4db52602a6074 ("vfio: Extend the device migration
+            # protocol with PRE_COPY" in v6.2
+            #
+            CODE="
+            #include <linux/vfio.h>
+            struct vfio_precopy_info precopy_info;
+            "
+
+            compile_check_conftest "$CODE" "NV_VFIO_PRECOPY_INFO_PRESENT" "" "types"
+        ;;
+
        vfio_log_ops)
            #
            # Determine if vfio_log_ops struct is present or not
@@ -1120,6 +1191,23 @@ compile_test() {
            compile_check_conftest "$CODE" "NV_VFIO_DEVICE_OPS_HAS_DMA_UNMAP" "" "types"
        ;;

+        vfio_device_ops_has_bind_iommufd)
+            #
+            # Determine if 'vfio_device_ops' struct has 'bind_iommufd' field.
+            #
+            # Added by commit a4d1f91db5021 ("vfio-iommufd: Support iommufd
+            # for physical VFIO devices") in v6.2
+            #
+            CODE="
+            #include <linux/pci.h>
+            #include <linux/vfio.h>
+            int conftest_vfio_device_ops_has_bind_iommufd(void) {
+                return offsetof(struct vfio_device_ops, bind_iommufd);
+            }"
+
+            compile_check_conftest "$CODE" "NV_VFIO_DEVICE_OPS_HAS_BIND_IOMMUFD" "" "types"
+        ;;
+
        pci_irq_vector_helpers)
            #
            # Determine if pci_alloc_irq_vectors(), pci_free_irq_vectors()
@@ -2395,6 +2483,10 @@ compile_test() {
            # commit 768ae309a961 ("mm: replace get_user_pages() write/force
            # parameters with gup_flags") in v4.9 (2016-10-13)
            #
+            # Removed vmas parameter from get_user_pages() by commit 7bbf9c8c99
+            # ("mm/gup: remove unused vmas parameter from get_user_pages()")
+            # in linux-next, expected in v6.5-rc1
+            #
            # linux-4.4.168 cherry-picked commit 768ae309a961 without
            # c12d2da56d0e which is covered in Conftest #3.
            #
@@ -2404,22 +2496,28 @@ compile_test() {
            # passing conftest's
            #
            set_get_user_pages_defines () {
-                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE" ]; then
-                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE" ]; then
-                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS" ]; then
-                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS_VMAS" | append_conftest "functions"
+                fi
+
+                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_FLAGS_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_HAS_ARGS_FLAGS_VMAS" | append_conftest "functions"
+                else
+                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_FLAGS_VMAS" | append_conftest "functions"
                fi

                if [ "$1" = "NV_GET_USER_PAGES_HAS_ARGS_FLAGS" ]; then
@@ -2427,6 +2525,7 @@ compile_test() {
                else
                    echo "#undef NV_GET_USER_PAGES_HAS_ARGS_FLAGS" | append_conftest "functions"
                fi
+
            }

            # Conftest #1: Check if get_user_pages accepts 6 arguments.
@@ -2447,14 +2546,15 @@ compile_test() {
            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
            rm -f conftest$$.c
            if [ -f conftest$$.o ]; then
-                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE"
+                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_WRITE_FORCE_VMAS"
                rm -f conftest$$.o
                return
            fi

            # Conftest #2: Check if get_user_pages has gup_flags instead of
            # write and force parameters. And that gup doesn't accept a
-            # task_struct and mm_struct as its first arguments.
+            # task_struct and mm_struct as its first arguments. get_user_pages
+            # has vm_area_struct as its last argument.
            # Return if available.
            # Fall through to conftest #3 on failure.

@@ -2472,16 +2572,17 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_FLAGS"
+                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_FLAGS_VMAS"
                rm -f conftest$$.o
                return
            fi

            # Conftest #3: Check if get_user_pages has gup_flags instead of
-            # write and force parameters AND that gup has task_struct and
-            # mm_struct as its first arguments.
+            # write and force parameters. The gup has task_struct and
+            # mm_struct as its first arguments. get_user_pages
+            # has vm_area_struct as its last argument.
            # Return if available.
-            # Fall through to default case if absent.
+            # Fall through to conftest #4 on failure.

            echo "$CONFTEST_PREAMBLE
            #include <linux/mm.h>
@@ -2499,12 +2600,35 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS"
+                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_TSK_FLAGS_VMAS"
                rm -f conftest$$.o
                return
            fi

-            set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE"
+            # Conftest #4: gup doesn't accept a task_struct and mm_struct as 
+            # its first arguments. check if get_user_pages() does not take
+            # vmas argument.
+            # Fall through to default case otherwise.
+
+            echo "$CONFTEST_PREAMBLE
+            #include <linux/mm.h>
+            long get_user_pages(unsigned long start,
+                                unsigned long nr_pages,
+                                unsigned int gup_flags,
+                                struct page **pages) {
+                return 0;
+            }" > conftest$$.c
+
+            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
+            rm -f conftest$$.c
+
+            if [ -f conftest$$.o ]; then
+                set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_FLAGS"
+                rm -f conftest$$.o
+                return
+            fi
+
+            set_get_user_pages_defines "NV_GET_USER_PAGES_HAS_ARGS_TSK_WRITE_FORCE_VMAS"

            return
        ;;
@@ -2531,6 +2655,10 @@ compile_test() {
            # commit 64019a2e467a ("mm/gup: remove task_struct pointer for
            # all gup code") in v5.9-rc1 (2020-08-11).
            #
+            # Removed vmas parameter from get_user_pages_remote() by commit
+            # a4bde14d549 ("mm/gup: remove vmas parameter from get_user_pages_remote()")
+            # in linux-next, expected in v6.5-rc1
+            #

            #
            # This function sets the NV_GET_USER_PAGES_REMOTE_* macros as per
@@ -2543,22 +2671,28 @@ compile_test() {
                    echo "#define NV_GET_USER_PAGES_REMOTE_PRESENT" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE" ]; then
-                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE_VMAS" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS" ]; then
-                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_VMAS" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED" ]; then
-                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED" | append_conftest "functions"
+                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED" | append_conftest "functions"
+                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED_VMAS" | append_conftest "functions"
+                fi
+
+                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED_VMAS" ]; then
+                    echo "#define NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED_VMAS" | append_conftest "functions"
+                else
+                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED_VMAS" | append_conftest "functions"
                fi

                if [ "$1" = "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED" ]; then
@@ -2566,6 +2700,7 @@ compile_test() {
                else
                    echo "#undef NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED" | append_conftest "functions"
                fi
+
            }

            # conftest #1: check if get_user_pages_remote() is available
@@ -2588,8 +2723,8 @@ compile_test() {
            fi

            #
-            # conftest #2: check if get_user_pages_remote() has write and
-            # force arguments. Return if these arguments are present
+            # conftest #2: check if get_user_pages_remote() has write, force
+            # and vmas arguments. Return if these arguments are present
            # Fall through to conftest #3 if these args are absent.
            #
            echo "$CONFTEST_PREAMBLE
@@ -2609,14 +2744,14 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE"
+                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_WRITE_FORCE_VMAS"
                rm -f conftest$$.o
                return
            fi

            #
-            # conftest #3: check if get_user_pages_remote() has gpu_flags
-            # arguments. Return if these arguments are present
+            # conftest #3: check if get_user_pages_remote() has gpu_flags and
+            # vmas arguments. Return if these arguments are present
            # Fall through to conftest #4 if these args are absent.
            #
            echo "$CONFTEST_PREAMBLE
@@ -2635,13 +2770,14 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS"
+                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_VMAS"
                rm -f conftest$$.o
                return
            fi

            #
-            # conftest #4: check if get_user_pages_remote() has locked argument
+            # conftest #4: check if get_user_pages_remote() has locked and 
+            # vmas argument
            # Return if these arguments are present. Fall through to conftest #5
            # if these args are absent.
            #
@@ -2662,7 +2798,7 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED"
+                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_TSK_FLAGS_LOCKED_VMAS"
                rm -f conftest$$.o
                return
            fi
@@ -2686,10 +2822,34 @@ compile_test() {
            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
            rm -f conftest$$.c

+            if [ -f conftest$$.o ]; then
+                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED_VMAS"
+                rm -f conftest$$.o
+            fi
+
+            #
+            # conftest #6: check if get_user_pages_remote() does not take
+            # vmas argument.
+            #
+            echo "$CONFTEST_PREAMBLE
+            #include <linux/mm.h>
+            long get_user_pages_remote(struct mm_struct *mm,
+                                       unsigned long start,
+                                       unsigned long nr_pages,
+                                       unsigned int gup_flags,
+                                       struct page **pages,
+                                       int *locked) {
+                return 0;
+            }" > conftest$$.c
+
+            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
+            rm -f conftest$$.c
+
            if [ -f conftest$$.o ]; then
                set_get_user_pages_remote_defines "NV_GET_USER_PAGES_REMOTE_HAS_ARGS_FLAGS_LOCKED"
                rm -f conftest$$.o
            fi
+
        ;;

        pin_user_pages)
@@ -2701,17 +2861,65 @@ compile_test() {
            # pin_user_pages() was added by commit eddb1c228f7951d399240
            # ("mm/gup: introduce pin_user_pages*() and FOLL_PIN") in
            # v5.6-rc1 (2020-01-30)
+            #
+            # Removed vmas parameter from pin_user_pages() by commit
+            # 40896a02751("mm/gup: remove vmas parameter from pin_user_pages()")
+            # in linux-next, expected in v6.5-rc1
+
+            set_pin_user_pages_defines () {
+                if [ "$1" = "" ]; then
+                    echo "#undef NV_PIN_USER_PAGES_PRESENT" | append_conftest "functions"
+                else
+                    echo "#define NV_PIN_USER_PAGES_PRESENT" | append_conftest "functions"
+                fi
+
+                if [ "$1" = "NV_PIN_USER_PAGES_HAS_ARGS_VMAS" ]; then
+                    echo "#define NV_PIN_USER_PAGES_HAS_ARGS_VMAS" | append_conftest "functions"
+                else
+                    echo "#undef NV_PIN_USER_PAGES_HAS_ARGS_VMAS" | append_conftest "functions"
+                fi
+
+            }

            # conftest #1: check if pin_user_pages() is available
            # return if not available.
+            # Fall through to conftest #2 if it is present
            #
-            CODE="
+            echo "$CONFTEST_PREAMBLE
            #include <linux/mm.h>
            void conftest_pin_user_pages(void) {
                pin_user_pages();
-            }"
+            }" > conftest$$.c

-            compile_check_conftest "$CODE" "NV_PIN_USER_PAGES_PRESENT" "" "functions"
+            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
+            rm -f conftest$$.c
+
+            if [ -f conftest$$.o ]; then
+                set_pin_user_pages_defines ""
+                rm -f conftest$$.o
+                return
+            fi
+
+            # conftest #2: Check if pin_user_pages() has vmas argument
+            echo "$CONFTEST_PREAMBLE
+            #include <linux/mm.h>
+            long pin_user_pages(unsigned long start,
+                                unsigned long nr_pages,
+                    		    unsigned int gup_flags,
+                                struct page **pages,
+                    		    struct vm_area_struct **vmas) {
+                return 0;
+            }" > conftest$$.c
+
+            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
+            rm -f conftest$$.c
+
+            if [ -f conftest$$.o ]; then
+                set_pin_user_pages_defines "NV_PIN_USER_PAGES_HAS_ARGS_VMAS"
+                rm -f conftest$$.o
+            else
+                set_pin_user_pages_defines "NV_PIN_USER_PAGES_PRESENT"
+            fi
        ;;

        pin_user_pages_remote)
@@ -2724,6 +2932,10 @@ compile_test() {
            # pin_user_pages_remote() removed 'tsk' parameter by
            # commit 64019a2e467a ("mm/gup: remove task_struct pointer for
            # all gup code") in v5.9-rc1 (2020-08-11).
+            #
+            # Removed unused vmas parameter from pin_user_pages_remote() by
+            # commit 83bcc2e132 ("mm/gup: remove unused vmas parameter from
+            # pin_user_pages_remote()") in linux-next, expected in v6.5-rc1

            #
            # This function sets the NV_PIN_USER_PAGES_REMOTE_* macros as per
@@ -2736,10 +2948,16 @@ compile_test() {
                    echo "#define NV_PIN_USER_PAGES_REMOTE_PRESENT" | append_conftest "functions"
                fi

-                if [ "$1" = "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK" ]; then
-                    echo "#define NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK" | append_conftest "functions"
+                if [ "$1" = "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS" ]; then
+                    echo "#define NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS" | append_conftest "functions"
                else
-                    echo "#undef NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK" | append_conftest "functions"
+                    echo "#undef NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS" | append_conftest "functions"
+                fi
+
+                if [ "$1" = "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_VMAS" ]; then
+                    echo "#define NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_VMAS" | append_conftest "functions"
+                else
+                    echo "#undef NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_VMAS" | append_conftest "functions"
                fi
            }

@@ -2762,7 +2980,11 @@ compile_test() {
                return
            fi

-            # conftest #2: Check if pin_user_pages_remote() has tsk argument
+            # conftest #2: Check if pin_user_pages_remote() has tsk and
+            # vmas argument
+            # Return if these arguments are present else fall through to
+            # conftest #3
+
            echo "$CONFTEST_PREAMBLE
            #include <linux/mm.h>
            long pin_user_pages_remote(struct task_struct *tsk,
@@ -2780,11 +3002,34 @@ compile_test() {
            rm -f conftest$$.c

            if [ -f conftest$$.o ]; then
-                set_pin_user_pages_remote_defines "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK"
+                set_pin_user_pages_remote_defines "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_TSK_VMAS"
+                rm -f conftest$$.o
+                return
+            fi
+
+            # conftest #3: Check if pin_user_pages_remote() has vmas argument
+            echo "$CONFTEST_PREAMBLE
+            #include <linux/mm.h>
+            long pin_user_pages_remote(struct mm_struct *mm,
+                                       unsigned long start,
+                                       unsigned long nr_pages,
+                                       unsigned int gup_flags,
+                                       struct page **pages,
+                                       struct vm_area_struct **vmas,
+                                       int *locked) {
+                return 0;
+            }" > conftest$$.c
+
+            $CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
+            rm -f conftest$$.c
+
+            if [ -f conftest$$.o ]; then
+                set_pin_user_pages_remote_defines "NV_PIN_USER_PAGES_REMOTE_HAS_ARGS_VMAS"
                rm -f conftest$$.o
            else
                set_pin_user_pages_remote_defines "NV_PIN_USER_PAGES_REMOTE_PRESENT"
            fi
+
        ;;

        vfio_pin_pages_has_vfio_device_arg)
@@ -4699,40 +4944,22 @@ compile_test() {
            compile_check_conftest "$CODE" "NV_PCI_CHANNEL_STATE_PRESENT" "" "types"
        ;;

-        pgprot_decrypted)
+        cc_platform_has)
            #
-            # Determine if the macro 'pgprot_decrypted()' is present.
+            # Determine if 'cc_platform_has()' is present.
            #
-            # Added by commit 21729f81ce8a ("x86/mm: Provide general kernel
-            # support for memory encryption") in v4.14 (2017-07-18)
+            # Added by commit aa5a461171f9 ("x86/sev: Add an x86 version of
+            # cc_platform_has()") in v5.15.3 (2021-10-04)
            CODE="
-            #include <asm/pgtable.h>
-
-            void conftest_pgprot_decrypted(void)
-                if(pgprot_decrypted()) {}
-            }"
-
-            compile_check_conftest "$CODE" "NV_PGPROT_DECRYPTED_PRESENT" "" "functions"
-
-        ;;
-
-        cc_mkdec)
-            #
-            # Determine if cc_mkdec() is present.
-            #
-            # cc_mkdec() by commit b577f542f93c ("x86/coco: Add API to handle
-            # encryption mask) in v5.18-rc1 (2022-02-22).
-            #
-            CODE="
-            #if defined(NV_ASM_COCO_H_PRESENT)
-            #include <asm/coco.h>
+            #if defined(NV_LINUX_CC_PLATFORM_H_PRESENT)
+            #include <linux/cc_platform.h>
            #endif

-            void conftest_cc_mkdec(void) {
-                cc_mkdec();
+            void conftest_cc_platfrom_has(void) {
+                cc_platform_has();
            }"

-            compile_check_conftest "$CODE" "NV_CC_MKDEC_PRESENT" "" "functions"
+            compile_check_conftest "$CODE" "NV_CC_PLATFORM_PRESENT" "" "functions"
        ;;

        drm_prime_pages_to_sg_has_drm_device_arg)
@@ -6052,6 +6279,83 @@ compile_test() {
            compile_check_conftest "$CODE" "NV_MEMORY_FAILURE_MF_SW_SIMULATED_DEFINED" "" "types"
        ;;

+        crypto)
+            #
+            # Determine if we support various crypto functions.
+            # This test is not complete and may return false positive.
+            #
+            CODE="
+	    #include <crypto/akcipher.h>
+	    #include <crypto/algapi.h>
+	    #include <crypto/ecc_curve.h>
+	    #include <crypto/ecdh.h>
+	    #include <crypto/hash.h>
+	    #include <crypto/internal/ecc.h>
+	    #include <crypto/kpp.h>
+	    #include <crypto/public_key.h>
+	    #include <crypto/sm3.h>
+	    #include <keys/asymmetric-type.h>
+	    #include <linux/crypto.h>
+            void conftest_crypto(void) {
+                struct shash_desc sd;
+                struct crypto_shash cs;
+                (void)crypto_shash_tfm_digest;
+            }"
+
+            compile_check_conftest "$CODE" "NV_CRYPTO_PRESENT" "" "symbols"
+        ;;
+
+        mempolicy_has_unified_nodes)
+            #
+            # Determine if the 'mempolicy' structure has
+            # nodes union.
+            #
+            # nodes field was added by commit 269fbe72cd ("mm/mempolicy:
+            # use unified 'nodes' for bind/interleave/prefer policies") in
+            # v5.14 (2021-06-30).
+            #
+            CODE="
+            #include <linux/mempolicy.h>
+            int conftest_mempolicy_has_unified_nodes(void) {
+                return offsetof(struct mempolicy, nodes);
+            }"
+
+            compile_check_conftest "$CODE" "NV_MEMPOLICY_HAS_UNIFIED_NODES" "" "types"
+        ;;
+
+        mempolicy_has_home_node)
+            #
+            # Determine if the 'mempolicy' structure has
+            # home_node field.
+            #
+            # home_node field was added by commit c6018b4b254
+            # ("mm/mempolicy: add set_mempolicy_home_node syscall") in v5.17
+            # (2022-01-14).
+            #
+            CODE="
+            #include <linux/mempolicy.h>
+            int conftest_mempolicy_has_home_node(void) {
+                return offsetof(struct mempolicy, home_node);
+            }"
+
+            compile_check_conftest "$CODE" "NV_MEMPOLICY_HAS_HOME_NODE" "" "types"
+        ;;
+
+        mmu_interval_notifier)
+            #
+            # Determine if mmu_interval_notifier struct is present or not
+            #
+            # Added by commit 99cb252f5 ("mm/mmu_notifier: add an interval tree
+            # notifier") in v5.10 (2019-11-12).
+            #
+            CODE="
+            #include <linux/mmu_notifier.h>
+            struct mmu_interval_notifier interval_notifier;
+            "
+
+            compile_check_conftest "$CODE" "NV_MMU_INTERVAL_NOTIFIER" "" "types"
+        ;;
+
        # When adding a new conftest entry, please use the correct format for
        # specifying the relevant upstream Linux kernel commit.
        #
@@ -6385,8 +6689,8 @@ case "$5" in
            if [ "$VFIO_IOMMU_PRESENT" != "0" ] && [ "$KVM_PRESENT" != "0" ] ; then

                # On x86_64, vGPU requires MDEV framework to be present.
-                # On aarch64, vGPU requires vfio-pci-core framework to be present.
-                if ([ "$ARCH" = "arm64" ] && [ "$VFIO_PCI_CORE_PRESENT" != "0" ]) ||
+                # On aarch64, vGPU requires MDEV or vfio-pci-core framework to be present.
+                if ([ "$ARCH" = "arm64" ] && ([ "$VFIO_MDEV_PRESENT" != "0" ] || [ "$VFIO_PCI_CORE_PRESENT" != "0" ])) ||
                   ([ "$ARCH" = "x86_64" ] && [ "$VFIO_MDEV_PRESENT" != "0" ];) then
                    exit 0
                fi
@@ -6398,8 +6702,8 @@ case "$5" in
                echo "CONFIG_VFIO_IOMMU_TYPE1";
            fi

-            if [ "$ARCH" = "arm64" ] &&  [ "$VFIO_PCI_CORE_PRESENT" = "0" ]; then
-                echo "CONFIG_VFIO_PCI_CORE";
+            if [ "$ARCH" = "arm64" ] && [ "$VFIO_MDEV_PRESENT" = "0" ] && [ "$VFIO_PCI_CORE_PRESENT" = "0" ]; then
+                echo "either CONFIG_VFIO_MDEV or CONFIG_VFIO_PCI_CORE";
            fi

            if [ "$ARCH" = "x86_64" ] && [ "$VFIO_MDEV_PRESENT" = "0" ]; then
--- a/kernel-open/nvidia-drm/nvidia-drm-drv.c
+++ b/kernel-open/nvidia-drm/nvidia-drm-drv.c
@@ -1367,8 +1367,23 @@ static struct drm_driver nv_drm_driver = {
    .ioctls                 = nv_drm_ioctls,
    .num_ioctls             = ARRAY_SIZE(nv_drm_ioctls),

+/*
+ * linux-next commit 71a7974ac701 ("drm/prime: Unexport helpers for fd/handle
+ * conversion") unexports drm_gem_prime_handle_to_fd() and
+ * drm_gem_prime_fd_to_handle().
+ *
+ * Prior linux-next commit 6b85aa68d9d5 ("drm: Enable PRIME import/export for
+ * all drivers") made these helpers the default when .prime_handle_to_fd /
+ * .prime_fd_to_handle are unspecified, so it's fine to just skip specifying
+ * them if the helpers aren't present.
+ */
+#if NV_IS_EXPORT_SYMBOL_PRESENT_drm_gem_prime_handle_to_fd
    .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
+#endif
+#if NV_IS_EXPORT_SYMBOL_PRESENT_drm_gem_prime_fd_to_handle
    .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
+#endif
+
    .gem_prime_import       = nv_drm_gem_prime_import,
    .gem_prime_import_sg_table = nv_drm_gem_prime_import_sg_table,

--- a/kernel-open/nvidia-drm/nvidia-drm-gem.h
+++ b/kernel-open/nvidia-drm/nvidia-drm-gem.h
@@ -179,6 +179,7 @@ static inline int nv_drm_gem_handle_create(struct drm_file *filp,
    return drm_gem_handle_create(filp, &nv_gem->base, handle);
 }

+#if defined(NV_DRM_FENCE_AVAILABLE)
 static inline nv_dma_resv_t *nv_drm_gem_res_obj(struct nv_drm_gem_object *nv_gem)
 {
 #if defined(NV_DRM_GEM_OBJECT_HAS_RESV)
@@ -187,6 +188,7 @@ static inline nv_dma_resv_t *nv_drm_gem_res_obj(struct nv_drm_gem_object *nv_gem
    return nv_gem->base.dma_buf ? nv_gem->base.dma_buf->resv : &nv_gem->resv;
 #endif
 }
+#endif

 void nv_drm_gem_object_init(struct nv_drm_device *nv_dev,
                            struct nv_drm_gem_object *nv_gem,
--- a/kernel-open/nvidia-drm/nvidia-drm.Kbuild
+++ b/kernel-open/nvidia-drm/nvidia-drm.Kbuild
@@ -54,6 +54,8 @@ NV_CONFTEST_GENERIC_COMPILE_TESTS += drm_atomic_available
 NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_gpl_refcount_inc
 NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_gpl_refcount_dec_and_test
 NV_CONFTEST_GENERIC_COMPILE_TESTS += drm_alpha_blending_available
+NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_present_drm_gem_prime_fd_to_handle
+NV_CONFTEST_GENERIC_COMPILE_TESTS += is_export_symbol_present_drm_gem_prime_handle_to_fd

 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_dev_unref
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_reinit_primary_mode_group
--- a/kernel-open/nvidia-modeset/nvidia-modeset-linux.c
+++ b/kernel-open/nvidia-modeset/nvidia-modeset-linux.c
@@ -65,6 +65,9 @@
 static bool output_rounding_fix = true;
 module_param_named(output_rounding_fix, output_rounding_fix, bool, 0400);

+static bool disable_vrr_memclk_switch = false;
+module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0400);
+
 /* These parameters are used for fault injection tests.  Normally the defaults
 * should be used. */
 MODULE_PARM_DESC(fail_malloc, "Fail the Nth call to nvkms_alloc");
@@ -91,6 +94,11 @@ NvBool nvkms_output_rounding_fix(void)
    return output_rounding_fix;
 }

+NvBool nvkms_disable_vrr_memclk_switch(void)
+{
+    return disable_vrr_memclk_switch;
+}
+
 #define NVKMS_SYNCPT_STUBS_NEEDED

 /*************************************************************************
--- a/kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h
+++ b/kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h
@@ -98,6 +98,8 @@ typedef struct {

 NvBool nvkms_output_rounding_fix(void);

+NvBool nvkms_disable_vrr_memclk_switch(void);
+
 void   nvkms_call_rm    (void *ops);
 void*  nvkms_alloc      (size_t size,
                         NvBool zero);
--- a/kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
+++ b/kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
@@ -108,5 +108,8 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_device_range
 NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
 NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_mm_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_pt_regs_arg
+NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
+NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
+NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_interval_notifier

 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg
--- a/kernel-open/nvidia-uvm/uvm_ampere_ce.c
+++ b/kernel-open/nvidia-uvm/uvm_ampere_ce.c
@@ -121,6 +121,8 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
        return true;

    if (uvm_channel_is_proxy(push->channel)) {
+        uvm_pushbuffer_t *pushbuffer;
+
        if (dst.is_virtual) {
            UVM_ERR_PRINT("Destination address of memcopy must be physical, not virtual\n");
            return false;
@@ -142,7 +144,8 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
            return false;
        }

-        push_begin_gpu_va = uvm_pushbuffer_get_gpu_va_for_push(push->channel->pool->manager->pushbuffer, push);
+        pushbuffer = uvm_channel_get_pushbuffer(push->channel);
+        push_begin_gpu_va = uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);

        if ((src.address < push_begin_gpu_va) || (src.address >= push_begin_gpu_va + uvm_push_get_size(push))) {
            UVM_ERR_PRINT("Source address of memcopy must point to pushbuffer\n");
@@ -177,10 +180,13 @@ bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t
 // irrespective of the virtualization mode.
 void uvm_hal_ampere_ce_memcopy_patch_src_c6b5(uvm_push_t *push, uvm_gpu_address_t *src)
 {
+    uvm_pushbuffer_t *pushbuffer;
+
    if (!uvm_channel_is_proxy(push->channel))
        return;

-    src->address -= uvm_pushbuffer_get_gpu_va_for_push(push->channel->pool->manager->pushbuffer, push);
+    pushbuffer = uvm_channel_get_pushbuffer(push->channel);
+    src->address -= uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
 }

 bool uvm_hal_ampere_ce_memset_is_valid_c6b5(uvm_push_t *push,
--- a/kernel-open/nvidia-uvm/uvm_ats.c
+++ b/kernel-open/nvidia-uvm/uvm_ats.c
@@ -44,6 +44,8 @@ void uvm_ats_init(const UvmPlatformInfo *platform_info)

 void uvm_ats_init_va_space(uvm_va_space_t *va_space)
 {
+    uvm_init_rwsem(&va_space->ats.lock, UVM_LOCK_ORDER_LEAF);
+
    if (UVM_ATS_IBM_SUPPORTED())
        uvm_ats_ibm_init_va_space(va_space);
 }
--- a/kernel-open/nvidia-uvm/uvm_ats.h
+++ b/kernel-open/nvidia-uvm/uvm_ats.h
@@ -28,17 +28,32 @@
 #include "uvm_forward_decl.h"
 #include "uvm_ats_ibm.h"
 #include "nv_uvm_types.h"
+#include "uvm_lock.h"

    #include "uvm_ats_sva.h"

    #define UVM_ATS_SUPPORTED() (UVM_ATS_IBM_SUPPORTED() || UVM_ATS_SVA_SUPPORTED())

+// ATS prefetcher uses hmm_range_fault() to query residency information.
+// hmm_range_fault() needs CONFIG_HMM_MIRROR. To detect racing CPU invalidates
+// of memory regions while hmm_range_fault() is being called, MMU interval
+// notifiers are needed.
+    #if defined(CONFIG_HMM_MIRROR) && defined(NV_MMU_INTERVAL_NOTIFIER)
+        #define UVM_ATS_PREFETCH_SUPPORTED() 1
+    #else
+        #define UVM_ATS_PREFETCH_SUPPORTED() 0
+    #endif
+
 typedef struct
 {
    // Mask of gpu_va_spaces which are registered for ATS access. The mask is
    // indexed by gpu->id. This mask is protected by the VA space lock.
    uvm_processor_mask_t registered_gpu_va_spaces;

+    // Protects racing invalidates in the VA space while hmm_range_fault() is
+    // being called in ats_compute_residency_mask().
+    uvm_rw_semaphore_t lock;
+
    union
    {
        uvm_ibm_va_space_t ibm;
--- a/kernel-open/nvidia-uvm/uvm_ats_faults.c
+++ b/kernel-open/nvidia-uvm/uvm_ats_faults.c
@@ -20,66 +20,26 @@
    DEALINGS IN THE SOFTWARE.
 *******************************************************************************/

+#include "uvm_api.h"
 #include "uvm_tools.h"
 #include "uvm_va_range.h"
+#include "uvm_ats.h"
 #include "uvm_ats_faults.h"
 #include "uvm_migrate_pageable.h"
+#include <linux/nodemask.h>
+#include <linux/mempolicy.h>
+#include <linux/mmu_notifier.h>

-// TODO: Bug 2103669: Implement a real prefetching policy and remove or adapt
-// these experimental parameters. These are intended to help guide that policy.
-static unsigned int uvm_exp_perf_prefetch_ats_order_replayable = 0;
-module_param(uvm_exp_perf_prefetch_ats_order_replayable, uint, 0644);
-MODULE_PARM_DESC(uvm_exp_perf_prefetch_ats_order_replayable,
-                 "Max order of pages (2^N) to prefetch on replayable ATS faults");
-
-static unsigned int uvm_exp_perf_prefetch_ats_order_non_replayable = 0;
-module_param(uvm_exp_perf_prefetch_ats_order_non_replayable, uint, 0644);
-MODULE_PARM_DESC(uvm_exp_perf_prefetch_ats_order_non_replayable,
-                 "Max order of pages (2^N) to prefetch on non-replayable ATS faults");
-
-// Expand the fault region to the naturally-aligned region with order given by
-// the module parameters, clamped to the vma containing fault_addr (if any).
-// Note that this means the region contains fault_addr but may not begin at
-// fault_addr.
-static void expand_fault_region(struct vm_area_struct *vma,
-                                NvU64 start,
-                                size_t length,
-                                uvm_fault_client_type_t client_type,
-                                unsigned long *migrate_start,
-                                unsigned long *migrate_length)
-{
-    unsigned int order;
-    unsigned long outer, aligned_start, aligned_size;
-
-    *migrate_start = start;
-    *migrate_length = length;
-
-    if (client_type == UVM_FAULT_CLIENT_TYPE_HUB)
-        order = uvm_exp_perf_prefetch_ats_order_non_replayable;
-    else
-        order = uvm_exp_perf_prefetch_ats_order_replayable;
-
-    if (order == 0)
-        return;
-
-    UVM_ASSERT(vma);
-    UVM_ASSERT(order < BITS_PER_LONG - PAGE_SHIFT);
-
-    aligned_size = (1UL << order) * PAGE_SIZE;
-
-    aligned_start = start & ~(aligned_size - 1);
-
-    *migrate_start = max(vma->vm_start, aligned_start);
-    outer = min(vma->vm_end, aligned_start + aligned_size);
-    *migrate_length = outer - *migrate_start;
-}
+#if UVM_ATS_PREFETCH_SUPPORTED()
+#include <linux/hmm.h>
+#endif

 static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,
                                    struct vm_area_struct *vma,
                                    NvU64 start,
                                    size_t length,
                                    uvm_fault_access_type_t access_type,
-                                    uvm_fault_client_type_t client_type)
+                                    uvm_ats_fault_context_t *ats_context)
 {
    uvm_va_space_t *va_space = gpu_va_space->va_space;
    struct mm_struct *mm = va_space->va_space_mm.mm;
@@ -95,17 +55,18 @@ static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,
    // 2) guest physical -> host physical
    //
    // The overall ATS translation will fault if either of those translations is
-    // invalid. The get_user_pages() call above handles translation #1, but not
-    // #2. We don't know if we're running as a guest, but in case we are we can
-    // force that translation to be valid by touching the guest physical address
-    // from the CPU. If the translation is not valid then the access will cause
-    // a hypervisor fault. Note that dma_map_page() can't establish mappings
-    // used by GPU ATS SVA translations. GPU accesses to host physical addresses
-    // obtained as a result of the address translation request uses the CPU
-    // address space instead of the IOMMU address space since the translated
-    // host physical address isn't necessarily an IOMMU address. The only way to
-    // establish guest physical to host physical mapping in the CPU address
-    // space is to touch the page from the CPU.
+    // invalid. The pin_user_pages() call within uvm_migrate_pageable() call
+    // below handles translation #1, but not #2. We don't know if we're running
+    // as a guest, but in case we are we can force that translation to be valid
+    // by touching the guest physical address from the CPU. If the translation
+    // is not valid then the access will cause a hypervisor fault. Note that
+    // dma_map_page() can't establish mappings used by GPU ATS SVA translations.
+    // GPU accesses to host physical addresses obtained as a result of the
+    // address translation request uses the CPU address space instead of the
+    // IOMMU address space since the translated host physical address isn't
+    // necessarily an IOMMU address. The only way to establish guest physical to
+    // host physical mapping in the CPU address space is to touch the page from
+    // the CPU.
    //
    // We assume that the hypervisor mappings are all VM_PFNMAP, VM_SHARED, and
    // VM_WRITE, meaning that the mappings are all granted write access on any
@@ -116,21 +77,22 @@ static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space,

    uvm_migrate_args_t uvm_migrate_args =
    {
-        .va_space               = va_space,
-        .mm                     = mm,
-        .dst_id                 = gpu_va_space->gpu->parent->id,
-        .dst_node_id            = -1,
-        .populate_permissions   = write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY,
-        .touch                  = true,
-        .skip_mapped            = true,
-        .user_space_start       = &user_space_start,
-        .user_space_length      = &user_space_length,
+        .va_space                       = va_space,
+        .mm                             = mm,
+        .dst_id                         = ats_context->residency_id,
+        .dst_node_id                    = ats_context->residency_node,
+        .start                          = start,
+        .length                         = length,
+        .populate_permissions           = write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY,
+        .touch                          = true,
+        .skip_mapped                    = true,
+        .populate_on_cpu_alloc_failures = true,
+        .user_space_start               = &user_space_start,
+        .user_space_length              = &user_space_length,
    };

    UVM_ASSERT(uvm_ats_can_service_faults(gpu_va_space, mm));

-    expand_fault_region(vma, start, length, client_type, &uvm_migrate_args.start, &uvm_migrate_args.length);
-
    // We are trying to use migrate_vma API in the kernel (if it exists) to
    // populate and map the faulting region on the GPU. We want to do this only
    // on the first touch. That is, pages which are not already mapped. So, we
@@ -165,6 +127,308 @@ static void flush_tlb_write_faults(uvm_gpu_va_space_t *gpu_va_space,
    uvm_tlb_batch_invalidate(&ats_invalidate->write_faults_tlb_batch, addr, size, PAGE_SIZE, UVM_MEMBAR_NONE);
 }

+static void ats_batch_select_residency(uvm_gpu_va_space_t *gpu_va_space,
+                                       struct vm_area_struct *vma,
+                                       uvm_ats_fault_context_t *ats_context)
+{
+    uvm_gpu_t *gpu = gpu_va_space->gpu;
+    int residency = uvm_gpu_numa_node(gpu);
+
+#if defined(NV_MEMPOLICY_HAS_UNIFIED_NODES)
+    struct mempolicy *vma_policy = vma_policy(vma);
+    unsigned short mode;
+
+    ats_context->prefetch_state.has_preferred_location = false;
+
+    // It's safe to read vma_policy since the mmap_lock is held in at least read
+    // mode in this path.
+    uvm_assert_mmap_lock_locked(vma->vm_mm);
+
+    if (!vma_policy)
+        goto done;
+
+    mode = vma_policy->mode;
+
+    if ((mode == MPOL_BIND) || (mode == MPOL_PREFERRED_MANY) || (mode == MPOL_PREFERRED)) {
+        int home_node = NUMA_NO_NODE;
+
+#if defined(NV_MEMPOLICY_HAS_HOME_NODE)
+        if ((mode != MPOL_PREFERRED) && (vma_policy->home_node != NUMA_NO_NODE))
+            home_node = vma_policy->home_node;
+#endif
+
+        // Prefer home_node if set. Otherwise, prefer the faulting GPU if it's
+        // in the list of preferred nodes, else prefer the closest_cpu_numa_node
+        // to the GPU if closest_cpu_numa_node is in the list of preferred
+        // nodes. Fallback to the faulting GPU if all else fails.
+        if (home_node != NUMA_NO_NODE) {
+            residency = home_node;
+        }
+        else if (!node_isset(residency, vma_policy->nodes)) {
+            int closest_cpu_numa_node = gpu->parent->closest_cpu_numa_node;
+
+            if ((closest_cpu_numa_node != NUMA_NO_NODE) && node_isset(closest_cpu_numa_node, vma_policy->nodes))
+                residency = gpu->parent->closest_cpu_numa_node;
+            else
+                residency = first_node(vma_policy->nodes);
+        }
+
+        if (!nodes_empty(vma_policy->nodes))
+            ats_context->prefetch_state.has_preferred_location = true;
+    }
+
+    // Update gpu if residency is not the faulting gpu.
+    if (residency != uvm_gpu_numa_node(gpu))
+        gpu = uvm_va_space_find_gpu_with_memory_node_id(gpu_va_space->va_space, residency);
+
+done:
+#else
+    ats_context->prefetch_state.has_preferred_location = false;
+#endif
+
+    ats_context->residency_id = gpu ? gpu->parent->id : UVM_ID_CPU;
+    ats_context->residency_node = residency;
+}
+
+static void get_range_in_vma(struct vm_area_struct *vma, NvU64 base, NvU64 *start, NvU64 *end)
+{
+    *start = max(vma->vm_start, (unsigned long) base);
+    *end = min(vma->vm_end, (unsigned long) (base + UVM_VA_BLOCK_SIZE));
+}
+
+static uvm_page_index_t uvm_ats_cpu_page_index(NvU64 base, NvU64 addr)
+{
+    UVM_ASSERT(addr >= base);
+    UVM_ASSERT(addr <= (base + UVM_VA_BLOCK_SIZE));
+
+    return (addr - base) / PAGE_SIZE;
+}
+
+// start and end must be aligned to PAGE_SIZE and must fall within
+// [base, base + UVM_VA_BLOCK_SIZE]
+static uvm_va_block_region_t uvm_ats_region_from_start_end(NvU64 start, NvU64 end)
+{
+    // base can be greater than, less than or equal to the start of a VMA.
+    NvU64 base = UVM_VA_BLOCK_ALIGN_DOWN(start);
+
+    UVM_ASSERT(start < end);
+    UVM_ASSERT(PAGE_ALIGNED(start));
+    UVM_ASSERT(PAGE_ALIGNED(end));
+    UVM_ASSERT(IS_ALIGNED(base, UVM_VA_BLOCK_SIZE));
+
+    return uvm_va_block_region(uvm_ats_cpu_page_index(base, start), uvm_ats_cpu_page_index(base, end));
+}
+
+static uvm_va_block_region_t uvm_ats_region_from_vma(struct vm_area_struct *vma, NvU64 base)
+{
+    NvU64 start;
+    NvU64 end;
+
+    get_range_in_vma(vma, base, &start, &end);
+
+    return uvm_ats_region_from_start_end(start, end);
+}
+
+#if UVM_ATS_PREFETCH_SUPPORTED()
+
+static bool uvm_ats_invalidate_notifier(struct mmu_interval_notifier *mni, unsigned long cur_seq)
+{
+    uvm_ats_fault_context_t *ats_context = container_of(mni, uvm_ats_fault_context_t, prefetch_state.notifier);
+    uvm_va_space_t *va_space = ats_context->prefetch_state.va_space;
+
+    // The following write lock protects against concurrent invalidates while
+    // hmm_range_fault() is being called in ats_compute_residency_mask().
+    uvm_down_write(&va_space->ats.lock);
+
+    mmu_interval_set_seq(mni, cur_seq);
+
+    uvm_up_write(&va_space->ats.lock);
+
+    return true;
+}
+
+static bool uvm_ats_invalidate_notifier_entry(struct mmu_interval_notifier *mni,
+                                              const struct mmu_notifier_range *range,
+                                              unsigned long cur_seq)
+{
+    UVM_ENTRY_RET(uvm_ats_invalidate_notifier(mni, cur_seq));
+}
+
+static const struct mmu_interval_notifier_ops uvm_ats_notifier_ops =
+{
+    .invalidate = uvm_ats_invalidate_notifier_entry,
+};
+
+#endif
+
+static NV_STATUS ats_compute_residency_mask(uvm_gpu_va_space_t *gpu_va_space,
+                                            struct vm_area_struct *vma,
+                                            NvU64 base,
+                                            uvm_ats_fault_context_t *ats_context)
+{
+    NV_STATUS status = NV_OK;
+
+#if UVM_ATS_PREFETCH_SUPPORTED()
+    int ret;
+    NvU64 start;
+    NvU64 end;
+    uvm_page_mask_t *residency_mask = &ats_context->prefetch_state.residency_mask;
+    struct hmm_range range;
+    uvm_page_index_t page_index;
+    uvm_va_block_region_t vma_region;
+    uvm_va_space_t *va_space = gpu_va_space->va_space;
+    struct mm_struct *mm = va_space->va_space_mm.mm;
+
+    uvm_assert_rwsem_locked_read(&va_space->lock);
+
+    ats_context->prefetch_state.first_touch = true;
+
+    uvm_page_mask_zero(residency_mask);
+
+    get_range_in_vma(vma, base, &start, &end);
+
+    vma_region = uvm_ats_region_from_start_end(start, end);
+
+    range.notifier = &ats_context->prefetch_state.notifier;
+    range.start = start;
+    range.end = end;
+    range.hmm_pfns = ats_context->prefetch_state.pfns;
+    range.default_flags = 0;
+    range.pfn_flags_mask = 0;
+    range.dev_private_owner = NULL;
+
+    ats_context->prefetch_state.va_space = va_space;
+
+    // mmu_interval_notifier_insert() will try to acquire mmap_lock for write
+    // and will deadlock since mmap_lock is already held for read in this path.
+    // This is prevented by calling __mmu_notifier_register() during va_space
+    // creation. See the comment in uvm_mmu_notifier_register() for more
+    // details.
+    ret = mmu_interval_notifier_insert(range.notifier, mm, start, end, &uvm_ats_notifier_ops);
+    if (ret)
+        return errno_to_nv_status(ret);
+
+    while (true) {
+        range.notifier_seq = mmu_interval_read_begin(range.notifier);
+        ret = hmm_range_fault(&range);
+        if (ret == -EBUSY)
+            continue;
+        if (ret) {
+            status = errno_to_nv_status(ret);
+            UVM_ASSERT(status != NV_OK);
+            break;
+        }
+
+        uvm_down_read(&va_space->ats.lock);
+
+        // Pages may have been freed or re-allocated after hmm_range_fault() is
+        // called. So the PTE might point to a different page or nothing. In the
+        // memory hot-unplug case it is not safe to call page_to_nid() on the
+        // page as the struct page itself may have been freed. To protect
+        // against these cases, uvm_ats_invalidate_entry() blocks on va_space
+        // ATS write lock for concurrent invalidates since va_space ATS lock is
+        // held for read in this path.
+        if (!mmu_interval_read_retry(range.notifier, range.notifier_seq))
+            break;
+
+        uvm_up_read(&va_space->ats.lock);
+    }
+
+    if (status == NV_OK) {
+        for_each_va_block_page_in_region(page_index, vma_region) {
+            unsigned long pfn = ats_context->prefetch_state.pfns[page_index - vma_region.first];
+
+            if (pfn & HMM_PFN_VALID) {
+                struct page *page = hmm_pfn_to_page(pfn);
+
+                if (page_to_nid(page) == ats_context->residency_node)
+                    uvm_page_mask_set(residency_mask, page_index);
+
+                ats_context->prefetch_state.first_touch = false;
+            }
+        }
+
+        uvm_up_read(&va_space->ats.lock);
+    }
+
+    mmu_interval_notifier_remove(range.notifier);
+
+#endif
+
+    return status;
+}
+
+static void ats_expand_fault_region(uvm_gpu_va_space_t *gpu_va_space,
+                                    struct vm_area_struct *vma,
+                                    uvm_ats_fault_context_t *ats_context,
+                                    uvm_va_block_region_t max_prefetch_region,
+                                    uvm_page_mask_t *faulted_mask)
+{
+    uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask;
+    uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask;
+    uvm_page_mask_t *residency_mask = &ats_context->prefetch_state.residency_mask;
+    uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask;
+    uvm_perf_prefetch_bitmap_tree_t *bitmap_tree = &ats_context->prefetch_state.bitmap_tree;
+
+    if (uvm_page_mask_empty(faulted_mask))
+        return;
+
+    uvm_perf_prefetch_compute_ats(gpu_va_space->va_space,
+                                  faulted_mask,
+                                  uvm_va_block_region_from_mask(NULL, faulted_mask),
+                                  max_prefetch_region,
+                                  residency_mask,
+                                  bitmap_tree,
+                                  prefetch_mask);
+
+    uvm_page_mask_or(read_fault_mask, read_fault_mask, prefetch_mask);
+
+    if (vma->vm_flags & VM_WRITE)
+        uvm_page_mask_or(write_fault_mask, write_fault_mask, prefetch_mask);
+}
+
+static NV_STATUS ats_fault_prefetch(uvm_gpu_va_space_t *gpu_va_space,
+                                    struct vm_area_struct *vma,
+                                    NvU64 base,
+                                    uvm_ats_fault_context_t *ats_context)
+{
+    NV_STATUS status = NV_OK;
+    uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask;
+    uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask;
+    uvm_page_mask_t *faulted_mask = &ats_context->faulted_mask;
+    uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask;
+    uvm_va_block_region_t max_prefetch_region = uvm_ats_region_from_vma(vma, base);
+
+    if (!uvm_perf_prefetch_enabled(gpu_va_space->va_space))
+        return status;
+
+    if (uvm_page_mask_empty(faulted_mask))
+        return status;
+
+    status = ats_compute_residency_mask(gpu_va_space, vma, base, ats_context);
+    if (status != NV_OK)
+        return status;
+
+    // Prefetch the entire region if none of the pages are resident on any node
+    // and if preferred_location is the faulting GPU.
+    if (ats_context->prefetch_state.has_preferred_location &&
+        ats_context->prefetch_state.first_touch &&
+        uvm_id_equal(ats_context->residency_id, gpu_va_space->gpu->parent->id)) {
+
+        uvm_page_mask_init_from_region(prefetch_mask, max_prefetch_region, NULL);
+        uvm_page_mask_or(read_fault_mask, read_fault_mask, prefetch_mask);
+
+        if (vma->vm_flags & VM_WRITE)
+            uvm_page_mask_or(write_fault_mask, write_fault_mask, prefetch_mask);
+
+        return status;
+    }
+
+    ats_expand_fault_region(gpu_va_space, vma, ats_context, max_prefetch_region, faulted_mask);
+
+    return status;
+}
+
 NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
                                 struct vm_area_struct *vma,
                                 NvU64 base,
@@ -205,6 +469,10 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
            uvm_page_mask_zero(write_fault_mask);
    }

+    ats_batch_select_residency(gpu_va_space, vma, ats_context);
+
+    ats_fault_prefetch(gpu_va_space, vma, base, ats_context);
+
    for_each_va_block_subregion_in_mask(subregion, write_fault_mask, region) {
        NvU64 start = base + (subregion.first * PAGE_SIZE);
        size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
@@ -215,7 +483,7 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
        UVM_ASSERT(start >= vma->vm_start);
        UVM_ASSERT((start + length) <= vma->vm_end);

-        status = service_ats_faults(gpu_va_space, vma, start, length, access_type, client_type);
+        status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context);
        if (status != NV_OK)
            return status;

@@ -244,11 +512,12 @@ NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space,
    for_each_va_block_subregion_in_mask(subregion, read_fault_mask, region) {
        NvU64 start = base + (subregion.first * PAGE_SIZE);
        size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE;
+        uvm_fault_access_type_t access_type = UVM_FAULT_ACCESS_TYPE_READ;

        UVM_ASSERT(start >= vma->vm_start);
        UVM_ASSERT((start + length) <= vma->vm_end);

-        status = service_ats_faults(gpu_va_space, vma, start, length, UVM_FAULT_ACCESS_TYPE_READ, client_type);
+        status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context);
        if (status != NV_OK)
            return status;

--- a/kernel-open/nvidia-uvm/uvm_ce_test.c
+++ b/kernel-open/nvidia-uvm/uvm_ce_test.c
@@ -338,11 +338,6 @@ static NV_STATUS test_memcpy_and_memset_inner(uvm_gpu_t *gpu,
        return NV_OK;
    }

-    if (!gpu->parent->ce_hal->memcopy_is_valid(&push, dst, src)) {
-        TEST_NV_CHECK_RET(uvm_push_end_and_wait(&push));
-        return NV_OK;
-    }
-
    // The input virtual addresses exist in UVM's internal address space, not
    // the proxy address space
    if (uvm_channel_is_proxy(push.channel)) {
@@ -401,7 +396,7 @@ static NV_STATUS test_memcpy_and_memset_inner(uvm_gpu_t *gpu,
 static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
 {
    NV_STATUS status = NV_OK;
-    bool is_proxy_va_space;
+    bool is_proxy_va_space = false;
    uvm_gpu_address_t gpu_verif_addr;
    void *cpu_verif_addr;
    uvm_mem_t *verif_mem = NULL;
@@ -437,6 +432,34 @@ static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
        }
    }

+    // Virtual address (in UVM's internal address space) backed by sysmem
+    TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_SYS, size, 0, &sys_rm_mem), done);
+    gpu_addresses[0] = uvm_rm_mem_get_gpu_va(sys_rm_mem, gpu, is_proxy_va_space);
+
+    if (uvm_conf_computing_mode_enabled(gpu)) {
+        for (i = 0; i < iterations; ++i) {
+            for (s = 0; s < ARRAY_SIZE(element_sizes); s++) {
+                TEST_NV_CHECK_GOTO(test_memcpy_and_memset_inner(gpu,
+                                                                gpu_addresses[0],
+                                                                gpu_addresses[0],
+                                                                size,
+                                                                element_sizes[s],
+                                                                gpu_verif_addr,
+                                                                cpu_verif_addr,
+                                                                i),
+                                    done);
+
+            }
+        }
+
+        // Because gpu_verif_addr is in sysmem, when the Confidential
+        // Computing feature is enabled, only the previous cases are valid.
+        // TODO: Bug 3839176: the test partially waived on Confidential
+        // Computing because it assumes that GPU can access system memory
+        // without using encryption.
+        goto done;
+    }
+
    // Using a page size equal to the allocation size ensures that the UVM
    // memories about to be allocated are physically contiguous. And since the
    // size is a valid GPU page size, the memories can be virtually mapped on
@@ -448,37 +471,22 @@ static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
    // Physical address in sysmem
    TEST_NV_CHECK_GOTO(uvm_mem_alloc(&mem_params, &sys_uvm_mem), done);
    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_phys(sys_uvm_mem, gpu), done);
-    gpu_addresses[0] = uvm_mem_gpu_address_physical(sys_uvm_mem, gpu, 0, size);
+    gpu_addresses[1] = uvm_mem_gpu_address_physical(sys_uvm_mem, gpu, 0, size);

    // Physical address in vidmem
    mem_params.backing_gpu = gpu;
    TEST_NV_CHECK_GOTO(uvm_mem_alloc(&mem_params, &gpu_uvm_mem), done);
-    gpu_addresses[1] = uvm_mem_gpu_address_physical(gpu_uvm_mem, gpu, 0, size);
+    gpu_addresses[2] = uvm_mem_gpu_address_physical(gpu_uvm_mem, gpu, 0, size);

    // Virtual address (in UVM's internal address space) backed by vidmem
    TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_GPU, size, 0, &gpu_rm_mem), done);
-    is_proxy_va_space = false;
-    gpu_addresses[2] = uvm_rm_mem_get_gpu_va(gpu_rm_mem, gpu, is_proxy_va_space);
+    gpu_addresses[3] = uvm_rm_mem_get_gpu_va(gpu_rm_mem, gpu, is_proxy_va_space);

-    // Virtual address (in UVM's internal address space) backed by sysmem
-    TEST_NV_CHECK_GOTO(uvm_rm_mem_alloc(gpu, UVM_RM_MEM_TYPE_SYS, size, 0, &sys_rm_mem), done);
-    gpu_addresses[3] = uvm_rm_mem_get_gpu_va(sys_rm_mem, gpu, is_proxy_va_space);

    for (i = 0; i < iterations; ++i) {
        for (j = 0; j < ARRAY_SIZE(gpu_addresses); ++j) {
            for (k = 0; k < ARRAY_SIZE(gpu_addresses); ++k) {
                for (s = 0; s < ARRAY_SIZE(element_sizes); s++) {
-                  // Because gpu_verif_addr is in sysmem, when the Confidential
-                  // Computing feature is enabled, only the following cases are
-                  // valid.
-                  //
-                  // TODO: Bug 3839176: the test partially waived on
-                  // Confidential Computing because it assumes that GPU can
-                  // access system memory without using encryption.
-                  if (uvm_conf_computing_mode_enabled(gpu) &&
-                      !(gpu_addresses[k].is_unprotected && gpu_addresses[j].is_unprotected)) {
-                        continue;
-                  }
                    TEST_NV_CHECK_GOTO(test_memcpy_and_memset_inner(gpu,
                                                                    gpu_addresses[k],
                                                                    gpu_addresses[j],
@@ -752,7 +760,7 @@ static NV_STATUS alloc_vidmem_protected(uvm_gpu_t *gpu, uvm_mem_t **mem, size_t

    *mem = NULL;

-    TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem_protected(size, gpu, mem));
+    TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem(size, gpu, mem));
    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
    TEST_NV_CHECK_GOTO(zero_vidmem(*mem), err);

--- a/kernel-open/nvidia-uvm/uvm_channel.c
+++ b/kernel-open/nvidia-uvm/uvm_channel.c
--- a/kernel-open/nvidia-uvm/uvm_channel.h
+++ b/kernel-open/nvidia-uvm/uvm_channel.h
@@ -104,16 +104,14 @@ typedef enum
    // ----------------------------------
    // Channel type with fixed schedules

-    // Work Launch Channel (WLC) is a specialized channel
-    // for launching work on other channels when
-    // Confidential Computing is enabled.
-    // It is paired with LCIC (below)
+    // Work Launch Channel (WLC) is a specialized channel for launching work on
+    // other channels when the Confidential Computing is feature enabled. It is
+    // paired with LCIC (below)
    UVM_CHANNEL_TYPE_WLC,

-    // Launch Confirmation Indicator Channel (LCIC) is a
-    // specialized channel with fixed schedule. It gets
-    // triggered by executing WLC work, and makes sure that
-    // WLC get/put pointers are up-to-date.
+    // Launch Confirmation Indicator Channel (LCIC) is a specialized channel
+    // with fixed schedule. It gets triggered by executing WLC work, and makes
+    // sure that WLC get/put pointers are up-to-date.
    UVM_CHANNEL_TYPE_LCIC,

    UVM_CHANNEL_TYPE_COUNT,
@@ -242,11 +240,9 @@ typedef struct
    DECLARE_BITMAP(push_locks, UVM_CHANNEL_MAX_NUM_CHANNELS_PER_POOL);

    // Counting semaphore for available and unlocked channels, it must be
-    // acquired before submitting work to a secure channel.
+    // acquired before submitting work to a channel when the Confidential
+    // Computing feature is enabled.
    uvm_semaphore_t push_sem;
-
-    // See uvm_channel_is_secure() documentation.
-    bool secure;
 } uvm_channel_pool_t;

 struct uvm_channel_struct
@@ -304,8 +300,9 @@ struct uvm_channel_struct
        // its internal operation and each push may modify this state.
        uvm_mutex_t push_lock;

-        // Every secure channel has cryptographic state in HW, which is
-        // mirrored here for CPU-side operations.
+        // When the Confidential Computing feature is enabled, every channel has
+        // cryptographic state in HW, which is mirrored here for CPU-side
+        // operations.
        UvmCslContext ctx;
        bool is_ctx_initialized;

@@ -355,6 +352,13 @@ struct uvm_channel_struct
        // Encryption auth tags have to be located in unprotected sysmem.
        void *launch_auth_tag_cpu;
        NvU64 launch_auth_tag_gpu_va;
+
+        // Used to decrypt the push back to protected sysmem.
+        // This happens when profilers register callbacks for migration data.
+        uvm_push_crypto_bundle_t *push_crypto_bundles;
+
+        // Accompanying authentication tags for the crypto bundles
+        uvm_rm_mem_t *push_crypto_bundle_auth_tags;
    } conf_computing;

    // RM channel information
@@ -452,46 +456,28 @@ struct uvm_channel_manager_struct
 // Create a channel manager for the GPU
 NV_STATUS uvm_channel_manager_create(uvm_gpu_t *gpu, uvm_channel_manager_t **manager_out);

-static bool uvm_channel_pool_is_ce(uvm_channel_pool_t *pool);
-
-// A channel is secure if it has HW encryption capabilities.
-//
-// Secure channels are treated differently in the UVM driver. Each secure
-// channel has a unique CSL context associated with it, has relatively
-// restrictive reservation policies (in comparison with non-secure channels),
-// it is requested to be allocated differently by RM, etc.
-static bool uvm_channel_pool_is_secure(uvm_channel_pool_t *pool)
+static bool uvm_pool_type_is_valid(uvm_channel_pool_type_t pool_type)
 {
-    return pool->secure;
-}
-
-static bool uvm_channel_is_secure(uvm_channel_t *channel)
-{
-    return uvm_channel_pool_is_secure(channel->pool);
+    return (is_power_of_2(pool_type) && (pool_type < UVM_CHANNEL_POOL_TYPE_MASK));
 }

 static bool uvm_channel_pool_is_sec2(uvm_channel_pool_t *pool)
 {
-    UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
+    UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));

    return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_SEC2);
 }

-static bool uvm_channel_pool_is_secure_ce(uvm_channel_pool_t *pool)
-{
-    return uvm_channel_pool_is_secure(pool) && uvm_channel_pool_is_ce(pool);
-}
-
 static bool uvm_channel_pool_is_wlc(uvm_channel_pool_t *pool)
 {
-    UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
+    UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));

    return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_WLC);
 }

 static bool uvm_channel_pool_is_lcic(uvm_channel_pool_t *pool)
 {
-    UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
+    UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));

    return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_LCIC);
 }
@@ -501,11 +487,6 @@ static bool uvm_channel_is_sec2(uvm_channel_t *channel)
    return uvm_channel_pool_is_sec2(channel->pool);
 }

-static bool uvm_channel_is_secure_ce(uvm_channel_t *channel)
-{
-    return uvm_channel_pool_is_secure_ce(channel->pool);
-}
-
 static bool uvm_channel_is_wlc(uvm_channel_t *channel)
 {
    return uvm_channel_pool_is_wlc(channel->pool);
@@ -516,12 +497,9 @@ static bool uvm_channel_is_lcic(uvm_channel_t *channel)
    return uvm_channel_pool_is_lcic(channel->pool);
 }

-bool uvm_channel_type_requires_secure_pool(uvm_gpu_t *gpu, uvm_channel_type_t channel_type);
-NV_STATUS uvm_channel_secure_init(uvm_gpu_t *gpu, uvm_channel_t *channel);
-
 static bool uvm_channel_pool_is_proxy(uvm_channel_pool_t *pool)
 {
-    UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
+    UVM_ASSERT(uvm_pool_type_is_valid(pool->pool_type));

    return pool->pool_type == UVM_CHANNEL_POOL_TYPE_CE_PROXY;
 }
@@ -533,11 +511,7 @@ static bool uvm_channel_is_proxy(uvm_channel_t *channel)

 static bool uvm_channel_pool_is_ce(uvm_channel_pool_t *pool)
 {
-    UVM_ASSERT(pool->pool_type < UVM_CHANNEL_POOL_TYPE_MASK);
-    if (uvm_channel_pool_is_wlc(pool) || uvm_channel_pool_is_lcic(pool))
-        return true;
-
-    return (pool->pool_type == UVM_CHANNEL_POOL_TYPE_CE) || uvm_channel_pool_is_proxy(pool);
+    return !uvm_channel_pool_is_sec2(pool);
 }

 static bool uvm_channel_is_ce(uvm_channel_t *channel)
@@ -679,6 +653,11 @@ static uvm_gpu_t *uvm_channel_get_gpu(uvm_channel_t *channel)
    return channel->pool->manager->gpu;
 }

+static uvm_pushbuffer_t *uvm_channel_get_pushbuffer(uvm_channel_t *channel)
+{
+    return channel->pool->manager->pushbuffer;
+}
+
 // Index of a channel within the owning pool
 static unsigned uvm_channel_index_in_pool(const uvm_channel_t *channel)
 {
--- a/kernel-open/nvidia-uvm/uvm_channel_test.c
+++ b/kernel-open/nvidia-uvm/uvm_channel_test.c
@@ -681,9 +681,10 @@ done:
 }

 // The following test is inspired by uvm_push_test.c:test_concurrent_pushes.
-// This test verifies that concurrent pushes using the same secure channel pool
-// select different channels.
-NV_STATUS test_secure_channel_selection(uvm_va_space_t *va_space)
+// This test verifies that concurrent pushes using the same channel pool
+// select different channels, when the Confidential Computing feature is
+// enabled.
+NV_STATUS test_conf_computing_channel_selection(uvm_va_space_t *va_space)
 {
    NV_STATUS status = NV_OK;
    uvm_channel_pool_t *pool;
@@ -703,9 +704,6 @@ NV_STATUS test_secure_channel_selection(uvm_va_space_t *va_space)
        uvm_channel_type_t channel_type;

        for (channel_type = 0; channel_type < UVM_CHANNEL_TYPE_COUNT; channel_type++) {
-            if (!uvm_channel_type_requires_secure_pool(gpu, channel_type))
-                continue;
-
            pool = gpu->channel_manager->pool_to_use.default_for_type[channel_type];
            TEST_CHECK_RET(pool != NULL);

@@ -997,7 +995,7 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct
    if (status != NV_OK)
        goto done;

-    status = test_secure_channel_selection(va_space);
+    status = test_conf_computing_channel_selection(va_space);
    if (status != NV_OK)
        goto done;

--- a/kernel-open/nvidia-uvm/uvm_conf_computing.c
+++ b/kernel-open/nvidia-uvm/uvm_conf_computing.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2021 NVIDIA Corporation
+    Copyright (c) 2021-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -26,6 +26,7 @@
 #include "uvm_conf_computing.h"
 #include "uvm_kvmalloc.h"
 #include "uvm_gpu.h"
+#include "uvm_hal.h"
 #include "uvm_mem.h"
 #include "uvm_processors.h"
 #include "uvm_tracker.h"
@@ -53,24 +54,26 @@ bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu)
    return uvm_conf_computing_get_mode(gpu->parent) == UVM_GPU_CONF_COMPUTE_MODE_HCC;
 }

-NV_STATUS uvm_conf_computing_init_parent_gpu(const uvm_parent_gpu_t *parent)
+void uvm_conf_computing_check_parent_gpu(const uvm_parent_gpu_t *parent)
 {
-    UvmGpuConfComputeMode cc, sys_cc;
-    uvm_gpu_t *first;
+    uvm_gpu_t *first_gpu;

    uvm_assert_mutex_locked(&g_uvm_global.global_lock);

-    // TODO: Bug 2844714.
-    // Since we have no routine to traverse parent gpus,
+    // The Confidential Computing state of the GPU should match that of the
+    // system.
+    UVM_ASSERT(uvm_conf_computing_mode_enabled_parent(parent) == g_uvm_global.conf_computing_enabled);
+
+    // TODO: Bug 2844714: since we have no routine to traverse parent GPUs,
    // find first child GPU and get its parent.
-    first = uvm_global_processor_mask_find_first_gpu(&g_uvm_global.retained_gpus);
-    if (!first)
-        return NV_OK;
+    first_gpu = uvm_global_processor_mask_find_first_gpu(&g_uvm_global.retained_gpus);
+    if (first_gpu == NULL)
+        return;

-    sys_cc = uvm_conf_computing_get_mode(first->parent);
-    cc = uvm_conf_computing_get_mode(parent);
-
-    return cc == sys_cc ? NV_OK : NV_ERR_NOT_SUPPORTED;
+    // All GPUs derive Confidential Computing status from their parent. By
+    // current policy all parent GPUs have identical Confidential Computing
+    // status.
+    UVM_ASSERT(uvm_conf_computing_get_mode(parent) == uvm_conf_computing_get_mode(first_gpu->parent));
 }

 static void dma_buffer_destroy_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
@@ -378,11 +381,12 @@ void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv)
    NV_STATUS status;

    uvm_mutex_lock(&channel->csl.ctx_lock);
-    status = nvUvmInterfaceCslLogDeviceEncryption(&channel->csl.ctx, iv);
+    status = nvUvmInterfaceCslIncrementIv(&channel->csl.ctx, UVM_CSL_OPERATION_DECRYPT, 1, iv);
    uvm_mutex_unlock(&channel->csl.ctx_lock);

-    // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
-    // overflows. This is not supposed to happen on CC.
+    // TODO: Bug 4014720: If nvUvmInterfaceCslIncrementIv returns with
+    // NV_ERR_INSUFFICIENT_RESOURCES then the IV needs to be rotated via
+    // nvUvmInterfaceCslRotateIv.
    UVM_ASSERT(status == NV_OK);
 }

@@ -391,11 +395,12 @@ void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *
    NV_STATUS status;

    uvm_mutex_lock(&channel->csl.ctx_lock);
-    status = nvUvmInterfaceCslAcquireEncryptionIv(&channel->csl.ctx, iv);
+    status = nvUvmInterfaceCslIncrementIv(&channel->csl.ctx, UVM_CSL_OPERATION_ENCRYPT, 1, iv);
    uvm_mutex_unlock(&channel->csl.ctx_lock);

-    // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
-    // overflows. This is not supposed to happen on CC.
+    // TODO: Bug 4014720: If nvUvmInterfaceCslIncrementIv returns with
+    // NV_ERR_INSUFFICIENT_RESOURCES then the IV needs to be rotated via
+    // nvUvmInterfaceCslRotateIv.
    UVM_ASSERT(status == NV_OK);
 }

@@ -439,8 +444,58 @@ NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
                                      (const NvU8 *) src_cipher,
                                      src_iv,
                                      (NvU8 *) dst_plain,
+                                      NULL,
+                                      0,
                                      (const NvU8 *) auth_tag_buffer);
    uvm_mutex_unlock(&channel->csl.ctx_lock);

    return status;
 }
+
+NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu,
+                                           void *dst_plain,
+                                           const void *src_cipher,
+                                           const void *auth_tag_buffer,
+                                           NvU8 valid)
+{
+    NV_STATUS status;
+
+    // There is no dedicated lock for the CSL context associated with replayable
+    // faults. The mutual exclusion required by the RM CSL API is enforced by
+    // relying on the GPU replayable service lock (ISR lock), since fault
+    // decryption is invoked as part of fault servicing.
+    UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
+
+    UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
+
+    status = nvUvmInterfaceCslDecrypt(&parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx,
+                                      parent_gpu->fault_buffer_hal->entry_size(parent_gpu),
+                                      (const NvU8 *) src_cipher,
+                                      NULL,
+                                      (NvU8 *) dst_plain,
+                                      &valid,
+                                      sizeof(valid),
+                                      (const NvU8 *) auth_tag_buffer);
+
+    if (status != NV_OK)
+        UVM_ERR_PRINT("nvUvmInterfaceCslDecrypt() failed: %s, GPU %s\n", nvstatusToString(status), parent_gpu->name);
+
+    return status;
+}
+
+void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu, NvU64 increment)
+{
+    NV_STATUS status;
+
+    // See comment in uvm_conf_computing_fault_decrypt
+    UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
+
+    UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
+
+    status = nvUvmInterfaceCslIncrementIv(&parent_gpu->fault_buffer_info.rm_info.replayable.cslCtx,
+                                          UVM_CSL_OPERATION_DECRYPT,
+                                          increment,
+                                          NULL);
+
+    UVM_ASSERT(status == NV_OK);
+}
--- a/kernel-open/nvidia-uvm/uvm_conf_computing.h
+++ b/kernel-open/nvidia-uvm/uvm_conf_computing.h
@@ -42,9 +42,11 @@
 // Use sizeof(UvmCslIv) to refer to the IV size.
 #define UVM_CONF_COMPUTING_IV_ALIGNMENT 16

-// SEC2 decrypt operation buffers are required to be 16-bytes aligned. CE
-// encrypt/decrypt can be unaligned if the buffer lies in a single 32B segment.
-// Otherwise, they need to be 32B aligned.
+// SEC2 decrypt operation buffers are required to be 16-bytes aligned.
+#define UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT 16
+
+// CE encrypt/decrypt can be unaligned if the entire buffer lies in a single
+// 32B segment. Otherwise, it needs to be 32B aligned.
 #define UVM_CONF_COMPUTING_BUF_ALIGNMENT 32

 #define UVM_CONF_COMPUTING_DMA_BUFFER_SIZE UVM_VA_BLOCK_SIZE
@@ -58,10 +60,8 @@
 // UVM_METHOD_SIZE * 2 * 10 = 80.
 #define UVM_CONF_COMPUTING_SIGN_BUF_MAX_SIZE 80

-// All GPUs derive confidential computing status from their parent.
-// By current policy all parent GPUs have identical confidential
-// computing status.
-NV_STATUS uvm_conf_computing_init_parent_gpu(const uvm_parent_gpu_t *parent);
+void uvm_conf_computing_check_parent_gpu(const uvm_parent_gpu_t *parent);
+
 bool uvm_conf_computing_mode_enabled_parent(const uvm_parent_gpu_t *parent);
 bool uvm_conf_computing_mode_enabled(const uvm_gpu_t *gpu);
 bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu);
@@ -175,4 +175,28 @@ NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
                                         const UvmCslIv *src_iv,
                                         size_t size,
                                         const void *auth_tag_buffer);
+
+// CPU decryption of a single replayable fault, encrypted by GSP-RM.
+//
+// Replayable fault decryption depends not only on the encrypted fault contents,
+// and the authentication tag, but also on the plaintext valid bit associated
+// with the fault.
+//
+// When decrypting data previously encrypted by the Copy Engine, use
+// uvm_conf_computing_cpu_decrypt instead.
+//
+// Locking: this function must be invoked while holding the replayable ISR lock.
+NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu,
+                                           void *dst_plain,
+                                           const void *src_cipher,
+                                           const void *auth_tag_buffer,
+                                           NvU8 valid);
+
+// Increment the CPU-side decrypt IV of the CSL context associated with
+// replayable faults. The function is a no-op if the given increment is zero.
+//
+// The IV associated with a fault CSL context is a 64-bit counter.
+//
+// Locking: this function must be invoked while holding the replayable ISR lock.
+void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu, NvU64 increment);
 #endif // __UVM_CONF_COMPUTING_H__
--- a/kernel-open/nvidia-uvm/uvm_forward_decl.h
+++ b/kernel-open/nvidia-uvm/uvm_forward_decl.h
@@ -50,6 +50,7 @@ typedef struct uvm_channel_struct uvm_channel_t;
 typedef struct uvm_user_channel_struct uvm_user_channel_t;
 typedef struct uvm_push_struct uvm_push_t;
 typedef struct uvm_push_info_struct uvm_push_info_t;
+typedef struct uvm_push_crypto_bundle_struct uvm_push_crypto_bundle_t;
 typedef struct uvm_push_acquire_info_struct uvm_push_acquire_info_t;
 typedef struct uvm_pushbuffer_struct uvm_pushbuffer_t;
 typedef struct uvm_gpfifo_entry_struct uvm_gpfifo_entry_t;
--- a/kernel-open/nvidia-uvm/uvm_global.c
+++ b/kernel-open/nvidia-uvm/uvm_global.c
@@ -71,11 +71,6 @@ static void uvm_unregister_callbacks(void)
    }
 }

-static void sev_init(const UvmPlatformInfo *platform_info)
-{
-    g_uvm_global.sev_enabled = platform_info->sevEnabled;
-}
-
 NV_STATUS uvm_global_init(void)
 {
    NV_STATUS status;
@@ -124,8 +119,7 @@ NV_STATUS uvm_global_init(void)

    uvm_ats_init(&platform_info);
    g_uvm_global.num_simulated_devices = 0;
-
-    sev_init(&platform_info);
+    g_uvm_global.conf_computing_enabled = platform_info.confComputingEnabled;

    status = uvm_gpu_init();
    if (status != NV_OK) {
--- a/kernel-open/nvidia-uvm/uvm_global.h
+++ b/kernel-open/nvidia-uvm/uvm_global.h
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2021 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -143,11 +143,16 @@ struct uvm_global_struct
        struct page *page;
    } unload_state;

-    // AMD Secure Encrypted Virtualization (SEV) status. True if VM has SEV
-    // enabled. This field is set once during global initialization
-    // (uvm_global_init), and can be read afterwards without acquiring any
-    // locks.
-    bool sev_enabled;
+    // True if the VM has AMD's SEV, or equivalent HW security extensions such
+    // as Intel's TDX, enabled. The flag is always false on the host.
+    //
+    // This value moves in tandem with that of Confidential Computing in the
+    // GPU(s) in all supported configurations, so it is used as a proxy for the
+    // Confidential Computing state.
+    //
+    // This field is set once during global initialization (uvm_global_init),
+    // and can be read afterwards without acquiring any locks.
+    bool conf_computing_enabled;
 };

 // Initialize global uvm state
--- a/kernel-open/nvidia-uvm/uvm_gpu.c
+++ b/kernel-open/nvidia-uvm/uvm_gpu.c
@@ -1099,12 +1099,7 @@ static NV_STATUS init_parent_gpu(uvm_parent_gpu_t *parent_gpu,
        return status;
    }

-    status = uvm_conf_computing_init_parent_gpu(parent_gpu);
-    if (status != NV_OK) {
-        UVM_ERR_PRINT("Confidential computing: %s, GPU %s\n",
-                      nvstatusToString(status), parent_gpu->name);
-        return status;
-    }
+    uvm_conf_computing_check_parent_gpu(parent_gpu);

    parent_gpu->pci_dev = gpu_platform_info->pci_dev;
    parent_gpu->closest_cpu_numa_node = dev_to_node(&parent_gpu->pci_dev->dev);
--- a/kernel-open/nvidia-uvm/uvm_gpu.h
+++ b/kernel-open/nvidia-uvm/uvm_gpu.h
@@ -46,6 +46,7 @@
 #include "uvm_rb_tree.h"
 #include "uvm_perf_prefetch.h"
 #include "nv-kthread-q.h"
+#include <linux/mmu_notifier.h>
 #include "uvm_conf_computing.h"

 // Buffer length to store uvm gpu id, RM device name and gpu uuid.
@@ -192,12 +193,52 @@ typedef struct
    // Mask of successfully serviced read faults on pages in write_fault_mask.
    uvm_page_mask_t reads_serviced_mask;

-    // Temporary mask used for uvm_page_mask_or_equal. This is used since
-    // bitmap_or_equal() isn't present in all linux kernel versions.
-    uvm_page_mask_t tmp_mask;
+    // Mask of all faulted pages in a UVM_VA_BLOCK_SIZE aligned region of a
+    // SAM VMA. This is used as input to the prefetcher.
+    uvm_page_mask_t faulted_mask;

    // Client type of the service requestor.
    uvm_fault_client_type_t client_type;
+
+    // New residency ID of the faulting region.
+    uvm_processor_id_t residency_id;
+
+    // New residency NUMA node ID of the faulting region.
+    int residency_node;
+
+    struct
+    {
+        // True if preferred_location was set on this faulting region.
+        // UVM_VA_BLOCK_SIZE sized region in the faulting region bound by the
+        // VMA is is prefetched if preferred_location was set and if first_touch
+        // is true;
+        bool has_preferred_location;
+
+        // True if the UVM_VA_BLOCK_SIZE sized region isn't resident on any
+        // node. False if any page in the region is resident somewhere.
+        bool first_touch;
+
+        // Mask of prefetched pages in a UVM_VA_BLOCK_SIZE aligned region of a
+        // SAM VMA.
+        uvm_page_mask_t prefetch_pages_mask;
+
+        // PFN info of the faulting region
+        unsigned long pfns[PAGES_PER_UVM_VA_BLOCK];
+
+        // Faulting/preferred processor residency mask of the faulting region.
+        uvm_page_mask_t residency_mask;
+
+#if defined(NV_MMU_INTERVAL_NOTIFIER)
+        // MMU notifier used to compute residency of this faulting region.
+        struct mmu_interval_notifier notifier;
+#endif
+
+        uvm_va_space_t *va_space;
+
+        // Prefetch temporary state.
+        uvm_perf_prefetch_bitmap_tree_t bitmap_tree;
+    } prefetch_state;
+
 } uvm_ats_fault_context_t;

 struct uvm_fault_service_batch_context_struct
--- a/kernel-open/nvidia-uvm/uvm_gpu_access_counters.c
+++ b/kernel-open/nvidia-uvm/uvm_gpu_access_counters.c
@@ -1009,6 +1009,7 @@ static NV_STATUS service_va_block_locked(uvm_processor_id_t processor,
        NvU64 address = uvm_va_block_cpu_page_address(va_block, page_index);
        bool read_duplicate = false;
        uvm_processor_id_t new_residency;
+        const uvm_va_policy_t *policy;

        // Ensure that the migratability iterator covers the current address
        while (iter.end < address)
@@ -1035,21 +1036,23 @@ static NV_STATUS service_va_block_locked(uvm_processor_id_t processor,

        // If the underlying VMA is gone, skip HMM migrations.
        if (uvm_va_block_is_hmm(va_block)) {
-            status = uvm_hmm_find_vma(&service_context->block_context, address);
+            status = uvm_hmm_find_vma(service_context->block_context.mm,
+                                      &service_context->block_context.hmm.vma,
+                                      address);
            if (status == NV_ERR_INVALID_ADDRESS)
                continue;

            UVM_ASSERT(status == NV_OK);
        }

-        service_context->block_context.policy = uvm_va_policy_get(va_block, address);
+        policy = uvm_va_policy_get(va_block, address);

        new_residency = uvm_va_block_select_residency(va_block,
                                                      &service_context->block_context,
                                                      page_index,
                                                      processor,
                                                      uvm_fault_access_type_mask_bit(UVM_FAULT_ACCESS_TYPE_PREFETCH),
-                                                      service_context->block_context.policy,
+                                                      policy,
                                                      &thrashing_hint,
                                                      UVM_SERVICE_OPERATION_ACCESS_COUNTERS,
                                                      &read_duplicate);
@@ -1094,12 +1097,17 @@ static NV_STATUS service_va_block_locked(uvm_processor_id_t processor,
        if (!uvm_processor_mask_empty(&service_context->resident_processors)) {
            while (first_page_index <= last_page_index) {
                uvm_page_index_t outer = last_page_index + 1;
+                const uvm_va_policy_t *policy;

                if (uvm_va_block_is_hmm(va_block)) {
-                    status = uvm_hmm_find_policy_vma_and_outer(va_block,
-                                                               &service_context->block_context,
-                                                               first_page_index,
-                                                               &outer);
+                    status = NV_ERR_INVALID_ADDRESS;
+                    if (service_context->block_context.mm) {
+                        status = uvm_hmm_find_policy_vma_and_outer(va_block,
+                                                                   &service_context->block_context.hmm.vma,
+                                                                   first_page_index,
+                                                                   &policy,
+                                                                   &outer);
+                    }
                    if (status != NV_OK)
                        break;
                }
--- a/kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c
+++ b/kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c
@@ -177,31 +177,34 @@ bool uvm_gpu_non_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu)
    return has_pending_faults == NV_TRUE;
 }

-static NvU32 fetch_non_replayable_fault_buffer_entries(uvm_gpu_t *gpu)
+static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *parent_gpu, NvU32 *cached_faults)
 {
    NV_STATUS status;
-    NvU32 i = 0;
-    NvU32 cached_faults = 0;
-    uvm_fault_buffer_entry_t *fault_cache;
-    NvU32 entry_size = gpu->parent->fault_buffer_hal->entry_size(gpu->parent);
-    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
+    NvU32 i;
+    NvU32 entry_size = parent_gpu->fault_buffer_hal->entry_size(parent_gpu);
+    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &parent_gpu->fault_buffer_info.non_replayable;
    char *current_hw_entry = (char *)non_replayable_faults->shadow_buffer_copy;
+    uvm_fault_buffer_entry_t *fault_entry = non_replayable_faults->fault_cache;

-    fault_cache = non_replayable_faults->fault_cache;
+    UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.non_replayable_faults.service_lock));
+    UVM_ASSERT(parent_gpu->non_replayable_faults_supported);

-    UVM_ASSERT(uvm_sem_is_locked(&gpu->parent->isr.non_replayable_faults.service_lock));
-    UVM_ASSERT(gpu->parent->non_replayable_faults_supported);
+    status = nvUvmInterfaceGetNonReplayableFaults(&parent_gpu->fault_buffer_info.rm_info,
+                                                  current_hw_entry,
+                                                  cached_faults);

-    status = nvUvmInterfaceGetNonReplayableFaults(&gpu->parent->fault_buffer_info.rm_info,
-                                                  non_replayable_faults->shadow_buffer_copy,
-                                                  &cached_faults);
-    UVM_ASSERT(status == NV_OK);
+    if (status != NV_OK) {
+        UVM_ERR_PRINT("nvUvmInterfaceGetNonReplayableFaults() failed: %s, GPU %s\n",
+                      nvstatusToString(status),
+                      parent_gpu->name);
+
+        uvm_global_set_fatal_error(status);
+        return status;
+    }

    // Parse all faults
-    for (i = 0; i < cached_faults; ++i) {
-        uvm_fault_buffer_entry_t *fault_entry = &non_replayable_faults->fault_cache[i];
-
-        gpu->parent->fault_buffer_hal->parse_non_replayable_entry(gpu->parent, current_hw_entry, fault_entry);
+    for (i = 0; i < *cached_faults; ++i) {
+        parent_gpu->fault_buffer_hal->parse_non_replayable_entry(parent_gpu, current_hw_entry, fault_entry);

        // The GPU aligns the fault addresses to 4k, but all of our tracking is
        // done in PAGE_SIZE chunks which might be larger.
@@ -226,9 +229,10 @@ static NvU32 fetch_non_replayable_fault_buffer_entries(uvm_gpu_t *gpu)
        }

        current_hw_entry += entry_size;
+        fault_entry++;
    }

-    return cached_faults;
+    return NV_OK;
 }

 // In SRIOV, the UVM (guest) driver does not have access to the privileged
@@ -339,6 +343,7 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_gpu_t *gpu,
    bool read_duplicate;
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
    uvm_non_replayable_fault_buffer_info_t *non_replayable_faults = &gpu->parent->fault_buffer_info.non_replayable;
+    const uvm_va_policy_t *policy;

    UVM_ASSERT(!fault_entry->is_fatal);

@@ -348,7 +353,7 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_gpu_t *gpu,
    UVM_ASSERT(fault_entry->fault_address >= va_block->start);
    UVM_ASSERT(fault_entry->fault_address <= va_block->end);

-    service_context->block_context.policy = uvm_va_policy_get(va_block, fault_entry->fault_address);
+    policy = uvm_va_policy_get(va_block, fault_entry->fault_address);

    if (service_context->num_retries == 0) {
        // notify event to tools/performance heuristics. For now we use a
@@ -357,7 +362,7 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_gpu_t *gpu,
        uvm_perf_event_notify_gpu_fault(&va_space->perf_events,
                                        va_block,
                                        gpu->id,
-                                        service_context->block_context.policy->preferred_location,
+                                        policy->preferred_location,
                                        fault_entry,
                                        ++non_replayable_faults->batch_id,
                                        false);
@@ -392,7 +397,7 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_gpu_t *gpu,
                                                  page_index,
                                                  gpu->id,
                                                  fault_entry->access_type_mask,
-                                                  service_context->block_context.policy,
+                                                  policy,
                                                  &thrashing_hint,
                                                  UVM_SERVICE_OPERATION_NON_REPLAYABLE_FAULTS,
                                                  &read_duplicate);
@@ -674,10 +679,17 @@ static NV_STATUS service_fault(uvm_gpu_t *gpu, uvm_fault_buffer_entry_t *fault_e
    fault_entry->fault_source.channel_id = user_channel->hw_channel_id;

    if (!fault_entry->is_fatal) {
-        status = uvm_va_block_find_create(fault_entry->va_space,
-                                          fault_entry->fault_address,
-                                          va_block_context,
-                                          &va_block);
+        if (mm) {
+            status = uvm_va_block_find_create(fault_entry->va_space,
+                                              fault_entry->fault_address,
+                                              &va_block_context->hmm.vma,
+                                              &va_block);
+        }
+        else {
+            status = uvm_va_block_find_create_managed(fault_entry->va_space,
+                                                      fault_entry->fault_address,
+                                                      &va_block);
+        }
        if (status == NV_OK)
            status = service_managed_fault_in_block(gpu_va_space->gpu, va_block, fault_entry);
        else
@@ -705,33 +717,35 @@ exit_no_channel:
    uvm_va_space_up_read(va_space);
    uvm_va_space_mm_release_unlock(va_space, mm);

+    if (status != NV_OK)
+        UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n", uvm_gpu_name(gpu));
+
    return status;
 }

 void uvm_gpu_service_non_replayable_fault_buffer(uvm_gpu_t *gpu)
 {
-    NV_STATUS status = NV_OK;
    NvU32 cached_faults;

    // If this handler is modified to handle fewer than all of the outstanding
    // faults, then special handling will need to be added to uvm_suspend()
    // to guarantee that fault processing has completed before control is
    // returned to the RM.
-    while ((cached_faults = fetch_non_replayable_fault_buffer_entries(gpu)) > 0) {
+    do {
+        NV_STATUS status;
        NvU32 i;

+        status = fetch_non_replayable_fault_buffer_entries(gpu->parent, &cached_faults);
+        if (status != NV_OK)
+            return;
+
        // Differently to replayable faults, we do not batch up and preprocess
        // non-replayable faults since getting multiple faults on the same
        // memory region is not very likely
-        //
-        // TODO: Bug 2103669: [UVM/ATS] Optimize ATS fault servicing
        for (i = 0; i < cached_faults; ++i) {
            status = service_fault(gpu, &gpu->parent->fault_buffer_info.non_replayable.fault_cache[i]);
            if (status != NV_OK)
-                break;
+                return;
        }
-    }
-
-    if (status != NV_OK)
-        UVM_DBG_PRINT("Error servicing non-replayable faults on GPU: %s\n", uvm_gpu_name(gpu));
+    } while (cached_faults > 0);
 }
--- a/kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.c
+++ b/kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.c
@@ -486,7 +486,9 @@ static NV_STATUS cancel_fault_precise_va(uvm_gpu_t *gpu,
    return status;
 }

-static NV_STATUS push_replay_on_gpu(uvm_gpu_t *gpu, uvm_fault_replay_type_t type, uvm_fault_service_batch_context_t *batch_context)
+static NV_STATUS push_replay_on_gpu(uvm_gpu_t *gpu,
+                                    uvm_fault_replay_type_t type,
+                                    uvm_fault_service_batch_context_t *batch_context)
 {
    NV_STATUS status;
    uvm_push_t push;
@@ -572,6 +574,19 @@ static NV_STATUS hw_fault_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu)
    return status;
 }

+static void fault_buffer_skip_replayable_entry(uvm_parent_gpu_t *parent_gpu, NvU32 index)
+{
+    UVM_ASSERT(parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, index));
+
+    // Flushed faults are never decrypted, but the decryption IV associated with
+    // replayable faults still requires manual adjustment so it is kept in sync
+    // with the encryption IV on the GSP-RM's side.
+    if (!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu))
+        uvm_conf_computing_fault_increment_decrypt_iv(parent_gpu, 1);
+
+    parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, index);
+}
+
 static NV_STATUS fault_buffer_flush_locked(uvm_gpu_t *gpu,
                                           uvm_gpu_buffer_flush_mode_t flush_mode,
                                           uvm_fault_replay_type_t fault_replay,
@@ -610,7 +625,7 @@ static NV_STATUS fault_buffer_flush_locked(uvm_gpu_t *gpu,
        // Wait until valid bit is set
        UVM_SPIN_WHILE(!parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, get), &spin);

-        parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, get);
+        fault_buffer_skip_replayable_entry(parent_gpu, get);
        ++get;
        if (get == replayable_faults->max_faults)
            get = 0;
@@ -785,9 +800,9 @@ static bool fetch_fault_buffer_try_merge_entry(uvm_fault_buffer_entry_t *current
 // This optimization cannot be performed during fault cancel on Pascal GPUs
 // (fetch_mode == FAULT_FETCH_MODE_ALL) since we need accurate tracking of all
 // the faults in each uTLB in order to guarantee precise fault attribution.
-static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
-                                       uvm_fault_service_batch_context_t *batch_context,
-                                       fault_fetch_mode_t fetch_mode)
+static NV_STATUS fetch_fault_buffer_entries(uvm_gpu_t *gpu,
+                                            uvm_fault_service_batch_context_t *batch_context,
+                                            fault_fetch_mode_t fetch_mode)
 {
    NvU32 get;
    NvU32 put;
@@ -796,6 +811,7 @@ static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
    NvU32 utlb_id;
    uvm_fault_buffer_entry_t *fault_cache;
    uvm_spin_loop_t spin;
+    NV_STATUS status = NV_OK;
    uvm_replayable_fault_buffer_info_t *replayable_faults = &gpu->parent->fault_buffer_info.replayable;
    const bool in_pascal_cancel_path = (!gpu->parent->fault_cancel_va_supported && fetch_mode == FAULT_FETCH_MODE_ALL);
    const bool may_filter = uvm_perf_fault_coalesce && !in_pascal_cancel_path;
@@ -851,7 +867,9 @@ static void fetch_fault_buffer_entries(uvm_gpu_t *gpu,
        smp_mb__after_atomic();

        // Got valid bit set. Let's cache.
-        gpu->parent->fault_buffer_hal->parse_entry(gpu->parent, get, current_entry);
+        status = gpu->parent->fault_buffer_hal->parse_replayable_entry(gpu->parent, get, current_entry);
+        if (status != NV_OK)
+            goto done;

        // The GPU aligns the fault addresses to 4k, but all of our tracking is
        // done in PAGE_SIZE chunks which might be larger.
@@ -918,6 +936,8 @@ done:

    batch_context->num_cached_faults = fault_index;
    batch_context->num_coalesced_faults = num_coalesced_faults;
+
+    return status;
 }

 // Sort comparator for pointers to fault buffer entries that sorts by
@@ -1302,6 +1322,7 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
    uvm_fault_buffer_entry_t **ordered_fault_cache = batch_context->ordered_fault_cache;
    uvm_service_block_context_t *block_context = &replayable_faults->block_service_context;
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
+    const uvm_va_policy_t *policy;
    NvU64 end;

    // Check that all uvm_fault_access_type_t values can fit into an NvU8
@@ -1327,13 +1348,13 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
    UVM_ASSERT(ordered_fault_cache[first_fault_index]->fault_address <= va_block->end);

    if (uvm_va_block_is_hmm(va_block)) {
-        uvm_hmm_find_policy_end(va_block,
-                                &block_context->block_context,
-                                ordered_fault_cache[first_fault_index]->fault_address,
-                                &end);
+        policy = uvm_hmm_find_policy_end(va_block,
+                                         block_context->block_context.hmm.vma,
+                                         ordered_fault_cache[first_fault_index]->fault_address,
+                                         &end);
    }
    else {
-        block_context->block_context.policy = uvm_va_range_get_policy(va_block->va_range);
+        policy = uvm_va_range_get_policy(va_block->va_range);
        end = va_block->end;
    }

@@ -1373,7 +1394,7 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
            update_batch_and_notify_fault(gpu,
                                          batch_context,
                                          va_block,
-                                          block_context->block_context.policy->preferred_location,
+                                          policy->preferred_location,
                                          current_entry,
                                          is_duplicate);
        }
@@ -1453,7 +1474,7 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
                                                      page_index,
                                                      gpu->id,
                                                      service_access_type_mask,
-                                                      block_context->block_context.policy,
+                                                      policy,
                                                      &thrashing_hint,
                                                      UVM_SERVICE_OPERATION_REPLAYABLE_FAULTS,
                                                      &read_duplicate);
@@ -1605,21 +1626,25 @@ static NV_STATUS service_fault_batch_ats_sub_vma(uvm_gpu_va_space_t *gpu_va_spac
    uvm_ats_fault_context_t *ats_context = &batch_context->ats_context;
    const uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask;
    const uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask;
-    const uvm_page_mask_t *faults_serviced_mask = &ats_context->faults_serviced_mask;
    const uvm_page_mask_t *reads_serviced_mask = &ats_context->reads_serviced_mask;
-    uvm_page_mask_t *tmp_mask = &ats_context->tmp_mask;
+    uvm_page_mask_t *faults_serviced_mask = &ats_context->faults_serviced_mask;
+    uvm_page_mask_t *faulted_mask = &ats_context->faulted_mask;

    UVM_ASSERT(vma);

    ats_context->client_type = UVM_FAULT_CLIENT_TYPE_GPC;

-    uvm_page_mask_or(tmp_mask, write_fault_mask, read_fault_mask);
+    uvm_page_mask_or(faulted_mask, write_fault_mask, read_fault_mask);

    status = uvm_ats_service_faults(gpu_va_space, vma, base, &batch_context->ats_context);

-    UVM_ASSERT(uvm_page_mask_subset(faults_serviced_mask, tmp_mask));
+    // Remove prefetched pages from the serviced mask since fault servicing
+    // failures belonging to prefetch pages need to be ignored.
+    uvm_page_mask_and(faults_serviced_mask, faults_serviced_mask, faulted_mask);

-    if ((status != NV_OK) || uvm_page_mask_equal(faults_serviced_mask, tmp_mask)) {
+    UVM_ASSERT(uvm_page_mask_subset(faults_serviced_mask, faulted_mask));
+
+    if ((status != NV_OK) || uvm_page_mask_equal(faults_serviced_mask, faulted_mask)) {
        (*block_faults) += (fault_index_end - fault_index_start);
        return status;
    }
@@ -1847,7 +1872,13 @@ static NV_STATUS service_fault_batch_dispatch(uvm_va_space_t *va_space,
        va_range_next = uvm_va_space_iter_next(va_range_next, ~0ULL);
    }

-    status = uvm_va_block_find_create_in_range(va_space, va_range, fault_address, va_block_context, &va_block);
+    if (va_range)
+        status = uvm_va_block_find_create_in_range(va_space, va_range, fault_address, &va_block);
+    else if (mm)
+        status = uvm_hmm_va_block_find_create(va_space, fault_address, &va_block_context->hmm.vma, &va_block);
+    else
+        status = NV_ERR_INVALID_ADDRESS;
+
    if (status == NV_OK) {
        status = service_fault_batch_block(gpu, va_block, batch_context, fault_index, block_faults);
    }
@@ -2475,7 +2506,10 @@ static NV_STATUS cancel_faults_precise_tlb(uvm_gpu_t *gpu, uvm_fault_service_bat
        batch_context->has_throttled_faults        = false;

        // 5) Fetch all faults from buffer
-        fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_ALL);
+        status = fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_ALL);
+        if (status != NV_OK)
+            break;
+
        ++batch_context->batch_id;

        UVM_ASSERT(batch_context->num_cached_faults == batch_context->num_coalesced_faults);
@@ -2612,7 +2646,10 @@ void uvm_gpu_service_replayable_faults(uvm_gpu_t *gpu)
        batch_context->has_fatal_faults            = false;
        batch_context->has_throttled_faults        = false;

-        fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_BATCH_READY);
+        status = fetch_fault_buffer_entries(gpu, batch_context, FAULT_FETCH_MODE_BATCH_READY);
+        if (status != NV_OK)
+            break;
+
        if (batch_context->num_cached_faults == 0)
            break;

--- a/kernel-open/nvidia-uvm/uvm_gpu_semaphore.c
+++ b/kernel-open/nvidia-uvm/uvm_gpu_semaphore.c
@@ -579,8 +579,10 @@ static void uvm_gpu_semaphore_encrypted_payload_update(uvm_channel_t *channel, u
    void *auth_tag_cpu_addr = uvm_rm_mem_get_cpu_va(semaphore->conf_computing.auth_tag);
    NvU32 *gpu_notifier_cpu_addr = (NvU32 *)uvm_rm_mem_get_cpu_va(semaphore->conf_computing.notifier);
    NvU32 *payload_cpu_addr = (NvU32 *)uvm_rm_mem_get_cpu_va(semaphore->conf_computing.encrypted_payload);
+    uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);

-    UVM_ASSERT(uvm_channel_is_secure_ce(channel));
+    UVM_ASSERT(uvm_conf_computing_mode_enabled(gpu));
+    UVM_ASSERT(uvm_channel_is_ce(channel));

    last_observed_notifier = semaphore->conf_computing.last_observed_notifier;
    gpu_notifier = UVM_READ_ONCE(*gpu_notifier_cpu_addr);
--- a/kernel-open/nvidia-uvm/uvm_gpu_semaphore.h
+++ b/kernel-open/nvidia-uvm/uvm_gpu_semaphore.h
@@ -91,9 +91,9 @@ struct uvm_gpu_tracking_semaphore_struct
 // Create a semaphore pool for a GPU.
 NV_STATUS uvm_gpu_semaphore_pool_create(uvm_gpu_t *gpu, uvm_gpu_semaphore_pool_t **pool_out);

-// When the Confidential Computing feature is enabled, pools associated with
-// secure CE channels are allocated in the CPR of vidmem and as such have
-// all the associated access restrictions. Because of this, they're called
+// When the Confidential Computing feature is enabled, semaphore pools
+// associated with CE channels are allocated in the CPR of vidmem and as such
+// have all the associated access restrictions. Because of this, they're called
 // secure pools and secure semaphores are allocated out of said secure pools.
 NV_STATUS uvm_gpu_semaphore_secure_pool_create(uvm_gpu_t *gpu, uvm_gpu_semaphore_pool_t **pool_out);

--- a/kernel-open/nvidia-uvm/uvm_hal.c
+++ b/kernel-open/nvidia-uvm/uvm_hal.c
@@ -373,7 +373,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
            .read_get = uvm_hal_maxwell_fault_buffer_read_get_unsupported,
            .write_get = uvm_hal_maxwell_fault_buffer_write_get_unsupported,
            .get_ve_id = uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported,
-            .parse_entry = uvm_hal_maxwell_fault_buffer_parse_entry_unsupported,
+            .parse_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported,
            .entry_is_valid = uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported,
            .entry_clear_valid = uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported,
            .entry_size = uvm_hal_maxwell_fault_buffer_entry_size_unsupported,
@@ -396,7 +396,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
            .read_put = uvm_hal_pascal_fault_buffer_read_put,
            .read_get = uvm_hal_pascal_fault_buffer_read_get,
            .write_get = uvm_hal_pascal_fault_buffer_write_get,
-            .parse_entry = uvm_hal_pascal_fault_buffer_parse_entry,
+            .parse_replayable_entry = uvm_hal_pascal_fault_buffer_parse_replayable_entry,
            .entry_is_valid = uvm_hal_pascal_fault_buffer_entry_is_valid,
            .entry_clear_valid = uvm_hal_pascal_fault_buffer_entry_clear_valid,
            .entry_size = uvm_hal_pascal_fault_buffer_entry_size,
@@ -411,7 +411,7 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
            .read_get = uvm_hal_volta_fault_buffer_read_get,
            .write_get = uvm_hal_volta_fault_buffer_write_get,
            .get_ve_id = uvm_hal_volta_fault_buffer_get_ve_id,
-            .parse_entry = uvm_hal_volta_fault_buffer_parse_entry,
+            .parse_replayable_entry = uvm_hal_volta_fault_buffer_parse_replayable_entry,
            .parse_non_replayable_entry = uvm_hal_volta_fault_buffer_parse_non_replayable_entry,
            .get_fault_type = uvm_hal_volta_fault_buffer_get_fault_type,
        }
--- a/kernel-open/nvidia-uvm/uvm_hal.h
+++ b/kernel-open/nvidia-uvm/uvm_hal.h
@@ -485,11 +485,24 @@ typedef NvU32 (*uvm_hal_fault_buffer_read_get_t)(uvm_parent_gpu_t *parent_gpu);
 typedef void (*uvm_hal_fault_buffer_write_get_t)(uvm_parent_gpu_t *parent_gpu, NvU32 get);
 typedef NvU8 (*uvm_hal_fault_buffer_get_ve_id_t)(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);

-// Parse the entry on the given buffer index. This also clears the valid bit of
-// the entry in the buffer.
-typedef void (*uvm_hal_fault_buffer_parse_entry_t)(uvm_parent_gpu_t *gpu,
-                                                   NvU32 index,
-                                                   uvm_fault_buffer_entry_t *buffer_entry);
+// Parse the replayable entry at the given buffer index. This also clears the
+// valid bit of the entry in the buffer.
+typedef NV_STATUS (*uvm_hal_fault_buffer_parse_replayable_entry_t)(uvm_parent_gpu_t *gpu,
+                                                                   NvU32 index,
+                                                                   uvm_fault_buffer_entry_t *buffer_entry);
+
+NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
+                                                                          NvU32 index,
+                                                                          uvm_fault_buffer_entry_t *buffer_entry);
+
+NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
+                                                             NvU32 index,
+                                                             uvm_fault_buffer_entry_t *buffer_entry);
+
+NV_STATUS uvm_hal_volta_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
+                                                            NvU32 index,
+                                                            uvm_fault_buffer_entry_t *buffer_entry);
+
 typedef bool (*uvm_hal_fault_buffer_entry_is_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 typedef void (*uvm_hal_fault_buffer_entry_clear_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 typedef NvU32 (*uvm_hal_fault_buffer_entry_size_t)(uvm_parent_gpu_t *parent_gpu);
@@ -508,9 +521,6 @@ NvU32 uvm_hal_maxwell_fault_buffer_read_put_unsupported(uvm_parent_gpu_t *parent
 NvU32 uvm_hal_maxwell_fault_buffer_read_get_unsupported(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_maxwell_fault_buffer_write_get_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-void uvm_hal_maxwell_fault_buffer_parse_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
-                                                          NvU32 index,
-                                                          uvm_fault_buffer_entry_t *buffer_entry);
 uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry);

 void uvm_hal_pascal_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
@@ -519,18 +529,14 @@ void uvm_hal_pascal_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32
 NvU32 uvm_hal_pascal_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
 NvU32 uvm_hal_pascal_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_pascal_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
-void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
-                                             NvU32 index,
-                                             uvm_fault_buffer_entry_t *buffer_entry);
+
 uvm_fault_type_t uvm_hal_pascal_fault_buffer_get_fault_type(const NvU32 *fault_entry);

 NvU32 uvm_hal_volta_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
 NvU32 uvm_hal_volta_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
 void uvm_hal_volta_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
 NvU8 uvm_hal_volta_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
-void uvm_hal_volta_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
-                                            NvU32 index,
-                                            uvm_fault_buffer_entry_t *buffer_entry);
+
 uvm_fault_type_t uvm_hal_volta_fault_buffer_get_fault_type(const NvU32 *fault_entry);

 void uvm_hal_turing_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
@@ -772,7 +778,7 @@ struct uvm_fault_buffer_hal_struct
    uvm_hal_fault_buffer_read_get_t read_get;
    uvm_hal_fault_buffer_write_get_t write_get;
    uvm_hal_fault_buffer_get_ve_id_t get_ve_id;
-    uvm_hal_fault_buffer_parse_entry_t parse_entry;
+    uvm_hal_fault_buffer_parse_replayable_entry_t parse_replayable_entry;
    uvm_hal_fault_buffer_entry_is_valid_t entry_is_valid;
    uvm_hal_fault_buffer_entry_clear_valid_t entry_clear_valid;
    uvm_hal_fault_buffer_entry_size_t entry_size;
--- a/kernel-open/nvidia-uvm/uvm_hal_types.h
+++ b/kernel-open/nvidia-uvm/uvm_hal_types.h
@@ -128,6 +128,13 @@ static uvm_gpu_address_t uvm_gpu_address_virtual(NvU64 va)
    return address;
 }

+static uvm_gpu_address_t uvm_gpu_address_virtual_unprotected(NvU64 va)
+{
+    uvm_gpu_address_t address = uvm_gpu_address_virtual(va);
+    address.is_unprotected = true;
+    return address;
+}
+
 // Create a physical GPU address
 static uvm_gpu_address_t uvm_gpu_address_physical(uvm_aperture_t aperture, NvU64 pa)
 {
--- a/kernel-open/nvidia-uvm/uvm_hmm.c
+++ b/kernel-open/nvidia-uvm/uvm_hmm.c
@@ -110,7 +110,20 @@ typedef struct

 bool uvm_hmm_is_enabled_system_wide(void)
 {
-    return !uvm_disable_hmm && !g_uvm_global.ats.enabled && uvm_va_space_mm_enabled_system();
+    if (uvm_disable_hmm)
+        return false;
+
+    if (g_uvm_global.ats.enabled)
+        return false;
+
+    // Confidential Computing and HMM impose mutually exclusive constraints. In
+    // Confidential Computing the GPU can only access pages resident in vidmem,
+    // but in HMM pages may be required to be resident in sysmem: file backed
+    // VMAs, huge pages, etc.
+    if (g_uvm_global.conf_computing_enabled)
+        return false;
+
+    return uvm_va_space_mm_enabled_system();
 }

 bool uvm_hmm_is_enabled(uvm_va_space_t *va_space)
@@ -127,32 +140,17 @@ static uvm_va_block_t *hmm_va_block_from_node(uvm_range_tree_node_t *node)
    return container_of(node, uvm_va_block_t, hmm.node);
 }

-NV_STATUS uvm_hmm_va_space_initialize(uvm_va_space_t *va_space)
+void uvm_hmm_va_space_initialize(uvm_va_space_t *va_space)
 {
    uvm_hmm_va_space_t *hmm_va_space = &va_space->hmm;
-    struct mm_struct *mm = va_space->va_space_mm.mm;
-    int ret;

    if (!uvm_hmm_is_enabled(va_space))
-        return NV_OK;
-
-    uvm_assert_mmap_lock_locked_write(mm);
-    uvm_assert_rwsem_locked_write(&va_space->lock);
+        return;

    uvm_range_tree_init(&hmm_va_space->blocks);
    uvm_mutex_init(&hmm_va_space->blocks_lock, UVM_LOCK_ORDER_LEAF);

-    // Initialize MMU interval notifiers for this process.
-    // This allows mmu_interval_notifier_insert() to be called without holding
-    // the mmap_lock for write.
-    // Note: there is no __mmu_notifier_unregister(), this call just allocates
-    // memory which is attached to the mm_struct and freed when the mm_struct
-    // is freed.
-    ret = __mmu_notifier_register(NULL, mm);
-    if (ret)
-        return errno_to_nv_status(ret);
-
-    return NV_OK;
+    return;
 }

 void uvm_hmm_va_space_destroy(uvm_va_space_t *va_space)
@@ -325,7 +323,6 @@ static bool hmm_invalidate(uvm_va_block_t *va_block,
    region = uvm_va_block_region_from_start_end(va_block, start, end);

    va_block_context->hmm.vma = NULL;
-    va_block_context->policy = NULL;

    // We only need to unmap GPUs since Linux handles the CPUs.
    for_each_gpu_id_in_mask(id, &va_block->mapped) {
@@ -444,11 +441,11 @@ static void hmm_va_block_init(uvm_va_block_t *va_block,
 static NV_STATUS hmm_va_block_find_create(uvm_va_space_t *va_space,
                                          NvU64 addr,
                                          bool allow_unreadable_vma,
-                                          uvm_va_block_context_t *va_block_context,
+                                          struct vm_area_struct **vma_out,
                                          uvm_va_block_t **va_block_ptr)
 {
-    struct mm_struct *mm = va_space->va_space_mm.mm;
-    struct vm_area_struct *vma;
+    struct mm_struct *mm;
+    struct vm_area_struct *va_block_vma;
    uvm_va_block_t *va_block;
    NvU64 start, end;
    NV_STATUS status;
@@ -457,15 +454,14 @@ static NV_STATUS hmm_va_block_find_create(uvm_va_space_t *va_space,
    if (!uvm_hmm_is_enabled(va_space))
        return NV_ERR_INVALID_ADDRESS;

-    UVM_ASSERT(mm);
-    UVM_ASSERT(!va_block_context || va_block_context->mm == mm);
+    mm = va_space->va_space_mm.mm;
    uvm_assert_mmap_lock_locked(mm);
    uvm_assert_rwsem_locked(&va_space->lock);
    UVM_ASSERT(PAGE_ALIGNED(addr));

    // Note that we have to allow PROT_NONE VMAs so that policies can be set.
-    vma = find_vma(mm, addr);
-    if (!uvm_hmm_vma_is_valid(vma, addr, allow_unreadable_vma))
+    va_block_vma = find_vma(mm, addr);
+    if (!uvm_hmm_vma_is_valid(va_block_vma, addr, allow_unreadable_vma))
        return NV_ERR_INVALID_ADDRESS;

    // Since we only hold the va_space read lock, there can be multiple
@@ -517,8 +513,8 @@ static NV_STATUS hmm_va_block_find_create(uvm_va_space_t *va_space,

 done:
    uvm_mutex_unlock(&va_space->hmm.blocks_lock);
-    if (va_block_context)
-        va_block_context->hmm.vma = vma;
+    if (vma_out)
+        *vma_out = va_block_vma;
    *va_block_ptr = va_block;
    return NV_OK;

@@ -532,43 +528,36 @@ err_unlock:

 NV_STATUS uvm_hmm_va_block_find_create(uvm_va_space_t *va_space,
                                       NvU64 addr,
-                                       uvm_va_block_context_t *va_block_context,
+                                       struct vm_area_struct **vma,
                                       uvm_va_block_t **va_block_ptr)
 {
-    return hmm_va_block_find_create(va_space, addr, false, va_block_context, va_block_ptr);
+    return hmm_va_block_find_create(va_space, addr, false, vma, va_block_ptr);
 }

-NV_STATUS uvm_hmm_find_vma(uvm_va_block_context_t *va_block_context, NvU64 addr)
+NV_STATUS uvm_hmm_find_vma(struct mm_struct *mm, struct vm_area_struct **vma_out, NvU64 addr)
 {
-    struct mm_struct *mm = va_block_context->mm;
-    struct vm_area_struct *vma;
-
    if (!mm)
        return NV_ERR_INVALID_ADDRESS;

    uvm_assert_mmap_lock_locked(mm);

-    vma = find_vma(mm, addr);
-    if (!uvm_hmm_vma_is_valid(vma, addr, false))
+    *vma_out = find_vma(mm, addr);
+    if (!uvm_hmm_vma_is_valid(*vma_out, addr, false))
        return NV_ERR_INVALID_ADDRESS;

-    va_block_context->hmm.vma = vma;
-
    return NV_OK;
 }

 bool uvm_hmm_check_context_vma_is_valid(uvm_va_block_t *va_block,
-                                        uvm_va_block_context_t *va_block_context,
+                                        struct vm_area_struct *vma,
                                        uvm_va_block_region_t region)
 {
    uvm_assert_mutex_locked(&va_block->lock);

    if (uvm_va_block_is_hmm(va_block)) {
-        struct vm_area_struct *vma = va_block_context->hmm.vma;
-
        UVM_ASSERT(vma);
-        UVM_ASSERT(va_block_context->mm == vma->vm_mm);
-        uvm_assert_mmap_lock_locked(va_block_context->mm);
+        UVM_ASSERT(va_block->hmm.va_space->va_space_mm.mm == vma->vm_mm);
+        uvm_assert_mmap_lock_locked(va_block->hmm.va_space->va_space_mm.mm);
        UVM_ASSERT(vma->vm_start <= uvm_va_block_region_start(va_block, region));
        UVM_ASSERT(vma->vm_end > uvm_va_block_region_end(va_block, region));
    }
@@ -619,8 +608,6 @@ static NV_STATUS hmm_migrate_range(uvm_va_block_t *va_block,
    uvm_mutex_lock(&va_block->lock);

    uvm_for_each_va_policy_in(policy, va_block, start, end, node, region) {
-        va_block_context->policy = policy;
-
        // Even though UVM_VA_BLOCK_RETRY_LOCKED() may unlock and relock the
        // va_block lock, the policy remains valid because we hold the mmap
        // lock so munmap can't remove the policy, and the va_space lock so the
@@ -670,7 +657,6 @@ void uvm_hmm_evict_va_blocks(uvm_va_space_t *va_space)
                continue;

            block_context->hmm.vma = vma;
-            block_context->policy = &uvm_va_policy_default;
            uvm_hmm_va_block_migrate_locked(va_block,
                                            NULL,
                                            block_context,
@@ -1046,11 +1032,7 @@ static NV_STATUS hmm_set_preferred_location_locked(uvm_va_block_t *va_block,
            uvm_processor_mask_test(&old_policy->accessed_by, old_policy->preferred_location))
            uvm_processor_mask_set(&set_accessed_by_processors, old_policy->preferred_location);

-        va_block_context->policy = uvm_va_policy_set_preferred_location(va_block,
-                                                                        region,
-                                                                        preferred_location,
-                                                                        old_policy);
-        if (!va_block_context->policy)
+        if (!uvm_va_policy_set_preferred_location(va_block, region, preferred_location, old_policy))
            return NV_ERR_NO_MEMORY;

        // Establish new remote mappings if the old preferred location had
@@ -1109,7 +1091,7 @@ NV_STATUS uvm_hmm_set_preferred_location(uvm_va_space_t *va_space,
    for (addr = base; addr < last_address; addr = va_block->end + 1) {
        NvU64 end;

-        status = hmm_va_block_find_create(va_space, addr, true, va_block_context, &va_block);
+        status = hmm_va_block_find_create(va_space, addr, true, &va_block_context->hmm.vma, &va_block);
        if (status != NV_OK)
            break;

@@ -1151,7 +1133,6 @@ static NV_STATUS hmm_set_accessed_by_start_end_locked(uvm_va_block_t *va_block,
        if (uvm_va_policy_is_read_duplicate(&node->policy, va_space))
            continue;

-        va_block_context->policy = &node->policy;
        region = uvm_va_block_region_from_start_end(va_block,
                                                    max(start, node->node.start),
                                                    min(end, node->node.end));
@@ -1196,7 +1177,7 @@ NV_STATUS uvm_hmm_set_accessed_by(uvm_va_space_t *va_space,
    for (addr = base; addr < last_address; addr = va_block->end + 1) {
        NvU64 end;

-        status = hmm_va_block_find_create(va_space, addr, true, va_block_context, &va_block);
+        status = hmm_va_block_find_create(va_space, addr, true, &va_block_context->hmm.vma, &va_block);
        if (status != NV_OK)
            break;

@@ -1249,8 +1230,6 @@ void uvm_hmm_block_add_eviction_mappings(uvm_va_space_t *va_space,
    uvm_mutex_lock(&va_block->lock);

    uvm_for_each_va_policy_node_in(node, va_block, va_block->start, va_block->end) {
-        block_context->policy = &node->policy;
-
        for_each_id_in_mask(id, &node->policy.accessed_by) {
            status = hmm_set_accessed_by_start_end_locked(va_block,
                                                          block_context,
@@ -1309,13 +1288,13 @@ void uvm_hmm_block_add_eviction_mappings(uvm_va_space_t *va_space,
    }
 }

-void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
-                             uvm_va_block_context_t *va_block_context,
-                             unsigned long addr,
-                             NvU64 *endp)
+const uvm_va_policy_t *uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
+                                               struct vm_area_struct *vma,
+                                               unsigned long addr,
+                                               NvU64 *endp)
 {
-    struct vm_area_struct *vma = va_block_context->hmm.vma;
    const uvm_va_policy_node_t *node;
+    const uvm_va_policy_t *policy;
    NvU64 end = va_block->end;

    uvm_assert_mmap_lock_locked(vma->vm_mm);
@@ -1326,40 +1305,45 @@ void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,

    node = uvm_va_policy_node_find(va_block, addr);
    if (node) {
-        va_block_context->policy = &node->policy;
+        policy = &node->policy;
        if (end > node->node.end)
            end = node->node.end;
    }
    else {
-        va_block_context->policy = &uvm_va_policy_default;
+        policy = &uvm_va_policy_default;
    }

    *endp = end;
+
+    return policy;
 }

 NV_STATUS uvm_hmm_find_policy_vma_and_outer(uvm_va_block_t *va_block,
-                                            uvm_va_block_context_t *va_block_context,
+                                            struct vm_area_struct **vma_out,
                                            uvm_page_index_t page_index,
+                                            const uvm_va_policy_t **policy,
                                            uvm_page_index_t *outerp)
 {
-    struct vm_area_struct *vma;
    unsigned long addr;
    NvU64 end;
    uvm_page_index_t outer;
+    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
+    struct mm_struct *mm = va_space->va_space_mm.mm;
+
+    if (!mm)
+        return NV_ERR_INVALID_ADDRESS;

    UVM_ASSERT(uvm_va_block_is_hmm(va_block));
-    uvm_assert_mmap_lock_locked(va_block_context->mm);
+    uvm_assert_mmap_lock_locked(mm);
    uvm_assert_mutex_locked(&va_block->lock);

    addr = uvm_va_block_cpu_page_address(va_block, page_index);

-    vma = vma_lookup(va_block_context->mm, addr);
-    if (!vma || !(vma->vm_flags & VM_READ))
+    *vma_out = vma_lookup(mm, addr);
+    if (!*vma_out || !((*vma_out)->vm_flags & VM_READ))
        return NV_ERR_INVALID_ADDRESS;

-    va_block_context->hmm.vma = vma;
-
-    uvm_hmm_find_policy_end(va_block, va_block_context, addr, &end);
+    *policy = uvm_hmm_find_policy_end(va_block, *vma_out, addr, &end);

    outer = uvm_va_block_cpu_page_index(va_block, end) + 1;
    if (*outerp > outer)
@@ -1379,8 +1363,6 @@ static NV_STATUS hmm_clear_thrashing_policy(uvm_va_block_t *va_block,
    uvm_mutex_lock(&va_block->lock);

    uvm_for_each_va_policy_in(policy, va_block, va_block->start, va_block->end, node, region) {
-        block_context->policy = policy;
-
        // Unmap may split PTEs and require a retry. Needs to be called
        // before the pinned pages information is destroyed.
        status = UVM_VA_BLOCK_RETRY_LOCKED(va_block,
@@ -1424,11 +1406,10 @@ NV_STATUS uvm_hmm_clear_thrashing_policy(uvm_va_space_t *va_space)
 }

 uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
-                                                  uvm_va_block_context_t *va_block_context,
+                                                  struct vm_area_struct *vma,
+                                                  const uvm_va_policy_t *policy,
                                                  NvU64 address)
 {
-    struct vm_area_struct *vma = va_block_context->hmm.vma;
-    const uvm_va_policy_t *policy = va_block_context->policy;
    NvU64 start, end;

    UVM_ASSERT(uvm_va_block_is_hmm(va_block));
@@ -1457,13 +1438,11 @@ uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
 }

 uvm_prot_t uvm_hmm_compute_logical_prot(uvm_va_block_t *va_block,
-                                        uvm_va_block_context_t *va_block_context,
+                                        struct vm_area_struct *vma,
                                        NvU64 addr)
 {
-    struct vm_area_struct *vma = va_block_context->hmm.vma;
-
    UVM_ASSERT(uvm_va_block_is_hmm(va_block));
-    uvm_assert_mmap_lock_locked(va_block_context->mm);
+    uvm_assert_mmap_lock_locked(va_block->hmm.va_space->va_space_mm.mm);
    UVM_ASSERT(vma && addr >= vma->vm_start && addr < vma->vm_end);

    if (!(vma->vm_flags & VM_READ))
@@ -2575,7 +2554,7 @@ static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block,
                continue;
            }

-            if (folio_test_swapcache(page_folio(src_page))) {
+            if (PageSwapCache(src_page)) {
                // TODO: Bug 4050579: Remove this when swap cached pages can be
                // migrated.
                if (service_context) {
@@ -2907,8 +2886,6 @@ static NV_STATUS uvm_hmm_migrate_alloc_and_copy(struct vm_area_struct *vma,
    if (status != NV_OK)
        return status;

-    UVM_ASSERT(!uvm_va_policy_is_read_duplicate(va_block_context->policy, va_block->hmm.va_space));
-
    status = uvm_va_block_make_resident_copy(va_block,
                                             va_block_retry,
                                             va_block_context,
@@ -3140,7 +3117,7 @@ NV_STATUS uvm_hmm_migrate_ranges(uvm_va_space_t *va_space,
    for (addr = base; addr < last_address; addr = end + 1) {
        struct vm_area_struct *vma;

-        status = hmm_va_block_find_create(va_space, addr, false, va_block_context, &va_block);
+        status = hmm_va_block_find_create(va_space, addr, false, &va_block_context->hmm.vma, &va_block);
        if (status != NV_OK)
            return status;

@@ -3232,7 +3209,6 @@ static NV_STATUS hmm_va_block_evict_chunks(uvm_va_block_t *va_block,
    uvm_for_each_va_policy_in(policy, va_block, start, end, node, region) {
        npages = uvm_va_block_region_num_pages(region);

-        va_block_context->policy = policy;
        if (out_accessed_by_set && uvm_processor_mask_get_count(&policy->accessed_by) > 0)
            *out_accessed_by_set = true;

--- a/kernel-open/nvidia-uvm/uvm_hmm.h
+++ b/kernel-open/nvidia-uvm/uvm_hmm.h
@@ -49,9 +49,7 @@ typedef struct
    bool uvm_hmm_is_enabled_system_wide(void);

    // Initialize HMM for the given the va_space.
-    // Locking: the va_space->va_space_mm.mm mmap_lock must be write locked
-    // and the va_space lock must be held in write mode.
-    NV_STATUS uvm_hmm_va_space_initialize(uvm_va_space_t *va_space);
+    void uvm_hmm_va_space_initialize(uvm_va_space_t *va_space);

    // Destroy any HMM state for the given the va_space.
    // Locking: va_space lock must be held in write mode.
@@ -90,31 +88,30 @@ typedef struct
    // address 'addr' or the VMA does not have at least PROT_READ permission.
    // The caller is also responsible for checking that there is no UVM
    // va_range covering the given address before calling this function.
-    // If va_block_context is not NULL, the VMA is cached in
-    // va_block_context->hmm.vma.
+    // The VMA is returned in vma_out if it's not NULL.
    // Locking: This function must be called with mm retained and locked for
    // at least read and the va_space lock at least for read.
    NV_STATUS uvm_hmm_va_block_find_create(uvm_va_space_t *va_space,
                                           NvU64 addr,
-                                           uvm_va_block_context_t *va_block_context,
+                                           struct vm_area_struct **vma_out,
                                           uvm_va_block_t **va_block_ptr);

-    // Find the VMA for the given address and set va_block_context->hmm.vma.
-    // Return NV_ERR_INVALID_ADDRESS if va_block_context->mm is NULL or there
-    // is no VMA associated with the address 'addr' or the VMA does not have at
-    // least PROT_READ permission.
+    // Find the VMA for the given address and return it in vma_out. Return
+    // NV_ERR_INVALID_ADDRESS if mm is NULL or there is no VMA associated with
+    // the address 'addr' or the VMA does not have at least PROT_READ
+    // permission.
    // Locking: This function must be called with mm retained and locked for
    // at least read or mm equal to NULL.
-    NV_STATUS uvm_hmm_find_vma(uvm_va_block_context_t *va_block_context, NvU64 addr);
+    NV_STATUS uvm_hmm_find_vma(struct mm_struct *mm, struct vm_area_struct **vma_out, NvU64 addr);

-    // If va_block is a HMM va_block, check that va_block_context->hmm.vma is
-    // not NULL and covers the given region. This always returns true and is
-    // intended to only be used with UVM_ASSERT().
+    // If va_block is a HMM va_block, check that vma is not NULL and covers the
+    // given region. This always returns true and is intended to only be used
+    // with UVM_ASSERT().
    // Locking: This function must be called with the va_block lock held and if
-    // va_block is a HMM block, va_block_context->mm must be retained and
-    // locked for at least read.
+    // va_block is a HMM block, va_space->va_space_mm.mm->mmap_lock must be
+    // retained and locked for at least read.
    bool uvm_hmm_check_context_vma_is_valid(uvm_va_block_t *va_block,
-                                            uvm_va_block_context_t *va_block_context,
+                                            struct vm_area_struct *vma,
                                            uvm_va_block_region_t region);

    // Initialize the HMM portion of the service_context.
@@ -225,31 +222,29 @@ typedef struct
        return NV_OK;
    }

-    // This function assigns va_block_context->policy to the policy covering
-    // the given address 'addr' and assigns the ending address '*endp' to the
-    // minimum of va_block->end, va_block_context->hmm.vma->vm_end - 1, and the
-    // ending address of the policy range. Note that va_block_context->hmm.vma
-    // is expected to be initialized before calling this function.
-    // Locking: This function must be called with
-    // va_block_context->hmm.vma->vm_mm retained and locked for least read and
-    // the va_block lock held.
-    void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
-                                 uvm_va_block_context_t *va_block_context,
-                                 unsigned long addr,
-                                 NvU64 *endp);
+    // This function returns the policy covering the given address 'addr' and
+    // assigns the ending address '*endp' to the minimum of va_block->end,
+    // vma->vm_end - 1, and the ending address of the policy range. Locking:
+    // This function must be called with vma->vm_mm retained and locked for at
+    // least read and the va_block and va_space lock held.
+    const uvm_va_policy_t *uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
+                                                   struct vm_area_struct *vma,
+                                                   unsigned long addr,
+                                                   NvU64 *endp);

-    // This function finds the VMA for the page index 'page_index' and assigns
-    // it to va_block_context->vma, sets va_block_context->policy to the policy
-    // covering the given address, and sets the ending page range '*outerp'
-    // to the minimum of *outerp, va_block_context->hmm.vma->vm_end - 1, the
-    // ending address of the policy range, and va_block->end.
-    // Return NV_ERR_INVALID_ADDRESS if no VMA is found; otherwise, NV_OK.
-    // Locking: This function must be called with
-    // va_block_context->hmm.vma->vm_mm retained and locked for least read and
-    // the va_block lock held.
+    // This function finds the VMA for the page index 'page_index' and returns
+    // it in vma_out which must not be NULL. Returns the policy covering the
+    // given address, and sets the ending page range '*outerp' to the minimum of
+    // *outerp, vma->vm_end - 1, the ending address of the policy range, and
+    // va_block->end.
+    // Return NV_ERR_INVALID_ADDRESS if no VMA is found; otherwise sets *vma
+    // and returns NV_OK.
+    // Locking: This function must be called with mm retained and locked for at
+    // least read and the va_block and va_space lock held.
    NV_STATUS uvm_hmm_find_policy_vma_and_outer(uvm_va_block_t *va_block,
-                                                uvm_va_block_context_t *va_block_context,
+                                                struct vm_area_struct **vma,
                                                uvm_page_index_t page_index,
+                                                const uvm_va_policy_t **policy,
                                                uvm_page_index_t *outerp);

    // Clear thrashing policy information from all HMM va_blocks.
@@ -258,24 +253,21 @@ typedef struct

    // Return the expanded region around 'address' limited to the intersection
    // of va_block start/end, vma start/end, and policy start/end.
-    // va_block_context must not be NULL, va_block_context->hmm.vma must be
-    // valid (this is usually set by uvm_hmm_va_block_find_create()), and
-    // va_block_context->policy must be valid.
-    // Locking: the caller must hold mm->mmap_lock in at least read mode, the
-    // va_space lock must be held in at least read mode, and the va_block lock
-    // held.
+    // Locking: the caller must hold va_space->va_space_mm.mm->mmap_lock in at
+    // least read mode, the va_space lock must be held in at least read mode,
+    // and the va_block lock held.
    uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
-                                                      uvm_va_block_context_t *va_block_context,
+                                                      struct vm_area_struct *vma,
+                                                      const uvm_va_policy_t *policy,
                                                      NvU64 address);

    // Return the logical protection allowed of a HMM va_block for the page at
-    // the given address.
-    // va_block_context must not be NULL and va_block_context->hmm.vma must be
-    // valid (this is usually set by uvm_hmm_va_block_find_create()).
-    // Locking: the caller must hold va_block_context->mm mmap_lock in at least
-    // read mode.
+    // the given address within the vma which must be valid. This is usually
+    // obtained from uvm_hmm_va_block_find_create()).
+    // Locking: the caller must hold va_space->va_space_mm.mm mmap_lock in at
+    // least read mode.
    uvm_prot_t uvm_hmm_compute_logical_prot(uvm_va_block_t *va_block,
-                                            uvm_va_block_context_t *va_block_context,
+                                            struct vm_area_struct *vma,
                                            NvU64 addr);

    // This is called to service a GPU fault.
@@ -288,9 +280,9 @@ typedef struct
                                              uvm_service_block_context_t *service_context);

    // This is called to migrate a region within a HMM va_block.
-    // va_block_context must not be NULL and va_block_context->policy and
-    // va_block_context->hmm.vma must be valid.
-    // Locking: the va_block_context->mm must be retained, mmap_lock must be
+    // va_block_context must not be NULL and va_block_context->hmm.vma
+    // must be valid.
+    // Locking: the va_space->va_space_mm.mm must be retained, mmap_lock must be
    // locked, and the va_block lock held.
    NV_STATUS uvm_hmm_va_block_migrate_locked(uvm_va_block_t *va_block,
                                              uvm_va_block_retry_t *va_block_retry,
@@ -303,7 +295,7 @@ typedef struct
    // UvmMigrate().
    //
    // va_block_context must not be NULL. The caller is not required to set
-    // va_block_context->policy or va_block_context->hmm.vma.
+    // va_block_context->hmm.vma.
    //
    // Locking: the va_space->va_space_mm.mm mmap_lock must be locked and
    // the va_space read lock must be held.
@@ -412,9 +404,8 @@ typedef struct
        return false;
    }

-    static NV_STATUS uvm_hmm_va_space_initialize(uvm_va_space_t *va_space)
+    static void uvm_hmm_va_space_initialize(uvm_va_space_t *va_space)
    {
-        return NV_OK;
    }

    static void uvm_hmm_va_space_destroy(uvm_va_space_t *va_space)
@@ -440,19 +431,19 @@ typedef struct

    static NV_STATUS uvm_hmm_va_block_find_create(uvm_va_space_t *va_space,
                                                  NvU64 addr,
-                                                  uvm_va_block_context_t *va_block_context,
+                                                  struct vm_area_struct **vma,
                                                  uvm_va_block_t **va_block_ptr)
    {
        return NV_ERR_INVALID_ADDRESS;
    }

-    static NV_STATUS uvm_hmm_find_vma(uvm_va_block_context_t *va_block_context, NvU64 addr)
+    static NV_STATUS uvm_hmm_find_vma(struct mm_struct *mm, struct vm_area_struct **vma, NvU64 addr)
    {
        return NV_OK;
    }

    static bool uvm_hmm_check_context_vma_is_valid(uvm_va_block_t *va_block,
-                                                   uvm_va_block_context_t *va_block_context,
+                                                   struct vm_area_struct *vma,
                                                   uvm_va_block_region_t region)
    {
        return true;
@@ -533,16 +524,19 @@ typedef struct
        return NV_ERR_INVALID_ADDRESS;
    }

-    static void uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
-                                        uvm_va_block_context_t *va_block_context,
-                                        unsigned long addr,
-                                        NvU64 *endp)
+    static const uvm_va_policy_t *uvm_hmm_find_policy_end(uvm_va_block_t *va_block,
+                                                          struct vm_area_struct *vma,
+                                                          unsigned long addr,
+                                                          NvU64 *endp)
    {
+        UVM_ASSERT(0);
+        return NULL;
    }

    static NV_STATUS uvm_hmm_find_policy_vma_and_outer(uvm_va_block_t *va_block,
-                                                       uvm_va_block_context_t *va_block_context,
+                                                       struct vm_area_struct **vma,
                                                       uvm_page_index_t page_index,
+                                                       const uvm_va_policy_t **policy,
                                                       uvm_page_index_t *outerp)
    {
        return NV_OK;
@@ -554,14 +548,15 @@ typedef struct
    }

    static uvm_va_block_region_t uvm_hmm_get_prefetch_region(uvm_va_block_t *va_block,
-                                                             uvm_va_block_context_t *va_block_context,
+                                                             struct vm_area_struct *vma,
+                                                             const uvm_va_policy_t *policy,
                                                             NvU64 address)
    {
        return (uvm_va_block_region_t){};
    }

    static uvm_prot_t uvm_hmm_compute_logical_prot(uvm_va_block_t *va_block,
-                                                   uvm_va_block_context_t *va_block_context,
+                                                   struct vm_area_struct *vma,
                                                   NvU64 addr)
    {
        return UVM_PROT_NONE;
--- a/kernel-open/nvidia-uvm/uvm_hopper.c
+++ b/kernel-open/nvidia-uvm/uvm_hopper.c
@@ -61,7 +61,11 @@ void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
    // GH180.
    parent_gpu->ce_phys_vidmem_write_supported = !uvm_gpu_is_coherent(parent_gpu);

-    parent_gpu->peer_copy_mode = g_uvm_global.peer_copy_mode;
+    // TODO: Bug 4174553: [HGX-SkinnyJoe][GH180] channel errors discussion/debug
+    //                    portion for the uvm tests became nonresponsive after
+    //                    some time and then failed even after reboot
+    parent_gpu->peer_copy_mode = uvm_gpu_is_coherent(parent_gpu) ?
+                                                           UVM_GPU_PEER_COPY_MODE_VIRTUAL : g_uvm_global.peer_copy_mode;

    // All GR context buffers may be mapped to 57b wide VAs. All "compute" units
    // accessing GR context buffers support the 57-bit VA range.
--- a/kernel-open/nvidia-uvm/uvm_hopper_ce.c
+++ b/kernel-open/nvidia-uvm/uvm_hopper_ce.c
@@ -491,7 +491,6 @@ void uvm_hal_hopper_ce_encrypt(uvm_push_t *push,
    uvm_gpu_t *gpu = uvm_push_get_gpu(push);

    UVM_ASSERT(uvm_conf_computing_mode_is_hcc(gpu));
-    UVM_ASSERT(uvm_push_is_fake(push) || uvm_channel_is_secure(push->channel));
    UVM_ASSERT(IS_ALIGNED(auth_tag.address, UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT));

    if (!src.is_virtual)
@@ -540,7 +539,6 @@ void uvm_hal_hopper_ce_decrypt(uvm_push_t *push,
    uvm_gpu_t *gpu = uvm_push_get_gpu(push);

    UVM_ASSERT(uvm_conf_computing_mode_is_hcc(gpu));
-    UVM_ASSERT(!push->channel || uvm_channel_is_secure(push->channel));
    UVM_ASSERT(IS_ALIGNED(auth_tag.address, UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT));

    // The addressing mode (and aperture, if applicable) of the source and
--- a/kernel-open/nvidia-uvm/uvm_hopper_sec2.c
+++ b/kernel-open/nvidia-uvm/uvm_hopper_sec2.c
@@ -166,6 +166,7 @@ void uvm_hal_hopper_sec2_decrypt(uvm_push_t *push, NvU64 dst_va, NvU64 src_va, N
    NvU32 *csl_sign_init = push->next;

    // Check that the provided alignment matches HW
+    BUILD_BUG_ON(UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT != (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)));
    BUILD_BUG_ON(UVM_CONF_COMPUTING_BUF_ALIGNMENT < (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)));
    BUILD_BUG_ON(UVM_CONF_COMPUTING_BUF_ALIGNMENT % (1 << HWSHIFT(CBA2, DECRYPT_COPY_DST_ADDR_LO, DATA)) != 0);

--- a/kernel-open/nvidia-uvm/uvm_linux.h
+++ b/kernel-open/nvidia-uvm/uvm_linux.h
@@ -153,6 +153,10 @@ static inline const struct cpumask *uvm_cpumask_of_node(int node)
 #define VM_MIXEDMAP    0x00000000
 #endif

+#if !defined(MPOL_PREFERRED_MANY)
+#define MPOL_PREFERRED_MANY    5
+#endif
+
 //
 // printk.h already defined pr_fmt, so we have to redefine it so the pr_*
 // routines pick up our version
--- a/kernel-open/nvidia-uvm/uvm_lock.h
+++ b/kernel-open/nvidia-uvm/uvm_lock.h
@@ -279,13 +279,14 @@
 //      Operations not allowed while holding the lock:
 //      - GPU memory allocation which can evict memory (would require nesting
 //        block locks)
+//
 // - GPU DMA Allocation pool lock (gpu->conf_computing.dma_buffer_pool.lock)
 //      Order: UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL
+//      Condition: The Confidential Computing feature is enabled
 //      Exclusive lock (mutex)
 //
 //      Protects:
 //      - Protect the state of the uvm_conf_computing_dma_buffer_pool_t
-//        when the Confidential Computing feature is enabled on the system.
 //
 // - Chunk mapping lock (gpu->root_chunk_mappings.bitlocks and
 //   gpu->sysmem_mappings.bitlock)
@@ -321,22 +322,25 @@
 //      Operations not allowed while holding this lock
 //      - GPU memory allocation which can evict
 //
-// - Secure channel CSL channel pool semaphore
+// - CE channel CSL channel pool semaphore
 //      Order: UVM_LOCK_ORDER_CSL_PUSH
-//      Semaphore per SEC2 channel pool
+//      Condition: The Confidential Computing feature is enabled
+//      Semaphore per CE channel pool
 //
-//      The semaphore controls concurrent pushes to secure channels. Secure work
-//      submission depends on channel availability in GPFIFO entries (as in any
-//      other channel type) but also on channel locking. Each secure channel has a
-//      lock to enforce ordering of pushes. The channel's CSL lock is taken on
-//      channel reservation until uvm_push_end. Secure channels are stateful
-//      channels and the CSL lock protects their CSL state/context.
+//      The semaphore controls concurrent pushes to CE channels that are not WCL
+//      channels. Secure work submission depends on channel availability in
+//      GPFIFO entries (as in any other channel type) but also on channel
+//      locking. Each channel has a lock to enforce ordering of pushes. The
+//      channel's CSL lock is taken on channel reservation until uvm_push_end.
+//      When the Confidential Computing feature is enabled, channels are
+//      stateful, and the CSL lock protects their CSL state/context.
 //
 //      Operations allowed while holding this lock
-//      - Pushing work to CE secure channels
+//      - Pushing work to CE channels (except for WLC channels)
 //
 // - WLC CSL channel pool semaphore
 //      Order: UVM_LOCK_ORDER_CSL_WLC_PUSH
+//      Condition: The Confidential Computing feature is enabled
 //      Semaphore per WLC channel pool
 //
 //      The semaphore controls concurrent pushes to WLC channels. WLC work
@@ -346,8 +350,8 @@
 //      channel reservation until uvm_push_end. SEC2 channels are stateful
 //      channels and the CSL lock protects their CSL state/context.
 //
-//      This lock ORDER is different and sits below generic secure channel CSL
-//      lock and above SEC2 CSL lock. This reflects the dual nature of WLC
+//      This lock ORDER is different and sits below the generic channel CSL
+//      lock and above the SEC2 CSL lock. This reflects the dual nature of WLC
 //      channels; they use SEC2 indirect work launch during initialization,
 //      and after their schedule is initialized they provide indirect launch
 //      functionality to other CE channels.
@@ -357,6 +361,7 @@
 //
 // - SEC2 CSL channel pool semaphore
 //      Order: UVM_LOCK_ORDER_SEC2_CSL_PUSH
+//      Condition: The Confidential Computing feature is enabled
 //      Semaphore per SEC2 channel pool
 //
 //      The semaphore controls concurrent pushes to SEC2 channels. SEC2 work
@@ -366,9 +371,9 @@
 //      channel reservation until uvm_push_end. SEC2 channels are stateful
 //      channels and the CSL lock protects their CSL state/context.
 //
-//      This lock ORDER is different and lower than the generic secure channel
-//      lock to allow secure work submission to use a SEC2 channel to submit
-//      work before releasing the CSL lock of the originating secure channel.
+//      This lock ORDER is different and lower than UVM_LOCK_ORDER_CSL_PUSH
+//      to allow secure work submission to use a SEC2 channel to submit
+//      work before releasing the CSL lock of the originating channel.
 //
 //      Operations allowed while holding this lock
 //      - Pushing work to SEC2 channels
@@ -408,16 +413,18 @@
 //
 // - WLC Channel lock
 //      Order: UVM_LOCK_ORDER_WLC_CHANNEL
+//      Condition: The Confidential Computing feature is enabled
 //      Spinlock (uvm_spinlock_t)
 //
 //      Lock protecting the state of WLC channels in a channel pool. This lock
-//      is separate from the above generic channel lock to allow for indirect
-//      worklaunch pushes while holding the main channel lock.
-//      (WLC pushes don't need any of the pushbuffer locks described above)
+//      is separate from the generic channel lock (UVM_LOCK_ORDER_CHANNEL)
+//      to allow for indirect worklaunch pushes while holding the main channel
+//      lock (WLC pushes don't need any of the pushbuffer locks described
+//      above)
 //
 // - Tools global VA space list lock (g_tools_va_space_list_lock)
 //      Order: UVM_LOCK_ORDER_TOOLS_VA_SPACE_LIST
-//      Reader/writer lock (rw_sempahore)
+//      Reader/writer lock (rw_semaphore)
 //
 //      This lock protects the list of VA spaces used when broadcasting
 //      UVM profiling events.
@@ -437,9 +444,10 @@
 //
 // - Tracking semaphores
 //      Order: UVM_LOCK_ORDER_SECURE_SEMAPHORE
-//      When the Confidential Computing feature is enabled, CE semaphores are
-//      encrypted, and require to take the CSL lock (UVM_LOCK_ORDER_LEAF) to
-//      decrypt the payload.
+//      Condition: The Confidential Computing feature is enabled
+//
+//      CE semaphore payloads are encrypted, and require to take the CSL lock
+//      (UVM_LOCK_ORDER_LEAF) to decrypt the payload.
 //
 // - Leaf locks
 //      Order: UVM_LOCK_ORDER_LEAF
--- a/kernel-open/nvidia-uvm/uvm_maxwell_fault_buffer.c
+++ b/kernel-open/nvidia-uvm/uvm_maxwell_fault_buffer.c
@@ -68,11 +68,12 @@ uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const N
    return UVM_FAULT_TYPE_COUNT;
 }

-void uvm_hal_maxwell_fault_buffer_parse_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
-                                                         NvU32 index,
-                                                         uvm_fault_buffer_entry_t *buffer_entry)
+NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
+                                                                          NvU32 index,
+                                                                          uvm_fault_buffer_entry_t *buffer_entry)
 {
    UVM_ASSERT_MSG(false, "fault_buffer_parse_entry is not supported on GPU: %s.\n", parent_gpu->name);
+    return NV_ERR_NOT_SUPPORTED;
 }

 bool uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index)
--- a/kernel-open/nvidia-uvm/uvm_mem.c
+++ b/kernel-open/nvidia-uvm/uvm_mem.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2022 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -93,8 +93,9 @@ static bool sysmem_can_be_mapped_on_gpu(uvm_mem_t *sysmem)
 {
    UVM_ASSERT(uvm_mem_is_sysmem(sysmem));

-    // If SEV is enabled, only unprotected memory can be mapped
-    if (g_uvm_global.sev_enabled)
+    // In Confidential Computing, only unprotected memory can be mapped on the
+    // GPU
+    if (g_uvm_global.conf_computing_enabled)
        return uvm_mem_is_sysmem_dma(sysmem);

    return true;
@@ -737,7 +738,7 @@ static NV_STATUS mem_map_cpu_to_sysmem_kernel(uvm_mem_t *mem)
            pages[page_index] = mem_cpu_page(mem, page_index * PAGE_SIZE);
    }

-    if (g_uvm_global.sev_enabled && uvm_mem_is_sysmem_dma(mem))
+    if (g_uvm_global.conf_computing_enabled && uvm_mem_is_sysmem_dma(mem))
        prot = uvm_pgprot_decrypted(PAGE_KERNEL_NOENC);

    mem->kernel.cpu_addr = vmap(pages, num_pages, VM_MAP, prot);
--- a/kernel-open/nvidia-uvm/uvm_mem.h
+++ b/kernel-open/nvidia-uvm/uvm_mem.h
@@ -392,12 +392,6 @@ static NV_STATUS uvm_mem_alloc_vidmem(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **me
    return uvm_mem_alloc(&params, mem_out);
 }

-// Helper for allocating protected vidmem with the default page size
-static NV_STATUS uvm_mem_alloc_vidmem_protected(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **mem_out)
-{
-    return uvm_mem_alloc_vidmem(size, gpu, mem_out);
-}
-
 // Helper for allocating sysmem and mapping it on the CPU
 static NV_STATUS uvm_mem_alloc_sysmem_and_map_cpu_kernel(NvU64 size, struct mm_struct *mm, uvm_mem_t **mem_out)
 {
--- a/kernel-open/nvidia-uvm/uvm_mem_test.c
+++ b/kernel-open/nvidia-uvm/uvm_mem_test.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2016-2021 NVIDIA Corporation
+    Copyright (c) 2016-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -44,10 +44,10 @@ static NvU32 first_page_size(NvU32 page_sizes)

 static inline NV_STATUS __alloc_map_sysmem(NvU64 size, uvm_gpu_t *gpu, uvm_mem_t **sys_mem)
 {
-    if (g_uvm_global.sev_enabled)
+    if (g_uvm_global.conf_computing_enabled)
        return uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(size, gpu, current->mm, sys_mem);
-    else
-        return uvm_mem_alloc_sysmem_and_map_cpu_kernel(size, current->mm, sys_mem);
+
+    return uvm_mem_alloc_sysmem_and_map_cpu_kernel(size, current->mm, sys_mem);
 }

 static NV_STATUS check_accessible_from_gpu(uvm_gpu_t *gpu, uvm_mem_t *mem)
@@ -335,9 +335,6 @@ error:

 static bool should_test_page_size(size_t alloc_size, NvU32 page_size)
 {
-    if (g_uvm_global.sev_enabled)
-        return false;
-
    if (g_uvm_global.num_simulated_devices == 0)
        return true;

--- a/kernel-open/nvidia-uvm/uvm_migrate.c
+++ b/kernel-open/nvidia-uvm/uvm_migrate.c
@@ -134,6 +134,22 @@ static NV_STATUS block_migrate_map_unmapped_pages(uvm_va_block_t *va_block,
    // first map operation
    uvm_page_mask_complement(&va_block_context->caller_page_mask, &va_block->maybe_mapped_pages);

+    if (uvm_va_block_is_hmm(va_block) && !UVM_ID_IS_CPU(dest_id)) {
+        // Do not map pages that are already resident on the CPU. This is in
+        // order to avoid breaking system-wide atomic operations on HMM. HMM's
+        // implementation of system-side atomic operations involves restricting
+        // mappings to one processor (CPU or a GPU) at a time. If we were to
+        // grant a GPU a mapping to system memory, this gets into trouble
+        // because, on the CPU side, Linux can silently upgrade PTE permissions
+        // (move from read-only, to read-write, without any MMU notifiers
+        // firing), thus breaking the model by allowing simultaneous read-write
+        // access from two separate processors. To avoid that, just don't map
+        // such pages at all, when migrating.
+        uvm_page_mask_andnot(&va_block_context->caller_page_mask,
+                             &va_block_context->caller_page_mask,
+                             uvm_va_block_resident_mask_get(va_block, UVM_ID_CPU));
+    }
+
    // Only map those pages that are not mapped anywhere else (likely due
    // to a first touch or a migration). We pass
    // UvmEventMapRemoteCauseInvalid since the destination processor of a
@@ -207,7 +223,7 @@ NV_STATUS uvm_va_block_migrate_locked(uvm_va_block_t *va_block,
    NV_STATUS status, tracker_status = NV_OK;

    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block, va_block_context, region));
+    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block, va_block_context->hmm.vma, region));

    if (uvm_va_block_is_hmm(va_block)) {
        status = uvm_hmm_va_block_migrate_locked(va_block,
@@ -218,9 +234,9 @@ NV_STATUS uvm_va_block_migrate_locked(uvm_va_block_t *va_block,
                                                 UVM_MAKE_RESIDENT_CAUSE_API_MIGRATE);
    }
    else {
-        va_block_context->policy = uvm_va_range_get_policy(va_block->va_range);
+        uvm_va_policy_t *policy = uvm_va_range_get_policy(va_block->va_range);

-        if (uvm_va_policy_is_read_duplicate(va_block_context->policy, va_space)) {
+        if (uvm_va_policy_is_read_duplicate(policy, va_space)) {
            status = uvm_va_block_make_resident_read_duplicate(va_block,
                                                               va_block_retry,
                                                               va_block_context,
@@ -355,8 +371,6 @@ static bool va_block_should_do_cpu_preunmap(uvm_va_block_t *va_block,
    if (!va_block)
        return true;

-    UVM_ASSERT(va_range_should_do_cpu_preunmap(va_block_context->policy, uvm_va_block_get_va_space(va_block)));
-
    region = uvm_va_block_region_from_start_end(va_block, max(start, va_block->start), min(end, va_block->end));

    uvm_mutex_lock(&va_block->lock);
@@ -480,11 +494,9 @@ static NV_STATUS uvm_va_range_migrate(uvm_va_range_t *va_range,
                                      uvm_tracker_t *out_tracker)
 {
    NvU64 preunmap_range_start = start;
+    uvm_va_policy_t *policy = uvm_va_range_get_policy(va_range);

-    UVM_ASSERT(va_block_context->policy == uvm_va_range_get_policy(va_range));
-
-    should_do_cpu_preunmap = should_do_cpu_preunmap && va_range_should_do_cpu_preunmap(va_block_context->policy,
-                                                                                       va_range->va_space);
+    should_do_cpu_preunmap = should_do_cpu_preunmap && va_range_should_do_cpu_preunmap(policy, va_range->va_space);

    // Divide migrations into groups of contiguous VA blocks. This is to trigger
    // CPU unmaps for that region before the migration starts.
@@ -561,8 +573,6 @@ static NV_STATUS uvm_migrate_ranges(uvm_va_space_t *va_space,
            break;
        }

-        va_block_context->policy = uvm_va_range_get_policy(va_range);
-
        // For UVM-Lite GPUs, the CUDA driver may suballocate a single va_range
        // into many range groups.  For this reason, we iterate over each va_range first
        // then through the range groups within.
@@ -637,6 +647,8 @@ static NV_STATUS uvm_migrate(uvm_va_space_t *va_space,

    if (mm)
        uvm_assert_mmap_lock_locked(mm);
+    else if (!first_va_range)
+        return NV_ERR_INVALID_ADDRESS;

    va_block_context = uvm_va_block_context_alloc(mm);
    if (!va_block_context)
@@ -944,17 +956,18 @@ NV_STATUS uvm_api_migrate(UVM_MIGRATE_PARAMS *params, struct file *filp)
        if (type == UVM_API_RANGE_TYPE_ATS) {
            uvm_migrate_args_t uvm_migrate_args =
            {
-                .va_space               = va_space,
-                .mm                     = mm,
-                .start                  = params->base,
-                .length                 = params->length,
-                .dst_id                 = (dest_gpu ? dest_gpu->id : UVM_ID_CPU),
-                .dst_node_id            = (int)params->cpuNumaNode,
-                .populate_permissions   = UVM_POPULATE_PERMISSIONS_INHERIT,
-                .touch                  = false,
-                .skip_mapped            = false,
-                .user_space_start       = &params->userSpaceStart,
-                .user_space_length      = &params->userSpaceLength,
+                .va_space                       = va_space,
+                .mm                             = mm,
+                .start                          = params->base,
+                .length                         = params->length,
+                .dst_id                         = (dest_gpu ? dest_gpu->id : UVM_ID_CPU),
+                .dst_node_id                    = (int)params->cpuNumaNode,
+                .populate_permissions           = UVM_POPULATE_PERMISSIONS_INHERIT,
+                .touch                          = false,
+                .skip_mapped                    = false,
+                .populate_on_cpu_alloc_failures = false,
+                .user_space_start               = &params->userSpaceStart,
+                .user_space_length              = &params->userSpaceLength,
            };

            status = uvm_migrate_pageable(&uvm_migrate_args);
--- a/kernel-open/nvidia-uvm/uvm_migrate_pageable.c
+++ b/kernel-open/nvidia-uvm/uvm_migrate_pageable.c
@@ -507,6 +507,22 @@ static NV_STATUS migrate_vma_copy_pages(struct vm_area_struct *vma,
    return NV_OK;
 }

+void migrate_vma_cleanup_pages(unsigned long *dst, unsigned long npages)
+{
+    unsigned long i;
+
+    for (i = 0; i < npages; i++) {
+        struct page *dst_page = migrate_pfn_to_page(dst[i]);
+
+        if (!dst_page)
+            continue;
+
+        unlock_page(dst_page);
+        __free_page(dst_page);
+        dst[i] = 0;
+    }
+}
+
 void uvm_migrate_vma_alloc_and_copy(struct migrate_vma *args, migrate_vma_state_t *state)
 {
    struct vm_area_struct *vma = args->vma;
@@ -531,6 +547,10 @@ void uvm_migrate_vma_alloc_and_copy(struct migrate_vma *args, migrate_vma_state_

    if (state->status == NV_OK)
        state->status = tracker_status;
+
+    // Mark all pages as not migrating if we're failing
+    if (state->status != NV_OK)
+        migrate_vma_cleanup_pages(args->dst, state->num_pages);
 }

 void uvm_migrate_vma_alloc_and_copy_helper(struct vm_area_struct *vma,
@@ -652,6 +672,14 @@ static NV_STATUS nv_migrate_vma(struct migrate_vma *args, migrate_vma_state_t *s
        .finalize_and_map = uvm_migrate_vma_finalize_and_map_helper,
    };

+    // WAR for Bug 4130089: [GH180][r535] WAR for kernel not issuing SMMU TLB
+    // invalidates on read-only to read-write upgrades
+    //
+    // This code path isn't used on GH180 but we need to maintain consistent
+    // behaviour on systems that do.
+    if (!vma_is_anonymous(args->vma))
+        return NV_WARN_NOTHING_TO_DO;
+
    ret = migrate_vma(&uvm_migrate_vma_ops, args->vma, args->start, args->end, args->src, args->dst, state);
    if (ret < 0)
        return errno_to_nv_status(ret);
@@ -665,6 +693,24 @@ static NV_STATUS nv_migrate_vma(struct migrate_vma *args, migrate_vma_state_t *s
    if (ret < 0)
        return errno_to_nv_status(ret);

+    // TODO: Bug 2419180: support file-backed pages in migrate_vma, when
+    //       support for it is added to the Linux kernel
+    //
+    // A side-effect of migrate_vma_setup() is it calls mmu notifiers even if a
+    // page can't be migrated (eg. because it's a non-anonymous mapping). We
+    // need this side-effect for SMMU on GH180 to ensure any cached read-only
+    // entries are flushed from SMMU on permission upgrade.
+    //
+    // TODO: Bug 4130089: [GH180][r535] WAR for kernel not issuing SMMU TLB
+    // invalidates on read-only to read-write upgrades
+    //
+    // The above WAR doesn't work for HugeTLBfs mappings because
+    // migrate_vma_setup() will fail in that case.
+    if (!vma_is_anonymous(args->vma)) {
+        migrate_vma_finalize(args);
+        return NV_WARN_NOTHING_TO_DO;
+    }
+
    uvm_migrate_vma_alloc_and_copy(args, state);
    if (state->status == NV_OK) {
        migrate_vma_pages(args);
@@ -802,7 +848,7 @@ static NV_STATUS migrate_pageable_vma_region(struct vm_area_struct *vma,
        // If the destination is the CPU, signal user-space to retry with a
        // different node. Otherwise, just try to populate anywhere in the
        // system
-        if (UVM_ID_IS_CPU(uvm_migrate_args->dst_id)) {
+        if (UVM_ID_IS_CPU(uvm_migrate_args->dst_id) && !uvm_migrate_args->populate_on_cpu_alloc_failures) {
            *next_addr = start + find_first_bit(state->scratch2_mask, num_pages) * PAGE_SIZE;
            return NV_ERR_MORE_PROCESSING_REQUIRED;
        }
@@ -838,9 +884,13 @@ static NV_STATUS migrate_pageable_vma(struct vm_area_struct *vma,
    start = max(start, vma->vm_start);
    outer = min(outer, vma->vm_end);

-    // TODO: Bug 2419180: support file-backed pages in migrate_vma, when
-    //       support for it is added to the Linux kernel
-    if (!vma_is_anonymous(vma))
+    // migrate_vma only supports anonymous VMAs. We check for those after
+    // calling migrate_vma_setup() to workaround Bug 4130089. We need to check
+    // for HugeTLB VMAs here because migrate_vma_setup() will return a fatal
+    // error for those.
+    // TODO: Bug 4130089: [GH180][r535] WAR for kernel not issuing SMMU TLB
+    // invalidates on read-only to read-write upgrades
+    if (is_vm_hugetlb_page(vma))
        return NV_WARN_NOTHING_TO_DO;

    if (uvm_processor_mask_empty(&va_space->registered_gpus))
@@ -961,13 +1011,10 @@ NV_STATUS uvm_migrate_pageable(uvm_migrate_args_t *uvm_migrate_args)
        // We only check that dst_node_id is a valid node in the system and it
        // doesn't correspond to a GPU node. This is fine because
        // alloc_pages_node will clamp the allocation to
-        // cpuset_current_mems_allowed, and uvm_migrate_pageable is only called
-        // from process context (uvm_migrate) when dst_id is CPU. UVM bottom
-        // half never calls uvm_migrate_pageable when dst_id is CPU. So, assert
-        // that we're in a user thread. However, this would need to change if we
-        // wanted to call this function from a bottom half with CPU dst_id.
-        UVM_ASSERT(!(current->flags & PF_KTHREAD));
-
+        // cpuset_current_mems_allowed when uvm_migrate_pageable is called from
+        // process context (uvm_migrate) when dst_id is CPU. UVM bottom half
+        // calls uvm_migrate_pageable with CPU dst_id only when the VMA memory
+        // policy is set to dst_node_id and dst_node_id is not NUMA_NO_NODE.
        if (!nv_numa_node_has_memory(dst_node_id) ||
            uvm_va_space_find_gpu_with_memory_node_id(va_space, dst_node_id) != NULL)
            return NV_ERR_INVALID_ARGUMENT;
--- a/kernel-open/nvidia-uvm/uvm_migrate_pageable.h
+++ b/kernel-open/nvidia-uvm/uvm_migrate_pageable.h
@@ -34,8 +34,8 @@ typedef struct
 {
    uvm_va_space_t                  *va_space;
    struct mm_struct                *mm;
-    unsigned long                   start;
-    unsigned long                   length;
+    const unsigned long             start;
+    const unsigned long             length;
    uvm_processor_id_t              dst_id;

    // dst_node_id may be clobbered by uvm_migrate_pageable().
@@ -43,6 +43,7 @@ typedef struct
    uvm_populate_permissions_t      populate_permissions;
    bool                            touch : 1;
    bool                            skip_mapped : 1;
+    bool                            populate_on_cpu_alloc_failures : 1;
    NvU64                           *user_space_start;
    NvU64                           *user_space_length;
 } uvm_migrate_args_t;
--- a/kernel-open/nvidia-uvm/uvm_mmu.c
+++ b/kernel-open/nvidia-uvm/uvm_mmu.c
@@ -906,11 +906,10 @@ error:
 // --------------|-------------------------||----------------|----------------
 //    vidmem     |           -             ||    vidmem      |      false
 //    sysmem     |           -             ||    sysmem      |      false
-//    default    |        <not set>        ||    vidmem      |      true (1)
+//    default    |        <not set>        ||    vidmem      |      true
 //    default    |         vidmem          ||    vidmem      |      false
 //    default    |         sysmem          ||    sysmem      |      false
 //
-// (1) When SEV mode is enabled, the fallback path is disabled.
 //
 // In SR-IOV heavy the the page tree must be in vidmem, to prevent guest drivers
 // from updating GPU page tables without hypervisor knowledge.
@@ -926,28 +925,27 @@ error:
 //
 static void page_tree_set_location(uvm_page_tree_t *tree, uvm_aperture_t location)
 {
-    bool should_location_be_vidmem;
    UVM_ASSERT(tree->gpu != NULL);
    UVM_ASSERT_MSG((location == UVM_APERTURE_VID) ||
                   (location == UVM_APERTURE_SYS) ||
                   (location == UVM_APERTURE_DEFAULT),
                   "Invalid location %s (%d)\n", uvm_aperture_string(location), (int)location);

-    should_location_be_vidmem = uvm_gpu_is_virt_mode_sriov_heavy(tree->gpu)
-                                || uvm_conf_computing_mode_enabled(tree->gpu);
-
    // The page tree of a "fake" GPU used during page tree testing can be in
-    // sysmem even if should_location_be_vidmem is true. A fake GPU can be
-    // identified by having no channel manager.
-    if ((tree->gpu->channel_manager != NULL) && should_location_be_vidmem)
-        UVM_ASSERT(location == UVM_APERTURE_VID);
+    // sysmem in scenarios where a "real" GPU must be in vidmem. Fake GPUs can
+    // be identified by having no channel manager.
+    if (tree->gpu->channel_manager != NULL) {
+
+        if (uvm_gpu_is_virt_mode_sriov_heavy(tree->gpu))
+            UVM_ASSERT(location == UVM_APERTURE_VID);
+        else if (uvm_conf_computing_mode_enabled(tree->gpu))
+            UVM_ASSERT(location == UVM_APERTURE_VID);
+    }

    if (location == UVM_APERTURE_DEFAULT) {
        if (page_table_aperture == UVM_APERTURE_DEFAULT) {
            tree->location = UVM_APERTURE_VID;
-
-            // See the comment (1) above.
-            tree->location_sys_fallback = !g_uvm_global.sev_enabled;
+            tree->location_sys_fallback = true;
        }
        else {
            tree->location = page_table_aperture;
--- a/kernel-open/nvidia-uvm/uvm_pascal_fault_buffer.c
+++ b/kernel-open/nvidia-uvm/uvm_pascal_fault_buffer.c
@@ -214,9 +214,9 @@ static UvmFaultMetadataPacket *get_fault_buffer_entry_metadata(uvm_parent_gpu_t
    return fault_entry_metadata + index;
 }

-void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
-                                             NvU32 index,
-                                             uvm_fault_buffer_entry_t *buffer_entry)
+NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
+                                                             NvU32 index,
+                                                             uvm_fault_buffer_entry_t *buffer_entry)
 {
    NvU32 *fault_entry;
    NvU64 addr_hi, addr_lo;
@@ -280,6 +280,8 @@ void uvm_hal_pascal_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,

    // Automatically clear valid bit for the entry in the fault buffer
    uvm_hal_pascal_fault_buffer_entry_clear_valid(parent_gpu, index);
+
+    return NV_OK;
 }

 bool uvm_hal_pascal_fault_buffer_entry_is_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index)
--- a/kernel-open/nvidia-uvm/uvm_perf_prefetch.c
+++ b/kernel-open/nvidia-uvm/uvm_perf_prefetch.c
@@ -218,57 +218,11 @@ static void grow_fault_granularity(uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
    }
 }

-// Within a block we only allow prefetching to a single processor. Therefore,
-// if two processors are accessing non-overlapping regions within the same
-// block they won't benefit from prefetching.
-//
-// TODO: Bug 1778034: [uvm] Explore prefetching to different processors within
-// a VA block.
-static NvU32 uvm_perf_prefetch_prenotify_fault_migrations(uvm_va_block_t *va_block,
-                                                          uvm_va_block_context_t *va_block_context,
-                                                          uvm_processor_id_t new_residency,
-                                                          const uvm_page_mask_t *faulted_pages,
-                                                          uvm_va_block_region_t faulted_region,
-                                                          uvm_page_mask_t *prefetch_pages,
-                                                          uvm_perf_prefetch_bitmap_tree_t *bitmap_tree)
+static void init_bitmap_tree_from_region(uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                         uvm_va_block_region_t max_prefetch_region,
+                                         const uvm_page_mask_t *resident_mask,
+                                         const uvm_page_mask_t *faulted_pages)
 {
-    uvm_page_index_t page_index;
-    const uvm_page_mask_t *resident_mask = NULL;
-    const uvm_page_mask_t *thrashing_pages = NULL;
-    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
-    const uvm_va_policy_t *policy = va_block_context->policy;
-    uvm_va_block_region_t max_prefetch_region;
-    NvU32 big_page_size;
-    uvm_va_block_region_t big_pages_region;
-
-    if (!uvm_id_equal(va_block->prefetch_info.last_migration_proc_id, new_residency)) {
-        va_block->prefetch_info.last_migration_proc_id = new_residency;
-        va_block->prefetch_info.fault_migrations_to_last_proc = 0;
-    }
-
-    // Compute the expanded region that prefetching is allowed from.
-    if (uvm_va_block_is_hmm(va_block)) {
-        max_prefetch_region = uvm_hmm_get_prefetch_region(va_block,
-                                                          va_block_context,
-                                                          uvm_va_block_region_start(va_block, faulted_region));
-    }
-    else {
-        max_prefetch_region = uvm_va_block_region_from_block(va_block);
-    }
-
-    uvm_page_mask_zero(prefetch_pages);
-
-    if (UVM_ID_IS_CPU(new_residency) || va_block->gpus[uvm_id_gpu_index(new_residency)] != NULL)
-        resident_mask = uvm_va_block_resident_mask_get(va_block, new_residency);
-
-    // If this is a first-touch fault and the destination processor is the
-    // preferred location, populate the whole max_prefetch_region.
-    if (uvm_processor_mask_empty(&va_block->resident) &&
-        uvm_id_equal(new_residency, policy->preferred_location)) {
-        uvm_page_mask_region_fill(prefetch_pages, max_prefetch_region);
-        goto done;
-    }
-
    if (resident_mask)
        uvm_page_mask_or(&bitmap_tree->pages, resident_mask, faulted_pages);
    else
@@ -277,6 +231,29 @@ static NvU32 uvm_perf_prefetch_prenotify_fault_migrations(uvm_va_block_t *va_blo
    // If we are using a subregion of the va_block, align bitmap_tree
    uvm_page_mask_shift_right(&bitmap_tree->pages, &bitmap_tree->pages, max_prefetch_region.first);

+    bitmap_tree->offset = 0;
+    bitmap_tree->leaf_count = uvm_va_block_region_num_pages(max_prefetch_region);
+    bitmap_tree->level_count = ilog2(roundup_pow_of_two(bitmap_tree->leaf_count)) + 1;
+}
+
+static void update_bitmap_tree_from_va_block(uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                             uvm_va_block_t *va_block,
+                                             uvm_va_block_context_t *va_block_context,
+                                             uvm_processor_id_t new_residency,
+                                             const uvm_page_mask_t *faulted_pages,
+                                             uvm_va_block_region_t max_prefetch_region)
+
+{
+    NvU32 big_page_size;
+    uvm_va_block_region_t big_pages_region;
+    uvm_va_space_t *va_space;
+    const uvm_page_mask_t *thrashing_pages;
+
+    UVM_ASSERT(va_block);
+    UVM_ASSERT(va_block_context);
+
+    va_space = uvm_va_block_get_va_space(va_block);
+
    // Get the big page size for the new residency.
    // Assume 64K size if the new residency is the CPU or no GPU va space is
    // registered in the current process for this GPU.
@@ -302,13 +279,9 @@ static NvU32 uvm_perf_prefetch_prenotify_fault_migrations(uvm_va_block_t *va_blo
        UVM_ASSERT(bitmap_tree->leaf_count <= PAGES_PER_UVM_VA_BLOCK);

        uvm_page_mask_shift_left(&bitmap_tree->pages, &bitmap_tree->pages, bitmap_tree->offset);
-    }
-    else {
-        bitmap_tree->offset = 0;
-        bitmap_tree->leaf_count = uvm_va_block_region_num_pages(max_prefetch_region);
-    }

-    bitmap_tree->level_count = ilog2(roundup_pow_of_two(bitmap_tree->leaf_count)) + 1;
+        bitmap_tree->level_count = ilog2(roundup_pow_of_two(bitmap_tree->leaf_count)) + 1;
+    }

    thrashing_pages = uvm_perf_thrashing_get_thrashing_pages(va_block);

@@ -320,25 +293,99 @@ static NvU32 uvm_perf_prefetch_prenotify_fault_migrations(uvm_va_block_t *va_blo
                           max_prefetch_region,
                           faulted_pages,
                           thrashing_pages);
+}

-    // Do not compute prefetch regions with faults on pages that are thrashing
-    if (thrashing_pages)
-        uvm_page_mask_andnot(&va_block_context->scratch_page_mask, faulted_pages, thrashing_pages);
-    else
-        uvm_page_mask_copy(&va_block_context->scratch_page_mask, faulted_pages);
+static void compute_prefetch_mask(uvm_va_block_region_t faulted_region,
+                                  uvm_va_block_region_t max_prefetch_region,
+                                  uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                  const uvm_page_mask_t *faulted_pages,
+                                  uvm_page_mask_t *out_prefetch_mask)
+{
+    uvm_page_index_t page_index;

-    // Update the tree using the scratch mask to compute the pages to prefetch
-    for_each_va_block_page_in_region_mask(page_index, &va_block_context->scratch_page_mask, faulted_region) {
+    uvm_page_mask_zero(out_prefetch_mask);
+
+    // Update the tree using the faulted mask to compute the pages to prefetch.
+    for_each_va_block_page_in_region_mask(page_index, faulted_pages, faulted_region) {
        uvm_va_block_region_t region = compute_prefetch_region(page_index, bitmap_tree, max_prefetch_region);

-        uvm_page_mask_region_fill(prefetch_pages, region);
+        uvm_page_mask_region_fill(out_prefetch_mask, region);

        // Early out if we have already prefetched until the end of the VA block
        if (region.outer == max_prefetch_region.outer)
            break;
    }
+}
+
+// Within a block we only allow prefetching to a single processor. Therefore,
+// if two processors are accessing non-overlapping regions within the same
+// block they won't benefit from prefetching.
+//
+// TODO: Bug 1778034: [uvm] Explore prefetching to different processors within
+// a VA block.
+static NvU32 uvm_perf_prefetch_prenotify_fault_migrations(uvm_va_block_t *va_block,
+                                                          uvm_va_block_context_t *va_block_context,
+                                                          uvm_processor_id_t new_residency,
+                                                          const uvm_page_mask_t *faulted_pages,
+                                                          uvm_va_block_region_t faulted_region,
+                                                          uvm_page_mask_t *prefetch_pages,
+                                                          uvm_perf_prefetch_bitmap_tree_t *bitmap_tree)
+{
+    const uvm_page_mask_t *resident_mask = NULL;
+    const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, faulted_region);
+    uvm_va_block_region_t max_prefetch_region;
+    const uvm_page_mask_t *thrashing_pages = uvm_perf_thrashing_get_thrashing_pages(va_block);
+
+    if (!uvm_id_equal(va_block->prefetch_info.last_migration_proc_id, new_residency)) {
+        va_block->prefetch_info.last_migration_proc_id = new_residency;
+        va_block->prefetch_info.fault_migrations_to_last_proc = 0;
+    }
+
+    // Compute the expanded region that prefetching is allowed from.
+    if (uvm_va_block_is_hmm(va_block)) {
+        max_prefetch_region = uvm_hmm_get_prefetch_region(va_block,
+                                                          va_block_context->hmm.vma,
+                                                          policy,
+                                                          uvm_va_block_region_start(va_block, faulted_region));
+    }
+    else {
+        max_prefetch_region = uvm_va_block_region_from_block(va_block);
+    }
+
+    uvm_page_mask_zero(prefetch_pages);
+
+    if (UVM_ID_IS_CPU(new_residency) || va_block->gpus[uvm_id_gpu_index(new_residency)] != NULL)
+        resident_mask = uvm_va_block_resident_mask_get(va_block, new_residency);
+
+    // If this is a first-touch fault and the destination processor is the
+    // preferred location, populate the whole max_prefetch_region.
+    if (uvm_processor_mask_empty(&va_block->resident) &&
+        uvm_id_equal(new_residency, policy->preferred_location)) {
+        uvm_page_mask_region_fill(prefetch_pages, max_prefetch_region);
+    }
+    else {
+        init_bitmap_tree_from_region(bitmap_tree, max_prefetch_region, resident_mask, faulted_pages);
+
+        update_bitmap_tree_from_va_block(bitmap_tree,
+                                         va_block,
+                                         va_block_context,
+                                         new_residency,
+                                         faulted_pages,
+                                         max_prefetch_region);
+
+        // Do not compute prefetch regions with faults on pages that are thrashing
+        if (thrashing_pages)
+            uvm_page_mask_andnot(&va_block_context->scratch_page_mask, faulted_pages, thrashing_pages);
+        else
+            uvm_page_mask_copy(&va_block_context->scratch_page_mask, faulted_pages);
+
+        compute_prefetch_mask(faulted_region,
+                              max_prefetch_region,
+                              bitmap_tree,
+                              &va_block_context->scratch_page_mask,
+                              prefetch_pages);
+    }

-done:
    // Do not prefetch pages that are going to be migrated/populated due to a
    // fault
    uvm_page_mask_andnot(prefetch_pages, prefetch_pages, faulted_pages);
@@ -364,31 +411,58 @@ done:
    return uvm_page_mask_weight(prefetch_pages);
 }

-void uvm_perf_prefetch_get_hint(uvm_va_block_t *va_block,
-                                uvm_va_block_context_t *va_block_context,
-                                uvm_processor_id_t new_residency,
-                                const uvm_page_mask_t *faulted_pages,
-                                uvm_va_block_region_t faulted_region,
-                                uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
-                                uvm_perf_prefetch_hint_t *out_hint)
+bool uvm_perf_prefetch_enabled(uvm_va_space_t *va_space)
+{
+    if (!g_uvm_perf_prefetch_enable)
+        return false;
+
+    UVM_ASSERT(va_space);
+
+    return va_space->test.page_prefetch_enabled;
+}
+
+void uvm_perf_prefetch_compute_ats(uvm_va_space_t *va_space,
+                                   const uvm_page_mask_t *faulted_pages,
+                                   uvm_va_block_region_t faulted_region,
+                                   uvm_va_block_region_t max_prefetch_region,
+                                   const uvm_page_mask_t *residency_mask,
+                                   uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                   uvm_page_mask_t *out_prefetch_mask)
+{
+    UVM_ASSERT(faulted_pages);
+    UVM_ASSERT(bitmap_tree);
+    UVM_ASSERT(out_prefetch_mask);
+
+    uvm_page_mask_zero(out_prefetch_mask);
+
+    if (!uvm_perf_prefetch_enabled(va_space))
+        return;
+
+    init_bitmap_tree_from_region(bitmap_tree, max_prefetch_region, residency_mask, faulted_pages);
+
+    compute_prefetch_mask(faulted_region, max_prefetch_region, bitmap_tree, faulted_pages, out_prefetch_mask);
+}
+
+void uvm_perf_prefetch_get_hint_va_block(uvm_va_block_t *va_block,
+                                         uvm_va_block_context_t *va_block_context,
+                                         uvm_processor_id_t new_residency,
+                                         const uvm_page_mask_t *faulted_pages,
+                                         uvm_va_block_region_t faulted_region,
+                                         uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                         uvm_perf_prefetch_hint_t *out_hint)
 {
-    const uvm_va_policy_t *policy = va_block_context->policy;
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
    uvm_page_mask_t *prefetch_pages = &out_hint->prefetch_pages_mask;
    NvU32 pending_prefetch_pages;

    uvm_assert_rwsem_locked(&va_space->lock);
    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, policy, faulted_region));
-    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block, va_block_context, faulted_region));
+    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block, va_block_context->hmm.vma, faulted_region));

    out_hint->residency = UVM_ID_INVALID;
    uvm_page_mask_zero(prefetch_pages);

-    if (!g_uvm_perf_prefetch_enable)
-        return;
-
-    if (!va_space->test.page_prefetch_enabled)
+    if (!uvm_perf_prefetch_enabled(va_space))
        return;

    pending_prefetch_pages = uvm_perf_prefetch_prenotify_fault_migrations(va_block,
--- a/kernel-open/nvidia-uvm/uvm_perf_prefetch.h
+++ b/kernel-open/nvidia-uvm/uvm_perf_prefetch.h
@@ -61,21 +61,41 @@ typedef struct
 // Global initialization function (no clean up needed).
 NV_STATUS uvm_perf_prefetch_init(void);

+// Returns whether prefetching is enabled in the VA space.
+// va_space cannot be NULL.
+bool uvm_perf_prefetch_enabled(uvm_va_space_t *va_space);
+
+// Return the prefetch mask with the pages that may be prefetched in a ATS
+// block. ATS block is a system allocated memory block with base aligned to
+// UVM_VA_BLOCK_SIZE and a maximum size of UVM_VA_BLOCK_SIZE. The faulted_pages
+// mask and faulted_region are the pages being faulted on the given residency.
+//
+// Only residency_mask can be NULL.
+//
+// Locking: The caller must hold the va_space lock.
+void uvm_perf_prefetch_compute_ats(uvm_va_space_t *va_space,
+                                   const uvm_page_mask_t *faulted_pages,
+                                   uvm_va_block_region_t faulted_region,
+                                   uvm_va_block_region_t max_prefetch_region,
+                                   const uvm_page_mask_t *residency_mask,
+                                   uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                   uvm_page_mask_t *out_prefetch_mask);
+
 // Return a hint with the pages that may be prefetched in the block.
 // The faulted_pages mask and faulted_region are the pages being migrated to
 // the given residency.
-// va_block_context must not be NULL, va_block_context->policy must be valid,
-// and if the va_block is a HMM block, va_block_context->hmm.vma must be valid
-// which also means the va_block_context->mm is not NULL, retained, and locked
-// for at least read.
+// va_block_context must not be NULL, and if the va_block is a HMM
+// block, va_block_context->hmm.vma must be valid which also means the
+// va_block_context->mm is not NULL, retained, and locked for at least
+// read.
 // Locking: The caller must hold the va_space lock and va_block lock.
-void uvm_perf_prefetch_get_hint(uvm_va_block_t *va_block,
-                                uvm_va_block_context_t *va_block_context,
-                                uvm_processor_id_t new_residency,
-                                const uvm_page_mask_t *faulted_pages,
-                                uvm_va_block_region_t faulted_region,
-                                uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
-                                uvm_perf_prefetch_hint_t *out_hint);
+void uvm_perf_prefetch_get_hint_va_block(uvm_va_block_t *va_block,
+                                         uvm_va_block_context_t *va_block_context,
+                                         uvm_processor_id_t new_residency,
+                                         const uvm_page_mask_t *faulted_pages,
+                                         uvm_va_block_region_t faulted_region,
+                                         uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
+                                         uvm_perf_prefetch_hint_t *out_hint);

 void uvm_perf_prefetch_bitmap_tree_iter_init(const uvm_perf_prefetch_bitmap_tree_t *bitmap_tree,
                                             uvm_page_index_t page_index,
--- a/kernel-open/nvidia-uvm/uvm_perf_thrashing.c
+++ b/kernel-open/nvidia-uvm/uvm_perf_thrashing.c
@@ -1095,7 +1095,7 @@ static NV_STATUS unmap_remote_pinned_pages(uvm_va_block_t *va_block,
    NV_STATUS tracker_status;
    uvm_tracker_t local_tracker = UVM_TRACKER_INIT();
    uvm_processor_id_t processor_id;
-    const uvm_va_policy_t *policy = va_block_context->policy;
+    const uvm_va_policy_t *policy = uvm_va_policy_get(va_block, uvm_va_block_region_start(va_block, region));

    uvm_assert_mutex_locked(&va_block->lock);

@@ -1141,10 +1141,9 @@ NV_STATUS uvm_perf_thrashing_unmap_remote_pinned_pages_all(uvm_va_block_t *va_bl
 {
    block_thrashing_info_t *block_thrashing;
    uvm_processor_mask_t unmap_processors;
-    const uvm_va_policy_t *policy = va_block_context->policy;
+    const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, region);

    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, policy, region));

    block_thrashing = thrashing_info_get(va_block);
    if (!block_thrashing || !block_thrashing->pages)
@@ -1455,7 +1454,18 @@ static uvm_perf_thrashing_hint_t get_hint_for_migration_thrashing(va_space_thras
    hint.type = UVM_PERF_THRASHING_HINT_TYPE_NONE;

    closest_resident_id = uvm_va_block_page_get_closest_resident(va_block, page_index, requester);
-    UVM_ASSERT(UVM_ID_IS_VALID(closest_resident_id));
+    if (uvm_va_block_is_hmm(va_block)) {
+        // HMM pages always start out resident on the CPU but may not be
+        // recorded in the va_block state because hmm_range_fault() or
+        // similar functions haven't been called to get an accurate snapshot
+        // of the Linux state. We can assume pages are CPU resident for the
+        // purpose of deciding where to migrate to reduce thrashing.
+        if (UVM_ID_IS_INVALID(closest_resident_id))
+            closest_resident_id = UVM_ID_CPU;
+    }
+    else {
+        UVM_ASSERT(UVM_ID_IS_VALID(closest_resident_id));
+    }

    if (thrashing_processors_can_access(va_space, page_thrashing, preferred_location)) {
        // The logic in uvm_va_block_select_residency chooses the preferred
@@ -1856,8 +1866,6 @@ static void thrashing_unpin_pages(struct work_struct *work)
            UVM_ASSERT(uvm_page_mask_test(&block_thrashing->pinned_pages.mask, page_index));

            uvm_va_block_context_init(va_block_context, NULL);
-            va_block_context->policy =
-                uvm_va_policy_get(va_block, uvm_va_block_cpu_page_address(va_block, page_index));

            uvm_perf_thrashing_unmap_remote_pinned_pages_all(va_block,
                                                             va_block_context,
@@ -2112,8 +2120,6 @@ NV_STATUS uvm_test_set_page_thrashing_policy(UVM_TEST_SET_PAGE_THRASHING_POLICY_
                uvm_va_block_region_t va_block_region = uvm_va_block_region_from_block(va_block);
                uvm_va_block_context_t *block_context = uvm_va_space_block_context(va_space, NULL);

-                block_context->policy = uvm_va_range_get_policy(va_range);
-
                uvm_mutex_lock(&va_block->lock);

                // Unmap may split PTEs and require a retry. Needs to be called
--- a/kernel-open/nvidia-uvm/uvm_perf_thrashing.h
+++ b/kernel-open/nvidia-uvm/uvm_perf_thrashing.h
@@ -103,11 +103,11 @@ void uvm_perf_thrashing_unload(uvm_va_space_t *va_space);
 // Destroy the thrashing detection struct for the given block.
 void uvm_perf_thrashing_info_destroy(uvm_va_block_t *va_block);

-// Unmap remote mappings from all processors on the pinned pages
-// described by region and block_thrashing->pinned pages.
-// va_block_context must not be NULL and va_block_context->policy must be valid.
-// See the comments for uvm_va_block_check_policy_is_valid() in uvm_va_block.h.
-// Locking: the va_block lock must be held.
+// Unmap remote mappings from all processors on the pinned pages described by
+// region and block_thrashing->pinned pages.  va_block_context must not be NULL
+// and policy for the region must match.  See the comments for
+// uvm_va_block_check_policy_is_valid() in uvm_va_block.h.  Locking: the
+// va_block lock must be held.
 NV_STATUS uvm_perf_thrashing_unmap_remote_pinned_pages_all(uvm_va_block_t *va_block,
                                                           uvm_va_block_context_t *va_block_context,
                                                           uvm_va_block_region_t region);
--- a/kernel-open/nvidia-uvm/uvm_pmm_gpu.c
+++ b/kernel-open/nvidia-uvm/uvm_pmm_gpu.c
@@ -3820,18 +3820,11 @@ NV_STATUS uvm_test_evict_chunk(UVM_TEST_EVICT_CHUNK_PARAMS *params, struct file
    // For virtual mode, look up and retain the block first so that eviction can
    // be started without the VA space lock held.
    if (params->eviction_mode == UvmTestEvictModeVirtual) {
-        uvm_va_block_context_t *block_context;
+        if (mm)
+            status = uvm_va_block_find_create(va_space, params->address, NULL, &block);
+        else
+            status = uvm_va_block_find_create_managed(va_space, params->address, &block);

-        block_context = uvm_va_block_context_alloc(mm);
-        if (!block_context) {
-            status = NV_ERR_NO_MEMORY;
-            uvm_va_space_up_read(va_space);
-            uvm_va_space_mm_release_unlock(va_space, mm);
-            goto out;
-        }
-
-        status = uvm_va_block_find_create(va_space, params->address, block_context, &block);
-        uvm_va_block_context_free(block_context);
        if (status != NV_OK) {
            uvm_va_space_up_read(va_space);
            uvm_va_space_mm_or_current_release_unlock(va_space, mm);
--- a/kernel-open/nvidia-uvm/uvm_pmm_test.c
+++ b/kernel-open/nvidia-uvm/uvm_pmm_test.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2022 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -324,7 +324,7 @@ static NV_STATUS gpu_mem_check(uvm_gpu_t *gpu,

    // TODO: Bug 3839176: [UVM][HCC][uvm_test] Update tests that assume GPU
    //                     engines can directly access sysmem
-    // Skip this test for now. To enable this test under SEV,
+    // Skip this test for now. To enable this test in Confidential Computing,
    // The GPU->CPU CE copy needs to be updated so it uses encryption when
    // CC is enabled.
    if (uvm_conf_computing_mode_enabled(gpu))
@@ -1223,8 +1223,6 @@ static NV_STATUS test_indirect_peers(uvm_gpu_t *owning_gpu, uvm_gpu_t *accessing
    if (!chunks)
        return NV_ERR_NO_MEMORY;

-    UVM_ASSERT(!g_uvm_global.sev_enabled);
-
    TEST_NV_CHECK_GOTO(uvm_mem_alloc_sysmem_and_map_cpu_kernel(UVM_CHUNK_SIZE_MAX, current->mm, &verif_mem), out);
    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(verif_mem, owning_gpu), out);
    TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(verif_mem, accessing_gpu), out);
--- a/kernel-open/nvidia-uvm/uvm_policy.c
+++ b/kernel-open/nvidia-uvm/uvm_policy.c
@@ -160,7 +160,7 @@ static NV_STATUS preferred_location_unmap_remote_pages(uvm_va_block_t *va_block,
    NV_STATUS status = NV_OK;
    NV_STATUS tracker_status;
    uvm_tracker_t local_tracker = UVM_TRACKER_INIT();
-    const uvm_va_policy_t *policy = va_block_context->policy;
+    const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, region);
    uvm_processor_id_t preferred_location = policy->preferred_location;
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
    const uvm_page_mask_t *mapped_mask;
@@ -279,6 +279,9 @@ static NV_STATUS preferred_location_set(uvm_va_space_t *va_space,
        return NV_OK;
    }

+    if (!mm)
+        return NV_ERR_INVALID_ADDRESS;
+
    return uvm_hmm_set_preferred_location(va_space, preferred_location, base, last_address, out_tracker);
 }

@@ -445,7 +448,6 @@ NV_STATUS uvm_va_block_set_accessed_by_locked(uvm_va_block_t *va_block,
    NV_STATUS tracker_status;

    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, va_block_context->policy, region));

    status = uvm_va_block_add_mappings(va_block,
                                       va_block_context,
@@ -467,13 +469,13 @@ NV_STATUS uvm_va_block_set_accessed_by(uvm_va_block_t *va_block,
    uvm_va_block_region_t region = uvm_va_block_region_from_block(va_block);
    NV_STATUS status;
    uvm_tracker_t local_tracker = UVM_TRACKER_INIT();
+    uvm_va_policy_t *policy = uvm_va_range_get_policy(va_block->va_range);

    UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
-    UVM_ASSERT(va_block_context->policy == uvm_va_range_get_policy(va_block->va_range));

    // Read duplication takes precedence over SetAccessedBy. Do not add mappings
    // if read duplication is enabled.
-    if (uvm_va_policy_is_read_duplicate(va_block_context->policy, va_space))
+    if (uvm_va_policy_is_read_duplicate(policy, va_space))
        return NV_OK;

    status = UVM_VA_BLOCK_LOCK_RETRY(va_block,
@@ -592,8 +594,15 @@ static NV_STATUS accessed_by_set(uvm_va_space_t *va_space,
        UVM_ASSERT(va_range_last->node.end >= last_address);
    }
    else {
+        // NULL mm case already filtered by uvm_api_range_type_check()
+        UVM_ASSERT(mm);
        UVM_ASSERT(type == UVM_API_RANGE_TYPE_HMM);
-        status = uvm_hmm_set_accessed_by(va_space, processor_id, set_bit, base, last_address, &local_tracker);
+        status = uvm_hmm_set_accessed_by(va_space,
+                                         processor_id,
+                                         set_bit,
+                                         base,
+                                         last_address,
+                                         &local_tracker);
    }

 done:
@@ -656,7 +665,6 @@ NV_STATUS uvm_va_block_set_read_duplication(uvm_va_block_t *va_block,

    // TODO: Bug 3660922: need to implement HMM read duplication support.
    UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
-    UVM_ASSERT(va_block_context->policy == uvm_va_range_get_policy(va_block->va_range));

    status = UVM_VA_BLOCK_LOCK_RETRY(va_block, &va_block_retry,
                                     va_block_set_read_duplication_locked(va_block,
@@ -675,7 +683,7 @@ static NV_STATUS va_block_unset_read_duplication_locked(uvm_va_block_t *va_block
    uvm_processor_id_t processor_id;
    uvm_va_block_region_t block_region = uvm_va_block_region_from_block(va_block);
    uvm_page_mask_t *break_read_duplication_pages = &va_block_context->caller_page_mask;
-    const uvm_va_policy_t *policy = va_block_context->policy;
+    const uvm_va_policy_t *policy = uvm_va_range_get_policy(va_block->va_range);
    uvm_processor_id_t preferred_location = policy->preferred_location;
    uvm_processor_mask_t accessed_by = policy->accessed_by;

@@ -757,7 +765,6 @@ NV_STATUS uvm_va_block_unset_read_duplication(uvm_va_block_t *va_block,
    uvm_tracker_t local_tracker = UVM_TRACKER_INIT();

    UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
-    UVM_ASSERT(va_block_context->policy == uvm_va_range_get_policy(va_block->va_range));

    // Restore all SetAccessedBy mappings
    status = UVM_VA_BLOCK_LOCK_RETRY(va_block, &va_block_retry,
@@ -915,7 +922,6 @@ static NV_STATUS system_wide_atomics_set(uvm_va_space_t *va_space, const NvProce
            if (va_range->type != UVM_VA_RANGE_TYPE_MANAGED)
                continue;

-            va_block_context->policy = uvm_va_range_get_policy(va_range);
            for_each_va_block_in_va_range(va_range, va_block) {
                uvm_page_mask_t *non_resident_pages = &va_block_context->caller_page_mask;

--- a/kernel-open/nvidia-uvm/uvm_push.c
+++ b/kernel-open/nvidia-uvm/uvm_push.c
@@ -391,11 +391,13 @@ uvm_gpu_address_t uvm_push_inline_data_end(uvm_push_inline_data_t *data)
        inline_data_address = (NvU64) (uintptr_t)(push->next + 1);
    }
    else {
+        uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
+
        // Offset of the inlined data within the push.
        inline_data_address = (push->next - push->begin + 1) * UVM_METHOD_SIZE;

        // Add GPU VA of the push begin
-        inline_data_address += uvm_pushbuffer_get_gpu_va_for_push(channel->pool->manager->pushbuffer, push);
+        inline_data_address += uvm_pushbuffer_get_gpu_va_for_push(pushbuffer, push);
    }

    // This will place a noop right before the inline data that was written.
@@ -438,10 +440,8 @@ NvU64 *uvm_push_timestamp(uvm_push_t *push)

    if (uvm_channel_is_ce(push->channel))
        gpu->parent->ce_hal->semaphore_timestamp(push, address.address);
-    else if (uvm_channel_is_sec2(push->channel))
-        gpu->parent->sec2_hal->semaphore_timestamp(push, address.address);
    else
-        UVM_ASSERT_MSG(0, "Semaphore release timestamp on an unsupported channel.\n");
+        gpu->parent->sec2_hal->semaphore_timestamp(push, address.address);

    return timestamp;
 }
--- a/kernel-open/nvidia-uvm/uvm_push.h
+++ b/kernel-open/nvidia-uvm/uvm_push.h
@@ -64,6 +64,14 @@ typedef enum
    UVM_PUSH_FLAG_COUNT,
 } uvm_push_flag_t;

+struct uvm_push_crypto_bundle_struct {
+    // Initialization vector used to decrypt the push
+    UvmCslIv iv;
+
+    // Size of the pushbuffer that is encrypted/decrypted
+    NvU32 push_size;
+};
+
 struct uvm_push_struct
 {
    // Location of the first method of the push
@@ -369,11 +377,6 @@ static bool uvm_push_has_space(uvm_push_t *push, NvU32 free_space)
 NV_STATUS uvm_push_begin_fake(uvm_gpu_t *gpu, uvm_push_t *push);
 void uvm_push_end_fake(uvm_push_t *push);

-static bool uvm_push_is_fake(uvm_push_t *push)
-{
-    return !push->channel;
-}
-
 // Begin an inline data fragment in the push
 //
 // The inline data will be ignored by the GPU, but can be referenced from
--- a/kernel-open/nvidia-uvm/uvm_push_test.c
+++ b/kernel-open/nvidia-uvm/uvm_push_test.c
@@ -40,10 +40,9 @@

 static NvU32 get_push_begin_size(uvm_channel_t *channel)
 {
-    if (uvm_channel_is_sec2(channel)) {
-        // SEC2 channels allocate CSL signature buffer at the beginning.
+    // SEC2 channels allocate CSL signature buffer at the beginning.
+    if (uvm_channel_is_sec2(channel))
        return UVM_CONF_COMPUTING_SIGN_BUF_MAX_SIZE + UVM_METHOD_SIZE;
-    }

    return 0;
 }
@@ -51,10 +50,14 @@ static NvU32 get_push_begin_size(uvm_channel_t *channel)
 // This is the storage required by a semaphore release.
 static NvU32 get_push_end_min_size(uvm_channel_t *channel)
 {
-    if (uvm_channel_is_ce(channel)) {
-        if (uvm_channel_is_wlc(channel)) {
-            // Space (in bytes) used by uvm_push_end() on a Secure CE channel.
-            // Note that Secure CE semaphore release pushes two memset and one
+    uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);
+
+    if (uvm_conf_computing_mode_enabled(gpu)) {
+        if (uvm_channel_is_ce(channel)) {
+            // Space (in bytes) used by uvm_push_end() on a CE channel when
+            // the Confidential Computing feature is enabled.
+            //
+            // Note that CE semaphore release pushes two memset and one
            // encryption method on top of the regular release.
            // Memset size
            // -------------
@@ -75,43 +78,44 @@ static NvU32 get_push_end_min_size(uvm_channel_t *channel)
            //
            // TOTAL                            : 144 Bytes

-            // Same as CE + LCIC GPPut update + LCIC doorbell
-            return 24 + 144 + 24 + 24;
-        }
-        else if (uvm_channel_is_secure_ce(channel)) {
+            if (uvm_channel_is_wlc(channel)) {
+                // Same as CE + LCIC GPPut update + LCIC doorbell
+                return 24 + 144 + 24 + 24;
+            }
+
            return 24 + 144;
        }
-        // Space (in bytes) used by uvm_push_end() on a CE channel.
-        return 24;
-    }
-    else if (uvm_channel_is_sec2(channel)) {
+
+        UVM_ASSERT(uvm_channel_is_sec2(channel));
+
        // A perfectly aligned inline buffer in SEC2 semaphore release.
        // We add UVM_METHOD_SIZE because of the NOP method to reserve
        // UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES (the inline buffer.)
        return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_METHOD_SIZE;
    }

-    return 0;
+    UVM_ASSERT(uvm_channel_is_ce(channel));
+
+    // Space (in bytes) used by uvm_push_end() on a CE channel.
+    return 24;
 }

 static NvU32 get_push_end_max_size(uvm_channel_t *channel)
 {
-    if (uvm_channel_is_ce(channel)) {
-        if (uvm_channel_is_wlc(channel)) {
-            // WLC pushes are always padded to UVM_MAX_WLC_PUSH_SIZE
-            return UVM_MAX_WLC_PUSH_SIZE;
-        }
-        // Space (in bytes) used by uvm_push_end() on a CE channel.
-        return get_push_end_min_size(channel);
-    }
-    else if (uvm_channel_is_sec2(channel)) {
-        // Space (in bytes) used by uvm_push_end() on a SEC2 channel.
-        // Note that SEC2 semaphore release uses an inline buffer with alignment
-        // requirements. This is the "worst" case semaphore_release storage.
-        return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT;
-    }
+    // WLC pushes are always padded to UVM_MAX_WLC_PUSH_SIZE
+    if (uvm_channel_is_wlc(channel))
+        return UVM_MAX_WLC_PUSH_SIZE;

-    return 0;
+    // Space (in bytes) used by uvm_push_end() on a SEC2 channel.
+    // Note that SEC2 semaphore release uses an inline buffer with alignment
+    // requirements. This is the "worst" case semaphore_release storage.
+    if (uvm_channel_is_sec2(channel))
+        return 48 + UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES + UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT;
+
+    UVM_ASSERT(uvm_channel_is_ce(channel));
+
+    // Space (in bytes) used by uvm_push_end() on a CE channel.
+    return get_push_end_min_size(channel);
 }

 static NV_STATUS test_push_end_size(uvm_va_space_t *va_space)
@@ -294,10 +298,19 @@ static NV_STATUS test_concurrent_pushes(uvm_va_space_t *va_space)
 {
    NV_STATUS status = NV_OK;
    uvm_gpu_t *gpu;
-    NvU32 i;
    uvm_push_t *pushes;
-    uvm_tracker_t tracker = UVM_TRACKER_INIT();
-    uvm_channel_type_t channel_type = UVM_CHANNEL_TYPE_GPU_INTERNAL;
+    uvm_tracker_t tracker;
+
+    // When the Confidential Computing feature is enabled, a channel reserved at
+    // the start of a push cannot be reserved again until that push ends. The
+    // test is waived, because the number of pushes it starts per pool exceeds
+    // the number of channels in the pool, so it would block indefinitely.
+    gpu = uvm_va_space_find_first_gpu(va_space);
+
+    if ((gpu != NULL) && uvm_conf_computing_mode_enabled(gpu))
+        return NV_OK;
+
+    uvm_tracker_init(&tracker);

    // As noted above, this test does unsafe things that would be detected by
    // lock tracking, opt-out.
@@ -310,16 +323,11 @@ static NV_STATUS test_concurrent_pushes(uvm_va_space_t *va_space)
    }

    for_each_va_space_gpu(gpu, va_space) {
+        NvU32 i;

-        // A secure channels reserved at the start of a push cannot be reserved
-        // again until that push ends. The test would block indefinitely
-        // if secure pools are not skipped, because the number of pushes started
-        // per pool exceeds the number of channels in the pool.
-        if (uvm_channel_type_requires_secure_pool(gpu, channel_type))
-            goto done;
        for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
            uvm_push_t *push = &pushes[i];
-            status = uvm_push_begin(gpu->channel_manager, channel_type, push, "concurrent push %u", i);
+            status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, push, "concurrent push %u", i);
            TEST_CHECK_GOTO(status == NV_OK, done);
        }
        for (i = 0; i < UVM_PUSH_MAX_CONCURRENT_PUSHES; ++i) {
@@ -776,15 +784,6 @@ static NV_STATUS test_timestamp_on_gpu(uvm_gpu_t *gpu)
    NvU32 i;
    NvU64 last_stamp = 0;

-    // TODO: Bug 3988992: [UVM][HCC] RFE - Support encrypted semaphore for secure CE channels
-    // This test is waived when Confidential Computing is enabled because it
-    // assumes that CPU can directly read the result of a semaphore timestamp
-    // operation. Instead the operation needs to be follower up by an encrypt
-    // -decrypt trip to be accessible to CPU. This will be cleaner and simpler
-    // once encrypted semaphores are available.
-    if (uvm_conf_computing_mode_enabled(gpu))
-        return NV_OK;
-
    for (i = 0; i < 10; ++i) {
        status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_INTERNAL, &push, "Releasing a timestamp");
        if (status != NV_OK)
--- a/kernel-open/nvidia-uvm/uvm_pushbuffer.c
+++ b/kernel-open/nvidia-uvm/uvm_pushbuffer.c
@@ -449,21 +449,68 @@ static uvm_pushbuffer_chunk_t *gpfifo_to_chunk(uvm_pushbuffer_t *pushbuffer, uvm
    return chunk;
 }

-void uvm_pushbuffer_mark_completed(uvm_pushbuffer_t *pushbuffer, uvm_gpfifo_entry_t *gpfifo)
+static void decrypt_push(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo)
+{
+    NV_STATUS status;
+    NvU32 auth_tag_offset;
+    void *auth_tag_cpu_va;
+    void *push_protected_cpu_va;
+    void *push_unprotected_cpu_va;
+    NvU32 pushbuffer_offset = gpfifo->pushbuffer_offset;
+    NvU32 push_info_index = gpfifo->push_info - channel->push_infos;
+    uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
+    uvm_push_crypto_bundle_t *crypto_bundle = channel->conf_computing.push_crypto_bundles + push_info_index;
+
+    if (channel->conf_computing.push_crypto_bundles == NULL)
+        return;
+
+    // When the crypto bundle is used, the push size cannot be zero
+    if (crypto_bundle->push_size == 0)
+        return;
+
+    UVM_ASSERT(!uvm_channel_is_wlc(channel));
+    UVM_ASSERT(!uvm_channel_is_lcic(channel));
+
+    push_protected_cpu_va = (char *)get_base_cpu_va(pushbuffer) + pushbuffer_offset;
+    push_unprotected_cpu_va = (char *)uvm_rm_mem_get_cpu_va(pushbuffer->memory_unprotected_sysmem) + pushbuffer_offset;
+    auth_tag_offset = push_info_index * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
+    auth_tag_cpu_va = (char *)uvm_rm_mem_get_cpu_va(channel->conf_computing.push_crypto_bundle_auth_tags) +
+                              auth_tag_offset;
+
+    status = uvm_conf_computing_cpu_decrypt(channel,
+                                            push_protected_cpu_va,
+                                            push_unprotected_cpu_va,
+                                            &crypto_bundle->iv,
+                                            crypto_bundle->push_size,
+                                            auth_tag_cpu_va);
+
+    // A decryption failure here is not fatal because it does not
+    // prevent UVM from running fine in the future and cannot be used
+    // maliciously to leak information or otherwise derail UVM from its
+    // regular duties.
+    UVM_ASSERT_MSG_RELEASE(status == NV_OK, "Pushbuffer decryption failure: %s\n", nvstatusToString(status));
+
+    // Avoid reusing the bundle across multiple pushes
+    crypto_bundle->push_size = 0;
+}
+
+void uvm_pushbuffer_mark_completed(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo)
 {
    uvm_pushbuffer_chunk_t *chunk;
-    uvm_push_info_t *push_info = gpfifo->push_info;
    bool need_to_update_chunk = false;
+    uvm_push_info_t *push_info = gpfifo->push_info;
+    uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);

    UVM_ASSERT(gpfifo->type == UVM_GPFIFO_ENTRY_TYPE_NORMAL);

    chunk = gpfifo_to_chunk(pushbuffer, gpfifo);

-    if (push_info->on_complete != NULL)
+    if (push_info->on_complete != NULL) {
+        decrypt_push(channel, gpfifo);
        push_info->on_complete(push_info->on_complete_data);
-
-    push_info->on_complete = NULL;
-    push_info->on_complete_data = NULL;
+        push_info->on_complete = NULL;
+        push_info->on_complete_data = NULL;
+    }

    uvm_spin_lock(&pushbuffer->lock);

--- a/kernel-open/nvidia-uvm/uvm_pushbuffer.h
+++ b/kernel-open/nvidia-uvm/uvm_pushbuffer.h
@@ -161,22 +161,22 @@
 // * WFI:                     8B
 // Total:                    64B
 //
-// Push space needed for secure work launch is 224B. The push is constructed
+// Push space needed for secure work launch is 364B. The push is constructed
 // in 'internal_channel_submit_work_indirect' and 'uvm_channel_end_push'
 // * CE decrypt (of indirect PB):                   56B
-// * 2*semaphore release (indirect GPFIFO entry): 2*24B
+// * memset_8 (indirect GPFIFO entry):              44B
 // * semaphore release (indirect GPPUT):            24B
 // * semaphore release (indirect doorbell):         24B
 // Appendix added in 'uvm_channel_end_push':
 // * semaphore release (WLC tracking):             168B
-//      * semaphore increment (memcopy):            24B
+//      * semaphore release (payload):              24B
 //      * notifier memset:                          40B
 //      * payload encryption:                       64B
 //      * notifier memset:                          40B
 // * semaphore increment (LCIC GPPUT):              24B
 // * semaphore release (LCIC doorbell):             24B
-// Total:                                          368B
-#define UVM_MAX_WLC_PUSH_SIZE (368)
+// Total:                                          364B
+#define UVM_MAX_WLC_PUSH_SIZE (364)

 // Push space needed for static LCIC schedule, as initialized in
 // 'setup_lcic_schedule':
@@ -184,7 +184,7 @@
 // * semaphore increment (WLC GPPUT):      24B
 // * semaphore increment (WLC GPPUT):      24B
 // * semaphore increment (LCIC tracking): 160B
-//      * semaphore increment (memcopy):   24B
+//      * semaphore increment (payload):   24B
 //      * notifier memcopy:                36B
 //      * payload encryption:              64B
 //      * notifier memcopy:                36B
@@ -258,7 +258,7 @@ NV_STATUS uvm_pushbuffer_begin_push(uvm_pushbuffer_t *pushbuffer, uvm_push_t *pu

 // Complete a pending push
 // Updates the chunk state the pending push used
-void uvm_pushbuffer_mark_completed(uvm_pushbuffer_t *pushbuffer, uvm_gpfifo_entry_t *gpfifo);
+void uvm_pushbuffer_mark_completed(uvm_channel_t *channel, uvm_gpfifo_entry_t *gpfifo);

 // Get the GPU VA for an ongoing push
 NvU64 uvm_pushbuffer_get_gpu_va_for_push(uvm_pushbuffer_t *pushbuffer, uvm_push_t *push);
--- a/kernel-open/nvidia-uvm/uvm_range_group.c
+++ b/kernel-open/nvidia-uvm/uvm_range_group.c
@@ -264,7 +264,6 @@ NV_STATUS uvm_range_group_va_range_migrate(uvm_va_range_t *va_range,
        return NV_ERR_NO_MEMORY;

    uvm_assert_rwsem_locked(&va_range->va_space->lock);
-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    // Iterate over blocks, populating them if necessary
    for (i = uvm_va_range_block_index(va_range, start); i <= uvm_va_range_block_index(va_range, end); ++i) {
--- a/kernel-open/nvidia-uvm/uvm_sec2_test.c
+++ b/kernel-open/nvidia-uvm/uvm_sec2_test.c
@@ -213,6 +213,7 @@ done:
 typedef enum
 {
    MEM_ALLOC_TYPE_SYSMEM_DMA,
+    MEM_ALLOC_TYPE_SYSMEM_PROTECTED,
    MEM_ALLOC_TYPE_VIDMEM_PROTECTED
 } mem_alloc_type_t;

@@ -269,14 +270,20 @@ static NV_STATUS alloc_and_init_mem(uvm_gpu_t *gpu, uvm_mem_t **mem, size_t size
    *mem = NULL;

    if (type == MEM_ALLOC_TYPE_VIDMEM_PROTECTED) {
-        TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem_protected(size, gpu, mem));
+        TEST_NV_CHECK_RET(uvm_mem_alloc_vidmem(size, gpu, mem));
        TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
        TEST_NV_CHECK_GOTO(ce_memset_gpu(gpu, *mem, size, 0xdead), err);
    }
    else {
-        TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem_dma(size, gpu, NULL, mem));
+        if (type == MEM_ALLOC_TYPE_SYSMEM_DMA) {
+            TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem_dma(size, gpu, NULL, mem));
+            TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
+        }
+        else {
+            TEST_NV_CHECK_RET(uvm_mem_alloc_sysmem(size, NULL, mem));
+        }
+
        TEST_NV_CHECK_GOTO(uvm_mem_map_cpu_kernel(*mem), err);
-        TEST_NV_CHECK_GOTO(uvm_mem_map_gpu_kernel(*mem, gpu), err);
        write_range_cpu(*mem, size, 0xdeaddead);
    }

@@ -341,9 +348,9 @@ static NV_STATUS cpu_decrypt(uvm_channel_t *channel,
    return NV_OK;
 }

-// gpu_encrypt uses a secure CE for encryption (instead of SEC2). SEC2 does not
-// support encryption. The following function is copied from uvm_ce_test.c and
-// adapted to SEC2 tests.
+// gpu_encrypt uses the Copy Engine for encryption, instead of SEC2. SEC2 does
+// not support encryption. The following function is copied from uvm_ce_test.c
+// and adapted to SEC2 tests.
 static void gpu_encrypt(uvm_push_t *push,
                        uvm_mem_t *dst_mem,
                        uvm_mem_t *src_mem,
@@ -405,48 +412,6 @@ static void gpu_decrypt(uvm_push_t *push,
    }
 }

-// This test only uses sysmem so that we can use the CPU for encryption and SEC2
-// for decryption, i.e., the test doesn't depend on any other GPU engine for
-// the encryption operation (refer to test_cpu_to_gpu_roundtrip()). This is not
-// how SEC2 is used in the driver. The intended SEC2 usage is to decrypt from
-// unprotected sysmem to protected vidmem, which is tested in
-// test_cpu_to_gpu_roundtrip().
-static NV_STATUS test_cpu_to_gpu_sysmem(uvm_gpu_t *gpu, size_t copy_size, size_t size)
-{
-    NV_STATUS status = NV_OK;
-    uvm_mem_t *src_plain = NULL;
-    uvm_mem_t *cipher = NULL;
-    uvm_mem_t *dst_plain = NULL;
-    uvm_mem_t *auth_tag_mem = NULL;
-    size_t auth_tag_buffer_size = (size / copy_size) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
-    uvm_push_t push;
-
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &auth_tag_mem, auth_tag_buffer_size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
-
-    write_range_cpu(src_plain, size, uvm_get_stale_thread_id());
-    write_range_cpu(dst_plain, size, 0xA5A5A5A5);
-
-    TEST_NV_CHECK_GOTO(uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_SEC2, &push, "enc(cpu)_dec(gpu)"), out);
-
-    cpu_encrypt(push.channel, cipher, src_plain, auth_tag_mem, size, copy_size);
-    gpu_decrypt(&push, dst_plain, cipher, auth_tag_mem, size, copy_size);
-
-    uvm_push_end_and_wait(&push);
-
-    TEST_CHECK_GOTO(mem_match(src_plain, dst_plain), out);
-
-out:
-    uvm_mem_free(auth_tag_mem);
-    uvm_mem_free(cipher);
-    uvm_mem_free(dst_plain);
-    uvm_mem_free(src_plain);
-
-    return status;
-}
-
 // This test depends on the CE for the encryption, so we assume tests from
 // uvm_ce_test.c have successfully passed.
 static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, size_t size)
@@ -461,19 +426,16 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, siz
    size_t auth_tag_buffer_size = (size / copy_size) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
    uvm_push_t push;
    UvmCslIv *decrypt_iv;
-    uvm_tracker_t tracker;

    decrypt_iv = uvm_kvmalloc_zero((size / copy_size) * sizeof(UvmCslIv));
    if (!decrypt_iv)
        return NV_ERR_NO_MEMORY;

-    uvm_tracker_init(&tracker);
-
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
+    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_plain, size, MEM_ALLOC_TYPE_SYSMEM_PROTECTED), out);
    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &src_cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_cipher, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain, size, MEM_ALLOC_TYPE_VIDMEM_PROTECTED), out);
-    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain_cpu, size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);
+    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &dst_plain_cpu, size, MEM_ALLOC_TYPE_SYSMEM_PROTECTED), out);
    TEST_NV_CHECK_GOTO(alloc_and_init_mem(gpu, &auth_tag_mem, auth_tag_buffer_size, MEM_ALLOC_TYPE_SYSMEM_DMA), out);

    write_range_cpu(src_plain, size, uvm_get_stale_thread_id());
@@ -483,14 +445,12 @@ static NV_STATUS test_cpu_to_gpu_roundtrip(uvm_gpu_t *gpu, size_t copy_size, siz
    cpu_encrypt(push.channel, src_cipher, src_plain, auth_tag_mem, size, copy_size);
    gpu_decrypt(&push, dst_plain, src_cipher, auth_tag_mem, size, copy_size);

-    uvm_push_end(&push);
-    TEST_NV_CHECK_GOTO(uvm_tracker_add_push(&tracker, &push), out);
+    // Wait for SEC2 before launching the CE part.
+    // SEC2 is only allowed to release semaphores in unprotected sysmem,
+    // and CE can only acquire semaphores in protected vidmem.
+    TEST_NV_CHECK_GOTO(uvm_push_end_and_wait(&push), out);

-    TEST_NV_CHECK_GOTO(uvm_push_begin_acquire(gpu->channel_manager,
-                                              UVM_CHANNEL_TYPE_GPU_TO_CPU,
-                                              &tracker,
-                                              &push,
-                                              "enc(gpu)_dec(cpu)"),
+    TEST_NV_CHECK_GOTO(uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_GPU_TO_CPU, &push, "enc(gpu)_dec(cpu)"),
                       out);

    gpu_encrypt(&push, dst_cipher, dst_plain, decrypt_iv, auth_tag_mem, size, copy_size);
@@ -521,8 +481,6 @@ out:

    uvm_kvfree(decrypt_iv);

-    uvm_tracker_deinit(&tracker);
-
    return status;
 }

@@ -545,7 +503,6 @@ static NV_STATUS test_encryption_decryption(uvm_gpu_t *gpu)

        UVM_ASSERT(size % copy_sizes[i] == 0);

-        TEST_NV_CHECK_RET(test_cpu_to_gpu_sysmem(gpu, copy_sizes[i], size));
        TEST_NV_CHECK_RET(test_cpu_to_gpu_roundtrip(gpu, copy_sizes[i], size));
    }

--- a/kernel-open/nvidia-uvm/uvm_tools.c
+++ b/kernel-open/nvidia-uvm/uvm_tools.c
@@ -229,6 +229,24 @@ static void unmap_user_pages(struct page **pages, void *addr, NvU64 size)
    uvm_kvfree(pages);
 }

+// This must be called with the mmap_lock held in read mode or better.
+static NV_STATUS check_vmas(struct mm_struct *mm, NvU64 start_va, NvU64 size)
+{
+    struct vm_area_struct *vma;
+    NvU64 addr = start_va;
+    NvU64 region_end = start_va + size;
+
+    do {
+        vma = find_vma(mm, addr);
+        if (!vma || !(addr >= vma->vm_start) || uvm_file_is_nvidia_uvm(vma->vm_file))
+            return NV_ERR_INVALID_ARGUMENT;
+
+        addr = vma->vm_end;
+    } while (addr < region_end);
+
+    return NV_OK;
+}
+
 // Map virtual memory of data from [user_va, user_va + size) of current process into kernel.
 // Sets *addr to kernel mapping and *pages to the array of struct pages that contain the memory.
 static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct page ***pages)
@@ -237,7 +255,6 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
    long ret = 0;
    long num_pages;
    long i;
-    struct vm_area_struct **vmas = NULL;

    *addr = NULL;
    *pages = NULL;
@@ -254,22 +271,30 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
        goto fail;
    }

-    vmas = uvm_kvmalloc(sizeof(struct vm_area_struct *) * num_pages);
-    if (vmas == NULL) {
-        status = NV_ERR_NO_MEMORY;
+    // Although uvm_down_read_mmap_lock() is preferable due to its participation
+    // in the UVM lock dependency tracker, it cannot be used here. That's
+    // because pin_user_pages() may fault in HMM pages which are GPU-resident.
+    // When that happens, the UVM page fault handler would record another
+    // mmap_read_lock() on the same thread as this one, leading to a false
+    // positive lock dependency report.
+    //
+    // Therefore, use the lower level nv_mmap_read_lock() here.
+    nv_mmap_read_lock(current->mm);
+    status = check_vmas(current->mm, user_va, size);
+    if (status != NV_OK) {
+        nv_mmap_read_unlock(current->mm);
        goto fail;
    }
-
-    nv_mmap_read_lock(current->mm);
-    ret = NV_PIN_USER_PAGES(user_va, num_pages, FOLL_WRITE, *pages, vmas);
+    ret = NV_PIN_USER_PAGES(user_va, num_pages, FOLL_WRITE, *pages, NULL);
    nv_mmap_read_unlock(current->mm);
+
    if (ret != num_pages) {
        status = NV_ERR_INVALID_ARGUMENT;
        goto fail;
    }

    for (i = 0; i < num_pages; i++) {
-        if (page_count((*pages)[i]) > MAX_PAGE_COUNT || uvm_file_is_nvidia_uvm(vmas[i]->vm_file)) {
+        if (page_count((*pages)[i]) > MAX_PAGE_COUNT) {
            status = NV_ERR_INVALID_ARGUMENT;
            goto fail;
        }
@@ -279,15 +304,12 @@ static NV_STATUS map_user_pages(NvU64 user_va, NvU64 size, void **addr, struct p
    if (*addr == NULL)
        goto fail;

-    uvm_kvfree(vmas);
    return NV_OK;

 fail:
    if (*pages == NULL)
        return status;

-    uvm_kvfree(vmas);
-
    if (ret > 0)
        uvm_put_user_pages_dirty(*pages, ret);
    else if (ret < 0)
@@ -2047,7 +2069,11 @@ static NV_STATUS tools_access_process_memory(uvm_va_space_t *va_space,

        // The RM flavor of the lock is needed to perform ECC checks.
        uvm_va_space_down_read_rm(va_space);
-        status = uvm_va_block_find_create(va_space, UVM_PAGE_ALIGN_DOWN(target_va_start), block_context, &block);
+        if (mm)
+            status = uvm_va_block_find_create(va_space, UVM_PAGE_ALIGN_DOWN(target_va_start), &block_context->hmm.vma, &block);
+        else
+            status = uvm_va_block_find_create_managed(va_space, UVM_PAGE_ALIGN_DOWN(target_va_start), &block);
+
        if (status != NV_OK)
            goto unlock_and_exit;

--- a/kernel-open/nvidia-uvm/uvm_tracker_test.c
+++ b/kernel-open/nvidia-uvm/uvm_tracker_test.c
@@ -69,6 +69,14 @@ static NV_STATUS test_tracker_completion(uvm_va_space_t *va_space)
    gpu = uvm_va_space_find_first_gpu(va_space);
    TEST_CHECK_RET(gpu != NULL);

+    // TODO: Bug 4008734: [UVM][HCC] Extend secure tracking semaphore mechanism
+    //                     to all semaphore
+    // This test allocates semaphore in vidmem and then releases it from the CPU
+    // SEC2 channels cannot target semaphores in vidmem. Moreover, CPU cannot
+    // directly release values to vidmem for CE channels.
+    if (uvm_conf_computing_mode_enabled(gpu))
+        return NV_OK;
+
    TEST_NV_CHECK_RET(uvm_gpu_semaphore_alloc(gpu->semaphore_pool, &sema));

    uvm_tracker_init(&tracker);
--- a/kernel-open/nvidia-uvm/uvm_va_block.c
+++ b/kernel-open/nvidia-uvm/uvm_va_block.c
@@ -106,36 +106,6 @@ uvm_va_space_t *uvm_va_block_get_va_space(uvm_va_block_t *va_block)
    return va_space;
 }

-bool uvm_va_block_check_policy_is_valid(uvm_va_block_t *va_block,
-                                        const uvm_va_policy_t *policy,
-                                        uvm_va_block_region_t region)
-{
-    uvm_assert_mutex_locked(&va_block->lock);
-
-    if (uvm_va_block_is_hmm(va_block)) {
-        const uvm_va_policy_node_t *node;
-
-        if (uvm_va_policy_is_default(policy)) {
-            // There should only be the default policy within the region.
-            node = uvm_va_policy_node_iter_first(va_block,
-                                                 uvm_va_block_region_start(va_block, region),
-                                                 uvm_va_block_region_end(va_block, region));
-            UVM_ASSERT(!node);
-        }
-        else {
-            // The policy node should cover the region.
-            node = uvm_va_policy_node_from_policy(policy);
-            UVM_ASSERT(node->node.start <= uvm_va_block_region_start(va_block, region));
-            UVM_ASSERT(node->node.end >= uvm_va_block_region_end(va_block, region));
-        }
-    }
-    else {
-        UVM_ASSERT(policy == uvm_va_range_get_policy(va_block->va_range));
-    }
-
-    return true;
-}
-
 static NvU64 block_gpu_pte_flag_cacheable(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_processor_id_t resident_id)
 {
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
@@ -2083,12 +2053,6 @@ static uvm_processor_id_t block_page_get_closest_resident_in_mask(uvm_va_block_t
            return id;
    }

-    // HMM va_blocks don't know if a page is CPU resident until either
-    // migrate_vma_setup() or hmm_range_fault() is called. If a page isn't
-    // resident anywhere, assume it is CPU resident.
-    if (uvm_va_block_is_hmm(va_block))
-        return UVM_ID_CPU;
-
    return UVM_ID_INVALID;
 }

@@ -2888,7 +2852,7 @@ static uvm_va_block_region_t block_phys_contig_region(uvm_va_block_t *block,
 {
    if (UVM_ID_IS_CPU(resident_id)) {
        uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_chunk_for_page(block, page_index);
-        return uvm_va_block_region(page_index, page_index + uvm_cpu_chunk_num_pages(chunk));
+        return uvm_cpu_chunk_block_region(block, chunk, page_index);
    }
    else {
        uvm_chunk_size_t chunk_size;
@@ -3061,7 +3025,7 @@ static NV_STATUS conf_computing_copy_pages_finish(uvm_va_block_t *block,
    void *auth_tag_buffer_base = uvm_mem_get_cpu_addr_kernel(dma_buffer->auth_tag);
    void *staging_buffer_base = uvm_mem_get_cpu_addr_kernel(dma_buffer->alloc);

-    UVM_ASSERT(uvm_channel_is_secure(push->channel));
+    UVM_ASSERT(uvm_conf_computing_mode_enabled(push->gpu));

    if (UVM_ID_IS_GPU(copy_state->dst.id))
        return NV_OK;
@@ -3112,7 +3076,7 @@ static void block_copy_push(uvm_va_block_t *block,

    uvm_push_set_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_NONE);

-    if (uvm_channel_is_secure(push->channel)) {
+    if (uvm_conf_computing_mode_enabled(gpu)) {
        if (UVM_ID_IS_CPU(copy_state->src.id))
            conf_computing_block_copy_push_cpu_to_gpu(block, copy_state, region, push);
        else
@@ -3140,19 +3104,18 @@ static NV_STATUS block_copy_end_push(uvm_va_block_t *block,
    //       at that point.
    uvm_push_end(push);

-    if ((push_status == NV_OK) && uvm_channel_is_secure(push->channel))
+    if ((push_status == NV_OK) && uvm_conf_computing_mode_enabled(push->gpu))
        push_status = conf_computing_copy_pages_finish(block, copy_state, push);

    tracker_status = uvm_tracker_add_push_safe(copy_tracker, push);
    if (push_status == NV_OK)
        push_status = tracker_status;

-    if (uvm_channel_is_secure(push->channel)) {
-        uvm_gpu_t *gpu = uvm_push_get_gpu(push);
+    if (uvm_conf_computing_mode_enabled(push->gpu)) {
        uvm_tracker_t local_tracker = UVM_TRACKER_INIT();

        uvm_tracker_overwrite_with_push(&local_tracker, push);
-        uvm_conf_computing_dma_buffer_free(&gpu->conf_computing.dma_buffer_pool,
+        uvm_conf_computing_dma_buffer_free(&push->gpu->conf_computing.dma_buffer_pool,
                                           copy_state->dma_buffer,
                                           &local_tracker);
        copy_state->dma_buffer = NULL;
@@ -3704,7 +3667,6 @@ NV_STATUS uvm_va_block_make_resident_copy(uvm_va_block_t *va_block,

    uvm_assert_mutex_locked(&va_block->lock);
    UVM_ASSERT(uvm_va_block_is_hmm(va_block) || va_block->va_range->type == UVM_VA_RANGE_TYPE_MANAGED);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, va_block_context->policy, region));

    resident_mask = block_resident_mask_get_alloc(va_block, dest_id);
    if (!resident_mask)
@@ -3951,7 +3913,6 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,

    // TODO: Bug 3660922: need to implement HMM read duplication support.
    UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
-    UVM_ASSERT(va_block_context->policy == uvm_va_range_get_policy(va_block->va_range));

    va_block_context->make_resident.dest_id = dest_id;
    va_block_context->make_resident.cause = cause;
@@ -4749,7 +4710,7 @@ static void block_unmap_cpu(uvm_va_block_t *block, uvm_va_block_region_t region,
 // Given a mask of mapped pages, returns true if any of the pages in the mask
 // are mapped remotely by the given GPU.
 static bool block_has_remote_mapping_gpu(uvm_va_block_t *block,
-                                         uvm_va_block_context_t *block_context,
+                                         uvm_page_mask_t *scratch_page_mask,
                                         uvm_gpu_id_t gpu_id,
                                         const uvm_page_mask_t *mapped_pages)
 {
@@ -4771,7 +4732,7 @@ static bool block_has_remote_mapping_gpu(uvm_va_block_t *block,
    }

    // Remote pages are pages which are mapped but not resident locally
-    return uvm_page_mask_andnot(&block_context->scratch_page_mask, mapped_pages, &gpu_state->resident);
+    return uvm_page_mask_andnot(scratch_page_mask, mapped_pages, &gpu_state->resident);
 }

 // Writes pte_clear_val to the 4k PTEs covered by clear_page_mask. If
@@ -6666,7 +6627,7 @@ static NV_STATUS block_unmap_gpu(uvm_va_block_t *block,
    if (status != NV_OK)
        return status;

-    only_local_mappings = !block_has_remote_mapping_gpu(block, block_context, gpu->id, pages_to_unmap);
+    only_local_mappings = !block_has_remote_mapping_gpu(block, &block_context->scratch_page_mask, gpu->id, pages_to_unmap);
    tlb_membar = uvm_hal_downgrade_membar_type(gpu, only_local_mappings);

    status = uvm_push_begin_acquire(gpu->channel_manager,
@@ -6801,16 +6762,15 @@ static NV_STATUS uvm_cpu_insert_page(struct vm_area_struct *vma,
 }

 static uvm_prot_t compute_logical_prot(uvm_va_block_t *va_block,
-                                       uvm_va_block_context_t *va_block_context,
+                                       struct vm_area_struct *hmm_vma,
                                       uvm_page_index_t page_index)
 {
-    struct vm_area_struct *vma;
    uvm_prot_t logical_prot;

    if (uvm_va_block_is_hmm(va_block)) {
        NvU64 addr = uvm_va_block_cpu_page_address(va_block, page_index);

-        logical_prot = uvm_hmm_compute_logical_prot(va_block, va_block_context, addr);
+        logical_prot = uvm_hmm_compute_logical_prot(va_block, hmm_vma, addr);
    }
    else {
        uvm_va_range_t *va_range = va_block->va_range;
@@ -6822,6 +6782,8 @@ static uvm_prot_t compute_logical_prot(uvm_va_block_t *va_block,
            logical_prot = UVM_PROT_NONE;
        }
        else {
+            struct vm_area_struct *vma;
+
            vma = uvm_va_range_vma(va_range);

            if (!(vma->vm_flags & VM_READ))
@@ -6871,13 +6833,15 @@ static struct page *block_page_get(uvm_va_block_t *block, block_phys_page_t bloc
 //    with new_prot permissions
 //  - Guarantee that vm_insert_page is safe to use (vma->vm_mm has a reference
 //    and mmap_lock is held in at least read mode)
+//  - For HMM blocks that vma is valid and safe to use, vma->vm_mm has a
+//    reference and mmap_lock is held in at least read mode
 //  - Ensure that the struct page corresponding to the physical memory being
 //    mapped exists
 //  - Manage the block's residency bitmap
 //  - Ensure that the block hasn't been killed (block->va_range is present)
 //  - Update the pte/mapping tracking state on success
 static NV_STATUS block_map_cpu_page_to(uvm_va_block_t *block,
-                                       uvm_va_block_context_t *va_block_context,
+                                       struct vm_area_struct *hmm_vma,
                                       uvm_processor_id_t resident_id,
                                       uvm_page_index_t page_index,
                                       uvm_prot_t new_prot)
@@ -6890,7 +6854,7 @@ static NV_STATUS block_map_cpu_page_to(uvm_va_block_t *block,
    NvU64 addr;
    struct page *page;

-    UVM_ASSERT(uvm_va_block_is_hmm(block) || va_range->type == UVM_VA_RANGE_TYPE_MANAGED);
+    UVM_ASSERT((uvm_va_block_is_hmm(block) && hmm_vma) || va_range->type == UVM_VA_RANGE_TYPE_MANAGED);
    UVM_ASSERT(new_prot != UVM_PROT_NONE);
    UVM_ASSERT(new_prot < UVM_PROT_MAX);
    UVM_ASSERT(uvm_processor_mask_test(&va_space->accessible_from[uvm_id_value(resident_id)], UVM_ID_CPU));
@@ -6911,7 +6875,7 @@ static NV_STATUS block_map_cpu_page_to(uvm_va_block_t *block,

    // Check for existing VMA permissions. They could have been modified after
    // the initial mmap by mprotect.
-    if (new_prot > compute_logical_prot(block, va_block_context, page_index))
+    if (new_prot > compute_logical_prot(block, hmm_vma, page_index))
        return NV_ERR_INVALID_ACCESS_TYPE;

    if (uvm_va_block_is_hmm(block)) {
@@ -7008,7 +6972,7 @@ static NV_STATUS block_map_cpu_to(uvm_va_block_t *block,

    for_each_va_block_page_in_region_mask(page_index, pages_to_map, region) {
        status = block_map_cpu_page_to(block,
-                                       block_context,
+                                       block_context->hmm.vma,
                                       resident_id,
                                       page_index,
                                       new_prot);
@@ -7189,6 +7153,7 @@ static NV_STATUS block_map_gpu_to(uvm_va_block_t *va_block,
 }

 static void map_get_allowed_destinations(uvm_va_block_t *block,
+                                         uvm_va_block_context_t *va_block_context,
                                         const uvm_va_policy_t *policy,
                                         uvm_processor_id_t id,
                                         uvm_processor_mask_t *allowed_mask)
@@ -7200,7 +7165,10 @@ static void map_get_allowed_destinations(uvm_va_block_t *block,
        uvm_processor_mask_zero(allowed_mask);
        uvm_processor_mask_set(allowed_mask, policy->preferred_location);
    }
-    else if ((uvm_va_policy_is_read_duplicate(policy, va_space) || uvm_id_equal(policy->preferred_location, id)) &&
+    else if ((uvm_va_policy_is_read_duplicate(policy, va_space) ||
+              (uvm_id_equal(policy->preferred_location, id) &&
+               !is_uvm_fault_force_sysmem_set() &&
+               !uvm_hmm_must_use_sysmem(block, va_block_context))) &&
             uvm_va_space_processor_has_memory(va_space, id)) {
        // When operating under read-duplication we should only map the local
        // processor to cause fault-and-duplicate of remote pages.
@@ -7237,13 +7205,13 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
    const uvm_page_mask_t *pte_mask;
    uvm_page_mask_t *running_page_mask = &va_block_context->mapping.map_running_page_mask;
    NV_STATUS status;
+    const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, region);

    va_block_context->mapping.cause = cause;

    UVM_ASSERT(new_prot != UVM_PROT_NONE);
    UVM_ASSERT(new_prot < UVM_PROT_MAX);
    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, va_block_context->policy, region));

    // Mapping is not supported on the eviction path that doesn't hold the VA
    // space lock.
@@ -7285,7 +7253,7 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,

    // Map per resident location so we can more easily detect physically-
    // contiguous mappings.
-    map_get_allowed_destinations(va_block, va_block_context->policy, id, &allowed_destinations);
+    map_get_allowed_destinations(va_block, va_block_context, policy, id, &allowed_destinations);

    for_each_closest_id(resident_id, &allowed_destinations, id, va_space) {
        if (UVM_ID_IS_CPU(id)) {
@@ -7591,8 +7559,6 @@ NV_STATUS uvm_va_block_map_mask(uvm_va_block_t *va_block,
    NV_STATUS tracker_status;
    uvm_processor_id_t id;

-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, va_block_context->policy, region));
-
    for_each_id_in_mask(id, map_processor_mask) {
        status = uvm_va_block_map(va_block,
                                  va_block_context,
@@ -9576,7 +9542,7 @@ static bool block_region_might_read_duplicate(uvm_va_block_t *va_block,
 //       could be changed in the future to optimize multiple faults/counters on
 //       contiguous pages.
 static uvm_prot_t compute_new_permission(uvm_va_block_t *va_block,
-                                         uvm_va_block_context_t *va_block_context,
+                                         struct vm_area_struct *hmm_vma,
                                         uvm_page_index_t page_index,
                                         uvm_processor_id_t fault_processor_id,
                                         uvm_processor_id_t new_residency,
@@ -9589,7 +9555,7 @@ static uvm_prot_t compute_new_permission(uvm_va_block_t *va_block,
    //       query_promote: upgrade access privileges to avoid future faults IF
    //       they don't trigger further revocations.
    new_prot = uvm_fault_access_type_to_prot(access_type);
-    logical_prot = compute_logical_prot(va_block, va_block_context, page_index);
+    logical_prot = compute_logical_prot(va_block, hmm_vma, page_index);

    UVM_ASSERT(logical_prot >= new_prot);

@@ -9614,15 +9580,9 @@ static uvm_prot_t compute_new_permission(uvm_va_block_t *va_block,
        if (uvm_processor_mask_empty(&revoke_processors))
            new_prot = UVM_PROT_READ_WRITE;
    }
-    if (logical_prot == UVM_PROT_READ_WRITE_ATOMIC) {
-        // HMM allocations with logical read/write/atomic permission can be
-        // upgraded without notifying the driver so assume read/write/atomic
-        // even if the fault is only for reading.
-        if (new_prot == UVM_PROT_READ_WRITE ||
-            (UVM_ID_IS_CPU(fault_processor_id) && uvm_va_block_is_hmm(va_block))) {
-            if (uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(new_residency)], fault_processor_id))
-                new_prot = UVM_PROT_READ_WRITE_ATOMIC;
-        }
+    if (logical_prot == UVM_PROT_READ_WRITE_ATOMIC && new_prot == UVM_PROT_READ_WRITE) {
+        if (uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(new_residency)], fault_processor_id))
+            new_prot = UVM_PROT_READ_WRITE_ATOMIC;
    }

    return new_prot;
@@ -9738,11 +9698,10 @@ NV_STATUS uvm_va_block_add_mappings_after_migration(uvm_va_block_t *va_block,
    uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
    const uvm_page_mask_t *final_page_mask = map_page_mask;
    uvm_tracker_t local_tracker = UVM_TRACKER_INIT();
-    const uvm_va_policy_t *policy = va_block_context->policy;
+    const uvm_va_policy_t *policy = uvm_va_policy_get_region(va_block, region);
    uvm_processor_id_t preferred_location;

    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, policy, region));

    // Read duplication takes precedence over SetAccessedBy.
    //
@@ -9859,8 +9818,6 @@ out:
    return status == NV_OK ? tracker_status : status;
 }

-// TODO: Bug 1750144: check logical permissions from HMM to know what's the
-//       maximum allowed.
 uvm_prot_t uvm_va_block_page_compute_highest_permission(uvm_va_block_t *va_block,
                                                        uvm_processor_id_t processor_id,
                                                        uvm_page_index_t page_index)
@@ -9937,14 +9894,18 @@ uvm_prot_t uvm_va_block_page_compute_highest_permission(uvm_va_block_t *va_block
        // Exclude the processor for which the mapping protections are being computed
        uvm_processor_mask_clear(&write_mappings, processor_id);

-        // At this point, any processor with atomic mappings either has native atomics support to the
-        // processor with the resident copy or has disabled system-wide atomics. If the requesting
-        // processor has disabled system-wide atomics or has native atomics to that processor, we can
-        // map with ATOMIC privileges. Likewise, if there are no other processors with WRITE or ATOMIC
-        // mappings, we can map with ATOMIC privileges.
+        // At this point, any processor with atomic mappings either has native
+        // atomics support to the processor with the resident copy or has
+        // disabled system-wide atomics. If the requesting processor has
+        // disabled system-wide atomics or has native atomics to that processor,
+        // we can map with ATOMIC privileges. Likewise, if there are no other
+        // processors with WRITE or ATOMIC mappings, we can map with ATOMIC
+        // privileges. For HMM, don't allow GPU atomic access to remote mapped
+        // system memory even if there are no write mappings since CPU access
+        // can be upgraded without notification.
        if (!uvm_processor_mask_test(&va_space->system_wide_atomics_enabled_processors, processor_id) ||
            uvm_processor_mask_test(&va_space->has_native_atomics[uvm_id_value(residency)], processor_id) ||
-            uvm_processor_mask_empty(&write_mappings)) {
+            (uvm_processor_mask_empty(&write_mappings) && !uvm_va_block_is_hmm(va_block))) {
            return UVM_PROT_READ_WRITE_ATOMIC;
        }

@@ -9966,8 +9927,6 @@ NV_STATUS uvm_va_block_add_mappings(uvm_va_block_t *va_block,
    uvm_range_group_range_iter_t iter;
    uvm_prot_t prot_to_map;

-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block, va_block_context->policy, region));
-
    if (UVM_ID_IS_CPU(processor_id) && !uvm_va_block_is_hmm(va_block)) {
        if (!uvm_va_range_vma_check(va_range, va_block_context->mm))
            return NV_OK;
@@ -10214,11 +10173,8 @@ uvm_processor_id_t uvm_va_block_select_residency(uvm_va_block_t *va_block,
 {
    uvm_processor_id_t id;

-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block,
-                                                  va_block_context->policy,
-                                                  uvm_va_block_region_for_page(page_index)));
    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block,
-                                                  va_block_context,
+                                                  va_block_context->hmm.vma,
                                                  uvm_va_block_region_for_page(page_index)));

    id = block_select_residency(va_block,
@@ -10262,6 +10218,7 @@ static bool check_access_counters_dont_revoke(uvm_va_block_t *block,
 // Update service_context->prefetch_hint, service_context->per_processor_masks,
 // and service_context->region.
 static void uvm_va_block_get_prefetch_hint(uvm_va_block_t *va_block,
+                                           const uvm_va_policy_t *policy,
                                           uvm_service_block_context_t *service_context)
 {
    uvm_processor_id_t new_residency;
@@ -10272,20 +10229,19 @@ static void uvm_va_block_get_prefetch_hint(uvm_va_block_t *va_block,
    if (uvm_processor_mask_get_count(&service_context->resident_processors) == 1) {
        uvm_page_index_t page_index;
        uvm_page_mask_t *new_residency_mask;
-        const uvm_va_policy_t *policy = service_context->block_context.policy;

        new_residency = uvm_processor_mask_find_first_id(&service_context->resident_processors);
        new_residency_mask = &service_context->per_processor_masks[uvm_id_value(new_residency)].new_residency;

        // Update prefetch tracking structure with the pages that will migrate
        // due to faults
-        uvm_perf_prefetch_get_hint(va_block,
-                                   &service_context->block_context,
-                                   new_residency,
-                                   new_residency_mask,
-                                   service_context->region,
-                                   &service_context->prefetch_bitmap_tree,
-                                   &service_context->prefetch_hint);
+        uvm_perf_prefetch_get_hint_va_block(va_block,
+                                            &service_context->block_context,
+                                            new_residency,
+                                            new_residency_mask,
+                                            service_context->region,
+                                            &service_context->prefetch_bitmap_tree,
+                                            &service_context->prefetch_hint);

        // Obtain the prefetch hint and give a fake fault access type to the
        // prefetched pages
@@ -10470,7 +10426,7 @@ NV_STATUS uvm_va_block_service_finish(uvm_processor_id_t processor_id,

    for_each_va_block_page_in_region_mask(page_index, new_residency_mask, service_context->region) {
        new_prot = compute_new_permission(va_block,
-                                          &service_context->block_context,
+                                          service_context->block_context.hmm.vma,
                                          page_index,
                                          processor_id,
                                          new_residency,
@@ -10713,11 +10669,8 @@ NV_STATUS uvm_va_block_service_locked(uvm_processor_id_t processor_id,
    NV_STATUS status = NV_OK;

    uvm_assert_mutex_locked(&va_block->lock);
-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block,
-                                                  service_context->block_context.policy,
-                                                  service_context->region));
    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block,
-                                                  &service_context->block_context,
+                                                  service_context->block_context.hmm.vma,
                                                  service_context->region));

    // GPU fault servicing must be done under the VA space read lock. GPU fault
@@ -10731,7 +10684,9 @@ NV_STATUS uvm_va_block_service_locked(uvm_processor_id_t processor_id,
    else
        uvm_assert_rwsem_locked_read(&va_space->lock);

-    uvm_va_block_get_prefetch_hint(va_block, service_context);
+    uvm_va_block_get_prefetch_hint(va_block,
+                                   uvm_va_policy_get_region(va_block, service_context->region),
+                                   service_context);

    for_each_id_in_mask(new_residency, &service_context->resident_processors) {
        if (uvm_va_block_is_hmm(va_block)) {
@@ -10764,11 +10719,8 @@ NV_STATUS uvm_va_block_check_logical_permissions(uvm_va_block_t *va_block,
    uvm_va_range_t *va_range = va_block->va_range;
    uvm_prot_t access_prot = uvm_fault_access_type_to_prot(access_type);

-    UVM_ASSERT(uvm_va_block_check_policy_is_valid(va_block,
-                                                  va_block_context->policy,
-                                                  uvm_va_block_region_for_page(page_index)));
    UVM_ASSERT(uvm_hmm_check_context_vma_is_valid(va_block,
-                                                  va_block_context,
+                                                  va_block_context->hmm.vma,
                                                  uvm_va_block_region_for_page(page_index)));

    // CPU permissions are checked later by block_map_cpu_page.
@@ -10786,8 +10738,8 @@ NV_STATUS uvm_va_block_check_logical_permissions(uvm_va_block_t *va_block,
        // vm_flags at any moment (for example on mprotect) and here we are not
        // guaranteed to have vma->vm_mm->mmap_lock. During tests we ensure that
        // this scenario does not happen.
-        if ((va_block_context->mm || uvm_enable_builtin_tests) &&
-            (access_prot > compute_logical_prot(va_block, va_block_context, page_index)))
+        if (((va_block->hmm.va_space && va_block->hmm.va_space->va_space_mm.mm) || uvm_enable_builtin_tests) &&
+            (access_prot > compute_logical_prot(va_block, va_block_context->hmm.vma, page_index)))
            return NV_ERR_INVALID_ACCESS_TYPE;
    }

@@ -10873,6 +10825,7 @@ static NV_STATUS block_cpu_fault_locked(uvm_va_block_t *va_block,
    uvm_perf_thrashing_hint_t thrashing_hint;
    uvm_processor_id_t new_residency;
    bool read_duplicate;
+    const uvm_va_policy_t *policy;

    uvm_assert_rwsem_locked(&va_space->lock);

@@ -10881,13 +10834,13 @@ static NV_STATUS block_cpu_fault_locked(uvm_va_block_t *va_block,

    uvm_assert_mmap_lock_locked(service_context->block_context.mm);

-    service_context->block_context.policy = uvm_va_policy_get(va_block, fault_addr);
+    policy = uvm_va_policy_get(va_block, fault_addr);

    if (service_context->num_retries == 0) {
        // notify event to tools/performance heuristics
        uvm_perf_event_notify_cpu_fault(&va_space->perf_events,
                                        va_block,
-                                        service_context->block_context.policy->preferred_location,
+                                        policy->preferred_location,
                                        fault_addr,
                                        fault_access_type > UVM_FAULT_ACCESS_TYPE_READ,
                                        KSTK_EIP(current));
@@ -10932,7 +10885,7 @@ static NV_STATUS block_cpu_fault_locked(uvm_va_block_t *va_block,
                                                  page_index,
                                                  UVM_ID_CPU,
                                                  uvm_fault_access_type_mask_bit(fault_access_type),
-                                                  service_context->block_context.policy,
+                                                  policy,
                                                  &thrashing_hint,
                                                  UVM_SERVICE_OPERATION_REPLAYABLE_FAULTS,
                                                  &read_duplicate);
@@ -11032,7 +10985,6 @@ NV_STATUS uvm_va_block_find(uvm_va_space_t *va_space, NvU64 addr, uvm_va_block_t
 NV_STATUS uvm_va_block_find_create_in_range(uvm_va_space_t *va_space,
                                            uvm_va_range_t *va_range,
                                            NvU64 addr,
-                                            uvm_va_block_context_t *va_block_context,
                                            uvm_va_block_t **out_block)
 {
    size_t index;
@@ -11040,12 +10992,7 @@ NV_STATUS uvm_va_block_find_create_in_range(uvm_va_space_t *va_space,
    if (uvm_enable_builtin_tests && atomic_dec_if_positive(&va_space->test.va_block_allocation_fail_nth) == 0)
        return NV_ERR_NO_MEMORY;

-    if (!va_range) {
-        if (!va_block_context || !va_block_context->mm)
-            return NV_ERR_INVALID_ADDRESS;
-        return uvm_hmm_va_block_find_create(va_space, addr, va_block_context, out_block);
-    }
-
+    UVM_ASSERT(va_range);
    UVM_ASSERT(addr >= va_range->node.start);
    UVM_ASSERT(addr <= va_range->node.end);

@@ -11059,14 +11006,32 @@ NV_STATUS uvm_va_block_find_create_in_range(uvm_va_space_t *va_space,
    return uvm_va_range_block_create(va_range, index, out_block);
 }

-NV_STATUS uvm_va_block_find_create(uvm_va_space_t *va_space,
+NV_STATUS uvm_va_block_find_create_managed(uvm_va_space_t *va_space,
                                   NvU64 addr,
-                                   uvm_va_block_context_t *va_block_context,
                                   uvm_va_block_t **out_block)
 {
    uvm_va_range_t *va_range = uvm_va_range_find(va_space, addr);

-    return uvm_va_block_find_create_in_range(va_space, va_range, addr, va_block_context, out_block);
+    if (va_range)
+        return uvm_va_block_find_create_in_range(va_space, va_range, addr, out_block);
+    else
+        return NV_ERR_INVALID_ADDRESS;
+}
+
+NV_STATUS uvm_va_block_find_create(uvm_va_space_t *va_space,
+                                   NvU64 addr,
+                                   struct vm_area_struct **hmm_vma,
+                                   uvm_va_block_t **out_block)
+{
+    uvm_va_range_t *va_range = uvm_va_range_find(va_space, addr);
+
+    if (hmm_vma)
+        *hmm_vma = NULL;
+
+    if (va_range)
+        return uvm_va_block_find_create_in_range(va_space, va_range, addr, out_block);
+    else
+        return uvm_hmm_va_block_find_create(va_space, addr, hmm_vma, out_block);
 }

 // Launch a synchronous, encrypted copy between GPU and CPU.
@@ -11243,8 +11208,6 @@ NV_STATUS uvm_va_block_write_from_cpu(uvm_va_block_t *va_block,
    if (UVM_ID_IS_INVALID(proc))
        proc = UVM_ID_CPU;

-    block_context->policy = uvm_va_policy_get(va_block, dst);
-
    // Use make_resident() in all cases to break read-duplication, but
    // block_retry can be NULL as if the page is not resident yet we will make
    // it resident on the CPU.
@@ -11413,7 +11376,6 @@ static void block_add_eviction_mappings(void *args)
        uvm_va_range_t *va_range = va_block->va_range;
        NV_STATUS status = NV_OK;

-        block_context->policy = uvm_va_range_get_policy(va_range);
        for_each_id_in_mask(id, &uvm_va_range_get_policy(va_range)->accessed_by) {
            status = uvm_va_block_set_accessed_by(va_block, block_context, id);
            if (status != NV_OK)
@@ -11564,8 +11526,8 @@ NV_STATUS uvm_va_block_evict_chunks(uvm_va_block_t *va_block,
                                               &accessed_by_set);
    }
    else {
-        block_context->policy = uvm_va_range_get_policy(va_block->va_range);
-        accessed_by_set = uvm_processor_mask_get_count(&block_context->policy->accessed_by) > 0;
+        const uvm_va_policy_t *policy = uvm_va_range_get_policy(va_block->va_range);
+        accessed_by_set = uvm_processor_mask_get_count(&policy->accessed_by) > 0;

        // TODO: Bug 1765193: make_resident() breaks read-duplication, but it's
        // not necessary to do so for eviction. Add a version that unmaps only
@@ -11756,19 +11718,16 @@ NV_STATUS uvm_test_va_block_inject_error(UVM_TEST_VA_BLOCK_INJECT_ERROR_PARAMS *
    struct mm_struct *mm;
    uvm_va_block_t *va_block;
    uvm_va_block_test_t *va_block_test;
-    uvm_va_block_context_t *block_context = NULL;
    NV_STATUS status = NV_OK;

    mm = uvm_va_space_mm_or_current_retain_lock(va_space);
    uvm_va_space_down_read(va_space);

-    block_context = uvm_va_block_context_alloc(mm);
-    if (!block_context) {
-        status = NV_ERR_NO_MEMORY;
-        goto out;
-    }
+    if (mm)
+        status = uvm_va_block_find_create(va_space, params->lookup_address, NULL, &va_block);
+    else
+        status = uvm_va_block_find_create_managed(va_space, params->lookup_address, &va_block);

-    status = uvm_va_block_find_create(va_space, params->lookup_address, block_context, &va_block);
    if (status != NV_OK)
        goto out;

@@ -11808,7 +11767,6 @@ block_unlock:
 out:
    uvm_va_space_up_read(va_space);
    uvm_va_space_mm_or_current_release_unlock(va_space, mm);
-    uvm_va_block_context_free(block_context);
    return status;
 }

@@ -11879,7 +11837,11 @@ NV_STATUS uvm_test_change_pte_mapping(UVM_TEST_CHANGE_PTE_MAPPING_PARAMS *params
        goto out;
    }

-    status = uvm_va_block_find_create(va_space, params->va, block_context, &block);
+    if (mm)
+        status = uvm_va_block_find_create(va_space, params->va, &block_context->hmm.vma, &block);
+    else
+        status = uvm_va_block_find_create_managed(va_space, params->va, &block);
+
    if (status != NV_OK)
        goto out;

@@ -11906,8 +11868,6 @@ NV_STATUS uvm_test_change_pte_mapping(UVM_TEST_CHANGE_PTE_MAPPING_PARAMS *params
        goto out_block;
    }

-    block_context->policy = uvm_va_policy_get(block, params->va);
-
    if (new_prot == UVM_PROT_NONE) {
        status = uvm_va_block_unmap(block, block_context, id, region, NULL, &block->tracker);
    }
--- a/kernel-open/nvidia-uvm/uvm_va_block.h
+++ b/kernel-open/nvidia-uvm/uvm_va_block.h
@@ -453,11 +453,12 @@ struct uvm_va_block_struct
        NvU16 fault_migrations_to_last_proc;
    } prefetch_info;

-#if UVM_IS_CONFIG_HMM()
    struct
    {
+#if UVM_IS_CONFIG_HMM()
        // The MMU notifier is registered per va_block.
        struct mmu_interval_notifier notifier;
+#endif

        // This is used to serialize migrations between CPU and GPU while
        // allowing the va_block lock to be dropped.
@@ -487,7 +488,6 @@ struct uvm_va_block_struct
        // Storage node for range tree of va_blocks.
        uvm_range_tree_node_t node;
    } hmm;
-#endif
 };

 // We define additional per-VA Block fields for testing. When
@@ -678,18 +678,8 @@ static void uvm_va_block_context_init(uvm_va_block_context_t *va_block_context,
        memset(va_block_context, 0xff, sizeof(*va_block_context));

    va_block_context->mm = mm;
-#if UVM_IS_CONFIG_HMM()
-    va_block_context->hmm.vma = NULL;
-#endif
 }

-// Check that a single policy covers the given region for the given va_block.
-// This always returns true and is intended to only be used with UVM_ASSERT().
-// Locking: the va_block lock must be held.
-bool uvm_va_block_check_policy_is_valid(uvm_va_block_t *va_block,
-                                        const uvm_va_policy_t *policy,
-                                        uvm_va_block_region_t region);
-
 // TODO: Bug 1766480: Using only page masks instead of a combination of regions
 //       and page masks could simplify the below APIs and their implementations
 //       at the cost of having to scan the whole mask for small regions.
@@ -734,15 +724,15 @@ bool uvm_va_block_check_policy_is_valid(uvm_va_block_t *va_block,
 // user memory is guaranteed not to happen. Allocation-retry of GPU page tables
 // can still occur though.
 //
-// va_block_context must not be NULL. This function will set a bit in
-// va_block_context->make_resident.pages_changed_residency for each page that
-// changed residency (due to a migration or first population) as a result of the
-// operation and va_block_context->make_resident.all_involved_processors for
-// each processor involved in the copy. This function only sets bits in those
-// masks. It is the caller's responsiblity to zero the masks or not first.
-//
-// va_block_context->policy must also be set by the caller for the given region.
-// See the comments for uvm_va_block_check_policy_is_valid().
+// va_block_context must not be NULL and policy for the region must
+// match. This function will set a bit in
+// va_block_context->make_resident.pages_changed_residency for each
+// page that changed residency (due to a migration or first
+// population) as a result of the operation and
+// va_block_context->make_resident.all_involved_processors for each
+// processor involved in the copy. This function only sets bits in
+// those masks. It is the caller's responsiblity to zero the masks or
+// not first.
 //
 // Notably any status other than NV_OK indicates that the block's lock might
 // have been unlocked and relocked.
@@ -839,7 +829,7 @@ void uvm_va_block_make_resident_finish(uvm_va_block_t *va_block,
 // pages because the earlier operation can cause a PTE split or merge which is
 // assumed by the later operation.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // If allocation-retry was required as part of the operation and was successful,
@@ -896,7 +886,7 @@ NV_STATUS uvm_va_block_map_mask(uvm_va_block_t *va_block,
 // pages because the earlier operation can cause a PTE split or merge which is
 // assumed by the later operation.
 //
-// va_block_context must not be NULL. The va_block_context->policy is unused.
+// va_block_context must not be NULL.
 //
 // If allocation-retry was required as part of the operation and was successful,
 // NV_ERR_MORE_PROCESSING_REQUIRED is returned. In this case, the entries in the
@@ -929,7 +919,7 @@ NV_STATUS uvm_va_block_unmap_mask(uvm_va_block_t *va_block,
 // - Unmap the preferred location's processor from any pages in this region
 //   which are not resident on the preferred location.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: The caller must hold the VA block lock.
@@ -941,7 +931,7 @@ NV_STATUS uvm_va_block_set_preferred_location_locked(uvm_va_block_t *va_block,
 // location and policy. Waits for the operation to complete before returning.
 // This function should only be called with managed va_blocks.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: This takes and releases the VA block lock. If va_block_context->mm
@@ -956,7 +946,7 @@ NV_STATUS uvm_va_block_set_accessed_by(uvm_va_block_t *va_block,
 // the tracker after all mappings have been started.
 // This function can be called with HMM and managed va_blocks.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: The caller must hold the va_block lock and
@@ -970,7 +960,7 @@ NV_STATUS uvm_va_block_set_accessed_by_locked(uvm_va_block_t *va_block,
 // Breaks SetAccessedBy and remote mappings
 // This function should only be called with managed va_blocks.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: This takes and releases the VA block lock. If va_block_context->mm
@@ -982,7 +972,7 @@ NV_STATUS uvm_va_block_set_read_duplication(uvm_va_block_t *va_block,
 // Restores SetAccessedBy mappings
 // This function should only be called with managed va_blocks.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: This takes and releases the VA block lock. If va_block_context->mm
@@ -1002,10 +992,9 @@ NV_STATUS uvm_va_block_unset_read_duplication(uvm_va_block_t *va_block,
 // NV_ERR_INVALID_OPERATION     The access would violate the policies specified
 //                              by UvmPreventMigrationRangeGroups.
 //
-// va_block_context must not be NULL, va_block_context->policy must be valid,
-// and if the va_block is a HMM block, va_block_context->hmm.vma must be valid
-// which also means the va_block_context->mm is not NULL, retained, and locked
-// for at least read.
+// va_block_context must not be NULL, policy must match, and if the va_block is
+// a HMM block, va_block_context->hmm.vma must be valid which also means the
+// va_block_context->mm is not NULL, retained, and locked for at least read.
 // Locking: the va_block lock must be held.
 NV_STATUS uvm_va_block_check_logical_permissions(uvm_va_block_t *va_block,
                                                 uvm_va_block_context_t *va_block_context,
@@ -1041,7 +1030,7 @@ NV_STATUS uvm_va_block_check_logical_permissions(uvm_va_block_t *va_block,
 // different pages because the earlier operation can cause a PTE split or merge
 // which is assumed by the later operation.
 //
-// va_block_context must not be NULL. The va_block_context->policy is unused.
+// va_block_context must not be NULL.
 //
 // If allocation-retry was required as part of the operation and was successful,
 // NV_ERR_MORE_PROCESSING_REQUIRED is returned. In this case, the entries in the
@@ -1081,7 +1070,7 @@ NV_STATUS uvm_va_block_revoke_prot_mask(uvm_va_block_t *va_block,
 // processor_id, which triggered the migration and should have already been
 // mapped).
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // This function acquires/waits for the va_block tracker and updates that
@@ -1112,7 +1101,7 @@ NV_STATUS uvm_va_block_add_mappings_after_migration(uvm_va_block_t *va_block,
 // Note that this can return NV_ERR_MORE_PROCESSING_REQUIRED just like
 // uvm_va_block_map() indicating that the operation needs to be retried.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
+// va_block_context must not be NULL and policy must for the region must match.
 // See the comments for uvm_va_block_check_policy_is_valid().
 //
 // LOCKING: The caller must hold the va block lock. If va_block_context->mm !=
@@ -1134,7 +1123,7 @@ NV_STATUS uvm_va_block_add_gpu_va_space(uvm_va_block_t *va_block, uvm_gpu_va_spa
 // If mm != NULL, that mm is used for any CPU mappings which may be created as
 // a result of this call. See uvm_va_block_context_t::mm for details.
 //
-// va_block_context must not be NULL. The va_block_context->policy is unused.
+// va_block_context must not be NULL.
 //
 // LOCKING: The caller must hold the va_block lock. If block_context->mm is not
 // NULL, the caller must hold mm->mmap_lock in at least read mode.
@@ -1225,7 +1214,6 @@ NV_STATUS uvm_va_block_split_locked(uvm_va_block_t *existing_va_block,
 //  - va_space lock must be held in at least read mode
 //
 // service_context->block_context.mm is ignored and vma->vm_mm is used instead.
-// service_context->block_context.policy is set by this function.
 //
 // Returns NV_ERR_INVALID_ACCESS_TYPE if a CPU mapping to fault_addr cannot be
 // accessed, for example because it's within a range group which is non-
@@ -1239,10 +1227,10 @@ NV_STATUS uvm_va_block_cpu_fault(uvm_va_block_t *va_block,
 // (migrations, cache invalidates, etc.) in response to the given service block
 // context.
 //
-// service_context must not be NULL and service_context->block_context.policy
-// must be valid. See the comments for uvm_va_block_check_policy_is_valid().
-// If va_block is a HMM block, va_block_context->hmm.vma must be valid.
-// See the comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
+// service_context must not be NULL and policy for service_context->region must
+// match. See the comments for uvm_va_block_check_policy_is_valid().  If
+// va_block is a HMM block, va_block_context->hmm.vma must be valid.  See the
+// comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
 // service_context->prefetch_hint is set by this function.
 //
 // Locking:
@@ -1267,10 +1255,10 @@ NV_STATUS uvm_va_block_service_locked(uvm_processor_id_t processor_id,
 // Performs population of the destination pages, unmapping and copying source
 // pages to new_residency.
 //
-// service_context must not be NULL and service_context->block_context.policy
-// must be valid. See the comments for uvm_va_block_check_policy_is_valid().
-// If va_block is a HMM block, va_block_context->hmm.vma must be valid.
-// See the comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
+// service_context must not be NULL and policy for service_context->region must
+// match.  See the comments for uvm_va_block_check_policy_is_valid().  If
+// va_block is a HMM block, va_block_context->hmm.vma must be valid.  See the
+// comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
 // service_context->prefetch_hint should be set before calling this function.
 //
 // Locking:
@@ -1296,10 +1284,10 @@ NV_STATUS uvm_va_block_service_copy(uvm_processor_id_t processor_id,
 // This updates the va_block residency state and maps the faulting processor_id
 // to the new residency (which may be remote).
 //
-// service_context must not be NULL and service_context->block_context.policy
-// must be valid. See the comments for uvm_va_block_check_policy_is_valid().
-// If va_block is a HMM block, va_block_context->hmm.vma must be valid.
-// See the comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
+// service_context must not be NULL and policy for service_context->region must
+// match. See the comments for uvm_va_block_check_policy_is_valid().  If
+// va_block is a HMM block, va_block_context->hmm.vma must be valid.  See the
+// comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
 // service_context must be initialized by calling uvm_va_block_service_copy()
 // before calling this function.
 //
@@ -1428,40 +1416,34 @@ const uvm_page_mask_t *uvm_va_block_map_mask_get(uvm_va_block_t *block, uvm_proc
 NV_STATUS uvm_va_block_find(uvm_va_space_t *va_space, NvU64 addr, uvm_va_block_t **out_block);

 // Same as uvm_va_block_find except that the block is created if not found.
-// If addr is covered by a UVM_VA_RANGE_TYPE_MANAGED va_range, a managed block
-// will be created. Otherwise, if addr is not covered by any va_range, HMM is
-// enabled in the va_space, and va_block_context and va_block_context->mm are
-// non-NULL, then a HMM block will be created and va_block_context->hmm.vma is
-// set to the VMA covering 'addr'. The va_block_context->policy field is left
-// unchanged.
-// In either case, if va_block_context->mm is non-NULL, it must be retained and
-// locked in at least read mode. Return values:
+// If addr is covered by a UVM_VA_RANGE_TYPE_MANAGED va_range a managed block
+// will be created. If addr is not covered by any va_range and HMM is
+// enabled in the va_space then a HMM block will be created and hmm_vma is
+// set to the VMA covering 'addr'. The va_space_mm must be retained and locked.
+// Otherwise hmm_vma is set to NULL.
+// Return values:
 // NV_ERR_INVALID_ADDRESS   addr is not a UVM_VA_RANGE_TYPE_MANAGED va_range nor
 //                          a HMM enabled VMA.
 // NV_ERR_NO_MEMORY         memory could not be allocated.
 NV_STATUS uvm_va_block_find_create(uvm_va_space_t *va_space,
                                   NvU64 addr,
-                                   uvm_va_block_context_t *va_block_context,
+                                   struct vm_area_struct **hmm_vma,
                                   uvm_va_block_t **out_block);

-// Same as uvm_va_block_find_create except that va_range lookup was already done
-// by the caller. If the supplied va_range is NULL, this function behaves just
-// like when the va_range lookup in uvm_va_block_find_create is NULL.
+// Same as uvm_va_block_find_create except that only managed va_blocks are
+// created if not already present in the VA range. Does not require va_space_mm
+// to be locked or retained.
+NV_STATUS uvm_va_block_find_create_managed(uvm_va_space_t *va_space,
+                                           NvU64 addr,
+                                           uvm_va_block_t **out_block);
+
+// Same as uvm_va_block_find_create_managed except that va_range lookup was
+// already done by the caller. The supplied va_range must not be NULL.
 NV_STATUS uvm_va_block_find_create_in_range(uvm_va_space_t *va_space,
                                            uvm_va_range_t *va_range,
                                            NvU64 addr,
-                                            uvm_va_block_context_t *va_block_context,
                                            uvm_va_block_t **out_block);

-// Same as uvm_va_block_find_create except that only managed va_blocks are
-// created if not already present in the VA range.
-static NV_STATUS uvm_va_block_find_create_managed(uvm_va_space_t *va_space,
-                                                  NvU64 addr,
-                                                  uvm_va_block_t **out_block)
-{
-    return uvm_va_block_find_create(va_space, addr, NULL, out_block);
-}
-
 // Look up a chunk backing a specific address within the VA block.
 // Returns NULL if none.
 uvm_gpu_chunk_t *uvm_va_block_lookup_gpu_chunk(uvm_va_block_t *va_block, uvm_gpu_t *gpu, NvU64 address);
@@ -1476,10 +1458,10 @@ uvm_gpu_chunk_t *uvm_va_block_lookup_gpu_chunk(uvm_va_block_t *va_block, uvm_gpu
 // The caller needs to handle allocation-retry. va_block_retry can be NULL if
 // the destination is the CPU.
 //
-// va_block_context must not be NULL and va_block_context->policy must be valid.
-// See the comments for uvm_va_block_check_policy_is_valid().
-// If va_block is a HMM block, va_block_context->hmm.vma must be valid.
-// See the comments for uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
+// va_block_context must not be NULL and policy for the region must match. See
+// the comments for uvm_va_block_check_policy_is_valid().  If va_block is a HMM
+// block, va_block_context->hmm.vma must be valid.  See the comments for
+// uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
 //
 // LOCKING: The caller must hold the va_block lock. If va_block_context->mm !=
 //          NULL, va_block_context->mm->mmap_lock must be held in at least
@@ -1497,7 +1479,7 @@ NV_STATUS uvm_va_block_migrate_locked(uvm_va_block_t *va_block,
 // The [dst, dst + size) range has to fit within a single PAGE_SIZE page.
 //
 // va_block_context must not be NULL. The caller is not required to set
-// va_block_context->policy or va_block_context->hmm.vma.
+// va_block_context->hmm.vma.
 //
 // The caller needs to support allocation-retry of page tables.
 //
@@ -1569,7 +1551,7 @@ void uvm_va_block_mark_cpu_dirty(uvm_va_block_t *va_block);
 // successful, NV_ERR_MORE_PROCESSING_REQUIRED is returned. In this case the
 // block's lock was unlocked and relocked.
 //
-// va_block_context must not be NULL. The va_block_context->policy is unused.
+// va_block_context must not be NULL.
 //
 // LOCKING: The caller must hold the va_block lock.
 NV_STATUS uvm_va_block_set_cancel(uvm_va_block_t *va_block, uvm_va_block_context_t *block_context, uvm_gpu_t *gpu);
@@ -1650,12 +1632,18 @@ static uvm_va_block_region_t uvm_va_block_region_from_block(uvm_va_block_t *va_b
    return uvm_va_block_region(0, uvm_va_block_num_cpu_pages(va_block));
 }

-// Create a block region from a va block and page mask. Note that the region
+// Create a block region from a va block and page mask. If va_block is NULL, the
+// region is assumed to cover the maximum va_block size. Note that the region
 // covers the first through the last set bit and may have unset bits in between.
 static uvm_va_block_region_t uvm_va_block_region_from_mask(uvm_va_block_t *va_block, const uvm_page_mask_t *page_mask)
 {
    uvm_va_block_region_t region;
-    uvm_page_index_t outer = uvm_va_block_num_cpu_pages(va_block);
+    uvm_page_index_t outer;
+
+    if (va_block)
+        outer = uvm_va_block_num_cpu_pages(va_block);
+    else
+        outer = PAGES_PER_UVM_VA_BLOCK;

    region.first = find_first_bit(page_mask->bitmap, outer);
    if (region.first >= outer) {
@@ -2140,15 +2128,14 @@ uvm_va_block_region_t uvm_va_block_big_page_region_subset(uvm_va_block_t *va_blo
 // MAX_BIG_PAGES_PER_UVM_VA_BLOCK is returned.
 size_t uvm_va_block_big_page_index(uvm_va_block_t *va_block, uvm_page_index_t page_index, NvU32 big_page_size);

-// Returns the new residency for a page that faulted or triggered access
-// counter notifications. The read_duplicate output parameter indicates if the
-// page meets the requirements to be read-duplicated
-// va_block_context must not be NULL, va_block_context->policy must be valid,
-// and if the va_block is a HMM block, va_block_context->hmm.vma must be valid
-// which also means the va_block_context->mm is not NULL, retained, and locked
-// for at least read. See the comments for uvm_va_block_check_policy_is_valid()
-// and uvm_hmm_check_context_vma_is_valid() in uvm_hmm.h.
-// Locking: the va_block lock must be held.
+// Returns the new residency for a page that faulted or triggered access counter
+// notifications. The read_duplicate output parameter indicates if the page
+// meets the requirements to be read-duplicated va_block_context must not be
+// NULL, and if the va_block is a HMM block, va_block_context->hmm.vma must be
+// valid which also means the va_block_context->mm is not NULL, retained, and
+// locked for at least read. See the comments for
+// uvm_va_block_check_policy_is_valid() and uvm_hmm_check_context_vma_is_valid()
+// in uvm_hmm.h.  Locking: the va_block lock must be held.
 uvm_processor_id_t uvm_va_block_select_residency(uvm_va_block_t *va_block,
                                                 uvm_va_block_context_t *va_block_context,
                                                 uvm_page_index_t page_index,
--- a/kernel-open/nvidia-uvm/uvm_va_block_types.h
+++ b/kernel-open/nvidia-uvm/uvm_va_block_types.h
@@ -29,9 +29,7 @@
 #include "uvm_tlb_batch.h"
 #include "uvm_forward_decl.h"

-#if UVM_IS_CONFIG_HMM()
 #include <linux/migrate.h>
-#endif

 // UVM_VA_BLOCK_BITS is 21, meaning the maximum block size is 2MB. Rationale:
 // - 2MB matches the largest Pascal GPU page size so it's a natural fit
@@ -234,9 +232,6 @@ typedef struct
    // the mm, such as creating CPU mappings.
    struct mm_struct *mm;

-    const uvm_va_policy_t *policy;
-
-#if UVM_IS_CONFIG_HMM()
    struct
    {
        // These are used for migrate_vma_*(), hmm_range_fault(), and
@@ -257,10 +252,11 @@ typedef struct
        // Cached VMA pointer. This is only valid while holding the mmap_lock.
        struct vm_area_struct *vma;

+#if UVM_IS_CONFIG_HMM()
        // Used for migrate_vma_*() to migrate pages to/from GPU/CPU.
        struct migrate_vma migrate_vma_args;
-    } hmm;
 #endif
+    } hmm;

    // Convenience buffer for page mask prints
    char page_mask_string_buffer[UVM_PAGE_MASK_PRINT_MIN_BUFFER_SIZE];
--- a/kernel-open/nvidia-uvm/uvm_va_policy.c
+++ b/kernel-open/nvidia-uvm/uvm_va_policy.c
@@ -54,6 +54,52 @@ const uvm_va_policy_t *uvm_va_policy_get(uvm_va_block_t *va_block, NvU64 addr)
    }
 }

+// HMM va_blocks can have different polices for different regions withing the
+// va_block. This function checks the given region is covered by the same policy
+// and asserts if the region is covered by different policies.
+// This always returns true and is intended to only be used with UVM_ASSERT() to
+// avoid calling it on release builds.
+// Locking: the va_block lock must be held.
+static bool uvm_hmm_va_block_assert_policy_is_valid(uvm_va_block_t *va_block,
+                                                    const uvm_va_policy_t *policy,
+                                                    uvm_va_block_region_t region)
+{
+    const uvm_va_policy_node_t *node;
+
+    if (uvm_va_policy_is_default(policy)) {
+        // There should only be the default policy within the region.
+        node = uvm_va_policy_node_iter_first(va_block,
+                                             uvm_va_block_region_start(va_block, region),
+                                             uvm_va_block_region_end(va_block, region));
+        UVM_ASSERT(!node);
+    }
+    else {
+        // The policy node should cover the region.
+        node = uvm_va_policy_node_from_policy(policy);
+        UVM_ASSERT(node->node.start <= uvm_va_block_region_start(va_block, region));
+        UVM_ASSERT(node->node.end >= uvm_va_block_region_end(va_block, region));
+    }
+
+    return true;
+}
+
+const uvm_va_policy_t *uvm_va_policy_get_region(uvm_va_block_t *va_block, uvm_va_block_region_t region)
+{
+    uvm_assert_mutex_locked(&va_block->lock);
+
+    if (uvm_va_block_is_hmm(va_block)) {
+        const uvm_va_policy_t *policy;
+        const uvm_va_policy_node_t *node = uvm_va_policy_node_find(va_block, uvm_va_block_region_start(va_block, region));
+
+        policy = node ? &node->policy : &uvm_va_policy_default;
+        UVM_ASSERT(uvm_hmm_va_block_assert_policy_is_valid(va_block, policy, region));
+        return policy;
+    }
+    else {
+        return uvm_va_range_get_policy(va_block->va_range);
+    }
+}
+
 #if UVM_IS_CONFIG_HMM()

 static struct kmem_cache *g_uvm_va_policy_node_cache __read_mostly;
--- a/kernel-open/nvidia-uvm/uvm_va_policy.h
+++ b/kernel-open/nvidia-uvm/uvm_va_policy.h
@@ -100,6 +100,9 @@ bool uvm_va_policy_is_read_duplicate(const uvm_va_policy_t *policy, uvm_va_space
 // Locking: The va_block lock must be held.
 const uvm_va_policy_t *uvm_va_policy_get(uvm_va_block_t *va_block, NvU64 addr);

+// Same as above but asserts the policy covers the whole region
+const uvm_va_policy_t *uvm_va_policy_get_region(uvm_va_block_t *va_block, uvm_va_block_region_t region);
+
 // Return a uvm_va_policy_node_t given a uvm_va_policy_t pointer.
 static const uvm_va_policy_node_t *uvm_va_policy_node_from_policy(const uvm_va_policy_t *policy)
 {
--- a/kernel-open/nvidia-uvm/uvm_va_range.c
+++ b/kernel-open/nvidia-uvm/uvm_va_range.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2015-2022 NVIDIA Corporation
+    Copyright (c) 2015-2023 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
@@ -376,7 +376,7 @@ NV_STATUS uvm_va_range_create_semaphore_pool(uvm_va_space_t *va_space,
        if (status != NV_OK)
            goto error;

-        if (i == 0 && g_uvm_global.sev_enabled)
+        if (i == 0 && g_uvm_global.conf_computing_enabled)
            mem_alloc_params.dma_owner = gpu;

        if (attrs.is_cacheable) {
@@ -608,7 +608,6 @@ static NV_STATUS va_range_add_gpu_va_space_managed(uvm_va_range_t *va_range,
        uvm_va_block_t *va_block;
        uvm_va_block_context_t *va_block_context = uvm_va_space_block_context(va_space, mm);

-        va_block_context->policy = uvm_va_range_get_policy(va_range);

        // TODO: Bug 2090378. Consolidate all per-VA block operations within
        // uvm_va_block_add_gpu_va_space so we only need to take the VA block
@@ -687,7 +686,6 @@ static void va_range_remove_gpu_va_space_managed(uvm_va_range_t *va_range,
    bool should_enable_read_duplicate;
    uvm_va_block_context_t *va_block_context = uvm_va_space_block_context(va_space, mm);

-    va_block_context->policy = uvm_va_range_get_policy(va_range);
    should_enable_read_duplicate =
        uvm_va_range_get_policy(va_range)->read_duplication == UVM_READ_DUPLICATION_ENABLED &&
        uvm_va_space_can_read_duplicate(va_space, NULL) != uvm_va_space_can_read_duplicate(va_space, gpu_va_space->gpu);
@@ -769,7 +767,6 @@ static NV_STATUS uvm_va_range_enable_peer_managed(uvm_va_range_t *va_range, uvm_
    uvm_va_space_t *va_space = va_range->va_space;
    uvm_va_block_context_t *va_block_context = uvm_va_space_block_context(va_space, NULL);

-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, va_block) {
        // TODO: Bug 1767224: Refactor the uvm_va_block_set_accessed_by logic
@@ -1322,7 +1319,6 @@ static NV_STATUS range_unmap_mask(uvm_va_range_t *va_range,
    if (uvm_processor_mask_empty(mask))
        return NV_OK;

-    block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, block) {
        NV_STATUS status;
@@ -1364,7 +1360,6 @@ static NV_STATUS range_map_uvm_lite_gpus(uvm_va_range_t *va_range, uvm_tracker_t
    if (uvm_processor_mask_empty(&va_range->uvm_lite_gpus))
        return NV_OK;

-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, va_block) {
        // UVM-Lite GPUs always map with RWA
@@ -1528,7 +1523,6 @@ NV_STATUS uvm_va_range_set_preferred_location(uvm_va_range_t *va_range,
    uvm_processor_mask_copy(&va_range->uvm_lite_gpus, &new_uvm_lite_gpus);

    va_block_context = uvm_va_space_block_context(va_space, mm);
-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, va_block) {
        uvm_processor_id_t id;
@@ -1610,7 +1604,6 @@ NV_STATUS uvm_va_range_set_accessed_by(uvm_va_range_t *va_range,

    uvm_processor_mask_copy(&va_range->uvm_lite_gpus, &new_uvm_lite_gpus);
    va_block_context = uvm_va_space_block_context(va_space, mm);
-    va_block_context->policy = policy;

    for_each_va_block_in_va_range(va_range, va_block) {
        status = uvm_va_block_set_accessed_by(va_block, va_block_context, processor_id);
@@ -1657,7 +1650,6 @@ NV_STATUS uvm_va_range_set_read_duplication(uvm_va_range_t *va_range, struct mm_
        return NV_OK;

    va_block_context = uvm_va_space_block_context(va_range->va_space, mm);
-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, va_block) {
        NV_STATUS status = uvm_va_block_set_read_duplication(va_block, va_block_context);
@@ -1679,7 +1671,6 @@ NV_STATUS uvm_va_range_unset_read_duplication(uvm_va_range_t *va_range, struct m
        return NV_OK;

    va_block_context = uvm_va_space_block_context(va_range->va_space, mm);
-    va_block_context->policy = uvm_va_range_get_policy(va_range);

    for_each_va_block_in_va_range(va_range, va_block) {
        status = uvm_va_block_unset_read_duplication(va_block, va_block_context);
@@ -1816,7 +1807,7 @@ NV_STATUS uvm_api_alloc_semaphore_pool(UVM_ALLOC_SEMAPHORE_POOL_PARAMS *params,
    if (params->gpuAttributesCount > UVM_MAX_GPUS)
        return NV_ERR_INVALID_ARGUMENT;

-    if (g_uvm_global.sev_enabled && params->gpuAttributesCount == 0)
+    if (g_uvm_global.conf_computing_enabled && params->gpuAttributesCount == 0)
        return NV_ERR_INVALID_ARGUMENT;

    // The mm needs to be locked in order to remove stale HMM va_blocks.
--- a/kernel-open/nvidia-uvm/uvm_va_space.c
+++ b/kernel-open/nvidia-uvm/uvm_va_space.c
@@ -242,9 +242,7 @@ NV_STATUS uvm_va_space_create(struct address_space *mapping, uvm_va_space_t **va
    if (status != NV_OK)
        goto fail;

-    status = uvm_hmm_va_space_initialize(va_space);
-    if (status != NV_OK)
-        goto fail;
+    uvm_hmm_va_space_initialize(va_space);

    uvm_va_space_up_write(va_space);
    uvm_up_write_mmap_lock(current->mm);
@@ -418,15 +416,6 @@ void uvm_va_space_destroy(uvm_va_space_t *va_space)
    uvm_global_processor_mask_t retained_gpus;
    LIST_HEAD(deferred_free_list);

-    // Normally we'd expect this to happen as part of uvm_mm_release()
-    // but if userspace never initialized uvm_mm_fd that won't happen.
-    // We don't have to take the va_space_mm spinlock and update state
-    // here because we know no other thread can be in or subsequently
-    // call uvm_api_mm_initialize successfully because the UVM
-    // file-descriptor has been released.
-    if (va_space->va_space_mm.state == UVM_VA_SPACE_MM_STATE_UNINITIALIZED)
-        uvm_va_space_mm_unregister(va_space);
-
    // Remove the VA space from the global list before we start tearing things
    // down so other threads can't see the VA space in a partially-valid state.
    uvm_mutex_lock(&g_uvm_global.va_spaces.lock);
@@ -532,7 +521,14 @@ void uvm_va_space_destroy(uvm_va_space_t *va_space)

    uvm_deferred_free_object_list(&deferred_free_list);

-    // MM FD teardown should already have destroyed va_space_mm
+    // Normally we'd expect this to happen as part of uvm_mm_release()
+    // but if userspace never initialized uvm_mm_fd that won't happen.
+    // We don't have to take the va_space_mm spinlock and update state
+    // here because we know no other thread can be in or subsequently
+    // call uvm_api_mm_initialize successfully because the UVM
+    // file-descriptor has been released.
+    if (va_space->va_space_mm.state == UVM_VA_SPACE_MM_STATE_UNINITIALIZED)
+        uvm_va_space_mm_unregister(va_space);
    UVM_ASSERT(!uvm_va_space_mm_alive(&va_space->va_space_mm));

    uvm_mutex_lock(&g_uvm_global.global_lock);
@@ -2228,11 +2224,12 @@ static vm_fault_t uvm_va_space_cpu_fault(uvm_va_space_t *va_space,
            // address with mremap() so create a new va_block if needed.
            status = uvm_hmm_va_block_find_create(va_space,
                                                  fault_addr,
-                                                  &service_context->block_context,
+                                                  &service_context->block_context.hmm.vma,
                                                  &va_block);
            if (status != NV_OK)
                break;

+            UVM_ASSERT(service_context->block_context.hmm.vma == vma);
            status = uvm_hmm_migrate_begin(va_block);
            if (status != NV_OK)
                break;
--- a/kernel-open/nvidia-uvm/uvm_va_space_mm.c
+++ b/kernel-open/nvidia-uvm/uvm_va_space_mm.c
@@ -274,6 +274,22 @@ NV_STATUS uvm_va_space_mm_register(uvm_va_space_t *va_space)
        }
    }

+    if ((UVM_IS_CONFIG_HMM() || UVM_ATS_PREFETCH_SUPPORTED()) && uvm_va_space_pageable_mem_access_supported(va_space)) {
+        #if UVM_CAN_USE_MMU_NOTIFIERS()
+            // Initialize MMU interval notifiers for this process. This allows
+            // mmu_interval_notifier_insert() to be called without holding the
+            // mmap_lock for write.
+            // Note: there is no __mmu_notifier_unregister(), this call just
+            // allocates memory which is attached to the mm_struct and freed
+            // when the mm_struct is freed.
+            ret = __mmu_notifier_register(NULL, current->mm);
+            if (ret)
+                return errno_to_nv_status(ret);
+        #else
+            UVM_ASSERT(0);
+        #endif
+    }
+
    return NV_OK;
 }

--- a/kernel-open/nvidia-uvm/uvm_volta_fault_buffer.c
+++ b/kernel-open/nvidia-uvm/uvm_volta_fault_buffer.c
@@ -25,7 +25,8 @@
 #include "uvm_global.h"
 #include "uvm_gpu.h"
 #include "uvm_hal.h"
-#include "uvm_push.h"
+#include "uvm_conf_computing.h"
+#include "nv_uvm_types.h"
 #include "hwref/volta/gv100/dev_fault.h"
 #include "hwref/volta/gv100/dev_fb.h"
 #include "clc369.h"
@@ -246,6 +247,20 @@ static NvU32 *get_fault_buffer_entry(uvm_parent_gpu_t *parent_gpu, NvU32 index)
    return fault_entry;
 }

+// See uvm_pascal_fault_buffer.c::get_fault_buffer_entry_metadata
+static UvmFaultMetadataPacket *get_fault_buffer_entry_metadata(uvm_parent_gpu_t *parent_gpu, NvU32 index)
+{
+    UvmFaultMetadataPacket *fault_entry_metadata;
+
+    UVM_ASSERT(index < parent_gpu->fault_buffer_info.replayable.max_faults);
+    UVM_ASSERT(!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu));
+
+    fault_entry_metadata = parent_gpu->fault_buffer_info.rm_info.replayable.bufferMetadata;
+    UVM_ASSERT(fault_entry_metadata != NULL);
+
+    return fault_entry_metadata + index;
+}
+
 static void parse_fault_entry_common(uvm_parent_gpu_t *parent_gpu,
                                     NvU32 *fault_entry,
                                     uvm_fault_buffer_entry_t *buffer_entry)
@@ -323,24 +338,47 @@ static void parse_fault_entry_common(uvm_parent_gpu_t *parent_gpu,
    UVM_ASSERT_MSG(replayable_fault_enabled, "Fault with REPLAYABLE_FAULT_EN bit unset\n");
 }

-void uvm_hal_volta_fault_buffer_parse_entry(uvm_parent_gpu_t *parent_gpu,
-                                            NvU32 index,
-                                            uvm_fault_buffer_entry_t *buffer_entry)
+NV_STATUS uvm_hal_volta_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
+                                                            NvU32 index,
+                                                            uvm_fault_buffer_entry_t *buffer_entry)
 {
+    fault_buffer_entry_c369_t entry;
    NvU32 *fault_entry;
-    BUILD_BUG_ON(NVC369_BUF_SIZE > UVM_GPU_MMU_MAX_FAULT_PACKET_SIZE);
+
+    BUILD_BUG_ON(sizeof(entry) > UVM_GPU_MMU_MAX_FAULT_PACKET_SIZE);

    // Valid bit must be set before this function is called
    UVM_ASSERT(parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, index));

    fault_entry = get_fault_buffer_entry(parent_gpu, index);

+    // When Confidential Computing is enabled, faults are encrypted by RM, so
+    // they need to be decrypted before they can be parsed
+    if (!uvm_parent_gpu_replayable_fault_buffer_is_uvm_owned(parent_gpu)) {
+        NV_STATUS status;
+        UvmFaultMetadataPacket *fault_entry_metadata = get_fault_buffer_entry_metadata(parent_gpu, index);
+
+        status = uvm_conf_computing_fault_decrypt(parent_gpu,
+                                                  &entry,
+                                                  fault_entry,
+                                                  fault_entry_metadata->authTag,
+                                                  fault_entry_metadata->valid);
+        if (status != NV_OK) {
+            uvm_global_set_fatal_error(status);
+            return status;
+        }
+
+        fault_entry = (NvU32 *) &entry;
+    }
+
    parse_fault_entry_common(parent_gpu, fault_entry, buffer_entry);

    UVM_ASSERT(buffer_entry->is_replayable);

    // Automatically clear valid bit for the entry in the fault buffer
    parent_gpu->fault_buffer_hal->entry_clear_valid(parent_gpu, index);
+
+    return NV_OK;
 }

 void uvm_hal_volta_fault_buffer_parse_non_replayable_entry(uvm_parent_gpu_t *parent_gpu,
--- a/kernel-open/nvidia/detect-self-hosted.h
+++ b/kernel-open/nvidia/detect-self-hosted.h
@@ -0,0 +1,33 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __DETECT_SELF_HOSTED_H__
+#define __DETECT_SELF_HOSTED_H__
+
+// PCI devIds 0x2340-0x237f are for Self-Hosted Hopper
+static inline int pci_devid_is_self_hosted(unsigned short devid)
+{
+    return devid >= 0x2340 && devid <= 0x237f;
+}
+
+#endif
--- a/kernel-open/nvidia/export_nvswitch.h
+++ b/kernel-open/nvidia/export_nvswitch.h
@@ -939,6 +939,12 @@ nvswitch_os_get_os_version
    NvU32 *pBuildNum
 );

+NvlStatus
+nvswitch_os_get_pid
+(
+    NvU32 *pPid
+);
+
 void
 nvswitch_lib_smbpbi_log_sxid
 (
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_aead.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_aead.h
@@ -0,0 +1,211 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_AEAD_H
+#define CRYPTLIB_AEAD_H
+
+/*=====================================================================================
+ *    Authenticated Encryption with Associated data (AEAD) Cryptography Primitives
+ *=====================================================================================
+ */
+
+#if LIBSPDM_AEAD_GCM_SUPPORT
+/**
+ * Performs AEAD AES-GCM authenticated encryption on a data buffer and additional authenticated
+ * data.
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 16 or 32, otherwise false is returned.
+ * tag_size must be 12, 13, 14, 15, 16, otherwise false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be encrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[out]  tag_out        Pointer to a buffer that receives the authentication tag output.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the encryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD AES-GCM authenticated encryption succeeded.
+ * @retval false  AEAD AES-GCM authenticated encryption failed.
+ **/
+extern bool libspdm_aead_aes_gcm_encrypt(const uint8_t *key, size_t key_size,
+                                         const uint8_t *iv, size_t iv_size,
+                                         const uint8_t *a_data, size_t a_data_size,
+                                         const uint8_t *data_in, size_t data_in_size,
+                                         uint8_t *tag_out, size_t tag_size,
+                                         uint8_t *data_out, size_t *data_out_size);
+
+/**
+ * Performs AEAD AES-GCM authenticated decryption on a data buffer and additional authenticated
+ * data.
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 16 or 32, otherwise false is returned.
+ * tag_size must be 12, 13, 14, 15, 16, otherwise false is returned.
+ *
+ * If data verification fails, false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be decrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[in]   tag            Pointer to a buffer that contains the authentication tag.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the decryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD AES-GCM authenticated decryption succeeded.
+ * @retval false  AEAD AES-GCM authenticated decryption failed.
+ **/
+extern bool libspdm_aead_aes_gcm_decrypt(const uint8_t *key, size_t key_size,
+                                         const uint8_t *iv, size_t iv_size,
+                                         const uint8_t *a_data, size_t a_data_size,
+                                         const uint8_t *data_in, size_t data_in_size,
+                                         const uint8_t *tag, size_t tag_size,
+                                         uint8_t *data_out, size_t *data_out_size);
+#endif /* LIBSPDM_AEAD_GCM_SUPPORT */
+
+#if LIBSPDM_AEAD_CHACHA20_POLY1305_SUPPORT
+/**
+ * Performs AEAD ChaCha20Poly1305 authenticated encryption on a data buffer and additional
+ * authenticated data.
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 32, otherwise false is returned.
+ * tag_size must be 16, otherwise false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be encrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[out]  tag_out        Pointer to a buffer that receives the authentication tag output.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the encryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD ChaCha20Poly1305 authenticated encryption succeeded.
+ * @retval false  AEAD ChaCha20Poly1305 authenticated encryption failed.
+ **/
+extern bool libspdm_aead_chacha20_poly1305_encrypt(
+    const uint8_t *key, size_t key_size, const uint8_t *iv,
+    size_t iv_size, const uint8_t *a_data, size_t a_data_size,
+    const uint8_t *data_in, size_t data_in_size, uint8_t *tag_out,
+    size_t tag_size, uint8_t *data_out, size_t *data_out_size);
+
+/**
+ * Performs AEAD ChaCha20Poly1305 authenticated decryption on a data buffer and additional authenticated data (AAD).
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 32, otherwise false is returned.
+ * tag_size must be 16, otherwise false is returned.
+ *
+ * If data verification fails, false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be decrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[in]   tag            Pointer to a buffer that contains the authentication tag.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the decryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD ChaCha20Poly1305 authenticated decryption succeeded.
+ * @retval false  AEAD ChaCha20Poly1305 authenticated decryption failed.
+ *
+ **/
+extern bool libspdm_aead_chacha20_poly1305_decrypt(
+    const uint8_t *key, size_t key_size, const uint8_t *iv,
+    size_t iv_size, const uint8_t *a_data, size_t a_data_size,
+    const uint8_t *data_in, size_t data_in_size, const uint8_t *tag,
+    size_t tag_size, uint8_t *data_out, size_t *data_out_size);
+#endif /* LIBSPDM_AEAD_CHACHA20_POLY1305_SUPPORT */
+
+#if LIBSPDM_AEAD_SM4_SUPPORT
+/**
+ * Performs AEAD SM4-GCM authenticated encryption on a data buffer and additional authenticated
+ * data.
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 16, otherwise false is returned.
+ * tag_size must be 16, otherwise false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be encrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[out]  tag_out        Pointer to a buffer that receives the authentication tag output.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the encryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD SM4-GCM authenticated encryption succeeded.
+ * @retval false  AEAD SM4-GCM authenticated encryption failed.
+ **/
+extern bool libspdm_aead_sm4_gcm_encrypt(const uint8_t *key, size_t key_size,
+                                         const uint8_t *iv, size_t iv_size,
+                                         const uint8_t *a_data, size_t a_data_size,
+                                         const uint8_t *data_in, size_t data_in_size,
+                                         uint8_t *tag_out, size_t tag_size,
+                                         uint8_t *data_out, size_t *data_out_size);
+
+/**
+ * Performs AEAD SM4-GCM authenticated decryption on a data buffer and additional authenticated
+ * data.
+ *
+ * iv_size must be 12, otherwise false is returned.
+ * key_size must be 16, otherwise false is returned.
+ * tag_size must be 16, otherwise false is returned.
+ *
+ * If data verification fails, false is returned.
+ *
+ * @param[in]   key            Pointer to the encryption key.
+ * @param[in]   key_size       Size of the encryption key in bytes.
+ * @param[in]   iv             Pointer to the IV value.
+ * @param[in]   iv_size        Size of the IV value in bytes.
+ * @param[in]   a_data         Pointer to the additional authenticated data.
+ * @param[in]   a_data_size    Size of the additional authenticated data in bytes.
+ * @param[in]   data_in        Pointer to the input data buffer to be decrypted.
+ * @param[in]   data_in_size   Size of the input data buffer in bytes.
+ * @param[in]   tag            Pointer to a buffer that contains the authentication tag.
+ * @param[in]   tag_size       Size of the authentication tag in bytes.
+ * @param[out]  data_out       Pointer to a buffer that receives the decryption output.
+ * @param[out]  data_out_size  Size of the output data buffer in bytes.
+ *
+ * @retval true   AEAD SM4-GCM authenticated decryption succeeded.
+ * @retval false  AEAD SM4-GCM authenticated decryption failed.
+ **/
+extern bool libspdm_aead_sm4_gcm_decrypt(const uint8_t *key, size_t key_size,
+                                         const uint8_t *iv, size_t iv_size,
+                                         const uint8_t *a_data, size_t a_data_size,
+                                         const uint8_t *data_in, size_t data_in_size,
+                                         const uint8_t *tag, size_t tag_size,
+                                         uint8_t *data_out, size_t *data_out_size);
+#endif /* LIBSPDM_AEAD_SM4_SUPPORT */
+
+#endif /* CRYPTLIB_AEAD_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_cert.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_cert.h
@@ -0,0 +1,416 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_CERT_H
+#define CRYPTLIB_CERT_H
+
+/**
+ * Retrieve the tag and length of the tag.
+ *
+ * @param ptr      The position in the ASN.1 data.
+ * @param end      End of data.
+ * @param length   The variable that will receive the length.
+ * @param tag      The expected tag.
+ *
+ * @retval  true   Get tag successful.
+ * @retval  false  Failed to get tag or tag not match.
+ **/
+extern bool libspdm_asn1_get_tag(uint8_t **ptr, const uint8_t *end, size_t *length, uint32_t tag);
+
+/**
+ * Retrieve the subject bytes from one X.509 certificate.
+ *
+ * If cert is NULL, then return false.
+ * If subject_size is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert          Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size     Size of the X509 certificate in bytes.
+ * @param[out]     cert_subject  Pointer to the retrieved certificate subject bytes.
+ * @param[in, out] subject_size  The size in bytes of the cert_subject buffer on input,
+ *                               and the size of buffer returned cert_subject on output.
+ *
+ * @retval  true   The certificate subject retrieved successfully.
+ * @retval  false  Invalid certificate, or the subject_size is too small for the result.
+ *                 The subject_size will be updated with the required size.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_x509_get_subject_name(const uint8_t *cert, size_t cert_size,
+                                          uint8_t *cert_subject,
+                                          size_t *subject_size);
+
+/**
+ * Retrieve the version from one X.509 certificate.
+ *
+ * If cert is NULL, then return false.
+ * If cert_size is 0, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert       Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size  Size of the X509 certificate in bytes.
+ * @param[out]     version    Pointer to the retrieved version integer.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_get_version(const uint8_t *cert, size_t cert_size, size_t *version);
+
+/**
+ * Retrieve the serialNumber from one X.509 certificate.
+ *
+ * If cert is NULL, then return false.
+ * If cert_size is 0, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert                Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size           Size of the X509 certificate in bytes.
+ * @param[out]     serial_number       Pointer to the retrieved certificate serial_number bytes.
+ * @param[in, out] serial_number_size  The size in bytes of the serial_number buffer on input,
+ *                                     and the size of buffer returned serial_number on output.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_get_serial_number(const uint8_t *cert, size_t cert_size,
+                                           uint8_t *serial_number,
+                                           size_t *serial_number_size);
+
+/**
+ * Retrieve the issuer bytes from one X.509 certificate.
+ *
+ * If cert is NULL, then return false.
+ * If issuer_size is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert         Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size    Size of the X509 certificate in bytes.
+ * @param[out]     cert_issuer  Pointer to the retrieved certificate subject bytes.
+ * @param[in, out] issuer_size  The size in bytes of the cert_issuer buffer on input,
+ *                              and the size of buffer returned cert_issuer on output.
+ *
+ * @retval  true   The certificate issuer retrieved successfully.
+ * @retval  false  Invalid certificate, or the issuer_size is too small for the result.
+ *                 The issuer_size will be updated with the required size.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_x509_get_issuer_name(const uint8_t *cert, size_t cert_size,
+                                         uint8_t *cert_issuer,
+                                         size_t *issuer_size);
+
+/**
+ * Retrieve Extension data from one X.509 certificate.
+ *
+ * @param[in]      cert                 Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size            Size of the X509 certificate in bytes.
+ * @param[in]      oid                  Object identifier buffer
+ * @param[in]      oid_size             Object identifier buffer size
+ * @param[out]     extension_data       Extension bytes.
+ * @param[in, out] extension_data_size  Extension bytes size.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_get_extension_data(const uint8_t *cert, size_t cert_size,
+                                            const uint8_t *oid, size_t oid_size,
+                                            uint8_t *extension_data,
+                                            size_t *extension_data_size);
+
+/**
+ * Retrieve the Validity from one X.509 certificate
+ *
+ * If cert is NULL, then return false.
+ * If CertIssuerSize is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert       Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size  Size of the X509 certificate in bytes.
+ * @param[out]     from       notBefore Pointer to date_time object.
+ * @param[in,out]  from_size  notBefore date_time object size.
+ * @param[out]     to         notAfter Pointer to date_time object.
+ * @param[in,out]  to_size    notAfter date_time object size.
+ *
+ * Note: libspdm_x509_compare_date_time to compare date_time oject
+ *       x509SetDateTime to get a date_time object from a date_time_str
+ *
+ * @retval  true   The certificate Validity retrieved successfully.
+ * @retval  false  Invalid certificate, or Validity retrieve failed.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_x509_get_validity(const uint8_t *cert, size_t cert_size,
+                                      uint8_t *from, size_t *from_size, uint8_t *to,
+                                      size_t *to_size);
+
+/**
+ * Format a date_time object into DataTime buffer
+ *
+ * If date_time_str is NULL, then return false.
+ * If date_time_size is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      date_time_str    date_time string like YYYYMMDDhhmmssZ
+ *                                  Ref: https://www.w3.org/TR/NOTE-datetime
+ *                                  Z stand for UTC time
+ * @param[out]     date_time       Pointer to a date_time object.
+ * @param[in,out]  date_time_size  date_time object buffer size.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_set_date_time(const char *date_time_str, void *date_time,
+                                       size_t *date_time_size);
+
+/**
+ * Compare date_time1 object and date_time2 object.
+ *
+ * If date_time1 is NULL, then return -2.
+ * If date_time2 is NULL, then return -2.
+ * If date_time1 == date_time2, then return 0
+ * If date_time1 > date_time2, then return 1
+ * If date_time1 < date_time2, then return -1
+ *
+ * @param[in]      date_time1  Pointer to a date_time Ojbect
+ * @param[in]      date_time2  Pointer to a date_time Object
+ *
+ * @retval   0  If date_time1 == date_time2
+ * @retval   1  If date_time1 > date_time2
+ * @retval  -1  If date_time1 < date_time2
+ **/
+extern int32_t libspdm_x509_compare_date_time(const void *date_time1, const void *date_time2);
+
+/**
+ * Retrieve the key usage from one X.509 certificate.
+ *
+ * @param[in]      cert       Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size  Size of the X509 certificate in bytes.
+ * @param[out]     usage      Key usage (LIBSPDM_CRYPTO_X509_KU_*)
+ *
+ * @retval  true   The certificate key usage retrieved successfully.
+ * @retval  false  Invalid certificate, or usage is NULL
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_x509_get_key_usage(const uint8_t *cert, size_t cert_size, size_t *usage);
+
+/**
+ * Retrieve the Extended key usage from one X.509 certificate.
+ *
+ * @param[in]      cert        Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size   Size of the X509 certificate in bytes.
+ * @param[out]     usage       Key usage bytes.
+ * @param[in, out] usage_size  Key usage buffer sizs in bytes.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_get_extended_key_usage(const uint8_t *cert,
+                                                size_t cert_size, uint8_t *usage,
+                                                size_t *usage_size);
+
+/**
+ * Retrieve the basic constraints from one X.509 certificate.
+ *
+ * @param[in]      cert                     Pointer to the DER-encoded X509 certificate.
+ * @param[in]      cert_size                Size of the X509 certificate in bytes.
+ * @param[out]     basic_constraints        Basic constraints bytes.
+ * @param[in, out] basic_constraints_size   Basic constraints buffer sizs in bytes.
+ *
+ * @retval  true
+ * @retval  false
+ **/
+extern bool libspdm_x509_get_extended_basic_constraints(const uint8_t *cert,
+                                                        size_t cert_size,
+                                                        uint8_t *basic_constraints,
+                                                        size_t *basic_constraints_size);
+
+/**
+ * Verify one X509 certificate was issued by the trusted CA.
+ *
+ * If cert is NULL, then return false.
+ * If ca_cert is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      cert          Pointer to the DER-encoded X509 certificate to be verified.
+ * @param[in]      cert_size     Size of the X509 certificate in bytes.
+ * @param[in]      ca_cert       Pointer to the DER-encoded trusted CA certificate.
+ * @param[in]      ca_cert_size  Size of the CA Certificate in bytes.
+ *
+ * @retval  true   The certificate was issued by the trusted CA.
+ * @retval  false  Invalid certificate or the certificate was not issued by the given
+ *                 trusted CA.
+ * @retval  false  This interface is not supported.
+ *
+ **/
+extern bool libspdm_x509_verify_cert(const uint8_t *cert, size_t cert_size,
+                                     const uint8_t *ca_cert, size_t ca_cert_size);
+
+/**
+ * Verify one X509 certificate was issued by the trusted CA.
+ *
+ * @param[in]      cert_chain         One or more ASN.1 DER-encoded X.509 certificates
+ *                                    where the first certificate is signed by the Root
+ *                                    Certificate or is the Root Cerificate itself. and
+ *                                    subsequent cerificate is signed by the preceding
+ *                                    cerificate.
+ * @param[in]      cert_chain_length  Total length of the certificate chain, in bytes.
+ *
+ * @param[in]      root_cert          Trusted Root Certificate buffer.
+ *
+ * @param[in]      root_cert_length   Trusted Root Certificate buffer length.
+ *
+ * @retval  true   All cerificates were issued by the first certificate in X509Certchain.
+ * @retval  false  Invalid certificate or the certificate was not issued by the given
+ *                 trusted CA.
+ **/
+extern bool libspdm_x509_verify_cert_chain(const uint8_t *root_cert, size_t root_cert_length,
+                                           const uint8_t *cert_chain,
+                                           size_t cert_chain_length);
+
+/**
+ * Get one X509 certificate from cert_chain.
+ *
+ * @param[in]      cert_chain         One or more ASN.1 DER-encoded X.509 certificates
+ *                                    where the first certificate is signed by the Root
+ *                                    Certificate or is the Root Cerificate itself. and
+ *                                    subsequent cerificate is signed by the preceding
+ *                                    cerificate.
+ * @param[in]      cert_chain_length  Total length of the certificate chain, in bytes.
+ *
+ * @param[in]      cert_index         Index of certificate. If index is -1 indecate the
+ *                                    last certificate in cert_chain.
+ *
+ * @param[out]     cert              The certificate at the index of cert_chain.
+ * @param[out]     cert_length       The length certificate at the index of cert_chain.
+ *
+ * @retval  true   Success.
+ * @retval  false  Failed to get certificate from certificate chain.
+ **/
+extern bool libspdm_x509_get_cert_from_cert_chain(const uint8_t *cert_chain,
+                                                  size_t cert_chain_length,
+                                                  const int32_t cert_index, const uint8_t **cert,
+                                                  size_t *cert_length);
+
+#if (LIBSPDM_RSA_SSA_SUPPORT) || (LIBSPDM_RSA_PSS_SUPPORT)
+/**
+ * Retrieve the RSA public key from one DER-encoded X509 certificate.
+ *
+ * If cert is NULL, then return false.
+ * If rsa_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  cert         Pointer to the DER-encoded X509 certificate.
+ * @param[in]  cert_size    Size of the X509 certificate in bytes.
+ * @param[out] rsa_context  Pointer to new-generated RSA context which contain the retrieved
+ *                          RSA public key component. Use libspdm_rsa_free() function to free the
+ *                          resource.
+ *
+ * @retval  true   RSA public key was retrieved successfully.
+ * @retval  false  Fail to retrieve RSA public key from X509 certificate.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_rsa_get_public_key_from_x509(const uint8_t *cert, size_t cert_size,
+                                                 void **rsa_context);
+#endif /* (LIBSPDM_RSA_SSA_SUPPORT) || (LIBSPDM_RSA_PSS_SUPPORT) */
+
+#if LIBSPDM_ECDSA_SUPPORT
+/**
+ * Retrieve the EC public key from one DER-encoded X509 certificate.
+ *
+ * @param[in]  cert        Pointer to the DER-encoded X509 certificate.
+ * @param[in]  cert_size   Size of the X509 certificate in bytes.
+ * @param[out] ec_context  Pointer to new-generated EC DSA context which contain the retrieved
+ *                         EC public key component. Use libspdm_ec_free() function to free the
+ *                         resource.
+ *
+ * If cert is NULL, then return false.
+ * If ec_context is NULL, then return false.
+ *
+ * @retval  true   EC public key was retrieved successfully.
+ * @retval  false  Fail to retrieve EC public key from X509 certificate.
+ *
+ **/
+extern bool libspdm_ec_get_public_key_from_x509(const uint8_t *cert, size_t cert_size,
+                                                void **ec_context);
+#endif /* LIBSPDM_ECDSA_SUPPORT */
+
+#if (LIBSPDM_EDDSA_ED25519_SUPPORT) || (LIBSPDM_EDDSA_ED448_SUPPORT)
+/**
+ * Retrieve the Ed public key from one DER-encoded X509 certificate.
+ *
+ * @param[in]  cert         Pointer to the DER-encoded X509 certificate.
+ * @param[in]  cert_size    Size of the X509 certificate in bytes.
+ * @param[out] ecd_context  Pointer to new-generated Ed DSA context which contain the retrieved
+ *                          Ed public key component. Use libspdm_ecd_free() function to free the
+ *                          resource.
+ *
+ * If cert is NULL, then return false.
+ * If ecd_context is NULL, then return false.
+ *
+ * @retval  true   Ed public key was retrieved successfully.
+ * @retval  false  Fail to retrieve Ed public key from X509 certificate.
+ *
+ **/
+extern bool libspdm_ecd_get_public_key_from_x509(const uint8_t *cert, size_t cert_size,
+                                                 void **ecd_context);
+#endif /* (LIBSPDM_EDDSA_ED25519_SUPPORT) || (LIBSPDM_EDDSA_ED448_SUPPORT) */
+
+#if LIBSPDM_SM2_DSA_SUPPORT
+/**
+ * Retrieve the sm2 public key from one DER-encoded X509 certificate.
+ *
+ * @param[in]  cert         Pointer to the DER-encoded X509 certificate.
+ * @param[in]  cert_size    Size of the X509 certificate in bytes.
+ * @param[out] sm2_context  Pointer to new-generated sm2 context which contain the retrieved
+ *                          sm2 public key component. Use sm2_free() function to free the
+ *                          resource.
+ *
+ * If cert is NULL, then return false.
+ * If sm2_context is NULL, then return false.
+ *
+ * @retval  true   sm2 public key was retrieved successfully.
+ * @retval  false  Fail to retrieve sm2 public key from X509 certificate.
+ *
+ **/
+extern bool libspdm_sm2_get_public_key_from_x509(const uint8_t *cert, size_t cert_size,
+                                                 void **sm2_context);
+#endif /* LIBSPDM_SM2_DSA_SUPPORT */
+
+#if LIBSPDM_ENABLE_CAPABILITY_GET_CSR_CAP
+/**
+ * Generate a CSR.
+ *
+ * @param[in]      hash_nid              hash algo for sign
+ * @param[in]      asym_nid              asym algo for sign
+ *
+ * @param[in]      requester_info        requester info to gen CSR
+ * @param[in]      requester_info_length The len of requester info
+ *
+ * @param[in]      context               Pointer to asymmetric context
+ * @param[in]      subject_name          Subject name: should be break with ',' in the middle
+ *                                       example: "C=AA,CN=BB"
+ *
+ * Subject names should contain a comma-separated list of OID types and values:
+ * The valid OID type name is in:
+ * {"CN", "commonName", "C", "countryName", "O", "organizationName","L",
+ * "OU", "organizationalUnitName", "ST", "stateOrProvinceName", "emailAddress",
+ * "serialNumber", "postalAddress", "postalCode", "dnQualifier", "title",
+ * "SN","givenName","GN", "initials", "pseudonym", "generationQualifier", "domainComponent", "DC"}.
+ * Note: The object of C and countryName should be CSR Supported Country Codes
+ *
+ * @param[in]      csr_len               For input, csr_len is the size of store CSR buffer.
+ *                                       For output, csr_len is CSR len for DER format
+ * @param[in]      csr_pointer           For input, csr_pointer is buffer address to store CSR.
+ *                                       For output, csr_pointer is address for stored CSR.
+ *                                       The csr_pointer address will be changed.
+ *
+ * @retval  true   Success.
+ * @retval  false  Failed to gen CSR.
+ **/
+extern bool libspdm_gen_x509_csr(size_t hash_nid, size_t asym_nid,
+                                 uint8_t *requester_info, size_t requester_info_length,
+                                 void *context, char *subject_name,
+                                 size_t *csr_len, uint8_t **csr_pointer);
+#endif /* LIBSPDM_ENABLE_CAPABILITY_GET_CSR_CAP */
+
+#endif /* CRYPTLIB_CERT_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_dh.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_dh.h
@@ -0,0 +1,98 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_DH_H
+#define CRYPTLIB_DH_H
+
+/*=====================================================================================
+ *    Diffie-Hellman Key Exchange Primitives
+ *=====================================================================================
+ */
+
+#if LIBSPDM_FFDHE_SUPPORT
+/**
+ * Allocates and initializes one Diffie-Hellman context for subsequent use with the NID.
+ *
+ * @param nid cipher NID
+ *
+ * @return  Pointer to the Diffie-Hellman context that has been initialized.
+ *          If the allocations fails, libspdm_dh_new_by_nid() returns NULL.
+ *          If the interface is not supported, libspdm_dh_new_by_nid() returns NULL.
+ **/
+extern void *libspdm_dh_new_by_nid(size_t nid);
+
+/**
+ * Release the specified DH context.
+ *
+ * @param[in]  dh_context  Pointer to the DH context to be released.
+ **/
+void libspdm_dh_free(void *dh_context);
+
+/**
+ * Generates DH public key.
+ *
+ * This function generates random secret exponent, and computes the public key, which is
+ * returned via parameter public_key and public_key_size. DH context is updated accordingly.
+ * If the public_key buffer is too small to hold the public key, false is returned and
+ * public_key_size is set to the required buffer size to obtain the public key.
+ *
+ * If dh_context is NULL, then return false.
+ * If public_key_size is NULL, then return false.
+ * If public_key_size is large enough but public_key is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * For FFDHE2048, the public_size is 256.
+ * For FFDHE3072, the public_size is 384.
+ * For FFDHE4096, the public_size is 512.
+ *
+ * @param[in, out]  dh_context       Pointer to the DH context.
+ * @param[out]      public_key       Pointer to the buffer to receive generated public key.
+ * @param[in, out]  public_key_size  On input, the size of public_key buffer in bytes.
+ *                                   On output, the size of data returned in public_key buffer in
+ *                                   bytes.
+ *
+ * @retval true   DH public key generation succeeded.
+ * @retval false  DH public key generation failed.
+ * @retval false  public_key_size is not large enough.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_dh_generate_key(void *dh_context, uint8_t *public_key, size_t *public_key_size);
+
+/**
+ * Computes exchanged common key.
+ *
+ * Given peer's public key, this function computes the exchanged common key, based on its own
+ * context including value of prime modulus and random secret exponent.
+ *
+ * If dh_context is NULL, then return false.
+ * If peer_public_key is NULL, then return false.
+ * If key_size is NULL, then return false.
+ * If key is NULL, then return false.
+ * If key_size is not large enough, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * For FFDHE2048, the peer_public_size and key_size is 256.
+ * For FFDHE3072, the peer_public_size and key_size is 384.
+ * For FFDHE4096, the peer_public_size and key_size is 512.
+ *
+ * @param[in, out]  dh_context            Pointer to the DH context.
+ * @param[in]       peer_public_key       Pointer to the peer's public key.
+ * @param[in]       peer_public_key_size  size of peer's public key in bytes.
+ * @param[out]      key                   Pointer to the buffer to receive generated key.
+ * @param[in, out]  key_size              On input, the size of key buffer in bytes.
+ *                                        On output, the size of data returned in key buffer in
+ *                                        bytes.
+ *
+ * @retval true   DH exchanged key generation succeeded.
+ * @retval false  DH exchanged key generation failed.
+ * @retval false  key_size is not large enough.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_dh_compute_key(void *dh_context, const uint8_t *peer_public_key,
+                                   size_t peer_public_key_size, uint8_t *key,
+                                   size_t *key_size);
+#endif /* LIBSPDM_FFDHE_SUPPORT */
+#endif /* CRYPTLIB_DH_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_ec.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_ec.h
@@ -0,0 +1,162 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_EC_H
+#define CRYPTLIB_EC_H
+
+/*=====================================================================================
+ *    Elliptic Curve Primitives
+ *=====================================================================================*/
+
+#if (LIBSPDM_ECDHE_SUPPORT) || (LIBSPDM_ECDSA_SUPPORT)
+/**
+ * Allocates and Initializes one Elliptic Curve context for subsequent use with the NID.
+ *
+ * @param nid cipher NID
+ *
+ * @return  Pointer to the Elliptic Curve context that has been initialized.
+ *          If the allocations fails, libspdm_ec_new_by_nid() returns NULL.
+ **/
+extern void *libspdm_ec_new_by_nid(size_t nid);
+
+/**
+ * Release the specified EC context.
+ *
+ * @param[in]  ec_context  Pointer to the EC context to be released.
+ **/
+extern void libspdm_ec_free(void *ec_context);
+#endif /* (LIBSPDM_ECDHE_SUPPORT) || (LIBSPDM_ECDSA_SUPPORT) */
+
+#if LIBSPDM_ECDHE_SUPPORT
+/**
+ * Generates EC key and returns EC public key (X, Y).
+ *
+ * This function generates random secret, and computes the public key (X, Y), which is
+ * returned via parameter public, public_size.
+ * X is the first half of public with size being public_size / 2,
+ * Y is the second half of public with size being public_size / 2.
+ * EC context is updated accordingly.
+ * If the public buffer is too small to hold the public X, Y, false is returned and
+ * public_size is set to the required buffer size to obtain the public X, Y.
+ *
+ * For P-256, the public_size is 64. first 32-byte is X, second 32-byte is Y.
+ * For P-384, the public_size is 96. first 48-byte is X, second 48-byte is Y.
+ * For P-521, the public_size is 132. first 66-byte is X, second 66-byte is Y.
+ *
+ * If ec_context is NULL, then return false.
+ * If public_size is NULL, then return false.
+ * If public_size is large enough but public is NULL, then return false.
+ *
+ * @param[in, out]  ec_context   Pointer to the EC context.
+ * @param[out]      public       Pointer to the buffer to receive generated public X,Y.
+ * @param[in, out]  public_size  On input, the size of public buffer in bytes.
+ *                               On output, the size of data returned in public buffer in bytes.
+ *
+ * @retval true   EC public X,Y generation succeeded.
+ * @retval false  EC public X,Y generation failed.
+ * @retval false  public_size is not large enough.
+ **/
+extern bool libspdm_ec_generate_key(void *ec_context, uint8_t *public_key, size_t *public_key_size);
+
+/**
+ * Computes exchanged common key.
+ *
+ * Given peer's public key (X, Y), this function computes the exchanged common key,
+ * based on its own context including value of curve parameter and random secret.
+ * X is the first half of peer_public with size being peer_public_size / 2,
+ * Y is the second half of peer_public with size being peer_public_size / 2.
+ *
+ * If ec_context is NULL, then return false.
+ * If peer_public is NULL, then return false.
+ * If peer_public_size is 0, then return false.
+ * If key is NULL, then return false.
+ * If key_size is not large enough, then return false.
+ *
+ * For P-256, the peer_public_size is 64. first 32-byte is X, second 32-byte is Y.
+ * The key_size is 32.
+ * For P-384, the peer_public_size is 96. first 48-byte is X, second 48-byte is Y.
+ * The key_size is 48.
+ * For P-521, the peer_public_size is 132. first 66-byte is X, second 66-byte is Y.
+ * The key_size is 66.
+ *
+ * @param[in, out]  ec_context        Pointer to the EC context.
+ * @param[in]       peer_public       Pointer to the peer's public X,Y.
+ * @param[in]       peer_public_size  Size of peer's public X,Y in bytes.
+ * @param[out]      key               Pointer to the buffer to receive generated key.
+ * @param[in, out]  key_size          On input, the size of key buffer in bytes.
+ *                                    On output, the size of data returned in key buffer in bytes.
+ *
+ * @retval true   EC exchanged key generation succeeded.
+ * @retval false  EC exchanged key generation failed.
+ * @retval false  key_size is not large enough.
+ **/
+extern bool libspdm_ec_compute_key(void *ec_context, const uint8_t *peer_public,
+                                   size_t peer_public_size, uint8_t *key,
+                                   size_t *key_size);
+#endif /* LIBSPDM_ECDHE_SUPPORT */
+
+#if LIBSPDM_ECDSA_SUPPORT
+/**
+ * Carries out the EC-DSA signature.
+ *
+ * This function carries out the EC-DSA signature.
+ * If the signature buffer is too small to hold the contents of signature, false
+ * is returned and sig_size is set to the required buffer size to obtain the signature.
+ *
+ * If ec_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If hash_size need match the hash_nid. hash_nid could be SHA256, SHA384, SHA512, SHA3_256,
+ * SHA3_384, SHA3_512.
+ * If sig_size is large enough but signature is NULL, then return false.
+ *
+ * For P-256, the sig_size is 64. first 32-byte is R, second 32-byte is S.
+ * For P-384, the sig_size is 96. first 48-byte is R, second 48-byte is S.
+ * For P-521, the sig_size is 132. first 66-byte is R, second 66-byte is S.
+ *
+ * @param[in]       ec_context    Pointer to EC context for signature generation.
+ * @param[in]       hash_nid      hash NID
+ * @param[in]       message_hash  Pointer to octet message hash to be signed.
+ * @param[in]       hash_size     Size of the message hash in bytes.
+ * @param[out]      signature     Pointer to buffer to receive EC-DSA signature.
+ * @param[in, out]  sig_size      On input, the size of signature buffer in bytes.
+ *                                On output, the size of data returned in signature buffer in bytes.
+ *
+ * @retval  true   signature successfully generated in EC-DSA.
+ * @retval  false  signature generation failed.
+ * @retval  false  sig_size is too small.
+ **/
+extern bool libspdm_ecdsa_sign(void *ec_context, size_t hash_nid,
+                               const uint8_t *message_hash, size_t hash_size,
+                               uint8_t *signature, size_t *sig_size);
+
+/**
+ * Verifies the EC-DSA signature.
+ *
+ * If ec_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If signature is NULL, then return false.
+ * If hash_size need match the hash_nid. hash_nid could be SHA256, SHA384, SHA512, SHA3_256,
+ * SHA3_384, SHA3_512.
+ *
+ * For P-256, the sig_size is 64. first 32-byte is R, second 32-byte is S.
+ * For P-384, the sig_size is 96. first 48-byte is R, second 48-byte is S.
+ * For P-521, the sig_size is 132. first 66-byte is R, second 66-byte is S.
+ *
+ * @param[in]  ec_context    Pointer to EC context for signature verification.
+ * @param[in]  hash_nid      hash NID
+ * @param[in]  message_hash  Pointer to octet message hash to be checked.
+ * @param[in]  hash_size     Size of the message hash in bytes.
+ * @param[in]  signature     Pointer to EC-DSA signature to be verified.
+ * @param[in]  sig_size      Size of signature in bytes.
+ *
+ * @retval  true   Valid signature encoded in EC-DSA.
+ * @retval  false  Invalid signature or invalid EC context.
+ **/
+extern bool libspdm_ecdsa_verify(void *ec_context, size_t hash_nid,
+                                 const uint8_t *message_hash, size_t hash_size,
+                                 const uint8_t *signature, size_t sig_size);
+#endif /* LIBSPDM_ECDSA_SUPPORT */
+#endif /* CRYPTLIB_EC_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_ecd.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_ecd.h
@@ -0,0 +1,100 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_ECD_H
+#define CRYPTLIB_ECD_H
+
+/*=====================================================================================
+ *    Edwards-Curve Primitives
+ *=====================================================================================*/
+
+#if (LIBSPDM_EDDSA_ED25519_SUPPORT) || (LIBSPDM_EDDSA_ED448_SUPPORT)
+/**
+ * Allocates and Initializes one Edwards-Curve context for subsequent use with the NID.
+ *
+ * @param nid cipher NID
+ *
+ * @return  Pointer to the Edwards-Curve context that has been initialized.
+ *          If the allocations fails, libspdm_ecd_new_by_nid() returns NULL.
+ **/
+extern void *libspdm_ecd_new_by_nid(size_t nid);
+
+/**
+ * Release the specified Ed context.
+ *
+ * @param[in]  ecd_context  Pointer to the Ed context to be released.
+ **/
+extern void libspdm_ecd_free(void *ecd_context);
+
+/**
+ * Carries out the Ed-DSA signature.
+ *
+ * This function carries out the Ed-DSA signature.
+ * If the signature buffer is too small to hold the contents of signature, false
+ * is returned and sig_size is set to the required buffer size to obtain the signature.
+ *
+ * If ecd_context is NULL, then return false.
+ * If message is NULL, then return false.
+ * hash_nid must be NULL.
+ * If sig_size is large enough but signature is NULL, then return false.
+ *
+ * For ed25519, context must be NULL and context_size must be 0.
+ * For ed448, context must be maximum of 255 octets.
+ *
+ * For ed25519, the sig_size is 64. first 32-byte is R, second 32-byte is S.
+ * For ed448, the sig_size is 114. first 57-byte is R, second 57-byte is S.
+ *
+ * @param[in]       ecd_context   Pointer to Ed context for signature generation.
+ * @param[in]       hash_nid      hash NID
+ * @param[in]       context       The EDDSA signing context.
+ * @param[in]       context_size  Size of EDDSA signing context.
+ * @param[in]       message       Pointer to octet message to be signed (before hash).
+ * @param[in]       size          size of the message in bytes.
+ * @param[out]      signature     Pointer to buffer to receive Ed-DSA signature.
+ * @param[in, out]  sig_size      On input, the size of signature buffer in bytes.
+ *                                On output, the size of data returned in signature buffer in bytes.
+ *
+ * @retval  true   signature successfully generated in Ed-DSA.
+ * @retval  false  signature generation failed.
+ * @retval  false  sig_size is too small.
+ **/
+extern bool libspdm_eddsa_sign(const void *ecd_context, size_t hash_nid,
+                               const uint8_t *context, size_t context_size,
+                               const uint8_t *message, size_t size, uint8_t *signature,
+                               size_t *sig_size);
+
+/**
+ * Verifies the Ed-DSA signature.
+ *
+ * If ecd_context is NULL, then return false.
+ * If message is NULL, then return false.
+ * If signature is NULL, then return false.
+ * hash_nid must be NULL.
+ *
+ * For ed25519, context must be NULL and context_size must be 0.
+ * For ed448, context must be maximum of 255 octets.
+ *
+ * For ed25519, the sig_size is 64. first 32-byte is R, second 32-byte is S.
+ * For ed448, the sig_size is 114. first 57-byte is R, second 57-byte is S.
+ *
+ * @param[in]  ecd_context   Pointer to Ed context for signature verification.
+ * @param[in]  hash_nid      hash NID
+ * @param[in]  context       The EDDSA signing context.
+ * @param[in]  context_size  Size of EDDSA signing context.
+ * @param[in]  message       Pointer to octet message to be checked (before hash).
+ * @param[in]  size          Size of the message in bytes.
+ * @param[in]  signature     Pointer to Ed-DSA signature to be verified.
+ * @param[in]  sig_size      Size of signature in bytes.
+ *
+ * @retval  true   Valid signature encoded in Ed-DSA.
+ * @retval  false  Invalid signature or invalid Ed context.
+ **/
+extern bool libspdm_eddsa_verify(const void *ecd_context, size_t hash_nid,
+                                 const uint8_t *context, size_t context_size,
+                                 const uint8_t *message, size_t size,
+                                 const uint8_t *signature, size_t sig_size);
+#endif /* (LIBSPDM_EDDSA_ED25519_SUPPORT) || (LIBSPDM_EDDSA_ED448_SUPPORT) */
+#endif /* CRYPTLIB_ECD_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_hash.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_hash.h
@@ -0,0 +1,772 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_HASH_H
+#define CRYPTLIB_HASH_H
+
+/* SHA-256 digest size in bytes. */
+#define LIBSPDM_SHA256_DIGEST_SIZE 32
+
+/* SHA-384 digest size in bytes. */
+#define LIBSPDM_SHA384_DIGEST_SIZE 48
+
+/* SHA-512 digest size in bytes. */
+#define LIBSPDM_SHA512_DIGEST_SIZE 64
+
+/* SHA3-256 digest size in bytes. */
+#define LIBSPDM_SHA3_256_DIGEST_SIZE 32
+
+/* SHA3-384 digest size in bytes. */
+#define LIBSPDM_SHA3_384_DIGEST_SIZE 48
+
+/* SHA3-512 digest size in bytes. */
+#define LIBSPDM_SHA3_512_DIGEST_SIZE 64
+
+/* SM3_256 digest size in bytes. */
+#define LIBSPDM_SM3_256_DIGEST_SIZE 32
+
+/*=====================================================================================
+ *    One-way cryptographic hash SHA2 primitives.
+ *=====================================================================================
+ */
+#if LIBSPDM_SHA256_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA-256 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, sha256_new() returns NULL. *
+ **/
+extern void *libspdm_sha256_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha256_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha256_free(void *sha256_context);
+
+/**
+ * Initializes user-supplied memory pointed to by sha256_context as SHA-256 hash context for
+ * subsequent use.
+ *
+ * If sha256_context is NULL, then return false.
+ *
+ * @param[out]  sha256_context  Pointer to SHA-256 context being initialized.
+ *
+ * @retval true   SHA-256 context initialization succeeded.
+ * @retval false  SHA-256 context initialization failed.
+ **/
+extern bool libspdm_sha256_init(void *sha256_context);
+
+/**
+ * Makes a copy of an existing SHA-256 context.
+ *
+ * If sha256_context is NULL, then return false.
+ * If new_sha256_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha256_context      Pointer to SHA-256 context being copied.
+ * @param[out] new_sha256_context  Pointer to new SHA-256 context.
+ *
+ * @retval true   SHA-256 context copy succeeded.
+ * @retval false  SHA-256 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha256_duplicate(const void *sha256_context, void *new_sha256_context);
+
+/**
+ * Digests the input data and updates SHA-256 context.
+ *
+ * This function performs SHA-256 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA-256 context should be already correctly initialized by libspdm_sha256_init(), and must not
+ * have been finalized by libspdm_sha256_final(). Behavior with invalid context is undefined.
+ *
+ * If sha256_context is NULL, then return false.
+ *
+ * @param[in, out]  sha256_context  Pointer to the SHA-256 context.
+ * @param[in]       data            Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size       Size of data buffer in bytes.
+ *
+ * @retval true   SHA-256 data digest succeeded.
+ * @retval false  SHA-256 data digest failed.
+ **/
+extern bool libspdm_sha256_update(void *sha256_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA-256 digest value.
+ *
+ * This function completes SHA-256 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA-256 context cannot
+ * be used again. SHA-256 context should be already correctly initialized by libspdm_sha256_init(),
+ * and must not have been finalized by libspdm_sha256_final(). Behavior with invalid SHA-256 context
+ * is undefined.
+ *
+ * If sha256_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha256_context  Pointer to the SHA-256 context.
+ * @param[out]      hash_value      Pointer to a buffer that receives the SHA-256 digest
+ *                                  value (32 bytes).
+ *
+ * @retval true   SHA-256 digest computation succeeded.
+ * @retval false  SHA-256 digest computation failed.
+ **/
+extern bool libspdm_sha256_final(void *sha256_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA-256 message digest of an input data buffer.
+ *
+ * This function performs the SHA-256 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA-256 digest value (32 bytes).
+ *
+ * @retval true   SHA-256 digest computation succeeded.
+ * @retval false  SHA-256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha256_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA256_SUPPORT */
+
+#if LIBSPDM_SHA384_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA-384 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sha384_new() returns NULL.
+ **/
+extern void *libspdm_sha384_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha384_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha384_free(void *sha384_context);
+
+/**
+ * Initializes user-supplied memory pointed to by sha384_context as SHA-384 hash context for
+ * subsequent use.
+ *
+ * If sha384_context is NULL, then return false.
+ *
+ * @param[out]  sha384_context  Pointer to SHA-384 context being initialized.
+ *
+ * @retval true   SHA-384 context initialization succeeded.
+ * @retval false  SHA-384 context initialization failed.
+ **/
+extern bool libspdm_sha384_init(void *sha384_context);
+
+/**
+ * Makes a copy of an existing SHA-384 context.
+ *
+ * If sha384_context is NULL, then return false.
+ * If new_sha384_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha384_context      Pointer to SHA-384 context being copied.
+ * @param[out] new_sha384_context  Pointer to new SHA-384 context.
+ *
+ * @retval true   SHA-384 context copy succeeded.
+ * @retval false  SHA-384 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha384_duplicate(const void *sha384_context, void *new_sha384_context);
+
+/**
+ * Digests the input data and updates SHA-384 context.
+ *
+ * This function performs SHA-384 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA-384 context should be already correctly initialized by libspdm_sha384_init(), and must not
+ * have been finalized by libspdm_sha384_final(). Behavior with invalid context is undefined.
+ *
+ * If sha384_context is NULL, then return false.
+ *
+ * @param[in, out]  sha384_context  Pointer to the SHA-384 context.
+ * @param[in]       data            Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size       Size of data buffer in bytes.
+ *
+ * @retval true   SHA-384 data digest succeeded.
+ * @retval false  SHA-384 data digest failed.
+ **/
+extern bool libspdm_sha384_update(void *sha384_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA-384 digest value.
+ *
+ * This function completes SHA-384 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA-384 context cannot
+ * be used again. SHA-384 context should be already correctly initialized by libspdm_sha384_init(),
+ * and must not have been finalized by libspdm_sha384_final(). Behavior with invalid SHA-384 context
+ * is undefined.
+ *
+ * If sha384_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha384_context  Pointer to the SHA-384 context.
+ * @param[out]      hash_value      Pointer to a buffer that receives the SHA-384 digest
+ *                                  value (48 bytes).
+ *
+ * @retval true   SHA-384 digest computation succeeded.
+ * @retval false  SHA-384 digest computation failed.
+ **/
+extern bool libspdm_sha384_final(void *sha384_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA-384 message digest of an input data buffer.
+ *
+ * This function performs the SHA-384 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA-384 digest value (48 bytes).
+ *
+ * @retval true   SHA-384 digest computation succeeded.
+ * @retval false  SHA-384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha384_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA384_SUPPORT */
+
+#if LIBSPDM_SHA512_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA-512 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sha512_new() returns NULL.
+ **/
+extern void *libspdm_sha512_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha512_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha512_free(void *sha512_context);
+
+/**
+ * Initializes user-supplied memory pointed by sha512_context as SHA-512 hash context for
+ * subsequent use.
+ *
+ * If sha512_context is NULL, then return false.
+ *
+ * @param[out]  sha512_context  Pointer to SHA-512 context being initialized.
+ *
+ * @retval true   SHA-512 context initialization succeeded.
+ * @retval false  SHA-512 context initialization failed.
+ **/
+extern bool libspdm_sha512_init(void *sha512_context);
+
+/**
+ * Makes a copy of an existing SHA-512 context.
+ *
+ * If sha512_context is NULL, then return false.
+ * If new_sha512_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha512_context      Pointer to SHA-512 context being copied.
+ * @param[out] new_sha512_context  Pointer to new SHA-512 context.
+ *
+ * @retval true   SHA-512 context copy succeeded.
+ * @retval false  SHA-512 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha512_duplicate(const void *sha512_context, void *new_sha512_context);
+
+/**
+ * Digests the input data and updates SHA-512 context.
+ *
+ * This function performs SHA-512 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA-512 context should be already correctly initialized by libspdm_sha512_init(), and must not
+ * have been finalized by libspdm_sha512_final(). Behavior with invalid context is undefined.
+ *
+ * If sha512_context is NULL, then return false.
+ *
+ * @param[in, out]  sha512_context  Pointer to the SHA-512 context.
+ * @param[in]       data            Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size       Size of data buffer in bytes.
+ *
+ * @retval true   SHA-512 data digest succeeded.
+ * @retval false  SHA-512 data digest failed.
+ **/
+extern bool libspdm_sha512_update(void *sha512_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA-512 digest value.
+ *
+ * This function completes SHA-512 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA-512 context cannot
+ * be used again. SHA-512 context should be already correctly initialized by libspdm_sha512_init(),
+ * and must not have been finalized by libspdm_sha512_final(). Behavior with invalid SHA-512 context
+ * is undefined.
+ *
+ * If sha512_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha512_context  Pointer to the SHA-512 context.
+ * @param[out]      hash_value      Pointer to a buffer that receives the SHA-512 digest
+ *                                value (64 bytes).
+ *
+ * @retval true   SHA-512 digest computation succeeded.
+ * @retval false  SHA-512 digest computation failed.
+ **/
+extern bool libspdm_sha512_final(void *sha512_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA-512 message digest of an input data buffer.
+ *
+ * This function performs the SHA-512 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA-512 digest value (64 bytes).
+ *
+ * @retval true   SHA-512 digest computation succeeded.
+ * @retval false  SHA-512 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha512_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA512_SUPPORT */
+
+/*=====================================================================================
+ *    One-way cryptographic hash SHA3 primitives.
+ *=====================================================================================
+ */
+#if LIBSPDM_SHA3_256_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA3-256 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sha3_256_new() returns NULL.
+ **/
+extern void *libspdm_sha3_256_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha3_256_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha3_256_free(void *sha3_256_context);
+
+/**
+ * Initializes user-supplied memory pointed by sha3_256_context as SHA3-256 hash context for
+ * subsequent use.
+ *
+ * If sha3_256_context is NULL, then return false.
+ *
+ * @param[out]  sha3_256_context  Pointer to SHA3-256 context being initialized.
+ *
+ * @retval true   SHA3-256 context initialization succeeded.
+ * @retval false  SHA3-256 context initialization failed.
+ **/
+extern bool libspdm_sha3_256_init(void *sha3_256_context);
+
+/**
+ * Makes a copy of an existing SHA3-256 context.
+ *
+ * If sha3_256_context is NULL, then return false.
+ * If new_sha3_256_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha3_256_context      Pointer to SHA3-256 context being copied.
+ * @param[out] new_sha3_256_context  Pointer to new SHA3-256 context.
+ *
+ * @retval true   SHA3-256 context copy succeeded.
+ * @retval false  SHA3-256 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha3_256_duplicate(const void *sha3_256_context, void *new_sha3_256_context);
+
+/**
+ * Digests the input data and updates SHA3-256 context.
+ *
+ * This function performs SHA3-256 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA3-256 context should be already correctly initialized by libspdm_sha3_256_init(), and must not
+ * have been finalized by libspdm_sha3_256_final(). Behavior with invalid context is undefined.
+ *
+ * If sha3_256_context is NULL, then return false.
+ *
+ * @param[in, out]  sha3_256_context  Pointer to the SHA3-256 context.
+ * @param[in]       data              Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size       size of data buffer in bytes.
+ *
+ * @retval true   SHA3-256 data digest succeeded.
+ * @retval false  SHA3-256 data digest failed.
+ **/
+extern bool libspdm_sha3_256_update(void *sha3_256_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA3-256 digest value.
+ *
+ * This function completes SHA3-256 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA3-512 context cannot
+ * be used again. SHA3-256 context should be already correctly initialized by
+ * libspdm_sha3_256_init(), and must not have been finalized by libspdm_sha3_256_final().
+ * Behavior with invalid SHA3-256 context is undefined.
+ *
+ * If sha3_256_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha3_256_context  Pointer to the SHA3-256 context.
+ * @param[out]      hash_value        Pointer to a buffer that receives the SHA3-256 digest
+ *                                    value (32 bytes).
+ *
+ * @retval true   SHA3-256 digest computation succeeded.
+ * @retval false  SHA3-256 digest computation failed.
+ **/
+extern bool libspdm_sha3_256_final(void *sha3_256_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA3-256 message digest of an input data buffer.
+ *
+ * This function performs the SHA3-256 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA3-256 digest value (32 bytes).
+ *
+ * @retval true   SHA3-256 digest computation succeeded.
+ * @retval false  SHA3-256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha3_256_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA3_256_SUPPORT */
+
+#if LIBSPDM_SHA3_384_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA3-384 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sha3_384_new() returns NULL.
+ **/
+extern void *libspdm_sha3_384_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha3_384_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha3_384_free(void *sha3_384_context);
+
+/**
+ * Initializes user-supplied memory pointed by sha3_384_context as SHA3-384 hash context for
+ * subsequent use.
+ *
+ * If sha3_384_context is NULL, then return false.
+ *
+ * @param[out]  sha3_384_context  Pointer to SHA3-384 context being initialized.
+ *
+ * @retval true   SHA3-384 context initialization succeeded.
+ * @retval false  SHA3-384 context initialization failed.
+ **/
+extern bool libspdm_sha3_384_init(void *sha3_384_context);
+
+/**
+ * Makes a copy of an existing SHA3-384 context.
+ *
+ * If sha3_384_context is NULL, then return false.
+ * If new_sha3_384_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha3_384_context      Pointer to SHA3-384 context being copied.
+ * @param[out] new_sha3_384_context  Pointer to new SHA3-384 context.
+ *
+ * @retval true   SHA3-384 context copy succeeded.
+ * @retval false  SHA3-384 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha3_384_duplicate(const void *sha3_384_context, void *new_sha3_384_context);
+
+/**
+ * Digests the input data and updates SHA3-384 context.
+ *
+ * This function performs SHA3-384 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA3-384 context should be already correctly initialized by libspdm_sha3_384_init(), and must not
+ * have been finalized by libspdm_sha3_384_final(). Behavior with invalid context is undefined.
+ *
+ * If sha3_384_context is NULL, then return false.
+ *
+ * @param[in, out]  sha3_384_context  Pointer to the SHA3-384 context.
+ * @param[in]       data              Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size         Size of data buffer in bytes.
+ *
+ * @retval true   SHA3-384 data digest succeeded.
+ * @retval false  SHA3-384 data digest failed.
+ **/
+extern bool libspdm_sha3_384_update(void *sha3_384_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA3-384 digest value.
+ *
+ * This function completes SHA3-384 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA3-384 context cannot
+ * be used again. SHA3-384 context should be already correctly initialized by
+ * libspdm_sha3_384_init(), and must not have been finalized by libspdm_sha3_384_final().
+ * Behavior with invalid SHA3-384 context is undefined.
+ *
+ * If sha3_384_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha3_384_context  Pointer to the SHA3-384 context.
+ * @param[out]      hash_value        Pointer to a buffer that receives the SHA3-384 digest
+ *                                    value (48 bytes).
+ *
+ * @retval true   SHA3-384 digest computation succeeded.
+ * @retval false  SHA3-384 digest computation failed.
+ *
+ **/
+extern bool libspdm_sha3_384_final(void *sha3_384_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA3-384 message digest of an input data buffer.
+ *
+ * This function performs the SHA3-384 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA3-384 digest value (48 bytes).
+ *
+ * @retval true   SHA3-384 digest computation succeeded.
+ * @retval false  SHA3-384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha3_384_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA3_384_SUPPORT */
+
+#if LIBSPDM_SHA3_512_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SHA3-512 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sha3_512_new() returns NULL.
+ **/
+extern void *libspdm_sha3_512_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sha3_512_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sha3_512_free(void *sha3_512_context);
+
+/**
+ * Initializes user-supplied memory pointed by sha3_512_context as SHA3-512 hash context for
+ * subsequent use.
+ *
+ * If sha3_512_context is NULL, then return false.
+ *
+ * @param[out]  sha3_512_context  Pointer to SHA3-512 context being initialized.
+ *
+ * @retval true   SHA3-512 context initialization succeeded.
+ * @retval false  SHA3-512 context initialization failed.
+ **/
+extern bool libspdm_sha3_512_init(void *sha3_512_context);
+
+/**
+ * Makes a copy of an existing SHA3-512 context.
+ *
+ * If sha3_512_context is NULL, then return false.
+ * If new_sha3_512_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sha3_512_context      Pointer to SHA3-512 context being copied.
+ * @param[out] new_sha3_512_context  Pointer to new SHA3-512 context.
+ *
+ * @retval true   SHA3-512 context copy succeeded.
+ * @retval false  SHA3-512 context copy failed.
+ * @retval false  This interface is not supported.
+ *
+ **/
+extern bool libspdm_sha3_512_duplicate(const void *sha3_512_context, void *new_sha3_512_context);
+
+/**
+ * Digests the input data and updates SHA3-512 context.
+ *
+ * This function performs SHA3-512 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SHA3-512 context should be already correctly initialized by libspdm_sha3_512_init(), and must not
+ * have been finalized by libspdm_sha3_512_final(). Behavior with invalid context is undefined.
+ *
+ * If sha3_512_context is NULL, then return false.
+ *
+ * @param[in, out]  sha3_512_context  Pointer to the SHA3-512 context.
+ * @param[in]       data              Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size         Size of data buffer in bytes.
+ *
+ * @retval true   SHA3-512 data digest succeeded.
+ * @retval false  SHA3-512 data digest failed.
+ **/
+extern bool libspdm_sha3_512_update(void *sha3_512_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SHA3-512 digest value.
+ *
+ * This function completes SHA3-512 hash computation and populates the digest value into
+ * the specified memory. After this function has been called, the SHA3-512 context cannot
+ * be used again. SHA3-512 context should be already correctly initialized by
+ * libspdm_sha3_512_init(), and must not have been finalized by libspdm_sha3_512_final().
+ * Behavior with invalid SHA3-512 context is undefined.
+ *
+ * If sha3_512_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sha3_512_context  Pointer to the SHA3-512 context.
+ * @param[out]      hash_value        Pointer to a buffer that receives the SHA3-512 digest
+ *                                    value (64 bytes).
+ *
+ * @retval true   SHA3-512 digest computation succeeded.
+ * @retval false  SHA3-512 digest computation failed.
+ **/
+extern bool libspdm_sha3_512_final(void *sha3_512_context, uint8_t *hash_value);
+
+/**
+ * Computes the SHA3-512 message digest of an input data buffer.
+ *
+ * This function performs the SHA3-512 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SHA3-512 digest value (64 bytes).
+ *
+ * @retval true   SHA3-512 digest computation succeeded.
+ * @retval false  SHA3-512 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sha3_512_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SHA3_512_SUPPORT */
+
+/*=====================================================================================
+ *    One-Way Cryptographic hash SM3 Primitives
+ *=====================================================================================
+ */
+
+#if LIBSPDM_SM3_256_SUPPORT
+/**
+ * Allocates and initializes one HASH_CTX context for subsequent SM3-256 use.
+ *
+ * @return  Pointer to the HASH_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_sm3_256_new() returns NULL.
+ **/
+extern void *libspdm_sm3_256_new(void);
+
+/**
+ * Release the specified HASH_CTX context.
+ *
+ * @param[in]  sm3_context  Pointer to the HASH_CTX context to be released.
+ **/
+extern void libspdm_sm3_256_free(void *sm3_context);
+
+/**
+ * Initializes user-supplied memory pointed by sm3_context as SM3 hash context for
+ * subsequent use.
+ *
+ * If sm3_context is NULL, then return false.
+ *
+ * @param[out]  sm3_context  Pointer to SM3 context being initialized.
+ *
+ * @retval true   SM3 context initialization succeeded.
+ * @retval false  SM3 context initialization failed.
+ **/
+extern bool libspdm_sm3_256_init(void *sm3_context);
+
+/**
+ * Makes a copy of an existing SM3 context.
+ *
+ * If sm3_context is NULL, then return false.
+ * If new_sm3_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  sm3_context      Pointer to SM3 context being copied.
+ * @param[out] new_sm3_context  Pointer to new SM3 context.
+ *
+ * @retval true   SM3 context copy succeeded.
+ * @retval false  SM3 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sm3_256_duplicate(const void *sm3_context, void *new_sm3_context);
+
+/**
+ * Digests the input data and updates SM3 context.
+ *
+ * This function performs SM3 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * SM3 context should be already correctly initialized by sm3_init(), and should not be finalized
+ * by sm3_final(). Behavior with invalid context is undefined.
+ *
+ * If sm3_context is NULL, then return false.
+ *
+ * @param[in, out]  sm3_context  Pointer to the SM3 context.
+ * @param[in]       data         Pointer to the buffer containing the data to be hashed.
+ * @param[in]       data_size    Size of data buffer in bytes.
+ *
+ * @retval true   SM3 data digest succeeded.
+ * @retval false  SM3 data digest failed.
+ **/
+extern bool libspdm_sm3_256_update(void *sm3_context, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the SM3 digest value.
+ *
+ * This function completes SM3 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the SM3 context cannot
+ * be used again. SM3 context should be already correctly initialized by sm3_init(), and should not
+ * be finalized by sm3_final(). Behavior with invalid SM3 context is undefined.
+ *
+ * If sm3_context is NULL, then return false.
+ * If hash_value is NULL, then return false.
+ *
+ * @param[in, out]  sm3_context  Pointer to the SM3 context.
+ * @param[out]      hash_value   Pointer to a buffer that receives the SM3 digest value (32 bytes).
+ *
+ * @retval true   SM3 digest computation succeeded.
+ * @retval false  SM3 digest computation failed.
+ **/
+extern bool libspdm_sm3_256_final(void *sm3_context, uint8_t *hash_value);
+
+/**
+ * Computes the SM3 message digest of an input data buffer.
+ *
+ * This function performs the SM3 message digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be hashed.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[out]  hash_value  Pointer to a buffer that receives the SM3 digest value (32 bytes).
+ *
+ * @retval true   SM3 digest computation succeeded.
+ * @retval false  SM3 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_sm3_256_hash_all(const void *data, size_t data_size, uint8_t *hash_value);
+#endif /* LIBSPDM_SM3_256_SUPPORT */
+
+#endif /* CRYPTLIB_HASH_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_hkdf.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_hkdf.h
@@ -0,0 +1,266 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_HKDF_H
+#define CRYPTLIB_HKDF_H
+
+/*=====================================================================================
+ *    Key Derivation Function Primitives
+ *=====================================================================================*/
+
+#if LIBSPDM_SHA256_SUPPORT
+/**
+ * Derive SHA-256 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive prk value.
+ * @param[in]   prk_out_size  Size of prk bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha256_extract(const uint8_t *key, size_t key_size,
+                                        const uint8_t *salt, size_t salt_size,
+                                        uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA256 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha256_expand(const uint8_t *prk, size_t prk_size,
+                                       const uint8_t *info, size_t info_size,
+                                       uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA256_SUPPORT */
+
+#if LIBSPDM_SHA384_SUPPORT
+/**
+ * Derive SHA384 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha384_extract(const uint8_t *key, size_t key_size,
+                                        const uint8_t *salt, size_t salt_size,
+                                        uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA384 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha384_expand(const uint8_t *prk, size_t prk_size,
+                                       const uint8_t *info, size_t info_size,
+                                       uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA384_SUPPORT */
+
+#if LIBSPDM_SHA512_SUPPORT
+/**
+ * Derive SHA512 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha512_extract(const uint8_t *key, size_t key_size,
+                                        const uint8_t *salt, size_t salt_size,
+                                        uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA512 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha512_expand(const uint8_t *prk, size_t prk_size,
+                                       const uint8_t *info, size_t info_size,
+                                       uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA512_SUPPORT */
+
+#if LIBSPDM_SHA3_256_SUPPORT
+/**
+ * Derive SHA3_256 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_256_extract(const uint8_t *key, size_t key_size,
+                                          const uint8_t *salt, size_t salt_size,
+                                          uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA3_256 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_256_expand(const uint8_t *prk, size_t prk_size,
+                                         const uint8_t *info, size_t info_size,
+                                         uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA3_256_SUPPORT */
+
+#if LIBSPDM_SHA3_384_SUPPORT
+/**
+ * Derive SHA3_384 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_384_extract(const uint8_t *key, size_t key_size,
+                                          const uint8_t *salt, size_t salt_size,
+                                          uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA3_384 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_384_expand(const uint8_t *prk, size_t prk_size,
+                                         const uint8_t *info, size_t info_size,
+                                         uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA3_384_SUPPORT */
+
+#if LIBSPDM_SHA3_512_SUPPORT
+/**
+ * Derive SHA3_512 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_512_extract(const uint8_t *key, size_t key_size,
+                                          const uint8_t *salt, size_t salt_size,
+                                          uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SHA3_512 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sha3_512_expand(const uint8_t *prk, size_t prk_size,
+                                         const uint8_t *info, size_t info_size,
+                                         uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SHA3_512_SUPPORT */
+
+#if LIBSPDM_SM3_256_SUPPORT
+/**
+ * Derive SM3_256 HMAC-based Extract key Derivation Function (HKDF).
+ *
+ * @param[in]   key           Pointer to the user-supplied key.
+ * @param[in]   key_size      Key size in bytes.
+ * @param[in]   salt          Pointer to the salt value.
+ * @param[in]   salt_size     Salt size in bytes.
+ * @param[out]  prk_out       Pointer to buffer to receive hkdf value.
+ * @param[in]   prk_out_size  Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sm3_256_extract(const uint8_t *key, size_t key_size,
+                                         const uint8_t *salt, size_t salt_size,
+                                         uint8_t *prk_out, size_t prk_out_size);
+
+/**
+ * Derive SM3_256 HMAC-based Expand key Derivation Function (HKDF).
+ *
+ * @param[in]   prk        Pointer to the user-supplied key.
+ * @param[in]   prk_size   Key size in bytes.
+ * @param[in]   info       Pointer to the application specific info.
+ * @param[in]   info_size  Info size in bytes.
+ * @param[out]  out        Pointer to buffer to receive hkdf value.
+ * @param[in]   out_size   Size of hkdf bytes to generate.
+ *
+ * @retval true   Hkdf generated successfully.
+ * @retval false  Hkdf generation failed.
+ **/
+extern bool libspdm_hkdf_sm3_256_expand(const uint8_t *prk, size_t prk_size,
+                                        const uint8_t *info, size_t info_size,
+                                        uint8_t *out, size_t out_size);
+#endif /* LIBSPDM_SM3_256_SUPPORT */
+
+#endif /* CRYPTLIB_HKDF_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_mac.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_mac.h
@@ -0,0 +1,833 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_MAC_H
+#define CRYPTLIB_MAC_H
+
+/*=====================================================================================
+ *    Message Authentication Code (MAC) Primitives
+ *=====================================================================================
+ */
+
+#if LIBSPDM_SHA256_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA256 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha256_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha256_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha256_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha256_free(void *hmac_sha256_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha256_update().
+ *
+ * If hmac_sha256_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  hmac_sha256_ctx  Pointer to HMAC-SHA256 context.
+ * @param[in]   key              Pointer to the user-supplied key.
+ * @param[in]   key_size         Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha256_set_key(void *hmac_sha256_ctx, const uint8_t *key, size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA256 context.
+ *
+ * If hmac_sha256_ctx is NULL, then return false.
+ * If new_hmac_sha256_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  hmac_sha256_ctx      Pointer to HMAC-SHA256 context being copied.
+ * @param[out] new_hmac_sha256_ctx  Pointer to new HMAC-SHA256 context.
+ *
+ * @retval true   HMAC-SHA256 context copy succeeded.
+ * @retval false  HMAC-SHA256 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha256_duplicate(const void *hmac_sha256_ctx, void *new_hmac_sha256_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA256 context.
+ *
+ * This function performs HMAC-SHA256 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA256 context should be initialized by libspdm_hmac_sha256_new(), and should not be
+ * finalized by libspdm_hmac_sha256_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha256_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha256_ctx Pointer to the HMAC-SHA256 context.
+ * @param[in]       data            Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size       Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA256 data digest succeeded.
+ * @retval false  HMAC-SHA256 data digest failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha256_update(void *hmac_sha256_ctx, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA256 digest value.
+ *
+ * This function completes HMAC-SHA256 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA256 context cannot
+ * be used again. HMAC-SHA256 context should be initialized by libspdm_hmac_sha256_new(), and should
+ * not be finalized by libspdm_hmac_sha256_final(). Behavior with invalid HMAC-SHA256 context is
+ * undefined.
+ *
+ * If hmac_sha256_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha256_ctx  Pointer to the HMAC-SHA256 context.
+ * @param[out]      hmac_value       Pointer to a buffer that receives the HMAC-SHA256 digest
+ *                                   value (32 bytes).
+ *
+ * @retval true   HMAC-SHA256 digest computation succeeded.
+ * @retval false  HMAC-SHA256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha256_final(void *hmac_sha256_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA256 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA256 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA256 digest
+ *                          value (32 bytes).
+ *
+ * @retval true   HMAC-SHA256 digest computation succeeded.
+ * @retval false  HMAC-SHA256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha256_all(const void *data, size_t data_size,
+                                    const uint8_t *key, size_t key_size,
+                                    uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA256_SUPPORT */
+
+#if LIBSPDM_SHA384_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA384 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha384_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha384_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha384_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha384_free(void *hmac_sha384_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha384_update().
+ *
+ * If hmac_sha384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  hmac_sha384_ctx  Pointer to HMAC-SHA384 context.
+ * @param[in]   key                Pointer to the user-supplied key.
+ * @param[in]   key_size            key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha384_set_key(void *hmac_sha384_ctx, const uint8_t *key, size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA384 context.
+ *
+ * If hmac_sha384_ctx is NULL, then return false.
+ * If new_hmac_sha384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  hmac_sha384_ctx      Pointer to HMAC-SHA384 context being copied.
+ * @param[out] new_hmac_sha384_ctx  Pointer to new HMAC-SHA384 context.
+ *
+ * @retval true   HMAC-SHA384 context copy succeeded.
+ * @retval false  HMAC-SHA384 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha384_duplicate(const void *hmac_sha384_ctx, void *new_hmac_sha384_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA384 context.
+ *
+ * This function performs HMAC-SHA384 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA384 context should be initialized by libspdm_hmac_sha384_new(), and should not be
+ * finalized by libspdm_hmac_sha384_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha384_ctx  Pointer to the HMAC-SHA384 context.
+ * @param[in]       data             Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size        Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA384 data digest succeeded.
+ * @retval false  HMAC-SHA384 data digest failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha384_update(void *hmac_sha384_ctx, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA384 digest value.
+ *
+ * This function completes HMAC-SHA384 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA384 context cannot
+ * be used again. HMAC-SHA384 context should be initialized by libspdm_hmac_sha384_new(), and should
+ * not be finalized by libspdm_hmac_sha384_final(). Behavior with invalid HMAC-SHA384 context is
+ * undefined.
+ *
+ * If hmac_sha384_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha384_ctx  Pointer to the HMAC-SHA384 context.
+ * @param[out]      hmac_value       Pointer to a buffer that receives the HMAC-SHA384 digest
+ *                                   value (48 bytes).
+ *
+ * @retval true   HMAC-SHA384 digest computation succeeded.
+ * @retval false  HMAC-SHA384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha384_final(void *hmac_sha384_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA384 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA384 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA384 digest
+ *                          value (48 bytes).
+ *
+ * @retval true   HMAC-SHA384 digest computation succeeded.
+ * @retval false  HMAC-SHA384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha384_all(const void *data, size_t data_size,
+                                    const uint8_t *key, size_t key_size,
+                                    uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA384_SUPPORT */
+
+#if LIBSPDM_SHA512_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA512 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha512_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha512_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha512_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha512_free(void *hmac_sha512_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha512_update().
+ *
+ * If hmac_sha512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  hmac_sha512_ctx  Pointer to HMAC-SHA512 context.
+ * @param[in]   key              Pointer to the user-supplied key.
+ * @param[in]   key_size         Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha512_set_key(void *hmac_sha512_ctx, const uint8_t *key, size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA512 context.
+ *
+ * If hmac_sha512_ctx is NULL, then return false.
+ * If new_hmac_sha512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  hmac_sha512_ctx      Pointer to HMAC-SHA512 context being copied.
+ * @param[out] new_hmac_sha512_ctx  Pointer to new HMAC-SHA512 context.
+ *
+ * @retval true   HMAC-SHA512 context copy succeeded.
+ * @retval false  HMAC-SHA512 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha512_duplicate(const void *hmac_sha512_ctx, void *new_hmac_sha512_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA512 context.
+ *
+ * This function performs HMAC-SHA512 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA512 context should be initialized by libspdm_hmac_sha512_new(), and should not be
+ * finalized by libspdm_hmac_sha512_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha512_ctx  Pointer to the HMAC-SHA512 context.
+ * @param[in]       data             Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size        Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA512 data digest succeeded.
+ * @retval false  HMAC-SHA512 data digest failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha512_update(void *hmac_sha512_ctx, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA512 digest value.
+ *
+ * This function completes HMAC-SHA512 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA512 context cannot
+ * be used again. HMAC-SHA512 context should be initialized by libspdm_hmac_sha512_new(), and should
+ * not be finalized by libspdm_hmac_sha512_final(). Behavior with invalid HMAC-SHA512 context is
+ * undefined.
+ *
+ * If hmac_sha512_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha512_ctx  Pointer to the HMAC-SHA512 context.
+ * @param[out]      hmac_value       Pointer to a buffer that receives the HMAC-SHA512 digest
+ *                                   value (64 bytes).
+ *
+ * @retval true   HMAC-SHA512 digest computation succeeded.
+ * @retval false  HMAC-SHA512 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha512_final(void *hmac_sha512_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA512 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA512 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA512 digest
+ *                          value (64 bytes).
+ *
+ * @retval true   HMAC-SHA512 digest computation succeeded.
+ * @retval false  HMAC-SHA512 digest computation failed.
+ * @retval false  This interface is not supported.
+ *
+ **/
+extern bool libspdm_hmac_sha512_all(const void *data, size_t data_size,
+                                    const uint8_t *key, size_t key_size,
+                                    uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA512_SUPPORT */
+
+#if LIBSPDM_SHA3_256_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA3-256 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha3_256_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha3_256_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha3_256_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha3_256_free(void *hmac_sha3_256_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha3_256_update().
+ *
+ * If hmac_sha3_256_ctx is NULL, then return false.
+ *
+ * @param[out]  hmac_sha3_256_ctx  Pointer to HMAC-SHA3-256 context.
+ * @param[in]   key                Pointer to the user-supplied key.
+ * @param[in]   key_size           Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ **/
+extern bool libspdm_hmac_sha3_256_set_key(void *hmac_sha3_256_ctx,
+                                          const uint8_t *key,
+                                          size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA3-256 context.
+ *
+ * If hmac_sha3_256_ctx is NULL, then return false.
+ * If new_hmac_sha3_256_ctx is NULL, then return false.
+ *
+ * @param[in]  hmac_sha3_256_ctx      Pointer to HMAC-SHA3-256 context being copied.
+ * @param[out] new_hmac_sha3_256_ctx  Pointer to new HMAC-SHA3-256 context.
+ *
+ * @retval true   HMAC-SHA3-256 context copy succeeded.
+ * @retval false  HMAC-SHA3-256 context copy failed.
+ **/
+extern bool libspdm_hmac_sha3_256_duplicate(const void *hmac_sha3_256_ctx,
+                                            void *new_hmac_sha3_256_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA3-256 context.
+ *
+ * This function performs HMAC-SHA3-256 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA3-256 context should be initialized by libspdm_hmac_sha3_256_new(), and should not be
+ * finalized by libspdm_hmac_sha3_256_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha3_256_ctx is NULL, then return false.
+ *
+ * @param[in, out]  hmac_sha3_256_ctx  Pointer to the HMAC-SHA3-256 context.
+ * @param[in]       data               Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size          Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA3-256 data digest succeeded.
+ * @retval false  HMAC-SHA3-256 data digest failed.
+ **/
+extern bool libspdm_hmac_sha3_256_update(void *hmac_sha3_256_ctx,
+                                         const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA3-256 digest value.
+ *
+ * This function completes HMAC-SHA3-256 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA3-256 context cannot
+ * be used again. HMAC-SHA3-256 context should be initialized by libspdm_hmac_sha3_256_new(), and
+ * should not be finalized by libspdm_hmac_sha3_256_final(). Behavior with invalid HMAC-SHA3-256
+ * context is undefined.
+ *
+ * If hmac_sha3_256_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ *
+ * @param[in, out]  hmac_sha3_256_ctx  Pointer to the HMAC-SHA3-256 context.
+ * @param[out]      hmac_value          Pointer to a buffer that receives the HMAC-SHA3-256 digest
+ *                                    value (32 bytes).
+ *
+ * @retval true   HMAC-SHA3-256 digest computation succeeded.
+ * @retval false  HMAC-SHA3-256 digest computation failed.
+ **/
+extern bool libspdm_hmac_sha3_256_final(void *hmac_sha3_256_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA3-256 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA3-256 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA3-256 digest
+ *                          value (32 bytes).
+ *
+ * @retval true   HMAC-SHA3-256 digest computation succeeded.
+ * @retval false  HMAC-SHA3-256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_256_all(const void *data, size_t data_size,
+                                      const uint8_t *key, size_t key_size,
+                                      uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA3_256_SUPPORT */
+
+#if LIBSPDM_SHA3_384_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA3-384 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha3_384_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha3_384_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha3_384_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha3_384_free(void *hmac_sha3_384_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha3_384_update().
+ *
+ * If hmac_sha3_384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  hmac_sha3_384_ctx  Pointer to HMAC-SHA3-384 context.
+ * @param[in]   key                Pointer to the user-supplied key.
+ * @param[in]   key_size           Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_384_set_key(void *hmac_sha3_384_ctx,
+                                          const uint8_t *key,
+                                          size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA3-384 context.
+ *
+ * If hmac_sha3_384_ctx is NULL, then return false.
+ * If new_hmac_sha3_384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  hmac_sha3_384_ctx      Pointer to HMAC-SHA3-384 context being copied.
+ * @param[out] new_hmac_sha3_384_ctx  Pointer to new HMAC-SHA3-384 context.
+ *
+ * @retval true   HMAC-SHA3-384 context copy succeeded.
+ * @retval false  HMAC-SHA3-384 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_384_duplicate(const void *hmac_sha3_384_ctx,
+                                            void *new_hmac_sha3_384_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA3-384 context.
+ *
+ * This function performs HMAC-SHA3-384 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA3-384 context should be initialized by libspdm_hmac_sha3_384_new(), and should not be
+ * finalized by libspdm_hmac_sha3_384_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha3_384_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha3_384_ctx  Pointer to the HMAC-SHA3-384 context.
+ * @param[in]       data               Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size          Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA3-384 data digest succeeded.
+ * @retval false  HMAC-SHA3-384 data digest failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_384_update(void *hmac_sha3_384_ctx, const void *data,
+                                         size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA3-384 digest value.
+ *
+ * This function completes HMAC-SHA3-384 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA3-384 context cannot
+ * be used again. HMAC-SHA3-384 context should be initialized by libspdm_hmac_sha3_384_new(), and
+ * should not be finalized by libspdm_hmac_sha3_384_final(). Behavior with invalid HMAC-SHA3-384
+ * context is undefined.
+ *
+ * If hmac_sha3_384_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha3_384_ctx  Pointer to the HMAC-SHA3-384 context.
+ * @param[out]      hmac_value         Pointer to a buffer that receives the HMAC-SHA3-384 digest
+ *                                     value (48 bytes).
+ *
+ * @retval true   HMAC-SHA3-384 digest computation succeeded.
+ * @retval false  HMAC-SHA3-384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_384_final(void *hmac_sha3_384_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA3-384 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA3-384 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA3-384 digest
+ *                          value (48 bytes).
+ *
+ * @retval true   HMAC-SHA3-384 digest computation succeeded.
+ * @retval false  HMAC-SHA3-384 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_384_all(const void *data, size_t data_size,
+                                      const uint8_t *key, size_t key_size,
+                                      uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA3_384_SUPPORT */
+
+#if LIBSPDM_SHA3_512_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SHA3-512 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sha3_512_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sha3_512_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sha3_512_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sha3_512_free(void *hmac_sha3_512_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sha3_512_update().
+ *
+ * If hmac_sha3_512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  hmac_sha3_512_ctx  Pointer to HMAC-SHA3-512 context.
+ * @param[in]   key                Pointer to the user-supplied key.
+ * @param[in]   key_size           Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_512_set_key(void *hmac_sha3_512_ctx,
+                                          const uint8_t *key,
+                                          size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SHA3-512 context.
+ *
+ * If hmac_sha3_512_ctx is NULL, then return false.
+ * If new_hmac_sha3_512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  hmac_sha3_512_ctx      Pointer to HMAC-SHA3-512 context being copied.
+ * @param[out] new_hmac_sha3_512_ctx  Pointer to new HMAC-SHA3-512 context.
+ *
+ * @retval true   HMAC-SHA3-512 context copy succeeded.
+ * @retval false  HMAC-SHA3-512 context copy failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_512_duplicate(const void *hmac_sha3_512_ctx,
+                                            void *new_hmac_sha3_512_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SHA3-512 context.
+ *
+ * This function performs HMAC-SHA3-512 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SHA3-512 context should be initialized by libspdm_hmac_sha3_512_new(), and should not be
+ * finalized by libspdm_hmac_sha3_512_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sha3_512_ctx is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha3_512_ctx  Pointer to the HMAC-SHA3-512 context.
+ * @param[in]       data               Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size          Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SHA3-512 data digest succeeded.
+ * @retval false  HMAC-SHA3-512 data digest failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_512_update(void *hmac_sha3_512_ctx,
+                                         const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SHA3-512 digest value.
+ *
+ * This function completes HMAC-SHA3-512 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SHA3-512 context cannot
+ * be used again. HMAC-SHA3-512 context should be initialized by libspdm_hmac_sha3_512_new(), and
+ * should not be finalized by libspdm_hmac_sha3_512_final(). Behavior with invalid HMAC-SHA3-512
+ * context is undefined.
+ *
+ * If hmac_sha3_512_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  hmac_sha3_512_ctx  Pointer to the HMAC-SHA3-512 context.
+ * @param[out]      hmac_value         Pointer to a buffer that receives the HMAC-SHA3-512 digest
+ *                                     value (64 bytes).
+ *
+ * @retval true   HMAC-SHA3-512 digest computation succeeded.
+ * @retval false  HMAC-SHA3-512 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_512_final(void *hmac_sha3_512_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SHA3-512 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SHA3-512 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SHA3-512 digest
+ *                          value (64 bytes).
+ *
+ * @retval true   HMAC-SHA3-512 digest computation succeeded.
+ * @retval false  HMAC-SHA3-512 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sha3_512_all(const void *data, size_t data_size,
+                                      const uint8_t *key, size_t key_size,
+                                      uint8_t *hmac_value);
+#endif /* LIBSPDM_SHA3_512_SUPPORT */
+
+#if LIBSPDM_SM3_256_SUPPORT
+/**
+ * Allocates and initializes one HMAC_CTX context for subsequent HMAC-SM3-256 use.
+ *
+ * @return  Pointer to the HMAC_CTX context that has been initialized.
+ *          If the allocations fails, libspdm_hmac_sm3_256_new() returns NULL.
+ **/
+extern void *libspdm_hmac_sm3_256_new(void);
+
+/**
+ * Release the specified HMAC_CTX context.
+ *
+ * @param[in]  hmac_sm3_256_ctx  Pointer to the HMAC_CTX context to be released.
+ **/
+extern void libspdm_hmac_sm3_256_free(void *hmac_sm3_256_ctx);
+
+/**
+ * Set user-supplied key for subsequent use. It must be done before any
+ * calling to libspdm_hmac_sm3_256_update().
+ *
+ * If hmac_sm3_256_ctx is NULL, then return false.
+ *
+ * @param[out]  hmac_sm3_256_ctx  Pointer to HMAC-SM3-256 context.
+ * @param[in]   key               Pointer to the user-supplied key.
+ * @param[in]   key_size          Key size in bytes.
+ *
+ * @retval true   The key is set successfully.
+ * @retval false  The key is set unsuccessfully.
+ **/
+extern bool libspdm_hmac_sm3_256_set_key(void *hmac_sm3_256_ctx,
+                                         const uint8_t *key, size_t key_size);
+
+/**
+ * Makes a copy of an existing HMAC-SM3-256 context.
+ *
+ * If hmac_sm3_256_ctx is NULL, then return false.
+ * If new_hmac_sm3_256_ctx is NULL, then return false.
+ *
+ * @param[in]  hmac_sm3_256_ctx      Pointer to HMAC-SM3-256 context being copied.
+ * @param[out] new_hmac_sm3_256_ctx  Pointer to new HMAC-SM3-256 context.
+ *
+ * @retval true   HMAC-SM3-256 context copy succeeded.
+ * @retval false  HMAC-SM3-256 context copy failed.
+ **/
+extern bool libspdm_hmac_sm3_256_duplicate(const void *hmac_sm3_256_ctx,
+                                           void *new_hmac_sm3_256_ctx);
+
+/**
+ * Digests the input data and updates HMAC-SM3-256 context.
+ *
+ * This function performs HMAC-SM3-256 digest on a data buffer of the specified size.
+ * It can be called multiple times to compute the digest of long or discontinuous data streams.
+ * HMAC-SM3-256 context should be initialized by libspdm_hmac_sm3_256_new(), and should not be
+ * finalized by libspdm_hmac_sm3_256_final(). Behavior with invalid context is undefined.
+ *
+ * If hmac_sm3_256_ctx is NULL, then return false.
+ *
+ * @param[in, out]  hmac_sm3_256_ctx  Pointer to the HMAC-SM3-256 context.
+ * @param[in]       data              Pointer to the buffer containing the data to be digested.
+ * @param[in]       data_size         Size of data buffer in bytes.
+ *
+ * @retval true   HMAC-SM3-256 data digest succeeded.
+ * @retval false  HMAC-SM3-256 data digest failed.
+ **/
+extern bool libspdm_hmac_sm3_256_update(void *hmac_sm3_256_ctx, const void *data, size_t data_size);
+
+/**
+ * Completes computation of the HMAC-SM3-256 digest value.
+ *
+ * This function completes HMAC-SM3-256 hash computation and retrieves the digest value into
+ * the specified memory. After this function has been called, the HMAC-SM3-256 context cannot
+ * be used again. HMAC-SM3-256 context should be initialized by libspdm_hmac_sm3_256_new(), and
+ * should not be finalized by libspdm_hmac_sm3_256_final(). Behavior with invalid HMAC-SM3-256
+ * context is undefined.
+ *
+ * If hmac_sm3_256_ctx is NULL, then return false.
+ * If hmac_value is NULL, then return false.
+ *
+ * @param[in, out]  hmac_sm3_256_ctx  Pointer to the HMAC-SM3-256 context.
+ * @param[out]      hmac_value        Pointer to a buffer that receives the HMAC-SM3-256 digest
+ *                                    value (32 bytes).
+ *
+ * @retval true   HMAC-SM3-256 digest computation succeeded.
+ * @retval false  HMAC-SM3-256 digest computation failed.
+ **/
+extern bool libspdm_hmac_sm3_256_final(void *hmac_sm3_256_ctx, uint8_t *hmac_value);
+
+/**
+ * Computes the HMAC-SM3-256 digest of a input data buffer.
+ *
+ * This function performs the HMAC-SM3-256 digest of a given data buffer, and places
+ * the digest value into the specified memory.
+ *
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]   data        Pointer to the buffer containing the data to be digested.
+ * @param[in]   data_size   Size of data buffer in bytes.
+ * @param[in]   key         Pointer to the user-supplied key.
+ * @param[in]   key_size    Key size in bytes.
+ * @param[out]  hmac_value  Pointer to a buffer that receives the HMAC-SM3-256 digest
+ *                          value (32 bytes).
+ *
+ * @retval true   HMAC-SM3-256 digest computation succeeded.
+ * @retval false  HMAC-SM3-256 digest computation failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_hmac_sm3_256_all(const void *data, size_t data_size,
+                                     const uint8_t *key, size_t key_size,
+                                     uint8_t *hmac_value);
+#endif /* LIBSPDM_SM3_256_SUPPORT */
+
+#endif /* CRYPTLIB_MAC_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_rng.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_rng.h
@@ -0,0 +1,30 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_RNG_H
+#define CRYPTLIB_RNG_H
+
+/*=====================================================================================
+ *    Random Number Generation Primitive
+ *=====================================================================================*/
+
+/**
+ * Generates a random byte stream of the specified size. If initialization, testing, or seeding of
+ * the (pseudo)random number generator is required it should be done before this function is called.
+ *
+ * If output is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[out]  output  Pointer to buffer to receive random value.
+ * @param[in]   size    Size of random bytes to generate.
+ *
+ * @retval true   Random byte stream generated successfully.
+ * @retval false  Generation of random byte stream failed.
+ * @retval false  This interface is not supported.
+ **/
+extern bool libspdm_random_bytes(uint8_t *output, size_t size);
+
+#endif /* CRYPTLIB_RNG_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_rsa.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_rsa.h
@@ -0,0 +1,264 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_RSA_H
+#define CRYPTLIB_RSA_H
+
+/*=====================================================================================
+ *    RSA Cryptography Primitives
+ *=====================================================================================
+ */
+
+#if (LIBSPDM_RSA_SSA_SUPPORT) || (LIBSPDM_RSA_PSS_SUPPORT)
+/* RSA key Tags Definition used in libspdm_rsa_set_key() function for key component
+ * identification.
+ */
+typedef enum {
+    LIBSPDM_RSA_KEY_N, /*< RSA public Modulus (N)*/
+    LIBSPDM_RSA_KEY_E, /*< RSA public exponent (e)*/
+    LIBSPDM_RSA_KEY_D, /*< RSA Private exponent (d)*/
+    LIBSPDM_RSA_KEY_P, /*< RSA secret prime factor of Modulus (p)*/
+    LIBSPDM_RSA_KEY_Q, /*< RSA secret prime factor of Modules (q)*/
+    LIBSPDM_RSA_KEY_DP, /*< p's CRT exponent (== d mod (p - 1))*/
+    LIBSPDM_RSA_KEY_DQ, /*< q's CRT exponent (== d mod (q - 1))*/
+    LIBSPDM_RSA_KEY_Q_INV /*< The CRT coefficient (== 1/q mod p)*/
+} libspdm_rsa_key_tag_t;
+
+/**
+ * Allocates and initializes one RSA context for subsequent use.
+ *
+ * @return  Pointer to the RSA context that has been initialized.
+ *          If the allocations fails, libspdm_rsa_new() returns NULL.
+ **/
+extern void *libspdm_rsa_new(void);
+
+/**
+ * Release the specified RSA context.
+ *
+ * If rsa_context is NULL, then return false.
+ *
+ * @param[in]  rsa_context  Pointer to the RSA context to be released.
+ **/
+extern void libspdm_rsa_free(void *rsa_context);
+
+/**
+ * Sets the tag-designated key component into the established RSA context.
+ *
+ * This function sets the tag-designated RSA key component into the established
+ * RSA context from the user-specified non-negative integer (octet string format
+ * represented in RSA PKCS#1).
+ * If big_number is NULL, then the specified key component in RSA context is cleared.
+ * If rsa_context is NULL, then return false.
+ *
+ * @param[in, out]  rsa_context  Pointer to RSA context being set.
+ * @param[in]       key_tag      tag of RSA key component being set.
+ * @param[in]       big_number   Pointer to octet integer buffer.
+ *                               If NULL, then the specified key component in RSA
+ *                               context is cleared.
+ * @param[in]       bn_size      Size of big number buffer in bytes.
+ *                               If big_number is NULL, then it is ignored.
+ *
+ * @retval  true   RSA key component was set successfully.
+ * @retval  false  Invalid RSA key component tag.
+ **/
+extern bool libspdm_rsa_set_key(void *rsa_context, const libspdm_rsa_key_tag_t key_tag,
+                                const uint8_t *big_number, size_t bn_size);
+
+/**
+ * Gets the tag-designated RSA key component from the established RSA context.
+ *
+ * This function retrieves the tag-designated RSA key component from the
+ * established RSA context as a non-negative integer (octet string format
+ * represented in RSA PKCS#1).
+ * If specified key component has not been set or has been cleared, then returned
+ * bn_size is set to 0.
+ * If the big_number buffer is too small to hold the contents of the key, false
+ * is returned and bn_size is set to the required buffer size to obtain the key.
+ *
+ * If rsa_context is NULL, then return false.
+ * If bn_size is NULL, then return false.
+ * If bn_size is large enough but big_number is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  rsa_context  Pointer to RSA context being set.
+ * @param[in]       key_tag      Tag of RSA key component being set.
+ * @param[out]      big_number   Pointer to octet integer buffer.
+ * @param[in, out]  bn_size      On input, the size of big number buffer in bytes.
+ *                               On output, the size of data returned in big number buffer in bytes.
+ *
+ * @retval  true   RSA key component was retrieved successfully.
+ * @retval  false  Invalid RSA key component tag.
+ * @retval  false  bn_size is too small.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_rsa_get_key(void *rsa_context, const libspdm_rsa_key_tag_t key_tag,
+                                uint8_t *big_number, size_t *bn_size);
+
+/**
+ * Generates RSA key components.
+ *
+ * This function generates RSA key components. It takes RSA public exponent E and
+ * length in bits of RSA modulus N as input, and generates all key components.
+ * If public_exponent is NULL, the default RSA public exponent (0x10001) will be used.
+ *
+ * If rsa_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in, out]  rsa_context           Pointer to RSA context being set.
+ * @param[in]       modulus_length        Length of RSA modulus N in bits.
+ * @param[in]       public_exponent       Pointer to RSA public exponent.
+ * @param[in]       public_exponent_size  Size of RSA public exponent buffer in bytes.
+ *
+ * @retval  true   RSA key component was generated successfully.
+ * @retval  false  Invalid RSA key component tag.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_rsa_generate_key(void *rsa_context, size_t modulus_length,
+                                     const uint8_t *public_exponent,
+                                     size_t public_exponent_size);
+
+/**
+ * Validates key components of RSA context.
+ * NOTE: This function performs integrity checks on all the RSA key material, so
+ *      the RSA key structure must contain all the private key data.
+ *
+ * This function validates key components of RSA context in following aspects:
+ * - Whether p is a prime
+ * - Whether q is a prime
+ * - Whether n = p * q
+ * - Whether d*e = 1  mod lcm(p-1,q-1)
+ *
+ * If rsa_context is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]  rsa_context  Pointer to RSA context to check.
+ *
+ * @retval  true   RSA key components are valid.
+ * @retval  false  RSA key components are not valid.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_rsa_check_key(void *rsa_context);
+#endif /* (LIBSPDM_RSA_SSA_SUPPORT) || (LIBSPDM_RSA_PSS_SUPPORT) */
+
+#if LIBSPDM_RSA_SSA_SUPPORT
+/**
+ * Carries out the RSA-SSA signature generation with EMSA-PKCS1-v1_5 encoding scheme.
+ *
+ * This function carries out the RSA-SSA signature generation with EMSA-PKCS1-v1_5 encoding scheme
+ * defined in RSA PKCS#1. If the signature buffer is too small to hold the contents of signature,
+ * false is returned and sig_size is set to the required buffer size to obtain the signature.
+ *
+ * If rsa_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If hash_size need match the hash_nid. hash_nid could be SHA256, SHA384, SHA512, SHA3_256,
+ * SHA3_384, SHA3_512.
+ * If sig_size is large enough but signature is NULL, then return false.
+ * If this interface is not supported, then return false.
+ *
+ * @param[in]      rsa_context   Pointer to RSA context for signature generation.
+ * @param[in]      hash_nid      hash NID
+ * @param[in]      message_hash  Pointer to octet message hash to be signed.
+ * @param[in]      hash_size     Size of the message hash in bytes.
+ * @param[out]     signature     Pointer to buffer to receive RSA PKCS1-v1_5 signature.
+ * @param[in, out] sig_size      On input, the size of signature buffer in bytes.
+ *                               On output, the size of data returned in signature buffer in bytes.
+ *
+ * @retval  true   signature successfully generated in PKCS1-v1_5.
+ * @retval  false  signature generation failed.
+ * @retval  false  sig_size is too small.
+ * @retval  false  This interface is not supported.
+ **/
+extern bool libspdm_rsa_pkcs1_sign_with_nid(void *rsa_context, size_t hash_nid,
+                                            const uint8_t *message_hash,
+                                            size_t hash_size, uint8_t *signature,
+                                            size_t *sig_size);
+
+/**
+ * Verifies the RSA-SSA signature with EMSA-PKCS1-v1_5 encoding scheme defined in RSA PKCS#1.
+ *
+ * If rsa_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If signature is NULL, then return false.
+ * If hash_size need match the hash_nid. hash_nid could be SHA256, SHA384, SHA512, SHA3_256,
+ * SHA3_384, SHA3_512.
+ *
+ * @param[in]  rsa_context   Pointer to RSA context for signature verification.
+ * @param[in]  hash_nid      hash NID
+ * @param[in]  message_hash  Pointer to octet message hash to be checked.
+ * @param[in]  hash_size     Size of the message hash in bytes.
+ * @param[in]  signature     Pointer to RSA PKCS1-v1_5 signature to be verified.
+ * @param[in]  sig_size      Size of signature in bytes.
+ *
+ * @retval  true   Valid signature encoded in PKCS1-v1_5.
+ * @retval  false  Invalid signature or invalid RSA context.
+ **/
+extern bool libspdm_rsa_pkcs1_verify_with_nid(void *rsa_context, size_t hash_nid,
+                                              const uint8_t *message_hash,
+                                              size_t hash_size, const uint8_t *signature,
+                                              size_t sig_size);
+#endif /* LIBSPDM_RSA_SSA_SUPPORT */
+
+#if LIBSPDM_RSA_PSS_SUPPORT
+/**
+ * Carries out the RSA-SSA signature generation with EMSA-PSS encoding scheme.
+ *
+ * This function carries out the RSA-SSA signature generation with EMSA-PSS encoding scheme defined
+ * in RSA PKCS#1 v2.2.
+ *
+ * The salt length is same as digest length.
+ *
+ * If the signature buffer is too small to hold the contents of signature, false
+ * is returned and sig_size is set to the required buffer size to obtain the signature.
+ *
+ * If rsa_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If hash_size need match the hash_nid. nid could be SHA256, SHA384, SHA512, SHA3_256, SHA3_384,
+ * SHA3_512.
+ * If sig_size is large enough but signature is NULL, then return false.
+ *
+ * @param[in]       rsa_context   Pointer to RSA context for signature generation.
+ * @param[in]       hash_nid      hash NID
+ * @param[in]       message_hash  Pointer to octet message hash to be signed.
+ * @param[in]       hash_size     Size of the message hash in bytes.
+ * @param[out]      signature     Pointer to buffer to receive RSA-SSA PSS signature.
+ * @param[in, out]  sig_size      On input, the size of signature buffer in bytes.
+ *                                On output, the size of data returned in signature buffer in bytes.
+ *
+ * @retval  true   signature successfully generated in RSA-SSA PSS.
+ * @retval  false  signature generation failed.
+ * @retval  false  sig_size is too small.
+ **/
+extern bool libspdm_rsa_pss_sign(void *rsa_context, size_t hash_nid,
+                                 const uint8_t *message_hash, size_t hash_size,
+                                 uint8_t *signature, size_t *sig_size);
+
+/**
+ * Verifies the RSA-SSA signature with EMSA-PSS encoding scheme defined in
+ * RSA PKCS#1 v2.2.
+ *
+ * The salt length is same as digest length.
+ *
+ * If rsa_context is NULL, then return false.
+ * If message_hash is NULL, then return false.
+ * If signature is NULL, then return false.
+ * If hash_size need match the hash_nid. nid could be SHA256, SHA384, SHA512, SHA3_256, SHA3_384,
+ * SHA3_512.
+ *
+ * @param[in]  rsa_context   Pointer to RSA context for signature verification.
+ * @param[in]  hash_nid      hash NID
+ * @param[in]  message_hash  Pointer to octet message hash to be checked.
+ * @param[in]  hash_size     Size of the message hash in bytes.
+ * @param[in]  signature     Pointer to RSA-SSA PSS signature to be verified.
+ * @param[in]  sig_size      Size of signature in bytes.
+ *
+ * @retval  true   Valid signature encoded in RSA-SSA PSS.
+ * @retval  false  Invalid signature or invalid RSA context.
+ **/
+extern bool libspdm_rsa_pss_verify(void *rsa_context, size_t hash_nid,
+                                   const uint8_t *message_hash, size_t hash_size,
+                                   const uint8_t *signature, size_t sig_size);
+#endif /* LIBSPDM_RSA_PSS_SUPPORT */
+#endif /* CRYPTLIB_RSA_H */
--- a/kernel-open/nvidia/hal/library/cryptlib/cryptlib_sm2.h
+++ b/kernel-open/nvidia/hal/library/cryptlib/cryptlib_sm2.h
@@ -0,0 +1,194 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef CRYPTLIB_SM2_H
+#define CRYPTLIB_SM2_H
+
+/*=====================================================================================
+ *    Shang-Mi2 Primitives
+ *=====================================================================================*/
+
+#if LIBSPDM_SM2_DSA_SUPPORT
+/**
+ * Allocates and Initializes one Shang-Mi2 context for subsequent use.
+ *
+ * @param nid cipher NID
+ *
+ * @return  Pointer to the Shang-Mi2 context that has been initialized.
+ *          If the allocations fails, sm2_new_by_nid() returns NULL.
+ **/
+extern void *libspdm_sm2_dsa_new_by_nid(size_t nid);
+
+/**
+ * Release the specified sm2 context.
+ *
+ * @param[in]  sm2_context  Pointer to the sm2 context to be released.
+ **/
+extern void libspdm_sm2_dsa_free(void *sm2_context);
+
+/**
+ * Carries out the SM2 signature, based upon GB/T 32918.2-2016: SM2 - Part2.
+ *
+ * This function carries out the SM2 signature.
+ * If the signature buffer is too small to hold the contents of signature, false
+ * is returned and sig_size is set to the required buffer size to obtain the signature.
+ *
+ * If sm2_context is NULL, then return false.
+ * If message is NULL, then return false.
+ * hash_nid must be SM3_256.
+ * If sig_size is large enough but signature is NULL, then return false.
+ *
+ * The id_a_size must be smaller than 2^16-1.
+ * The sig_size is 64. first 32-byte is R, second 32-byte is S.
+ *
+ * @param[in]       sm2_context  Pointer to sm2 context for signature generation.
+ * @param[in]       hash_nid     hash NID
+ * @param[in]       id_a         The ID-A of the signing context.
+ * @param[in]       id_a_size    Size of ID-A signing context.
+ * @param[in]       message      Pointer to octet message to be signed (before hash).
+ * @param[in]       size         Size of the message in bytes.
+ * @param[out]      signature    Pointer to buffer to receive SM2 signature.
+ * @param[in, out]  sig_size     On input, the size of signature buffer in bytes.
+ *                               On output, the size of data returned in signature buffer in bytes.
+ *
+ * @retval  true   signature successfully generated in SM2.
+ * @retval  false  signature generation failed.
+ * @retval  false  sig_size is too small.
+ **/
+extern bool libspdm_sm2_dsa_sign(const void *sm2_context, size_t hash_nid,
+                                 const uint8_t *id_a, size_t id_a_size,
+                                 const uint8_t *message, size_t size,
+                                 uint8_t *signature, size_t *sig_size);
+
+/**
+ * Verifies the SM2 signature, based upon GB/T 32918.2-2016: SM2 - Part2.
+ *
+ * If sm2_context is NULL, then return false.
+ * If message is NULL, then return false.
+ * If signature is NULL, then return false.
+ * hash_nid must be SM3_256.
+ *
+ * The id_a_size must be smaller than 2^16-1.
+ * The sig_size is 64. first 32-byte is R, second 32-byte is S.
+ *
+ * @param[in]  sm2_context  Pointer to SM2 context for signature verification.
+ * @param[in]  hash_nid     hash NID
+ * @param[in]  id_a         The ID-A of the signing context.
+ * @param[in]  id_a_size    Size of ID-A signing context.
+ * @param[in]  message      Pointer to octet message to be checked (before hash).
+ * @param[in]  size         Size of the message in bytes.
+ * @param[in]  signature    Pointer to SM2 signature to be verified.
+ * @param[in]  sig_size     Size of signature in bytes.
+ *
+ * @retval  true   Valid signature encoded in SM2.
+ * @retval  false  Invalid signature or invalid sm2 context.
+ *
+ **/
+extern bool libspdm_sm2_dsa_verify(const void *sm2_context, size_t hash_nid,
+                                   const uint8_t *id_a, size_t id_a_size,
+                                   const uint8_t *message, size_t size,
+                                   const uint8_t *signature, size_t sig_size);
+#endif /* LIBSPDM_SM2_DSA_SUPPORT */
+
+#if LIBSPDM_SM2_KEY_EXCHANGE_SUPPORT
+/**
+ * Allocates and Initializes one Shang-Mi2 context for subsequent use.
+ *
+ * @param nid cipher NID
+ *
+ * @return  Pointer to the Shang-Mi2 context that has been initialized.
+ *          If the allocations fails, sm2_new_by_nid() returns NULL.
+ **/
+extern void *libspdm_sm2_key_exchange_new_by_nid(size_t nid);
+
+/**
+ * Release the specified sm2 context.
+ *
+ * @param[in]  sm2_context  Pointer to the sm2 context to be released.
+ *
+ **/
+extern void libspdm_sm2_key_exchange_free(void *sm2_context);
+
+/**
+ * Initialize the specified sm2 context.
+ *
+ * @param[in]  sm2_context   Pointer to the sm2 context to be released.
+ * @param[in]  hash_nid      hash NID, only SM3 is valid.
+ * @param[in]  id_a          The ID-A of the key exchange context.
+ * @param[in]  id_a_size     Size of ID-A key exchange context.
+ * @param[in]  id_b          The ID-B of the key exchange context.
+ * @param[in]  id_b_size     Size of ID-B key exchange context.
+ * @param[in]  is_initiator  If the caller is initiator.
+ *
+ * @retval true   sm2 context is initialized.
+ * @retval false  sm2 context is not initialized.
+ **/
+extern bool libspdm_sm2_key_exchange_init(const void *sm2_context, size_t hash_nid,
+                                          const uint8_t *id_a, size_t id_a_size,
+                                          const uint8_t *id_b, size_t id_b_size,
+                                          bool is_initiator);
+
+/**
+ * Generates sm2 key and returns sm2 public key (X, Y), based upon GB/T 32918.3-2016: SM2 - Part3.
+ *
+ * This function generates random secret, and computes the public key (X, Y), which is
+ * returned via parameter public, public_size.
+ * X is the first half of public with size being public_size / 2,
+ * Y is the second half of public with size being public_size / 2.
+ * sm2 context is updated accordingly.
+ * If the public buffer is too small to hold the public X, Y, false is returned and
+ * public_size is set to the required buffer size to obtain the public X, Y.
+ *
+ * The public_size is 64. first 32-byte is X, second 32-byte is Y.
+ *
+ * If sm2_context is NULL, then return false.
+ * If public_size is NULL, then return false.
+ * If public_size is large enough but public is NULL, then return false.
+ *
+ * @param[in, out]  sm2_context  Pointer to the sm2 context.
+ * @param[out]      public_data  Pointer to the buffer to receive generated public X,Y.
+ * @param[in, out]  public_size  On input, the size of public buffer in bytes.
+ *                               On output, the size of data returned in public buffer in bytes.
+ *
+ * @retval true   sm2 public X,Y generation succeeded.
+ * @retval false  sm2 public X,Y generation failed.
+ * @retval false  public_size is not large enough.
+ **/
+extern bool libspdm_sm2_key_exchange_generate_key(void *sm2_context, uint8_t *public_data,
+                                                  size_t *public_size);
+
+/**
+ * Computes exchanged common key, based upon GB/T 32918.3-2016: SM2 - Part3.
+ *
+ * Given peer's public key (X, Y), this function computes the exchanged common key,
+ * based on its own context including value of curve parameter and random secret.
+ * X is the first half of peer_public with size being peer_public_size / 2,
+ * Y is the second half of peer_public with size being peer_public_size / 2.
+ *
+ * If sm2_context is NULL, then return false.
+ * If peer_public is NULL, then return false.
+ * If peer_public_size is 0, then return false.
+ * If key is NULL, then return false.
+ *
+ * The id_a_size and id_b_size must be smaller than 2^16-1.
+ * The peer_public_size is 64. first 32-byte is X, second 32-byte is Y.
+ * The key_size must be smaller than 2^32-1, limited by KDF function.
+ *
+ * @param[in, out]  sm2_context       Pointer to the sm2 context.
+ * @param[in]       peer_public       Pointer to the peer's public X,Y.
+ * @param[in]       peer_public_size  Size of peer's public X,Y in bytes.
+ * @param[out]      key               Pointer to the buffer to receive generated key.
+ * @param[in]       key_size          On input, the size of key buffer in bytes.
+ *
+ * @retval true   sm2 exchanged key generation succeeded.
+ * @retval false  sm2 exchanged key generation failed.
+ **/
+extern bool libspdm_sm2_key_exchange_compute_key(void *sm2_context,
+                                                 const uint8_t *peer_public,
+                                                 size_t peer_public_size, uint8_t *key,
+                                                 size_t *key_size);
+#endif /* LIBSPDM_SM2_KEY_EXCHANGE_SUPPORT */
+#endif /* CRYPTLIB_SM2_H */
--- a/kernel-open/nvidia/internal/libspdm_lib_config.h
+++ b/kernel-open/nvidia/internal/libspdm_lib_config.h
@@ -0,0 +1,71 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef LIBSPDM_LIB_CONFIG_H
+#define LIBSPDM_LIB_CONFIG_H
+
+#ifndef LIBSPDM_CONFIG
+#include "library/spdm_lib_config.h"
+#else
+#include LIBSPDM_CONFIG
+#endif
+
+#if defined(LIBSPDM_ENABLE_SET_CERTIFICATE_CAP) && \
+    !defined(LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP)
+#ifdef _MSC_VER
+#pragma message("LIBSPDM_ENABLE_SET_CERTIFICATE_CAP is deprecated. Use " \
+    "LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP instead. This warning will be removed in a " \
+    "future release.")
+#else
+#warning LIBSPDM_ENABLE_SET_CERTIFICATE_CAP is deprecated. Use \
+    LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP instead. This warning will be removed in a \
+    future release.
+#endif /* _MSC_VER */
+#endif /* defined(LIBSPDM_ENABLE_SET_CERTIFICATE_CAP) */
+
+#if defined(LIBSPDM_ENABLE_CHUNK_CAP) && !defined(LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP)
+#ifdef _MSC_VER
+#pragma message("LIBSPDM_ENABLE_CHUNK_CAP is deprecated. Use LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP " \
+    "instead. This warning will be removed in a future release.")
+#else
+#warning LIBSPDM_ENABLE_CHUNK_CAP is deprecated. Use LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP \
+    instead. This warning will be removed in a future release.
+#endif /* _MSC_VER */
+#endif /* defined(LIBSPDM_ENABLE_CHUNK_CAP) */
+
+#if defined(MDEPKG_NDEBUG) && !defined(LIBSPDM_DEBUG_ENABLE)
+#ifdef _MSC_VER
+#pragma message("MDEPKG_NDEBUG is deprecated. Use LIBSPDM_DEBUG_ENABLE " \
+    "instead. This warning will be removed in a future release.")
+#else
+#warning MDEPKG_NDEBUG is deprecated. Use LIBSPDM_DEBUG_ENABLE \
+    instead. This warning will be removed in a future release.
+#endif /* _MSC_VER */
+#endif /* defined(MDEPKG_NDEBUG) */
+
+#if defined(LIBSPDM_DEBUG_ENABLE)
+#undef LIBSPDM_DEBUG_ASSERT_ENABLE
+#undef LIBSPDM_DEBUG_PRINT_ENABLE
+#undef LIBSPDM_DEBUG_BLOCK_ENABLE
+
+#define LIBSPDM_DEBUG_ASSERT_ENABLE (LIBSPDM_DEBUG_ENABLE)
+#define LIBSPDM_DEBUG_PRINT_ENABLE (LIBSPDM_DEBUG_ENABLE)
+#define LIBSPDM_DEBUG_BLOCK_ENABLE (LIBSPDM_DEBUG_ENABLE)
+#elif defined(MDEPKG_NDEBUG)
+#undef LIBSPDM_DEBUG_ASSERT_ENABLE
+#undef LIBSPDM_DEBUG_PRINT_ENABLE
+#undef LIBSPDM_DEBUG_BLOCK_ENABLE
+
+#define LIBSPDM_DEBUG_ASSERT_ENABLE 0
+#define LIBSPDM_DEBUG_PRINT_ENABLE 0
+#define LIBSPDM_DEBUG_BLOCK_ENABLE 0
+#endif /* defined(LIBSPDM_DEBUG_ENABLE) */
+
+#if LIBSPDM_CHECK_MACRO
+#include "internal/libspdm_macro_check.h"
+#endif /* LIBSPDM_CHECK_MACRO */
+
+#endif /* LIBSPDM_LIB_CONFIG_H */
--- a/kernel-open/nvidia/internal_crypt_lib.h
+++ b/kernel-open/nvidia/internal_crypt_lib.h
@@ -0,0 +1,154 @@
+/*
+* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+* SPDX-License-Identifier: MIT
+*
+* Permission is hereby granted, free of charge, to any person obtaining a
+* copy of this software and associated documentation files (the "Software"),
+* to deal in the Software without restriction, including without limitation
+* the rights to use, copy, modify, merge, publish, distribute, sublicense,
+* and/or sell copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+* DEALINGS IN THE SOFTWARE.
+*/
+
+#ifndef __INTERNAL_CRYPT_LIB_H__
+#define __INTERNAL_CRYPT_LIB_H__
+
+/*
+ * This code uses Linux Kernel Crypto API extensively. Web page written by
+ * Stephan Mueller and Marek Vasut is a good starting reference on how linux
+ * kernel provides crypto api.
+ */
+#include "conftest.h"
+
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/random.h>
+#include <linux/string.h>
+
+// Check if ECDH/ECDSA are there, on some platforms they might not be...
+#ifndef AUTOCONF_INCLUDED
+#if defined(NV_GENERATED_AUTOCONF_H_PRESENT)
+#include <generated/autoconf.h>
+#else
+#include <linux/autoconf.h>
+#endif
+#endif
+#if \
+    (defined(CONFIG_CRYPTO_AEAD) || defined(CONFIG_CRYPTO_AEAD_MODULE)) && \
+    (defined(CONFIG_CRYPTO_AKCIPHER) || defined(CONFIG_CRYPTO_AKCIPHER_MODULE)) && \
+    (defined(CONFIG_CRYPTO_SKCIPHER) || defined(CONFIG_CRYPTO_SKCIPHER_MODULE)) && \
+    (defined(CONFIG_CRYPTO_HASH) || defined(CONFIG_CRYPTO_HASH_MODULE)) && \
+    (defined(CONFIG_CRYPTO_HMAC) || defined(CONFIG_CRYPTO_HMAC_MODULE)) && \
+    (defined(CONFIG_CRYPTO_ECDH) || defined(CONFIG_CRYPTO_ECDH_MODULE)) && \
+    (defined(CONFIG_CRYPTO_ECDSA) || defined(CONFIG_CRYPTO_ECDSA_MODULE)) && \
+    (defined(CONFIG_X509_CERTIFICATE_PARSER) || defined(CONFIG_X509_CERTIFICATE_PARSER_MODULE))
+#define NV_CONFIG_CRYPTO_PRESENT 1
+#endif
+
+/*
+ * It is possible that we don't have access to all the functions we have. This
+ * could be either because we are running non-gpl kernel, because kernel is too
+ * old or even just user disabled. If we should use LKCA, include headers, else
+ * define stubs to return errors.
+ */
+#if defined(NV_CRYPTO_PRESENT) && defined (NV_CONFIG_CRYPTO_PRESENT)
+#define USE_LKCA 1
+#endif
+
+#ifdef USE_LKCA
+#include <linux/crypto.h>
+#include <linux/scatterlist.h>
+#include <crypto/aead.h>
+#include <crypto/algapi.h>
+#include <crypto/hash.h>
+#include <crypto/sm3.h>
+
+// HASH_MAX_DIGESTSIZE is available since 4.20.
+// This value is accurate as of 6.1
+#ifndef HASH_MAX_DIGESTSIZE
+#define HASH_MAX_DIGESTSIZE 64
+#endif
+
+#else
+// Just stub everything out
+struct shash_desc;
+struct crypto_shash;
+#define crypto_shash_setkey(...) -ENOMEM
+#define crypto_shash_init(...) -ENOMEM
+#define crypto_shash_update(...) -ENOMEM
+#define crypto_shash_update(...) -ENOMEM
+#define crypto_shash_final(...) -ENOMEM
+#endif
+
+#define CHAR_BIT 8U
+#undef SIZE_MAX
+#define SIZE_MAX 8
+
+#include "library/cryptlib.h"
+
+#define LIBSPDM_ASSERT(...)
+struct lkca_aead_ctx;
+int lkca_aead_alloc(struct lkca_aead_ctx **ctx, char const *alg);
+void lkca_aead_free(struct lkca_aead_ctx *ctx);
+int lkca_aead_ex(struct lkca_aead_ctx *ctx,
+                 const uint8_t *key, size_t key_size,
+                 uint8_t *iv, size_t iv_size,
+                 const uint8_t *data_in, size_t data_in_size,
+                 uint8_t *tag, size_t tag_size,
+                 uint8_t *data_out, size_t *data_out_size,
+                 bool enc);
+
+int libspdm_aead(const uint8_t *key, size_t key_size,
+                 const uint8_t *iv, size_t iv_size,
+                 const uint8_t *a_data, size_t a_data_size,
+                 const uint8_t *data_in, size_t data_in_size,
+                 const uint8_t *tag, size_t tag_size,
+                 uint8_t *data_out, size_t *data_out_size,
+                 bool enc, char const *alg);
+
+void *lkca_hash_new(const char* alg_name);
+void lkca_hash_free(struct shash_desc *ctx);
+bool lkca_hash_duplicate(struct shash_desc *dst, struct shash_desc const *src);
+bool lkca_hash_all(const char* alg_name, const void *data,
+                   size_t data_size, uint8_t *hash_value);
+bool lkca_hmac_duplicate(struct shash_desc *dst, struct shash_desc const *src);
+bool lkca_hmac_set_key(struct shash_desc *ctx, const uint8_t *key, size_t key_size);
+bool lkca_hmac_all(const char* alg_name, const uint8_t *key, size_t key_size,
+                   const uint8_t *data, size_t data_size, uint8_t *hash_value);
+bool lkca_hkdf_extract_and_expand(const char *alg_name,
+                                  const uint8_t *key, size_t key_size,
+                                  const uint8_t *salt, size_t salt_size,
+                                  const uint8_t *info, size_t info_size,
+                                  uint8_t *out, size_t out_size);
+bool lkca_hkdf_expand(const char *alg_name,
+                      const uint8_t *prk, size_t prk_size,
+                      const uint8_t *info, size_t info_size,
+                      uint8_t *out, size_t out_size);
+
+
+bool lkca_ecdsa_set_priv_key(void *context, uint8_t *key, size_t key_size);
+bool lkca_ec_set_pub_key(void *ec_context, const uint8_t *public_key,
+                         size_t public_key_size);
+bool lkca_ec_get_pub_key(void *ec_context, uint8_t *public_key,
+                         size_t *public_key_size);
+bool lkca_ec_generate_key(void *ec_context, uint8_t *public_data,
+                          size_t *public_size);
+bool lkca_ec_compute_key(void *ec_context, const uint8_t *peer_public,
+                         size_t peer_public_size, uint8_t *key,
+                         size_t *key_size);
+bool lkca_ecdsa_verify(void *ec_context, size_t hash_nid,
+                       const uint8_t *message_hash, size_t hash_size,
+                       const uint8_t *signature, size_t sig_size);
+#endif
--- a/kernel-open/nvidia/library/cryptlib.h
+++ b/kernel-open/nvidia/library/cryptlib.h
@@ -0,0 +1,109 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+/** @file
+ * Defines base cryptographic library APIs.
+ * The Base Cryptographic Library provides implementations of basic cryptography
+ * primitives (hash Serials, HMAC, AES, RSA, Diffie-Hellman, Elliptic Curve, etc) for security
+ * functionality enabling.
+ **/
+
+#ifndef CRYPTLIB_H
+#define CRYPTLIB_H
+
+#include "internal/libspdm_lib_config.h"
+
+#define LIBSPDM_CRYPTO_NID_NULL 0x0000
+
+/* Hash */
+#define LIBSPDM_CRYPTO_NID_SHA256 0x0001
+#define LIBSPDM_CRYPTO_NID_SHA384 0x0002
+#define LIBSPDM_CRYPTO_NID_SHA512 0x0003
+#define LIBSPDM_CRYPTO_NID_SHA3_256 0x0004
+#define LIBSPDM_CRYPTO_NID_SHA3_384 0x0005
+#define LIBSPDM_CRYPTO_NID_SHA3_512 0x0006
+#define LIBSPDM_CRYPTO_NID_SM3_256 0x0007
+
+/* Signing */
+#define LIBSPDM_CRYPTO_NID_RSASSA2048 0x0101
+#define LIBSPDM_CRYPTO_NID_RSASSA3072 0x0102
+#define LIBSPDM_CRYPTO_NID_RSASSA4096 0x0103
+#define LIBSPDM_CRYPTO_NID_RSAPSS2048 0x0104
+#define LIBSPDM_CRYPTO_NID_RSAPSS3072 0x0105
+#define LIBSPDM_CRYPTO_NID_RSAPSS4096 0x0106
+#define LIBSPDM_CRYPTO_NID_ECDSA_NIST_P256 0x0107
+#define LIBSPDM_CRYPTO_NID_ECDSA_NIST_P384 0x0108
+#define LIBSPDM_CRYPTO_NID_ECDSA_NIST_P521 0x0109
+#define LIBSPDM_CRYPTO_NID_SM2_DSA_P256 0x010A
+#define LIBSPDM_CRYPTO_NID_EDDSA_ED25519 0x010B
+#define LIBSPDM_CRYPTO_NID_EDDSA_ED448 0x010C
+
+/* Key Exchange */
+#define LIBSPDM_CRYPTO_NID_FFDHE2048 0x0201
+#define LIBSPDM_CRYPTO_NID_FFDHE3072 0x0202
+#define LIBSPDM_CRYPTO_NID_FFDHE4096 0x0203
+#define LIBSPDM_CRYPTO_NID_SECP256R1 0x0204
+#define LIBSPDM_CRYPTO_NID_SECP384R1 0x0205
+#define LIBSPDM_CRYPTO_NID_SECP521R1 0x0206
+#define LIBSPDM_CRYPTO_NID_SM2_KEY_EXCHANGE_P256 0x0207
+#define LIBSPDM_CRYPTO_NID_CURVE_X25519 0x0208
+#define LIBSPDM_CRYPTO_NID_CURVE_X448 0x0209
+
+/* AEAD */
+#define LIBSPDM_CRYPTO_NID_AES_128_GCM 0x0301
+#define LIBSPDM_CRYPTO_NID_AES_256_GCM 0x0302
+#define LIBSPDM_CRYPTO_NID_CHACHA20_POLY1305 0x0303
+#define LIBSPDM_CRYPTO_NID_SM4_128_GCM 0x0304
+
+/* X.509 v3 key usage extension flags. */
+#define LIBSPDM_CRYPTO_X509_KU_DIGITAL_SIGNATURE 0x80 /* bit 0 */
+#define LIBSPDM_CRYPTO_X509_KU_NON_REPUDIATION 0x40 /* bit 1 */
+#define LIBSPDM_CRYPTO_X509_KU_KEY_ENCIPHERMENT 0x20 /* bit 2 */
+#define LIBSPDM_CRYPTO_X509_KU_DATA_ENCIPHERMENT 0x10 /* bit 3 */
+#define LIBSPDM_CRYPTO_X509_KU_KEY_AGREEMENT 0x08 /* bit 4 */
+#define LIBSPDM_CRYPTO_X509_KU_KEY_CERT_SIGN 0x04 /* bit 5 */
+#define LIBSPDM_CRYPTO_X509_KU_CRL_SIGN 0x02 /* bit 6 */
+#define LIBSPDM_CRYPTO_X509_KU_ENCIPHER_ONLY 0x01 /* bit 7 */
+#define LIBSPDM_CRYPTO_X509_KU_DECIPHER_ONLY 0x8000 /* bit 8 */
+
+/* These constants comply with the DER encoded ASN.1 type tags. */
+#define LIBSPDM_CRYPTO_ASN1_BOOLEAN 0x01
+#define LIBSPDM_CRYPTO_ASN1_INTEGER 0x02
+#define LIBSPDM_CRYPTO_ASN1_BIT_STRING 0x03
+#define LIBSPDM_CRYPTO_ASN1_OCTET_STRING 0x04
+#define LIBSPDM_CRYPTO_ASN1_NULL 0x05
+#define LIBSPDM_CRYPTO_ASN1_OID 0x06
+#define LIBSPDM_CRYPTO_ASN1_UTF8_STRING 0x0C
+#define LIBSPDM_CRYPTO_ASN1_SEQUENCE 0x10
+#define LIBSPDM_CRYPTO_ASN1_SET 0x11
+#define LIBSPDM_CRYPTO_ASN1_PRINTABLE_STRING 0x13
+#define LIBSPDM_CRYPTO_ASN1_T61_STRING 0x14
+#define LIBSPDM_CRYPTO_ASN1_IA5_STRING 0x16
+#define LIBSPDM_CRYPTO_ASN1_UTC_TIME 0x17
+#define LIBSPDM_CRYPTO_ASN1_GENERALIZED_TIME 0x18
+#define LIBSPDM_CRYPTO_ASN1_UNIVERSAL_STRING 0x1C
+#define LIBSPDM_CRYPTO_ASN1_BMP_STRING 0x1E
+#define LIBSPDM_CRYPTO_ASN1_PRIMITIVE 0x00
+#define LIBSPDM_CRYPTO_ASN1_CONSTRUCTED 0x20
+#define LIBSPDM_CRYPTO_ASN1_CONTEXT_SPECIFIC 0x80
+
+#define LIBSPDM_CRYPTO_ASN1_TAG_CLASS_MASK 0xC0
+#define LIBSPDM_CRYPTO_ASN1_TAG_PC_MASK 0x20
+#define LIBSPDM_CRYPTO_ASN1_TAG_VALUE_MASK 0x1F
+
+#include "hal/library/cryptlib/cryptlib_hash.h"
+#include "hal/library/cryptlib/cryptlib_mac.h"
+#include "hal/library/cryptlib/cryptlib_aead.h"
+#include "hal/library/cryptlib/cryptlib_cert.h"
+#include "hal/library/cryptlib/cryptlib_hkdf.h"
+#include "hal/library/cryptlib/cryptlib_rsa.h"
+#include "hal/library/cryptlib/cryptlib_ec.h"
+#include "hal/library/cryptlib/cryptlib_dh.h"
+#include "hal/library/cryptlib/cryptlib_ecd.h"
+#include "hal/library/cryptlib/cryptlib_sm2.h"
+#include "hal/library/cryptlib/cryptlib_rng.h"
+
+#endif /* CRYPTLIB_H */
--- a/kernel-open/nvidia/library/spdm_lib_config.h
+++ b/kernel-open/nvidia/library/spdm_lib_config.h
@@ -0,0 +1,415 @@
+/**
+ *  Copyright Notice:
+ *  Copyright 2021-2022 DMTF. All rights reserved.
+ *  License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+ **/
+
+#ifndef SPDM_LIB_CONFIG_H
+#define SPDM_LIB_CONFIG_H
+
+/* Enables assertions and debug printing. When `LIBSPDM_DEBUG_ENABLE` is defined it overrides or
+ * sets the values of `LIBSPDM_DEBUG_PRINT_ENABLE`, `LIBSPDM_DEBUG_ASSERT_ENABLE`, and
+ * `LIBSPDM_BLOCK_ENABLE` to the value of `LIBSPDM_DEBUG_ENABLE`.
+ *
+ * Note that if this file is used with CMake and `DTARGET=Release` is defined, then all debugging
+ * is disabled.
+ */
+#ifndef LIBSPDM_DEBUG_ENABLE
+#define LIBSPDM_DEBUG_ENABLE 1
+#endif
+
+/* The SPDM specification allows a Responder to return up to 256 version entries in the `VERSION`
+ * response to the Requester, including duplicate entries. For a Requester this value specifies the
+ * maximum number of entries that libspdm will tolerate in a `VERSION` response before returning an
+ * error. A similiar macro, `SPDM_MAX_VERSION_COUNT`, exists for the Responder. However this macro
+ * is not meant to be configured by the Integrator.
+ */
+#ifndef LIBSPDM_MAX_VERSION_COUNT
+#define LIBSPDM_MAX_VERSION_COUNT 5
+#endif
+
+/* This value specifies the maximum size, in bytes, of the `PSK_EXCHANGE.RequesterContext` and,
+ * if supported by the Responder, `PSK_EXCHANGE_RSP.ResponderContext` fields. The fields are
+ * typically random or monotonically increasing numbers.
+ */
+#ifndef LIBSPDM_PSK_CONTEXT_LENGTH
+#define LIBSPDM_PSK_CONTEXT_LENGTH LIBSPDM_MAX_HASH_SIZE
+#endif
+/* This value specifies the maximum size, in bytes, of the `PSK_EXCHANGE.PSKHint` field.*/
+#ifndef LIBSPDM_PSK_MAX_HINT_LENGTH
+#define LIBSPDM_PSK_MAX_HINT_LENGTH 16
+#endif
+
+/* libspdm allows an Integrator to specify multiple root certificates as trust anchors when
+ * verifying certificate chains from an endpoint. This value specifies the maximum number of root
+ * certificates that libspdm can support.
+ */
+#ifndef LIBSPDM_MAX_ROOT_CERT_SUPPORT
+#define LIBSPDM_MAX_ROOT_CERT_SUPPORT 10
+#endif
+
+/* If the Responder supports it a Requester is allowed to establish multiple secure sessions with
+ * the Responder. This value specifies the maximum number of sessions libspdm can support.
+ */
+#ifndef LIBSPDM_MAX_SESSION_COUNT
+#define LIBSPDM_MAX_SESSION_COUNT 4
+#endif
+/* This value specifies the maximum size, in bytes, of a certificate chain that can be stored in a
+ * libspdm context.
+ */
+#ifndef LIBSPDM_MAX_CERT_CHAIN_SIZE
+#define LIBSPDM_MAX_CERT_CHAIN_SIZE 0x1000
+#endif
+#ifndef LIBSPDM_MAX_MEASUREMENT_RECORD_SIZE
+#define LIBSPDM_MAX_MEASUREMENT_RECORD_SIZE 0x1000
+#endif
+/* Partial certificates can be retrieved from a Requester or Responder and through multiple messages
+ * the complete certificate chain can be constructed. This value specifies the maximum size,
+ * in bytes, of a partial certificate that can be sent or received.
+ */
+#ifndef LIBSPDM_MAX_CERT_CHAIN_BLOCK_LEN
+#define LIBSPDM_MAX_CERT_CHAIN_BLOCK_LEN 1024
+#endif
+
+#ifndef LIBSPDM_MAX_MESSAGE_BUFFER_SIZE
+#define LIBSPDM_MAX_MESSAGE_BUFFER_SIZE 0x1200
+#endif
+#ifndef LIBSPDM_MAX_MESSAGE_SMALL_BUFFER_SIZE
+#define LIBSPDM_MAX_MESSAGE_SMALL_BUFFER_SIZE 0x100  /* to hold message_a before negotiate*/
+#endif
+#ifndef LIBSPDM_MAX_MESSAGE_MEDIUM_BUFFER_SIZE
+#define LIBSPDM_MAX_MESSAGE_MEDIUM_BUFFER_SIZE 0x300 /* to hold message_k before finished_key is ready*/
+#endif
+
+/* If the Responder replies with a Busy `ERROR` response to a request then the Requester is free to
+ * retry sending the request. This value specifies the maximum number of times libspdm will retry
+ * sending the request before returning an error. If its value is 0 then libspdm will not send any
+ * retry requests.
+ */
+#ifndef LIBSPDM_MAX_REQUEST_RETRY_TIMES
+#define LIBSPDM_MAX_REQUEST_RETRY_TIMES 3
+#endif
+#ifndef LIBSPDM_MAX_SESSION_STATE_CALLBACK_NUM
+#define LIBSPDM_MAX_SESSION_STATE_CALLBACK_NUM 4
+#endif
+#ifndef LIBSPDM_MAX_CONNECTION_STATE_CALLBACK_NUM
+#define LIBSPDM_MAX_CONNECTION_STATE_CALLBACK_NUM 4
+#endif
+#ifndef LIBSPDM_MAX_KEY_UPDATE_CALLBACK_NUM
+#define LIBSPDM_MAX_KEY_UPDATE_CALLBACK_NUM 4
+#endif
+
+#ifndef LIBSPDM_MAX_CSR_SIZE
+#define LIBSPDM_MAX_CSR_SIZE 0x1000
+#endif
+
+/* To ensure integrity in communication between the Requester and the Responder libspdm calculates
+ * cryptographic digests and signatures over multiple requests and responses. This value specifies
+ * whether libspdm will use a running calculation over the transcript, where requests and responses
+ * are discarded as they are cryptographically consumed, or whether libspdm will buffer the entire
+ * transcript before calculating the digest or signature.
+ */
+#ifndef LIBSPDM_RECORD_TRANSCRIPT_DATA_SUPPORT
+#define LIBSPDM_RECORD_TRANSCRIPT_DATA_SUPPORT 0
+#endif
+
+
+/* Cryptography Configuration
+ * In each category, at least one should be selected.
+ * NOTE: Not all combination can be supported. E.g. Don't mix NIST algo with SMx.*/
+
+#ifndef LIBSPDM_RSA_SSA_SUPPORT
+#define LIBSPDM_RSA_SSA_SUPPORT 1
+#endif
+#ifndef LIBSPDM_RSA_PSS_SUPPORT
+#define LIBSPDM_RSA_PSS_SUPPORT 1
+#endif
+#ifndef LIBSPDM_ECDSA_SUPPORT
+#define LIBSPDM_ECDSA_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SM2_DSA_SUPPORT
+#define LIBSPDM_SM2_DSA_SUPPORT 1
+#endif
+#ifndef LIBSPDM_EDDSA_ED25519_SUPPORT
+#define LIBSPDM_EDDSA_ED25519_SUPPORT 1
+#endif
+#ifndef LIBSPDM_EDDSA_ED448_SUPPORT
+#define LIBSPDM_EDDSA_ED448_SUPPORT 1
+#endif
+
+#ifndef LIBSPDM_FFDHE_SUPPORT
+#define LIBSPDM_FFDHE_SUPPORT 1
+#endif
+#ifndef LIBSPDM_ECDHE_SUPPORT
+#define LIBSPDM_ECDHE_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SM2_KEY_EXCHANGE_SUPPORT
+#define LIBSPDM_SM2_KEY_EXCHANGE_SUPPORT 1
+#endif
+
+#ifndef LIBSPDM_AEAD_GCM_SUPPORT
+#define LIBSPDM_AEAD_GCM_SUPPORT 1
+#endif
+#ifndef LIBSPDM_AEAD_CHACHA20_POLY1305_SUPPORT
+#define LIBSPDM_AEAD_CHACHA20_POLY1305_SUPPORT 1
+#endif
+#ifndef LIBSPDM_AEAD_SM4_SUPPORT
+#define LIBSPDM_AEAD_SM4_SUPPORT 1
+#endif
+
+#ifndef LIBSPDM_SHA256_SUPPORT
+#define LIBSPDM_SHA256_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SHA384_SUPPORT
+#define LIBSPDM_SHA384_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SHA512_SUPPORT
+#define LIBSPDM_SHA512_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SHA3_256_SUPPORT
+#define LIBSPDM_SHA3_256_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SHA3_384_SUPPORT
+#define LIBSPDM_SHA3_384_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SHA3_512_SUPPORT
+#define LIBSPDM_SHA3_512_SUPPORT 1
+#endif
+#ifndef LIBSPDM_SM3_256_SUPPORT
+#define LIBSPDM_SM3_256_SUPPORT 1
+#endif
+
+/* Code space optimization for Optional request/response messages.*/
+
+/* Consumers of libspdm may wish to not fully implement all of the optional
+ * SPDM request/response messages. Therefore we have provided these
+ * SPDM_ENABLE_CAPABILITY_***_CAP compile time switches as an optimization
+ * disable the code (#if 0) related to said optional capability, thereby
+ * reducing the code space used in the image.*/
+
+/* A single switch may enable/disable a single capability or group of related
+ * capabilities.*/
+
+/* LIBSPDM_ENABLE_CAPABILITY_CERT_CAP - Enable/Disable single CERT capability.
+ * LIBSPDM_ENABLE_CAPABILITY_CHAL_CAP - Enable/Disable single CHAL capability.
+ * LIBSPDM_ENABLE_CAPABILTIY_MEAS_CAP - Enable/Disables multiple MEAS capabilities:
+ *                                  (MEAS_CAP_NO_SIG, MEAS_CAP_SIG, MEAS_FRESH_CAP)*/
+
+/* LIBSPDM_ENABLE_CAPABILITY_KEY_EX_CAP - Enable/Disable single Key Exchange capability.
+ * LIBSPDM_ENABLE_CAPABILITY_PSK_EX_CAP - Enable/Disable PSK_EX and PSK_FINISH.*/
+
+/* LIBSPDM_ENABLE_CAPABILITY_MUT_AUTH_CAP - Enable/Disable mutual authentication.
+* LIBSPDM_ENABLE_CAPABILITY_ENCAP_CAP    - Enable/Disable encapsulated message.*/
+
+/* LIBSPDM_ENABLE_CAPABILITY_GET_CSR_CAP - Enable/Disable get csr capability.
+ * LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP - Enable/Disable set certificate capability. */
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_CERT_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_CERT_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_CHAL_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_CHAL_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_MEAS_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_MEAS_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_KEY_EX_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_KEY_EX_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_PSK_EX_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_PSK_EX_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_HBEAT_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_HBEAT_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_MUT_AUTH_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_MUT_AUTH_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_ENCAP_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_ENCAP_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_GET_CSR_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_GET_CSR_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_SET_CERTIFICATE_CAP 1
+#endif
+
+#ifndef LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP
+#define LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP 1
+#endif
+
+/*
+ * MinDataTransferSize = 42
+ *
+ * H = HashLen = HmacLen = [32, 64]
+ * S = SigLen = [64, 512]
+ * D = ExchangeDataLen = [64, 512]
+ * R = RequesterContextLen >= 32
+ * R = ResponderContextLen >= 0
+ * O = OpaqueDataLen <= 1024
+ *
+ * Max Chunk No = 1, if (message size <= 42)
+ * Max Chunk No = [(message size + 4) / 30] roundup, if (message size > 42)
+ *
+ * +==========================+==========================================+=========+
+ * |  Command                 |   Size                                   |MaxChunk |
+ * +==========================+==========================================+=========+
+ * | GET_VERSION              | 4                                        | 1       |
+ * | VERSION {1.0, 1.1, 1.2}  | 6 + 2 * 3 = 12                           | 1       |
+ * +--------------------------+------------------------------------------+---------+
+ * | GET_CAPABILITIES 1.2     | 20                                       | 1       |
+ * | CAPABILITIES 1.2         | 20                                       | 1       |
+ * +--------------------------+------------------------------------------+---------+
+ * | ERROR                    | 4                                        | 1       |
+ * | ERROR(ResponseTooLarge)  | 4 + 4 = 8                                | 1       |
+ * | ERROR(LargeResponse)     | 4 + 1 = 5                                | 1       |
+ * | ERROR(ResponseNotReady)  | 4 + 4 = 8                                | 1       |
+ * +--------------------------+------------------------------------------+---------+
+ * | CHUNK_SEND header        | 12 + L0 (0 or 4)                         | 1       |
+ * | CHUNK_RESPONSE header    | 12 + L0 (0 or 4)                         | 1       |
+ * +==========================+==========================================+=========+
+ * | NEGOTIATE_ALGORITHMS 1.2 | 32 + 4 * 4 = 48                          | 2       |
+ * | ALGORITHMS 1.2           | 36 + 4 * 4 = 52                          | 2       |
+ * +--------------------------+------------------------------------------+---------+
+ * | GET_DIGESTS 1.2          | 4                                        | 1       |
+ * | DIGESTS 1.2              | 4 + H * SlotNum = [36, 516]              | [1, 18] |
+ * +--------------------------+------------------------------------------+---------+
+ * | GET_CERTIFICATE 1.2      | 8                                        | 1       |
+ * | CERTIFICATE 1.2          | 8 + PortionLen                           | [1, ]   |
+ * +--------------------------+------------------------------------------+---------+
+ * | CHALLENGE 1.2            | 40                                       | 1       |
+ * | CHALLENGE_AUTH 1.2       | 38 + H * 2 + S [+ O] = [166, 678]        | [6, 23] |
+ * +--------------------------+------------------------------------------+---------+
+ * | GET_MEASUREMENTS 1.2     | 5 + Nounce (0 or 32)                     | 1       |
+ * | MEASUREMENTS 1.2         | 42 + MeasRecLen (+ S) [+ O] = [106, 554] | [4, 19] |
+ * +--------------------------+------------------------------------------+---------+
+ * | KEY_EXCHANGE 1.2         | 42 + D [+ O] = [106, 554]                | [4, 19] |
+ * | KEY_EXCHANGE_RSP 1.2     | 42 + D + H + S (+ H) [+ O] = [234, 1194] | [8, 40] |
+ * +--------------------------+------------------------------------------+---------+
+ * | FINISH 1.2               | 4 (+ S) + H = [100, 580]                 | [4, 20] |
+ * | FINISH_RSP 1.2           | 4 (+ H) = [36, 69]                       | [1, 3]  |
+ * +--------------------------+------------------------------------------+---------+
+ * | PSK_EXCHANGE 1.2         | 12 [+ PSKHint] + R [+ O] = 44            | 2       |
+ * | PSK_EXCHANGE_RSP 1.2     | 12 + R + H (+ H) [+ O] = [108, 172]      | [4, 6]  |
+ * +--------------------------+------------------------------------------+---------+
+ * | PSK_FINISH 1.2           | 4 + H = [36, 68]                         | [1, 3]  |
+ * | PSK_FINISH_RSP 1.2       | 4                                        | 1       |
+ * +--------------------------+------------------------------------------+---------+
+ * | GET_CSR 1.2              | 8 + RequesterInfoLen [+ O]               | [1, ]   |
+ * | CSR 1.2                  | 8 + CSRLength                            | [1, ]   |
+ * +--------------------------+------------------------------------------+---------+
+ * | SET_CERTIFICATE 1.2      | 4 + CertChainLen                         | [1, ]   |
+ * | SET_CERTIFICATE_RSP 1.2  | 4                                        | 1       |
+ * +==========================+==========================================+=========+
+ */
+
+/* Maximum size of a large SPDM message.
+ * If chunk is unsupported, it must be same as LIBSPDM_DATA_TRANSFER_SIZE.
+ * If chunk is supported, it must be larger than LIBSPDM_DATA_TRANSFER_SIZE.
+ * It matches MaxSPDMmsgSize in SPDM specification. */
+#ifndef LIBSPDM_MAX_SPDM_MSG_SIZE
+#define LIBSPDM_MAX_SPDM_MSG_SIZE LIBSPDM_MAX_MESSAGE_BUFFER_SIZE
+#endif
+
+/* Maximum size of a single SPDM message.
+ * It matches DataTransferSize in SPDM specification. */
+#ifndef LIBSPDM_DATA_TRANSFER_SIZE
+#define LIBSPDM_DATA_TRANSFER_SIZE LIBSPDM_MAX_MESSAGE_BUFFER_SIZE
+#endif
+
+/* Required sender/receive buffer in device io.
+ * NOTE: This is transport specific. Below configuration is just an example.
+ * +-------+--------+---------------------------+------+--+------+---+--------+-----+
+ * | TYPE  |TransHdr|      EncryptionHeader     |AppHdr|  |Random|MAC|AlignPad|FINAL|
+ * |       |        |SessionId|SeqNum|Len|AppLen|      |  |      |   |        |     |
+ * +-------+--------+---------------------------+------+  +------+---+--------+-----+
+ * | MCTP  |    1   |    4    |   2  | 2 |   2  |   1  |  |  32  | 12|   0    |  56 |
+ * |PCI_DOE|    8   |    4    |   0  | 2 |   2  |   0  |  |   0  | 12|   3    |  31 |
+ * +-------+--------+---------------------------+------+--+------+---+--------+-----+
+ */
+#ifndef LIBSPDM_TRANSPORT_ADDITIONAL_SIZE
+#define LIBSPDM_TRANSPORT_ADDITIONAL_SIZE    64
+#endif
+#ifndef LIBSPDM_SENDER_RECEIVE_BUFFER_SIZE
+#define LIBSPDM_SENDER_RECEIVE_BUFFER_SIZE (LIBSPDM_DATA_TRANSFER_SIZE + \
+                                            LIBSPDM_TRANSPORT_ADDITIONAL_SIZE)
+#endif
+
+
+/* Required scratch buffer size for libspdm internal usage.
+ * It may be used to hold the encrypted/decrypted message and/or last sent/received message.
+ * It may be used to hold the large request/response and intermediate send/receive buffer
+ * in case of chunking.
+ *
+ * If chunking is not supported, it may be just LIBSPDM_SENDER_RECEIVE_BUFFER_SIZE.
+ * If chunking is supported, it should be at least below.
+ *
+ * +---------------+--------------+--------------------------+------------------------------+
+ * |SECURE_MESSAGE |LARGE_MESSAGE |    SENDER_RECEIVER       | LARGE_SENDER_RECEIVER        |
+ * +---------------+--------------+--------------------------+------------------------------+
+ * |<-Secure msg ->|<-Large msg ->|<-Snd/Rcv buf for chunk ->|<-Snd/Rcv buf for large msg ->|
+ *
+ * The value is NOT configurable.
+ * The value MAY be changed in different libspdm version.
+ * It is exposed here, just in case the libspdm consumer wants to configure the setting at build time.
+ */
+#if LIBSPDM_ENABLE_CAPABILITY_CHUNK_CAP
+
+/* first section */
+#define LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_OFFSET 0
+
+#define LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_CAPACITY (LIBSPDM_MAX_SPDM_MSG_SIZE)
+
+/* second section */
+#define LIBSPDM_SCRATCH_BUFFER_LARGE_MESSAGE_OFFSET (LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_CAPACITY)
+
+#define LIBSPDM_SCRATCH_BUFFER_LARGE_MESSAGE_CAPACITY (LIBSPDM_MAX_SPDM_MSG_SIZE)
+
+/* third section */
+#define LIBSPDM_SCRATCH_BUFFER_SENDER_RECEIVER_OFFSET  \
+    (LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_CAPACITY + \
+     LIBSPDM_SCRATCH_BUFFER_LARGE_MESSAGE_CAPACITY)
+
+#define LIBSPDM_SCRATCH_BUFFER_SENDER_RECEIVER_CAPACITY (LIBSPDM_MAX_SPDM_MSG_SIZE)
+
+/* fourth section */
+#define LIBSPDM_SCRATCH_BUFFER_LARGE_SENDER_RECEIVER_OFFSET  \
+    (LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_CAPACITY + \
+     LIBSPDM_SCRATCH_BUFFER_LARGE_MESSAGE_CAPACITY + \
+     LIBSPDM_SCRATCH_BUFFER_SENDER_RECEIVER_CAPACITY)
+
+#define LIBSPDM_SCRATCH_BUFFER_LARGE_SENDER_RECEIVER_CAPACITY (LIBSPDM_MAX_SPDM_MSG_SIZE)
+
+#define LIBSPDM_SCRATCH_BUFFER_SIZE (LIBSPDM_SCRATCH_BUFFER_SECURE_MESSAGE_CAPACITY + \
+                                     LIBSPDM_SCRATCH_BUFFER_LARGE_MESSAGE_CAPACITY + \
+                                     LIBSPDM_SCRATCH_BUFFER_SENDER_RECEIVER_CAPACITY + \
+                                     LIBSPDM_SCRATCH_BUFFER_LARGE_SENDER_RECEIVER_CAPACITY \
+                                     )
+
+#else
+#define LIBSPDM_SCRATCH_BUFFER_SIZE (LIBSPDM_SENDER_RECEIVE_BUFFER_SIZE)
+#endif
+
+/* Enable message logging.
+ * See https://github.com/DMTF/libspdm/blob/main/doc/user_guide.md#message-logging
+ * for more information */
+#ifndef LIBSPDM_ENABLE_MSG_LOG
+#define LIBSPDM_ENABLE_MSG_LOG 1
+#endif
+
+/* Enable macro checking during compilation. */
+#ifndef LIBSPDM_CHECK_MACRO
+#define LIBSPDM_CHECK_MACRO 0
+#endif
+
+#endif /* SPDM_LIB_CONFIG_H */
--- a/kernel-open/nvidia/libspdm_aead.c
+++ b/kernel-open/nvidia/libspdm_aead.c
@@ -0,0 +1,470 @@
+/*
+* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+* SPDX-License-Identifier: MIT
+*
+* Permission is hereby granted, free of charge, to any person obtaining a
+* copy of this software and associated documentation files (the "Software"),
+* to deal in the Software without restriction, including without limitation
+* the rights to use, copy, modify, merge, publish, distribute, sublicense,
+* and/or sell copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+* DEALINGS IN THE SOFTWARE.
+*/
+
+#include "internal_crypt_lib.h"
+#include "nvspdm_cryptlib_extensions.h"
+
+#ifdef USE_LKCA
+#define BUFFER_SIZE (2 * 1024 * 1024)
+#define AUTH_TAG_SIZE 16
+struct lkca_aead_ctx
+{
+    struct crypto_aead *aead;
+    struct aead_request *req;
+    char *a_data_buffer;
+    char *in_buffer;
+    char *out_buffer;
+    char tag[AUTH_TAG_SIZE];
+};
+#endif
+
+int libspdm_aead_prealloc(void **context, char const *alg)
+{
+#ifndef USE_LKCA
+    return -ENODEV;
+#else
+    struct lkca_aead_ctx *ctx;
+
+    ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    memset(ctx, 0, sizeof(*ctx));
+
+    ctx->aead = crypto_alloc_aead(alg, CRYPTO_ALG_TYPE_AEAD, 0);
+    if (IS_ERR(ctx->aead)) {
+        pr_notice("could not allocate AEAD algorithm\n");
+        kfree(ctx);
+        return -ENODEV;
+    }
+
+    ctx->req = aead_request_alloc(ctx->aead, GFP_KERNEL);
+    if (ctx->req == NULL) {
+        pr_info("could not allocate skcipher request\n");
+        crypto_free_aead(ctx->aead);
+        kfree(ctx);
+        return -ENOMEM;
+    }
+
+    ctx->a_data_buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
+    if (ctx->a_data_buffer == NULL) {
+        aead_request_free(ctx->req);
+        crypto_free_aead(ctx->aead);
+        kfree(ctx);
+        return -ENOMEM;
+    }
+
+    ctx->in_buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
+    if (ctx->in_buffer == NULL) {
+        kfree(ctx->a_data_buffer);
+        aead_request_free(ctx->req);
+        crypto_free_aead(ctx->aead);
+        kfree(ctx);
+        return -ENOMEM;
+    }
+
+    ctx->out_buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
+    if (ctx->out_buffer == NULL) {
+        kfree(ctx->a_data_buffer);
+        kfree(ctx->in_buffer);
+        aead_request_free(ctx->req);
+        crypto_free_aead(ctx->aead);
+        kfree(ctx);
+        return -ENOMEM;
+    }
+
+    *context = ctx;
+    return 0;
+#endif
+}
+
+void libspdm_aead_free(void *context)
+{
+#ifdef USE_LKCA
+    struct lkca_aead_ctx *ctx = context;
+    crypto_free_aead(ctx->aead);
+    aead_request_free(ctx->req);
+    kfree(ctx->a_data_buffer);
+    kfree(ctx->in_buffer);
+    kfree(ctx->out_buffer);
+    kfree(ctx);
+#endif
+}
+
+#define SG_AEAD_AAD 0
+#define SG_AEAD_TEXT 1
+#define SG_AEAD_SIG 2
+// Number of fields in AEAD scatterlist
+#define SG_AEAD_LEN 3
+
+#ifdef USE_LKCA
+// This function doesn't do any allocs, it uses temp buffers instead
+static int lkca_aead_internal(struct crypto_aead *aead,
+                              struct aead_request *req,
+                              const uint8_t *key, size_t key_size,
+                              const uint8_t *iv, size_t iv_size,
+                              struct scatterlist sg_in[],
+                              struct scatterlist sg_out[],
+                              size_t a_data_size,
+                              size_t data_in_size,
+                              size_t *data_out_size,
+                              size_t tag_size,
+                              bool enc)
+{
+    DECLARE_CRYPTO_WAIT(wait);
+    int rc = 0;
+
+    if (crypto_aead_setkey(aead, key, key_size)) {
+        pr_info("key could not be set\n");
+        return -EINVAL;
+    }
+
+    if (crypto_aead_ivsize(aead) != iv_size) {
+        pr_info("iv could not be set\n");
+        return -EINVAL;
+    }
+
+    aead_request_set_ad(req, a_data_size);
+
+    aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
+            CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait);
+
+    if (enc) {
+        aead_request_set_crypt(req, sg_in, sg_out, data_in_size, (u8 *) iv);
+        rc = crypto_wait_req(crypto_aead_encrypt(req), &wait);
+    } else {
+        aead_request_set_crypt(req, sg_in, sg_out, data_in_size + tag_size, (u8 *) iv);
+        rc = crypto_wait_req(crypto_aead_decrypt(req), &wait);
+    }
+
+    if (rc != 0) {
+        pr_info("Encryption FAILED\n");
+    }
+
+    *data_out_size = data_in_size;
+
+    return rc;
+}
+#endif
+
+int libspdm_aead_prealloced(void *context,
+                            const uint8_t *key, size_t key_size,
+                            const uint8_t *iv, size_t iv_size,
+                            const uint8_t *a_data, size_t a_data_size,
+                            const uint8_t *data_in, size_t data_in_size,
+                            uint8_t *tag, size_t tag_size,
+                            uint8_t *data_out, size_t *data_out_size,
+                            bool enc)
+{
+#ifndef USE_LKCA
+    return -ENODEV;
+#else
+    int rc = 0;
+    struct scatterlist sg_in[SG_AEAD_LEN];
+    struct scatterlist sg_out[SG_AEAD_LEN];
+    struct lkca_aead_ctx *ctx = context;
+
+
+    sg_init_table(sg_in, SG_AEAD_LEN);
+    sg_init_table(sg_out, SG_AEAD_LEN);
+
+    if (!virt_addr_valid(a_data)) {
+        if (a_data_size > BUFFER_SIZE) {
+            return -ENOMEM;
+        }
+        sg_set_buf(&sg_in[SG_AEAD_AAD], ctx->a_data_buffer, a_data_size);
+        sg_set_buf(&sg_out[SG_AEAD_AAD], ctx->a_data_buffer, a_data_size);
+
+        memcpy(ctx->a_data_buffer, a_data, a_data_size);
+    } else {
+        sg_set_buf(&sg_in[SG_AEAD_AAD], a_data, a_data_size);
+        sg_set_buf(&sg_out[SG_AEAD_AAD], a_data, a_data_size);
+    }
+
+    if (!virt_addr_valid(data_in)) {
+        if (data_in_size > BUFFER_SIZE) {
+            return -ENOMEM;
+        }
+        sg_set_buf(&sg_in[SG_AEAD_TEXT], ctx->in_buffer, data_in_size);
+        memcpy(ctx->in_buffer, data_in, data_in_size);
+    } else {
+        sg_set_buf(&sg_in[SG_AEAD_TEXT], data_in, data_in_size);
+    }
+
+    if (!virt_addr_valid(data_out)) {
+        if (data_in_size > BUFFER_SIZE) {
+            return -ENOMEM;
+        }
+        sg_set_buf(&sg_out[SG_AEAD_TEXT], ctx->out_buffer, data_in_size);
+    } else {
+        sg_set_buf(&sg_out[SG_AEAD_TEXT], data_out, data_in_size);
+    }
+
+    // Tag is small enough that memcpy is cheaper than checking if page is virtual
+    if(tag_size > AUTH_TAG_SIZE) {
+        return -ENOMEM;
+    }
+    sg_set_buf(&sg_in[SG_AEAD_SIG], ctx->tag, tag_size);
+    sg_set_buf(&sg_out[SG_AEAD_SIG], ctx->tag, tag_size);
+
+    if(!enc)
+        memcpy(ctx->tag, tag, tag_size);
+
+    rc = lkca_aead_internal(ctx->aead, ctx->req, key, key_size, iv, iv_size,
+                            sg_in, sg_out, a_data_size, data_in_size,
+                            data_out_size, tag_size, enc);
+
+    if (enc) {
+        memcpy(tag, ctx->tag, tag_size);
+    }
+
+    if (!virt_addr_valid(data_out)) {
+        memcpy(data_out, ctx->out_buffer, data_in_size);
+    }
+
+    return rc;
+#endif
+}
+
+int libspdm_aead(const uint8_t *key, size_t key_size,
+                 const uint8_t *iv, size_t iv_size,
+                 const uint8_t *a_data, size_t a_data_size,
+                 const uint8_t *data_in, size_t data_in_size,
+                 const uint8_t *tag, size_t tag_size,
+                 uint8_t *data_out, size_t *data_out_size,
+                 bool enc, char const *alg)
+{
+#ifndef USE_LKCA
+    return -ENODEV;
+#else
+    struct crypto_aead *aead = NULL;
+    struct aead_request *req = NULL;
+    struct scatterlist sg_in[SG_AEAD_LEN];
+    struct scatterlist sg_out[SG_AEAD_LEN];
+    uint8_t *a_data_shadow = NULL;
+    uint8_t *data_in_shadow = NULL;
+    uint8_t *data_out_shadow = NULL;
+    uint8_t *tag_shadow = NULL;
+    int rc = 0;
+
+    aead = crypto_alloc_aead(alg, CRYPTO_ALG_TYPE_AEAD, 0);
+    if (IS_ERR(aead)) {
+        pr_notice("could not allocate AEAD algorithm\n");
+        return -ENODEV;
+    }
+
+    req = aead_request_alloc(aead, GFP_KERNEL);
+    if (req == NULL) {
+        pr_info("could not allocate skcipher request\n");
+        rc = -ENOMEM;
+        goto out;
+    }
+
+    sg_init_table(sg_in, SG_AEAD_LEN);
+    sg_init_table(sg_out, SG_AEAD_LEN);
+
+    if (!virt_addr_valid(a_data)) {
+        a_data_shadow = kmalloc(a_data_size, GFP_KERNEL);
+        if (a_data_shadow == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
+
+        sg_set_buf(&sg_in[SG_AEAD_AAD], a_data_shadow, a_data_size);
+        sg_set_buf(&sg_out[SG_AEAD_AAD], a_data_shadow, a_data_size);
+
+        memcpy(a_data_shadow, a_data, a_data_size);
+    } else {
+        sg_set_buf(&sg_in[SG_AEAD_AAD], a_data, a_data_size);
+        sg_set_buf(&sg_out[SG_AEAD_AAD], a_data, a_data_size);
+    }
+
+    if (!virt_addr_valid(data_in)) {
+        data_in_shadow = kmalloc(data_in_size, GFP_KERNEL);
+        if (data_in_shadow == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
+
+        sg_set_buf(&sg_in[SG_AEAD_TEXT], data_in_shadow, data_in_size);
+
+        memcpy(data_in_shadow, data_in, data_in_size);
+    } else {
+        sg_set_buf(&sg_in[SG_AEAD_TEXT], data_in, data_in_size);
+    }
+
+    if (!virt_addr_valid(data_out)) {
+        data_out_shadow = kmalloc(data_in_size, GFP_KERNEL);
+        if (data_out_shadow == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
+
+        sg_set_buf(&sg_out[SG_AEAD_TEXT], data_out_shadow, data_in_size);
+    } else {
+        sg_set_buf(&sg_out[SG_AEAD_TEXT], data_out, data_in_size);
+    }
+
+    if (!virt_addr_valid(tag)) {
+        tag_shadow = kmalloc(tag_size, GFP_KERNEL);
+        if (tag_shadow == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
+
+        sg_set_buf(&sg_in[SG_AEAD_SIG], tag_shadow, tag_size);
+        sg_set_buf(&sg_out[SG_AEAD_SIG], tag_shadow, tag_size);
+
+        if(!enc)
+            memcpy(tag_shadow, tag, tag_size);
+    } else {
+        sg_set_buf(&sg_in[SG_AEAD_SIG], tag, tag_size);
+        sg_set_buf(&sg_out[SG_AEAD_SIG], tag, tag_size);
+    }
+
+    rc = lkca_aead_internal(aead, req, key, key_size, iv, iv_size,
+                            sg_in, sg_out, a_data_size, data_in_size,
+                            data_out_size, tag_size, enc);
+
+    if (enc && (tag_shadow != NULL))
+        memcpy((uint8_t *) tag, tag_shadow, tag_size);
+
+    if (data_out_shadow != NULL)
+        memcpy(data_out, data_out_shadow, data_in_size);
+
+out:
+    if (a_data_shadow != NULL)
+        kfree(a_data_shadow);
+    if (data_in_shadow != NULL)
+        kfree(data_in_shadow);
+    if (data_out != NULL)
+       kfree(data_out_shadow);
+    if (tag != NULL)
+        kfree(tag_shadow);
+    if (aead != NULL)
+        crypto_free_aead(aead);
+    if (req != NULL)
+        aead_request_free(req);
+    return rc;
+#endif
+}
+
+// Wrapper to make look like libspdm
+bool libspdm_aead_gcm_prealloc(void **context)
+{
+    return libspdm_aead_prealloc(context, "gcm(aes)") == 0;
+}
+
+bool libspdm_aead_aes_gcm_encrypt_prealloc(void *context,
+                                  const uint8_t *key, size_t key_size,
+                                  const uint8_t *iv, size_t iv_size,
+                                  const uint8_t *a_data, size_t a_data_size,
+                                  const uint8_t *data_in, size_t data_in_size,
+                                  uint8_t *tag_out, size_t tag_size,
+                                  uint8_t *data_out, size_t *data_out_size)
+{
+    int32_t ret;
+
+    if (data_in_size > INT_MAX) {
+        return false;
+    }
+    if (a_data_size > INT_MAX) {
+        return false;
+    }
+    if (iv_size != 12) {
+        return false;
+    }
+    switch (key_size) {
+    case 16:
+    case 24:
+    case 32:
+        break;
+    default:
+        return false;
+    }
+    if ((tag_size < 12) || (tag_size > 16)) {
+        return false;
+    }
+    if (data_out_size != NULL) {
+        if ((*data_out_size > INT_MAX) ||
+            (*data_out_size < data_in_size)) {
+            return false;
+        }
+    }
+
+    ret = libspdm_aead_prealloced(context, key, key_size, iv, iv_size,
+                       a_data, a_data_size, data_in, data_in_size,
+                       tag_out, tag_size, data_out, data_out_size, true);
+
+    *data_out_size = data_in_size;
+
+    return ret == 0;
+}
+
+bool libspdm_aead_aes_gcm_decrypt_prealloc(void *context,
+                                  const uint8_t *key, size_t key_size,
+                                  const uint8_t *iv, size_t iv_size,
+                                  const uint8_t *a_data, size_t a_data_size,
+                                  const uint8_t *data_in, size_t data_in_size,
+                                  const uint8_t *tag, size_t tag_size,
+                                  uint8_t *data_out, size_t *data_out_size)
+{
+    int ret;
+    if (data_in_size > INT_MAX) {
+        return false;
+    }
+    if (a_data_size > INT_MAX) {
+        return false;
+    }
+    if (iv_size != 12) {
+        return false;
+    }
+    switch (key_size) {
+    case 16:
+    case 24:
+    case 32:
+        break;
+    default:
+        return false;
+    }
+    if ((tag_size < 12) || (tag_size > 16)) {
+        return false;
+    }
+    if (data_out_size != NULL) {
+        if ((*data_out_size > INT_MAX) ||
+            (*data_out_size < data_in_size)) {
+            return false;
+        }
+    }
+
+    ret = libspdm_aead_prealloced(context, key, key_size, iv, iv_size,
+                       a_data, a_data_size, data_in, data_in_size,
+                       (uint8_t *) tag, tag_size, data_out, data_out_size, false);
+
+    *data_out_size = data_in_size;
+
+    return ret == 0;
+
+}
+
--- a/kernel-open/nvidia/libspdm_aead_aes_gcm.c
+++ b/kernel-open/nvidia/libspdm_aead_aes_gcm.c
@@ -0,0 +1,117 @@
+/*
+* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+* SPDX-License-Identifier: MIT
+*
+* Permission is hereby granted, free of charge, to any person obtaining a
+* copy of this software and associated documentation files (the "Software"),
+* to deal in the Software without restriction, including without limitation
+* the rights to use, copy, modify, merge, publish, distribute, sublicense,
+* and/or sell copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+* DEALINGS IN THE SOFTWARE.
+*
+* Prototypes and checks taken from DMTF: Copyright 2021-2022 DMTF. All rights reserved.
+* License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
+*/
+
+#include "internal_crypt_lib.h"
+
+bool libspdm_aead_aes_gcm_encrypt(const uint8_t *key, size_t key_size,
+                                  const uint8_t *iv, size_t iv_size,
+                                  const uint8_t *a_data, size_t a_data_size,
+                                  const uint8_t *data_in, size_t data_in_size,
+                                  uint8_t *tag_out, size_t tag_size,
+                                  uint8_t *data_out, size_t *data_out_size)
+{
+    int32_t ret;
+
+    if (data_in_size > INT_MAX) {
+        return false;
+    }
+    if (a_data_size > INT_MAX) {
+        return false;
+    }
+    if (iv_size != 12) {
+        return false;
+    }
+    switch (key_size) {
+    case 16:
+    case 24:
+    case 32:
+        break;
+    default:
+        return false;
+    }
+    if ((tag_size < 12) || (tag_size > 16)) {
+        return false;
+    }
+    if (data_out_size != NULL) {
+        if ((*data_out_size > INT_MAX) ||
+            (*data_out_size < data_in_size)) {
+            return false;
+        }
+    }
+
+    ret = libspdm_aead(key, key_size, iv, iv_size, a_data, a_data_size,
+                       data_in, data_in_size, tag_out, tag_size,
+                       data_out, data_out_size, true, "gcm(aes)");
+
+    *data_out_size = data_in_size;
+
+    return ret == 0;
+}
+
+bool libspdm_aead_aes_gcm_decrypt(const uint8_t *key, size_t key_size,
+                                  const uint8_t *iv, size_t iv_size,
+                                  const uint8_t *a_data, size_t a_data_size,
+                                  const uint8_t *data_in, size_t data_in_size,
+                                  const uint8_t *tag, size_t tag_size,
+                                  uint8_t *data_out, size_t *data_out_size)
+{
+    int ret;
+    if (data_in_size > INT_MAX) {
+        return false;
+    }
+    if (a_data_size > INT_MAX) {
+        return false;
+    }
+    if (iv_size != 12) {
+        return false;
+    }
+    switch (key_size) {
+    case 16:
+    case 24:
+    case 32:
+        break;
+    default:
+        return false;
+    }
+    if ((tag_size < 12) || (tag_size > 16)) {
+        return false;
+    }
+    if (data_out_size != NULL) {
+        if ((*data_out_size > INT_MAX) ||
+            (*data_out_size < data_in_size)) {
+            return false;
+        }
+    }
+
+    ret = libspdm_aead(key, key_size, iv, iv_size, a_data, a_data_size,
+                       data_in, data_in_size, tag, tag_size,
+                       data_out, data_out_size, false, "gcm(aes)");
+
+    *data_out_size = data_in_size;
+
+    return ret == 0;
+
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
russellcnv	2a3b58b8c8	535.43.23	2024-01-24 13:46:26 -08:00
Andy Ritger	a86fb2f213	535.43.22	2023-12-19 16:03:49 -08:00
Andy Ritger	6c07c9ac89	535.43.20	2023-12-08 11:25:50 -08:00
russellcnv	0447c5d731	535.43.19	2023-12-01 13:41:12 -08:00
russellcnv	2684f27397	535.43.16	2023-11-06 16:08:36 -08:00
russellcnv	d20b85afd8	535.43.15	2023-10-24 10:48:38 -07:00
russellcnv	ca884aa672	535.43.13	2023-10-11 11:22:16 -07:00
russellcnv	c7d7d95a5a	535.43.11	2023-10-05 23:44:51 -07:00
russellcnv	dfd3023173	535.43.10	2023-09-27 20:59:18 -07:00
russellcnv	17546dbdda	535.43.09	2023-09-01 21:36:45 -07:00
russellcnv	18b7303c54	535.43.08	2023-08-17 13:03:06 -07:00