535.261.03

kernel-open/nvidia-uvm/nvidia-uvm.Kbuild

@@ -87,6 +87,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += mmgrab
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_fault_to_errno
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += folio_test_swapcache
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += page_pgmap
 
 NV_CONFTEST_TYPE_COMPILE_TESTS += backing_dev_info
 NV_CONFTEST_TYPE_COMPILE_TESTS += mm_context_t
@@ -116,3 +117,5 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_interval_notifier
 
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_migrate_vma_setup
+NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present___iowrite64_lo_hi
+NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_make_device_exclusive

kernel-open/nvidia-uvm/uvm_ats_sva.c

@@ -139,7 +139,11 @@ NvU32 smmu_vcmdq_read32(void __iomem *smmu_cmdqv_base, int reg)
 
 static void smmu_vcmdq_write64(void __iomem *smmu_cmdqv_base, int reg, NvU64 val)
 {
+#if NV_IS_EXPORT_SYMBOL_PRESENT___iowrite64_lo_hi
+    __iowrite64_lo_hi(val, SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
+#else
     iowrite64(val, SMMU_VCMDQ_BASE_ADDR(smmu_cmdqv_base, VCMDQ) + reg);
+#endif
 }
 
 // Fix for Bug 4130089: [GH180][r535] WAR for kernel not issuing SMMU
@@ -304,12 +308,13 @@ void uvm_ats_smmu_invalidate_tlbs(uvm_gpu_va_space_t *gpu_va_space, NvU64 addr,
 
 NV_STATUS uvm_ats_sva_add_gpu(uvm_parent_gpu_t *parent_gpu)
 {
+#if NV_IS_EXPORT_SYMBOL_GPL_iommu_dev_enable_feature
     int ret;
 
     ret = iommu_dev_enable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA);
     if (ret)
         return errno_to_nv_status(ret);
-
+#endif
     if (UVM_ATS_SMMU_WAR_REQUIRED())
         return uvm_ats_smmu_war_init(parent_gpu);
     else
@@ -321,7 +326,9 @@ void uvm_ats_sva_remove_gpu(uvm_parent_gpu_t *parent_gpu)
     if (UVM_ATS_SMMU_WAR_REQUIRED())
         uvm_ats_smmu_war_deinit(parent_gpu);
 
+#if NV_IS_EXPORT_SYMBOL_GPL_iommu_dev_disable_feature
     iommu_dev_disable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA);
+#endif
 }
 
 NV_STATUS uvm_ats_sva_bind_gpu(uvm_gpu_va_space_t *gpu_va_space)

kernel-open/nvidia-uvm/uvm_hmm.c

@@ -1864,7 +1864,7 @@ static void fill_dst_pfn(uvm_va_block_t *va_block,
 
     dpage = pfn_to_page(pfn);
     UVM_ASSERT(is_device_private_page(dpage));
-    UVM_ASSERT(dpage->pgmap->owner == &g_uvm_global);
+    UVM_ASSERT(page_pgmap(dpage)->owner == &g_uvm_global);
 
     hmm_mark_gpu_chunk_referenced(va_block, gpu, gpu_chunk);
     UVM_ASSERT(!page_count(dpage));
@@ -2281,6 +2281,39 @@ static void hmm_release_atomic_pages(uvm_va_block_t *va_block,
     }
 }
 
+static int hmm_make_device_exclusive_range(struct mm_struct *mm,
+                                           unsigned long start,
+                                           unsigned long end,
+                                           struct page **pages)
+{
+#if NV_IS_EXPORT_SYMBOL_PRESENT_make_device_exclusive
+    unsigned long addr;
+    int npages = 0;
+
+    for (addr = start; addr < end; addr += PAGE_SIZE) {
+        struct folio *folio;
+        struct page *page;
+
+        page = make_device_exclusive(mm, addr, &g_uvm_global, &folio);
+        if (IS_ERR(page)) {
+            while (npages) {
+                page = pages[--npages];
+                unlock_page(page);
+                put_page(page);
+            }
+            npages = PTR_ERR(page);
+            break;
+        }
+
+        pages[npages++] = page;
+    }
+
+    return npages;
+#else
+    return make_device_exclusive_range(mm, start, end, pages, &g_uvm_global);
+#endif
+}
+
 static NV_STATUS hmm_block_atomic_fault_locked(uvm_processor_id_t processor_id,
                                                uvm_va_block_t *va_block,
                                                uvm_va_block_retry_t *va_block_retry,
@@ -2336,11 +2369,10 @@ static NV_STATUS hmm_block_atomic_fault_locked(uvm_processor_id_t processor_id,
 
     uvm_mutex_unlock(&va_block->lock);
 
-    npages = make_device_exclusive_range(service_context->block_context.mm,
+    npages = hmm_make_device_exclusive_range(service_context->block_context.mm,
         uvm_va_block_cpu_page_address(va_block, region.first),
         uvm_va_block_cpu_page_address(va_block, region.outer - 1) + PAGE_SIZE,
-        pages + region.first,
-        &g_uvm_global);
+        pages + region.first);
 
     uvm_mutex_lock(&va_block->lock);
 

kernel-open/nvidia-uvm/uvm_linux.h

@@ -590,4 +590,9 @@ static inline pgprot_t uvm_pgprot_decrypted(pgprot_t prot)
   #include <asm/page.h>
   #define page_to_virt(x)    __va(PFN_PHYS(page_to_pfn(x)))
 #endif
+
+#ifndef NV_PAGE_PGMAP_PRESENT
+#define page_pgmap(page) (page)->pgmap
+#endif
+
 #endif // _UVM_LINUX_H

kernel-open/nvidia-uvm/uvm_pmm_gpu.c

@@ -3314,7 +3314,7 @@ NvU32 uvm_pmm_gpu_phys_to_virt(uvm_pmm_gpu_t *pmm, NvU64 phys_addr, NvU64 region
 
 static uvm_pmm_gpu_t *devmem_page_to_pmm(struct page *page)
 {
-    return container_of(page->pgmap, uvm_pmm_gpu_t, devmem.pagemap);
+    return container_of(page_pgmap(page), uvm_pmm_gpu_t, devmem.pagemap);
 }
 
 static uvm_gpu_chunk_t *devmem_page_to_chunk_locked(struct page *page)