590.48.01

kernel-open/nvidia/nv-dma.c

@@ -729,7 +729,11 @@ static NvBool nv_dma_use_map_resource
 #endif
     }
 
+#if defined(NV_DMA_MAP_OPS_HAS_MAP_PHYS)
+    return (ops->map_phys != NULL);
+#else
     return (ops->map_resource != NULL);
+#endif
 }
 
 /* DMA-map a peer device's C2C aperture for peer access. */

kernel-open/nvidia/nv-dmabuf.c

@@ -468,9 +468,28 @@ nv_dma_buf_dup_mem_handles(
     return NV_OK;
 
 failed:
-    nv_dma_buf_undup_mem_handles_unlocked(sp, params->index, count, priv);
+    if (!priv->acquire_release_all_gpu_lock_on_dup)
+    {
+        //
+        // Undup requires taking all-GPUs lock.
+        // So if single GPU lock was taken,
+        // release it first so all-GPUs lock can be taken in
+        // nv_dma_buf_undup_mem_handles().
+        //
+        nv_dma_buf_release_gpu_lock(sp, priv);
 
-    nv_dma_buf_release_gpu_lock(sp, priv);
+        nv_dma_buf_undup_mem_handles(sp, params->index, count, priv);
+    }
+    else
+    {
+        //
+        // Here, all-GPUs lock is already taken, so undup the handles under
+        // the unlocked version of the function and then release the locks.
+        //
+        nv_dma_buf_undup_mem_handles_unlocked(sp, params->index, count, priv);
+
+        nv_dma_buf_release_gpu_lock(sp, priv);
+    }
 
 unlock_api_lock:
     rm_release_api_lock(sp);

kernel-open/nvidia/nv-mmap.c

@@ -72,7 +72,7 @@ nvidia_vma_open(struct vm_area_struct *vma)
 
     if (at != NULL)
     {
-        NV_ATOMIC_INC(at->usage_count);
+        atomic64_inc(&at->usage_count);
 
         NV_PRINT_AT(NV_DBG_MEMINFO, at);
     }
@@ -414,7 +414,7 @@ static int nvidia_mmap_sysmem(
     int ret = 0;
     unsigned long start = 0;
 
-    NV_ATOMIC_INC(at->usage_count);
+    atomic64_inc(&at->usage_count);
 
     start = vma->vm_start;
     for (j = page_index; j < (page_index + pages); j++)
@@ -450,7 +450,7 @@ static int nvidia_mmap_sysmem(
 
         if (ret)
         {
-            NV_ATOMIC_DEC(at->usage_count);
+            atomic64_dec(&at->usage_count);
             nv_printf(NV_DBG_ERRORS,
                       "NVRM: Userspace mapping creation failed [%d]!\n", ret);
             return -EAGAIN;

kernel-open/nvidia/nv-pci.c

@@ -244,7 +244,11 @@ static int nv_resize_pcie_bars(struct pci_dev *pci_dev) {
 
 resize:
     /* Attempt to resize BAR1 to the largest supported size */
+#if defined(NV_PCI_RESIZE_RESOURCE_HAS_EXCLUDE_BARS_ARG)
+    r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size, 0);
+#else
     r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size);
+#endif
 
     if (r) {
         if (r == -ENOSPC)
@@ -1687,11 +1691,6 @@ nv_pci_probe
     nv_printf(NV_DBG_SETUP, "NVRM: probing 0x%x 0x%x, class 0x%x\n",
         pci_dev->vendor, pci_dev->device, pci_dev->class);
 
-    if (nv_kmem_cache_alloc_stack(&sp) != 0)
-    {
-        return -1;
-    }
-
 #ifdef NV_PCI_SRIOV_SUPPORT
     if (pci_dev->is_virtfn)
     {
@@ -1707,21 +1706,25 @@ nv_pci_probe
                       "since IOMMU is not present on the system.\n",
                        NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
                        NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
-            goto failed;
+            return -1;
         }
 
-        nv_kmem_cache_free_stack(sp);
         return 0;
 #else
         nv_printf(NV_DBG_ERRORS, "NVRM: Ignoring probe for VF %04x:%02x:%02x.%x ",
                   NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
                   NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
 
-        goto failed;
+        return -1;
 #endif /* NV_VGPU_KVM_BUILD */
     }
 #endif /* NV_PCI_SRIOV_SUPPORT */
 
+    if (nv_kmem_cache_alloc_stack(&sp) != 0)
+    {
+        return -1;
+    }
+
     if (!rm_wait_for_bar_firewall(
                 sp,
                 NV_PCI_DOMAIN_NUMBER(pci_dev),
@@ -2178,7 +2181,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
      * For eGPU, fall off the bus along with clients active is a valid scenario.
      * Hence skipping the sanity check for eGPU.
      */
-    if ((NV_ATOMIC_READ(nvl->usage_count) != 0) && !(nv->is_external_gpu))
+    if ((atomic64_read(&nvl->usage_count) != 0) && !(nv->is_external_gpu))
     {
         nv_printf(NV_DBG_ERRORS,
                   "NVRM: Attempting to remove device %04x:%02x:%02x.%x with non-zero usage count!\n",
@@ -2189,7 +2192,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
          * We can't return from this function without corrupting state, so we wait for
          * the usage count to go to zero.
          */
-        while (NV_ATOMIC_READ(nvl->usage_count) != 0)
+        while (atomic64_read(&nvl->usage_count) != 0)
         {
 
             /*
@@ -2267,7 +2270,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
         nvl->sysfs_config_file = NULL;
     }
 
-    if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+    if (atomic64_read(&nvl->usage_count) == 0)
     {
         nv_lock_destroy_locks(sp, nv);
     }
@@ -2283,7 +2286,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
 
     num_nv_devices--;
 
-    if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+    if (atomic64_read(&nvl->usage_count) == 0)
     {
         NV_PCI_DISABLE_DEVICE(pci_dev);
         NV_KFREE(nvl, sizeof(nv_linux_state_t));

kernel-open/nvidia/nv-procfs.c

@@ -890,7 +890,7 @@ nv_procfs_close_unbind_lock(
         down(&nvl->ldata_lock);
         if ((value == 1) && !(nv->flags & NV_FLAG_UNBIND_LOCK))
         {
-            if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+            if (atomic64_read(&nvl->usage_count) == 0)
                 rm_unbind_lock(sp, nv);
 
             if (nv->flags & NV_FLAG_UNBIND_LOCK)

kernel-open/nvidia/nv.c

@@ -419,7 +419,7 @@ nv_alloc_t *nvos_create_alloc(
         return NULL;
     }
 
-    NV_ATOMIC_SET(at->usage_count, 0);
+    atomic64_set(&at->usage_count, 0);
     at->pid = os_get_current_process();
     at->dev = dev;
 
@@ -434,7 +434,7 @@ int nvos_free_alloc(
     if (at == NULL)
         return -1;
 
-    if (NV_ATOMIC_READ(at->usage_count))
+    if (atomic64_read(&at->usage_count))
         return 1;
 
     kvfree(at->page_table);
@@ -1656,13 +1656,10 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
         return -ENODEV;
     }
 
-    if (unlikely(NV_ATOMIC_READ(nvl->usage_count) >= NV_S32_MAX))
-        return -EMFILE;
-
     if ( ! (nv->flags & NV_FLAG_OPEN))
     {
         /* Sanity check: !NV_FLAG_OPEN requires usage_count == 0 */
-        if (NV_ATOMIC_READ(nvl->usage_count) != 0)
+        if (atomic64_read(&nvl->usage_count) != 0)
         {
             NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
                           "Minor device %u is referenced without being open!\n",
@@ -1684,7 +1681,7 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
 
     nv_assert_not_in_gpu_exclusion_list(sp, nv);
 
-    NV_ATOMIC_INC(nvl->usage_count);
+    atomic64_inc(&nvl->usage_count);
 
     return 0;
 }
@@ -2100,7 +2097,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
 {
     nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
 
-    if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+    if (atomic64_read(&nvl->usage_count) == 0)
     {
         nv_printf(NV_DBG_ERRORS,
                   "NVRM: Attempting to close unopened minor device %u!\n",
@@ -2109,7 +2106,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
         return;
     }
 
-    if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
+    if (atomic64_dec_and_test(&nvl->usage_count))
         nv_stop_device(nv, sp);
 }
 
@@ -2154,7 +2151,7 @@ nvidia_close_callback(
     nv_close_device(nv, sp);
 
     bRemove = (!NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv)) &&
-              (NV_ATOMIC_READ(nvl->usage_count) == 0) &&
+              (atomic64_read(&nvl->usage_count) == 0) &&
               rm_get_device_remove_flag(sp, nv->gpu_id);
 
     nv_free_file_private(nvlfp);
@@ -2173,7 +2170,7 @@ nvidia_close_callback(
      * any cleanup related to linux layer locks and nv linux state struct.
      * nvidia_pci_remove when scheduled will do necessary cleanup.
      */
-    if ((NV_ATOMIC_READ(nvl->usage_count) == 0) && nv->removed)
+    if ((atomic64_read(&nvl->usage_count) == 0) && nv->removed)
     {
         nv_lock_destroy_locks(sp, nv);
         NV_KFREE(nvl, sizeof(nv_linux_state_t));
@@ -2693,7 +2690,7 @@ nvidia_ioctl(
                      * Only the current client should have an open file
                      * descriptor for the device, to allow safe offlining.
                      */
-                    if (NV_ATOMIC_READ(nvl->usage_count) > 1)
+                    if (atomic64_read(&nvl->usage_count) > 1)
                     {
                         status = -EBUSY;
                         goto unlock;
@@ -3082,12 +3079,12 @@ nvidia_ctl_open(
     /* save the nv away in file->private_data */
     nvlfp->nvptr = nvl;
 
-    if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+    if (atomic64_read(&nvl->usage_count) == 0)
     {
         nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
     }
 
-    NV_ATOMIC_INC(nvl->usage_count);
+    atomic64_inc(&nvl->usage_count);
     up(&nvl->ldata_lock);
 
     return 0;
@@ -3112,7 +3109,7 @@ nvidia_ctl_close(
     nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_close\n");
 
     down(&nvl->ldata_lock);
-    if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
+    if (atomic64_dec_and_test(&nvl->usage_count))
     {
         nv->flags &= ~NV_FLAG_OPEN;
     }
@@ -3275,7 +3272,7 @@ nv_alias_pages(
 
     at->guest_id = guest_id;
     *priv_data = at;
-    NV_ATOMIC_INC(at->usage_count);
+    atomic64_inc(&at->usage_count);
 
     NV_PRINT_AT(NV_DBG_MEMINFO, at);
 
@@ -3588,7 +3585,7 @@ NV_STATUS NV_API_CALL nv_register_sgt(
 
     at->order = get_order(at->num_pages * PAGE_SIZE);
 
-    NV_ATOMIC_INC(at->usage_count);
+    atomic64_inc(&at->usage_count);
 
     *priv_data = at;
 
@@ -3619,7 +3616,7 @@ void NV_API_CALL nv_unregister_sgt(
         *import_priv = at->import_priv;
     }
 
-    if (NV_ATOMIC_DEC_AND_TEST(at->usage_count))
+    if (atomic64_dec_and_test(&at->usage_count))
     {
         nvos_free_alloc(at);
     }
@@ -3892,7 +3889,7 @@ NV_STATUS NV_API_CALL nv_alloc_pages(
     }
 
     *priv_data = at;
-    NV_ATOMIC_INC(at->usage_count);
+    atomic64_inc(&at->usage_count);
 
     NV_PRINT_AT(NV_DBG_MEMINFO, at);
 
@@ -3928,7 +3925,7 @@ NV_STATUS NV_API_CALL nv_free_pages(
      * This is described in greater detail in the comments above the
      * nvidia_vma_(open|release)() callbacks in nv-mmap.c.
      */
-    if (!NV_ATOMIC_DEC_AND_TEST(at->usage_count))
+    if (!atomic64_dec_and_test(&at->usage_count))
         return NV_OK;
 
     if (!at->flags.guest && !at->import_sgt)
@@ -3957,7 +3954,7 @@ NvBool nv_lock_init_locks
     NV_INIT_MUTEX(&nvl->mmap_lock);
     NV_INIT_MUTEX(&nvl->open_q_lock);
 
-    NV_ATOMIC_SET(nvl->usage_count, 0);
+    atomic64_set(&nvl->usage_count, 0);
 
     if (!rm_init_event_locks(sp, nv))
         return NV_FALSE;

kernel-open/nvidia/nvidia.Kbuild

@@ -141,6 +141,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_get
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += devm_of_icc_get
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_put
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_set_bw
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_map_ops_has_map_phys
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map_atomic
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_attachment_has_peer2peer
@@ -159,6 +160,8 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += ioasid_get
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += mm_pasid_drop
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
 NV_CONFTEST_FUNCTION_COMPILE_TESTS += shrinker_alloc
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_flags_set
+NV_CONFTEST_FUNCTION_COMPILE_TESTS += vma_flags_set_word
 
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_gpl_sme_active
 NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_swiotlb_map_sg_attrs
@@ -206,7 +209,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += remove_memory_has_nid_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
 NV_CONFTEST_TYPE_COMPILE_TESTS += pci_driver_has_driver_managed_dma
-NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
 NV_CONFTEST_TYPE_COMPILE_TESTS += memory_failure_queue_has_trapno_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += foll_longterm_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += bus_type_has_iommu_ops
@@ -220,6 +222,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += devfreq_has_suspend_freq
 NV_CONFTEST_TYPE_COMPILE_TESTS += has_enum_pidtype_tgid
 NV_CONFTEST_TYPE_COMPILE_TESTS += bpmp_mrq_has_strap_set
 NV_CONFTEST_TYPE_COMPILE_TESTS += register_shrinker_has_format_arg
+NV_CONFTEST_TYPE_COMPILE_TESTS += pci_resize_resource_has_exclude_bars_arg
 
 NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
 NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build

kernel-open/nvidia/os-interface.c

@@ -371,7 +371,7 @@ NvBool NV_API_CALL os_semaphore_may_sleep(void)
 
 NvBool NV_API_CALL os_is_isr(void)
 {
-    return (in_irq());
+    return (nv_in_hardirq());
 }
 
 // return TRUE if the caller is the super-user