570.148.08

kernel-open/Kbuild

@@ -86,7 +86,7 @@ EXTRA_CFLAGS += -I$(src)/common/inc
 EXTRA_CFLAGS += -I$(src)
 EXTRA_CFLAGS += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
 EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
-EXTRA_CFLAGS += -DNV_VERSION_STRING=\"570.144\"
+EXTRA_CFLAGS += -DNV_VERSION_STRING=\"570.148.08\"
 
 ifneq ($(SYSSRCHOST1X),)
  EXTRA_CFLAGS += -I$(SYSSRCHOST1X)

kernel-open/conftest.sh

@@ -3132,6 +3132,21 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_FOLL_LONGTERM_PRESENT" "" "types"
         ;;
 
+        has_enum_pidtype_tgid)
+            # Determine if PIDTYPE_TGID is present in the kernel as an enum
+            #
+            # Added by commit 6883f81aac6f ("pid: Implement PIDTYPE_TGID")
+            # in v4.19
+            #
+            CODE="
+            #include <linux/pid.h>
+
+            enum pid_type type = PIDTYPE_TGID;
+            "
+
+            compile_check_conftest "$CODE" "NV_HAS_ENUM_PIDTYPE_TGID" "" "types"
+        ;;
+
         vfio_pin_pages_has_vfio_device_arg)
             #
             # Determine if vfio_pin_pages() kABI accepts "struct vfio_device *"

kernel-open/nvidia-uvm/uvm_blackwell_host.c

@@ -254,3 +254,31 @@ void uvm_hal_blackwell_host_tlb_invalidate_test(uvm_push_t *push,
                                    HWVALUE(C96F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
     }
 }
+
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb100(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries)
+{
+    if (parent_gpu->rm_info.accessCntrBufferCount > 1) {
+        NvU32 i;
+
+        for (i = 0; i < num_entries; i++) {
+            const uvm_access_counter_buffer_entry_t *entry = buffer_entries[i];
+
+            // The LSb identifies the die ID.
+            if ((entry->tag & 0x1) == 1)
+                return UVM_ACCESS_COUNTER_CLEAR_OP_ALL;
+        }
+    }
+
+    return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
+}
+
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb20x(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries)
+{
+    return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
+}

kernel-open/nvidia-uvm/uvm_gpu.c

@@ -1197,6 +1197,8 @@ static NV_STATUS alloc_parent_gpu(const NvProcessorUuid *gpu_uuid,
     uvm_sema_init(&parent_gpu->isr.replayable_faults.service_lock, 1, UVM_LOCK_ORDER_ISR);
     uvm_sema_init(&parent_gpu->isr.non_replayable_faults.service_lock, 1, UVM_LOCK_ORDER_ISR);
     uvm_mutex_init(&parent_gpu->access_counters_enablement_lock, UVM_LOCK_ORDER_ACCESS_COUNTERS);
+    uvm_mutex_init(&parent_gpu->access_counters_clear_tracker_lock, UVM_LOCK_ACCESS_COUNTERS_CLEAR_OPS);
+    uvm_tracker_init(&parent_gpu->access_counters_clear_tracker);
     uvm_spin_lock_irqsave_init(&parent_gpu->isr.interrupts_lock, UVM_LOCK_ORDER_LEAF);
     uvm_spin_lock_init(&parent_gpu->instance_ptr_table_lock, UVM_LOCK_ORDER_LEAF);
     uvm_rb_tree_init(&parent_gpu->instance_ptr_table);
@@ -1214,6 +1216,7 @@ static NV_STATUS alloc_parent_gpu(const NvProcessorUuid *gpu_uuid,
     return NV_OK;
 
 cleanup:
+    uvm_tracker_deinit(&parent_gpu->access_counters_clear_tracker);
     uvm_kvfree(parent_gpu);
 
     return status;
@@ -1644,19 +1647,12 @@ static void sync_parent_gpu_trackers(uvm_parent_gpu_t *parent_gpu,
 
     // Sync the access counter clear tracker too.
     if (parent_gpu->access_counters_supported && parent_gpu->access_counter_buffer) {
-        NvU32 notif_buf_index;
-        for (notif_buf_index = 0; notif_buf_index < parent_gpu->rm_info.accessCntrBufferCount; notif_buf_index++) {
-            uvm_access_counter_buffer_t *access_counters = &parent_gpu->access_counter_buffer[notif_buf_index];
+        uvm_mutex_lock(&parent_gpu->access_counters_clear_tracker_lock);
+        status = uvm_tracker_wait(&parent_gpu->access_counters_clear_tracker);
+        uvm_mutex_unlock(&parent_gpu->access_counters_clear_tracker_lock);
 
-            if (access_counters->rm_info.accessCntrBufferHandle != 0) {
-                uvm_access_counters_isr_lock(access_counters);
-                status = uvm_tracker_wait(&access_counters->clear_tracker);
-                uvm_access_counters_isr_unlock(access_counters);
-
-                if (status != NV_OK)
-                    UVM_ASSERT(status == uvm_global_get_status());
-            }
-        }
+        if (status != NV_OK)
+            UVM_ASSERT(status == uvm_global_get_status());
     }
 }
 
@@ -1787,6 +1783,8 @@ static void uvm_parent_gpu_destroy(nv_kref_t *nv_kref)
     for_each_sub_processor_index(sub_processor_index)
         UVM_ASSERT(!parent_gpu->gpus[sub_processor_index]);
 
+    uvm_tracker_deinit(&parent_gpu->access_counters_clear_tracker);
+
     uvm_kvfree(parent_gpu);
 }
 
@@ -2881,6 +2879,9 @@ static NV_STATUS gpu_retain_by_uuid_locked(const NvProcessorUuid *gpu_uuid,
     if (status != NV_OK)
         goto error_unregister;
 
+    if (gpu_info->accessCntrBufferCount > 1)
+        gpu_info->accessCntrBufferCount = 1;
+
     if (parent_gpu != NULL) {
         // If the UUID has been seen before, and if SMC is enabled, then check
         // if this specific partition has been seen previously. The UUID-based

kernel-open/nvidia-uvm/uvm_gpu.h

@@ -522,10 +522,6 @@ struct uvm_access_counter_buffer_struct
     // PCIe
     NvU32 cached_put;
 
-    // Tracker used to aggregate access counters clear operations, needed for
-    // GPU removal
-    uvm_tracker_t clear_tracker;
-
     // Current access counter configuration. During normal operation this
     // information is computed once during GPU initialization. However, tests
     // may override it to try different configuration values.
@@ -1205,6 +1201,11 @@ struct uvm_parent_gpu_struct
     uvm_access_counter_buffer_t *access_counter_buffer;
     uvm_mutex_t access_counters_enablement_lock;
 
+    // Tracker used to aggregate access counters clear operations, needed for
+    // GPU removal. It is only used when supports_access_counters is set.
+    uvm_tracker_t access_counters_clear_tracker;
+    uvm_mutex_t access_counters_clear_tracker_lock;
+
     // Number of uTLBs per GPC. This information is only valid on Pascal+ GPUs.
     NvU32 utlb_per_gpc_count;
 

kernel-open/nvidia-uvm/uvm_gpu_access_counters.c

@@ -216,38 +216,19 @@ static NV_STATUS config_granularity_to_bytes(UVM_ACCESS_COUNTER_GRANULARITY gran
     return NV_OK;
 }
 
-// Clear the access counter notifications and add it to the per-GPU
-// per-notification-buffer clear tracker.
-static NV_STATUS access_counter_clear_notifications(uvm_gpu_t *gpu,
-                                                    uvm_access_counter_buffer_t *access_counters,
-                                                    uvm_access_counter_buffer_entry_t **notification_start,
-                                                    NvU32 num_notifications)
+static NV_STATUS parent_gpu_clear_tracker_wait(uvm_parent_gpu_t *parent_gpu)
 {
-    NvU32 i;
     NV_STATUS status;
-    uvm_push_t push;
 
-    status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_MEMOPS, &push, "Clear access counter batch");
-    if (status != NV_OK) {
-        UVM_ERR_PRINT("Error creating push to clear access counters: %s, GPU %s, notif buf index %u\n",
-                      nvstatusToString(status),
-                      uvm_gpu_name(gpu),
-                      access_counters->index);
-        return status;
-    }
+    uvm_mutex_lock(&parent_gpu->access_counters_clear_tracker_lock);
+    status = uvm_tracker_wait(&parent_gpu->access_counters_clear_tracker);
+    uvm_mutex_unlock(&parent_gpu->access_counters_clear_tracker_lock);
 
-    for (i = 0; i < num_notifications; i++)
-        gpu->parent->host_hal->access_counter_clear_targeted(&push, notification_start[i]);
-
-    uvm_push_end(&push);
-
-    uvm_tracker_remove_completed(&access_counters->clear_tracker);
-
-    return uvm_tracker_add_push_safe(&access_counters->clear_tracker, &push);
+    return status;
 }
 
-// Clear all access counters and add the operation to the per-GPU
-// per-notification-buffer clear tracker
+// Clear all access counters and add the operation to the per-GPU clear
+// tracker.
 static NV_STATUS access_counter_clear_all(uvm_gpu_t *gpu, uvm_access_counter_buffer_t *access_counters)
 {
     NV_STATUS status;
@@ -269,8 +250,52 @@ static NV_STATUS access_counter_clear_all(uvm_gpu_t *gpu, uvm_access_counter_buf
 
     uvm_push_end(&push);
 
-    uvm_tracker_remove_completed(&access_counters->clear_tracker);
-    return uvm_tracker_add_push_safe(&access_counters->clear_tracker, &push);
+    uvm_mutex_lock(&gpu->parent->access_counters_clear_tracker_lock);
+    uvm_tracker_remove_completed(&gpu->parent->access_counters_clear_tracker);
+    status = uvm_tracker_add_push_safe(&gpu->parent->access_counters_clear_tracker, &push);
+    uvm_mutex_unlock(&gpu->parent->access_counters_clear_tracker_lock);
+
+    return status;
+}
+
+// Clear the access counter notifications and add it to the per-GPU clear
+// tracker.
+static NV_STATUS access_counter_clear_notifications(uvm_gpu_t *gpu,
+                                                    uvm_access_counter_buffer_t *access_counters,
+                                                    uvm_access_counter_buffer_entry_t **notification_start,
+                                                    NvU32 num_notifications)
+{
+    NvU32 i;
+    NV_STATUS status;
+    uvm_push_t push;
+    uvm_access_counter_clear_op_t clear_op;
+
+    clear_op = gpu->parent->host_hal->access_counter_query_clear_op(gpu->parent, notification_start, num_notifications);
+    if (clear_op == UVM_ACCESS_COUNTER_CLEAR_OP_ALL)
+        return access_counter_clear_all(gpu, access_counters);
+
+    UVM_ASSERT(clear_op == UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED);
+
+    status = uvm_push_begin(gpu->channel_manager, UVM_CHANNEL_TYPE_MEMOPS, &push, "Clear access counter batch");
+    if (status != NV_OK) {
+        UVM_ERR_PRINT("Error creating push to clear access counters: %s, GPU %s, notif buf index %u\n",
+                      nvstatusToString(status),
+                      uvm_gpu_name(gpu),
+                      access_counters->index);
+        return status;
+    }
+
+    for (i = 0; i < num_notifications; i++)
+        gpu->parent->host_hal->access_counter_clear_targeted(&push, notification_start[i]);
+
+    uvm_push_end(&push);
+
+    uvm_mutex_lock(&gpu->parent->access_counters_clear_tracker_lock);
+    uvm_tracker_remove_completed(&gpu->parent->access_counters_clear_tracker);
+    status = uvm_tracker_add_push_safe(&gpu->parent->access_counters_clear_tracker, &push);
+    uvm_mutex_unlock(&gpu->parent->access_counters_clear_tracker_lock);
+
+    return status;
 }
 
 bool uvm_parent_gpu_access_counters_pending(uvm_parent_gpu_t *parent_gpu, NvU32 index)
@@ -373,8 +398,6 @@ NV_STATUS uvm_parent_gpu_init_access_counters(uvm_parent_gpu_t *parent_gpu, NvU3
     access_counters->notifications_ignored_count = 0;
     access_counters->test.reconfiguration_owner = NULL;
 
-    uvm_tracker_init(&access_counters->clear_tracker);
-
     access_counters->max_notifications = access_counters->rm_info.bufferSize /
                                          parent_gpu->access_counter_buffer_hal->entry_size(parent_gpu);
 
@@ -442,8 +465,6 @@ void uvm_parent_gpu_deinit_access_counters(uvm_parent_gpu_t *parent_gpu, NvU32 n
         UVM_ASSERT(status == NV_OK);
 
         access_counters->rm_info.accessCntrBufferHandle = 0;
-        uvm_tracker_deinit(&access_counters->clear_tracker);
-
         uvm_kvfree(batch_context->notification_cache);
         uvm_kvfree(batch_context->notifications);
         batch_context->notification_cache = NULL;
@@ -487,7 +508,7 @@ static NV_STATUS access_counters_take_ownership(uvm_gpu_t *gpu, NvU32 index, con
     if (status != NV_OK)
         goto error;
 
-    status = uvm_tracker_wait(&access_counters->clear_tracker);
+    status = parent_gpu_clear_tracker_wait(gpu->parent);
     if (status != NV_OK)
         goto error;
 
@@ -521,7 +542,7 @@ static void access_counters_yield_ownership(uvm_parent_gpu_t *parent_gpu, NvU32
     UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.access_counters[index].service_lock));
 
     // Wait for any pending clear operation before releasing ownership
-    status = uvm_tracker_wait(&access_counters->clear_tracker);
+    status = parent_gpu_clear_tracker_wait(parent_gpu);
     if (status != NV_OK)
         UVM_ASSERT(status == uvm_global_get_status());
 
@@ -1750,28 +1771,21 @@ NV_STATUS uvm_api_clear_all_access_counters(UVM_CLEAR_ALL_ACCESS_COUNTERS_PARAMS
     uvm_va_space_up_read(va_space);
 
     for_each_gpu_in_mask(gpu, retained_gpus) {
-        NvU32 notif_buf_index;
+        uvm_access_counter_buffer_t *access_counters;
 
         if (!gpu->parent->access_counters_supported)
             continue;
 
-        for (notif_buf_index = 0; notif_buf_index < gpu->parent->rm_info.accessCntrBufferCount; notif_buf_index++) {
-            uvm_access_counter_buffer_t *access_counters = parent_gpu_access_counter_buffer_get(gpu->parent,
-                                                                                                notif_buf_index);
-            uvm_access_counters_isr_lock(access_counters);
+        // clear_all affects all the notification buffers, we issue it for
+        // the notif_buf_index 0.
+        access_counters = parent_gpu_access_counter_buffer_get(gpu->parent, 0);
+        status = access_counter_clear_all(gpu, access_counters);
+        if (status == NV_OK)
+            status = parent_gpu_clear_tracker_wait(gpu->parent);
 
-            // Access counters are not enabled. Nothing to clear.
-            if (gpu->parent->isr.access_counters[notif_buf_index].handling_ref_count) {
-                status = access_counter_clear_all(gpu, access_counters);
-                if (status == NV_OK)
-                    status = uvm_tracker_wait(&access_counters->clear_tracker);
-            }
-
-            uvm_access_counters_isr_unlock(access_counters);
-
-            if (status != NV_OK)
-                break;
-        }
+        // Break the loop if clear_all failed in any of the retained gpus.
+        if (status != NV_OK)
+            break;
     }
 
     for_each_gpu_in_mask(gpu, retained_gpus)
@@ -2054,7 +2068,9 @@ NV_STATUS uvm_test_reset_access_counters(UVM_TEST_RESET_ACCESS_COUNTERS_PARAMS *
     NV_STATUS status = NV_OK;
     uvm_gpu_t *gpu = NULL;
     uvm_va_space_t *va_space = uvm_va_space_get(filp);
+    uvm_access_counter_buffer_t *access_counters;
     NvU32 notif_buf_index;
+    NvBool index0_state;
 
     if (params->mode >= UVM_TEST_ACCESS_COUNTER_RESET_MODE_MAX)
         return NV_ERR_INVALID_ARGUMENT;
@@ -2068,51 +2084,52 @@ NV_STATUS uvm_test_reset_access_counters(UVM_TEST_RESET_ACCESS_COUNTERS_PARAMS *
         goto exit_release_gpu;
     }
 
-    for (notif_buf_index = 0;
-         notif_buf_index < gpu->parent->rm_info.accessCntrBufferCount && status == NV_OK;
-         notif_buf_index++) {
-        uvm_access_counter_buffer_t *access_counters = parent_gpu_access_counter_buffer_get(gpu->parent,
-                                                                                            notif_buf_index);
+    uvm_mutex_lock(&gpu->parent->access_counters_enablement_lock);
 
-        uvm_access_counters_isr_lock(access_counters);
+    // Access counters not enabled. Nothing to reset
+    if (!uvm_parent_processor_mask_test(&va_space->access_counters_enabled_processors, gpu->parent->id)) {
+        uvm_mutex_unlock(&gpu->parent->access_counters_enablement_lock);
+        goto exit_release_gpu;
+    }
 
-        // Access counters not enabled. Nothing to reset
-        if (gpu->parent->isr.access_counters[notif_buf_index].handling_ref_count == 0)
-            goto exit_isr_unlock;
+    uvm_mutex_unlock(&gpu->parent->access_counters_enablement_lock);
 
-        if (params->mode == UVM_TEST_ACCESS_COUNTER_RESET_MODE_ALL) {
-            status = access_counter_clear_all(gpu, access_counters);
-        }
-        else {
-            uvm_access_counter_buffer_entry_t entry = { 0 };
-            uvm_access_counter_buffer_entry_t *notification = &entry;
+    // Clear operations affect all notification buffers, we use the
+    // notif_buf_index = 0;
+    notif_buf_index = 0;
+    access_counters = parent_gpu_access_counter_buffer_get(gpu->parent, notif_buf_index);
 
-            entry.bank = params->bank;
-            entry.tag = params->tag;
+    uvm_access_counters_isr_lock(access_counters);
 
-            status = access_counter_clear_notifications(gpu, access_counters, &notification, 1);
-        }
+    // Recheck access counters are enabled.
+    index0_state = gpu->parent->isr.access_counters[notif_buf_index].handling_ref_count == 0;
+    if (index0_state) {
+        NvU32 i;
 
-        if (status == NV_OK)
-            status = uvm_tracker_wait(&access_counters->clear_tracker);
+        for (i = notif_buf_index + 1; i < gpu->parent->rm_info.accessCntrBufferCount; i++)
+            UVM_ASSERT((gpu->parent->isr.access_counters[i].handling_ref_count == 0) == index0_state);
+
+        goto exit_isr_unlock;
+    }
+
+    if (params->mode == UVM_TEST_ACCESS_COUNTER_RESET_MODE_ALL) {
+        status = access_counter_clear_all(gpu, access_counters);
+    }
+    else {
+        uvm_access_counter_buffer_entry_t entry = { 0 };
+        uvm_access_counter_buffer_entry_t *notification = &entry;
+
+        entry.bank = params->bank;
+        entry.tag = params->tag;
+
+        status = access_counter_clear_notifications(gpu, access_counters, &notification, 1);
+    }
+
+    if (status == NV_OK)
+        status = parent_gpu_clear_tracker_wait(gpu->parent);
 
 exit_isr_unlock:
-        uvm_access_counters_isr_unlock(access_counters);
-
-        // We only need to clear_all() once.
-        if (params->mode == UVM_TEST_ACCESS_COUNTER_RESET_MODE_ALL) {
-            NvU32 i;
-
-            // Early exit of the main loop; since we only need to clear_all()
-            // once. Check that all the remaining notification buffers have
-            // access counters in same state.
-            NvBool index0_state = (gpu->parent->isr.access_counters[notif_buf_index].handling_ref_count == 0);
-            for (i = notif_buf_index + 1; i < gpu->parent->rm_info.accessCntrBufferCount; i++)
-                UVM_ASSERT((gpu->parent->isr.access_counters[i].handling_ref_count == 0) == index0_state);
-
-            break;
-        }
-    }
+    uvm_access_counters_isr_unlock(access_counters);
 
 exit_release_gpu:
     uvm_gpu_release(gpu);

kernel-open/nvidia-uvm/uvm_hal.c

@@ -218,6 +218,7 @@ static uvm_hal_class_ops_t host_table[] =
             .clear_faulted_channel_register = uvm_hal_maxwell_host_clear_faulted_channel_register_unsupported,
             .access_counter_clear_all = uvm_hal_maxwell_access_counter_clear_all_unsupported,
             .access_counter_clear_targeted = uvm_hal_maxwell_access_counter_clear_targeted_unsupported,
+            .access_counter_query_clear_op = uvm_hal_maxwell_access_counter_query_clear_op_unsupported,
             .get_time = uvm_hal_maxwell_get_time,
         }
     },
@@ -269,6 +270,7 @@ static uvm_hal_class_ops_t host_table[] =
             .tlb_invalidate_test = uvm_hal_turing_host_tlb_invalidate_test,
             .access_counter_clear_all = uvm_hal_turing_access_counter_clear_all,
             .access_counter_clear_targeted = uvm_hal_turing_access_counter_clear_targeted,
+            .access_counter_query_clear_op = uvm_hal_turing_access_counter_query_clear_op,
         }
     },
     {
@@ -308,12 +310,15 @@ static uvm_hal_class_ops_t host_table[] =
             .tlb_invalidate_all = uvm_hal_blackwell_host_tlb_invalidate_all,
             .tlb_invalidate_va = uvm_hal_blackwell_host_tlb_invalidate_va,
             .tlb_invalidate_test = uvm_hal_blackwell_host_tlb_invalidate_test,
+            .access_counter_query_clear_op = uvm_hal_blackwell_access_counter_query_clear_op_gb100,
         }
     },
     {
         .id = BLACKWELL_CHANNEL_GPFIFO_B,
         .parent_id = BLACKWELL_CHANNEL_GPFIFO_A,
-        .u.host_ops = {}
+        .u.host_ops = {
+            .access_counter_query_clear_op = uvm_hal_blackwell_access_counter_query_clear_op_gb20x
+        }
     },
 };
 

kernel-open/nvidia-uvm/uvm_hal.h

@@ -703,6 +703,10 @@ typedef NvU32 (*uvm_hal_access_counter_buffer_entry_size_t)(uvm_parent_gpu_t *pa
 typedef void (*uvm_hal_access_counter_clear_all_t)(uvm_push_t *push);
 typedef void (*uvm_hal_access_counter_clear_targeted_t)(uvm_push_t *push,
                                                         const uvm_access_counter_buffer_entry_t *buffer_entry);
+typedef uvm_access_counter_clear_op_t
+        (*uvm_hal_access_counter_query_clear_op_t)(uvm_parent_gpu_t *parent_gpu,
+                                                   uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                   NvU32 num_entries);
 
 void uvm_hal_maxwell_enable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters);
 void uvm_hal_maxwell_disable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters);
@@ -719,6 +723,10 @@ NvU32 uvm_hal_maxwell_access_counter_buffer_entry_size_unsupported(uvm_parent_gp
 void uvm_hal_maxwell_access_counter_clear_all_unsupported(uvm_push_t *push);
 void uvm_hal_maxwell_access_counter_clear_targeted_unsupported(uvm_push_t *push,
                                                                const uvm_access_counter_buffer_entry_t *buffer_entry);
+uvm_access_counter_clear_op_t
+uvm_hal_maxwell_access_counter_query_clear_op_unsupported(uvm_parent_gpu_t *parent_gpu,
+                                                          uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                          NvU32 num_entries);
 
 void uvm_hal_turing_enable_access_counter_notifications(uvm_access_counter_buffer_t *access_counters);
 void uvm_hal_turing_disable_access_counter_notifications(uvm_access_counter_buffer_t *access_counters);
@@ -732,6 +740,18 @@ NvU32 uvm_hal_turing_access_counter_buffer_entry_size(uvm_parent_gpu_t *parent_g
 void uvm_hal_turing_access_counter_clear_all(uvm_push_t *push);
 void uvm_hal_turing_access_counter_clear_targeted(uvm_push_t *push,
                                                   const uvm_access_counter_buffer_entry_t *buffer_entry);
+uvm_access_counter_clear_op_t
+uvm_hal_turing_access_counter_query_clear_op(uvm_parent_gpu_t *parent_gpu,
+                                             uvm_access_counter_buffer_entry_t **buffer_entries,
+                                             NvU32 num_entries);
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb100(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries);
+uvm_access_counter_clear_op_t
+uvm_hal_blackwell_access_counter_query_clear_op_gb20x(uvm_parent_gpu_t *parent_gpu,
+                                                      uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                      NvU32 num_entries);
 
 // The source and destination addresses must be 16-byte aligned. Note that the
 // best performance is achieved with 256-byte alignment. The decrypt size must
@@ -785,6 +805,7 @@ struct uvm_host_hal_struct
     uvm_hal_host_clear_faulted_channel_register_t clear_faulted_channel_register;
     uvm_hal_access_counter_clear_all_t access_counter_clear_all;
     uvm_hal_access_counter_clear_targeted_t access_counter_clear_targeted;
+    uvm_hal_access_counter_query_clear_op_t access_counter_query_clear_op;
     uvm_hal_get_time_t get_time;
 };
 

kernel-open/nvidia-uvm/uvm_hal_types.h

@@ -471,6 +471,13 @@ static uvm_membar_t uvm_membar_max(uvm_membar_t membar_1, uvm_membar_t membar_2)
     return max(membar_1, membar_2);
 }
 
+typedef enum
+{
+    UVM_ACCESS_COUNTER_CLEAR_OP_NONE = 0,
+    UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED,
+    UVM_ACCESS_COUNTER_CLEAR_OP_ALL
+} uvm_access_counter_clear_op_t;
+
 struct uvm_access_counter_buffer_entry_struct
 {
     // Address of the region for which a notification was sent

kernel-open/nvidia-uvm/uvm_lock.c

@@ -27,7 +27,7 @@
 
 const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
 {
-    BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 37);
+    BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 38);
 
     switch (lock_order) {
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_INVALID);
@@ -58,6 +58,7 @@ const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_PMM);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_PMM_PMA);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_PMM_ROOT_CHUNK);
+        UVM_ENUM_STRING_CASE(UVM_LOCK_ACCESS_COUNTERS_CLEAR_OPS);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_CHANNEL);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_WLC_CHANNEL);
         UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_TOOLS_VA_SPACE_LIST);

kernel-open/nvidia-uvm/uvm_lock.h

@@ -432,6 +432,11 @@
 //      Order: UVM_LOCK_ORDER_PMM_ROOT_CHUNK
 //      Exclusive bitlock (mutex) per each root chunk internal to PMM.
 //
+// - Access counters clear operations
+//     Order: UVM_LOCK_ACCESS_COUNTERS_CLEAR_OPS
+//
+//     It protects the parent_gpu's access counters clear tracker.
+//
 // - Channel lock
 //      Order: UVM_LOCK_ORDER_CHANNEL
 //      Spinlock (uvm_spinlock_t) or exclusive lock (mutex)
@@ -477,7 +482,7 @@
 //
 //      CE semaphore payloads are encrypted, and require to take the CSL lock
 //      (UVM_LOCK_ORDER_LEAF) to decrypt the payload.
-
+//
 // - CSL Context
 //      Order: UVM_LOCK_ORDER_CSL_CTX
 //      When the Confidential Computing feature is enabled, encrypt/decrypt
@@ -523,6 +528,7 @@ typedef enum
     UVM_LOCK_ORDER_PMM,
     UVM_LOCK_ORDER_PMM_PMA,
     UVM_LOCK_ORDER_PMM_ROOT_CHUNK,
+    UVM_LOCK_ACCESS_COUNTERS_CLEAR_OPS,
     UVM_LOCK_ORDER_CHANNEL,
     UVM_LOCK_ORDER_WLC_CHANNEL,
     UVM_LOCK_ORDER_TOOLS_VA_SPACE_LIST,

kernel-open/nvidia-uvm/uvm_maxwell_host.c

@@ -336,6 +336,15 @@ void uvm_hal_maxwell_access_counter_clear_targeted_unsupported(uvm_push_t *push,
     UVM_ASSERT_MSG(false, "host access_counter_clear_targeted called on Maxwell GPU\n");
 }
 
+uvm_access_counter_clear_op_t
+uvm_hal_maxwell_access_counter_query_clear_op_unsupported(uvm_parent_gpu_t *parent_gpu,
+                                                          uvm_access_counter_buffer_entry_t **buffer_entries,
+                                                          NvU32 num_entries)
+{
+    UVM_ASSERT_MSG(false, "host access_counter_query_clear_op called on Maxwell GPU\n");
+    return UVM_ACCESS_COUNTER_CLEAR_OP_NONE;
+}
+
 NvU64 uvm_hal_maxwell_get_time(uvm_gpu_t *gpu)
 {
     NvU32 time0;

kernel-open/nvidia-uvm/uvm_turing_host.c

@@ -1,5 +1,5 @@
 /*******************************************************************************
-    Copyright (c) 2017-2024 NVIDIA Corporation
+    Copyright (c) 2017-2025 NVIDIA Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to
@@ -382,3 +382,11 @@ void uvm_hal_turing_access_counter_clear_targeted(uvm_push_t *push,
                                HWCONST(C46F, MEM_OP_D, ACCESS_COUNTER_CLR_TARGETED_TYPE, MIMC) |
                                HWVALUE(C46F, MEM_OP_D, ACCESS_COUNTER_CLR_TARGETED_BANK, buffer_entry->bank));
 }
+
+uvm_access_counter_clear_op_t
+uvm_hal_turing_access_counter_query_clear_op(uvm_parent_gpu_t *parent_gpu,
+                                             uvm_access_counter_buffer_entry_t **buffer_entries,
+                                             NvU32 num_entries)
+{
+    return UVM_ACCESS_COUNTER_CLEAR_OP_TARGETED;
+}

kernel-open/nvidia/nvidia.Kbuild

@@ -260,6 +260,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += foll_longterm_present
 NV_CONFTEST_TYPE_COMPILE_TESTS += bus_type_has_iommu_ops
 NV_CONFTEST_TYPE_COMPILE_TESTS += class_create_has_no_owner_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += class_devnode_has_const_arg
+NV_CONFTEST_TYPE_COMPILE_TESTS += has_enum_pidtype_tgid
 
 NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
 NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build

kernel-open/nvidia/os-interface.c

@@ -2644,7 +2644,11 @@ NV_STATUS NV_API_CALL os_offline_page_at_address
 
 void* NV_API_CALL os_get_pid_info(void)
 {
-    return get_task_pid(current, PIDTYPE_PID);
+#if defined(NV_HAS_ENUM_PIDTYPE_TGID)
+    return get_task_pid(current, PIDTYPE_TGID);
+#else
+    return get_task_pid(current->group_leader, PIDTYPE_PID);
+#endif
 }
 
 void NV_API_CALL os_put_pid_info(void *pid_info)