NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-04-29 10:41:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

565.57.01

Browse Source
This commit is contained in:
Bernhard Stoeckner
2024-10-22 17:38:58 +02:00
parent ed4be64962
commit d5a0858f90
1049 changed files with 209491 additions and 167508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
kernel-open/nvidia-uvm/uvm_user_channel.c
View File

@@ -237,23 +237,22 @@ static uvm_user_channel_t *find_user_channel(uvm_va_space_t *va_space, uvm_rm_us
// 1) gpu_va_space must match
// 2) rm_descriptor must match
// 3) new_user_channel's TSG matches existing mappings
static uvm_va_range_t *find_va_range(uvm_user_channel_t *new_user_channel, NvP64 rm_descriptor)
static uvm_va_range_channel_t *find_va_range(uvm_user_channel_t *new_user_channel, NvP64 rm_descriptor)
{
uvm_gpu_va_space_t *gpu_va_space = new_user_channel->gpu_va_space;
uvm_va_range_t *range;
uvm_va_range_channel_t *channel_range;
// We can only allow sharing within a TSG
if (!new_user_channel->tsg.valid)
return NULL;
list_for_each_entry(range, &gpu_va_space->channel_va_ranges, channel.list_node) {
UVM_ASSERT(range->type == UVM_VA_RANGE_TYPE_CHANNEL);
UVM_ASSERT(range->channel.ref_count > 0);
list_for_each_entry(channel_range, &gpu_va_space->channel_va_ranges, list_node) {
UVM_ASSERT(channel_range->ref_count > 0);
if (range->channel.tsg.valid &&
range->channel.tsg.id == new_user_channel->tsg.id &&
range->channel.rm_descriptor == rm_descriptor)
return range;
if (channel_range->tsg.valid &&
channel_range->tsg.id == new_user_channel->tsg.id &&
channel_range->rm_descriptor == rm_descriptor)
return channel_range;
}
return NULL;
@@ -290,8 +289,8 @@ static NvU64 find_va_in_range(uvm_va_space_t *va_space, NvU64 base, NvU64 end, N
}
}
// Allocate or reuse a VA range for the given channel resource, but don't map
// it. If a new VA range is allocated, the VA used is the first unallocated VA
// Allocate or reuse a range for the given channel resource, but don't map
// it. If a new range is allocated, the VA used is the first unallocated VA
// in the range [base, end] which has the appropriate alignment and size for the
// given resource.
static NV_STATUS create_va_range(struct mm_struct *mm,
@@ -303,7 +302,7 @@ static NV_STATUS create_va_range(struct mm_struct *mm,
uvm_gpu_va_space_t *gpu_va_space = user_channel->gpu_va_space;
UvmGpuChannelResourceInfo *resource = &user_channel->resources[resource_index];
UvmGpuMemoryInfo *mem_info = &resource->resourceInfo;
uvm_va_range_t *range = NULL;
uvm_va_range_channel_t *channel_range = NULL;
NvU64 start;
uvm_aperture_t aperture;
NV_STATUS status;
@@ -316,19 +315,19 @@ static NV_STATUS create_va_range(struct mm_struct *mm,
aperture = UVM_APERTURE_VID;
// See if we've already mapped this resource
range = find_va_range(user_channel, resource->resourceDescriptor);
if (range) {
channel_range = find_va_range(user_channel, resource->resourceDescriptor);
if (channel_range) {
// We've already mapped this resource, so just bump the ref count
UVM_ASSERT(IS_ALIGNED(range->node.start, resource->alignment));
UVM_ASSERT(uvm_va_range_size(range) >= mem_info->size);
UVM_ASSERT(range->channel.aperture == aperture);
UVM_ASSERT(IS_ALIGNED(channel_range->va_range.node.start, resource->alignment));
UVM_ASSERT(uvm_va_range_size(&channel_range->va_range) >= mem_info->size);
UVM_ASSERT(channel_range->aperture == aperture);
++range->channel.ref_count;
user_channel->va_ranges[resource_index] = range;
++channel_range->ref_count;
user_channel->channel_ranges[resource_index] = channel_range;
return NV_OK;
}
// This is a new VA range. Find an available VA in the input region and
// This is a new channel range. Find an available VA in the input region and
// allocate it there.
start = find_va_in_range(gpu_va_space->va_space, base, end, mem_info->size, resource->alignment);
if (!start) {
@@ -346,28 +345,28 @@ static NV_STATUS create_va_range(struct mm_struct *mm,
mm,
start,
start + mem_info->size - 1,
&range);
&channel_range);
if (status != NV_OK) {
UVM_ASSERT(status != NV_ERR_UVM_ADDRESS_IN_USE);
goto error;
}
range->channel.gpu_va_space = gpu_va_space;
range->channel.aperture = aperture;
range->channel.rm_descriptor = resource->resourceDescriptor;
range->channel.rm_id = resource->resourceId;
range->channel.tsg.valid = user_channel->tsg.valid;
range->channel.tsg.id = user_channel->tsg.id;
range->channel.ref_count = 1;
list_add(&range->channel.list_node, &gpu_va_space->channel_va_ranges);
channel_range->gpu_va_space = gpu_va_space;
channel_range->aperture = aperture;
channel_range->rm_descriptor = resource->resourceDescriptor;
channel_range->rm_id = resource->resourceId;
channel_range->tsg.valid = user_channel->tsg.valid;
channel_range->tsg.id = user_channel->tsg.id;
channel_range->ref_count = 1;
list_add(&channel_range->list_node, &gpu_va_space->channel_va_ranges);
user_channel->va_ranges[resource_index] = range;
user_channel->channel_ranges[resource_index] = channel_range;
return NV_OK;
error:
if (range) {
range->channel.ref_count = 0; // Destroy assumes this
uvm_va_range_destroy(range, NULL);
if (channel_range) {
channel_range->ref_count = 0; // Destroy assumes this
uvm_va_range_destroy(&channel_range->va_range, NULL);
}
return status;
}
@@ -376,32 +375,31 @@ static void destroy_va_ranges(uvm_user_channel_t *user_channel)
{
size_t i;
if (!user_channel || !user_channel->va_ranges)
if (!user_channel || !user_channel->channel_ranges)
return;
for (i = 0; i < user_channel->num_resources; i++) {
uvm_va_range_t *resource_range = user_channel->va_ranges[i];
uvm_va_range_channel_t *resource_range = user_channel->channel_ranges[i];
if (!resource_range)
continue;
UVM_ASSERT(resource_range->type == UVM_VA_RANGE_TYPE_CHANNEL);
UVM_ASSERT(resource_range->channel.rm_descriptor == user_channel->resources[i].resourceDescriptor);
UVM_ASSERT(resource_range->channel.rm_id == user_channel->resources[i].resourceId);
UVM_ASSERT(resource_range->channel.tsg.valid == user_channel->tsg.valid);
UVM_ASSERT(resource_range->channel.tsg.id == user_channel->tsg.id);
UVM_ASSERT(resource_range->rm_descriptor == user_channel->resources[i].resourceDescriptor);
UVM_ASSERT(resource_range->rm_id == user_channel->resources[i].resourceId);
UVM_ASSERT(resource_range->tsg.valid == user_channel->tsg.valid);
UVM_ASSERT(resource_range->tsg.id == user_channel->tsg.id);
// Drop the ref count on each each range
UVM_ASSERT(resource_range->channel.ref_count > 0);
if (!resource_range->channel.tsg.valid)
UVM_ASSERT(resource_range->channel.ref_count == 1);
UVM_ASSERT(resource_range->ref_count > 0);
if (!resource_range->tsg.valid)
UVM_ASSERT(resource_range->ref_count == 1);
--resource_range->channel.ref_count;
if (resource_range->channel.ref_count == 0)
uvm_va_range_destroy(resource_range, NULL);
--resource_range->ref_count;
if (resource_range->ref_count == 0)
uvm_va_range_destroy(&resource_range->va_range, NULL);
}
uvm_kvfree(user_channel->va_ranges);
user_channel->va_ranges = NULL;
uvm_kvfree(user_channel->channel_ranges);
user_channel->channel_ranges = NULL;
}
// Channels need virtual allocations to operate, but we don't know about them.
@@ -415,8 +413,9 @@ static NV_STATUS create_va_ranges(struct mm_struct *mm,
NvU32 i;
NV_STATUS status;
user_channel->va_ranges = uvm_kvmalloc_zero(user_channel->num_resources * sizeof(user_channel->va_ranges[0]));
if (!user_channel->va_ranges)
user_channel->channel_ranges = uvm_kvmalloc_zero(user_channel->num_resources *
sizeof(user_channel->channel_ranges[0]));
if (!user_channel->channel_ranges)
return NV_ERR_NO_MEMORY;
for (i = 0; i < user_channel->num_resources; i++) {
@@ -447,17 +446,17 @@ static NV_STATUS bind_channel_resources(uvm_user_channel_t *user_channel)
}
for (i = 0; i < user_channel->num_resources; i++) {
uvm_va_range_t *resource_range = user_channel->va_ranges[i];
uvm_va_range_channel_t *resource_range = user_channel->channel_ranges[i];
UVM_ASSERT(resource_range);
UVM_ASSERT(resource_range->type == UVM_VA_RANGE_TYPE_CHANNEL);
UVM_ASSERT(resource_range->channel.rm_descriptor == user_channel->resources[i].resourceDescriptor);
UVM_ASSERT(resource_range->channel.rm_id == user_channel->resources[i].resourceId);
UVM_ASSERT(resource_range->channel.tsg.valid == user_channel->tsg.valid);
UVM_ASSERT(resource_range->channel.tsg.id == user_channel->tsg.id);
UVM_ASSERT(resource_range->va_range.type == UVM_VA_RANGE_TYPE_CHANNEL);
UVM_ASSERT(resource_range->rm_descriptor == user_channel->resources[i].resourceDescriptor);
UVM_ASSERT(resource_range->rm_id == user_channel->resources[i].resourceId);
UVM_ASSERT(resource_range->tsg.valid == user_channel->tsg.valid);
UVM_ASSERT(resource_range->tsg.id == user_channel->tsg.id);
resource_va_list[i].resourceId = resource_range->channel.rm_id;
resource_va_list[i].resourceVa = resource_range->node.start;
resource_va_list[i].resourceId = resource_range->rm_id;
resource_va_list[i].resourceVa = resource_range->va_range.node.start;
}
status = uvm_rm_locked_call(nvUvmInterfaceBindChannelResources(user_channel->rm_retained_channel,
@@ -500,23 +499,28 @@ static NV_STATUS uvm_user_channel_map_resources(uvm_user_channel_t *user_channel
for (i = 0; i < user_channel->num_resources; i++) {
UvmGpuMemoryInfo *mem_info;
uvm_va_range_t *range = user_channel->va_ranges[i];
uvm_va_range_channel_t *range = user_channel->channel_ranges[i];
// Skip already-mapped VA ranges. Note that the ref count might not be
// Skip already-mapped ranges. Note that the ref count might not be
// 1 even if the range is unmapped, because a thread which fails to map
// will drop and re-take the VA space lock in uvm_register_channel
// leaving a shareable VA range in the list unmapped. This thread could
// leaving a shareable range in the list unmapped. This thread could
// have attached to it during that window, so we'll do the mapping
// instead.
if (range->channel.pt_range_vec.ranges) {
UVM_ASSERT(range->channel.ref_count >= 1);
if (range->pt_range_vec.ranges) {
UVM_ASSERT(range->ref_count >= 1);
continue;
}
mem_info = &user_channel->resources[i].resourceInfo;
status = uvm_va_range_map_rm_allocation(range, user_channel->gpu, mem_info, &map_rm_params, NULL, &tracker);
status = uvm_va_range_map_rm_allocation(&range->va_range,
user_channel->gpu,
mem_info,
&map_rm_params,
NULL,
&tracker);
if (status != NV_OK) {
// We can't destroy the VA ranges here since we only have the VA
// We can't destroy the ranges here since we only have the VA
// space lock in read mode, so let the caller handle it.
break;
}
@@ -742,7 +746,7 @@ static void free_user_channel(nv_kref_t *nv_kref)
{
uvm_user_channel_t *user_channel = container_of(nv_kref, uvm_user_channel_t, kref);
UVM_ASSERT(!user_channel->gpu_va_space);
UVM_ASSERT(!user_channel->va_ranges);
UVM_ASSERT(!user_channel->channel_ranges);
UVM_ASSERT(!atomic_read(&user_channel->is_bound));
uvm_kvfree(user_channel);
}
@@ -839,7 +843,7 @@ void uvm_user_channel_destroy_detached(uvm_user_channel_t *user_channel)
uvm_tracker_wait_deinit(&user_channel->clear_faulted_tracker);
if (user_channel->resources) {
UVM_ASSERT(!user_channel->va_ranges);
UVM_ASSERT(!user_channel->channel_ranges);
uvm_kvfree(user_channel->resources);
}