NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-05 23:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

580.65.06

Browse Source
This commit is contained in:
Maneet Singh
2025-08-04 11:15:02 -07:00
parent d890313300
commit 307159f262
1315 changed files with 477791 additions and 279973 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
kernel-open/nvidia-uvm/uvm_channel.c
View File

@@ -34,6 +34,7 @@
#include "uvm_kvmalloc.h"
#include "nv_uvm_types.h"
#include "nv_uvm_user_types.h"
#include "nv_uvm_interface.h"
#include "clb06f.h"
#include "uvm_conf_computing.h"
@@ -381,7 +382,9 @@ static void lock_channel_for_push(uvm_channel_t *channel)
__set_bit(index, channel->pool->conf_computing.push_locks);
}
static bool test_claim_and_lock_channel(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
static bool test_claim_and_lock_channel(uvm_channel_t *channel,
NvU32 num_gpfifo_entries,
uvm_channel_reserve_type_t reserve_type)
{
UVM_ASSERT(g_uvm_global.conf_computing_enabled);
uvm_channel_pool_assert_locked(channel->pool);
@@ -390,8 +393,7 @@ static bool test_claim_and_lock_channel(uvm_channel_t *channel, NvU32 num_gpfifo
if (uvm_channel_is_locked_for_push(channel))
return false;
// Confidential compute is not using p2p ops, reserve without p2p
if (try_claim_channel_locked(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P)) {
if (try_claim_channel_locked(channel, num_gpfifo_entries, reserve_type)) {
lock_channel_for_push(channel);
return true;
}
@@ -504,7 +506,9 @@ NV_STATUS uvm_channel_pool_rotate_key(uvm_channel_pool_t *pool)
// Reserve a channel in the specified pool. The channel is locked until the push
// ends
static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, uvm_channel_t **channel_out)
static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool,
uvm_channel_reserve_type_t reserve_type,
uvm_channel_t **channel_out)
{
uvm_channel_t *channel;
uvm_spin_loop_t spin;
@@ -533,8 +537,7 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, uvm_
for_each_clear_bit(index, pool->conf_computing.push_locks, pool->num_channels) {
channel = &pool->channels[index];
// Confidential compute is not using p2p ops, reserve without p2p
if (try_claim_channel_locked(channel, 1, UVM_CHANNEL_RESERVE_NO_P2P)) {
if (try_claim_channel_locked(channel, 1, reserve_type)) {
lock_channel_for_push(channel);
goto done;
}
@@ -551,7 +554,7 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, uvm_
channel_pool_lock(pool);
if (test_claim_and_lock_channel(channel, 1))
if (test_claim_and_lock_channel(channel, 1, reserve_type))
goto done;
channel_pool_unlock(pool);
@@ -562,6 +565,11 @@ static NV_STATUS channel_reserve_and_lock_in_pool(uvm_channel_pool_t *pool, uvm_
return status;
}
if (reserve_type == UVM_CHANNEL_RESERVE_WITH_P2P && channel->suspended_p2p) {
uvm_up(&pool->conf_computing.push_sem);
return NV_ERR_BUSY_RETRY;
}
UVM_SPIN_LOOP(&spin);
}
}
@@ -583,7 +591,7 @@ static NV_STATUS channel_reserve_in_pool(uvm_channel_pool_t *pool,
UVM_ASSERT(pool);
if (g_uvm_global.conf_computing_enabled)
return channel_reserve_and_lock_in_pool(pool, channel_out);
return channel_reserve_and_lock_in_pool(pool, reserve_type, channel_out);
uvm_for_each_channel_in_pool(channel, pool) {
// TODO: Bug 1764953: Prefer idle/less busy channels
@@ -1799,7 +1807,9 @@ NV_STATUS uvm_channel_write_ctrl_gpfifo(uvm_channel_t *channel, NvU64 ctrl_fifo_
return NV_OK;
}
static NV_STATUS channel_reserve_and_lock(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
static NV_STATUS channel_reserve_and_lock(uvm_channel_t *channel,
NvU32 num_gpfifo_entries,
uvm_channel_reserve_type_t reserve_type)
{
NV_STATUS status;
uvm_spin_loop_t spin;
@@ -1822,7 +1832,7 @@ static NV_STATUS channel_reserve_and_lock(uvm_channel_t *channel, NvU32 num_gpfi
channel_pool_lock(pool);
if (test_claim_and_lock_channel(channel, num_gpfifo_entries))
if (test_claim_and_lock_channel(channel, num_gpfifo_entries, reserve_type))
goto out;
channel_pool_unlock(pool);
@@ -1833,7 +1843,7 @@ static NV_STATUS channel_reserve_and_lock(uvm_channel_t *channel, NvU32 num_gpfi
channel_pool_lock(pool);
if (test_claim_and_lock_channel(channel, num_gpfifo_entries))
if (test_claim_and_lock_channel(channel, num_gpfifo_entries, reserve_type))
goto out;
channel_pool_unlock(pool);
@@ -1857,8 +1867,9 @@ NV_STATUS uvm_channel_reserve(uvm_channel_t *channel, NvU32 num_gpfifo_entries)
NV_STATUS status = NV_OK;
uvm_spin_loop_t spin;
// Direct channel reservations don't use p2p
if (g_uvm_global.conf_computing_enabled)
return channel_reserve_and_lock(channel, num_gpfifo_entries);
return channel_reserve_and_lock(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P);
// Direct channel reservations don't use p2p
if (try_claim_channel(channel, num_gpfifo_entries, UVM_CHANNEL_RESERVE_NO_P2P))
@@ -3058,7 +3069,7 @@ static int compare_ce_for_channel_type(const UvmGpuCopyEngineCaps *ce_caps,
// Select the preferred CE for the given channel types.
static void pick_ces_for_channel_types(uvm_channel_manager_t *manager,
const UvmGpuCopyEngineCaps *ce_caps,
uvm_channel_type_t *channel_types,
const uvm_channel_type_t *channel_types,
unsigned num_channel_types,
unsigned *preferred_ce)
{
@@ -3067,7 +3078,7 @@ static void pick_ces_for_channel_types(uvm_channel_manager_t *manager,
for (i = 0; i < num_channel_types; ++i) {
unsigned ce;
unsigned best_ce = UVM_COPY_ENGINE_COUNT_MAX;
uvm_channel_type_t type = channel_types[i];
const uvm_channel_type_t type = channel_types[i];
for (ce = 0; ce < UVM_COPY_ENGINE_COUNT_MAX; ++ce) {
if (!ce_is_usable(ce_caps + ce))
@@ -3104,11 +3115,11 @@ static void pick_ces(uvm_channel_manager_t *manager, const UvmGpuCopyEngineCaps
// the usage count of each CE and it increases every time a CE
// is selected. MEMOPS has the least priority as it only cares about
// low usage of the CE to improve latency
uvm_channel_type_t types[] = {UVM_CHANNEL_TYPE_CPU_TO_GPU,
UVM_CHANNEL_TYPE_GPU_TO_CPU,
UVM_CHANNEL_TYPE_GPU_INTERNAL,
UVM_CHANNEL_TYPE_GPU_TO_GPU,
UVM_CHANNEL_TYPE_MEMOPS};
static const uvm_channel_type_t types[] = { UVM_CHANNEL_TYPE_CPU_TO_GPU,
UVM_CHANNEL_TYPE_GPU_TO_CPU,
UVM_CHANNEL_TYPE_GPU_INTERNAL,
UVM_CHANNEL_TYPE_GPU_TO_GPU,
UVM_CHANNEL_TYPE_MEMOPS };
UVM_ASSERT(!g_uvm_global.conf_computing_enabled);
@@ -3123,11 +3134,11 @@ static void pick_ces_conf_computing(uvm_channel_manager_t *manager,
uvm_gpu_t *gpu = manager->gpu;
// The WLC type must go last so an unused CE is chosen, if available
uvm_channel_type_t types[] = {UVM_CHANNEL_TYPE_CPU_TO_GPU,
UVM_CHANNEL_TYPE_GPU_TO_CPU,
UVM_CHANNEL_TYPE_GPU_INTERNAL,
UVM_CHANNEL_TYPE_MEMOPS,
UVM_CHANNEL_TYPE_WLC};
static const uvm_channel_type_t types[] = { UVM_CHANNEL_TYPE_CPU_TO_GPU,
UVM_CHANNEL_TYPE_GPU_TO_CPU,
UVM_CHANNEL_TYPE_GPU_INTERNAL,
UVM_CHANNEL_TYPE_MEMOPS,
UVM_CHANNEL_TYPE_WLC };
UVM_ASSERT(g_uvm_global.conf_computing_enabled);