580.94.06

This commit is contained in:
russellcnv
2025-10-27 13:41:40 -07:00
parent 1babfa3dab
commit e2dbb3d99c
74 changed files with 55029 additions and 53585 deletions

View File

@@ -79,7 +79,7 @@ ccflags-y += -I$(src)/common/inc
ccflags-y += -I$(src)
ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
ccflags-y += -DNV_VERSION_STRING=\"580.94.03\"
ccflags-y += -DNV_VERSION_STRING=\"580.94.06\"
# Include and link Tegra out-of-tree modules.
ifneq ($(wildcard /usr/src/nvidia/nvidia-oot),)

View File

@@ -58,6 +58,12 @@ typedef struct {
*/
void (*suspend)(NvU32 gpu_id);
void (*resume)(NvU32 gpu_id);
/* Remove callback, called when a device is going away completely. */
void (*remove)(NvU32 gpu_id);
/* Probe callback, called when a device is being hotplugged. */
void (*probe)(const nv_gpu_info_t *gpu_info);
} nvidia_modeset_callbacks_t;
/*

View File

@@ -78,6 +78,8 @@ NV_STATUS nv_set_system_power_state (nv_power_state_t, nv_pm_action_depth_t)
void nvidia_modeset_suspend (NvU32 gpuId);
void nvidia_modeset_resume (NvU32 gpuId);
void nvidia_modeset_remove (NvU32 gpuId);
void nvidia_modeset_probe (const nv_linux_state_t *nvl);
NvBool nv_is_uuid_in_gpu_exclusion_list (const char *);
NV_STATUS nv_parse_per_device_option_string(nvidia_stack_t *sp);

View File

@@ -688,6 +688,13 @@ typedef struct UvmGpuInfo_tag
// GPU setup in CDMM mode
NvBool cdmmEnabled;
// If nvlinkDirectConnect is NV_TRUE,
// nvlDirectConnectMemoryWindowStart is the GPA base address for the GPU's vidmem over
// direct-connect NVLink. It is used when creating PTEs of GPU memory mappings
// to direct NVLink peers.
NvBool nvlDirectConnect;
NvU64 nvlDirectConnectMemoryWindowStart;
} UvmGpuInfo;
typedef struct UvmGpuFbInfo_tag

View File

@@ -598,13 +598,17 @@ typedef enum NvKmsKapiRegisterWaiterResultRec {
NVKMS_KAPI_REG_WAITER_ALREADY_SIGNALLED,
} NvKmsKapiRegisterWaiterResult;
typedef void NvKmsKapiSuspendResumeCallbackFunc(NvBool suspend);
struct NvKmsKapiGpuInfo {
nv_gpu_info_t gpuInfo;
MIGDeviceId migDevice;
};
struct NvKmsKapiCallbacks {
void (*suspendResume)(NvBool suspend);
void (*remove)(NvU32 gpuId);
void (*probe)(const struct NvKmsKapiGpuInfo *gpu_info);
};
struct NvKmsKapiFunctionsTable {
/*!
@@ -1473,12 +1477,12 @@ struct NvKmsKapiFunctionsTable {
);
/*!
* Set the callback function for suspending and resuming the display system.
* Set the pointer to the callback function table.
*/
void
(*setSuspendResumeCallback)
(*setCallbacks)
(
NvKmsKapiSuspendResumeCallbackFunc *function
const struct NvKmsKapiCallbacks *callbacks
);
/*!

View File

@@ -101,6 +101,7 @@ static int nv_drm_revoke_modeset_permission(struct drm_device *dev,
NvU32 dpyId);
static int nv_drm_revoke_sub_ownership(struct drm_device *dev);
static DEFINE_MUTEX(dev_list_mutex);
static struct nv_drm_device *dev_list = NULL;
static const char* nv_get_input_colorspace_name(
@@ -2067,8 +2068,10 @@ void nv_drm_register_drm_device(const struct NvKmsKapiGpuInfo *gpu_info)
/* Add NVIDIA-DRM device into list */
mutex_lock(&dev_list_mutex);
nv_dev->next = dev_list;
dev_list = nv_dev;
mutex_unlock(&dev_list_mutex);
return; /* Success */
@@ -2106,22 +2109,81 @@ int nv_drm_probe_devices(void)
}
#endif
static struct nv_drm_device*
nv_drm_pop_device(void)
{
struct nv_drm_device *nv_dev;
mutex_lock(&dev_list_mutex);
nv_dev = dev_list;
if (nv_dev) {
dev_list = nv_dev->next;
nv_dev->next = NULL;
}
mutex_unlock(&dev_list_mutex);
return nv_dev;
}
static struct nv_drm_device*
nv_drm_find_and_remove_device(NvU32 gpuId)
{
struct nv_drm_device **pPrev = &dev_list;
struct nv_drm_device *nv_dev;
mutex_lock(&dev_list_mutex);
nv_dev = *pPrev;
while (nv_dev) {
if (nv_dev->gpu_info.gpu_id == gpuId) {
/* Remove it from the linked list */
*pPrev = nv_dev->next;
nv_dev->next = NULL;
break;
}
pPrev = &nv_dev->next;
nv_dev = *pPrev;
}
mutex_unlock(&dev_list_mutex);
return nv_dev;
}
static void nv_drm_dev_destroy(struct nv_drm_device *nv_dev)
{
struct drm_device *dev = nv_dev->dev;
nv_drm_dev_unload(dev);
drm_dev_put(dev);
nv_drm_free(nv_dev);
}
/*
* Unregister a single NVIDIA DRM device.
*/
void nv_drm_remove(NvU32 gpuId)
{
struct nv_drm_device *nv_dev = nv_drm_find_and_remove_device(gpuId);
if (nv_dev) {
NV_DRM_DEV_LOG_INFO(nv_dev, "Removing device");
drm_dev_unplug(nv_dev->dev);
nv_drm_dev_destroy(nv_dev);
}
}
/*
* Unregister all NVIDIA DRM devices.
*/
void nv_drm_remove_devices(void)
{
while (dev_list != NULL) {
struct nv_drm_device *next = dev_list->next;
struct drm_device *dev = dev_list->dev;
struct nv_drm_device *nv_dev;
drm_dev_unregister(dev);
nv_drm_dev_unload(dev);
drm_dev_put(dev);
nv_drm_free(dev_list);
dev_list = next;
while ((nv_dev = nv_drm_pop_device())) {
drm_dev_unregister(nv_dev->dev);
nv_drm_dev_destroy(nv_dev);
}
}
@@ -2143,11 +2205,10 @@ void nv_drm_remove_devices(void)
*/
void nv_drm_suspend_resume(NvBool suspend)
{
static DEFINE_MUTEX(nv_drm_suspend_mutex);
static NvU32 nv_drm_suspend_count = 0;
struct nv_drm_device *nv_dev;
mutex_lock(&nv_drm_suspend_mutex);
mutex_lock(&dev_list_mutex);
/*
* Count the number of times the driver is asked to suspend. Suspend all DRM
@@ -2195,7 +2256,7 @@ void nv_drm_suspend_resume(NvBool suspend)
}
done:
mutex_unlock(&nv_drm_suspend_mutex);
mutex_unlock(&dev_list_mutex);
}
#endif /* NV_DRM_AVAILABLE */

View File

@@ -31,6 +31,7 @@ struct NvKmsKapiGpuInfo;
int nv_drm_probe_devices(void);
void nv_drm_remove(NvU32 gpuId);
void nv_drm_remove_devices(void);
void nv_drm_suspend_resume(NvBool suspend);

View File

@@ -33,6 +33,12 @@ static struct NvKmsKapiFunctionsTable nvKmsFuncsTable = {
const struct NvKmsKapiFunctionsTable* const nvKms = &nvKmsFuncsTable;
const struct NvKmsKapiCallbacks nv_drm_kapi_callbacks = {
.suspendResume = nv_drm_suspend_resume,
.remove = nv_drm_remove,
.probe = nv_drm_register_drm_device,
};
#endif
int nv_drm_init(void)
@@ -45,7 +51,7 @@ int nv_drm_init(void)
return -EINVAL;
}
nvKms->setSuspendResumeCallback(nv_drm_suspend_resume);
nvKms->setCallbacks(&nv_drm_kapi_callbacks);
return nv_drm_probe_devices();
#else
return 0;
@@ -55,7 +61,7 @@ int nv_drm_init(void)
void nv_drm_exit(void)
{
#if defined(NV_DRM_AVAILABLE)
nvKms->setSuspendResumeCallback(NULL);
nvKms->setCallbacks(NULL);
nv_drm_remove_devices();
#endif
}

View File

@@ -820,6 +820,20 @@ static void nvkms_resume(NvU32 gpuId)
nvKmsKapiSuspendResume(NV_FALSE /* suspend */);
}
static void nvkms_remove(NvU32 gpuId)
{
nvKmsKapiRemove(gpuId);
// Eventually, this function should also terminate all NVKMS clients and
// free the NVDevEvoRec. Until that is implemented, all NVKMS clients must
// be closed before a device is removed.
}
static void nvkms_probe(const nv_gpu_info_t *gpu_info)
{
nvKmsKapiProbe(gpu_info);
}
/*************************************************************************
* Interface with resman.
@@ -828,7 +842,9 @@ static void nvkms_resume(NvU32 gpuId)
static nvidia_modeset_rm_ops_t __rm_ops = { 0 };
static nvidia_modeset_callbacks_t nvkms_rm_callbacks = {
.suspend = nvkms_suspend,
.resume = nvkms_resume
.resume = nvkms_resume,
.remove = nvkms_remove,
.probe = nvkms_probe,
};
static int nvkms_alloc_rm(void)

View File

@@ -104,6 +104,8 @@ NvBool nvKmsKapiGetFunctionsTableInternal
);
void nvKmsKapiSuspendResume(NvBool suspend);
void nvKmsKapiRemove(NvU32 gpuId);
void nvKmsKapiProbe(const nv_gpu_info_t *gpu_info);
NvBool nvKmsGetBacklight(NvU32 display_id, void *drv_priv, NvU32 *brightness);
NvBool nvKmsSetBacklight(NvU32 display_id, void *drv_priv, NvU32 brightness);

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2018-2023 NVIDIA Corporation
Copyright (c) 2018-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -43,8 +43,11 @@
// commit cb4789b0d19ff231ce9f73376a023341300aed96 (11/23/2020). Commit
// 701fac40384f07197b106136012804c3cae0b3de (02/15/2022) removed ioasid_get()
// and added mm_pasid_drop().
//
// CONFIG_IOMMU_SVA_LIB was renamed to CONFIG_IOMMU_SVA with commit 7ba5647
// (02/07/2022).
#if UVM_CAN_USE_MMU_NOTIFIERS() && (defined(NV_IOASID_GET_PRESENT) || defined(NV_MM_PASID_DROP_PRESENT))
#if defined(CONFIG_IOMMU_SVA)
#if defined(CONFIG_IOMMU_SVA) || defined(CONFIG_IOMMU_SVA_LIB)
#define UVM_ATS_SVA_SUPPORTED() 1
#else
#define UVM_ATS_SVA_SUPPORTED() 0

View File

@@ -144,10 +144,13 @@ static NV_STATUS verify_mapping_info(uvm_va_space_t *va_space,
phys_offset = mapping_offset;
// Add the physical offset for nvswitch connected peer mappings
if (uvm_aperture_is_peer(aperture) &&
uvm_parent_gpus_are_nvswitch_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
// Add the physical offset for peer mappings
if (uvm_aperture_is_peer(aperture)) {
if (uvm_parent_gpus_are_direct_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->peer_address_info.peer_gpa_memory_window_start;
else if (uvm_parent_gpus_are_nvswitch_connected(memory_mapping_gpu->parent, memory_owning_gpu->parent))
phys_offset += memory_owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
}
for (index = 0; index < ext_mapping_info->numWrittenPtes; index++) {

View File

@@ -107,6 +107,7 @@ static void fill_parent_gpu_info(uvm_parent_gpu_t *parent_gpu, const UvmGpuInfo
}
parent_gpu->nvswitch_info.is_nvswitch_connected = gpu_info->connectedToSwitch;
parent_gpu->peer_address_info.is_direct_connected = gpu_info->nvlDirectConnect;
// nvswitch is routed via physical pages, where the upper 13-bits of the
// 47-bit address space holds the routing information for each peer.
@@ -115,6 +116,9 @@ static void fill_parent_gpu_info(uvm_parent_gpu_t *parent_gpu, const UvmGpuInfo
parent_gpu->nvswitch_info.fabric_memory_window_start = gpu_info->nvswitchMemoryWindowStart;
parent_gpu->nvswitch_info.egm_fabric_memory_window_start = gpu_info->nvswitchEgmMemoryWindowStart;
}
else if (parent_gpu->peer_address_info.is_direct_connected) {
parent_gpu->peer_address_info.peer_gpa_memory_window_start = gpu_info->nvlDirectConnectMemoryWindowStart;
}
parent_gpu->ats.non_pasid_ats_enabled = gpu_info->nonPasidAtsSupport;
@@ -2110,6 +2114,16 @@ bool uvm_parent_gpus_are_nvswitch_connected(const uvm_parent_gpu_t *parent_gpu0,
return false;
}
bool uvm_parent_gpus_are_direct_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1)
{
if (parent_gpu0 != parent_gpu1 &&
parent_gpu0->peer_address_info.is_direct_connected &&
parent_gpu1->peer_address_info.is_direct_connected)
return true;
return false;
}
NV_STATUS uvm_gpu_check_ecc_error_no_rm(uvm_gpu_t *gpu)
{
// We may need to call service_interrupts() which cannot be done in the top
@@ -3068,7 +3082,9 @@ uvm_gpu_phys_address_t uvm_gpu_peer_phys_address(uvm_gpu_t *owning_gpu, NvU64 ad
{
uvm_aperture_t aperture = uvm_gpu_peer_aperture(accessing_gpu, owning_gpu);
if (uvm_parent_gpus_are_nvswitch_connected(accessing_gpu->parent, owning_gpu->parent))
if (uvm_parent_gpus_are_direct_connected(accessing_gpu->parent, owning_gpu->parent))
address += owning_gpu->parent->peer_address_info.peer_gpa_memory_window_start;
else if (uvm_parent_gpus_are_nvswitch_connected(accessing_gpu->parent, owning_gpu->parent))
address += owning_gpu->parent->nvswitch_info.fabric_memory_window_start;
return uvm_gpu_phys_address(aperture, address);

View File

@@ -1365,6 +1365,20 @@ struct uvm_parent_gpu_struct
NvU64 base_address;
} egm;
// Peer VIDMEM base offset used when creating GPA PTEs for
// peer mappings. RM will set this offset on systems where
// peer accesses are not zero-based (NVLINK 5+).
struct
{
// Is the GPU directly connected to peer GPUs.
bool is_direct_connected;
// 48-bit fabric memory physical offset that peer gpus need in order
// access to be rounted to the correct peer.
// Each memory window is 4TB, so the upper 6 bits are used for rounting.
NvU64 peer_gpa_memory_window_start;
} peer_address_info;
uvm_test_parent_gpu_inject_error_t test;
// PASID ATS
@@ -1619,6 +1633,8 @@ uvm_aperture_t uvm_gpu_egm_peer_aperture(uvm_parent_gpu_t *local_gpu, uvm_parent
bool uvm_parent_gpus_are_nvswitch_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1);
bool uvm_parent_gpus_are_direct_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1);
static bool uvm_gpus_are_smc_peers(const uvm_gpu_t *gpu0, const uvm_gpu_t *gpu1)
{
UVM_ASSERT(gpu0 != gpu1);

View File

@@ -32,7 +32,8 @@ static inline int pci_devid_is_self_hosted_hopper(unsigned short devid)
static inline int pci_devid_is_self_hosted_blackwell(unsigned short devid)
{
return (devid >= 0x2940 && devid <= 0x297f) // GB100 Self-Hosted
|| (devid >= 0x31c0 && devid <= 0x31ff); // GB110 Self-Hosted
|| (devid >= 0x31c0 && devid <= 0x31ff) // GB110 Self-Hosted
|| (devid == 0x31a1); //
}
static inline int pci_devid_is_self_hosted(unsigned short devid)

View File

@@ -737,8 +737,6 @@ static NV_STATUS nv_acpi_evaluate_dsm_method(
rmStatus = nv_acpi_extract_object(dsm, pOutData, *pSize, &data_size);
*pSize = data_size;
kfree(output.pointer);
}
else
{
@@ -751,6 +749,7 @@ static NV_STATUS nv_acpi_evaluate_dsm_method(
"NVRM: %s: DSM data invalid!\n", __FUNCTION__);
}
kfree(output.pointer);
return rmStatus;
}
@@ -1183,6 +1182,7 @@ NvBool nv_acpi_power_resource_method_present(
(object_package->package.count != 0x1))
{
nv_printf(NV_DBG_ERRORS,"NVRM: _PR3 object is not a type 'package'\n");
kfree(buf.pointer);
return NV_FALSE;
}
@@ -1194,8 +1194,10 @@ NvBool nv_acpi_power_resource_method_present(
{
nv_printf(NV_DBG_ERRORS,
"NVRM: _PR3 object does not contain POWER Reference\n");
kfree(buf.pointer);
return NV_FALSE;
}
kfree(buf.pointer);
return NV_TRUE;
}
@@ -1325,6 +1327,7 @@ static acpi_status nv_acpi_find_battery_info(
if (object_package->type != ACPI_TYPE_PACKAGE)
{
nv_printf(NV_DBG_INFO, "NVRM: Battery method output is not package\n");
kfree(buf.pointer);
return AE_OK;
}
@@ -1350,11 +1353,13 @@ static acpi_status nv_acpi_find_battery_info(
if ((object_package->package.elements[battery_technology_offset].type != ACPI_TYPE_INTEGER) ||
(object_package->package.elements[battery_technology_offset].integer.value != BATTERY_RECHARGABLE))
{
kfree(buf.pointer);
return AE_OK;
}
battery_present = NV_TRUE;
kfree(buf.pointer);
/* Stop traversing acpi tree. */
return AE_CTRL_TERMINATE;
}

View File

@@ -71,6 +71,45 @@ void nvidia_modeset_resume(NvU32 gpuId)
}
}
void nvidia_modeset_remove(NvU32 gpuId)
{
if (nv_modeset_callbacks && nv_modeset_callbacks->remove)
{
nv_modeset_callbacks->remove(gpuId);
}
}
static void nvidia_modeset_get_gpu_info(nv_gpu_info_t *gpu_info,
const nv_linux_state_t *nvl)
{
nv_state_t *nv = NV_STATE_PTR(nvl);
int numa_status = nv_get_numa_status(nvl);
gpu_info->gpu_id = nv->gpu_id;
gpu_info->pci_info.domain = nv->pci_info.domain;
gpu_info->pci_info.bus = nv->pci_info.bus;
gpu_info->pci_info.slot = nv->pci_info.slot;
gpu_info->pci_info.function = nv->pci_info.function;
gpu_info->needs_numa_setup =
numa_status != NV_IOCTL_NUMA_STATUS_DISABLED &&
numa_status != NV_IOCTL_NUMA_STATUS_ONLINE;
gpu_info->os_device_ptr = nvl->dev;
}
void nvidia_modeset_probe(const nv_linux_state_t *nvl)
{
if (nv_modeset_callbacks && nv_modeset_callbacks->probe)
{
nv_gpu_info_t gpu_info;
nvidia_modeset_get_gpu_info(&gpu_info, nvl);
nv_modeset_callbacks->probe(&gpu_info);
}
}
static NvU32 nvidia_modeset_enumerate_gpus(nv_gpu_info_t *gpu_info)
{
nv_linux_state_t *nvl;
@@ -82,9 +121,6 @@ static NvU32 nvidia_modeset_enumerate_gpus(nv_gpu_info_t *gpu_info)
for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
{
nv_state_t *nv = NV_STATE_PTR(nvl);
int numa_status = nv_get_numa_status(nvl);
/*
* The gpu_info[] array has NV_MAX_GPUS elements. Fail if there
* are more GPUs than that.
@@ -96,19 +132,7 @@ static NvU32 nvidia_modeset_enumerate_gpus(nv_gpu_info_t *gpu_info)
break;
}
gpu_info[count].gpu_id = nv->gpu_id;
gpu_info[count].pci_info.domain = nv->pci_info.domain;
gpu_info[count].pci_info.bus = nv->pci_info.bus;
gpu_info[count].pci_info.slot = nv->pci_info.slot;
gpu_info[count].pci_info.function = nv->pci_info.function;
gpu_info->needs_numa_setup =
numa_status != NV_IOCTL_NUMA_STATUS_DISABLED &&
numa_status != NV_IOCTL_NUMA_STATUS_ONLINE;
gpu_info[count].os_device_ptr = nvl->dev;
nvidia_modeset_get_gpu_info(&gpu_info[count], nvl);
count++;
}

View File

@@ -1661,6 +1661,8 @@ nv_pci_probe
nv_kmem_cache_free_stack(sp);
nvidia_modeset_probe(nvl);
return 0;
goto err_free_all;
@@ -1760,6 +1762,8 @@ nv_pci_remove(struct pci_dev *pci_dev)
*/
nv_linux_stop_open_q(nvl);
nvidia_modeset_remove(nv->gpu_id);
LOCK_NV_LINUX_DEVICES();
down(&nvl->ldata_lock);
nv->flags |= NV_FLAG_PCI_REMOVE_IN_PROGRESS;
@@ -1899,14 +1903,18 @@ nv_pci_shutdown(struct pci_dev *pci_dev)
{
nv_linux_state_t *nvl = pci_get_drvdata(pci_dev);
if ((nvl != NULL) && nvl->is_forced_shutdown)
{
nvl->is_forced_shutdown = NV_FALSE;
return;
}
if (nvl != NULL)
{
nv_state_t *nv = NV_STATE_PTR(nvl);
if (nvl->is_forced_shutdown)
{
nvl->is_forced_shutdown = NV_FALSE;
return;
}
nvidia_modeset_remove(nv->gpu_id);
nvl->nv_state.is_shutdown = NV_TRUE;
}

View File

@@ -1335,6 +1335,18 @@ static int nv_platform_device_remove_wrapper(struct platform_device *pdev)
}
#endif
static void nv_platform_device_shutdown(struct platform_device *pdev)
{
nv_linux_state_t *nvl = platform_get_drvdata(pdev);
if (nvl != NULL && !nvl->is_forced_shutdown)
{
nv_state_t *nv = NV_STATE_PTR(nvl);
nvidia_modeset_remove(nv->gpu_id);
}
}
const struct of_device_id nv_platform_device_table[] =
{
{ .compatible = "nvidia,tegra234-display",},
@@ -1358,6 +1370,7 @@ struct platform_driver nv_platform_driver = {
},
.probe = nv_platform_device_probe,
.remove = nv_platform_device_remove_wrapper,
.shutdown = nv_platform_device_shutdown,
};
int nv_platform_count_devices(void)