NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-30 04:59:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

580.94.16

Browse Source
This commit is contained in:
russellcnv
2026-01-22 14:46:29 -08:00
parent a3af2867b7
commit c273d84a8b
68 changed files with 160855 additions and 160151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kernel-open/Kbuild
View File

@@ -79,7 +79,7 @@ ccflags-y += -I$(src)/common/inc
ccflags-y += -I$(src)
ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
ccflags-y += -DNV_VERSION_STRING=\"580.94.13\"
ccflags-y += -DNV_VERSION_STRING=\"580.94.16\"
# Include and link Tegra out-of-tree modules.
ifneq ($(wildcard /usr/src/nvidia/nvidia-oot),)

16
kernel-open/common/inc/nv-linux.h
View File

@@ -693,9 +693,9 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
#define NV_PRINT_AT(nv_debug_level,at) \
{ \
nv_printf(nv_debug_level, \
"NVRM: VM: %s:%d: 0x%p, %d page(s), count = %d, " \
"NVRM: VM: %s:%d: 0x%p, %d page(s), count = %lld, " \
"page_table = 0x%p\n", __FUNCTION__, __LINE__, at, \
at->num_pages, NV_ATOMIC_READ(at->usage_count), \
at->num_pages, (long long)atomic64_read(&at->usage_count), \
at->page_table); \
}
@@ -970,7 +970,7 @@ struct nv_dma_buf
typedef struct nv_alloc_s {
struct nv_alloc_s *next;
struct device *dev;
atomic_t usage_count;
atomic64_t usage_count;
struct {
NvBool contig : 1;
NvBool guest : 1;
@@ -1258,6 +1258,10 @@ typedef struct coherent_link_info_s {
* of virutalized OS environment it is Intermediate Physical Address(IPA) */
NvU64 gpu_mem_pa;
/* Size of the GPU memory mappable through coherent link. It is possible
that less than whole FB is mapped to CPU. */
NvU64 gpu_mem_size;
/* Physical address of the reserved portion of the GPU memory, applicable
* only in Grace Hopper self hosted passthrough virtualizatioan platform. */
NvU64 rsvd_mem_pa;
@@ -1294,7 +1298,7 @@ struct nv_pci_tegra_devfreq_dev;
typedef struct nv_linux_state_s {
nv_state_t nv_state;
atomic_t usage_count;
atomic64_t usage_count;
NvU32 suspend_count;
@@ -1670,9 +1674,9 @@ static inline NvBool nv_alloc_release(nv_linux_file_private_t *nvlfp, nv_alloc_t
{
NV_PRINT_AT(NV_DBG_MEMINFO, at);
if (NV_ATOMIC_DEC_AND_TEST(at->usage_count))
if (atomic64_dec_and_test(&at->usage_count))
{
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
at->next = nvlfp->free_list;
nvlfp->free_list = at;

30
kernel-open/common/inc/nv-mm.h
View File

@@ -196,14 +196,33 @@ static inline struct rw_semaphore *nv_mmap_get_lock(struct mm_struct *mm)
* Commit 45ad9f5290dc updated vma_start_write() to call __vma_start_write().
*/
void nv_vma_start_write(struct vm_area_struct *);
static inline void nv_vma_flags_set_word(struct vm_area_struct *vma, unsigned long flags)
{
nv_vma_start_write(vma);
#if defined(NV_VMA_FLAGS_SET_WORD_PRESENT)
vma_flags_set_word(&vma->flags, flags);
#else
ACCESS_PRIVATE(vma, __vm_flags) |= flags;
#endif
}
static inline void nv_vma_flags_clear_word(struct vm_area_struct *vma, unsigned long flags)
{
nv_vma_start_write(vma);
#if defined(NV_VMA_FLAGS_SET_WORD_PRESENT)
vma_flags_clear_word(&vma->flags, flags);
#else
ACCESS_PRIVATE(vma, __vm_flags) &= ~flags;
#endif
}
#endif // !NV_CAN_CALL_VMA_START_WRITE
static inline void nv_vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags)
{
#if !NV_CAN_CALL_VMA_START_WRITE
nv_vma_start_write(vma);
ACCESS_PRIVATE(vma, __vm_flags) |= flags;
#elif defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
nv_vma_flags_set_word(vma, flags);
#elif defined(NV_VM_FLAGS_SET_PRESENT)
vm_flags_set(vma, flags);
#else
vma->vm_flags |= flags;
@@ -213,9 +232,8 @@ static inline void nv_vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags)
static inline void nv_vm_flags_clear(struct vm_area_struct *vma, vm_flags_t flags)
{
#if !NV_CAN_CALL_VMA_START_WRITE
nv_vma_start_write(vma);
ACCESS_PRIVATE(vma, __vm_flags) &= ~flags;
#elif defined(NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS)
nv_vma_flags_clear_word(vma, flags);
#elif defined(NV_VM_FLAGS_SET_PRESENT)
vm_flags_clear(vma, flags);
#else
vma->vm_flags &= ~flags;

2
kernel-open/common/inc/nv.h
View File

@@ -959,7 +959,7 @@ void NV_API_CALL nv_put_firmware(const void *);
nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **);
void NV_API_CALL nv_put_file_private(void *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
void NV_API_CALL nv_p2p_free_platform_data(void *data);

163
kernel-open/conftest.sh
View File

@@ -1139,6 +1139,45 @@ compile_test() {
compile_check_conftest "$CODE" "NV_PFN_ADDRESS_SPACE_STRUCT_PRESENT" "" "types"
;;
irq_bypass_producer_has_token)
#
# Determine if 'struct irq_bypass_producer' has 'token' field
#
# Added by commit 2b521d86ee80 ("irqbypass: Take ownership of
# producer/consumer token tracking") in v6.17
#
CODE="
#include <linux/irqbypass.h>
int conftest_irq_bypass_producer_has_token(void) {
return offsetof(struct irq_bypass_producer, token);
}"
compile_check_conftest "$CODE" "NV_IRQ_BYPASS_PRODUCER_HAS_TOKEN" "" "types"
;;
irq_bypass_register_producer_has_eventfd_and_irq_args)
#
# Determine if irq_bypass_register_producer() function has
# additional 'eventfd' and 'irq' arguments.
#
# Added by commits 2b521d86ee80 ("irqbypass: Take ownership of
# producer/consumer token tracking") and 23b54381cee2
# ("irqbypass: Require producers to pass in Linux IRQ number
# during registration") in v6.17
#
CODE="
#include <linux/irqbypass.h>
#include <linux/eventfd.h>
void conftest_irq_bypass_register_producer_has_eventfd_and_irq_args(void) {
struct irq_bypass_producer *prod = NULL;
struct eventfd_ctx *eventfd = NULL;
int irq = 0;
irq_bypass_register_producer(prod, eventfd, irq);
}"
compile_check_conftest "$CODE" "NV_IRQ_BYPASS_REGISTER_PRODUCER_HAS_EVENTFD_AND_IRQ_ARGS" "" "types"
;;
egm_module_helper_api_present)
#
# Determine if egm management api are present or not.
@@ -1351,6 +1390,45 @@ compile_test() {
compile_check_conftest "$CODE" "NV_GET_DEV_PAGEMAP_HAS_PGMAP_ARG" "" "types"
;;
zone_device_page_init_has_order_arg)
#
# Determine if the zone_device_page_init() has an extra argument
#
# This change was introduced by d245f9b4ab80
# ("mm/zone_device: support large zone device private folios")
#
# in linux-next, expected in v6.19.
#
CODE="
#include <linux/memremap.h>
void init_page(void) {
struct page *page;
zone_device_page_init(page, 0);
}"
compile_check_conftest "$CODE" "NV_ZONE_DEVICE_PAGE_INIT_HAS_ORDER_ARG" "" "types"
;;
dev_pagemap_ops_has_folio_free)
#
# Determine if the zone device now uses a folio_free() as the callback
# function instead of page_free()
#
# This change was introduced by 3a5a06554566
# (mm/zone_device: rename page_free callback to folio_free)
#
# in linux-next, expected in v6.19.
#
CODE="
#include <linux/memremap.h>
void test_folio_free(struct folio *folio) {
}
void set_folio_free_ops(void) {
struct dev_pagemap_ops ops;
ops.folio_free = test_folio_free;
}"
compile_check_conftest "$CODE" "NV_PAGEMAP_OPS_HAS_FOLIO_FREE" "" "types"
;;
drm_available)
# Determine if the DRM subsystem is usable
CODE="
@@ -2223,6 +2301,35 @@ compile_test() {
compile_check_conftest "$CODE" "NV_GET_BACKLIGHT_DEVICE_BY_NAME_PRESENT" "" "functions"
;;
dma_map_ops_has_map_phys)
#
# Determine if .map_phys exists in struct dma_map_ops.
#
# Commit 14cb413af00c ("dma-mapping: remove unused mapping resource callbacks")
# removed .map_resource operation and replaced it with .map_phys.
#
echo "$CONFTEST_PREAMBLE
#include <linux/dma-map-ops.h>
int conftest_dma_map_ops_has_map_phys(void) {
return offsetof(struct dma_map_ops, map_phys);
}
int conftest_dma_map_ops_has_unmap_phys(void) {
return offsetof(struct dma_map_ops, unmap_phys);
}" > conftest$$.c
$CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
rm -f conftest$$.c
if [ -f conftest$$.o ]; then
echo "#define NV_DMA_MAP_OPS_HAS_MAP_PHYS" | append_conftest "types"
rm -f conftest$$.o
return
else
echo "#undef NV_DMA_MAP_OPS_HAS_MAP_PHYS" | append_conftest "types"
return
fi
;;
dma_buf_ops_has_map)
#
# Determine if .map exists in dma_buf_ops.
@@ -3938,6 +4045,27 @@ compile_test() {
compile_check_conftest "$CODE" "NV_PCI_REBAR_GET_POSSIBLE_SIZES_PRESENT" "" "functions"
;;
pci_resize_resource_has_exclude_bars_arg)
#
# Determine if pci_resize_resource() has exclude_bars argument.
#
# exclude_bars argument was added to pci_resize_resource by commit
# 337b1b566db0 (11/14/2025) ("PCI: Fix restoring BARs on BAR resize rollback path")
# in linux-next.
#
CODE="
#include <linux/pci.h>
typeof(pci_resize_resource) conftest_pci_resize_resource_has_exclude_bars_arg;
int __must_check conftest_pci_resize_resource_has_exclude_bars_arg(struct pci_dev *dev,
int i, int size,
int exclude_bars) {
return 0;
}"
compile_check_conftest "$CODE" "NV_PCI_RESIZE_RESOURCE_HAS_EXCLUDE_BARS_ARG" "" "types"
;;
drm_connector_has_override_edid)
#
# Determine if 'struct drm_connector' has an 'override_edid' member.
@@ -3976,22 +4104,39 @@ compile_test() {
compile_check_conftest "$CODE" "NV_IOMMU_SVA_BIND_DEVICE_HAS_DRVDATA_ARG" "" "types"
;;
vm_area_struct_has_const_vm_flags)
vm_flags_set)
#
# Determine if the 'vm_area_struct' structure has
# const 'vm_flags'.
# Determine if the vm_flags_set() function is present. The
# presence of this function indicates that the vm_flags_clear()
# function is also present.
#
# A union of '__vm_flags' and 'const vm_flags' was added by
# The functions vm_flags_set()/ vm_flags_clear() were added by
# commit bc292ab00f6c ("mm: introduce vma->vm_flags wrapper
# functions") in v6.3.
# functions") in v6.3-rc1 (2023-02-09).
#
CODE="
#include <linux/mm_types.h>
int conftest_vm_area_struct_has_const_vm_flags(void) {
return offsetof(struct vm_area_struct, __vm_flags);
#include <linux/mm.h>
void conftest_vm_flags_set(void) {
vm_flags_set();
}"
compile_check_conftest "$CODE" "NV_VM_AREA_STRUCT_HAS_CONST_VM_FLAGS" "" "types"
compile_check_conftest "$CODE" "NV_VM_FLAGS_SET_PRESENT" "" "functions"
;;
vma_flags_set_word)
#
# Determine if the vma_flags_set_word() function is present.
#
# Added by commit c3f7c506e8f1 ("mm: introduce VMA flags bitmap type")
# in v6.19-rc1.
#
CODE="
#include <linux/mm.h>
void conftest_vma_flags_set_word(void) {
vma_flags_set_word();
}"
compile_check_conftest "$CODE" "NV_VMA_FLAGS_SET_WORD_PRESENT" "" "functions"
;;
drm_driver_has_dumb_destroy)

2
kernel-open/nvidia-drm/nvidia-drm-sources.mk
View File

@@ -62,6 +62,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_plane_create_color_properties
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_atomic_helper_legacy_gamma_set
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vmf_insert_mixed
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_gem_prime_mmap
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_flags_set
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_legacy_dev_list
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_ops_fault_removed_vma_arg
@@ -91,7 +92,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += reservation_object_reserve_shared_has_num_fenc
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_has_override_edid
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_file_get_master
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_modeset_lock_all_end
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_dumb_destroy
NV_CONFTEST_TYPE_COMPILE_TESTS += fence_ops_use_64bit_seqno
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_aperture_remove_conflicting_framebuffers_has_driver_arg

2
kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
View File

@@ -62,6 +62,8 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_vma_added_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_device_range
NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_pt_regs_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += get_dev_pagemap_has_pgmap_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += dev_pagemap_ops_has_folio_free
NV_CONFTEST_TYPE_COMPILE_TESTS += zone_device_page_init_has_order_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
NV_CONFTEST_TYPE_COMPILE_TESTS += mpol_preferred_many_present

12
kernel-open/nvidia-uvm/uvm_hmm.c
View File

@@ -74,6 +74,16 @@ module_param(uvm_disable_hmm, bool, 0444);
#include "uvm_va_policy.h"
#include "uvm_tools.h"
//
// Pass 0 as the order, when actual large order support is added this
// function will need to be revisited
//
#if defined(NV_ZONE_DEVICE_PAGE_INIT_HAS_ORDER_ARG)
#define ZONE_DEVICE_PAGE_INIT(page) zone_device_page_init(page, 0)
#else
#define ZONE_DEVICE_PAGE_INIT(page) zone_device_page_init(page)
#endif
// The function nv_PageSwapCache() wraps the check for page swap cache flag in
// order to support a wide variety of kernel versions.
// The function PageSwapCache() is removed after 32f51ead3d77 ("mm: remove
@@ -2146,7 +2156,7 @@ static void fill_dst_pfn(uvm_va_block_t *va_block,
UVM_ASSERT(!page_count(dpage));
UVM_ASSERT(!dpage->zone_device_data);
zone_device_page_init(dpage);
ZONE_DEVICE_PAGE_INIT(dpage);
dpage->zone_device_data = gpu_chunk;
atomic64_inc(&va_block->hmm.va_space->hmm.allocated_page_count);
}

35
kernel-open/nvidia-uvm/uvm_pmm_gpu.c
View File

@@ -3159,6 +3159,11 @@ static void devmem_page_free(struct page *page)
&gpu->pmm.root_chunks.va_block_lazy_free_q_item);
}
static void devmem_folio_free(struct folio *folio)
{
devmem_page_free(&folio->page);
}
// This is called by HMM when the CPU faults on a ZONE_DEVICE private entry.
static vm_fault_t devmem_fault(struct vm_fault *vmf)
{
@@ -3177,7 +3182,11 @@ static vm_fault_t devmem_fault_entry(struct vm_fault *vmf)
static const struct dev_pagemap_ops uvm_pmm_devmem_ops =
{
#if defined(NV_PAGEMAP_OPS_HAS_FOLIO_FREE)
.folio_free = devmem_folio_free,
#else
.page_free = devmem_page_free,
#endif
.migrate_to_ram = devmem_fault_entry,
};
@@ -3272,6 +3281,11 @@ static void device_p2p_page_free(struct page *page)
page->zone_device_data = NULL;
nv_kref_put(&p2p_mem->refcount, device_p2p_page_free_wake);
}
static void device_p2p_folio_free(struct folio *folio)
{
device_p2p_page_free(&folio->page);
}
#endif
#if UVM_CDMM_PAGES_SUPPORTED()
@@ -3280,9 +3294,18 @@ static void device_coherent_page_free(struct page *page)
device_p2p_page_free(page);
}
static void device_coherent_folio_free(struct folio *folio)
{
device_p2p_page_free(&folio->page);
}
static const struct dev_pagemap_ops uvm_device_coherent_pgmap_ops =
{
#if defined(NV_PAGEMAP_OPS_HAS_FOLIO_FREE)
.folio_free = device_coherent_folio_free,
#else
.page_free = device_coherent_page_free,
#endif
};
static NV_STATUS uvm_pmm_cdmm_init(uvm_parent_gpu_t *parent_gpu)
@@ -3419,7 +3442,11 @@ static bool uvm_pmm_gpu_check_orphan_pages(uvm_pmm_gpu_t *pmm)
static const struct dev_pagemap_ops uvm_device_p2p_pgmap_ops =
{
#if defined(NV_PAGEMAP_OPS_HAS_FOLIO_FREE)
.folio_free = device_p2p_folio_free,
#else
.page_free = device_p2p_page_free,
#endif
};
void uvm_pmm_gpu_device_p2p_init(uvm_parent_gpu_t *parent_gpu)
@@ -3477,12 +3504,10 @@ void uvm_pmm_gpu_device_p2p_init(uvm_parent_gpu_t *parent_gpu)
void uvm_pmm_gpu_device_p2p_deinit(uvm_parent_gpu_t *parent_gpu)
{
unsigned long pci_start_pfn = pci_resource_start(parent_gpu->pci_dev,
uvm_device_p2p_static_bar(parent_gpu)) >> PAGE_SHIFT;
struct page *p2p_page;
if (parent_gpu->device_p2p_initialised && !uvm_parent_gpu_is_coherent(parent_gpu)) {
p2p_page = pfn_to_page(pci_start_pfn);
struct page *p2p_page = pfn_to_page(pci_resource_start(parent_gpu->pci_dev,
uvm_device_p2p_static_bar(parent_gpu)) >> PAGE_SHIFT);
devm_memunmap_pages(&parent_gpu->pci_dev->dev, page_pgmap(p2p_page));
}

8
kernel-open/nvidia-uvm/uvm_populate_pageable.c
View File

@@ -21,6 +21,8 @@
*******************************************************************************/
#include <linux/sched.h>
#include "uvm_common.h"
#include "uvm_ioctl.h"
#include "uvm_linux.h"
@@ -63,6 +65,12 @@ static NV_STATUS handle_fault(struct vm_area_struct *vma, unsigned long start, u
ret = UVM_HANDLE_MM_FAULT(vma, start + (i * PAGE_SIZE), fault_flags);
if (ret & VM_FAULT_ERROR)
return errno_to_nv_status(vm_fault_to_errno(ret, fault_flags));
// Depending on the kernel version and the active preemption model,
// calls to handle_mm_fault might not have had a chance to check for
// for scheduling points. Insert an explicit yield point here to prevent
// large buffers from triggering soft lockup timeout.
cond_resched();
}
return NV_OK;

4
kernel-open/nvidia/nv-dma.c
View File

@@ -718,7 +718,11 @@ static NvBool nv_dma_use_map_resource
#endif
}
#if defined(NV_DMA_MAP_OPS_HAS_MAP_PHYS)
return (ops->map_phys != NULL);
#else
return (ops->map_resource != NULL);
#endif
}
/* DMA-map a peer device's C2C aperture for peer access. */

37
kernel-open/nvidia/nv-dmabuf.c
View File

@@ -253,7 +253,7 @@ nv_inc_and_check_one_phys_refcount(
return is_one;
}
// Must be called with RMAPI lock and GPU lock taken
// Must be called with RMAPI lock and all-GPU lock taken
static void
nv_dma_buf_undup_mem_handles_unlocked(
nvidia_stack_t *sp,
@@ -284,6 +284,28 @@ nv_dma_buf_undup_mem_handles_unlocked(
}
}
// Must be called with RMAPI lock taken
static void
nv_dma_buf_undup_mem_handles_gpus_locked(
nvidia_stack_t *sp,
NvU32 start_index,
NvU32 num_objects,
nv_dma_buf_file_private_t *priv
)
{
NV_STATUS status;
status = rm_acquire_all_gpus_lock(sp);
if (WARN_ON(status != NV_OK))
{
return;
}
nv_dma_buf_undup_mem_handles_unlocked(sp, start_index, num_objects, priv);
rm_release_all_gpus_lock(sp);
}
static void
nv_dma_buf_undup_mem_handles(
nvidia_stack_t *sp,
@@ -300,17 +322,8 @@ nv_dma_buf_undup_mem_handles(
return;
}
status = rm_acquire_all_gpus_lock(sp);
if (WARN_ON(status != NV_OK))
{
goto unlock_api_lock;
}
nv_dma_buf_undup_mem_handles_gpus_locked(sp, index, num_objects, priv);
nv_dma_buf_undup_mem_handles_unlocked(sp, index, num_objects, priv);
rm_release_all_gpus_lock(sp);
unlock_api_lock:
rm_release_api_lock(sp);
}
@@ -478,7 +491,7 @@ failed:
//
nv_dma_buf_release_gpu_lock(sp, priv);
nv_dma_buf_undup_mem_handles(sp, params->index, count, priv);
nv_dma_buf_undup_mem_handles_gpus_locked(sp, params->index, count, priv);
}
else
{

18
kernel-open/nvidia/nv-mmap.c
View File

@@ -72,7 +72,7 @@ nvidia_vma_open(struct vm_area_struct *vma)
if (at != NULL)
{
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
}
@@ -128,6 +128,9 @@ nvidia_vma_access(
void *kernel_mapping;
const nv_alloc_mapping_context_t *mmap_context = &nvlfp->mmap_context;
NvU64 offsInVma = addr - vma->vm_start;
NvBool bIsNuma = NV_FALSE;
bIsNuma = pfn_valid(mmap_context->access_start >> PAGE_SHIFT);
pageIndex = (offsInVma >> PAGE_SHIFT);
pageOffset = (offsInVma & ~PAGE_MASK);
@@ -164,10 +167,17 @@ nvidia_vma_access(
pageIndex = nv_array_index_no_speculate(pageIndex, at->num_pages);
kernel_mapping = (void *)(at->page_table[pageIndex].virt_addr + pageOffset);
}
else if (bIsNuma)
{
struct page *pPage = NV_GET_PAGE_STRUCT(mmap_context->page_array[pageIndex]);
NvU8 *pPagePtr = (NvU8 *) page_address(pPage);
kernel_mapping = &pPagePtr[pageOffset];
}
else
{
NvU64 idx = 0;
NvU64 curOffs = 0;
for(; idx < mmap_context->memArea.numRanges; idx++)
{
NvU64 nextOffs = mmap_context->memArea.pRanges[idx].size + curOffs;
@@ -195,7 +205,7 @@ found:
else
memcpy(buffer, kernel_mapping, length);
if (at == NULL)
if (at == NULL && !bIsNuma)
{
kernel_mapping = ((char *)kernel_mapping - pageOffset);
os_unmap_kernel_space(kernel_mapping, PAGE_SIZE);
@@ -408,7 +418,7 @@ static int nvidia_mmap_sysmem(
int ret = 0;
unsigned long start = 0;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
start = vma->vm_start;
for (j = page_index; j < (page_index + pages); j++)
@@ -444,7 +454,7 @@ static int nvidia_mmap_sysmem(
if (ret)
{
NV_ATOMIC_DEC(at->usage_count);
atomic64_dec(&at->usage_count);
nv_printf(NV_DBG_ERRORS,
"NVRM: Userspace mapping creation failed [%d]!\n", ret);
return -EAGAIN;

18
kernel-open/nvidia/nv-pci.c
View File

@@ -229,7 +229,11 @@ static int nv_resize_pcie_bars(struct pci_dev *pci_dev) {
resize:
/* Attempt to resize BAR1 to the largest supported size */
#if defined(NV_PCI_RESIZE_RESOURCE_HAS_EXCLUDE_BARS_ARG)
r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size, 0);
#else
r = pci_resize_resource(pci_dev, NV_GPU_BAR1, requested_size);
#endif
if (r) {
if (r == -ENOSPC)
@@ -420,6 +424,12 @@ nv_init_coherent_link_info
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-base-pa", &pa) == 0)
{
nvl->coherent_link_info.gpu_mem_pa = pa;
NvU64 gpu_mem_size;
if (device_property_read_u64(nvl->dev, "nvidia,gpu-mem-size", &gpu_mem_size) == 0)
{
nvl->coherent_link_info.gpu_mem_size = gpu_mem_size;
}
}
else
{
@@ -1774,7 +1784,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
* For eGPU, fall off the bus along with clients active is a valid scenario.
* Hence skipping the sanity check for eGPU.
*/
if ((NV_ATOMIC_READ(nvl->usage_count) != 0) && !(nv->is_external_gpu))
if ((atomic64_read(&nvl->usage_count) != 0) && !(nv->is_external_gpu))
{
nv_printf(NV_DBG_ERRORS,
"NVRM: Attempting to remove device %04x:%02x:%02x.%x with non-zero usage count!\n",
@@ -1785,7 +1795,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
* We can't return from this function without corrupting state, so we wait for
* the usage count to go to zero.
*/
while (NV_ATOMIC_READ(nvl->usage_count) != 0)
while (atomic64_read(&nvl->usage_count) != 0)
{
/*
@@ -1863,7 +1873,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
nvl->sysfs_config_file = NULL;
}
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv_lock_destroy_locks(sp, nv);
}
@@ -1879,7 +1889,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
num_nv_devices--;
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
NV_PCI_DISABLE_DEVICE(pci_dev);
NV_KFREE(nvl, sizeof(nv_linux_state_t));

2
kernel-open/nvidia/nv-procfs.c
View File

@@ -889,7 +889,7 @@ nv_procfs_close_unbind_lock(
down(&nvl->ldata_lock);
if ((value == 1) && !(nv->flags & NV_FLAG_UNBIND_LOCK))
{
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
rm_unbind_lock(sp, nv);
if (nv->flags & NV_FLAG_UNBIND_LOCK)

44
kernel-open/nvidia/nv.c
View File

@@ -413,7 +413,7 @@ nv_alloc_t *nvos_create_alloc(
return NULL;
}
NV_ATOMIC_SET(at->usage_count, 0);
atomic64_set(&at->usage_count, 0);
at->pid = os_get_current_process();
at->dev = dev;
@@ -428,7 +428,7 @@ int nvos_free_alloc(
if (at == NULL)
return -1;
if (NV_ATOMIC_READ(at->usage_count))
if (atomic64_read(&at->usage_count))
return 1;
kvfree(at->page_table);
@@ -1646,13 +1646,10 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
return -ENODEV;
}
if (unlikely(NV_ATOMIC_READ(nvl->usage_count) >= NV_S32_MAX))
return -EMFILE;
if ( ! (nv->flags & NV_FLAG_OPEN))
{
/* Sanity check: !NV_FLAG_OPEN requires usage_count == 0 */
if (NV_ATOMIC_READ(nvl->usage_count) != 0)
if (atomic64_read(&nvl->usage_count) != 0)
{
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
"Minor device %u is referenced without being open!\n",
@@ -1674,7 +1671,7 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
nv_assert_not_in_gpu_exclusion_list(sp, nv);
NV_ATOMIC_INC(nvl->usage_count);
atomic64_inc(&nvl->usage_count);
return 0;
}
@@ -2102,7 +2099,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv_printf(NV_DBG_ERRORS,
"NVRM: Attempting to close unopened minor device %u!\n",
@@ -2111,7 +2108,7 @@ static void nv_close_device(nv_state_t *nv, nvidia_stack_t *sp)
return;
}
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
if (atomic64_dec_and_test(&nvl->usage_count))
nv_stop_device(nv, sp);
}
@@ -2156,7 +2153,7 @@ nvidia_close_callback(
nv_close_device(nv, sp);
bRemove = (!NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv)) &&
(NV_ATOMIC_READ(nvl->usage_count) == 0) &&
(atomic64_read(&nvl->usage_count) == 0) &&
rm_get_device_remove_flag(sp, nv->gpu_id);
nv_free_file_private(nvlfp);
@@ -2175,7 +2172,7 @@ nvidia_close_callback(
* any cleanup related to linux layer locks and nv linux state struct.
* nvidia_pci_remove when scheduled will do necessary cleanup.
*/
if ((NV_ATOMIC_READ(nvl->usage_count) == 0) && nv->removed)
if ((atomic64_read(&nvl->usage_count) == 0) && nv->removed)
{
nv_lock_destroy_locks(sp, nv);
NV_KFREE(nvl, sizeof(nv_linux_state_t));
@@ -2694,7 +2691,7 @@ nvidia_ioctl(
* Only the current client should have an open file
* descriptor for the device, to allow safe offlining.
*/
if (NV_ATOMIC_READ(nvl->usage_count) > 1)
if (atomic64_read(&nvl->usage_count) > 1)
{
status = -EBUSY;
goto unlock;
@@ -3083,12 +3080,12 @@ nvidia_ctl_open(
/* save the nv away in file->private_data */
nvlfp->nvptr = nvl;
if (NV_ATOMIC_READ(nvl->usage_count) == 0)
if (atomic64_read(&nvl->usage_count) == 0)
{
nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
}
NV_ATOMIC_INC(nvl->usage_count);
atomic64_inc(&nvl->usage_count);
up(&nvl->ldata_lock);
return 0;
@@ -3113,7 +3110,7 @@ nvidia_ctl_close(
nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_close\n");
down(&nvl->ldata_lock);
if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
if (atomic64_dec_and_test(&nvl->usage_count))
{
nv->flags &= ~NV_FLAG_OPEN;
}
@@ -3276,7 +3273,7 @@ nv_alias_pages(
at->guest_id = guest_id;
*priv_data = at;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
@@ -3589,7 +3586,7 @@ NV_STATUS NV_API_CALL nv_register_sgt(
at->order = get_order(at->num_pages * PAGE_SIZE);
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
*priv_data = at;
@@ -3620,7 +3617,7 @@ void NV_API_CALL nv_unregister_sgt(
*import_priv = at->import_priv;
}
if (NV_ATOMIC_DEC_AND_TEST(at->usage_count))
if (atomic64_dec_and_test(&at->usage_count))
{
nvos_free_alloc(at);
}
@@ -3893,7 +3890,7 @@ NV_STATUS NV_API_CALL nv_alloc_pages(
}
*priv_data = at;
NV_ATOMIC_INC(at->usage_count);
atomic64_inc(&at->usage_count);
NV_PRINT_AT(NV_DBG_MEMINFO, at);
@@ -3929,7 +3926,7 @@ NV_STATUS NV_API_CALL nv_free_pages(
* This is described in greater detail in the comments above the
* nvidia_vma_(open|release)() callbacks in nv-mmap.c.
*/
if (!NV_ATOMIC_DEC_AND_TEST(at->usage_count))
if (!atomic64_dec_and_test(&at->usage_count))
return NV_OK;
if (!at->flags.guest && !at->import_sgt)
@@ -3958,7 +3955,7 @@ NvBool nv_lock_init_locks
NV_INIT_MUTEX(&nvl->mmap_lock);
NV_INIT_MUTEX(&nvl->open_q_lock);
NV_ATOMIC_SET(nvl->usage_count, 0);
atomic64_set(&nvl->usage_count, 0);
if (!rm_init_event_locks(sp, nv))
return NV_FALSE;
@@ -5342,6 +5339,7 @@ NV_STATUS NV_API_CALL nv_get_device_memory_config(
nv_state_t *nv,
NvU64 *compr_addr_sys_phys,
NvU64 *addr_guest_phys,
NvU64 *size_guest_phys,
NvU64 *rsvd_phys,
NvU32 *addr_width,
NvS32 *node_id
@@ -5365,6 +5363,10 @@ NV_STATUS NV_API_CALL nv_get_device_memory_config(
{
*addr_guest_phys = nvl->coherent_link_info.gpu_mem_pa;
}
if (size_guest_phys)
{
*size_guest_phys = nvl->coherent_link_info.gpu_mem_size;
}
if (rsvd_phys)
{
*rsvd_phys = nvl->coherent_link_info.rsvd_mem_pa;

5
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -145,6 +145,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += devm_of_icc_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_put
NV_CONFTEST_FUNCTION_COMPILE_TESTS += icc_set_bw
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_map_ops_has_map_phys
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_ops_has_map_atomic
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_buf_attachment_has_peer2peer
@@ -158,6 +159,8 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += assign_str
NV_CONFTEST_FUNCTION_COMPILE_TESTS += ioasid_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += mm_pasid_drop
NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vma_flags_set_word
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_flags_set
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_gpl_sme_active
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_swiotlb_map_sg_attrs
@@ -205,7 +208,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += remove_memory_has_nid_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_driver_has_driver_managed_dma
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_area_struct_has_const_vm_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += memory_failure_queue_has_trapno_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += foll_longterm_present
NV_CONFTEST_TYPE_COMPILE_TESTS += bus_type_has_iommu_ops
@@ -215,6 +217,7 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += class_create_has_no_owner_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += class_devnode_has_const_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += devfreq_has_freq_table
NV_CONFTEST_TYPE_COMPILE_TESTS += has_enum_pidtype_tgid
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_resize_resource_has_exclude_bars_arg
NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build