NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-20 23:13:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

575.51.02

Browse Source
This commit is contained in:
Bernhard Stoeckner
2025-04-17 19:35:38 +02:00
parent e8113f665d
commit 4159579888
1142 changed files with 309085 additions and 272273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
kernel-open/common/inc/nv-linux.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2001-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2001-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -36,8 +36,7 @@
#include "nv-timer.h"
#include "nv-time.h"
#include "nv-chardev-numbers.h"
#define NV_KERNEL_NAME "Linux"
#include "nv-platform.h"
#ifndef AUTOCONF_INCLUDED
#if defined(NV_GENERATED_AUTOCONF_H_PRESENT)
@@ -239,7 +238,7 @@ NV_STATUS nvos_forward_error_to_cray(struct pci_dev *, NvU32,
#undef NV_SET_PAGES_UC_PRESENT
#endif
#if !defined(NVCPU_AARCH64) && !defined(NVCPU_PPC64LE) && !defined(NVCPU_RISCV64)
#if !defined(NVCPU_AARCH64) && !defined(NVCPU_RISCV64)
#if !defined(NV_SET_MEMORY_UC_PRESENT) && !defined(NV_SET_PAGES_UC_PRESENT)
#error "This driver requires the ability to change memory types!"
#endif
@@ -403,7 +402,7 @@ typedef enum
NV_MEMORY_TYPE_DEVICE_MMIO, /* All kinds of MMIO referred by NVRM e.g. BARs and MCFG of device */
} nv_memory_type_t;
#if defined(NVCPU_AARCH64) || defined(NVCPU_PPC64LE) || defined(NVCPU_RISCV64)
#if defined(NVCPU_AARCH64) || defined(NVCPU_RISCV64)
#define NV_ALLOW_WRITE_COMBINING(mt) 1
#elif defined(NVCPU_X86_64)
#if defined(NV_ENABLE_PAT_SUPPORT)
@@ -461,10 +460,7 @@ static inline void *nv_vmalloc(unsigned long size)
#else
void *ptr = __vmalloc(size, GFP_KERNEL);
#endif
if (ptr)
{
NV_MEMDBG_ADD(ptr, size);
}
NV_MEMDBG_ADD(ptr, size);
return ptr;
}
@@ -481,10 +477,7 @@ static inline void *nv_ioremap(NvU64 phys, NvU64 size)
#else
void *ptr = ioremap(phys, size);
#endif
if (ptr)
{
NV_MEMDBG_ADD(ptr, size);
}
NV_MEMDBG_ADD(ptr, size);
return ptr;
}
@@ -500,29 +493,12 @@ static inline void *nv_ioremap_cache(NvU64 phys, NvU64 size)
ptr = ioremap_cache_shared(phys, size);
#elif defined(NV_IOREMAP_CACHE_PRESENT)
ptr = ioremap_cache(phys, size);
#elif defined(NVCPU_PPC64LE)
//
// ioremap_cache() has been only implemented correctly for ppc64le with
// commit f855b2f544d6 in April 2017 (kernel 4.12+). Internally, the kernel
// does provide a default implementation of ioremap_cache() that would be
// incorrect for our use (creating an uncached mapping) before the
// referenced commit, but that implementation is not exported and the
// NV_IOREMAP_CACHE_PRESENT conftest doesn't pick it up, and we end up in
// this #elif branch.
//
// At the same time, ppc64le have supported ioremap_prot() since May 2011
// (commit 40f1ce7fb7e8, kernel 3.0+) and that covers all kernels we
// support on power.
//
ptr = ioremap_prot(phys, size, pgprot_val(PAGE_KERNEL));
#else
return nv_ioremap(phys, size);
#endif
if (ptr)
{
NV_MEMDBG_ADD(ptr, size);
}
NV_MEMDBG_ADD(ptr, size);
return ptr;
}
@@ -537,10 +513,8 @@ static inline void *nv_ioremap_wc(NvU64 phys, NvU64 size)
return nv_ioremap_nocache(phys, size);
#endif
if (ptr)
{
NV_MEMDBG_ADD(ptr, size);
}
NV_MEMDBG_ADD(ptr, size);
return ptr;
}
@@ -560,22 +534,19 @@ static NvBool nv_numa_node_has_memory(int node_id)
#define NV_KMALLOC(ptr, size) \
{ \
(ptr) = kmalloc(size, NV_GFP_KERNEL); \
if (ptr) \
NV_MEMDBG_ADD(ptr, size); \
NV_MEMDBG_ADD(ptr, size); \
}
#define NV_KZALLOC(ptr, size) \
{ \
(ptr) = kzalloc(size, NV_GFP_KERNEL); \
if (ptr) \
NV_MEMDBG_ADD(ptr, size); \
NV_MEMDBG_ADD(ptr, size); \
}
#define NV_KMALLOC_ATOMIC(ptr, size) \
{ \
(ptr) = kmalloc(size, NV_GFP_ATOMIC); \
if (ptr) \
NV_MEMDBG_ADD(ptr, size); \
NV_MEMDBG_ADD(ptr, size); \
}
#if defined(__GFP_RETRY_MAYFAIL)
@@ -589,8 +560,7 @@ static NvBool nv_numa_node_has_memory(int node_id)
#define NV_KMALLOC_NO_OOM(ptr, size) \
{ \
(ptr) = kmalloc(size, NV_GFP_NO_OOM); \
if (ptr) \
NV_MEMDBG_ADD(ptr, size); \
NV_MEMDBG_ADD(ptr, size); \
}
#define NV_KFREE(ptr, size) \
@@ -623,9 +593,9 @@ static inline pgprot_t nv_sme_clr(pgprot_t prot)
#endif // __sme_clr
}
static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot, NvU32 extra)
static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot)
{
pgprot_t prot = __pgprot(pgprot_val(vm_prot) | extra);
pgprot_t prot = __pgprot(pgprot_val(vm_prot));
#if defined(pgprot_decrypted)
return pgprot_decrypted(prot);
@@ -646,41 +616,6 @@ static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot, NvU32 extra)
#endif
#endif
static inline NvUPtr nv_vmap(struct page **pages, NvU32 page_count,
NvBool cached, NvBool unencrypted)
{
void *ptr;
pgprot_t prot = PAGE_KERNEL;
#if defined(NVCPU_X86_64)
#if defined(PAGE_KERNEL_NOENC)
if (unencrypted)
{
prot = cached ? nv_adjust_pgprot(PAGE_KERNEL_NOENC, 0) :
nv_adjust_pgprot(NV_PAGE_KERNEL_NOCACHE_NOENC, 0);
}
else
#endif
{
prot = cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE;
}
#elif defined(NVCPU_AARCH64)
prot = cached ? PAGE_KERNEL : NV_PGPROT_UNCACHED(PAGE_KERNEL);
#endif
/* All memory cached in PPC64LE; can't honor 'cached' input. */
ptr = vmap(pages, page_count, VM_MAP, prot);
if (ptr)
{
NV_MEMDBG_ADD(ptr, page_count * PAGE_SIZE);
}
return (NvUPtr)ptr;
}
static inline void nv_vunmap(NvUPtr vaddr, NvU32 page_count)
{
vunmap((void *)vaddr);
NV_MEMDBG_REMOVE((void *)vaddr, page_count * PAGE_SIZE);
}
#if defined(NV_GET_NUM_PHYSPAGES_PRESENT)
#define NV_NUM_PHYSPAGES get_num_physpages()
#else
@@ -705,6 +640,47 @@ static inline void nv_vunmap(NvUPtr vaddr, NvU32 page_count)
#define NV_NUM_CPUS() num_possible_cpus()
#define NV_HAVE_MEMORY_ENCRYPT_DECRYPT 0
#if defined(NVCPU_X86_64) && \
NV_IS_EXPORT_SYMBOL_GPL_set_memory_encrypted && \
NV_IS_EXPORT_SYMBOL_GPL_set_memory_decrypted
#undef NV_HAVE_MEMORY_ENCRYPT_DECRYPT
#define NV_HAVE_MEMORY_ENCRYPT_DECRYPT 1
#endif
static inline void nv_set_memory_decrypted_zeroed(NvBool unencrypted,
unsigned long virt_addr,
int num_native_pages,
size_t size)
{
if (virt_addr == 0)
return;
#if NV_HAVE_MEMORY_ENCRYPT_DECRYPT
if (unencrypted)
{
set_memory_decrypted(virt_addr, num_native_pages);
memset((void *)virt_addr, 0, size);
}
#endif
}
static inline void nv_set_memory_encrypted(NvBool unencrypted,
unsigned long virt_addr,
int num_native_pages)
{
if (virt_addr == 0)
return;
#if NV_HAVE_MEMORY_ENCRYPT_DECRYPT
if (unencrypted)
{
set_memory_encrypted(virt_addr, num_native_pages);
}
#endif
}
static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
{
#if defined(NV_PHYS_TO_DMA_PRESENT)
@@ -885,94 +861,42 @@ typedef void irqreturn_t;
(((addr) >> NV_RM_PAGE_SHIFT) == \
(((addr) + (size) - 1) >> NV_RM_PAGE_SHIFT)))
/*
* The kernel may have a workaround for this, by providing a method to isolate
* a single 4K page in a given mapping.
*/
#if (PAGE_SIZE > NV_RM_PAGE_SIZE) && defined(NVCPU_PPC64LE) && defined(NV_PAGE_4K_PFN)
#define NV_4K_PAGE_ISOLATION_PRESENT
#define NV_4K_PAGE_ISOLATION_MMAP_ADDR(addr) \
((NvP64)((void*)(((addr) >> NV_RM_PAGE_SHIFT) << PAGE_SHIFT)))
#define NV_4K_PAGE_ISOLATION_MMAP_LEN(size) PAGE_SIZE
#define NV_4K_PAGE_ISOLATION_ACCESS_START(addr) \
((NvP64)((void*)((addr) & ~NV_RM_PAGE_MASK)))
#define NV_4K_PAGE_ISOLATION_ACCESS_LEN(addr, size) \
((((addr) & NV_RM_PAGE_MASK) + size + NV_RM_PAGE_MASK) & \
~NV_RM_PAGE_MASK)
#define NV_PROT_4K_PAGE_ISOLATION NV_PAGE_4K_PFN
#endif
static inline int nv_remap_page_range(struct vm_area_struct *vma,
unsigned long virt_addr, NvU64 phys_addr, NvU64 size, pgprot_t prot)
{
int ret = -1;
#if defined(NV_4K_PAGE_ISOLATION_PRESENT) && defined(NV_PROT_4K_PAGE_ISOLATION)
if ((size == PAGE_SIZE) &&
((pgprot_val(prot) & NV_PROT_4K_PAGE_ISOLATION) != 0))
{
/*
* remap_4k_pfn() hardcodes the length to a single OS page, and checks
* whether applying the page isolation workaround will cause PTE
* corruption (in which case it will fail, and this is an unsupported
* configuration).
*/
#if defined(NV_HASH__REMAP_4K_PFN_PRESENT)
ret = hash__remap_4k_pfn(vma, virt_addr, (phys_addr >> PAGE_SHIFT), prot);
#else
ret = remap_4k_pfn(vma, virt_addr, (phys_addr >> PAGE_SHIFT), prot);
#endif
}
else
#endif
{
ret = remap_pfn_range(vma, virt_addr, (phys_addr >> PAGE_SHIFT), size,
return remap_pfn_range(vma, virt_addr, (phys_addr >> PAGE_SHIFT), size,
prot);
}
return ret;
}
static inline int nv_io_remap_page_range(struct vm_area_struct *vma,
NvU64 phys_addr, NvU64 size, NvU32 extra_prot, NvU64 start)
NvU64 phys_addr, NvU64 size, NvU64 start)
{
int ret = -1;
#if !defined(NV_XEN_SUPPORT_FULLY_VIRTUALIZED_KERNEL)
ret = nv_remap_page_range(vma, start, phys_addr, size,
nv_adjust_pgprot(vma->vm_page_prot, extra_prot));
nv_adjust_pgprot(vma->vm_page_prot));
#else
ret = io_remap_pfn_range(vma, start, (phys_addr >> PAGE_SHIFT),
size, nv_adjust_pgprot(vma->vm_page_prot, extra_prot));
size, nv_adjust_pgprot(vma->vm_page_prot));
#endif
return ret;
}
static inline vm_fault_t nv_insert_pfn(struct vm_area_struct *vma,
NvU64 virt_addr, NvU64 pfn, NvU32 extra_prot)
NvU64 virt_addr, NvU64 pfn)
{
/*
* vm_insert_pfn{,_prot} replaced with vmf_insert_pfn{,_prot} in Linux 4.20
*/
#if defined(NV_VMF_INSERT_PFN_PROT_PRESENT)
return vmf_insert_pfn_prot(vma, virt_addr, pfn,
__pgprot(pgprot_val(vma->vm_page_prot) | extra_prot));
__pgprot(pgprot_val(vma->vm_page_prot)));
#else
int ret = -EINVAL;
/*
* Only PPC64LE (NV_4K_PAGE_ISOLATION_PRESENT) requires extra_prot to be
* used when remapping.
*
* vm_insert_pfn_prot() was added in Linux 4.4, whereas POWER9 support
* was added in Linux 4.8.
*
* Rather than tampering with the vma to make use of extra_prot with
* vm_insert_pfn() on older kernels, for now, just fail in this case, as
* it's not expected to be used currently.
*/
#if defined(NV_VM_INSERT_PFN_PROT_PRESENT)
ret = vm_insert_pfn_prot(vma, virt_addr, pfn,
__pgprot(pgprot_val(vma->vm_page_prot) | extra_prot));
#elif !defined(NV_4K_PAGE_ISOLATION_PRESENT)
__pgprot(pgprot_val(vma->vm_page_prot)));
#else
ret = vm_insert_pfn(vma, virt_addr, pfn);
#endif
switch (ret)
@@ -1158,7 +1082,6 @@ static inline void nv_kmem_cache_free_stack(nvidia_stack_t *stack)
typedef struct nvidia_pte_s {
NvU64 phys_addr;
unsigned long virt_addr;
NvU64 dma_addr;
} nvidia_pte_t;
#if defined(CONFIG_DMA_SHARED_BUFFER)
@@ -1199,6 +1122,7 @@ typedef struct nv_alloc_s {
NvS32 node_id; /* Node id for memory allocation when node is set in flags */
void *import_priv;
struct sg_table *import_sgt;
dma_addr_t dma_handle; /* dma handle used by dma_alloc_coherent(), dma_free_coherent() */
} nv_alloc_t;
/**
@@ -1424,6 +1348,23 @@ struct os_wait_queue {
struct completion q;
};
/*!
* @brief Mapping between clock names and clock handles.
*
* TEGRA_DISP_WHICH_CLK_MAX: maximum number of clocks
* defined in below enum.
*
* arch/nvalloc/unix/include/nv.h
* enum TEGRASOC_WHICH_CLK_MAX;
*
*/
typedef struct nvsoc_clks_s {
struct {
struct clk *handles;
const char *clkName;
} clk[TEGRASOC_WHICH_CLK_MAX];
} nvsoc_clks_t;
/*
* To report error in msi/msix when unhandled count reaches a threshold
*/
@@ -1583,6 +1524,8 @@ typedef struct nv_linux_state_s {
nv_acpi_t* nv_acpi_object;
#endif
nvsoc_clks_t soc_clk_handles;
/* Lock serializing ISRs for different SOC vectors */
nv_spinlock_t soc_isr_lock;
void *soc_bh_mutex;
@@ -1782,12 +1725,10 @@ static inline struct kmem_cache *nv_kmem_cache_create(const char *name, unsigned
*/
static inline NV_STATUS nv_check_gpu_state(nv_state_t *nv)
{
#if !defined(NVCPU_PPC64LE)
if (NV_IS_DEVICE_IN_SURPRISE_REMOVAL(nv))
{
return NV_ERR_GPU_IS_LOST;
}
#endif
return NV_OK;
}