NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-05-13 01:06:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

595.44.02

Browse Source
This commit is contained in:
Andy Ritger
2026-03-09 13:13:35 -07:00
parent 2ccbad25e1
commit 2c7bfb4706
1210 changed files with 436452 additions and 437093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -1,7 +1,7 @@
# NVIDIA Linux Open GPU Kernel Module Source
This is the source release of the NVIDIA Linux open GPU kernel modules,
version 590.48.01.
version 595.44.02.
## How to Build
@@ -17,7 +17,7 @@ as root:
Note that the kernel modules built here must be used with GSP
firmware and user-space NVIDIA GPU driver components from a corresponding
590.48.01 driver release. This can be achieved by installing
595.44.02 driver release. This can be achieved by installing
the NVIDIA GPU driver from the .run file using the `--no-kernel-modules`
option. E.g.,
@@ -185,7 +185,7 @@ table below).
For details on feature support and limitations, see the NVIDIA GPU driver
end user README here:
https://us.download.nvidia.com/XFree86/Linux-x86_64/590.48.01/README/kernel_open.html
https://us.download.nvidia.com/XFree86/Linux-x86_64/595.44.02/README/kernel_open.html
For vGPU support, please refer to the README.vgpu packaged in the vGPU Host
Package for more details.
@@ -845,6 +845,7 @@ Subsystem Device ID.
| NVIDIA RTX A400 | 25B2 1028 1879 |
| NVIDIA RTX A400 | 25B2 103C 1879 |
| NVIDIA RTX A400 | 25B2 103C 8D95 |
| NVIDIA RTX A400 | 25B2 103C 8F5B |
| NVIDIA RTX A400 | 25B2 10DE 1879 |
| NVIDIA RTX A400 | 25B2 17AA 1879 |
| NVIDIA A16 | 25B6 10DE 14A9 |
@@ -971,6 +972,7 @@ Subsystem Device ID.
| NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition | 2BB4 10DE 204C |
| NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition | 2BB4 17AA 204C |
| NVIDIA RTX PRO 6000 Blackwell Server Edition | 2BB5 10DE 204E |
| NVIDIA RTX PRO 6000 Blackwell Server Edition | 2BB5 10DE 220B |
| NVIDIA RTX 6000D | 2BB9 10DE 2091 |
| NVIDIA GeForce RTX 5080 | 2C02 |
| NVIDIA GeForce RTX 5070 Ti | 2C05 |

6
kernel-open/Kbuild
View File

@@ -79,11 +79,11 @@ ccflags-y += -I$(src)/common/inc
ccflags-y += -I$(src)
ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
ccflags-y += -DNV_VERSION_STRING=\"590.48.01\"
ccflags-y += -DNV_VERSION_STRING=\"595.44.02\"
# Include and link Tegra out-of-tree modules.
ifneq ($(wildcard /usr/src/nvidia/nvidia-oot),)
SYSSRCNVOOT ?= /usr/src/nvidia/nvidia-oot
ifneq ($(wildcard /usr/src/nvidia/nvidia-public),)
SYSSRCNVOOT ?= /usr/src/nvidia/nvidia-public
endif
ifneq ($(SYSSRCHOST1X),)

5
kernel-open/common/inc/cpuopsys.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2018-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2018-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -28,7 +28,7 @@
*
* cpuopsys.h
*
* Copyright (c) 2001, Nvidia Corporation. All rights reserved.
* Copyright (c) 2001-2025, Nvidia Corporation. All rights reserved.
*/
/*!
@@ -134,7 +134,6 @@
#endif
/* ***** Processor type variations */
/* Note: The prefix NV_CPU_* is taken by Nvcm.h */
#if ((defined(_M_IX86) || defined(__i386__) || defined(__i386)) && !defined(NVCPU_X86)) /* XXX until removed from Makefiles */
/* _M_IX86 for windows, __i386__ for Linux (or any x86 using gcc) */

7
kernel-open/common/inc/nv-firmware-registry.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -59,8 +59,9 @@
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_ENABLED 0x00000001
#define NV_REG_ENABLE_GPU_FIRMWARE_MODE_DEFAULT 0x00000002
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_MASK 0x000000F0
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK 0x00000010
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_MASK 0x000000F0
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK 0x00000010
#define NV_REG_ENABLE_GPU_FIRMWARE_POLICY_DEFAULT_ON_WS_SERVER 0x00000020
#define NV_REG_ENABLE_GPU_FIRMWARE_DEFAULT_VALUE 0x00000012

1
kernel-open/common/inc/nv-gpu-info.h
View File

@@ -33,6 +33,7 @@ typedef struct {
} pci_info;
NvBool needs_numa_setup;
NvBool is_soc_disp;
/*
* opaque OS-specific pointer; on Linux, this is a pointer to the

6
kernel-open/common/inc/nv-ioctl-numbers.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -26,6 +26,10 @@
#define NV_IOCTL_NUMBERS_H
/* NOTE: using an ioctl() number > 55 will overflow! */
/*
* NOTE: Any new ioctls added here should also be added to nv_validate_ioctls() in nv.c or
* rm_validate_ioctls() in osapi.c
*/
#define NV_IOCTL_MAGIC 'F'
#define NV_IOCTL_BASE 200
#define NV_ESC_CARD_INFO (NV_IOCTL_BASE + 0)

43
kernel-open/common/inc/nv-linux.h
View File

@@ -129,6 +129,7 @@
#include <asm/pgtable.h> /* pte bit definitions */
#include <asm/bitops.h> /* __set_bit() */
#include <linux/time.h> /* FD_SET() */
#include <linux/memremap.h>
/*
* Use current->cred->euid, instead of calling current_euid().
@@ -503,7 +504,7 @@ static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot)
#define NV_HAVE_MEMORY_ENCRYPT_DECRYPT 0
#if defined(NVCPU_X86_64) && \
#if (defined(NVCPU_X86_64) || defined(NVCPU_AARCH64)) && \
NV_IS_EXPORT_SYMBOL_GPL_set_memory_encrypted && \
NV_IS_EXPORT_SYMBOL_GPL_set_memory_decrypted
#undef NV_HAVE_MEMORY_ENCRYPT_DECRYPT
@@ -553,7 +554,7 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
#endif
}
#define NV_GET_PAGE_STRUCT(phys_page) virt_to_page(__va(phys_page))
#define NV_GET_PAGE_STRUCT(phys_page) pfn_to_page(phys_page >> PAGE_SHIFT)
#define NV_VMA_PGOFF(vma) ((vma)->vm_pgoff)
#define NV_VMA_SIZE(vma) ((vma)->vm_end - (vma)->vm_start)
#define NV_VMA_OFFSET(vma) (((NvU64)(vma)->vm_pgoff) << PAGE_SHIFT)
@@ -562,6 +563,12 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
#define NV_DEVICE_MINOR_NUMBER(x) minor((x)->i_rdev)
#if defined(NV_GET_DEV_PAGEMAP_HAS_PGMAP_ARG)
#define NV_GET_DEV_PAGEMAP(pfn) get_dev_pagemap(pfn, NULL)
#else
#define NV_GET_DEV_PAGEMAP get_dev_pagemap
#endif
#define NV_PCI_DISABLE_DEVICE(pci_dev) \
{ \
NvU16 __cmd[2]; \
@@ -671,18 +678,6 @@ static inline dma_addr_t nv_phys_to_dma(struct device *dev, NvU64 pa)
#define PCI_CAP_ID_EXP 0x10
#endif
/*
* If the host OS has page sizes larger than 4KB, we may have a security
* problem. Registers are typically grouped in 4KB pages, but if there are
* larger pages, then the smallest userspace mapping possible (e.g., a page)
* may give more access than intended to the user.
*/
#define NV_4K_PAGE_ISOLATION_REQUIRED(addr, size) \
((PAGE_SIZE > NV_RM_PAGE_SIZE) && \
((size) <= NV_RM_PAGE_SIZE) && \
(((addr) >> NV_RM_PAGE_SHIFT) == \
(((addr) + (size) - 1) >> NV_RM_PAGE_SHIFT)))
static inline int nv_remap_page_range(struct vm_area_struct *vma,
unsigned long virt_addr, NvU64 phys_addr, NvU64 size, pgprot_t prot)
{
@@ -1212,6 +1207,10 @@ typedef struct coherent_link_info_s {
* of virutalized OS environment it is Intermediate Physical Address(IPA) */
NvU64 gpu_mem_pa;
/* Size of the GPU memory mappable through coherent link. It is possible
that less than whole FB is mapped to CPU. */
NvU64 gpu_mem_size;
/* Physical address of the reserved portion of the GPU memory, applicable
* only in Grace Hopper self hosted passthrough virtualizatioan platform. */
NvU64 rsvd_mem_pa;
@@ -1374,6 +1373,13 @@ typedef struct nv_linux_state_s {
/* Lock serializing ISRs for different SOC vectors */
nv_spinlock_t soc_isr_lock;
/*
* Lock serializing access to the soc_isr_info struct across top and
* bottom halves for SOC vectors.
*/
nv_spinlock_t soc_isr_info_lock;
void *soc_bh_mutex;
struct nv_timer snapshot_timer;
@@ -1428,6 +1434,9 @@ typedef struct nv_linux_state_s {
int (*devfreq_enable_boost)(struct device *dev, unsigned int duration);
int (*devfreq_disable_boost)(struct device *dev);
#endif
/* Per-device GPU init on probe setting, initialized from global NVreg_GpuInitOnProbe */
NvBool init_on_probe;
} nv_linux_state_t;
extern nv_linux_state_t *nv_linux_devices;
@@ -1485,9 +1494,11 @@ typedef struct
nv_kthread_q_item_t deferred_close_q_item;
NvU32 *attached_gpus;
size_t num_attached_gpus;
nv_alloc_mapping_context_t mmap_context;
struct address_space mapping;
struct rw_semaphore fileVaLock;
nv_alloc_mapping_list_node_t *file_mapping_list;
nv_kthread_q_item_t open_q_item;
struct completion open_complete;
nv_linux_state_t *deferred_open_nvl;
@@ -1599,6 +1610,8 @@ extern NvU32 NVreg_RegisterPlatformDeviceDriver;
extern NvU32 NVreg_EnableResizableBar;
extern NvU32 NVreg_TegraGpuPgMask;
extern NvU32 NVreg_EnableNonblockingOpen;
extern NvU32 NVreg_UseKernelSuspendNotifiers;
extern NvU32 NVreg_GpuInitOnProbe;
extern NvU32 num_probed_nv_devices;
extern NvU32 num_nv_devices;

4
kernel-open/common/inc/nv-modeset-interface.h
View File

@@ -114,8 +114,8 @@ typedef struct {
* specified GPU. This is equivalent to opening and closing a
* /dev/nvidiaN device file from user-space.
*/
int (*open_gpu)(NvU32 gpu_id, nvidia_modeset_stack_ptr sp);
void (*close_gpu)(NvU32 gpu_id, nvidia_modeset_stack_ptr sp);
int (*open_gpu)(NvU32 gpu_id, nvidia_modeset_stack_ptr sp, NvBool reset_aware);
void (*close_gpu)(NvU32 gpu_id, nvidia_modeset_stack_ptr sp, NvBool reset_aware);
void (*op)(nvidia_modeset_stack_ptr sp, void *ops_cmd);

1
kernel-open/common/inc/nv-pci.h
View File

@@ -38,4 +38,5 @@ int nvidia_dev_get_pci_info(const NvU8 *, struct pci_dev **, NvU64 *, NvU64 *);
nv_linux_state_t * find_pci(NvU32, NvU8, NvU8, NvU8);
NvBool nv_pci_is_valid_topology_for_direct_pci(nv_state_t *, struct pci_dev *);
NvBool nv_pci_has_common_pci_switch(nv_state_t *nv, struct pci_dev *);
void nv_pci_tegra_boost_clocks(struct device *dev);
#endif

12
kernel-open/common/inc/nv-platform.h
View File

@@ -45,4 +45,16 @@ void nv_soc_free_irqs(nv_state_t *nv);
#define NV_SUPPORTS_PLATFORM_DISPLAY_DEVICE (NV_SUPPORTS_PLATFORM_DEVICE && NV_SUPPORTS_DCE_CLIENT_IPC)
#if defined(CONFIG_OF)
NV_STATUS nv_platform_get_screen_info_dt(
NvU64 *pPhysicalAddress,
NvU32 *pFbWidth,
NvU32 *pFbHeight,
NvU32 *pFbDepth,
NvU32 *pFbPitch,
NvU64 *pFbSize
);
#endif // CONFIG_OF
#endif

14
kernel-open/common/inc/nv-proto.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -40,6 +40,8 @@ void nv_procfs_add_warning (const char *, const char *);
int nv_procfs_add_gpu (nv_linux_state_t *);
void nv_procfs_remove_gpu (nv_linux_state_t *);
extern nv_pm_action_depth_t nv_procfs_pm_action_depth;
int nvidia_mmap (struct file *, struct vm_area_struct *);
int nvidia_mmap_helper (nv_state_t *, nv_linux_file_private_t *, nvidia_stack_t *, struct vm_area_struct *, void *);
int nv_encode_caching (pgprot_t *, NvU32, nv_memory_type_t);
@@ -63,8 +65,8 @@ NV_STATUS nv_uvm_resume_P2P (const NvU8 *uuid);
/* Move these to nv.h once implemented by other UNIX platforms */
NvBool nvidia_get_gpuid_list (NvU32 *gpu_ids, NvU32 *gpu_count);
int nvidia_dev_get (NvU32, nvidia_stack_t *);
void nvidia_dev_put (NvU32, nvidia_stack_t *);
int nvidia_dev_get (NvU32, nvidia_stack_t *, NvBool reset_aware);
void nvidia_dev_put (NvU32, nvidia_stack_t *, NvBool reset_aware);
int nvidia_dev_get_uuid (const NvU8 *, nvidia_stack_t *);
void nvidia_dev_put_uuid (const NvU8 *, nvidia_stack_t *);
int nvidia_dev_block_gc6 (const NvU8 *, nvidia_stack_t *);
@@ -87,11 +89,15 @@ void nv_shutdown_adapter(nvidia_stack_t *, nv_state_t *, nv_linux_state
void nv_dev_free_stacks(nv_linux_state_t *);
NvBool nv_lock_init_locks(nvidia_stack_t *, nv_state_t *);
void nv_lock_destroy_locks(nvidia_stack_t *, nv_state_t *);
int nv_linux_add_device_locked(nv_linux_state_t *);
void nv_linux_add_device_locked(nv_linux_state_t *);
int nv_linux_assign_minor_locked(nv_linux_state_t *);
void nv_linux_remove_minor_locked(nv_linux_state_t *);
void nv_linux_remove_device_locked(nv_linux_state_t *);
NvBool nv_acpi_power_resource_method_present(struct pci_dev *);
int nv_linux_init_open_q(nv_linux_state_t *);
void nv_linux_stop_open_q(nv_linux_state_t *);
int nv_start_device(nv_state_t *nv, nvidia_stack_t *sp);
void nv_stop_device(nv_state_t *nv, nvidia_stack_t *sp);
#endif /* _NV_PROTO_H_ */

89
kernel-open/common/inc/nv.h
View File

@@ -46,6 +46,12 @@
#include <nvmisc.h>
#include <os/nv_memory_area.h>
#if !defined(NV_KERNEL_INTERFACE_LAYER) && defined(NVRM) && !defined(NV_DRM_FREEBSD_LKPI)
#include <nvport/nvport.h>
#else
#define PORT_ATOMIC
#endif
extern nv_cap_t *nvidia_caps_root;
extern const NvBool nv_is_rm_firmware_supported_os;
@@ -312,6 +318,13 @@ typedef struct
((((NvUPtr)(count)) >> NV_RM_TO_OS_PAGE_SHIFT) + \
((((count) & ((1 << NV_RM_TO_OS_PAGE_SHIFT) - 1)) != 0) ? 1 : 0))
// for dynamic granularity page array feature.
// Notice: dst_granu should be larger than src_granu.
#define NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu) (GET_PAGE_SHIFT(dst_granu) - GET_PAGE_SHIFT(src_granu))
#define NV_SRC_TO_DST_PAGE_COUNT(src_page_count,src_granu, dst_granu) \
((((NvUPtr)(src_page_count)) >> NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu)) + \
((((src_page_count) & ((1 << NV_SRC_TO_DST_PAGE_SHIFT(src_granu, dst_granu)) - 1)) != 0) ? 1 : 0))
#if defined(NVCPU_X86_64)
#define NV_STACK_SIZE (NV_RM_PAGE_SIZE * 3)
#else
@@ -389,6 +402,15 @@ typedef struct nv_alloc_mapping_context_s {
NvU32 caching;
} nv_alloc_mapping_context_t;
struct nv_alloc_mapping_list_node_s;
typedef struct nv_alloc_mapping_list_node_s {
nv_alloc_mapping_context_t context;
MemoryRange fileRange;
struct nv_alloc_mapping_list_node_s *pNext;
struct nv_alloc_mapping_list_node_s *pPrev;
} nv_alloc_mapping_list_node_t;
typedef enum
{
NV_SOC_IRQ_DISPLAY_TYPE = 0x1,
@@ -492,6 +514,7 @@ typedef struct nv_state_t
NvBool supports_tegra_igpu_rg;
NvBool is_tegra_pci_igpu_rg_enabled;
NvU32 tegra_pci_igpu_pg_mask;
NvU32 gpc_fuse_status_offset;
NvBool primary_vga;
@@ -562,9 +585,6 @@ typedef struct nv_state_t
/* Bool to check if dma-buf is supported */
NvBool dma_buf_supported;
/* Check if NVPCF DSM function is implemented under NVPCF or GPU device scope */
NvBool nvpcf_dsm_in_gpu_scope;
/* Bool to check if the device received a shutdown notification */
NvBool is_shutdown;
@@ -578,6 +598,14 @@ typedef struct nv_state_t
*/
NvBool mem_has_struct_page;
/*
* Coherent GPU Memory Mode, one of:
* NV_COHERENT_GPU_MEM_MODE_NONE
* NV_COHERENT_GPU_MEM_MODE_NUMA
* NV_COHERENT_GPU_MEM_MODE_DRIVER
*/
NvBool coherent_gpu_mem_mode;
/* OS detected GPU has ATS capability */
NvBool ats_support;
/*
@@ -596,9 +624,18 @@ typedef struct nv_state_t
NvU32 dispNisoStreamId;
} iommus;
struct {
NvU32 max_dispclk_rate_using_disppllkhz;
NvU32 max_dispclk_rate_using_sppll0clkoutakhz;
NvU32 max_hubclk_rate_using_sppll0clkoutbkhz;
} clocks;
/* Console is managed by drm drivers or NVKMS */
NvBool client_managed_console;
/* Console is mapped to sysmem instead of GPU PCIe BAR1 or BAR2 */
NvBool sysmem_mapped_console;
/* Struct to cache the gpu info details */
nv_cached_gpu_info_t cached_gpu_info;
@@ -607,9 +644,9 @@ typedef struct nv_state_t
} nv_state_t;
#define NVFP_TYPE_NONE 0x0
#define NVFP_TYPE_REFCOUNTED 0x1
#define NVFP_TYPE_REGISTERED 0x2
#define NVFP_TYPE_NONE ((NvU32)0x0)
#define NVFP_TYPE_REFCOUNTED ((NvU32)0x1)
#define NVFP_TYPE_REGISTERED ((NvU32)0x2)
struct nv_file_private_t
{
@@ -619,9 +656,9 @@ struct nv_file_private_t
NvU32 gpuInstanceId;
NvU8 metadata[64];
nv_file_private_t *ctl_nvfp;
nv_file_private_t * PORT_ATOMIC ctl_nvfp;
void *ctl_nvfp_priv;
NvU32 register_or_refcount;
PORT_ATOMIC NvU32 register_or_refcount;
//
// True if a client or an event was ever allocated on this fd.
@@ -671,7 +708,7 @@ typedef struct UvmGpuAccessBitsBufferAlloc_tag *nvgpuAccessBitBufferAlloc_t
* flags
*/
#define NV_FLAG_OPEN 0x0001
#define NV_FLAG_INITIALIZED 0x0001
#define NV_FLAG_EXCLUDE 0x0002
#define NV_FLAG_CONTROL 0x0004
#define NV_FLAG_PCI_P2P_UNSUPPORTED_CHIPSET 0x0008
@@ -685,7 +722,7 @@ typedef struct UvmGpuAccessBitsBufferAlloc_tag *nvgpuAccessBitBufferAlloc_t
#define NV_FLAG_TRIGGER_FLR 0x0400
#define NV_FLAG_PERSISTENT_SW_STATE 0x0800
#define NV_FLAG_IN_RECOVERY 0x1000
#define NV_FLAG_PCI_REMOVE_IN_PROGRESS 0x2000
// Unused 0x2000
#define NV_FLAG_UNBIND_LOCK 0x4000
/* To be set when GPU is not present on the bus, to help device teardown */
#define NV_FLAG_IN_SURPRISE_REMOVAL 0x8000
@@ -782,6 +819,13 @@ typedef enum
#define NV_EVAL_ACPI_METHOD_NVIF 0x01
#define NV_EVAL_ACPI_METHOD_WMMX 0x02
/*
* Coherent GPU Memory Mode
*/
#define NV_COHERENT_GPU_MEM_MODE_NONE 0
#define NV_COHERENT_GPU_MEM_MODE_NUMA 1
#define NV_COHERENT_GPU_MEM_MODE_DRIVER 2
typedef enum {
NV_I2C_CMD_READ = 1,
NV_I2C_CMD_WRITE,
@@ -795,7 +839,7 @@ typedef enum {
NV_I2C_CMD_BLOCK_WRITE
} nv_i2c_cmd_t;
// Flags needed by OSAllocPagesNode
// Flags needed by osAllocPagesNode / os_alloc_pages_node
#define NV_ALLOC_PAGES_NODE_NONE 0x0
#define NV_ALLOC_PAGES_NODE_SKIP_RECLAIM 0x1
@@ -930,13 +974,14 @@ NV_STATUS NV_API_CALL nv_dma_map_alloc (nv_dma_device_t *, NvU64, NvU6
NV_STATUS NV_API_CALL nv_dma_unmap_alloc (nv_dma_device_t *, NvU64, NvU64 *, void **);
NV_STATUS NV_API_CALL nv_dma_map_peer (nv_dma_device_t *, nv_dma_device_t *, NvU8, NvU64, NvU64 *);
NV_STATUS NV_API_CALL nv_dma_map_non_pci_peer (nv_dma_device_t *, NvU64, NvU64 *);
void NV_API_CALL nv_dma_unmap_peer (nv_dma_device_t *, NvU64, NvU64);
NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU64 *);
void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64);
void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *);
void* NV_API_CALL nv_dma_get_dev_pagemap (NvU64);
void NV_API_CALL nv_dma_put_dev_pagemap (void *);
NvBool NV_API_CALL nv_grdma_pci_topology_supported(nv_state_t *, nv_dma_device_t *);
NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *);
@@ -974,7 +1019,7 @@ void NV_API_CALL nv_put_firmware(const void *);
nv_file_private_t* NV_API_CALL nv_get_file_private(NvS32, NvBool, void **);
void NV_API_CALL nv_put_file_private(void *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_device_memory_config(nv_state_t *, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_get_egm_info(nv_state_t *, NvU64 *, NvU64 *, NvS32 *);
void NV_API_CALL nv_p2p_free_platform_data(void *data);
@@ -985,6 +1030,11 @@ void NV_API_CALL nv_release_mmap_lock (nv_state_t *);
NvBool NV_API_CALL nv_get_all_mappings_revoked_locked (nv_state_t *);
void NV_API_CALL nv_set_safe_to_mmap_locked (nv_state_t *, NvBool);
#if !defined(NV_VMWARE)
nv_alloc_mapping_list_node_t** NV_API_CALL nv_acquire_file_va(nv_file_private_t *, NvBool);
void NV_API_CALL nv_release_file_va(nv_file_private_t *, NvBool);
#endif
NV_STATUS NV_API_CALL nv_indicate_idle (nv_state_t *);
NV_STATUS NV_API_CALL nv_indicate_not_idle (nv_state_t *);
void NV_API_CALL nv_idle_holdoff (nv_state_t *);
@@ -1004,6 +1054,7 @@ NvBool NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *);
void NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end);
void NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *);
void NV_API_CALL nv_set_gpu_pg_mask(nv_state_t *);
void NV_API_CALL nv_trigger_gpu_flr(nv_state_t *);
struct dma_buf;
typedef struct nv_dma_buf nv_dma_buf_t;
@@ -1045,6 +1096,7 @@ NV_STATUS NV_API_CALL nv_i2c_transfer(nv_state_t *, NvU32, NvU8, nv_i2c_msg_t *,
void NV_API_CALL nv_i2c_unregister_clients(nv_state_t *);
NV_STATUS NV_API_CALL nv_i2c_bus_status(nv_state_t *, NvU32, NvS32 *, NvS32 *);
NV_STATUS NV_API_CALL nv_imp_get_import_data (TEGRA_IMP_IMPORT_DATA *);
NV_STATUS NV_API_CALL nv_imp_get_uefi_data (nv_state_t *nv, NvU32 *iso_bw_kbps, NvU32 *floor_bw_kbps);
NV_STATUS NV_API_CALL nv_imp_enable_disable_rfl (nv_state_t *nv, NvBool bEnable);
NV_STATUS NV_API_CALL nv_imp_icc_set_bw (nv_state_t *nv, NvU32 avg_bw_kbps, NvU32 floor_bw_kbps);
NV_STATUS NV_API_CALL nv_get_num_dpaux_instances(nv_state_t *nv, NvU32 *num_instances);
@@ -1119,6 +1171,7 @@ NV_STATUS NV_API_CALL rm_release_gpu_lock (nvidia_stack_t *, nv_state_t *
NV_STATUS NV_API_CALL rm_acquire_all_gpus_lock (nvidia_stack_t *);
NV_STATUS NV_API_CALL rm_release_all_gpus_lock (nvidia_stack_t *);
NV_STATUS NV_API_CALL rm_ioctl (nvidia_stack_t *, nv_state_t *, nv_file_private_t *, NvU32, void *, NvU32);
NV_STATUS NV_API_CALL rm_validate_ioctls (NvU32, NvU32);
NvBool NV_API_CALL rm_isr (nvidia_stack_t *, nv_state_t *, NvU32 *);
void NV_API_CALL rm_isr_bh (nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_isr_bh_unlocked (nvidia_stack_t *, nv_state_t *);
@@ -1197,7 +1250,6 @@ void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd);
NvBool NV_API_CALL rm_get_device_remove_flag(nvidia_stack_t *sp, NvU32 gpu_id);
NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *);
NV_STATUS NV_API_CALL rm_gpu_handle_mmu_faults(nvidia_stack_t *, nv_state_t *, NvU32 *);
NvBool NV_API_CALL rm_gpu_need_4k_page_isolation(nv_state_t *);
NvBool NV_API_CALL rm_is_chipset_io_coherent(nv_stack_t *);
NvBool NV_API_CALL rm_init_event_locks(nvidia_stack_t *, nv_state_t *);
void NV_API_CALL rm_destroy_event_locks(nvidia_stack_t *, nv_state_t *);
@@ -1241,7 +1293,6 @@ NV_STATUS NV_API_CALL nv_vgpu_process_vf_info(nvidia_stack_t *, nv_state_t *,
NV_STATUS NV_API_CALL nv_gpu_bind_event(nvidia_stack_t *, NvU32, NvBool *);
NV_STATUS NV_API_CALL nv_gpu_unbind_event(nvidia_stack_t *, NvU32, NvBool *);
NV_STATUS NV_API_CALL nv_check_usermap_access_params(nv_state_t*, const nv_usermap_access_params_t*);
nv_soc_irq_type_t NV_API_CALL nv_get_current_irq_type(nv_state_t*);
void NV_API_CALL nv_flush_coherent_cpu_cache_range(nv_state_t *nv, NvU64 cpu_virtual, NvU64 size);
@@ -1287,6 +1338,14 @@ static inline NvU64 nv_rdtsc(void)
#endif
static inline NvBool nv_dev_needs_vidmem_preservation(const nv_state_t *nv)
{
/*
* Tegra iGPUs and SoC display devices don't need vidmem preservation.
*/
return !nv->is_tegra_pci_igpu && !NV_IS_SOC_DISPLAY_DEVICE(nv);
}
#endif /* NVRM */
static inline int nv_count_bits(NvU64 word)

29
kernel-open/common/inc/nvkms-kapi.h
View File

@@ -209,6 +209,7 @@ struct NvKmsKapiConnectorInfo {
NvU32 numIncompatibleConnectors;
NvKmsKapiConnector incompatibleConnectorHandles[NVKMS_KAPI_MAX_CONNECTORS];
NvBool dynamicDpyIdListValid;
NVDpyIdList dynamicDpyIdList;
};
@@ -518,6 +519,10 @@ struct NvKmsKapiDynamicDisplayParams {
NvU8 buffer[NVKMS_KAPI_EDID_BUFFER_SIZE];
} edid;
/* [OUT] Max resolution allowed for modelist */
NvU32 maxWidthInPixels;
NvU32 maxHeightInPixels;
/* [IN] Set true to override EDID */
NvBool overrideEdid;
@@ -1028,6 +1033,30 @@ struct NvKmsKapiFunctionsTable {
const struct NvKmsKapiMemory *memory
);
/*!
* Increment the GC6 blocker reference count to prevent the GPU from
* entering GC6 power state.
*
* \param [in] device A device allocated using allocateDevice().
*
* \return NV_TRUE on success, NV_FALSE on failure.
*/
NvBool (*gc6BlockerRefCntInc)(
const struct NvKmsKapiDevice *device
);
/*!
* Decrement the GC6 blocker reference count to allow the GPU to
* enter GC6 power state.
*
* \param [in] device A device allocated using allocateDevice().
*
* \return NV_TRUE on success, NV_FALSE on failure.
*/
NvBool (*gc6BlockerRefCntDec)(
const struct NvKmsKapiDevice *device
);
/*!
* Create a formatted surface from an NvKmsKapiMemory object.
*

141
kernel-open/common/inc/nvmisc.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -27,6 +27,8 @@
#ifndef __NV_MISC_H
#define __NV_MISC_H
#if !defined(NVRISCV_LIBFSP_BUILD) || !NVRISCV_LIBFSP_BUILD
#ifdef __cplusplus
extern "C" {
#endif //__cplusplus
@@ -991,10 +993,145 @@ static NV_FORCEINLINE void *NV_NVUPTR_TO_PTR(NvUPtr address)
#define BIT64(b) ((NvU64)1U<<(b))
#endif
#endif
//! 1 if @p x needs more than @p bits to represent
#define NV_BITFIELD_SIZE_ADD_ONE(x, bits) \
((!!(((NvU64)(x)) >> (bits))) ? 1 : 0)
/*!
* Minimum bits required to represent a value between 1 and @p x inclusive.
*
* @param[in] x Highest unsigned value to represent: 1 <= x <= NV_U64_MAX
*
* Example
* @code
* enum Foo {
* E_A,
* E_B,
* E_C,
* E_D,
* E_LAST = E_D
* };
*
* struct Bar {
* NvU32 fooValue : NV_BITFIELD_SIZE_64(E_LAST);
* // note, it is the highest value of the enum that needs to be
* // represented.
* // If you define a _COUNT as a convenient value for declaring arrays
* // indexed by the enum, remember to subtract 1.
* }
* @endcode
*
* @note This function assumes that the enums are defined from 0..x (inclusive,
* not necessarily contiguous) It is technically possible to have negative enum
* values, so to be safe, the underlying type should always be unsigned.
*
* If @p x is larger than what can fit in the underlying type, a
* 'width of <bitfield> exceeds its type'
* compiler warning is expected.
*
* @return -1 if @p x <= 0. This causes a compile time error if used as
* a bitfield size.
*/
#define NV_BITFIELD_SIZE_64(x) \
(((x) <= 0) ? -1 : \
(NV_BITFIELD_SIZE_ADD_ONE((x), 0) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 1) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 2) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 3) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 4) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 5) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 6) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 7) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 8) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 9) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 10) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 11) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 12) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 13) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 14) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 15) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 16) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 17) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 18) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 19) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 20) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 21) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 22) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 23) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 24) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 25) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 26) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 27) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 28) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 29) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 30) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 31) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 32) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 33) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 34) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 35) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 36) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 37) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 38) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 39) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 40) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 41) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 42) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 43) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 44) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 45) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 46) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 47) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 48) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 49) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 50) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 51) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 52) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 53) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 54) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 55) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 56) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 57) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 58) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 59) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 60) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 61) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 62) + \
NV_BITFIELD_SIZE_ADD_ONE((x), 63)))
//
// Bug 4851259: Newly added functions must be hidden from certain HS-signed
// ucode compilers to avoid signature mismatch.
//
#ifndef NVDEC_1_0
/*!
* Find the Greatest Common Denominator of two NvU64s
*
* @param[in] a first number
* @param[in] b second number
*
* @return GCD of a and b *
*/
static NV_FORCEINLINE NvU64 nvFindGcdU64(NvU64 a, NvU64 b)
{
while (1)
{
NvU64 temp;
if (a == 0)
return b;
temp = a;
a = b % a;
b = temp;
}
}
#endif // NVDEC_1_0
#ifdef __cplusplus
}
#endif //__cplusplus
#else
#include <misc/nvmisc_drf.h>
#include <misc/bitops.h>
#endif // !defined(NVRISCV_LIBFSP_BUILD) || !NVRISCV_LIBFSP_BUILD
#endif // __NV_MISC_H

5
kernel-open/common/inc/nvstatuscodes.h
View File

@@ -132,7 +132,7 @@ NV_STATUS_CODE(NV_ERR_TIMEOUT_RETRY, 0x00000066, "Call timed o
NV_STATUS_CODE(NV_ERR_TOO_MANY_PRIMARIES, 0x00000067, "Too many primaries")
NV_STATUS_CODE(NV_ERR_UVM_ADDRESS_IN_USE, 0x00000068, "Unified virtual memory requested address already in use")
NV_STATUS_CODE(NV_ERR_MAX_SESSION_LIMIT_REACHED, 0x00000069, "Maximum number of sessions reached")
NV_STATUS_CODE(NV_ERR_LIB_RM_VERSION_MISMATCH, 0x0000006A, "Library version doesn't match driver version") //Contained within the RMAPI library
NV_STATUS_CODE(NV_ERR_LIB_RM_VERSION_MISMATCH, 0x0000006A, "Library version doesn't match driver version") // Contained within the RMAPI library
NV_STATUS_CODE(NV_ERR_PRIV_SEC_VIOLATION, 0x0000006B, "Priv security violation")
NV_STATUS_CODE(NV_ERR_GPU_IN_DEBUG_MODE, 0x0000006C, "GPU currently in debug mode")
NV_STATUS_CODE(NV_ERR_FEATURE_NOT_ENABLED, 0x0000006D, "Requested Feature functionality is not enabled")
@@ -165,7 +165,8 @@ NV_STATUS_CODE(NV_ERR_FABRIC_STATE_OUT_OF_SYNC, 0x00000087, "NVLink fabri
NV_STATUS_CODE(NV_ERR_BUFFER_FULL, 0x00000088, "Buffer is full")
NV_STATUS_CODE(NV_ERR_BUFFER_EMPTY, 0x00000089, "Buffer is empty")
NV_STATUS_CODE(NV_ERR_MC_FLA_OFFSET_TABLE_FULL, 0x0000008A, "Multicast FLA offset table has no available slots")
NV_STATUS_CODE(NV_ERR_DMA_XFER_FAILED, 0x0000008B, "DMA transfer failed")
NV_STATUS_CODE(NV_ERR_OPERATION_ABORTED, 0x0000008B, "Operation has aborted")
NV_STATUS_CODE(NV_ERR_DMA_XFER_FAILED, 0x0000008C, "DMA transfer failed")
// Warnings:
NV_STATUS_CODE(NV_WARN_HOT_SWITCH, 0x00010001, "WARNING Hot switch")

21
kernel-open/common/inc/nvtypes.h
View File

@@ -664,6 +664,27 @@ typedef struct
#endif // defined(_MSC_VER)
/***************************************************************************\
|* *|
|* Definitions of binary/bytes representations of floating point values *|
|* *|
|* Intended for use when passing values across kernel contexts in which *|
|* floating point use is forbidden *|
|* *|
\***************************************************************************/
/*!
* A struct containing the 32-bit representation of an (IEEE-754 binary32)
* floating point value, i.e., the raw underlying bytes of an @ref NvF32
*
* This struct must be used in places where a value may be passed across a
* kernel boundary, as use of floating point within some kernels is, in general,
* banned.
*/
typedef struct
{
NvU32 value;
} NvF32Bytes;
#ifdef __cplusplus
}
#endif

2
kernel-open/common/inc/os-interface.h
View File

@@ -163,6 +163,7 @@ void NV_API_CALL os_release_rwlock_write (void *);
NvBool NV_API_CALL os_semaphore_may_sleep (void);
NV_STATUS NV_API_CALL os_get_version_info (os_version_info*);
NV_STATUS NV_API_CALL os_get_is_openrm (NvBool *);
NvBool NV_API_CALL os_is_bif_reset_supported (void *);
NvBool NV_API_CALL os_is_isr (void);
NvBool NV_API_CALL os_pat_supported (void);
void NV_API_CALL os_dump_stack (void);
@@ -233,6 +234,7 @@ NV_STATUS NV_API_CALL os_find_ns_pid(void *pid_info, NvU32 *ns_pid);
NvBool NV_API_CALL os_is_init_ns(void);
NV_STATUS NV_API_CALL os_iommu_sva_bind(void *arg, void **handle, NvU32 *pasid);
void NV_API_CALL os_iommu_sva_unbind(void *handle);
NvBool NV_API_CALL os_supports_kernel_suspend_notifiers(void);
extern NvU64 os_page_size;
extern NvU64 os_max_page_size;

161
kernel-open/conftest.sh
View File

@@ -14,7 +14,7 @@ OUTPUT=$4
XEN_PRESENT=1
PREEMPT_RT_PRESENT=0
NVIDIA_OOT_PATH="/usr/src/nvidia/nvidia-oot"
NVIDIA_OOT_PATH="/usr/src/nvidia/nvidia-public"
MODULE_SYMVERS_PATHS="$OUTPUT/Module.symvers"
# Also search in out-of-tree Module.symvers on Tegra
@@ -507,6 +507,7 @@ compile_test() {
#include <asm/pgtable_types.h>
#endif
#include <asm/page.h>
#include <linux/percpu.h>
#include <asm/set_memory.h>
#else
#include <asm/cacheflush.h>
@@ -530,6 +531,7 @@ compile_test() {
#include <asm/pgtable_types.h>
#endif
#include <asm/page.h>
#include <linux/percpu.h>
#include <asm/set_memory.h>
#else
#include <asm/cacheflush.h>
@@ -588,6 +590,7 @@ compile_test() {
#include <asm/pgtable_types.h>
#endif
#include <asm/page.h>
#include <linux/percpu.h>
#include <asm/set_memory.h>
#else
#include <asm/cacheflush.h>
@@ -614,6 +617,7 @@ compile_test() {
#include <asm/pgtable_types.h>
#endif
#include <asm/page.h>
#include <linux/percpu.h>
#include <asm/set_memory.h>
#else
#include <asm/cacheflush.h>
@@ -1082,30 +1086,99 @@ compile_test() {
compile_check_conftest "$CODE" "NV_VFIO_DEVICE_OPS_HAS_DETACH_IOAS" "" "types"
;;
vfio_device_ops_has_get_region_info_caps)
#
# Determine if 'struct vfio_device_ops' has 'get_region_info_caps'
# callback.
#
# Added by commit 775f726a742a ("vfio: Add get_region_info_caps op")
# in v6.19
#
CODE="
#include <linux/pci.h>
#include <linux/vfio.h>
int conftest_vfio_device_ops_has_get_region_info_caps(void) {
return offsetof(struct vfio_device_ops, get_region_info_caps);
}"
compile_check_conftest "$CODE" "NV_VFIO_DEVICE_OPS_HAS_GET_REGION_INFO_CAPS" "" "types"
;;
pfn_address_space)
#
# Determine if 'struct pfn_address_space' structure is present or not.
#
# Added by commit 2ec41967189c ("mm: handle poisoning of pfn without
# struct pages") in v6.19
# Added by commit e6dbcb7c0e7b ("mm: fixup pfnmap memory failure
# handling to use pgoff in v6.19
#
CODE="
#include <linux/memory-failure.h>
void conftest_pfn_address_space() {
struct pfn_address_space pfn_address_space;
pfn_address_space.pfn_to_vma_pgoff = NULL;
register_pfn_address_space(&pfn_address_space);
unregister_pfn_address_space(&pfn_address_space);
}"
compile_check_conftest "$CODE" "NV_PFN_ADDRESS_SPACE_STRUCT_PRESENT" "" "types"
;;
irq_bypass_producer_has_token)
#
# Determine if 'struct irq_bypass_producer' has 'token' field
#
# Added by commit 2b521d86ee80 ("irqbypass: Take ownership of
# producer/consumer token tracking") in v6.17
#
CODE="
#include <linux/irqbypass.h>
int conftest_irq_bypass_producer_has_token(void) {
return offsetof(struct irq_bypass_producer, token);
}"
compile_check_conftest "$CODE" "NV_PFN_ADDRESS_SPACE_STRUCT_PRESENT" "" "types"
compile_check_conftest "$CODE" "NV_IRQ_BYPASS_PRODUCER_HAS_TOKEN" "" "types"
;;
irq_bypass_register_producer_has_eventfd_and_irq_args)
#
# Determine if irq_bypass_register_producer() function has
# additional 'eventfd' and 'irq' arguments.
#
# Added by commits 2b521d86ee80 ("irqbypass: Take ownership of
# producer/consumer token tracking") and 23b54381cee2
# ("irqbypass: Require producers to pass in Linux IRQ number
# during registration") in v6.17
#
CODE="
#include <linux/irqbypass.h>
#include <linux/eventfd.h>
void conftest_irq_bypass_register_producer_has_eventfd_and_irq_args(void) {
struct irq_bypass_producer *prod = NULL;
struct eventfd_ctx *eventfd = NULL;
int irq = 0;
irq_bypass_register_producer(prod, eventfd, irq);
}"
compile_check_conftest "$CODE" "NV_IRQ_BYPASS_REGISTER_PRODUCER_HAS_EVENTFD_AND_IRQ_ARGS" "" "types"
;;
egm_module_helper_api_present)
#
# Determine if egm management api are present or not.
#
# unregister_egm_node() function signature changed by commit
# 9863aeed3a2d ("NVIDIA: SAUCE: vfio/nvgrace-egm: Update EGM
# unregistration API") to use PCI device as its parameter
# https://github.com/NVIDIA/NV-Kernels.git
#
CODE="
#include <linux/pci.h>
#include <linux/nvgrace-egm.h>
void conftest_egm_module_helper_api_present() {
struct pci_dev *pdev;
struct pci_dev *pdev = NULL;
register_egm_node(pdev);
unregister_egm_node(0);
unregister_egm_node(pdev);
}
"
compile_check_conftest "$CODE" "NV_EGM_MODULE_HELPER_API_PRESENT" "" "types"
@@ -1307,6 +1380,51 @@ compile_test() {
compile_check_conftest "$CODE" "NV_GET_DEV_PAGEMAP_HAS_PGMAP_ARG" "" "types"
;;
zone_device_page_init_has_pgmap_and_order_args)
#
# Determine if the zone_device_page_init() has two additional
# arguments
#
# This change was introduced by d245f9b4ab80
# ("mm/zone_device: support large zone device private folios")
#
# It was further amended in 9387a71ec62c
# (mm/zone_device: reinitialize large zone device private folios)
#
# both commits are in linux-next, expected in v6.19.
#
CODE="
#include <linux/memremap.h>
void init_page(void) {
struct page *page;
struct dev_pagemap *pgmap;
zone_device_page_init(page, pgmap, 0);
}"
compile_check_conftest "$CODE" "NV_ZONE_DEVICE_PAGE_INIT_HAS_PGMAP_AND_ORDER_ARGS" "" "types"
;;
dev_pagemap_ops_has_folio_free)
#
# Determine if the zone device now uses a folio_free() as the callback
# function instead of page_free()
#
# This change was introduced by 3a5a06554566
# (mm/zone_device: rename page_free callback to folio_free)
#
# in linux-next, expected in v6.19.
#
CODE="
#include <linux/memremap.h>
void test_folio_free(struct folio *folio) {
}
void set_folio_free_ops(void) {
struct dev_pagemap_ops ops;
ops.folio_free = test_folio_free;
}"
compile_check_conftest "$CODE" "NV_PAGEMAP_OPS_HAS_FOLIO_FREE" "" "types"
;;
drm_sysfs_connector_property_event)
#
# Determine if drm_sysfs_connector_property_event() is present.
@@ -1945,24 +2063,6 @@ compile_test() {
fi
;;
enable_apicv)
#
# Determine if enable_apicv boolean is exported by kernel.
#
# Added by commit fdf513e37a3b ("KVM: x86: Use common
# 'enable_apicv' variable for both APICv and AVIC") in v5.14.
#
CODE="
$CONFTEST_PREAMBLE
#include <asm/kvm_host.h>
bool is_enable_apicv_present() {
return enable_apicv;
}"
compile_check_conftest "$CODE" "NV_ENABLE_APICV_PRESENT" "" "types"
;;
pci_driver_has_driver_managed_dma)
#
# Determine if "struct pci_driver" has .driver_managed_dma member.
@@ -4999,6 +5099,23 @@ compile_test() {
compile_check_conftest "$CODE" "NV_MEMORY_DEVICE_COHERENT_PRESENT" "" "types"
;;
drm_crtc_funcs_has_get_vblank_timestamp)
#
# Determine if the 'drm_crtc_funcs' structure has a
# 'get_vblank_timestamp' field.
#
# Added by commit 7fe3f0d15aac ("drm: Add get_vblank_timestamp()
# to struct drm_crtc_funcs.") in v5.0
#
CODE="
#include <drm/drm_crtc.h>
int conftest_drm_crtc_funcs_has_get_vblank_timestamp(void) {
return offsetof(struct drm_crtc_funcs, get_vblank_timestamp);
}"
compile_check_conftest "$CODE" "NV_DRM_CRTC_FUNCS_HAS_GET_VBLANK_TIMESTAMP" "" "types"
;;
# When adding a new conftest entry, please use the correct format for
# specifying the relevant upstream Linux kernel commit. Please

1
kernel-open/header-presence-tests.mk
View File

@@ -41,6 +41,7 @@ NV_HEADER_PRESENCE_TESTS = \
linux/vfio_pci_core.h \
linux/cc_platform.h \
linux/slub_def.h \
asm/cpufeature.h \
asm/mshyperv.h \
crypto/sig.h

187
kernel-open/nvidia-drm/nv_drm_common_ioctl.h
View File

@@ -53,7 +53,21 @@
#define DRM_NVIDIA_SEMSURF_FENCE_WAIT 0x16
#define DRM_NVIDIA_SEMSURF_FENCE_ATTACH 0x17
#define DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID 0x18
#define DRM_NVIDIA_REGISTER_ROI 0x19 /* Register ROI */
#define DRM_NVIDIA_UNREGISTER_ROI 0x1a /* Unregister ROI */
#define DRM_NVIDIA_GET_CRTC_ROI_CRCS 0x1b /* Read CRCs for registered ROIs */
#define DRM_NVIDIA_GET_ROI_CAPABILITIES 0x1c /* Get ROI related capabilities */
/* Maximum possible telltale CRCs per plane (used for array size)
* actual value is queried from the plane property NV_DRM_NUM_PLANE_TELLTALE_CRCS*/
#define NV_DRM_MAX_TELLTALES_PER_PLANE 64
/* Maximum possible ROIs per CRTC (used for array size)
* actual value is queried from DRM_NVIDIA_GET_ROI_CAPABILITIES */
#define NV_DRM_MAX_ROIS_PER_CRTC 64
/* IOCTLs */
#define DRM_IOCTL_NVIDIA_GEM_IMPORT_NVKMS_MEMORY \
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GEM_IMPORT_NVKMS_MEMORY), \
struct drm_nvidia_gem_import_nvkms_memory_params)
@@ -163,6 +177,22 @@
DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID), \
struct drm_nvidia_get_drm_file_unique_id_params)
#define DRM_IOCTL_NVIDIA_REGISTER_ROI \
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_REGISTER_ROI), \
struct drm_nvidia_register_roi_params)
#define DRM_IOCTL_NVIDIA_UNREGISTER_ROI \
DRM_IOW((DRM_COMMAND_BASE + DRM_NVIDIA_UNREGISTER_ROI), \
struct drm_nvidia_unregister_roi_params)
#define DRM_IOCTL_NVIDIA_GET_CRTC_ROI_CRCS \
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GET_CRTC_ROI_CRCS), \
struct drm_nvidia_read_crc_params)
#define DRM_IOCTL_NVIDIA_GET_ROI_CAPABILITIES \
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GET_ROI_CAPABILITIES), \
struct drm_nvidia_get_roi_capabilities_params)
struct drm_nvidia_gem_import_nvkms_memory_params {
uint64_t mem_size; /* IN */
@@ -396,4 +426,161 @@ struct drm_nvidia_get_drm_file_unique_id_params {
uint64_t id; /* OUT Unique ID of the DRM file */
};
/**
* @brief Parameters for getting ROI capabilities
*
* This structure is used with the DRM_NVIDIA_GET_ROI_CAPABILITIES ioctl to
* get ROI related capabilities.
*
* @param[out] max_rois Maximum number of ROIs that can be registered
*/
struct drm_nvidia_get_roi_capabilities_params {
uint32_t max_registered_rois; /* OUT Maximum number of ROIs that can be registered */
uint32_t reserved[7];
};
/**
* @brief Rectangle structure for ROI
*
* This structure is used to define a rectangle region of interest.
*
* @param[in] x X coordinate of the top-left corner
* @param[in] y Y coordinate of the top-left corner
* @param[in] width Width of the ROI
* @param[in] height Height of the ROI
*/
struct drm_nvidia_roi_rect {
uint32_t x;
uint32_t y;
uint32_t width;
uint32_t height;
};
/**
* @brief Parameters for registering a Region of Interest (ROI)
*
* This structure is used to register an ROI region in the Region RAM which can
* then be referenced by its region handle when configuring a window to enable CRC.
* The region handle is treated as a separate resource and must be registered
* outside of the regular commit cycle.
*
* @param[in] rect Rectangle defining the ROI coordinates
* @param[out] region_handle Unique handle for the registered ROI
*/
struct drm_nvidia_register_roi_params {
struct drm_nvidia_roi_rect rect; /* IN */
uint64_t region_handle; /* OUT */
};
/**
* @brief Parameters for unregistering a Region of Interest (ROI)
*
* This structure is used to unregister a previously registered ROI region.
* Note: A region handle that is actively in use (configured in the CRC enablement
* property) cannot be unregistered. Only a handle that was previously registered
* by this client can be unregistered.
*
* @param[in] region_handle Handle of the ROI to unregister
*/
struct drm_nvidia_unregister_roi_params {
uint64_t region_handle; /* IN */
};
/**
* @brief Per-plane telltale CRC configuration
*
* This structure defines the configuration for telltale CRC on a per-plane basis.
* Each plane can have multiple telltale CRC regions configured, with each region
* referencing a previously registered ROI via its region handle.
*
* @param[in] region_handle Handle of registered ROI to use
* @param[in] golden_crc Expected CRC value for safety interrupts (0 if not configured)
*/
struct drm_nvidia_telltale_per_plane_config {
uint64_t region_handle; /* IN */
uint64_t golden_crc; /* IN */
};
/* NV_DRM_NUM_PLANE_TELLTALE_CRCS is a plane property that can be read to get
* the max number of telltale CRCs that can be enabled on a plane */
/**
* @brief Blob property structure for NV_DRM_PLANE_TELLTALES
*
* This structure is used as the content for the NV_DRM_PLANE_TELLTALES blob property
* that is set on a plane before atomic commit. It contains an array of telltale
* configurations, one for each telltale CRC region that should be enabled on the plane.
* The number of configurations should not exceed the value read from the
* NV_DRM_NUM_PLANE_TELLTALE_CRCS plane property.
* This property is expected to be set in the regular atomic commit operation.
*
* @param[in] telltale_configs Array of telltale CRC configurations
*/
struct drm_nvidia_plane_telltales {
struct drm_nvidia_telltale_per_plane_config \
telltale_configs[NV_DRM_MAX_TELLTALES_PER_PLANE]; /* IN */
};
/**
* @brief ROI CRC data structure
*
* This structure contains the CRC value for a specific ROI region. It is used
* as part of the array returned by the DRM_NVIDIA_GET_CRTC_ROI_CRCS ioctl to
* provide CRC values for all active ROI regions on a CRTC.
*
* @param[out] region_handle Handle of the ROI region
* @param[out] crc Computed CRC value for this ROI
* @param[in/out] reserved Reserved fields for future use
*/
struct drm_nvidia_roi_crc {
uint64_t region_handle; /* OUT */
uint64_t crc; /* OUT */
uint64_t reserved[4];
};
/**
* @brief Parameters for reading CRC data from CRTC
*
* This structure is used with the DRM_NVIDIA_GET_CRTC_ROI_CRCS ioctl to read
* the CRC values for all registered ROI regions that are currently active on
* the specified CRTC. The ioctl will populate the roi_crcs array with CRC data
* for each active region and set num_collected_crcs to indicate how many valid
* entries are in the array.
* Please note that only the last collected CRC is returned per active region handle.
*
* @param[in] crtc_id CRTC identifier to read CRCs from
* @param[out] num_collected_crcs Number of valid CRC entries returned
* @param[in/out] reserved Reserved fields for future use
* @param[out] roi_crcs Array of ROI CRC data
*/
struct drm_nvidia_read_crc_params {
int32_t crtc_id; /* IN */
int32_t num_collected_crcs; /* OUT */
struct drm_nvidia_roi_crc roi_crcs[NV_DRM_MAX_ROIS_PER_CRTC]; /* OUT */
uint64_t reserved[4];
};
/**
* @brief Named transfer function enum
*
* This enum defines the named transfer functions that can be used to set
* the degamma and regamma transfer functions properties.
*/
enum nv_drm_transfer_function {
NV_DRM_TRANSFER_FUNCTION_DEFAULT,
NV_DRM_TRANSFER_FUNCTION_LINEAR,
NV_DRM_TRANSFER_FUNCTION_PQ,
/*
* nvidia-drm only supports the transfer function types defined above. The
* below transfer functions are currently only supported by tegradisp-drm,
* and not by nvidia-drm. Do not define a transfer function above
* NV_DRM_TRANSFER_FUNCTION_SRGB without updating
* NV_DRM_TRANSFER_FUNCTION_MAX.
*/
NV_DRM_TRANSFER_FUNCTION_SRGB,
NV_DRM_TRANSFER_FUNCTION_SMPTE_170M,
NV_DRM_TRANSFER_FUNCTION_TEGRA_MAX,
NV_DRM_TRANSFER_FUNCTION_MAX = NV_DRM_TRANSFER_FUNCTION_SRGB,
};
#endif /* _NV_DRM_COMMON_IOCTL_H_ */

3
kernel-open/nvidia-drm/nvidia-drm-conftest.h
View File

@@ -57,7 +57,8 @@
#include <linux/kconfig.h> // for IS_ENABLED()
#if IS_ENABLED(CONFIG_DRM) || defined(__FreeBSD__)
#if (IS_ENABLED(CONFIG_DRM) && IS_ENABLED(CONFIG_DRM_KMS_HELPER)) || \
defined(__FreeBSD__)
#define NV_DRM_AVAILABLE
#endif

62
kernel-open/nvidia-drm/nvidia-drm-connector.c
View File

@@ -124,6 +124,9 @@ __nv_drm_detect_encoder(struct NvKmsKapiDynamicDisplayParams *pDetectParams,
return false;
}
dev->mode_config.max_width = pDetectParams->maxWidthInPixels;
dev->mode_config.max_height = pDetectParams->maxHeightInPixels;
#if defined(NV_DRM_CONNECTOR_HAS_VRR_CAPABLE_PROPERTY)
drm_connector_attach_vrr_capable_property(&nv_connector->base);
drm_connector_set_vrr_capable_property(&nv_connector->base, pDetectParams->vrrSupported ? true : false);
@@ -216,6 +219,52 @@ done:
return status;
}
static void nv_drm_connector_reset(struct drm_connector *connector)
{
struct nv_drm_connector_state * nv_connector_state =
nv_drm_calloc(1, sizeof(*nv_connector_state));
if (!nv_connector_state) {
return;
}
if (connector->state)
{
struct nv_drm_connector_state *nv_drm_connector_state_old =
to_nv_drm_connector_state(connector->state);
__drm_atomic_helper_connector_destroy_state(connector->state);
nv_drm_free(nv_drm_connector_state_old);
}
__drm_atomic_helper_connector_reset(connector, &nv_connector_state->base);
}
static struct drm_connector_state* nv_drm_connector_atomic_duplicate_state(struct drm_connector *connector)
{
struct nv_drm_connector_state *nv_drm_new_connector_state =
nv_drm_calloc(1, sizeof(*nv_drm_new_connector_state));
if (!nv_drm_new_connector_state) {
return NULL;
}
__drm_atomic_helper_connector_duplicate_state(connector, &nv_drm_new_connector_state->base);
return &nv_drm_new_connector_state->base;
}
static void nv_drm_connector_atomic_destroy_state(
struct drm_connector *connector,
struct drm_connector_state *state)
{
struct nv_drm_connector_state *nv_drm_connector_state =
to_nv_drm_connector_state(state);
__drm_atomic_helper_connector_destroy_state(state);
nv_drm_free(nv_drm_connector_state);
}
static void __nv_drm_connector_force(struct drm_connector *connector)
{
__nv_drm_connector_detect_internal(connector);
@@ -229,12 +278,12 @@ nv_drm_connector_detect(struct drm_connector *connector, bool force)
static struct drm_connector_funcs nv_connector_funcs = {
.destroy = nv_drm_connector_destroy,
.reset = drm_atomic_helper_connector_reset,
.reset = nv_drm_connector_reset,
.force = __nv_drm_connector_force,
.detect = nv_drm_connector_detect,
.fill_modes = drm_helper_probe_single_connector_modes,
.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
.atomic_duplicate_state = nv_drm_connector_atomic_duplicate_state,
.atomic_destroy_state = nv_drm_connector_atomic_destroy_state,
};
static int nv_drm_connector_get_modes(struct drm_connector *connector)
@@ -487,16 +536,19 @@ nv_drm_connector_new(struct drm_device *dev,
{
struct nv_drm_device *nv_dev = to_nv_device(dev);
struct nv_drm_connector *nv_connector = NULL;
struct nv_drm_connector_state *nv_connector_state = NULL;
int ret = -ENOMEM;
if ((nv_connector = nv_drm_calloc(1, sizeof(*nv_connector))) == NULL) {
goto failed;
}
if ((nv_connector->base.state =
nv_drm_calloc(1, sizeof(*nv_connector->base.state))) == NULL) {
if ((nv_connector_state =
nv_drm_calloc(1, sizeof(*nv_connector_state))) == NULL) {
goto failed_state_alloc;
}
nv_connector->base.state = &nv_connector_state->base;
nv_connector->base.state->connector = &nv_connector->base;
nv_connector->physicalIndex = physicalIndex;

16
kernel-open/nvidia-drm/nvidia-drm-connector.h
View File

@@ -75,6 +75,22 @@ static inline struct nv_drm_connector *to_nv_connector(
return container_of(connector, struct nv_drm_connector, base);
}
struct nv_drm_connector_state {
struct drm_connector_state base;
};
static inline struct nv_drm_connector_state *to_nv_drm_connector_state(
struct drm_connector_state *state)
{
return container_of(state, struct nv_drm_connector_state, base);
}
static inline const struct nv_drm_connector_state *to_nv_drm_connector_state_const(
const struct drm_connector_state *state)
{
return container_of(state, const struct nv_drm_connector_state, base);
}
static inline void nv_drm_connector_mark_connection_status_dirty(
struct nv_drm_connector *nv_connector)
{

77
kernel-open/nvidia-drm/nvidia-drm-crtc.c
View File

@@ -282,7 +282,7 @@ static void ctm_3x4_to_csc(struct NvKmsCscMatrix *nvkms_csc,
}
}
static void
static int
cursor_plane_req_config_update(struct drm_plane *plane,
struct drm_plane_state *plane_state,
struct NvKmsKapiCursorRequestedConfig *req_config)
@@ -292,11 +292,14 @@ cursor_plane_req_config_update(struct drm_plane *plane,
if (plane_state->fb == NULL) {
cursor_req_config_disable(req_config);
return;
return 0;
}
memset(req_config, 0, sizeof(*req_config));
req_config->surface = to_nv_framebuffer(plane_state->fb)->pSurface;
if (req_config->surface == NULL) {
return -EINVAL;
}
req_config->dstX = plane_state->crtc_x;
req_config->dstY = plane_state->crtc_y;
@@ -312,6 +315,10 @@ cursor_plane_req_config_update(struct drm_plane *plane,
req_config->compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_NON_PREMULT_SURFACE_ALPHA;
break;
case DRM_MODE_BLEND_PIXEL_NONE:
req_config->compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE;
break;
default:
/*
* We should not hit this, because
@@ -338,6 +345,10 @@ cursor_plane_req_config_update(struct drm_plane *plane,
req_config->compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_NON_PREMULT_ALPHA;
break;
case DRM_MODE_BLEND_PIXEL_NONE:
req_config->compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE;
break;
default:
/*
* We should not hit this, because
@@ -368,12 +379,14 @@ cursor_plane_req_config_update(struct drm_plane *plane,
if (old_config.surface == NULL &&
old_config.surface != req_config->surface) {
req_config->flags.dstXYChanged = NV_TRUE;
return;
return 0;
}
req_config->flags.dstXYChanged =
old_config.dstX != req_config->dstX ||
old_config.dstY != req_config->dstY;
return 0;
}
static void release_drm_nvkms_surface(struct nv_drm_nvkms_surface *drm_nvkms_surface)
@@ -1171,6 +1184,9 @@ plane_req_config_update(struct drm_plane *plane,
memset(req_config, 0, sizeof(*req_config));
req_config->config.surface = to_nv_framebuffer(plane_state->fb)->pSurface;
if (req_config->config.surface == NULL) {
return -EINVAL;
}
/* Source values are 16.16 fixed point */
req_config->config.srcX = plane_state->src_x >> 16;
@@ -1235,6 +1251,10 @@ plane_req_config_update(struct drm_plane *plane,
req_config->config.compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_NON_PREMULT_SURFACE_ALPHA;
break;
case DRM_MODE_BLEND_PIXEL_NONE:
req_config->config.compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE;
break;
default:
/*
* We should not hit this, because
@@ -1261,6 +1281,10 @@ plane_req_config_update(struct drm_plane *plane,
req_config->config.compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_NON_PREMULT_ALPHA;
break;
case DRM_MODE_BLEND_PIXEL_NONE:
req_config->config.compParams.compMode =
NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE;
break;
default:
/*
* We should not hit this, because
@@ -1439,6 +1463,17 @@ plane_req_config_update(struct drm_plane *plane,
}
}
}
}
/*
* Send ILUT configuration to NVKMS if:
* - degamma properties changed, OR
* - plane is being re-enabled and old req_config had ILUT disabled
* but DRM plane state has ILUT (blob ID didn't change, but NVKMS
* needs to be notified)
*/
if (nv_drm_plane_state->degamma_changed ||
(!old_config.ilut.enabled && (nv_drm_plane_state->degamma_drm_lut_surface != NULL))) {
if (nv_drm_plane_state->degamma_drm_lut_surface != NULL) {
req_config->config.ilut.enabled = NV_TRUE;
@@ -1455,7 +1490,6 @@ plane_req_config_update(struct drm_plane *plane,
req_config->config.ilut.offset = 0;
req_config->config.ilut.vssSegments = 0;
req_config->config.ilut.lutEntries = 0;
}
req_config->flags.ilutChanged = NV_TRUE;
}
@@ -1475,6 +1509,17 @@ plane_req_config_update(struct drm_plane *plane,
return -1;
}
}
}
/*
* Send TMO configuration to NVKMS if:
* - tmo properties changed, OR
* - plane is being re-enabled and old req_config had TMO disabled
* but DRM plane state has TMO (blob ID didn't change, but NVKMS
* needs to be notified)
*/
if (nv_drm_plane_state->tmo_changed ||
(!old_config.tmo.enabled && (nv_drm_plane_state->tmo_drm_lut_surface != NULL))) {
if (nv_drm_plane_state->tmo_drm_lut_surface != NULL) {
req_config->config.tmo.enabled = NV_TRUE;
@@ -1548,7 +1593,7 @@ static int __nv_drm_cursor_atomic_check(struct drm_plane *plane,
struct drm_plane_state *plane_state)
{
struct nv_drm_plane *nv_plane = to_nv_plane(plane);
int i;
int i, ret;
struct drm_crtc *crtc;
struct drm_crtc_state *crtc_state;
@@ -1568,8 +1613,11 @@ static int __nv_drm_cursor_atomic_check(struct drm_plane *plane,
}
if (plane_state->crtc == crtc) {
cursor_plane_req_config_update(plane, plane_state,
cursor_req_config);
ret = cursor_plane_req_config_update(plane, plane_state,
cursor_req_config);
if (ret != 0) {
return ret;
}
}
}
@@ -1621,7 +1669,7 @@ static int nv_drm_plane_atomic_check(struct drm_plane *plane,
return ret;
}
if (crtc_state->color_mgmt_changed) {
if (crtc_state->color_mgmt_changed || (plane->state->crtc != plane_state->crtc)) {
/*
* According to the comment in the Linux kernel's
* drivers/gpu/drm/drm_color_mgmt.c, if this property is NULL,
@@ -2206,6 +2254,7 @@ nv_drm_atomic_crtc_duplicate_state(struct drm_crtc *crtc)
drm_property_blob_get(nv_state->regamma_lut);
}
nv_state->regamma_divisor = nv_old_state->regamma_divisor;
nv_state->regamma_drm_lut_surface = nv_old_state->regamma_drm_lut_surface;
if (nv_state->regamma_drm_lut_surface) {
kref_get(&nv_state->regamma_drm_lut_surface->base.refcount);
}
@@ -2687,7 +2736,8 @@ __nv_drm_plane_create_alpha_blending_properties(struct drm_plane *plane,
drm_plane_create_alpha_property(plane);
drm_plane_create_blend_mode_property(plane,
NVBIT(DRM_MODE_BLEND_PREMULTI) |
NVBIT(DRM_MODE_BLEND_COVERAGE));
NVBIT(DRM_MODE_BLEND_COVERAGE) |
NVBIT(DRM_MODE_BLEND_PIXEL_NONE));
} else if ((validCompModes &
NVBIT(NVKMS_COMPOSITION_BLENDING_MODE_PREMULT_ALPHA)) != 0x0 &&
(validCompModes &
@@ -2695,7 +2745,8 @@ __nv_drm_plane_create_alpha_blending_properties(struct drm_plane *plane,
drm_plane_create_blend_mode_property(plane,
NVBIT(DRM_MODE_BLEND_PREMULTI) |
NVBIT(DRM_MODE_BLEND_COVERAGE));
NVBIT(DRM_MODE_BLEND_COVERAGE) |
NVBIT(DRM_MODE_BLEND_PIXEL_NONE));
}
#endif
}
@@ -2785,7 +2836,11 @@ nv_drm_plane_create(struct drm_device *dev,
(plane_type == DRM_PLANE_TYPE_CURSOR) ?
0x0 : pResInfo->caps.layer[layer_idx].validRRTransforms;
if ((validCompositionModes &
/* Primary planes should always use OPAQUE mode, while overlay and cursor
* planes prefer PREMULT_ALPHA.
*/
if ((plane_type == DRM_PLANE_TYPE_PRIMARY) &&
(validCompositionModes &
NVBIT(NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE)) != 0x0) {
defaultCompositionMode = NVKMS_COMPOSITION_BLENDING_MODE_OPAQUE;
} else if ((validCompositionModes &

8
kernel-open/nvidia-drm/nvidia-drm-crtc.h
View File

@@ -37,13 +37,7 @@
#include "nvtypes.h"
#include "nvkms-kapi.h"
enum nv_drm_transfer_function {
NV_DRM_TRANSFER_FUNCTION_DEFAULT,
NV_DRM_TRANSFER_FUNCTION_LINEAR,
NV_DRM_TRANSFER_FUNCTION_PQ,
NV_DRM_TRANSFER_FUNCTION_MAX,
};
#include "nv_drm_common_ioctl.h"
struct nv_drm_crtc {
NvU32 head;

18
kernel-open/nvidia-drm/nvidia-drm-drv.c
View File

@@ -283,6 +283,10 @@ static int nv_drm_get_mst_display_infos
goto done;
}
if (!connectorInfo->dynamicDpyIdListValid) {
ret = -ETIMEDOUT;
goto done;
}
*nDynamicDisplays = nvCountDpyIdsInDpyIdList(connectorInfo->dynamicDpyIdList);
@@ -444,9 +448,11 @@ static void nv_drm_enumerate_encoders_and_connectors
if (nv_drm_get_mst_display_infos(nv_dev, hDisplays[i],
&displayInfos, &nDynamicDisplays)) {
NV_DRM_DEV_LOG_ERR(
NV_DRM_DEV_LOG_INFO(
nv_dev,
"Failed to get dynamic displays");
"Failed to get dynamic displays during device "
"registration. Dynamic displays may be probed in "
"non-deterministic order.");
} else if (nDynamicDisplays) {
nv_drm_sort_dynamic_displays_by_dp_addr(displayInfos, nDynamicDisplays);
@@ -2059,6 +2065,7 @@ void nv_drm_register_drm_device(const struct NvKmsKapiGpuInfo *gpu_info)
#elif defined(NV_APERTURE_REMOVE_CONFLICTING_DEVICES_PRESENT)
aperture_remove_conflicting_devices(base, size, nv_drm_driver.name);
#endif
nvKms->framebufferConsoleDisabled(nv_dev->pDevice);
} else {
NV_DRM_DEV_LOG_INFO(nv_dev, "Invalid framebuffer console info");
}
@@ -2070,6 +2077,13 @@ void nv_drm_register_drm_device(const struct NvKmsKapiGpuInfo *gpu_info)
#elif defined(NV_DRM_FBDEV_GENERIC_AVAILABLE)
drm_fbdev_generic_setup(dev, 32);
#endif
/*
* The fbdev setup functions above may override the
* `pm_vt_switch_required` state of our PCI device. For now, a VT switch
* is required during suspend and resume.
*/
pm_vt_switch_required(dev->dev, true);
}
#endif /* defined(NV_DRM_FBDEV_AVAILABLE) */

22
kernel-open/nvidia-drm/nvidia-drm-fb.c
View File

@@ -63,7 +63,9 @@ static void nv_drm_framebuffer_destroy(struct drm_framebuffer *fb)
/* Free NvKmsKapiSurface associated with this framebuffer object */
nvKms->destroySurface(nv_dev->pDevice, nv_fb->pSurface);
if (nv_fb->pSurface != NULL) {
nvKms->destroySurface(nv_dev->pDevice, nv_fb->pSurface);
}
__nv_drm_framebuffer_free(nv_fb);
}
@@ -139,6 +141,7 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
struct NvKmsKapiCreateSurfaceParams params = { };
struct nv_drm_gem_object *nv_gem;
struct drm_framebuffer *fb = &nv_fb->base;
bool non_scanout_mem_backed = false;
uint32_t i;
int ret;
@@ -160,6 +163,10 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
params.planes[i].memory = nv_gem->pMemory;
params.planes[i].offset = fb->offsets[i];
params.planes[i].pitch = fb->pitches[i];
if (!nvKms->isVidmem(nv_gem->pMemory) && nv_dev->hasVideoMemory) {
non_scanout_mem_backed = true;
}
}
}
params.height = fb->height;
@@ -226,10 +233,15 @@ static int nv_drm_framebuffer_init(struct drm_device *dev,
/* Create NvKmsKapiSurface */
nv_fb->pSurface = nvKms->createSurface(nv_dev->pDevice, &params);
if (nv_fb->pSurface == NULL) {
NV_DRM_DEV_DEBUG_DRIVER(nv_dev, "Failed to create NvKmsKapiSurface");
goto fail;
if (non_scanout_mem_backed) {
/* Do not register drm_framebuffer against nvkms */
nv_fb->pSurface = NULL;
} else {
nv_fb->pSurface = nvKms->createSurface(nv_dev->pDevice, &params);
if (nv_fb->pSurface == NULL) {
NV_DRM_DEV_DEBUG_DRIVER(nv_dev, "Failed to create NvKmsKapiSurface");
goto fail;
}
}
return 0;

40
kernel-open/nvidia-drm/nvidia-drm-gem-nvkms-memory.c
View File

@@ -57,6 +57,11 @@ static void __nv_drm_gem_nvkms_memory_free(struct nv_drm_gem_object *nv_gem)
nvKms->freeMemoryPages((NvU64 *)nv_nvkms_memory->pages);
}
/* Decrement GC6 blocker if it's still held */
if (nv_nvkms_memory->was_mmapped) {
nvKms->gc6BlockerRefCntDec(nv_dev->pDevice);
}
/* Free NvKmsKapiMemory handle associated with this gem object */
nvKms->freeMemory(nv_dev->pDevice, nv_nvkms_memory->base.pMemory);
@@ -78,6 +83,23 @@ static int __nv_drm_gem_nvkms_mmap(struct nv_drm_gem_object *nv_gem,
return ret;
}
if (nvKms->isVidmem(nv_nvkms_memory->base.pMemory)) {
/*
* Usermode clients can't be trusted not to access mappings while the GPU
* is in GC6. Increment the GC6 blocker refcount to keep the GPU awake
* while this mapping is active.
*/
mutex_lock(&nv_nvkms_memory->map_lock);
if (!nv_nvkms_memory->was_mmapped) {
if (!nvKms->gc6BlockerRefCntInc(nv_gem->nv_dev->pDevice)) {
mutex_unlock(&nv_nvkms_memory->map_lock);
return -ENOMEM;
}
nv_nvkms_memory->was_mmapped = true;
}
mutex_unlock(&nv_nvkms_memory->map_lock);
}
return drm_gem_mmap_obj(&nv_gem->base,
drm_vma_node_size(&nv_gem->base.vma_node) << PAGE_SHIFT, vma);
}
@@ -218,6 +240,14 @@ static void *__nv_drm_gem_nvkms_prime_vmap(
}
if (nv_nvkms_memory->physically_mapped) {
/*
* For CPU mappings of vidmem, increment the GC6 blocker refcount to
* keep the GPU awake while the kernel is using the mapping.
*/
if (nvKms->isVidmem(nv_gem->pMemory) &&
!nvKms->gc6BlockerRefCntInc(nv_gem->nv_dev->pDevice)) {
return ERR_PTR(-ENOMEM);
}
return nv_nvkms_memory->pWriteCombinedIORemapAddress;
}
@@ -246,6 +276,15 @@ static void __nv_drm_gem_nvkms_prime_vunmap(
nv_nvkms_memory->pages_count > 0) {
nv_drm_vunmap(address);
}
if (nv_nvkms_memory->physically_mapped &&
nvKms->isVidmem(nv_gem->pMemory)) {
/*
* Decrement the GC6 blocker refcount now that the mapping is no longer
* needed.
*/
nvKms->gc6BlockerRefCntDec(nv_gem->nv_dev->pDevice);
}
}
static int __nv_drm_gem_map_nvkms_memory_offset(
@@ -312,6 +351,7 @@ static int __nv_drm_nvkms_gem_obj_init(
nv_nvkms_memory->pPhysicalAddress = NULL;
nv_nvkms_memory->pWriteCombinedIORemapAddress = NULL;
nv_nvkms_memory->physically_mapped = false;
nv_nvkms_memory->was_mmapped = false;
if (!nvKms->isVidmem(pMemory) &&
!nvKms->getMemoryPages(nv_dev->pDevice,

7
kernel-open/nvidia-drm/nvidia-drm-gem-nvkms-memory.h
View File

@@ -34,15 +34,16 @@ struct nv_drm_gem_nvkms_memory {
/*
* Lock to protect concurrent writes to physically_mapped, pPhysicalAddress,
* and pWriteCombinedIORemapAddress.
* pWriteCombinedIORemapAddress, and was_mmapped.
*
* __nv_drm_gem_nvkms_map(), the sole writer, is structured such that
* readers are not required to hold the lock.
* __nv_drm_gem_nvkms_map() and __nv_drm_gem_nvkms_mmap are structured such
* that readers are not required to hold the lock.
*/
struct mutex map_lock;
bool physically_mapped;
void *pPhysicalAddress;
void *pWriteCombinedIORemapAddress;
bool was_mmapped;
struct page **pages;
unsigned long pages_count;

2
kernel-open/nvidia-drm/nvidia-drm-linux.c
View File

@@ -31,7 +31,7 @@
MODULE_PARM_DESC(
modeset,
"Enable atomic kernel modesetting (1 = enable, 0 = disable (default))");
"Enable atomic kernel modesetting (1 = enable (default), 0 = disable)");
module_param_named(modeset, nv_drm_modeset_module_param, bool, 0400);
#if defined(NV_DRM_FBDEV_AVAILABLE)

2
kernel-open/nvidia-drm/nvidia-drm-os-interface.c
View File

@@ -38,7 +38,7 @@
#include <drm/drmP.h>
#endif
bool nv_drm_modeset_module_param = false;
bool nv_drm_modeset_module_param = true;
bool nv_drm_fbdev_module_param = true;
void *nv_drm_calloc(size_t nmemb, size_t size)

16
kernel-open/nvidia-modeset/nvidia-modeset-linux.c
View File

@@ -81,9 +81,6 @@ module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0
static bool hdmi_deepcolor = true;
module_param_named(hdmi_deepcolor, hdmi_deepcolor, bool, 0400);
static bool vblank_sem_control = true;
module_param_named(vblank_sem_control, vblank_sem_control, bool, 0400);
static bool opportunistic_display_sync = true;
module_param_named(opportunistic_display_sync, opportunistic_display_sync, bool, 0400);
@@ -180,11 +177,6 @@ NvBool nvkms_hdmi_deepcolor(void)
return hdmi_deepcolor;
}
NvBool nvkms_vblank_sem_control(void)
{
return vblank_sem_control;
}
NvBool nvkms_opportunistic_display_sync(void)
{
return opportunistic_display_sync;
@@ -1179,7 +1171,7 @@ done:
return ret;
}
NvBool nvkms_open_gpu(NvU32 gpuId)
NvBool nvkms_open_gpu(NvU32 gpuId, NvBool reset_aware)
{
nvidia_modeset_stack_ptr stack = NULL;
NvBool ret;
@@ -1188,14 +1180,14 @@ NvBool nvkms_open_gpu(NvU32 gpuId)
return NV_FALSE;
}
ret = __rm_ops.open_gpu(gpuId, stack) == 0;
ret = __rm_ops.open_gpu(gpuId, stack, reset_aware) == 0;
__rm_ops.free_stack(stack);
return ret;
}
void nvkms_close_gpu(NvU32 gpuId)
void nvkms_close_gpu(NvU32 gpuId, NvBool reset_aware)
{
nvidia_modeset_stack_ptr stack = NULL;
@@ -1203,7 +1195,7 @@ void nvkms_close_gpu(NvU32 gpuId)
return;
}
__rm_ops.close_gpu(gpuId, stack);
__rm_ops.close_gpu(gpuId, stack, reset_aware);
__rm_ops.free_stack(stack);
}

5
kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h
View File

@@ -116,7 +116,6 @@ NvBool nvkms_output_rounding_fix(void);
NvBool nvkms_disable_hdmi_frl(void);
NvBool nvkms_disable_vrr_memclk_switch(void);
NvBool nvkms_hdmi_deepcolor(void);
NvBool nvkms_vblank_sem_control(void);
NvBool nvkms_opportunistic_display_sync(void);
enum NvKmsDebugForceColorSpace nvkms_debug_force_color_space(void);
NvBool nvkms_enable_overlay_layers(void);
@@ -307,8 +306,8 @@ void* nvkms_get_per_open_data(int fd);
/*!
* Raise and lower the reference count of the specified GPU.
*/
NvBool nvkms_open_gpu(NvU32 gpuId);
void nvkms_close_gpu(NvU32 gpuId);
NvBool nvkms_open_gpu(NvU32 gpuId, NvBool reset_aware);
void nvkms_close_gpu(NvU32 gpuId, NvBool reset_aware);
/*!

254
kernel-open/nvidia-uvm/cla16f.h
View File

@@ -1,254 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _cla16f_h_
#define _cla16f_h_
#ifdef __cplusplus
extern "C" {
#endif
#include "nvtypes.h"
/* class KEPLER_CHANNEL_GPFIFO */
/*
* Documentation for KEPLER_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
* chapter "User Control Registers". It is documented as device NV_UDMA.
* The GPFIFO format itself is also documented in dev_pbdma.ref,
* NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
* chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
*
*/
#define KEPLER_CHANNEL_GPFIFO_B (0x0000A16F)
/* pio method data structure */
typedef volatile struct _cla16f_tag0 {
NvV32 Reserved00[0x7c0];
} NvA16FTypedef, KEPLER_ChannelGPFifoB;
#define NVA16F_TYPEDEF KEPLER_CHANNELChannelGPFifo
/* dma flow control data structure */
typedef volatile struct _cla16f_tag1 {
NvU32 Ignored00[0x010]; /* 0000-003f*/
NvU32 Put; /* put offset, read/write 0040-0043*/
NvU32 Get; /* get offset, read only 0044-0047*/
NvU32 Reference; /* reference value, read only 0048-004b*/
NvU32 PutHi; /* high order put offset bits 004c-004f*/
NvU32 Ignored01[0x002]; /* 0050-0057*/
NvU32 TopLevelGet; /* top level get offset, read only 0058-005b*/
NvU32 TopLevelGetHi; /* high order top level get bits 005c-005f*/
NvU32 GetHi; /* high order get offset bits 0060-0063*/
NvU32 Ignored02[0x007]; /* 0064-007f*/
NvU32 Ignored03; /* used to be engine yield 0080-0083*/
NvU32 Ignored04[0x001]; /* 0084-0087*/
NvU32 GPGet; /* GP FIFO get offset, read only 0088-008b*/
NvU32 GPPut; /* GP FIFO put offset 008c-008f*/
NvU32 Ignored05[0x5c];
} NvA16FControl, KeplerBControlGPFifo;
/* fields and values */
#define NVA16F_NUMBER_OF_SUBCHANNELS (8)
#define NVA16F_SET_OBJECT (0x00000000)
#define NVA16F_SET_OBJECT_NVCLASS 15:0
#define NVA16F_SET_OBJECT_ENGINE 20:16
#define NVA16F_SET_OBJECT_ENGINE_SW 0x0000001f
#define NVA16F_ILLEGAL (0x00000004)
#define NVA16F_ILLEGAL_HANDLE 31:0
#define NVA16F_NOP (0x00000008)
#define NVA16F_NOP_HANDLE 31:0
#define NVA16F_SEMAPHOREA (0x00000010)
#define NVA16F_SEMAPHOREA_OFFSET_UPPER 7:0
#define NVA16F_SEMAPHOREB (0x00000014)
#define NVA16F_SEMAPHOREB_OFFSET_LOWER 31:2
#define NVA16F_SEMAPHOREC (0x00000018)
#define NVA16F_SEMAPHOREC_PAYLOAD 31:0
#define NVA16F_SEMAPHORED (0x0000001C)
#define NVA16F_SEMAPHORED_OPERATION 4:0
#define NVA16F_SEMAPHORED_OPERATION_ACQUIRE 0x00000001
#define NVA16F_SEMAPHORED_OPERATION_RELEASE 0x00000002
#define NVA16F_SEMAPHORED_OPERATION_ACQ_GEQ 0x00000004
#define NVA16F_SEMAPHORED_OPERATION_ACQ_AND 0x00000008
#define NVA16F_SEMAPHORED_OPERATION_REDUCTION 0x00000010
#define NVA16F_SEMAPHORED_ACQUIRE_SWITCH 12:12
#define NVA16F_SEMAPHORED_ACQUIRE_SWITCH_DISABLED 0x00000000
#define NVA16F_SEMAPHORED_ACQUIRE_SWITCH_ENABLED 0x00000001
#define NVA16F_SEMAPHORED_RELEASE_WFI 20:20
#define NVA16F_SEMAPHORED_RELEASE_WFI_EN 0x00000000
#define NVA16F_SEMAPHORED_RELEASE_WFI_DIS 0x00000001
#define NVA16F_SEMAPHORED_RELEASE_SIZE 24:24
#define NVA16F_SEMAPHORED_RELEASE_SIZE_16BYTE 0x00000000
#define NVA16F_SEMAPHORED_RELEASE_SIZE_4BYTE 0x00000001
#define NVA16F_SEMAPHORED_REDUCTION 30:27
#define NVA16F_SEMAPHORED_REDUCTION_MIN 0x00000000
#define NVA16F_SEMAPHORED_REDUCTION_MAX 0x00000001
#define NVA16F_SEMAPHORED_REDUCTION_XOR 0x00000002
#define NVA16F_SEMAPHORED_REDUCTION_AND 0x00000003
#define NVA16F_SEMAPHORED_REDUCTION_OR 0x00000004
#define NVA16F_SEMAPHORED_REDUCTION_ADD 0x00000005
#define NVA16F_SEMAPHORED_REDUCTION_INC 0x00000006
#define NVA16F_SEMAPHORED_REDUCTION_DEC 0x00000007
#define NVA16F_SEMAPHORED_FORMAT 31:31
#define NVA16F_SEMAPHORED_FORMAT_SIGNED 0x00000000
#define NVA16F_SEMAPHORED_FORMAT_UNSIGNED 0x00000001
#define NVA16F_NON_STALL_INTERRUPT (0x00000020)
#define NVA16F_NON_STALL_INTERRUPT_HANDLE 31:0
#define NVA16F_FB_FLUSH (0x00000024)
#define NVA16F_FB_FLUSH_HANDLE 31:0
#define NVA16F_MEM_OP_A (0x00000028)
#define NVA16F_MEM_OP_A_OPERAND_LOW 31:2
#define NVA16F_MEM_OP_A_TLB_INVALIDATE_ADDR 29:2
#define NVA16F_MEM_OP_A_TLB_INVALIDATE_TARGET 31:30
#define NVA16F_MEM_OP_A_TLB_INVALIDATE_TARGET_VID_MEM 0x00000000
#define NVA16F_MEM_OP_A_TLB_INVALIDATE_TARGET_SYS_MEM_COHERENT 0x00000002
#define NVA16F_MEM_OP_A_TLB_INVALIDATE_TARGET_SYS_MEM_NONCOHERENT 0x00000003
#define NVA16F_MEM_OP_B (0x0000002c)
#define NVA16F_MEM_OP_B_OPERAND_HIGH 7:0
#define NVA16F_MEM_OP_B_OPERATION 31:27
#define NVA16F_MEM_OP_B_OPERATION_SYSMEMBAR_FLUSH 0x00000005
#define NVA16F_MEM_OP_B_OPERATION_SOFT_FLUSH 0x00000006
#define NVA16F_MEM_OP_B_OPERATION_MMU_TLB_INVALIDATE 0x00000009
#define NVA16F_MEM_OP_B_OPERATION_L2_PEERMEM_INVALIDATE 0x0000000d
#define NVA16F_MEM_OP_B_OPERATION_L2_SYSMEM_INVALIDATE 0x0000000e
#define NVA16F_MEM_OP_B_OPERATION_L2_CLEAN_COMPTAGS 0x0000000f
#define NVA16F_MEM_OP_B_OPERATION_L2_FLUSH_DIRTY 0x00000010
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_PDB 0:0
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_PDB_ONE 0x00000000
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_PDB_ALL 0x00000001
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_GPC 1:1
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_GPC_ENABLE 0x00000000
#define NVA16F_MEM_OP_B_MMU_TLB_INVALIDATE_GPC_DISABLE 0x00000001
#define NVA16F_SET_REFERENCE (0x00000050)
#define NVA16F_SET_REFERENCE_COUNT 31:0
#define NVA16F_WFI (0x00000078)
#define NVA16F_WFI_HANDLE 31:0
#define NVA16F_CRC_CHECK (0x0000007c)
#define NVA16F_CRC_CHECK_VALUE 31:0
#define NVA16F_YIELD (0x00000080)
#define NVA16F_YIELD_OP 1:0
#define NVA16F_YIELD_OP_NOP 0x00000000
/* GPFIFO entry format */
#define NVA16F_GP_ENTRY__SIZE 8
#define NVA16F_GP_ENTRY0_FETCH 0:0
#define NVA16F_GP_ENTRY0_FETCH_UNCONDITIONAL 0x00000000
#define NVA16F_GP_ENTRY0_FETCH_CONDITIONAL 0x00000001
#define NVA16F_GP_ENTRY0_GET 31:2
#define NVA16F_GP_ENTRY0_OPERAND 31:0
#define NVA16F_GP_ENTRY1_GET_HI 7:0
#define NVA16F_GP_ENTRY1_PRIV 8:8
#define NVA16F_GP_ENTRY1_PRIV_USER 0x00000000
#define NVA16F_GP_ENTRY1_PRIV_KERNEL 0x00000001
#define NVA16F_GP_ENTRY1_LEVEL 9:9
#define NVA16F_GP_ENTRY1_LEVEL_MAIN 0x00000000
#define NVA16F_GP_ENTRY1_LEVEL_SUBROUTINE 0x00000001
#define NVA16F_GP_ENTRY1_LENGTH 30:10
#define NVA16F_GP_ENTRY1_SYNC 31:31
#define NVA16F_GP_ENTRY1_SYNC_PROCEED 0x00000000
#define NVA16F_GP_ENTRY1_SYNC_WAIT 0x00000001
#define NVA16F_GP_ENTRY1_OPCODE 7:0
#define NVA16F_GP_ENTRY1_OPCODE_NOP 0x00000000
#define NVA16F_GP_ENTRY1_OPCODE_ILLEGAL 0x00000001
#define NVA16F_GP_ENTRY1_OPCODE_GP_CRC 0x00000002
#define NVA16F_GP_ENTRY1_OPCODE_PB_CRC 0x00000003
/* dma method formats */
#define NVA16F_DMA_METHOD_ADDRESS_OLD 12:2
#define NVA16F_DMA_METHOD_ADDRESS 11:0
#define NVA16F_DMA_SUBDEVICE_MASK 15:4
#define NVA16F_DMA_METHOD_SUBCHANNEL 15:13
#define NVA16F_DMA_TERT_OP 17:16
#define NVA16F_DMA_TERT_OP_GRP0_INC_METHOD (0x00000000)
#define NVA16F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK (0x00000001)
#define NVA16F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK (0x00000002)
#define NVA16F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK (0x00000003)
#define NVA16F_DMA_TERT_OP_GRP2_NON_INC_METHOD (0x00000000)
#define NVA16F_DMA_METHOD_COUNT_OLD 28:18
#define NVA16F_DMA_METHOD_COUNT 28:16
#define NVA16F_DMA_IMMD_DATA 28:16
#define NVA16F_DMA_SEC_OP 31:29
#define NVA16F_DMA_SEC_OP_GRP0_USE_TERT (0x00000000)
#define NVA16F_DMA_SEC_OP_INC_METHOD (0x00000001)
#define NVA16F_DMA_SEC_OP_GRP2_USE_TERT (0x00000002)
#define NVA16F_DMA_SEC_OP_NON_INC_METHOD (0x00000003)
#define NVA16F_DMA_SEC_OP_IMMD_DATA_METHOD (0x00000004)
#define NVA16F_DMA_SEC_OP_ONE_INC (0x00000005)
#define NVA16F_DMA_SEC_OP_RESERVED6 (0x00000006)
#define NVA16F_DMA_SEC_OP_END_PB_SEGMENT (0x00000007)
/* dma incrementing method format */
#define NVA16F_DMA_INCR_ADDRESS 11:0
#define NVA16F_DMA_INCR_SUBCHANNEL 15:13
#define NVA16F_DMA_INCR_COUNT 28:16
#define NVA16F_DMA_INCR_OPCODE 31:29
#define NVA16F_DMA_INCR_OPCODE_VALUE (0x00000001)
#define NVA16F_DMA_INCR_DATA 31:0
/* dma non-incrementing method format */
#define NVA16F_DMA_NONINCR_ADDRESS 11:0
#define NVA16F_DMA_NONINCR_SUBCHANNEL 15:13
#define NVA16F_DMA_NONINCR_COUNT 28:16
#define NVA16F_DMA_NONINCR_OPCODE 31:29
#define NVA16F_DMA_NONINCR_OPCODE_VALUE (0x00000003)
#define NVA16F_DMA_NONINCR_DATA 31:0
/* dma increment-once method format */
#define NVA16F_DMA_ONEINCR_ADDRESS 11:0
#define NVA16F_DMA_ONEINCR_SUBCHANNEL 15:13
#define NVA16F_DMA_ONEINCR_COUNT 28:16
#define NVA16F_DMA_ONEINCR_OPCODE 31:29
#define NVA16F_DMA_ONEINCR_OPCODE_VALUE (0x00000005)
#define NVA16F_DMA_ONEINCR_DATA 31:0
/* dma no-operation format */
#define NVA16F_DMA_NOP (0x00000000)
/* dma immediate-data format */
#define NVA16F_DMA_IMMD_ADDRESS 11:0
#define NVA16F_DMA_IMMD_SUBCHANNEL 15:13
#define NVA16F_DMA_IMMD_DATA 28:16
#define NVA16F_DMA_IMMD_OPCODE 31:29
#define NVA16F_DMA_IMMD_OPCODE_VALUE (0x00000004)
/* dma set sub-device mask format */
#define NVA16F_DMA_SET_SUBDEVICE_MASK_VALUE 15:4
#define NVA16F_DMA_SET_SUBDEVICE_MASK_OPCODE 31:16
#define NVA16F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE (0x00000001)
/* dma store sub-device mask format */
#define NVA16F_DMA_STORE_SUBDEVICE_MASK_VALUE 15:4
#define NVA16F_DMA_STORE_SUBDEVICE_MASK_OPCODE 31:16
#define NVA16F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000002)
/* dma use sub-device mask format */
#define NVA16F_DMA_USE_SUBDEVICE_MASK_OPCODE 31:16
#define NVA16F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000003)
/* dma end-segment format */
#define NVA16F_DMA_ENDSEG_OPCODE 31:29
#define NVA16F_DMA_ENDSEG_OPCODE_VALUE (0x00000007)
/* dma legacy incrementing/non-incrementing formats */
#define NVA16F_DMA_ADDRESS 12:2
#define NVA16F_DMA_SUBCH 15:13
#define NVA16F_DMA_OPCODE3 17:16
#define NVA16F_DMA_OPCODE3_NONE (0x00000000)
#define NVA16F_DMA_COUNT 28:18
#define NVA16F_DMA_OPCODE 31:29
#define NVA16F_DMA_OPCODE_METHOD (0x00000000)
#define NVA16F_DMA_OPCODE_NONINC_METHOD (0x00000002)
#define NVA16F_DMA_DATA 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif /* _cla16F_h_ */

260
kernel-open/nvidia-uvm/clb06f.h
View File

@@ -1,260 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _clb06f_h_
#define _clb06f_h_
#ifdef __cplusplus
extern "C" {
#endif
#include "nvtypes.h"
/* class MAXWELL_CHANNEL_GPFIFO */
/*
* Documentation for MAXWELL_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
* chapter "User Control Registers". It is documented as device NV_UDMA.
* The GPFIFO format itself is also documented in dev_pbdma.ref,
* NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
* chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
*
*/
#define MAXWELL_CHANNEL_GPFIFO_A (0x0000B06F)
#define NVB06F_TYPEDEF MAXWELL_CHANNELChannelGPFifoA
/* dma flow control data structure */
typedef volatile struct _clb06f_tag0 {
NvU32 Ignored00[0x010]; /* 0000-003f*/
NvU32 Put; /* put offset, read/write 0040-0043*/
NvU32 Get; /* get offset, read only 0044-0047*/
NvU32 Reference; /* reference value, read only 0048-004b*/
NvU32 PutHi; /* high order put offset bits 004c-004f*/
NvU32 Ignored01[0x002]; /* 0050-0057*/
NvU32 TopLevelGet; /* top level get offset, read only 0058-005b*/
NvU32 TopLevelGetHi; /* high order top level get bits 005c-005f*/
NvU32 GetHi; /* high order get offset bits 0060-0063*/
NvU32 Ignored02[0x007]; /* 0064-007f*/
NvU32 Ignored03; /* used to be engine yield 0080-0083*/
NvU32 Ignored04[0x001]; /* 0084-0087*/
NvU32 GPGet; /* GP FIFO get offset, read only 0088-008b*/
NvU32 GPPut; /* GP FIFO put offset 008c-008f*/
NvU32 Ignored05[0x5c];
} Nvb06FControl, MaxwellAControlGPFifo;
/* fields and values */
#define NVB06F_NUMBER_OF_SUBCHANNELS (8)
#define NVB06F_SET_OBJECT (0x00000000)
#define NVB06F_SET_OBJECT_NVCLASS 15:0
#define NVB06F_SET_OBJECT_ENGINE 20:16
#define NVB06F_SET_OBJECT_ENGINE_SW 0x0000001f
#define NVB06F_ILLEGAL (0x00000004)
#define NVB06F_ILLEGAL_HANDLE 31:0
#define NVB06F_NOP (0x00000008)
#define NVB06F_NOP_HANDLE 31:0
#define NVB06F_SEMAPHOREA (0x00000010)
#define NVB06F_SEMAPHOREA_OFFSET_UPPER 7:0
#define NVB06F_SEMAPHOREB (0x00000014)
#define NVB06F_SEMAPHOREB_OFFSET_LOWER 31:2
#define NVB06F_SEMAPHOREC (0x00000018)
#define NVB06F_SEMAPHOREC_PAYLOAD 31:0
#define NVB06F_SEMAPHORED (0x0000001C)
#define NVB06F_SEMAPHORED_OPERATION 4:0
#define NVB06F_SEMAPHORED_OPERATION_ACQUIRE 0x00000001
#define NVB06F_SEMAPHORED_OPERATION_RELEASE 0x00000002
#define NVB06F_SEMAPHORED_OPERATION_ACQ_GEQ 0x00000004
#define NVB06F_SEMAPHORED_OPERATION_ACQ_AND 0x00000008
#define NVB06F_SEMAPHORED_OPERATION_REDUCTION 0x00000010
#define NVB06F_SEMAPHORED_ACQUIRE_SWITCH 12:12
#define NVB06F_SEMAPHORED_ACQUIRE_SWITCH_DISABLED 0x00000000
#define NVB06F_SEMAPHORED_ACQUIRE_SWITCH_ENABLED 0x00000001
#define NVB06F_SEMAPHORED_RELEASE_WFI 20:20
#define NVB06F_SEMAPHORED_RELEASE_WFI_EN 0x00000000
#define NVB06F_SEMAPHORED_RELEASE_WFI_DIS 0x00000001
#define NVB06F_SEMAPHORED_RELEASE_SIZE 24:24
#define NVB06F_SEMAPHORED_RELEASE_SIZE_16BYTE 0x00000000
#define NVB06F_SEMAPHORED_RELEASE_SIZE_4BYTE 0x00000001
#define NVB06F_SEMAPHORED_REDUCTION 30:27
#define NVB06F_SEMAPHORED_REDUCTION_MIN 0x00000000
#define NVB06F_SEMAPHORED_REDUCTION_MAX 0x00000001
#define NVB06F_SEMAPHORED_REDUCTION_XOR 0x00000002
#define NVB06F_SEMAPHORED_REDUCTION_AND 0x00000003
#define NVB06F_SEMAPHORED_REDUCTION_OR 0x00000004
#define NVB06F_SEMAPHORED_REDUCTION_ADD 0x00000005
#define NVB06F_SEMAPHORED_REDUCTION_INC 0x00000006
#define NVB06F_SEMAPHORED_REDUCTION_DEC 0x00000007
#define NVB06F_SEMAPHORED_FORMAT 31:31
#define NVB06F_SEMAPHORED_FORMAT_SIGNED 0x00000000
#define NVB06F_SEMAPHORED_FORMAT_UNSIGNED 0x00000001
#define NVB06F_NON_STALL_INTERRUPT (0x00000020)
#define NVB06F_NON_STALL_INTERRUPT_HANDLE 31:0
#define NVB06F_FB_FLUSH (0x00000024)
#define NVB06F_FB_FLUSH_HANDLE 31:0
// NOTE - MEM_OP_A and MEM_OP_B have been removed for gm20x to make room for
// possible future MEM_OP features. MEM_OP_C/D have identical functionality
// to the previous MEM_OP_A/B methods.
#define NVB06F_MEM_OP_C (0x00000030)
#define NVB06F_MEM_OP_C_OPERAND_LOW 31:2
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_PDB 0:0
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE 0x00000000
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL 0x00000001
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_GPC 1:1
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE 0x00000000
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE 0x00000001
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_TARGET 11:10
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_TARGET_VID_MEM 0x00000000
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_TARGET_SYS_MEM_COHERENT 0x00000002
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_TARGET_SYS_MEM_NONCOHERENT 0x00000003
#define NVB06F_MEM_OP_C_TLB_INVALIDATE_ADDR_LO 31:12
#define NVB06F_MEM_OP_D (0x00000034)
#define NVB06F_MEM_OP_D_OPERAND_HIGH 7:0
#define NVB06F_MEM_OP_D_OPERATION 31:27
#define NVB06F_MEM_OP_D_OPERATION_MEMBAR 0x00000005
#define NVB06F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE 0x00000009
#define NVB06F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE 0x0000000d
#define NVB06F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE 0x0000000e
#define NVB06F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS 0x0000000f
#define NVB06F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY 0x00000010
#define NVB06F_MEM_OP_D_TLB_INVALIDATE_ADDR_HI 7:0
#define NVB06F_SET_REFERENCE (0x00000050)
#define NVB06F_SET_REFERENCE_COUNT 31:0
#define NVB06F_WFI (0x00000078)
#define NVB06F_WFI_SCOPE 0:0
#define NVB06F_WFI_SCOPE_CURRENT_SCG_TYPE 0x00000000
#define NVB06F_WFI_SCOPE_ALL 0x00000001
#define NVB06F_CRC_CHECK (0x0000007c)
#define NVB06F_CRC_CHECK_VALUE 31:0
#define NVB06F_YIELD (0x00000080)
#define NVB06F_YIELD_OP 1:0
#define NVB06F_YIELD_OP_NOP 0x00000000
#define NVB06F_YIELD_OP_PBDMA_TIMESLICE 0x00000001
#define NVB06F_YIELD_OP_RUNLIST_TIMESLICE 0x00000002
#define NVB06F_YIELD_OP_TSG 0x00000003
/* GPFIFO entry format */
#define NVB06F_GP_ENTRY__SIZE 8
#define NVB06F_GP_ENTRY0_FETCH 0:0
#define NVB06F_GP_ENTRY0_FETCH_UNCONDITIONAL 0x00000000
#define NVB06F_GP_ENTRY0_FETCH_CONDITIONAL 0x00000001
#define NVB06F_GP_ENTRY0_GET 31:2
#define NVB06F_GP_ENTRY0_OPERAND 31:0
#define NVB06F_GP_ENTRY1_GET_HI 7:0
#define NVB06F_GP_ENTRY1_PRIV 8:8
#define NVB06F_GP_ENTRY1_PRIV_USER 0x00000000
#define NVB06F_GP_ENTRY1_PRIV_KERNEL 0x00000001
#define NVB06F_GP_ENTRY1_LEVEL 9:9
#define NVB06F_GP_ENTRY1_LEVEL_MAIN 0x00000000
#define NVB06F_GP_ENTRY1_LEVEL_SUBROUTINE 0x00000001
#define NVB06F_GP_ENTRY1_LENGTH 30:10
#define NVB06F_GP_ENTRY1_SYNC 31:31
#define NVB06F_GP_ENTRY1_SYNC_PROCEED 0x00000000
#define NVB06F_GP_ENTRY1_SYNC_WAIT 0x00000001
#define NVB06F_GP_ENTRY1_OPCODE 7:0
#define NVB06F_GP_ENTRY1_OPCODE_NOP 0x00000000
#define NVB06F_GP_ENTRY1_OPCODE_ILLEGAL 0x00000001
#define NVB06F_GP_ENTRY1_OPCODE_GP_CRC 0x00000002
#define NVB06F_GP_ENTRY1_OPCODE_PB_CRC 0x00000003
/* dma method formats */
#define NVB06F_DMA_METHOD_ADDRESS_OLD 12:2
#define NVB06F_DMA_METHOD_ADDRESS 11:0
#define NVB06F_DMA_SUBDEVICE_MASK 15:4
#define NVB06F_DMA_METHOD_SUBCHANNEL 15:13
#define NVB06F_DMA_TERT_OP 17:16
#define NVB06F_DMA_TERT_OP_GRP0_INC_METHOD (0x00000000)
#define NVB06F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK (0x00000001)
#define NVB06F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK (0x00000002)
#define NVB06F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK (0x00000003)
#define NVB06F_DMA_TERT_OP_GRP2_NON_INC_METHOD (0x00000000)
#define NVB06F_DMA_METHOD_COUNT_OLD 28:18
#define NVB06F_DMA_METHOD_COUNT 28:16
#define NVB06F_DMA_IMMD_DATA 28:16
#define NVB06F_DMA_SEC_OP 31:29
#define NVB06F_DMA_SEC_OP_GRP0_USE_TERT (0x00000000)
#define NVB06F_DMA_SEC_OP_INC_METHOD (0x00000001)
#define NVB06F_DMA_SEC_OP_GRP2_USE_TERT (0x00000002)
#define NVB06F_DMA_SEC_OP_NON_INC_METHOD (0x00000003)
#define NVB06F_DMA_SEC_OP_IMMD_DATA_METHOD (0x00000004)
#define NVB06F_DMA_SEC_OP_ONE_INC (0x00000005)
#define NVB06F_DMA_SEC_OP_RESERVED6 (0x00000006)
#define NVB06F_DMA_SEC_OP_END_PB_SEGMENT (0x00000007)
/* dma incrementing method format */
#define NVB06F_DMA_INCR_ADDRESS 11:0
#define NVB06F_DMA_INCR_SUBCHANNEL 15:13
#define NVB06F_DMA_INCR_COUNT 28:16
#define NVB06F_DMA_INCR_OPCODE 31:29
#define NVB06F_DMA_INCR_OPCODE_VALUE (0x00000001)
#define NVB06F_DMA_INCR_DATA 31:0
/* dma non-incrementing method format */
#define NVB06F_DMA_NONINCR_ADDRESS 11:0
#define NVB06F_DMA_NONINCR_SUBCHANNEL 15:13
#define NVB06F_DMA_NONINCR_COUNT 28:16
#define NVB06F_DMA_NONINCR_OPCODE 31:29
#define NVB06F_DMA_NONINCR_OPCODE_VALUE (0x00000003)
#define NVB06F_DMA_NONINCR_DATA 31:0
/* dma increment-once method format */
#define NVB06F_DMA_ONEINCR_ADDRESS 11:0
#define NVB06F_DMA_ONEINCR_SUBCHANNEL 15:13
#define NVB06F_DMA_ONEINCR_COUNT 28:16
#define NVB06F_DMA_ONEINCR_OPCODE 31:29
#define NVB06F_DMA_ONEINCR_OPCODE_VALUE (0x00000005)
#define NVB06F_DMA_ONEINCR_DATA 31:0
/* dma no-operation format */
#define NVB06F_DMA_NOP (0x00000000)
/* dma immediate-data format */
#define NVB06F_DMA_IMMD_ADDRESS 11:0
#define NVB06F_DMA_IMMD_SUBCHANNEL 15:13
#define NVB06F_DMA_IMMD_DATA 28:16
#define NVB06F_DMA_IMMD_OPCODE 31:29
#define NVB06F_DMA_IMMD_OPCODE_VALUE (0x00000004)
/* dma set sub-device mask format */
#define NVB06F_DMA_SET_SUBDEVICE_MASK_VALUE 15:4
#define NVB06F_DMA_SET_SUBDEVICE_MASK_OPCODE 31:16
#define NVB06F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE (0x00000001)
/* dma store sub-device mask format */
#define NVB06F_DMA_STORE_SUBDEVICE_MASK_VALUE 15:4
#define NVB06F_DMA_STORE_SUBDEVICE_MASK_OPCODE 31:16
#define NVB06F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000002)
/* dma use sub-device mask format */
#define NVB06F_DMA_USE_SUBDEVICE_MASK_OPCODE 31:16
#define NVB06F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000003)
/* dma end-segment format */
#define NVB06F_DMA_ENDSEG_OPCODE 31:29
#define NVB06F_DMA_ENDSEG_OPCODE_VALUE (0x00000007)
/* dma legacy incrementing/non-incrementing formats */
#define NVB06F_DMA_ADDRESS 12:2
#define NVB06F_DMA_SUBCH 15:13
#define NVB06F_DMA_OPCODE3 17:16
#define NVB06F_DMA_OPCODE3_NONE (0x00000000)
#define NVB06F_DMA_COUNT 28:18
#define NVB06F_DMA_OPCODE 31:29
#define NVB06F_DMA_OPCODE_METHOD (0x00000000)
#define NVB06F_DMA_OPCODE_NONINC_METHOD (0x00000002)
#define NVB06F_DMA_DATA 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif /* _clb06f_h_ */

261
kernel-open/nvidia-uvm/clb0b5.h
View File

@@ -1,261 +0,0 @@
/*******************************************************************************
Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "nvtypes.h"
#ifndef _clb0b5_h_
#define _clb0b5_h_
#ifdef __cplusplus
extern "C" {
#endif
#define MAXWELL_DMA_COPY_A (0x0000B0B5)
#define NVB0B5_NOP (0x00000100)
#define NVB0B5_NOP_PARAMETER 31:0
#define NVB0B5_PM_TRIGGER (0x00000140)
#define NVB0B5_PM_TRIGGER_V 31:0
#define NVB0B5_SET_SEMAPHORE_A (0x00000240)
#define NVB0B5_SET_SEMAPHORE_A_UPPER 7:0
#define NVB0B5_SET_SEMAPHORE_B (0x00000244)
#define NVB0B5_SET_SEMAPHORE_B_LOWER 31:0
#define NVB0B5_SET_SEMAPHORE_PAYLOAD (0x00000248)
#define NVB0B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD 31:0
#define NVB0B5_SET_RENDER_ENABLE_A (0x00000254)
#define NVB0B5_SET_RENDER_ENABLE_A_UPPER 7:0
#define NVB0B5_SET_RENDER_ENABLE_B (0x00000258)
#define NVB0B5_SET_RENDER_ENABLE_B_LOWER 31:0
#define NVB0B5_SET_RENDER_ENABLE_C (0x0000025C)
#define NVB0B5_SET_RENDER_ENABLE_C_MODE 2:0
#define NVB0B5_SET_RENDER_ENABLE_C_MODE_FALSE (0x00000000)
#define NVB0B5_SET_RENDER_ENABLE_C_MODE_TRUE (0x00000001)
#define NVB0B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL (0x00000002)
#define NVB0B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL (0x00000003)
#define NVB0B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL (0x00000004)
#define NVB0B5_SET_SRC_PHYS_MODE (0x00000260)
#define NVB0B5_SET_SRC_PHYS_MODE_TARGET 1:0
#define NVB0B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVB0B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVB0B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVB0B5_SET_DST_PHYS_MODE (0x00000264)
#define NVB0B5_SET_DST_PHYS_MODE_TARGET 1:0
#define NVB0B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVB0B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVB0B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVB0B5_LAUNCH_DMA (0x00000300)
#define NVB0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE 1:0
#define NVB0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE (0x00000000)
#define NVB0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED (0x00000001)
#define NVB0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED (0x00000002)
#define NVB0B5_LAUNCH_DMA_FLUSH_ENABLE 2:2
#define NVB0B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE (0x00000000)
#define NVB0B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE (0x00000001)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_TYPE 4:3
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE (0x00000000)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE (0x00000001)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE (0x00000002)
#define NVB0B5_LAUNCH_DMA_INTERRUPT_TYPE 6:5
#define NVB0B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE (0x00000000)
#define NVB0B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING (0x00000001)
#define NVB0B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING (0x00000002)
#define NVB0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT 7:7
#define NVB0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVB0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVB0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT 8:8
#define NVB0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVB0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVB0B5_LAUNCH_DMA_MULTI_LINE_ENABLE 9:9
#define NVB0B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE (0x00000000)
#define NVB0B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE (0x00000001)
#define NVB0B5_LAUNCH_DMA_REMAP_ENABLE 10:10
#define NVB0B5_LAUNCH_DMA_REMAP_ENABLE_FALSE (0x00000000)
#define NVB0B5_LAUNCH_DMA_REMAP_ENABLE_TRUE (0x00000001)
#define NVB0B5_LAUNCH_DMA_FORCE_RMWDISABLE 11:11
#define NVB0B5_LAUNCH_DMA_FORCE_RMWDISABLE_FALSE (0x00000000)
#define NVB0B5_LAUNCH_DMA_FORCE_RMWDISABLE_TRUE (0x00000001)
#define NVB0B5_LAUNCH_DMA_SRC_TYPE 12:12
#define NVB0B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL (0x00000000)
#define NVB0B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL (0x00000001)
#define NVB0B5_LAUNCH_DMA_DST_TYPE 13:13
#define NVB0B5_LAUNCH_DMA_DST_TYPE_VIRTUAL (0x00000000)
#define NVB0B5_LAUNCH_DMA_DST_TYPE_PHYSICAL (0x00000001)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION 17:14
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN (0x00000000)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX (0x00000001)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR (0x00000002)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND (0x00000003)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR (0x00000004)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD (0x00000005)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC (0x00000006)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC (0x00000007)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD (0x0000000A)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN 18:18
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED (0x00000000)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED (0x00000001)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE 19:19
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE (0x00000000)
#define NVB0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE (0x00000001)
#define NVB0B5_LAUNCH_DMA_BYPASS_L2 20:20
#define NVB0B5_LAUNCH_DMA_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVB0B5_LAUNCH_DMA_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVB0B5_OFFSET_IN_UPPER (0x00000400)
#define NVB0B5_OFFSET_IN_UPPER_UPPER 7:0
#define NVB0B5_OFFSET_IN_LOWER (0x00000404)
#define NVB0B5_OFFSET_IN_LOWER_VALUE 31:0
#define NVB0B5_OFFSET_OUT_UPPER (0x00000408)
#define NVB0B5_OFFSET_OUT_UPPER_UPPER 7:0
#define NVB0B5_OFFSET_OUT_LOWER (0x0000040C)
#define NVB0B5_OFFSET_OUT_LOWER_VALUE 31:0
#define NVB0B5_PITCH_IN (0x00000410)
#define NVB0B5_PITCH_IN_VALUE 31:0
#define NVB0B5_PITCH_OUT (0x00000414)
#define NVB0B5_PITCH_OUT_VALUE 31:0
#define NVB0B5_LINE_LENGTH_IN (0x00000418)
#define NVB0B5_LINE_LENGTH_IN_VALUE 31:0
#define NVB0B5_LINE_COUNT (0x0000041C)
#define NVB0B5_LINE_COUNT_VALUE 31:0
#define NVB0B5_SET_REMAP_CONST_A (0x00000700)
#define NVB0B5_SET_REMAP_CONST_A_V 31:0
#define NVB0B5_SET_REMAP_CONST_B (0x00000704)
#define NVB0B5_SET_REMAP_CONST_B_V 31:0
#define NVB0B5_SET_REMAP_COMPONENTS (0x00000708)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X 2:0
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_SRC_X (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_SRC_W (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_CONST_A (0x00000004)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_CONST_B (0x00000005)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE (0x00000006)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y 6:4
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A (0x00000004)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B (0x00000005)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE (0x00000006)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z 10:8
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A (0x00000004)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B (0x00000005)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE (0x00000006)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W 14:12
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_SRC_X (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_SRC_W (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_CONST_A (0x00000004)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_CONST_B (0x00000005)
#define NVB0B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE (0x00000006)
#define NVB0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE 17:16
#define NVB0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS 21:20
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR (0x00000003)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS 25:24
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE (0x00000000)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO (0x00000001)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE (0x00000002)
#define NVB0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR (0x00000003)
#define NVB0B5_SET_DST_BLOCK_SIZE (0x0000070C)
#define NVB0B5_SET_DST_BLOCK_SIZE_WIDTH 3:0
#define NVB0B5_SET_DST_BLOCK_SIZE_WIDTH_QUARTER_GOB (0x0000000E)
#define NVB0B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT 7:4
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVB0B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH 11:8
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVB0B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVB0B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVB0B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_TESLA_4 (0x00000000)
#define NVB0B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVB0B5_SET_DST_WIDTH (0x00000710)
#define NVB0B5_SET_DST_WIDTH_V 31:0
#define NVB0B5_SET_DST_HEIGHT (0x00000714)
#define NVB0B5_SET_DST_HEIGHT_V 31:0
#define NVB0B5_SET_DST_DEPTH (0x00000718)
#define NVB0B5_SET_DST_DEPTH_V 31:0
#define NVB0B5_SET_DST_LAYER (0x0000071C)
#define NVB0B5_SET_DST_LAYER_V 31:0
#define NVB0B5_SET_DST_ORIGIN (0x00000720)
#define NVB0B5_SET_DST_ORIGIN_X 15:0
#define NVB0B5_SET_DST_ORIGIN_Y 31:16
#define NVB0B5_SET_SRC_BLOCK_SIZE (0x00000728)
#define NVB0B5_SET_SRC_BLOCK_SIZE_WIDTH 3:0
#define NVB0B5_SET_SRC_BLOCK_SIZE_WIDTH_QUARTER_GOB (0x0000000E)
#define NVB0B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT 7:4
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVB0B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH 11:8
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVB0B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVB0B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVB0B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_TESLA_4 (0x00000000)
#define NVB0B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVB0B5_SET_SRC_WIDTH (0x0000072C)
#define NVB0B5_SET_SRC_WIDTH_V 31:0
#define NVB0B5_SET_SRC_HEIGHT (0x00000730)
#define NVB0B5_SET_SRC_HEIGHT_V 31:0
#define NVB0B5_SET_SRC_DEPTH (0x00000734)
#define NVB0B5_SET_SRC_DEPTH_V 31:0
#define NVB0B5_SET_SRC_LAYER (0x00000738)
#define NVB0B5_SET_SRC_LAYER_V 31:0
#define NVB0B5_SET_SRC_ORIGIN (0x0000073C)
#define NVB0B5_SET_SRC_ORIGIN_X 15:0
#define NVB0B5_SET_SRC_ORIGIN_Y 31:16
#define NVB0B5_PM_TRIGGER_END (0x00001114)
#define NVB0B5_PM_TRIGGER_END_V 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif // _clb0b5_h

312
kernel-open/nvidia-uvm/clc06f.h
View File

@@ -1,312 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _clc06f_h_
#define _clc06f_h_
#ifdef __cplusplus
extern "C" {
#endif
#include "nvtypes.h"
/* class PASCAL_CHANNEL_GPFIFO */
/*
* Documentation for PASCAL_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
* chapter "User Control Registers". It is documented as device NV_UDMA.
* The GPFIFO format itself is also documented in dev_pbdma.ref,
* NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
* chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
*
* Note there is no .mfs file for this class.
*/
#define PASCAL_CHANNEL_GPFIFO_A (0x0000C06F)
#define NVC06F_TYPEDEF PASCAL_CHANNELChannelGPFifoA
/* dma flow control data structure */
typedef volatile struct Nvc06fControl_struct {
NvU32 Ignored00[0x010]; /* 0000-003f*/
NvU32 Put; /* put offset, read/write 0040-0043*/
NvU32 Get; /* get offset, read only 0044-0047*/
NvU32 Reference; /* reference value, read only 0048-004b*/
NvU32 PutHi; /* high order put offset bits 004c-004f*/
NvU32 Ignored01[0x002]; /* 0050-0057*/
NvU32 TopLevelGet; /* top level get offset, read only 0058-005b*/
NvU32 TopLevelGetHi; /* high order top level get bits 005c-005f*/
NvU32 GetHi; /* high order get offset bits 0060-0063*/
NvU32 Ignored02[0x007]; /* 0064-007f*/
NvU32 Ignored03; /* used to be engine yield 0080-0083*/
NvU32 Ignored04[0x001]; /* 0084-0087*/
NvU32 GPGet; /* GP FIFO get offset, read only 0088-008b*/
NvU32 GPPut; /* GP FIFO put offset 008c-008f*/
NvU32 Ignored05[0x5c];
} Nvc06fControl, PascalAControlGPFifo;
/* fields and values */
#define NVC06F_NUMBER_OF_SUBCHANNELS (8)
#define NVC06F_SET_OBJECT (0x00000000)
#define NVC06F_SET_OBJECT_NVCLASS 15:0
#define NVC06F_SET_OBJECT_ENGINE 20:16
#define NVC06F_SET_OBJECT_ENGINE_SW 0x0000001f
#define NVC06F_ILLEGAL (0x00000004)
#define NVC06F_ILLEGAL_HANDLE 31:0
#define NVC06F_NOP (0x00000008)
#define NVC06F_NOP_HANDLE 31:0
#define NVC06F_SEMAPHOREA (0x00000010)
#define NVC06F_SEMAPHOREA_OFFSET_UPPER 7:0
#define NVC06F_SEMAPHOREB (0x00000014)
#define NVC06F_SEMAPHOREB_OFFSET_LOWER 31:2
#define NVC06F_SEMAPHOREC (0x00000018)
#define NVC06F_SEMAPHOREC_PAYLOAD 31:0
#define NVC06F_SEMAPHORED (0x0000001C)
#define NVC06F_SEMAPHORED_OPERATION 4:0
#define NVC06F_SEMAPHORED_OPERATION_ACQUIRE 0x00000001
#define NVC06F_SEMAPHORED_OPERATION_RELEASE 0x00000002
#define NVC06F_SEMAPHORED_OPERATION_ACQ_GEQ 0x00000004
#define NVC06F_SEMAPHORED_OPERATION_ACQ_AND 0x00000008
#define NVC06F_SEMAPHORED_OPERATION_REDUCTION 0x00000010
#define NVC06F_SEMAPHORED_ACQUIRE_SWITCH 12:12
#define NVC06F_SEMAPHORED_ACQUIRE_SWITCH_DISABLED 0x00000000
#define NVC06F_SEMAPHORED_ACQUIRE_SWITCH_ENABLED 0x00000001
#define NVC06F_SEMAPHORED_RELEASE_WFI 20:20
#define NVC06F_SEMAPHORED_RELEASE_WFI_EN 0x00000000
#define NVC06F_SEMAPHORED_RELEASE_WFI_DIS 0x00000001
#define NVC06F_SEMAPHORED_RELEASE_SIZE 24:24
#define NVC06F_SEMAPHORED_RELEASE_SIZE_16BYTE 0x00000000
#define NVC06F_SEMAPHORED_RELEASE_SIZE_4BYTE 0x00000001
#define NVC06F_SEMAPHORED_REDUCTION 30:27
#define NVC06F_SEMAPHORED_REDUCTION_MIN 0x00000000
#define NVC06F_SEMAPHORED_REDUCTION_MAX 0x00000001
#define NVC06F_SEMAPHORED_REDUCTION_XOR 0x00000002
#define NVC06F_SEMAPHORED_REDUCTION_AND 0x00000003
#define NVC06F_SEMAPHORED_REDUCTION_OR 0x00000004
#define NVC06F_SEMAPHORED_REDUCTION_ADD 0x00000005
#define NVC06F_SEMAPHORED_REDUCTION_INC 0x00000006
#define NVC06F_SEMAPHORED_REDUCTION_DEC 0x00000007
#define NVC06F_SEMAPHORED_FORMAT 31:31
#define NVC06F_SEMAPHORED_FORMAT_SIGNED 0x00000000
#define NVC06F_SEMAPHORED_FORMAT_UNSIGNED 0x00000001
#define NVC06F_NON_STALL_INTERRUPT (0x00000020)
#define NVC06F_NON_STALL_INTERRUPT_HANDLE 31:0
#define NVC06F_FB_FLUSH (0x00000024) // Deprecated - use MEMBAR TYPE SYS_MEMBAR
#define NVC06F_FB_FLUSH_HANDLE 31:0
// NOTE - MEM_OP_A and MEM_OP_B have been replaced in gp100 with methods for
// specifying the page address for a targeted TLB invalidate and the uTLB for
// a targeted REPLAY_CANCEL for UVM.
// The previous MEM_OP_A/B functionality is in MEM_OP_C/D, with slightly
// rearranged fields.
#define NVC06F_MEM_OP_A (0x00000028)
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID 5:0 // only relevant for REPLAY_CANCEL_TARGETED
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID 10:6 // only relevant for REPLAY_CANCEL_TARGETED
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR 11:11
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN 0x00000001
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS 0x00000000
#define NVC06F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO 31:12
#define NVC06F_MEM_OP_B (0x0000002c)
#define NVC06F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI 31:0
#define NVC06F_MEM_OP_C (0x00000030)
#define NVC06F_MEM_OP_C_MEMBAR_TYPE 2:0
#define NVC06F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR 0x00000000
#define NVC06F_MEM_OP_C_MEMBAR_TYPE_MEMBAR 0x00000001
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB 0:0
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE 0x00000000
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL 0x00000001 // Probably nonsensical for MMU_TLB_INVALIDATE_TARGETED
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_GPC 1:1
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE 0x00000000
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE 0x00000001
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY 4:2 // only relevant if GPC ENABLE
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE 0x00000000
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START 0x00000001
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL 0x00000002
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED 0x00000003
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL 0x00000004
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE 6:5 // only relevant if GPC ENABLE
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE 0x00000000
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY 0x00000001
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE 0x00000002
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL 9:7 // Invalidate affects this level and all below
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL 0x00000000 // Invalidate tlb caches at all levels of the page table
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY 0x00000001
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0 0x00000002
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1 0x00000003
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2 0x00000004
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 0x00000005
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 0x00000006
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5 0x00000007
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE 11:10 // only relevant if PDB_ONE
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM 0x00000000
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT 0x00000002
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT 0x00000003
#define NVC06F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO 31:12 // only relevant if PDB_ONE
// MEM_OP_D MUST be preceded by MEM_OPs A-C.
#define NVC06F_MEM_OP_D (0x00000034)
#define NVC06F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI 26:0 // only relevant if PDB_ONE
#define NVC06F_MEM_OP_D_OPERATION 31:27
#define NVC06F_MEM_OP_D_OPERATION_MEMBAR 0x00000005
#define NVC06F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE 0x00000009
#define NVC06F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED 0x0000000a
#define NVC06F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE 0x0000000d
#define NVC06F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE 0x0000000e
// CLEAN_LINES is an alias for Tegra/GPU IP usage
#define NVC06F_MEM_OP_D_OPERATION_L2_INVALIDATE_CLEAN_LINES 0x0000000e
// This B alias is confusing but it was missed as part of the update. Left here
// for compatibility.
#define NVC06F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES 0x0000000e
#define NVC06F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS 0x0000000f
#define NVC06F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY 0x00000010
#define NVC06F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS 0x00000015
#define NVC06F_SET_REFERENCE (0x00000050)
#define NVC06F_SET_REFERENCE_COUNT 31:0
// Syncpoint methods are only available on Tegra parts. Attempting to use
// them on discrete GPUs will result in Host raising NV_PPBDMA_INTR_0_METHOD.
#define NVC06F_SYNCPOINTA (0x00000070)
#define NVC06F_SYNCPOINTA_PAYLOAD 31:0
#define NVC06F_SYNCPOINTB (0x00000074)
#define NVC06F_SYNCPOINTB_OPERATION 0:0
#define NVC06F_SYNCPOINTB_OPERATION_WAIT 0x00000000
#define NVC06F_SYNCPOINTB_OPERATION_INCR 0x00000001
#define NVC06F_SYNCPOINTB_WAIT_SWITCH 4:4
#define NVC06F_SYNCPOINTB_WAIT_SWITCH_DIS 0x00000000
#define NVC06F_SYNCPOINTB_WAIT_SWITCH_EN 0x00000001
#define NVC06F_SYNCPOINTB_SYNCPT_INDEX 19:8
#define NVC06F_WFI (0x00000078)
#define NVC06F_WFI_SCOPE 0:0
#define NVC06F_WFI_SCOPE_CURRENT_SCG_TYPE 0x00000000
#define NVC06F_WFI_SCOPE_ALL 0x00000001
#define NVC06F_CRC_CHECK (0x0000007c)
#define NVC06F_CRC_CHECK_VALUE 31:0
#define NVC06F_YIELD (0x00000080)
#define NVC06F_YIELD_OP 1:0
#define NVC06F_YIELD_OP_NOP 0x00000000
#define NVC06F_YIELD_OP_PBDMA_TIMESLICE 0x00000001
#define NVC06F_YIELD_OP_RUNLIST_TIMESLICE 0x00000002
#define NVC06F_YIELD_OP_TSG 0x00000003
/* GPFIFO entry format */
#define NVC06F_GP_ENTRY__SIZE 8
#define NVC06F_GP_ENTRY0_FETCH 0:0
#define NVC06F_GP_ENTRY0_FETCH_UNCONDITIONAL 0x00000000
#define NVC06F_GP_ENTRY0_FETCH_CONDITIONAL 0x00000001
#define NVC06F_GP_ENTRY0_GET 31:2
#define NVC06F_GP_ENTRY0_OPERAND 31:0
#define NVC06F_GP_ENTRY1_GET_HI 7:0
#define NVC06F_GP_ENTRY1_PRIV 8:8
#define NVC06F_GP_ENTRY1_PRIV_USER 0x00000000
#define NVC06F_GP_ENTRY1_PRIV_KERNEL 0x00000001
#define NVC06F_GP_ENTRY1_LEVEL 9:9
#define NVC06F_GP_ENTRY1_LEVEL_MAIN 0x00000000
#define NVC06F_GP_ENTRY1_LEVEL_SUBROUTINE 0x00000001
#define NVC06F_GP_ENTRY1_LENGTH 30:10
#define NVC06F_GP_ENTRY1_SYNC 31:31
#define NVC06F_GP_ENTRY1_SYNC_PROCEED 0x00000000
#define NVC06F_GP_ENTRY1_SYNC_WAIT 0x00000001
#define NVC06F_GP_ENTRY1_OPCODE 7:0
#define NVC06F_GP_ENTRY1_OPCODE_NOP 0x00000000
#define NVC06F_GP_ENTRY1_OPCODE_ILLEGAL 0x00000001
#define NVC06F_GP_ENTRY1_OPCODE_GP_CRC 0x00000002
#define NVC06F_GP_ENTRY1_OPCODE_PB_CRC 0x00000003
/* dma method formats */
#define NVC06F_DMA_METHOD_ADDRESS_OLD 12:2
#define NVC06F_DMA_METHOD_ADDRESS 11:0
#define NVC06F_DMA_SUBDEVICE_MASK 15:4
#define NVC06F_DMA_METHOD_SUBCHANNEL 15:13
#define NVC06F_DMA_TERT_OP 17:16
#define NVC06F_DMA_TERT_OP_GRP0_INC_METHOD (0x00000000)
#define NVC06F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK (0x00000001)
#define NVC06F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK (0x00000002)
#define NVC06F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK (0x00000003)
#define NVC06F_DMA_TERT_OP_GRP2_NON_INC_METHOD (0x00000000)
#define NVC06F_DMA_METHOD_COUNT_OLD 28:18
#define NVC06F_DMA_METHOD_COUNT 28:16
#define NVC06F_DMA_IMMD_DATA 28:16
#define NVC06F_DMA_SEC_OP 31:29
#define NVC06F_DMA_SEC_OP_GRP0_USE_TERT (0x00000000)
#define NVC06F_DMA_SEC_OP_INC_METHOD (0x00000001)
#define NVC06F_DMA_SEC_OP_GRP2_USE_TERT (0x00000002)
#define NVC06F_DMA_SEC_OP_NON_INC_METHOD (0x00000003)
#define NVC06F_DMA_SEC_OP_IMMD_DATA_METHOD (0x00000004)
#define NVC06F_DMA_SEC_OP_ONE_INC (0x00000005)
#define NVC06F_DMA_SEC_OP_RESERVED6 (0x00000006)
#define NVC06F_DMA_SEC_OP_END_PB_SEGMENT (0x00000007)
/* dma incrementing method format */
#define NVC06F_DMA_INCR_ADDRESS 11:0
#define NVC06F_DMA_INCR_SUBCHANNEL 15:13
#define NVC06F_DMA_INCR_COUNT 28:16
#define NVC06F_DMA_INCR_OPCODE 31:29
#define NVC06F_DMA_INCR_OPCODE_VALUE (0x00000001)
#define NVC06F_DMA_INCR_DATA 31:0
/* dma non-incrementing method format */
#define NVC06F_DMA_NONINCR_ADDRESS 11:0
#define NVC06F_DMA_NONINCR_SUBCHANNEL 15:13
#define NVC06F_DMA_NONINCR_COUNT 28:16
#define NVC06F_DMA_NONINCR_OPCODE 31:29
#define NVC06F_DMA_NONINCR_OPCODE_VALUE (0x00000003)
#define NVC06F_DMA_NONINCR_DATA 31:0
/* dma increment-once method format */
#define NVC06F_DMA_ONEINCR_ADDRESS 11:0
#define NVC06F_DMA_ONEINCR_SUBCHANNEL 15:13
#define NVC06F_DMA_ONEINCR_COUNT 28:16
#define NVC06F_DMA_ONEINCR_OPCODE 31:29
#define NVC06F_DMA_ONEINCR_OPCODE_VALUE (0x00000005)
#define NVC06F_DMA_ONEINCR_DATA 31:0
/* dma no-operation format */
#define NVC06F_DMA_NOP (0x00000000)
/* dma immediate-data format */
#define NVC06F_DMA_IMMD_ADDRESS 11:0
#define NVC06F_DMA_IMMD_SUBCHANNEL 15:13
#define NVC06F_DMA_IMMD_DATA 28:16
#define NVC06F_DMA_IMMD_OPCODE 31:29
#define NVC06F_DMA_IMMD_OPCODE_VALUE (0x00000004)
/* dma set sub-device mask format */
#define NVC06F_DMA_SET_SUBDEVICE_MASK_VALUE 15:4
#define NVC06F_DMA_SET_SUBDEVICE_MASK_OPCODE 31:16
#define NVC06F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE (0x00000001)
/* dma store sub-device mask format */
#define NVC06F_DMA_STORE_SUBDEVICE_MASK_VALUE 15:4
#define NVC06F_DMA_STORE_SUBDEVICE_MASK_OPCODE 31:16
#define NVC06F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000002)
/* dma use sub-device mask format */
#define NVC06F_DMA_USE_SUBDEVICE_MASK_OPCODE 31:16
#define NVC06F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000003)
/* dma end-segment format */
#define NVC06F_DMA_ENDSEG_OPCODE 31:29
#define NVC06F_DMA_ENDSEG_OPCODE_VALUE (0x00000007)
/* dma legacy incrementing/non-incrementing formats */
#define NVC06F_DMA_ADDRESS 12:2
#define NVC06F_DMA_SUBCH 15:13
#define NVC06F_DMA_OPCODE3 17:16
#define NVC06F_DMA_OPCODE3_NONE (0x00000000)
#define NVC06F_DMA_COUNT 28:18
#define NVC06F_DMA_OPCODE 31:29
#define NVC06F_DMA_OPCODE_METHOD (0x00000000)
#define NVC06F_DMA_OPCODE_NONINC_METHOD (0x00000002)
#define NVC06F_DMA_DATA 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif /* _clc06f_h_ */

261
kernel-open/nvidia-uvm/clc0b5.h
View File

@@ -1,261 +0,0 @@
/*******************************************************************************
Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "nvtypes.h"
#ifndef _clc0b5_h_
#define _clc0b5_h_
#ifdef __cplusplus
extern "C" {
#endif
#define PASCAL_DMA_COPY_A (0x0000C0B5)
#define NVC0B5_NOP (0x00000100)
#define NVC0B5_NOP_PARAMETER 31:0
#define NVC0B5_PM_TRIGGER (0x00000140)
#define NVC0B5_PM_TRIGGER_V 31:0
#define NVC0B5_SET_SEMAPHORE_A (0x00000240)
#define NVC0B5_SET_SEMAPHORE_A_UPPER 16:0
#define NVC0B5_SET_SEMAPHORE_B (0x00000244)
#define NVC0B5_SET_SEMAPHORE_B_LOWER 31:0
#define NVC0B5_SET_SEMAPHORE_PAYLOAD (0x00000248)
#define NVC0B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD 31:0
#define NVC0B5_SET_RENDER_ENABLE_A (0x00000254)
#define NVC0B5_SET_RENDER_ENABLE_A_UPPER 7:0
#define NVC0B5_SET_RENDER_ENABLE_B (0x00000258)
#define NVC0B5_SET_RENDER_ENABLE_B_LOWER 31:0
#define NVC0B5_SET_RENDER_ENABLE_C (0x0000025C)
#define NVC0B5_SET_RENDER_ENABLE_C_MODE 2:0
#define NVC0B5_SET_RENDER_ENABLE_C_MODE_FALSE (0x00000000)
#define NVC0B5_SET_RENDER_ENABLE_C_MODE_TRUE (0x00000001)
#define NVC0B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL (0x00000002)
#define NVC0B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL (0x00000003)
#define NVC0B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL (0x00000004)
#define NVC0B5_SET_SRC_PHYS_MODE (0x00000260)
#define NVC0B5_SET_SRC_PHYS_MODE_TARGET 1:0
#define NVC0B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC0B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC0B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC0B5_SET_DST_PHYS_MODE (0x00000264)
#define NVC0B5_SET_DST_PHYS_MODE_TARGET 1:0
#define NVC0B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC0B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC0B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC0B5_LAUNCH_DMA (0x00000300)
#define NVC0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE 1:0
#define NVC0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE (0x00000000)
#define NVC0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED (0x00000001)
#define NVC0B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED (0x00000002)
#define NVC0B5_LAUNCH_DMA_FLUSH_ENABLE 2:2
#define NVC0B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE (0x00000000)
#define NVC0B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE (0x00000001)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_TYPE 4:3
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE (0x00000000)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE (0x00000001)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE (0x00000002)
#define NVC0B5_LAUNCH_DMA_INTERRUPT_TYPE 6:5
#define NVC0B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE (0x00000000)
#define NVC0B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING (0x00000001)
#define NVC0B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING (0x00000002)
#define NVC0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT 7:7
#define NVC0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC0B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT 8:8
#define NVC0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC0B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC0B5_LAUNCH_DMA_MULTI_LINE_ENABLE 9:9
#define NVC0B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE (0x00000000)
#define NVC0B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE (0x00000001)
#define NVC0B5_LAUNCH_DMA_REMAP_ENABLE 10:10
#define NVC0B5_LAUNCH_DMA_REMAP_ENABLE_FALSE (0x00000000)
#define NVC0B5_LAUNCH_DMA_REMAP_ENABLE_TRUE (0x00000001)
#define NVC0B5_LAUNCH_DMA_FORCE_RMWDISABLE 11:11
#define NVC0B5_LAUNCH_DMA_FORCE_RMWDISABLE_FALSE (0x00000000)
#define NVC0B5_LAUNCH_DMA_FORCE_RMWDISABLE_TRUE (0x00000001)
#define NVC0B5_LAUNCH_DMA_SRC_TYPE 12:12
#define NVC0B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL (0x00000000)
#define NVC0B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL (0x00000001)
#define NVC0B5_LAUNCH_DMA_DST_TYPE 13:13
#define NVC0B5_LAUNCH_DMA_DST_TYPE_VIRTUAL (0x00000000)
#define NVC0B5_LAUNCH_DMA_DST_TYPE_PHYSICAL (0x00000001)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION 17:14
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN (0x00000000)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX (0x00000001)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR (0x00000002)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND (0x00000003)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR (0x00000004)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD (0x00000005)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC (0x00000006)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC (0x00000007)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD (0x0000000A)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN 18:18
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED (0x00000000)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED (0x00000001)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE 19:19
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE (0x00000000)
#define NVC0B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE (0x00000001)
#define NVC0B5_LAUNCH_DMA_SRC_BYPASS_L2 20:20
#define NVC0B5_LAUNCH_DMA_SRC_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC0B5_LAUNCH_DMA_SRC_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC0B5_LAUNCH_DMA_DST_BYPASS_L2 21:21
#define NVC0B5_LAUNCH_DMA_DST_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC0B5_LAUNCH_DMA_DST_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC0B5_LAUNCH_DMA_RESERVED 31:28
#define NVC0B5_OFFSET_IN_UPPER (0x00000400)
#define NVC0B5_OFFSET_IN_UPPER_UPPER 16:0
#define NVC0B5_OFFSET_IN_LOWER (0x00000404)
#define NVC0B5_OFFSET_IN_LOWER_VALUE 31:0
#define NVC0B5_OFFSET_OUT_UPPER (0x00000408)
#define NVC0B5_OFFSET_OUT_UPPER_UPPER 16:0
#define NVC0B5_OFFSET_OUT_LOWER (0x0000040C)
#define NVC0B5_OFFSET_OUT_LOWER_VALUE 31:0
#define NVC0B5_PITCH_IN (0x00000410)
#define NVC0B5_PITCH_IN_VALUE 31:0
#define NVC0B5_PITCH_OUT (0x00000414)
#define NVC0B5_PITCH_OUT_VALUE 31:0
#define NVC0B5_LINE_LENGTH_IN (0x00000418)
#define NVC0B5_LINE_LENGTH_IN_VALUE 31:0
#define NVC0B5_LINE_COUNT (0x0000041C)
#define NVC0B5_LINE_COUNT_VALUE 31:0
#define NVC0B5_SET_REMAP_CONST_A (0x00000700)
#define NVC0B5_SET_REMAP_CONST_A_V 31:0
#define NVC0B5_SET_REMAP_CONST_B (0x00000704)
#define NVC0B5_SET_REMAP_CONST_B_V 31:0
#define NVC0B5_SET_REMAP_COMPONENTS (0x00000708)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X 2:0
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_SRC_X (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_SRC_W (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_CONST_A (0x00000004)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_CONST_B (0x00000005)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE (0x00000006)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y 6:4
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A (0x00000004)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B (0x00000005)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE (0x00000006)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z 10:8
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A (0x00000004)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B (0x00000005)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE (0x00000006)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W 14:12
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_SRC_X (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_SRC_W (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_CONST_A (0x00000004)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_CONST_B (0x00000005)
#define NVC0B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE (0x00000006)
#define NVC0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE 17:16
#define NVC0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS 21:20
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR (0x00000003)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS 25:24
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE (0x00000000)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO (0x00000001)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE (0x00000002)
#define NVC0B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR (0x00000003)
#define NVC0B5_SET_DST_BLOCK_SIZE (0x0000070C)
#define NVC0B5_SET_DST_BLOCK_SIZE_WIDTH 3:0
#define NVC0B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT 7:4
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC0B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH 11:8
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC0B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC0B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC0B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC0B5_SET_DST_WIDTH (0x00000710)
#define NVC0B5_SET_DST_WIDTH_V 31:0
#define NVC0B5_SET_DST_HEIGHT (0x00000714)
#define NVC0B5_SET_DST_HEIGHT_V 31:0
#define NVC0B5_SET_DST_DEPTH (0x00000718)
#define NVC0B5_SET_DST_DEPTH_V 31:0
#define NVC0B5_SET_DST_LAYER (0x0000071C)
#define NVC0B5_SET_DST_LAYER_V 31:0
#define NVC0B5_SET_DST_ORIGIN (0x00000720)
#define NVC0B5_SET_DST_ORIGIN_X 15:0
#define NVC0B5_SET_DST_ORIGIN_Y 31:16
#define NVC0B5_SET_SRC_BLOCK_SIZE (0x00000728)
#define NVC0B5_SET_SRC_BLOCK_SIZE_WIDTH 3:0
#define NVC0B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT 7:4
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC0B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH 11:8
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC0B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC0B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC0B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC0B5_SET_SRC_WIDTH (0x0000072C)
#define NVC0B5_SET_SRC_WIDTH_V 31:0
#define NVC0B5_SET_SRC_HEIGHT (0x00000730)
#define NVC0B5_SET_SRC_HEIGHT_V 31:0
#define NVC0B5_SET_SRC_DEPTH (0x00000734)
#define NVC0B5_SET_SRC_DEPTH_V 31:0
#define NVC0B5_SET_SRC_LAYER (0x00000738)
#define NVC0B5_SET_SRC_LAYER_V 31:0
#define NVC0B5_SET_SRC_ORIGIN (0x0000073C)
#define NVC0B5_SET_SRC_ORIGIN_X 15:0
#define NVC0B5_SET_SRC_ORIGIN_Y 31:16
#define NVC0B5_PM_TRIGGER_END (0x00001114)
#define NVC0B5_PM_TRIGGER_END_V 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif // _clc0b5_h

273
kernel-open/nvidia-uvm/clc1b5.h
View File

@@ -1,273 +0,0 @@
/*******************************************************************************
Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "nvtypes.h"
#ifndef _clc1b5_h_
#define _clc1b5_h_
#ifdef __cplusplus
extern "C" {
#endif
#define PASCAL_DMA_COPY_B (0x0000C1B5)
#define NVC1B5_NOP (0x00000100)
#define NVC1B5_NOP_PARAMETER 31:0
#define NVC1B5_PM_TRIGGER (0x00000140)
#define NVC1B5_PM_TRIGGER_V 31:0
#define NVC1B5_SET_SEMAPHORE_A (0x00000240)
#define NVC1B5_SET_SEMAPHORE_A_UPPER 16:0
#define NVC1B5_SET_SEMAPHORE_B (0x00000244)
#define NVC1B5_SET_SEMAPHORE_B_LOWER 31:0
#define NVC1B5_SET_SEMAPHORE_PAYLOAD (0x00000248)
#define NVC1B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD 31:0
#define NVC1B5_SET_RENDER_ENABLE_A (0x00000254)
#define NVC1B5_SET_RENDER_ENABLE_A_UPPER 7:0
#define NVC1B5_SET_RENDER_ENABLE_B (0x00000258)
#define NVC1B5_SET_RENDER_ENABLE_B_LOWER 31:0
#define NVC1B5_SET_RENDER_ENABLE_C (0x0000025C)
#define NVC1B5_SET_RENDER_ENABLE_C_MODE 2:0
#define NVC1B5_SET_RENDER_ENABLE_C_MODE_FALSE (0x00000000)
#define NVC1B5_SET_RENDER_ENABLE_C_MODE_TRUE (0x00000001)
#define NVC1B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL (0x00000002)
#define NVC1B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL (0x00000003)
#define NVC1B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL (0x00000004)
#define NVC1B5_SET_SRC_PHYS_MODE (0x00000260)
#define NVC1B5_SET_SRC_PHYS_MODE_TARGET 1:0
#define NVC1B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC1B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC1B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC1B5_SET_DST_PHYS_MODE (0x00000264)
#define NVC1B5_SET_DST_PHYS_MODE_TARGET 1:0
#define NVC1B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC1B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC1B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC1B5_LAUNCH_DMA (0x00000300)
#define NVC1B5_LAUNCH_DMA_DATA_TRANSFER_TYPE 1:0
#define NVC1B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE (0x00000000)
#define NVC1B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED (0x00000001)
#define NVC1B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED (0x00000002)
#define NVC1B5_LAUNCH_DMA_FLUSH_ENABLE 2:2
#define NVC1B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE (0x00000000)
#define NVC1B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE (0x00000001)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_TYPE 4:3
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE (0x00000000)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE (0x00000001)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE (0x00000002)
#define NVC1B5_LAUNCH_DMA_INTERRUPT_TYPE 6:5
#define NVC1B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE (0x00000000)
#define NVC1B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING (0x00000001)
#define NVC1B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING (0x00000002)
#define NVC1B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT 7:7
#define NVC1B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC1B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC1B5_LAUNCH_DMA_DST_MEMORY_LAYOUT 8:8
#define NVC1B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC1B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC1B5_LAUNCH_DMA_MULTI_LINE_ENABLE 9:9
#define NVC1B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE (0x00000000)
#define NVC1B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE (0x00000001)
#define NVC1B5_LAUNCH_DMA_REMAP_ENABLE 10:10
#define NVC1B5_LAUNCH_DMA_REMAP_ENABLE_FALSE (0x00000000)
#define NVC1B5_LAUNCH_DMA_REMAP_ENABLE_TRUE (0x00000001)
#define NVC1B5_LAUNCH_DMA_FORCE_RMWDISABLE 11:11
#define NVC1B5_LAUNCH_DMA_FORCE_RMWDISABLE_FALSE (0x00000000)
#define NVC1B5_LAUNCH_DMA_FORCE_RMWDISABLE_TRUE (0x00000001)
#define NVC1B5_LAUNCH_DMA_SRC_TYPE 12:12
#define NVC1B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL (0x00000000)
#define NVC1B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL (0x00000001)
#define NVC1B5_LAUNCH_DMA_DST_TYPE 13:13
#define NVC1B5_LAUNCH_DMA_DST_TYPE_VIRTUAL (0x00000000)
#define NVC1B5_LAUNCH_DMA_DST_TYPE_PHYSICAL (0x00000001)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION 17:14
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN (0x00000000)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX (0x00000001)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR (0x00000002)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND (0x00000003)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR (0x00000004)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD (0x00000005)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC (0x00000006)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC (0x00000007)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD (0x0000000A)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN 18:18
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED (0x00000000)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED (0x00000001)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE 19:19
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE (0x00000000)
#define NVC1B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE (0x00000001)
#define NVC1B5_LAUNCH_DMA_SRC_BYPASS_L2 20:20
#define NVC1B5_LAUNCH_DMA_SRC_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC1B5_LAUNCH_DMA_SRC_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC1B5_LAUNCH_DMA_DST_BYPASS_L2 21:21
#define NVC1B5_LAUNCH_DMA_DST_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC1B5_LAUNCH_DMA_DST_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC1B5_LAUNCH_DMA_VPRMODE 23:22
#define NVC1B5_LAUNCH_DMA_VPRMODE_VPR_NONE (0x00000000)
#define NVC1B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID (0x00000001)
#define NVC1B5_LAUNCH_DMA_RESERVED_START_OF_COPY 24:24
#define NVC1B5_LAUNCH_DMA_RESERVED_ERR_CODE 31:28
#define NVC1B5_OFFSET_IN_UPPER (0x00000400)
#define NVC1B5_OFFSET_IN_UPPER_UPPER 16:0
#define NVC1B5_OFFSET_IN_LOWER (0x00000404)
#define NVC1B5_OFFSET_IN_LOWER_VALUE 31:0
#define NVC1B5_OFFSET_OUT_UPPER (0x00000408)
#define NVC1B5_OFFSET_OUT_UPPER_UPPER 16:0
#define NVC1B5_OFFSET_OUT_LOWER (0x0000040C)
#define NVC1B5_OFFSET_OUT_LOWER_VALUE 31:0
#define NVC1B5_PITCH_IN (0x00000410)
#define NVC1B5_PITCH_IN_VALUE 31:0
#define NVC1B5_PITCH_OUT (0x00000414)
#define NVC1B5_PITCH_OUT_VALUE 31:0
#define NVC1B5_LINE_LENGTH_IN (0x00000418)
#define NVC1B5_LINE_LENGTH_IN_VALUE 31:0
#define NVC1B5_LINE_COUNT (0x0000041C)
#define NVC1B5_LINE_COUNT_VALUE 31:0
#define NVC1B5_SET_REMAP_CONST_A (0x00000700)
#define NVC1B5_SET_REMAP_CONST_A_V 31:0
#define NVC1B5_SET_REMAP_CONST_B (0x00000704)
#define NVC1B5_SET_REMAP_CONST_B_V 31:0
#define NVC1B5_SET_REMAP_COMPONENTS (0x00000708)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X 2:0
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_SRC_X (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_SRC_W (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_CONST_A (0x00000004)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_CONST_B (0x00000005)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE (0x00000006)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y 6:4
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A (0x00000004)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B (0x00000005)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE (0x00000006)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z 10:8
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A (0x00000004)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B (0x00000005)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE (0x00000006)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W 14:12
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_SRC_X (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_SRC_W (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_CONST_A (0x00000004)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_CONST_B (0x00000005)
#define NVC1B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE (0x00000006)
#define NVC1B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE 17:16
#define NVC1B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS 21:20
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR (0x00000003)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS 25:24
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE (0x00000000)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO (0x00000001)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE (0x00000002)
#define NVC1B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR (0x00000003)
#define NVC1B5_SET_DST_BLOCK_SIZE (0x0000070C)
#define NVC1B5_SET_DST_BLOCK_SIZE_WIDTH 3:0
#define NVC1B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT 7:4
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC1B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH 11:8
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC1B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC1B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC1B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC1B5_SET_DST_WIDTH (0x00000710)
#define NVC1B5_SET_DST_WIDTH_V 31:0
#define NVC1B5_SET_DST_HEIGHT (0x00000714)
#define NVC1B5_SET_DST_HEIGHT_V 31:0
#define NVC1B5_SET_DST_DEPTH (0x00000718)
#define NVC1B5_SET_DST_DEPTH_V 31:0
#define NVC1B5_SET_DST_LAYER (0x0000071C)
#define NVC1B5_SET_DST_LAYER_V 31:0
#define NVC1B5_SET_DST_ORIGIN (0x00000720)
#define NVC1B5_SET_DST_ORIGIN_X 15:0
#define NVC1B5_SET_DST_ORIGIN_Y 31:16
#define NVC1B5_SET_SRC_BLOCK_SIZE (0x00000728)
#define NVC1B5_SET_SRC_BLOCK_SIZE_WIDTH 3:0
#define NVC1B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT 7:4
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC1B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH 11:8
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC1B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC1B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC1B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC1B5_SET_SRC_WIDTH (0x0000072C)
#define NVC1B5_SET_SRC_WIDTH_V 31:0
#define NVC1B5_SET_SRC_HEIGHT (0x00000730)
#define NVC1B5_SET_SRC_HEIGHT_V 31:0
#define NVC1B5_SET_SRC_DEPTH (0x00000734)
#define NVC1B5_SET_SRC_DEPTH_V 31:0
#define NVC1B5_SET_SRC_LAYER (0x00000738)
#define NVC1B5_SET_SRC_LAYER_V 31:0
#define NVC1B5_SET_SRC_ORIGIN (0x0000073C)
#define NVC1B5_SET_SRC_ORIGIN_X 15:0
#define NVC1B5_SET_SRC_ORIGIN_Y 31:16
#define NVC1B5_SRC_ORIGIN_X (0x00000744)
#define NVC1B5_SRC_ORIGIN_X_VALUE 31:0
#define NVC1B5_SRC_ORIGIN_Y (0x00000748)
#define NVC1B5_SRC_ORIGIN_Y_VALUE 31:0
#define NVC1B5_DST_ORIGIN_X (0x0000074C)
#define NVC1B5_DST_ORIGIN_X_VALUE 31:0
#define NVC1B5_DST_ORIGIN_Y (0x00000750)
#define NVC1B5_DST_ORIGIN_Y_VALUE 31:0
#define NVC1B5_PM_TRIGGER_END (0x00001114)
#define NVC1B5_PM_TRIGGER_END_V 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif // _clc1b5_h

366
kernel-open/nvidia-uvm/clc36f.h
View File

@@ -1,366 +0,0 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _clc36f_h_
#define _clc36f_h_
#ifdef __cplusplus
extern "C" {
#endif
#include "nvtypes.h"
/* class VOLTA_CHANNEL_GPFIFO */
/*
* Documentation for VOLTA_CHANNEL_GPFIFO can be found in dev_pbdma.ref,
* chapter "User Control Registers". It is documented as device NV_UDMA.
* The GPFIFO format itself is also documented in dev_pbdma.ref,
* NV_PPBDMA_GP_ENTRY_*. The pushbuffer format is documented in dev_ram.ref,
* chapter "FIFO DMA RAM", NV_FIFO_DMA_*.
*
* Note there is no .mfs file for this class.
*/
#define VOLTA_CHANNEL_GPFIFO_A (0x0000C36F)
#define NVC36F_TYPEDEF VOLTA_CHANNELChannelGPFifoA
/* dma flow control data structure */
typedef volatile struct Nvc36fControl_struct {
NvU32 Ignored00[0x010]; /* 0000-003f*/
NvU32 Put; /* put offset, read/write 0040-0043*/
NvU32 Get; /* get offset, read only 0044-0047*/
NvU32 Reference; /* reference value, read only 0048-004b*/
NvU32 PutHi; /* high order put offset bits 004c-004f*/
NvU32 Ignored01[0x002]; /* 0050-0057*/
NvU32 TopLevelGet; /* top level get offset, read only 0058-005b*/
NvU32 TopLevelGetHi; /* high order top level get bits 005c-005f*/
NvU32 GetHi; /* high order get offset bits 0060-0063*/
NvU32 Ignored02[0x007]; /* 0064-007f*/
NvU32 Ignored03; /* used to be engine yield 0080-0083*/
NvU32 Ignored04[0x001]; /* 0084-0087*/
NvU32 GPGet; /* GP FIFO get offset, read only 0088-008b*/
NvU32 GPPut; /* GP FIFO put offset 008c-008f*/
NvU32 Ignored05[0x5c];
} Nvc36fControl, VoltaAControlGPFifo;
/* fields and values */
#define NVC36F_NUMBER_OF_SUBCHANNELS (8)
#define NVC36F_SET_OBJECT (0x00000000)
#define NVC36F_SET_OBJECT_NVCLASS 15:0
#define NVC36F_SET_OBJECT_ENGINE 20:16
#define NVC36F_SET_OBJECT_ENGINE_SW 0x0000001f
#define NVC36F_ILLEGAL (0x00000004)
#define NVC36F_ILLEGAL_HANDLE 31:0
#define NVC36F_NOP (0x00000008)
#define NVC36F_NOP_HANDLE 31:0
#define NVC36F_SEMAPHOREA (0x00000010)
#define NVC36F_SEMAPHOREA_OFFSET_UPPER 7:0
#define NVC36F_SEMAPHOREB (0x00000014)
#define NVC36F_SEMAPHOREB_OFFSET_LOWER 31:2
#define NVC36F_SEMAPHOREC (0x00000018)
#define NVC36F_SEMAPHOREC_PAYLOAD 31:0
#define NVC36F_SEMAPHORED (0x0000001C)
#define NVC36F_SEMAPHORED_OPERATION 4:0
#define NVC36F_SEMAPHORED_OPERATION_ACQUIRE 0x00000001
#define NVC36F_SEMAPHORED_OPERATION_RELEASE 0x00000002
#define NVC36F_SEMAPHORED_OPERATION_ACQ_GEQ 0x00000004
#define NVC36F_SEMAPHORED_OPERATION_ACQ_AND 0x00000008
#define NVC36F_SEMAPHORED_OPERATION_REDUCTION 0x00000010
#define NVC36F_SEMAPHORED_ACQUIRE_SWITCH 12:12
#define NVC36F_SEMAPHORED_ACQUIRE_SWITCH_DISABLED 0x00000000
#define NVC36F_SEMAPHORED_ACQUIRE_SWITCH_ENABLED 0x00000001
#define NVC36F_SEMAPHORED_RELEASE_WFI 20:20
#define NVC36F_SEMAPHORED_RELEASE_WFI_EN 0x00000000
#define NVC36F_SEMAPHORED_RELEASE_WFI_DIS 0x00000001
#define NVC36F_SEMAPHORED_RELEASE_SIZE 24:24
#define NVC36F_SEMAPHORED_RELEASE_SIZE_16BYTE 0x00000000
#define NVC36F_SEMAPHORED_RELEASE_SIZE_4BYTE 0x00000001
#define NVC36F_SEMAPHORED_REDUCTION 30:27
#define NVC36F_SEMAPHORED_REDUCTION_MIN 0x00000000
#define NVC36F_SEMAPHORED_REDUCTION_MAX 0x00000001
#define NVC36F_SEMAPHORED_REDUCTION_XOR 0x00000002
#define NVC36F_SEMAPHORED_REDUCTION_AND 0x00000003
#define NVC36F_SEMAPHORED_REDUCTION_OR 0x00000004
#define NVC36F_SEMAPHORED_REDUCTION_ADD 0x00000005
#define NVC36F_SEMAPHORED_REDUCTION_INC 0x00000006
#define NVC36F_SEMAPHORED_REDUCTION_DEC 0x00000007
#define NVC36F_SEMAPHORED_FORMAT 31:31
#define NVC36F_SEMAPHORED_FORMAT_SIGNED 0x00000000
#define NVC36F_SEMAPHORED_FORMAT_UNSIGNED 0x00000001
#define NVC36F_NON_STALL_INTERRUPT (0x00000020)
#define NVC36F_NON_STALL_INTERRUPT_HANDLE 31:0
#define NVC36F_FB_FLUSH (0x00000024) // Deprecated - use MEMBAR TYPE SYS_MEMBAR
#define NVC36F_FB_FLUSH_HANDLE 31:0
// NOTE - MEM_OP_A and MEM_OP_B have been replaced in gp100 with methods for
// specifying the page address for a targeted TLB invalidate and the uTLB for
// a targeted REPLAY_CANCEL for UVM.
// The previous MEM_OP_A/B functionality is in MEM_OP_C/D, with slightly
// rearranged fields.
#define NVC36F_MEM_OP_A (0x00000028)
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_CLIENT_UNIT_ID 5:0 // only relevant for REPLAY_CANCEL_TARGETED
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_INVALIDATION_SIZE 5:0 // Used to specify size of invalidate, used for invalidates which are not of the REPLAY_CANCEL_TARGETED type
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_CANCEL_TARGET_GPC_ID 10:6 // only relevant for REPLAY_CANCEL_TARGETED
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_CANCEL_MMU_ENGINE_ID 6:0 // only relevant for REPLAY_CANCEL_VA_GLOBAL
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR 11:11
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_EN 0x00000001
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_SYSMEMBAR_DIS 0x00000000
#define NVC36F_MEM_OP_A_TLB_INVALIDATE_TARGET_ADDR_LO 31:12
#define NVC36F_MEM_OP_B (0x0000002c)
#define NVC36F_MEM_OP_B_TLB_INVALIDATE_TARGET_ADDR_HI 31:0
#define NVC36F_MEM_OP_C (0x00000030)
#define NVC36F_MEM_OP_C_MEMBAR_TYPE 2:0
#define NVC36F_MEM_OP_C_MEMBAR_TYPE_SYS_MEMBAR 0x00000000
#define NVC36F_MEM_OP_C_MEMBAR_TYPE_MEMBAR 0x00000001
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB 0:0
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_ONE 0x00000000
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_ALL 0x00000001 // Probably nonsensical for MMU_TLB_INVALIDATE_TARGETED
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_GPC 1:1
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_GPC_ENABLE 0x00000000
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_GPC_DISABLE 0x00000001
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY 4:2 // only relevant if GPC ENABLE
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_NONE 0x00000000
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START 0x00000001
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_START_ACK_ALL 0x00000002
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_TARGETED 0x00000003
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_GLOBAL 0x00000004
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_REPLAY_CANCEL_VA_GLOBAL 0x00000005
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE 6:5 // only relevant if GPC ENABLE
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_NONE 0x00000000
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_GLOBALLY 0x00000001
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACK_TYPE_INTRANODE 0x00000002
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE 9:7 //only relevant for REPLAY_CANCEL_VA_GLOBAL
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_READ 0
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE 1
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_STRONG 2
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_RSVRVD 3
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_WEAK 4
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ATOMIC_ALL 5
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_WRITE_AND_ATOMIC 6
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_ACCESS_TYPE_VIRT_ALL 7
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL 9:7 // Invalidate affects this level and all below
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_ALL 0x00000000 // Invalidate tlb caches at all levels of the page table
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_PTE_ONLY 0x00000001
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE0 0x00000002
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE1 0x00000003
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE2 0x00000004
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 0x00000005
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE4 0x00000006
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE5 0x00000007
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE 11:10 // only relevant if PDB_ONE
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_VID_MEM 0x00000000
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_COHERENT 0x00000002
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_APERTURE_SYS_MEM_NONCOHERENT 0x00000003
#define NVC36F_MEM_OP_C_TLB_INVALIDATE_PDB_ADDR_LO 31:12 // only relevant if PDB_ONE
#define NVC36F_MEM_OP_C_ACCESS_COUNTER_CLR_TARGETED_NOTIFY_TAG 19:0
// MEM_OP_D MUST be preceded by MEM_OPs A-C.
#define NVC36F_MEM_OP_D (0x00000034)
#define NVC36F_MEM_OP_D_TLB_INVALIDATE_PDB_ADDR_HI 26:0 // only relevant if PDB_ONE
#define NVC36F_MEM_OP_D_OPERATION 31:27
#define NVC36F_MEM_OP_D_OPERATION_MEMBAR 0x00000005
#define NVC36F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE 0x00000009
#define NVC36F_MEM_OP_D_OPERATION_MMU_TLB_INVALIDATE_TARGETED 0x0000000a
#define NVC36F_MEM_OP_D_OPERATION_L2_PEERMEM_INVALIDATE 0x0000000d
#define NVC36F_MEM_OP_D_OPERATION_L2_SYSMEM_INVALIDATE 0x0000000e
// CLEAN_LINES is an alias for Tegra/GPU IP usage
#define NVC36F_MEM_OP_B_OPERATION_L2_INVALIDATE_CLEAN_LINES 0x0000000e
#define NVC36F_MEM_OP_D_OPERATION_L2_CLEAN_COMPTAGS 0x0000000f
#define NVC36F_MEM_OP_D_OPERATION_L2_FLUSH_DIRTY 0x00000010
#define NVC36F_MEM_OP_D_OPERATION_L2_WAIT_FOR_SYS_PENDING_READS 0x00000015
#define NVC36F_MEM_OP_D_OPERATION_ACCESS_COUNTER_CLR 0x00000016
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE 1:0
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MIMC 0x00000000
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_MOMC 0x00000001
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_ALL 0x00000002
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TYPE_TARGETED 0x00000003
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE 2:2
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MIMC 0x00000000
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_TYPE_MOMC 0x00000001
#define NVC36F_MEM_OP_D_ACCESS_COUNTER_CLR_TARGETED_BANK 6:3
#define NVC36F_SET_REFERENCE (0x00000050)
#define NVC36F_SET_REFERENCE_COUNT 31:0
#define NVC36F_SEM_ADDR_LO (0x0000005c)
#define NVC36F_SEM_ADDR_LO_OFFSET 31:2
#define NVC36F_SEM_ADDR_HI (0x00000060)
#define NVC36F_SEM_ADDR_HI_OFFSET 7:0
#define NVC36F_SEM_PAYLOAD_LO (0x00000064)
#define NVC36F_SEM_PAYLOAD_LO_PAYLOAD 31:0
#define NVC36F_SEM_PAYLOAD_HI (0x00000068)
#define NVC36F_SEM_PAYLOAD_HI_PAYLOAD 31:0
#define NVC36F_SEM_EXECUTE (0x0000006c)
#define NVC36F_SEM_EXECUTE_OPERATION 2:0
#define NVC36F_SEM_EXECUTE_OPERATION_ACQUIRE 0x00000000
#define NVC36F_SEM_EXECUTE_OPERATION_RELEASE 0x00000001
#define NVC36F_SEM_EXECUTE_OPERATION_ACQ_STRICT_GEQ 0x00000002
#define NVC36F_SEM_EXECUTE_OPERATION_ACQ_CIRC_GEQ 0x00000003
#define NVC36F_SEM_EXECUTE_OPERATION_ACQ_AND 0x00000004
#define NVC36F_SEM_EXECUTE_OPERATION_ACQ_NOR 0x00000005
#define NVC36F_SEM_EXECUTE_OPERATION_REDUCTION 0x00000006
#define NVC36F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG 12:12
#define NVC36F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_DIS 0x00000000
#define NVC36F_SEM_EXECUTE_ACQUIRE_SWITCH_TSG_EN 0x00000001
#define NVC36F_SEM_EXECUTE_RELEASE_WFI 20:20
#define NVC36F_SEM_EXECUTE_RELEASE_WFI_DIS 0x00000000
#define NVC36F_SEM_EXECUTE_RELEASE_WFI_EN 0x00000001
#define NVC36F_SEM_EXECUTE_PAYLOAD_SIZE 24:24
#define NVC36F_SEM_EXECUTE_PAYLOAD_SIZE_32BIT 0x00000000
#define NVC36F_SEM_EXECUTE_PAYLOAD_SIZE_64BIT 0x00000001
#define NVC36F_SEM_EXECUTE_RELEASE_TIMESTAMP 25:25
#define NVC36F_SEM_EXECUTE_RELEASE_TIMESTAMP_DIS 0x00000000
#define NVC36F_SEM_EXECUTE_RELEASE_TIMESTAMP_EN 0x00000001
#define NVC36F_SEM_EXECUTE_REDUCTION 30:27
#define NVC36F_SEM_EXECUTE_REDUCTION_IMIN 0x00000000
#define NVC36F_SEM_EXECUTE_REDUCTION_IMAX 0x00000001
#define NVC36F_SEM_EXECUTE_REDUCTION_IXOR 0x00000002
#define NVC36F_SEM_EXECUTE_REDUCTION_IAND 0x00000003
#define NVC36F_SEM_EXECUTE_REDUCTION_IOR 0x00000004
#define NVC36F_SEM_EXECUTE_REDUCTION_IADD 0x00000005
#define NVC36F_SEM_EXECUTE_REDUCTION_INC 0x00000006
#define NVC36F_SEM_EXECUTE_REDUCTION_DEC 0x00000007
#define NVC36F_SEM_EXECUTE_REDUCTION_FORMAT 31:31
#define NVC36F_SEM_EXECUTE_REDUCTION_FORMAT_SIGNED 0x00000000
#define NVC36F_SEM_EXECUTE_REDUCTION_FORMAT_UNSIGNED 0x00000001
#define NVC36F_WFI (0x00000078)
#define NVC36F_WFI_SCOPE 0:0
#define NVC36F_WFI_SCOPE_CURRENT_SCG_TYPE 0x00000000
#define NVC36F_WFI_SCOPE_CURRENT_VEID 0x00000000
#define NVC36F_WFI_SCOPE_ALL 0x00000001
#define NVC36F_CRC_CHECK (0x0000007c)
#define NVC36F_CRC_CHECK_VALUE 31:0
#define NVC36F_YIELD (0x00000080)
#define NVC36F_YIELD_OP 1:0
#define NVC36F_YIELD_OP_NOP 0x00000000
#define NVC36F_YIELD_OP_RUNLIST_TIMESLICE 0x00000002
#define NVC36F_YIELD_OP_TSG 0x00000003
#define NVC36F_CLEAR_FAULTED (0x00000084)
#define NVC36F_CLEAR_FAULTED_CHID 11:0
#define NVC36F_CLEAR_FAULTED_TYPE 31:31
#define NVC36F_CLEAR_FAULTED_TYPE_PBDMA_FAULTED 0x00000000
#define NVC36F_CLEAR_FAULTED_TYPE_ENG_FAULTED 0x00000001
/* GPFIFO entry format */
#define NVC36F_GP_ENTRY__SIZE 8
#define NVC36F_GP_ENTRY0_FETCH 0:0
#define NVC36F_GP_ENTRY0_FETCH_UNCONDITIONAL 0x00000000
#define NVC36F_GP_ENTRY0_FETCH_CONDITIONAL 0x00000001
#define NVC36F_GP_ENTRY0_GET 31:2
#define NVC36F_GP_ENTRY0_OPERAND 31:0
#define NVC36F_GP_ENTRY1_GET_HI 7:0
#define NVC36F_GP_ENTRY1_PRIV 8:8
#define NVC36F_GP_ENTRY1_PRIV_USER 0x00000000
#define NVC36F_GP_ENTRY1_PRIV_KERNEL 0x00000001
#define NVC36F_GP_ENTRY1_LEVEL 9:9
#define NVC36F_GP_ENTRY1_LEVEL_MAIN 0x00000000
#define NVC36F_GP_ENTRY1_LEVEL_SUBROUTINE 0x00000001
#define NVC36F_GP_ENTRY1_LENGTH 30:10
#define NVC36F_GP_ENTRY1_SYNC 31:31
#define NVC36F_GP_ENTRY1_SYNC_PROCEED 0x00000000
#define NVC36F_GP_ENTRY1_SYNC_WAIT 0x00000001
#define NVC36F_GP_ENTRY1_OPCODE 7:0
#define NVC36F_GP_ENTRY1_OPCODE_NOP 0x00000000
#define NVC36F_GP_ENTRY1_OPCODE_ILLEGAL 0x00000001
#define NVC36F_GP_ENTRY1_OPCODE_GP_CRC 0x00000002
#define NVC36F_GP_ENTRY1_OPCODE_PB_CRC 0x00000003
/* dma method formats */
#define NVC36F_DMA_METHOD_ADDRESS_OLD 12:2
#define NVC36F_DMA_METHOD_ADDRESS 11:0
#define NVC36F_DMA_SUBDEVICE_MASK 15:4
#define NVC36F_DMA_METHOD_SUBCHANNEL 15:13
#define NVC36F_DMA_TERT_OP 17:16
#define NVC36F_DMA_TERT_OP_GRP0_INC_METHOD (0x00000000)
#define NVC36F_DMA_TERT_OP_GRP0_SET_SUB_DEV_MASK (0x00000001)
#define NVC36F_DMA_TERT_OP_GRP0_STORE_SUB_DEV_MASK (0x00000002)
#define NVC36F_DMA_TERT_OP_GRP0_USE_SUB_DEV_MASK (0x00000003)
#define NVC36F_DMA_TERT_OP_GRP2_NON_INC_METHOD (0x00000000)
#define NVC36F_DMA_METHOD_COUNT_OLD 28:18
#define NVC36F_DMA_METHOD_COUNT 28:16
#define NVC36F_DMA_IMMD_DATA 28:16
#define NVC36F_DMA_SEC_OP 31:29
#define NVC36F_DMA_SEC_OP_GRP0_USE_TERT (0x00000000)
#define NVC36F_DMA_SEC_OP_INC_METHOD (0x00000001)
#define NVC36F_DMA_SEC_OP_GRP2_USE_TERT (0x00000002)
#define NVC36F_DMA_SEC_OP_NON_INC_METHOD (0x00000003)
#define NVC36F_DMA_SEC_OP_IMMD_DATA_METHOD (0x00000004)
#define NVC36F_DMA_SEC_OP_ONE_INC (0x00000005)
#define NVC36F_DMA_SEC_OP_RESERVED6 (0x00000006)
#define NVC36F_DMA_SEC_OP_END_PB_SEGMENT (0x00000007)
/* dma incrementing method format */
#define NVC36F_DMA_INCR_ADDRESS 11:0
#define NVC36F_DMA_INCR_SUBCHANNEL 15:13
#define NVC36F_DMA_INCR_COUNT 28:16
#define NVC36F_DMA_INCR_OPCODE 31:29
#define NVC36F_DMA_INCR_OPCODE_VALUE (0x00000001)
#define NVC36F_DMA_INCR_DATA 31:0
/* dma non-incrementing method format */
#define NVC36F_DMA_NONINCR_ADDRESS 11:0
#define NVC36F_DMA_NONINCR_SUBCHANNEL 15:13
#define NVC36F_DMA_NONINCR_COUNT 28:16
#define NVC36F_DMA_NONINCR_OPCODE 31:29
#define NVC36F_DMA_NONINCR_OPCODE_VALUE (0x00000003)
#define NVC36F_DMA_NONINCR_DATA 31:0
/* dma increment-once method format */
#define NVC36F_DMA_ONEINCR_ADDRESS 11:0
#define NVC36F_DMA_ONEINCR_SUBCHANNEL 15:13
#define NVC36F_DMA_ONEINCR_COUNT 28:16
#define NVC36F_DMA_ONEINCR_OPCODE 31:29
#define NVC36F_DMA_ONEINCR_OPCODE_VALUE (0x00000005)
#define NVC36F_DMA_ONEINCR_DATA 31:0
/* dma no-operation format */
#define NVC36F_DMA_NOP (0x00000000)
/* dma immediate-data format */
#define NVC36F_DMA_IMMD_ADDRESS 11:0
#define NVC36F_DMA_IMMD_SUBCHANNEL 15:13
#define NVC36F_DMA_IMMD_DATA 28:16
#define NVC36F_DMA_IMMD_OPCODE 31:29
#define NVC36F_DMA_IMMD_OPCODE_VALUE (0x00000004)
/* dma set sub-device mask format */
#define NVC36F_DMA_SET_SUBDEVICE_MASK_VALUE 15:4
#define NVC36F_DMA_SET_SUBDEVICE_MASK_OPCODE 31:16
#define NVC36F_DMA_SET_SUBDEVICE_MASK_OPCODE_VALUE (0x00000001)
/* dma store sub-device mask format */
#define NVC36F_DMA_STORE_SUBDEVICE_MASK_VALUE 15:4
#define NVC36F_DMA_STORE_SUBDEVICE_MASK_OPCODE 31:16
#define NVC36F_DMA_STORE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000002)
/* dma use sub-device mask format */
#define NVC36F_DMA_USE_SUBDEVICE_MASK_OPCODE 31:16
#define NVC36F_DMA_USE_SUBDEVICE_MASK_OPCODE_VALUE (0x00000003)
/* dma end-segment format */
#define NVC36F_DMA_ENDSEG_OPCODE 31:29
#define NVC36F_DMA_ENDSEG_OPCODE_VALUE (0x00000007)
/* dma legacy incrementing/non-incrementing formats */
#define NVC36F_DMA_ADDRESS 12:2
#define NVC36F_DMA_SUBCH 15:13
#define NVC36F_DMA_OPCODE3 17:16
#define NVC36F_DMA_OPCODE3_NONE (0x00000000)
#define NVC36F_DMA_COUNT 28:18
#define NVC36F_DMA_OPCODE 31:29
#define NVC36F_DMA_OPCODE_METHOD (0x00000000)
#define NVC36F_DMA_OPCODE_NONINC_METHOD (0x00000002)
#define NVC36F_DMA_DATA 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif /* _clc36f_h_ */

278
kernel-open/nvidia-uvm/clc3b5.h
View File

@@ -1,278 +0,0 @@
/*******************************************************************************
Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "nvtypes.h"
#ifndef _clc3b5_h_
#define _clc3b5_h_
#ifdef __cplusplus
extern "C" {
#endif
#define VOLTA_DMA_COPY_A (0x0000C3B5)
#define NVC3B5_NOP (0x00000100)
#define NVC3B5_NOP_PARAMETER 31:0
#define NVC3B5_PM_TRIGGER (0x00000140)
#define NVC3B5_PM_TRIGGER_V 31:0
#define NVC3B5_SET_SEMAPHORE_A (0x00000240)
#define NVC3B5_SET_SEMAPHORE_A_UPPER 16:0
#define NVC3B5_SET_SEMAPHORE_B (0x00000244)
#define NVC3B5_SET_SEMAPHORE_B_LOWER 31:0
#define NVC3B5_SET_SEMAPHORE_PAYLOAD (0x00000248)
#define NVC3B5_SET_SEMAPHORE_PAYLOAD_PAYLOAD 31:0
#define NVC3B5_SET_RENDER_ENABLE_A (0x00000254)
#define NVC3B5_SET_RENDER_ENABLE_A_UPPER 7:0
#define NVC3B5_SET_RENDER_ENABLE_B (0x00000258)
#define NVC3B5_SET_RENDER_ENABLE_B_LOWER 31:0
#define NVC3B5_SET_RENDER_ENABLE_C (0x0000025C)
#define NVC3B5_SET_RENDER_ENABLE_C_MODE 2:0
#define NVC3B5_SET_RENDER_ENABLE_C_MODE_FALSE (0x00000000)
#define NVC3B5_SET_RENDER_ENABLE_C_MODE_TRUE (0x00000001)
#define NVC3B5_SET_RENDER_ENABLE_C_MODE_CONDITIONAL (0x00000002)
#define NVC3B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_EQUAL (0x00000003)
#define NVC3B5_SET_RENDER_ENABLE_C_MODE_RENDER_IF_NOT_EQUAL (0x00000004)
#define NVC3B5_SET_SRC_PHYS_MODE (0x00000260)
#define NVC3B5_SET_SRC_PHYS_MODE_TARGET 1:0
#define NVC3B5_SET_SRC_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC3B5_SET_SRC_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC3B5_SET_SRC_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC3B5_SET_SRC_PHYS_MODE_BASIC_KIND 5:2
#define NVC3B5_SET_DST_PHYS_MODE (0x00000264)
#define NVC3B5_SET_DST_PHYS_MODE_TARGET 1:0
#define NVC3B5_SET_DST_PHYS_MODE_TARGET_LOCAL_FB (0x00000000)
#define NVC3B5_SET_DST_PHYS_MODE_TARGET_COHERENT_SYSMEM (0x00000001)
#define NVC3B5_SET_DST_PHYS_MODE_TARGET_NONCOHERENT_SYSMEM (0x00000002)
#define NVC3B5_SET_DST_PHYS_MODE_BASIC_KIND 5:2
#define NVC3B5_LAUNCH_DMA (0x00000300)
#define NVC3B5_LAUNCH_DMA_DATA_TRANSFER_TYPE 1:0
#define NVC3B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NONE (0x00000000)
#define NVC3B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_PIPELINED (0x00000001)
#define NVC3B5_LAUNCH_DMA_DATA_TRANSFER_TYPE_NON_PIPELINED (0x00000002)
#define NVC3B5_LAUNCH_DMA_FLUSH_ENABLE 2:2
#define NVC3B5_LAUNCH_DMA_FLUSH_ENABLE_FALSE (0x00000000)
#define NVC3B5_LAUNCH_DMA_FLUSH_ENABLE_TRUE (0x00000001)
#define NVC3B5_LAUNCH_DMA_FLUSH_TYPE 25:25
#define NVC3B5_LAUNCH_DMA_FLUSH_TYPE_SYS (0x00000000)
#define NVC3B5_LAUNCH_DMA_FLUSH_TYPE_GL (0x00000001)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_TYPE 4:3
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_TYPE_NONE (0x00000000)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_ONE_WORD_SEMAPHORE (0x00000001)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_TYPE_RELEASE_FOUR_WORD_SEMAPHORE (0x00000002)
#define NVC3B5_LAUNCH_DMA_INTERRUPT_TYPE 6:5
#define NVC3B5_LAUNCH_DMA_INTERRUPT_TYPE_NONE (0x00000000)
#define NVC3B5_LAUNCH_DMA_INTERRUPT_TYPE_BLOCKING (0x00000001)
#define NVC3B5_LAUNCH_DMA_INTERRUPT_TYPE_NON_BLOCKING (0x00000002)
#define NVC3B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT 7:7
#define NVC3B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC3B5_LAUNCH_DMA_SRC_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC3B5_LAUNCH_DMA_DST_MEMORY_LAYOUT 8:8
#define NVC3B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_BLOCKLINEAR (0x00000000)
#define NVC3B5_LAUNCH_DMA_DST_MEMORY_LAYOUT_PITCH (0x00000001)
#define NVC3B5_LAUNCH_DMA_MULTI_LINE_ENABLE 9:9
#define NVC3B5_LAUNCH_DMA_MULTI_LINE_ENABLE_FALSE (0x00000000)
#define NVC3B5_LAUNCH_DMA_MULTI_LINE_ENABLE_TRUE (0x00000001)
#define NVC3B5_LAUNCH_DMA_REMAP_ENABLE 10:10
#define NVC3B5_LAUNCH_DMA_REMAP_ENABLE_FALSE (0x00000000)
#define NVC3B5_LAUNCH_DMA_REMAP_ENABLE_TRUE (0x00000001)
#define NVC3B5_LAUNCH_DMA_FORCE_RMWDISABLE 11:11
#define NVC3B5_LAUNCH_DMA_FORCE_RMWDISABLE_FALSE (0x00000000)
#define NVC3B5_LAUNCH_DMA_FORCE_RMWDISABLE_TRUE (0x00000001)
#define NVC3B5_LAUNCH_DMA_SRC_TYPE 12:12
#define NVC3B5_LAUNCH_DMA_SRC_TYPE_VIRTUAL (0x00000000)
#define NVC3B5_LAUNCH_DMA_SRC_TYPE_PHYSICAL (0x00000001)
#define NVC3B5_LAUNCH_DMA_DST_TYPE 13:13
#define NVC3B5_LAUNCH_DMA_DST_TYPE_VIRTUAL (0x00000000)
#define NVC3B5_LAUNCH_DMA_DST_TYPE_PHYSICAL (0x00000001)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION 17:14
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMIN (0x00000000)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IMAX (0x00000001)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IXOR (0x00000002)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IAND (0x00000003)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IOR (0x00000004)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_IADD (0x00000005)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_INC (0x00000006)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_DEC (0x00000007)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_FADD (0x0000000A)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN 18:18
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_SIGNED (0x00000000)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_SIGN_UNSIGNED (0x00000001)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE 19:19
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_FALSE (0x00000000)
#define NVC3B5_LAUNCH_DMA_SEMAPHORE_REDUCTION_ENABLE_TRUE (0x00000001)
#define NVC3B5_LAUNCH_DMA_SRC_BYPASS_L2 20:20
#define NVC3B5_LAUNCH_DMA_SRC_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC3B5_LAUNCH_DMA_SRC_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC3B5_LAUNCH_DMA_DST_BYPASS_L2 21:21
#define NVC3B5_LAUNCH_DMA_DST_BYPASS_L2_USE_PTE_SETTING (0x00000000)
#define NVC3B5_LAUNCH_DMA_DST_BYPASS_L2_FORCE_VOLATILE (0x00000001)
#define NVC3B5_LAUNCH_DMA_VPRMODE 23:22
#define NVC3B5_LAUNCH_DMA_VPRMODE_VPR_NONE (0x00000000)
#define NVC3B5_LAUNCH_DMA_VPRMODE_VPR_VID2VID (0x00000001)
#define NVC3B5_LAUNCH_DMA_RESERVED_START_OF_COPY 24:24
#define NVC3B5_LAUNCH_DMA_RESERVED_ERR_CODE 31:28
#define NVC3B5_OFFSET_IN_UPPER (0x00000400)
#define NVC3B5_OFFSET_IN_UPPER_UPPER 16:0
#define NVC3B5_OFFSET_IN_LOWER (0x00000404)
#define NVC3B5_OFFSET_IN_LOWER_VALUE 31:0
#define NVC3B5_OFFSET_OUT_UPPER (0x00000408)
#define NVC3B5_OFFSET_OUT_UPPER_UPPER 16:0
#define NVC3B5_OFFSET_OUT_LOWER (0x0000040C)
#define NVC3B5_OFFSET_OUT_LOWER_VALUE 31:0
#define NVC3B5_PITCH_IN (0x00000410)
#define NVC3B5_PITCH_IN_VALUE 31:0
#define NVC3B5_PITCH_OUT (0x00000414)
#define NVC3B5_PITCH_OUT_VALUE 31:0
#define NVC3B5_LINE_LENGTH_IN (0x00000418)
#define NVC3B5_LINE_LENGTH_IN_VALUE 31:0
#define NVC3B5_LINE_COUNT (0x0000041C)
#define NVC3B5_LINE_COUNT_VALUE 31:0
#define NVC3B5_SET_REMAP_CONST_A (0x00000700)
#define NVC3B5_SET_REMAP_CONST_A_V 31:0
#define NVC3B5_SET_REMAP_CONST_B (0x00000704)
#define NVC3B5_SET_REMAP_CONST_B_V 31:0
#define NVC3B5_SET_REMAP_COMPONENTS (0x00000708)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X 2:0
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_SRC_X (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_SRC_Y (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_SRC_Z (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_SRC_W (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_CONST_A (0x00000004)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_CONST_B (0x00000005)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_X_NO_WRITE (0x00000006)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y 6:4
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_SRC_X (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Y (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_SRC_Z (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_SRC_W (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_CONST_A (0x00000004)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_CONST_B (0x00000005)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Y_NO_WRITE (0x00000006)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z 10:8
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_SRC_X (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Y (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_SRC_Z (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_SRC_W (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_CONST_A (0x00000004)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_CONST_B (0x00000005)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_Z_NO_WRITE (0x00000006)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W 14:12
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_SRC_X (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_SRC_Y (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_SRC_Z (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_SRC_W (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_CONST_A (0x00000004)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_CONST_B (0x00000005)
#define NVC3B5_SET_REMAP_COMPONENTS_DST_W_NO_WRITE (0x00000006)
#define NVC3B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE 17:16
#define NVC3B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_ONE (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_TWO (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_THREE (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_COMPONENT_SIZE_FOUR (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS 21:20
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_ONE (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_TWO (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_THREE (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_SRC_COMPONENTS_FOUR (0x00000003)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS 25:24
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_ONE (0x00000000)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_TWO (0x00000001)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_THREE (0x00000002)
#define NVC3B5_SET_REMAP_COMPONENTS_NUM_DST_COMPONENTS_FOUR (0x00000003)
#define NVC3B5_SET_DST_BLOCK_SIZE (0x0000070C)
#define NVC3B5_SET_DST_BLOCK_SIZE_WIDTH 3:0
#define NVC3B5_SET_DST_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT 7:4
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC3B5_SET_DST_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH 11:8
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC3B5_SET_DST_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC3B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC3B5_SET_DST_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC3B5_SET_DST_WIDTH (0x00000710)
#define NVC3B5_SET_DST_WIDTH_V 31:0
#define NVC3B5_SET_DST_HEIGHT (0x00000714)
#define NVC3B5_SET_DST_HEIGHT_V 31:0
#define NVC3B5_SET_DST_DEPTH (0x00000718)
#define NVC3B5_SET_DST_DEPTH_V 31:0
#define NVC3B5_SET_DST_LAYER (0x0000071C)
#define NVC3B5_SET_DST_LAYER_V 31:0
#define NVC3B5_SET_DST_ORIGIN (0x00000720)
#define NVC3B5_SET_DST_ORIGIN_X 15:0
#define NVC3B5_SET_DST_ORIGIN_Y 31:16
#define NVC3B5_SET_SRC_BLOCK_SIZE (0x00000728)
#define NVC3B5_SET_SRC_BLOCK_SIZE_WIDTH 3:0
#define NVC3B5_SET_SRC_BLOCK_SIZE_WIDTH_ONE_GOB (0x00000000)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT 7:4
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_ONE_GOB (0x00000000)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_TWO_GOBS (0x00000001)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_FOUR_GOBS (0x00000002)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_EIGHT_GOBS (0x00000003)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_SIXTEEN_GOBS (0x00000004)
#define NVC3B5_SET_SRC_BLOCK_SIZE_HEIGHT_THIRTYTWO_GOBS (0x00000005)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH 11:8
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_ONE_GOB (0x00000000)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_TWO_GOBS (0x00000001)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_FOUR_GOBS (0x00000002)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_EIGHT_GOBS (0x00000003)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_SIXTEEN_GOBS (0x00000004)
#define NVC3B5_SET_SRC_BLOCK_SIZE_DEPTH_THIRTYTWO_GOBS (0x00000005)
#define NVC3B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT 15:12
#define NVC3B5_SET_SRC_BLOCK_SIZE_GOB_HEIGHT_GOB_HEIGHT_FERMI_8 (0x00000001)
#define NVC3B5_SET_SRC_WIDTH (0x0000072C)
#define NVC3B5_SET_SRC_WIDTH_V 31:0
#define NVC3B5_SET_SRC_HEIGHT (0x00000730)
#define NVC3B5_SET_SRC_HEIGHT_V 31:0
#define NVC3B5_SET_SRC_DEPTH (0x00000734)
#define NVC3B5_SET_SRC_DEPTH_V 31:0
#define NVC3B5_SET_SRC_LAYER (0x00000738)
#define NVC3B5_SET_SRC_LAYER_V 31:0
#define NVC3B5_SET_SRC_ORIGIN (0x0000073C)
#define NVC3B5_SET_SRC_ORIGIN_X 15:0
#define NVC3B5_SET_SRC_ORIGIN_Y 31:16
#define NVC3B5_SRC_ORIGIN_X (0x00000744)
#define NVC3B5_SRC_ORIGIN_X_VALUE 31:0
#define NVC3B5_SRC_ORIGIN_Y (0x00000748)
#define NVC3B5_SRC_ORIGIN_Y_VALUE 31:0
#define NVC3B5_DST_ORIGIN_X (0x0000074C)
#define NVC3B5_DST_ORIGIN_X_VALUE 31:0
#define NVC3B5_DST_ORIGIN_Y (0x00000750)
#define NVC3B5_DST_ORIGIN_Y_VALUE 31:0
#define NVC3B5_PM_TRIGGER_END (0x00001114)
#define NVC3B5_PM_TRIGGER_END_V 31:0
#ifdef __cplusplus
}; /* extern "C" */
#endif
#endif // _clc3b5_h

339
kernel-open/nvidia-uvm/hwref/maxwell/gm107/dev_mmu.h
View File

@@ -1,339 +0,0 @@
/*******************************************************************************
Copyright (c) 2003-2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __gm107_dev_mmu_h__
#define __gm107_dev_mmu_h__
/* This file is autogenerated. Do not edit */
#define NV_MMU_PDE /* ----G */
#define NV_MMU_PDE_APERTURE_BIG (0*32+1):(0*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_SIZE (0*32+3):(0*32+2) /* RWXVF */
#define NV_MMU_PDE_SIZE_FULL 0x00000000 /* RW--V */
#define NV_MMU_PDE_SIZE_HALF 0x00000001 /* RW--V */
#define NV_MMU_PDE_SIZE_QUARTER 0x00000002 /* RW--V */
#define NV_MMU_PDE_SIZE_EIGHTH 0x00000003 /* RW--V */
#define NV_MMU_PDE_ADDRESS_BIG_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL (1*32+1):(1*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL (1*32+2):(1*32+2) /* RWXVF */
#define NV_MMU_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_VOL_BIG (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SMALL_SYS (1*32+31):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID (1*32+31-3):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER (1*32+31):(1*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PDE__SIZE 8
#define NV_MMU_PTE /* ----G */
#define NV_MMU_PTE_VALID (0*32+0):(0*32+0) /* RWXVF */
#define NV_MMU_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE (0*32+1):(0*32+1) /* RWXVF */
#define NV_MMU_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_READ_ONLY (0*32+2):(0*32+2) /* RWXVF */
#define NV_MMU_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ENCRYPTED (0*32+3):(0*32+3) /* RWXVF */
#define NV_MMU_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_PTE_ADDRESS_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_PTE_VOL (1*32+0):(1*32+0) /* RWXVF */
#define NV_MMU_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE (1*32+2):(1*32+1) /* RWXVF */
#define NV_MMU_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PTE_LOCK (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PTE_LOCK_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_LOCK_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_COMPTAGLINE (1*32+28):(1*32+12) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE (1*32+30):(1*32+30) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE (1*32+31):(1*32+31) /* RWXVF */
#define NV_MMU_PTE_WRITE_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PTE__SIZE 8
#define NV_MMU_PTE_COMPTAGS_NONE 0x0 /* */
#define NV_MMU_PTE_COMPTAGS_1 0x1 /* */
#define NV_MMU_PTE_COMPTAGS_2 0x2 /* */
#define NV_MMU_PTE_KIND (1*32+11):(1*32+4) /* RWXVF */
#define NV_MMU_PTE_KIND_INVALID 0xff /* R---V */
#define NV_MMU_PTE_KIND_PITCH 0x00 /* R---V */
#define NV_MMU_PTE_KIND_Z16 0x01 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2C 0x02 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2C 0x03 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2C 0x04 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2C 0x05 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2C 0x06 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2Z 0x07 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2Z 0x08 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2Z 0x09 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2Z 0x0a /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2Z 0x0b /* R---V */
#define NV_MMU_PTE_KIND_Z16_4CZ 0x0c /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_4CZ 0x0d /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_4CZ 0x0e /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_4CZ 0x0f /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_4CZ 0x10 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24 0x11 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_1Z 0x12 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_1Z 0x13 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_1Z 0x14 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_1Z 0x15 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_1Z 0x16 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CZ 0x17 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CZ 0x18 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CZ 0x19 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CZ 0x1a /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CZ 0x1b /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CS 0x1c /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CS 0x1d /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CS 0x1e /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CS 0x1f /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CS 0x20 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_4CSZV 0x21 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_4CSZV 0x22 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_4CSZV 0x23 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_4CSZV 0x24 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_4CSZV 0x25 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12 0x26 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4 0x27 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8 0x28 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24 0x29 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_1ZV 0x2e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_1ZV 0x2f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_1ZV 0x30 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_1ZV 0x31 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CS 0x32 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CS 0x33 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CS 0x34 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CS 0x35 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CZV 0x3a /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CZV 0x3b /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CZV 0x3c /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CZV 0x3d /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2ZV 0x3e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2ZV 0x3f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2ZV 0x40 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2ZV 0x41 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_4CSZV 0x42 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_4CSZV 0x43 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_4CSZV 0x44 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_4CSZV 0x45 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8 0x46 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_1Z 0x47 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_1Z 0x48 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_1Z 0x49 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_1Z 0x4a /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_1Z 0x4b /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CS 0x4c /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CS 0x4d /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CS 0x4e /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CS 0x4f /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CS 0x50 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CZ 0x51 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CZ 0x52 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CZ 0x53 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CZ 0x54 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CZ 0x55 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_4CSZV 0x56 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_4CSZV 0x57 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_4CSZV 0x58 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_4CSZV 0x59 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_4CSZV 0x5a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12 0x5b /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4 0x5c /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8 0x5d /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24 0x5e /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_1ZV 0x63 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_1ZV 0x64 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_1ZV 0x65 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_1ZV 0x66 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CS 0x67 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CS 0x68 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CS 0x69 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CS 0x6a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CZV 0x6f /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CZV 0x70 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CZV 0x71 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CZV 0x72 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2ZV 0x73 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2ZV 0x74 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2ZV 0x75 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2ZV 0x76 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_4CSZV 0x77 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_4CSZV 0x78 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_4CSZV 0x79 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_4CSZV 0x7a /* R---V */
#define NV_MMU_PTE_KIND_ZF32 0x7b /* R---V */
#define NV_MMU_PTE_KIND_ZF32_1Z 0x7c /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_1Z 0x7d /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_1Z 0x7e /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_1Z 0x7f /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_1Z 0x80 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CS 0x81 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CS 0x82 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CS 0x83 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CS 0x84 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CS 0x85 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CZ 0x86 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CZ 0x87 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CZ 0x88 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CZ 0x89 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CZ 0x8a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12 0x8b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4 0x8c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8 0x8d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24 0x8e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CS 0x8f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CS 0x90 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CS 0x91 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CS 0x92 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1ZV 0x97 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1ZV 0x98 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1ZV 0x99 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1ZV 0x9a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CZV 0x9b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CZV 0x9c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CZV 0x9d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CZV 0x9e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CS 0x9f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CS 0xa0 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CS 0xa1 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CS 0xa2 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CSZV 0xa3 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CSZV 0xa4 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CSZV 0xa5 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CSZV 0xa6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12 0xa7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4 0xa8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8 0xa9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24 0xaa /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CS 0xab /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CS 0xac /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CS 0xad /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CS 0xae /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1ZV 0xb3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1ZV 0xb4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1ZV 0xb5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1ZV 0xb6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CZV 0xb7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CZV 0xb8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CZV 0xb9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CZV 0xba /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CS 0xbb /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CS 0xbc /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CS 0xbd /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CS 0xbe /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CSZV 0xbf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CSZV 0xc0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CSZV 0xc1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CSZV 0xc2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8 0xc3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_1CS 0xc4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_1CS 0xc5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_1CS 0xc6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_1CS 0xc7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_1CS 0xc8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CSZV 0xce /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CSZV 0xcf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CSZV 0xd0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CSZV 0xd1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CSZV 0xd2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CS 0xd3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CS 0xd4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CS 0xd5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CS 0xd6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CS 0xd7 /* R---V */
#define NV_MMU_PTE_KIND_GENERIC_16BX2 0xfe /* R---V */
#define NV_MMU_PTE_KIND_C32_2C 0xd8 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBR 0xd9 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBA 0xda /* R---V */
#define NV_MMU_PTE_KIND_C32_2CRA 0xdb /* R---V */
#define NV_MMU_PTE_KIND_C32_2BRA 0xdc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2C 0xdd /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2CBR 0xde /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2CRA 0xcc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2C 0xdf /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBR 0xe0 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBA 0xe1 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CRA 0xe2 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2BRA 0xe3 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2C 0xe4 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2CRA 0xe5 /* R---V */
#define NV_MMU_PTE_KIND_C64_2C 0xe6 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBR 0xe7 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBA 0xe8 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CRA 0xe9 /* R---V */
#define NV_MMU_PTE_KIND_C64_2BRA 0xea /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2C 0xeb /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2CBR 0xec /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2CRA 0xcd /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2C 0xed /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBR 0xee /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBA 0xef /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CRA 0xf0 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2BRA 0xf1 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2C 0xf2 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2CRA 0xf3 /* R---V */
#define NV_MMU_PTE_KIND_C128_2C 0xf4 /* R---V */
#define NV_MMU_PTE_KIND_C128_2CR 0xf5 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2C 0xf6 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2CR 0xf7 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2C 0xf8 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2CR 0xf9 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2C 0xfa /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2CR 0xfb /* R---V */
#define NV_MMU_PTE_KIND_X8C24 0xfc /* R---V */
#define NV_MMU_PTE_KIND_PITCH_NO_SWIZZLE 0xfd /* R---V */
#define NV_MMU_PTE_KIND_SMSKED_MESSAGE 0xca /* R---V */
#define NV_MMU_PTE_KIND_SMHOST_MESSAGE 0xcb /* R---V */
#endif // __gm107_dev_mmu_h__

203
kernel-open/nvidia-uvm/hwref/pascal/gp100/dev_fault.h
View File

@@ -1,203 +0,0 @@
/*******************************************************************************
Copyright (c) 2003-2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __gp100_dev_fault_h__
#define __gp100_dev_fault_h__
/* This file is autogenerated. Do not edit */
#define NV_PFAULT_MMU_ENG_ID_GRAPHICS 0 /* */
#define NV_PFAULT_MMU_ENG_ID_DISPLAY 1 /* */
#define NV_PFAULT_MMU_ENG_ID_IFB 3 /* */
#define NV_PFAULT_MMU_ENG_ID_BAR1 4 /* */
#define NV_PFAULT_MMU_ENG_ID_BAR2 5 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST0 6 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST1 7 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST2 8 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST3 9 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST4 10 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST5 11 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST6 12 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST7 13 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST8 14 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST9 15 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST10 16 /* */
#define NV_PFAULT_MMU_ENG_ID_SEC 18 /* */
#define NV_PFAULT_MMU_ENG_ID_PERF 19 /* */
#define NV_PFAULT_MMU_ENG_ID_NVDEC 2 /* */
#define NV_PFAULT_MMU_ENG_ID_GRCOPY 27 /* */
#define NV_PFAULT_MMU_ENG_ID_CE0 21 /* */
#define NV_PFAULT_MMU_ENG_ID_CE1 22 /* */
#define NV_PFAULT_MMU_ENG_ID_CE2 27 /* */
#define NV_PFAULT_MMU_ENG_ID_CE3 28 /* */
#define NV_PFAULT_MMU_ENG_ID_CE4 29 /* */
#define NV_PFAULT_MMU_ENG_ID_CE5 30 /* */
#define NV_PFAULT_MMU_ENG_ID_PWR_PMU 23 /* */
#define NV_PFAULT_MMU_ENG_ID_PTP 24 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC 25 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC0 25 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC1 17 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC2 20 /* */
#define NV_PFAULT_MMU_ENG_ID_PHYSICAL 31 /* */
#define NV_PFAULT_FAULT_TYPE 4:0 /* */
#define NV_PFAULT_FAULT_TYPE_PDE 0x00000000 /* */
#define NV_PFAULT_FAULT_TYPE_PDE_SIZE 0x00000001 /* */
#define NV_PFAULT_FAULT_TYPE_PTE 0x00000002 /* */
#define NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION 0x00000003 /* */
#define NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK 0x00000004 /* */
#define NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION 0x00000005 /* */
#define NV_PFAULT_FAULT_TYPE_RO_VIOLATION 0x00000006 /* */
#define NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION 0x00000008 /* */
#define NV_PFAULT_FAULT_TYPE_WORK_CREATION 0x00000009 /* */
#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE 0x0000000a /* */
#define NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE 0x0000000b /* */
#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND 0x0000000c /* */
#define NV_PFAULT_FAULT_TYPE_REGION_VIOLATION 0x0000000d /* */
#define NV_PFAULT_FAULT_TYPE_POISONED 0x0000000e /* */
#define NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION 0x0000000f /* */
#define NV_PFAULT_CLIENT 14:8 /* */
#define NV_PFAULT_CLIENT_GPC_L1_0 0x00000000 /* */
#define NV_PFAULT_CLIENT_GPC_T1_0 0x00000001 /* */
#define NV_PFAULT_CLIENT_GPC_PE_0 0x00000002 /* */
#define NV_PFAULT_CLIENT_GPC_L1_1 0x00000003 /* */
#define NV_PFAULT_CLIENT_GPC_T1_1 0x00000004 /* */
#define NV_PFAULT_CLIENT_GPC_PE_1 0x00000005 /* */
#define NV_PFAULT_CLIENT_GPC_L1_2 0x00000006 /* */
#define NV_PFAULT_CLIENT_GPC_T1_2 0x00000007 /* */
#define NV_PFAULT_CLIENT_GPC_PE_2 0x00000008 /* */
#define NV_PFAULT_CLIENT_GPC_L1_3 0x00000009 /* */
#define NV_PFAULT_CLIENT_GPC_T1_3 0x0000000A /* */
#define NV_PFAULT_CLIENT_GPC_PE_3 0x0000000B /* */
#define NV_PFAULT_CLIENT_GPC_RAST 0x0000000C /* */
#define NV_PFAULT_CLIENT_GPC_GCC 0x0000000D /* */
#define NV_PFAULT_CLIENT_GPC_GPCCS 0x0000000E /* */
#define NV_PFAULT_CLIENT_GPC_PROP_0 0x0000000F /* */
#define NV_PFAULT_CLIENT_GPC_PROP_1 0x00000010 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_2 0x00000011 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_3 0x00000012 /* */
#define NV_PFAULT_CLIENT_GPC_L1_4 0x00000014 /* */
#define NV_PFAULT_CLIENT_GPC_T1_4 0x00000015 /* */
#define NV_PFAULT_CLIENT_GPC_PE_4 0x00000016 /* */
#define NV_PFAULT_CLIENT_GPC_L1_5 0x00000017 /* */
#define NV_PFAULT_CLIENT_GPC_T1_5 0x00000018 /* */
#define NV_PFAULT_CLIENT_GPC_PE_5 0x00000019 /* */
#define NV_PFAULT_CLIENT_GPC_L1_6 0x0000001A /* */
#define NV_PFAULT_CLIENT_GPC_T1_6 0x0000001B /* */
#define NV_PFAULT_CLIENT_GPC_PE_6 0x0000001C /* */
#define NV_PFAULT_CLIENT_GPC_L1_7 0x0000001D /* */
#define NV_PFAULT_CLIENT_GPC_T1_7 0x0000001E /* */
#define NV_PFAULT_CLIENT_GPC_PE_7 0x0000001F /* */
#define NV_PFAULT_CLIENT_GPC_L1_8 0x00000020 /* */
#define NV_PFAULT_CLIENT_GPC_T1_8 0x00000021 /* */
#define NV_PFAULT_CLIENT_GPC_PE_8 0x00000022 /* */
#define NV_PFAULT_CLIENT_GPC_L1_9 0x00000023 /* */
#define NV_PFAULT_CLIENT_GPC_T1_9 0x00000024 /* */
#define NV_PFAULT_CLIENT_GPC_PE_9 0x00000025 /* */
#define NV_PFAULT_CLIENT_GPC_L1_10 0x00000026 /* */
#define NV_PFAULT_CLIENT_GPC_T1_10 0x00000027 /* */
#define NV_PFAULT_CLIENT_GPC_PE_10 0x00000028 /* */
#define NV_PFAULT_CLIENT_GPC_L1_11 0x00000029 /* */
#define NV_PFAULT_CLIENT_GPC_T1_11 0x0000002A /* */
#define NV_PFAULT_CLIENT_GPC_PE_11 0x0000002B /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_0 0x00000030 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_1 0x00000031 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_2 0x00000032 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_3 0x00000033 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_4 0x00000034 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_5 0x00000035 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_6 0x00000036 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_7 0x00000037 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_8 0x00000038 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_9 0x00000039 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_10 0x0000003A /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_11 0x0000003B /* */
#define NV_PFAULT_CLIENT_GPC_GPM 0x00000013 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_0 0x00000014 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_1 0x00000015 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_2 0x00000016 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_3 0x00000017 /* */
#define NV_PFAULT_CLIENT_GPC_RGG_UTLB 0x00000018 /* */
#define NV_PFAULT_CLIENT_HUB_CE0 0x00000001 /* */
#define NV_PFAULT_CLIENT_HUB_CE1 0x00000002 /* */
#define NV_PFAULT_CLIENT_HUB_DNISO 0x00000003 /* */
#define NV_PFAULT_CLIENT_HUB_FE 0x00000004 /* */
#define NV_PFAULT_CLIENT_HUB_FECS 0x00000005 /* */
#define NV_PFAULT_CLIENT_HUB_HOST 0x00000006 /* */
#define NV_PFAULT_CLIENT_HUB_HOST_CPU 0x00000007 /* */
#define NV_PFAULT_CLIENT_HUB_HOST_CPU_NB 0x00000008 /* */
#define NV_PFAULT_CLIENT_HUB_ISO 0x00000009 /* */
#define NV_PFAULT_CLIENT_HUB_MMU 0x0000000A /* */
#define NV_PFAULT_CLIENT_HUB_NVDEC 0x0000000B /* */
#define NV_PFAULT_CLIENT_HUB_NVENC1 0x0000000D /* */
#define NV_PFAULT_CLIENT_HUB_NVENC2 0x00000033 /* */
#define NV_PFAULT_CLIENT_HUB_NISO 0x0000000E /* */
#define NV_PFAULT_CLIENT_HUB_P2P 0x0000000F /* */
#define NV_PFAULT_CLIENT_HUB_PD 0x00000010 /* */
#define NV_PFAULT_CLIENT_HUB_PERF 0x00000011 /* */
#define NV_PFAULT_CLIENT_HUB_PMU 0x00000012 /* */
#define NV_PFAULT_CLIENT_HUB_RASTERTWOD 0x00000013 /* */
#define NV_PFAULT_CLIENT_HUB_SCC 0x00000014 /* */
#define NV_PFAULT_CLIENT_HUB_SCC_NB 0x00000015 /* */
#define NV_PFAULT_CLIENT_HUB_SEC 0x00000016 /* */
#define NV_PFAULT_CLIENT_HUB_SSYNC 0x00000017 /* */
#define NV_PFAULT_CLIENT_HUB_VIP 0x00000000 /* */
#define NV_PFAULT_CLIENT_HUB_GRCOPY 0x00000018 /* */
#define NV_PFAULT_CLIENT_HUB_CE2 0x00000018 /* */
#define NV_PFAULT_CLIENT_HUB_XV 0x00000019 /* */
#define NV_PFAULT_CLIENT_HUB_MMU_NB 0x0000001A /* */
#define NV_PFAULT_CLIENT_HUB_NVENC 0x0000001B /* */
#define NV_PFAULT_CLIENT_HUB_NVENC0 0x0000001B /* */
#define NV_PFAULT_CLIENT_HUB_DFALCON 0x0000001C /* */
#define NV_PFAULT_CLIENT_HUB_SKED 0x0000001D /* */
#define NV_PFAULT_CLIENT_HUB_AFALCON 0x0000001E /* */
#define NV_PFAULT_CLIENT_HUB_HSCE0 0x00000020 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE1 0x00000021 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE2 0x00000022 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE3 0x00000023 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE4 0x00000024 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE5 0x00000025 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE6 0x00000026 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE7 0x00000027 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE8 0x00000028 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE9 0x00000029 /* */
#define NV_PFAULT_CLIENT_HUB_HSHUB 0x0000002A /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X0 0x0000002B /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X1 0x0000002C /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X2 0x0000002D /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X3 0x0000002E /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X4 0x0000002F /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X5 0x00000030 /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X6 0x00000031 /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X7 0x00000032 /* */
#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 0x00000034 /* */
#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 0x00000035 /* */
#define NV_PFAULT_CLIENT_HUB_DONT_CARE 0x0000001F /* */
#define NV_PFAULT_ACCESS_TYPE 18:16 /* */
#define NV_PFAULT_ACCESS_TYPE_READ 0x00000000 /* */
#define NV_PFAULT_ACCESS_TYPE_WRITE 0x00000001 /* */
#define NV_PFAULT_ACCESS_TYPE_ATOMIC 0x00000002 /* */
#define NV_PFAULT_ACCESS_TYPE_PREFETCH 0x00000003 /* */
#define NV_PFAULT_MMU_CLIENT_TYPE 20:20 /* */
#define NV_PFAULT_MMU_CLIENT_TYPE_GPC 0x00000000 /* */
#define NV_PFAULT_MMU_CLIENT_TYPE_HUB 0x00000001 /* */
#define NV_PFAULT_GPC_ID 28:24 /* */
#endif // __gp100_dev_fault_h__

71
kernel-open/nvidia-uvm/hwref/pascal/gp100/dev_fb.h
View File

@@ -1,71 +0,0 @@
/*******************************************************************************
Copyright (c) 2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
// Excerpt of gp100/dev_fb.h
#ifndef __dev_fb_h__
#define __dev_fb_h__
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_VA 0:0 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_VA_FALSE 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_VA_TRUE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_PDB 1:1 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_PDB_FALSE 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ALL_PDB_TRUE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY 5:3 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_NONE 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_START 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_START_ACK_ALL 0x00000002 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_CANCEL_TARGETED 0x00000003 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_CANCEL_GLOBAL 0x00000004 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_REPLAY_CANCEL 0x00000004 /* */
#define NV_PFB_PRI_MMU_INVALIDATE_SYS_MEMBAR 6:6 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_SYS_MEMBAR_FALSE 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_SYS_MEMBAR_TRUE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ACK 8:7 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_ACK_NONE_REQUIRED 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ACK_INTRANODE 0x00000002 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_ACK_GLOBALLY 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CANCEL_CLIENT_ID 14:9 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_CANCEL_GPC_ID 19:15 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_CANCEL_CLIENT_TYPE 20:20 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_CANCEL_CLIENT_TYPE_GPC 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CANCEL_CLIENT_TYPE_HUB 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL 26:24 /* RWXVF */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_ALL 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_PTE_ONLY 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE0 0x00000002 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE1 0x00000003 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE2 0x00000004 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE3 0x00000005 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE4 0x00000006 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_CACHE_LEVEL_UP_TO_PDE5 0x00000007 /* RW--V */
#define NV_PFB_PRI_MMU_INVALIDATE_TRIGGER 31:31 /* -WEVF */
#define NV_PFB_PRI_MMU_INVALIDATE_TRIGGER_FALSE 0x00000000 /* -WE-V */
#define NV_PFB_PRI_MMU_INVALIDATE_TRIGGER_TRUE 0x00000001 /* -W--T */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER 1:0 /* RWEVF */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER_SEND_ALL 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER_SEND_NONE 0x00000003 /* RW--V */
#endif // __dev_fb_h__

625
kernel-open/nvidia-uvm/hwref/pascal/gp100/dev_mmu.h
View File

@@ -1,625 +0,0 @@
/*******************************************************************************
Copyright (c) 2003-2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __gp100_dev_mmu_h__
#define __gp100_dev_mmu_h__
/* This file is autogenerated. Do not edit */
#define NV_MMU_PDE /* ----G */
#define NV_MMU_PDE_APERTURE_BIG (0*32+1):(0*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_SIZE (0*32+3):(0*32+2) /* RWXVF */
#define NV_MMU_PDE_SIZE_FULL 0x00000000 /* RW--V */
#define NV_MMU_PDE_SIZE_HALF 0x00000001 /* RW--V */
#define NV_MMU_PDE_SIZE_QUARTER 0x00000002 /* RW--V */
#define NV_MMU_PDE_SIZE_EIGHTH 0x00000003 /* RW--V */
#define NV_MMU_PDE_ADDRESS_BIG_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL (1*32+1):(1*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL (1*32+2):(1*32+2) /* RWXVF */
#define NV_MMU_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_VOL_BIG (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SMALL_SYS (1*32+31):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID (1*32+31-3):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER (1*32+31):(1*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PDE__SIZE 8
#define NV_MMU_PTE /* ----G */
#define NV_MMU_PTE_VALID (0*32+0):(0*32+0) /* RWXVF */
#define NV_MMU_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE (0*32+1):(0*32+1) /* RWXVF */
#define NV_MMU_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_READ_ONLY (0*32+2):(0*32+2) /* RWXVF */
#define NV_MMU_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ENCRYPTED (0*32+3):(0*32+3) /* RWXVF */
#define NV_MMU_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_PTE_ADDRESS_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_PTE_VOL (1*32+0):(1*32+0) /* RWXVF */
#define NV_MMU_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE (1*32+2):(1*32+1) /* RWXVF */
#define NV_MMU_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PTE_LOCK (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PTE_LOCK_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_LOCK_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ATOMIC_DISABLE (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_COMPTAGLINE (1*32+18+11):(1*32+12) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE (1*32+30):(1*32+30) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE (1*32+31):(1*32+31) /* RWXVF */
#define NV_MMU_PTE_WRITE_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PTE__SIZE 8
#define NV_MMU_PTE_COMPTAGS_NONE 0x0 /* */
#define NV_MMU_PTE_COMPTAGS_1 0x1 /* */
#define NV_MMU_PTE_COMPTAGS_2 0x2 /* */
#define NV_MMU_PTE_KIND (1*32+11):(1*32+4) /* RWXVF */
#define NV_MMU_PTE_KIND_INVALID 0xff /* R---V */
#define NV_MMU_PTE_KIND_PITCH 0x00 /* R---V */
#define NV_MMU_PTE_KIND_Z16 0x01 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2C 0x02 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2C 0x03 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2C 0x04 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2C 0x05 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2C 0x06 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2Z 0x07 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2Z 0x08 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2Z 0x09 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2Z 0x0a /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2Z 0x0b /* R---V */
#define NV_MMU_PTE_KIND_Z16_2CZ 0x36 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2CZ 0x37 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2CZ 0x38 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2CZ 0x39 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2CZ 0x5f /* R---V */
#define NV_MMU_PTE_KIND_Z16_4CZ 0x0c /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_4CZ 0x0d /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_4CZ 0x0e /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_4CZ 0x0f /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_4CZ 0x10 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24 0x11 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_1Z 0x12 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_1Z 0x13 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_1Z 0x14 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_1Z 0x15 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_1Z 0x16 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CZ 0x17 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CZ 0x18 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CZ 0x19 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CZ 0x1a /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CZ 0x1b /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CS 0x1c /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CS 0x1d /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CS 0x1e /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CS 0x1f /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CS 0x20 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_4CSZV 0x21 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_4CSZV 0x22 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_4CSZV 0x23 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_4CSZV 0x24 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_4CSZV 0x25 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12 0x26 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4 0x27 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8 0x28 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24 0x29 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_1ZV 0x2e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_1ZV 0x2f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_1ZV 0x30 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_1ZV 0x31 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CS 0x32 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CS 0x33 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CS 0x34 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CS 0x35 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CZV 0x3a /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CZV 0x3b /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CZV 0x3c /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CZV 0x3d /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2ZV 0x3e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2ZV 0x3f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2ZV 0x40 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2ZV 0x41 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_4CSZV 0x42 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_4CSZV 0x43 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_4CSZV 0x44 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_4CSZV 0x45 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8 0x46 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_1Z 0x47 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_1Z 0x48 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_1Z 0x49 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_1Z 0x4a /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_1Z 0x4b /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CS 0x4c /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CS 0x4d /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CS 0x4e /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CS 0x4f /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CS 0x50 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CZ 0x51 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CZ 0x52 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CZ 0x53 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CZ 0x54 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CZ 0x55 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_4CSZV 0x56 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_4CSZV 0x57 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_4CSZV 0x58 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_4CSZV 0x59 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_4CSZV 0x5a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12 0x5b /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4 0x5c /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8 0x5d /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24 0x5e /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_1ZV 0x63 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_1ZV 0x64 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_1ZV 0x65 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_1ZV 0x66 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CS 0x67 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CS 0x68 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CS 0x69 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CS 0x6a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CZV 0x6f /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CZV 0x70 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CZV 0x71 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CZV 0x72 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2ZV 0x73 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2ZV 0x74 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2ZV 0x75 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2ZV 0x76 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_4CSZV 0x77 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_4CSZV 0x78 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_4CSZV 0x79 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_4CSZV 0x7a /* R---V */
#define NV_MMU_PTE_KIND_ZF32 0x7b /* R---V */
#define NV_MMU_PTE_KIND_ZF32_1Z 0x7c /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_1Z 0x7d /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_1Z 0x7e /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_1Z 0x7f /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_1Z 0x80 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CS 0x81 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CS 0x82 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CS 0x83 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CS 0x84 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CS 0x85 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CZ 0x86 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CZ 0x87 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CZ 0x88 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CZ 0x89 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CZ 0x8a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12 0x8b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4 0x8c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8 0x8d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24 0x8e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CS 0x8f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CS 0x90 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CS 0x91 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CS 0x92 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1ZV 0x97 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1ZV 0x98 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1ZV 0x99 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1ZV 0x9a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CZV 0x9b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CZV 0x9c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CZV 0x9d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CZV 0x9e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CS 0x9f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CS 0xa0 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CS 0xa1 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CS 0xa2 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CSZV 0xa3 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CSZV 0xa4 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CSZV 0xa5 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CSZV 0xa6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12 0xa7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4 0xa8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8 0xa9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24 0xaa /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CS 0xab /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CS 0xac /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CS 0xad /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CS 0xae /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1ZV 0xb3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1ZV 0xb4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1ZV 0xb5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1ZV 0xb6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CZV 0xb7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CZV 0xb8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CZV 0xb9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CZV 0xba /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CS 0xbb /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CS 0xbc /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CS 0xbd /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CS 0xbe /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CSZV 0xbf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CSZV 0xc0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CSZV 0xc1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CSZV 0xc2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8 0xc3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_1CS 0xc4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_1CS 0xc5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_1CS 0xc6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_1CS 0xc7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_1CS 0xc8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CSZV 0xce /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CSZV 0xcf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CSZV 0xd0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CSZV 0xd1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CSZV 0xd2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CS 0xd3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CS 0xd4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CS 0xd5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CS 0xd6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CS 0xd7 /* R---V */
#define NV_MMU_PTE_KIND_S8 0x2a /* R---V */
#define NV_MMU_PTE_KIND_S8_2S 0x2b /* R---V */
#define NV_MMU_PTE_KIND_GENERIC_16BX2 0xfe /* R---V */
#define NV_MMU_PTE_KIND_C32_2C 0xd8 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBR 0xd9 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBA 0xda /* R---V */
#define NV_MMU_PTE_KIND_C32_2CRA 0xdb /* R---V */
#define NV_MMU_PTE_KIND_C32_2BRA 0xdc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2C 0xdd /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2CBR 0xde /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2CRA 0xcc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2C 0xdf /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBR 0xe0 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBA 0xe1 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CRA 0xe2 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2BRA 0xe3 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_4CBRA 0x2c /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2C 0xe4 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2CRA 0xe5 /* R---V */
#define NV_MMU_PTE_KIND_C64_2C 0xe6 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBR 0xe7 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBA 0xe8 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CRA 0xe9 /* R---V */
#define NV_MMU_PTE_KIND_C64_2BRA 0xea /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2C 0xeb /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2CBR 0xec /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2CRA 0xcd /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2C 0xed /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBR 0xee /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBA 0xef /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CRA 0xf0 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2BRA 0xf1 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_4CBRA 0x2d /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2C 0xf2 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2CRA 0xf3 /* R---V */
#define NV_MMU_PTE_KIND_C128_2C 0xf4 /* R---V */
#define NV_MMU_PTE_KIND_C128_2CR 0xf5 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2C 0xf6 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2CR 0xf7 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2C 0xf8 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2CR 0xf9 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2C 0xfa /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2CR 0xfb /* R---V */
#define NV_MMU_PTE_KIND_X8C24 0xfc /* R---V */
#define NV_MMU_PTE_KIND_PITCH_NO_SWIZZLE 0xfd /* R---V */
#define NV_MMU_PTE_KIND_SMSKED_MESSAGE 0xca /* R---V */
#define NV_MMU_PTE_KIND_SMHOST_MESSAGE 0xcb /* R---V */
#define NV_MMU_VER1_PDE /* ----G */
#define NV_MMU_VER1_PDE_APERTURE_BIG (0*32+1):(0*32+0) /* RWXVF */
#define NV_MMU_VER1_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE (0*32+3):(0*32+2) /* RWXVF */
#define NV_MMU_VER1_PDE_SIZE_FULL 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_HALF 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_QUARTER 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_EIGHTH 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL (1*32+1):(1*32+0) /* RWXVF */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_SMALL (1*32+2):(1*32+2) /* RWXVF */
#define NV_MMU_VER1_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_BIG (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_VER1_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_SYS (1*32+31):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID (1*32+31-3):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER (1*32+31):(1*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER1_PDE__SIZE 8
#define NV_MMU_VER1_PTE /* ----G */
#define NV_MMU_VER1_PTE_VALID (0*32+0):(0*32+0) /* RWXVF */
#define NV_MMU_VER1_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_PRIVILEGE (0*32+1):(0*32+1) /* RWXVF */
#define NV_MMU_VER1_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_READ_ONLY (0*32+2):(0*32+2) /* RWXVF */
#define NV_MMU_VER1_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_ENCRYPTED (0*32+3):(0*32+3) /* RWXVF */
#define NV_MMU_VER1_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_VER1_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_VER1_PTE_ADDRESS_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_VER1_PTE_VOL (1*32+0):(1*32+0) /* RWXVF */
#define NV_MMU_VER1_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE (1*32+2):(1*32+1) /* RWXVF */
#define NV_MMU_VER1_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_COMPTAGLINE (1*32+18+11):(1*32+12) /* RWXVF */
#define NV_MMU_VER1_PTE_KIND (1*32+11):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER1_PTE__SIZE 8
#define NV_MMU_VER1_PTE_COMPTAGS_NONE 0x0 /* */
#define NV_MMU_VER1_PTE_COMPTAGS_1 0x1 /* */
#define NV_MMU_VER1_PTE_COMPTAGS_2 0x2 /* */
#define NV_MMU_NEW_PDE /* ----G */
#define NV_MMU_NEW_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_NEW_PDE_APERTURE_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_PDE_VOL 3:3 /* RWXVF */
#define NV_MMU_NEW_PDE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_PDE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_PDE__SIZE 8
#define NV_MMU_NEW_DUAL_PDE /* ----G */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG 2:1 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG 3:3 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SYS 53:(8-4) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID (35-3):(8-4) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL 66:65 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL 67:67 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_SYS 117:72 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID (99-3):72 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /* */
#define NV_MMU_NEW_DUAL_PDE__SIZE 16
#define NV_MMU_NEW_PTE /* ----G */
#define NV_MMU_NEW_PTE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_NEW_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_PTE_VOL 3:3 /* RWXVF */
#define NV_MMU_NEW_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_ENCRYPTED 4:4 /* RWXVF */
#define NV_MMU_NEW_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_NEW_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_NEW_PTE_PRIVILEGE 5:5 /* RWXVF */
#define NV_MMU_NEW_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_READ_ONLY 6:6 /* RWXVF */
#define NV_MMU_NEW_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE 7:7 /* RWXVF */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_NEW_PTE_COMPTAGLINE (18+35):36 /* RWXVF */
#define NV_MMU_NEW_PTE_KIND 63:56 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_PTE__SIZE 8
#define NV_MMU_VER2_PDE /* ----G */
#define NV_MMU_VER2_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_VER2_PDE_APERTURE_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_PDE_VOL 3:3 /* RWXVF */
#define NV_MMU_VER2_PDE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_PDE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_PDE__SIZE 8
#define NV_MMU_VER2_DUAL_PDE /* ----G */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG 2:1 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG 3:3 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SYS 53:(8-4) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID (35-3):(8-4) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL 66:65 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL 67:67 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_SYS 117:72 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID (99-3):72 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /* */
#define NV_MMU_VER2_DUAL_PDE__SIZE 16
#define NV_MMU_VER2_PTE /* ----G */
#define NV_MMU_VER2_PTE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_PTE_VOL 3:3 /* RWXVF */
#define NV_MMU_VER2_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_ENCRYPTED 4:4 /* RWXVF */
#define NV_MMU_VER2_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_VER2_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_VER2_PTE_PRIVILEGE 5:5 /* RWXVF */
#define NV_MMU_VER2_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_READ_ONLY 6:6 /* RWXVF */
#define NV_MMU_VER2_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE 7:7 /* RWXVF */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_VER2_PTE_COMPTAGLINE (18+35):36 /* RWXVF */
#define NV_MMU_VER2_PTE_KIND 63:56 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_PTE__SIZE 8
#endif // __gp100_dev_mmu_h__

103
kernel-open/nvidia-uvm/hwref/turing/tu102/dev_fb.h Normal file
View File

@@ -0,0 +1,103 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2003-2022 NVIDIA CORPORATION & AFFILIATES
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __tu102_dev_fb_h__
#define __tu102_dev_fb_h__
#define NV_PFB_NISO_ACCESS_COUNTER_NOTIFY_BUFFER_INFO 0x00100A18 /* R--4R */
#define NV_PFB_NISO_ACCESS_COUNTER_NOTIFY_BUFFER_INFO_FULL 0:0 /* R-IVF */
#define NV_PFB_NISO_ACCESS_COUNTER_NOTIFY_BUFFER_INFO_FULL_FALSE 0x0 /* R-I-V */
#define NV_PFB_NISO_ACCESS_COUNTER_NOTIFY_BUFFER_INFO_FULL_TRUE 0x1 /* R---V */
#define NV_PFB_PRI_MMU_INT_VECTOR_FAULT_NOTIFY_REPLAYABLE 64 /* R---V */
#define NV_PFB_PRI_MMU_INT_VECTOR_FAULT_NOTIFY_NON_REPLAYABLE 132 /* R---V */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL 0x00100CF8 /* RW-4R */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER 1:0 /* RWEVF */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER_SEND_ALL 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_PAGE_FAULT_CTRL_PRF_FILTER_SEND_NONE 0x00000003 /* RWE-V */
#define NV_PFB_PRI_MMU_WPR2_ADDR_LO 0x001FA824 /* RW-4R */
#define NV_PFB_PRI_MMU_WPR2_ADDR_LO_VAL 31:4 /* RWEVF */
#define NV_PFB_PRI_MMU_WPR2_ADDR_LO_ALIGNMENT 0x0000000c /* */
#define NV_PFB_PRI_MMU_WPR2_ADDR_HI 0x001FA828 /* RW-4R */
#define NV_PFB_PRI_MMU_WPR2_ADDR_HI_VAL 31:4 /* RWEVF */
#define NV_PFB_PRI_MMU_WPR2_ADDR_HI_ALIGNMENT 0x0000000c /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET(i) (0x00100E2C+(i)*20) /* RW-4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_PTR 19:0 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_PTR_RESET 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED 30:30 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_CLEAR 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW 31:31 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_CLEAR 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT(i) (0x00100E30+(i)*20) /* R--4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_PTR 19:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_PTR_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED 30:30 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED_NO 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED_YES 0x00000001 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW 31:31 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW_NO 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW_YES 0x00000001 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_VAL 19:0 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_VAL_RESET 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR 29:29 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR_DISABLE 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR_ENABLE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT 30:30 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE 31:31 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE_FALSE 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE_TRUE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT 0x00100E78 /* RW-4R */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT 0x00100E78 /* RW-4R */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT_TOTAL 15:0 /* RWEVF */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT_TOTAL_INIT 0 /* RWE-V */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT_UNIQUE 31:16 /* RWEVF */
#define NV_PFB_PRI_MMU_L2TLB_ECC_UNCORRECTED_ERR_COUNT_UNIQUE_INIT 0 /* RWE-V */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT 0x00100E8C /* RW-4R */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT 0x00100E8C /* RW-4R */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT_TOTAL 15:0 /* RWEVF */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT_TOTAL_INIT 0 /* RWE-V */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT_UNIQUE 31:16 /* RWEVF */
#define NV_PFB_PRI_MMU_HUBTLB_ECC_UNCORRECTED_ERR_COUNT_UNIQUE_INIT 0 /* RWE-V */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT 0x00100EA0 /* RW-4R */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT 0x00100EA0 /* RW-4R */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT_TOTAL 15:0 /* RWEVF */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT_TOTAL_INIT 0 /* RWE-V */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT_UNIQUE 31:16 /* RWEVF */
#define NV_PFB_PRI_MMU_FILLUNIT_ECC_UNCORRECTED_ERR_COUNT_UNIQUE_INIT 0 /* RWE-V */
#endif // __tu102_dev_fb_h__

263
kernel-open/nvidia-uvm/hwref/volta/gv100/dev_fault.h
View File

@@ -1,263 +0,0 @@
/*******************************************************************************
Copyright (c) 2003-2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __gv100_dev_fault_h__
#define __gv100_dev_fault_h__
/* This file is autogenerated. Do not edit */
#define NV_PFAULT_MMU_ENG_ID_GRAPHICS 64 /* */
#define NV_PFAULT_MMU_ENG_ID_DISPLAY 1 /* */
#define NV_PFAULT_MMU_ENG_ID_GSP 2 /* */
#define NV_PFAULT_MMU_ENG_ID_IFB 8 /* */
#define NV_PFAULT_MMU_ENG_ID_BAR1 4 /* */
#define NV_PFAULT_MMU_ENG_ID_BAR2 5 /* */
#define NV_PFAULT_MMU_ENG_ID_SEC 14 /* */
#define NV_PFAULT_MMU_ENG_ID_PERF 9 /* */
#define NV_PFAULT_MMU_ENG_ID_NVDEC 10 /* */
#define NV_PFAULT_MMU_ENG_ID_GRCOPY 15 /* */
#define NV_PFAULT_MMU_ENG_ID_CE0 15 /* */
#define NV_PFAULT_MMU_ENG_ID_CE1 16 /* */
#define NV_PFAULT_MMU_ENG_ID_CE2 17 /* */
#define NV_PFAULT_MMU_ENG_ID_CE3 18 /* */
#define NV_PFAULT_MMU_ENG_ID_CE4 19 /* */
#define NV_PFAULT_MMU_ENG_ID_CE5 20 /* */
#define NV_PFAULT_MMU_ENG_ID_CE6 21 /* */
#define NV_PFAULT_MMU_ENG_ID_CE7 22 /* */
#define NV_PFAULT_MMU_ENG_ID_CE8 23 /* */
#define NV_PFAULT_MMU_ENG_ID_PWR_PMU 6 /* */
#define NV_PFAULT_MMU_ENG_ID_PTP 3 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC0 11 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC1 12 /* */
#define NV_PFAULT_MMU_ENG_ID_NVENC2 13 /* */
#define NV_PFAULT_MMU_ENG_ID_PHYSICAL 31 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST0 32 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST1 33 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST2 34 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST3 35 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST4 36 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST5 37 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST6 38 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST7 39 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST8 40 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST9 41 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST10 42 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST11 43 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST12 44 /* */
#define NV_PFAULT_MMU_ENG_ID_HOST13 45 /* */
#define NV_PFAULT_FAULT_TYPE 4:0 /* */
#define NV_PFAULT_FAULT_TYPE_PDE 0x00000000 /* */
#define NV_PFAULT_FAULT_TYPE_PDE_SIZE 0x00000001 /* */
#define NV_PFAULT_FAULT_TYPE_PTE 0x00000002 /* */
#define NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION 0x00000003 /* */
#define NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK 0x00000004 /* */
#define NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION 0x00000005 /* */
#define NV_PFAULT_FAULT_TYPE_RO_VIOLATION 0x00000006 /* */
#define NV_PFAULT_FAULT_TYPE_WO_VIOLATION 0x00000007 /* */
#define NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION 0x00000008 /* */
#define NV_PFAULT_FAULT_TYPE_WORK_CREATION 0x00000009 /* */
#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE 0x0000000a /* */
#define NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE 0x0000000b /* */
#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND 0x0000000c /* */
#define NV_PFAULT_FAULT_TYPE_REGION_VIOLATION 0x0000000d /* */
#define NV_PFAULT_FAULT_TYPE_POISONED 0x0000000e /* */
#define NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION 0x0000000f /* */
#define NV_PFAULT_CLIENT 14:8 /* */
#define NV_PFAULT_CLIENT_GPC_T1_0 0x00000000 /* */
#define NV_PFAULT_CLIENT_GPC_T1_1 0x00000001 /* */
#define NV_PFAULT_CLIENT_GPC_T1_2 0x00000002 /* */
#define NV_PFAULT_CLIENT_GPC_T1_3 0x00000003 /* */
#define NV_PFAULT_CLIENT_GPC_T1_4 0x00000004 /* */
#define NV_PFAULT_CLIENT_GPC_T1_5 0x00000005 /* */
#define NV_PFAULT_CLIENT_GPC_T1_6 0x00000006 /* */
#define NV_PFAULT_CLIENT_GPC_T1_7 0x00000007 /* */
#define NV_PFAULT_CLIENT_GPC_PE_0 0x00000008 /* */
#define NV_PFAULT_CLIENT_GPC_PE_1 0x00000009 /* */
#define NV_PFAULT_CLIENT_GPC_PE_2 0x0000000A /* */
#define NV_PFAULT_CLIENT_GPC_PE_3 0x0000000B /* */
#define NV_PFAULT_CLIENT_GPC_PE_4 0x0000000C /* */
#define NV_PFAULT_CLIENT_GPC_PE_5 0x0000000D /* */
#define NV_PFAULT_CLIENT_GPC_PE_6 0x0000000E /* */
#define NV_PFAULT_CLIENT_GPC_PE_7 0x0000000F /* */
#define NV_PFAULT_CLIENT_GPC_RAST 0x00000010 /* */
#define NV_PFAULT_CLIENT_GPC_GCC 0x00000011 /* */
#define NV_PFAULT_CLIENT_GPC_GPCCS 0x00000012 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_0 0x00000013 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_1 0x00000014 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_2 0x00000015 /* */
#define NV_PFAULT_CLIENT_GPC_PROP_3 0x00000016 /* */
#define NV_PFAULT_CLIENT_GPC_T1_8 0x00000021 /* */
#define NV_PFAULT_CLIENT_GPC_T1_9 0x00000022 /* */
#define NV_PFAULT_CLIENT_GPC_T1_10 0x00000023 /* */
#define NV_PFAULT_CLIENT_GPC_T1_11 0x00000024 /* */
#define NV_PFAULT_CLIENT_GPC_T1_12 0x00000025 /* */
#define NV_PFAULT_CLIENT_GPC_T1_13 0x00000026 /* */
#define NV_PFAULT_CLIENT_GPC_T1_14 0x00000027 /* */
#define NV_PFAULT_CLIENT_GPC_T1_15 0x00000028 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_0 0x00000029 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_1 0x0000002A /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_2 0x0000002B /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_3 0x0000002C /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_4 0x0000002D /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_5 0x0000002E /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_6 0x0000002F /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_7 0x00000030 /* */
#define NV_PFAULT_CLIENT_GPC_PE_8 0x00000031 /* */
#define NV_PFAULT_CLIENT_GPC_PE_9 0x00000032 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_8 0x00000033 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_9 0x00000034 /* */
#define NV_PFAULT_CLIENT_GPC_T1_16 0x00000035 /* */
#define NV_PFAULT_CLIENT_GPC_T1_17 0x00000036 /* */
#define NV_PFAULT_CLIENT_GPC_T1_18 0x00000037 /* */
#define NV_PFAULT_CLIENT_GPC_T1_19 0x00000038 /* */
#define NV_PFAULT_CLIENT_GPC_PE_10 0x00000039 /* */
#define NV_PFAULT_CLIENT_GPC_PE_11 0x0000003A /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_10 0x0000003B /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_11 0x0000003C /* */
#define NV_PFAULT_CLIENT_GPC_T1_20 0x0000003D /* */
#define NV_PFAULT_CLIENT_GPC_T1_21 0x0000003E /* */
#define NV_PFAULT_CLIENT_GPC_T1_22 0x0000003F /* */
#define NV_PFAULT_CLIENT_GPC_T1_23 0x00000040 /* */
#define NV_PFAULT_CLIENT_GPC_PE_12 0x00000041 /* */
#define NV_PFAULT_CLIENT_GPC_PE_13 0x00000042 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_12 0x00000043 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_13 0x00000044 /* */
#define NV_PFAULT_CLIENT_GPC_T1_24 0x00000045 /* */
#define NV_PFAULT_CLIENT_GPC_T1_25 0x00000046 /* */
#define NV_PFAULT_CLIENT_GPC_T1_26 0x00000047 /* */
#define NV_PFAULT_CLIENT_GPC_T1_27 0x00000048 /* */
#define NV_PFAULT_CLIENT_GPC_PE_14 0x00000049 /* */
#define NV_PFAULT_CLIENT_GPC_PE_15 0x0000004A /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_14 0x0000004B /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_15 0x0000004C /* */
#define NV_PFAULT_CLIENT_GPC_T1_28 0x0000004D /* */
#define NV_PFAULT_CLIENT_GPC_T1_29 0x0000004E /* */
#define NV_PFAULT_CLIENT_GPC_T1_30 0x0000004F /* */
#define NV_PFAULT_CLIENT_GPC_T1_31 0x00000050 /* */
#define NV_PFAULT_CLIENT_GPC_PE_16 0x00000051 /* */
#define NV_PFAULT_CLIENT_GPC_PE_17 0x00000052 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_16 0x00000053 /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_17 0x00000054 /* */
#define NV_PFAULT_CLIENT_GPC_T1_32 0x00000055 /* */
#define NV_PFAULT_CLIENT_GPC_T1_33 0x00000056 /* */
#define NV_PFAULT_CLIENT_GPC_T1_34 0x00000057 /* */
#define NV_PFAULT_CLIENT_GPC_T1_35 0x00000058 /* */
#define NV_PFAULT_CLIENT_GPC_PE_18 0x00000059 /* */
#define NV_PFAULT_CLIENT_GPC_PE_19 0x0000005A /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_18 0x0000005B /* */
#define NV_PFAULT_CLIENT_GPC_TPCCS_19 0x0000005C /* */
#define NV_PFAULT_CLIENT_GPC_T1_36 0x0000005D /* */
#define NV_PFAULT_CLIENT_GPC_T1_37 0x0000005E /* */
#define NV_PFAULT_CLIENT_GPC_T1_38 0x0000005F /* */
#define NV_PFAULT_CLIENT_GPC_T1_39 0x00000060 /* */
#define NV_PFAULT_CLIENT_GPC_GPM 0x00000017 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_0 0x00000018 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_1 0x00000019 /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_2 0x0000001A /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_3 0x0000001B /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_4 0x0000001C /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_5 0x0000001D /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_6 0x0000001E /* */
#define NV_PFAULT_CLIENT_GPC_LTP_UTLB_7 0x0000001F /* */
#define NV_PFAULT_CLIENT_GPC_RGG_UTLB 0x00000020 /* */
#define NV_PFAULT_CLIENT_HUB_CE0 0x00000001 /* */
#define NV_PFAULT_CLIENT_HUB_CE1 0x00000002 /* */
#define NV_PFAULT_CLIENT_HUB_DNISO 0x00000003 /* */
#define NV_PFAULT_CLIENT_HUB_FE 0x00000004 /* */
#define NV_PFAULT_CLIENT_HUB_FECS 0x00000005 /* */
#define NV_PFAULT_CLIENT_HUB_HOST 0x00000006 /* */
#define NV_PFAULT_CLIENT_HUB_HOST_CPU 0x00000007 /* */
#define NV_PFAULT_CLIENT_HUB_HOST_CPU_NB 0x00000008 /* */
#define NV_PFAULT_CLIENT_HUB_ISO 0x00000009 /* */
#define NV_PFAULT_CLIENT_HUB_MMU 0x0000000A /* */
#define NV_PFAULT_CLIENT_HUB_NVDEC 0x0000000B /* */
#define NV_PFAULT_CLIENT_HUB_NVENC1 0x0000000D /* */
#define NV_PFAULT_CLIENT_HUB_NVENC2 0x00000033 /* */
#define NV_PFAULT_CLIENT_HUB_NISO 0x0000000E /* */
#define NV_PFAULT_CLIENT_HUB_P2P 0x0000000F /* */
#define NV_PFAULT_CLIENT_HUB_PD 0x00000010 /* */
#define NV_PFAULT_CLIENT_HUB_PERF 0x00000011 /* */
#define NV_PFAULT_CLIENT_HUB_PMU 0x00000012 /* */
#define NV_PFAULT_CLIENT_HUB_RASTERTWOD 0x00000013 /* */
#define NV_PFAULT_CLIENT_HUB_SCC 0x00000014 /* */
#define NV_PFAULT_CLIENT_HUB_SCC_NB 0x00000015 /* */
#define NV_PFAULT_CLIENT_HUB_SEC 0x00000016 /* */
#define NV_PFAULT_CLIENT_HUB_SSYNC 0x00000017 /* */
#define NV_PFAULT_CLIENT_HUB_VIP 0x00000000 /* */
#define NV_PFAULT_CLIENT_HUB_GRCOPY 0x00000018 /* */
#define NV_PFAULT_CLIENT_HUB_CE2 0x00000018 /* */
#define NV_PFAULT_CLIENT_HUB_XV 0x00000019 /* */
#define NV_PFAULT_CLIENT_HUB_MMU_NB 0x0000001A /* */
#define NV_PFAULT_CLIENT_HUB_NVENC 0x0000001B /* */
#define NV_PFAULT_CLIENT_HUB_NVENC0 0x0000001B /* */
#define NV_PFAULT_CLIENT_HUB_DFALCON 0x0000001C /* */
#define NV_PFAULT_CLIENT_HUB_SKED 0x0000001D /* */
#define NV_PFAULT_CLIENT_HUB_AFALCON 0x0000001E /* */
#define NV_PFAULT_CLIENT_HUB_HSCE0 0x00000020 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE1 0x00000021 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE2 0x00000022 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE3 0x00000023 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE4 0x00000024 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE5 0x00000025 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE6 0x00000026 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE7 0x00000027 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE8 0x00000028 /* */
#define NV_PFAULT_CLIENT_HUB_HSCE9 0x00000029 /* */
#define NV_PFAULT_CLIENT_HUB_HSHUB 0x0000002A /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X0 0x0000002B /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X1 0x0000002C /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X2 0x0000002D /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X3 0x0000002E /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X4 0x0000002F /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X5 0x00000030 /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X6 0x00000031 /* */
#define NV_PFAULT_CLIENT_HUB_PTP_X7 0x00000032 /* */
#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER0 0x00000034 /* */
#define NV_PFAULT_CLIENT_HUB_VPR_SCRUBBER1 0x00000035 /* */
#define NV_PFAULT_CLIENT_HUB_DWBIF 0x00000036 /* */
#define NV_PFAULT_CLIENT_HUB_FBFALCON 0x00000037 /* */
#define NV_PFAULT_CLIENT_HUB_CE_SHIM 0x00000038 /* */
#define NV_PFAULT_CLIENT_HUB_GSP 0x00000039 /* */
#define NV_PFAULT_CLIENT_HUB_DONT_CARE 0x0000001F /* */
#define NV_PFAULT_ACCESS_TYPE 19:16 /* */
#define NV_PFAULT_ACCESS_TYPE_READ 0x00000000 /* */
#define NV_PFAULT_ACCESS_TYPE_WRITE 0x00000001 /* */
#define NV_PFAULT_ACCESS_TYPE_ATOMIC 0x00000002 /* */
#define NV_PFAULT_ACCESS_TYPE_PREFETCH 0x00000003 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_READ 0x00000000 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_WRITE 0x00000001 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC 0x00000002 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG 0x00000002 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH 0x00000003 /* */
#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK 0x00000004 /* */
#define NV_PFAULT_ACCESS_TYPE_PHYS_READ 0x00000008 /* */
#define NV_PFAULT_ACCESS_TYPE_PHYS_WRITE 0x00000009 /* */
#define NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC 0x0000000a /* */
#define NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH 0x0000000b /* */
#define NV_PFAULT_MMU_CLIENT_TYPE 20:20 /* */
#define NV_PFAULT_MMU_CLIENT_TYPE_GPC 0x00000000 /* */
#define NV_PFAULT_MMU_CLIENT_TYPE_HUB 0x00000001 /* */
#define NV_PFAULT_GPC_ID 28:24 /* */
#define NV_PFAULT_PROTECTED_MODE 29:29 /* */
#define NV_PFAULT_ATS_FAULT 30:30 /* */
#define NV_PFAULT_VALID 31:31 /* */
#endif // __gv100_dev_fault_h__

103
kernel-open/nvidia-uvm/hwref/volta/gv100/dev_fb.h
View File

@@ -1,103 +0,0 @@
/*******************************************************************************
Copyright (c) 2017 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
// Excerpt of gv100/dev_fb.h
#ifndef __dev_fb_h__
#define __dev_fb_h__
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO(i) (0x00100E24+(i)*20) /* RW-4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_ADDR_MODE 0:0 /* RW-VF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_ADDR_MODE_VIRTUAL 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_ADDR_MODE_PHYSICAL 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_PHYS_APERTURE 2:1 /* RW-VF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_PHYS_APERTURE_LOCAL 0x00000000 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_PHYS_APERTURE_SYS_COH 0x00000002 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_PHYS_APERTURE_SYS_NCOH 0x00000003 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_PHYS_VOL 3:3 /* RW-VF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_LO_ADDR 31:12 /* RW-VF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_HI(i) (0x00100E28+(i)*20) /* RW-4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_HI__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_HI_ADDR 31:0 /* RW-VF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET(i) (0x00100E2C+(i)*20) /* RW-4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_PTR 19:0 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_PTR_RESET 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED 30:30 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_GETPTR_CORRUPTED_CLEAR 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW 31:31 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_GET_OVERFLOW_CLEAR 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT(i) (0x00100E30+(i)*20) /* R--4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_PTR 19:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_PTR_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED 30:30 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED_NO 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_GETPTR_CORRUPTED_YES 0x00000001 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW 31:31 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW_NO 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_PUT_OVERFLOW_YES 0x00000001 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE(i) (0x00100E34+(i)*20) /* RW-4A */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE__SIZE_1 2 /* */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_VAL 19:0 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_VAL_RESET 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR 29:29 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR_DISABLE 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_OVERFLOW_INTR_ENABLE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT 30:30 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT_NO 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_SET_DEFAULT_YES 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE 31:31 /* RWEVF */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE_FALSE 0x00000000 /* RWE-V */
#define NV_PFB_PRI_MMU_FAULT_BUFFER_SIZE_ENABLE_TRUE 0x00000001 /* RW--V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO 0x00100E4C /* R--4R */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_PHYS_APERTURE 1:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_PHYS_APERTURE_LOCAL 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_PHYS_APERTURE_PEER 0x00000001 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_PHYS_APERTURE_SYS_COH 0x00000002 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_PHYS_APERTURE_SYS_NCOH 0x00000003 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_ADDR 31:12 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_ADDR_LO_ADDR_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_ADDR_HI 0x00100E50 /* R--4R */
#define NV_PFB_PRI_MMU_FAULT_ADDR_HI_ADDR 31:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_ADDR_HI_ADDR_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO 0x00100E54 /* R--4R */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_ENGINE_ID 8:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_ENGINE_ID_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_APERTURE 11:10 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_APERTURE_VID_MEM 0x00000000 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_APERTURE_SYS_MEM_COHERENT 0x00000002 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_APERTURE_SYS_MEM_NONCOHERENT 0x00000003 /* R---V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_APERTURE_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_ADDR 31:12 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_INST_LO_ADDR_RESET 0x00000000 /* R-E-V */
#define NV_PFB_PRI_MMU_FAULT_INST_HI 0x00100E58 /* R--4R */
#define NV_PFB_PRI_MMU_FAULT_INST_HI_ADDR 31:0 /* R-EVF */
#define NV_PFB_PRI_MMU_FAULT_INST_HI_ADDR_RESET 0x00000000 /* R-E-V */
#endif

661
kernel-open/nvidia-uvm/hwref/volta/gv100/dev_mmu.h
View File

@@ -1,661 +0,0 @@
/*******************************************************************************
Copyright (c) 2003-2016 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __gv100_dev_mmu_h__
#define __gv100_dev_mmu_h__
/* This file is autogenerated. Do not edit */
#define NV_MMU_PDE /* ----G */
#define NV_MMU_PDE_APERTURE_BIG (0*32+1):(0*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_SIZE (0*32+3):(0*32+2) /* RWXVF */
#define NV_MMU_PDE_SIZE_FULL 0x00000000 /* RW--V */
#define NV_MMU_PDE_SIZE_HALF 0x00000001 /* RW--V */
#define NV_MMU_PDE_SIZE_QUARTER 0x00000002 /* RW--V */
#define NV_MMU_PDE_SIZE_EIGHTH 0x00000003 /* RW--V */
#define NV_MMU_PDE_ADDRESS_BIG_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL (1*32+1):(1*32+0) /* RWXVF */
#define NV_MMU_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL (1*32+2):(1*32+2) /* RWXVF */
#define NV_MMU_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_VOL_BIG (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SMALL_SYS (1*32+31):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID (1*32+31-3):(1*32+4) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER (1*32+31):(1*32+32-3) /* RWXVF */
#define NV_MMU_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PDE__SIZE 8
#define NV_MMU_PTE /* ----G */
#define NV_MMU_PTE_VALID (0*32+0):(0*32+0) /* RWXVF */
#define NV_MMU_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE (0*32+1):(0*32+1) /* RWXVF */
#define NV_MMU_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_READ_ONLY (0*32+2):(0*32+2) /* RWXVF */
#define NV_MMU_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ENCRYPTED (0*32+3):(0*32+3) /* RWXVF */
#define NV_MMU_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_PTE_ADDRESS_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_PTE_VOL (1*32+0):(1*32+0) /* RWXVF */
#define NV_MMU_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE (1*32+2):(1*32+1) /* RWXVF */
#define NV_MMU_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_PTE_LOCK (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PTE_LOCK_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_LOCK_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ATOMIC_DISABLE (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_COMPTAGLINE (1*32+18+11):(1*32+12) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE (1*32+30):(1*32+30) /* RWXVF */
#define NV_MMU_PTE_READ_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_READ_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE (1*32+31):(1*32+31) /* RWXVF */
#define NV_MMU_PTE_WRITE_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_PTE_WRITE_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_PTE__SIZE 8
#define NV_MMU_PTE_COMPTAGS_NONE 0x0 /* */
#define NV_MMU_PTE_COMPTAGS_1 0x1 /* */
#define NV_MMU_PTE_COMPTAGS_2 0x2 /* */
#define NV_MMU_PTE_KIND (1*32+11):(1*32+4) /* RWXVF */
#define NV_MMU_PTE_KIND_INVALID 0xff /* R---V */
#define NV_MMU_PTE_KIND_PITCH 0x00 /* R---V */
#define NV_MMU_PTE_KIND_Z16 0x01 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2C 0x02 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2C 0x03 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2C 0x04 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2C 0x05 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2C 0x06 /* R---V */
#define NV_MMU_PTE_KIND_Z16_2Z 0x07 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2Z 0x08 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2Z 0x09 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2Z 0x0a /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2Z 0x0b /* R---V */
#define NV_MMU_PTE_KIND_Z16_2CZ 0x36 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_2CZ 0x37 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_2CZ 0x38 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_2CZ 0x39 /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_2CZ 0x5f /* R---V */
#define NV_MMU_PTE_KIND_Z16_4CZ 0x0c /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS2_4CZ 0x0d /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS4_4CZ 0x0e /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS8_4CZ 0x0f /* R---V */
#define NV_MMU_PTE_KIND_Z16_MS16_4CZ 0x10 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24 0x11 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_1Z 0x12 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_1Z 0x13 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_1Z 0x14 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_1Z 0x15 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_1Z 0x16 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CZ 0x17 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CZ 0x18 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CZ 0x19 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CZ 0x1a /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CZ 0x1b /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_2CS 0x1c /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_2CS 0x1d /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_2CS 0x1e /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_2CS 0x1f /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_2CS 0x20 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_4CSZV 0x21 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS2_4CSZV 0x22 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS4_4CSZV 0x23 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS8_4CSZV 0x24 /* R---V */
#define NV_MMU_PTE_KIND_S8Z24_MS16_4CSZV 0x25 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12 0x26 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4 0x27 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8 0x28 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24 0x29 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_1ZV 0x2e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_1ZV 0x2f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_1ZV 0x30 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_1ZV 0x31 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CS 0x32 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CS 0x33 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CS 0x34 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CS 0x35 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2CZV 0x3a /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2CZV 0x3b /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2CZV 0x3c /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2CZV 0x3d /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_2ZV 0x3e /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_2ZV 0x3f /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_2ZV 0x40 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_2ZV 0x41 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC12_4CSZV 0x42 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS4_VC4_4CSZV 0x43 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC8_4CSZV 0x44 /* R---V */
#define NV_MMU_PTE_KIND_V8Z24_MS8_VC24_4CSZV 0x45 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8 0x46 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_1Z 0x47 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_1Z 0x48 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_1Z 0x49 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_1Z 0x4a /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_1Z 0x4b /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CS 0x4c /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CS 0x4d /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CS 0x4e /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CS 0x4f /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CS 0x50 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_2CZ 0x51 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_2CZ 0x52 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_2CZ 0x53 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_2CZ 0x54 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_2CZ 0x55 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_4CSZV 0x56 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS2_4CSZV 0x57 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS4_4CSZV 0x58 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS8_4CSZV 0x59 /* R---V */
#define NV_MMU_PTE_KIND_Z24S8_MS16_4CSZV 0x5a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12 0x5b /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4 0x5c /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8 0x5d /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24 0x5e /* R---V */
#define NV_MMU_PTE_KIND_YUV_B8C1_2Y 0x60 /* R---V */
#define NV_MMU_PTE_KIND_YUV_B8C2_2Y 0x61 /* R---V */
#define NV_MMU_PTE_KIND_YUV_B10C1_2Y 0x62 /* R---V */
#define NV_MMU_PTE_KIND_YUV_B10C2_2Y 0x6b /* R---V */
#define NV_MMU_PTE_KIND_YUV_B12C1_2Y 0x6c /* R---V */
#define NV_MMU_PTE_KIND_YUV_B12C2_2Y 0x6d /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_1ZV 0x63 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_1ZV 0x64 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_1ZV 0x65 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_1ZV 0x66 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CS 0x67 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CS 0x68 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CS 0x69 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CS 0x6a /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2CZV 0x6f /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2CZV 0x70 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2CZV 0x71 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2CZV 0x72 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_2ZV 0x73 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_2ZV 0x74 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_2ZV 0x75 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_2ZV 0x76 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC12_4CSZV 0x77 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS4_VC4_4CSZV 0x78 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC8_4CSZV 0x79 /* R---V */
#define NV_MMU_PTE_KIND_Z24V8_MS8_VC24_4CSZV 0x7a /* R---V */
#define NV_MMU_PTE_KIND_ZF32 0x7b /* R---V */
#define NV_MMU_PTE_KIND_ZF32_1Z 0x7c /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_1Z 0x7d /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_1Z 0x7e /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_1Z 0x7f /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_1Z 0x80 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CS 0x81 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CS 0x82 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CS 0x83 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CS 0x84 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CS 0x85 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_2CZ 0x86 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS2_2CZ 0x87 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS4_2CZ 0x88 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS8_2CZ 0x89 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_MS16_2CZ 0x8a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12 0x8b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4 0x8c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8 0x8d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24 0x8e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CS 0x8f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CS 0x90 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CS 0x91 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CS 0x92 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1ZV 0x97 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1ZV 0x98 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1ZV 0x99 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1ZV 0x9a /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_1CZV 0x9b /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_1CZV 0x9c /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_1CZV 0x9d /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_1CZV 0x9e /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CS 0x9f /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CS 0xa0 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CS 0xa1 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CS 0xa2 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC12_2CSZV 0xa3 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS4_VC4_2CSZV 0xa4 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC8_2CSZV 0xa5 /* R---V */
#define NV_MMU_PTE_KIND_X8Z24_X16V8S8_MS8_VC24_2CSZV 0xa6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12 0xa7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4 0xa8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8 0xa9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24 0xaa /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CS 0xab /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CS 0xac /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CS 0xad /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CS 0xae /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1ZV 0xb3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1ZV 0xb4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1ZV 0xb5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1ZV 0xb6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_1CZV 0xb7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_1CZV 0xb8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_1CZV 0xb9 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_1CZV 0xba /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CS 0xbb /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CS 0xbc /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CS 0xbd /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CS 0xbe /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC12_2CSZV 0xbf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS4_VC4_2CSZV 0xc0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC8_2CSZV 0xc1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X16V8S8_MS8_VC24_2CSZV 0xc2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8 0xc3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_1CS 0xc4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_1CS 0xc5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_1CS 0xc6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_1CS 0xc7 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_1CS 0xc8 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CSZV 0xce /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CSZV 0xcf /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CSZV 0xd0 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CSZV 0xd1 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CSZV 0xd2 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_2CS 0xd3 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS2_2CS 0xd4 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS4_2CS 0xd5 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS8_2CS 0xd6 /* R---V */
#define NV_MMU_PTE_KIND_ZF32_X24S8_MS16_2CS 0xd7 /* R---V */
#define NV_MMU_PTE_KIND_S8 0x2a /* R---V */
#define NV_MMU_PTE_KIND_S8_2S 0x2b /* R---V */
#define NV_MMU_PTE_KIND_GENERIC_16BX2 0xfe /* R---V */
#define NV_MMU_PTE_KIND_C32_2C 0xd8 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBR 0xd9 /* R---V */
#define NV_MMU_PTE_KIND_C32_2CBA 0xda /* R---V */
#define NV_MMU_PTE_KIND_C32_2CRA 0xdb /* R---V */
#define NV_MMU_PTE_KIND_C32_2BRA 0xdc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2C 0xdd /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_2CBR 0xde /* R---V */
#define NV_MMU_PTE_KIND_C32_MS2_4CBRA 0xcc /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2C 0xdf /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBR 0xe0 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CBA 0xe1 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2CRA 0xe2 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_2BRA 0xe3 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS4_4CBRA 0x2c /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2C 0xe4 /* R---V */
#define NV_MMU_PTE_KIND_C32_MS8_MS16_2CRA 0xe5 /* R---V */
#define NV_MMU_PTE_KIND_C64_2C 0xe6 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBR 0xe7 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CBA 0xe8 /* R---V */
#define NV_MMU_PTE_KIND_C64_2CRA 0xe9 /* R---V */
#define NV_MMU_PTE_KIND_C64_2BRA 0xea /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2C 0xeb /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_2CBR 0xec /* R---V */
#define NV_MMU_PTE_KIND_C64_MS2_4CBRA 0xcd /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2C 0xed /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBR 0xee /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CBA 0xef /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2CRA 0xf0 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_2BRA 0xf1 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS4_4CBRA 0x2d /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2C 0xf2 /* R---V */
#define NV_MMU_PTE_KIND_C64_MS8_MS16_2CRA 0xf3 /* R---V */
#define NV_MMU_PTE_KIND_C128_2C 0xf4 /* R---V */
#define NV_MMU_PTE_KIND_C128_2CR 0xf5 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2C 0xf6 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS2_2CR 0xf7 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2C 0xf8 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS4_2CR 0xf9 /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2C 0xfa /* R---V */
#define NV_MMU_PTE_KIND_C128_MS8_MS16_2CR 0xfb /* R---V */
#define NV_MMU_PTE_KIND_X8C24 0xfc /* R---V */
#define NV_MMU_PTE_KIND_PITCH_NO_SWIZZLE 0xfd /* R---V */
#define NV_MMU_PTE_KIND_SMSKED_MESSAGE 0xca /* R---V */
#define NV_MMU_PTE_KIND_SMHOST_MESSAGE 0xcb /* R---V */
#define NV_MMU_VER1_PDE /* ----G */
#define NV_MMU_VER1_PDE_APERTURE_BIG (0*32+1):(0*32+0) /* RWXVF */
#define NV_MMU_VER1_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE (0*32+3):(0*32+2) /* RWXVF */
#define NV_MMU_VER1_PDE_SIZE_FULL 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_HALF 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_QUARTER 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_SIZE_EIGHTH 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL (1*32+1):(1*32+0) /* RWXVF */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_SMALL (1*32+2):(1*32+2) /* RWXVF */
#define NV_MMU_VER1_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_BIG (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_VER1_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_SYS (1*32+31):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID (1*32+31-3):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER (1*32+31):(1*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER1_PDE__SIZE 8
#define NV_MMU_VER1_PTE /* ----G */
#define NV_MMU_VER1_PTE_VALID (0*32+0):(0*32+0) /* RWXVF */
#define NV_MMU_VER1_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_PRIVILEGE (0*32+1):(0*32+1) /* RWXVF */
#define NV_MMU_VER1_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_READ_ONLY (0*32+2):(0*32+2) /* RWXVF */
#define NV_MMU_VER1_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_ENCRYPTED (0*32+3):(0*32+3) /* RWXVF */
#define NV_MMU_VER1_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_VER1_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_VER1_PTE_ADDRESS_SYS (0*32+31):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID (0*32+31-3):(0*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER (0*32+31):(0*32+32-3) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_VER1_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_VER1_PTE_VOL (1*32+0):(1*32+0) /* RWXVF */
#define NV_MMU_VER1_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE (1*32+2):(1*32+1) /* RWXVF */
#define NV_MMU_VER1_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER1_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE (1*32+3):(1*32+3) /* RWXVF */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER1_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER1_PTE_COMPTAGLINE (1*32+18+11):(1*32+12) /* RWXVF */
#define NV_MMU_VER1_PTE_KIND (1*32+11):(1*32+4) /* RWXVF */
#define NV_MMU_VER1_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER1_PTE__SIZE 8
#define NV_MMU_VER1_PTE_COMPTAGS_NONE 0x0 /* */
#define NV_MMU_VER1_PTE_COMPTAGS_1 0x1 /* */
#define NV_MMU_VER1_PTE_COMPTAGS_2 0x2 /* */
#define NV_MMU_NEW_PDE /* ----G */
#define NV_MMU_NEW_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_NEW_PDE_APERTURE_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_PDE_VOL 3:3 /* RWXVF */
#define NV_MMU_NEW_PDE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_PDE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_NO_ATS 5:5 /* RWXVF */
#define NV_MMU_NEW_PDE_NO_ATS_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PDE_NO_ATS_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PDE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_PDE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_PDE__SIZE 8
#define NV_MMU_NEW_DUAL_PDE /* ----G */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG 2:1 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG 3:3 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_NO_ATS 5:5 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_NO_ATS_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_NO_ATS_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SYS 53:(8-4) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID (35-3):(8-4) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL 66:65 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL 67:67 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_SYS 117:72 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID (99-3):72 /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /* */
#define NV_MMU_NEW_DUAL_PDE__SIZE 16
#define NV_MMU_NEW_PTE /* ----G */
#define NV_MMU_NEW_PTE_VALID 0:0 /* RWXVF */
#define NV_MMU_NEW_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_NEW_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_NEW_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_NEW_PTE_VOL 3:3 /* RWXVF */
#define NV_MMU_NEW_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_ENCRYPTED 4:4 /* RWXVF */
#define NV_MMU_NEW_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_NEW_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_NEW_PTE_PRIVILEGE 5:5 /* RWXVF */
#define NV_MMU_NEW_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_READ_ONLY 6:6 /* RWXVF */
#define NV_MMU_NEW_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE 7:7 /* RWXVF */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_NEW_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_NEW_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_NEW_PTE_COMPTAGLINE (18+35):36 /* RWXVF */
#define NV_MMU_NEW_PTE_KIND 63:56 /* RWXVF */
#define NV_MMU_NEW_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_NEW_PTE__SIZE 8
#define NV_MMU_VER2_PDE /* ----G */
#define NV_MMU_VER2_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_VER2_PDE_APERTURE_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_PDE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_PDE_VOL 3:3 /* RWXVF */
#define NV_MMU_VER2_PDE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_PDE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_NO_ATS 5:5 /* RWXVF */
#define NV_MMU_VER2_PDE_NO_ATS_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PDE_NO_ATS_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PDE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_PDE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_PDE__SIZE 8
#define NV_MMU_VER2_DUAL_PDE /* ----G */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PTE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE_TRUE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_IS_PDE_FALSE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG 2:1 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_BIG_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG 3:3 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_BIG_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_NO_ATS 5:5 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_NO_ATS_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_NO_ATS_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SYS 53:(8-4) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID (35-3):(8-4) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL 66:65 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_INVALID 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_VIDEO_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_APERTURE_SMALL_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL 67:67 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_VOL_SMALL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_SYS 117:72 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID (99-3):72 /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER 99:(100-3) /* RWXVF */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SMALL_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SHIFT 8 /* */
#define NV_MMU_VER2_DUAL_PDE__SIZE 16
#define NV_MMU_VER2_PTE /* ----G */
#define NV_MMU_VER2_PTE_VALID 0:0 /* RWXVF */
#define NV_MMU_VER2_PTE_VALID_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_VALID_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE 2:1 /* RWXVF */
#define NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY 0x00000002 /* RW--V */
#define NV_MMU_VER2_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY 0x00000003 /* RW--V */
#define NV_MMU_VER2_PTE_VOL 3:3 /* RWXVF */
#define NV_MMU_VER2_PTE_VOL_TRUE 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_VOL_FALSE 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_ENCRYPTED 4:4 /* RWXVF */
#define NV_MMU_VER2_PTE_ENCRYPTED_TRUE 0x00000001 /* R---V */
#define NV_MMU_VER2_PTE_ENCRYPTED_FALSE 0x00000000 /* R---V */
#define NV_MMU_VER2_PTE_PRIVILEGE 5:5 /* RWXVF */
#define NV_MMU_VER2_PTE_PRIVILEGE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_PRIVILEGE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_READ_ONLY 6:6 /* RWXVF */
#define NV_MMU_VER2_PTE_READ_ONLY_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_READ_ONLY_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE 7:7 /* RWXVF */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_TRUE 0x1 /* RW--V */
#define NV_MMU_VER2_PTE_ATOMIC_DISABLE_FALSE 0x0 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_SYS 53:8 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID (35-3):8 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER 35:(36-3) /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_0 0x00000000 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_1 0x00000001 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_2 0x00000002 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_3 0x00000003 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_4 0x00000004 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_5 0x00000005 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_6 0x00000006 /* RW--V */
#define NV_MMU_VER2_PTE_ADDRESS_VID_PEER_7 0x00000007 /* RW--V */
#define NV_MMU_VER2_PTE_COMPTAGLINE (18+35):36 /* RWXVF */
#define NV_MMU_VER2_PTE_KIND 63:56 /* RWXVF */
#define NV_MMU_VER2_PTE_ADDRESS_SHIFT 0x0000000c /* */
#define NV_MMU_VER2_PTE__SIZE 8
#define NV_MMU_BASIC /* ----G */
#define NV_MMU_BASIC_KIND 3:0 /* RWXVF */
#define NV_MMU_BASIC_KIND_TRANSPARENT 0x0 /* R---V */
#define NV_MMU_BASIC_KIND_GENERIC 0x1 /* R---V */
#define NV_MMU_BASIC_KIND_S8 0x2 /* R---V */
#define NV_MMU_BASIC_KIND_Z16 0x3 /* R---V */
#define NV_MMU_BASIC_KIND_Z24S8 0x4 /* R---V */
#define NV_MMU_BASIC_KIND_ZF32 0x5 /* R---V */
#define NV_MMU_BASIC_KIND_ZF32_X24S8 0x6 /* R---V */
#define NV_MMU_BASIC_KIND_RSVRD0 0x7 /* R---V */
#define NV_MMU_BASIC_KIND_PITCH 0x8 /* R---V */
#define NV_MMU_BASIC_KIND_GENERIC_C 0x9 /* R---V */
#define NV_MMU_BASIC_KIND_S8_C 0xa /* R---V */
#define NV_MMU_BASIC_KIND_Z16_C 0xb /* R---V */
#define NV_MMU_BASIC_KIND_Z24S8_C 0xc /* R---V */
#define NV_MMU_BASIC_KIND_ZF32_C 0xd /* R---V */
#define NV_MMU_BASIC_KIND_ZF32_X24S8_C 0xe /* R---V */
#define NV_MMU_BASIC_KIND_INVALID 0xf /* R---V */
#endif // __gv100_dev_mmu_h__

19
kernel-open/nvidia-uvm/nvidia-uvm-sources.Kbuild
View File

@@ -43,28 +43,13 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_push.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pushbuffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_thread_context.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_tracker.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_host.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_ce.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_sec2.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_mmu.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_fault_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_maxwell_access_counter_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pascal.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pascal_ce.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pascal_host.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pascal_mmu.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_pascal_fault_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_ce.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_host.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_mmu.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_volta_fault_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_ce.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_access_counter_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_fault_buffer.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_mmu.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_host.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_turing_sec2.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_ce.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_ampere_host.c

3
kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
View File

@@ -61,7 +61,8 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_notifier_ops_arch_invalidate_secondary_tlb
NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_vma_added_flags
NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_device_range
NV_CONFTEST_TYPE_COMPILE_TESTS += handle_mm_fault_has_pt_regs_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += get_dev_pagemap_has_pgmap_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += dev_pagemap_ops_has_folio_free
NV_CONFTEST_TYPE_COMPILE_TESTS += zone_device_page_init_has_pgmap_and_order_args
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_unified_nodes
NV_CONFTEST_TYPE_COMPILE_TESTS += mempolicy_has_home_node
NV_CONFTEST_TYPE_COMPILE_TESTS += mpol_preferred_many_present

2
kernel-open/nvidia-uvm/uvm.c
View File

@@ -1030,8 +1030,6 @@ static long uvm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_ENABLE_READ_DUPLICATION, uvm_api_enable_read_duplication);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_DISABLE_READ_DUPLICATION, uvm_api_disable_read_duplication);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_MIGRATE, uvm_api_migrate);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_ENABLE_SYSTEM_WIDE_ATOMICS, uvm_api_enable_system_wide_atomics);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_DISABLE_SYSTEM_WIDE_ATOMICS, uvm_api_disable_system_wide_atomics);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TOOLS_READ_PROCESS_MEMORY, uvm_api_tools_read_process_memory);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TOOLS_WRITE_PROCESS_MEMORY, uvm_api_tools_write_process_memory);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TOOLS_GET_PROCESSOR_UUID_TABLE, uvm_api_tools_get_processor_uuid_table);

130
kernel-open/nvidia-uvm/uvm.h
View File

@@ -45,20 +45,20 @@
// #endif
// 3) Do the same thing for the function definition, and for any structs that
// are taken as arguments to these functions.
// 4) Let this change propagate over to cuda_a and bugfix_main, so that the CUDA
// 4) Let this change propagate over to bugfix_main, so that the CUDA
// and nvidia-cfg libraries can start using the new API by bumping up the API
// version number it's using.
// Places where UVM_API_REVISION is defined are:
// drivers/gpgpu/cuda/cuda.nvmk (cuda_a)
// drivers/gpgpu/cuda/cuda.nvmk (bugfix_main)
// drivers/setup/linux/nvidia-cfg/makefile.nvmk (bugfix_main)
// 5) Once the bugfix_main and cuda_a changes have made it back into chips_a,
// 5) Once the bugfix_main changes have made it back into chips_a,
// remove the old API declaration, definition, and any old structs that were
// in use.
#ifndef _UVM_H_
#define _UVM_H_
#define UVM_API_LATEST_REVISION 13
#define UVM_API_LATEST_REVISION 14
#if !defined(UVM_API_REVISION)
#error "please define UVM_API_REVISION macro to a desired version number or UVM_API_LATEST_REVISION macro"
@@ -335,6 +335,12 @@ NV_STATUS UvmIsPageableMemoryAccessSupportedOnGpu(const NvProcessorUuid *gpuUuid
// NvRmControl(NVC637_CTRL_CMD_GET_UUID). Otherwise, if the GPU is not SMC
// capable or SMC enabled, the physical GPU UUID must be used.
//
// GPU registration may fail if certain features, such as those required to
// support pageable memory on systems without hardware support, could not be
// initialised. Registration may succeed if callers retry after explicitly
// requesting these features be disabled when calling UvmInitialize(), for
// example by passing UVM_INIT_FLAGS_DISABLE_HMM.
//
// Arguments:
// gpuUuid: (INPUT)
// UUID of the physical GPU to register.
@@ -344,6 +350,12 @@ NV_STATUS UvmIsPageableMemoryAccessSupportedOnGpu(const NvProcessorUuid *gpuUuid
// This should be NULL if the GPU is not SMC capable or SMC enabled.
//
// Error codes:
//
// NV_ERR_BUSY_RETRY:
// Registration of the GPU failed because not all features required
// to support the VA space could be enabled. Caller should retry after
// disabling certain features when creating the VA space.
//
// NV_ERR_NO_MEMORY:
// Internal memory allocation failed.
//
@@ -1123,7 +1135,6 @@ NV_STATUS UvmAllowMigrationRangeGroups(const NvU64 *rangeGroupIds,
// for (i = 0; i < accessedByCount; i++) {
// UvmSetAccessedBy(base, length, &accessedByUuids[i]);
// }
// UvmSetRangeGroup(base, length, rangeGroupId);
// UvmMigrate(base, length, preferredLocationUuid, 0);
//
// Please see those APIs for further details on their behavior. If an error is
@@ -1183,8 +1194,7 @@ NV_STATUS UvmAlloc(void *base,
NvLength length,
const NvProcessorUuid *preferredLocationUuid,
const NvProcessorUuid *accessedByUuids,
NvLength accessedByCount,
NvU64 rangeGroupId);
NvLength accessedByCount);
//------------------------------------------------------------------------------
// UvmFree
@@ -2339,19 +2349,6 @@ NV_STATUS UvmDisableReadDuplication(void *base,
// a file backed shared mapping and least one GPU in the system supports
// transparent access to pageable memory, the behavior below is not guaranteed.
//
// If any pages in the VA range are associated with a range group that was made
// non-migratable via UvmPreventMigrationRangeGroups, then those pages are
// migrated immediately to the specified preferred location and mapped according
// to policies specified in UvmPreventMigrationRangeGroups. Otherwise, this API
// neither migrates pages nor does it populate unpopulated pages. Note that if
// the specified preferred location is a fault-capable GPU and at least one page
// in the VA range is associated with a non-migratable range group, then an
// error is returned. Additionally, if the specified preferred location is a
// non-fault capable GPU and at least one page in the VA range is associated
// with a non-migratable range group, an error is returned if another
// non-fault-capable GPU is present in the accessed-by list of that page but P2P
// support has not been enabled between both GPUs.
//
// When a page is in its preferred location, a fault from another processor will
// not cause a migration if a mapping for that page from that processor can be
// established without migrating the page. Individual faulting pages will still
@@ -2784,99 +2781,6 @@ NV_STATUS UvmUnsetAccessedBy(void *base,
//------------------------------------------------------------------------------
NV_STATUS UvmDiscard(void *base, NvLength length, NvU64 flags);
//------------------------------------------------------------------------------
// UvmEnableSystemWideAtomics
//
// Enables software-assisted system-wide atomics support on the specified GPU.
// Any system-wide atomic operation issued from this GPU is now guaranteed to be
// atomic with respect to all accesses from other processors that also support
// system-wide atomics regardless of whether that support is enabled on those
// other processors or not.
//
// The class of atomic operations from the GPU that are considered system-wide
// is GPU architecture dependent. All atomic operations from the CPU are always
// considered to be system-wide and support for system-wide atomics on the CPU
// is always considered to be enabled.
//
// System-wide atomics which cannot be natively supported in hardware are
// emulated using virtual mappings and page faults. For example, assume a
// virtual address which is resident in CPU memory and has CPU memory as its
// preferred location. A GPU with system-wide atomics enabled but without native
// atomics support to CPU memory will not have atomics enabled in its virtual
// mapping of the page that contains that address. If that GPU performs an
// atomic operation, the access will fault, all other processors' mappings to
// that page will have their write permissions revoked, the faulting GPU will be
// granted atomic permissions in its virtual mapping, and the faulting GPU will
// retry its access. Further atomic accesses from that GPU will not cause page
// faults until another processor attempts a write access to the same page.
//
// Multiple calls to this API for the same GPU are not refcounted, i.e. calling
// this API for a GPU for which software-assisted system-wide atomics support
// has already been enabled results in a no-op.
//
// The GPU must have been registered using UvmRegisterGpu prior to making this
// call. By default, software-assisted system-wide atomics support is enabled
// when a GPU is registered.
//
// Arguments:
// gpuUuid: (INPUT)
// UUID of the physical GPU if the GPU is not SMC capable or SMC
// enabled, or the GPU instance UUID of the partition to enable
// software-assisted system-wide atomics on.
//
// Error codes:
// NV_ERR_NO_MEMORY:
// Internal memory allocation failed.
//
// NV_ERR_INVALID_DEVICE:
// The GPU referred to by gpuUuid was not registered.
//
// NV_ERR_NOT_SUPPORTED:
// The GPU does not support system-wide atomic operations, or the GPU
// has hardware support for scoped atomic operations.
//
// NV_ERR_GENERIC:
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
//------------------------------------------------------------------------------
NV_STATUS UvmEnableSystemWideAtomics(const NvProcessorUuid *gpuUuid);
//------------------------------------------------------------------------------
// UvmDisableSystemWideAtomics
//
// Disables software-assisted system-wide atomics support on the specified GPU.
// Any atomic operation from this GPU is no longer guaranteed to be atomic with
// respect to accesses from other processors in the system, even if the
// operation has system-wide scope at the instruction level.
//
// The GPU must have been registered using UvmRegisterGpu prior to making this
// call. It is however ok to call this API for GPUs that do not have support for
// system-wide atomic operations enabled. If the GPU is unregistered via
// UvmUnregisterGpu and then registered again via UvmRegisterGpu, support for
// software-assisted system-wide atomics will be enabled.
//
// Arguments:
// gpuUuid: (INPUT)
// UUID of the physical GPU if the GPU is not SMC capable or SMC
// enabled, or the GPU instance UUID of the partition to disable
// software-assisted system-wide atomics on.
//
// Error codes:
// NV_ERR_INVALID_DEVICE:
// The GPU referred to by gpuUuid was not registered.
//
// NV_ERR_NOT_SUPPORTED:
// The GPU does not support system-wide atomic operations, or the GPU
// has hardware support for scoped atomic operations.
//
// NV_ERR_GENERIC:
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
//------------------------------------------------------------------------------
NV_STATUS UvmDisableSystemWideAtomics(const NvProcessorUuid *gpuUuid);
//------------------------------------------------------------------------------
// UvmGetFileDescriptor
//

25
kernel-open/nvidia-uvm/uvm_ada.c
View File

@@ -29,10 +29,6 @@
void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = true;
parent_gpu->tlb_batch.va_range_invalidate_supported = true;
// TODO: Bug 1767241: Run benchmarks to figure out a good number
parent_gpu->tlb_batch.max_ranges = 8;
@@ -65,39 +61,18 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->max_host_va = 1ULL << 40;
// Ada can map sysmem with any page size
parent_gpu->can_map_sysmem_with_large_pages = true;
// Prefetch instructions will generate faults
parent_gpu->prefetch_fault_supported = true;
// Ada can place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = true;
parent_gpu->replayable_faults_supported = true;
parent_gpu->non_replayable_faults_supported = true;
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->fault_cancel_va_supported = true;
parent_gpu->scoped_atomics_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
parent_gpu->has_clear_faulted_channel_method = false;
parent_gpu->smc.supported = true;
parent_gpu->sparse_mappings_supported = true;
parent_gpu->map_remap_larger_page_promotion = false;
parent_gpu->plc_supported = true;
parent_gpu->ats.no_ats_range_required = false;
parent_gpu->ats.gmmu_pt_depth0_init_required = false;

25
kernel-open/nvidia-uvm/uvm_ampere.c
View File

@@ -29,10 +29,6 @@
void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = true;
parent_gpu->tlb_batch.va_range_invalidate_supported = true;
// TODO: Bug 1767241: Run benchmarks to figure out a good number
parent_gpu->tlb_batch.max_ranges = 8;
@@ -69,33 +65,14 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->max_host_va = 1ULL << 40;
// Ampere can map sysmem with any page size
parent_gpu->can_map_sysmem_with_large_pages = true;
// Prefetch instructions will generate faults
parent_gpu->prefetch_fault_supported = true;
// Ampere can place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = true;
parent_gpu->replayable_faults_supported = true;
parent_gpu->non_replayable_faults_supported = true;
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->fault_cancel_va_supported = true;
parent_gpu->scoped_atomics_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
parent_gpu->has_clear_faulted_channel_method = false;
parent_gpu->smc.supported = true;
parent_gpu->sparse_mappings_supported = true;
parent_gpu->access_bits_supported = false;
UVM_ASSERT(parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100);
@@ -107,8 +84,6 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->map_remap_larger_page_promotion = false;
}
parent_gpu->plc_supported = true;
parent_gpu->ats.no_ats_range_required = false;
parent_gpu->ats.gmmu_pt_depth0_init_required = false;

13
kernel-open/nvidia-uvm/uvm_ampere_mmu.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2018-2024 NVIDIA Corporation
Copyright (c) 2018-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -54,19 +54,12 @@ static NvU64 page_sizes_ampere(void)
static uvm_mmu_mode_hal_t ampere_mmu_mode_hal;
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU64 big_page_size)
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(void)
{
static bool initialized = false;
UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
// TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
// 128K big page size for Pascal+ GPUs
if (big_page_size == UVM_PAGE_SIZE_128K)
return NULL;
if (!initialized) {
uvm_mmu_mode_hal_t *turing_mmu_mode_hal = uvm_hal_mmu_mode_turing(big_page_size);
uvm_mmu_mode_hal_t *turing_mmu_mode_hal = uvm_hal_mmu_mode_turing();
UVM_ASSERT(turing_mmu_mode_hal);
// The assumption made is that arch_hal->mmu_mode_hal() will be

12
kernel-open/nvidia-uvm/uvm_api.h
View File

@@ -188,6 +188,16 @@ static bool uvm_api_range_invalid(NvU64 base, NvU64 length)
return uvm_api_range_invalid_aligned(base, length, PAGE_SIZE);
}
// Similar to uvm_api_range_invalid() but also validates that an offset is
// aligned and that the offset and length won't overflow.
static bool uvm_api_range_offset_invalid(NvU64 base, NvU64 offset, NvU64 length)
{
return uvm_api_range_invalid(base, offset + length) ||
!IS_ALIGNED(offset, PAGE_SIZE) ||
!IS_ALIGNED(length, PAGE_SIZE) ||
offset + length < offset;
}
// Some APIs can only enforce 4K alignment as it's the smallest GPU page size
// even when the smallest host page is larger.
static bool uvm_api_range_invalid_4k(NvU64 base, NvU64 length)
@@ -246,8 +256,6 @@ NV_STATUS uvm_api_unregister_channel(UVM_UNREGISTER_CHANNEL_PARAMS *params, stru
NV_STATUS uvm_api_enable_read_duplication(const UVM_ENABLE_READ_DUPLICATION_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_disable_read_duplication(const UVM_DISABLE_READ_DUPLICATION_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_migrate(UVM_MIGRATE_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_enable_system_wide_atomics(UVM_ENABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_disable_system_wide_atomics(UVM_DISABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_tools_init_event_tracker(UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_tools_init_event_tracker_v2(UVM_TOOLS_INIT_EVENT_TRACKER_V2_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_tools_set_notification_threshold(UVM_TOOLS_SET_NOTIFICATION_THRESHOLD_PARAMS *params, struct file *filp);

25
kernel-open/nvidia-uvm/uvm_blackwell.c
View File

@@ -30,10 +30,6 @@
void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = true;
parent_gpu->tlb_batch.va_range_invalidate_supported = true;
// TODO: Bug 1767241: Run benchmarks to figure out a good number
parent_gpu->tlb_batch.max_ranges = 8;
@@ -69,19 +65,6 @@ void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->max_host_va = 1ull << 57;
// Blackwell can map sysmem with any page size
parent_gpu->can_map_sysmem_with_large_pages = true;
// Prefetch instructions will generate faults
parent_gpu->prefetch_fault_supported = true;
// Blackwell can place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = true;
parent_gpu->replayable_faults_supported = true;
parent_gpu->non_replayable_faults_supported = true;
parent_gpu->access_counters_serialize_clear_ops_by_type = parent_gpu->rm_info.accessCntrBufferCount == 2;
// TODO: Bug 5262806: Remove this WAR once the bug is fixed.
@@ -92,22 +75,14 @@ void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
if (parent_gpu->rm_info.accessCntrBufferCount > 1)
parent_gpu->rm_info.accessCntrBufferCount = 1;
parent_gpu->fault_cancel_va_supported = true;
parent_gpu->scoped_atomics_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
parent_gpu->has_clear_faulted_channel_method = false;
parent_gpu->smc.supported = true;
parent_gpu->sparse_mappings_supported = true;
parent_gpu->map_remap_larger_page_promotion = false;
parent_gpu->plc_supported = true;
parent_gpu->ats.no_ats_range_required = true;
parent_gpu->ats.gmmu_pt_depth0_init_required = parent_gpu->ats.non_pasid_ats_enabled;

27
kernel-open/nvidia-uvm/uvm_blackwell_mmu.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2022-2024 NVIDIA Corporation
Copyright (c) 2022-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -65,14 +65,11 @@ static NvU64 page_sizes_blackwell_integrated(void)
return UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
}
static uvm_mmu_mode_hal_t *__uvm_hal_mmu_mode_blackwell(uvm_mmu_mode_hal_t *mmu_mode_hal,
NvU64 big_page_size)
static uvm_mmu_mode_hal_t *__uvm_hal_mmu_mode_blackwell(uvm_mmu_mode_hal_t *mmu_mode_hal)
{
uvm_mmu_mode_hal_t *hopper_mmu_mode_hal;
UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
hopper_mmu_mode_hal = uvm_hal_mmu_mode_hopper(big_page_size);
hopper_mmu_mode_hal = uvm_hal_mmu_mode_hopper();
UVM_ASSERT(hopper_mmu_mode_hal);
// The assumption made is that arch_hal->mmu_mode_hal() will be called
@@ -85,19 +82,14 @@ static uvm_mmu_mode_hal_t *__uvm_hal_mmu_mode_blackwell(uvm_mmu_mode_hal_t *mmu_
return mmu_mode_hal;
}
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size)
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(void)
{
static bool initialized = false;
// TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
// 128K big page size for Pascal+ GPUs
if (big_page_size == UVM_PAGE_SIZE_128K)
return NULL;
if (!initialized) {
uvm_mmu_mode_hal_t *mmu_mode_hal;
mmu_mode_hal = __uvm_hal_mmu_mode_blackwell(&blackwell_mmu_mode_hal, big_page_size);
mmu_mode_hal = __uvm_hal_mmu_mode_blackwell(&blackwell_mmu_mode_hal);
mmu_mode_hal->page_sizes = page_sizes_blackwell;
initialized = true;
}
@@ -105,19 +97,14 @@ uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size)
return &blackwell_mmu_mode_hal;
}
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell_integrated(NvU64 big_page_size)
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell_integrated(void)
{
static bool initialized = false;
// TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
// 128K big page size for Pascal+ GPUs
if (big_page_size == UVM_PAGE_SIZE_128K)
return NULL;
if (!initialized) {
uvm_mmu_mode_hal_t *mmu_mode_hal;
mmu_mode_hal = __uvm_hal_mmu_mode_blackwell(&blackwell_integrated_mmu_mode_hal, big_page_size);
mmu_mode_hal = __uvm_hal_mmu_mode_blackwell(&blackwell_integrated_mmu_mode_hal);
mmu_mode_hal->page_sizes = page_sizes_blackwell_integrated;
initialized = true;
}

4
kernel-open/nvidia-uvm/uvm_ce_test.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2023 NVIDIA Corporation
Copyright (c) 2015-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -409,7 +409,7 @@ static NV_STATUS test_memcpy_and_memset(uvm_gpu_t *gpu)
uvm_rm_mem_t *gpu_rm_mem = NULL;
uvm_gpu_address_t gpu_addresses[4] = {0};
size_t gpu_addresses_length = 0;
size_t size = gpu->big_page.internal_size;
size_t size = UVM_BIG_PAGE_SIZE;
static const size_t element_sizes[] = {1, 4, 8};
const size_t iterations = 4;
size_t i, j, k, s;

50
kernel-open/nvidia-uvm/uvm_channel.c
View File

@@ -36,7 +36,7 @@
#include "nv_uvm_types.h"
#include "nv_uvm_user_types.h"
#include "nv_uvm_interface.h"
#include "clb06f.h"
#include "clc46f.h"
#include "uvm_conf_computing.h"
@@ -989,7 +989,7 @@ static void internal_channel_submit_work(uvm_push_t *push, NvU32 push_size, NvU3
uvm_pushbuffer_t *pushbuffer = uvm_channel_get_pushbuffer(channel);
uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);
BUILD_BUG_ON(sizeof(*gpfifo_entry) != NVB06F_GP_ENTRY__SIZE);
BUILD_BUG_ON(sizeof(*gpfifo_entry) != NVC46F_GP_ENTRY__SIZE);
UVM_ASSERT(!uvm_channel_is_proxy(channel));
gpfifo_entry = (NvU64*)channel->channel_info.gpFifoEntries + channel->cpu_put;
@@ -3018,10 +3018,16 @@ static int compare_ce_for_channel_type(const UvmGpuCopyEngineCaps *ce_caps,
break;
// For GPU_INTERNAL we want the max possible bandwidth for CEs. For now
// assume that the number of PCEs is a good measure.
// For GPU_INTERNAL we opt for an LCE optimized for scrubbing, if
// available. Otherwise, we want the max possible bandwidth for CEs. For
// now assume that the number of PCEs is a good measure.
// If not, we fall back to less-used LCEs.
// TODO: Bug 1735254: Add a direct CE query for local FB bandwidth
case UVM_CHANNEL_TYPE_GPU_INTERNAL:
// Always prefer a scrub-optimized LCE.
if (cap0->scrub != cap1->scrub)
return cap1->scrub - cap0->scrub;
{
int pce_diff = (int)hweight32(cap1->cePceMask) - (int)hweight32(cap0->cePceMask);
@@ -3040,6 +3046,7 @@ static int compare_ce_for_channel_type(const UvmGpuCopyEngineCaps *ce_caps,
// system-wide) so just break out to get the default ordering which
// prioritizes usage count.
case UVM_CHANNEL_TYPE_MEMOPS:
// For WLC we only care about using a dedicated CE, which requires
// knowing the global CE mappings. For now just rely on the default
// ordering, which results on selecting an unused CE (if available).
@@ -3348,21 +3355,6 @@ static void init_channel_manager_conf(uvm_channel_manager_t *manager)
}
}
// 3- GPFIFO/GPPut location
// Only support the knobs for GPFIFO/GPPut on Volta+
if (!gpu->parent->gpfifo_in_vidmem_supported) {
if (manager->conf.gpput_loc == UVM_BUFFER_LOCATION_SYS) {
UVM_INFO_PRINT("CAUTION: allocating GPPut in sysmem is NOT supported and may crash the system, using %s "
"instead\n",
buffer_location_to_string(UVM_BUFFER_LOCATION_DEFAULT));
}
manager->conf.gpfifo_loc = UVM_BUFFER_LOCATION_DEFAULT;
manager->conf.gpput_loc = UVM_BUFFER_LOCATION_DEFAULT;
return;
}
gpfifo_loc_value = uvm_channel_gpfifo_loc;
if (!is_string_valid_location(gpfifo_loc_value)) {
gpfifo_loc_value = UVM_CHANNEL_GPFIFO_LOC_DEFAULT;
@@ -4120,6 +4112,26 @@ static const char *get_gpput_location_string(uvm_channel_t *channel)
return buffer_location_to_string(channel->pool->manager->conf.gpput_loc);
}
void uvm_gpu_print_ce_mapping(const uvm_gpu_t *gpu, struct seq_file *s)
{
uvm_channel_type_t type;
UVM_SEQ_OR_DBG_PRINT(s, "CE channel_type to engine index mapping:\n");
for (type = 0; type < UVM_CHANNEL_TYPE_CE_COUNT; type++) {
// For the UVM_CHANNEL_TYPE_GPU_TO_GPU type, we print the default pool,
// which may not be the optimal CE (in gpu_to_gpu) on NVLINK-based
// systems, as we may have different CEs per peer GPU.
uvm_channel_pool_t *channel_pool = gpu->channel_manager->pool_to_use.default_for_type[type];
if (channel_pool) {
UVM_SEQ_OR_DBG_PRINT(s, " %-30s ce %-2u #channels %-2u\n",
uvm_channel_type_to_string(type),
channel_pool->engine_index,
channel_pool->num_channels);
}
}
}
static void uvm_channel_print_info(uvm_channel_t *channel, struct seq_file *s)
{
UVM_SEQ_OR_DBG_PRINT(s, "Channel %s\n", channel->name);

1
kernel-open/nvidia-uvm/uvm_channel.h
View File

@@ -725,6 +725,7 @@ const char *uvm_channel_pool_type_to_string(uvm_channel_pool_type_t channel_pool
// acquires the channel pool lock.
NvU32 uvm_channel_get_available_gpfifo_entries(uvm_channel_t *channel);
void uvm_gpu_print_ce_mapping(const uvm_gpu_t *gpu, struct seq_file *s);
void uvm_channel_print_pending_pushes(uvm_channel_t *channel);
bool uvm_channel_is_locked_for_push(uvm_channel_t *channel);

6
kernel-open/nvidia-uvm/uvm_fault_buffer_flush_test.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2016-2023 NVIDIA Corporation
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -45,7 +45,7 @@ NV_STATUS uvm_test_fault_buffer_flush(UVM_TEST_FAULT_BUFFER_FLUSH_PARAMS *params
uvm_va_space_down_read(va_space);
uvm_processor_mask_and(retained_gpus, &va_space->faultable_processors, &va_space->registered_gpus);
uvm_processor_mask_copy(retained_gpus, &va_space->registered_gpus);
uvm_global_gpu_retain(retained_gpus);
@@ -63,7 +63,7 @@ NV_STATUS uvm_test_fault_buffer_flush(UVM_TEST_FAULT_BUFFER_FLUSH_PARAMS *params
}
for_each_gpu_in_mask(gpu, retained_gpus)
TEST_CHECK_GOTO(uvm_gpu_fault_buffer_flush(gpu) == NV_OK, out);
TEST_CHECK_GOTO(uvm_gpu_replayable_buffer_flush(gpu) == NV_OK, out);
}
out:

7
kernel-open/nvidia-uvm/uvm_global.c
View File

@@ -90,6 +90,7 @@ NV_STATUS uvm_global_init(void)
INIT_LIST_HEAD(&g_uvm_global.va_spaces.list);
uvm_mutex_init(&g_uvm_global.devmem_ranges.lock, UVM_LOCK_ORDER_LEAF);
INIT_LIST_HEAD(&g_uvm_global.devmem_ranges.list);
INIT_LIST_HEAD(&g_uvm_global.pci_p2pdma_devices.list);
status = uvm_kvmalloc_init();
if (status != NV_OK) {
@@ -238,6 +239,7 @@ void uvm_global_exit(void)
uvm_mem_global_exit();
uvm_pmm_sysmem_exit();
uvm_pmm_devmem_exit();
uvm_pmm_pci_p2pdma_cache_exit();
uvm_gpu_exit();
uvm_processor_mask_cache_exit();
@@ -311,10 +313,7 @@ static NV_STATUS uvm_suspend(void)
// guarantees that all user channels have been preempted before
// uvm_suspend() is called, which implies that no user channels can be
// stalled on faults when this point is reached.
if (gpu->parent->replayable_faults_supported)
uvm_gpu_fault_buffer_flush(gpu);
// TODO: Bug 2535118: flush the non-replayable fault buffer
uvm_gpu_replayable_buffer_flush(gpu);
// Stop access counter interrupt processing for the duration of this
// sleep cycle to defend against potential interrupt storms in

10
kernel-open/nvidia-uvm/uvm_global.h
View File

@@ -156,8 +156,18 @@ struct uvm_global_struct
uvm_mutex_t lock;
struct list_head list;
} devmem_ranges;
// List of all pci devices and the device information if the PCI P2PDMA registration
// is already done for the device. This is an optimization to prevent having to
// unregister and re-register on every add_gpu()/remove_gpu(). Depending on the
// size of BAR1 region, the cost of registration can be really high.
struct {
struct list_head list;
} pci_p2pdma_devices;
};
// Initialize global uvm state
NV_STATUS uvm_global_init(void);

92
kernel-open/nvidia-uvm/uvm_gpu.c
View File

@@ -239,9 +239,8 @@ static NV_STATUS alloc_and_init_address_space(uvm_gpu_t *gpu)
if (status != NV_OK)
return status;
UVM_ASSERT(gpu_address_space_info.bigPageSize <= NV_U32_MAX);
UVM_ASSERT(gpu_address_space_info.bigPageSize == UVM_BIG_PAGE_SIZE);
gpu->big_page.internal_size = gpu_address_space_info.bigPageSize;
gpu->time.time0_register = gpu_address_space_info.time0Offset;
gpu->time.time1_register = gpu_address_space_info.time1Offset;
@@ -456,7 +455,7 @@ NvU64 uvm_parent_gpu_canonical_address(uvm_parent_gpu_t *parent_gpu, NvU64 addr)
// On x86: the CPU uses 5-level paging and the GPU is pre-Hopper.
// We sign-extend on the 48b on ARM and on the 47b on x86 to mirror the
// behavior of CPUs with smaller (than GPU) VA widths.
gpu_addr_shift = parent_gpu->arch_hal->mmu_mode_hal(UVM_PAGE_SIZE_64K)->num_va_bits();
gpu_addr_shift = parent_gpu->arch_hal->mmu_mode_hal()->num_va_bits();
cpu_addr_shift = uvm_cpu_num_va_bits();
if (cpu_addr_shift > gpu_addr_shift)
@@ -503,7 +502,7 @@ static void gpu_info_print_ce_caps(uvm_gpu_t *gpu, struct seq_file *s)
continue;
UVM_SEQ_OR_DBG_PRINT(s, " ce %u pce mask 0x%08x grce %u shared %u sysmem read %u sysmem write %u sysmem %u "
"nvlink p2p %u p2p %u secure %u\n",
"nvlink p2p %u p2p %u secure %u scrub %u\n",
i,
ce_caps->cePceMask,
ce_caps->grce,
@@ -513,7 +512,8 @@ static void gpu_info_print_ce_caps(uvm_gpu_t *gpu, struct seq_file *s)
ce_caps->sysmem,
ce_caps->nvlinkP2p,
ce_caps->p2p,
ce_caps->secure);
ce_caps->secure,
ce_caps->scrub);
}
out:
@@ -590,7 +590,7 @@ static void gpu_info_print_common(uvm_gpu_t *gpu, struct seq_file *s)
UVM_SEQ_OR_DBG_PRINT(s, "ce_class 0x%X\n", gpu_info->ceClass);
UVM_SEQ_OR_DBG_PRINT(s, "virtualization_mode %s\n",
uvm_gpu_virt_type_string(gpu_info->virtMode));
UVM_SEQ_OR_DBG_PRINT(s, "big_page_size %u\n", gpu->big_page.internal_size);
UVM_SEQ_OR_DBG_PRINT(s, "big_page_size %llu\n", UVM_BIG_PAGE_SIZE);
UVM_SEQ_OR_DBG_PRINT(s, "rm_va_base 0x%llx\n", gpu->parent->rm_va_base);
UVM_SEQ_OR_DBG_PRINT(s, "rm_va_size 0x%llx\n", gpu->parent->rm_va_size);
UVM_SEQ_OR_DBG_PRINT(s, "vidmem_start %llu (%llu MBs)\n",
@@ -699,6 +699,7 @@ static void gpu_info_print_common(uvm_gpu_t *gpu, struct seq_file *s)
mapped_cpu_pages_size / (1024u * 1024u));
gpu_info_print_ce_caps(gpu, s);
uvm_gpu_print_ce_mapping(gpu, s);
if (g_uvm_global.conf_computing_enabled) {
UVM_SEQ_OR_DBG_PRINT(s, "dma_buffer_pool_num_buffers %lu\n",
@@ -1252,6 +1253,9 @@ static NV_STATUS alloc_parent_gpu(const NvProcessorUuid *gpu_uuid,
uvm_rb_tree_init(&parent_gpu->instance_ptr_table);
uvm_rb_tree_init(&parent_gpu->tsg_table);
if (uvm_enable_builtin_tests)
INIT_LIST_HEAD(&parent_gpu->test.dead_channels);
// TODO: Bug 3881835: revisit whether to use nv_kthread_q_t or workqueue.
status = errno_to_nv_status(nv_kthread_q_init(&parent_gpu->lazy_free_q, "vidmem lazy free"));
if (status != NV_OK)
@@ -1308,7 +1312,6 @@ static NV_STATUS configure_address_space(uvm_gpu_t *gpu)
status = uvm_page_tree_init(gpu,
NULL,
UVM_PAGE_TREE_TYPE_KERNEL,
gpu->big_page.internal_size,
uvm_get_page_tree_location(gpu),
&gpu->address_space_tree);
if (status != NV_OK) {
@@ -1442,7 +1445,8 @@ NV_STATUS uvm_gpu_check_nvlink_error(uvm_gpu_t *gpu)
static NV_STATUS init_parent_gpu(uvm_parent_gpu_t *parent_gpu,
const NvProcessorUuid *gpu_uuid,
const UvmGpuInfo *gpu_info,
const UvmGpuPlatformInfo *gpu_platform_info)
const UvmGpuPlatformInfo *gpu_platform_info,
bool enable_hmm)
{
NV_STATUS status;
UvmGpuFbInfo fb_info = {0};
@@ -1524,12 +1528,14 @@ static NV_STATUS init_parent_gpu(uvm_parent_gpu_t *parent_gpu,
uvm_mmu_init_gpu_chunk_sizes(parent_gpu);
status = uvm_pmm_devmem_init(parent_gpu);
if (status != NV_OK) {
UVM_ERR_PRINT("failed to intialize device private memory: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
if (enable_hmm) {
status = uvm_pmm_devmem_init(parent_gpu);
if (status != NV_OK) {
UVM_ERR_PRINT("failed to intialize device private memory: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
}
uvm_pmm_gpu_device_p2p_init(parent_gpu);
@@ -2176,7 +2182,8 @@ bool uvm_parent_gpus_are_bar1_peers(const uvm_parent_gpu_t *parent_gpu0, const u
return false;
}
bool uvm_parent_gpus_are_nvlink_direct_connected(const uvm_parent_gpu_t *parent_gpu0, const uvm_parent_gpu_t *parent_gpu1)
bool uvm_parent_gpus_are_nvlink_direct_connected(const uvm_parent_gpu_t *parent_gpu0,
const uvm_parent_gpu_t *parent_gpu1)
{
if (parent_gpu0 != parent_gpu1 &&
parent_gpu0->peer_address_info.is_nvlink_direct_connected &&
@@ -2771,7 +2778,7 @@ static NV_STATUS uvm_gpu_init_access_bits(uvm_parent_gpu_t *parent_gpu)
return uvm_rm_locked_call(nvUvmInterfaceAccessBitsBufAlloc(parent_gpu->rm_device, &parent_gpu->vab_info));
}
static NV_STATUS uvm_gpu_update_access_bits(uvm_parent_gpu_t *parent_gpu, UVM_ACCESS_BITS_DUMP_MODE mode)
static NV_STATUS uvm_gpu_update_access_bits(uvm_parent_gpu_t *parent_gpu, UVM_ACCESS_BITS_DUMP_MODE mode)
{
return nvUvmInterfaceAccessBitsDump(parent_gpu->rm_device, &parent_gpu->vab_info, mode);
}
@@ -2867,6 +2874,7 @@ static NV_STATUS add_gpu(const NvProcessorUuid *gpu_uuid,
const UvmGpuPlatformInfo *gpu_platform_info,
uvm_parent_gpu_t *parent_gpu,
const uvm_test_parent_gpu_inject_error_t *parent_gpu_error,
bool enable_hmm,
uvm_gpu_t **gpu_out)
{
NV_STATUS status;
@@ -2881,7 +2889,7 @@ static NV_STATUS add_gpu(const NvProcessorUuid *gpu_uuid,
return status;
if (uvm_enable_builtin_tests)
parent_gpu->test = *parent_gpu_error;
parent_gpu->test.inject_error = *parent_gpu_error;
}
gpu = alloc_gpu(parent_gpu, gpu_id);
@@ -2906,10 +2914,18 @@ static NV_STATUS add_gpu(const NvProcessorUuid *gpu_uuid,
}
if (alloc_parent) {
status = init_parent_gpu(parent_gpu, gpu_uuid, gpu_info, gpu_platform_info);
status = init_parent_gpu(parent_gpu, gpu_uuid, gpu_info, gpu_platform_info, enable_hmm);
if (status != NV_OK)
goto error;
}
#if UVM_IS_CONFIG_HMM()
// HMM was disabled when first initialising the parent so we can't support
// it now. Tell the caller to retry with it disabled.
else if (!parent_gpu->devmem && enable_hmm) {
status = NV_ERR_BUSY_RETRY;
goto error;
}
#endif
status = init_gpu(gpu, gpu_info);
if (status != NV_OK)
@@ -3001,6 +3017,7 @@ error:
static NV_STATUS gpu_retain_by_uuid_locked(const NvProcessorUuid *gpu_uuid,
const uvm_rm_user_object_t *user_rm_device,
const uvm_test_parent_gpu_inject_error_t *parent_gpu_error,
bool enable_hmm,
uvm_gpu_t **gpu_out)
{
NV_STATUS status = NV_OK;
@@ -3052,7 +3069,7 @@ static NV_STATUS gpu_retain_by_uuid_locked(const NvProcessorUuid *gpu_uuid,
if (status != NV_OK)
goto error_unregister;
status = add_gpu(gpu_uuid, gpu_id, gpu_info, &gpu_platform_info, parent_gpu, parent_gpu_error, &gpu);
status = add_gpu(gpu_uuid, gpu_id, gpu_info, &gpu_platform_info, parent_gpu, parent_gpu_error, enable_hmm, &gpu);
if (status != NV_OK)
goto error_unregister;
}
@@ -3078,11 +3095,12 @@ error_free_gpu_info:
NV_STATUS uvm_gpu_retain_by_uuid(const NvProcessorUuid *gpu_uuid,
const uvm_rm_user_object_t *user_rm_device,
const uvm_test_parent_gpu_inject_error_t *parent_gpu_error,
bool enable_hmm,
uvm_gpu_t **gpu_out)
{
NV_STATUS status;
uvm_mutex_lock(&g_uvm_global.global_lock);
status = gpu_retain_by_uuid_locked(gpu_uuid, user_rm_device, parent_gpu_error, gpu_out);
status = gpu_retain_by_uuid_locked(gpu_uuid, user_rm_device, parent_gpu_error, enable_hmm, gpu_out);
uvm_mutex_unlock(&g_uvm_global.global_lock);
return status;
}
@@ -3191,7 +3209,7 @@ uvm_gpu_phys_address_t uvm_gpu_peer_phys_address(uvm_gpu_t *owning_gpu, NvU64 ad
// depending on atomic capabilities of the peer devices.
uvm_parent_gpu_peer_t *parent_peer_caps = parent_gpu_peer_caps(accessing_gpu->parent, owning_gpu->parent);
int peer_index = (uvm_id_cmp(accessing_gpu->id, owning_gpu->id) < 0) ? 0 : 1;
UVM_ASSERT(parent_peer_caps->link_type == UVM_GPU_LINK_PCIE_BAR1);
UVM_ASSERT(parent_peer_caps->bar1_p2p_dma_size[peer_index] != 0);
@@ -3631,6 +3649,28 @@ NV_STATUS uvm_parent_gpu_add_user_channel(uvm_parent_gpu_t *parent_gpu, uvm_user
return NV_OK;
}
// Debugging aid to detect notifications that report instance pointers which
// should be dead and no longer observed.
static void instance_ptr_check_dead(uvm_parent_gpu_t *parent_gpu, uvm_gpu_phys_address_t instance_ptr)
{
uvm_user_channel_t *user_channel;
uvm_assert_spinlock_locked(&parent_gpu->instance_ptr_table_lock);
if (!uvm_enable_builtin_tests)
return;
list_for_each_entry(user_channel, &parent_gpu->test.dead_channels, test.dead.node) {
if (uvm_gpu_phys_addr_cmp(instance_ptr, user_channel->instance_ptr.addr) == 0) {
// We shouldn't have seen this instance pointer. Inform user space.
atomic64_t *ptr = (atomic64_t *)user_channel->test.dead.ptr;
UVM_ASSERT(ptr);
atomic64_inc(ptr);
break;
}
}
}
static uvm_user_channel_t *instance_ptr_to_user_channel(uvm_parent_gpu_t *parent_gpu,
uvm_gpu_phys_address_t instance_ptr)
{
@@ -3640,8 +3680,10 @@ static uvm_user_channel_t *instance_ptr_to_user_channel(uvm_parent_gpu_t *parent
uvm_assert_spinlock_locked(&parent_gpu->instance_ptr_table_lock);
instance_node = uvm_rb_tree_find(&parent_gpu->instance_ptr_table, key);
if (!instance_node)
if (!instance_node) {
instance_ptr_check_dead(parent_gpu, instance_ptr);
return NULL;
}
return get_user_channel(instance_node);
}
@@ -4015,7 +4057,7 @@ NV_STATUS uvm_api_pageable_mem_access_on_gpu(UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARA
return NV_ERR_INVALID_DEVICE;
}
if (uvm_va_space_pageable_mem_access_enabled(va_space) && gpu->parent->replayable_faults_supported)
if (uvm_va_space_pageable_mem_access_enabled(va_space))
params->pageableMemAccess = NV_TRUE;
else
params->pageableMemAccess = NV_FALSE;
@@ -4041,7 +4083,7 @@ NV_STATUS uvm_test_set_prefetch_filtering(UVM_TEST_SET_PREFETCH_FILTERING_PARAMS
goto done;
}
if (!gpu->parent->isr.replayable_faults.handling || !gpu->parent->prefetch_fault_supported) {
if (!gpu->parent->isr.replayable_faults.handling) {
status = NV_ERR_INVALID_DEVICE;
goto done;
}
@@ -4118,7 +4160,7 @@ NV_STATUS uvm_test_dump_access_bits(UVM_TEST_DUMP_ACCESS_BITS_PARAMS *params, st
}
// Copy the bits to user space
if (copy_to_user(params->current_bits,
if (copy_to_user(params->current_bits,
gpu->parent->vab_info.currentBits,
sizeof(NvU64) * params->current_bits_length)) {
status = NV_ERR_INVALID_ADDRESS;

67
kernel-open/nvidia-uvm/uvm_gpu.h
View File

@@ -467,6 +467,14 @@ typedef struct
// Information required to invalidate stale ATS PTEs from the GPU TLBs
uvm_ats_fault_invalidate_t ats_invalidate;
struct
{
// Not protected by any lock. Can only be set by one VA space at a
// time. When the VA space that set this unregisters this GPU, this
// is cleared.
atomic_t sleep_per_iteration_us;
} test;
} non_replayable;
// Flag that tells if prefetch faults are enabled in HW
@@ -714,14 +722,6 @@ struct uvm_gpu_struct
} mem_info;
struct
{
// Big page size used by the internal UVM VA space
// Notably it may be different than the big page size used by a user's
// VA space in general.
NvU32 internal_size;
} big_page;
// Mapped registers needed to obtain the current GPU timestamp
struct
{
@@ -940,6 +940,8 @@ typedef struct
bool isr_access_counters_alloc_stats_cpu;
bool access_counters_batch_context_notifications;
bool access_counters_batch_context_notification_cache;
bool disable_devmem;
} uvm_test_parent_gpu_inject_error_t;
// In order to support SMC/MIG GPU partitions, we split UVM GPUs into two
@@ -1054,23 +1056,9 @@ struct uvm_parent_gpu_struct
// Virtualization mode of the GPU.
UVM_VIRT_MODE virt_mode;
// Pascal+ GPUs can trigger faults on prefetch instructions. If false, this
// feature must be disabled at all times in GPUs of the given architecture.
// If true, the feature can be toggled at will by SW.
//
// The field should not be used unless the GPU supports replayable faults.
bool prefetch_fault_supported;
// Number of membars required to flush out HSHUB following a TLB invalidate
NvU32 num_hshub_tlb_invalidate_membars;
// Whether the channels can configure GPFIFO in vidmem
bool gpfifo_in_vidmem_supported;
bool replayable_faults_supported;
bool non_replayable_faults_supported;
bool access_counters_supported;
// True when HW does not allow mixing different clear types concurrently.
@@ -1078,11 +1066,6 @@ struct uvm_parent_gpu_struct
bool access_bits_supported;
bool fault_cancel_va_supported;
// True if the GPU has hardware support for scoped atomics
bool scoped_atomics_supported;
// If true, a HW method can be used to clear a faulted channel.
// If false, then the GPU supports clearing faulted channels using registers
// instead of a HW method.
@@ -1096,23 +1079,13 @@ struct uvm_parent_gpu_struct
// This value is only defined for GPUs that support non-replayable faults.
bool has_clear_faulted_channel_sw_method;
bool sparse_mappings_supported;
// Ampere(GA100) requires map->invalidate->remap->invalidate for page size
// promotion
bool map_remap_larger_page_promotion;
bool plc_supported;
// Parameters used by the TLB batching API
struct
{
// Is the targeted (single page) VA invalidate supported at all?
NvBool va_invalidate_supported;
// Is the VA range invalidate supported?
NvBool va_range_invalidate_supported;
union
{
// Maximum (inclusive) number of single page invalidations before
@@ -1131,9 +1104,6 @@ struct uvm_parent_gpu_struct
// Largest VA (exclusive) which Host can operate.
NvU64 max_host_va;
// Indicates whether the GPU can map sysmem with pages larger than 4k
bool can_map_sysmem_with_large_pages;
// An integrated GPU has no vidmem and coherent access to sysmem. Note
// integrated GPUs have a write-back L2 cache (cf. discrete GPUs
// write-through cache.)
@@ -1211,7 +1181,6 @@ struct uvm_parent_gpu_struct
// Interrupt handling state and locks
uvm_isr_info_t isr;
// This is only valid if supports_replayable_faults is set to true.
uvm_fault_buffer_t fault_buffer;
// PMM lazy free processing queue.
@@ -1406,7 +1375,14 @@ struct uvm_parent_gpu_struct
NvU64 peer_gpa_memory_window_start;
} peer_address_info;
uvm_test_parent_gpu_inject_error_t test;
struct
{
uvm_test_parent_gpu_inject_error_t inject_error;
// Channels whose instance pointers should never be observed in any
// fault notification. Synchronized using instance_ptr_table_lock.
struct list_head dead_channels;
} test;
// PASID ATS
bool ats_supported;
@@ -1602,6 +1578,7 @@ uvm_parent_gpu_t *uvm_parent_gpu_get_by_uuid_locked(const NvProcessorUuid *gpu_u
NV_STATUS uvm_gpu_retain_by_uuid(const NvProcessorUuid *gpu_uuid,
const uvm_rm_user_object_t *user_rm_device,
const uvm_test_parent_gpu_inject_error_t *parent_gpu_error,
bool enable_hmm,
uvm_gpu_t **gpu_out);
// Retain a gpu which is known to already be retained. Does NOT require the
@@ -1860,12 +1837,6 @@ static bool uvm_parent_gpu_needs_pushbuffer_segments(uvm_parent_gpu_t *parent_gp
return parent_gpu->max_host_va > (1ull << 40);
}
static bool uvm_parent_gpu_supports_eviction(uvm_parent_gpu_t *parent_gpu)
{
// Eviction is supported only if the GPU supports replayable faults
return parent_gpu->replayable_faults_supported;
}
static bool uvm_parent_gpu_is_virt_mode_sriov_heavy(const uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->virt_mode == UVM_VIRT_MODE_SRIOV_HEAVY;

4
kernel-open/nvidia-uvm/uvm_gpu_access_counters.c
View File

@@ -459,7 +459,7 @@ NV_STATUS uvm_parent_gpu_init_access_counters(uvm_parent_gpu_t *parent_gpu, NvU3
access_counters->max_batch_size);
}
if (uvm_enable_builtin_tests && parent_gpu->test.access_counters_batch_context_notification_cache) {
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.access_counters_batch_context_notification_cache) {
status = NV_ERR_NO_MEMORY;
goto fail;
}
@@ -471,7 +471,7 @@ NV_STATUS uvm_parent_gpu_init_access_counters(uvm_parent_gpu_t *parent_gpu, NvU3
goto fail;
}
if (uvm_enable_builtin_tests && parent_gpu->test.access_counters_batch_context_notifications) {
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.access_counters_batch_context_notifications) {
status = NV_ERR_NO_MEMORY;
goto fail;
}

184
kernel-open/nvidia-uvm/uvm_gpu_isr.c
View File

@@ -118,7 +118,7 @@ static unsigned schedule_replayable_faults_handler(uvm_parent_gpu_t *parent_gpu)
return 1;
}
static unsigned schedule_non_replayable_faults_handler(uvm_parent_gpu_t *parent_gpu)
unsigned uvm_parent_gpu_schedule_non_replayable_faults_handler(uvm_parent_gpu_t *parent_gpu)
{
bool scheduled;
@@ -261,7 +261,7 @@ static NV_STATUS uvm_isr_top_half(const NvProcessorUuid *gpu_uuid)
++parent_gpu->isr.interrupt_count;
num_handlers_scheduled += schedule_replayable_faults_handler(parent_gpu);
num_handlers_scheduled += schedule_non_replayable_faults_handler(parent_gpu);
num_handlers_scheduled += uvm_parent_gpu_schedule_non_replayable_faults_handler(parent_gpu);
for (i = 0; i < parent_gpu->rm_info.accessCntrBufferCount; i++)
num_handlers_scheduled += schedule_access_counters_handler(parent_gpu, i);
@@ -320,7 +320,7 @@ static NV_STATUS uvm_isr_init_access_counters(uvm_parent_gpu_t *parent_gpu, NvU3
return status;
}
if (uvm_enable_builtin_tests && parent_gpu->test.access_counters_alloc_block_context)
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.access_counters_alloc_block_context)
return NV_ERR_NO_MEMORY;
block_context = uvm_va_block_context_alloc(NULL);
@@ -338,7 +338,7 @@ static NV_STATUS uvm_isr_init_access_counters(uvm_parent_gpu_t *parent_gpu, NvU3
// dynamically enabled when the GPU is registered on a VA space.
parent_gpu->isr.access_counters[notif_buf_index].handling_ref_count = 0;
if (uvm_enable_builtin_tests && parent_gpu->test.isr_access_counters_alloc_stats_cpu)
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.isr_access_counters_alloc_stats_cpu)
return NV_ERR_NO_MEMORY;
parent_gpu->isr.access_counters[notif_buf_index].stats.cpu_exec_count =
@@ -356,98 +356,94 @@ NV_STATUS uvm_parent_gpu_init_isr(uvm_parent_gpu_t *parent_gpu)
char kthread_name[TASK_COMM_LEN + 1];
uvm_va_block_context_t *block_context;
if (parent_gpu->replayable_faults_supported) {
status = uvm_parent_gpu_fault_buffer_init(parent_gpu);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed to initialize GPU fault buffer: %s, GPU: %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
status = uvm_parent_gpu_fault_buffer_init(parent_gpu);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed to initialize GPU fault buffer: %s, GPU: %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
nv_kthread_q_item_init(&parent_gpu->isr.replayable_faults.bottom_half_q_item,
replayable_faults_isr_bottom_half_entry,
parent_gpu);
nv_kthread_q_item_init(&parent_gpu->isr.replayable_faults.bottom_half_q_item,
replayable_faults_isr_bottom_half_entry,
parent_gpu);
parent_gpu->isr.replayable_faults.stats.cpu_exec_count =
uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.replayable_faults.stats.cpu_exec_count) * num_possible_cpus());
if (!parent_gpu->isr.replayable_faults.stats.cpu_exec_count)
parent_gpu->isr.replayable_faults.stats.cpu_exec_count =
uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.replayable_faults.stats.cpu_exec_count) * num_possible_cpus());
if (!parent_gpu->isr.replayable_faults.stats.cpu_exec_count)
return NV_ERR_NO_MEMORY;
block_context = uvm_va_block_context_alloc(NULL);
if (!block_context)
return NV_ERR_NO_MEMORY;
parent_gpu->fault_buffer.replayable.block_service_context.block_context = block_context;
parent_gpu->isr.replayable_faults.handling = true;
snprintf(kthread_name, sizeof(kthread_name), "UVM GPU%u BH", uvm_parent_id_value(parent_gpu->id));
status = init_queue_on_node(&parent_gpu->isr.bottom_half_q, kthread_name, parent_gpu->closest_cpu_numa_node);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed in nv_kthread_q_init for bottom_half_q: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
nv_kthread_q_item_init(&parent_gpu->isr.non_replayable_faults.bottom_half_q_item,
non_replayable_faults_isr_bottom_half_entry,
parent_gpu);
parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count =
uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count) *
num_possible_cpus());
if (!parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count)
return NV_ERR_NO_MEMORY;
block_context = uvm_va_block_context_alloc(NULL);
if (!block_context)
return NV_ERR_NO_MEMORY;
parent_gpu->fault_buffer.non_replayable.block_service_context.block_context = block_context;
parent_gpu->isr.non_replayable_faults.handling = true;
snprintf(kthread_name, sizeof(kthread_name), "UVM GPU%u KC", uvm_parent_id_value(parent_gpu->id));
status = init_queue_on_node(&parent_gpu->isr.kill_channel_q,
kthread_name,
parent_gpu->closest_cpu_numa_node);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed in nv_kthread_q_init for kill_channel_q: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
if (parent_gpu->access_counters_supported) {
NvU32 index_count = parent_gpu->rm_info.accessCntrBufferCount;
NvU32 notif_buf_index;
UVM_ASSERT(index_count > 0);
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.access_counters_alloc_buffer)
return NV_ERR_NO_MEMORY;
block_context = uvm_va_block_context_alloc(NULL);
if (!block_context)
parent_gpu->access_counters.buffer = uvm_kvmalloc_zero(sizeof(*parent_gpu->access_counters.buffer) *
index_count);
if (!parent_gpu->access_counters.buffer)
return NV_ERR_NO_MEMORY;
parent_gpu->fault_buffer.replayable.block_service_context.block_context = block_context;
if (uvm_enable_builtin_tests && parent_gpu->test.inject_error.isr_access_counters_alloc)
return NV_ERR_NO_MEMORY;
parent_gpu->isr.replayable_faults.handling = true;
parent_gpu->isr.access_counters = uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.access_counters) * index_count);
if (!parent_gpu->isr.access_counters)
return NV_ERR_NO_MEMORY;
snprintf(kthread_name, sizeof(kthread_name), "UVM GPU%u BH", uvm_parent_id_value(parent_gpu->id));
status = init_queue_on_node(&parent_gpu->isr.bottom_half_q, kthread_name, parent_gpu->closest_cpu_numa_node);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed in nv_kthread_q_init for bottom_half_q: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
return status;
}
if (parent_gpu->non_replayable_faults_supported) {
nv_kthread_q_item_init(&parent_gpu->isr.non_replayable_faults.bottom_half_q_item,
non_replayable_faults_isr_bottom_half_entry,
parent_gpu);
parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count =
uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count) *
num_possible_cpus());
if (!parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count)
return NV_ERR_NO_MEMORY;
block_context = uvm_va_block_context_alloc(NULL);
if (!block_context)
return NV_ERR_NO_MEMORY;
parent_gpu->fault_buffer.non_replayable.block_service_context.block_context = block_context;
parent_gpu->isr.non_replayable_faults.handling = true;
snprintf(kthread_name, sizeof(kthread_name), "UVM GPU%u KC", uvm_parent_id_value(parent_gpu->id));
status = init_queue_on_node(&parent_gpu->isr.kill_channel_q,
kthread_name,
parent_gpu->closest_cpu_numa_node);
if (status != NV_OK) {
UVM_ERR_PRINT("Failed in nv_kthread_q_init for kill_channel_q: %s, GPU %s\n",
nvstatusToString(status),
uvm_parent_gpu_name(parent_gpu));
for (notif_buf_index = 0; notif_buf_index < index_count; notif_buf_index++) {
status = uvm_isr_init_access_counters(parent_gpu, notif_buf_index);
if (status != NV_OK)
return status;
}
}
if (parent_gpu->access_counters_supported) {
NvU32 index_count = parent_gpu->rm_info.accessCntrBufferCount;
NvU32 notif_buf_index;
UVM_ASSERT(index_count > 0);
if (uvm_enable_builtin_tests && parent_gpu->test.access_counters_alloc_buffer)
return NV_ERR_NO_MEMORY;
parent_gpu->access_counters.buffer = uvm_kvmalloc_zero(sizeof(*parent_gpu->access_counters.buffer) *
index_count);
if (!parent_gpu->access_counters.buffer)
return NV_ERR_NO_MEMORY;
if (uvm_enable_builtin_tests && parent_gpu->test.isr_access_counters_alloc)
return NV_ERR_NO_MEMORY;
parent_gpu->isr.access_counters = uvm_kvmalloc_zero(sizeof(*parent_gpu->isr.access_counters) * index_count);
if (!parent_gpu->isr.access_counters)
return NV_ERR_NO_MEMORY;
for (notif_buf_index = 0; notif_buf_index < index_count; notif_buf_index++) {
status = uvm_isr_init_access_counters(parent_gpu, notif_buf_index);
if (status != NV_OK)
return status;
}
}
}
@@ -549,12 +545,10 @@ void uvm_parent_gpu_deinit_isr(uvm_parent_gpu_t *parent_gpu)
uvm_kvfree(parent_gpu->access_counters.buffer);
}
if (parent_gpu->non_replayable_faults_supported) {
block_context = parent_gpu->fault_buffer.non_replayable.block_service_context.block_context;
uvm_va_block_context_free(block_context);
block_context = parent_gpu->fault_buffer.non_replayable.block_service_context.block_context;
uvm_va_block_context_free(block_context);
uvm_kvfree(parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count);
}
uvm_kvfree(parent_gpu->isr.non_replayable_faults.stats.cpu_exec_count);
block_context = parent_gpu->fault_buffer.replayable.block_service_context.block_context;
uvm_va_block_context_free(block_context);
@@ -600,8 +594,6 @@ static void replayable_faults_isr_bottom_half(void *args)
uvm_parent_gpu_t *parent_gpu = (uvm_parent_gpu_t *)args;
unsigned int cpu;
UVM_ASSERT(parent_gpu->replayable_faults_supported);
// Record the lock ownership
// The service_lock semaphore is taken in the top half using a raw
// semaphore call (down_trylock()). Here, the lock "ownership" is recorded,
@@ -640,8 +632,6 @@ static void non_replayable_faults_isr_bottom_half(void *args)
uvm_parent_gpu_t *parent_gpu = (uvm_parent_gpu_t *)args;
unsigned int cpu;
UVM_ASSERT(parent_gpu->non_replayable_faults_supported);
uvm_parent_gpu_non_replayable_faults_isr_lock(parent_gpu);
// Multiple bottom halves for non-replayable faults can be running
@@ -724,7 +714,7 @@ static void access_counters_isr_bottom_half_entry(void *args)
//
// While in the typical case the retriggering happens within a replayable
// fault bottom half, it can also happen within a non-interrupt path such as
// uvm_gpu_fault_buffer_flush.
// uvm_gpu_replayable_buffer_flush.
static void replayable_faults_retrigger_bottom_half(uvm_parent_gpu_t *parent_gpu)
{
if (!g_uvm_global.conf_computing_enabled)

8
kernel-open/nvidia-uvm/uvm_gpu_isr.h
View File

@@ -198,4 +198,12 @@ void uvm_access_counters_intr_enable(uvm_access_counter_buffer_t *access_counter
// g_uvm_global.global_lock is held so that the returned pointer remains valid.
uvm_gpu_t *uvm_parent_gpu_find_first_valid_gpu(uvm_parent_gpu_t *parent_gpu);
// Check if any non-replayable faults are pending, and if so schedule a bottom
// half to service them.
//
// Returns 1 if a new bottom half instance was scheduled, 0 if not.
//
// parent_gpu->isr.interrupts_lock must be held to call this function.
unsigned uvm_parent_gpu_schedule_non_replayable_faults_handler(uvm_parent_gpu_t *parent_gpu);
#endif // __UVM_GPU_ISR_H__

87
kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c
View File

@@ -121,8 +121,6 @@ NV_STATUS uvm_parent_gpu_fault_buffer_init_non_replayable_faults(uvm_parent_gpu_
{
uvm_non_replayable_fault_buffer_t *non_replayable_faults = &parent_gpu->fault_buffer.non_replayable;
UVM_ASSERT(parent_gpu->non_replayable_faults_supported);
non_replayable_faults->shadow_buffer_copy = NULL;
non_replayable_faults->fault_cache = NULL;
@@ -170,13 +168,43 @@ bool uvm_parent_gpu_non_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu)
UVM_ASSERT(parent_gpu->isr.non_replayable_faults.handling);
status = nvUvmInterfaceHasPendingNonReplayableFaults(&parent_gpu->fault_buffer.rm_info,
&has_pending_faults);
// RM requires the caller to serialize calls on the same GPU
uvm_assert_spinlock_locked(&parent_gpu->isr.interrupts_lock);
status = nvUvmInterfaceHasPendingNonReplayableFaults(&parent_gpu->fault_buffer.rm_info, &has_pending_faults);
UVM_ASSERT(status == NV_OK);
return has_pending_faults == NV_TRUE;
}
void uvm_parent_gpu_non_replayable_buffer_flush(uvm_parent_gpu_t *parent_gpu)
{
// If faults are pending we need to schedule a bottom half invocation. It's
// not enough to wait for in-flight bottom halves because we might have
// checked after RM added the faults but before UVM's top half was invoked
// to schedule the bottom half.
// uvm_parent_gpu_schedule_non_replayable_faults_handler() does that check
// for us.
uvm_spin_lock_irqsave(&parent_gpu->isr.interrupts_lock);
uvm_parent_gpu_schedule_non_replayable_faults_handler(parent_gpu);
uvm_spin_unlock_irqrestore(&parent_gpu->isr.interrupts_lock);
// Wait for the non-replayable bottom half to finish servicing. This relies
// on the greedy nature of the non-replayable bottom half:
// fetch_non_replayable_fault_buffer_entries() always pulls all available
// faults in one go and services them in that instance. Those fetched faults
// are the ones we need to flush. Therefore we just need to be sure that
// bottom halves scheduled prior to this call are complete.
//
// We have to wait even if no pending fault was detected, because we might
// have caught the window after fetch but before service.
//
// This is overkill since it will wait for all bottom half types, not just
// the non-replayable one, but this is not expected to be on the critical
// path.
nv_kthread_q_flush(&parent_gpu->isr.bottom_half_q);
}
static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *parent_gpu, NvU32 *cached_faults)
{
NV_STATUS status;
@@ -187,7 +215,6 @@ static NV_STATUS fetch_non_replayable_fault_buffer_entries(uvm_parent_gpu_t *par
uvm_fault_buffer_entry_t *fault_entry = non_replayable_faults->fault_cache;
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.non_replayable_faults.service_lock));
UVM_ASSERT(parent_gpu->non_replayable_faults_supported);
status = nvUvmInterfaceGetNonReplayableFaults(&parent_gpu->fault_buffer.rm_info,
current_hw_entry,
@@ -387,9 +414,7 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_va_block_t *va_block,
gpu->id,
uvm_va_block_cpu_page_index(va_block,
fault_entry->fault_address),
fault_entry->fault_access_type,
uvm_range_group_address_migratable(va_space,
fault_entry->fault_address));
fault_entry->fault_access_type);
if (status != NV_OK) {
fault_entry->is_fatal = true;
fault_entry->fatal_reason = uvm_tools_status_to_fatal_fault_reason(status);
@@ -786,6 +811,12 @@ void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent
NV_STATUS status;
NvU32 i;
if (uvm_enable_builtin_tests) {
NvU32 sleep_us = atomic_read(&parent_gpu->fault_buffer.non_replayable.test.sleep_per_iteration_us);
if (sleep_us)
usleep_range(sleep_us, sleep_us * 2);
}
status = fetch_non_replayable_fault_buffer_entries(parent_gpu, &cached_faults);
if (status != NV_OK)
return;
@@ -800,3 +831,43 @@ void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent
}
} while (cached_faults > 0);
}
NV_STATUS uvm_test_set_non_replayable_delay(UVM_TEST_SET_NON_REPLAYABLE_DELAY_PARAMS *params, struct file *filp)
{
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_gpu_t *gpu;
NV_STATUS status = NV_OK;
uvm_va_space_down_write(va_space);
gpu = uvm_va_space_get_gpu_by_uuid(va_space, &params->gpu_uuid);
if (!gpu) {
status = NV_ERR_INVALID_DEVICE;
goto out;
}
if (uvm_parent_processor_mask_test(&va_space->test.non_replayable_delay_set, gpu->parent->id)) {
// If we already own the setting, just clobber it
atomic_set(&gpu->parent->fault_buffer.non_replayable.test.sleep_per_iteration_us,
params->sleep_per_iteration_us);
if (params->sleep_per_iteration_us == 0)
uvm_parent_processor_mask_clear(&va_space->test.non_replayable_delay_set, gpu->parent->id);
}
else if (params->sleep_per_iteration_us == 0) {
status = NV_ERR_INVALID_ARGUMENT;
}
else {
// We don't own the setting. Try to take control.
int old = atomic_cmpxchg(&gpu->parent->fault_buffer.non_replayable.test.sleep_per_iteration_us,
0,
params->sleep_per_iteration_us);
if (old == 0)
uvm_parent_processor_mask_set(&va_space->test.non_replayable_delay_set, gpu->parent->id);
else
status = NV_ERR_IN_USE;
}
out:
uvm_va_space_up_write(va_space);
return status;
}

18
kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.h
View File

@@ -28,10 +28,28 @@
bool uvm_parent_gpu_non_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu);
// Flush the non-replayable shadow buffer (see the comment at the top of
// uvm_gpu_non_replayable_faults.c for an explanation of the shadow buffer).
// All faults in the shadow buffer when this function is called will have been
// removed and serviced upon return.
//
// For faults that are flushed, it is not guaranteed that the HW channel faulted
// state will have been cleared on return, nor that fatal faults have been
// serviced by RM.
//
// Faults added to the shadow buffer during this call are not guaranteed to be
// flushed.
//
// LOCKING: This waits for faults to be serviced, so it must not be called while
// holding the ISR or VA space locks.
void uvm_parent_gpu_non_replayable_buffer_flush(uvm_parent_gpu_t *parent_gpu);
void uvm_parent_gpu_service_non_replayable_fault_buffer(uvm_parent_gpu_t *parent_gpu);
NV_STATUS uvm_parent_gpu_fault_buffer_init_non_replayable_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_parent_gpu_fault_buffer_deinit_non_replayable_faults(uvm_parent_gpu_t *parent_gpu);
NV_STATUS uvm_test_set_non_replayable_delay(UVM_TEST_SET_NON_REPLAYABLE_DELAY_PARAMS *params, struct file *filp);
#endif // __UVM_GPU_NON_REPLAYABLE_FAULTS_H__

67
kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.c
View File

@@ -213,7 +213,7 @@ static NV_STATUS fault_buffer_init_replayable_faults(uvm_parent_gpu_t *parent_gp
}
// Re-enable fault prefetching just in case it was disabled in a previous run
parent_gpu->fault_buffer.prefetch_faults_enabled = parent_gpu->prefetch_fault_supported;
parent_gpu->fault_buffer.prefetch_faults_enabled = true;
fault_buffer_reinit_replayable_faults(parent_gpu);
@@ -232,7 +232,7 @@ static void fault_buffer_deinit_replayable_faults(uvm_parent_gpu_t *parent_gpu)
if (parent_gpu->fault_buffer.rm_info.faultBufferHandle) {
// Re-enable prefetch faults in case we disabled them
if (parent_gpu->prefetch_fault_supported && !parent_gpu->fault_buffer.prefetch_faults_enabled)
if (!parent_gpu->fault_buffer.prefetch_faults_enabled)
parent_gpu->arch_hal->enable_prefetch_faults(parent_gpu);
}
@@ -249,7 +249,6 @@ NV_STATUS uvm_parent_gpu_fault_buffer_init(uvm_parent_gpu_t *parent_gpu)
NV_STATUS status = NV_OK;
uvm_assert_mutex_locked(&g_uvm_global.global_lock);
UVM_ASSERT(parent_gpu->replayable_faults_supported);
status = uvm_rm_locked_call(nvUvmInterfaceInitFaultInfo(parent_gpu->rm_device,
&parent_gpu->fault_buffer.rm_info));
@@ -270,11 +269,9 @@ NV_STATUS uvm_parent_gpu_fault_buffer_init(uvm_parent_gpu_t *parent_gpu)
if (status != NV_OK)
goto fail;
if (parent_gpu->non_replayable_faults_supported) {
status = uvm_parent_gpu_fault_buffer_init_non_replayable_faults(parent_gpu);
if (status != NV_OK)
goto fail;
}
status = uvm_parent_gpu_fault_buffer_init_non_replayable_faults(parent_gpu);
if (status != NV_OK)
goto fail;
return NV_OK;
@@ -288,8 +285,6 @@ fail:
// from a power management cycle.
void uvm_parent_gpu_fault_buffer_resume(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT(parent_gpu->replayable_faults_supported);
fault_buffer_reinit_replayable_faults(parent_gpu);
}
@@ -299,8 +294,7 @@ void uvm_parent_gpu_fault_buffer_deinit(uvm_parent_gpu_t *parent_gpu)
uvm_assert_mutex_locked(&g_uvm_global.global_lock);
if (parent_gpu->non_replayable_faults_supported)
uvm_parent_gpu_fault_buffer_deinit_non_replayable_faults(parent_gpu);
uvm_parent_gpu_fault_buffer_deinit_non_replayable_faults(parent_gpu);
fault_buffer_deinit_replayable_faults(parent_gpu);
@@ -319,8 +313,6 @@ bool uvm_parent_gpu_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu)
{
uvm_replayable_fault_buffer_t *replayable_faults = &parent_gpu->fault_buffer.replayable;
UVM_ASSERT(parent_gpu->replayable_faults_supported);
// Fast path 1: we left some faults unserviced in the buffer in the last
// pass
if (replayable_faults->cached_get != replayable_faults->cached_put)
@@ -447,7 +439,6 @@ static NV_STATUS cancel_fault_precise_va(uvm_fault_buffer_entry_t *fault_entry,
uvm_replayable_fault_buffer_t *replayable_faults = &gpu->parent->fault_buffer.replayable;
NvU64 offset;
UVM_ASSERT(gpu->parent->replayable_faults_supported);
UVM_ASSERT(fault_entry->fatal_reason != UvmEventFatalReasonInvalid);
UVM_ASSERT(!fault_entry->filtered);
@@ -623,7 +614,6 @@ static NV_STATUS fault_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu,
NV_STATUS status;
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
UVM_ASSERT(parent_gpu->replayable_faults_supported);
// Wait for the prior replay to flush out old fault messages
if (flush_mode == UVM_GPU_BUFFER_FLUSH_MODE_WAIT_UPDATE_PUT) {
@@ -671,12 +661,10 @@ static NV_STATUS fault_buffer_flush_locked(uvm_parent_gpu_t *parent_gpu,
return push_replay_on_parent_gpu(parent_gpu, fault_replay, batch_context);
}
NV_STATUS uvm_gpu_fault_buffer_flush(uvm_gpu_t *gpu)
NV_STATUS uvm_gpu_replayable_buffer_flush(uvm_gpu_t *gpu)
{
NV_STATUS status = NV_OK;
UVM_ASSERT(gpu->parent->replayable_faults_supported);
// Disables replayable fault interrupts and fault servicing
uvm_parent_gpu_replayable_faults_isr_lock(gpu->parent);
@@ -854,11 +842,9 @@ static NV_STATUS fetch_fault_buffer_entries(uvm_parent_gpu_t *parent_gpu,
uvm_spin_loop_t spin;
NV_STATUS status = NV_OK;
uvm_replayable_fault_buffer_t *replayable_faults = &parent_gpu->fault_buffer.replayable;
const bool in_pascal_cancel_path = (!parent_gpu->fault_cancel_va_supported && fetch_mode == FAULT_FETCH_MODE_ALL);
const bool may_filter = uvm_perf_fault_coalesce && !in_pascal_cancel_path;
const bool may_filter = uvm_perf_fault_coalesce;
UVM_ASSERT(uvm_sem_is_locked(&parent_gpu->isr.replayable_faults.service_lock));
UVM_ASSERT(parent_gpu->replayable_faults_supported);
fault_cache = batch_context->fault_cache;
@@ -1294,8 +1280,7 @@ static uvm_fault_access_type_t check_fault_access_permissions(uvm_gpu_t *gpu,
uvm_fault_service_batch_context_t *batch_context,
uvm_va_block_t *va_block,
uvm_service_block_context_t *service_block_context,
uvm_fault_buffer_entry_t *fault_entry,
bool allow_migration)
uvm_fault_buffer_entry_t *fault_entry)
{
NV_STATUS perm_status;
UvmEventFatalReason fatal_reason;
@@ -1308,8 +1293,7 @@ static uvm_fault_access_type_t check_fault_access_permissions(uvm_gpu_t *gpu,
gpu->id,
uvm_va_block_cpu_page_index(va_block,
fault_entry->fault_address),
fault_entry->fault_access_type,
allow_migration);
fault_entry->fault_access_type);
if (perm_status == NV_OK)
return fault_entry->fault_access_type;
@@ -1338,8 +1322,7 @@ static uvm_fault_access_type_t check_fault_access_permissions(uvm_gpu_t *gpu,
gpu->id,
uvm_va_block_cpu_page_index(va_block,
fault_entry->fault_address),
UVM_FAULT_ACCESS_TYPE_READ,
allow_migration);
UVM_FAULT_ACCESS_TYPE_READ);
if (perm_status == NV_OK) {
ret = UVM_FAULT_ACCESS_TYPE_READ;
}
@@ -1494,8 +1477,7 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
batch_context,
va_block,
block_context,
current_entry,
iter.migratable);
current_entry);
// Do not exit early due to logical errors such as access permission
// violation.
@@ -1590,9 +1572,6 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_t *gpu,
++block_context->num_retries;
if (status == NV_OK && batch_context->fatal_va_space)
status = uvm_va_block_set_cancel(va_block, block_context->block_context, gpu);
return status;
}
@@ -2053,7 +2032,6 @@ static NV_STATUS service_fault_batch_for_cancel(uvm_fault_service_batch_context_
UVM_ASSERT(va_space);
UVM_ASSERT(gpu);
UVM_ASSERT(gpu->parent->replayable_faults_supported);
// Perform the flush and re-fetch while holding the mmap_lock and the
// VA space lock. This avoids stale faults because it prevents any vma
@@ -2246,8 +2224,6 @@ static NV_STATUS service_fault_batch(uvm_parent_gpu_t *parent_gpu,
uvm_va_block_context_t *va_block_context = service_context->block_context;
bool hmm_migratable = true;
UVM_ASSERT(parent_gpu->replayable_faults_supported);
ats_invalidate->tlb_batch_pending = false;
for (i = 0; i < batch_context->num_coalesced_faults;) {
@@ -2693,10 +2669,7 @@ static void cancel_fault_batch(uvm_parent_gpu_t *parent_gpu,
{
// Return code is ignored since we're on a global error path and wouldn't be
// able to recover anyway.
if (parent_gpu->fault_cancel_va_supported)
cancel_faults_all(batch_context, reason);
else
cancel_fault_batch_tlb(batch_context, reason);
cancel_faults_all(batch_context, reason);
}
@@ -2731,8 +2704,6 @@ static NV_STATUS cancel_faults_precise_tlb(uvm_gpu_t *gpu, uvm_fault_service_bat
uvm_replayable_fault_buffer_t *replayable_faults = &gpu->parent->fault_buffer.replayable;
bool first = true;
UVM_ASSERT(gpu->parent->replayable_faults_supported);
// 1) Disable prefetching to avoid new requests keep coming and flooding
// the buffer
if (gpu->parent->fault_buffer.prefetch_faults_enabled)
@@ -2868,18 +2839,12 @@ static NV_STATUS cancel_faults_precise(uvm_fault_service_batch_context_t *batch_
UVM_ASSERT(batch_context->fatal_gpu);
gpu = batch_context->fatal_gpu;
if (gpu->parent->fault_cancel_va_supported)
return service_fault_batch_for_cancel(batch_context);
return cancel_faults_precise_tlb(gpu, batch_context);
return service_fault_batch_for_cancel(batch_context);
}
static void enable_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu,
uvm_fault_service_batch_context_t *batch_context)
{
if (!parent_gpu->prefetch_fault_supported)
return;
// If more than 66% of faults are invalid prefetch accesses, disable
// prefetch faults for a while.
// num_invalid_prefetch_faults may be higher than the actual count. See the
@@ -2912,8 +2877,6 @@ void uvm_parent_gpu_service_replayable_faults(uvm_parent_gpu_t *parent_gpu)
uvm_replayable_fault_buffer_t *replayable_faults = &parent_gpu->fault_buffer.replayable;
uvm_fault_service_batch_context_t *batch_context = &replayable_faults->batch_service_context;
UVM_ASSERT(parent_gpu->replayable_faults_supported);
uvm_tracker_init(&batch_context->tracker);
// Process all faults in the buffer
@@ -3030,7 +2993,6 @@ void uvm_parent_gpu_service_replayable_faults(uvm_parent_gpu_t *parent_gpu)
void uvm_parent_gpu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT(parent_gpu->isr.replayable_faults.handling);
UVM_ASSERT(parent_gpu->prefetch_fault_supported);
if (!parent_gpu->fault_buffer.prefetch_faults_enabled) {
parent_gpu->arch_hal->enable_prefetch_faults(parent_gpu);
@@ -3041,7 +3003,6 @@ void uvm_parent_gpu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu)
void uvm_parent_gpu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT(parent_gpu->isr.replayable_faults.handling);
UVM_ASSERT(parent_gpu->prefetch_fault_supported);
if (parent_gpu->fault_buffer.prefetch_faults_enabled) {
parent_gpu->arch_hal->disable_prefetch_faults(parent_gpu);

4
kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.h
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2024 NVIDIA Corporation
Copyright (c) 2015-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -65,7 +65,7 @@ bool uvm_parent_gpu_replayable_faults_pending(uvm_parent_gpu_t *parent_gpu);
// to the GPU's replay_tracker.
//
// LOCKING: Takes gpu->isr_lock
NV_STATUS uvm_gpu_fault_buffer_flush(uvm_gpu_t *gpu);
NV_STATUS uvm_gpu_replayable_buffer_flush(uvm_gpu_t *gpu);
// Enable/disable HW support for prefetch-initiated faults
void uvm_parent_gpu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);

366
kernel-open/nvidia-uvm/uvm_hal.c
View File

@@ -25,18 +25,7 @@
#include "uvm_global.h"
#include "uvm_kvmalloc.h"
#include "cla16f.h"
#include "clb069.h"
#include "clb06f.h"
#include "clb0b5.h"
#include "clc06f.h"
#include "clc0b5.h"
#include "clc1b5.h"
#include "ctrl2080mc.h"
#include "clc3b5.h"
#include "clc36f.h"
#include "clc369.h"
#include "clc365.h"
#include "clc46f.h"
#include "clc5b5.h"
#include "clc6b5.h"
@@ -67,66 +56,32 @@ MODULE_PARM_DESC(uvm_downgrade_force_membar_sys, "Force all TLB invalidation dow
// load. The parent class must appear earlier in the array than the child.
static uvm_hal_class_ops_t ce_table[] =
{
{
.id = MAXWELL_DMA_COPY_A,
.u.ce_ops = {
.init = uvm_hal_maxwell_ce_init,
.method_is_valid = uvm_hal_method_is_valid_stub,
.semaphore_release = uvm_hal_maxwell_ce_semaphore_release,
.semaphore_timestamp = uvm_hal_maxwell_ce_semaphore_timestamp,
.semaphore_reduction_inc = uvm_hal_maxwell_ce_semaphore_reduction_inc,
.semaphore_target_is_valid = uvm_hal_maxwell_semaphore_target_is_valid,
.offset_out = uvm_hal_maxwell_ce_offset_out,
.offset_in_out = uvm_hal_maxwell_ce_offset_in_out,
.phys_mode = uvm_hal_maxwell_ce_phys_mode,
.plc_mode = uvm_hal_maxwell_ce_plc_mode,
.memcopy_copy_type = uvm_hal_maxwell_ce_memcopy_copy_type,
.memcopy_is_valid = uvm_hal_maxwell_ce_memcopy_is_valid,
.memcopy_patch_src = uvm_hal_ce_memcopy_patch_src_stub,
.memcopy = uvm_hal_maxwell_ce_memcopy,
.memcopy_v_to_v = uvm_hal_maxwell_ce_memcopy_v_to_v,
.memset_is_valid = uvm_hal_maxwell_ce_memset_is_valid,
.memset_1 = uvm_hal_maxwell_ce_memset_1,
.memset_4 = uvm_hal_maxwell_ce_memset_4,
.memset_8 = uvm_hal_maxwell_ce_memset_8,
.memset_v_4 = uvm_hal_maxwell_ce_memset_v_4,
.encrypt = uvm_hal_maxwell_ce_encrypt_unsupported,
.decrypt = uvm_hal_maxwell_ce_decrypt_unsupported,
}
},
{
.id = PASCAL_DMA_COPY_A,
.parent_id = MAXWELL_DMA_COPY_A,
.u.ce_ops = {
.semaphore_release = uvm_hal_pascal_ce_semaphore_release,
.semaphore_timestamp = uvm_hal_pascal_ce_semaphore_timestamp,
.semaphore_reduction_inc = uvm_hal_pascal_ce_semaphore_reduction_inc,
.offset_out = uvm_hal_pascal_ce_offset_out,
.offset_in_out = uvm_hal_pascal_ce_offset_in_out,
}
},
{
.id = PASCAL_DMA_COPY_B,
.parent_id = PASCAL_DMA_COPY_A,
.u.ce_ops = {}
},
{
.id = VOLTA_DMA_COPY_A,
.parent_id = PASCAL_DMA_COPY_B,
.u.ce_ops = {
.semaphore_release = uvm_hal_volta_ce_semaphore_release,
.semaphore_timestamp = uvm_hal_volta_ce_semaphore_timestamp,
.semaphore_reduction_inc = uvm_hal_volta_ce_semaphore_reduction_inc,
.memcopy = uvm_hal_volta_ce_memcopy,
.memset_1 = uvm_hal_volta_ce_memset_1,
.memset_4 = uvm_hal_volta_ce_memset_4,
.memset_8 = uvm_hal_volta_ce_memset_8,
},
},
{
.id = TURING_DMA_COPY_A,
.parent_id = VOLTA_DMA_COPY_A,
.u.ce_ops = {},
.u.ce_ops = {
.init = uvm_hal_turing_ce_init,
.method_is_valid = uvm_hal_method_is_valid_stub,
.semaphore_release = uvm_hal_turing_ce_semaphore_release,
.semaphore_timestamp = uvm_hal_turing_ce_semaphore_timestamp,
.semaphore_reduction_inc = uvm_hal_turing_ce_semaphore_reduction_inc,
.semaphore_target_is_valid = uvm_hal_turing_semaphore_target_is_valid,
.offset_out = uvm_hal_turing_ce_offset_out,
.offset_in_out = uvm_hal_turing_ce_offset_in_out,
.phys_mode = uvm_hal_turing_ce_phys_mode,
.plc_mode = uvm_hal_turing_ce_plc_mode,
.memcopy_copy_type = uvm_hal_turing_ce_memcopy_copy_type,
.memcopy_is_valid = uvm_hal_turing_ce_memcopy_is_valid,
.memcopy_patch_src = uvm_hal_ce_memcopy_patch_src_stub,
.memcopy = uvm_hal_turing_ce_memcopy,
.memcopy_v_to_v = uvm_hal_turing_ce_memcopy_v_to_v,
.memset_is_valid = uvm_hal_turing_ce_memset_is_valid,
.memset_1 = uvm_hal_turing_ce_memset_1,
.memset_4 = uvm_hal_turing_ce_memset_4,
.memset_8 = uvm_hal_turing_ce_memset_8,
.memset_v_4 = uvm_hal_turing_ce_memset_v_4,
.encrypt = uvm_hal_turing_ce_encrypt_unsupported,
.decrypt = uvm_hal_turing_ce_decrypt_unsupported,
},
},
{
.id = AMPERE_DMA_COPY_A,
@@ -145,9 +100,9 @@ static uvm_hal_class_ops_t ce_table[] =
.u.ce_ops = {
.method_is_valid = uvm_hal_method_is_valid_stub,
.plc_mode = uvm_hal_ampere_ce_plc_mode_c7b5,
.memcopy_is_valid = uvm_hal_maxwell_ce_memcopy_is_valid,
.memcopy_is_valid = uvm_hal_turing_ce_memcopy_is_valid,
.memcopy_patch_src = uvm_hal_ce_memcopy_patch_src_stub,
.memset_is_valid = uvm_hal_maxwell_ce_memset_is_valid,
.memset_is_valid = uvm_hal_turing_ce_memset_is_valid,
},
},
{
@@ -188,95 +143,41 @@ static uvm_hal_class_ops_t ce_table[] =
static uvm_hal_class_ops_t host_table[] =
{
{
// This host class is reported for GM10x
.id = KEPLER_CHANNEL_GPFIFO_B,
.id = TURING_CHANNEL_GPFIFO_A,
.u.host_ops = {
.init = uvm_hal_maxwell_host_init_noop,
.init = uvm_hal_turing_host_init,
.method_is_valid = uvm_hal_method_is_valid_stub,
.sw_method_is_valid = uvm_hal_method_is_valid_stub,
.wait_for_idle = uvm_hal_maxwell_host_wait_for_idle,
.membar_sys = uvm_hal_maxwell_host_membar_sys,
// No MEMBAR GPU until Pascal, just do a MEMBAR SYS.
.membar_gpu = uvm_hal_maxwell_host_membar_sys,
.noop = uvm_hal_maxwell_host_noop,
.interrupt = uvm_hal_maxwell_host_interrupt,
.semaphore_acquire = uvm_hal_maxwell_host_semaphore_acquire,
.semaphore_release = uvm_hal_maxwell_host_semaphore_release,
.semaphore_timestamp = uvm_hal_maxwell_host_semaphore_timestamp,
.semaphore_target_is_valid = uvm_hal_maxwell_semaphore_target_is_valid,
.set_gpfifo_entry = uvm_hal_maxwell_host_set_gpfifo_entry,
.set_gpfifo_noop = uvm_hal_maxwell_host_set_gpfifo_noop,
.set_gpfifo_pushbuffer_segment_base = uvm_hal_maxwell_host_set_gpfifo_pushbuffer_segment_base_unsupported,
.write_gpu_put = uvm_hal_maxwell_host_write_gpu_put,
.tlb_invalidate_all = uvm_hal_maxwell_host_tlb_invalidate_all_a16f,
.tlb_invalidate_va = uvm_hal_maxwell_host_tlb_invalidate_va,
.tlb_invalidate_phys = uvm_hal_maxwell_host_tlb_invalidate_phys_unsupported,
.tlb_invalidate_test = uvm_hal_maxwell_host_tlb_invalidate_test,
.tlb_flush_prefetch = uvm_hal_maxwell_host_tlb_flush_prefetch_unsupported,
.replay_faults = uvm_hal_maxwell_replay_faults_unsupported,
.cancel_faults_global = uvm_hal_maxwell_cancel_faults_global_unsupported,
.cancel_faults_targeted = uvm_hal_maxwell_cancel_faults_targeted_unsupported,
.cancel_faults_va = uvm_hal_maxwell_cancel_faults_va_unsupported,
.clear_faulted_channel_sw_method = uvm_hal_maxwell_host_clear_faulted_channel_sw_method_unsupported,
.clear_faulted_channel_method = uvm_hal_maxwell_host_clear_faulted_channel_method_unsupported,
.clear_faulted_channel_register = uvm_hal_maxwell_host_clear_faulted_channel_register_unsupported,
.access_counter_clear_all = uvm_hal_maxwell_access_counter_clear_all_unsupported,
.access_counter_clear_targeted = uvm_hal_maxwell_access_counter_clear_targeted_unsupported,
.access_counter_query_clear_op = uvm_hal_maxwell_access_counter_query_clear_op_unsupported,
.l2_invalidate = uvm_hal_host_l2_invalidate_unsupported,
.get_time = uvm_hal_maxwell_get_time,
}
},
{
// This host class is reported for GM20x
.id = MAXWELL_CHANNEL_GPFIFO_A,
.parent_id = KEPLER_CHANNEL_GPFIFO_B,
.u.host_ops = {
.tlb_invalidate_all = uvm_hal_maxwell_host_tlb_invalidate_all_b06f,
}
},
{
.id = PASCAL_CHANNEL_GPFIFO_A,
.parent_id = MAXWELL_CHANNEL_GPFIFO_A,
.u.host_ops = {
.init = uvm_hal_pascal_host_init,
.membar_sys = uvm_hal_pascal_host_membar_sys,
.membar_gpu = uvm_hal_pascal_host_membar_gpu,
.tlb_invalidate_all = uvm_hal_pascal_host_tlb_invalidate_all,
.tlb_invalidate_va = uvm_hal_pascal_host_tlb_invalidate_va,
.tlb_invalidate_test = uvm_hal_pascal_host_tlb_invalidate_test,
.replay_faults = uvm_hal_pascal_replay_faults,
.cancel_faults_global = uvm_hal_pascal_cancel_faults_global,
.cancel_faults_targeted = uvm_hal_pascal_cancel_faults_targeted,
}
},
{
.id = VOLTA_CHANNEL_GPFIFO_A,
.parent_id = PASCAL_CHANNEL_GPFIFO_A,
.u.host_ops = {
.write_gpu_put = uvm_hal_volta_host_write_gpu_put,
.tlb_invalidate_va = uvm_hal_volta_host_tlb_invalidate_va,
.replay_faults = uvm_hal_volta_replay_faults,
.cancel_faults_va = uvm_hal_volta_cancel_faults_va,
.clear_faulted_channel_method = uvm_hal_volta_host_clear_faulted_channel_method,
.semaphore_timestamp = uvm_hal_volta_host_semaphore_timestamp,
}
},
{
.id = TURING_CHANNEL_GPFIFO_A,
.parent_id = VOLTA_CHANNEL_GPFIFO_A,
.u.host_ops = {
.wait_for_idle = uvm_hal_turing_host_wait_for_idle,
.membar_sys = uvm_hal_turing_host_membar_sys,
.membar_gpu = uvm_hal_turing_host_membar_gpu,
.noop = uvm_hal_turing_host_noop,
.interrupt = uvm_hal_turing_host_interrupt,
.semaphore_acquire = uvm_hal_turing_host_semaphore_acquire,
.semaphore_release = uvm_hal_turing_host_semaphore_release,
.clear_faulted_channel_method = uvm_hal_turing_host_clear_faulted_channel_method,
.semaphore_timestamp = uvm_hal_turing_host_semaphore_timestamp,
.semaphore_target_is_valid = uvm_hal_turing_semaphore_target_is_valid,
.set_gpfifo_entry = uvm_hal_turing_host_set_gpfifo_entry,
.set_gpfifo_noop = uvm_hal_turing_host_set_gpfifo_noop,
.set_gpfifo_pushbuffer_segment_base = uvm_hal_turing_host_set_gpfifo_pushbuffer_segment_base_unsupported,
.write_gpu_put = uvm_hal_turing_host_write_gpu_put,
.tlb_invalidate_all = uvm_hal_turing_host_tlb_invalidate_all,
.tlb_invalidate_va = uvm_hal_turing_host_tlb_invalidate_va,
.tlb_invalidate_phys = uvm_hal_turing_host_tlb_invalidate_phys_unsupported,
.tlb_invalidate_test = uvm_hal_turing_host_tlb_invalidate_test,
.tlb_flush_prefetch = uvm_hal_turing_host_tlb_flush_prefetch_unsupported,
.replay_faults = uvm_hal_turing_replay_faults,
.cancel_faults_global = uvm_hal_turing_cancel_faults_global,
.cancel_faults_targeted = uvm_hal_turing_cancel_faults_targeted,
.cancel_faults_va = uvm_hal_turing_cancel_faults_va,
.clear_faulted_channel_sw_method = uvm_hal_turing_host_clear_faulted_channel_sw_method_unsupported,
.clear_faulted_channel_method = uvm_hal_turing_host_clear_faulted_channel_method,
.clear_faulted_channel_register = uvm_hal_turing_host_clear_faulted_channel_register_unsupported,
.access_counter_clear_all = uvm_hal_turing_access_counter_clear_all,
.access_counter_clear_targeted = uvm_hal_turing_access_counter_clear_targeted,
.access_counter_query_clear_op = uvm_hal_turing_access_counter_query_clear_op,
.l2_invalidate = uvm_hal_host_l2_invalidate_unsupported,
.get_time = uvm_hal_turing_get_time,
}
},
{
@@ -334,47 +235,15 @@ static uvm_hal_class_ops_t host_table[] =
static uvm_hal_class_ops_t arch_table[] =
{
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.arch_ops = {
.init_properties = uvm_hal_maxwell_arch_init_properties,
.mmu_mode_hal = uvm_hal_mmu_mode_maxwell,
.enable_prefetch_faults = uvm_hal_maxwell_mmu_enable_prefetch_faults_unsupported,
.disable_prefetch_faults = uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported,
.mmu_client_id_to_utlb_id = uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.arch_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.u.arch_ops = {
.init_properties = uvm_hal_pascal_arch_init_properties,
.mmu_mode_hal = uvm_hal_mmu_mode_pascal,
.enable_prefetch_faults = uvm_hal_pascal_mmu_enable_prefetch_faults,
.disable_prefetch_faults = uvm_hal_pascal_mmu_disable_prefetch_faults,
.mmu_client_id_to_utlb_id = uvm_hal_pascal_mmu_client_id_to_utlb_id,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.u.arch_ops = {
.init_properties = uvm_hal_volta_arch_init_properties,
.mmu_mode_hal = uvm_hal_mmu_mode_volta,
.mmu_client_id_to_utlb_id = uvm_hal_volta_mmu_client_id_to_utlb_id,
},
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.u.arch_ops = {
.init_properties = uvm_hal_turing_arch_init_properties,
.mmu_mode_hal = uvm_hal_mmu_mode_turing,
.enable_prefetch_faults = uvm_hal_turing_mmu_enable_prefetch_faults,
.disable_prefetch_faults = uvm_hal_turing_mmu_disable_prefetch_faults,
.mmu_client_id_to_utlb_id = uvm_hal_turing_mmu_client_id_to_utlb_id,
},
},
{
@@ -450,68 +319,23 @@ static uvm_hal_class_ops_t chip_table[] =
static uvm_hal_class_ops_t fault_buffer_table[] =
{
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.fault_buffer_ops = {
.enable_replayable_faults = uvm_hal_maxwell_enable_replayable_faults_unsupported,
.disable_replayable_faults = uvm_hal_maxwell_disable_replayable_faults_unsupported,
.clear_replayable_faults = uvm_hal_maxwell_clear_replayable_faults_unsupported,
.read_put = uvm_hal_maxwell_fault_buffer_read_put_unsupported,
.read_get = uvm_hal_maxwell_fault_buffer_read_get_unsupported,
.write_get = uvm_hal_maxwell_fault_buffer_write_get_unsupported,
.get_ve_id = uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported,
.get_mmu_engine_type = uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported,
.parse_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported,
.entry_is_valid = uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported,
.entry_clear_valid = uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported,
.entry_size = uvm_hal_maxwell_fault_buffer_entry_size_unsupported,
.parse_non_replayable_entry = uvm_hal_maxwell_fault_buffer_parse_non_replayable_entry_unsupported,
.get_fault_type = uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.fault_buffer_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.u.fault_buffer_ops = {
.enable_replayable_faults = uvm_hal_pascal_enable_replayable_faults,
.disable_replayable_faults = uvm_hal_pascal_disable_replayable_faults,
.clear_replayable_faults = uvm_hal_pascal_clear_replayable_faults,
.read_put = uvm_hal_pascal_fault_buffer_read_put,
.read_get = uvm_hal_pascal_fault_buffer_read_get,
.write_get = uvm_hal_pascal_fault_buffer_write_get,
.parse_replayable_entry = uvm_hal_pascal_fault_buffer_parse_replayable_entry,
.entry_is_valid = uvm_hal_pascal_fault_buffer_entry_is_valid,
.entry_clear_valid = uvm_hal_pascal_fault_buffer_entry_clear_valid,
.entry_size = uvm_hal_pascal_fault_buffer_entry_size,
.get_fault_type = uvm_hal_pascal_fault_buffer_get_fault_type,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.u.fault_buffer_ops = {
.read_put = uvm_hal_volta_fault_buffer_read_put,
.read_get = uvm_hal_volta_fault_buffer_read_get,
.write_get = uvm_hal_volta_fault_buffer_write_get,
.get_ve_id = uvm_hal_volta_fault_buffer_get_ve_id,
.get_mmu_engine_type = uvm_hal_volta_fault_buffer_get_mmu_engine_type,
.parse_replayable_entry = uvm_hal_volta_fault_buffer_parse_replayable_entry,
.parse_non_replayable_entry = uvm_hal_volta_fault_buffer_parse_non_replayable_entry,
.get_fault_type = uvm_hal_volta_fault_buffer_get_fault_type,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.u.fault_buffer_ops = {
.enable_replayable_faults = uvm_hal_turing_enable_replayable_faults,
.disable_replayable_faults = uvm_hal_turing_disable_replayable_faults,
.clear_replayable_faults = uvm_hal_turing_clear_replayable_faults,
.read_put = uvm_hal_turing_fault_buffer_read_put,
.read_get = uvm_hal_turing_fault_buffer_read_get,
.write_get = uvm_hal_turing_fault_buffer_write_get,
.get_ve_id = uvm_hal_turing_fault_buffer_get_ve_id,
.get_mmu_engine_type = uvm_hal_turing_fault_buffer_get_mmu_engine_type,
.parse_replayable_entry = uvm_hal_turing_fault_buffer_parse_replayable_entry,
.entry_is_valid = uvm_hal_turing_fault_buffer_entry_is_valid,
.entry_clear_valid = uvm_hal_turing_fault_buffer_entry_clear_valid,
.entry_size = uvm_hal_turing_fault_buffer_entry_size,
.parse_non_replayable_entry = uvm_hal_turing_fault_buffer_parse_non_replayable_entry,
.get_fault_type = uvm_hal_turing_fault_buffer_get_fault_type,
}
},
{
@@ -551,36 +375,8 @@ static uvm_hal_class_ops_t fault_buffer_table[] =
static uvm_hal_class_ops_t access_counter_buffer_table[] =
{
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.access_counter_buffer_ops = {
.enable_access_counter_notifications = uvm_hal_maxwell_enable_access_counter_notifications_unsupported,
.disable_access_counter_notifications = uvm_hal_maxwell_disable_access_counter_notifications_unsupported,
.clear_access_counter_notifications = uvm_hal_maxwell_clear_access_counter_notifications_unsupported,
.parse_entry = uvm_hal_maxwell_access_counter_buffer_parse_entry_unsupported,
.entry_is_valid = uvm_hal_maxwell_access_counter_buffer_entry_is_valid_unsupported,
.entry_clear_valid = uvm_hal_maxwell_access_counter_buffer_entry_clear_valid_unsupported,
.entry_size = uvm_hal_maxwell_access_counter_buffer_entry_size_unsupported,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.access_counter_buffer_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.u.access_counter_buffer_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.u.access_counter_buffer_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.u.access_counter_buffer_ops = {
.enable_access_counter_notifications = uvm_hal_turing_enable_access_counter_notifications,
.disable_access_counter_notifications = uvm_hal_turing_disable_access_counter_notifications,
@@ -620,35 +416,15 @@ static uvm_hal_class_ops_t access_counter_buffer_table[] =
static uvm_hal_class_ops_t sec2_table[] =
{
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.sec2_ops = {
.init = uvm_hal_maxwell_sec2_init_noop,
.decrypt = uvm_hal_maxwell_sec2_decrypt_unsupported,
.semaphore_release = uvm_hal_maxwell_sec2_semaphore_release_unsupported,
.semaphore_timestamp = uvm_hal_maxwell_sec2_semaphore_timestamp_unsupported,
.semaphore_target_is_valid = uvm_hal_maxwell_semaphore_target_is_valid,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM000,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GM200,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100,
.u.sec2_ops = {}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_TU100,
.parent_id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GV100,
.u.sec2_ops = {}
.u.sec2_ops = {
.init = uvm_hal_turing_sec2_init_noop,
.decrypt = uvm_hal_turing_sec2_decrypt_unsupported,
.semaphore_release = uvm_hal_turing_sec2_semaphore_release_unsupported,
.semaphore_timestamp = uvm_hal_turing_sec2_semaphore_timestamp_unsupported,
.semaphore_target_is_valid = uvm_hal_turing_semaphore_target_is_valid,
}
},
{
.id = NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GA100,

323
kernel-open/nvidia-uvm/uvm_hal.h
View File

@@ -37,10 +37,9 @@
bool uvm_hal_method_is_valid_stub(uvm_push_t *push, NvU32 method_address, NvU32 method_data);
typedef void (*uvm_hal_init_t)(uvm_push_t *push);
void uvm_hal_maxwell_ce_init(uvm_push_t *push);
void uvm_hal_maxwell_host_init_noop(uvm_push_t *push);
void uvm_hal_pascal_host_init(uvm_push_t *push);
void uvm_hal_maxwell_sec2_init_noop(uvm_push_t *push);
void uvm_hal_turing_ce_init(uvm_push_t *push);
void uvm_hal_turing_host_init(uvm_push_t *push);
void uvm_hal_turing_sec2_init_noop(uvm_push_t *push);
void uvm_hal_hopper_sec2_init(uvm_push_t *push);
// Host method validation
@@ -53,21 +52,20 @@ bool uvm_hal_ampere_host_sw_method_is_valid(uvm_push_t *push, NvU32 method_addre
// Wait for idle
typedef void (*uvm_hal_wait_for_idle_t)(uvm_push_t *push);
void uvm_hal_maxwell_host_wait_for_idle(uvm_push_t *push);
void uvm_hal_turing_host_wait_for_idle(uvm_push_t *push);
// Membar SYS
typedef void (*uvm_hal_membar_sys_t)(uvm_push_t *push);
void uvm_hal_maxwell_host_membar_sys(uvm_push_t *push);
void uvm_hal_pascal_host_membar_sys(uvm_push_t *push);
void uvm_hal_turing_host_membar_sys(uvm_push_t *push);
// Membar GPU
typedef void (*uvm_hal_membar_gpu_t)(uvm_push_t *push);
void uvm_hal_pascal_host_membar_gpu(uvm_push_t *push);
void uvm_hal_turing_host_membar_gpu(uvm_push_t *push);
// Put a noop in the pushbuffer of the given size in bytes.
// The size needs to be a multiple of 4.
typedef void (*uvm_hal_noop_t)(uvm_push_t *push, NvU32 size);
void uvm_hal_maxwell_host_noop(uvm_push_t *push, NvU32 size);
void uvm_hal_turing_host_noop(uvm_push_t *push, NvU32 size);
// Host-generated interrupt method. This will generate a call to
// uvm_isr_top_half_entry.
@@ -80,7 +78,7 @@ void uvm_hal_maxwell_host_noop(uvm_push_t *push, NvU32 size);
// We must set up software state before pushing the interrupt, and check any
// possible interrupt condition on receiving an interrupt callback.
typedef void (*uvm_hal_interrupt_t)(uvm_push_t *push);
void uvm_hal_maxwell_host_interrupt(uvm_push_t *push);
void uvm_hal_turing_host_interrupt(uvm_push_t *push);
// Issue a TLB invalidate applying to all VAs in a PDB.
//
@@ -88,9 +86,9 @@ void uvm_hal_maxwell_host_interrupt(uvm_push_t *push);
// the PDB are invalidated from the specified depth down to the PTEs. This
// allows for optimizations if the caller isn't writing all levels of the PDEs.
// Depth follows the MMU code convention where depth 0 is the top level and here
// means to invalidate everything. See uvm_pascal_mmu.c for an example of depth
// means to invalidate everything. See uvm_turing_mmu.c for an example of depth
// mapping to HW PDE levels. Notably 2M PTEs are considered PDEs as far as the
// TLBs are concerned and hence on Pascal the depth needs to be at most 3 for
// TLBs are concerned and hence on Turing the depth needs to be at most 3 for
// them to be included in the invalidation.
//
// If the membar parameter is not UVM_MEMBAR_NONE, the specified membar is
@@ -100,18 +98,6 @@ typedef void (*uvm_hal_host_tlb_invalidate_all_t)(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar);
void uvm_hal_maxwell_host_tlb_invalidate_all_a16f(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar);
void uvm_hal_maxwell_host_tlb_invalidate_all_b06f(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar);
void uvm_hal_pascal_host_tlb_invalidate_all(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar);
void uvm_hal_turing_host_tlb_invalidate_all(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
@@ -140,9 +126,9 @@ void uvm_hal_blackwell_host_tlb_invalidate_all(uvm_push_t *push,
// invalidated.
//
// Depth follows the MMU code convention where depth 0 is the top level and here
// means to invalidate all levels. See uvm_pascal_mmu.c for an example of depth
// means to invalidate all levels. See uvm_turing_mmu.c for an example of depth
// mapping to HW PDE levels. Notably 2M PTEs are considered PDEs as far as the
// TLBs are concerned and hence on Pascal the depth needs to be at most 3 for
// TLBs are concerned and hence on Turing the depth needs to be at most 3 for
// them to be included in the invalidation.
//
// If the membar parameter is not UVM_MEMBAR_NONE, the specified membar is
@@ -159,27 +145,6 @@ typedef void (*uvm_hal_host_tlb_invalidate_va_t)(uvm_push_t *push,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar);
void uvm_hal_maxwell_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
NvU64 base,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar);
void uvm_hal_pascal_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
NvU64 base,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar);
void uvm_hal_volta_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
NvU64 base,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar);
void uvm_hal_turing_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
@@ -215,18 +180,12 @@ typedef void (*uvm_hal_host_tlb_invalidate_phys_t)(uvm_push_t *push);
// Volta was the first GPU which enabled caching of physical translations, but
// we have no need to invalidate manually until Blackwell.
void uvm_hal_maxwell_host_tlb_invalidate_phys_unsupported(uvm_push_t *push);
void uvm_hal_turing_host_tlb_invalidate_phys_unsupported(uvm_push_t *push);
void uvm_hal_blackwell_host_tlb_invalidate_phys(uvm_push_t *push);
typedef void (*uvm_hal_host_tlb_invalidate_test_t)(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params);
void uvm_hal_maxwell_host_tlb_invalidate_test(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params);
void uvm_hal_pascal_host_tlb_invalidate_test(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params);
void uvm_hal_turing_host_tlb_invalidate_test(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params);
@@ -245,7 +204,7 @@ void uvm_hal_blackwell_host_tlb_invalidate_test(uvm_push_t *push,
typedef void (*uvm_hal_host_tlb_flush_prefetch_t)(uvm_push_t *push);
// Blackwell is the first GPU which needs prefetch flushing
void uvm_hal_maxwell_host_tlb_flush_prefetch_unsupported(uvm_push_t *push);
void uvm_hal_turing_host_tlb_flush_prefetch_unsupported(uvm_push_t *push);
void uvm_hal_blackwell_host_tlb_flush_prefetch(uvm_push_t *push);
// Performs L2 cache invalidation for peer or system memory.
@@ -259,12 +218,9 @@ void uvm_hal_host_l2_invalidate_unsupported(uvm_push_t *push, uvm_aperture_t ape
// operation. This can be affected by using UVM_PUSH_FLAG_NEXT_* flags with
// uvm_push_set_flag().
typedef void (*uvm_hal_semaphore_release_t)(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_maxwell_host_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_maxwell_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_maxwell_sec2_semaphore_release_unsupported(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_pascal_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_volta_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_turing_host_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_turing_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_turing_sec2_semaphore_release_unsupported(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_hopper_sec2_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_hopper_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_hopper_host_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
@@ -275,20 +231,16 @@ void uvm_hal_hopper_host_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32
// aligned. The value of the released payload is unspecified and shouldn't be
// relied on, only the timestamp should be of interest.
typedef void (*uvm_hal_semaphore_timestamp_t)(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_maxwell_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_pascal_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_volta_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_turing_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_hopper_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_maxwell_host_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_volta_host_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_turing_host_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_hopper_host_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_maxwell_sec2_semaphore_timestamp_unsupported(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_turing_sec2_semaphore_timestamp_unsupported(uvm_push_t *push, NvU64 gpu_va);
void uvm_hal_hopper_sec2_semaphore_timestamp_unsupported(uvm_push_t *push, NvU64 gpu_va);
typedef void (*uvm_hal_semaphore_acquire_t)(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_maxwell_host_semaphore_acquire(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_turing_host_semaphore_acquire(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_hopper_host_semaphore_acquire(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
@@ -299,16 +251,12 @@ void uvm_hal_hopper_host_semaphore_acquire(uvm_push_t *push, NvU64 gpu_va, NvU32
// reduction operations, semaphore acquire, and semaphore release
// operations with timestamp.
typedef bool (*uvm_hal_semaphore_target_is_valid_t)(uvm_push_t *push, NvU64 gpu_va);
bool uvm_hal_maxwell_semaphore_target_is_valid(uvm_push_t *push, NvU64 gpu_va);
bool uvm_hal_turing_semaphore_target_is_valid(uvm_push_t *push, NvU64 gpu_va);
typedef void (*uvm_hal_host_set_gpfifo_entry_t)(NvU64 *fifo_entry,
NvU64 pushbuffer_va,
NvU32 pushbuffer_length,
uvm_gpfifo_sync_t sync_flag);
void uvm_hal_maxwell_host_set_gpfifo_entry(NvU64 *fifo_entry,
NvU64 pushbuffer_va,
NvU32 pushbuffer_length,
uvm_gpfifo_sync_t sync_flag);
void uvm_hal_turing_host_set_gpfifo_entry(NvU64 *fifo_entry,
NvU64 pushbuffer_va,
NvU32 pushbuffer_length,
@@ -319,43 +267,40 @@ void uvm_hal_hopper_host_set_gpfifo_entry(NvU64 *fifo_entry,
uvm_gpfifo_sync_t sync_flag);
typedef void (*uvm_hal_host_set_gpfifo_noop_t)(NvU64 *fifo_entry);
void uvm_hal_maxwell_host_set_gpfifo_noop(NvU64 *fifo_entry);
void uvm_hal_turing_host_set_gpfifo_noop(NvU64 *fifo_entry);
typedef void (*uvm_hal_host_set_gpfifo_pushbuffer_segment_base_t)(NvU64 *fifo_entry, NvU64 pushbuffer_va);
void uvm_hal_maxwell_host_set_gpfifo_pushbuffer_segment_base_unsupported(NvU64 *fifo_entry, NvU64 pushbuffer_va);
void uvm_hal_turing_host_set_gpfifo_pushbuffer_segment_base_unsupported(NvU64 *fifo_entry, NvU64 pushbuffer_va);
void uvm_hal_hopper_host_set_gpfifo_pushbuffer_segment_base(NvU64 *fifo_entry, NvU64 pushbuffer_va);
typedef void (*uvm_hal_host_write_gpu_put_t)(uvm_channel_t *channel, NvU32 gpu_put);
void uvm_hal_maxwell_host_write_gpu_put(uvm_channel_t *channel, NvU32 gpu_put);
void uvm_hal_volta_host_write_gpu_put(uvm_channel_t *channel, NvU32 gpu_put);
void uvm_hal_turing_host_write_gpu_put(uvm_channel_t *channel, NvU32 gpu_put);
// Return the current GPU time in nanoseconds
typedef NvU64 (*uvm_hal_get_time_t)(uvm_gpu_t *gpu);
NvU64 uvm_hal_maxwell_get_time(uvm_gpu_t *gpu);
NvU64 uvm_hal_turing_get_time(uvm_gpu_t *gpu);
// Internal helpers used by the CE hal
// Used to handle the offset encoding differences between architectures
typedef void (*uvm_hal_ce_offset_out_t)(uvm_push_t *push, NvU64 offset);
void uvm_hal_maxwell_ce_offset_out(uvm_push_t *push, NvU64 offset);
void uvm_hal_pascal_ce_offset_out(uvm_push_t *push, NvU64 offset);
void uvm_hal_turing_ce_offset_out(uvm_push_t *push, NvU64 offset);
void uvm_hal_hopper_ce_offset_out(uvm_push_t *push, NvU64 offset);
typedef void (*uvm_hal_ce_offset_in_out_t)(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out);
void uvm_hal_maxwell_ce_offset_in_out(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out);
void uvm_hal_pascal_ce_offset_in_out(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out);
void uvm_hal_turing_ce_offset_in_out(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out);
void uvm_hal_hopper_ce_offset_in_out(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out);
typedef NvU32 (*uvm_hal_ce_phys_mode_t)(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_maxwell_ce_phys_mode(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_turing_ce_phys_mode(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_ampere_ce_phys_mode(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_hopper_ce_phys_mode(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
typedef NvU32 (*uvm_hal_ce_plc_mode_t)(void);
NvU32 uvm_hal_maxwell_ce_plc_mode(void);
NvU32 uvm_hal_turing_ce_plc_mode(void);
NvU32 uvm_hal_ampere_ce_plc_mode_c7b5(void);
typedef NvU32 (*uvm_hal_ce_memcopy_type_t)(uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_maxwell_ce_memcopy_copy_type(uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_turing_ce_memcopy_copy_type(uvm_gpu_address_t dst, uvm_gpu_address_t src);
NvU32 uvm_hal_hopper_ce_memcopy_copy_type(uvm_gpu_address_t dst, uvm_gpu_address_t src);
// CE method validation
@@ -366,7 +311,7 @@ bool uvm_hal_ampere_ce_method_is_valid_c6b5(uvm_push_t *push, NvU32 method_addre
// The validation happens at the start of the memcopy (uvm_hal_memcopy_t)
// execution.
typedef bool (*uvm_hal_ce_memcopy_is_valid)(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
bool uvm_hal_maxwell_ce_memcopy_is_valid(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
bool uvm_hal_turing_ce_memcopy_is_valid(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
bool uvm_hal_ampere_ce_memcopy_is_valid_c6b5(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
bool uvm_hal_hopper_ce_memcopy_is_valid(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
bool uvm_hal_blackwell_ce_memcopy_is_valid(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src);
@@ -383,12 +328,11 @@ void uvm_hal_ampere_ce_memcopy_patch_src_c6b5(uvm_push_t *push, uvm_gpu_address_
// operation and are not pipelined. This can be affected by using
// UVM_PUSH_FLAG_NEXT_CE_* flags with uvm_push_set_flag().
typedef void (*uvm_hal_memcopy_t)(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src, size_t size);
void uvm_hal_maxwell_ce_memcopy(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src, size_t size);
void uvm_hal_volta_ce_memcopy(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src, size_t size);
void uvm_hal_turing_ce_memcopy(uvm_push_t *push, uvm_gpu_address_t dst, uvm_gpu_address_t src, size_t size);
// Simple wrapper for uvm_hal_memcopy_t with both addresses being virtual
typedef void (*uvm_hal_memcopy_v_to_v_t)(uvm_push_t *push, NvU64 dst, NvU64 src, size_t size);
void uvm_hal_maxwell_ce_memcopy_v_to_v(uvm_push_t *push, NvU64 dst, NvU64 src, size_t size);
void uvm_hal_turing_ce_memcopy_v_to_v(uvm_push_t *push, NvU64 dst, NvU64 src, size_t size);
// Memset validation.
// The validation happens at the start of the memset (uvm_hal_memset_*_t)
@@ -397,10 +341,10 @@ typedef bool (*uvm_hal_ce_memset_is_valid)(uvm_push_t *push,
uvm_gpu_address_t dst,
size_t num_elements,
size_t element_size);
bool uvm_hal_maxwell_ce_memset_is_valid(uvm_push_t *push,
uvm_gpu_address_t dst,
size_t num_elements,
size_t element_size);
bool uvm_hal_turing_ce_memset_is_valid(uvm_push_t *push,
uvm_gpu_address_t dst,
size_t num_elements,
size_t element_size);
bool uvm_hal_ampere_ce_memset_is_valid_c6b5(uvm_push_t *push,
uvm_gpu_address_t dst,
size_t num_elements,
@@ -425,14 +369,10 @@ typedef void (*uvm_hal_memset_8_t)(uvm_push_t *push, uvm_gpu_address_t dst, NvU6
// Simple wrapper for uvm_hal_memset_4_t with the address being virtual.
typedef void (*uvm_hal_memset_v_4_t)(uvm_push_t *push, NvU64 dst_va, NvU32 value, size_t size);
void uvm_hal_maxwell_ce_memset_1(uvm_push_t *push, uvm_gpu_address_t dst, NvU8 value, size_t size);
void uvm_hal_maxwell_ce_memset_4(uvm_push_t *push, uvm_gpu_address_t dst, NvU32 value, size_t size);
void uvm_hal_maxwell_ce_memset_8(uvm_push_t *push, uvm_gpu_address_t dst, NvU64 value, size_t size);
void uvm_hal_maxwell_ce_memset_v_4(uvm_push_t *push, NvU64 dst_va, NvU32 value, size_t size);
void uvm_hal_volta_ce_memset_1(uvm_push_t *push, uvm_gpu_address_t dst, NvU8 value, size_t size);
void uvm_hal_volta_ce_memset_4(uvm_push_t *push, uvm_gpu_address_t dst, NvU32 value, size_t size);
void uvm_hal_volta_ce_memset_8(uvm_push_t *push, uvm_gpu_address_t dst, NvU64 value, size_t size);
void uvm_hal_turing_ce_memset_1(uvm_push_t *push, uvm_gpu_address_t dst, NvU8 value, size_t size);
void uvm_hal_turing_ce_memset_4(uvm_push_t *push, uvm_gpu_address_t dst, NvU32 value, size_t size);
void uvm_hal_turing_ce_memset_8(uvm_push_t *push, uvm_gpu_address_t dst, NvU64 value, size_t size);
void uvm_hal_turing_ce_memset_v_4(uvm_push_t *push, NvU64 dst_va, NvU32 value, size_t size);
void uvm_hal_hopper_ce_memset_1(uvm_push_t *push, uvm_gpu_address_t dst, NvU8 value, size_t size);
void uvm_hal_hopper_ce_memset_4(uvm_push_t *push, uvm_gpu_address_t dst, NvU32 value, size_t size);
@@ -465,16 +405,16 @@ typedef void (*uvm_hal_ce_decrypt_t)(uvm_push_t *push,
NvU32 size,
uvm_gpu_address_t auth_tag);
void uvm_hal_maxwell_ce_encrypt_unsupported(uvm_push_t *push,
uvm_gpu_address_t dst,
uvm_gpu_address_t src,
NvU32 size,
uvm_gpu_address_t auth_tag);
void uvm_hal_maxwell_ce_decrypt_unsupported(uvm_push_t *push,
uvm_gpu_address_t dst,
uvm_gpu_address_t src,
NvU32 size,
uvm_gpu_address_t auth_tag);
void uvm_hal_turing_ce_encrypt_unsupported(uvm_push_t *push,
uvm_gpu_address_t dst,
uvm_gpu_address_t src,
NvU32 size,
uvm_gpu_address_t auth_tag);
void uvm_hal_turing_ce_decrypt_unsupported(uvm_push_t *push,
uvm_gpu_address_t dst,
uvm_gpu_address_t src,
NvU32 size,
uvm_gpu_address_t auth_tag);
void uvm_hal_hopper_ce_encrypt(uvm_push_t *push,
uvm_gpu_address_t dst,
uvm_gpu_address_t src,
@@ -493,16 +433,11 @@ void uvm_hal_hopper_ce_decrypt(uvm_push_t *push,
// semaphore operation. This can be affected by using UVM_PUSH_FLAG_NEXT_CE_*
// flags with uvm_push_set_flag().
typedef void (*uvm_hal_semaphore_reduction_inc_t)(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_maxwell_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_pascal_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_volta_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_turing_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
void uvm_hal_hopper_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload);
// Initialize GPU architecture dependent properties
typedef void (*uvm_hal_arch_init_properties_t)(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_volta_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
@@ -510,27 +445,20 @@ void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu);
// Retrieve the page-tree HAL for a given big page size
typedef uvm_mmu_mode_hal_t *(*uvm_hal_lookup_mode_hal_t)(NvU64 big_page_size);
typedef uvm_mmu_mode_hal_t *(*uvm_hal_lookup_mode_hal_t)(void);
typedef void (*uvm_hal_mmu_enable_prefetch_faults_t)(uvm_parent_gpu_t *parent_gpu);
typedef void (*uvm_hal_mmu_disable_prefetch_faults_t)(uvm_parent_gpu_t *parent_gpu);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_pascal(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_volta(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_turing(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell_integrated(NvU64 big_page_size);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_turing(void);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_ampere(void);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(void);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell(void);
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_blackwell_integrated(void);
void uvm_hal_maxwell_mmu_enable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_mmu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_mmu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_mmu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_mmu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu);
typedef NvU16 (*uvm_hal_mmu_client_id_to_utlb_id_t)(NvU16 client_id);
NvU16 uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported(NvU16 client_id);
NvU16 uvm_hal_pascal_mmu_client_id_to_utlb_id(NvU16 client_id);
NvU16 uvm_hal_volta_mmu_client_id_to_utlb_id(NvU16 client_id);
NvU16 uvm_hal_turing_mmu_client_id_to_utlb_id(NvU16 client_id);
NvU16 uvm_hal_ampere_mmu_client_id_to_utlb_id(NvU16 client_id);
NvU16 uvm_hal_hopper_mmu_client_id_to_utlb_id(NvU16 client_id);
NvU16 uvm_hal_blackwell_mmu_client_id_to_utlb_id(NvU16 client_id);
@@ -553,18 +481,10 @@ typedef NV_STATUS (*uvm_hal_fault_buffer_parse_replayable_entry_t)(uvm_parent_gp
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NV_STATUS uvm_hal_turing_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
NV_STATUS uvm_hal_volta_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry);
typedef bool (*uvm_hal_fault_buffer_entry_is_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
typedef void (*uvm_hal_fault_buffer_entry_clear_valid_t)(uvm_parent_gpu_t *parent_gpu, NvU32 index);
typedef NvU32 (*uvm_hal_fault_buffer_entry_size_t)(uvm_parent_gpu_t *parent_gpu);
@@ -576,41 +496,20 @@ typedef void (*uvm_hal_fault_cancel_targeted_t)(uvm_push_t *push,
NvU32 gpc_id,
NvU32 client_id);
void uvm_hal_maxwell_enable_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_disable_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_clear_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 get);
NvU32 uvm_hal_maxwell_fault_buffer_read_put_unsupported(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_maxwell_fault_buffer_read_get_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_fault_buffer_write_get_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
uvm_mmu_engine_type_t uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported(NvU16 mmu_engine_id,
uvm_fault_client_type_t client_type,
NvU16 client_id);
uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry);
void uvm_hal_pascal_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32 get);
NvU32 uvm_hal_pascal_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_pascal_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_pascal_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
uvm_fault_type_t uvm_hal_pascal_fault_buffer_get_fault_type(const NvU32 *fault_entry);
NvU32 uvm_hal_volta_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_volta_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_volta_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU8 uvm_hal_volta_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
uvm_mmu_engine_type_t uvm_hal_volta_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
uvm_fault_client_type_t client_type,
NvU16 client_id);
uvm_fault_type_t uvm_hal_volta_fault_buffer_get_fault_type(const NvU32 *fault_entry);
void uvm_hal_turing_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32 get);
NvU32 uvm_hal_turing_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu);
NvU32 uvm_hal_turing_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_turing_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU8 uvm_hal_turing_fault_buffer_get_ve_id(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type);
uvm_mmu_engine_type_t uvm_hal_turing_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
uvm_fault_client_type_t client_type,
NvU16 client_id);
uvm_fault_client_type_t client_type,
NvU16 client_id);
bool uvm_hal_turing_fault_buffer_entry_is_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index);
void uvm_hal_turing_fault_buffer_entry_clear_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU32 uvm_hal_turing_fault_buffer_entry_size(uvm_parent_gpu_t *parent_gpu);
uvm_fault_type_t uvm_hal_turing_fault_buffer_get_fault_type(const NvU32 *fault_entry);
uvm_mmu_engine_type_t uvm_hal_ampere_fault_buffer_get_mmu_engine_type(NvU16 mmu_engine_id,
uvm_fault_client_type_t client_type,
@@ -626,34 +525,17 @@ uvm_mmu_engine_type_t uvm_hal_blackwell_fault_buffer_get_mmu_engine_type(NvU16 m
NvU16 client_id);
uvm_fault_type_t uvm_hal_blackwell_fault_buffer_get_fault_type(const NvU32 *fault_entry);
bool uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
void uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU32 uvm_hal_maxwell_fault_buffer_entry_size_unsupported(uvm_parent_gpu_t *parent_gpu);
bool uvm_hal_pascal_fault_buffer_entry_is_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index);
void uvm_hal_pascal_fault_buffer_entry_clear_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index);
NvU32 uvm_hal_pascal_fault_buffer_entry_size(uvm_parent_gpu_t *parent_gpu);
typedef void (*uvm_hal_fault_buffer_parse_non_replayable_entry_t)(uvm_parent_gpu_t *parent_gpu,
void *fault_packet,
uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_maxwell_fault_buffer_parse_non_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
void *fault_packet,
uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_volta_fault_buffer_parse_non_replayable_entry(uvm_parent_gpu_t *parent_gpu,
void *fault_packet,
uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_turing_fault_buffer_parse_non_replayable_entry(uvm_parent_gpu_t *parent_gpu,
void *fault_packet,
uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_maxwell_cancel_faults_global_unsupported(uvm_push_t *push, uvm_gpu_phys_address_t instance_ptr);
void uvm_hal_pascal_cancel_faults_global(uvm_push_t *push, uvm_gpu_phys_address_t instance_ptr);
void uvm_hal_turing_cancel_faults_global(uvm_push_t *push, uvm_gpu_phys_address_t instance_ptr);
// Trigger fault replay on the GPU where the given pushbuffer is located.
void uvm_hal_maxwell_replay_faults_unsupported(uvm_push_t *push, uvm_fault_replay_type_t type);
void uvm_hal_maxwell_cancel_faults_targeted_unsupported(uvm_push_t *push,
uvm_gpu_phys_address_t instance_ptr,
NvU32 gpc_id,
NvU32 client_id);
void uvm_hal_pascal_replay_faults(uvm_push_t *push, uvm_fault_replay_type_t type);
void uvm_hal_pascal_cancel_faults_targeted(uvm_push_t *push,
void uvm_hal_turing_cancel_faults_targeted(uvm_push_t *push,
uvm_gpu_phys_address_t instance_ptr,
NvU32 gpc_id,
NvU32 client_id);
@@ -663,13 +545,8 @@ typedef void (*uvm_hal_fault_cancel_va_t)(uvm_push_t *push,
const uvm_fault_buffer_entry_t *fault_entry,
uvm_fault_cancel_va_mode_t cancel_va_mode);
void uvm_hal_maxwell_cancel_faults_va_unsupported(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
const uvm_fault_buffer_entry_t *fault_entry,
uvm_fault_cancel_va_mode_t cancel_va_mode);
void uvm_hal_volta_replay_faults(uvm_push_t *push, uvm_fault_replay_type_t type);
void uvm_hal_volta_cancel_faults_va(uvm_push_t *push,
void uvm_hal_turing_replay_faults(uvm_push_t *push, uvm_fault_replay_type_t type);
void uvm_hal_turing_cancel_faults_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
const uvm_fault_buffer_entry_t *fault_entry,
uvm_fault_cancel_va_mode_t cancel_va_mode);
@@ -683,26 +560,20 @@ typedef void (*uvm_hal_host_clear_faulted_channel_method_t)(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_maxwell_host_clear_faulted_channel_method_unsupported(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_volta_host_clear_faulted_channel_method(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_turing_host_clear_faulted_channel_method(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
typedef void (*uvm_hal_host_clear_faulted_channel_register_t)(uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_maxwell_host_clear_faulted_channel_register_unsupported(uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_turing_host_clear_faulted_channel_register_unsupported(uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_ampere_host_clear_faulted_channel_register(uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
typedef void (*uvm_hal_host_clear_faulted_channel_sw_method_t)(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_maxwell_host_clear_faulted_channel_sw_method_unsupported(uvm_push_t *push,
void uvm_hal_turing_host_clear_faulted_channel_sw_method_unsupported(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry);
void uvm_hal_ampere_host_clear_faulted_channel_sw_method(uvm_push_t *push,
@@ -735,26 +606,6 @@ typedef uvm_access_counter_clear_op_t
uvm_access_counter_buffer_entry_t **buffer_entries,
NvU32 num_entries);
void uvm_hal_maxwell_enable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters);
void uvm_hal_maxwell_disable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters);
void uvm_hal_maxwell_clear_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 get);
void uvm_hal_maxwell_access_counter_buffer_parse_entry_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index,
uvm_access_counter_buffer_entry_t *buffer_entry);
bool uvm_hal_maxwell_access_counter_buffer_entry_is_valid_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index);
void uvm_hal_maxwell_access_counter_buffer_entry_clear_valid_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index);
NvU32 uvm_hal_maxwell_access_counter_buffer_entry_size_unsupported(uvm_parent_gpu_t *parent_gpu);
void uvm_hal_maxwell_access_counter_clear_all_unsupported(uvm_push_t *push);
void uvm_hal_maxwell_access_counter_clear_targeted_unsupported(uvm_push_t *push,
const uvm_access_counter_buffer_entry_t *buffer_entry);
uvm_access_counter_clear_op_t
uvm_hal_maxwell_access_counter_query_clear_op_unsupported(uvm_parent_gpu_t *parent_gpu,
uvm_access_counter_buffer_entry_t **buffer_entries,
NvU32 num_entries);
void uvm_hal_turing_enable_access_counter_notifications(uvm_access_counter_buffer_t *access_counters);
void uvm_hal_turing_disable_access_counter_notifications(uvm_access_counter_buffer_t *access_counters);
void uvm_hal_turing_clear_access_counter_notifications(uvm_access_counter_buffer_t *access_counters, NvU32 get);
@@ -795,11 +646,11 @@ uvm_hal_blackwell_access_counter_query_clear_op_gb20x(uvm_parent_gpu_t *parent_g
// Note: SEC2 does not support encryption.
typedef void (*uvm_hal_sec2_decrypt_t)(uvm_push_t *push, NvU64 dst_va, NvU64 src_va, NvU32 size, NvU64 auth_tag_va);
void uvm_hal_maxwell_sec2_decrypt_unsupported(uvm_push_t *push,
NvU64 dst_va,
NvU64 src_va,
NvU32 size,
NvU64 auth_tag_va);
void uvm_hal_turing_sec2_decrypt_unsupported(uvm_push_t *push,
NvU64 dst_va,
NvU64 src_va,
NvU32 size,
NvU64 auth_tag_va);
void uvm_hal_hopper_sec2_decrypt(uvm_push_t *push, NvU64 dst_va, NvU64 src_va, NvU32 size, NvU64 auth_tag_va);
struct uvm_host_hal_struct

5
kernel-open/nvidia-uvm/uvm_hal_types.h
View File

@@ -225,9 +225,8 @@ static int uvm_gpu_addr_cmp(uvm_gpu_address_t a, uvm_gpu_address_t b)
}
}
// For processors with no concept of an atomic fault (the CPU and pre-Pascal
// GPUs), UVM_PROT_READ_WRITE and UVM_PROT_READ_WRITE_ATOMIC are
// interchangeable.
// For processors with no concept of an atomic fault (the CPU),
// UVM_PROT_READ_WRITE and UVM_PROT_READ_WRITE_ATOMIC are interchangeable.
typedef enum
{
UVM_PROT_NONE,

18
kernel-open/nvidia-uvm/uvm_hmm.c
View File

@@ -74,6 +74,19 @@ module_param(uvm_disable_hmm, bool, 0444);
#include "uvm_va_policy.h"
#include "uvm_tools.h"
//
// Pass 0 as the order, when actual large order support is added this
// function will need to be revisited
//
static __always_inline void nv_zone_device_page_init(struct page *page)
{
#if defined(NV_ZONE_DEVICE_PAGE_INIT_HAS_PGMAP_AND_ORDER_ARGS)
zone_device_page_init(page, page_pgmap(page), 0);
#else
zone_device_page_init(page);
#endif
}
// The function nv_PageSwapCache() wraps the check for page swap cache flag in
// order to support a wide variety of kernel versions.
// The function PageSwapCache() is removed after 32f51ead3d77 ("mm: remove
@@ -1253,7 +1266,6 @@ static NV_STATUS hmm_set_accessed_by_start_end_locked(uvm_va_block_t *va_block,
NvU64 end,
uvm_tracker_t *out_tracker)
{
uvm_va_space_t *va_space = va_block->hmm.va_space;
uvm_va_policy_node_t *node;
uvm_va_block_region_t region;
NV_STATUS status = NV_OK;
@@ -1261,7 +1273,7 @@ static NV_STATUS hmm_set_accessed_by_start_end_locked(uvm_va_block_t *va_block,
uvm_for_each_va_policy_node_in(node, va_block, start, end) {
// Read duplication takes precedence over SetAccessedBy.
// Do not add mappings if read duplication is enabled.
if (uvm_va_policy_is_read_duplicate(&node->policy, va_space))
if (uvm_va_policy_is_read_duplicate(&node->policy))
continue;
region = uvm_va_block_region_from_start_end(va_block,
@@ -2140,7 +2152,7 @@ static void fill_dst_pfn(uvm_va_block_t *va_block,
UVM_ASSERT(!page_count(dpage));
UVM_ASSERT(!dpage->zone_device_data);
zone_device_page_init(dpage);
nv_zone_device_page_init(dpage);
dpage->zone_device_data = gpu_chunk;
atomic64_inc(&va_block->hmm.va_space->hmm.allocated_page_count);
}

25
kernel-open/nvidia-uvm/uvm_hopper.c
View File

@@ -41,10 +41,6 @@ static uvm_gpu_peer_copy_mode_t hopper_peer_copy_mode(uvm_parent_gpu_t *parent_g
void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = true;
parent_gpu->tlb_batch.va_range_invalidate_supported = true;
// TODO: Bug 1767241: Run benchmarks to figure out a good number
parent_gpu->tlb_batch.max_ranges = 8;
@@ -82,39 +78,18 @@ void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->max_host_va = 1ull << 57;
// Hopper can map sysmem with any page size
parent_gpu->can_map_sysmem_with_large_pages = true;
// Prefetch instructions will generate faults
parent_gpu->prefetch_fault_supported = true;
// Hopper can place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = true;
parent_gpu->replayable_faults_supported = true;
parent_gpu->non_replayable_faults_supported = true;
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->fault_cancel_va_supported = true;
parent_gpu->scoped_atomics_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
parent_gpu->has_clear_faulted_channel_method = false;
parent_gpu->smc.supported = true;
parent_gpu->sparse_mappings_supported = true;
parent_gpu->map_remap_larger_page_promotion = false;
parent_gpu->plc_supported = true;
parent_gpu->ats.no_ats_range_required = true;
parent_gpu->ats.gmmu_pt_depth0_init_required = false;

15
kernel-open/nvidia-uvm/uvm_hopper_mmu.c
View File

@@ -239,13 +239,11 @@ static NvU64 make_sparse_pte_hopper(void)
HWCONST64(_MMU_VER3, PTE, PCF, SPARSE);
}
static NvU64 unmapped_pte_hopper(NvU64 page_size)
static NvU64 unmapped_pte_hopper(void)
{
// Setting PCF to NO_VALID_4KB_PAGE on an otherwise-zeroed big PTE causes
// the corresponding 4k PTEs to be ignored. This allows the invalidation of
// a mixed PDE range to be much faster.
if (page_size != UVM_PAGE_SIZE_64K)
return 0;
// When VALID == 0, GMMU still reads the PCF field, which indicates the PTE
// is sparse (make_sparse_pte_hopper) or an unmapped big-page PTE.
@@ -509,19 +507,12 @@ static void make_pde_hopper(void *entry,
static uvm_mmu_mode_hal_t hopper_mmu_mode_hal;
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(NvU64 big_page_size)
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_hopper(void)
{
static bool initialized = false;
UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
// TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
// 128K big page size for Pascal+ GPUs
if (big_page_size == UVM_PAGE_SIZE_128K)
return NULL;
if (!initialized) {
uvm_mmu_mode_hal_t *ampere_mmu_mode_hal = uvm_hal_mmu_mode_ampere(big_page_size);
uvm_mmu_mode_hal_t *ampere_mmu_mode_hal = uvm_hal_mmu_mode_ampere();
UVM_ASSERT(ampere_mmu_mode_hal);
// The assumption made is that arch_hal->mmu_mode_hal() will be called

8
kernel-open/nvidia-uvm/uvm_hopper_sec2.c
View File

@@ -29,7 +29,7 @@
#include "nv_uvm_interface.h"
#include "clcba2.h"
#include "clc86f.h"
#include "clb06f.h"
#include "clc46f.h"
#define UVM_CSL_SIGN_AUTH_TAG_ALIGNMENT_BYTES (1 << HWSHIFT(CBA2, METHOD_STREAM_AUTH_TAG_ADDR_LO, DATA))
@@ -42,11 +42,11 @@ static void sign_push(uvm_push_t *push, NvU32 *init_method, NvU8 *auth_tag)
UVM_ASSERT(init_method < push->next);
while (init_method < push->next) {
NvU8 subch = READ_HWVALUE(*init_method, B06F, DMA, METHOD_SUBCHANNEL);
NvU32 count = READ_HWVALUE(*init_method, B06F, DMA, METHOD_COUNT);
NvU8 subch = READ_HWVALUE(*init_method, C46F, DMA, METHOD_SUBCHANNEL);
NvU32 count = READ_HWVALUE(*init_method, C46F, DMA, METHOD_COUNT);
if (subch == UVM_SUBCHANNEL_CBA2) {
NvU32 method_addr = READ_HWVALUE(*init_method, B06F, DMA, METHOD_ADDRESS) << 2;
NvU32 method_addr = READ_HWVALUE(*init_method, C46F, DMA, METHOD_ADDRESS) << 2;
UVM_ASSERT(count == 1);
UVM_ASSERT((sign_size + 2) * UVM_METHOD_SIZE <= UVM_CONF_COMPUTING_SIGN_BUF_MAX_SIZE);

22
kernel-open/nvidia-uvm/uvm_ioctl.h
View File

@@ -694,28 +694,6 @@ typedef struct
NV_STATUS rmStatus; // OUT
} UVM_MIGRATE_RANGE_GROUP_PARAMS;
//
// UvmEnableSystemWideAtomics
//
#define UVM_ENABLE_SYSTEM_WIDE_ATOMICS UVM_IOCTL_BASE(54)
typedef struct
{
NvProcessorUuid gpu_uuid; // IN
NV_STATUS rmStatus; // OUT
} UVM_ENABLE_SYSTEM_WIDE_ATOMICS_PARAMS;
//
// UvmDisableSystemWideAtomics
//
#define UVM_DISABLE_SYSTEM_WIDE_ATOMICS UVM_IOCTL_BASE(55)
typedef struct
{
NvProcessorUuid gpu_uuid; // IN
NV_STATUS rmStatus; // OUT
} UVM_DISABLE_SYSTEM_WIDE_ATOMICS_PARAMS;
//
// Initialize any tracker object such as a queue or counter
// UvmToolsCreateEventQueue, UvmToolsCreateProcessAggregateCounters,

12
kernel-open/nvidia-uvm/uvm_linux.h
View File

@@ -103,7 +103,11 @@
#define UVM_HMM_RANGE_FAULT_SUPPORTED() 0
#endif
#define UVM_CAN_USE_MMU_NOTIFIERS() 1
#if defined(CONFIG_MMU_NOTIFIER)
#define UVM_CAN_USE_MMU_NOTIFIERS() 1
#else
#define UVM_CAN_USE_MMU_NOTIFIERS() 0
#endif
//
// printk.h already defined pr_fmt, so we have to redefine it so the pr_*
@@ -115,12 +119,6 @@
#define NV_UVM_GFP_FLAGS (GFP_KERNEL | __GFP_NOMEMALLOC)
#if defined(NV_GET_DEV_PAGEMAP_HAS_PGMAP_ARG)
#define NV_GET_DEV_PAGEMAP(pfn) get_dev_pagemap(pfn, NULL)
#else
#define NV_GET_DEV_PAGEMAP get_dev_pagemap
#endif
/* Return a nanosecond-precise value */
static inline NvU64 NV_GETTIME(void)
{

9
kernel-open/nvidia-uvm/uvm_map_external.c
View File

@@ -858,9 +858,6 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_external_t *ext
uvm_assert_rwsem_locked_read(&va_space->lock);
if ((map_rm_params->compression_type == UvmGpuCompressionTypeEnabledNoPlc) && !mapping_gpu->parent->plc_supported)
return NV_ERR_INVALID_DEVICE;
// Check if the GPU can access the VA
if (!uvm_gpu_can_address(mapping_gpu, base, length))
return NV_ERR_OUT_OF_RANGE;
@@ -1192,12 +1189,6 @@ static NV_STATUS uvm_map_external_sparse(uvm_va_space_t *va_space, UVM_MAP_EXTER
goto out;
}
// Sparse mappings are unsupported on GPUs prior to Pascal.
if (!mapping_gpu->parent->sparse_mappings_supported) {
status = NV_ERR_INVALID_DEVICE;
goto out;
}
status = uvm_map_external_sparse_on_gpu(external_range,
mapping_gpu,
params->base,

85
kernel-open/nvidia-uvm/uvm_maxwell.c
View File

@@ -1,85 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_hal.h"
#include "uvm_gpu.h"
#include "uvm_mem.h"
void uvm_hal_maxwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = false;
// 128 GB should be enough for all current RM allocations and leaves enough
// space for UVM internal mappings.
// A single top level PDE covers 64 or 128 MB on Maxwell so 128 GB is fine to use.
parent_gpu->rm_va_base = 0;
parent_gpu->rm_va_size = 128 * UVM_SIZE_1GB;
parent_gpu->peer_va_base = 0;
parent_gpu->peer_va_size = 0;
parent_gpu->uvm_mem_va_base = 768 * UVM_SIZE_1GB;
parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
parent_gpu->ce_phys_vidmem_write_supported = true;
// We don't have a compelling use case in UVM-Lite for direct peer
// migrations between GPUs, so don't bother setting them up.
parent_gpu->peer_copy_mode = UVM_GPU_PEER_COPY_MODE_UNSUPPORTED;
parent_gpu->max_channel_va = 1ULL << 40;
parent_gpu->max_host_va = 1ULL << 40;
// Maxwell can only map sysmem with 4K pages
parent_gpu->can_map_sysmem_with_large_pages = false;
// Maxwell cannot place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = false;
parent_gpu->replayable_faults_supported = false;
parent_gpu->non_replayable_faults_supported = false;
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->fault_cancel_va_supported = false;
parent_gpu->scoped_atomics_supported = false;
parent_gpu->sparse_mappings_supported = false;
parent_gpu->map_remap_larger_page_promotion = false;
parent_gpu->smc.supported = false;
parent_gpu->plc_supported = false;
parent_gpu->ats.no_ats_range_required = false;
parent_gpu->ats.gmmu_pt_depth0_init_required = false;
parent_gpu->conf_computing.per_channel_key_rotation = false;
}

87
kernel-open/nvidia-uvm/uvm_maxwell_access_counter_buffer.c
View File

@@ -1,87 +0,0 @@
/*******************************************************************************
Copyright (c) 2021-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_gpu.h"
#include "uvm_hal.h"
void uvm_hal_maxwell_enable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters)
{
UVM_ASSERT_MSG(false,
"enable_access_counter_notifications is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
}
void uvm_hal_maxwell_disable_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters)
{
UVM_ASSERT_MSG(false,
"disable_access_counter_notifications is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
}
void uvm_hal_maxwell_clear_access_counter_notifications_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 get)
{
UVM_ASSERT_MSG(false,
"clear_access_counter_notifications is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
}
NvU32 uvm_hal_maxwell_access_counter_buffer_entry_size_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"access_counter_buffer_entry_size is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return 0;
}
bool uvm_hal_maxwell_access_counter_buffer_entry_is_valid_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index)
{
UVM_ASSERT_MSG(false,
"access_counter_buffer_entry_is_valid is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
return false;
}
void uvm_hal_maxwell_access_counter_buffer_entry_clear_valid_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index)
{
UVM_ASSERT_MSG(false,
"access_counter_buffer_entry_clear_valid is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
}
void uvm_hal_maxwell_access_counter_buffer_parse_entry_unsupported(uvm_access_counter_buffer_t *access_counters,
NvU32 index,
uvm_access_counter_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false,
"access_counter_buffer_parse_entry is not supported on GPU: %s notif buf index: %u.\n",
uvm_parent_gpu_name(access_counters->parent_gpu),
access_counters->index);
}

132
kernel-open/nvidia-uvm/uvm_maxwell_fault_buffer.c
View File

@@ -1,132 +0,0 @@
/*******************************************************************************
Copyright (c) 2021-2024 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_gpu.h"
#include "uvm_hal.h"
void uvm_hal_maxwell_enable_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"enable_replayable_faults is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}
void uvm_hal_maxwell_disable_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"disable_replayable_faults is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}
void uvm_hal_maxwell_clear_replayable_faults_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 get)
{
UVM_ASSERT_MSG(false,
"clear_replayable_faults is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}
NvU32 uvm_hal_maxwell_fault_buffer_read_put_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"fault_buffer_read_put is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return 0;
}
NvU32 uvm_hal_maxwell_fault_buffer_read_get_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"fault_buffer_read_get is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return 0;
}
void uvm_hal_maxwell_fault_buffer_write_get_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UVM_ASSERT_MSG(false,
"fault_buffer_write_get is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}
NvU8 uvm_hal_maxwell_fault_buffer_get_ve_id_unsupported(NvU16 mmu_engine_id, uvm_mmu_engine_type_t mmu_engine_type)
{
UVM_ASSERT_MSG(false, "fault_buffer_get_ve_id is not supported on Maxwell GPUs.\n");
return 0;
}
uvm_mmu_engine_type_t uvm_hal_maxwell_fault_buffer_get_mmu_engine_type_unsupported(NvU16 mmu_engine_id,
uvm_fault_client_type_t client_type,
NvU16 client_id)
{
UVM_ASSERT_MSG(false, "fault_buffer_get_mmu_engine_type is not supported on Maxwell GPUs.\n");
return UVM_MMU_ENGINE_TYPE_GRAPHICS;
}
uvm_fault_type_t uvm_hal_maxwell_fault_buffer_get_fault_type_unsupported(const NvU32 *fault_entry)
{
UVM_ASSERT_MSG(false, "fault_buffer_get_fault_type is not supported.\n");
return UVM_FAULT_TYPE_COUNT;
}
NV_STATUS uvm_hal_maxwell_fault_buffer_parse_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false,
"fault_buffer_parse_entry is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return NV_ERR_NOT_SUPPORTED;
}
bool uvm_hal_maxwell_fault_buffer_entry_is_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UVM_ASSERT_MSG(false,
"fault_buffer_entry_is_valid is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return false;
}
void uvm_hal_maxwell_fault_buffer_entry_clear_valid_unsupported(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UVM_ASSERT_MSG(false,
"fault_buffer_entry_clear_valid is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}
NvU32 uvm_hal_maxwell_fault_buffer_entry_size_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false,
"fault_buffer_entry_size is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
return 0;
}
void uvm_hal_maxwell_fault_buffer_parse_non_replayable_entry_unsupported(uvm_parent_gpu_t *parent_gpu,
void *fault_packet,
uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false,
"fault_buffer_parse_non_replayable_entry is not supported on GPU: %s.\n",
uvm_parent_gpu_name(parent_gpu));
}

377
kernel-open/nvidia-uvm/uvm_maxwell_host.c
View File

@@ -1,377 +0,0 @@
/*******************************************************************************
Copyright (c) 2021-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_linux.h"
#include "uvm_global.h"
#include "uvm_hal_types.h"
#include "uvm_hal.h"
#include "uvm_push.h"
#include "cla16f.h"
#include "clb06f.h"
void uvm_hal_maxwell_host_wait_for_idle(uvm_push_t *push)
{
NV_PUSH_1U(A16F, WFI, 0);
}
void uvm_hal_maxwell_host_membar_sys(uvm_push_t *push)
{
NV_PUSH_1U(A16F, MEM_OP_B,
HWCONST(A16F, MEM_OP_B, OPERATION, SYSMEMBAR_FLUSH));
}
void uvm_hal_maxwell_host_tlb_invalidate_all_a16f(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar)
{
NvU32 target;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
// Only Pascal+ supports invalidating down from a specific depth.
(void)depth;
(void)membar;
if (pdb.aperture == UVM_APERTURE_VID)
target = HWCONST(A16F, MEM_OP_A, TLB_INVALIDATE_TARGET, VID_MEM);
else
target = HWCONST(A16F, MEM_OP_A, TLB_INVALIDATE_TARGET, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
NV_PUSH_2U(A16F, MEM_OP_A, target |
HWVALUE(A16F, MEM_OP_A, TLB_INVALIDATE_ADDR, pdb.address),
MEM_OP_B, HWCONST(A16F, MEM_OP_B, OPERATION, MMU_TLB_INVALIDATE) |
HWCONST(A16F, MEM_OP_B, MMU_TLB_INVALIDATE_PDB, ONE) |
HWCONST(A16F, MEM_OP_B, MMU_TLB_INVALIDATE_GPC, ENABLE));
}
void uvm_hal_maxwell_host_tlb_invalidate_all_b06f(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
uvm_membar_t membar)
{
NvU32 target;
NvU32 pdb_lo;
NvU32 pdb_hi;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
// Only Pascal+ supports invalidating down from a specific depth.
(void)depth;
(void)membar;
if (pdb.aperture == UVM_APERTURE_VID)
target = HWCONST(B06F, MEM_OP_C, TLB_INVALIDATE_TARGET, VID_MEM);
else
target = HWCONST(B06F, MEM_OP_C, TLB_INVALIDATE_TARGET, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
pdb_lo = pdb.address & HWMASK(B06F, MEM_OP_C, TLB_INVALIDATE_ADDR_LO);
pdb_hi = pdb.address >> HWSIZE(B06F, MEM_OP_C, TLB_INVALIDATE_ADDR_LO);
NV_PUSH_2U(B06F, MEM_OP_C, target |
HWCONST(B06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWCONST(B06F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
HWVALUE(B06F, MEM_OP_C, TLB_INVALIDATE_ADDR_LO, pdb_lo),
MEM_OP_D, HWCONST(B06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
HWVALUE(B06F, MEM_OP_D, TLB_INVALIDATE_ADDR_HI, pdb_hi));
}
void uvm_hal_maxwell_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
NvU64 base,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar)
{
// No per VA invalidate on Maxwell, redirect to invalidate all.
uvm_push_get_gpu(push)->parent->host_hal->tlb_invalidate_all(push, pdb, depth, membar);
}
void uvm_hal_maxwell_host_tlb_invalidate_phys_unsupported(uvm_push_t *push)
{
UVM_ASSERT_MSG(false, "GPA caching is not supported on Maxwell\n");
}
void uvm_hal_maxwell_host_tlb_invalidate_test(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params)
{
NvU32 target_pdb = 0;
NvU32 invalidate_gpc_value;
// Only Pascal+ supports invalidating down from a specific depth. We
// invalidate all
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
if (pdb.aperture == UVM_APERTURE_VID)
target_pdb = HWCONST(A16F, MEM_OP_A, TLB_INVALIDATE_TARGET, VID_MEM);
else
target_pdb = HWCONST(A16F, MEM_OP_A, TLB_INVALIDATE_TARGET, SYS_MEM_COHERENT);
target_pdb |= HWVALUE(A16F, MEM_OP_A, TLB_INVALIDATE_ADDR, pdb.address);
if (params->disable_gpc_invalidate)
invalidate_gpc_value = HWCONST(A16F, MEM_OP_B, MMU_TLB_INVALIDATE_GPC, DISABLE);
else
invalidate_gpc_value = HWCONST(A16F, MEM_OP_B, MMU_TLB_INVALIDATE_GPC, ENABLE);
NV_PUSH_2U(A16F, MEM_OP_A, target_pdb,
MEM_OP_B, HWCONST(A16F, MEM_OP_B, OPERATION, MMU_TLB_INVALIDATE) |
HWCONST(A16F, MEM_OP_B, MMU_TLB_INVALIDATE_PDB, ONE) |
invalidate_gpc_value);
}
void uvm_hal_maxwell_host_tlb_flush_prefetch_unsupported(uvm_push_t *push)
{
UVM_ASSERT_MSG(false, "Prefetch flushing not needed on Maxwell\n");
}
void uvm_hal_maxwell_host_noop(uvm_push_t *push, NvU32 size)
{
UVM_ASSERT_MSG(size % 4 == 0, "size %u\n", size);
if (size == 0)
return;
// size is in bytes so divide by the method size (4 bytes)
size /= 4;
while (size > 0) {
// noop_this_time includes the NOP method itself and hence can be
// up to COUNT_MAX + 1.
NvU32 noop_this_time = min(UVM_METHOD_COUNT_MAX + 1, size);
// -1 for the NOP method itself.
NV_PUSH_NU_NONINC(A16F, NOP, noop_this_time - 1);
size -= noop_this_time;
}
}
void uvm_hal_maxwell_host_interrupt(uvm_push_t *push)
{
NV_PUSH_1U(A16F, NON_STALL_INTERRUPT, 0);
}
void uvm_hal_maxwell_host_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload)
{
NvU32 sem_lo;
UVM_ASSERT_MSG(uvm_push_get_gpu(push)->parent->host_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(uvm_push_get_gpu(push)));
UVM_ASSERT(!(NvOffset_LO32(gpu_va) & ~HWSHIFTMASK(A16F, SEMAPHOREB, OFFSET_LOWER)));
sem_lo = READ_HWVALUE(NvOffset_LO32(gpu_va), A16F, SEMAPHOREB, OFFSET_LOWER);
uvm_hal_wfi_membar(push, uvm_push_get_and_reset_membar_flag(push));
NV_PUSH_4U(A16F, SEMAPHOREA, HWVALUE(A16F, SEMAPHOREA, OFFSET_UPPER, NvOffset_HI32(gpu_va)),
SEMAPHOREB, HWVALUE(A16F, SEMAPHOREB, OFFSET_LOWER, sem_lo),
SEMAPHOREC, payload,
SEMAPHORED, HWCONST(A16F, SEMAPHORED, OPERATION, RELEASE) |
HWCONST(A16F, SEMAPHORED, RELEASE_SIZE, 4BYTE)|
HWCONST(A16F, SEMAPHORED, RELEASE_WFI, DIS));
}
void uvm_hal_maxwell_host_semaphore_acquire(uvm_push_t *push, NvU64 gpu_va, NvU32 payload)
{
NvU32 sem_lo;
UVM_ASSERT_MSG(uvm_push_get_gpu(push)->parent->host_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(uvm_push_get_gpu(push)));
UVM_ASSERT(!(NvOffset_LO32(gpu_va) & ~HWSHIFTMASK(A16F, SEMAPHOREB, OFFSET_LOWER)));
sem_lo = READ_HWVALUE(NvOffset_LO32(gpu_va), A16F, SEMAPHOREB, OFFSET_LOWER);
NV_PUSH_4U(A16F, SEMAPHOREA, HWVALUE(A16F, SEMAPHOREA, OFFSET_UPPER, NvOffset_HI32(gpu_va)),
SEMAPHOREB, HWVALUE(A16F, SEMAPHOREB, OFFSET_LOWER, sem_lo),
SEMAPHOREC, payload,
SEMAPHORED, HWCONST(A16F, SEMAPHORED, ACQUIRE_SWITCH, ENABLED) |
HWCONST(A16F, SEMAPHORED, OPERATION, ACQ_GEQ));
}
void uvm_hal_maxwell_host_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va)
{
NvU32 sem_lo;
UVM_ASSERT_MSG(uvm_push_get_gpu(push)->parent->host_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(uvm_push_get_gpu(push)));
UVM_ASSERT(!(NvOffset_LO32(gpu_va) & ~HWSHIFTMASK(A16F, SEMAPHOREB, OFFSET_LOWER)));
sem_lo = READ_HWVALUE(NvOffset_LO32(gpu_va), A16F, SEMAPHOREB, OFFSET_LOWER);
uvm_hal_wfi_membar(push, uvm_push_get_and_reset_membar_flag(push));
NV_PUSH_4U(A16F, SEMAPHOREA, HWVALUE(A16F, SEMAPHOREA, OFFSET_UPPER, NvOffset_HI32(gpu_va)),
SEMAPHOREB, HWVALUE(A16F, SEMAPHOREB, OFFSET_LOWER, sem_lo),
SEMAPHOREC, 0xdeadbeef,
SEMAPHORED, HWCONST(A16F, SEMAPHORED, OPERATION, RELEASE) |
HWCONST(A16F, SEMAPHORED, RELEASE_SIZE, 16BYTE)|
HWCONST(A16F, SEMAPHORED, RELEASE_WFI, DIS));
}
void uvm_hal_maxwell_host_set_gpfifo_entry(NvU64 *fifo_entry,
NvU64 pushbuffer_va,
NvU32 pushbuffer_length,
uvm_gpfifo_sync_t sync_flag)
{
NvU64 fifo_entry_value;
const NvU32 sync_value = (sync_flag == UVM_GPFIFO_SYNC_WAIT) ? HWCONST(A16F, GP_ENTRY1, SYNC, WAIT) :
HWCONST(A16F, GP_ENTRY1, SYNC, PROCEED);
UVM_ASSERT(!uvm_global_is_suspended());
UVM_ASSERT_MSG(pushbuffer_va % 4 == 0, "pushbuffer va unaligned: %llu\n", pushbuffer_va);
UVM_ASSERT_MSG(pushbuffer_length % 4 == 0, "pushbuffer length unaligned: %u\n", pushbuffer_length);
fifo_entry_value = HWVALUE(A16F, GP_ENTRY0, GET, NvU64_LO32(pushbuffer_va) >> 2);
fifo_entry_value |= (NvU64)(HWVALUE(A16F, GP_ENTRY1, GET_HI, NvU64_HI32(pushbuffer_va)) |
HWVALUE(A16F, GP_ENTRY1, LENGTH, pushbuffer_length >> 2) |
HWCONST(A16F, GP_ENTRY1, PRIV, KERNEL) |
sync_value) << 32;
*fifo_entry = fifo_entry_value;
}
void uvm_hal_maxwell_host_set_gpfifo_noop(NvU64 *fifo_entry)
{
UVM_ASSERT(!uvm_global_is_suspended());
// A NOOP control GPFIFO does not require a GP_ENTRY0.
*fifo_entry = (NvU64)(HWVALUE(A16F, GP_ENTRY1, LENGTH, 0) | HWCONST(A16F, GP_ENTRY1, OPCODE, NOP)) << 32;
}
void uvm_hal_maxwell_host_set_gpfifo_pushbuffer_segment_base_unsupported(NvU64 *fifo_entry, NvU64 pushbuffer_va)
{
UVM_ASSERT_MSG(false, "host set_gpfifo_pushbuffer_segment_base called on Maxwell GPU\n");
}
void uvm_hal_maxwell_host_write_gpu_put(uvm_channel_t *channel, NvU32 gpu_put)
{
UVM_GPU_WRITE_ONCE(*channel->channel_info.gpPut, gpu_put);
}
void uvm_hal_maxwell_host_init_noop(uvm_push_t *push)
{
}
void uvm_hal_maxwell_replay_faults_unsupported(uvm_push_t *push, uvm_fault_replay_type_t type)
{
UVM_ASSERT_MSG(false, "host replay_faults called on Maxwell GPU\n");
}
void uvm_hal_maxwell_cancel_faults_global_unsupported(uvm_push_t *push, uvm_gpu_phys_address_t instance_ptr)
{
UVM_ASSERT_MSG(false, "host cancel_faults_global called on Maxwell GPU\n");
}
void uvm_hal_maxwell_cancel_faults_targeted_unsupported(uvm_push_t *push,
uvm_gpu_phys_address_t instance_ptr,
NvU32 gpc_id,
NvU32 client_id)
{
UVM_ASSERT_MSG(false, "host cancel_faults_targeted called on Maxwell GPU\n");
}
void uvm_hal_maxwell_cancel_faults_va_unsupported(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
const uvm_fault_buffer_entry_t *fault_entry,
uvm_fault_cancel_va_mode_t cancel_va_mode)
{
UVM_ASSERT_MSG(false, "host cancel_faults_va called on Maxwell GPU\n");
}
void uvm_hal_maxwell_host_clear_faulted_channel_sw_method_unsupported(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false, "host clear_faulted_channel_sw_method called on Maxwell GPU\n");
}
void uvm_hal_maxwell_host_clear_faulted_channel_method_unsupported(uvm_push_t *push,
uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false, "host clear_faulted_channel_method called on Maxwell GPU\n");
}
void uvm_hal_maxwell_host_clear_faulted_channel_register_unsupported(uvm_user_channel_t *user_channel,
const uvm_fault_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false, "host clear_faulted_channel_register called on Maxwell GPU\n");
}
void uvm_hal_maxwell_access_counter_clear_all_unsupported(uvm_push_t *push)
{
UVM_ASSERT_MSG(false, "host access_counter_clear_all called on Maxwell GPU\n");
}
void uvm_hal_maxwell_access_counter_clear_targeted_unsupported(uvm_push_t *push,
const uvm_access_counter_buffer_entry_t *buffer_entry)
{
UVM_ASSERT_MSG(false, "host access_counter_clear_targeted called on Maxwell GPU\n");
}
uvm_access_counter_clear_op_t
uvm_hal_maxwell_access_counter_query_clear_op_unsupported(uvm_parent_gpu_t *parent_gpu,
uvm_access_counter_buffer_entry_t **buffer_entries,
NvU32 num_entries)
{
UVM_ASSERT_MSG(false, "host access_counter_query_clear_op called on Maxwell GPU\n");
return UVM_ACCESS_COUNTER_CLEAR_OP_COUNT;
}
NvU64 uvm_hal_maxwell_get_time(uvm_gpu_t *gpu)
{
NvU32 time0;
NvU32 time1_first, time1_second;
// When reading the TIME, TIME_1 should be read first, followed by TIME_0,
// then a second reading of TIME_1 should be done. If the two readings of
// do not match, this process should be repeated.
//
// Doing that will catch the 4-second wrap-around
do {
time1_first = UVM_GPU_READ_ONCE(*gpu->time.time1_register);
rmb();
time0 = UVM_GPU_READ_ONCE(*gpu->time.time0_register);
rmb();
time1_second = UVM_GPU_READ_ONCE(*gpu->time.time1_register);
} while (time1_second != time1_first);
return (((NvU64)time1_first) << 32) + time0;
}

388
kernel-open/nvidia-uvm/uvm_maxwell_mmu.c
View File

@@ -1,388 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
// For Maxwell, UVM page tree 'depth' maps to hardware as follows:
//
// 64k Big page size:
// UVM depth HW level VA bits
// 0 PDE 39:26
// 1 PTE Big / PTE 4k 25:16 / 25:12
//
// 128k Big page size (same levels, just different VA bits):
// UVM depth HW level VA bits
// 0 PDE 39:27
// 1 PTE Big / PTE 4k 26:17 / 26:12
#include "uvm_types.h"
#include "uvm_forward_decl.h"
#include "uvm_gpu.h"
#include "uvm_mmu.h"
#include "uvm_hal.h"
#include "uvm_push_macros.h"
#include "hwref/maxwell/gm107/dev_mmu.h"
#define MMU_BIG 0
#define MMU_SMALL 1
static NvU32 entries_per_index_maxwell(NvU32 depth)
{
UVM_ASSERT(depth < 2);
if (depth == 0)
return 2;
return 1;
}
static NvLength entry_offset_maxwell(NvU32 depth, NvU64 page_size)
{
UVM_ASSERT(depth < 2);
if (page_size == UVM_PAGE_SIZE_4K && depth == 0)
return MMU_SMALL;
return MMU_BIG;
}
static NvU64 big_half_pde_maxwell(uvm_mmu_page_table_alloc_t *phys_alloc)
{
NvU64 pde_bits = 0;
if (phys_alloc != NULL) {
NvU64 address = phys_alloc->addr.address >> NV_MMU_PDE_ADDRESS_SHIFT;
pde_bits |= HWCONST64(_MMU, PDE, VOL_BIG, TRUE);
switch (phys_alloc->addr.aperture) {
case UVM_APERTURE_SYS:
pde_bits |= HWCONST64(_MMU, PDE, APERTURE_BIG, SYSTEM_COHERENT_MEMORY) |
HWVALUE64(_MMU, PDE, ADDRESS_BIG_SYS, address);
break;
case UVM_APERTURE_VID:
pde_bits |= HWCONST64(_MMU, PDE, APERTURE_BIG, VIDEO_MEMORY) |
HWVALUE64(_MMU, PDE, ADDRESS_BIG_VID, phys_alloc->addr.address >> NV_MMU_PDE_ADDRESS_SHIFT);
break;
default:
UVM_ASSERT_MSG(0, "Invalid big aperture: %d\n", phys_alloc->addr.aperture);
break;
}
}
return pde_bits;
}
static NvU64 small_half_pde_maxwell(uvm_mmu_page_table_alloc_t *phys_alloc)
{
NvU64 pde_bits = 0;
if (phys_alloc != NULL) {
NvU64 address = phys_alloc->addr.address >> NV_MMU_PDE_ADDRESS_SHIFT;
pde_bits |= HWCONST64(_MMU, PDE, VOL_SMALL, TRUE);
switch (phys_alloc->addr.aperture) {
case UVM_APERTURE_SYS:
pde_bits |= HWCONST64(_MMU, PDE, APERTURE_SMALL, SYSTEM_COHERENT_MEMORY) |
HWVALUE64(_MMU, PDE, ADDRESS_SMALL_SYS, address);
break;
case UVM_APERTURE_VID:
pde_bits |= HWCONST64(_MMU, PDE, APERTURE_SMALL, VIDEO_MEMORY) |
HWVALUE64(_MMU, PDE, ADDRESS_SMALL_VID, address);
break;
default:
UVM_ASSERT_MSG(0, "Invalid small aperture: %d\n", phys_alloc->addr.aperture);
break;
}
}
return pde_bits;
}
static void make_pde_maxwell(void *entry,
uvm_mmu_page_table_alloc_t **phys_allocs,
uvm_page_directory_t *dir,
NvU32 child_index)
{
NvU64 pde_bits = 0;
UVM_ASSERT(dir);
UVM_ASSERT(dir->depth == 0);
pde_bits |= HWCONST64(_MMU, PDE, SIZE, FULL);
pde_bits |= big_half_pde_maxwell(phys_allocs[MMU_BIG]) | small_half_pde_maxwell(phys_allocs[MMU_SMALL]);
*(NvU64 *)entry = pde_bits;
}
static NvLength entry_size_maxwell(NvU32 depth)
{
UVM_ASSERT(depth < 2);
return 8;
}
static NvU32 index_bits_maxwell_64(NvU32 depth, NvU64 page_size)
{
UVM_ASSERT(depth < 2);
UVM_ASSERT(page_size == UVM_PAGE_SIZE_4K ||
page_size == UVM_PAGE_SIZE_64K ||
(depth == 0 && page_size == UVM_PAGE_SIZE_AGNOSTIC));
if (depth == 0) {
return 14;
}
else {
if (page_size == UVM_PAGE_SIZE_4K)
return 14;
else
return 10;
}
}
static NvU32 index_bits_maxwell_128(NvU32 depth, NvU64 page_size)
{
UVM_ASSERT(depth < 2);
UVM_ASSERT(page_size == UVM_PAGE_SIZE_4K ||
page_size == UVM_PAGE_SIZE_128K ||
(depth == 0 && page_size == UVM_PAGE_SIZE_AGNOSTIC));
if (depth == 0) {
return 13;
}
else {
if (page_size == UVM_PAGE_SIZE_4K)
return 15;
else
return 10;
}
}
static NvU32 num_va_bits_maxwell(void)
{
return 40;
}
static NvLength allocation_size_maxwell_64(NvU32 depth, NvU64 page_size)
{
return entry_size_maxwell(depth) << index_bits_maxwell_64(depth, page_size);
}
static NvLength allocation_size_maxwell_128(NvU32 depth, NvU64 page_size)
{
return entry_size_maxwell(depth) << index_bits_maxwell_128(depth, page_size);
}
static NvU32 page_table_depth_maxwell(NvU64 page_size)
{
return 1;
}
static NvU64 page_sizes_maxwell_128(void)
{
return UVM_PAGE_SIZE_128K | UVM_PAGE_SIZE_4K;
}
static NvU64 page_sizes_maxwell_64(void)
{
return UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
}
static NvU64 unmapped_pte_maxwell(NvU64 page_size)
{
// Setting the privilege bit on an otherwise-zeroed big PTE causes the
// corresponding 4k PTEs to be ignored. This allows the invalidation of a
// mixed PDE range to be much faster.
if (page_size == UVM_PAGE_SIZE_4K)
return 0;
// When VALID == 0, MMU still reads the VOL and PRIV fields. VOL == 1
// indicates that the PTE is sparse, so make sure we don't use it.
return HWCONST64(_MMU, PTE, VALID, FALSE) |
HWCONST64(_MMU, PTE, PRIVILEGE, TRUE) |
HWCONST64(_MMU, PTE, VOL, FALSE);
}
static NvU64 make_pte_maxwell(uvm_aperture_t aperture, NvU64 address, uvm_prot_t prot, NvU64 flags)
{
NvU64 pte_bits = 0;
NvU8 aperture_bits = 0;
UVM_ASSERT(prot != UVM_PROT_NONE);
UVM_ASSERT((flags & ~UVM_MMU_PTE_FLAGS_MASK) == 0);
// valid 0:0
pte_bits |= HWCONST64(_MMU, PTE, VALID, TRUE);
// privilege 1:1
pte_bits |= HWCONST64(_MMU, PTE, PRIVILEGE, FALSE);
// read-only 2:2 (used by everything except L1 and GCC)
if (prot == UVM_PROT_READ_ONLY)
pte_bits |= HWCONST64(_MMU, PTE, READ_ONLY, TRUE);
else
pte_bits |= HWCONST64(_MMU, PTE, READ_ONLY, FALSE);
// encrypted 3:3
pte_bits |= HWCONST64(_MMU, PTE, ENCRYPTED, FALSE);
address >>= NV_MMU_PTE_ADDRESS_SHIFT;
if (aperture == UVM_APERTURE_SYS) {
// sys address 31:4
pte_bits |= HWVALUE64(_MMU, PTE, ADDRESS_SYS, address);
}
else {
// vid address 28:4
pte_bits |= HWVALUE64(_MMU, PTE, ADDRESS_VID, address);
// peer 29:31
if (aperture != UVM_APERTURE_VID)
pte_bits |= HWVALUE64(_MMU, PTE, ADDRESS_VID_PEER, UVM_APERTURE_PEER_ID(aperture));
}
// volatile 32:32
if (flags & UVM_MMU_PTE_FLAGS_CACHED)
pte_bits |= HWCONST64(_MMU, PTE, VOL, FALSE);
else
pte_bits |= HWCONST64(_MMU, PTE, VOL, TRUE);
// aperture 34:33
if (aperture == UVM_APERTURE_SYS)
aperture_bits = NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_VID)
aperture_bits = NV_MMU_PTE_APERTURE_VIDEO_MEMORY;
else if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7)
aperture_bits = NV_MMU_PTE_APERTURE_PEER_MEMORY;
else
UVM_ASSERT_MSG(NV_FALSE, "Invalid location: %d", aperture);
pte_bits |= HWVALUE64(_MMU, PTE, APERTURE, aperture_bits);
// lock 35:35
pte_bits |= HWCONST64(_MMU, PTE, LOCK, FALSE);
// kind 43:36
pte_bits |= HWCONST64(_MMU, PTE, KIND, PITCH);
// comptagline 61:44
pte_bits |= HWVALUE64(_MMU, PTE, COMPTAGLINE, 0);
// read disable 62:62 (used only by L1 and GCC)
pte_bits |= HWCONST64(_MMU, PTE, READ_DISABLE, FALSE);
// write disable 63:63 (used only by L1 and GCC: everything else uses READ_ONLY)
if (prot == UVM_PROT_READ_ONLY)
pte_bits |= HWCONST64(_MMU, PTE, WRITE_DISABLE, TRUE);
else
pte_bits |= HWCONST64(_MMU, PTE, WRITE_DISABLE, FALSE);
return pte_bits;
}
static NvU64 make_sked_reflected_pte_maxwell(void)
{
NvU64 pte_bits = 0;
pte_bits |= HWCONST64(_MMU, PTE, VALID, TRUE);
pte_bits |= HWCONST64(_MMU, PTE, KIND, SMSKED_MESSAGE);
return pte_bits;
}
static NvU64 poisoned_pte_maxwell(uvm_page_tree_t *tree)
{
// An invalid PTE is also fatal on Maxwell, but a PRIV violation will
// immediately identify bad PTE usage.
// Engines with priv accesses won't fault on the priv PTE, so add a backup
// mechanism using an impossible memory address. This will trigger an
// interrupt starting with GM20x. On earlier GPUs the upper bits will
// silently be dropped.
//
// This address has to fit within 37 bits (max address width of vidmem) and
// be aligned to page_size.
NvU64 phys_addr = 0x1bad000000ULL;
NvU64 pte_bits = tree->hal->make_pte(UVM_APERTURE_VID, phys_addr, UVM_PROT_READ_ONLY, UVM_MMU_PTE_FLAGS_NONE);
return WRITE_HWCONST64(pte_bits, _MMU, PTE, PRIVILEGE, TRUE);
}
// Sparse mappings are not supported.
static NvU64 make_sparse_pte_maxwell_unsupported(void)
{
NvU64 pte_bits;
UVM_ASSERT_MSG(0, "Sparse mappings unsupported on pre-Pascal GPUs\n");
pte_bits = HWCONST64(_MMU, PTE, VALID, FALSE);
return pte_bits;
}
static uvm_mmu_mode_hal_t maxwell_64_mmu_mode_hal =
{
.make_pte = make_pte_maxwell,
.make_sked_reflected_pte = make_sked_reflected_pte_maxwell,
.make_sparse_pte = make_sparse_pte_maxwell_unsupported,
.unmapped_pte = unmapped_pte_maxwell,
.poisoned_pte = poisoned_pte_maxwell,
.make_pde = make_pde_maxwell,
.entry_size = entry_size_maxwell,
.index_bits = index_bits_maxwell_64,
.entries_per_index = entries_per_index_maxwell,
.entry_offset = entry_offset_maxwell,
.num_va_bits = num_va_bits_maxwell,
.allocation_size = allocation_size_maxwell_64,
.page_table_depth = page_table_depth_maxwell,
.page_sizes = page_sizes_maxwell_64
};
static uvm_mmu_mode_hal_t maxwell_128_mmu_mode_hal =
{
.make_pte = make_pte_maxwell,
.make_sked_reflected_pte = make_sked_reflected_pte_maxwell,
.make_sparse_pte = make_sparse_pte_maxwell_unsupported,
.unmapped_pte = unmapped_pte_maxwell,
.poisoned_pte = poisoned_pte_maxwell,
.make_pde = make_pde_maxwell,
.entry_size = entry_size_maxwell,
.index_bits = index_bits_maxwell_128,
.entries_per_index = entries_per_index_maxwell,
.entry_offset = entry_offset_maxwell,
.num_va_bits = num_va_bits_maxwell,
.allocation_size = allocation_size_maxwell_128,
.page_table_depth = page_table_depth_maxwell,
.page_sizes = page_sizes_maxwell_128
};
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_maxwell(NvU64 big_page_size)
{
UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
if (big_page_size == UVM_PAGE_SIZE_64K)
return &maxwell_64_mmu_mode_hal;
return &maxwell_128_mmu_mode_hal;
}
void uvm_hal_maxwell_mmu_enable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false, "mmu enable_prefetch_faults called on Maxwell GPU\n");
}
void uvm_hal_maxwell_mmu_disable_prefetch_faults_unsupported(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT_MSG(false, "mmu disable_prefetch_faults called on Maxwell GPU\n");
}
NvU16 uvm_hal_maxwell_mmu_client_id_to_utlb_id_unsupported(NvU16 client_id)
{
UVM_ASSERT(0);
return 0;
}

19
kernel-open/nvidia-uvm/uvm_mem.c
View File

@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2016-2024 NVIDIA Corporation
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -289,7 +289,7 @@ uvm_chunk_sizes_mask_t uvm_mem_kernel_chunk_sizes(uvm_gpu_t *gpu)
{
// Get the mmu mode hal directly as the internal address space tree has not
// been created yet.
uvm_mmu_mode_hal_t *hal = gpu->parent->arch_hal->mmu_mode_hal(gpu->big_page.internal_size);
uvm_mmu_mode_hal_t *hal = gpu->parent->arch_hal->mmu_mode_hal();
NvU64 page_sizes = hal->page_sizes();
return (uvm_chunk_sizes_mask_t)(page_sizes & UVM_CHUNK_SIZES_MASK);
@@ -305,10 +305,10 @@ static NvU64 mem_pick_chunk_size(uvm_mem_t *mem)
biggest_page_size = uvm_mmu_biggest_page_size_up_to(&mem->backing_gpu->address_space_tree, UVM_CHUNK_SIZE_MAX);
if (mem->size < mem->backing_gpu->big_page.internal_size)
if (mem->size < UVM_BIG_PAGE_SIZE)
chunk_size = UVM_PAGE_SIZE_4K;
else if (mem->size < biggest_page_size)
chunk_size = mem->backing_gpu->big_page.internal_size;
chunk_size = UVM_BIG_PAGE_SIZE;
else
chunk_size = biggest_page_size;
@@ -328,15 +328,8 @@ static NvU64 mem_pick_gpu_page_size(uvm_mem_t *mem, uvm_gpu_t *gpu, uvm_page_tre
return mem->chunk_size;
}
// For sysmem, check whether the GPU supports mapping it with large pages.
if (gpu->parent->can_map_sysmem_with_large_pages) {
// If it's supported, pick the largest page size not bigger than
// the chunk size.
return uvm_mmu_biggest_page_size_up_to(gpu_page_tree, mem->chunk_size);
}
// Otherwise just use 4K.
return UVM_PAGE_SIZE_4K;
// For sysmem, pick the largest page size not bigger than the chunk size.
return uvm_mmu_biggest_page_size_up_to(gpu_page_tree, mem->chunk_size);
}
static void mem_free_vidmem_chunks(uvm_mem_t *mem)

18
kernel-open/nvidia-uvm/uvm_migrate.c
View File

@@ -201,7 +201,6 @@ NV_STATUS uvm_va_block_migrate_locked(uvm_va_block_t *va_block,
uvm_migrate_mode_t mode,
uvm_tracker_t *out_tracker)
{
uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block);
uvm_va_block_context_t *va_block_context = service_context->block_context;
NV_STATUS status = NV_OK;
NV_STATUS tracker_status = NV_OK;
@@ -226,7 +225,7 @@ NV_STATUS uvm_va_block_migrate_locked(uvm_va_block_t *va_block,
uvm_page_mask_init_from_region(make_resident_mask, region, NULL);
if (uvm_va_policy_is_read_duplicate(policy, va_space)) {
if (uvm_va_policy_is_read_duplicate(policy)) {
if (uvm_page_mask_andnot(make_resident_mask, make_resident_mask, &va_block->discarded_pages)) {
status = uvm_va_block_make_resident_read_duplicate(va_block,
va_block_retry,
@@ -353,10 +352,9 @@ static bool migration_should_do_cpu_preunmap(uvm_va_space_t *va_space,
// read-duplication is enabled in the VA range. This is because, when migrating
// read-duplicated VA blocks, the source processor doesn't need to be unmapped
// (though it may need write access revoked).
static bool va_range_should_do_cpu_preunmap(const uvm_va_policy_t *policy,
uvm_va_space_t *va_space)
static bool va_range_should_do_cpu_preunmap(const uvm_va_policy_t *policy)
{
return !uvm_va_policy_is_read_duplicate(policy, va_space);
return !uvm_va_policy_is_read_duplicate(policy);
}
// Function that determines if the VA block to be migrated contains pages with
@@ -504,8 +502,7 @@ static NV_STATUS uvm_va_range_migrate(uvm_va_range_managed_t *managed_range,
NvU64 preunmap_range_start = start;
uvm_va_policy_t *policy = &managed_range->policy;
should_do_cpu_preunmap = should_do_cpu_preunmap &&
va_range_should_do_cpu_preunmap(policy, managed_range->va_range.va_space);
should_do_cpu_preunmap = should_do_cpu_preunmap && va_range_should_do_cpu_preunmap(policy);
// Divide migrations into groups of contiguous VA blocks. This is to trigger
// CPU unmaps for that region before the migration starts.
@@ -588,13 +585,6 @@ static NV_STATUS uvm_migrate_ranges(uvm_va_space_t *va_space,
service_context->block_context->make_resident.dest_nid))
skipped_migrate = true;
}
else if (uvm_processor_mask_test(&managed_range->uvm_lite_gpus, dest_id) &&
!uvm_va_policy_preferred_location_equal(policy, dest_id, NUMA_NO_NODE)) {
// Don't migrate to a non-faultable GPU that is in UVM-Lite mode,
// unless it's the preferred location
status = NV_ERR_INVALID_DEVICE;
break;
}
else {
status = uvm_va_range_migrate(managed_range,
service_context,

25
kernel-open/nvidia-uvm/uvm_mmu.c
View File

@@ -635,9 +635,9 @@ static void pde_clear(uvm_page_tree_t *tree,
pde_write(tree, dir, entry_index, false, push);
}
static uvm_chunk_sizes_mask_t allocation_sizes_for_big_page_size(uvm_parent_gpu_t *parent_gpu, NvU64 big_page_size)
static uvm_chunk_sizes_mask_t allocation_sizes(uvm_parent_gpu_t *parent_gpu)
{
uvm_mmu_mode_hal_t *hal = parent_gpu->arch_hal->mmu_mode_hal(big_page_size);
uvm_mmu_mode_hal_t *hal = parent_gpu->arch_hal->mmu_mode_hal();
unsigned long page_sizes, page_size_log2;
uvm_chunk_sizes_mask_t alloc_sizes;
@@ -659,9 +659,9 @@ static uvm_chunk_sizes_mask_t allocation_sizes_for_big_page_size(uvm_parent_gpu_
return alloc_sizes;
}
static NvU64 page_sizes_for_big_page_size(uvm_parent_gpu_t *parent_gpu, NvU64 big_page_size)
static NvU64 page_sizes(uvm_parent_gpu_t *parent_gpu)
{
uvm_mmu_mode_hal_t *hal = parent_gpu->arch_hal->mmu_mode_hal(big_page_size);
uvm_mmu_mode_hal_t *hal = parent_gpu->arch_hal->mmu_mode_hal();
if (hal != NULL)
return hal->page_sizes();
@@ -1136,7 +1136,6 @@ static void page_tree_set_location(uvm_page_tree_t *tree, uvm_aperture_t locatio
NV_STATUS uvm_page_tree_init(uvm_gpu_t *gpu,
uvm_gpu_va_space_t *gpu_va_space,
uvm_page_tree_type_t type,
NvU64 big_page_size,
uvm_aperture_t location,
uvm_page_tree_t *tree)
{
@@ -1148,15 +1147,12 @@ NV_STATUS uvm_page_tree_init(uvm_gpu_t *gpu,
memset(tree, 0, sizeof(*tree));
uvm_mutex_init(&tree->lock, UVM_LOCK_ORDER_PAGE_TREE);
tree->hal = gpu->parent->arch_hal->mmu_mode_hal(big_page_size);
tree->hal = gpu->parent->arch_hal->mmu_mode_hal();
UVM_ASSERT(tree->hal != NULL);
UVM_ASSERT(MAX_OPERATION_DEPTH >= tree->hal->page_table_depth(UVM_PAGE_SIZE_AGNOSTIC));
tree->gpu = gpu;
tree->type = type;
tree->gpu_va_space = gpu_va_space;
tree->big_page_size = big_page_size;
UVM_ASSERT(uvm_mmu_page_size_supported(tree, big_page_size));
page_tree_set_location(tree, location);
@@ -2446,9 +2442,7 @@ void uvm_mmu_destroy_peer_identity_mappings(uvm_gpu_t *gpu, uvm_gpu_t *peer)
void uvm_mmu_init_gpu_chunk_sizes(uvm_parent_gpu_t *parent_gpu)
{
uvm_chunk_sizes_mask_t sizes = page_sizes_for_big_page_size(parent_gpu, UVM_PAGE_SIZE_64K) |
page_sizes_for_big_page_size(parent_gpu, UVM_PAGE_SIZE_128K) |
PAGE_SIZE;
uvm_chunk_sizes_mask_t sizes = page_sizes(parent_gpu) | PAGE_SIZE;
// Although we may have to map PTEs smaller than PAGE_SIZE, user (managed)
// memory is never allocated with granularity smaller than PAGE_SIZE. Force
@@ -2461,8 +2455,7 @@ void uvm_mmu_init_gpu_chunk_sizes(uvm_parent_gpu_t *parent_gpu)
// the chunk size list.
parent_gpu->mmu_user_chunk_sizes &= UVM_CHUNK_SIZES_MASK;
parent_gpu->mmu_kernel_chunk_sizes = allocation_sizes_for_big_page_size(parent_gpu, UVM_PAGE_SIZE_64K) |
allocation_sizes_for_big_page_size(parent_gpu, UVM_PAGE_SIZE_128K);
parent_gpu->mmu_kernel_chunk_sizes = allocation_sizes(parent_gpu);
}
void uvm_mmu_init_gpu_peer_addresses(uvm_gpu_t *gpu)
@@ -3001,8 +2994,10 @@ NV_STATUS uvm_mmu_l2_invalidate(uvm_gpu_t *gpu, uvm_aperture_t aperture)
UVM_CHANNEL_TYPE_MEMOPS,
&push,
"L2 cache invalidate");
if (status != NV_OK)
if (status != NV_OK) {
UVM_ERR_PRINT("L2 cache invalidation: Failed to begin push, status: %s\n", nvstatusToString(status));
return status;
}
gpu->parent->host_hal->l2_invalidate(&push, aperture);

4
kernel-open/nvidia-uvm/uvm_mmu.h
View File

@@ -212,7 +212,7 @@ struct uvm_mmu_mode_hal_struct
// This is an optimization which reduces TLB pressure, reduces the number of
// TLB invalidates we must issue, and means we don't have to initialize the
// 4k PTEs which are covered by big PTEs since the MMU will never read them.
NvU64 (*unmapped_pte)(NvU64 page_size);
NvU64 (*unmapped_pte)(void);
// Bit pattern used for debug purposes to clobber PTEs which ought to be
// unused. In practice this will generate a PRIV violation or a physical
@@ -279,7 +279,6 @@ struct uvm_page_tree_struct
uvm_page_directory_t *root;
uvm_mmu_mode_hal_t *hal;
uvm_page_tree_type_t type;
NvU64 big_page_size;
// Pointer to the GPU VA space containing the page tree.
// This pointer is set only for page trees of type
@@ -379,7 +378,6 @@ void uvm_mmu_init_gpu_peer_addresses(uvm_gpu_t *gpu);
NV_STATUS uvm_page_tree_init(uvm_gpu_t *gpu,
uvm_gpu_va_space_t *gpu_va_space,
uvm_page_tree_type_t type,
NvU64 big_page_size,
uvm_aperture_t location,
uvm_page_tree_t *tree_out);

777
kernel-open/nvidia-uvm/uvm_page_tree_test.c
View File

File diff suppressed because it is too large Load Diff

114
kernel-open/nvidia-uvm/uvm_pascal.c
View File

@@ -1,114 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_hal.h"
#include "uvm_gpu.h"
#include "uvm_mem.h"
#include "uvm_pascal_fault_buffer.h"
static unsigned uvm_force_prefetch_fault_support = 0;
module_param(uvm_force_prefetch_fault_support, uint, S_IRUGO);
void uvm_hal_pascal_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
{
parent_gpu->tlb_batch.va_invalidate_supported = true;
parent_gpu->tlb_batch.va_range_invalidate_supported = false;
// TODO: Bug 1767241: Run benchmarks to figure out a good number
parent_gpu->tlb_batch.max_pages = 32;
parent_gpu->utlb_per_gpc_count = uvm_pascal_get_utlbs_per_gpc(parent_gpu);
parent_gpu->fault_buffer.replayable.utlb_count = parent_gpu->rm_info.gpcCount * parent_gpu->utlb_per_gpc_count;
{
uvm_fault_buffer_entry_t *dummy;
UVM_ASSERT(parent_gpu->fault_buffer.replayable.utlb_count <= (1 << (sizeof(dummy->fault_source.utlb_id) * 8)));
}
// A single top level PDE on Pascal covers 128 TB and that's the minimum
// size that can be used.
parent_gpu->rm_va_base = 0;
parent_gpu->rm_va_size = 128 * UVM_SIZE_1TB;
parent_gpu->peer_va_base = parent_gpu->rm_va_base + parent_gpu->rm_va_size;
parent_gpu->peer_va_size = NV_MAX_DEVICES * UVM_PEER_IDENTITY_VA_SIZE;
parent_gpu->uvm_mem_va_base = 384 * UVM_SIZE_1TB;
parent_gpu->uvm_mem_va_size = UVM_MEM_VA_SIZE;
parent_gpu->ce_phys_vidmem_write_supported = true;
parent_gpu->peer_copy_mode = UVM_GPU_PEER_COPY_MODE_VIRTUAL;
// Not all units on Pascal support 49-bit addressing, including those which
// access channel buffers.
parent_gpu->max_channel_va = 1ULL << 40;
parent_gpu->max_host_va = 1ULL << 40;
// Pascal can map sysmem with any page size
parent_gpu->can_map_sysmem_with_large_pages = true;
// Prefetch faults are disabled by default in Pascal
parent_gpu->prefetch_fault_supported = uvm_force_prefetch_fault_support != 0;
// Pascal and Volta require post-invalidate membars to flush out HSHUB. See
// bug 1975028. GP10x chips do not have HSHUB, so they don't need any.
UVM_ASSERT(parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GP100);
if (parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GP100 ||
parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GP000) {
parent_gpu->num_hshub_tlb_invalidate_membars = 2;
}
else
parent_gpu->num_hshub_tlb_invalidate_membars = 0;
// Pascal cannot place GPFIFO in vidmem
parent_gpu->gpfifo_in_vidmem_supported = false;
parent_gpu->replayable_faults_supported = true;
parent_gpu->non_replayable_faults_supported = false;
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->fault_cancel_va_supported = false;
parent_gpu->scoped_atomics_supported = false;
parent_gpu->sparse_mappings_supported = true;
parent_gpu->map_remap_larger_page_promotion = false;
parent_gpu->smc.supported = false;
parent_gpu->plc_supported = false;
parent_gpu->ats.no_ats_range_required = false;
parent_gpu->ats.gmmu_pt_depth0_init_required = false;
parent_gpu->conf_computing.per_channel_key_rotation = false;
}

137
kernel-open/nvidia-uvm/uvm_pascal_ce.c
View File

@@ -1,137 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2020 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_hal.h"
#include "uvm_push.h"
#include "clc0b5.h"
void uvm_hal_pascal_ce_offset_out(uvm_push_t *push, NvU64 offset_out)
{
NV_PUSH_2U(C0B5, OFFSET_OUT_UPPER, HWVALUE(C0B5, OFFSET_OUT_UPPER, UPPER, NvOffset_HI32(offset_out)),
OFFSET_OUT_LOWER, HWVALUE(C0B5, OFFSET_OUT_LOWER, VALUE, NvOffset_LO32(offset_out)));
}
void uvm_hal_pascal_ce_offset_in_out(uvm_push_t *push, NvU64 offset_in, NvU64 offset_out)
{
NV_PUSH_4U(C0B5, OFFSET_IN_UPPER, HWVALUE(C0B5, OFFSET_IN_UPPER, UPPER, NvOffset_HI32(offset_in)),
OFFSET_IN_LOWER, HWVALUE(C0B5, OFFSET_IN_LOWER, VALUE, NvOffset_LO32(offset_in)),
OFFSET_OUT_UPPER, HWVALUE(C0B5, OFFSET_OUT_UPPER, UPPER, NvOffset_HI32(offset_out)),
OFFSET_OUT_LOWER, HWVALUE(C0B5, OFFSET_OUT_LOWER, VALUE, NvOffset_LO32(offset_out)));
}
void uvm_hal_pascal_ce_semaphore_release(uvm_push_t *push, NvU64 gpu_va, NvU32 payload)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
NvU32 flush_value;
NvU32 launch_dma_plc_mode;
bool use_flush;
UVM_ASSERT_MSG(gpu->parent->ce_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(gpu));
use_flush = uvm_hal_membar_before_semaphore(push);
if (use_flush)
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE);
else
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, FALSE);
NV_PUSH_3U(C0B5, SET_SEMAPHORE_A, HWVALUE(C0B5, SET_SEMAPHORE_A, UPPER, NvOffset_HI32(gpu_va)),
SET_SEMAPHORE_B, HWVALUE(C0B5, SET_SEMAPHORE_B, LOWER, NvOffset_LO32(gpu_va)),
SET_SEMAPHORE_PAYLOAD, payload);
launch_dma_plc_mode = gpu->parent->ce_hal->plc_mode();
NV_PUSH_1U(C0B5, LAUNCH_DMA, flush_value |
HWCONST(C0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NONE) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_TYPE, RELEASE_ONE_WORD_SEMAPHORE) |
launch_dma_plc_mode);
}
void uvm_hal_pascal_ce_semaphore_reduction_inc(uvm_push_t *push, NvU64 gpu_va, NvU32 payload)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
NvU32 flush_value;
NvU32 launch_dma_plc_mode;
bool use_flush;
UVM_ASSERT_MSG(gpu->parent->ce_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(gpu));
use_flush = uvm_hal_membar_before_semaphore(push);
if (use_flush)
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE);
else
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, FALSE);
NV_PUSH_3U(C0B5, SET_SEMAPHORE_A, HWVALUE(C0B5, SET_SEMAPHORE_A, UPPER, NvOffset_HI32(gpu_va)),
SET_SEMAPHORE_B, HWVALUE(C0B5, SET_SEMAPHORE_B, LOWER, NvOffset_LO32(gpu_va)),
SET_SEMAPHORE_PAYLOAD, payload);
launch_dma_plc_mode = gpu->parent->ce_hal->plc_mode();
NV_PUSH_1U(C0B5, LAUNCH_DMA, flush_value |
HWCONST(C0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NONE) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_TYPE, RELEASE_ONE_WORD_SEMAPHORE) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_REDUCTION, INC) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_REDUCTION_SIGN, UNSIGNED) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_REDUCTION_ENABLE, TRUE) |
launch_dma_plc_mode);
}
void uvm_hal_pascal_ce_semaphore_timestamp(uvm_push_t *push, NvU64 gpu_va)
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
NvU32 flush_value;
NvU32 launch_dma_plc_mode;
bool use_flush;
UVM_ASSERT_MSG(gpu->parent->ce_hal->semaphore_target_is_valid(push, gpu_va),
"Semaphore target validation failed in channel %s, GPU %s.\n",
push->channel->name,
uvm_gpu_name(gpu));
use_flush = uvm_hal_membar_before_semaphore(push);
if (use_flush)
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE);
else
flush_value = HWCONST(C0B5, LAUNCH_DMA, FLUSH_ENABLE, FALSE);
NV_PUSH_3U(C0B5, SET_SEMAPHORE_A, HWVALUE(C0B5, SET_SEMAPHORE_A, UPPER, NvOffset_HI32(gpu_va)),
SET_SEMAPHORE_B, HWVALUE(C0B5, SET_SEMAPHORE_B, LOWER, NvOffset_LO32(gpu_va)),
SET_SEMAPHORE_PAYLOAD, 0xdeadbeef);
launch_dma_plc_mode = gpu->parent->ce_hal->plc_mode();
NV_PUSH_1U(C0B5, LAUNCH_DMA, flush_value |
HWCONST(C0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NONE) |
HWCONST(C0B5, LAUNCH_DMA, SEMAPHORE_TYPE, RELEASE_FOUR_WORD_SEMAPHORE) |
launch_dma_plc_mode);
}

329
kernel-open/nvidia-uvm/uvm_pascal_fault_buffer.c
View File

@@ -1,329 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_linux.h"
#include "uvm_global.h"
#include "uvm_gpu.h"
#include "uvm_hal.h"
#include "uvm_push.h"
#include "hwref/pascal/gp100/dev_fault.h"
#include "clb069.h"
#include "uvm_pascal_fault_buffer.h"
typedef struct {
NvU8 bufferEntry[NVB069_FAULT_BUF_SIZE];
} fault_buffer_entry_b069_t;
void uvm_hal_pascal_clear_replayable_faults(uvm_parent_gpu_t *parent_gpu, NvU32 get)
{
// No-op, this function is only used by pulse-based interrupt GPUs.
}
void uvm_hal_pascal_enable_replayable_faults(uvm_parent_gpu_t *parent_gpu)
{
volatile NvU32 *reg;
NvU32 mask;
reg = parent_gpu->fault_buffer.rm_info.replayable.pPmcIntrEnSet;
mask = parent_gpu->fault_buffer.rm_info.replayable.replayableFaultMask;
UVM_GPU_WRITE_ONCE(*reg, mask);
}
void uvm_hal_pascal_disable_replayable_faults(uvm_parent_gpu_t *parent_gpu)
{
volatile NvU32 *reg;
NvU32 mask;
reg = parent_gpu->fault_buffer.rm_info.replayable.pPmcIntrEnClear;
mask = parent_gpu->fault_buffer.rm_info.replayable.replayableFaultMask;
UVM_GPU_WRITE_ONCE(*reg, mask);
}
NvU32 uvm_hal_pascal_fault_buffer_read_put(uvm_parent_gpu_t *parent_gpu)
{
NvU32 put = UVM_GPU_READ_ONCE(*parent_gpu->fault_buffer.rm_info.replayable.pFaultBufferPut);
UVM_ASSERT(put < parent_gpu->fault_buffer.replayable.max_faults);
return put;
}
NvU32 uvm_hal_pascal_fault_buffer_read_get(uvm_parent_gpu_t *parent_gpu)
{
NvU32 get = UVM_GPU_READ_ONCE(*parent_gpu->fault_buffer.rm_info.replayable.pFaultBufferGet);
UVM_ASSERT(get < parent_gpu->fault_buffer.replayable.max_faults);
return get;
}
void uvm_hal_pascal_fault_buffer_write_get(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UVM_ASSERT(index < parent_gpu->fault_buffer.replayable.max_faults);
UVM_GPU_WRITE_ONCE(*parent_gpu->fault_buffer.rm_info.replayable.pFaultBufferGet, index);
}
static uvm_fault_access_type_t get_fault_access_type(const NvU32 *fault_entry)
{
NvU32 hw_access_type_value = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, ACCESS_TYPE);
switch (hw_access_type_value)
{
case NV_PFAULT_ACCESS_TYPE_READ:
return UVM_FAULT_ACCESS_TYPE_READ;
case NV_PFAULT_ACCESS_TYPE_WRITE:
return UVM_FAULT_ACCESS_TYPE_WRITE;
case NV_PFAULT_ACCESS_TYPE_ATOMIC:
return UVM_FAULT_ACCESS_TYPE_ATOMIC_STRONG;
case NV_PFAULT_ACCESS_TYPE_PREFETCH:
return UVM_FAULT_ACCESS_TYPE_PREFETCH;
}
UVM_ASSERT_MSG(false, "Invalid fault access type value: %d\n", hw_access_type_value);
return UVM_FAULT_ACCESS_TYPE_COUNT;
}
uvm_fault_type_t uvm_hal_pascal_fault_buffer_get_fault_type(const NvU32 *fault_entry)
{
NvU32 hw_fault_type_value = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, FAULT_TYPE);
switch (hw_fault_type_value)
{
case NV_PFAULT_FAULT_TYPE_PDE:
return UVM_FAULT_TYPE_INVALID_PDE;
case NV_PFAULT_FAULT_TYPE_PTE:
return UVM_FAULT_TYPE_INVALID_PTE;
case NV_PFAULT_FAULT_TYPE_RO_VIOLATION:
return UVM_FAULT_TYPE_WRITE;
case NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION:
return UVM_FAULT_TYPE_ATOMIC;
case NV_PFAULT_FAULT_TYPE_PDE_SIZE:
return UVM_FAULT_TYPE_PDE_SIZE;
case NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION:
return UVM_FAULT_TYPE_VA_LIMIT_VIOLATION;
case NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK:
return UVM_FAULT_TYPE_UNBOUND_INST_BLOCK;
case NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION:
return UVM_FAULT_TYPE_PRIV_VIOLATION;
case NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION:
return UVM_FAULT_TYPE_PITCH_MASK_VIOLATION;
case NV_PFAULT_FAULT_TYPE_WORK_CREATION:
return UVM_FAULT_TYPE_WORK_CREATION;
case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE:
return UVM_FAULT_TYPE_UNSUPPORTED_APERTURE;
case NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE:
return UVM_FAULT_TYPE_COMPRESSION_FAILURE;
case NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND:
return UVM_FAULT_TYPE_UNSUPPORTED_KIND;
case NV_PFAULT_FAULT_TYPE_REGION_VIOLATION:
return UVM_FAULT_TYPE_REGION_VIOLATION;
case NV_PFAULT_FAULT_TYPE_POISONED:
return UVM_FAULT_TYPE_POISONED;
}
UVM_ASSERT_MSG(false, "Invalid fault type value: %d\n", hw_fault_type_value);
return UVM_FAULT_TYPE_COUNT;
}
static uvm_fault_client_type_t get_fault_client_type(const NvU32 *fault_entry)
{
NvU32 hw_client_type_value = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, MMU_CLIENT_TYPE);
switch (hw_client_type_value)
{
case NV_PFAULT_MMU_CLIENT_TYPE_GPC:
return UVM_FAULT_CLIENT_TYPE_GPC;
case NV_PFAULT_MMU_CLIENT_TYPE_HUB:
return UVM_FAULT_CLIENT_TYPE_HUB;
}
UVM_ASSERT_MSG(false, "Invalid mmu client type value: %d\n", hw_client_type_value);
return UVM_FAULT_CLIENT_TYPE_COUNT;
}
static uvm_aperture_t get_fault_inst_aperture(NvU32 *fault_entry)
{
NvU32 hw_aperture_value = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, INST_APERTURE);
switch (hw_aperture_value)
{
case NVB069_FAULT_BUF_ENTRY_INST_APERTURE_VID_MEM:
return UVM_APERTURE_VID;
case NVB069_FAULT_BUF_ENTRY_INST_APERTURE_SYS_MEM_NONCOHERENT:
// UVM does not use SYS_NON_COHERENT aperture for sysmem addresses
// but RM might. The value returned here denotes sysmem location
// to UVM so it's safe to return UVM_APERTURE_SYS even for
// SYS_MEM_NONCOHERENT.
/* falls through */
case NVB069_FAULT_BUF_ENTRY_INST_APERTURE_SYS_MEM_COHERENT:
return UVM_APERTURE_SYS;
}
UVM_ASSERT_MSG(false, "Invalid inst aperture value: %d\n", hw_aperture_value);
return UVM_APERTURE_MAX;
}
static NvU32 *get_fault_buffer_entry(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
fault_buffer_entry_b069_t *buffer_start;
NvU32 *fault_entry;
UVM_ASSERT(index < parent_gpu->fault_buffer.replayable.max_faults);
buffer_start = (fault_buffer_entry_b069_t *)parent_gpu->fault_buffer.rm_info.replayable.bufferAddress;
fault_entry = (NvU32 *)&buffer_start[index];
return fault_entry;
}
// When Confidential Computing is enabled, fault entries are encrypted. Each
// fault has (unencrypted) metadata containing the authentication tag, and a
// valid bit that allows UVM to check if an encrypted fault is valid, without
// having to decrypt it first.
static UvmFaultMetadataPacket *get_fault_buffer_entry_metadata(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
UvmFaultMetadataPacket *fault_entry_metadata;
UVM_ASSERT(index < parent_gpu->fault_buffer.replayable.max_faults);
UVM_ASSERT(g_uvm_global.conf_computing_enabled);
fault_entry_metadata = parent_gpu->fault_buffer.rm_info.replayable.bufferMetadata;
UVM_ASSERT(fault_entry_metadata != NULL);
return fault_entry_metadata + index;
}
NV_STATUS uvm_hal_pascal_fault_buffer_parse_replayable_entry(uvm_parent_gpu_t *parent_gpu,
NvU32 index,
uvm_fault_buffer_entry_t *buffer_entry)
{
NvU32 *fault_entry;
NvU64 addr_hi, addr_lo;
NvU64 timestamp_hi, timestamp_lo;
NvU16 gpc_utlb_id;
NvU32 utlb_id;
BUILD_BUG_ON(NVB069_FAULT_BUF_SIZE > UVM_GPU_MMU_MAX_FAULT_PACKET_SIZE);
// Valid bit must be set before this function is called
UVM_ASSERT(parent_gpu->fault_buffer_hal->entry_is_valid(parent_gpu, index));
fault_entry = get_fault_buffer_entry(parent_gpu, index);
addr_hi = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, INST_HI);
addr_lo = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, INST_LO);
buffer_entry->instance_ptr.address = addr_lo + (addr_hi << HWSIZE_MW(B069, FAULT_BUF_ENTRY, INST_LO));
// HW value contains the 4K page number. Shift to build the full address
buffer_entry->instance_ptr.address <<= 12;
buffer_entry->instance_ptr.aperture = get_fault_inst_aperture(fault_entry);
addr_hi = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, ADDR_HI);
addr_lo = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, ADDR_LO);
buffer_entry->fault_address = addr_lo + (addr_hi << HWSIZE_MW(B069, FAULT_BUF_ENTRY, ADDR_LO));
buffer_entry->fault_address = uvm_parent_gpu_canonical_address(parent_gpu, buffer_entry->fault_address);
timestamp_hi = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, TIMESTAMP_HI);
timestamp_lo = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, TIMESTAMP_LO);
buffer_entry->timestamp = timestamp_lo + (timestamp_hi << HWSIZE_MW(B069, FAULT_BUF_ENTRY, TIMESTAMP_LO));
buffer_entry->fault_type = parent_gpu->fault_buffer_hal->get_fault_type(fault_entry);
buffer_entry->fault_access_type = get_fault_access_type(fault_entry);
buffer_entry->fault_source.client_type = get_fault_client_type(fault_entry);
if (buffer_entry->fault_source.client_type == UVM_FAULT_CLIENT_TYPE_HUB)
UVM_ASSERT_MSG(false, "Invalid client type: HUB\n");
buffer_entry->fault_source.client_id = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, CLIENT);
BUILD_BUG_ON(sizeof(buffer_entry->fault_source.client_id) * 8 < DRF_SIZE_MW(NVB069_FAULT_BUF_ENTRY_CLIENT));
buffer_entry->fault_source.gpc_id = READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, GPC_ID);
BUILD_BUG_ON(sizeof(buffer_entry->fault_source.gpc_id) * 8 < DRF_SIZE_MW(NVB069_FAULT_BUF_ENTRY_GPC_ID));
gpc_utlb_id = parent_gpu->arch_hal->mmu_client_id_to_utlb_id(buffer_entry->fault_source.client_id);
UVM_ASSERT(gpc_utlb_id < parent_gpu->utlb_per_gpc_count);
// Compute global uTLB id
utlb_id = buffer_entry->fault_source.gpc_id * parent_gpu->utlb_per_gpc_count + gpc_utlb_id;
UVM_ASSERT(utlb_id < parent_gpu->fault_buffer.replayable.utlb_count);
buffer_entry->fault_source.utlb_id = utlb_id;
buffer_entry->is_replayable = true;
buffer_entry->is_virtual = true;
buffer_entry->in_protected_mode = false;
buffer_entry->fault_source.mmu_engine_type = UVM_MMU_ENGINE_TYPE_GRAPHICS;
buffer_entry->fault_source.mmu_engine_id = NV_PFAULT_MMU_ENG_ID_GRAPHICS;
buffer_entry->fault_source.ve_id = 0;
// Automatically clear valid bit for the entry in the fault buffer
uvm_hal_pascal_fault_buffer_entry_clear_valid(parent_gpu, index);
return NV_OK;
}
bool uvm_hal_pascal_fault_buffer_entry_is_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
if (g_uvm_global.conf_computing_enabled) {
// Use the valid bit present in the encryption metadata, which is
// unencrypted, instead of the valid bit present in the (encrypted)
// fault itself.
UvmFaultMetadataPacket *fault_entry_metadata = get_fault_buffer_entry_metadata(parent_gpu, index);
return fault_entry_metadata->valid;
}
else {
NvU32 *fault_entry = get_fault_buffer_entry(parent_gpu, index);
return READ_HWVALUE_MW(fault_entry, B069, FAULT_BUF_ENTRY, VALID);
}
}
void uvm_hal_pascal_fault_buffer_entry_clear_valid(uvm_parent_gpu_t *parent_gpu, NvU32 index)
{
if (g_uvm_global.conf_computing_enabled) {
// Use the valid bit present in the encryption metadata, which is
// unencrypted, instead of the valid bit present in the (encrypted)
// fault itself.
UvmFaultMetadataPacket *fault_entry_metadata = get_fault_buffer_entry_metadata(parent_gpu, index);
fault_entry_metadata->valid = false;
}
else {
NvU32 *fault_entry = get_fault_buffer_entry(parent_gpu, index);
WRITE_HWCONST_MW(fault_entry, B069, FAULT_BUF_ENTRY, VALID, FALSE);
}
}
NvU32 uvm_hal_pascal_fault_buffer_entry_size(uvm_parent_gpu_t *parent_gpu)
{
return NVB069_FAULT_BUF_SIZE;
}

56
kernel-open/nvidia-uvm/uvm_pascal_fault_buffer.h
View File

@@ -1,56 +0,0 @@
/*******************************************************************************
Copyright (c) 2016-2019 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#ifndef __UVM_HAL_PASCAL_FAULT_BUFFER_H__
#define __UVM_HAL_PASCAL_FAULT_BUFFER_H__
#include "nvtypes.h"
#include "uvm_common.h"
#include "uvm_gpu.h"
// There are up to 5 TPCs per GPC in Pascal, and there is 1 LTP uTLB per TPC. Besides, there is one RGG uTLB per GPC.
// Each TPC has a number of clients that can make requests to its uTLB: 1xTPCCS, 1xPE, 2xT1. The client ids are local
// to their GPC and the id mapping is linear across TPCs:
// TPC_n has TPCCS_n, PE_n, T1_p, and T1_q, where p=2*n and q=p+1.
//
// NV_PFAULT_CLIENT_GPC_LTP_UTLB_n and NV_PFAULT_CLIENT_GPC_RGG_UTLB enums can be ignored. These will never be reported
// in a fault message, and should never be used in an invalidate. Therefore, we define our own values.
typedef enum {
UVM_PASCAL_GPC_UTLB_ID_RGG = 0,
UVM_PASCAL_GPC_UTLB_ID_LTP0 = 1,
UVM_PASCAL_GPC_UTLB_ID_LTP1 = 2,
UVM_PASCAL_GPC_UTLB_ID_LTP2 = 3,
UVM_PASCAL_GPC_UTLB_ID_LTP3 = 4,
UVM_PASCAL_GPC_UTLB_ID_LTP4 = 5,
UVM_PASCAL_GPC_UTLB_COUNT,
} uvm_pascal_gpc_utlb_id_t;
static NvU32 uvm_pascal_get_utlbs_per_gpc(uvm_parent_gpu_t *parent_gpu)
{
NvU32 utlbs = parent_gpu->rm_info.maxTpcPerGpcCount + 1;
UVM_ASSERT(utlbs <= UVM_PASCAL_GPC_UTLB_COUNT);
return utlbs;
}
#endif

364
kernel-open/nvidia-uvm/uvm_pascal_host.c
View File

@@ -1,364 +0,0 @@
/*******************************************************************************
Copyright (c) 2015-2021 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_linux.h"
#include "uvm_hal_types.h"
#include "uvm_hal.h"
#include "uvm_push.h"
#include "uvm_channel.h"
#include "clc06f.h"
#include "clc076.h"
void uvm_hal_pascal_host_membar_sys(uvm_push_t *push)
{
NV_PUSH_4U(C06F, MEM_OP_A, 0,
MEM_OP_B, 0,
MEM_OP_C, HWCONST(C06F, MEM_OP_C, MEMBAR_TYPE, SYS_MEMBAR),
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MEMBAR));
}
void uvm_hal_pascal_host_membar_gpu(uvm_push_t *push)
{
NV_PUSH_4U(C06F, MEM_OP_A, 0,
MEM_OP_B, 0,
MEM_OP_C, HWCONST(C06F, MEM_OP_C, MEMBAR_TYPE, MEMBAR),
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MEMBAR));
}
void uvm_hal_pascal_host_tlb_invalidate_all(uvm_push_t *push, uvm_gpu_phys_address_t pdb, NvU32 depth, uvm_membar_t membar)
{
NvU32 aperture_value;
NvU32 page_table_level;
NvU32 pdb_lo;
NvU32 pdb_hi;
NvU32 ack_value = 0;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
if (pdb.aperture == UVM_APERTURE_VID)
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
else
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
pdb_lo = pdb.address & HWMASK(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
pdb_hi = pdb.address >> HWSIZE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
// PDE3 is the highest level on Pascal, see the comment in uvm_pascal_mmu.c for details.
UVM_ASSERT_MSG(depth < NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
page_table_level = NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
if (membar != UVM_MEMBAR_NONE) {
// If a GPU or SYS membar is needed, ACK_TYPE needs to be set to
// GLOBALLY to make sure all the pending accesses can be picked up by
// the membar.
ack_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
}
NV_PUSH_4U(C06F, MEM_OP_A, HWCONST(C06F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS),
MEM_OP_B, 0,
MEM_OP_C, HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
aperture_value |
ack_value,
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
HWVALUE(C06F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
uvm_hal_tlb_invalidate_membar(push, membar);
}
void uvm_hal_pascal_host_tlb_invalidate_va(uvm_push_t *push,
uvm_gpu_phys_address_t pdb,
NvU32 depth,
NvU64 base,
NvU64 size,
NvU64 page_size,
uvm_membar_t membar)
{
NvU32 aperture_value;
NvU32 page_table_level;
NvU32 pdb_lo;
NvU32 pdb_hi;
NvU32 ack_value = 0;
NvU64 va;
NvU32 va_lo;
NvU32 va_hi;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
if (pdb.aperture == UVM_APERTURE_VID)
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
else
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
pdb_lo = pdb.address & HWMASK(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
pdb_hi = pdb.address >> HWSIZE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
// PDE3 is the highest level on Pascal, see the comment in uvm_pascal_mmu.c for details.
UVM_ASSERT_MSG(depth < NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3, "depth %u", depth);
page_table_level = NVC06F_MEM_OP_C_TLB_INVALIDATE_PAGE_TABLE_LEVEL_UP_TO_PDE3 - depth;
if (membar != UVM_MEMBAR_NONE) {
// If a GPU or SYS membar is needed, ACK_TYPE needs to be set to
// GLOBALLY to make sure all the pending accesses can be picked up by
// the membar.
ack_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
}
UVM_ASSERT_MSG(IS_ALIGNED(page_size, 1 << 12), "page_size 0x%llx\n", page_size);
UVM_ASSERT_MSG(IS_ALIGNED(base, page_size), "base 0x%llx page_size 0x%llx\n", base, page_size);
UVM_ASSERT_MSG(IS_ALIGNED(size, page_size), "size 0x%llx page_size 0x%llx\n", size, page_size);
UVM_ASSERT_MSG(size > 0, "size 0x%llx\n", size);
base >>= 12;
size >>= 12;
page_size >>= 12;
for (va = base; va < base + size; va += page_size) {
va_lo = va & HWMASK(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
va_hi = va >> HWSIZE(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
NV_PUSH_4U(C06F, MEM_OP_A, HWCONST(C06F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
HWVALUE(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
MEM_OP_B, HWVALUE(C06F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
MEM_OP_C, HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
aperture_value |
ack_value,
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
HWVALUE(C06F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
}
uvm_hal_tlb_invalidate_membar(push, membar);
}
void uvm_hal_pascal_host_tlb_invalidate_test(uvm_push_t *push, uvm_gpu_phys_address_t pdb,
UVM_TEST_INVALIDATE_TLB_PARAMS *params)
{
NvU32 ack_value = 0;
NvU32 invalidate_gpc_value = 0;
NvU32 aperture_value = 0;
NvU32 pdb_lo = 0;
NvU32 pdb_hi = 0;
NvU32 page_table_level = 0;
uvm_membar_t membar;
UVM_ASSERT_MSG(pdb.aperture == UVM_APERTURE_VID || pdb.aperture == UVM_APERTURE_SYS, "aperture: %u", pdb.aperture);
if (pdb.aperture == UVM_APERTURE_VID)
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
else
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
pdb_lo = pdb.address & HWMASK(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
pdb_hi = pdb.address >> HWSIZE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
if (params->page_table_level != UvmInvalidatePageTableLevelAll) {
// PDE3 is the highest level on Pascal, see the comment in
// uvm_pascal_mmu.c for details.
page_table_level = min((NvU32)UvmInvalidatePageTableLevelPde3, params->page_table_level) - 1;
}
if (params->membar != UvmInvalidateTlbMemBarNone) {
// If a GPU or SYS membar is needed, ack_value needs to be set to
// GLOBALLY to make sure all the pending accesses can be picked up by
// the membar.
ack_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_ACK_TYPE, GLOBALLY);
}
if (params->disable_gpc_invalidate)
invalidate_gpc_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_GPC, DISABLE);
else
invalidate_gpc_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE);
if (params->target_va_mode == UvmTargetVaModeTargeted) {
NvU64 va = params->va >> 12;
NvU32 va_lo = va & HWMASK(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
NvU32 va_hi = va >> HWSIZE(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO);
NV_PUSH_4U(C06F, MEM_OP_A, HWCONST(C06F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
HWVALUE(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
MEM_OP_B, HWVALUE(C06F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
MEM_OP_C, HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
invalidate_gpc_value |
aperture_value |
ack_value,
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
HWVALUE(C06F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
}
else {
NV_PUSH_4U(C06F, MEM_OP_A, HWCONST(C06F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS),
MEM_OP_B, 0,
MEM_OP_C, HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, NONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, page_table_level) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
invalidate_gpc_value |
aperture_value |
ack_value,
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE) |
HWVALUE(C06F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
}
if (params->membar == UvmInvalidateTlbMemBarSys)
membar = UVM_MEMBAR_SYS;
else if (params->membar == UvmInvalidateTlbMemBarLocal)
membar = UVM_MEMBAR_GPU;
else
membar = UVM_MEMBAR_NONE;
uvm_hal_tlb_invalidate_membar(push, membar);
}
void uvm_hal_pascal_replay_faults(uvm_push_t *push, uvm_fault_replay_type_t type)
{
NvU32 aperture_value;
NvU32 replay_value = 0;
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
uvm_gpu_phys_address_t pdb;
NvU32 va_lo = 0;
NvU32 va_hi = 0;
NvU32 pdb_lo;
NvU32 pdb_hi;
// MMU will not forward the replay to the uTLBs if the PDB is not in the MMU PDB_ID cache. If
// we have stale entries filling the fault buffer and a context switch has happened, the new
// context may have faulted without writing its entries into the buffer. To force a replay
// regardless of which faults happen to be in the uTLB replay lists, we use the PDB of the
// channel used to push the replay, which is guaranteed to be in the cache as it is required
// to be resident for the channel to push the method. In order to minimize the performance hit
// of the invalidation, we just invalidate PTEs for address 0x0.
UVM_ASSERT_MSG(type == UVM_FAULT_REPLAY_TYPE_START || type == UVM_FAULT_REPLAY_TYPE_START_ACK_ALL,
"replay_type: %u\n", type);
pdb = uvm_page_tree_pdb_address(&gpu->address_space_tree);
if (pdb.aperture == UVM_APERTURE_VID)
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, VID_MEM);
else
aperture_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_APERTURE, SYS_MEM_COHERENT);
UVM_ASSERT_MSG(IS_ALIGNED(pdb.address, 1 << 12), "pdb 0x%llx\n", pdb.address);
pdb.address >>= 12;
pdb_lo = pdb.address & HWMASK(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
pdb_hi = pdb.address >> HWSIZE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO);
if (type == UVM_FAULT_REPLAY_TYPE_START)
replay_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, START);
else if (type == UVM_FAULT_REPLAY_TYPE_START_ACK_ALL)
replay_value = HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_REPLAY, START_ACK_ALL);
NV_PUSH_4U(C06F, MEM_OP_A, HWCONST(C06F, MEM_OP_A, TLB_INVALIDATE_SYSMEMBAR, DIS) |
HWVALUE(C06F, MEM_OP_A, TLB_INVALIDATE_TARGET_ADDR_LO, va_lo),
MEM_OP_B, HWVALUE(C06F, MEM_OP_B, TLB_INVALIDATE_TARGET_ADDR_HI, va_hi),
MEM_OP_C, HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PDB, ONE) |
HWVALUE(C06F, MEM_OP_C, TLB_INVALIDATE_PDB_ADDR_LO, pdb_lo) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_GPC, ENABLE) |
HWCONST(C06F, MEM_OP_C, TLB_INVALIDATE_PAGE_TABLE_LEVEL, PTE_ONLY) |
aperture_value |
replay_value,
MEM_OP_D, HWCONST(C06F, MEM_OP_D, OPERATION, MMU_TLB_INVALIDATE_TARGETED) |
HWVALUE(C06F, MEM_OP_D, TLB_INVALIDATE_PDB_ADDR_HI, pdb_hi));
}
static NvU32 instance_ptr_aperture_type_to_hw_value(uvm_aperture_t aperture)
{
switch (aperture)
{
case UVM_APERTURE_SYS:
return HWCONST(C076, FAULT_CANCEL_A, INST_APERTURE, SYS_MEM_COHERENT);
case UVM_APERTURE_VID:
return HWCONST(C076, FAULT_CANCEL_A, INST_APERTURE, VID_MEM);
default:
UVM_ASSERT_MSG(false, "Invalid aperture_type %d\n", aperture);
}
return 0;
}
void uvm_hal_pascal_host_init(uvm_push_t *push)
{
if (uvm_channel_is_ce(push->channel))
NV_PUSH_1U(C076, SET_OBJECT, GP100_UVM_SW);
}
static void instance_ptr_address_to_hw_values(NvU64 instance_ptr_address,
NvU32 *instance_ptr_lo,
NvU32 *instance_ptr_hi)
{
// instance_ptr must be 4K aligned
UVM_ASSERT_MSG(IS_ALIGNED(instance_ptr_address, 1 << 12), "instance_ptr 0x%llx\n", instance_ptr_address);
instance_ptr_address >>= 12;
*instance_ptr_lo = instance_ptr_address & HWMASK(C076, FAULT_CANCEL_A, INST_LOW);
*instance_ptr_hi = instance_ptr_address >> HWSIZE(C076, FAULT_CANCEL_A, INST_LOW);
}
void uvm_hal_pascal_cancel_faults_global(uvm_push_t *push, uvm_gpu_phys_address_t instance_ptr)
{
NvU32 instance_ptr_lo, instance_ptr_hi;
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
// Global cancellations affect all MIG instances, not only the one that
// originated the fault to be cancelled
UVM_ASSERT(!gpu->parent->smc.enabled);
instance_ptr_address_to_hw_values(instance_ptr.address, &instance_ptr_lo, &instance_ptr_hi);
NV_PUSH_3U(C076, FAULT_CANCEL_A, HWVALUE(C076, FAULT_CANCEL_A, INST_LOW, instance_ptr_lo) |
instance_ptr_aperture_type_to_hw_value(instance_ptr.aperture),
FAULT_CANCEL_B, HWVALUE(C076, FAULT_CANCEL_B, INST_HI, instance_ptr_hi),
FAULT_CANCEL_C, HWCONST(C076, FAULT_CANCEL_C, MODE, GLOBAL));
}
void uvm_hal_pascal_cancel_faults_targeted(uvm_push_t *push,
uvm_gpu_phys_address_t instance_ptr,
NvU32 gpc_id,
NvU32 client_id)
{
NvU32 instance_ptr_lo, instance_ptr_hi;
instance_ptr_address_to_hw_values(instance_ptr.address, &instance_ptr_lo, &instance_ptr_hi);
NV_PUSH_3U(C076, FAULT_CANCEL_A, HWVALUE(C076, FAULT_CANCEL_A, INST_LOW, instance_ptr_lo) |
instance_ptr_aperture_type_to_hw_value(instance_ptr.aperture),
FAULT_CANCEL_B, HWVALUE(C076, FAULT_CANCEL_B, INST_HI, instance_ptr_hi),
FAULT_CANCEL_C, HWVALUE(C076, FAULT_CANCEL_C, CLIENT_ID, client_id) |
HWVALUE(C076, FAULT_CANCEL_C, GPC_ID, gpc_id) |
HWCONST(C076, FAULT_CANCEL_C, MODE, TARGETED));
}

465
kernel-open/nvidia-uvm/uvm_pascal_mmu.c
View File

@@ -1,465 +0,0 @@
/*******************************************************************************
Copyright (c) 2015-2025 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
// For Pascal, UVM page tree 'depth' maps to hardware as follows:
//
// UVM depth HW level VA bits
// 0 PDE3 48:47
// 1 PDE2 46:38
// 2 PDE1 37:29
// 3 PDE0 (dual 64k/4k PDE, or 2M PTE) 28:21
// 4 PTE_64K / PTE_4K 20:16 / 20:12
#include "uvm_types.h"
#include "uvm_forward_decl.h"
#include "nv_uvm_interface.h"
#include "uvm_global.h"
#include "uvm_gpu.h"
#include "uvm_mmu.h"
#include "uvm_hal.h"
#include "uvm_push_macros.h"
#include "uvm_pascal_fault_buffer.h"
#include "hwref/pascal/gp100/dev_fault.h"
#include "hwref/pascal/gp100/dev_fb.h"
#include "hwref/pascal/gp100/dev_mmu.h"
#define MMU_BIG 0
#define MMU_SMALL 1
static NvU32 entries_per_index_pascal(NvU32 depth)
{
UVM_ASSERT(depth < 5);
if (depth == 3)
return 2;
return 1;
}
static NvLength entry_offset_pascal(NvU32 depth, NvU64 page_size)
{
UVM_ASSERT(depth < 5);
if (page_size == UVM_PAGE_SIZE_4K && depth == 3)
return MMU_SMALL;
return MMU_BIG;
}
static NvU64 single_pde_pascal(uvm_mmu_page_table_alloc_t *phys_alloc)
{
NvU64 pde_bits = 0;
if (phys_alloc != NULL) {
NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_PDE_ADDRESS_SHIFT;
pde_bits |= HWCONST64(_MMU_VER2, PDE, IS_PDE, TRUE) |
HWCONST64(_MMU_VER2, PDE, VOL, TRUE);
switch (phys_alloc->addr.aperture) {
case UVM_APERTURE_SYS:
pde_bits |= HWCONST64(_MMU_VER2, PDE, APERTURE, SYSTEM_COHERENT_MEMORY) |
HWVALUE64(_MMU_VER2, PDE, ADDRESS_SYS, address);
break;
case UVM_APERTURE_VID:
pde_bits |= HWCONST64(_MMU_VER2, PDE, APERTURE, VIDEO_MEMORY) |
HWVALUE64(_MMU_VER2, PDE, ADDRESS_VID, address);
break;
default:
UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", phys_alloc->addr.aperture);
break;
}
}
return pde_bits;
}
static NvU64 big_half_pde_pascal(uvm_mmu_page_table_alloc_t *phys_alloc)
{
NvU64 pde_bits = 0;
if (phys_alloc != NULL) {
NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SHIFT;
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, VOL_BIG, TRUE);
switch (phys_alloc->addr.aperture) {
case UVM_APERTURE_SYS:
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_BIG, SYSTEM_COHERENT_MEMORY) |
HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_BIG_SYS, address);
break;
case UVM_APERTURE_VID:
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_BIG, VIDEO_MEMORY) |
HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_BIG_VID, address);
break;
default:
UVM_ASSERT_MSG(0, "Invalid big aperture %d\n", phys_alloc->addr.aperture);
break;
}
}
return pde_bits;
}
static NvU64 small_half_pde_pascal(uvm_mmu_page_table_alloc_t *phys_alloc)
{
NvU64 pde_bits = 0;
if (phys_alloc != NULL) {
NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_DUAL_PDE_ADDRESS_SHIFT;
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, VOL_SMALL, TRUE);
switch (phys_alloc->addr.aperture) {
case UVM_APERTURE_SYS:
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_SMALL, SYSTEM_COHERENT_MEMORY);
pde_bits |= HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_SMALL_SYS, address);
break;
case UVM_APERTURE_VID:
pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_SMALL, VIDEO_MEMORY);
pde_bits |= HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_SMALL_VID, address);
break;
default:
UVM_ASSERT_MSG(0, "Invalid small aperture %d\n", phys_alloc->addr.aperture);
break;
}
}
return pde_bits;
}
static void make_pde_pascal(void *entry,
uvm_mmu_page_table_alloc_t **phys_allocs,
uvm_page_directory_t *dir,
NvU32 child_index)
{
NvU32 entry_count;
NvU64 *entry_bits = (NvU64 *)entry;
UVM_ASSERT(dir);
entry_count = entries_per_index_pascal(dir->depth);
if (entry_count == 1) {
*entry_bits = single_pde_pascal(*phys_allocs);
}
else if (entry_count == 2) {
entry_bits[MMU_BIG] = big_half_pde_pascal(phys_allocs[MMU_BIG]);
entry_bits[MMU_SMALL] = small_half_pde_pascal(phys_allocs[MMU_SMALL]);
// This entry applies to the whole dual PDE but is stored in the lower
// bits.
entry_bits[MMU_BIG] |= HWCONST64(_MMU_VER2, DUAL_PDE, IS_PDE, TRUE);
}
else {
UVM_ASSERT_MSG(0, "Invalid number of entries per index: %d\n", entry_count);
}
}
static NvLength entry_size_pascal(NvU32 depth)
{
UVM_ASSERT(depth < 5);
if (depth == 3)
return 16;
else
return 8;
}
static NvU32 index_bits_pascal(NvU32 depth, NvU64 page_size)
{
static const NvU32 bit_widths[] = {2, 9, 9, 8};
// some code paths keep on querying this until they get a 0, meaning only the page offset remains.
UVM_ASSERT(depth < 5);
if (depth < 4) {
return bit_widths[depth];
}
else if (depth == 4) {
switch (page_size) {
case UVM_PAGE_SIZE_4K:
return 9;
case UVM_PAGE_SIZE_64K:
return 5;
default:
break;
}
}
return 0;
}
static NvU32 num_va_bits_pascal(void)
{
return 49;
}
static NvLength allocation_size_pascal(NvU32 depth, NvU64 page_size)
{
UVM_ASSERT(depth < 5);
if (depth == 4 && page_size == UVM_PAGE_SIZE_64K)
return 256;
// depth 0 requires only a 32 byte allocation, but it must be 4k aligned
return 4096;
}
static NvU32 page_table_depth_pascal(NvU64 page_size)
{
if (page_size == UVM_PAGE_SIZE_2M)
return 3;
else
return 4;
}
static NvU64 page_sizes_pascal(void)
{
return UVM_PAGE_SIZE_2M | UVM_PAGE_SIZE_64K | UVM_PAGE_SIZE_4K;
}
static NvU64 unmapped_pte_pascal(NvU64 page_size)
{
// Setting the privilege bit on an otherwise-zeroed big PTE causes the
// corresponding 4k PTEs to be ignored. This allows the invalidation of a
// mixed PDE range to be much faster.
if (page_size != UVM_PAGE_SIZE_64K)
return 0;
// When VALID == 0, MMU still reads the VOL and PRIV fields. VOL == 1
// indicates that the PTE is sparse, so make sure we don't use it.
return HWCONST64(_MMU_VER2, PTE, VALID, FALSE) |
HWCONST64(_MMU_VER2, PTE, VOL, FALSE) |
HWCONST64(_MMU_VER2, PTE, PRIVILEGE, TRUE);
}
static NvU64 make_pte_pascal(uvm_aperture_t aperture, NvU64 address, uvm_prot_t prot, NvU64 flags)
{
NvU8 aperture_bits = 0;
NvU64 pte_bits = 0;
UVM_ASSERT(prot != UVM_PROT_NONE);
UVM_ASSERT((flags & ~UVM_MMU_PTE_FLAGS_MASK) == 0);
// valid 0:0
pte_bits |= HWCONST64(_MMU_VER2, PTE, VALID, TRUE);
// aperture 2:1
if (aperture == UVM_APERTURE_SYS)
aperture_bits = NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_VID)
aperture_bits = NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY;
else if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7)
aperture_bits = NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY;
else
UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", aperture);
pte_bits |= HWVALUE64(_MMU_VER2, PTE, APERTURE, aperture_bits);
// volatile 3:3
if (flags & UVM_MMU_PTE_FLAGS_CACHED)
pte_bits |= HWCONST64(_MMU_VER2, PTE, VOL, FALSE);
else
pte_bits |= HWCONST64(_MMU_VER2, PTE, VOL, TRUE);
// encrypted 4:4
pte_bits |= HWCONST64(_MMU_VER2, PTE, ENCRYPTED, FALSE);
// privilege 5:5
pte_bits |= HWCONST64(_MMU_VER2, PTE, PRIVILEGE, FALSE);
// read only 6:6
if (prot == UVM_PROT_READ_ONLY)
pte_bits |= HWCONST64(_MMU_VER2, PTE, READ_ONLY, TRUE);
else
pte_bits |= HWCONST64(_MMU_VER2, PTE, READ_ONLY, FALSE);
// atomic disable 7:7
if (prot == UVM_PROT_READ_WRITE_ATOMIC)
pte_bits |= HWCONST64(_MMU_VER2, PTE, ATOMIC_DISABLE, FALSE);
else
pte_bits |= HWCONST64(_MMU_VER2, PTE, ATOMIC_DISABLE, TRUE);
address >>= NV_MMU_VER2_PTE_ADDRESS_SHIFT;
if (aperture == UVM_APERTURE_SYS) {
// sys address 53:8
pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_SYS, address);
}
else {
// vid address 32:8
pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_VID, address);
// peer id 35:33
if (aperture != UVM_APERTURE_VID)
pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_VID_PEER, UVM_APERTURE_PEER_ID(aperture));
// comptagline 53:36
pte_bits |= HWVALUE64(_MMU_VER2, PTE, COMPTAGLINE, 0);
}
// kind 63:56
pte_bits |= HWVALUE64(_MMU_VER2, PTE, KIND, NV_MMU_PTE_KIND_PITCH);
return pte_bits;
}
static NvU64 make_sked_reflected_pte_pascal(void)
{
NvU64 pte_bits = 0;
pte_bits |= HWCONST64(_MMU_VER2, PTE, VALID, TRUE);
pte_bits |= HWVALUE64(_MMU_VER2, PTE, KIND, NV_MMU_PTE_KIND_SMSKED_MESSAGE);
return pte_bits;
}
static NvU64 make_sparse_pte_pascal(void)
{
return HWCONST64(_MMU_VER2, PTE, VALID, FALSE) |
HWCONST64(_MMU_VER2, PTE, VOL, TRUE);
}
static NvU64 poisoned_pte_pascal(uvm_page_tree_t *tree)
{
// An invalid PTE won't be fatal from faultable units like SM, which is the
// most likely source of bad PTE accesses.
// Engines with priv accesses won't fault on the priv PTE, so add a backup
// mechanism using an impossible memory address. MMU will trigger an
// interrupt when it detects a bad physical address.
//
// This address has to fit within 37 bits (max address width of vidmem) and
// be aligned to page_size.
NvU64 phys_addr = 0x1bad000000ULL;
NvU64 pte_bits = tree->hal->make_pte(UVM_APERTURE_VID, phys_addr, UVM_PROT_READ_ONLY, UVM_MMU_PTE_FLAGS_NONE);
return WRITE_HWCONST64(pte_bits, _MMU_VER2, PTE, PRIVILEGE, TRUE);
}
static uvm_mmu_mode_hal_t pascal_mmu_mode_hal =
{
.make_pte = make_pte_pascal,
.make_sked_reflected_pte = make_sked_reflected_pte_pascal,
.make_sparse_pte = make_sparse_pte_pascal,
.unmapped_pte = unmapped_pte_pascal,
.poisoned_pte = poisoned_pte_pascal,
.make_pde = make_pde_pascal,
.entry_size = entry_size_pascal,
.index_bits = index_bits_pascal,
.entries_per_index = entries_per_index_pascal,
.entry_offset = entry_offset_pascal,
.num_va_bits = num_va_bits_pascal,
.allocation_size = allocation_size_pascal,
.page_table_depth = page_table_depth_pascal,
.page_sizes = page_sizes_pascal
};
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_pascal(NvU64 big_page_size)
{
UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K);
// TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with
// 128K big page size for Pascal+ GPUs
if (big_page_size == UVM_PAGE_SIZE_128K)
return NULL;
return &pascal_mmu_mode_hal;
}
static void mmu_set_prefetch_faults(uvm_parent_gpu_t *parent_gpu, bool enable)
{
volatile NvU32 *prefetch_ctrl = parent_gpu->fault_buffer.rm_info.replayable.pPrefetchCtrl;
// A null prefetch control mapping indicates that UVM should toggle the
// register's value using the RM API, instead of performing a direct access.
if (prefetch_ctrl == NULL) {
NV_STATUS status;
// Access to the register is currently blocked only in Confidential
// Computing.
UVM_ASSERT(g_uvm_global.conf_computing_enabled);
status = nvUvmInterfaceTogglePrefetchFaults(&parent_gpu->fault_buffer.rm_info, (NvBool)enable);
UVM_ASSERT(status == NV_OK);
}
else {
NvU32 prefetch_ctrl_value = UVM_GPU_READ_ONCE(*prefetch_ctrl);
if (enable)
prefetch_ctrl_value = WRITE_HWCONST(prefetch_ctrl_value, _PFB_PRI_MMU_PAGE, FAULT_CTRL, PRF_FILTER, SEND_ALL);
else
prefetch_ctrl_value = WRITE_HWCONST(prefetch_ctrl_value, _PFB_PRI_MMU_PAGE, FAULT_CTRL, PRF_FILTER, SEND_NONE);
UVM_GPU_WRITE_ONCE(*prefetch_ctrl, prefetch_ctrl_value);
}
}
void uvm_hal_pascal_mmu_enable_prefetch_faults(uvm_parent_gpu_t *parent_gpu)
{
UVM_ASSERT(parent_gpu->prefetch_fault_supported);
mmu_set_prefetch_faults(parent_gpu, true);
}
void uvm_hal_pascal_mmu_disable_prefetch_faults(uvm_parent_gpu_t *parent_gpu)
{
mmu_set_prefetch_faults(parent_gpu, false);
}
NvU16 uvm_hal_pascal_mmu_client_id_to_utlb_id(NvU16 client_id)
{
switch (client_id) {
case NV_PFAULT_CLIENT_GPC_RAST:
case NV_PFAULT_CLIENT_GPC_GCC:
case NV_PFAULT_CLIENT_GPC_GPCCS:
return UVM_PASCAL_GPC_UTLB_ID_RGG;
case NV_PFAULT_CLIENT_GPC_PE_0:
case NV_PFAULT_CLIENT_GPC_TPCCS_0:
case NV_PFAULT_CLIENT_GPC_L1_0:
case NV_PFAULT_CLIENT_GPC_T1_0:
case NV_PFAULT_CLIENT_GPC_L1_1:
case NV_PFAULT_CLIENT_GPC_T1_1:
return UVM_PASCAL_GPC_UTLB_ID_LTP0;
case NV_PFAULT_CLIENT_GPC_PE_1:
case NV_PFAULT_CLIENT_GPC_TPCCS_1:
case NV_PFAULT_CLIENT_GPC_L1_2:
case NV_PFAULT_CLIENT_GPC_T1_2:
case NV_PFAULT_CLIENT_GPC_L1_3:
case NV_PFAULT_CLIENT_GPC_T1_3:
return UVM_PASCAL_GPC_UTLB_ID_LTP1;
case NV_PFAULT_CLIENT_GPC_PE_2:
case NV_PFAULT_CLIENT_GPC_TPCCS_2:
case NV_PFAULT_CLIENT_GPC_L1_4:
case NV_PFAULT_CLIENT_GPC_T1_4:
case NV_PFAULT_CLIENT_GPC_L1_5:
case NV_PFAULT_CLIENT_GPC_T1_5:
return UVM_PASCAL_GPC_UTLB_ID_LTP2;
case NV_PFAULT_CLIENT_GPC_PE_3:
case NV_PFAULT_CLIENT_GPC_TPCCS_3:
case NV_PFAULT_CLIENT_GPC_L1_6:
case NV_PFAULT_CLIENT_GPC_T1_6:
case NV_PFAULT_CLIENT_GPC_L1_7:
case NV_PFAULT_CLIENT_GPC_T1_7:
return UVM_PASCAL_GPC_UTLB_ID_LTP3;
case NV_PFAULT_CLIENT_GPC_PE_4:
case NV_PFAULT_CLIENT_GPC_TPCCS_4:
case NV_PFAULT_CLIENT_GPC_L1_8:
case NV_PFAULT_CLIENT_GPC_T1_8:
case NV_PFAULT_CLIENT_GPC_L1_9:
case NV_PFAULT_CLIENT_GPC_T1_9:
return UVM_PASCAL_GPC_UTLB_ID_LTP4;
default:
UVM_ASSERT_MSG(false, "Invalid client value: 0x%x\n", client_id);
}
return 0;
}

Some files were not shown because too many files have changed in this diff Show More