NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-27 03:29:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

515.76

Browse Source
This commit is contained in:
Andy Ritger
2022-09-20 13:54:59 -07:00
parent 9855350159
commit fe0728787f
50 changed files with 822 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
kernel-open/Kbuild
View File

@@ -72,7 +72,7 @@ EXTRA_CFLAGS += -I$(src)/common/inc
EXTRA_CFLAGS += -I$(src)
EXTRA_CFLAGS += -Wall -MD $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-error -Wno-format-extra-args
EXTRA_CFLAGS += -D__KERNEL__ -DMODULE -DNVRM
EXTRA_CFLAGS += -DNV_VERSION_STRING=\"515.65.01\"
EXTRA_CFLAGS += -DNV_VERSION_STRING=\"515.76\"
EXTRA_CFLAGS += -Wno-unused-function
@@ -203,9 +203,108 @@ $(obj)/conftest/patches.h: $(NV_CONFTEST_SCRIPT)
@mkdir -p $(obj)/conftest
@$(NV_CONFTEST_CMD) patch_check > $@
$(obj)/conftest/headers.h: $(NV_CONFTEST_SCRIPT)
@mkdir -p $(obj)/conftest
@$(NV_CONFTEST_CMD) test_kernel_headers '$(NV_CONFTEST_CFLAGS)' > $@
# Each of these headers is checked for presence with a test #include; a
# corresponding #define will be generated in conftest/headers.h.
NV_HEADER_PRESENCE_TESTS = \
asm/system.h \
drm/drmP.h \
drm/drm_auth.h \
drm/drm_gem.h \
drm/drm_crtc.h \
drm/drm_atomic.h \
drm/drm_atomic_helper.h \
drm/drm_encoder.h \
drm/drm_atomic_uapi.h \
drm/drm_drv.h \
drm/drm_framebuffer.h \
drm/drm_connector.h \
drm/drm_probe_helper.h \
drm/drm_blend.h \
drm/drm_fourcc.h \
drm/drm_prime.h \
drm/drm_plane.h \
drm/drm_vblank.h \
drm/drm_file.h \
drm/drm_ioctl.h \
drm/drm_device.h \
drm/drm_mode_config.h \
dt-bindings/interconnect/tegra_icc_id.h \
generated/autoconf.h \
generated/compile.h \
generated/utsrelease.h \
linux/efi.h \
linux/kconfig.h \
linux/platform/tegra/mc_utils.h \
linux/semaphore.h \
linux/printk.h \
linux/ratelimit.h \
linux/prio_tree.h \
linux/log2.h \
linux/of.h \
linux/bug.h \
linux/sched/signal.h \
linux/sched/task.h \
linux/sched/task_stack.h \
xen/ioemu.h \
linux/fence.h \
linux/dma-resv.h \
soc/tegra/chip-id.h \
soc/tegra/fuse.h \
soc/tegra/tegra_bpmp.h \
video/nv_internal.h \
linux/platform/tegra/dce/dce-client-ipc.h \
linux/nvhost.h \
linux/nvhost_t194.h \
asm/book3s/64/hash-64k.h \
asm/set_memory.h \
asm/prom.h \
asm/powernv.h \
linux/atomic.h \
asm/barrier.h \
asm/opal-api.h \
sound/hdaudio.h \
asm/pgtable_types.h \
linux/stringhash.h \
linux/dma-map-ops.h \
rdma/peer_mem.h \
sound/hda_codec.h \
linux/dma-buf.h \
linux/time.h \
linux/platform_device.h \
linux/mutex.h \
linux/reset.h \
linux/of_platform.h \
linux/of_device.h \
linux/of_gpio.h \
linux/gpio.h \
linux/gpio/consumer.h \
linux/interconnect.h \
linux/pm_runtime.h \
linux/clk.h \
linux/clk-provider.h \
linux/ioasid.h \
linux/stdarg.h \
linux/iosys-map.h \
asm/coco.h
# Filename to store the define for the header in $(1); this is only consumed by
# the rule below that concatenates all of these together.
NV_HEADER_PRESENCE_PART = $(addprefix $(obj)/conftest/header_presence/,$(addsuffix .part,$(1)))
# Define a rule to check the header $(1).
define NV_HEADER_PRESENCE_CHECK
$$(call NV_HEADER_PRESENCE_PART,$(1)): $$(NV_CONFTEST_SCRIPT) $(obj)/conftest/uts_release
@mkdir -p $$(dir $$@)
@$$(NV_CONFTEST_CMD) test_kernel_header '$$(NV_CONFTEST_CFLAGS)' '$(1)' > $$@
endef
# Evaluate the rule above for each header in the list.
$(foreach header,$(NV_HEADER_PRESENCE_TESTS),$(eval $(call NV_HEADER_PRESENCE_CHECK,$(header))))
# Concatenate all of the parts into headers.h.
$(obj)/conftest/headers.h: $(call NV_HEADER_PRESENCE_PART,$(NV_HEADER_PRESENCE_TESTS))
@cat $^ > $@
clean-dirs := $(obj)/conftest

1
kernel-open/common/inc/nv-linux.h
View File

@@ -227,6 +227,7 @@ static inline uid_t __kuid_val(uid_t uid)
#endif
#include <linux/fb.h> /* fb_info struct */
#include <linux/screen_info.h> /* screen_info */
#if !defined(CONFIG_PCI)
#warning "Attempting to build driver for a platform with no PCI support!"

5
kernel-open/common/inc/nv-pgprot.h
View File

@@ -78,13 +78,8 @@ static inline pgprot_t pgprot_modify_writecombine(pgprot_t old_prot)
#define NV_PGPROT_UNCACHED_DEVICE(old_prot) pgprot_noncached(old_prot)
#if defined(NVCPU_AARCH64)
#if defined(NV_MT_DEVICE_GRE_PRESENT)
#define NV_PROT_WRITE_COMBINED_DEVICE (PROT_DEFAULT | PTE_PXN | PTE_UXN | \
PTE_ATTRINDX(MT_DEVICE_GRE))
#else
#define NV_PROT_WRITE_COMBINED_DEVICE (PROT_DEFAULT | PTE_PXN | PTE_UXN | \
PTE_ATTRINDX(MT_DEVICE_nGnRE))
#endif
#define NV_PGPROT_WRITE_COMBINED_DEVICE(old_prot) \
__pgprot_modify(old_prot, PTE_ATTRINDX_MASK, NV_PROT_WRITE_COMBINED_DEVICE)
#define NV_PGPROT_WRITE_COMBINED(old_prot) NV_PGPROT_UNCACHED(old_prot)

54
kernel-open/common/inc/nv.h
View File

@@ -624,27 +624,45 @@ typedef enum
#define NV_GET_NV_STATE(pGpu) \
(nv_state_t *)((pGpu) ? (pGpu)->pOsGpuInfo : NULL)
#define IS_REG_OFFSET(nv, offset, length) \
(((offset) >= (nv)->regs->cpu_address) && \
(((offset) + ((length)-1)) <= \
(nv)->regs->cpu_address + ((nv)->regs->size-1)))
static inline NvBool IS_REG_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
{
return ((offset >= nv->regs->cpu_address) &&
#define IS_FB_OFFSET(nv, offset, length) \
(((nv)->fb) && ((offset) >= (nv)->fb->cpu_address) && \
(((offset) + ((length)-1)) <= (nv)->fb->cpu_address + ((nv)->fb->size-1)))
#define IS_UD_OFFSET(nv, offset, length) \
(((nv)->ud.cpu_address != 0) && ((nv)->ud.size != 0) && \
((offset) >= (nv)->ud.cpu_address) && \
(((offset) + ((length)-1)) <= (nv)->ud.cpu_address + ((nv)->ud.size-1)))
#define IS_IMEM_OFFSET(nv, offset, length) \
(((nv)->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address != 0) && \
((nv)->bars[NV_GPU_BAR_INDEX_IMEM].size != 0) && \
((offset) >= (nv)->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address) && \
(((offset) + ((length) - 1)) <= \
(nv)->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address + \
((nv)->bars[NV_GPU_BAR_INDEX_IMEM].size - 1)))
((offset + (length - 1)) <= (nv->regs->cpu_address + (nv->regs->size - 1))));
}
static inline NvBool IS_FB_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
{
return ((nv->fb) && (offset >= nv->fb->cpu_address) &&
((offset + (length - 1)) <= (nv->fb->cpu_address + (nv->fb->size - 1))));
}
static inline NvBool IS_UD_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
{
return ((nv->ud.cpu_address != 0) && (nv->ud.size != 0) &&
(offset >= nv->ud.cpu_address) &&
((offset + (length - 1)) <= (nv->ud.cpu_address + (nv->ud.size - 1))));
}
static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
{
return ((nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address != 0) &&
(nv->bars[NV_GPU_BAR_INDEX_IMEM].size != 0) &&
(offset >= nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address) &&
((offset + (length - 1)) <= (nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address +
(nv->bars[NV_GPU_BAR_INDEX_IMEM].size - 1))));
}
#define NV_RM_MAX_MSIX_LINES 8

170
kernel-open/conftest.sh
View File

@@ -55,9 +55,13 @@ append_conftest() {
done
}
translate_and_preprocess_header_files() {
# Inputs:
# $1: list of relative file paths
test_header_presence() {
#
# Determine if the given header file (which may or may not be
# present) is provided by the target kernel.
#
# Input:
# $1: relative file path
#
# This routine creates an upper case, underscore version of each of the
# relative file paths, and uses that as the token to either define or
@@ -73,115 +77,25 @@ translate_and_preprocess_header_files() {
# strings, without special handling of the beginning or the end of the line.
TEST_CFLAGS=`echo "-E -M $CFLAGS " | sed -e 's/\( -M[DG]\)* / /g'`
for file in "$@"; do
file_define=NV_`echo $file | tr '/.' '_' | tr '-' '_' | tr 'a-z' 'A-Z'`_PRESENT
file="$1"
file_define=NV_`echo $file | tr '/.' '_' | tr '-' '_' | tr 'a-z' 'A-Z'`_PRESENT
CODE="#include <$file>"
CODE="#include <$file>"
if echo "$CODE" | $CC $TEST_CFLAGS - > /dev/null 2>&1; then
echo "#define $file_define"
if echo "$CODE" | $CC $TEST_CFLAGS - > /dev/null 2>&1; then
echo "#define $file_define"
else
# If preprocessing failed, it could have been because the header
# file under test is not present, or because it is present but
# depends upon the inclusion of other header files. Attempting
# preprocessing again with -MG will ignore a missing header file
# but will still fail if the header file is present.
if echo "$CODE" | $CC $TEST_CFLAGS -MG - > /dev/null 2>&1; then
echo "#undef $file_define"
else
# If preprocessing failed, it could have been because the header
# file under test is not present, or because it is present but
# depends upon the inclusion of other header files. Attempting
# preprocessing again with -MG will ignore a missing header file
# but will still fail if the header file is present.
if echo "$CODE" | $CC $TEST_CFLAGS -MG - > /dev/null 2>&1; then
echo "#undef $file_define"
else
echo "#define $file_define"
fi
echo "#define $file_define"
fi
done
}
test_headers() {
#
# Determine which header files (of a set that may or may not be
# present) are provided by the target kernel.
#
FILES="asm/system.h"
FILES="$FILES drm/drmP.h"
FILES="$FILES drm/drm_auth.h"
FILES="$FILES drm/drm_gem.h"
FILES="$FILES drm/drm_crtc.h"
FILES="$FILES drm/drm_atomic.h"
FILES="$FILES drm/drm_atomic_helper.h"
FILES="$FILES drm/drm_encoder.h"
FILES="$FILES drm/drm_atomic_uapi.h"
FILES="$FILES drm/drm_drv.h"
FILES="$FILES drm/drm_framebuffer.h"
FILES="$FILES drm/drm_connector.h"
FILES="$FILES drm/drm_probe_helper.h"
FILES="$FILES drm/drm_blend.h"
FILES="$FILES drm/drm_fourcc.h"
FILES="$FILES drm/drm_prime.h"
FILES="$FILES drm/drm_plane.h"
FILES="$FILES drm/drm_vblank.h"
FILES="$FILES drm/drm_file.h"
FILES="$FILES drm/drm_ioctl.h"
FILES="$FILES drm/drm_device.h"
FILES="$FILES drm/drm_mode_config.h"
FILES="$FILES dt-bindings/interconnect/tegra_icc_id.h"
FILES="$FILES generated/autoconf.h"
FILES="$FILES generated/compile.h"
FILES="$FILES generated/utsrelease.h"
FILES="$FILES linux/efi.h"
FILES="$FILES linux/kconfig.h"
FILES="$FILES linux/platform/tegra/mc_utils.h"
FILES="$FILES linux/semaphore.h"
FILES="$FILES linux/printk.h"
FILES="$FILES linux/ratelimit.h"
FILES="$FILES linux/prio_tree.h"
FILES="$FILES linux/log2.h"
FILES="$FILES linux/of.h"
FILES="$FILES linux/bug.h"
FILES="$FILES linux/sched/signal.h"
FILES="$FILES linux/sched/task.h"
FILES="$FILES linux/sched/task_stack.h"
FILES="$FILES xen/ioemu.h"
FILES="$FILES linux/fence.h"
FILES="$FILES linux/dma-resv.h"
FILES="$FILES soc/tegra/chip-id.h"
FILES="$FILES soc/tegra/fuse.h"
FILES="$FILES soc/tegra/tegra_bpmp.h"
FILES="$FILES video/nv_internal.h"
FILES="$FILES linux/platform/tegra/dce/dce-client-ipc.h"
FILES="$FILES linux/nvhost.h"
FILES="$FILES linux/nvhost_t194.h"
FILES="$FILES asm/book3s/64/hash-64k.h"
FILES="$FILES asm/set_memory.h"
FILES="$FILES asm/prom.h"
FILES="$FILES asm/powernv.h"
FILES="$FILES linux/atomic.h"
FILES="$FILES asm/barrier.h"
FILES="$FILES asm/opal-api.h"
FILES="$FILES sound/hdaudio.h"
FILES="$FILES asm/pgtable_types.h"
FILES="$FILES linux/stringhash.h"
FILES="$FILES linux/dma-map-ops.h"
FILES="$FILES rdma/peer_mem.h"
FILES="$FILES sound/hda_codec.h"
FILES="$FILES linux/dma-buf.h"
FILES="$FILES linux/time.h"
FILES="$FILES linux/platform_device.h"
FILES="$FILES linux/mutex.h"
FILES="$FILES linux/reset.h"
FILES="$FILES linux/of_platform.h"
FILES="$FILES linux/of_device.h"
FILES="$FILES linux/of_gpio.h"
FILES="$FILES linux/gpio.h"
FILES="$FILES linux/gpio/consumer.h"
FILES="$FILES linux/interconnect.h"
FILES="$FILES linux/pm_runtime.h"
FILES="$FILES linux/clk.h"
FILES="$FILES linux/clk-provider.h"
FILES="$FILES linux/ioasid.h"
FILES="$FILES linux/stdarg.h"
FILES="$FILES linux/iosys-map.h"
FILES="$FILES asm/coco.h"
translate_and_preprocess_header_files $FILES
fi
}
build_cflags() {
@@ -2420,23 +2334,6 @@ compile_test() {
compile_check_conftest "$CODE" "NV_PCI_DEV_HAS_ATS_ENABLED" "" "types"
;;
mt_device_gre)
#
# Determine if MT_DEVICE_GRE flag is present.
#
# MT_DEVICE_GRE flag is removed by commit 58cc6b72a21274
# ("arm64: mm: Remove unused support for Device-GRE memory type") in v5.14-rc1
# (2021-06-01).
#
CODE="
#include <asm/memory.h>
unsigned int conftest_mt_device_gre(void) {
return MT_DEVICE_GRE;
}"
compile_check_conftest "$CODE" "NV_MT_DEVICE_GRE_PRESENT" "" "types"
;;
get_user_pages)
#
# Conftest for get_user_pages()
@@ -5366,6 +5263,23 @@ compile_test() {
compile_check_conftest "$CODE" "NV_GET_TASK_IOPRIO_PRESENT" "" "functions"
;;
num_registered_fb)
#
# Determine if 'num_registered_fb' variable is present.
#
# 'num_registered_fb' was removed by commit 5727dcfd8486
# ("fbdev: Make registered_fb[] private to fbmem.c) for
# v5.20 linux-next (2022-07-27).
#
CODE="
#include <linux/fb.h>
int conftest_num_registered_fb(void) {
return num_registered_fb;
}"
compile_check_conftest "$CODE" "NV_NUM_REGISTERED_FB_PRESENT" "" "types"
;;
# When adding a new conftest entry, please use the correct format for
# specifying the relevant upstream Linux kernel commit.
#
@@ -5764,14 +5678,14 @@ case "$5" in
;;
test_kernel_headers)
test_kernel_header)
#
# Check for the availability of certain kernel headers
# Check for the availability of the given kernel header
#
CFLAGS=$6
test_headers
test_header_presence "${7}"
for file in conftest*.d; do
rm -f $file > /dev/null 2>&1

13
kernel-open/nvidia-drm/nvidia-drm-helper.c
View File

@@ -41,6 +41,19 @@
#include <drm/drm_atomic_uapi.h>
#endif
/*
* The inclusion of drm_framebuffer.h was removed from drm_crtc.h by commit
* 720cf96d8fecde29b72e1101f8a567a0ce99594f ("drm: Drop drm_framebuffer.h from
* drm_crtc.h") in linux-next, expected in v5.19-rc7.
*
* We only need drm_framebuffer.h for drm_framebuffer_put(), and it is always
* present (v4.9+) when drm_framebuffer_{put,get}() is present (v4.12+), so it
* is safe to unconditionally include it when drm_framebuffer_get() is present.
*/
#if defined(NV_DRM_FRAMEBUFFER_GET_PRESENT)
#include <drm/drm_framebuffer.h>
#endif
static void __nv_drm_framebuffer_put(struct drm_framebuffer *fb)
{
#if defined(NV_DRM_FRAMEBUFFER_GET_PRESENT)

7
kernel-open/nvidia-modeset/nvidia-modeset-linux.c
View File

@@ -59,6 +59,9 @@
#define NVKMS_LOG_PREFIX "nvidia-modeset: "
static bool output_rounding_fix = false;
module_param_named(output_rounding_fix, output_rounding_fix, bool, 0400);
/* These parameters are used for fault injection tests. Normally the defaults
* should be used. */
MODULE_PARM_DESC(fail_malloc, "Fail the Nth call to nvkms_alloc");
@@ -71,6 +74,10 @@ module_param_named(malloc_verbose, malloc_verbose, bool, 0400);
static atomic_t nvkms_alloc_called_count;
NvBool nvkms_output_rounding_fix(void)
{
return output_rounding_fix;
}
#define NVKMS_SYNCPT_STUBS_NEEDED

1
kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h
View File

@@ -110,6 +110,7 @@ typedef struct {
} set_maxval;
} NvKmsSyncPtOpParams;
NvBool nvkms_output_rounding_fix(void);
void nvkms_call_rm (void *ops);
void* nvkms_alloc (size_t size,

19
kernel-open/nvidia-uvm/uvm_channel.c
View File

@@ -35,10 +35,6 @@
#include "nv_uvm_interface.h"
#include "clb06f.h"
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT 1024
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MIN 32
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (1024 * 1024)
static unsigned uvm_channel_num_gpfifo_entries = UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT;
#define UVM_CHANNEL_GPFIFO_LOC_DEFAULT "auto"
@@ -86,6 +82,12 @@ static NvU32 uvm_channel_update_progress_with_max(uvm_channel_t *channel,
uvm_spin_lock(&channel->pool->lock);
// Completed value should never exceed the queued value
UVM_ASSERT_MSG_RELEASE(completed_value <= channel->tracking_sem.queued_value,
"GPU %s channel %s unexpected completed_value 0x%llx > queued_value 0x%llx\n",
channel->pool->manager->gpu->parent->name, channel->name, completed_value,
channel->tracking_sem.queued_value);
cpu_put = channel->cpu_put;
gpu_get = channel->gpu_get;
@@ -395,6 +397,14 @@ static void uvm_channel_semaphore_release(uvm_push_t *push, NvU64 semaphore_va,
{
uvm_gpu_t *gpu = uvm_push_get_gpu(push);
// We used to skip the membar or use membar GPU for the semaphore release
// for a few pushes, but that doesn't provide sufficient ordering guarantees
// in some cases (e.g. ga100 with an LCE with PCEs from both HSHUBs) for the
// semaphore writes. To be safe, just always uses a membar sys for now.
// TODO bug 3770539: Optimize membars used by end of push semaphore releases
(void)uvm_push_get_and_reset_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_GPU);
(void)uvm_push_get_and_reset_flag(push, UVM_PUSH_FLAG_NEXT_MEMBAR_NONE);
if (uvm_channel_is_ce(push->channel))
gpu->parent->ce_hal->semaphore_release(push, semaphore_va, new_payload);
@@ -1562,6 +1572,7 @@ static void uvm_channel_print_info(uvm_channel_t *channel, struct seq_file *s)
UVM_SEQ_OR_DBG_PRINT(s, "get %u\n", channel->gpu_get);
UVM_SEQ_OR_DBG_PRINT(s, "put %u\n", channel->cpu_put);
UVM_SEQ_OR_DBG_PRINT(s, "Semaphore GPU VA 0x%llx\n", uvm_channel_tracking_semaphore_get_gpu_va(channel));
UVM_SEQ_OR_DBG_PRINT(s, "Semaphore CPU VA 0x%llx\n", (NvU64)(uintptr_t)channel->tracking_sem.semaphore.payload);
uvm_spin_unlock(&channel->pool->lock);
}

15
kernel-open/nvidia-uvm/uvm_channel.h
View File

@@ -46,6 +46,21 @@
// wait for a GPFIFO entry to free up.
//
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_DEFAULT 1024
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MIN 32
#define UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (1024 * 1024)
// Semaphore payloads cannot advance too much between calls to
// uvm_gpu_tracking_semaphore_update_completed_value(). In practice the jumps
// are bound by gpfifo sizing as we have to update the completed value to
// reclaim gpfifo entries. Set a limit based on the max gpfifo entries we could
// ever see.
//
// Logically this define belongs to uvm_gpu_semaphore.h but it depends on the
// channel GPFIFO sizing defined here so it's easiest to just have it here as
// uvm_channel.h includes uvm_gpu_semaphore.h.
#define UVM_GPU_SEMAPHORE_MAX_JUMP (2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX)
// Channel types
typedef enum
{

39
kernel-open/nvidia-uvm/uvm_channel_test.c
View File

@@ -151,6 +151,37 @@ done:
return status;
}
static NV_STATUS test_unexpected_completed_values(uvm_va_space_t *va_space)
{
NV_STATUS status;
uvm_gpu_t *gpu;
for_each_va_space_gpu(gpu, va_space) {
uvm_channel_t *channel;
NvU64 completed_value;
// The GPU channel manager is destroyed and then re-created after
// the test, so this test requires exclusive access to the GPU.
TEST_CHECK_RET(uvm_gpu_retained_count(gpu) == 1);
channel = &gpu->channel_manager->channel_pools[0].channels[0];
completed_value = uvm_channel_update_completed_value(channel);
uvm_gpu_semaphore_set_payload(&channel->tracking_sem.semaphore, (NvU32)completed_value + 1);
TEST_CHECK_RET(uvm_global_get_status() == NV_OK);
uvm_channel_update_progress_all(channel);
TEST_CHECK_RET(uvm_global_reset_fatal_error() == NV_ERR_INVALID_STATE);
uvm_channel_manager_destroy(gpu->channel_manager);
// Destruction will hit the error again, so clear one more time.
uvm_global_reset_fatal_error();
TEST_NV_CHECK_RET(uvm_channel_manager_create(gpu, &gpu->channel_manager));
}
return NV_OK;
}
static NV_STATUS uvm_test_rc_for_gpu(uvm_gpu_t *gpu)
{
uvm_push_t push;
@@ -712,6 +743,14 @@ NV_STATUS uvm_test_channel_sanity(UVM_TEST_CHANNEL_SANITY_PARAMS *params, struct
g_uvm_global.disable_fatal_error_assert = true;
uvm_release_asserts_set_global_error_for_tests = true;
status = test_unexpected_completed_values(va_space);
uvm_release_asserts_set_global_error_for_tests = false;
g_uvm_global.disable_fatal_error_assert = false;
if (status != NV_OK)
goto done;
if (g_uvm_global.num_simulated_devices == 0) {
status = test_rc(va_space);
if (status != NV_OK)

27
kernel-open/nvidia-uvm/uvm_common.c
View File

@@ -48,6 +48,33 @@ module_param(uvm_enable_builtin_tests, int, S_IRUGO);
MODULE_PARM_DESC(uvm_enable_builtin_tests,
"Enable the UVM built-in tests. (This is a security risk)");
// Default to release asserts being enabled.
int uvm_release_asserts __read_mostly = 1;
// Make the module param writable so that release asserts can be enabled or
// disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts, "Enable uvm asserts included in release builds.");
// Default to failed release asserts not dumping stack.
int uvm_release_asserts_dump_stack __read_mostly = 0;
// Make the module param writable so that dumping the stack can be enabled and
// disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts_dump_stack, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts_dump_stack, "dump_stack() on failed UVM release asserts.");
// Default to failed release asserts not setting the global UVM error.
int uvm_release_asserts_set_global_error __read_mostly = 0;
// Make the module param writable so that setting the global fatal error can be
// enabled and disabled at any time by modifying the module parameter.
module_param(uvm_release_asserts_set_global_error, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(uvm_release_asserts_set_global_error, "Set UVM global fatal error on failed release asserts.");
// A separate flag to enable setting global error, to be used by tests only.
bool uvm_release_asserts_set_global_error_for_tests __read_mostly = false;
//
// Convert kernel errno codes to corresponding NV_STATUS
//

43
kernel-open/nvidia-uvm/uvm_common.h
View File

@@ -80,6 +80,9 @@ bool uvm_debug_prints_enabled(void);
#define UVM_ASSERT_PRINT(fmt, ...) \
UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_PRINT_RL(fmt, ...) \
UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
#define UVM_ERR_PRINT(fmt, ...) \
UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
@@ -146,9 +149,7 @@ void on_uvm_test_fail(void);
// Unlike on_uvm_test_fail it provides 'panic' coverity semantics
void on_uvm_assert(void);
// UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always enabled, even on
// release builds.
#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \
#define _UVM_ASSERT_MSG(expr, cond, fmt, ...) \
do { \
if (unlikely(!(expr))) { \
UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
@@ -157,9 +158,6 @@ void on_uvm_assert(void);
} \
} while (0)
#define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n")
// Prevent function calls in expr and the print argument list from being
// evaluated.
#define UVM_ASSERT_MSG_IGNORE(expr, fmt, ...) \
@@ -170,13 +168,42 @@ void on_uvm_assert(void);
// UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds
#if UVM_IS_DEBUG() || defined __COVERITY__
#define UVM_ASSERT_MSG UVM_ASSERT_MSG_RELEASE
#define UVM_ASSERT UVM_ASSERT_RELEASE
#define UVM_ASSERT_MSG(expr, fmt, ...) _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT(expr) _UVM_ASSERT_MSG(expr, #expr, "\n")
#else
#define UVM_ASSERT_MSG(expr, fmt, ...) UVM_ASSERT_MSG_IGNORE(expr, fmt, ##__VA_ARGS__)
#define UVM_ASSERT(expr) UVM_ASSERT_MSG_IGNORE(expr, "\n")
#endif
// UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always included in the
// build, even on release builds. They are skipped at runtime if
// uvm_release_asserts is 0.
// Whether release asserts are enabled and whether they should dump the stack
// and set the global error.
extern int uvm_release_asserts;
extern int uvm_release_asserts_dump_stack;
extern int uvm_release_asserts_set_global_error;
extern bool uvm_release_asserts_set_global_error_for_tests;
// Given these are enabled for release builds, we need to be more cautious than
// in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module
// param is enabled.
#define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \
do { \
if (uvm_release_asserts && unlikely(!(expr))) { \
UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \
uvm_global_set_fatal_error(NV_ERR_INVALID_STATE); \
if (uvm_release_asserts_dump_stack) \
dump_stack(); \
on_uvm_assert(); \
} \
} while (0)
#define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
#define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n")
// Provide a short form of UUID's, typically for use in debug printing:
#define ABBREV_UUID(uuid) (unsigned)(uuid)

10
kernel-open/nvidia-uvm/uvm_gpu_semaphore.c
View File

@@ -25,6 +25,7 @@
#include "uvm_lock.h"
#include "uvm_global.h"
#include "uvm_kvmalloc.h"
#include "uvm_channel.h" // For UVM_GPU_SEMAPHORE_MAX_JUMP
#define UVM_SEMAPHORE_SIZE 4
#define UVM_SEMAPHORE_PAGE_SIZE PAGE_SIZE
@@ -467,9 +468,16 @@ static NvU64 update_completed_value_locked(uvm_gpu_tracking_semaphore_t *trackin
// push, it's easily guaranteed because of the small number of GPFIFO
// entries available per channel (there could be at most as many pending
// pushes as GPFIFO entries).
if (new_sem_value < old_sem_value)
if (unlikely(new_sem_value < old_sem_value))
new_value += 1ULL << 32;
// Check for unexpected large jumps of the semaphore value
UVM_ASSERT_MSG_RELEASE(new_value - old_value <= UVM_GPU_SEMAPHORE_MAX_JUMP,
"GPU %s unexpected semaphore (CPU VA 0x%llx) jump from 0x%llx to 0x%llx\n",
tracking_semaphore->semaphore.page->pool->gpu->parent->name,
(NvU64)(uintptr_t)tracking_semaphore->semaphore.payload,
old_value, new_value);
// Use an atomic write even though the spinlock is held so that the value can
// be (carefully) read atomically outside of the lock.
//

78
kernel-open/nvidia-uvm/uvm_gpu_semaphore_test.c
View File

@@ -27,6 +27,18 @@
#include "uvm_va_space.h"
#include "uvm_kvmalloc.h"
static NV_STATUS set_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 new_value)
{
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem) == new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value - 1));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value + 1));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
return NV_OK;
}
static NV_STATUS add_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU32 increment_by)
{
NvU64 new_value;
@@ -43,13 +55,45 @@ static NV_STATUS add_and_test(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU32
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)new_value);
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem) == new_value);
TEST_NV_CHECK_RET(set_and_test(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, completed));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value - 1));
TEST_CHECK_RET(!uvm_gpu_tracking_semaphore_is_value_completed(tracking_sem, new_value + 1));
TEST_CHECK_RET(uvm_gpu_tracking_semaphore_is_completed(tracking_sem));
return NV_OK;
}
// Set the current state of the sema, avoiding UVM_GPU_SEMAPHORE_MAX_JUMP
// detection.
static void manual_set(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 value)
{
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, (NvU32)value);
atomic64_set(&tracking_sem->completed_value, value);
tracking_sem->queued_value = value;
}
// Set the starting value and payload and expect a global error
static NV_STATUS set_and_expect_error(uvm_gpu_tracking_semaphore_t *tracking_sem, NvU64 starting_value, NvU32 payload)
{
manual_set(tracking_sem, starting_value);
uvm_gpu_semaphore_set_payload(&tracking_sem->semaphore, payload);
TEST_CHECK_RET(uvm_global_get_status() == NV_OK);
uvm_gpu_tracking_semaphore_update_completed_value(tracking_sem);
TEST_CHECK_RET(uvm_global_reset_fatal_error() == NV_ERR_INVALID_STATE);
return NV_OK;
}
static NV_STATUS test_invalid_jumps(uvm_gpu_tracking_semaphore_t *tracking_sem)
{
int i;
for (i = 0; i < 10; ++i) {
NvU64 base = (1ULL<<32) * i;
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base, UVM_GPU_SEMAPHORE_MAX_JUMP + 1));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base, UINT_MAX));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + i + 1, i));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + UINT_MAX / 2, UINT_MAX / 2 + UVM_GPU_SEMAPHORE_MAX_JUMP + 1));
TEST_NV_CHECK_RET(set_and_expect_error(tracking_sem, base + UINT_MAX / 2, UINT_MAX / 2 - i - 1));
}
return NV_OK;
}
@@ -73,11 +117,31 @@ static NV_STATUS test_tracking(uvm_va_space_t *va_space)
goto done;
for (i = 0; i < 100; ++i) {
status = add_and_test(&tracking_sem, UINT_MAX - 1);
status = add_and_test(&tracking_sem, UVM_GPU_SEMAPHORE_MAX_JUMP - i);
if (status != NV_OK)
goto done;
}
// Test wrap-around cases
for (i = 0; i < 100; ++i) {
// Start with a value right before wrap-around
NvU64 starting_value = (1ULL<<32) * (i + 1) - i - 1;
manual_set(&tracking_sem, starting_value);
// And set payload to after wrap-around
status = set_and_test(&tracking_sem, (1ULL<<32) * (i + 1) + i);
if (status != NV_OK)
goto done;
}
g_uvm_global.disable_fatal_error_assert = true;
uvm_release_asserts_set_global_error_for_tests = true;
status = test_invalid_jumps(&tracking_sem);
uvm_release_asserts_set_global_error_for_tests = false;
g_uvm_global.disable_fatal_error_assert = false;
if (status != NV_OK)
goto done;
done:
uvm_gpu_tracking_semaphore_free(&tracking_sem);
return status;

10
kernel-open/nvidia-uvm/uvm_push.h
View File

@@ -52,11 +52,21 @@ typedef enum
// By default all operations include a membar sys after any transfer and
// before a semaphore operation.
// This flag indicates that next operation should use no membar at all.
//
// For end of push semaphore release, this flag indicates that the push
// itself does not need a membar to be used (membar sys is the default). A
// membar may still be used, if needed to order the semaphore release
// write. See comments in uvm_channel_end_push().
UVM_PUSH_FLAG_NEXT_MEMBAR_NONE,
// By default all operations include a membar sys after any transfer and
// before a semaphore operation.
// This flag indicates that next operation should use a membar gpu instead.
//
// For end of push semaphore release, this flag indicates that the push
// itself only needs a membar gpu (the default is membar sys). A membar sys
// may still be used, if needed to order the semaphore release write. See
// comments in uvm_channel_end_push().
UVM_PUSH_FLAG_NEXT_MEMBAR_GPU,
UVM_PUSH_FLAG_COUNT,

5
kernel-open/nvidia/nv-dmabuf.c
View File

@@ -820,8 +820,13 @@ nv_dma_buf_reuse(
goto cleanup_dmabuf;
}
if (params->index > (priv->total_objects - params->numObjects))
{
status = NV_ERR_INVALID_ARGUMENT;
goto unlock_priv;
}

16
kernel-open/nvidia/nv-mmap.c
View File

@@ -132,6 +132,13 @@ nvidia_vma_access(
pageIndex = ((addr - vma->vm_start) >> PAGE_SHIFT);
pageOffset = (addr & ~PAGE_MASK);
if (!mmap_context->valid)
{
nv_printf(NV_DBG_ERRORS, "NVRM: VM: invalid mmap context\n");
@@ -430,7 +437,7 @@ static int nvidia_mmap_numa(
const nv_alloc_mapping_context_t *mmap_context)
{
NvU64 start, addr;
unsigned int pages;
NvU64 pages;
NvU64 i;
pages = NV_VMA_SIZE(vma) >> PAGE_SHIFT;
@@ -509,6 +516,13 @@ int nvidia_mmap_helper(
NvU64 access_start = mmap_context->access_start;
NvU64 access_len = mmap_context->access_size;
if (IS_REG_OFFSET(nv, access_start, access_len))
{
if (nv_encode_caching(&vma->vm_page_prot, NV_MEMORY_UNCACHED,

5
kernel-open/nvidia/nv.c
View File

@@ -1467,6 +1467,11 @@ static int nv_open_device(nv_state_t *nv, nvidia_stack_t *sp)
return -ENODEV;
}
if ( ! (nv->flags & NV_FLAG_OPEN))
{
/* Sanity check: !NV_FLAG_OPEN requires usage_count == 0 */

3
kernel-open/nvidia/nvidia.Kbuild
View File

@@ -219,6 +219,7 @@ NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_dram_clk_to_mc_clk
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_get_dram_num_channels
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_tegra_dram_types
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_pxm_to_node
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_screen_info
NV_CONFTEST_TYPE_COMPILE_TESTS += file_operations
NV_CONFTEST_TYPE_COMPILE_TESTS += kuid_t
@@ -242,9 +243,9 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += vmalloc_has_pgprot_t_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += mm_has_mmap_lock
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_channel_state
NV_CONFTEST_TYPE_COMPILE_TESTS += pci_dev_has_ats_enabled
NV_CONFTEST_TYPE_COMPILE_TESTS += mt_device_gre
NV_CONFTEST_TYPE_COMPILE_TESTS += remove_memory_has_nid_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += add_memory_driver_managed_has_mhp_flags_arg
NV_CONFTEST_TYPE_COMPILE_TESTS += num_registered_fb
NV_CONFTEST_GENERIC_COMPILE_TESTS += dom0_kernel_present
NV_CONFTEST_GENERIC_COMPILE_TESTS += nvidia_vgpu_kvm_build

7
kernel-open/nvidia/nvlink_linux.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2015-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -207,7 +207,10 @@ static int nvlink_fops_release(struct inode *inode, struct file *filp)
nvlink_print(NVLINK_DBG_INFO, "nvlink driver close\n");
WARN_ON(private == NULL);
mutex_lock(&nvlink_drvctx.lock);

61
kernel-open/nvidia/os-interface.c
View File

@@ -1120,31 +1120,58 @@ void NV_API_CALL os_get_screen_info(
NvU64 consoleBar2Address
)
{
#if defined(CONFIG_FB)
int i;
*pPhysicalAddress = 0;
*pFbWidth = *pFbHeight = *pFbDepth = *pFbPitch = 0;
for (i = 0; i < num_registered_fb; i++)
#if defined(CONFIG_FB) && defined(NV_NUM_REGISTERED_FB_PRESENT)
if (num_registered_fb > 0)
{
if (!registered_fb[i])
continue;
int i;
/* Make sure base address is mapped to GPU BAR */
if ((registered_fb[i]->fix.smem_start == consoleBar1Address) ||
(registered_fb[i]->fix.smem_start == consoleBar2Address))
for (i = 0; i < num_registered_fb; i++)
{
*pPhysicalAddress = registered_fb[i]->fix.smem_start;
*pFbWidth = registered_fb[i]->var.xres;
*pFbHeight = registered_fb[i]->var.yres;
*pFbDepth = registered_fb[i]->var.bits_per_pixel;
*pFbPitch = registered_fb[i]->fix.line_length;
break;
if (!registered_fb[i])
continue;
/* Make sure base address is mapped to GPU BAR */
if ((registered_fb[i]->fix.smem_start == consoleBar1Address) ||
(registered_fb[i]->fix.smem_start == consoleBar2Address))
{
*pPhysicalAddress = registered_fb[i]->fix.smem_start;
*pFbWidth = registered_fb[i]->var.xres;
*pFbHeight = registered_fb[i]->var.yres;
*pFbDepth = registered_fb[i]->var.bits_per_pixel;
*pFbPitch = registered_fb[i]->fix.line_length;
break;
}
}
}
#elif NV_IS_EXPORT_SYMBOL_PRESENT_screen_info
/*
* If there is not a framebuffer console, return 0 size.
*
* orig_video_isVGA is set to 1 during early Linux kernel
* initialization, and then will be set to a value, such as
* VIDEO_TYPE_VLFB or VIDEO_TYPE_EFI if an fbdev console is used.
*/
if (screen_info.orig_video_isVGA > 1)
{
NvU64 physAddr = screen_info.lfb_base;
#if defined(VIDEO_CAPABILITY_64BIT_BASE)
physAddr |= (NvU64)screen_info.ext_lfb_base << 32;
#endif
/* Make sure base address is mapped to GPU BAR */
if ((physAddr == consoleBar1Address) ||
(physAddr == consoleBar2Address))
{
*pPhysicalAddress = physAddr;
*pFbWidth = screen_info.lfb_width;
*pFbHeight = screen_info.lfb_height;
*pFbDepth = screen_info.lfb_depth;
*pFbPitch = screen_info.lfb_linelength;
}
}
#else
*pPhysicalAddress = 0;
*pFbWidth = *pFbHeight = *pFbDepth = *pFbPitch = 0;
#endif
}