diff --git a/README.md b/README.md index 356532fb6..ef3130d15 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # NVIDIA Linux Open GPU Kernel Module Source This is the source release of the NVIDIA Linux open GPU kernel modules, -version 580.76.05. +version 580.82.07. ## How to Build @@ -17,7 +17,7 @@ as root: Note that the kernel modules built here must be used with GSP firmware and user-space NVIDIA GPU driver components from a corresponding -580.76.05 driver release. This can be achieved by installing +580.82.07 driver release. This can be achieved by installing the NVIDIA GPU driver from the .run file using the `--no-kernel-modules` option. E.g., @@ -185,7 +185,7 @@ table below). For details on feature support and limitations, see the NVIDIA GPU driver end user README here: -https://us.download.nvidia.com/XFree86/Linux-x86_64/580.76.05/README/kernel_open.html +https://us.download.nvidia.com/XFree86/Linux-x86_64/580.82.07/README/kernel_open.html For vGPU support, please refer to the README.vgpu packaged in the vGPU Host Package for more details. @@ -749,6 +749,7 @@ Subsystem Device ID. | NVIDIA A10 | 2236 10DE 1482 | | NVIDIA A10G | 2237 10DE 152F | | NVIDIA A10M | 2238 10DE 1677 | +| NVIDIA H20 NVL16 | 230E 10DE 20DF | | NVIDIA H100 NVL | 2321 10DE 1839 | | NVIDIA H800 PCIe | 2322 10DE 17A4 | | NVIDIA H800 | 2324 10DE 17A6 | @@ -949,9 +950,10 @@ Subsystem Device ID. | NVIDIA GB200 | 2941 10DE 20D5 | | NVIDIA GB200 | 2941 10DE 21C9 | | NVIDIA GB200 | 2941 10DE 21CA | +| NVIDIA DRIVE P2021 | 29BB 10DE 207C | | NVIDIA GeForce RTX 5090 | 2B85 | | NVIDIA GeForce RTX 5090 D | 2B87 | -| NVIDIA GeForce RTX 5090 D V2 | 2B8C | +| NVIDIA GeForce RTX 5090 D v2 | 2B8C | | NVIDIA RTX PRO 6000 Blackwell Workstation Edition | 2BB1 1028 204B | | NVIDIA RTX PRO 6000 Blackwell Workstation Edition | 2BB1 103C 204B | | NVIDIA RTX PRO 6000 Blackwell Workstation Edition | 2BB1 10DE 204B | @@ -965,6 +967,7 @@ Subsystem Device ID. | NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition | 2BB4 10DE 204C | | NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition | 2BB4 17AA 204C | | NVIDIA RTX PRO 6000 Blackwell Server Edition | 2BB5 10DE 204E | +| NVIDIA RTX 6000D | 2BB9 10DE 2091 | | NVIDIA GeForce RTX 5080 | 2C02 | | NVIDIA GeForce RTX 5070 Ti | 2C05 | | NVIDIA GeForce RTX 5090 Laptop GPU | 2C18 | @@ -1005,3 +1008,5 @@ Subsystem Device ID. | NVIDIA GeForce RTX 5070 Ti Laptop GPU | 2F18 | | NVIDIA RTX PRO 3000 Blackwell Generation Laptop GPU | 2F38 | | NVIDIA GeForce RTX 5070 Ti Laptop GPU | 2F58 | +| NVIDIA B300 SXM6 AC | 3182 10DE 20E6 | +| NVIDIA GB300 | 31C2 10DE 21F1 | diff --git a/kernel-open/Kbuild b/kernel-open/Kbuild index 0e74a6175..910434b2a 100644 --- a/kernel-open/Kbuild +++ b/kernel-open/Kbuild @@ -79,7 +79,7 @@ ccflags-y += -I$(src)/common/inc ccflags-y += -I$(src) ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args ccflags-y += -D__KERNEL__ -DMODULE -DNVRM -ccflags-y += -DNV_VERSION_STRING=\"580.76.05\" +ccflags-y += -DNV_VERSION_STRING=\"580.82.07\" # Include and link Tegra out-of-tree modules. ifneq ($(wildcard /usr/src/nvidia/nvidia-oot),) diff --git a/kernel-open/common/inc/nv-linux.h b/kernel-open/common/inc/nv-linux.h index 28972bb9f..c707c97f3 100644 --- a/kernel-open/common/inc/nv-linux.h +++ b/kernel-open/common/inc/nv-linux.h @@ -827,6 +827,10 @@ extern void *nvidia_stack_t_cache; * d50d82faa0c964e31f7a946ba8aba7c715ca7ab0 (4.18) fixes this issue by cleaning * up sysfs entry within slab_mutex, so the entry is deleted before a cache with * the same attributes could be created. + * The definition for sysfs_slab_unlink() was moved to mm/slab.h in commit + * 19975f83412f ("mm/slab: move the rest of slub_def.h to mm/slab.h") (6.8). + * Since we can't conftest mm/slab.h, use the fact that linux/slub_def.h was + * removed by the commit. * * To workaround this kernel issue, we take two steps: * - Create unmergeable caches: a kmem_cache with a constructor is unmergeable. @@ -838,7 +842,7 @@ extern void *nvidia_stack_t_cache; * wait for the timestamp to increment by at least one to ensure that we do * not hit a name conflict in cache create -> destroy (async) -> create cycle. */ -#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) +#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) && defined(NV_LINUX_SLUB_DEF_H_PRESENT) static inline void nv_kmem_ctor_dummy(void *arg) { (void)arg; @@ -866,7 +870,7 @@ static inline void nv_kmem_ctor_dummy(void *arg) static inline void *nv_kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) { -#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) +#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) && defined(NV_LINUX_SLUB_DEF_H_PRESENT) /* * We cannot call kmem_cache_zalloc directly as it adds the __GFP_ZERO * flag. This flag together with the presence of a slab constructor is @@ -1577,7 +1581,7 @@ static inline struct kmem_cache *nv_kmem_cache_create(const char *name, unsigned char *name_unique; struct kmem_cache *cache; -#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) +#if !defined(NV_SYSFS_SLAB_UNLINK_PRESENT) && defined(NV_LINUX_SLUB_DEF_H_PRESENT) size_t len; NvU64 tm_ns = nv_ktime_get_raw_ns(); diff --git a/kernel-open/conftest.sh b/kernel-open/conftest.sh index 5078d26c6..7de4f4af8 100755 --- a/kernel-open/conftest.sh +++ b/kernel-open/conftest.sh @@ -2302,6 +2302,43 @@ compile_test() { fi ;; + drm_fb_create_takes_format_info) + # + # Determine if a `struct drm_format_info *` is passed into + # the .fb_create callback. If so, it will have 4 arguments. + # This parameter was added in commit 81112eaac559 ("drm: + # Pass the format info to .fb_create") in linux-next + # (2025-07-16) + CODE=" + #include + #include + + static const struct drm_mode_config_funcs funcs; + void conftest_drm_fb_create_takes_format_info(void) { + funcs.fb_create(NULL, NULL, NULL, NULL); + }" + + compile_check_conftest "$CODE" "NV_DRM_FB_CREATE_TAKES_FORMAT_INFO" "" "types" + ;; + + drm_fill_fb_struct_takes_format_info) + # + # Determine if a `struct drm_format_info *` is passed into + # drm_helper_mode_fill_fb_struct(). If so, it will have 4 arguments. + # This parameter was added in commit a34cc7bf1034 ("drm: + # Allow the caller to pass in the format info to + # drm_helper_mode_fill_fb_struct()") in linux-next + # (2025-07-16) + CODE=" + #include + + void conftest_drm_fill_fb_struct_takes_format_info(void) { + drm_helper_mode_fill_fb_struct(NULL, NULL, NULL, NULL); + }" + + compile_check_conftest "$CODE" "NV_DRM_FILL_FB_STRUCT_TAKES_FORMAT_INFO" "" "types" + ;; + drm_connector_funcs_have_mode_in_name) # # Determine if _mode_ is present in connector function names. We diff --git a/kernel-open/header-presence-tests.mk b/kernel-open/header-presence-tests.mk index 0374b230e..6242277ae 100644 --- a/kernel-open/header-presence-tests.mk +++ b/kernel-open/header-presence-tests.mk @@ -37,6 +37,7 @@ NV_HEADER_PRESENCE_TESTS = \ linux/iosys-map.h \ linux/vfio_pci_core.h \ linux/cc_platform.h \ + linux/slub_def.h \ asm/mshyperv.h \ crypto/sig.h diff --git a/kernel-open/nvidia-drm/nvidia-drm-fb.c b/kernel-open/nvidia-drm/nvidia-drm-fb.c index 5977953c3..20602ed53 100644 --- a/kernel-open/nvidia-drm/nvidia-drm-fb.c +++ b/kernel-open/nvidia-drm/nvidia-drm-fb.c @@ -242,6 +242,9 @@ fail: struct drm_framebuffer *nv_drm_framebuffer_create( struct drm_device *dev, struct drm_file *file, +#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO) + const struct drm_format_info *info, +#endif const struct drm_mode_fb_cmd2 *cmd) { struct nv_drm_device *nv_dev = to_nv_device(dev); @@ -289,6 +292,9 @@ struct drm_framebuffer *nv_drm_framebuffer_create( drm_helper_mode_fill_fb_struct( dev, &nv_fb->base, +#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO) + info, +#endif cmd); /* diff --git a/kernel-open/nvidia-drm/nvidia-drm-fb.h b/kernel-open/nvidia-drm/nvidia-drm-fb.h index a7a83048c..2f914f94c 100644 --- a/kernel-open/nvidia-drm/nvidia-drm-fb.h +++ b/kernel-open/nvidia-drm/nvidia-drm-fb.h @@ -53,6 +53,9 @@ static inline struct nv_drm_framebuffer *to_nv_framebuffer( struct drm_framebuffer *nv_drm_framebuffer_create( struct drm_device *dev, struct drm_file *file, +#if defined(NV_DRM_FB_CREATE_TAKES_FORMAT_INFO) + const struct drm_format_info *info, +#endif const struct drm_mode_fb_cmd2 *cmd); #endif /* NV_DRM_AVAILABLE */ diff --git a/kernel-open/nvidia-drm/nvidia-drm-modeset.c b/kernel-open/nvidia-drm/nvidia-drm-modeset.c index 2ec089177..57116712f 100644 --- a/kernel-open/nvidia-drm/nvidia-drm-modeset.c +++ b/kernel-open/nvidia-drm/nvidia-drm-modeset.c @@ -702,6 +702,14 @@ int nv_drm_atomic_commit(struct drm_device *dev, return ret; } + /* + * Used to update legacy modeset state pointers to support UAPIs not updated + * by the core atomic modeset infrastructure. + * + * Example: /sys/class/drm//enabled + */ + drm_atomic_helper_update_legacy_modeset_state(dev, state); + /* * nv_drm_atomic_commit_internal() must not return failure after * calling drm_atomic_helper_swap_state(). diff --git a/kernel-open/nvidia-drm/nvidia-drm-sources.mk b/kernel-open/nvidia-drm/nvidia-drm-sources.mk index d8c9271ae..926eb76ee 100644 --- a/kernel-open/nvidia-drm/nvidia-drm-sources.mk +++ b/kernel-open/nvidia-drm/nvidia-drm-sources.mk @@ -106,3 +106,4 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_gem_prime_mmap NV_CONFTEST_TYPE_COMPILE_TESTS += drm_output_poll_changed NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_date NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_helper_funcs_mode_valid_has_const_mode_arg +NV_CONFTEST_TYPE_COMPILE_TESTS += drm_fb_create_takes_format_info diff --git a/kernel-open/nvidia-uvm/uvm_blackwell.c b/kernel-open/nvidia-uvm/uvm_blackwell.c index 7b4b0647e..4e28c43f7 100644 --- a/kernel-open/nvidia-uvm/uvm_blackwell.c +++ b/kernel-open/nvidia-uvm/uvm_blackwell.c @@ -143,8 +143,13 @@ void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu) // by UVM. if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100 && parent_gpu->rm_info.gpuImplementation == - NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10B) + NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10B) { parent_gpu->is_integrated_gpu = true; + // GB10B has sticky L2 coherent cache lines. + // For details, refer to the comments in uvm_gpu.h + // where this field is declared. + parent_gpu->sticky_l2_coherent_cache_lines = true; + } if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB200 && parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB20B) diff --git a/kernel-open/nvidia-uvm/uvm_gpu.h b/kernel-open/nvidia-uvm/uvm_gpu.h index 760266684..c2f424a50 100644 --- a/kernel-open/nvidia-uvm/uvm_gpu.h +++ b/kernel-open/nvidia-uvm/uvm_gpu.h @@ -1132,6 +1132,15 @@ struct uvm_parent_gpu_struct // by UVM. bool is_integrated_gpu; + // True if the GPU has sticky L2 coherent cache lines that prevent + // caching of system memory. GB10B experiences "sticky" lines. + // Bug 4577236 outlines the issue. Essentially normal eviction of coherent + // cache lines is prevented, causing "sticky" lines that persist until + // invalidate/snoop. This limits L2 cache availability and can cause + // cross-context interference. This is fixed in GB20B/GB20C. This field is + // set for specific GPU implementations that have this limitation i.e. GB10B. + bool sticky_l2_coherent_cache_lines; + struct { // If true, the granularity of key rotation is a single channel. If @@ -1560,6 +1569,14 @@ static NvU64 uvm_gpu_retained_count(uvm_gpu_t *gpu) void uvm_parent_gpu_kref_put(uvm_parent_gpu_t *gpu); // Returns a GPU peer pair index in the range [0 .. UVM_MAX_UNIQUE_GPU_PAIRS). + +static bool uvm_parent_gpu_supports_full_coherence(uvm_parent_gpu_t *parent_gpu) +{ + // TODO: Bug 5310178: Replace this with the value returned by RM to check + // if the GPU supports full coherence. + return parent_gpu->is_integrated_gpu; +} + NvU32 uvm_gpu_pair_index(const uvm_gpu_id_t id0, const uvm_gpu_id_t id1); // Either retains an existing PCIe peer entry or creates a new one. In both diff --git a/kernel-open/nvidia-uvm/uvm_hmm.c b/kernel-open/nvidia-uvm/uvm_hmm.c index 7fde671da..3e995e2de 100644 --- a/kernel-open/nvidia-uvm/uvm_hmm.c +++ b/kernel-open/nvidia-uvm/uvm_hmm.c @@ -92,10 +92,6 @@ static __always_inline bool nv_PageSwapCache(struct page *page) #endif } -static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block, - uvm_page_index_t page_index, - struct page *page); - typedef struct { uvm_processor_id_t processor_id; @@ -364,9 +360,10 @@ void uvm_hmm_unregister_gpu(uvm_va_space_t *va_space, uvm_gpu_t *gpu, struct mm_ // This check is racy because nothing stops the page being freed and // even reused. That doesn't matter though - worst case the // migration fails, we retry and find the va_space doesn't match. - if (uvm_pmm_devmem_page_to_va_space(page) == va_space) + if (uvm_pmm_devmem_page_to_va_space(page) == va_space) { if (uvm_hmm_pmm_gpu_evict_pfn(pfn) != NV_OK) retry = true; + } } } while (retry); @@ -1353,6 +1350,7 @@ void uvm_hmm_block_add_eviction_mappings(uvm_va_space_t *va_space, uvm_tracker_t local_tracker = UVM_TRACKER_INIT(); uvm_va_policy_node_t *node; uvm_va_block_region_t region; + const uvm_va_policy_t *policy; uvm_processor_mask_t *map_processors = &block_context->hmm.map_processors_eviction; uvm_processor_id_t id; NV_STATUS tracker_status; @@ -1364,8 +1362,8 @@ void uvm_hmm_block_add_eviction_mappings(uvm_va_space_t *va_space, uvm_mutex_lock(&va_block->lock); - uvm_for_each_va_policy_node_in(node, va_block, va_block->start, va_block->end) { - for_each_id_in_mask(id, &node->policy.accessed_by) { + uvm_for_each_va_policy_in(policy, va_block, va_block->start, va_block->end, node, region) { + for_each_id_in_mask(id, &policy->accessed_by) { status = hmm_set_accessed_by_start_end_locked(va_block, block_context, id, @@ -1380,7 +1378,7 @@ void uvm_hmm_block_add_eviction_mappings(uvm_va_space_t *va_space, // Exclude the processors that have been already mapped due to // AccessedBy. - uvm_processor_mask_andnot(map_processors, &va_block->evicted_gpus, &node->policy.accessed_by); + uvm_processor_mask_andnot(map_processors, &va_block->evicted_gpus, &policy->accessed_by); for_each_gpu_id_in_mask(id, map_processors) { uvm_gpu_t *gpu = uvm_gpu_get(id); @@ -1613,7 +1611,7 @@ static NV_STATUS hmm_va_block_cpu_page_populate(uvm_va_block_t *va_block, status = uvm_va_block_map_cpu_chunk_on_gpus(va_block, chunk); if (status != NV_OK) { - uvm_cpu_chunk_remove_from_block(va_block, page_to_nid(page), page_index); + uvm_cpu_chunk_remove_from_block(va_block, chunk, page_to_nid(page), page_index); uvm_cpu_chunk_free(chunk); } @@ -1632,7 +1630,7 @@ static void hmm_va_block_cpu_unpopulate_chunk(uvm_va_block_t *va_block, !uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); UVM_ASSERT(uvm_cpu_chunk_get_size(chunk) == PAGE_SIZE); - uvm_cpu_chunk_remove_from_block(va_block, chunk_nid, page_index); + uvm_cpu_chunk_remove_from_block(va_block, chunk, chunk_nid, page_index); uvm_va_block_unmap_cpu_chunk_on_gpus(va_block, chunk); uvm_cpu_chunk_free(chunk); } @@ -1657,14 +1655,45 @@ static void hmm_va_block_cpu_page_unpopulate(uvm_va_block_t *va_block, uvm_page_ } } -static bool hmm_va_block_cpu_page_is_same(uvm_va_block_t *va_block, - uvm_page_index_t page_index, - struct page *page) +// Insert the given sysmem page. +// Note that we might have a driver allocated sysmem page for staged GPU to GPU +// copies and that Linux may independently have allocated a page. +// If so, we have to free the driver page and use the one from Linux. +static NV_STATUS hmm_va_block_cpu_page_insert_or_replace(uvm_va_block_t *va_block, + uvm_page_index_t page_index, + struct page *page, + uvm_page_mask_t *populated_page_mask) { - struct page *old_page = uvm_va_block_get_cpu_page(va_block, page_index); + NV_STATUS status; - UVM_ASSERT(uvm_cpu_chunk_is_hmm(uvm_cpu_chunk_get_chunk_for_page(va_block, page_to_nid(page), page_index))); - return old_page == page; + if (uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { + uvm_cpu_chunk_t *cpu_chunk = uvm_cpu_chunk_get_chunk_for_page(va_block, page_to_nid(page), page_index); + + // Check to see if the CPU chunk already refers to the given page. + if (cpu_chunk && + uvm_cpu_chunk_is_hmm(cpu_chunk) && + uvm_cpu_chunk_get_cpu_page(va_block, cpu_chunk, page_index) == page) { + + UVM_ASSERT(uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU)); + UVM_ASSERT(uvm_va_block_cpu_is_page_resident_on(va_block, page_to_nid(page), page_index)); + + return NV_OK; + } + + // A driver allocated CPU chunk could have a different NUMA node ID. + hmm_va_block_cpu_page_unpopulate(va_block, page_index, NULL); + } + + status = hmm_va_block_cpu_page_populate(va_block, page_index, page); + if (status != NV_OK) + return status; + + // Record that we populated this page. hmm_block_cpu_fault_locked() + // uses this to ensure pages that don't migrate get remote mapped. + if (populated_page_mask) + uvm_page_mask_set(populated_page_mask, page_index); + + return NV_OK; } // uvm_va_block_service_copy() and uvm_va_block_service_finish() expect the @@ -1718,6 +1747,67 @@ static void cpu_mapping_clear(uvm_va_block_t *va_block, uvm_page_index_t page_in uvm_processor_mask_clear(&va_block->mapped, UVM_ID_CPU); } +static void gpu_chunk_free(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, + uvm_va_block_gpu_state_t *gpu_state, + uvm_page_index_t page_index) +{ + uvm_gpu_chunk_t *gpu_chunk = gpu_state->chunks[page_index]; + + if (gpu_chunk->state != UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED || gpu_chunk->is_referenced) + return; + + UVM_ASSERT(gpu_chunk->va_block == va_block); + UVM_ASSERT(gpu_chunk->va_block_page_index == page_index); + + uvm_mmu_chunk_unmap(gpu_chunk, &va_block->tracker); + gpu_state->chunks[page_index] = NULL; + if (va_block_retry) { + list_move_tail(&gpu_chunk->list, &va_block_retry->free_chunks); + } + else { + list_del_init(&gpu_chunk->list); + uvm_pmm_gpu_free(&uvm_gpu_chunk_get_gpu(gpu_chunk)->pmm, gpu_chunk, NULL); + } +} + +static void gpu_chunk_free_region(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, + uvm_gpu_id_t gpu_id, + uvm_va_block_region_t region, + const uvm_page_mask_t *page_mask) +{ + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(va_block, gpu_id); + uvm_page_index_t page_index; + + for_each_va_block_page_in_region_mask(page_index, page_mask, region) + gpu_chunk_free(va_block, va_block_retry, gpu_state, page_index); +} + +static void gpu_chunk_free_preallocated(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry) +{ + uvm_gpu_chunk_t *gpu_chunk, *next_chunk; + + list_for_each_entry_safe(gpu_chunk, next_chunk, &va_block_retry->used_chunks, list) { + uvm_gpu_t *gpu = uvm_gpu_chunk_get_gpu(gpu_chunk); + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(va_block, gpu->id); + uvm_page_index_t page_index = gpu_chunk->va_block_page_index; + + UVM_ASSERT(gpu_state); + + UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); + UVM_ASSERT(gpu_chunk->va_block == va_block); + UVM_ASSERT(!gpu_chunk->is_referenced); + + uvm_mmu_chunk_unmap(gpu_chunk, &va_block->tracker); + gpu_state->chunks[page_index] = NULL; + + list_del_init(&gpu_chunk->list); + uvm_pmm_gpu_free(&gpu->pmm, gpu_chunk, NULL); + } +} + static void gpu_chunk_remove(uvm_va_block_t *va_block, uvm_page_index_t page_index, struct page *page) @@ -1726,20 +1816,23 @@ static void gpu_chunk_remove(uvm_va_block_t *va_block, uvm_gpu_chunk_t *gpu_chunk; uvm_gpu_id_t id; - id = uvm_gpu_chunk_get_gpu(uvm_pmm_devmem_page_to_chunk(page))->id; + gpu_chunk = uvm_pmm_devmem_page_to_chunk(page); + id = uvm_gpu_chunk_get_gpu(gpu_chunk)->id; gpu_state = uvm_va_block_gpu_state_get(va_block, id); UVM_ASSERT(gpu_state); - gpu_chunk = gpu_state->chunks[page_index]; - if (!gpu_chunk) { + if (!gpu_state->chunks[page_index]) { // If we didn't find a chunk it's because the page was unmapped for // mremap and no fault has established a new mapping. UVM_ASSERT(!uvm_page_mask_test(&gpu_state->resident, page_index)); return; } - UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); + UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED || + gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); + UVM_ASSERT(gpu_chunk->va_block == va_block); UVM_ASSERT(gpu_chunk->is_referenced); + UVM_ASSERT(gpu_chunk == gpu_state->chunks[page_index]); uvm_page_mask_clear(&gpu_state->resident, page_index); @@ -1748,42 +1841,42 @@ static void gpu_chunk_remove(uvm_va_block_t *va_block, } static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, uvm_page_index_t page_index, struct page *page) { uvm_va_block_gpu_state_t *gpu_state; uvm_gpu_chunk_t *gpu_chunk; - uvm_gpu_id_t id; + uvm_gpu_t *gpu; NV_STATUS status; - id = uvm_gpu_chunk_get_gpu(uvm_pmm_devmem_page_to_chunk(page))->id; - gpu_state = uvm_va_block_gpu_state_get(va_block, id); + gpu_chunk = uvm_pmm_devmem_page_to_chunk(page); + gpu = uvm_gpu_chunk_get_gpu(gpu_chunk); + gpu_state = uvm_va_block_gpu_state_get_alloc(va_block, gpu); - // It's possible that this is a fresh va_block we're trying to add an - // existing gpu_chunk to. This occurs for example when a GPU faults on a - // virtual address that has been remapped with mremap(). - if (!gpu_state) { - status = uvm_va_block_gpu_state_alloc(va_block); - if (status != NV_OK) - return status; - gpu_state = uvm_va_block_gpu_state_get(va_block, id); - } - - UVM_ASSERT(gpu_state); + if (!gpu_state) + return NV_ERR_NO_MEMORY; // Note that a mremap() might be to a CPU virtual address that is nolonger // aligned with a larger GPU chunk size. We would need to allocate a new // aligned GPU chunk and copy from old to new. // TODO: Bug 3368756: add support for large GPU pages. - gpu_chunk = uvm_pmm_devmem_page_to_chunk(page); UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); UVM_ASSERT(gpu_chunk->is_referenced); UVM_ASSERT(uvm_pmm_devmem_page_to_va_space(page) == va_block->hmm.va_space); - if (gpu_state->chunks[page_index] == gpu_chunk) + if (gpu_state->chunks[page_index] == gpu_chunk) { + UVM_ASSERT(gpu_chunk->va_block == va_block); + UVM_ASSERT(gpu_chunk->va_block_page_index == page_index); return NV_OK; + } - UVM_ASSERT(!gpu_state->chunks[page_index]); + if (gpu_state->chunks[page_index]) { + // In the mremap() case, if we pre-allocated a new GPU chunk for the + // destination of a potential migration but we need to free it because + // we are replacing it with the old chunk from the mremap() source. + gpu_chunk_free(va_block, va_block_retry, gpu_state, page_index); + } // In some configurations such as SR-IOV heavy, the chunk cannot be // referenced using its physical address. Create a virtual mapping. @@ -1791,7 +1884,7 @@ static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block, if (status != NV_OK) return status; - uvm_processor_mask_set(&va_block->resident, id); + uvm_processor_mask_set(&va_block->resident, gpu->id); uvm_page_mask_set(&gpu_state->resident, page_index); // It is safe to modify the page index field without holding any PMM locks @@ -1826,34 +1919,50 @@ static NV_STATUS sync_page_and_chunk_state(uvm_va_block_t *va_block, // Wait for the GPU to finish. migrate_vma_finalize() will release the // migrated source pages (or non migrating destination pages), so GPU - // opererations must be finished by then. + // opererations must be finished by then. Also, we unmap the source or + // destination so DMAs must be complete before DMA unmapping. status = uvm_tracker_wait(&va_block->tracker); for_each_va_block_page_in_region(page_index, region) { - struct page *page; + struct page *src_page; + struct page *dst_page; if (uvm_page_mask_test(same_devmem_page_mask, page_index)) continue; - // If a page migrated, clean up the source page. - // Otherwise, clean up the destination page. - if (uvm_page_mask_test(migrated_pages, page_index)) - page = migrate_pfn_to_page(src_pfns[page_index]); - else - page = migrate_pfn_to_page(dst_pfns[page_index]); - - if (!page) - continue; - - if (is_device_private_page(page)) { - gpu_chunk_remove(va_block, page_index, page); + // If the source page migrated, we have to remove our pointers to it + // because migrate_vma_finalize() will release the reference. + // TODO: Bug 3660922: Need to handle read duplication at some point. + src_page = migrate_pfn_to_page(src_pfns[page_index]); + if (src_page && uvm_page_mask_test(migrated_pages, page_index)) { + if (is_device_private_page(src_page)) + gpu_chunk_remove(va_block, page_index, src_page); + else + hmm_va_block_cpu_page_unpopulate(va_block, page_index, src_page); } - else { - // If the source page is a system memory page, - // migrate_vma_finalize() will release the reference so we should - // clear our pointer to it. - // TODO: Bug 3660922: Need to handle read duplication at some point. - hmm_va_block_cpu_page_unpopulate(va_block, page_index, page); + + dst_page = migrate_pfn_to_page(dst_pfns[page_index]); + if (dst_page) { + if (is_device_private_page(dst_page)) { + uvm_gpu_chunk_t *gpu_chunk = uvm_pmm_devmem_page_to_chunk(dst_page); + + UVM_ASSERT(gpu_chunk); + UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED || + gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); + UVM_ASSERT(gpu_chunk->is_referenced); + + // If a page migrated to the GPU, we have to unpin the + // gpu_chunk. Otherwise, clear pointers to temporary pinned + // pages that aren't migrating. + if (uvm_page_mask_test(migrated_pages, page_index)) + uvm_pmm_gpu_unpin_allocated(&uvm_gpu_chunk_get_gpu(gpu_chunk)->pmm, gpu_chunk, va_block); + else + gpu_chunk_remove(va_block, page_index, dst_page); + } + else if (!uvm_page_mask_test(migrated_pages, page_index)) { + // Clear pointer to sysmem page that will be released. + hmm_va_block_cpu_page_unpopulate(va_block, page_index, dst_page); + } } } @@ -1862,7 +1971,6 @@ static NV_STATUS sync_page_and_chunk_state(uvm_va_block_t *va_block, // Update va_block state to reflect that the page isn't migrating. static void clean_up_non_migrating_page(uvm_va_block_t *va_block, - const unsigned long *src_pfns, unsigned long *dst_pfns, uvm_page_index_t page_index) { @@ -1888,10 +1996,9 @@ static void clean_up_non_migrating_page(uvm_va_block_t *va_block, } static void clean_up_non_migrating_pages(uvm_va_block_t *va_block, - const unsigned long *src_pfns, unsigned long *dst_pfns, uvm_va_block_region_t region, - uvm_page_mask_t *page_mask) + const uvm_page_mask_t *page_mask) { uvm_page_index_t page_index; NV_STATUS status; @@ -1900,23 +2007,47 @@ static void clean_up_non_migrating_pages(uvm_va_block_t *va_block, UVM_ASSERT(status == NV_OK); for_each_va_block_page_in_region_mask(page_index, page_mask, region) { - clean_up_non_migrating_page(va_block, src_pfns, dst_pfns, page_index); + clean_up_non_migrating_page(va_block, dst_pfns, page_index); } } // CPU page fault handling. -// Fill in the dst_pfns[page_index] entry given that there is an allocated -// CPU page. -static void lock_block_cpu_page(uvm_va_block_t *va_block, - uvm_page_index_t page_index, - struct page *src_page, - unsigned long *dst_pfns, - uvm_page_mask_t *same_devmem_page_mask) +// Fill in the dst_pfns[page_index] entry with a CPU page. +// The src_pfns[page_index] page, if present, is page locked. +static NV_STATUS alloc_page_on_cpu(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, + uvm_page_index_t page_index, + const unsigned long *src_pfns, + unsigned long *dst_pfns, + uvm_page_mask_t *same_devmem_page_mask, + uvm_va_block_context_t *block_context) { - uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_any_chunk_for_page(va_block, page_index); - uvm_va_block_region_t chunk_region; + struct page *src_page; struct page *dst_page; + uvm_cpu_chunk_t *chunk; + uvm_va_block_region_t chunk_region; + + if (!uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { + NV_STATUS status; + + UVM_ASSERT(!uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU) || + !uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); + + status = uvm_va_block_populate_page_cpu(va_block, page_index, block_context); + if (status != NV_OK) + return status; + } + + // This is the page that will be copied to system memory. + src_page = migrate_pfn_to_page(src_pfns[page_index]); + + // mremap may have caused us to lose the gpu_chunk associated with + // this va_block/page_index so make sure we have the correct chunk. + if (src_page && is_device_private_page(src_page)) + gpu_chunk_add(va_block, va_block_retry, page_index, src_page); + + chunk = uvm_cpu_chunk_get_any_chunk_for_page(va_block, page_index); UVM_ASSERT(chunk); UVM_ASSERT(chunk->page); @@ -1932,61 +2063,63 @@ static void lock_block_cpu_page(uvm_va_block_t *va_block, // remote mapped system memory page. It could also be a driver allocated // page for GPU-to-GPU staged copies (i.e., not a resident copy and owned // by the driver). - if (is_device_private_page(src_page)) { - // Since the page isn't mirrored, it was allocated by alloc_pages() + if (!src_page || is_device_private_page(src_page)) { + UVM_ASSERT(!uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU) || + !uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); + + // If the page isn't mirrored, it was allocated by alloc_pages() // and UVM owns the reference. We leave the reference count unchanged // and mark the page pointer as mirrored since UVM is transferring // ownership to Linux and we don't want UVM to double free the page in // hmm_va_block_cpu_page_unpopulate() or block_kill(). If the page // does not migrate, it will be freed though. - UVM_ASSERT(!uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU) || - !uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); - UVM_ASSERT(chunk->type == UVM_CPU_CHUNK_TYPE_PHYSICAL); - UVM_ASSERT(page_ref_count(dst_page) == 1); - uvm_cpu_chunk_make_hmm(chunk); + if (chunk->type == UVM_CPU_CHUNK_TYPE_PHYSICAL) { + UVM_ASSERT(page_ref_count(dst_page) == 1); + uvm_cpu_chunk_make_hmm(chunk); + } + + lock_page(dst_page); + dst_pfns[page_index] = migrate_pfn(page_to_pfn(dst_page)); } else { + if (src_page != dst_page) { + // This must be a driver allocated staging page that doesn't match + // the page that migrate_vma_setup() locked. + hmm_va_block_cpu_unpopulate_chunk(va_block, chunk, page_to_nid(dst_page), page_index); + hmm_va_block_cpu_page_populate(va_block, page_index, src_page); + } + + UVM_ASSERT(uvm_cpu_chunk_is_hmm(chunk)); UVM_ASSERT(same_devmem_page_mask); - UVM_ASSERT(src_page == dst_page); uvm_page_mask_set(same_devmem_page_mask, page_index); // The call to migrate_vma_setup() will have inserted a migration PTE // so the CPU has no access. cpu_mapping_clear(va_block, page_index); - return; } - lock_page(dst_page); - dst_pfns[page_index] = migrate_pfn(page_to_pfn(dst_page)); -} - -static void hmm_mark_gpu_chunk_referenced(uvm_va_block_t *va_block, - uvm_gpu_t *gpu, - uvm_gpu_chunk_t *gpu_chunk) -{ - // Tell PMM to expect a callback from Linux to free the page since the - // device private struct page reference count will determine when the - // GPU chunk is free. - UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); - list_del_init(&gpu_chunk->list); - uvm_pmm_gpu_unpin_referenced(&gpu->pmm, gpu_chunk, va_block); + return NV_OK; } static void fill_dst_pfn(uvm_va_block_t *va_block, + uvm_va_block_gpu_state_t *gpu_state, uvm_gpu_t *gpu, const unsigned long *src_pfns, unsigned long *dst_pfns, uvm_page_index_t page_index, + const uvm_page_mask_t *page_mask, uvm_page_mask_t *same_devmem_page_mask) { unsigned long src_pfn = src_pfns[page_index]; - uvm_gpu_chunk_t *gpu_chunk; + uvm_gpu_chunk_t *gpu_chunk = gpu_state->chunks[page_index]; unsigned long pfn; struct page *dpage; - gpu_chunk = uvm_va_block_lookup_gpu_chunk(va_block, gpu, uvm_va_block_cpu_page_address(va_block, page_index)); UVM_ASSERT(gpu_chunk); + UVM_ASSERT(uvm_gpu_chunk_is_user(gpu_chunk)); UVM_ASSERT(gpu_chunk->log2_size == PAGE_SHIFT); + UVM_ASSERT(gpu_chunk->va_block == va_block); + pfn = uvm_pmm_gpu_devmem_get_pfn(&gpu->pmm, gpu_chunk); // If the same GPU page is both source and destination, migrate_vma_pages() @@ -1994,6 +2127,8 @@ static void fill_dst_pfn(uvm_va_block_t *va_block, // mark it as not migrating but we keep track of this so we don't confuse // it with a page that migrate_vma_pages() actually does not migrate. if ((src_pfn & MIGRATE_PFN_VALID) && (src_pfn >> MIGRATE_PFN_SHIFT) == pfn) { + UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); + UVM_ASSERT(gpu_chunk->is_referenced); uvm_page_mask_set(same_devmem_page_mask, page_index); return; } @@ -2002,90 +2137,32 @@ static void fill_dst_pfn(uvm_va_block_t *va_block, UVM_ASSERT(is_device_private_page(dpage)); UVM_ASSERT(page_pgmap(dpage)->owner == &g_uvm_global); - hmm_mark_gpu_chunk_referenced(va_block, gpu, gpu_chunk); - UVM_ASSERT(!page_count(dpage)); - zone_device_page_init(dpage); - dpage->zone_device_data = gpu_chunk; - atomic64_inc(&va_block->hmm.va_space->hmm.allocated_page_count); + if (gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) { + UVM_ASSERT(!gpu_chunk->is_referenced); + gpu_chunk->is_referenced = true; + + // Remove the GPU chunk from the retry->used_chunks list. + list_del_init(&gpu_chunk->list); + + UVM_ASSERT(!page_count(dpage)); + UVM_ASSERT(!dpage->zone_device_data); + zone_device_page_init(dpage); + dpage->zone_device_data = gpu_chunk; + atomic64_inc(&va_block->hmm.va_space->hmm.allocated_page_count); + } + else { + UVM_ASSERT(gpu_chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); + UVM_ASSERT(gpu_chunk->is_referenced); + UVM_ASSERT(uvm_pmm_devmem_page_to_chunk(dpage) == gpu_chunk); + UVM_ASSERT(page_count(dpage) == 1); + } dst_pfns[page_index] = migrate_pfn(pfn); } -static void fill_dst_pfns(uvm_va_block_t *va_block, - const unsigned long *src_pfns, - unsigned long *dst_pfns, - uvm_va_block_region_t region, - uvm_page_mask_t *page_mask, - uvm_page_mask_t *same_devmem_page_mask, - uvm_processor_id_t dest_id) -{ - uvm_gpu_t *gpu = uvm_gpu_get(dest_id); - uvm_page_index_t page_index; - - uvm_page_mask_zero(same_devmem_page_mask); - - for_each_va_block_page_in_region_mask(page_index, page_mask, region) { - if (!(src_pfns[page_index] & MIGRATE_PFN_MIGRATE)) - continue; - - fill_dst_pfn(va_block, - gpu, - src_pfns, - dst_pfns, - page_index, - same_devmem_page_mask); - } -} - -static NV_STATUS alloc_page_on_cpu(uvm_va_block_t *va_block, - uvm_page_index_t page_index, - const unsigned long *src_pfns, - unsigned long *dst_pfns, - uvm_page_mask_t *same_devmem_page_mask, - uvm_va_block_context_t *block_context) -{ - NV_STATUS status; - struct page *src_page; - struct page *dst_page; - - // This is the page that will be copied to system memory. - src_page = migrate_pfn_to_page(src_pfns[page_index]); - - if (src_page) { - // mremap may have caused us to lose the gpu_chunk associated with - // this va_block/page_index so make sure we have the correct chunk. - if (is_device_private_page(src_page)) - gpu_chunk_add(va_block, page_index, src_page); - - if (uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { - lock_block_cpu_page(va_block, page_index, src_page, dst_pfns, same_devmem_page_mask); - return NV_OK; - } - } - - UVM_ASSERT(!uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU) || - !uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); - - status = uvm_va_block_populate_page_cpu(va_block, page_index, block_context); - if (status != NV_OK) - return status; - - // TODO: Bug 3368756: add support for transparent huge pages - // Support for large CPU pages means the page_index may need fixing - // Note that we don't call get_page(dst_page) since alloc_page_vma() - // returns with a page reference count of one and we are passing - // ownership to Linux. Also, uvm_va_block_cpu_page_populate() recorded - // the page as "mirrored" so that migrate_vma_finalize() and - // hmm_va_block_cpu_page_unpopulate() don't double free the page. - dst_pfns[page_index] = block_context->hmm.dst_pfns[page_index]; - dst_page = migrate_pfn_to_page(dst_pfns[page_index]); - lock_page(dst_page); - - return NV_OK; -} - // Allocates pages on the CPU to handle migration due to a page fault static NV_STATUS fault_alloc_on_cpu(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, const unsigned long *src_pfns, unsigned long *dst_pfns, uvm_va_block_region_t region, @@ -2118,7 +2195,13 @@ static NV_STATUS fault_alloc_on_cpu(uvm_va_block_t *va_block, goto clr_mask; } - status = alloc_page_on_cpu(va_block, page_index, src_pfns, dst_pfns, same_devmem_page_mask, service_context->block_context); + status = alloc_page_on_cpu(va_block, + va_block_retry, + page_index, + src_pfns, + dst_pfns, + same_devmem_page_mask, + service_context->block_context); if (status != NV_OK) { // Ignore errors if the page is only for prefetching. if (service_context && @@ -2135,7 +2218,7 @@ static NV_STATUS fault_alloc_on_cpu(uvm_va_block_t *va_block, } if (status != NV_OK) - clean_up_non_migrating_pages(va_block, src_pfns, dst_pfns, region, page_mask); + clean_up_non_migrating_pages(va_block, dst_pfns, region, page_mask); else if (uvm_page_mask_empty(page_mask)) return NV_WARN_MORE_PROCESSING_REQUIRED; @@ -2144,6 +2227,7 @@ static NV_STATUS fault_alloc_on_cpu(uvm_va_block_t *va_block, // Allocates pages on the CPU for explicit migration calls. static NV_STATUS migrate_alloc_on_cpu(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, const unsigned long *src_pfns, unsigned long *dst_pfns, uvm_va_block_region_t region, @@ -2166,7 +2250,7 @@ static NV_STATUS migrate_alloc_on_cpu(uvm_va_block_t *va_block, continue; } - status = alloc_page_on_cpu(va_block, page_index, src_pfns, dst_pfns, same_devmem_page_mask, block_context); + status = alloc_page_on_cpu(va_block, va_block_retry, page_index, src_pfns, dst_pfns, same_devmem_page_mask, block_context); if (status != NV_OK) { // Try to migrate other pages if we can't allocate this one. if (status != NV_ERR_NO_MEMORY) @@ -2177,7 +2261,7 @@ static NV_STATUS migrate_alloc_on_cpu(uvm_va_block_t *va_block, } if (status != NV_OK) - clean_up_non_migrating_pages(va_block, src_pfns, dst_pfns, region, page_mask); + clean_up_non_migrating_pages(va_block, dst_pfns, region, page_mask); else if (uvm_page_mask_empty(page_mask)) return NV_WARN_MORE_PROCESSING_REQUIRED; @@ -2210,6 +2294,7 @@ static NV_STATUS uvm_hmm_devmem_fault_alloc_and_copy(uvm_hmm_devmem_fault_contex uvm_page_mask_copy(page_mask, &service_context->per_processor_masks[UVM_ID_CPU_VALUE].new_residency); status = fault_alloc_on_cpu(va_block, + va_block_retry, src_pfns, dst_pfns, service_context->region, @@ -2224,7 +2309,7 @@ static NV_STATUS uvm_hmm_devmem_fault_alloc_and_copy(uvm_hmm_devmem_fault_contex // location yet. status = uvm_va_block_service_copy(processor_id, UVM_ID_CPU, va_block, va_block_retry, service_context); if (status != NV_OK) - clean_up_non_migrating_pages(va_block, src_pfns, dst_pfns, service_context->region, page_mask); + clean_up_non_migrating_pages(va_block, dst_pfns, service_context->region, page_mask); return status; } @@ -2234,7 +2319,7 @@ static NV_STATUS uvm_hmm_devmem_fault_finalize_and_map(uvm_hmm_devmem_fault_cont uvm_processor_id_t processor_id; uvm_service_block_context_t *service_context; const unsigned long *src_pfns; - unsigned long *dst_pfns; + const unsigned long *dst_pfns; uvm_page_mask_t *page_mask; uvm_va_block_t *va_block; uvm_va_block_region_t region; @@ -2282,6 +2367,7 @@ static NV_STATUS uvm_hmm_devmem_fault_finalize_and_map(uvm_hmm_devmem_fault_cont } static NV_STATUS populate_region(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, unsigned long *pfns, uvm_va_block_region_t region, uvm_page_mask_t *populated_page_mask) @@ -2289,12 +2375,6 @@ static NV_STATUS populate_region(uvm_va_block_t *va_block, uvm_page_index_t page_index; NV_STATUS status; - // Make sure GPU state is allocated or else the GPU DMA mappings to - // system memory won't be saved. - status = uvm_va_block_gpu_state_alloc(va_block); - if (status != NV_OK) - return status; - for_each_va_block_page_in_region(page_index, region) { struct page *page; @@ -2328,30 +2408,18 @@ static NV_STATUS populate_region(uvm_va_block_t *va_block, // not release the device private struct page reference. Since // hmm_range_fault() did find a device private PTE, we can // re-establish the GPU chunk pointer. - status = gpu_chunk_add(va_block, page_index, page); + status = gpu_chunk_add(va_block, va_block_retry, page_index, page); if (status != NV_OK) return status; continue; } - // If a CPU chunk is already allocated, check to see it matches what - // hmm_range_fault() found. - if (uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { - UVM_ASSERT(hmm_va_block_cpu_page_is_same(va_block, page_index, page)); - } - else { - status = hmm_va_block_cpu_page_populate(va_block, page_index, page); - if (status != NV_OK) - return status; + status = hmm_va_block_cpu_page_insert_or_replace(va_block, page_index, page, populated_page_mask); + if (status != NV_OK) + return status; - // Record that we populated this page. hmm_block_cpu_fault_locked() - // uses this to ensure pages that don't migrate get remote mapped. - if (populated_page_mask) - uvm_page_mask_set(populated_page_mask, page_index); - } - - // Since we have a stable snapshot of the CPU pages, we can - // update the residency and protection information. + // Since we have a stable snapshot of the CPU pages, we can update the + // residency and mapping information. uvm_va_block_cpu_set_resident_page(va_block, page_to_nid(page), page_index); cpu_mapping_set(va_block, pfns[page_index] & HMM_PFN_WRITE, page_index); @@ -2379,6 +2447,7 @@ static bool hmm_range_fault_retry(uvm_va_block_t *va_block) // Make the region be resident on the CPU by calling hmm_range_fault() to fault // in CPU pages. static NV_STATUS hmm_make_resident_cpu(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, struct vm_area_struct *vma, unsigned long *hmm_pfns, uvm_va_block_region_t region, @@ -2426,6 +2495,7 @@ static NV_STATUS hmm_make_resident_cpu(uvm_va_block_t *va_block, return NV_WARN_MORE_PROCESSING_REQUIRED; return populate_region(va_block, + va_block_retry, hmm_pfns, region, populated_page_mask); @@ -2560,27 +2630,15 @@ static NV_STATUS hmm_block_atomic_fault_locked(uvm_processor_id_t processor_id, for_each_va_block_page_in_region(page_index, region) { struct page *page = pages[page_index]; - if (!page) { + if (!page || hmm_va_block_cpu_page_insert_or_replace(va_block, page_index, page, NULL) != NV_OK) { // Record that one of the pages isn't exclusive but keep converting // the others. status = NV_WARN_MORE_PROCESSING_REQUIRED; continue; } - // If a CPU chunk is already allocated, check to see it matches what - // make_device_exclusive_range() found. - if (uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { - UVM_ASSERT(hmm_va_block_cpu_page_is_same(va_block, page_index, page)); - UVM_ASSERT(uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU)); - UVM_ASSERT(uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); - } - else { - NV_STATUS s = hmm_va_block_cpu_page_populate(va_block, page_index, page); - - if (s == NV_OK) - uvm_va_block_cpu_set_resident_page(va_block, page_to_nid(page), page_index); - } - + // Since we have a stable snapshot of the CPU pages, we can update the + // mapping information. cpu_mapping_clear(va_block, page_index); } @@ -2641,6 +2699,7 @@ static NV_STATUS hmm_block_cpu_fault_locked(uvm_processor_id_t processor_id, .va_block = va_block, .va_block_retry = va_block_retry, .service_context = service_context, + .same_devmem_page_mask = {} }; // Normally the source page will be a device private page that is being @@ -2667,6 +2726,7 @@ static NV_STATUS hmm_block_cpu_fault_locked(uvm_processor_id_t processor_id, } status = hmm_make_resident_cpu(va_block, + va_block_retry, service_context->block_context->hmm.vma, service_context->block_context->hmm.src_pfns, region, @@ -2736,20 +2796,27 @@ static NV_STATUS hmm_block_cpu_fault_locked(uvm_processor_id_t processor_id, return status; } -static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block, - struct vm_area_struct *vma, - const unsigned long *src_pfns, - unsigned long *dst_pfns, - uvm_va_block_region_t region, - uvm_page_mask_t *page_mask, - uvm_processor_id_t dest_id, - uvm_service_block_context_t *service_context) +static NV_STATUS dmamap_src_sysmem_and_fill_dst(uvm_va_block_t *va_block, + uvm_va_block_retry_t *va_block_retry, + const unsigned long *src_pfns, + unsigned long *dst_pfns, + uvm_va_block_region_t region, + uvm_page_mask_t *page_mask, + uvm_page_mask_t *same_devmem_page_mask, + uvm_processor_id_t dest_id, + uvm_service_block_context_t *service_context) { + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(va_block, dest_id); + uvm_gpu_t *gpu = uvm_gpu_get(dest_id); uvm_page_index_t page_index; NV_STATUS status = NV_OK; + UVM_ASSERT(gpu_state); + UVM_ASSERT(gpu); UVM_ASSERT(service_context); + uvm_page_mask_zero(same_devmem_page_mask); + for_each_va_block_page_in_region_mask(page_index, page_mask, region) { struct page *src_page; @@ -2764,46 +2831,28 @@ static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block, src_page = migrate_pfn_to_page(src_pfns[page_index]); if (src_page) { if (is_device_private_page(src_page)) { - status = gpu_chunk_add(va_block, page_index, src_page); + status = gpu_chunk_add(va_block, va_block_retry, page_index, src_page); if (status != NV_OK) - break; - continue; + goto clr_mask; + + goto fill_dst; } if (nv_PageSwapCache(src_page)) { // TODO: Bug 4050579: Remove this when swap cached pages can be // migrated. + gpu_chunk_free_region(va_block, va_block_retry, dest_id, region, page_mask); status = NV_WARN_MISMATCHED_TARGET; break; } - // If the page is already allocated, it is most likely a mirrored - // page. Check to be sure it matches what we have recorded. The - // page shouldn't be a staging page from a GPU to GPU migration - // or a remote mapped atomic sysmem page because migrate_vma_setup() - // found a normal page and non-mirrored pages are only known - // privately to the UVM driver. - if (uvm_page_mask_test(&va_block->cpu.allocated, page_index)) { - UVM_ASSERT(hmm_va_block_cpu_page_is_same(va_block, page_index, src_page)); - UVM_ASSERT(uvm_processor_mask_test(&va_block->resident, UVM_ID_CPU)); - UVM_ASSERT(uvm_va_block_cpu_is_page_resident_on(va_block, NUMA_NO_NODE, page_index)); - } - else { - status = hmm_va_block_cpu_page_populate(va_block, page_index, src_page); - if (status != NV_OK) - goto clr_mask; + status = hmm_va_block_cpu_page_insert_or_replace(va_block, page_index, src_page, NULL); + if (status != NV_OK) + goto clr_mask; - // Since there is a CPU resident page, there shouldn't be one - // anywhere else. TODO: Bug 3660922: Need to handle read - // duplication at some point. - UVM_ASSERT(!uvm_va_block_page_resident_processors_count(va_block, - service_context->block_context, - page_index)); - - // migrate_vma_setup() was able to isolate and lock the page; - // therefore, it is CPU resident and not mapped. - uvm_va_block_cpu_set_resident_page(va_block, page_to_nid(src_page), page_index); - } + // Since we have a stable snapshot of the CPU pages, we can update + // the residency information. + uvm_va_block_cpu_set_resident_page(va_block, page_to_nid(src_page), page_index); // The call to migrate_vma_setup() will have inserted a migration // PTE so the CPU has no access. @@ -2822,26 +2871,37 @@ static NV_STATUS dmamap_src_sysmem_pages(uvm_va_block_t *va_block, } } + fill_dst: + fill_dst_pfn(va_block, + gpu_state, + gpu, + src_pfns, + dst_pfns, + page_index, + page_mask, + same_devmem_page_mask); + continue; clr_mask: + // Free the pre-allocated GPU chunk for non-migrating pages. + gpu_chunk_free(va_block, va_block_retry, gpu_state, page_index); + // TODO: Bug 3900774: clean up murky mess of mask clearing. uvm_page_mask_clear(page_mask, page_index); if (service_context) clear_service_context_masks(service_context, dest_id, page_index); } - if (uvm_page_mask_empty(page_mask)) - status = NV_WARN_MORE_PROCESSING_REQUIRED; + gpu_chunk_free_preallocated(va_block, va_block_retry); - if (status != NV_OK) - clean_up_non_migrating_pages(va_block, src_pfns, dst_pfns, region, page_mask); + if (status == NV_OK && uvm_page_mask_empty(page_mask)) + status = NV_WARN_MORE_PROCESSING_REQUIRED; return status; } -static NV_STATUS uvm_hmm_gpu_fault_alloc_and_copy(struct vm_area_struct *vma, - uvm_hmm_gpu_fault_event_t *uvm_hmm_gpu_fault_event) +static NV_STATUS uvm_hmm_gpu_fault_alloc_and_copy(uvm_hmm_gpu_fault_event_t *uvm_hmm_gpu_fault_event) { uvm_processor_id_t processor_id; uvm_processor_id_t new_residency; @@ -2870,14 +2930,15 @@ static NV_STATUS uvm_hmm_gpu_fault_alloc_and_copy(struct vm_area_struct *vma, uvm_page_mask_copy(page_mask, &service_context->per_processor_masks[uvm_id_value(new_residency)].new_residency); - status = dmamap_src_sysmem_pages(va_block, - vma, - src_pfns, - dst_pfns, - region, - page_mask, - new_residency, - service_context); + status = dmamap_src_sysmem_and_fill_dst(va_block, + va_block_retry, + src_pfns, + dst_pfns, + region, + page_mask, + &uvm_hmm_gpu_fault_event->same_devmem_page_mask, + new_residency, + service_context); if (status != NV_OK) return status; @@ -2885,17 +2946,7 @@ static NV_STATUS uvm_hmm_gpu_fault_alloc_and_copy(struct vm_area_struct *vma, // new location yet. status = uvm_va_block_service_copy(processor_id, new_residency, va_block, va_block_retry, service_context); if (status != NV_OK) - return status; - - // Record the destination PFNs of device private struct pages now that - // uvm_va_block_service_copy() has populated the GPU destination pages. - fill_dst_pfns(va_block, - src_pfns, - dst_pfns, - region, - page_mask, - &uvm_hmm_gpu_fault_event->same_devmem_page_mask, - new_residency); + clean_up_non_migrating_pages(va_block, dst_pfns, region, page_mask); return status; } @@ -2907,7 +2958,7 @@ static NV_STATUS uvm_hmm_gpu_fault_finalize_and_map(uvm_hmm_gpu_fault_event_t *u uvm_va_block_t *va_block; uvm_service_block_context_t *service_context; const unsigned long *src_pfns; - unsigned long *dst_pfns; + const unsigned long *dst_pfns; uvm_va_block_region_t region; uvm_page_index_t page_index; uvm_page_mask_t *page_mask; @@ -2966,6 +3017,8 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_processor_id_t processor_id, uvm_va_block_region_t region = service_context->region; uvm_hmm_gpu_fault_event_t uvm_hmm_gpu_fault_event; struct migrate_vma *args = &service_context->block_context->hmm.migrate_vma_args; + const uvm_page_mask_t *new_residency_mask = + &service_context->per_processor_masks[uvm_id_value(new_residency)].new_residency; int ret; NV_STATUS status = NV_ERR_INVALID_ADDRESS; @@ -2979,8 +3032,66 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_processor_id_t processor_id, UVM_ASSERT(vma); // If the desired destination is the CPU, try to fault in CPU pages. - if (UVM_ID_IS_CPU(new_residency)) + if (UVM_ID_IS_CPU(new_residency)) { + if (va_block_retry && !list_empty(&va_block_retry->used_chunks)) + gpu_chunk_free_preallocated(va_block, va_block_retry); + return hmm_block_cpu_fault_locked(processor_id, va_block, va_block_retry, service_context); + } + + UVM_ASSERT(va_block_retry); + + // The overall process here is to migrate pages from the CPU or GPUs to the + // faulting GPU. This is only safe because we hold the va_block lock across + // the calls to migrate_vma_pages(), uvm_hmm_gpu_fault_alloc_and_copy(), + // uvm_hmm_gpu_fault_finalize_and_map(), and migrate_vma_finalize(). + // If the va_block lock were to be dropped, eviction callbacks from RM, + // migration callbacks from CPU faults, or invalidation callbacks from + // Linux could change the va_block state which would require careful + // revalidation of the state. Also, pages are page locked which leads to + // inefficiency or potential deadlocks. + + // We pre-allocate the destination GPU pages because otherwise, + // migrate_vma_setup() could page lock the source pages and then try to + // allocate destination pages with block_alloc_gpu_chunk() which might + // unlock the va_block lock and try to evict the source page and fail. + // Note that by preallocating, we introduce 3 states instead of 2 for + // GPU chunks: + // UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED, !is_referenced + // UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED, is_referenced + // UVM_PMM_GPU_CHUNK_STATE_ALLOCATED, is_referenced + // The first state is when uvm_va_block_populate_pages_gpu() returns and + // we should call uvm_pmm_gpu_free() if the chunk isn't needed. + // The second state is after the source pages are pinned and we know which + // chunks will be used for DMA and passed to migrate_vma_pages() dst_pfns[]. + // The third state is when migrate_vma_pages() commits to the migration and + // the GPU chunk will be marked resident. + // The is_referenced flag is just for sanity checking so it is clear when + // ownership for freeing the chunk changes from the driver to Linux's + // page_free() callback. The TEMP_PINNED/is_referenced state could be + // replaced with ALLOCATED/is_referenced but ALLOCATED implies the chunk + // could be evicted (except we hold the va_block lock) and seems safer + // to leave it in the pinned state until we are about to call + // migrate_vma_finalize(). + // Also note that we have to free any pre-allocated pages because otherwise + // they would be marked ALLOCATED in uvm_va_block_retry_deinit() and + // we can't free them in uvm_va_block_retry_deinit() because the va_block + // lock might not be held and freeing the GPU chunk requires unmapping and + // clearing the gpu_state->chunks[] entry. + // Also note that the new_residency and new_residency_mask can change each + // time uvm_va_block_populate_pages_gpu() returns + // NV_ERR_MORE_PROCESSING_REQUIRED (based on thrashing and other reasons) + // so there might be pre-allocated chunks not in region. + status = uvm_va_block_populate_pages_gpu(va_block, + va_block_retry, + new_residency, + region, + new_residency_mask); + if (status != NV_OK) { + if (status != NV_ERR_MORE_PROCESSING_REQUIRED) + gpu_chunk_free_preallocated(va_block, va_block_retry); + return status; + } uvm_hmm_gpu_fault_event.processor_id = processor_id; uvm_hmm_gpu_fault_event.new_residency = new_residency; @@ -3000,21 +3111,7 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_processor_id_t processor_id, ret = migrate_vma_setup_locked(args, va_block); UVM_ASSERT(!ret); - // The overall process here is to migrate pages from the CPU or GPUs to the - // faulting GPU. - // This is safe because we hold the va_block lock across the calls to - // uvm_hmm_gpu_fault_alloc_and_copy(), migrate_vma_pages(), - // uvm_hmm_gpu_fault_finalize_and_map(), and migrate_vma_finalize(). - // If uvm_hmm_gpu_fault_alloc_and_copy() needs to drop the va_block - // lock, a sequence number is used to tell if an invalidate() callback - // occurred while not holding the lock. If the sequence number changes, - // all the locks need to be dropped (mm, va_space, va_block) and the whole - // uvm_va_block_service_locked() called again. Otherwise, there were no - // conflicting invalidate callbacks and our snapshots of the CPU page - // tables are accurate and can be used to DMA pages and update GPU page - // tables. TODO: Bug 3901904: there might be better ways of handling no - // page being migrated. - status = uvm_hmm_gpu_fault_alloc_and_copy(vma, &uvm_hmm_gpu_fault_event); + status = uvm_hmm_gpu_fault_alloc_and_copy(&uvm_hmm_gpu_fault_event); if (status == NV_WARN_MORE_PROCESSING_REQUIRED) { migrate_vma_finalize(args); @@ -3023,12 +3120,16 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_processor_id_t processor_id, // We do know that none of the pages in the region are zero pages // since migrate_vma_setup() would have reported that information. // Try to make it resident in system memory and retry the migration. + // TODO: Bug 3901904: there might be better ways of handling no page + // being migrated. status = hmm_make_resident_cpu(va_block, + va_block_retry, service_context->block_context->hmm.vma, service_context->block_context->hmm.src_pfns, region, service_context->access_type, NULL); + return NV_WARN_MORE_PROCESSING_REQUIRED; } @@ -3045,8 +3146,7 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_processor_id_t processor_id, return status; } -static NV_STATUS uvm_hmm_migrate_alloc_and_copy(struct vm_area_struct *vma, - uvm_hmm_migrate_event_t *uvm_hmm_migrate_event) +static NV_STATUS uvm_hmm_migrate_alloc_and_copy(uvm_hmm_migrate_event_t *uvm_hmm_migrate_event) { uvm_va_block_t *va_block; uvm_va_block_retry_t *va_block_retry; @@ -3073,6 +3173,7 @@ static NV_STATUS uvm_hmm_migrate_alloc_and_copy(struct vm_area_struct *vma, if (UVM_ID_IS_CPU(dest_id)) { status = migrate_alloc_on_cpu(va_block, + va_block_retry, src_pfns, dst_pfns, region, @@ -3081,14 +3182,15 @@ static NV_STATUS uvm_hmm_migrate_alloc_and_copy(struct vm_area_struct *vma, service_context->block_context); } else { - status = dmamap_src_sysmem_pages(va_block, - vma, - src_pfns, - dst_pfns, - region, - page_mask, - dest_id, - service_context); + status = dmamap_src_sysmem_and_fill_dst(va_block, + va_block_retry, + src_pfns, + dst_pfns, + region, + page_mask, + &uvm_hmm_migrate_event->same_devmem_page_mask, + dest_id, + service_context); } if (status != NV_OK) @@ -3103,20 +3205,7 @@ static NV_STATUS uvm_hmm_migrate_alloc_and_copy(struct vm_area_struct *vma, NULL, uvm_hmm_migrate_event->cause); if (status != NV_OK) - return status; - - if (!UVM_ID_IS_CPU(dest_id)) { - // Record the destination PFNs of device private struct pages now that - // uvm_va_block_make_resident_copy() has populated the GPU destination - // pages. - fill_dst_pfns(va_block, - src_pfns, - dst_pfns, - region, - page_mask, - &uvm_hmm_migrate_event->same_devmem_page_mask, - dest_id); - } + clean_up_non_migrating_pages(va_block, dst_pfns, region, page_mask); return status; } @@ -3130,7 +3219,7 @@ static NV_STATUS uvm_hmm_migrate_finalize(uvm_hmm_migrate_event_t *uvm_hmm_migra uvm_page_index_t page_index; uvm_page_mask_t *page_mask; const unsigned long *src_pfns; - unsigned long *dst_pfns; + const unsigned long *dst_pfns; va_block = uvm_hmm_migrate_event->va_block; va_block_context = uvm_hmm_migrate_event->service_context->block_context; @@ -3195,6 +3284,45 @@ NV_STATUS uvm_hmm_va_block_migrate_locked(uvm_va_block_t *va_block, uvm_assert_mutex_locked(&va_block->hmm.migrate_lock); uvm_assert_mutex_locked(&va_block->lock); + // Save some time and effort if we can't migrate to a GPU. + if (UVM_ID_IS_GPU(dest_id) && uvm_hmm_must_use_sysmem(va_block, vma)) { + return hmm_make_resident_cpu(va_block, + va_block_retry, + vma, + va_block_context->hmm.src_pfns, + region, + NULL, + NULL); + } + + // The overall process here is to migrate pages from the CPU or GPUs to the + // destination processor. Note that block_migrate_add_mappings() handles + // updating GPU mappings after the migration. + // This is only safe because we hold the va_block lock across the calls to + // uvm_hmm_migrate_alloc_and_copy(), migrate_vma_pages(), + // uvm_hmm_migrate_finalize(), migrate_vma_finalize() and + // block_migrate_add_mappings(). + // If the va_block lock were to be dropped, eviction callbacks from RM, + // migration callbacks from CPU faults, or invalidation callbacks from + // Linux could change the va_block state which would require careful + // revalidation of the state. Also, pages are page locked which leads to + // inefficiency or potential deadlocks. + // tables are accurate and can be used to DMA pages and update GPU page + // tables. + + // We pre-allocate the destination GPU pages because otherwise, + // migrate_vma_setup() could page lock the source pages and then try to + // allocate destination pages with block_alloc_gpu_chunk() which might + // unlock the va_block lock and try to evict the source page and fail. + if (UVM_ID_IS_GPU(dest_id)) { + status = uvm_va_block_populate_pages_gpu(va_block, va_block_retry, dest_id, region, NULL); + if (status != NV_OK) { + if (status != NV_ERR_MORE_PROCESSING_REQUIRED) + gpu_chunk_free_preallocated(va_block, va_block_retry); + return status; + } + } + start = uvm_va_block_region_start(va_block, region); end = uvm_va_block_region_end(va_block, region); UVM_ASSERT(vma->vm_start <= start && end < vma->vm_end); @@ -3220,30 +3348,20 @@ NV_STATUS uvm_hmm_va_block_migrate_locked(uvm_va_block_t *va_block, // VMAs so if UvmMigrate() tries to migrate such a region, -EINVAL will // be returned and we will only try to make the pages be CPU resident. ret = migrate_vma_setup_locked(args, va_block); - if (ret) + if (ret) { + if (va_block_retry && !list_empty(&va_block_retry->used_chunks)) + gpu_chunk_free_preallocated(va_block, va_block_retry); + return hmm_make_resident_cpu(va_block, + va_block_retry, vma, va_block_context->hmm.src_pfns, region, NULL, NULL); + } - // The overall process here is to migrate pages from the CPU or GPUs to the - // destination processor. Note that block_migrate_add_mappings() handles - // updating GPU mappings after the migration. - // This is safe because we hold the va_block lock across the calls to - // uvm_hmm_migrate_alloc_and_copy(), migrate_vma_pages(), - // uvm_hmm_migrate_finalize(), migrate_vma_finalize() and - // block_migrate_add_mappings(). - // If uvm_hmm_migrate_alloc_and_copy() needs to drop the va_block - // lock, a sequence number is used to tell if an invalidate() callback - // occurred while not holding the lock. If the sequence number changes, - // all the locks need to be dropped (mm, va_space, va_block) and the whole - // uvm_hmm_va_block_migrate_locked() called again. Otherwise, there were no - // conflicting invalidate callbacks and our snapshots of the CPU page - // tables are accurate and can be used to DMA pages and update GPU page - // tables. - status = uvm_hmm_migrate_alloc_and_copy(vma, &uvm_hmm_migrate_event); + status = uvm_hmm_migrate_alloc_and_copy(&uvm_hmm_migrate_event); if (status == NV_WARN_MORE_PROCESSING_REQUIRED) { uvm_processor_id_t id; uvm_page_mask_t *page_mask; @@ -3266,6 +3384,7 @@ NV_STATUS uvm_hmm_va_block_migrate_locked(uvm_va_block_t *va_block, } return hmm_make_resident_cpu(va_block, + va_block_retry, vma, va_block_context->hmm.src_pfns, region, @@ -3356,6 +3475,17 @@ NV_STATUS uvm_hmm_va_block_evict_chunk_prep(uvm_va_block_t *va_block, if (ret) return errno_to_nv_status(ret); + if (!(src_pfns[page_index] & MIGRATE_PFN_MIGRATE)) + return NV_WARN_MORE_PROCESSING_REQUIRED; + + if (UVM_IS_DEBUG()) { + struct page *src_page = migrate_pfn_to_page(src_pfns[page_index]); + + UVM_ASSERT(is_device_private_page(src_page)); + UVM_ASSERT(page_pgmap(src_page)->owner == &g_uvm_global); + UVM_ASSERT(uvm_pmm_devmem_page_to_chunk(src_page) == gpu_chunk); + } + return NV_OK; } @@ -3408,9 +3538,10 @@ static NV_STATUS hmm_va_block_evict_chunks(uvm_va_block_t *va_block, // Pages resident on the GPU should not have a resident page in system // memory. // TODO: Bug 3660922: Need to handle read duplication at some point. - UVM_ASSERT(uvm_page_mask_region_empty(cpu_resident_mask, region)); + UVM_ASSERT(!uvm_page_mask_intersects(cpu_resident_mask, page_mask)); status = migrate_alloc_on_cpu(va_block, + NULL, src_pfns, dst_pfns, region, @@ -3684,7 +3815,7 @@ NV_STATUS uvm_hmm_va_block_update_residency_info(uvm_va_block_t *va_block, // Update the va_block CPU state based on the snapshot. // Note that we have to adjust the pfns address since it will be indexed // by region.first. - status = populate_region(va_block, &pfn - region.first, region, NULL); + status = populate_region(va_block, NULL, &pfn - region.first, region, NULL); uvm_mutex_unlock(&va_block->lock); uvm_hmm_migrate_finish(va_block); diff --git a/kernel-open/nvidia-uvm/uvm_migrate_pageable.c b/kernel-open/nvidia-uvm/uvm_migrate_pageable.c index ca6ca4e97..d31fa038a 100644 --- a/kernel-open/nvidia-uvm/uvm_migrate_pageable.c +++ b/kernel-open/nvidia-uvm/uvm_migrate_pageable.c @@ -330,6 +330,9 @@ static struct page *uvm_migrate_vma_alloc_page(migrate_vma_state_t *state) } } + if (!dst_page) + state->out_of_memory = true; + return dst_page; } @@ -489,7 +492,11 @@ static NV_STATUS uvm_migrate_vma_populate_anon_pages(struct vm_area_struct *vma, // Pre-allocate the dst pages and mark the ones that failed for_each_set_bit(i, page_mask, state->num_pages) { - struct page *dst_page = uvm_migrate_vma_alloc_page(state); + struct page *dst_page = NULL; + + if (!state->out_of_memory) + dst_page = uvm_migrate_vma_alloc_page(state); + if (!dst_page) { __set_bit(i, state->allocation_failed_mask.page_mask); continue; @@ -734,7 +741,11 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma, // Pre-allocate the dst pages and mark the ones that failed for_each_set_bit(i, page_mask, state->num_pages) { - struct page *dst_page = uvm_migrate_vma_alloc_page(state); + struct page *dst_page = NULL; + + if (!state->out_of_memory) + dst_page = uvm_migrate_vma_alloc_page(state); + if (!dst_page) { __set_bit(i, state->allocation_failed_mask.page_mask); continue; @@ -1486,7 +1497,7 @@ NV_STATUS uvm_migrate_pageable(uvm_migrate_args_t *uvm_migrate_args) uvm_migrate_args->dst_node_id = uvm_gpu_numa_node(gpu); } - state = kmem_cache_alloc(g_uvm_migrate_vma_state_cache, NV_UVM_GFP_FLAGS); + state = nv_kmem_cache_zalloc(g_uvm_migrate_vma_state_cache, NV_UVM_GFP_FLAGS); if (!state) return NV_ERR_NO_MEMORY; diff --git a/kernel-open/nvidia-uvm/uvm_migrate_pageable.h b/kernel-open/nvidia-uvm/uvm_migrate_pageable.h index 5a9ab0bdd..b2abfef94 100644 --- a/kernel-open/nvidia-uvm/uvm_migrate_pageable.h +++ b/kernel-open/nvidia-uvm/uvm_migrate_pageable.h @@ -177,6 +177,9 @@ typedef struct // Number of pages that are directly populated on the destination unsigned long num_populate_anon_pages; + + // Tracks if OOM condition was encountered. + bool out_of_memory; } migrate_vma_state_t; #if defined(CONFIG_MIGRATE_VMA_HELPER) diff --git a/kernel-open/nvidia-uvm/uvm_pmm_gpu.c b/kernel-open/nvidia-uvm/uvm_pmm_gpu.c index 5a8f2e491..438791861 100644 --- a/kernel-open/nvidia-uvm/uvm_pmm_gpu.c +++ b/kernel-open/nvidia-uvm/uvm_pmm_gpu.c @@ -406,7 +406,10 @@ static void chunk_pin(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) { uvm_gpu_root_chunk_t *root_chunk = root_chunk_from_chunk(pmm, chunk); - uvm_assert_spinlock_locked(&pmm->list_lock); + // The PMM list_lock must be held, but calling uvm_assert_spinlock_locked() + // is not possible here due to the absence of the UVM context pointer in + // the interrupt context when called from devmem_page_free(). + UVM_ASSERT(chunk->state != UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); chunk->state = UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED; @@ -434,7 +437,6 @@ static void chunk_unpin(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_pmm_gpu_ uvm_assert_spinlock_locked(&pmm->list_lock); UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); - UVM_ASSERT(chunk->va_block == NULL); UVM_ASSERT(chunk_is_root_chunk_pinned(pmm, chunk)); UVM_ASSERT(new_state != UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); @@ -609,8 +611,6 @@ NV_STATUS uvm_pmm_gpu_alloc_kernel(uvm_pmm_gpu_t *pmm, return status; for (i = 0; i < num_chunks; ++i) { - UVM_ASSERT(chunks[i]->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); - uvm_spin_lock(&pmm->list_lock); chunk_unpin(pmm, chunks[i], UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); chunks[i]->is_referenced = false; @@ -656,45 +656,29 @@ static void chunk_update_lists_locked(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk list_del_init(&chunk->list); } -static void gpu_unpin_temp(uvm_pmm_gpu_t *pmm, - uvm_gpu_chunk_t *chunk, - uvm_va_block_t *va_block, - bool is_referenced) +void uvm_pmm_gpu_unpin_allocated(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_va_block_t *va_block) { UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); UVM_ASSERT(uvm_gpu_chunk_is_user(chunk)); - - INIT_LIST_HEAD(&chunk->list); + UVM_ASSERT(list_empty(&chunk->list)); + UVM_ASSERT(va_block); + UVM_ASSERT(chunk->va_block == va_block); + UVM_ASSERT(chunk->va_block_page_index < uvm_va_block_num_cpu_pages(va_block)); uvm_spin_lock(&pmm->list_lock); - UVM_ASSERT(!chunk->va_block); - UVM_ASSERT(va_block); - UVM_ASSERT(chunk->va_block_page_index < uvm_va_block_num_cpu_pages(va_block)); - chunk_unpin(pmm, chunk, UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); - chunk->is_referenced = is_referenced; - chunk->va_block = va_block; chunk_update_lists_locked(pmm, chunk); uvm_spin_unlock(&pmm->list_lock); } -void uvm_pmm_gpu_unpin_allocated(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_va_block_t *va_block) -{ - gpu_unpin_temp(pmm, chunk, va_block, false); -} - -void uvm_pmm_gpu_unpin_referenced(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_va_block_t *va_block) -{ - gpu_unpin_temp(pmm, chunk, va_block, true); -} - void uvm_pmm_gpu_free(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_tracker_t *tracker) { NV_STATUS status; - if (!chunk) + // Referenced chunks are freed by Linux when the reference is released. + if (!chunk || chunk->is_referenced) return; UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED || @@ -760,6 +744,10 @@ static bool assert_chunk_mergeable(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) size_t i; UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_IS_SPLIT); + UVM_ASSERT_MSG(chunk->suballoc->allocated == num_subchunks(chunk), + "%u != %u\n", + chunk->suballoc->allocated, + num_subchunks(chunk)); UVM_ASSERT(first_child->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED || first_child->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); @@ -778,16 +766,6 @@ static bool assert_chunk_mergeable(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) } } - if (first_child->state == UVM_PMM_GPU_CHUNK_STATE_FREE) { - UVM_ASSERT(chunk->suballoc->allocated == 0); - } - else { - UVM_ASSERT_MSG(chunk->suballoc->allocated == num_subchunks(chunk), - "%u != %u\n", - chunk->suballoc->allocated, - num_subchunks(chunk)); - } - return true; } @@ -826,6 +804,7 @@ static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) else if (child_state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) { UVM_ASSERT(root_chunk->chunk.suballoc->pinned_leaf_chunks >= num_sub); root_chunk->chunk.suballoc->pinned_leaf_chunks += 1 - num_sub; + chunk->va_block = subchunk->va_block; } chunk->state = child_state; @@ -849,7 +828,7 @@ static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) UVM_ASSERT(list_empty(&subchunk->list)); if ((child_state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) && uvm_gpu_chunk_is_user(subchunk)) - UVM_ASSERT(subchunk->va_block != NULL); + UVM_ASSERT(subchunk->va_block); kmem_cache_free(CHUNK_CACHE, subchunk); } @@ -1216,7 +1195,7 @@ void uvm_pmm_gpu_mark_chunk_evicted(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) UVM_ASSERT(chunk_is_in_eviction(pmm, chunk)); UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); - UVM_ASSERT(chunk->va_block != NULL); + UVM_ASSERT(chunk->va_block); chunk->va_block = NULL; chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; @@ -1885,9 +1864,9 @@ static void init_root_chunk(uvm_pmm_gpu_t *pmm, uvm_pmm_gpu_chunk_state_string(chunk->state), uvm_gpu_name(gpu)); - UVM_ASSERT(chunk->parent == NULL); - UVM_ASSERT(chunk->suballoc == NULL); - UVM_ASSERT(chunk->va_block == NULL); + UVM_ASSERT(!chunk->parent); + UVM_ASSERT(!chunk->suballoc); + UVM_ASSERT(!chunk->va_block); UVM_ASSERT(chunk->va_block_page_index == PAGES_PER_UVM_VA_BLOCK); UVM_ASSERT(list_empty(&chunk->list)); UVM_ASSERT(uvm_gpu_chunk_get_size(chunk) == UVM_CHUNK_SIZE_MAX); @@ -2145,6 +2124,9 @@ NV_STATUS split_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) subchunk->va_block_page_index = chunk->va_block_page_index + (i * subchunk_size) / PAGE_SIZE; subchunk->is_referenced = chunk->is_referenced; } + else if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) { + subchunk->va_block = chunk->va_block; + } } // We're splitting an allocated or pinned chunk in-place. @@ -2170,6 +2152,10 @@ NV_STATUS split_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) // accounting for the root chunk itself so add the 1 back. if (chunk_is_root_chunk(chunk)) root_chunk->chunk.suballoc->pinned_leaf_chunks += 1; + + chunk->va_block = NULL; + chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; + chunk->is_referenced = false; } chunk->state = UVM_PMM_GPU_CHUNK_STATE_IS_SPLIT; @@ -2251,16 +2237,16 @@ static void chunk_free_locked(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) if (root_chunk->chunk.in_eviction) { // A root chunk with pinned subchunks would never be picked for eviction - // so this one has to be in the allocated state. Pin it and let the - // evicting thread pick it up. - UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED); - UVM_ASSERT(chunk->va_block != NULL); - UVM_ASSERT(chunk->va_block_page_index != PAGES_PER_UVM_VA_BLOCK); - UVM_ASSERT(list_empty(&chunk->list)); - chunk->va_block = NULL; - chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; - chunk->is_zero = false; - chunk_pin(pmm, chunk); + // but HMM evictions will end up here so leave the chunk pinned (or pin + // it) and let the eviction thread pick it up. + if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) { + UVM_ASSERT(chunk->va_block); + UVM_ASSERT(list_empty(&chunk->list)); + chunk->va_block = NULL; + chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; + chunk->is_zero = false; + chunk_pin(pmm, chunk); + } return; } @@ -2274,17 +2260,15 @@ static void chunk_free_locked(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk) } } - if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) { - chunk_unpin(pmm, chunk, UVM_PMM_GPU_CHUNK_STATE_FREE); - } - else { - chunk->state = UVM_PMM_GPU_CHUNK_STATE_FREE; - chunk->va_block = NULL; - } - + chunk->va_block = NULL; chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; chunk->is_zero = false; + if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) + chunk_unpin(pmm, chunk, UVM_PMM_GPU_CHUNK_STATE_FREE); + else + chunk->state = UVM_PMM_GPU_CHUNK_STATE_FREE; + chunk_update_lists_locked(pmm, chunk); } @@ -3117,6 +3101,11 @@ static bool uvm_pmm_gpu_check_orphan_pages(uvm_pmm_gpu_t *pmm) break; } + if (page->zone_device_data) { + ret = false; + break; + } + if (page_count(page)) { ret = false; break; @@ -3128,12 +3117,17 @@ static bool uvm_pmm_gpu_check_orphan_pages(uvm_pmm_gpu_t *pmm) static void devmem_page_free(struct page *page) { - uvm_va_space_t *va_space = uvm_pmm_devmem_page_to_va_space(page); uvm_gpu_chunk_t *chunk = uvm_pmm_devmem_page_to_chunk(page); uvm_gpu_t *gpu = uvm_gpu_chunk_get_gpu(chunk); - atomic64_dec(&va_space->hmm.allocated_page_count); - UVM_ASSERT(atomic64_read(&va_space->hmm.allocated_page_count) >= 0); + if (chunk->va_block) { + uvm_va_space_t *va_space = chunk->va_block->hmm.va_space; + + UVM_ASSERT(va_space); + atomic64_dec(&va_space->hmm.allocated_page_count); + UVM_ASSERT(atomic64_read(&va_space->hmm.allocated_page_count) >= 0); + } + page->zone_device_data = NULL; // We should be calling free_chunk() except that it acquires a mutex and @@ -3143,7 +3137,20 @@ static void devmem_page_free(struct page *page) spin_lock(&gpu->pmm.list_lock.lock); UVM_ASSERT(chunk->is_referenced); + + chunk->va_block = NULL; + chunk->va_block_page_index = PAGES_PER_UVM_VA_BLOCK; chunk->is_referenced = false; + + if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) { + list_del_init(&chunk->list); + chunk_pin(&gpu->pmm, chunk); + } + else { + UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); + UVM_ASSERT(list_empty(&chunk->list)); + } + list_add_tail(&chunk->list, &gpu->pmm.root_chunks.va_block_lazy_free); spin_unlock(&gpu->pmm.list_lock.lock); @@ -3430,6 +3437,7 @@ static void process_lazy_free(uvm_pmm_gpu_t *pmm) // is empty. while (!list_empty(&pmm->root_chunks.va_block_lazy_free)) { chunk = list_first_entry(&pmm->root_chunks.va_block_lazy_free, uvm_gpu_chunk_t, list); + UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED); list_del_init(&chunk->list); uvm_spin_unlock(&pmm->list_lock); @@ -3587,9 +3595,9 @@ void uvm_pmm_gpu_deinit(uvm_pmm_gpu_t *pmm) gpu = uvm_pmm_to_gpu(pmm); - UVM_ASSERT(uvm_pmm_gpu_check_orphan_pages(pmm)); nv_kthread_q_flush(&gpu->parent->lazy_free_q); UVM_ASSERT(list_empty(&pmm->root_chunks.va_block_lazy_free)); + UVM_ASSERT(uvm_pmm_gpu_check_orphan_pages(pmm)); release_free_root_chunks(pmm); if (gpu->mem_info.size != 0 && gpu_supports_pma_eviction(gpu)) diff --git a/kernel-open/nvidia-uvm/uvm_pmm_gpu.h b/kernel-open/nvidia-uvm/uvm_pmm_gpu.h index bb3199814..802257a1b 100644 --- a/kernel-open/nvidia-uvm/uvm_pmm_gpu.h +++ b/kernel-open/nvidia-uvm/uvm_pmm_gpu.h @@ -264,6 +264,11 @@ struct uvm_gpu_chunk_struct // This flag indicates an allocated user chunk is referenced by a device // private struct page PTE and therefore expects a page_free() callback. + // The flag is only for sanity checking since uvm_pmm_gpu_free() + // shouldn't be called if Linux has a device private reference to this + // chunk and devmem_page_free() should only be called from the Linux + // callback if a reference was created. + // See uvm_hmm_va_block_service_locked() and fill_dst_pfn() for details. // // This field is always false in kernel chunks. bool is_referenced : 1; @@ -293,6 +298,9 @@ struct uvm_gpu_chunk_struct // The VA block using the chunk, if any. // User chunks that are not backed by a VA block are considered to be // temporarily pinned and cannot be evicted. + // Note that the chunk state is normally UVM_PMM_GPU_CHUNK_STATE_ALLOCATED + // but can also be UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED if an HMM va_block + // and device private struct page have a pointer to this chunk. // // This field is always NULL in kernel chunks. uvm_va_block_t *va_block; @@ -441,17 +449,16 @@ NvU64 uvm_gpu_chunk_to_sys_addr(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk); // Allocates num_chunks chunks of size chunk_size in caller-supplied array // (chunks). // -// Returned chunks are in the TEMP_PINNED state, requiring a call to either -// uvm_pmm_gpu_unpin_allocated, uvm_pmm_gpu_unpin_referenced, or -// uvm_pmm_gpu_free. If a tracker is passed in, all -// the pending operations on the allocated chunks will be added to it +// Returned chunks are in the TEMP_PINNED state, requiring a call to +// uvm_pmm_gpu_unpin_allocated or uvm_pmm_gpu_free. If a tracker is passed in, +// all the pending operations on the allocated chunks will be added to it // guaranteeing that all the entries come from the same GPU as the PMM. // Otherwise, when tracker is NULL, all the pending operations will be // synchronized before returning to the caller. // // Each of the allocated chunks list nodes (uvm_gpu_chunk_t::list) can be used -// by the caller until the chunk is unpinned (uvm_pmm_gpu_unpin_allocated, -// uvm_pmm_gpu_unpin_referenced) or freed (uvm_pmm_gpu_free). If used, the list +// by the caller until the chunk is unpinned (uvm_pmm_gpu_unpin_allocated) +// or freed (uvm_pmm_gpu_free). If used, the list // node has to be returned to a valid state before calling either of the APIs. // // In case of an error, the chunks array is guaranteed to be cleared. @@ -484,12 +491,6 @@ NV_STATUS uvm_pmm_gpu_alloc_kernel(uvm_pmm_gpu_t *pmm, // Can only be used on user memory. void uvm_pmm_gpu_unpin_allocated(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_va_block_t *va_block); -// Unpin a temporarily pinned chunk, set its reverse map to a VA block, and -// mark it as referenced. -// -// Can only be used on user memory. -void uvm_pmm_gpu_unpin_referenced(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk, uvm_va_block_t *va_block); - // Free a user or kernel chunk. Temporarily pinned chunks are unpinned. // // The tracker is optional and a NULL tracker indicates that no new operation diff --git a/kernel-open/nvidia-uvm/uvm_va_block.c b/kernel-open/nvidia-uvm/uvm_va_block.c index 4b38e326d..1feed8844 100644 --- a/kernel-open/nvidia-uvm/uvm_va_block.c +++ b/kernel-open/nvidia-uvm/uvm_va_block.c @@ -124,6 +124,18 @@ uvm_va_space_t *uvm_va_block_get_va_space(uvm_va_block_t *va_block) return va_space; } +// Check if the GPU should cache system memory. This depends on whether the GPU +// supports full coherence and whether it has sticky L2 coherent cache lines. +// Also, if the module parameter is set, we will cache system memory. +static bool gpu_should_cache_sysmem(uvm_gpu_t *gpu) +{ + if (uvm_parent_gpu_supports_full_coherence(gpu->parent) && + !gpu->parent->sticky_l2_coherent_cache_lines) { + return true; + } + return uvm_exp_gpu_cache_sysmem != 0; +} + static NvU64 block_gpu_pte_flag_cacheable(uvm_va_block_t *block, uvm_gpu_t *gpu, uvm_processor_id_t resident_id) { uvm_va_space_t *va_space = uvm_va_block_get_va_space(block); @@ -135,7 +147,7 @@ static NvU64 block_gpu_pte_flag_cacheable(uvm_va_block_t *block, uvm_gpu_t *gpu, return UVM_MMU_PTE_FLAGS_CACHED; if (UVM_ID_IS_CPU(resident_id)) - return uvm_exp_gpu_cache_sysmem == 0 ? UVM_MMU_PTE_FLAGS_NONE : UVM_MMU_PTE_FLAGS_CACHED; + return gpu_should_cache_sysmem(gpu) ? UVM_MMU_PTE_FLAGS_CACHED : UVM_MMU_PTE_FLAGS_NONE; UVM_ASSERT(uvm_processor_mask_test(&va_space->can_access[uvm_id_value(gpu->id)], resident_id)); @@ -420,11 +432,13 @@ static uvm_cpu_chunk_t *uvm_cpu_chunk_get_chunk_for_page_resident(uvm_va_block_t return chunk; } -void uvm_cpu_chunk_remove_from_block(uvm_va_block_t *va_block, int nid, uvm_page_index_t page_index) +void uvm_cpu_chunk_remove_from_block(uvm_va_block_t *va_block, + uvm_cpu_chunk_t *chunk, + int nid, + uvm_page_index_t page_index) { uvm_va_block_cpu_node_state_t *node_state = block_node_state_get(va_block, nid); uvm_cpu_chunk_storage_mixed_t *mixed; - uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_chunk_for_page(va_block, nid, page_index); uvm_va_block_region_t chunk_region = uvm_cpu_chunk_block_region(va_block, chunk, page_index); size_t slot_index; uvm_cpu_chunk_t **chunks; @@ -759,7 +773,7 @@ static bool block_check_cpu_chunks(uvm_va_block_t *block) int nid; uvm_page_mask_t *temp_resident_mask; - temp_resident_mask = kmem_cache_alloc(g_uvm_page_mask_cache, NV_UVM_GFP_FLAGS | __GFP_ZERO); + temp_resident_mask = nv_kmem_cache_zalloc(g_uvm_page_mask_cache, NV_UVM_GFP_FLAGS); for_each_possible_uvm_node(nid) { uvm_cpu_chunk_t *chunk; @@ -821,16 +835,16 @@ void uvm_va_block_retry_deinit(uvm_va_block_retry_t *retry, uvm_va_block_t *va_b uvm_pmm_gpu_free(&gpu->pmm, gpu_chunk, NULL); } + // HMM should have already moved allocated GPU chunks to the referenced + // state or freed them. + if (uvm_va_block_is_hmm(va_block)) + UVM_ASSERT(list_empty(&retry->used_chunks)); + // Unpin all the used chunks now that we are done list_for_each_entry_safe(gpu_chunk, next_chunk, &retry->used_chunks, list) { list_del_init(&gpu_chunk->list); gpu = uvm_gpu_chunk_get_gpu(gpu_chunk); - // HMM should have already moved allocated blocks to the referenced - // state so any left over were not migrated and should be freed. - if (uvm_va_block_is_hmm(va_block)) - uvm_pmm_gpu_free(&gpu->pmm, gpu_chunk, NULL); - else - uvm_pmm_gpu_unpin_allocated(&gpu->pmm, gpu_chunk, va_block); + uvm_pmm_gpu_unpin_allocated(&gpu->pmm, gpu_chunk, va_block); } } @@ -1152,6 +1166,8 @@ static size_t block_gpu_chunk_index(uvm_va_block_t *block, UVM_ASSERT(gpu_state->chunks); chunk = gpu_state->chunks[index]; if (chunk) { + UVM_ASSERT(uvm_gpu_chunk_is_user(chunk)); + UVM_ASSERT(uvm_id_equal(uvm_gpu_id_from_index(chunk->gpu_index), gpu->id)); UVM_ASSERT(uvm_gpu_chunk_get_size(chunk) == size); UVM_ASSERT(chunk->state != UVM_PMM_GPU_CHUNK_STATE_PMA_OWNED); UVM_ASSERT(chunk->state != UVM_PMM_GPU_CHUNK_STATE_FREE); @@ -1390,10 +1406,7 @@ error: return status; } -// Retrieves the gpu_state for the given GPU. The returned pointer is -// internally managed and will be allocated (and freed) automatically, -// rather than by the caller. -static uvm_va_block_gpu_state_t *block_gpu_state_get_alloc(uvm_va_block_t *block, uvm_gpu_t *gpu) +uvm_va_block_gpu_state_t *uvm_va_block_gpu_state_get_alloc(uvm_va_block_t *block, uvm_gpu_t *gpu) { NV_STATUS status; uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get(block, gpu->id); @@ -1425,22 +1438,6 @@ error: return NULL; } -NV_STATUS uvm_va_block_gpu_state_alloc(uvm_va_block_t *va_block) -{ - uvm_va_space_t *va_space = uvm_va_block_get_va_space(va_block); - uvm_gpu_id_t gpu_id; - - UVM_ASSERT(uvm_va_block_is_hmm(va_block)); - uvm_assert_mutex_locked(&va_block->lock); - - for_each_gpu_id_in_mask(gpu_id, &va_space->registered_gpus) { - if (!block_gpu_state_get_alloc(va_block, uvm_gpu_get(gpu_id))) - return NV_ERR_NO_MEMORY; - } - - return NV_OK; -} - void uvm_va_block_unmap_cpu_chunk_on_gpus(uvm_va_block_t *block, uvm_cpu_chunk_t *chunk) { @@ -1495,7 +1492,7 @@ void uvm_va_block_remove_cpu_chunks(uvm_va_block_t *va_block, uvm_va_block_regio uvm_page_mask_region_clear(&va_block->cpu.pte_bits[UVM_PTE_BITS_CPU_READ], chunk_region); uvm_page_mask_region_clear(&va_block->cpu.pte_bits[UVM_PTE_BITS_CPU_WRITE], chunk_region); uvm_va_block_cpu_clear_resident_region(va_block, nid, chunk_region); - uvm_cpu_chunk_remove_from_block(va_block, nid, page_index); + uvm_cpu_chunk_remove_from_block(va_block, chunk, nid, page_index); uvm_va_block_unmap_cpu_chunk_on_gpus(va_block, chunk); uvm_cpu_chunk_free(chunk); } @@ -1591,26 +1588,6 @@ static NV_STATUS block_alloc_cpu_chunk(uvm_va_block_t *block, return status; } -// Same as block_alloc_cpu_chunk() but allocate a chunk suitable for use as -// a HMM destination page. The main difference is UVM does not own the reference -// on the struct page backing these chunks. -static NV_STATUS block_alloc_hmm_cpu_chunk(uvm_va_block_t *block, - uvm_chunk_sizes_mask_t cpu_allocation_sizes, - uvm_cpu_chunk_alloc_flags_t flags, - int nid, - uvm_cpu_chunk_t **chunk) -{ - NV_STATUS status; - - UVM_ASSERT(uvm_va_block_is_hmm(block)); - - status = block_alloc_cpu_chunk(block, cpu_allocation_sizes, flags, nid, chunk); - if (status == NV_OK) - (*chunk)->type = UVM_CPU_CHUNK_TYPE_HMM; - - return status; -} - // Find the largest allocation size we can use for the given page_index in the // given block. Returns the mask of possible sizes and region covered by the // largest. Callers may also elect to use a smaller size. @@ -1842,7 +1819,7 @@ static NV_STATUS block_add_cpu_chunk(uvm_va_block_t *block, status = uvm_va_block_map_cpu_chunk_on_gpus(block, chunk); if (status != NV_OK) { - uvm_cpu_chunk_remove_from_block(block, uvm_cpu_chunk_get_numa_node(chunk), page_index); + uvm_cpu_chunk_remove_from_block(block, chunk, uvm_cpu_chunk_get_numa_node(chunk), page_index); goto out; } } @@ -1866,8 +1843,7 @@ out: static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block, const uvm_page_mask_t *populate_page_mask, uvm_va_block_region_t populate_region, - uvm_va_block_context_t *block_context, - bool staged) + uvm_va_block_context_t *block_context) { NV_STATUS status = NV_OK; uvm_cpu_chunk_t *chunk; @@ -1965,13 +1941,7 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block, if (!uvm_page_mask_region_full(resident_mask, region)) chunk_alloc_flags |= UVM_CPU_CHUNK_ALLOC_FLAGS_ZERO; - // Management of a page used for a staged migration is never handed off - // to the kernel and is really just a driver managed page. Therefore - // don't allocate a HMM chunk in this case. - if (uvm_va_block_is_hmm(block) && !staged) - status = block_alloc_hmm_cpu_chunk(block, allocation_sizes, chunk_alloc_flags, preferred_nid, &chunk); - else - status = block_alloc_cpu_chunk(block, allocation_sizes, chunk_alloc_flags, preferred_nid, &chunk); + status = block_alloc_cpu_chunk(block, allocation_sizes, chunk_alloc_flags, preferred_nid, &chunk); if (status == NV_WARN_MORE_PROCESSING_REQUIRED) { alloc_flags &= ~UVM_CPU_CHUNK_ALLOC_FLAGS_STRICT; @@ -1991,11 +1961,6 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block, // Skip iterating over all pages covered by the allocated chunk. page_index = region.outer - 1; - -#if UVM_IS_CONFIG_HMM() - if (uvm_va_block_is_hmm(block) && block_context) - block_context->hmm.dst_pfns[page_index] = migrate_pfn(page_to_pfn(chunk->page)); -#endif } return NV_OK; @@ -2003,7 +1968,7 @@ static NV_STATUS block_populate_pages_cpu(uvm_va_block_t *block, NV_STATUS uvm_va_block_populate_page_cpu(uvm_va_block_t *va_block, uvm_page_index_t page_index, uvm_va_block_context_t *block_context) { - return block_populate_pages_cpu(va_block, NULL, uvm_va_block_region_for_page(page_index), block_context, false); + return block_populate_pages_cpu(va_block, NULL, uvm_va_block_region_for_page(page_index), block_context); } // Try allocating a chunk. If eviction was required, @@ -2392,7 +2357,7 @@ static uvm_page_mask_t *block_resident_mask_get_alloc(uvm_va_block_t *block, uvm if (UVM_ID_IS_CPU(processor)) return uvm_va_block_resident_mask_get(block, processor, nid); - gpu_state = block_gpu_state_get_alloc(block, uvm_gpu_get(processor)); + gpu_state = uvm_va_block_gpu_state_get_alloc(block, uvm_gpu_get(processor)); if (!gpu_state) return NULL; @@ -2432,9 +2397,15 @@ void uvm_va_block_unmapped_pages_get(uvm_va_block_t *va_block, return; } + uvm_page_mask_zero(out_mask); uvm_page_mask_region_fill(out_mask, region); - for_each_id_in_mask(id, &va_block->mapped) { + // UVM-HMM doesn't always know when CPU pages are mapped or not since there + // is no notification when CPU page tables are upgraded. If the page is + // resident, assume the CPU has some mapping. + uvm_page_mask_andnot(out_mask, out_mask, uvm_va_block_resident_mask_get(va_block, UVM_ID_CPU, NUMA_NO_NODE)); + + for_each_gpu_id_in_mask(id, &va_block->mapped) { uvm_page_mask_andnot(out_mask, out_mask, uvm_va_block_map_mask_get(va_block, id)); } } @@ -2931,7 +2902,7 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block, size_t chunk_index, uvm_va_block_region_t chunk_region) { - uvm_va_block_gpu_state_t *gpu_state = block_gpu_state_get_alloc(block, gpu); + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get_alloc(block, gpu); uvm_gpu_chunk_t *chunk = NULL; uvm_chunk_size_t chunk_size = uvm_va_block_region_size(chunk_region); uvm_va_block_test_t *block_test = uvm_va_block_get_test(block); @@ -2974,11 +2945,11 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block, if (status != NV_OK) goto chunk_free; - // An allocation retry might cause another thread to call UvmDiscard when it drops - // the block lock. As a result, inconsistent residency bits would trigger redundant - // zeroing of the chunk. Current implementation chooses to do the redundant zeroing - // as for better performance tradeoffs (tracking to avoid this case could cost more) - // and security. + // An allocation retry might cause another thread to call UvmDiscard when it + // drops the block lock. As a result, inconsistent residency bits would + // trigger redundant zeroing of the chunk. Current implementation chooses + // to do the redundant zeroing as for better performance tradeoffs + // (tracking to avoid this case could cost more) and security. status = block_zero_new_gpu_chunk(block, gpu, chunk, chunk_region, &retry->tracker); if (status != NV_OK) goto chunk_unmap; @@ -3002,8 +2973,10 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block, } // Record the used chunk so that it can be unpinned at the end of the whole - // operation. + // operation. HMM chunks are unpinned after a successful migration. block_retry_add_used_chunk(retry, chunk); + + chunk->va_block = block; gpu_state->chunks[chunk_index] = chunk; return NV_OK; @@ -3020,12 +2993,13 @@ chunk_free: } // Populate all chunks which cover the given region and page mask. -static NV_STATUS block_populate_pages_gpu(uvm_va_block_t *block, +NV_STATUS uvm_va_block_populate_pages_gpu(uvm_va_block_t *block, uvm_va_block_retry_t *retry, - uvm_gpu_t *gpu, + uvm_gpu_id_t gpu_id, uvm_va_block_region_t region, const uvm_page_mask_t *populate_mask) { + uvm_gpu_t *gpu = uvm_gpu_get(gpu_id); uvm_va_block_region_t chunk_region, check_region; size_t chunk_index; uvm_page_index_t page_index; @@ -3102,7 +3076,7 @@ static NV_STATUS block_populate_pages(uvm_va_block_t *block, if (!tmp_processor_mask) return NV_ERR_NO_MEMORY; - status = block_populate_pages_gpu(block, retry, uvm_gpu_get(dest_id), region, populate_page_mask); + status = uvm_va_block_populate_pages_gpu(block, retry, dest_id, region, populate_page_mask); if (status != NV_OK) { uvm_processor_mask_cache_free(tmp_processor_mask); return status; @@ -3150,7 +3124,7 @@ static NV_STATUS block_populate_pages(uvm_va_block_t *block, } uvm_memcg_context_start(&memcg_context, block_context->mm); - status = block_populate_pages_cpu(block, cpu_populate_mask, region, block_context, UVM_ID_IS_GPU(dest_id)); + status = block_populate_pages_cpu(block, cpu_populate_mask, region, block_context); uvm_memcg_context_end(&memcg_context); return status; } @@ -4199,7 +4173,7 @@ static NV_STATUS block_copy_resident_pages_between(uvm_va_block_t *block, // Ensure that there is GPU state that can be used for CPU-to-CPU copies if (UVM_ID_IS_CPU(dst_id) && uvm_id_equal(src_id, dst_id)) { - uvm_va_block_gpu_state_t *gpu_state = block_gpu_state_get_alloc(block, copying_gpu); + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get_alloc(block, copying_gpu); if (!gpu_state) { status = NV_ERR_NO_MEMORY; break; @@ -4893,6 +4867,7 @@ static void block_cleanup_temp_pinned_gpu_chunks(uvm_va_block_t *va_block, uvm_g // block_populate_pages above. Release them since the copy // failed and they won't be mapped to userspace. if (chunk && chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) { + list_del_init(&chunk->list); uvm_mmu_chunk_unmap(chunk, &va_block->tracker); uvm_pmm_gpu_free(&gpu->pmm, chunk, &va_block->tracker); gpu_state->chunks[i] = NULL; @@ -4993,7 +4968,8 @@ NV_STATUS uvm_va_block_make_resident_copy(uvm_va_block_t *va_block, prefetch_page_mask, UVM_VA_BLOCK_TRANSFER_MODE_MOVE); - if (status != NV_OK) { + // HMM does its own clean up. + if (status != NV_OK && !uvm_va_block_is_hmm(va_block)) { if (UVM_ID_IS_GPU(dest_id)) block_cleanup_temp_pinned_gpu_chunks(va_block, dest_id); @@ -7971,7 +7947,7 @@ static NV_STATUS block_pre_populate_pde1_gpu(uvm_va_block_t *block, gpu = gpu_va_space->gpu; big_page_size = gpu_va_space->page_tables.big_page_size; - gpu_state = block_gpu_state_get_alloc(block, gpu); + gpu_state = uvm_va_block_gpu_state_get_alloc(block, gpu); if (!gpu_state) return NV_ERR_NO_MEMORY; @@ -8705,12 +8681,12 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block, gpu = uvm_gpu_get(id); - // Although this GPU UUID is registered in the VA space, it might not have a - // GPU VA space registered. + // Although this GPU UUID is registered in the VA space, it might not + // have a GPU VA space registered. if (!uvm_gpu_va_space_get(va_space, gpu)) return NV_OK; - gpu_state = block_gpu_state_get_alloc(va_block, gpu); + gpu_state = uvm_va_block_gpu_state_get_alloc(va_block, gpu); if (!gpu_state) return NV_ERR_NO_MEMORY; @@ -9760,7 +9736,7 @@ static void block_kill(uvm_va_block_t *block) if (!uvm_va_block_is_hmm(block)) uvm_cpu_chunk_mark_dirty(chunk, 0); - uvm_cpu_chunk_remove_from_block(block, nid, page_index); + uvm_cpu_chunk_remove_from_block(block, chunk, nid, page_index); uvm_cpu_chunk_free(chunk); } @@ -9824,13 +9800,12 @@ void uvm_va_block_kill(uvm_va_block_t *va_block) static void block_gpu_release_region(uvm_va_block_t *va_block, uvm_gpu_id_t gpu_id, uvm_va_block_gpu_state_t *gpu_state, - uvm_page_mask_t *page_mask, uvm_va_block_region_t region) { uvm_page_index_t page_index; uvm_gpu_t *gpu = uvm_gpu_get(gpu_id); - for_each_va_block_page_in_region_mask(page_index, page_mask, region) { + for_each_va_block_page_in_region(page_index, region) { size_t chunk_index = block_gpu_chunk_index(va_block, gpu, page_index, NULL); uvm_gpu_chunk_t *gpu_chunk = gpu_state->chunks[chunk_index]; @@ -9875,7 +9850,7 @@ void uvm_va_block_munmap_region(uvm_va_block_t *va_block, uvm_processor_mask_clear(&va_block->evicted_gpus, gpu_id); if (gpu_state->chunks) { - block_gpu_release_region(va_block, gpu_id, gpu_state, NULL, region); + block_gpu_release_region(va_block, gpu_id, gpu_state, region); // TODO: bug 3660922: Need to update the read duplicated pages mask // when read duplication is supported for HMM. @@ -10446,7 +10421,7 @@ static NV_STATUS block_split_preallocate_no_retry(uvm_va_block_t *existing, uvm_ if (status != NV_OK) goto error; - if (!block_gpu_state_get_alloc(new, gpu)) { + if (!uvm_va_block_gpu_state_get_alloc(new, gpu)) { status = NV_ERR_NO_MEMORY; goto error; } @@ -10620,7 +10595,7 @@ static void block_split_cpu(uvm_va_block_t *existing, uvm_va_block_t *new) uvm_page_index_t new_chunk_page_index; NV_STATUS status; - uvm_cpu_chunk_remove_from_block(existing, nid, page_index); + uvm_cpu_chunk_remove_from_block(existing, chunk, nid, page_index); // The chunk has to be adjusted for the new block before inserting it. new_chunk_page_index = page_index - split_page_index; @@ -13532,7 +13507,7 @@ out: static NV_STATUS block_gpu_force_4k_ptes(uvm_va_block_t *block, uvm_va_block_context_t *block_context, uvm_gpu_t *gpu) { - uvm_va_block_gpu_state_t *gpu_state = block_gpu_state_get_alloc(block, gpu); + uvm_va_block_gpu_state_t *gpu_state = uvm_va_block_gpu_state_get_alloc(block, gpu); uvm_push_t push; NV_STATUS status; diff --git a/kernel-open/nvidia-uvm/uvm_va_block.h b/kernel-open/nvidia-uvm/uvm_va_block.h index 3eb504b3a..6a3b2c5df 100644 --- a/kernel-open/nvidia-uvm/uvm_va_block.h +++ b/kernel-open/nvidia-uvm/uvm_va_block.h @@ -1375,9 +1375,11 @@ NV_STATUS uvm_va_block_service_finish(uvm_processor_id_t processor_id, uvm_va_block_t *va_block, uvm_service_block_context_t *service_context); -// Allocate GPU state for the given va_block and registered GPUs. +// Returns the gpu_state for the given GPU. The returned pointer is +// internally managed and will be allocated (and freed) automatically, +// rather than by the caller. Returns NULL if there is no memory. // Locking: The block lock must be held. -NV_STATUS uvm_va_block_gpu_state_alloc(uvm_va_block_t *va_block); +uvm_va_block_gpu_state_t *uvm_va_block_gpu_state_get_alloc(uvm_va_block_t *va_block, uvm_gpu_t *gpu); // Release any GPU or policy data associated with the given region in response // to munmap(). @@ -2152,10 +2154,13 @@ bool uvm_va_block_cpu_is_region_resident_on(uvm_va_block_t *va_block, int nid, u // Locking: The va_block lock must be held. NV_STATUS uvm_cpu_chunk_insert_in_block(uvm_va_block_t *va_block, uvm_cpu_chunk_t *chunk, uvm_page_index_t page_index); -// Remove a CPU chunk at the given page_index from the va_block. +// Remove the given CPU chunk at the given page_index from the va_block. // nid cannot be NUMA_NO_NODE. // Locking: The va_block lock must be held. -void uvm_cpu_chunk_remove_from_block(uvm_va_block_t *va_block, int nid, uvm_page_index_t page_index); +void uvm_cpu_chunk_remove_from_block(uvm_va_block_t *va_block, + uvm_cpu_chunk_t *chunk, + int nid, + uvm_page_index_t page_index); // Return the CPU chunk at the given page_index on the given NUMA node from the // va_block. nid cannot be NUMA_NO_NODE. @@ -2288,6 +2293,13 @@ NV_STATUS uvm_va_block_populate_page_cpu(uvm_va_block_t *va_block, uvm_page_index_t page_index, uvm_va_block_context_t *block_context); +// Populate all GPU chunks which cover the given region and page mask. +NV_STATUS uvm_va_block_populate_pages_gpu(uvm_va_block_t *block, + uvm_va_block_retry_t *retry, + uvm_gpu_id_t gpu_id, + uvm_va_block_region_t region, + const uvm_page_mask_t *populate_mask); + // A helper macro for handling allocation-retry // // The macro takes a VA block, uvm_va_block_retry_t struct and a function call diff --git a/kernel-open/nvidia/nv.c b/kernel-open/nvidia/nv.c index ed4d7dea0..02bb9a67c 100644 --- a/kernel-open/nvidia/nv.c +++ b/kernel-open/nvidia/nv.c @@ -2532,8 +2532,13 @@ nvidia_ioctl( NV_CTL_DEVICE_ONLY(nv); - if (num_arg_gpus == 0 || nvlfp->num_attached_gpus != 0 || - arg_size % sizeof(NvU32) != 0) + if ((num_arg_gpus == 0) || (arg_size % sizeof(NvU32) != 0)) + { + status = -EINVAL; + goto done; + } + + if (nvlfp->num_attached_gpus != 0) { status = -EINVAL; goto done; @@ -2562,6 +2567,7 @@ nvidia_ioctl( if (nvlfp->attached_gpus[i] != 0) nvidia_dev_put(nvlfp->attached_gpus[i], sp); } + NV_KFREE(nvlfp->attached_gpus, arg_size); nvlfp->num_attached_gpus = 0; diff --git a/src/common/displayport/inc/dp_regkeydatabase.h b/src/common/displayport/inc/dp_regkeydatabase.h index c2fe2c69c..0d1113bab 100644 --- a/src/common/displayport/inc/dp_regkeydatabase.h +++ b/src/common/displayport/inc/dp_regkeydatabase.h @@ -47,7 +47,7 @@ #define NV_DP_REGKEY_APPLY_MAX_LINK_RATE_OVERRIDES "APPLY_OVERRIDES_FOR_BUG_2489143" #define NV_DP_REGKEY_DISABLE_DSC "DISABLE_DSC" #define NV_DP_REGKEY_SKIP_ASSESSLINK_FOR_EDP "HP_WAR_2189772" -#define NV_DP_REGKEY_HDCP_AUTH_ONLY_ON_DEMAND "DP_HDCP_AUTH_ONLY_ON_DEMAND" +#define NV_DP_REGKEY_MST_AUTO_HDCP_AUTH_AT_ATTACH "DP_MST_AUTO_HDCP_AUTH_AT_ATTACH" #define NV_DP_REGKEY_ENABLE_MSA_OVER_MST "ENABLE_MSA_OVER_MST" #define NV_DP_REGKEY_DISABLE_DOWNSPREAD "DISABLE_DOWNSPREAD" @@ -126,7 +126,7 @@ struct DP_REGKEY_DATABASE NvU32 applyMaxLinkRateOverrides; bool bDscDisabled; bool bAssesslinkForEdpSkipped; - bool bHdcpAuthOnlyOnDemand; + bool bMstAutoHdcpAuthAtAttach; bool bMsaOverMstEnabled; bool bOptLinkKeptAlive; bool bOptLinkKeptAliveMst; diff --git a/src/common/displayport/src/dp_connectorimpl.cpp b/src/common/displayport/src/dp_connectorimpl.cpp index e34fde0da..7f9a8f8a2 100644 --- a/src/common/displayport/src/dp_connectorimpl.cpp +++ b/src/common/displayport/src/dp_connectorimpl.cpp @@ -52,13 +52,13 @@ #include "dp_tracing.h" /* - * This is needed by Synaptics to disable DisplayExpand feature - * in some of their docking station based on if GPU supports DSC. + * This is needed by Synaptics to disable DisplayExpand feature + * in some of their docking station based on if GPU supports DSC. * Feature is not needed if DSC is supported. * Customers reported problems with the feature enabled on GB20x devices * and requested GPU DSC detection to disable DisplayExpand feature. * DSC is supported in Turing and later SKUs hence - * exposing Turing DevId to customers to address their requirement. + * exposing Turing DevId to customers to address their requirement. */ #define TURING_DEV_ID 0x1E @@ -90,7 +90,7 @@ ConnectorImpl::ConnectorImpl(MainLink * main, AuxBus * auxBus, Timer * timer, Co hdcpCpIrqRxStatusRetries(0), bFromResumeToNAB(false), bAttachOnResume(false), - bHdcpAuthOnlyOnDemand(false), + bHdcpAuthOnlyOnDemand(true), constructorFailed(false), policyModesetOrderMitigation(false), policyForceLTAtNAB(false), @@ -170,8 +170,8 @@ void ConnectorImpl::applyRegkeyOverrides(const DP_REGKEY_DATABASE& dpRegkeyDatab this->bSkipAssessLinkForEDP = dpRegkeyDatabase.bAssesslinkForEdpSkipped; - // If Hdcp authenticatoin on demand regkey is set, override to the provided value. - this->bHdcpAuthOnlyOnDemand = dpRegkeyDatabase.bHdcpAuthOnlyOnDemand; + // default bHdcpAuthOnlyOnDemand is true and override to false if regkey bMstAutoHdcpAuthAtAttach set as true. + this->bHdcpAuthOnlyOnDemand = !dpRegkeyDatabase.bMstAutoHdcpAuthAtAttach; if (dpRegkeyDatabase.bOptLinkKeptAlive) { @@ -5687,6 +5687,12 @@ bool ConnectorImpl::train(const LinkConfiguration & lConfig, bool force, return false; } + // + // Cancel pending HDCP authentication callbacks if have or may interrupt + // active link training that violates spec. + // + cancelHdcpCallbacks(); + if (!lConfig.multistream) { for (Device * i = enumDevices(0); i; i=enumDevices(i)) @@ -7428,6 +7434,12 @@ void ConnectorImpl::notifyShortPulse() activeLinkConfig.peakRate > dp2LinkRate_10_0Gbps && main->isCableVconnSourceUnknown()) { + // + // Cancel pending HDCP authentication callbacks if have or may interrupt + // active link training that violates spec. + // + cancelHdcpCallbacks(); + if (activeLinkConfig.isValid() && enableFlush()) { train(originalActiveLinkConfig, true); diff --git a/src/common/displayport/src/dp_evoadapter.cpp b/src/common/displayport/src/dp_evoadapter.cpp index e1b26fae5..a8619b082 100644 --- a/src/common/displayport/src/dp_evoadapter.cpp +++ b/src/common/displayport/src/dp_evoadapter.cpp @@ -88,7 +88,7 @@ const struct {NV_DP_REGKEY_APPLY_MAX_LINK_RATE_OVERRIDES, &dpRegkeyDatabase.applyMaxLinkRateOverrides, DP_REG_VAL_U32}, {NV_DP_REGKEY_DISABLE_DSC, &dpRegkeyDatabase.bDscDisabled, DP_REG_VAL_BOOL}, {NV_DP_REGKEY_SKIP_ASSESSLINK_FOR_EDP, &dpRegkeyDatabase.bAssesslinkForEdpSkipped, DP_REG_VAL_BOOL}, - {NV_DP_REGKEY_HDCP_AUTH_ONLY_ON_DEMAND, &dpRegkeyDatabase.bHdcpAuthOnlyOnDemand, DP_REG_VAL_BOOL}, + {NV_DP_REGKEY_MST_AUTO_HDCP_AUTH_AT_ATTACH, &dpRegkeyDatabase.bMstAutoHdcpAuthAtAttach, DP_REG_VAL_BOOL}, {NV_DP_REGKEY_ENABLE_MSA_OVER_MST, &dpRegkeyDatabase.bMsaOverMstEnabled, DP_REG_VAL_BOOL}, {NV_DP_REGKEY_KEEP_OPT_LINK_ALIVE, &dpRegkeyDatabase.bOptLinkKeptAlive, DP_REG_VAL_BOOL}, {NV_DP_REGKEY_KEEP_OPT_LINK_ALIVE_MST, &dpRegkeyDatabase.bOptLinkKeptAliveMst, DP_REG_VAL_BOOL}, diff --git a/src/common/inc/nvBldVer.h b/src/common/inc/nvBldVer.h index 1eb75eb7f..d1ccc839d 100644 --- a/src/common/inc/nvBldVer.h +++ b/src/common/inc/nvBldVer.h @@ -36,25 +36,25 @@ // and then checked back in. You cannot make changes to these sections without // corresponding changes to the buildmeister script #ifndef NV_BUILD_BRANCH - #define NV_BUILD_BRANCH r580_92 + #define NV_BUILD_BRANCH r581_07 #endif #ifndef NV_PUBLIC_BRANCH - #define NV_PUBLIC_BRANCH r580_92 + #define NV_PUBLIC_BRANCH r581_07 #endif #if defined(NV_LINUX) || defined(NV_BSD) || defined(NV_SUNOS) -#define NV_BUILD_BRANCH_VERSION "rel/gpu_drv/r580/r580_92-206" -#define NV_BUILD_CHANGELIST_NUM (36361462) +#define NV_BUILD_BRANCH_VERSION "rel/gpu_drv/r580/r581_07-226" +#define NV_BUILD_CHANGELIST_NUM (36455674) #define NV_BUILD_TYPE "Official" -#define NV_BUILD_NAME "rel/gpu_drv/r580/r580_92-206" -#define NV_LAST_OFFICIAL_CHANGELIST_NUM (36361462) +#define NV_BUILD_NAME "rel/gpu_drv/r580/r581_07-226" +#define NV_LAST_OFFICIAL_CHANGELIST_NUM (36455674) #else /* Windows builds */ -#define NV_BUILD_BRANCH_VERSION "r580_92-5" -#define NV_BUILD_CHANGELIST_NUM (36361462) +#define NV_BUILD_BRANCH_VERSION "r581_07-4" +#define NV_BUILD_CHANGELIST_NUM (36452923) #define NV_BUILD_TYPE "Official" -#define NV_BUILD_NAME "580.97" -#define NV_LAST_OFFICIAL_CHANGELIST_NUM (36361462) +#define NV_BUILD_NAME "581.18" +#define NV_LAST_OFFICIAL_CHANGELIST_NUM (36452923) #define NV_BUILD_BRANCH_BASE_VERSION R580 #endif // End buildmeister python edited section diff --git a/src/common/inc/nvUnixVersion.h b/src/common/inc/nvUnixVersion.h index 3e7e407a4..48d89e4d1 100644 --- a/src/common/inc/nvUnixVersion.h +++ b/src/common/inc/nvUnixVersion.h @@ -4,7 +4,7 @@ #if defined(NV_LINUX) || defined(NV_BSD) || defined(NV_SUNOS) || defined(NV_VMWARE) || defined(NV_QNX) || defined(NV_INTEGRITY) || \ (defined(RMCFG_FEATURE_PLATFORM_GSP) && RMCFG_FEATURE_PLATFORM_GSP == 1) -#define NV_VERSION_STRING "580.76.05" +#define NV_VERSION_STRING "580.82.07" #else diff --git a/src/common/modeset/timing/nvt_edid.c b/src/common/modeset/timing/nvt_edid.c index 1adf60e41..8e6ddfc4e 100644 --- a/src/common/modeset/timing/nvt_edid.c +++ b/src/common/modeset/timing/nvt_edid.c @@ -1200,6 +1200,10 @@ NVT_STATUS NV_STDCALL NvTiming_ParseEDIDInfo(NvU8 *pEdid, NvU32 length, NVT_EDID getEdidHDM1_4bVsdbTiming(pInfo); +#if defined(NVT_USE_NVKMS) + prioritizeEdidHDMIExtTiming(pInfo); +#endif + // Assert if no timings were found (due to a bad EDID) or if we mistakenly // assigned more timings than we allocated space for (due to bad logic above) nvt_assert(pInfo->total_timings && diff --git a/src/common/modeset/timing/nvt_edidext_861.c b/src/common/modeset/timing/nvt_edidext_861.c index 24f6173fc..5f8b1b66f 100644 --- a/src/common/modeset/timing/nvt_edidext_861.c +++ b/src/common/modeset/timing/nvt_edidext_861.c @@ -1,6 +1,6 @@ //***************************************************************************** // -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: MIT // // Permission is hereby granted, free of charge, to any person obtaining a @@ -2838,122 +2838,148 @@ NVT_STATUS NvTiming_ConstructVendorSpecificInfoframe(NVT_EDID_INFO *pEdidInfo, N return NVT_STATUS_ERR; } + if ((pCtrl->VSIFVersion != NVT_VSIF_VERSION_HF_VSIF) && (pCtrl->VSIFVersion != NVT_VSIF_VERSION_H14B_VSIF)) + { + return NVT_STATUS_INVALID_PARAMETER; + } // initialize the infoframe buffer nvt_nvu8_set_bits(pInfoFrame->Header.type, NVT_HDMI_VS_HB0_VALUE, NVT_HDMI_VS_HB0_MASK, NVT_HDMI_VS_HB0_SHIFT); nvt_nvu8_set_bits(pInfoFrame->Header.version, NVT_HDMI_VS_HB1_VALUE, NVT_HDMI_VS_HB1_MASK, NVT_HDMI_VS_HB1_SHIFT); nvt_nvu8_set_bits(pInfoFrame->Header.length, NVT_HDMI_VS_HB2_VALUE, NVT_HDMI_VS_HB2_MASK, NVT_HDMI_VS_HB2_SHIFT); - if (pCtrl->VSIFVersion == NVT_VSIF_VERSION_H14B_VSIF) - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte1, NVT_HDMI_VS_BYTE1_OUI_VER_1_4, NVT_HDMI_VS_BYTE1_OUI_MASK, NVT_HDMI_VS_BYTE1_OUI_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.byte2, NVT_HDMI_VS_BYTE2_OUI_VER_1_4, NVT_HDMI_VS_BYTE2_OUI_MASK, NVT_HDMI_VS_BYTE2_OUI_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.byte3, NVT_HDMI_VS_BYTE3_OUI_VER_1_4, NVT_HDMI_VS_BYTE3_OUI_MASK, NVT_HDMI_VS_BYTE3_OUI_SHIFT); - } - else if (pCtrl->VSIFVersion == NVT_VSIF_VERSION_HF_VSIF) - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte1, NVT_HDMI_VS_BYTE1_OUI_VER_2_0, NVT_HDMI_VS_BYTE1_OUI_MASK, NVT_HDMI_VS_BYTE1_OUI_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.byte2, NVT_HDMI_VS_BYTE2_OUI_VER_2_0, NVT_HDMI_VS_BYTE2_OUI_MASK, NVT_HDMI_VS_BYTE2_OUI_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.byte3, NVT_HDMI_VS_BYTE3_OUI_VER_2_0, NVT_HDMI_VS_BYTE3_OUI_MASK, NVT_HDMI_VS_BYTE3_OUI_SHIFT); - } - // init the header (mostly done in default Infoframe) pInfoFrame->Header.length = offsetof(NVT_VENDOR_SPECIFIC_INFOFRAME_PAYLOAD, optionalBytes); - // construct the desired infoframe contents based on the control - - // clear all static reserved fields - nvt_nvu8_set_bits(pInfoFrame->Data.byte4, 0, NVT_HDMI_VS_BYTE4_RSVD_MASK, NVT_HDMI_VS_BYTE4_RSVD_SHIFT); - - // setup the parameters - nvt_nvu8_set_bits(pInfoFrame->Data.byte4, pCtrl->HDMIFormat, NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_MASK, NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_SHIFT); - - // determine what the format is -- if disabled, force the format to NONE. - if (pCtrl->Enable) + if (pCtrl->VSIFVersion == NVT_VSIF_VERSION_HF_VSIF) { - HDMIFormat = pCtrl->HDMIFormat; - } - else - { - HDMIFormat = NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE; - } - - switch(HDMIFormat) - { - case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE: + // infoframe is only supported on 861F and later + if (pEdidInfo->ext861.revision < NVT_CEA861_REV_F) { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, 0, NVT_HDMI_VS_BYTENv_RSVD_MASK, NVT_HDMI_VS_BYTENv_RSVD_SHIFT); - break; + return NVT_STATUS_ERR; } - case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_EXT: - { - // Note: extended resolution frames are not yet fully supported - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, pCtrl->HDMI_VIC, NVT_HDMI_VS_BYTE5_HDMI_VIC_MASK, NVT_HDMI_VS_BYTE5_HDMI_VIC_SHIFT); - break; - } - case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D: - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, 0, NVT_HDMI_VS_BYTE5_HDMI_RSVD_MASK, NVT_HDMI_VS_BYTE5_HDMI_RSVD_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, pCtrl->ThreeDStruc, NVT_HDMI_VS_BYTE5_HDMI_3DS_MASK, NVT_HDMI_VS_BYTE5_HDMI_3DS_SHIFT); - // side by side half requires additional format data in the infoframe. - if (NVT_HDMI_VS_BYTE5_HDMI_3DS_SIDEBYSIDEHALF == pCtrl->ThreeDStruc) + if (pCtrl->Enable && (pCtrl->HDMIFormat != NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE)) + { + return NVT_STATUS_INVALID_PARAMETER; + } + + nvt_nvu8_set_bits(pInfoFrame->Data.byte1, NVT_HDMI_VS_BYTE1_OUI_VER_2_0, NVT_HDMI_VS_BYTE1_OUI_MASK, NVT_HDMI_VS_BYTE1_OUI_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte2, NVT_HDMI_VS_BYTE2_OUI_VER_2_0, NVT_HDMI_VS_BYTE2_OUI_MASK, NVT_HDMI_VS_BYTE2_OUI_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte3, NVT_HDMI_VS_BYTE3_OUI_VER_2_0, NVT_HDMI_VS_BYTE3_OUI_MASK, NVT_HDMI_VS_BYTE3_OUI_SHIFT); + + // construct the desired HF-VSIF infoframe contents based on the control + + nvt_nvu8_set_bits(pInfoFrame->Data.byte4, NVT_HDMI_HF_VS_BYTE4_VER_1_0, NVT_HDMI_HF_VS_BYTE4_VER_MASK, NVT_HDMI_HF_VS_BYTE4_VER_SHIFT); + + if (pCtrl->Enable) + { + if (pCtrl->ALLMEnable == 1) { - nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], pCtrl->ThreeDDetail, NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_MASK, NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_SHIFT); - optIdx++; + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_ALLM_MODE_EN, NVT_HDMI_VS_BYTE5_ALLM_MODE_MASK, NVT_HDMI_VS_BYTE5_ALLM_MODE_SHIFT); } - if (pCtrl->MetadataPresent) + else if (pCtrl->ALLMEnable == 0) { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_PRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_ALLM_MODE_DIS, NVT_HDMI_VS_BYTE5_ALLM_MODE_MASK, NVT_HDMI_VS_BYTE5_ALLM_MODE_SHIFT); + } + } + else + { + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTENv_RSVD, NVT_HDMI_VS_BYTENv_RSVD_MASK, NVT_HDMI_VS_BYTENv_RSVD_SHIFT); + } + } + else if (pCtrl->VSIFVersion == NVT_VSIF_VERSION_H14B_VSIF) + { + // construct the desired H14b-VSIF infoframe contents based on the control - switch(pCtrl->MetadataType) + nvt_nvu8_set_bits(pInfoFrame->Data.byte1, NVT_HDMI_VS_BYTE1_OUI_VER_1_4, NVT_HDMI_VS_BYTE1_OUI_MASK, NVT_HDMI_VS_BYTE1_OUI_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte2, NVT_HDMI_VS_BYTE2_OUI_VER_1_4, NVT_HDMI_VS_BYTE2_OUI_MASK, NVT_HDMI_VS_BYTE2_OUI_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte3, NVT_HDMI_VS_BYTE3_OUI_VER_1_4, NVT_HDMI_VS_BYTE3_OUI_MASK, NVT_HDMI_VS_BYTE3_OUI_SHIFT); + + // clear all static reserved fields + nvt_nvu8_set_bits(pInfoFrame->Data.byte4, 0, NVT_HDMI_VS_BYTE4_RSVD_MASK, NVT_HDMI_VS_BYTE4_RSVD_SHIFT); + + // setup the parameters + nvt_nvu8_set_bits(pInfoFrame->Data.byte4, pCtrl->HDMIFormat, NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_MASK, NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_SHIFT); + + // determine what the format is -- if disabled, force the format to NONE. + if (pCtrl->Enable) + { + HDMIFormat = pCtrl->HDMIFormat; + } + else + { + HDMIFormat = NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE; + } + + switch(HDMIFormat) + { + case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE: + { + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, 0, NVT_HDMI_VS_BYTENv_RSVD_MASK, NVT_HDMI_VS_BYTENv_RSVD_SHIFT); + break; + } + case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_EXT: + { + // Note: extended resolution frames are not yet fully supported + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, pCtrl->HDMI_VIC, NVT_HDMI_VS_BYTE5_HDMI_VIC_MASK, NVT_HDMI_VS_BYTE5_HDMI_VIC_SHIFT); + break; + } + case NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D: + { + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, 0, NVT_HDMI_VS_BYTE5_HDMI_RSVD_MASK, NVT_HDMI_VS_BYTE5_HDMI_RSVD_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, pCtrl->ThreeDStruc, NVT_HDMI_VS_BYTE5_HDMI_3DS_MASK, NVT_HDMI_VS_BYTE5_HDMI_3DS_SHIFT); + + // side by side half requires additional format data in the infoframe. + if (NVT_HDMI_VS_BYTE5_HDMI_3DS_SIDEBYSIDEHALF == pCtrl->ThreeDStruc) { - case NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_PARALLAX: - { - if (sizeof(pCtrl->Metadata) >= NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX && - sizeof(pInfoFrame->Data.optionalBytes) - (optIdx + 1) >= NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX) - { - nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_MASK, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_SHIFT); - nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_PARALLAX, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_MASK, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_SHIFT); - ++optIdx; - - NVMISC_MEMCPY(pCtrl->Metadata, &pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX); - optIdx += NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX; - } - else - { - // not enough data in the control struct or not enough room in the infoframe -- BOTH compile time issues!! - // ignore metadata. - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); - } - break; - } - default: - { - // unrecognised metadata, recover the best we can. - // note -- can not copy whatever is there because type implies length. - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); - RetCode = NVT_STATUS_ERR; - } + nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], pCtrl->ThreeDDetail, NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_MASK, NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_SHIFT); + optIdx++; } + if (pCtrl->MetadataPresent) + { + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_PRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); + switch(pCtrl->MetadataType) + { + case NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_PARALLAX: + { + if (sizeof(pCtrl->Metadata) >= NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX && + sizeof(pInfoFrame->Data.optionalBytes) - (optIdx + 1) >= NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX) + { + nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_MASK, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_SHIFT); + nvt_nvu8_set_bits(pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_PARALLAX, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_MASK, NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_SHIFT); + ++optIdx; + + NVMISC_MEMCPY(pCtrl->Metadata, &pInfoFrame->Data.optionalBytes[optIdx], NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX); + optIdx += NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_LEN_PARALLAX; + } + else + { + // not enough data in the control struct or not enough room in the infoframe -- BOTH compile time issues!! + // ignore metadata. + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); + } + break; + } + default: + { + // unrecognised metadata, recover the best we can. + // note -- can not copy whatever is there because type implies length. + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); + RetCode = NVT_STATUS_ERR; + } + } + + } + else + { + nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); + } + break; } - else - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_HDMI_META_PRESENT_NOTPRES, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_MASK, NVT_HDMI_VS_BYTE5_3D_META_PRESENT_SHIFT); - } - break; + } - - } - - if (pCtrl->ALLMEnable == 1) - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_ALLM_MODE_EN, NVT_HDMI_VS_BYTE5_ALLM_MODE_MASK, NVT_HDMI_VS_BYTE5_ALLM_MODE_SHIFT); - } - else if (pCtrl->ALLMEnable == 0) - { - nvt_nvu8_set_bits(pInfoFrame->Data.byte5, NVT_HDMI_VS_BYTE5_ALLM_MODE_DIS, NVT_HDMI_VS_BYTE5_ALLM_MODE_MASK, NVT_HDMI_VS_BYTE5_ALLM_MODE_SHIFT); } // clear last byte of infoframe (reserved per spec). @@ -3599,6 +3625,27 @@ void parseEdidHDMILLCTiming(NVT_EDID_INFO *pInfo, VSDB_DATA *pVsdb, NvU32 *pMapS } } +CODE_SEGMENT(PAGE_DD_CODE) +void prioritizeEdidHDMIExtTiming(NVT_EDID_INFO *pInfo) +{ + NvU16 i, j; + + nvt_assert(pInfo->total_timings <= COUNT(pInfo->timing)); + + for (i = 0; i < pInfo->total_timings; ++i) + { + if (NVT_GET_TIMING_STATUS_TYPE(pInfo->timing[i].etc.status) == NVT_TYPE_HDMI_EXT) + { + const NVT_TIMING extTimings = pInfo->timing[i]; + for (j = i; j > 0; j--) + { + pInfo->timing[j] = pInfo->timing[j - 1]; + } + pInfo->timing[0] = extTimings; + } + } +} + // get HDMI 1.4 3D mandatory stereo format datail base on the input vic. // If the vic is not in the mandatory format list, return error. CODE_SEGMENT(PAGE_DD_CODE) diff --git a/src/common/modeset/timing/nvtiming.h b/src/common/modeset/timing/nvtiming.h index 718f88e21..a28405250 100644 --- a/src/common/modeset/timing/nvtiming.h +++ b/src/common/modeset/timing/nvtiming.h @@ -3807,6 +3807,11 @@ typedef struct tagNVT_VENDOR_SPECIFIC_INFOFRAME #define NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_NONE 0x00 #define NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_EXT 0x01 #define NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D 0x02 + +#define NVT_HDMI_HF_VS_BYTE4_VER_MASK 0xff +#define NVT_HDMI_HF_VS_BYTE4_VER_SHIFT 0x00 +#define NVT_HDMI_HF_VS_BYTE4_VER_1_0 0x01 + // 0x03-0x07 reserved // #define NVT_HDMI_VS_BYTE5_HDMI_VIC_MASK 0xff // HDMI_VID_FMT = HDMI_VID_FMT_EXT diff --git a/src/common/modeset/timing/nvtiming_pvt.h b/src/common/modeset/timing/nvtiming_pvt.h index 04aca8cd0..1a54d0211 100644 --- a/src/common/modeset/timing/nvtiming_pvt.h +++ b/src/common/modeset/timing/nvtiming_pvt.h @@ -88,6 +88,7 @@ void parseEdidHdmiLlcBasicInfo(VSDB_DATA *pVsdb, NVT_HDMI_LLC_INFO *pHdmiL void parseEdidHdmiForumVSDB(VSDB_DATA *pVsdb, NVT_HDMI_FORUM_INFO *pHdmiInfo); void getEdidHDM1_4bVsdbTiming(NVT_EDID_INFO *pInfo); void parseEdidHDMILLCTiming(NVT_EDID_INFO *pInfo, VSDB_DATA *pVsdb, NvU32 *pSupported, HDMI3DSUPPORTMAP * pM); +void prioritizeEdidHDMIExtTiming(NVT_EDID_INFO *pInfo); void parseEdidNvidiaVSDBBlock(VSDB_DATA *pVsdb, NVDA_VSDB_PARSED_INFO *vsdbInfo); void parseCta861HdrStaticMetadataDataBlock(NVT_EDID_CEA861_INFO *pExt861, void *pRawInfo, NVT_CTA861_ORIGIN flag); void parseCta861DvStaticMetadataDataBlock(VSVDB_DATA* pVsvdb, NVT_DV_STATIC_METADATA* pDvInfo); diff --git a/src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h b/src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h index b20cba6ac..48921f258 100644 --- a/src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h +++ b/src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h @@ -364,8 +364,9 @@ typedef NVXXXX_CTRL_XXX_INFO NV2080_CTRL_FB_INFO; #define NV2080_CTRL_FB_INFO_INDEX_LTC_MASK_1 (0x00000038U) #define NV2080_CTRL_FB_INFO_INDEX_ACCESS_COUNTER_BUFFER_COUNT (0x00000039U) #define NV2080_CTRL_FB_INFO_INDEX_COHERENCE_INFO (0x0000003AU) +#define NV2080_CTRL_FB_INFO_INDEX_NUMA_NODE_ID (0x0000003BU) -#define NV2080_CTRL_FB_INFO_INDEX_MAX NV2080_CTRL_FB_INFO_INDEX_ACCESS_COUNTER_BUFFER_COUNT +#define NV2080_CTRL_FB_INFO_INDEX_MAX NV2080_CTRL_FB_INFO_INDEX_NUMA_NODE_ID /* Intentionally picking a value much bigger than NV2080_CTRL_FB_INFO_INDEX_MAX to prevent VGPU plumbing updates */ #define NV2080_CTRL_FB_INFO_MAX_LIST_SIZE (0x00000080U) diff --git a/src/common/shared/inc/g_vgpu_chip_flags.h b/src/common/shared/inc/g_vgpu_chip_flags.h index 579cae878..3007d365f 100644 --- a/src/common/shared/inc/g_vgpu_chip_flags.h +++ b/src/common/shared/inc/g_vgpu_chip_flags.h @@ -621,6 +621,25 @@ ENTRY(0x2238, 0x16B8, 0x10de, "NVIDIA A10M-10C"), ENTRY(0x2238, 0x16B9, 0x10de, "NVIDIA A10M-20C"), ENTRY(0x2238, 0x16E6, 0x10de, "NVIDIA A10M-1"), ENTRY(0x2238, 0x2208, 0x10de, "NVIDIA A10M-3B"), +ENTRY(0x230E, 0x20F5, 0x10de, "NVIDIA H20L-1-15CME"), +ENTRY(0x230E, 0x20F6, 0x10de, "NVIDIA H20L-1-15C"), +ENTRY(0x230E, 0x20F7, 0x10de, "NVIDIA H20L-1-30C"), +ENTRY(0x230E, 0x20F8, 0x10de, "NVIDIA H20L-2-30C"), +ENTRY(0x230E, 0x20F9, 0x10de, "NVIDIA H20L-3-60C"), +ENTRY(0x230E, 0x20FA, 0x10de, "NVIDIA H20L-4-60C"), +ENTRY(0x230E, 0x20FB, 0x10de, "NVIDIA H20L-7-120C"), +ENTRY(0x230E, 0x20FC, 0x10de, "NVIDIA H20L-4C"), +ENTRY(0x230E, 0x20FD, 0x10de, "NVIDIA H20L-5C"), +ENTRY(0x230E, 0x20FE, 0x10de, "NVIDIA H20L-6C"), +ENTRY(0x230E, 0x20FF, 0x10de, "NVIDIA H20L-8C"), +ENTRY(0x230E, 0x2100, 0x10de, "NVIDIA H20L-10C"), +ENTRY(0x230E, 0x2101, 0x10de, "NVIDIA H20L-12C"), +ENTRY(0x230E, 0x2102, 0x10de, "NVIDIA H20L-15C"), +ENTRY(0x230E, 0x2103, 0x10de, "NVIDIA H20L-20C"), +ENTRY(0x230E, 0x2104, 0x10de, "NVIDIA H20L-30C"), +ENTRY(0x230E, 0x2105, 0x10de, "NVIDIA H20L-40C"), +ENTRY(0x230E, 0x2106, 0x10de, "NVIDIA H20L-60C"), +ENTRY(0x230E, 0x2107, 0x10de, "NVIDIA H20L-120C"), ENTRY(0x2321, 0x1853, 0x10de, "NVIDIA H100L-1-12CME"), ENTRY(0x2321, 0x1854, 0x10de, "NVIDIA H100L-1-12C"), ENTRY(0x2321, 0x1855, 0x10de, "NVIDIA H100L-1-24C"), diff --git a/src/common/shared/inc/g_vgpu_resman_specific.h b/src/common/shared/inc/g_vgpu_resman_specific.h index d1cba0f29..314a3d199 100644 --- a/src/common/shared/inc/g_vgpu_resman_specific.h +++ b/src/common/shared/inc/g_vgpu_resman_specific.h @@ -17,6 +17,7 @@ static inline void _get_chip_id_for_alias_pgpu(NvU32 *dev_id, NvU32 *subdev_id) { 0x20B7, 0x1804, 0x20B7, 0x1532 }, { 0x20B9, 0x157F, 0x20B7, 0x1532 }, { 0x20FD, 0x17F8, 0x20F5, 0x0 }, + { 0x230E, 0x20DF, 0x230E, 0x20DF }, { 0x2324, 0x17A8, 0x2324, 0x17A6 }, { 0x2329, 0x198C, 0x2329, 0x198B }, { 0x232C, 0x2064, 0x232C, 0x2063 }, @@ -121,6 +122,13 @@ static const struct { {0x20F610DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_HALF_GPU , 1094}, // GRID A800-4-20C {0x20F610DE, NV2080_CTRL_GPU_PARTITION_FLAG_FULL_GPU , 1095}, // GRID A800-7-40C {0x20F610DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_MINI_QUARTER_GPU , 1091}, // GRID A800-1-10C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_EIGHTHED_GPU | DRF_DEF(2080, _CTRL_GPU_PARTITION_FLAG, _REQ_DEC_JPG_OFA, _ENABLE), 1499}, // NVIDIA H20L-1-15CME + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_EIGHTHED_GPU , 1500}, // NVIDIA H20L-1-15C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_MINI_QUARTER_GPU , 1501}, // NVIDIA H20L-1-30C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_QUARTER_GPU , 1502}, // NVIDIA H20L-2-30C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_MINI_HALF_GPU , 1503}, // NVIDIA H20L-3-60C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_HALF_GPU , 1504}, // NVIDIA H20L-4-60C + {0x230E10DE, NV2080_CTRL_GPU_PARTITION_FLAG_FULL_GPU , 1505}, // NVIDIA H20L-7-120C {0x232110DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_EIGHTHED_GPU | DRF_DEF(2080, _CTRL_GPU_PARTITION_FLAG, _REQ_DEC_JPG_OFA, _ENABLE), 1061}, // NVIDIA H100L-1-12CME {0x232110DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_EIGHTHED_GPU , 1062}, // NVIDIA H100L-1-12C {0x232110DE, NV2080_CTRL_GPU_PARTITION_FLAG_ONE_MINI_QUARTER_GPU , 1063}, // NVIDIA H100L-1-24C diff --git a/src/nvidia-modeset/include/nvkms-modepool.h b/src/nvidia-modeset/include/nvkms-modepool.h index 18e779eee..3ef6c1520 100644 --- a/src/nvidia-modeset/include/nvkms-modepool.h +++ b/src/nvidia-modeset/include/nvkms-modepool.h @@ -51,7 +51,8 @@ NvBool nvValidateModeForModeset(NVDpyEvoRec *pDpyEvo, const struct NvKmsRect *pViewPortOut, NVDpyAttributeColor *pDpyColor, NVHwModeTimingsEvo *pTimingsEvo, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl); + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl); const NVT_TIMING *nvFindEdidNVT_TIMING( const NVDpyEvoRec *pDpyEvo, diff --git a/src/nvidia-modeset/include/nvkms-modeset.h b/src/nvidia-modeset/include/nvkms-modeset.h index 0ae03d38d..35bd66462 100644 --- a/src/nvidia-modeset/include/nvkms-modeset.h +++ b/src/nvidia-modeset/include/nvkms-modeset.h @@ -37,7 +37,8 @@ nvGetHwModeTimings(const NVDispEvoRec *pDispEvo, const struct NvKmsSetModeOneHeadRequest *pRequestHead, NVHwModeTimingsEvo *pTimings, NVDpyAttributeColor *pDpyColor, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl); + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl); NvBool nvGetAllowHeadSurfaceInNvKms(const NVDevEvoRec *pDevEvo, const struct NvKmsPerOpenDev *pOpenDev, diff --git a/src/nvidia-modeset/include/nvkms-types.h b/src/nvidia-modeset/include/nvkms-types.h index 61a48c020..f7724d5ea 100644 --- a/src/nvidia-modeset/include/nvkms-types.h +++ b/src/nvidia-modeset/include/nvkms-types.h @@ -1843,6 +1843,7 @@ typedef struct _NVDispHeadAudioStateEvoRec { typedef struct _NVDispHeadInfoFrameStateEvoRec { NVT_VIDEO_INFOFRAME_CTRL ctrl; + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL vendorSpecificCtrl; NvBool hdTimings; } NVDispHeadInfoFrameStateEvoRec; @@ -2568,6 +2569,15 @@ static inline NvBool nvIs3DVisionStereoEvo(const enum NvKmsStereoMode stereo) stereo == NVKMS_STEREO_NVIDIA_3D_VISION_PRO); } +static inline NvBool nvGetPreferHdmiFrlMode( + const NVDevEvoRec *pDevEvo, + const struct NvKmsModeValidationParams *pParams) +{ + const NvBool preferHDMIFrlMode = + !!(pParams->overrides & NVKMS_MODE_VALIDATION_PREFER_HDMI_FRL_MODE); + return preferHDMIFrlMode; +} + /* * Utility macro for iterating over all head bits set in a head bit mask */ diff --git a/src/nvidia-modeset/interface/nvkms-api.h b/src/nvidia-modeset/interface/nvkms-api.h index 37a5ac11b..85c035cb7 100644 --- a/src/nvidia-modeset/interface/nvkms-api.h +++ b/src/nvidia-modeset/interface/nvkms-api.h @@ -362,6 +362,7 @@ enum NvKmsModeValidationOverrides { NVKMS_MODE_VALIDATION_ALLOW_DP_INTERLACED = (1 << 17), NVKMS_MODE_VALIDATION_NO_INTERLACED_MODES = (1 << 18), NVKMS_MODE_VALIDATION_MAX_ONE_HARDWARE_HEAD = (1 << 19), + NVKMS_MODE_VALIDATION_PREFER_HDMI_FRL_MODE = (1 << 20), }; /*! diff --git a/src/nvidia-modeset/src/dp/nvdp-host.cpp b/src/nvidia-modeset/src/dp/nvdp-host.cpp index 275a5e98c..88f4c7f17 100644 --- a/src/nvidia-modeset/src/dp/nvdp-host.cpp +++ b/src/nvidia-modeset/src/dp/nvdp-host.cpp @@ -42,8 +42,21 @@ void dpFree(void *p) static NVEvoLogType dpSeverityToNvkmsMap(DP_LOG_LEVEL severity) { - NVEvoLogType level = EVO_LOG_INFO; - return level; + switch (severity) { + case DP_SILENT: + case DP_INFO: + case DP_NOTICE: + return EVO_LOG_INFO; + case DP_WARNING: + return EVO_LOG_WARN; + case DP_ERROR: + case DP_HW_ERROR: + case DP_FATAL: + return EVO_LOG_ERROR; + } + + nvAssert(!"Invalid DP_LOG_LEVEL enumerant passed"); + return EVO_LOG_ERROR; } void dpPrintf(DP_LOG_LEVEL severity, const char *format, ...) diff --git a/src/nvidia-modeset/src/nvkms-evo.c b/src/nvidia-modeset/src/nvkms-evo.c index f0f542f14..a71441c3f 100644 --- a/src/nvidia-modeset/src/nvkms-evo.c +++ b/src/nvidia-modeset/src/nvkms-evo.c @@ -6893,17 +6893,18 @@ static NvBool GetDefaultFrlDpyColor( return FALSE; } -static NvBool GetDfpHdmiProtocol(const NVDpyEvoRec *pDpyEvo, - const NvU32 overrides, - NVDpyAttributeColor *pDpyColor, - NVHwModeTimingsEvoPtr pTimings, - enum nvKmsTimingsProtocol *pTimingsProtocol) +static NvBool GetDfpHdmiProtocol( + const NVDpyEvoRec *pDpyEvo, + const struct NvKmsModeValidationParams *pValidationParams, + NVDpyAttributeColor *pDpyColor, + NVHwModeTimingsEvoPtr pTimings, + enum nvKmsTimingsProtocol *pTimingsProtocol) { NVConnectorEvoPtr pConnectorEvo = pDpyEvo->pConnectorEvo; + const NVDevEvoRec *pDevEvo = pConnectorEvo->pDispEvo->pDevEvo; const NvU32 rmProtocol = pConnectorEvo->or.protocol; const NvKmsDpyOutputColorFormatInfo colorFormatsInfo = nvDpyGetOutputColorFormatInfo(pDpyEvo); - const NvBool forceHdmiFrlIfSupported = FALSE; nvAssert(rmProtocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_DUAL_TMDS || rmProtocol == NV0073_CTRL_SPECIFIC_OR_PROTOCOL_SOR_SINGLE_TMDS_A || @@ -6911,11 +6912,12 @@ static NvBool GetDfpHdmiProtocol(const NVDpyEvoRec *pDpyEvo, /* Override protocol if this mode requires HDMI FRL. */ /* If we don't require boot clocks... */ - if (((overrides & NVKMS_MODE_VALIDATION_REQUIRE_BOOT_CLOCKS) == 0) && + if (((pValidationParams->overrides & + NVKMS_MODE_VALIDATION_REQUIRE_BOOT_CLOCKS) == 0) && (!nvHdmiIsTmdsPossible(pDpyEvo, pTimings, pDpyColor) || - forceHdmiFrlIfSupported) && - /* If FRL is supported... */ - nvHdmiDpySupportsFrl(pDpyEvo)) { + nvGetPreferHdmiFrlMode(pDevEvo, pValidationParams)) && + /* If FRL is supported... */ + nvHdmiDpySupportsFrl(pDpyEvo)) { /* Hardware does not support HDMI FRL with YUV422 */ if ((pDpyColor->format == @@ -6977,7 +6979,7 @@ static NvBool GetDfpProtocol(const NVDpyEvoRec *pDpyEvo, if (pConnectorEvo->or.type == NV0073_CTRL_SPECIFIC_OR_TYPE_SOR) { if (nvDpyIsHdmiEvo(pDpyEvo)) { - if (!GetDfpHdmiProtocol(pDpyEvo, overrides, pDpyColor, pTimings, + if (!GetDfpHdmiProtocol(pDpyEvo, pParams, pDpyColor, pTimings, &timingsProtocol)) { return FALSE; } diff --git a/src/nvidia-modeset/src/nvkms-hdmi.c b/src/nvidia-modeset/src/nvkms-hdmi.c index 8e2b14aaf..53a561a0a 100644 --- a/src/nvidia-modeset/src/nvkms-hdmi.c +++ b/src/nvidia-modeset/src/nvkms-hdmi.c @@ -427,38 +427,28 @@ static void SendVideoInfoFrame(const NVDispEvoRec *pDispEvo, } /* - * SendHDMI3DVendorSpecificInfoFrame() - Construct vendor specific infoframe - * using provided EDID and call ->SendHdmiInfoFrame() to send it to RM. Currently - * hardcoded to send the infoframe necessary for HDMI 3D. + * SendVendorSpecificInfoFrame() - Construct vendor specific infoframe using + * provided EDID and call ->SendHdmiInfoFrame() to send it to RM. */ static void -SendHDMI3DVendorSpecificInfoFrame(const NVDispEvoRec *pDispEvo, - const NvU32 head, NVT_EDID_INFO *pEdidInfo) +SendVendorSpecificInfoFrame(const NVDispEvoRec *pDispEvo, + const NvU32 head, + const NVDispHeadInfoFrameStateEvoRec *pInfoFrameState, + NVT_EDID_INFO *pEdidInfo) { const NVDevEvoRec *pDevEvo = pDispEvo->pDevEvo; - const NVDispHeadStateEvoRec *pHeadState = - &pDispEvo->headState[head]; - NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL vendorCtrl = { - .Enable = 1, - .HDMIFormat = NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D, - .HDMI_VIC = NVT_HDMI_VS_BYTE5_HDMI_VIC_NA, - .ThreeDStruc = NVT_HDMI_VS_BYTE5_HDMI_3DS_FRAMEPACK, - .ThreeDDetail = NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_NA, - .MetadataPresent = 0, - .MetadataType = NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_NA, - }; + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL vendorCtrl = pInfoFrameState->vendorSpecificCtrl; NVT_VENDOR_SPECIFIC_INFOFRAME vendorInfoFrame; NVT_STATUS status; - if (!pEdidInfo->HDMI3DSupported) { - // Only send the HDMI 3D infoframe if the display supports HDMI 3D - return; - } - - // Send the infoframe with HDMI 3D configured if we're setting an HDMI 3D - // mode. - if (!pHeadState->timings.hdmi3D) { + /* + * Disable the vendor specific infoframe if not requested to be + * enabled, or if HDMI 3D is requested but not supported by the monitor. + */ + if (!vendorCtrl.Enable || + ((vendorCtrl.HDMIFormat == NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D) && + !pEdidInfo->HDMI3DSupported)) { pDevEvo->hal->DisableHdmiInfoFrame(pDispEvo, head, NVT_INFOFRAME_TYPE_VENDOR_SPECIFIC); return; @@ -575,9 +565,10 @@ void nvUpdateHdmiInfoFrames(const NVDispEvoRec *pDispEvo, pInfoFrameState, &pDpyEvo->parsedEdid.info); - SendHDMI3DVendorSpecificInfoFrame(pDispEvo, - head, - &pDpyEvo->parsedEdid.info); + SendVendorSpecificInfoFrame(pDispEvo, + head, + pInfoFrameState, + &pDpyEvo->parsedEdid.info); SendHDRInfoFrame(pDispEvo, head, @@ -2146,7 +2137,9 @@ static NvBool nvHdmiFrlQueryConfigOneBpc( nvAssert(nvDpyIsHdmiEvo(pDpyEvo)); nvAssert(nvHdmiDpySupportsFrl(pDpyEvo)); - nvAssert(!nvHdmiIsTmdsPossible(pDpyEvo, pHwTimings, pDpyColor)); + + nvAssert(!nvHdmiIsTmdsPossible(pDpyEvo, pHwTimings, pDpyColor) || + nvGetPreferHdmiFrlMode(pDevEvo, pValidationParams)); /* See if we can find an NVT_TIMING for this mode from the EDID. */ pNvtTiming = nvFindEdidNVT_TIMING(pDpyEvo, pModeTimings, pValidationParams); diff --git a/src/nvidia-modeset/src/nvkms-headsurface-config.c b/src/nvidia-modeset/src/nvkms-headsurface-config.c index 56dde60cd..c7d359a81 100644 --- a/src/nvidia-modeset/src/nvkms-headsurface-config.c +++ b/src/nvidia-modeset/src/nvkms-headsurface-config.c @@ -833,7 +833,8 @@ NvBool nvHsConfigInitModeset( if (!nvGetHwModeTimings(pDispEvo, apiHead, pRequestHead, pTimings, NULL /* pDpyColor */, - NULL /* pInfoFrameCtrl */)) { + NULL /* pInfoFrameCtrl */, + NULL /* pVSInfoFrameCtrl */)) { nvPreallocRelease(pDevEvo, PREALLOC_TYPE_HS_INIT_CONFIG_HW_TIMINGS); return FALSE; } @@ -1250,7 +1251,8 @@ NvBool nvHsConfigDowngrade( if (!nvGetHwModeTimings(pDispEvo, apiHead, pRequestHead, pTimings, NULL /* pDpyColor */, - NULL /* pInfoFrameCtrl */)) { + NULL /* pInfoFrameCtrl */, + NULL /* pVSInfoFrameCtrl */)) { nvPreallocRelease(pDevEvo, PREALLOC_TYPE_HS_INIT_CONFIG_HW_TIMINGS); return FALSE; } diff --git a/src/nvidia-modeset/src/nvkms-modepool.c b/src/nvidia-modeset/src/nvkms-modepool.c index 8dcc23be5..baeb394b4 100644 --- a/src/nvidia-modeset/src/nvkms-modepool.c +++ b/src/nvidia-modeset/src/nvkms-modepool.c @@ -71,7 +71,8 @@ static NvBool ConstructModeTimingsMetaData( const struct NvKmsModeValidationParams *pParams, struct NvKmsMode *pKmsMode, EvoValidateModeFlags *pFlags, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl); + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl); static NvBool ValidateMode(NVDpyEvoPtr pDpyEvo, const struct NvKmsMode *pKmsMode, @@ -143,7 +144,6 @@ nvValidateModeEvo(NVDpyEvoPtr pDpyEvo, .timings = pRequest->mode.timings, }; EvoValidateModeFlags evoFlags; - NVT_VIDEO_INFOFRAME_CTRL dummyInfoFrameCtrl; nvkms_memset(pReply, 0, sizeof(*pReply)); @@ -151,7 +151,8 @@ nvValidateModeEvo(NVDpyEvoPtr pDpyEvo, &pRequest->modeValidation, &kmsMode, &evoFlags, - &dummyInfoFrameCtrl)) { + NULL /* pInfoFrameCtrl */, + NULL /* pVSInfoFrameCtrl */)) { pReply->valid = FALSE; return; } @@ -1837,13 +1838,27 @@ const NVT_TIMING *nvFindEdidNVT_TIMING const struct NvKmsModeValidationParams *pParams ) { + const NVParsedEdidEvoRec *pParsedEdid = &pDpyEvo->parsedEdid; + const NVT_HDMI_FORUM_INFO *pHdmiInfo = &pParsedEdid->info.hdmiForumInfo; NvModeTimings tmpModeTimings; + int match861stOnly; int i; if (!pDpyEvo->parsedEdid.valid) { return NULL; } + /* + * In the first pass, attempt to match the pModeTimings with CEA/CTA 861 + * video formats if the monitor prefers them, or if HDMI 3D is requested. + */ + if (nvDpyIsHdmiEvo(pDpyEvo) && + (pModeTimings->hdmi3D || pHdmiInfo->uhd_vic)) { + match861stOnly = 1; + } else { + match861stOnly = 0; + } + tmpModeTimings = *pModeTimings; /* @@ -1860,22 +1875,62 @@ const NVT_TIMING *nvFindEdidNVT_TIMING */ tmpModeTimings.yuv420Mode = NV_YUV420_MODE_NONE; - for (i = 0; i < pDpyEvo->parsedEdid.info.total_timings; i++) { - const NVT_TIMING *pTiming = &pDpyEvo->parsedEdid.info.timing[i]; - if (NVT_TIMINGmatchesNvModeTimings(pTiming, &tmpModeTimings, pParams) && - /* - * Only consider the mode a match if the yuv420 - * configuration of pTiming would match pModeTimings. - */ - (pModeTimings->yuv420Mode == - GetYUV420Value(pDpyEvo, pParams, pTiming))) { - return pTiming; + for (; match861stOnly >= 0; match861stOnly--) { + for (i = 0; i < pDpyEvo->parsedEdid.info.total_timings; i++) { + const NVT_TIMING *pTiming = &pDpyEvo->parsedEdid.info.timing[i]; + + if (match861stOnly && + (NVT_GET_TIMING_STATUS_TYPE(pTiming->etc.status) != + NVT_TYPE_EDID_861ST)) { + continue; + } + + if (NVT_TIMINGmatchesNvModeTimings(pTiming, &tmpModeTimings, pParams) && + /* + * Only consider the mode a match if the yuv420 + * configuration of pTiming would match pModeTimings. + */ + (pModeTimings->yuv420Mode == + GetYUV420Value(pDpyEvo, pParams, pTiming))) { + return pTiming; + } } } return NULL; } +static void ConstructVSInfoFrameCtrls( + const NVT_TIMING *pTiming, + const NvBool hdmi3D, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSCtrl) +{ + if (!hdmi3D && (NVT_GET_TIMING_STATUS_TYPE(pTiming->etc.status) != + NVT_TYPE_HDMI_EXT)) { + pVSCtrl->Enable = FALSE; + pVSCtrl->VSIFVersion = NVT_VSIF_VERSION_NONE; + return; + } + + pVSCtrl->Enable = TRUE; + pVSCtrl->VSIFVersion = NVT_VSIF_VERSION_H14B_VSIF; + + if (hdmi3D) { + pVSCtrl->HDMIFormat = NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_3D; + pVSCtrl->HDMI_VIC = NVT_HDMI_VS_BYTE5_HDMI_VIC_NA; + pVSCtrl->ThreeDStruc = NVT_HDMI_VS_BYTE5_HDMI_3DS_FRAMEPACK; + } else if (NVT_GET_TIMING_STATUS_TYPE(pTiming->etc.status) == + NVT_TYPE_HDMI_EXT) { + pVSCtrl->HDMIFormat = NVT_HDMI_VS_BYTE4_HDMI_VID_FMT_EXT; + pVSCtrl->HDMI_VIC = NVT_GET_TIMING_STATUS_SEQ(pTiming->etc.status); + pVSCtrl->ThreeDStruc = NVT_HDMI_VS_BYTE5_HDMI_3DS_NA; + } + + pVSCtrl->ThreeDDetail = NVT_HDMI_VS_BYTE_OPT1_HDMI_3DEX_NA; + pVSCtrl->MetadataPresent = 0; + pVSCtrl->MetadataType = NVT_HDMI_VS_BYTE_OPT2_HDMI_METADATA_TYPE_NA; +} + /*! * Construct mode-timing's meta data required for mode validation * logic. This meta data involves EvoValidateModeFlags, patched stereo @@ -1896,11 +1951,13 @@ static NvBool ConstructModeTimingsMetaData( const struct NvKmsModeValidationParams *pParams, struct NvKmsMode *pKmsMode, EvoValidateModeFlags *pFlags, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl) + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl) { const NVDispEvoRec *pDispEvo = pDpyEvo->pDispEvo; EvoValidateModeFlags flags = { 0 }; NVT_VIDEO_INFOFRAME_CTRL infoFrameCtrl; + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL vsInfoFrameCtrl = { }; NvModeTimings modeTimings = pKmsMode->timings; const NVT_TIMING *pTiming; @@ -1981,6 +2038,7 @@ static NvBool ConstructModeTimingsMetaData( */ if (nvDpyIsHdmiEvo(pDpyEvo)) { NvTiming_ConstructVideoInfoframeCtrl(&timing, &infoFrameCtrl); + ConstructVSInfoFrameCtrls(&timing, hdmi3D, &vsInfoFrameCtrl); } goto done; @@ -2000,7 +2058,12 @@ static NvBool ConstructModeTimingsMetaData( done: *pFlags = flags; - *pInfoFrameCtrl = infoFrameCtrl; + if (pInfoFrameCtrl != NULL) { + *pInfoFrameCtrl = infoFrameCtrl; + } + if (pVSInfoFrameCtrl != NULL) { + *pVSInfoFrameCtrl = vsInfoFrameCtrl; + } pKmsMode->timings = modeTimings; return TRUE; @@ -2024,11 +2087,11 @@ NvBool nvValidateModeForModeset(NVDpyEvoRec *pDpyEvo, const struct NvKmsRect *pViewPortOut, NVDpyAttributeColor *pDpyColor, NVHwModeTimingsEvo *pTimingsEvo, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl) + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl) { EvoValidateModeFlags flags; struct NvKmsMode kmsMode = *pKmsMode; - NVT_VIDEO_INFOFRAME_CTRL infoFrameCtrl; struct NvKmsModeValidationValidSyncs dummyValidSyncs; nvkms_memset(pTimingsEvo, 0, sizeof(*pTimingsEvo)); @@ -2037,7 +2100,8 @@ NvBool nvValidateModeForModeset(NVDpyEvoRec *pDpyEvo, pParams, &kmsMode, &flags, - &infoFrameCtrl)) { + pInfoFrameCtrl, + pVSInfoFrameCtrl)) { return FALSE; } @@ -2062,9 +2126,5 @@ NvBool nvValidateModeForModeset(NVDpyEvoRec *pDpyEvo, return FALSE; } - if (pInfoFrameCtrl != NULL) { - *pInfoFrameCtrl = infoFrameCtrl; - } - return TRUE; } diff --git a/src/nvidia-modeset/src/nvkms-modeset.c b/src/nvidia-modeset/src/nvkms-modeset.c index c22cbadb1..03d28604e 100644 --- a/src/nvidia-modeset/src/nvkms-modeset.c +++ b/src/nvidia-modeset/src/nvkms-modeset.c @@ -180,7 +180,8 @@ nvGetHwModeTimings(const NVDispEvoRec *pDispEvo, const struct NvKmsSetModeOneHeadRequest *pRequestHead, NVHwModeTimingsEvo *pTimings, NVDpyAttributeColor *pDpyColor, - NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl) + NVT_VIDEO_INFOFRAME_CTRL *pInfoFrameCtrl, + NVT_VENDOR_SPECIFIC_INFOFRAME_CTRL *pVSInfoFrameCtrl) { NVDpyEvoPtr pDpyEvo; NVDpyAttributeColor dpyColor = { }; @@ -208,7 +209,8 @@ nvGetHwModeTimings(const NVDispEvoRec *pDispEvo, &pRequestHead->viewPortOut : NULL, &dpyColor, pTimings, - pInfoFrameCtrl)) { + pInfoFrameCtrl, + pVSInfoFrameCtrl)) { return FALSE; } @@ -1128,7 +1130,8 @@ AssignProposedModeSetHwState(NVDevEvoRec *pDevEvo, pRequestHead, &pProposedApiHead->timings, &pProposedApiHead->attributes.color, - &pProposedApiHead->infoFrame.ctrl)) { + &pProposedApiHead->infoFrame.ctrl, + &pProposedApiHead->infoFrame.vendorSpecificCtrl)) { pReply->disp[sd].head[apiHead].status = NVKMS_SET_MODE_ONE_HEAD_STATUS_INVALID_MODE; ret = FALSE; diff --git a/src/nvidia/arch/nvalloc/unix/src/osapi.c b/src/nvidia/arch/nvalloc/unix/src/osapi.c index ef766512a..8efb3db78 100644 --- a/src/nvidia/arch/nvalloc/unix/src/osapi.c +++ b/src/nvidia/arch/nvalloc/unix/src/osapi.c @@ -4307,9 +4307,24 @@ void NV_API_CALL rm_power_source_change_event( ) { THREAD_STATE_NODE threadState; - void *fp; - nv_state_t *nv; - NV_STATUS rmStatus = NV_OK; + void *fp; + nv_state_t *nv; + static NvBool first_event_seen = NV_FALSE; + static NvU32 last_event_val = 0; + NV_STATUS rmStatus = NV_OK; + + // Some systems generate spurious power source changed ACPI events. + // Before entering the PM runtime and waking the GPU, see if the power + // state has actually changed. If not, return without waking the GPU. + if ((first_event_seen != NV_FALSE) && (last_event_val == event_val)) + { + return; + } + else + { + first_event_seen = NV_TRUE; + last_event_val = event_val; + } NV_ENTER_RM_RUNTIME(sp,fp); threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); diff --git a/src/nvidia/generated/g_all_dcl_pb.h b/src/nvidia/generated/g_all_dcl_pb.h index 47abe37f0..045079036 100644 --- a/src/nvidia/generated/g_all_dcl_pb.h +++ b/src/nvidia/generated/g_all_dcl_pb.h @@ -20,8 +20,8 @@ extern const PRB_MSG_DESC prb_messages_dcl[]; // Message maximum lengths // Does not include repeated fields, strings and byte arrays. #define DCL_ENGINES_LEN 142 -#define DCL_DCLMSG_LEN 2262 -#define DCL_ERRORBLOCK_LEN 2266 +#define DCL_DCLMSG_LEN 2298 +#define DCL_ERRORBLOCK_LEN 2302 extern const PRB_FIELD_DESC prb_fields_dcl_engines[]; @@ -58,8 +58,8 @@ extern const PRB_FIELD_DESC prb_fields_dcl_dclmsg[]; #define DCL_DCLMSG_ENGINE_LEN 145 #define DCL_DCLMSG_RC_DIAG_RECS_LEN 42 #define DCL_DCLMSG_CRASHCAT_REPORT_LEN 564 -#define DCL_DCLMSG_GSP_RPCDEBUGINFO_LEN 245 -#define DCL_DCLMSG_GSP_XIDREPORT_LEN 828 +#define DCL_DCLMSG_GSP_RPCDEBUGINFO_LEN 263 +#define DCL_DCLMSG_GSP_XIDREPORT_LEN 846 extern const PRB_FIELD_DESC prb_fields_dcl_errorblock[]; @@ -67,7 +67,7 @@ extern const PRB_FIELD_DESC prb_fields_dcl_errorblock[]; #define DCL_ERRORBLOCK_DATA (&prb_fields_dcl_errorblock[0]) // 'ErrorBlock' field lengths -#define DCL_ERRORBLOCK_DATA_LEN 2265 +#define DCL_ERRORBLOCK_DATA_LEN 2301 extern const PRB_SERVICE_DESC prb_services_dcl[]; diff --git a/src/nvidia/generated/g_engines_pb.c b/src/nvidia/generated/g_engines_pb.c index d0cbc6ba0..f3a3d1902 100644 --- a/src/nvidia/generated/g_engines_pb.c +++ b/src/nvidia/generated/g_engines_pb.c @@ -380,6 +380,18 @@ const PRB_FIELD_DESC prb_fields_nvdebug_eng_kgsp_rpcinfo[] = { PRB_MAYBE_FIELD_NAME("data1") PRB_MAYBE_FIELD_DEFAULT(0) }, + { + 6, + { + PRB_OPTIONAL, + PRB_UINT32, + 0, + }, + 0, + 0, + PRB_MAYBE_FIELD_NAME("sequence") + PRB_MAYBE_FIELD_DEFAULT(0) + }, }; // Message descriptors @@ -415,7 +427,7 @@ const PRB_MSG_DESC prb_messages_nvdebug_eng[] = { PRB_MAYBE_MESSAGE_NAME("NvDebug.Eng.Mc.PciBarInfo") }, { - 5, + 6, prb_fields_nvdebug_eng_kgsp_rpcinfo, PRB_MAYBE_MESSAGE_NAME("NvDebug.Eng.KGsp.RpcInfo") }, diff --git a/src/nvidia/generated/g_engines_pb.h b/src/nvidia/generated/g_engines_pb.h index 2817c35a2..ada513bb0 100644 --- a/src/nvidia/generated/g_engines_pb.h +++ b/src/nvidia/generated/g_engines_pb.h @@ -21,10 +21,10 @@ extern const PRB_MSG_DESC prb_messages_nvdebug_eng[]; #define NVDEBUG_ENG_MC_LEN 72 #define NVDEBUG_ENG_GPU_LEN 62 #define NVDEBUG_ENG_NVD_LEN 30 -#define NVDEBUG_ENG_KGSP_LEN 88 +#define NVDEBUG_ENG_KGSP_LEN 100 #define NVDEBUG_ENG_MC_RMDATA_LEN 12 #define NVDEBUG_ENG_MC_PCIBARINFO_LEN 22 -#define NVDEBUG_ENG_KGSP_RPCINFO_LEN 40 +#define NVDEBUG_ENG_KGSP_RPCINFO_LEN 46 extern const PRB_FIELD_DESC prb_fields_nvdebug_eng_mc[]; @@ -85,8 +85,8 @@ extern const PRB_FIELD_DESC prb_fields_nvdebug_eng_kgsp[]; #define NVDEBUG_ENG_KGSP_EVENT_HISTORY (&prb_fields_nvdebug_eng_kgsp[1]) // 'KGsp' field lengths -#define NVDEBUG_ENG_KGSP_RPC_HISTORY_LEN 43 -#define NVDEBUG_ENG_KGSP_EVENT_HISTORY_LEN 43 +#define NVDEBUG_ENG_KGSP_RPC_HISTORY_LEN 49 +#define NVDEBUG_ENG_KGSP_EVENT_HISTORY_LEN 49 extern const PRB_FIELD_DESC prb_fields_nvdebug_eng_mc_rmdata[]; @@ -116,6 +116,7 @@ extern const PRB_FIELD_DESC prb_fields_nvdebug_eng_kgsp_rpcinfo[]; #define NVDEBUG_ENG_KGSP_RPCINFO_TS_END (&prb_fields_nvdebug_eng_kgsp_rpcinfo[2]) #define NVDEBUG_ENG_KGSP_RPCINFO_DATA0 (&prb_fields_nvdebug_eng_kgsp_rpcinfo[3]) #define NVDEBUG_ENG_KGSP_RPCINFO_DATA1 (&prb_fields_nvdebug_eng_kgsp_rpcinfo[4]) +#define NVDEBUG_ENG_KGSP_RPCINFO_SEQUENCE (&prb_fields_nvdebug_eng_kgsp_rpcinfo[5]) // 'RpcInfo' field lengths #define NVDEBUG_ENG_KGSP_RPCINFO_FUNCTION_LEN 5 @@ -123,6 +124,7 @@ extern const PRB_FIELD_DESC prb_fields_nvdebug_eng_kgsp_rpcinfo[]; #define NVDEBUG_ENG_KGSP_RPCINFO_TS_END_LEN 10 #define NVDEBUG_ENG_KGSP_RPCINFO_DATA0_LEN 5 #define NVDEBUG_ENG_KGSP_RPCINFO_DATA1_LEN 5 +#define NVDEBUG_ENG_KGSP_RPCINFO_SEQUENCE_LEN 5 extern const PRB_SERVICE_DESC prb_services_nvdebug_eng[]; diff --git a/src/nvidia/generated/g_gsp_pb.c b/src/nvidia/generated/g_gsp_pb.c index db1d3b3e5..0dd6715b7 100644 --- a/src/nvidia/generated/g_gsp_pb.c +++ b/src/nvidia/generated/g_gsp_pb.c @@ -104,6 +104,18 @@ const PRB_FIELD_DESC prb_fields_gsp_rpcentry[] = { PRB_MAYBE_FIELD_NAME("duration") PRB_MAYBE_FIELD_DEFAULT(0) }, + { + 9, + { + PRB_OPTIONAL, + PRB_UINT32, + 0, + }, + 0, + 0, + PRB_MAYBE_FIELD_NAME("sequence") + PRB_MAYBE_FIELD_DEFAULT(0) + }, }; // 'RpcHistoryCpuToGsp' field defaults @@ -253,7 +265,7 @@ const PRB_FIELD_DESC prb_fields_gsp_xidreport[] = { // Message descriptors const PRB_MSG_DESC prb_messages_gsp[] = { { - 8, + 9, prb_fields_gsp_rpcentry, PRB_MAYBE_MESSAGE_NAME("Gsp.RpcEntry") }, diff --git a/src/nvidia/generated/g_gsp_pb.h b/src/nvidia/generated/g_gsp_pb.h index 34779e53b..37c7f15c8 100644 --- a/src/nvidia/generated/g_gsp_pb.h +++ b/src/nvidia/generated/g_gsp_pb.h @@ -16,11 +16,11 @@ extern const PRB_MSG_DESC prb_messages_gsp[]; // Message maximum lengths // Does not include repeated fields, strings and byte arrays. -#define GSP_RPCENTRY_LEN 74 -#define GSP_RPCHISTORYCPUTOGSP_LEN 78 -#define GSP_RPCHISTORYGSPTOCPU_LEN 78 -#define GSP_RPCDEBUGINFO_LEN 242 -#define GSP_XIDREPORT_LEN 825 +#define GSP_RPCENTRY_LEN 80 +#define GSP_RPCHISTORYCPUTOGSP_LEN 84 +#define GSP_RPCHISTORYGSPTOCPU_LEN 84 +#define GSP_RPCDEBUGINFO_LEN 260 +#define GSP_XIDREPORT_LEN 843 extern const PRB_FIELD_DESC prb_fields_gsp_rpcentry[]; @@ -33,6 +33,7 @@ extern const PRB_FIELD_DESC prb_fields_gsp_rpcentry[]; #define GSP_RPCENTRY_STARTTIMESTAMP (&prb_fields_gsp_rpcentry[5]) #define GSP_RPCENTRY_ENDTIMESTAMP (&prb_fields_gsp_rpcentry[6]) #define GSP_RPCENTRY_DURATION (&prb_fields_gsp_rpcentry[7]) +#define GSP_RPCENTRY_SEQUENCE (&prb_fields_gsp_rpcentry[8]) // 'RpcEntry' field lengths #define GSP_RPCENTRY_HISTORYINDEX_LEN 5 @@ -43,6 +44,7 @@ extern const PRB_FIELD_DESC prb_fields_gsp_rpcentry[]; #define GSP_RPCENTRY_STARTTIMESTAMP_LEN 10 #define GSP_RPCENTRY_ENDTIMESTAMP_LEN 10 #define GSP_RPCENTRY_DURATION_LEN 10 +#define GSP_RPCENTRY_SEQUENCE_LEN 5 extern const PRB_FIELD_DESC prb_fields_gsp_rpchistorycputogsp[]; @@ -50,7 +52,7 @@ extern const PRB_FIELD_DESC prb_fields_gsp_rpchistorycputogsp[]; #define GSP_RPCHISTORYCPUTOGSP_RPCENTRY (&prb_fields_gsp_rpchistorycputogsp[0]) // 'RpcHistoryCpuToGsp' field lengths -#define GSP_RPCHISTORYCPUTOGSP_RPCENTRY_LEN 77 +#define GSP_RPCHISTORYCPUTOGSP_RPCENTRY_LEN 83 extern const PRB_FIELD_DESC prb_fields_gsp_rpchistorygsptocpu[]; @@ -58,7 +60,7 @@ extern const PRB_FIELD_DESC prb_fields_gsp_rpchistorygsptocpu[]; #define GSP_RPCHISTORYGSPTOCPU_RPCENTRY (&prb_fields_gsp_rpchistorygsptocpu[0]) // 'RpcHistoryGspToCpu' field lengths -#define GSP_RPCHISTORYGSPTOCPU_RPCENTRY_LEN 77 +#define GSP_RPCHISTORYGSPTOCPU_RPCENTRY_LEN 83 extern const PRB_FIELD_DESC prb_fields_gsp_rpcdebuginfo[]; @@ -68,9 +70,9 @@ extern const PRB_FIELD_DESC prb_fields_gsp_rpcdebuginfo[]; #define GSP_RPCDEBUGINFO_RPCHISTORYGSPTOCPU (&prb_fields_gsp_rpcdebuginfo[2]) // 'RpcDebugInfo' field lengths -#define GSP_RPCDEBUGINFO_ACTIVERPC_LEN 77 -#define GSP_RPCDEBUGINFO_RPCHISTORYCPUTOGSP_LEN 81 -#define GSP_RPCDEBUGINFO_RPCHISTORYGSPTOCPU_LEN 81 +#define GSP_RPCDEBUGINFO_ACTIVERPC_LEN 83 +#define GSP_RPCDEBUGINFO_RPCHISTORYCPUTOGSP_LEN 87 +#define GSP_RPCDEBUGINFO_RPCHISTORYGSPTOCPU_LEN 87 extern const PRB_FIELD_DESC prb_fields_gsp_xidreport[]; @@ -86,7 +88,7 @@ extern const PRB_FIELD_DESC prb_fields_gsp_xidreport[]; #define GSP_XIDREPORT_GPUINSTANCE_LEN 5 #define GSP_XIDREPORT_BUILDID_LEN 1 #define GSP_XIDREPORT_CRASHCATREPORT_LEN 564 -#define GSP_XIDREPORT_RPCDEBUGINFO_LEN 245 +#define GSP_XIDREPORT_RPCDEBUGINFO_LEN 263 extern const PRB_SERVICE_DESC prb_services_gsp[]; diff --git a/src/nvidia/generated/g_kern_bus_nvoc.h b/src/nvidia/generated/g_kern_bus_nvoc.h index 145a4d5dd..28027069d 100644 --- a/src/nvidia/generated/g_kern_bus_nvoc.h +++ b/src/nvidia/generated/g_kern_bus_nvoc.h @@ -116,6 +116,14 @@ typedef enum #define BUS_MAP_FB_FLAGS_PRE_INIT NVBIT(7) #define BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG NVBIT(8) #define BUS_MAP_FB_FLAGS_UNMANAGED_MEM_AREA NVBIT(9) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_4K NVBIT(10) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_64K NVBIT(11) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_2M NVBIT(12) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_512M NVBIT(13) +// Reserve 3 bits for future expansion of page size. +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_RESERVED_2 NVBIT(14) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_RESERVED_3 NVBIT(15) +#define BUS_MAP_FB_FLAGS_PAGE_SIZE_RESERVED_4 NVBIT(16) #define BUS_MAP_FB_FLAGS_ALL_FLAGS \ (BUS_MAP_FB_FLAGS_MAP_RSVD_BAR1 |\ @@ -127,6 +135,10 @@ typedef enum BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED |\ BUS_MAP_FB_FLAGS_PRE_INIT |\ BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_4K |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_64K |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_2M |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_512M |\ BUS_MAP_FB_FLAGS_UNMANAGED_MEM_AREA) #define BUS_MAP_FB_FLAGS_FERMI_INVALID ((~BUS_MAP_FB_FLAGS_ALL_FLAGS) | BUS_MAP_FB_FLAGS_MAP_RSVD_BAR1) @@ -233,6 +245,7 @@ typedef struct Bar1VaInfo NvU64 apertureLength; // Aperture length that is visible to CPU NvU64 mappableLength; // Total mappable aperture length after WARs struct OBJVASPACE *pVAS; + NvU64 vasFreeSize; // Cached value of the BAR1 VAS's free size used by PMA NvU64 instBlockBase; MEMORY_DESCRIPTOR *pInstBlkMemDesc; ReuseMappingDb reuseDb; diff --git a/src/nvidia/generated/g_nv_name_released.h b/src/nvidia/generated/g_nv_name_released.h index 2ead7be19..dacb110a7 100644 --- a/src/nvidia/generated/g_nv_name_released.h +++ b/src/nvidia/generated/g_nv_name_released.h @@ -5214,6 +5214,7 @@ static const CHIPS_RELEASED sChipsReleased[] = { { 0x2236, 0x1482, 0x10de, "NVIDIA A10" }, { 0x2237, 0x152f, 0x10de, "NVIDIA A10G" }, { 0x2238, 0x1677, 0x10de, "NVIDIA A10M" }, + { 0x230E, 0x20df, 0x10de, "NVIDIA H20 NVL16" }, { 0x2321, 0x1839, 0x10de, "NVIDIA H100 NVL" }, { 0x2322, 0x17a4, 0x10de, "NVIDIA H800 PCIe" }, { 0x2324, 0x17a6, 0x10de, "NVIDIA H800" }, @@ -5414,9 +5415,10 @@ static const CHIPS_RELEASED sChipsReleased[] = { { 0x2941, 0x20d5, 0x10de, "NVIDIA GB200" }, { 0x2941, 0x21c9, 0x10de, "NVIDIA GB200" }, { 0x2941, 0x21ca, 0x10de, "NVIDIA GB200" }, + { 0x29BB, 0x207c, 0x10de, "NVIDIA DRIVE P2021" }, { 0x2B85, 0x0000, 0x0000, "NVIDIA GeForce RTX 5090" }, { 0x2B87, 0x0000, 0x0000, "NVIDIA GeForce RTX 5090 D" }, - { 0x2B8C, 0x0000, 0x0000, "NVIDIA GeForce RTX 5090 D V2" }, + { 0x2B8C, 0x0000, 0x0000, "NVIDIA GeForce RTX 5090 D v2" }, { 0x2BB1, 0x204b, 0x1028, "NVIDIA RTX PRO 6000 Blackwell Workstation Edition" }, { 0x2BB1, 0x204b, 0x103c, "NVIDIA RTX PRO 6000 Blackwell Workstation Edition" }, { 0x2BB1, 0x204b, 0x10de, "NVIDIA RTX PRO 6000 Blackwell Workstation Edition" }, @@ -5430,6 +5432,7 @@ static const CHIPS_RELEASED sChipsReleased[] = { { 0x2BB4, 0x204c, 0x10de, "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition" }, { 0x2BB4, 0x204c, 0x17aa, "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition" }, { 0x2BB5, 0x204e, 0x10de, "NVIDIA RTX PRO 6000 Blackwell Server Edition" }, + { 0x2BB9, 0x2091, 0x10de, "NVIDIA RTX 6000D" }, { 0x2C02, 0x0000, 0x0000, "NVIDIA GeForce RTX 5080" }, { 0x2C05, 0x0000, 0x0000, "NVIDIA GeForce RTX 5070 Ti" }, { 0x2C18, 0x0000, 0x0000, "NVIDIA GeForce RTX 5090 Laptop GPU" }, @@ -5470,6 +5473,8 @@ static const CHIPS_RELEASED sChipsReleased[] = { { 0x2F18, 0x0000, 0x0000, "NVIDIA GeForce RTX 5070 Ti Laptop GPU" }, { 0x2F38, 0x0000, 0x0000, "NVIDIA RTX PRO 3000 Blackwell Generation Laptop GPU" }, { 0x2F58, 0x0000, 0x0000, "NVIDIA GeForce RTX 5070 Ti Laptop GPU" }, + { 0x3182, 0x20e6, 0x10de, "NVIDIA B300 SXM6 AC" }, + { 0x31C2, 0x21f1, 0x10de, "NVIDIA GB300" }, { 0x13BD, 0x11cc, 0x10DE, "GRID M10-0B" }, { 0x13BD, 0x11cd, 0x10DE, "GRID M10-1B" }, { 0x13BD, 0x11ce, 0x10DE, "GRID M10-0Q" }, @@ -6070,6 +6075,25 @@ static const CHIPS_RELEASED sChipsReleased[] = { { 0x2238, 0x16b9, 0x10DE, "NVIDIA A10M-20C" }, { 0x2238, 0x16e6, 0x10DE, "NVIDIA A10M-1" }, { 0x2238, 0x2208, 0x10DE, "NVIDIA A10M-3B" }, + { 0x230E, 0x20f5, 0x10DE, "NVIDIA H20L-1-15CME" }, + { 0x230E, 0x20f6, 0x10DE, "NVIDIA H20L-1-15C" }, + { 0x230E, 0x20f7, 0x10DE, "NVIDIA H20L-1-30C" }, + { 0x230E, 0x20f8, 0x10DE, "NVIDIA H20L-2-30C" }, + { 0x230E, 0x20f9, 0x10DE, "NVIDIA H20L-3-60C" }, + { 0x230E, 0x20fa, 0x10DE, "NVIDIA H20L-4-60C" }, + { 0x230E, 0x20fb, 0x10DE, "NVIDIA H20L-7-120C" }, + { 0x230E, 0x20fc, 0x10DE, "NVIDIA H20L-4C" }, + { 0x230E, 0x20fd, 0x10DE, "NVIDIA H20L-5C" }, + { 0x230E, 0x20fe, 0x10DE, "NVIDIA H20L-6C" }, + { 0x230E, 0x20ff, 0x10DE, "NVIDIA H20L-8C" }, + { 0x230E, 0x2100, 0x10DE, "NVIDIA H20L-10C" }, + { 0x230E, 0x2101, 0x10DE, "NVIDIA H20L-12C" }, + { 0x230E, 0x2102, 0x10DE, "NVIDIA H20L-15C" }, + { 0x230E, 0x2103, 0x10DE, "NVIDIA H20L-20C" }, + { 0x230E, 0x2104, 0x10DE, "NVIDIA H20L-30C" }, + { 0x230E, 0x2105, 0x10DE, "NVIDIA H20L-40C" }, + { 0x230E, 0x2106, 0x10DE, "NVIDIA H20L-60C" }, + { 0x230E, 0x2107, 0x10DE, "NVIDIA H20L-120C" }, { 0x2321, 0x1853, 0x10DE, "NVIDIA H100L-1-12CME" }, { 0x2321, 0x1854, 0x10DE, "NVIDIA H100L-1-12C" }, { 0x2321, 0x1855, 0x10DE, "NVIDIA H100L-1-24C" }, diff --git a/src/nvidia/generated/g_nvdebug_pb.h b/src/nvidia/generated/g_nvdebug_pb.h index 5de397079..1fd84206d 100644 --- a/src/nvidia/generated/g_nvdebug_pb.h +++ b/src/nvidia/generated/g_nvdebug_pb.h @@ -42,8 +42,8 @@ extern const PRB_MSG_DESC prb_messages_nvdebug[]; // Message maximum lengths // Does not include repeated fields, strings and byte arrays. #define NVDEBUG_SYSTEMINFO_LEN 354 -#define NVDEBUG_GPUINFO_LEN 268 -#define NVDEBUG_NVDUMP_LEN 3265 +#define NVDEBUG_GPUINFO_LEN 280 +#define NVDEBUG_NVDUMP_LEN 3313 #define NVDEBUG_SYSTEMINFO_NORTHBRIDGEINFO_LEN 12 #define NVDEBUG_SYSTEMINFO_SOCINFO_LEN 12 #define NVDEBUG_SYSTEMINFO_CPUINFO_LEN 24 @@ -101,7 +101,7 @@ extern const PRB_FIELD_DESC prb_fields_nvdebug_gpuinfo[]; #define NVDEBUG_GPUINFO_ENG_GPU_LEN 65 #define NVDEBUG_GPUINFO_ENG_MC_LEN 75 #define NVDEBUG_GPUINFO_ENG_NVD_LEN 33 -#define NVDEBUG_GPUINFO_ENG_KGSP_LEN 91 +#define NVDEBUG_GPUINFO_ENG_KGSP_LEN 103 extern const PRB_FIELD_DESC prb_fields_nvdebug_nvdump[]; @@ -114,8 +114,8 @@ extern const PRB_FIELD_DESC prb_fields_nvdebug_nvdump[]; // 'NvDump' field lengths #define NVDEBUG_NVDUMP_SYSTEM_INFO_LEN 357 -#define NVDEBUG_NVDUMP_DCL_MSG_LEN 2265 -#define NVDEBUG_NVDUMP_GPU_INFO_LEN 271 +#define NVDEBUG_NVDUMP_DCL_MSG_LEN 2301 +#define NVDEBUG_NVDUMP_GPU_INFO_LEN 283 #define NVDEBUG_NVDUMP_EXCEPTION_ADDRESS_LEN 10 #define NVDEBUG_NVDUMP_SYSTEM_INFO_GSPRM_LEN 357 diff --git a/src/nvidia/generated/g_rpc_hal.h b/src/nvidia/generated/g_rpc_hal.h index 803d22258..84b23114d 100644 --- a/src/nvidia/generated/g_rpc_hal.h +++ b/src/nvidia/generated/g_rpc_hal.h @@ -16,8 +16,8 @@ typedef NV_STATUS RpcConstruct(POBJGPU, POBJRPC); typedef void RpcDestroy(POBJGPU, POBJRPC); -typedef NV_STATUS RpcSendMessage(POBJGPU, POBJRPC); -typedef NV_STATUS RpcRecvPoll(POBJGPU, POBJRPC, NvU32); +typedef NV_STATUS RpcSendMessage(POBJGPU, POBJRPC, NvU32 *); +typedef NV_STATUS RpcRecvPoll(POBJGPU, POBJRPC, NvU32, NvU32); // @@ -42,10 +42,10 @@ typedef struct RPC_OBJ_IFACES { (_pRpc)->obj.__rpcConstruct__(_pGpu, _pRpc) #define rpcDestroy(_pGpu, _pRpc) \ (_pRpc)->obj.__rpcDestroy__(_pGpu, _pRpc) -#define rpcSendMessage(_pGpu, _pRpc) \ - (_pRpc)->obj.__rpcSendMessage__(_pGpu, _pRpc) -#define rpcRecvPoll(_pGpu, _pRpc, _arg0) \ - (_pRpc)->obj.__rpcRecvPoll__(_pGpu, _pRpc, _arg0) +#define rpcSendMessage(_pGpu, _pRpc, _pArg0) \ + (_pRpc)->obj.__rpcSendMessage__(_pGpu, _pRpc, _pArg0) +#define rpcRecvPoll(_pGpu, _pRpc, _arg0, _arg1) \ + (_pRpc)->obj.__rpcRecvPoll__(_pGpu, _pRpc, _arg0, _arg1) // diff --git a/src/nvidia/inc/kernel/gpu/rpc/objrpc.h b/src/nvidia/inc/kernel/gpu/rpc/objrpc.h index cc9d31533..67db98c43 100644 --- a/src/nvidia/inc/kernel/gpu/rpc/objrpc.h +++ b/src/nvidia/inc/kernel/gpu/rpc/objrpc.h @@ -58,6 +58,7 @@ TYPEDEF_BITVECTOR(MC_ENGINE_BITVECTOR); typedef struct RpcHistoryEntry { NvU32 function; + NvU32 sequence; NvU64 data[2]; NvU64 ts_start; NvU64 ts_end; @@ -89,6 +90,9 @@ struct OBJRPC{ NvU32 rpcHistoryCurrent; RpcHistoryEntry rpcEventHistory[RPC_HISTORY_DEPTH]; NvU32 rpcEventHistoryCurrent; + + /* sequence number for RPC */ + NvU32 sequence; NvU32 timeoutCount; NvBool bQuietPrints; diff --git a/src/nvidia/interface/nvrm_registry.h b/src/nvidia/interface/nvrm_registry.h index bd560bdbd..b4886c9d6 100644 --- a/src/nvidia/interface/nvrm_registry.h +++ b/src/nvidia/interface/nvrm_registry.h @@ -1270,6 +1270,15 @@ #define NV_REG_STR_FERMI_BIG_PAGE_SIZE_64KB (64 * 1024) #define NV_REG_STR_FERMI_BIG_PAGE_SIZE_128KB (128 * 1024) +// +// Type: DWORD +// Regkey to enable/disable sysmem allocation retry. +// +#define NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT "RmEnableLargePageSizeSysmemDefault" +#define NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT_DEFAULT NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT_DISABLE +#define NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT_ENABLE (0x00000001) +#define NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT_DISABLE (0x00000000) + // // TYPE DWORD // This setting will disable big page size per address space diff --git a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c index 97037fcae..d1e49249e 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c @@ -1095,6 +1095,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) NvU64 fbPhysOffset = 0; PMEMORY_DESCRIPTOR pConsoleMemDesc = NULL; MEMORY_DESCRIPTOR memdesc; + NvU32 mapFlags = BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED | BUS_MAP_FB_FLAGS_PAGE_SIZE_4K; if (bSmoothTransitionEnabled) { @@ -1106,7 +1107,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) pConsoleMemDesc = &memdesc; memdescCreateExisting(pConsoleMemDesc, pGpu, pGpu->uefiScanoutSurfaceSizeInMB * 1024 * 1024, ADDR_FBMEM, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); memdescDescribe(pConsoleMemDesc, ADDR_FBMEM, 0, pGpu->uefiScanoutSurfaceSizeInMB * 1024 * 1024); - pConsoleMemDesc->_pageSize = RM_PAGE_SIZE; + memdescSetPageSize(pConsoleMemDesc, AT_GPU, RM_PAGE_SIZE); } else if (kbusIsPreserveBar1ConsoleEnabled(pKernelBus)) { @@ -1123,7 +1124,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) rmStatus = kbusMapFbApertureSingle(pGpu, pKernelBus, pConsoleMemDesc, fbPhysOffset, &bar1VAOffset, &consoleSize, - BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED, + mapFlags, NULL); if (rmStatus != NV_OK) { @@ -1203,7 +1204,8 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) { // // To support 512MB page sizes, we need to align the BAR1 offset to 512MB. - // This will lose up to (but not including) 512MB of BAR1 space. + // This up to (but not including) 512MB of BAR1 space will still be allocatable in + // the dynamic region // NvU64 bar1Offset = NV_ALIGN_UP(consoleSize + pKernelBus->p2pPcie.writeMailboxTotalSize, RM_PAGE_SIZE_512M); // Enable the static BAR1 mapping for the BAR1 P2P @@ -2997,6 +2999,10 @@ _kbusDestroyMemdescBar1Cb BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED |\ BUS_MAP_FB_FLAGS_PRE_INIT |\ BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_4K |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_64K |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_2M |\ + BUS_MAP_FB_FLAGS_PAGE_SIZE_512M |\ BUS_MAP_FB_FLAGS_UNMANAGED_MEM_AREA) ct_assert((BUS_FLAGS_AFFECTING_MAPPING_MASK & BUS_FLAGS_NOT_AFFECTING_MAPPING_MASK) == 0); diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c index 7aa112ed6..d2db661d2 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c @@ -29,6 +29,7 @@ #include "gpu/fifo/kernel_fifo.h" #include "gpu/mem_mgr/virt_mem_allocator.h" #include "nvrm_registry.h" +#include "kernel/virtualization/hypervisor/hypervisor.h" #include "published/turing/tu102/dev_bus.h" #include "published/turing/tu102/dev_vm.h" @@ -396,6 +397,15 @@ kbusIsStaticBar1Supported_TU102 return NV_ERR_NOT_SUPPORTED; } + if (hypervisorIsVgxHyper()) + { + // + // We don't want to consume BAR1 from the PF if + // we're not going to use it ourselves + // + return NV_ERR_NOT_SUPPORTED; + } + // // BAR1 mappings not supported in CC/PPCIE mode // TODO: Bug 5201018 @@ -525,6 +535,7 @@ kbusEnableStaticBar1Mapping_TU102 NV_STATUS status = NV_OK; NvU64 bar1MapSize; NvU64 bar1BusAddr; + NvU32 mapFlags = BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED; // // But use memmgrGetClientFbAddrSpaceSize @@ -552,17 +563,22 @@ kbusEnableStaticBar1Mapping_TU102 // Set to use RM_PAGE_SIZE_HUGE, 2MB memdescSetPageSize(pMemDesc, AT_GPU, RM_PAGE_SIZE_HUGE); + mapFlags |= BUS_MAP_FB_FLAGS_PAGE_SIZE_2M; pKernelBus->staticBar1DefaultKind = NV_MMU_PTE_KIND_GENERIC_MEMORY; // Setup GMK PTE type for this memory memdescSetPteKind(pMemDesc, pKernelBus->staticBar1DefaultKind); - // Deploy the static mapping. + // + // Deploy the static mapping. The RUSD statistics will read incorrectly + // until the subsequent call to kbusUpdateRusdStatistics at the end of + // kbusStatePostLoad_GM107 with bStaticBar1Enabled set + // NV_ASSERT_OK_OR_GOTO(status, kbusMapFbApertureSingle(pGpu, pKernelBus, pMemDesc, 0, &bar1Offset, &bar1MapSize, - BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED, + mapFlags, NV01_NULL_OBJECT), cleanup_mem); diff --git a/src/nvidia/src/kernel/gpu/bus/kern_bus.c b/src/nvidia/src/kernel/gpu/bus/kern_bus.c index 60216d903..728893836 100644 --- a/src/nvidia/src/kernel/gpu/bus/kern_bus.c +++ b/src/nvidia/src/kernel/gpu/bus/kern_bus.c @@ -941,6 +941,7 @@ NvU32 kbusConvertBusMapFlagsToDmaFlags(KernelBus *pKernelBus, MEMORY_DESCRIPTOR *pMemDesc, NvU32 busMapFlags) { NvU32 dmaFlags = DRF_DEF(OS46, _FLAGS, _DMA_UNICAST_REUSE_ALLOC, _FALSE); + NvBool bPageSizeLocked = NV_FALSE; if (busMapFlags & BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED) { @@ -974,6 +975,33 @@ kbusConvertBusMapFlagsToDmaFlags(KernelBus *pKernelBus, MEMORY_DESCRIPTOR *pMemD dmaFlags = FLD_SET_DRF(OS46, _FLAGS, _ACCESS, _WRITE_ONLY, dmaFlags); } + // Resolve explicit map page size request. Required for fixed offset mapping at specific page size. + if (busMapFlags & BUS_MAP_FB_FLAGS_PAGE_SIZE_4K) + { + NV_ASSERT(!bPageSizeLocked); + dmaFlags = FLD_SET_DRF(OS46, _FLAGS, _PAGE_SIZE, _4KB, dmaFlags); + bPageSizeLocked = NV_TRUE; + } + if (!bPageSizeLocked && (busMapFlags & BUS_MAP_FB_FLAGS_PAGE_SIZE_64K)) + { + NV_ASSERT(!bPageSizeLocked); + dmaFlags = FLD_SET_DRF(OS46, _FLAGS, _PAGE_SIZE, _BIG, dmaFlags); + bPageSizeLocked = NV_TRUE; + } + if (!bPageSizeLocked && (busMapFlags & BUS_MAP_FB_FLAGS_PAGE_SIZE_2M)) + { + NV_ASSERT(!bPageSizeLocked); + dmaFlags = FLD_SET_DRF(OS46, _FLAGS, _PAGE_SIZE, _HUGE, dmaFlags); + bPageSizeLocked = NV_TRUE; + } + if (!bPageSizeLocked && (busMapFlags & BUS_MAP_FB_FLAGS_PAGE_SIZE_512M)) + { + NV_ASSERT(!bPageSizeLocked); + dmaFlags = FLD_SET_DRF(OS46, _FLAGS, _PAGE_SIZE, _512M, dmaFlags); + bPageSizeLocked = NV_TRUE; + } + + return dmaFlags; } @@ -1203,12 +1231,6 @@ kbusUpdateRusdStatistics_IMPL NvU64 bar1AvailSize = 0; NV_RANGE bar1VARange = NV_RANGE_EMPTY; NvBool bZeroRusd = kbusIsBar1Disabled(pKernelBus); - - if (kbusIsStaticBar1Enabled(pGpu, pKernelBus)) - { - // PMA owns BAR1 info, so don't update. - return NV_OK; - } bZeroRusd = bZeroRusd || IS_MIG_ENABLED(pGpu); @@ -1227,6 +1249,29 @@ kbusUpdateRusdStatistics_IMPL pVASHeap->eheapInfoForRange(pVASHeap, bar1VARange, NULL, NULL, NULL, &freeSize); bar1AvailSize = (NvU32)(freeSize / 1024); + + if (kbusIsStaticBar1Enabled(pGpu, pKernelBus)) + { + RUSD_PMA_MEMORY_INFO *pPmaInfo; + NvU64 freeMem = 0; + NvU64 totalMem = 0; + NvU64 fbInUse; + NvU32 gfid = GPU_GFID_PF; + + NV_ASSERT_OK(vgpuGetCallingContextGfid(pGpu, &gfid)); + + // Cache off the vasFreeSize for PMA to use when it updates the memory + pKernelBus->bar1[gfid].vasFreeSize = freeSize; + + pPmaInfo = gpushareddataWriteStart(pGpu, pmaMemoryInfo); + gpushareddataWriteFinish(pGpu, pmaMemoryInfo); + totalMem = MEM_RD64(&pPmaInfo->totalPmaMemory); + freeMem = MEM_RD64(&pPmaInfo->freePmaMemory); + fbInUse = totalMem - freeMem; + + // Available size in KB + bar1AvailSize = (freeSize + pKernelBus->bar1[gfid].staticBar1.size - fbInUse) / 1024; + } } } diff --git a/src/nvidia/src/kernel/gpu/fifo/arch/maxwell/kernel_fifo_gm107.c b/src/nvidia/src/kernel/gpu/fifo/arch/maxwell/kernel_fifo_gm107.c index fe87b94de..7e5332778 100644 --- a/src/nvidia/src/kernel/gpu/fifo/arch/maxwell/kernel_fifo_gm107.c +++ b/src/nvidia/src/kernel/gpu/fifo/arch/maxwell/kernel_fifo_gm107.c @@ -1222,6 +1222,7 @@ kfifoPreAllocUserD_GM107 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pUserdInfo->userdPhysDesc[currentGpuInst], AT_GPU, RM_ATTR_PAGE_SIZE_4KB); + mapFlags |= BUS_MAP_FB_FLAGS_PAGE_SIZE_4K; if (bFifoFirstInit) { pUserdInfo->userdBar1MapStartOffset = kfifoGetUserdBar1MapStartOffset_HAL(pGpu, pKernelFifo); @@ -1243,6 +1244,7 @@ kfifoPreAllocUserD_GM107 // Force page size to 4KB in broadcast to match host phys access memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pUserdInfo->userdPhysDesc[currentGpuInst], AT_GPU, RM_ATTR_PAGE_SIZE_4KB); + mapFlags |= BUS_MAP_FB_FLAGS_PAGE_SIZE_4K; // // If coherent link is available, just get a coherent mapping to USERD and diff --git a/src/nvidia/src/kernel/gpu/gsp/kernel_gsp.c b/src/nvidia/src/kernel/gpu/gsp/kernel_gsp.c index 435cbe79f..e262e8fc3 100644 --- a/src/nvidia/src/kernel/gpu/gsp/kernel_gsp.c +++ b/src/nvidia/src/kernel/gpu/gsp/kernel_gsp.c @@ -144,9 +144,9 @@ static void _kgspFreeRpcInfrastructure(OBJGPU *, KernelGsp *); static NV_STATUS _kgspConstructRpcObject(OBJGPU *, KernelGsp *, MESSAGE_QUEUE_INFO *, OBJRPC **); -static NV_STATUS _kgspRpcSendMessage(OBJGPU *, OBJRPC *); -static NV_STATUS _kgspRpcRecvPoll(OBJGPU *, OBJRPC *, NvU32); -static NV_STATUS _kgspRpcDrainEvents(OBJGPU *, KernelGsp *, NvU32, KernelGspRpcEventHandlerContext); +static NV_STATUS _kgspRpcSendMessage(OBJGPU *, OBJRPC *, NvU32 *); +static NV_STATUS _kgspRpcRecvPoll(OBJGPU *, OBJRPC *, NvU32, NvU32); +static NV_STATUS _kgspRpcDrainEvents(OBJGPU *, KernelGsp *, NvU32, NvU32, KernelGspRpcEventHandlerContext); static void _kgspRpcIncrementTimeoutCountAndRateLimitPrints(OBJGPU *, OBJRPC *); static NV_STATUS _kgspAllocSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp); @@ -331,12 +331,14 @@ _kgspAddRpcHistoryEntry ) { NvU32 func = RPC_HDR->function; + NvU32 sequence = RPC_HDR->sequence; NvU32 entry; entry = *pCurrent = (*pCurrent + 1) % RPC_HISTORY_DEPTH; portMemSet(&pHistory[entry], 0, sizeof(pHistory[0])); pHistory[entry].function = func; + pHistory[entry].sequence = sequence; pHistory[entry].ts_start = osGetTimestamp(); _kgspGetActiveRpcDebugData(pRpc, func, @@ -386,7 +388,8 @@ static NV_STATUS _kgspRpcSendMessage ( OBJGPU *pGpu, - OBJRPC *pRpc + OBJRPC *pRpc, + NvU32 *pSequence ) { NV_STATUS nvStatus; @@ -395,6 +398,11 @@ _kgspRpcSendMessage NV_ASSERT(rmGpuGroupLockIsOwner(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, &gpuMaskUnused)); + if (pSequence) + vgpu_rpc_message_header_v->sequence = *pSequence = pRpc->sequence++; + else + vgpu_rpc_message_header_v->sequence = 0; + NV_CHECK_OK_OR_RETURN(LEVEL_SILENT, _kgspRpcSanityCheck(pGpu, pKernelGsp, pRpc)); nvStatus = GspMsgQueueSendCommand(pRpc->pMessageQueueInfo, pGpu); @@ -1605,8 +1613,8 @@ _kgspProcessRpcEvent // eventually comes in as an unexpected event. The error handling // for the timeout should have already happened. // - NV_PRINTF(LEVEL_ERROR, "Unexpected RPC event from GPU%d: 0x%x (%s)\n", - gpuGetInstance(pGpu), event, _getRpcName(event)); + NV_PRINTF(LEVEL_ERROR, "Unexpected RPC event from GPU%d: 0x%x (%s), sequence: %u\n", + gpuGetInstance(pGpu), event, _getRpcName(event), pMsgHdr->sequence); break; } @@ -1745,6 +1753,7 @@ _kgspRpcDrainOneEvent OBJGPU *pGpu, OBJRPC *pRpc, NvU32 expectedFunc, + NvU32 expectedSequence, KernelGspRpcEventHandlerContext rpcHandlerContext ) { @@ -1761,8 +1770,11 @@ _kgspRpcDrainOneEvent { rpc_message_header_v *pMsgHdr = RPC_HDR; - if (pMsgHdr->function == expectedFunc) + if (pMsgHdr->function == expectedFunc && + pMsgHdr->sequence == expectedSequence) + { return NV_WARN_MORE_PROCESSING_REQUIRED; + } _kgspProcessRpcEvent(pGpu, pRpc, rpcHandlerContext); } @@ -1795,6 +1807,7 @@ _kgspRpcDrainEvents OBJGPU *pGpu, KernelGsp *pKernelGsp, NvU32 expectedFunc, + NvU32 expectedSequence, KernelGspRpcEventHandlerContext rpcHandlerContext ) { @@ -1803,7 +1816,7 @@ _kgspRpcDrainEvents while (nvStatus == NV_OK) { - nvStatus = _kgspRpcDrainOneEvent(pGpu, pRpc, expectedFunc, rpcHandlerContext); + nvStatus = _kgspRpcDrainOneEvent(pGpu, pRpc, expectedFunc, expectedSequence, rpcHandlerContext); kgspDumpGspLogs(pKernelGsp, NV_FALSE); } @@ -1898,11 +1911,12 @@ _kgspLogRpcHistoryEntry duration = _tsDiffToDuration(duration, &durationUnitsChar); NV_ERROR_LOG_DATA(pGpu, errorNum, - " %c%-4d %-4d %-21.21s 0x%016llx 0x%016llx 0x%016llx 0x%016llx %6llu%cs %c\n", + " %c%-4d %-4d %-21.21s %10u 0x%016llx 0x%016llx 0x%016llx 0x%016llx %6llu%cs %c\n", ((historyIndex == 0) ? ' ' : '-'), historyIndex, pEntry->function, _getRpcName(pEntry->function), + pEntry->sequence, pEntry->data[0], pEntry->data[1], pEntry->ts_start, @@ -1913,11 +1927,12 @@ _kgspLogRpcHistoryEntry else { NV_ERROR_LOG_DATA(pGpu, errorNum, - " %c%-4d %-4d %-21.21s 0x%016llx 0x%016llx 0x%016llx 0x%016llx %c\n", + " %c%-4d %-4d %-21.21s %10u 0x%016llx 0x%016llx 0x%016llx 0x%016llx %c\n", ((historyIndex == 0) ? ' ' : '-'), historyIndex, pEntry->function, _getRpcName(pEntry->function), + pEntry->sequence, pEntry->data[0], pEntry->data[1], pEntry->ts_start, @@ -1980,6 +1995,7 @@ kgspLogRpcDebugInfoToProtobuf prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_HISTORYINDEX, historyIndex); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_FUNCTION, pEntry->function); prbEncAddString(pProtobufData, GSP_RPCENTRY_RPCNAME, _getRpcName(pEntry->function)); + prbEncAddUInt32(pProtobufData, GSP_RPCENTRY_SEQUENCE, pEntry->sequence); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_DATA0, pEntry->data[0]); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_DATA1, pEntry->data[1]); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_STARTTIMESTAMP, pEntry->ts_start); @@ -2007,6 +2023,7 @@ kgspLogRpcDebugInfoToProtobuf prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_HISTORYINDEX, historyIndex); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_FUNCTION, pEntry->function); prbEncAddString(pProtobufData, GSP_RPCENTRY_RPCNAME, _getRpcName(pEntry->function)); + prbEncAddUInt32(pProtobufData, GSP_RPCENTRY_SEQUENCE, pEntry->sequence); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_DATA0, pEntry->data[0]); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_DATA1, pEntry->data[1]); prbEncAddUInt64(pProtobufData, GSP_RPCENTRY_STARTTIMESTAMP, pEntry->ts_start); @@ -2041,16 +2058,16 @@ kgspLogRpcDebugInfo _kgspGetActiveRpcDebugData(pRpc, pMsgHdr->function, &activeData[0], &activeData[1]); NV_ERROR_LOG_DATA(pGpu, errorNum, - "GPU%d GSP RPC buffer contains function %d (%s) and data 0x%016llx 0x%016llx.\n", + "GPU%d GSP RPC buffer contains function %d (%s) sequence %u and data 0x%016llx 0x%016llx.\n", gpuGetInstance(pGpu), - pMsgHdr->function, _getRpcName(pMsgHdr->function), + pMsgHdr->function, _getRpcName(pMsgHdr->function), pMsgHdr->sequence, activeData[0], activeData[1]); NV_ERROR_LOG_DATA(pGpu, errorNum, "GPU%d RPC history (CPU -> GSP):\n", gpuGetInstance(pGpu)); NV_ERROR_LOG_DATA(pGpu, errorNum, - " entry function data0 data1 ts_start ts_end duration actively_polling\n"); + " entry function sequence data0 data1 ts_start ts_end duration actively_polling\n"); for (historyIndex = 0; historyIndex < rpcEntriesToLog; historyIndex++) { historyEntry = (pRpc->rpcHistoryCurrent + RPC_HISTORY_DEPTH - historyIndex) % RPC_HISTORY_DEPTH; @@ -2062,7 +2079,7 @@ kgspLogRpcDebugInfo "GPU%d RPC event history (CPU <- GSP):\n", gpuGetInstance(pGpu)); NV_ERROR_LOG_DATA(pGpu, errorNum, - " entry function data0 data1 ts_start ts_end duration during_incomplete_rpc\n"); + " entry function sequence data0 data1 ts_start ts_end duration during_incomplete_rpc\n"); for (historyIndex = 0; historyIndex < rpcEntriesToLog; historyIndex++) { historyEntry = (pRpc->rpcEventHistoryCurrent + RPC_HISTORY_DEPTH - historyIndex) % RPC_HISTORY_DEPTH; @@ -2081,7 +2098,8 @@ _kgspLogXid119 ( OBJGPU *pGpu, OBJRPC *pRpc, - NvU32 expectedFunc + NvU32 expectedFunc, + NvU32 expectedSequence ) { RpcHistoryEntry *pHistoryEntry = &pRpc->rpcHistory[pRpc->rpcHistoryCurrent]; @@ -2104,11 +2122,12 @@ _kgspLogXid119 duration = _tsDiffToDuration(ts_end - pHistoryEntry->ts_start, &durationUnitsChar); NV_ERROR_LOG(pGpu, GSP_RPC_TIMEOUT, - "Timeout after %llus of waiting for RPC response from GPU%d GSP! Expected function %d (%s) (0x%llx 0x%llx).", + "Timeout after %llus of waiting for RPC response from GPU%d GSP! Expected function %d (%s) sequence %u (0x%llx 0x%llx).", (durationUnitsChar == 'm' ? duration / 1000 : duration), gpuGetInstance(pGpu), expectedFunc, _getRpcName(expectedFunc), + expectedSequence, pHistoryEntry->data[0], pHistoryEntry->data[1]); @@ -2141,7 +2160,8 @@ _kgspLogRpcSanityCheckFailure OBJGPU *pGpu, OBJRPC *pRpc, NvU32 rpcStatus, - NvU32 expectedFunc + NvU32 expectedFunc, + NvU32 expectedSequence ) { RpcHistoryEntry *pHistoryEntry = &pRpc->rpcHistory[pRpc->rpcHistoryCurrent]; @@ -2149,11 +2169,12 @@ _kgspLogRpcSanityCheckFailure NV_ASSERT(expectedFunc == pHistoryEntry->function); NV_PRINTF(LEVEL_ERROR, - "GPU%d sanity check failed 0x%x waiting for RPC response from GSP. Expected function %d (%s) (0x%llx 0x%llx).\n", + "GPU%d sanity check failed 0x%x waiting for RPC response from GSP. Expected function %d (%s) sequence %u (0x%llx 0x%llx).\n", gpuGetInstance(pGpu), rpcStatus, expectedFunc, _getRpcName(expectedFunc), + expectedSequence, pHistoryEntry->data[0], pHistoryEntry->data[1]); @@ -2200,7 +2221,8 @@ _kgspRpcRecvPoll ( OBJGPU *pGpu, OBJRPC *pRpc, - NvU32 expectedFunc + NvU32 expectedFunc, + NvU32 expectedSequence ) { KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); @@ -2294,7 +2316,7 @@ _kgspRpcRecvPoll // timeoutStatus = gpuCheckTimeout(pGpu, &timeout); - rpcStatus = _kgspRpcDrainEvents(pGpu, pKernelGsp, expectedFunc, rpcHandlerContext); + rpcStatus = _kgspRpcDrainEvents(pGpu, pKernelGsp, expectedFunc, expectedSequence, rpcHandlerContext); switch (rpcStatus) { case NV_WARN_MORE_PROCESSING_REQUIRED: @@ -2320,7 +2342,7 @@ _kgspRpcRecvPoll { if (!pRpc->bQuietPrints) { - _kgspLogRpcSanityCheckFailure(pGpu, pRpc, rpcStatus, expectedFunc); + _kgspLogRpcSanityCheckFailure(pGpu, pRpc, rpcStatus, expectedFunc, expectedSequence); pRpc->bQuietPrints = NV_TRUE; } goto done; @@ -2334,7 +2356,7 @@ _kgspRpcRecvPoll if (!pRpc->bQuietPrints) { - _kgspLogXid119(pGpu, pRpc, expectedFunc); + _kgspLogXid119(pGpu, pRpc, expectedFunc, expectedSequence); } // Detect for 3 back to back GSP RPC timeout @@ -4946,7 +4968,7 @@ kgspRpcRecvEvents_IMPL // If we do the assert will fail on NV_WARN_MORE_PROCESSING_REQUIRED, // in addition to general error codes. // - NV_ASSERT_OK(_kgspRpcDrainEvents(pGpu, pKernelGsp, NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS, KGSP_RPC_EVENT_HANDLER_CONTEXT_INTERRUPT)); + NV_ASSERT_OK(_kgspRpcDrainEvents(pGpu, pKernelGsp, NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS, 0, KGSP_RPC_EVENT_HANDLER_CONTEXT_INTERRUPT)); } /*! @@ -4968,7 +4990,7 @@ kgspWaitForRmInitDone_IMPL threadStateResetTimeout(pGpu); NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, - rpcRecvPoll(pGpu, pRpc, NV_VGPU_MSG_EVENT_GSP_INIT_DONE)); + rpcRecvPoll(pGpu, pRpc, NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 0)); // // Now check if RPC really succeeded (NV_VGPU_MSG_RESULT_* are defined to @@ -5479,6 +5501,7 @@ static NV_STATUS _kgspDumpEngineFunc prbEncNestedStart(pPrbEnc, NVDEBUG_ENG_KGSP_RPC_HISTORY)); prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_FUNCTION, entry->function); + prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_SEQUENCE, entry->sequence); prbEncAddUInt64(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_TS_START, entry->ts_start); prbEncAddUInt64(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_TS_END, entry->ts_end); prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_DATA0, entry->data[0]); @@ -5499,6 +5522,7 @@ static NV_STATUS _kgspDumpEngineFunc prbEncNestedStart(pPrbEnc, NVDEBUG_ENG_KGSP_EVENT_HISTORY)); prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_FUNCTION, entry->function); + prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_SEQUENCE, entry->sequence); prbEncAddUInt64(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_TS_START, entry->ts_start); prbEncAddUInt64(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_TS_END, entry->ts_end); prbEncAddUInt32(pPrbEnc, NVDEBUG_ENG_KGSP_RPCINFO_DATA0, entry->data[0]); diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/arch/maxwell/virt_mem_allocator_gm107.c b/src/nvidia/src/kernel/gpu/mem_mgr/arch/maxwell/virt_mem_allocator_gm107.c index 756432b0c..9d1c96ae2 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/arch/maxwell/virt_mem_allocator_gm107.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/arch/maxwell/virt_mem_allocator_gm107.c @@ -178,6 +178,91 @@ typedef struct VASINFO_MAXWELL VA_MANAGEMENT management; // Level of management. } VASINFO_MAXWELL, *PVASINFO_MAXWELL; +static NV_STATUS _dmaGetMaxVAPageSize +( + NvU64 vaAddr, + VirtualMemory *pVirtualMemory, + NvU64 physStartAddr, + NvU64 physSize, + NvU64 *pVaMaxPageSize, + NvU64 vaspaceBigPageSize +) +{ + NvU64 vaMaxPageSize; + NvU64 targetSpaceLength; + NvU64 targetSpaceBase; + NvU64 targetSpaceLimit; + NvU64 physOffset; + NvU64 alignedVA; + NvBool bFoundPageSize = NV_FALSE; + + if (pVirtualMemory == NULL) + { + return NV_ERR_INVALID_ARGUMENT; + } + + virtmemGetAddressAndSize(pVirtualMemory, &targetSpaceBase, &targetSpaceLength); + targetSpaceLimit = targetSpaceBase + targetSpaceLength - 1; + + // + // Check VA and PA compatibility with a specific page size. + // This check is used to determine the default mapping page size. + // RM will look to make sure that: + // 1. The page size aligned down vaAddr does not underflow the VA surface. + // 2. The physical size of the allocation, when aligned up to the page size does not overflow the end of the VA surface. + // + // Passing both checks ensures that the VA surface is compatible for mapping the surface + // at the selected (and smaller) page size(s). + // + if (!bFoundPageSize) + { + alignedVA = NV_ALIGN_DOWN64(vaAddr, RM_PAGE_SIZE_2M); + physOffset = physStartAddr & GET_PAGE_MASK(RM_PAGE_SIZE_2M); + + if ((alignedVA >= targetSpaceBase) && + (NV_ALIGN_UP64(alignedVA + physOffset + physSize, RM_PAGE_SIZE_2M) - 1) <= targetSpaceLimit) + { + vaMaxPageSize = RM_PAGE_SIZE_2M; + bFoundPageSize = NV_TRUE; + } + } + + if (!bFoundPageSize) + { + alignedVA = NV_ALIGN_DOWN64(vaAddr, vaspaceBigPageSize); + physOffset = physStartAddr & GET_PAGE_MASK(vaspaceBigPageSize); + + if ((alignedVA >= targetSpaceBase) && + (NV_ALIGN_UP64(alignedVA + physOffset + physSize, vaspaceBigPageSize) - 1) <= targetSpaceLimit) + { + vaMaxPageSize = vaspaceBigPageSize; + bFoundPageSize = NV_TRUE; + } + } + + if (!bFoundPageSize) + { + alignedVA = NV_ALIGN_DOWN64(vaAddr, RM_PAGE_SIZE); + physOffset = physStartAddr & GET_PAGE_MASK(RM_PAGE_SIZE); + + if ((alignedVA >= targetSpaceBase) && + (NV_ALIGN_UP64(alignedVA + physOffset + physSize, RM_PAGE_SIZE) - 1) <= targetSpaceLimit) + { + vaMaxPageSize = RM_PAGE_SIZE; + bFoundPageSize = NV_TRUE; + } + } + + if (!bFoundPageSize) + { + return NV_ERR_INVALID_STATE; + } + + *pVaMaxPageSize = vaMaxPageSize; + + return NV_OK; +} + NvBool dmaIsDefaultGpuUncached_GM107 ( @@ -446,7 +531,86 @@ dmaAllocMapping_GM107 { case NVOS46_FLAGS_PAGE_SIZE_DEFAULT: case NVOS46_FLAGS_PAGE_SIZE_BOTH: - pLocals->pageSize = memdescGetPageSize(pLocals->pTempMemDesc, addressTranslation); + if (pMemoryManager->bSysmemPageSizeDefaultAllowLargePages) + { + // + // On Fixed offset allocation, make sure that the least amount of VA is allocated + // while using the largest possible page size that completely covers the user requested physical surface. + // + if (FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, flags) || + (pCliMapInfo == NULL && !pLocals->bAllocVASpace)) + { + NvU64 vaddrAligned = RM_ALIGN_DOWN(*pVaddr, RM_PAGE_SIZE); + // physAddr[0] - PteAdjust should always be at least 4K aligned. + NvU64 physStartAddr = memdescGetPhysAddr(pLocals->pTempMemDesc, addressTranslation, 0) - pLocals->pTempMemDesc->PteAdjust; + // Aligning up the size to 4K as that is the smallest page size that RM can map. + NvU64 physEndAddr = RM_ALIGN_UP(memdescGetPhysAddr(pLocals->pTempMemDesc, addressTranslation, 0) + + memdescGetSize(pLocals->pTempMemDesc), RM_PAGE_SIZE); + + NV_PRINTF(LEVEL_SILENT, "Choosing default map pagesize through fixed offset path.\n"); + + if (NV_IS_ALIGNED64(physStartAddr, RM_PAGE_SIZE_2M) && + NV_IS_ALIGNED64(physEndAddr, RM_PAGE_SIZE_2M) && + NV_IS_ALIGNED64(vaddrAligned, RM_PAGE_SIZE_2M)) + { + pLocals->pageSize = RM_PAGE_SIZE_2M; + } + else if (NV_IS_ALIGNED64(physStartAddr, pLocals->vaspaceBigPageSize) && + NV_IS_ALIGNED64(physEndAddr, pLocals->vaspaceBigPageSize) && + NV_IS_ALIGNED64(vaddrAligned, pLocals->vaspaceBigPageSize)) + { + pLocals->pageSize = pLocals->vaspaceBigPageSize; + } + else + { + pLocals->pageSize = RM_PAGE_SIZE; + } + } + else + { + // + // When a client provides a CliMapInfo RM is provided with the exact VA range that + // the mapping is performed in. + // + // RM will parse the VA and PA range to find a compatible page size. + // + if (pCliMapInfo != NULL && !pLocals->bAllocVASpace) + { + NvU64 vaMaxPageSize = 0; + + NV_PRINTF(LEVEL_SILENT, "Choosing default map pagesize based on client provided VA range.\n"); + + NV_ASSERT_OK_OR_GOTO(status, _dmaGetMaxVAPageSize(*pVaddr, pCliMapInfo->pVirtualMemory, + memdescGetPhysAddr(pLocals->pTempMemDesc, addressTranslation, 0), + memdescGetSize(pLocals->pTempMemDesc), + &vaMaxPageSize, + // TODO: Remove once Maxwell is deprecated. + pLocals->vaspaceBigPageSize), + cleanup); + + // + // The output of _dmaGetMaxVAPageSize guarantees tha the VA surface + // is compatible with all page sizes <= vaMaxPageSize. + // + pLocals->pageSize = NV_MIN(vaMaxPageSize, pLocals->physPageSize); + } + else + { + // + // If the mapping call creates the VA surface, it has full control over the allocated VA range. + // Optimize VA surface to match the physical page size. + // + NV_PRINTF(LEVEL_SILENT, "Choosing default map pagesize based on the map call allocating VA for the mapping.\n"); + pLocals->pageSize = memdescGetPageSize(pLocals->pTempMemDesc, addressTranslation); + } + } + } + else + { + pLocals->pageSize = pLocals->physPageSize; + } + + NV_ASSERT_OR_GOTO(pLocals->pageSize != 0, cleanup); break; case NVOS46_FLAGS_PAGE_SIZE_4KB: pLocals->pageSize = RM_PAGE_SIZE; @@ -541,7 +705,8 @@ dmaAllocMapping_GM107 // // Skipping it for Raw mode, See bug 4036809 // - if ((pLocals->pageSize == RM_PAGE_SIZE) && + if (!(pMemoryManager->bSysmemPageSizeDefaultAllowLargePages) && + (pLocals->pageSize == RM_PAGE_SIZE) && memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pLocals->kind) && !(pMemorySystemConfig->bUseRawModeComptaglineAllocation)) { diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c b/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c index 209b093ec..e940890d8 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c @@ -310,8 +310,6 @@ memdescCreate // (4k >> 12 = 1). This modification helps us to avoid overflow of variable // allocSize, in case caller of this function passes highest value of NvU64. // - // If allocSize is passed as 0, PageCount should be returned as 0. - // if (allocSize == 0) { PageCount = 0; diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/mem_mgr.c b/src/nvidia/src/kernel/gpu/mem_mgr/mem_mgr.c index 8016a2f1a..6b8aeb417 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/mem_mgr.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/mem_mgr.c @@ -150,6 +150,21 @@ _memmgrInitRegistryOverridesAtConstruct { NvU32 NV_ATTRIBUTE_UNUSED data32; + if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT, &data32) == NV_OK) + { + if (data32 == NV_REG_STR_RM_ENABLE_LARGE_PAGE_SYSMEM_DEFAULT_ENABLE) + { + pMemoryManager->bSysmemPageSizeDefaultAllowLargePages = NV_TRUE; + } + else + { + pMemoryManager->bSysmemPageSizeDefaultAllowLargePages = NV_FALSE; + } + + NV_PRINTF(LEVEL_NOTICE, "Large page sysmem default override to 0x%x via regkey.\n", + pMemoryManager->bSysmemPageSizeDefaultAllowLargePages); + } + if (osReadRegistryDword( pGpu, NV_REG_STR_RM_FORCE_ENABLE_FLA_SYSMEM, &data32) == NV_OK) { @@ -3203,34 +3218,31 @@ _memmgrPmaStatsUpdateCb // static BAR1 update { - OBJVASPACE *pBar1VAS; - OBJEHEAP *pVASHeap; NvU64 freeSize = 0; NvU64 bar1AvailSize = 0; - NV_RANGE bar1VARange = NV_RANGE_EMPTY; NvBool bZeroRusd = kbusIsBar1Disabled(pKernelBus); NvU32 gfid; - NvU32 fbInUse; + NvU64 fbInUse; NV_STATUS status; NV_ASSERT_OK_OR_ELSE(status, vgpuGetCallingContextGfid(pGpu, &gfid), return); bZeroRusd = bZeroRusd || IS_MIG_ENABLED(pGpu); - if (!bZeroRusd) + // bar1Size is not owned here, no need to update anything if zero RUSD + if (bZeroRusd) { - pBar1VAS = kbusGetBar1VASpace_HAL(pGpu, pKernelBus); - NV_ASSERT_OR_ELSE(pBar1VAS != NULL, return); - pVASHeap = vaspaceGetHeap(pBar1VAS); - bar1VARange = rangeMake(vaspaceGetVaStart(pBar1VAS), vaspaceGetVaLimit(pBar1VAS)); - - // no need to update bar1size - if (pVASHeap != NULL) - { - pVASHeap->eheapInfoForRange(pVASHeap, bar1VARange, NULL, NULL, NULL, &freeSize); - } + return; } + // no need to update bar1size + // + // We access this structure locklessly and can't iterate the + // VASpace's structures which may change underneath us + // Use the cached value of the free size which always has a valid value + // + freeSize = pKernelBus->bar1[gfid].vasFreeSize; + totalMem = MEM_RD64(&pSharedData->totalPmaMemory); fbInUse = totalMem - freeMem; diff --git a/src/nvidia/src/kernel/gpu/mem_sys/kern_mem_sys_ctrl.c b/src/nvidia/src/kernel/gpu/mem_sys/kern_mem_sys_ctrl.c index 522b449d1..082d0af8e 100644 --- a/src/nvidia/src/kernel/gpu/mem_sys/kern_mem_sys_ctrl.c +++ b/src/nvidia/src/kernel/gpu/mem_sys/kern_mem_sys_ctrl.c @@ -900,6 +900,33 @@ _kmemsysGetFbInfos break; } + case NV2080_CTRL_FB_INFO_INDEX_NUMA_NODE_ID: + { + NvU32 nodeId = NV0000_CTRL_NO_NUMA_NODE; + + if (bIsMIGInUse) + { + MIG_INSTANCE_REF ref; + + if (kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref) == NV_OK) + { + NvU32 swizzId = ref.pKernelMIGGpuInstance->swizzId; + + if (pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse) + { + nodeId = pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId; + } + } + } + else + { + nodeId = pGpu->numaNodeId; + } + + data = nodeId; + break; + } + default: { if (status != NV_OK) diff --git a/src/nvidia/src/kernel/mem_mgr/standard_mem.c b/src/nvidia/src/kernel/mem_mgr/standard_mem.c index 5a68ead94..00311fabf 100644 --- a/src/nvidia/src/kernel/mem_mgr/standard_mem.c +++ b/src/nvidia/src/kernel/mem_mgr/standard_mem.c @@ -56,7 +56,7 @@ NV_STATUS stdmemValidateParams return NV_ERR_INVALID_ARGUMENT; } - // + // // These flags don't do anything in this path. No mapping on alloc and // kernel map is controlled by TYPE // diff --git a/src/nvidia/src/kernel/mem_mgr/system_mem.c b/src/nvidia/src/kernel/mem_mgr/system_mem.c index bb732e16b..bb39fdd77 100644 --- a/src/nvidia/src/kernel/mem_mgr/system_mem.c +++ b/src/nvidia/src/kernel/mem_mgr/system_mem.c @@ -346,6 +346,7 @@ sysmemConstruct_IMPL // Freeing memdescs to avoid leaks on retry. memdescFree(pMemDesc); memdescDestroy(pMemDesc); + pAllocRequest->pMemDesc = pMemDesc = NULL; } else { diff --git a/src/nvidia/src/kernel/rmapi/rpc_common.c b/src/nvidia/src/kernel/rmapi/rpc_common.c index 7b446638c..9bc62d736 100644 --- a/src/nvidia/src/kernel/rmapi/rpc_common.c +++ b/src/nvidia/src/kernel/rmapi/rpc_common.c @@ -95,6 +95,7 @@ OBJRPC *initRpcObject(OBJGPU *pGpu) pRpc->timeoutCount = 0; pRpc->bQuietPrints = NV_FALSE; + pRpc->sequence = 0; if (!IS_DCE_CLIENT(pGpu)) { // VIRTUALIZATION is disabled on DCE. Only run the below code on VGPU and GSP. diff --git a/src/nvidia/src/kernel/vgpu/rpc.c b/src/nvidia/src/kernel/vgpu/rpc.c index 82670b39b..b7dcb07e4 100644 --- a/src/nvidia/src/kernel/vgpu/rpc.c +++ b/src/nvidia/src/kernel/vgpu/rpc.c @@ -121,8 +121,8 @@ static NvU64 startTimeInNs, endTimeInNs, elapsedTimeInNs; static NV_STATUS updateHostVgpuFbUsage(OBJGPU *pGpu, NvHandle hClient, NvHandle hDevice, NvHandle hSubdevice); -static NV_STATUS _rpcSendMessage_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC); -static NV_STATUS _rpcRecvPoll_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC, NvU32 expectedFunc); +static NV_STATUS _rpcSendMessage_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC, NvU32 *pSequence); +static NV_STATUS _rpcRecvPoll_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC, NvU32 expectedFunc, NvU32 expectedSequence); void setGuestEccStatus(OBJGPU *pGpu); typedef NV_STATUS dma_control_copy_params_to_rpc_buffer_v(NvU32 cmd, void *Params, void *params_in); @@ -1792,28 +1792,29 @@ NV_STATUS freeRpcInfrastructure_VGPU(OBJGPU *pGpu) return rmStatus; } -NV_STATUS rpcSendMessage_IMPL(OBJGPU *pGpu, OBJRPC *pRpc) +NV_STATUS rpcSendMessage_IMPL(OBJGPU *pGpu, OBJRPC *pRpc, NvU32 *pSequence) { NV_PRINTF(LEVEL_ERROR, "virtual function not implemented.\n"); return NV_ERR_NOT_SUPPORTED; } -NV_STATUS rpcRecvPoll_IMPL(OBJGPU *pGpu, OBJRPC *pRpc, NvU32 expectedFunc) +NV_STATUS rpcRecvPoll_IMPL(OBJGPU *pGpu, OBJRPC *pRpc, NvU32 expectedFunc, NvU32 expectedSequence) { NV_PRINTF(LEVEL_ERROR, "virtual function not implemented.\n"); return NV_ERR_NOT_SUPPORTED; } -static NV_STATUS _rpcSendMessage_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRpc) +static NV_STATUS _rpcSendMessage_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRpc, NvU32 *pSequence) { OBJVGPU *pVGpu = GPU_GET_VGPU(pGpu); - vgpu_rpc_message_header_v->sequence = pVGpu->sequence_base++; + NV_ASSERT(pSequence != NULL); + vgpu_rpc_message_header_v->sequence = *pSequence = pVGpu->sequence_base++; return _vgpuGspSendRpcRequest(pGpu, pRpc); } -static NV_STATUS _rpcRecvPoll_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC, NvU32 expectedFunc) +static NV_STATUS _rpcRecvPoll_VGPUGSP(OBJGPU *pGpu, OBJRPC *pRPC, NvU32 expectedFunc, NvU32 expectedSequence) { return _vgpuGspWaitForResponse(pGpu); } @@ -1851,6 +1852,15 @@ static NV_STATUS _issueRpcAndWait(OBJGPU *pGpu, OBJRPC *pRpc) pNewEntry->rpcData.rpcDataTag = vgpu_rpc_message_header_v->function; + switch (vgpu_rpc_message_header_v->function) + { + case NV_VGPU_MSG_FUNCTION_RM_API_CONTROL: + pNewEntry->rpcData.rpcExtraData = rpc_message->rm_api_control_v.params.cmd; + break; + default: + break; + } + rpcProfilerEntryCount++; osGetPerformanceCounter(&pNewEntry->rpcData.startTimeInNs); @@ -1858,13 +1868,14 @@ static NV_STATUS _issueRpcAndWait(OBJGPU *pGpu, OBJRPC *pRpc) // For HCC, cache expectedFunc value before encrypting. NvU32 expectedFunc = vgpu_rpc_message_header_v->function; + NvU32 expectedSequence = 0; - status = rpcSendMessage(pGpu, pRpc); + status = rpcSendMessage(pGpu, pRpc, &expectedSequence); if (status != NV_OK) { NV_PRINTF_COND(pRpc->bQuietPrints, LEVEL_INFO, LEVEL_ERROR, - "rpcSendMessage failed with status 0x%08x for fn %d!\n", - status, vgpu_rpc_message_header_v->function); + "rpcSendMessage failed with status 0x%08x for fn %d sequence %d!\n", + status, expectedFunc, expectedSequence); // // It has been observed that returning NV_ERR_BUSY_RETRY in a bad state (RPC // buffers full and not being serviced) can make things worse, i.e. turn RPC @@ -1959,20 +1970,20 @@ static NV_STATUS _issueRpcAndWait(OBJGPU *pGpu, OBJRPC *pRpc) } // Use cached expectedFunc here because vgpu_rpc_message_header_v is encrypted for HCC. - status = rpcRecvPoll(pGpu, pRpc, expectedFunc); + status = rpcRecvPoll(pGpu, pRpc, expectedFunc, expectedSequence); if (status != NV_OK) { if (status == NV_ERR_TIMEOUT) { NV_PRINTF_COND(pRpc->bQuietPrints, LEVEL_INFO, LEVEL_ERROR, - "rpcRecvPoll timedout for fn %d!\n", - vgpu_rpc_message_header_v->function); + "rpcRecvPoll timedout for fn %d sequence %u!\n", + expectedFunc, expectedSequence); } else { NV_PRINTF_COND(pRpc->bQuietPrints, LEVEL_INFO, LEVEL_ERROR, - "rpcRecvPoll failed with status 0x%08x for fn %d!\n", - status, vgpu_rpc_message_header_v->function); + "rpcRecvPoll failed with status 0x%08x for fn %d sequence %u!\n", + status, expectedFunc, expectedSequence); } return status; } @@ -2007,10 +2018,10 @@ static NV_STATUS _issueRpcAsync(OBJGPU *pGpu, OBJRPC *pRpc) // should not be called in broadcast mode NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); - status = rpcSendMessage(pGpu, pRpc); + status = rpcSendMessage(pGpu, pRpc, NULL); if (status != NV_OK) { - NV_PRINTF(LEVEL_ERROR, "rpcSendMessage failed with status 0x%08x for fn %d!\n", + NV_PRINTF(LEVEL_ERROR, "rpcSendMessage async failed with status 0x%08x for fn %d!\n", status, vgpu_rpc_message_header_v->function); NV_ASSERT(0); // @@ -2038,6 +2049,8 @@ static NV_STATUS _issueRpcLarge NvU8 *pBuf8 = (NvU8 *)pBuffer; NV_STATUS nvStatus = NV_OK; NvU32 expectedFunc = vgpu_rpc_message_header_v->function; + NvU32 firstSequence = pRpc->sequence; + NvU32 lastSequence, waitSequence; NvU32 entryLength; NvU32 remainingSize = bufSize; NvU32 recordCount = 0; @@ -2054,7 +2067,7 @@ static NV_STATUS _issueRpcLarge // Set the correct length for this queue entry. vgpu_rpc_message_header_v->length = entryLength; - nvStatus = rpcSendMessage(pGpu, pRpc); + nvStatus = rpcSendMessage(pGpu, pRpc, &firstSequence); if (nvStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "rpcSendMessage failed with status 0x%08x for fn %d!\n", @@ -2090,7 +2103,7 @@ static NV_STATUS _issueRpcLarge vgpu_rpc_message_header_v->length = entryLength + sizeof(rpc_message_header_v); vgpu_rpc_message_header_v->function = NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD; - nvStatus = rpcSendMessage(pGpu, pRpc); + nvStatus = rpcSendMessage(pGpu, pRpc, &lastSequence); if (nvStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, @@ -2111,6 +2124,8 @@ static NV_STATUS _issueRpcLarge recordCount++; } + NV_ASSERT(lastSequence == (firstSequence + recordCount)); + if (!bWait) { // In case of Async RPC, we are done here. @@ -2118,18 +2133,20 @@ static NV_STATUS _issueRpcLarge } // Always receive at least one.. - nvStatus = rpcRecvPoll(pGpu, pRpc, expectedFunc); + waitSequence = firstSequence; + + nvStatus = rpcRecvPoll(pGpu, pRpc, expectedFunc, waitSequence); if (nvStatus != NV_OK) { if (nvStatus == NV_ERR_TIMEOUT) { - NV_PRINTF(LEVEL_ERROR, "rpcRecvPoll timedout for fn %d!\n", - vgpu_rpc_message_header_v->function); + NV_PRINTF(LEVEL_ERROR, "rpcRecvPoll timedout for fn %d sequence %d!\n", + expectedFunc, waitSequence); } else { - NV_PRINTF(LEVEL_ERROR, "rpcRecvPoll failed with status 0x%08x for fn %d!\n", - nvStatus, vgpu_rpc_message_header_v->function); + NV_PRINTF(LEVEL_ERROR, "rpcRecvPoll failed with status 0x%08x for fn %d sequence %d!\n", + nvStatus, expectedFunc, waitSequence); } NV_ASSERT(0); return nvStatus; @@ -2145,26 +2162,27 @@ static NV_STATUS _issueRpcLarge remainingSize -= entryLength; pBuf8 += entryLength; + waitSequence++; // For bidirectional transfer messages, need to receive all other frames as well if (bBidirectional && (recordCount > 0)) { while (remainingSize > 0) { - nvStatus = rpcRecvPoll(pGpu, pRpc, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD); + nvStatus = rpcRecvPoll(pGpu, pRpc, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, waitSequence); if (nvStatus != NV_OK) { if (nvStatus == NV_ERR_TIMEOUT) { NV_PRINTF(LEVEL_ERROR, - "rpcRecvPoll timedout for fn %d continuation record (remainingSize=0x%x)!\n", - vgpu_rpc_message_header_v->function, remainingSize); + "rpcRecvPoll timedout for fn %d sequence %d continuation record (remainingSize=0x%x)!\n", + expectedFunc, waitSequence, remainingSize); } else { NV_PRINTF(LEVEL_ERROR, - "rpcRecvPoll failed with status 0x%08x for fn %d continuation record! (remainingSize=0x%x)\n", - nvStatus, vgpu_rpc_message_header_v->function, remainingSize); + "rpcRecvPoll failed with status 0x%08x for fn %d sequence %d continuation record! (remainingSize=0x%x)\n", + nvStatus, expectedFunc, waitSequence, remainingSize); } NV_ASSERT(0); return nvStatus; @@ -2182,9 +2200,11 @@ static NV_STATUS _issueRpcLarge remainingSize -= entryLength; pBuf8 += entryLength; recordCount--; + waitSequence++; } vgpu_rpc_message_header_v->function = expectedFunc; NV_ASSERT(recordCount == 0); + NV_ASSERT(waitSequence - 1 == lastSequence); } // Now check if RPC really succeeded diff --git a/version.mk b/version.mk index 76c9bf3c6..134a8cb91 100644 --- a/version.mk +++ b/version.mk @@ -1,5 +1,5 @@ -NVIDIA_VERSION = 580.76.05 -NVIDIA_NVID_VERSION = 580.76.05 +NVIDIA_VERSION = 580.82.07 +NVIDIA_NVID_VERSION = 580.82.07 NVIDIA_NVID_EXTRA = # This file.