515.43.04

This commit is contained in:
Andy Ritger
2022-05-09 13:18:59 -07:00
commit 1739a20efc
2519 changed files with 1060036 additions and 0 deletions

View File

@@ -0,0 +1,3 @@
#include "g_console_mem_nvoc.h"

View File

@@ -0,0 +1,83 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2016-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _CTX_BUF_POOL_H_
#define _CTX_BUF_POOL_H_
/*!
* @file ctx_buf_pool.h
*
* @brief Describes interfaces used for creating RM memory pools for
* RM internal allocations like global (engine-specific) and
* local (context-specific) context buffers.
*/
/* ---------------------------------Includes ------------------------------------ */
#include "core/core.h"
#include "mem_mgr/vaspace.h"
#include "mem_mgr/pool_alloc.h"
#include "gpu/mem_mgr/virt_mem_allocator_common.h"
// state of context buffer pools
struct CTX_BUF_POOL_INFO
{
//
// Each array index corresponds to a pointer to memory pool with
// page size corresponding to RM_ATTR_PAGE_SIZE_*
// Pool corresponding to RM_ATTR_PAGE_SIZE_DEFAULT will be left unused
//
RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemPool[RM_ATTR_PAGE_SIZE_INVALID];
};
// List of all context buffers supported by memory pools
typedef enum CTX_BUF_ID
{
CTX_BUF_ID_INST_MEM,
CTX_BUF_ID_RUNLIST,
CTX_BUF_ID_GR_GLOBAL,
CTX_BUF_ID_GR_LOCAL
}CTX_BUF_ID;
// context buffer info
struct CTX_BUF_INFO
{
NvU64 size;
NvU64 align;
RM_ATTR_PAGE_SIZE attr;
NvBool bContig;
};
typedef struct CTX_BUF_INFO CTX_BUF_INFO;
NV_STATUS ctxBufPoolInit(OBJGPU *pGpu, Heap *pHeap, CTX_BUF_POOL_INFO **ppCtxBufPool);
NV_STATUS ctxBufPoolReserve(OBJGPU *pGpu, CTX_BUF_POOL_INFO *pCtxBufPool, CTX_BUF_INFO *pBufInfoList, NvU32 bufCount);
NV_STATUS ctxBufPoolTrim(CTX_BUF_POOL_INFO *pCtxBufPool);
NV_STATUS ctxBufPoolAllocate(CTX_BUF_POOL_INFO *pCtxBufPool, PMEMORY_DESCRIPTOR pMemDesc);
NV_STATUS ctxBufPoolFree(CTX_BUF_POOL_INFO *pCtxBufPool, PMEMORY_DESCRIPTOR pMemDesc);
void ctxBufPoolRelease(CTX_BUF_POOL_INFO *pCtxBufPool);
void ctxBufPoolDestroy(CTX_BUF_POOL_INFO **ppCtxBufPool);
NvBool ctxBufPoolIsSupported(OBJGPU *pGpu);
NV_STATUS ctxBufPoolGetSizeAndPageSize(CTX_BUF_POOL_INFO *pCtxBufPool, OBJGPU *pGpu, NvU64 alignment, RM_ATTR_PAGE_SIZE attr, NvBool bContig, NvU64 *pSize, NvU32 *pPageSize);
NV_STATUS ctxBufPoolGetGlobalPool(OBJGPU *pGpu, CTX_BUF_ID bufId, NvU32 engineType, CTX_BUF_POOL_INFO **ppCtxBufPool);
NvBool ctxBufPoolIsScrubSkipped(CTX_BUF_POOL_INFO *pCtxBufPool);
void ctxBufPoolSetScrubSkip(CTX_BUF_POOL_INFO *pCtxBufPool, NvBool bSkipScrub);
#endif // _CTX_BUF_POOL_H_

View File

@@ -0,0 +1,3 @@
#include "g_fabric_vaspace_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_fla_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_gpu_vaspace_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_hw_resources_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_io_vaspace_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_mem_fabric_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_no_device_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_os_desc_mem_nvoc.h"

View File

@@ -0,0 +1,37 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _P2P_H_
#define _P2P_H_
#include "core/core.h"
NV_STATUS RmP2PGetPages (NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, OBJGPU **, void *, void (*)(void *), void *);
NV_STATUS RmP2PGetPagesWithoutCallbackRegistration (NvU64, NvU32, NvU64, NvU64, NvU64 *, NvU32 *, NvU32 *, NvU32 *, OBJGPU **, void *);
NV_STATUS RmP2PGetPagesPersistent (NvU64, NvU64, void **, NvU64 *, NvU32 *, void *, void *);
NV_STATUS RmP2PRegisterCallback (NvU64, NvU64, NvU64, void *, void (*)(void *), void *);
NV_STATUS RmP2PPutPages (NvU64, NvU32, NvU64, void *);
NV_STATUS RmP2PGetGpuByAddress (NvU64, NvU64, OBJGPU **);
NV_STATUS RmP2PPutPagesPersistent (void *, void *);
#endif

View File

@@ -0,0 +1,3 @@
#include "g_phys_mem_nvoc.h"

View File

@@ -0,0 +1,201 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2016-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _RM_POOL_ALLOC_H_
#define _RM_POOL_ALLOC_H_
/*!
* @file pool_alloc.h
*
* @brief Describes utilities for creating pools for RM internal usage,
* allocating memory from the pools, freeing memory to the pools
* and destroying the pools.
* Uses:
* 1. RM allocations for client page tables
* 2. Context-specific RM allocations like context buffers, instance memory, fault buffers, GR local buffers
* 3. Engine-specific RM allocations like runlist buffers and GR global buffers
*/
/* ---------------------------------Includes ------------------------------------ */
#include "core/core.h"
/* ------------------------------- Public Interface ----------------------------- */
/*!
* Static configurations for various pools.
*/
typedef enum
{
POOL_CONFIG_GMMU_FMT_1 = 0, // configure pool for client page tables with version = GMMU_FMT_VERSION_1
POOL_CONFIG_GMMU_FMT_2 = 1, // configure pool for client page tables with version = GMMU_FMT_VERSION_2
POOL_CONFIG_CTXBUF_512M = 2, // configure pool for RM internal allocations like ctx buffers with 512MB page size
POOL_CONFIG_CTXBUF_2M = 3, // configure pool for RM internal allocations like ctx buffers with 2MB page size
POOL_CONFIG_CTXBUF_64K = 4, // configure pool for RM internal allocations like ctx buffers with 64KB page size
POOL_CONFIG_CTXBUF_4K = 5, // configure pool for RM internal allocations like ctx buffers with 4KB page size
POOL_CONFIG_MAX_SUPPORTED = 6
}POOL_CONFIG_MODE;
/* ------------------------------------ Datatypes ---------------------------------- */
/*!
* Opaque library-defined state of a pool reserve.
*/
typedef struct RM_POOL_ALLOC_MEM_RESERVE_INFO RM_POOL_ALLOC_MEM_RESERVE_INFO;
/*!
* Opaque user-defined state describing a block of physical memory.
* We reference these as the backing memory for the allocation.
*/
typedef struct RM_POOL_ALLOC_MEMDESC RM_POOL_ALLOC_MEMDESC;
/* ------------------------------- Public Interface ----------------------------- */
/*!
* @brief Sets up the memory pool and the tracking structure.
*
* @param[in] pCtx Pointer to some user context data
* @param[in] ppMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] configMode Mode to configure the pool
*
* @return
* NV_OK:
* Internal memory allocation failed.
*/
NV_STATUS rmMemPoolSetup(void *pCtx, RM_POOL_ALLOC_MEM_RESERVE_INFO **ppMemReserve, POOL_CONFIG_MODE configMode);
/*!
* @brief Reserve memory for the allocation in vidmem. Physical frames are
* added to the memory pool on need basis only when a mapping is
* made in the VAS.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] poolSize pool size
* @param[in] flags VASpace flags to skip scrubbing in PMA for internal clients
*
* @return
* NV_ERR_NO_MEMORY:
* Internal memory allocation failed
* NV_ERR_INVALID_ARGUMENT:
* Invalid argument
*/
NV_STATUS rmMemPoolReserve(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserve,
NvU64 poolSize, NvU32 flags);
/*!
* @brief Releases the memory pool memory to the PMA once all allocations are
* returned back to it.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] flags VASpace flags to skip scrubbing in PMA for internal clients
*
* @return
*/
void rmMemPoolRelease(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, NvU32 flags);
/*!
* @brief Returns any unused nodes from the topmost level of a pool hierarchy
* back to PMA.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] nodesToPreserve Number of nodes to preserve in the topmost pool
* @param[in] flags VASpace flags to skip scrubbing in PMA for internal clients
* @return
*/
void rmMemPoolTrim (RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo,
NvU32 nodesToPreserve, NvU32 flags);
/*!
* @brief Suballocate memory for an allocation from the pool created
* by @see rmMemPoolReserve.
*
* @param[in] pMemReserveInfo Pointer to RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] pMemDesc Pointer to the allocations mem descriptor
*
* @return
* NV_ERR_NO_MEMORY:
* Internal memory allocation failed.
* NV_ERR_GENERIC:
* Unexpected error.
*/
NV_STATUS rmMemPoolAllocate(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo,
RM_POOL_ALLOC_MEMDESC *pPoolMemDesc);
/*!
* @brief Returns the allocation's memory back to the pool from
* which it was borrowed.
*
* @param[in] pMemReserveInfo Pointer to RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] pMemDesc Pointer to the allocations mem descriptor
* @param[in] flags VASpace flags to skip scrubbing in PMA for internal clients
*
* @return
*/
void rmMemPoolFree(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo,
RM_POOL_ALLOC_MEMDESC *pPoolMemDesc, NvU32 flags);
/*!
* @brief Destroys the memory pool once all allocations are returned
* back to it.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
*
* @return
*/
void rmMemPoolDestroy(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo);
/*!
* @brief Setup pool to skip scrubber.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[in] bSkipScrub skip scrubber
*
* @return
*/
void rmMemPoolSkipScrub(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, NvBool bSkipScrub);
/*!
* @brief Get pool setting for skipping scrubber.
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
*
* @return
* NV_TRUE Scrubbing is skipped
* NV_FALSE Scrubbing not skipped
*/
NvBool rmMemPoolIsScrubSkipped(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo);
/*!
* @brief Get page size and chunk size for a pool
*
* @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data
* @param[out] chunkSize
* @param[out] pageSize
*
* @return
* NV_ERR_INVALID_ARGUMENT
* NV_OK
*/
NV_STATUS rmMemPoolGetChunkAndPageSize(RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, NvU64*, NvU32*);
#endif //_RM_POOL_ALLOC_H_

View File

@@ -0,0 +1,3 @@
#include "g_reg_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_standard_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_system_mem_nvoc.h"

View File

@@ -0,0 +1,88 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2016-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/*!
* Structure to track virtual memory mapping
*/
#ifndef VADDR_LIST_H
#define VADDR_LIST_H
#include "containers/map.h"
#include "mem_mgr/vaspace.h"
/*!
* Map info
*/
typedef struct
{
/*!
* Indicate whether caller should release the VA.
* Default is set to TRUE. i.e caller should free the VA.
* Use vaListSetManaged() to change the value.
*/
NvBool bRelease;
}VADDR_LIST_INFO;
/*!
* Virtual memory info
*/
typedef struct
{
/*! virtual address */
NvU64 vAddr;
/*! No: of active references to the virtual address */
NvU64 refCnt;
/*!
* To track common properties.
* Key 0 (pVAS == NULL) is used to store this info
*/
VADDR_LIST_INFO *pVaListInfo;
}VA_INFO;
/*!
* Dictionary that tracks active virtual memory mappings.
* Indexed by the vaspace object pointer.
*/
MAKE_MAP(VA_LIST, VA_INFO);
/*! Init the tracker object */
NV_STATUS vaListInit(VA_LIST *);
/*! Init the tracker object */
void vaListDestroy(VA_LIST *);
/*!
* Set VA lifecycle property.
* TRUE means caller should free the VA. e.g RM managed mappings
* FALSE means caller shouldn't free the VA. e.g UVM or KMD managed mappings.
*/
NV_STATUS vaListSetManaged(VA_LIST *, NvBool bManaged);
NvBool vaListGetManaged(VA_LIST *);
/*! Add a vas mapping to the tracker */
NV_STATUS vaListAddVa(VA_LIST *, OBJVASPACE *, NvU64 vaddr);
/*! Remove a vas mapping to the tracker */
NV_STATUS vaListRemoveVa(VA_LIST *, OBJVASPACE *);
/*! Get the vas mapping */
NV_STATUS vaListFindVa(VA_LIST *, OBJVASPACE *, NvU64 *vaddr);
/*! Get the vas refCount */
NV_STATUS vaListGetRefCount(VA_LIST *, OBJVASPACE *, NvU64 *refCount);
#endif // VADDR_LIST_H

View File

@@ -0,0 +1,3 @@
#include "g_vaspace_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_video_mem_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_virt_mem_mgr_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_virt_mem_range_nvoc.h"

View File

@@ -0,0 +1,3 @@
#include "g_virtual_mem_nvoc.h"