NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-02-10 10:09:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

535.43.02

Browse Source
This commit is contained in:
Andy Ritger
2023-05-30 10:11:36 -07:00
parent 6dd092ddb7
commit eb5c7665a1
1403 changed files with 295367 additions and 86235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
src/nvidia/inc/kernel/gpu/mem_mgr/ce_utils.h Normal file
View File

@@ -0,0 +1,122 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include "g_ce_utils_nvoc.h"
#ifndef CE_UTILS_H
#define CE_UTILS_H
#include "gpu/gpu_resource.h" // GpuResource
#include "class/cl0050.h"
#include "ctrl/ctrl0050.h"
#include "kernel/gpu/mem_mgr/channel_utils.h"
typedef struct
{
MEMORY_DESCRIPTOR *pMemDesc;
NvU64 offset;
NvU64 length;
NvU32 pattern;
NvU64 flags;
NvU64 submittedWorkId; // Payload to poll for async completion
} CEUTILS_MEMSET_PARAMS;
typedef struct
{
MEMORY_DESCRIPTOR *pSrcMemDesc;
MEMORY_DESCRIPTOR *pDstMemDesc;
NvU64 dstOffset;
NvU64 srcOffset;
NvU64 length;
NvU64 flags;
NvU64 submittedWorkId; // Payload to poll for async completion
} CEUTILS_MEMCOPY_PARAMS;
NVOC_PREFIX(ceutils) class CeUtils : Object
{
public:
NV_STATUS ceutilsConstruct(CeUtils *pCeUtils, OBJGPU *pGpu, NV0050_ALLOCATION_PARAMETERS *pAllocParams);
void ceutilsDestruct(CeUtils *pCeUtils);
NV_STATUS ceutilsInitialize(CeUtils *pCeUtils, OBJGPU *pGpu, NV0050_ALLOCATION_PARAMETERS *pAllocParams);
void ceutilsDeinit(CeUtils *pCeUtils);
void ceutilsRegisterGPUInstance(CeUtils *pCeUtils, KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance);
NV_STATUS ceutilsMemset(CeUtils *pCeUtils, CEUTILS_MEMSET_PARAMS *pParams);
NV_STATUS ceutilsMemcopy(CeUtils *pCeUtils, CEUTILS_MEMCOPY_PARAMS *pParams);
NvU64 ceutilsUpdateProgress(CeUtils *pCeUtils);
void ceutilsServiceInterrupts(CeUtils *pCeUtils);
//
// Internal states
//
NvHandle hClient;
NvHandle hDevice;
NvHandle hSubdevice;
OBJCHANNEL *pChannel;
KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance;
OBJGPU *pGpu;
KernelCE *pKCe;
NvBool bUseVasForCeCopy;
NvU32 hTdCopyClass;
NvU64 lastSubmittedPayload;
NvU64 lastCompletedPayload;
};
#if defined(DEBUG) || defined (DEVELOP)
NVOC_PREFIX(ceutilsapi) class CeUtilsApi : GpuResource
{
public:
NV_STATUS ceutilsapiConstruct(CeUtilsApi *pCeUtilsApi, CALL_CONTEXT *pCallContext,
RS_RES_ALLOC_PARAMS_INTERNAL *pParams)
: GpuResource(pCallContext, pParams);
void ceutilsapiDestruct(CeUtilsApi *pCeUtilsApi);
//
// Below APIs are only provided for SRT testing, thus only available for debug or
// develop driver builds
//
//
RMCTRL_EXPORT(NV0050_CTRL_CMD_MEMSET, RMCTRL_FLAGS(PRIVILEGED, API_LOCK_READONLY))
NV_STATUS ceutilsapiCtrlCmdMemset(CeUtilsApi *pCeUtilsApi, NV0050_CTRL_MEMSET_PARAMS *pParams);
RMCTRL_EXPORT(NV0050_CTRL_CMD_MEMCOPY, RMCTRL_FLAGS(PRIVILEGED, API_LOCK_READONLY))
NV_STATUS ceutilsapiCtrlCmdMemcopy(CeUtilsApi *pCeUtilsApi, NV0050_CTRL_MEMCOPY_PARAMS *pParams);
RMCTRL_EXPORT(NV0050_CTRL_CMD_CHECK_PROGRESS, RMCTRL_FLAGS(PRIVILEGED, API_LOCK_READONLY))
NV_STATUS ceutilsapiCtrlCmdCheckProgress(CeUtilsApi *pCeUtilsApi, NV0050_CTRL_CHECK_PROGRESS_PARAMS *pParams);
CeUtils *pCeUtils;
};
#endif
#endif // CE_UTILS_H

36
src/nvidia/inc/kernel/gpu/mem_mgr/ce_utils_sizes.h Normal file
View File

@@ -0,0 +1,36 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _CE_UTILS_SIZES_H
#define _CE_UTILS_SIZES_H
#define CE_MAX_BYTES_PER_LINE 0xffffffffULL
#define CE_NUM_COPY_BLOCKS 4096
#define CE_CHANNEL_SEMAPHORE_SIZE 8
#define CE_GPFIFO_SIZE NV906F_GP_ENTRY__SIZE * CE_NUM_COPY_BLOCKS
#define CE_CHANNEL_NOTIFIER_SIZE (sizeof(NvNotification) * \
NV_CHANNELGPFIFO_NOTIFICATION_TYPE__SIZE_1)
#define CE_METHOD_SIZE_PER_BLOCK 0x64
#define FAST_SCRUBBER_METHOD_SIZE_PER_BLOCK 0x78
#endif // _CE_UTILS_SIZES_H

157
src/nvidia/inc/kernel/gpu/mem_mgr/channel_utils.h Normal file
View File

@@ -0,0 +1,157 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _CHANNEL_UTILS_H_
#define _CHANNEL_UTILS_H_
#include "core/core.h"
#include "gpu/gpu.h"
#include "gpu/mem_mgr/mem_mgr.h"
#include "gpu/ce/kernel_ce.h"
#include "gpu/bus/kern_bus.h"
#include "core/prelude.h"
#include "rmapi/rs_utils.h"
#include "nvos.h"
#include "class/cl906f.h"
#include "class/cl906f.h"
#include "class/cl906fsw.h"
#include "class/clb0b5.h" // MAXWELL_DMA_COPY_A
#include "class/clc0b5.h" // PASCAL_DMA_COPY_A
#include "class/clc1b5.h" // PASCAL_DMA_COPY_B
#include "class/clc3b5.h" // VOLTA_DMA_COPY_A
#include "class/clc5b5.h" // TURING_DMA_COPY_A
#include "class/clc6b5.h" // AMPERE_DMA_COPY_A
#include "class/clc7b5.h" // AMPERE_DMA_COPY_B
#include "class/clc8b5.h" // HOPPER_DMA_COPY_A
#include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A
#include "nvctassert.h"
#include "vgpu/vgpu_guest_pma_scrubber.h"
#define RM_SUBCHANNEL 0x0
#define NV_PUSH_METHOD(OpType, SubCh, Method, Count) \
(DRF_DEF(906F, _DMA, _SEC_OP, OpType) | \
DRF_NUM(906F, _DMA, _METHOD_ADDRESS, (Method) >> 2) | \
DRF_NUM(906F, _DMA, _METHOD_SUBCHANNEL, (SubCh)) | \
DRF_NUM(906F, _DMA, _METHOD_COUNT, (Count)))
#define _NV_ASSERT_CONTIGUOUS_METHOD(a1, a2) NV_ASSERT((a2) - (a1) == 4)
#define NV_PUSH_DATA(Data) MEM_WR32(pPtr++, (Data))
#define _NV_PUSH_INC_1U(SubCh, a1, d1, Count) \
do \
{ \
NV_PUSH_DATA(NV_PUSH_METHOD(_INC_METHOD, SubCh, a1, Count)); \
NV_PUSH_DATA(d1); \
} while (0)
#define NV_PUSH_INC_1U(SubCh, a1, d1) \
do \
{ \
_NV_PUSH_INC_1U (SubCh, a1, d1, 1); \
} while (0)
#define NV_PUSH_INC_2U(SubCh, a1, d1, a2, d2) \
do \
{ \
_NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \
_NV_PUSH_INC_1U(SubCh, a1, d1, 2); \
NV_PUSH_DATA(d2); \
} while (0)
#define NV_PUSH_INC_3U(SubCh, a1, d1, a2, d2, a3, d3) \
do \
{ \
_NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \
_NV_ASSERT_CONTIGUOUS_METHOD(a2, a3); \
_NV_PUSH_INC_1U(SubCh, a1, d1, 3); \
NV_PUSH_DATA(d2); \
NV_PUSH_DATA(d3); \
} while (0)
#define NV_PUSH_INC_4U(SubCh, a1, d1, a2, d2, a3, d3, a4, d4) \
do \
{ \
_NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \
_NV_ASSERT_CONTIGUOUS_METHOD(a2, a3); \
_NV_ASSERT_CONTIGUOUS_METHOD(a3, a4); \
_NV_PUSH_INC_1U(SubCh, a1, d1, 4); \
NV_PUSH_DATA(d2); \
NV_PUSH_DATA(d3); \
NV_PUSH_DATA(d4); \
} while (0)
#define READ_CHANNEL_PAYLOAD_SEMA(channel) MEM_RD32((NvU8*)channel->pbCpuVA + \
channel->finishPayloadOffset)
#define READ_CHANNEL_PB_SEMA(channel) MEM_RD32((NvU8*)channel->pbCpuVA + \
channel->semaOffset)
#define WRITE_CHANNEL_PB_SEMA(channel, val) MEM_WR32((NvU8*)channel->pbCpuVA + \
channel->semaOffset, val);
#define WRITE_CHANNEL_PAYLOAD_SEMA(channel,val) MEM_WR32((NvU8*)channel->pbCpuVA + \
channel->finishPayloadOffset, val);
//
// This struct contains parameters needed to send a pushbuffer for a CE
// operation. This interface only supports contiguous operations.
//
typedef struct
{
NvBool bCeMemcopy; // Whether this is a CE memcopy;
// If set to false, this will be a memset operation
NvU64 dstAddr; // Physical address of the source address
NvU64 srcAddr; // Physical address of the source address; only valid for memcopy
NvU32 size;
NvU32 pattern; // Fixed pattern to memset to. Only valid for memset
NvU32 payload; // Payload value used to release semaphore
NvU64 clientSemaAddr;
NV_ADDRESS_SPACE dstAddressSpace;
NV_ADDRESS_SPACE srcAddressSpace;
NvU32 dstCpuCacheAttrib;
NvU32 srcCpuCacheAttrib;
} CHANNEL_PB_INFO;
NV_STATUS channelSetupIDs(OBJCHANNEL *pChannel, OBJGPU *pGpu, NvBool bUseVasForCeCopy, NvBool bMIGInUse);
void channelSetupChannelBufferSizes(OBJCHANNEL *pChannel);
// Needed for pushbuffer management
NV_STATUS channelWaitForFreeEntry(OBJCHANNEL *pChannel, NvU32 *pPutIndex);
NV_STATUS channelFillGpFifo(OBJCHANNEL *pChannel, NvU32 putIndex, NvU32 methodsLength);
NvU32 channelFillPb(OBJCHANNEL *pChannel, NvU32 putIndex, NvBool bPipelined,
NvBool bInsertFinishPayload, CHANNEL_PB_INFO *pChannelPbInfo);
NvU32 channelFillPbFastScrub(OBJCHANNEL *pChannel, NvU32 putIndex, NvBool bPipelined,
NvBool bInsertFinishPayload, CHANNEL_PB_INFO *pChannelPbInfo);
// Needed for work tracking
NV_STATUS channelWaitForFinishPayload(OBJCHANNEL *pChannel, NvU64 targetPayload);
NvU64 channelGetFinishPayload(OBJCHANNEL *pChannel);
#endif // _CHANNEL_UTILS_H_

2
src/nvidia/inc/kernel/gpu/mem_mgr/heap_base.h
View File

@@ -125,7 +125,7 @@ typedef struct PMA_ALLOC_INFO
{
NvBool bContig;
NvU32 pageCount;
NvU32 pageSize;
NvU64 pageSize;
NvU32 refCount;
NvU64 allocSize;
NvU32 flags;

73
src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h
View File

@@ -34,16 +34,18 @@
#include "nvctassert.h"
#include "vgpu/vgpu_guest_pma_scrubber.h"
#if !defined(SRT_BUILD)
#include "gpu/mem_mgr/ce_utils.h"
#endif
struct OBJGPU;
struct Heap;
struct OBJCHANNEL;
#define RM_SUBCHANNEL 0x0
#define MEMSET_PATTERN 0x00000000
#define SCRUBBER_NUM_PAYLOAD_SEMAPHORES (2)
#define SCRUBBER_SEMAPHORE_SIZE_INBYTES (4)
#define SCRUBBER_CHANNEL_SEMAPHORE_SIZE (SCRUBBER_SEMAPHORE_SIZE_INBYTES *\
#define SCRUBBER_CHANNEL_SEMAPHORE_SIZE (SCRUBBER_SEMAPHORE_SIZE_INBYTES * \
SCRUBBER_NUM_PAYLOAD_SEMAPHORES)
#define SCRUBBER_CHANNEL_NOTIFIER_SIZE (sizeof(NvNotification) * NV_CHANNELGPFIFO_NOTIFICATION_TYPE__SIZE_1)
@@ -52,67 +54,6 @@ struct OBJCHANNEL;
#define SCRUB_MAX_BYTES_PER_LINE 0xffffffffULL
#define MAX_SCRUB_ITEMS 4096 // 4K scrub items
#define READ_SCRUBBER_PAYLOAD_SEMA(channel) MEM_RD32((NvU8*)channel->pbCpuVA +\
channel->finishPayloadOffset)
#define READ_SCRUBBER_PB_SEMA(channel) MEM_RD32((NvU8*)channel->pbCpuVA +\
channel->semaOffset)
#define WRITE_SCRUBBER_PB_SEMA(channel, val) MEM_WR32((NvU8*)channel->pbCpuVA +\
channel->semaOffset, val);
#define WRITE_SCRUBBER_PAYLOAD_SEMA(channel,val) MEM_WR32((NvU8*)channel->pbCpuVA +\
channel->finishPayloadOffset, val);
// Use Incrementing Methods to save the PB Space
#define _NV_ASSERT_CONTIGUOUS_METHODS(a1, a2) NV_ASSERT((a2) - (a1) == 4)
#define NV_PUSH_METHOD(OpType, SubCh, Method, Count) \
(DRF_DEF(906F, _DMA, _SEC_OP, OpType) |\
DRF_NUM(906F, _DMA, _METHOD_ADDRESS, (Method) >> 2) |\
DRF_NUM(906F, _DMA, _METHOD_SUBCHANNEL, (SubCh)) |\
DRF_NUM(906F, _DMA, _METHOD_COUNT, (Count)))
#define NV_PUSH_DATA(Data) MEM_WR32(pPtr++, (Data))
#define _NV_PUSH_INC_1U(SubCh, a1,d1, Count) \
do{ \
NV_PUSH_DATA(NV_PUSH_METHOD(_INC_METHOD, SubCh, a1, Count));\
NV_PUSH_DATA(d1); \
} while(0)
#define NV_PUSH_INC_1U(SubCh, a1,d1) \
do{ \
_NV_PUSH_INC_1U (SubCh, a1,d1, 1);\
} while(0)
#define NV_PUSH_INC_2U(SubCh, a1,d1, a2,d2) \
do{ \
_NV_ASSERT_CONTIGUOUS_METHODS(a1, a2);\
_NV_PUSH_INC_1U(SubCh, a1,d1, 2); \
NV_PUSH_DATA(d2); \
} while(0)
#define NV_PUSH_INC_3U(SubCh, a1,d1, a2,d2, a3,d3) \
do{ \
_NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
_NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
_NV_PUSH_INC_1U(SubCh, a1,d1, 3); \
NV_PUSH_DATA(d2); \
NV_PUSH_DATA(d3); \
} while(0)
#define NV_PUSH_INC_4U(SubCh, a1,d1, a2,d2, a3,d3, a4,d4) \
do{ \
_NV_ASSERT_CONTIGUOUS_METHODS(a1,a2);\
_NV_ASSERT_CONTIGUOUS_METHODS(a2,a3);\
_NV_ASSERT_CONTIGUOUS_METHODS(a3,a4);\
_NV_PUSH_INC_1U(SubCh, a1,d1, 4); \
NV_PUSH_DATA(d2); \
NV_PUSH_DATA(d3); \
NV_PUSH_DATA(d4); \
} while(0)
// structure to store the details of a scrubbing work
typedef struct SCRUB_NODE {
// The 64 bit ID assigned to each work
@@ -144,8 +85,10 @@ typedef struct OBJMEMSCRUB {
NvLength scrubListSize;
// Pre-allocated Free Scrub List
PSCRUB_NODE pScrubList;
// Scrubber Channel
struct OBJCHANNEL *pChannel;
#if !defined(SRT_BUILD)
// Scrubber uses ceUtils to manage CE channel
CeUtils ceUtilsObject;
#endif
struct OBJGPU *pGpu;
VGPU_GUEST_PMA_SCRUB_BUFFER_RING vgpuScrubBuffRing;
NvBool bVgpuScrubberEnabled;

4
src/nvidia/inc/kernel/gpu/mem_mgr/mem_utils.h
View File

@@ -26,12 +26,12 @@
#include "core/prelude.h"
#define CLEAR_HAL_ATTR(a) \
#define CLEAR_HAL_ATTR(a) \
a = (a &~(DRF_NUM(OS32, _ATTR, _COMPR, 0x3) | \
DRF_NUM(OS32, _ATTR, _TILED, 0x3) | \
DRF_NUM(OS32, _ATTR, _ZCULL, 0x3)));
#define CLEAR_HAL_ATTR2(a) \
#define CLEAR_HAL_ATTR2(a) \
a = (a & ~(DRF_SHIFTMASK(NVOS32_ATTR2_ZBC) | \
DRF_SHIFTMASK(NVOS32_ATTR2_GPU_CACHEABLE)));

6
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/addrtree.h
View File

@@ -127,12 +127,12 @@ void pmaAddrtreeSetEvictingFrames(void *pMap, NvU64 frameEvictionsInProcess);
*/
NV_STATUS pmaAddrtreeScanContiguous(
void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
NV_STATUS pmaAddrtreeScanDiscontiguous(
void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
void pmaAddrtreePrintTree(void *pMap, const char* str);
@@ -154,7 +154,7 @@ void pmaAddrtreePrintTree(void *pMap, const char* str);
void pmaAddrtreeChangeState(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
void pmaAddrtreeChangeStateAttrib(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib);
void pmaAddrtreeChangeStateAttribEx(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState,PMA_PAGESTATUS newStateMask);
void pmaAddrtreeChangePageStateAttrib(void * pMap, NvU64 startFrame, NvU32 pageSize,
void pmaAddrtreeChangePageStateAttrib(void * pMap, NvU64 startFrame, NvU64 pageSize,
PMA_PAGESTATUS newState, NvBool writeAttrib);
/*!

5
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/map_defines.h
View File

@@ -90,7 +90,7 @@ typedef NvU32 PMA_PAGESTATUS;
#define ATTRIB_PERSISTENT NVBIT(MAP_IDX_PERSISTENT)
#define ATTRIB_NUMA_REUSE NVBIT(MAP_IDX_NUMA_REUSE)
#define ATTRIB_BLACKLIST NVBIT(MAP_IDX_BLACKLIST)
#define ATTRIB_MASK (ATTRIB_EVICTING | ATTRIB_SCRUBBING \
#define ATTRIB_MASK (ATTRIB_EVICTING | ATTRIB_SCRUBBING \
| ATTRIB_PERSISTENT | ATTRIB_NUMA_REUSE \
| ATTRIB_BLACKLIST)
@@ -114,6 +114,9 @@ typedef struct _PMA_STATS
NvU64 numFreeFrames; // PMA-wide free 64KB frame count
NvU64 numFree2mbPages; // PMA-wide free 2MB pages count
#if !defined(NVWATCH)
NvU64 num2mbPagesProtected; // PMA-wide total number of 2MB pages in protected memory
NvU64 numFreeFramesProtected; // PMA-wide free 64KB frame count in protected memory
NvU64 numFree2mbPagesProtected; // PMA-wide free 2MB pages count in protected memory
#endif // !defined(NVWATCH)
} PMA_STATS;

2
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/numa.h
View File

@@ -59,7 +59,7 @@ extern "C" {
* is used for allocations coming from the Linux kernel.
* The perf implication is under further study. See bug #1999793.
*/
NV_STATUS pmaNumaAllocate(PMA *pPma, NvLength allocationCount, NvU32 pageSize,
NV_STATUS pmaNumaAllocate(PMA *pPma, NvLength allocationCount, NvU64 pageSize,
PMA_ALLOCATION_OPTIONS *allocationOptions, NvU64 *pPages);
/*!

70
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -73,6 +73,7 @@ typedef struct SCRUB_NODE SCRUB_NODE;
#define PMA_INIT_INTERNAL NVBIT(3) // Used after heap is removed
#define PMA_INIT_FORCE_PERSISTENCE NVBIT(4)
#define PMA_INIT_ADDRTREE NVBIT(5)
#define PMA_INIT_NUMA_AUTO_ONLINE NVBIT(6)
// These flags are used for querying PMA's config and/or state.
#define PMA_QUERY_SCRUB_ENABLED NVBIT(0)
@@ -166,7 +167,7 @@ typedef enum
/*!
* @brief Callbacks to UVM for eviction
*/
typedef NV_STATUS (*pmaEvictPagesCb_t)(void *ctxPtr, NvU32 pageSize, NvU64 *pPages,
typedef NV_STATUS (*pmaEvictPagesCb_t)(void *ctxPtr, NvU64 pageSize, NvU64 *pPages,
NvU32 count, NvU64 physBegin, NvU64 physEnd,
MEMORY_PROTECTION prot);
typedef NV_STATUS (*pmaEvictRangeCb_t)(void *ctxPtr, NvU64 physBegin, NvU64 physEnd,
@@ -180,13 +181,13 @@ typedef void (*pmaMapDestroy_t)(void *pMap);
typedef void (*pmaMapChangeState_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState);
typedef void (*pmaMapChangeStateAttrib_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib);
typedef void (*pmaMapChangeStateAttribEx_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask);
typedef void (*pmaMapChangePageStateAttrib_t)(void *pMap, NvU64 startFrame, NvU32 pageSize, PMA_PAGESTATUS newState, NvBool writeAttrib);
typedef void (*pmaMapChangePageStateAttrib_t)(void *pMap, NvU64 startFrame, NvU64 pageSize, PMA_PAGESTATUS newState, NvBool writeAttrib);
typedef PMA_PAGESTATUS (*pmaMapRead_t)(void *pMap, NvU64 frameNum, NvBool readAttrib);
typedef NV_STATUS (*pmaMapScanContiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
typedef NV_STATUS (*pmaMapScanDiscontiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
typedef void (*pmaMapGetSize_t)(void *pMap, NvU64 *pBytesTotal);
typedef void (*pmaMapGetLargestFree_t)(void *pMap, NvU64 *pLargestFree);
@@ -251,6 +252,7 @@ struct _PMA
NvU64 coherentCpuFbBase; // Used to calculate FB offset from bus address
NvU64 coherentCpuFbSize; // Used for error checking only
NvU32 numaReclaimSkipThreshold; // percent value below which __GFP_RECLAIM will not be used.
NvBool bNumaAutoOnline; // If NUMA memory is auto-onlined
// Blacklist related states
PMA_BLACKLIST_CHUNK *pBlacklistChunks; // Tracking for blacklist pages
@@ -433,12 +435,12 @@ NV_STATUS pmaRegisterRegion(PMA *pPma, NvU32 id, NvBool bAsyncEccScrub,
* code,because it is not very informative.
*
*/
NV_STATUS pmaAllocatePages(PMA *pPma, NvLength pageCount, NvU32 pageSize,
NV_STATUS pmaAllocatePages(PMA *pPma, NvLength pageCount, NvU64 pageSize,
PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages);
// allocate on multiple GPU, thus pmaCount
NV_STATUS pmaAllocatePagesBroadcast(PMA **pPma, NvU32 pmaCount, NvLength allocationCount,
NvU32 pageSize, PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages);
NvU64 pageSize, PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages);
/*!
@@ -472,7 +474,7 @@ NV_STATUS pmaAllocatePagesBroadcast(PMA **pPma, NvU32 pmaCount, NvLength allocat
* TODO some error for rollback
*
*/
NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize);
NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize);
/*!
@@ -498,7 +500,7 @@ NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSi
* TODO some error for rollback
*
*/
NV_STATUS pmaUnpinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU32 pageSize);
NV_STATUS pmaUnpinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize);
/*!
@@ -815,7 +817,7 @@ void pmaNumaOfflined(PMA *pPma);
* @return
* void
*/
void pmaGetClientBlacklistedPages(PMA *pPma, NvU64 *pChunks, NvU32 *pPageSize, NvU32 *pNumChunks);
void pmaGetClientBlacklistedPages(PMA *pPma, NvU64 *pChunks, NvU64 *pPageSize, NvU32 *pNumChunks);
/*!
* @brief Returns the PMA blacklist size in bytes for
@@ -865,6 +867,54 @@ void pmaPrintMapState(PMA *pPma);
*/
NV_STATUS pmaAddToBlacklistTracking(PMA *pPma, NvU64 physBase);
/*!
* @brief Returns total protected video memory.
*
* @param[in] pPma PMA pointer
* @param[in] pBytesTotal Pointer that will return the total FB memory size.
*
* @return
* void
*/
void pmaGetTotalProtectedMemory(PMA *pPma, NvU64 *pBytesTotal);
/*!
* @brief Returns total unprotected video memory.
*
* @param[in] pPma PMA pointer
* @param[in] pBytesTotal Pointer that will return the total FB memory size.
*
* @return
* void
*/
void pmaGetTotalUnprotectedMemory(PMA *pPma, NvU64 *pBytesTotal);
/*!
* @brief Returns information about the total free protected FB memory.
* In confidential compute use cases, memory will be split into
* protected and unprotected regions
*
* @param[in] pPma PMA pointer
* @param[in] pBytesFree Pointer that will return the free protected memory size.
*
* @return
* void
*/
void pmaGetFreeProtectedMemory(PMA *pPma, NvU64 *pBytesFree);
/*!
* @brief Returns information about the total free unprotected FB memory.
* In confidential compute use cases, memory will be split into
* protected and unprotected regions
*
* @param[in] pPma PMA pointer
* @param[in] pBytesFree Pointer that will return the free unprotected memory size.
*
* @return
* void
*/
void pmaGetFreeUnprotectedMemory(PMA *pPma, NvU64 *pBytesFree);
#ifdef __cplusplus
}
#endif

4
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.h
View File

@@ -45,11 +45,11 @@ NvBool pmaStateCheck(PMA *pPma);
NV_STATUS _pmaEvictContiguous(PMA *pPma, void *pMap, NvU64 evictStart, NvU64 evictEnd,
MEMORY_PROTECTION prot);
NV_STATUS _pmaEvictPages(PMA *pPma, void *pMap, NvU64 *evictPages, NvU64 evictPageCount,
NvU64 *allocPages, NvU64 allocPageCount, NvU32 pageSize,
NvU64 *allocPages, NvU64 allocPageCount, NvU64 pageSize,
NvU64 physBegin, NvU64 physEnd, MEMORY_PROTECTION prot);
void _pmaClearScrubBit(PMA *pPma, SCRUB_NODE *pPmaScrubList, NvU64 count);
NV_STATUS _pmaCheckScrubbedPages(PMA *pPma, NvU64 chunkSize, NvU64 *pPages, NvU32 pageCount);
NV_STATUS _pmaPredictOutOfMemory(PMA *pPma, NvLength allocationCount, NvU32 pageSize,
NV_STATUS _pmaPredictOutOfMemory(PMA *pPma, NvLength allocationCount, NvU64 pageSize,
PMA_ALLOCATION_OPTIONS *allocationOptions);
NV_STATUS pmaSelector(PMA *pPma, PMA_ALLOCATION_OPTIONS *allocationOptions, NvS32 *regionList);
void _pmaReallocBlacklistPages (PMA *pPma, NvU32 regId, NvU64 rangeBegin, NvU64 rangeSize);

6
src/nvidia/inc/kernel/gpu/mem_mgr/phys_mem_allocator/regmap.h
View File

@@ -131,7 +131,7 @@ void pmaRegmapChangeStateAttrib(void *pMap, NvU64 frameNum,
*
* @return void
*/
void pmaRegmapChangePageStateAttrib(void * pMap, NvU64 frameNumStart, NvU32 pageSize,
void pmaRegmapChangePageStateAttrib(void * pMap, NvU64 frameNumStart, NvU64 pageSize,
PMA_PAGESTATUS newState, NvBool writeAttrib);
/*!
@@ -188,7 +188,7 @@ PMA_PAGESTATUS pmaRegmapRead(void *pMap, NvU64 frameNum, NvBool readAttrib);
*/
NV_STATUS pmaRegmapScanContiguous(
void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
/*!
@@ -215,7 +215,7 @@ NV_STATUS pmaRegmapScanContiguous(
*/
NV_STATUS pmaRegmapScanDiscontiguous(
void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd,
NvU64 numPages, NvU64 *freelist, NvU32 pageSize, NvU64 alignment,
NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment,
NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc);
/*!

80
src/nvidia/inc/kernel/gpu/mem_mgr/rm_page_size.h Normal file
View File

@@ -0,0 +1,80 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef RM_PAGE_SIZE_H
#define RM_PAGE_SIZE_H
//---------------------------------------------------------------------------
//
// Memory page defines.
//
// These correspond to the granularity understood by the hardware
// for address mapping; the system page size can be larger.
//
//---------------------------------------------------------------------------
#define RM_PAGE_SIZE_INVALID 0
#define RM_PAGE_SIZE 4096
#define RM_PAGE_SIZE_64K (64 * 1024)
#define RM_PAGE_SIZE_128K (128 * 1024)
#define RM_PAGE_MASK 0x0FFF
#define RM_PAGE_SHIFT 12
#define RM_PAGE_SHIFT_64K 16
#define RM_PAGE_SHIFT_128K 17
#define RM_PAGE_SHIFT_2M 21
#define RM_PAGE_SIZE_2M (1 << RM_PAGE_SHIFT_2M)
// Huge page size is 2 MB
#define RM_PAGE_SHIFT_HUGE RM_PAGE_SHIFT_2M
#define RM_PAGE_SIZE_HUGE (1ULL << RM_PAGE_SHIFT_HUGE)
#define RM_PAGE_MASK_HUGE ((1ULL << RM_PAGE_SHIFT_HUGE) - 1)
// 512MB page size
#define RM_PAGE_SHIFT_512M 29
#define RM_PAGE_SIZE_512M (1ULL << RM_PAGE_SHIFT_512M)
#define RM_PAGE_MASK_512M (RM_PAGE_SIZE_512M - 1)
//---------------------------------------------------------------------------
//
// Memory page attributes.
//
// These attributes are used by software for page size mapping;
// Big pages can be of 64/128KB[Fermi/Kepler/Pascal]
// Huge page is 2 MB[Pascal+]
// 512MB page is Ampere+
// Default page attribute lets driver decide the optimal page size
//
//---------------------------------------------------------------------------
typedef enum
{
RM_ATTR_PAGE_SIZE_DEFAULT,
RM_ATTR_PAGE_SIZE_4KB,
RM_ATTR_PAGE_SIZE_BIG,
RM_ATTR_PAGE_SIZE_HUGE,
RM_ATTR_PAGE_SIZE_512MB,
RM_ATTR_PAGE_SIZE_INVALID
}
RM_ATTR_PAGE_SIZE;
#endif // RM_PAGE_SIZE_H

3
src/nvidia/inc/kernel/gpu/mem_mgr/sem_surf.h Normal file
View File

@@ -0,0 +1,3 @@
#include "g_sem_surf_nvoc.h"

60
src/nvidia/inc/kernel/gpu/mem_mgr/virt_mem_allocator_common.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -33,60 +33,14 @@
#include "nvtypes.h"
#include "nvgputypes.h"
#include "nvstatus.h"
#include "resserv/rs_client.h"
#include "gpu/mem_mgr/rm_page_size.h"
typedef struct OBJGPU OBJGPU;
typedef struct ChannelDescendant ChannelDescendant;
typedef struct ContextDma ContextDma;
typedef struct Memory Memory;
typedef struct EVENTNOTIFICATION EVENTNOTIFICATION;
//---------------------------------------------------------------------------
//
// Memory page defines.
//
// These correspond to the granularity understood by the hardware
// for address mapping; the system page size can be larger.
//
//---------------------------------------------------------------------------
#define RM_PAGE_SIZE_INVALID 0
#define RM_PAGE_SIZE 4096
#define RM_PAGE_SIZE_64K (64 * 1024)
#define RM_PAGE_SIZE_128K (128 * 1024)
#define RM_PAGE_MASK 0x0FFF
#define RM_PAGE_SHIFT 12
#define RM_PAGE_SHIFT_64K 16
#define RM_PAGE_SHIFT_128K 17
// Huge page size is 2 MB
#define RM_PAGE_SHIFT_HUGE 21
#define RM_PAGE_SIZE_HUGE (1 << RM_PAGE_SHIFT_HUGE)
#define RM_PAGE_MASK_HUGE ((1 << RM_PAGE_SHIFT_HUGE) - 1)
// 512MB page size
#define RM_PAGE_SHIFT_512M 29
#define RM_PAGE_SIZE_512M (1 << RM_PAGE_SHIFT_512M)
#define RM_PAGE_MASK_512M (RM_PAGE_SIZE_512M - 1)
//---------------------------------------------------------------------------
//
// Memory page attributes.
//
// These attributes are used by software for page size mapping;
// Big pages can be of 64/128KB[Fermi/Kepler/Pascal]
// Huge page is 2 MB[Pascal+]
// 512MB page is Ampere+
// Default page attribute lets driver decide the optimal page size
//
//---------------------------------------------------------------------------
typedef enum
{
RM_ATTR_PAGE_SIZE_DEFAULT = 0x0,
RM_ATTR_PAGE_SIZE_4KB = 0x1,
RM_ATTR_PAGE_SIZE_BIG = 0x2,
RM_ATTR_PAGE_SIZE_HUGE = 0x3,
RM_ATTR_PAGE_SIZE_512MB = 0x4,
RM_ATTR_PAGE_SIZE_INVALID = 0x5
}
RM_ATTR_PAGE_SIZE;
//---------------------------------------------------------------------------
//
@@ -133,8 +87,8 @@ void notifyFillNOTIFICATION(OBJGPU *pGpu,
NV_STATUS CompletionStatus,
NvBool TimeSupplied,
NvU64 Time);
NV_STATUS notifyFillNotifierGPUVA (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32);
NV_STATUS notifyFillNotifierGPUVATimestamp (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
NV_STATUS notifyFillNotifierGPUVA (OBJGPU*, RsClient*, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32);
NV_STATUS notifyFillNotifierGPUVATimestamp (OBJGPU*, RsClient*, NvHandle, NvU64, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
NV_STATUS notifyFillNotifierMemory (OBJGPU*, Memory *, NvV32, NvV16, NV_STATUS, NvU32);
NV_STATUS notifyFillNotifierMemoryTimestamp(OBJGPU*, Memory *, NvV32, NvV16, NV_STATUS, NvU32, NvU64);
void notifyFillNvNotification(OBJGPU *pGpu,
@@ -145,8 +99,8 @@ void notifyFillNvNotification(OBJGPU *pGpu,
NvBool TimeSupplied,
NvU64 Time);
NV_STATUS semaphoreFillGPUVA (OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV32, NvBool);
NV_STATUS semaphoreFillGPUVATimestamp(OBJGPU*, NvHandle, NvHandle, NvU64, NvV32, NvV32, NvBool, NvU64);
NV_STATUS semaphoreFillGPUVA (OBJGPU*, RsClient*, NvHandle, NvU64, NvV32, NvV32, NvBool);
NV_STATUS semaphoreFillGPUVATimestamp(OBJGPU*, RsClient*, NvHandle, NvU64, NvV32, NvV32, NvBool, NvU64);
RM_ATTR_PAGE_SIZE dmaNvos32ToPageSizeAttr(NvU32 attr, NvU32 attr2);