NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-04-28 18:21:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

525.53

Browse Source
This commit is contained in:
Andy Ritger
2022-11-10 08:39:33 -08:00
parent 7c345b838b
commit 758b4ee818
1323 changed files with 262135 additions and 60754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
kernel-open/nvidia/nv-dmabuf.c
View File

@@ -24,6 +24,15 @@
#include "nv-dmabuf.h"
#if defined(CONFIG_DMA_SHARED_BUFFER)
//
// The Linux kernel's dma_length in struct scatterlist is unsigned int
// which limits the maximum sg length to 4GB - 1.
// To get around this limitation, the BAR1 scatterlist returned by RM
// is split into (4GB - PAGE_SIZE) sized chunks to build the sg_table.
//
#define NV_DMA_BUF_SG_MAX_LEN ((NvU32)(NVBIT64(32) - PAGE_SIZE))
typedef struct nv_dma_buf_mem_handle
{
NvHandle h_memory;
@@ -77,24 +86,20 @@ nv_dma_buf_alloc_file_private(
{
nv_dma_buf_file_private_t *priv = NULL;
NV_KMALLOC(priv, sizeof(nv_dma_buf_file_private_t));
NV_KZALLOC(priv, sizeof(nv_dma_buf_file_private_t));
if (priv == NULL)
{
return NULL;
}
memset(priv, 0, sizeof(nv_dma_buf_file_private_t));
mutex_init(&priv->lock);
NV_KMALLOC(priv->handles, num_handles * sizeof(priv->handles[0]));
NV_KZALLOC(priv->handles, num_handles * sizeof(priv->handles[0]));
if (priv->handles == NULL)
{
goto failed;
}
memset(priv->handles, 0, num_handles * sizeof(priv->handles[0]));
return priv;
failed:
@@ -257,26 +262,36 @@ nv_dma_buf_unmap_unlocked(
nv_dma_device_t *peer_dma_dev,
nv_dma_buf_file_private_t *priv,
struct sg_table *sgt,
NvU32 count
NvU32 mapped_handle_count
)
{
NV_STATUS status;
NvU32 i;
NvU64 dma_len;
NvU64 dma_addr;
NvU64 bar1_va;
NvBool bar1_unmap_needed;
struct scatterlist *sg = NULL;
bar1_unmap_needed = (priv->bar1_va_ref_count == 0);
for_each_sg(sgt->sgl, sg, count, i)
sg = sgt->sgl;
for (i = 0; i < mapped_handle_count; i++)
{
dma_addr = sg_dma_address(sg);
dma_len = priv->handles[i].size;
bar1_va = priv->handles[i].bar1_va;
NvU64 handle_size = priv->handles[i].size;
WARN_ON(sg_dma_len(sg) != priv->handles[i].size);
dma_addr = sg_dma_address(sg);
dma_len = 0;
//
// Seek ahead in the scatterlist until the handle size is covered.
// IOVA unmap can then be done all at once instead of doing it
// one sg at a time.
//
while(handle_size != dma_len)
{
dma_len += sg_dma_len(sg);
sg = sg_next(sg);
}
nv_dma_unmap_peer(peer_dma_dev, (dma_len / os_page_size), dma_addr);
@@ -307,7 +322,8 @@ nv_dma_buf_map(
nv_dma_device_t peer_dma_dev = {{ 0 }};
NvBool bar1_map_needed;
NvBool bar1_unmap_needed;
NvU32 count = 0;
NvU32 mapped_handle_count = 0;
NvU32 num_sg_entries = 0;
NvU32 i = 0;
int rc = 0;
@@ -352,20 +368,29 @@ nv_dma_buf_map(
goto unlock_api_lock;
}
NV_KMALLOC(sgt, sizeof(struct sg_table));
NV_KZALLOC(sgt, sizeof(struct sg_table));
if (sgt == NULL)
{
goto unlock_gpu_lock;
}
memset(sgt, 0, sizeof(struct sg_table));
//
// Pre-calculate number of sg entries we need based on handle size.
// This is needed to allocate sg_table.
//
for (i = 0; i < priv->num_objects; i++)
{
NvU64 count = priv->handles[i].size + NV_DMA_BUF_SG_MAX_LEN - 1;
do_div(count, NV_DMA_BUF_SG_MAX_LEN);
num_sg_entries += count;
}
//
// RM currently returns contiguous BAR1, so we create as many
// sg entries as the number of handles being mapped.
// sg entries as num_sg_entries calculated above.
// When RM can alloc discontiguous BAR1, this code will need to be revisited.
//
rc = sg_alloc_table(sgt, priv->num_objects, GFP_KERNEL);
rc = sg_alloc_table(sgt, num_sg_entries, GFP_KERNEL);
if (rc != 0)
{
goto free_sgt;
@@ -375,7 +400,8 @@ nv_dma_buf_map(
peer_dma_dev.addressable_range.limit = (NvU64)dev->dma_mask;
bar1_map_needed = bar1_unmap_needed = (priv->bar1_va_ref_count == 0);
for_each_sg(sgt->sgl, sg, priv->num_objects, i)
sg = sgt->sgl;
for (i = 0; i < priv->num_objects; i++)
{
NvU64 dma_addr;
NvU64 dma_len;
@@ -393,9 +419,15 @@ nv_dma_buf_map(
}
}
mapped_handle_count++;
dma_addr = priv->handles[i].bar1_va;
dma_len = priv->handles[i].size;
//
// IOVA map the full handle at once and then breakdown the range
// (dma_addr, dma_addr + dma_len) into smaller sg entries.
//
status = nv_dma_map_peer(&peer_dma_dev, priv->nv->dma_dev,
0x1, (dma_len / os_page_size), &dma_addr);
if (status != NV_OK)
@@ -409,14 +441,23 @@ nv_dma_buf_map(
priv->handles[i].bar1_va);
}
mapped_handle_count--;
// Unmap remaining memory handles
goto unmap_handles;
}
sg_set_page(sg, NULL, dma_len, 0);
sg_dma_address(sg) = (dma_addr_t)dma_addr;
sg_dma_len(sg) = dma_len;
count++;
while(dma_len != 0)
{
NvU32 sg_len = NV_MIN(dma_len, NV_DMA_BUF_SG_MAX_LEN);
sg_set_page(sg, NULL, sg_len, 0);
sg_dma_address(sg) = (dma_addr_t)dma_addr;
sg_dma_len(sg) = sg_len;
dma_addr += sg_len;
dma_len -= sg_len;
sg = sg_next(sg);
}
}
priv->bar1_va_ref_count++;
@@ -432,7 +473,7 @@ nv_dma_buf_map(
return sgt;
unmap_handles:
nv_dma_buf_unmap_unlocked(sp, &peer_dma_dev, priv, sgt, count);
nv_dma_buf_unmap_unlocked(sp, &peer_dma_dev, priv, sgt, mapped_handle_count);
sg_free_table(sgt);