NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-01-31 13:39:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

550.54.14

Browse Source
This commit is contained in:
Bernhard Stoeckner
2024-02-23 16:37:56 +01:00
parent 91676d6628
commit 476bd34534
186 changed files with 42509 additions and 37629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
kernel-open/nvidia/internal_crypt_lib.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -53,6 +53,7 @@
(defined(CONFIG_CRYPTO_HMAC) || defined(CONFIG_CRYPTO_HMAC_MODULE)) && \
(defined(CONFIG_CRYPTO_ECDH) || defined(CONFIG_CRYPTO_ECDH_MODULE)) && \
(defined(CONFIG_CRYPTO_ECDSA) || defined(CONFIG_CRYPTO_ECDSA_MODULE)) && \
(defined(CONFIG_CRYPTO_RSA) || defined(CONFIG_CRYPTO_RSA_MODULE)) && \
(defined(CONFIG_X509_CERTIFICATE_PARSER) || defined(CONFIG_X509_CERTIFICATE_PARSER_MODULE))
#define NV_CONFIG_CRYPTO_PRESENT 1
#endif
@@ -151,4 +152,17 @@ bool lkca_ec_compute_key(void *ec_context, const uint8_t *peer_public,
bool lkca_ecdsa_verify(void *ec_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
const uint8_t *signature, size_t sig_size);
bool lkca_rsa_verify(void *rsa_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
const uint8_t *signature, size_t sig_size);
bool lkca_rsa_pkcs1_sign(void *rsa_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
uint8_t *signature, size_t *sig_size);
bool lkca_rsa_pss_sign(void *rsa_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
uint8_t *signature, size_t *sig_size);
#endif

611
kernel-open/nvidia/libspdm_rsa.c Normal file
View File

@@ -0,0 +1,611 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include "internal_crypt_lib.h"
#include "library/cryptlib.h"
#ifdef USE_LKCA
#include <linux/module.h>
#include <linux/mpi.h>
#include <linux/random.h>
#include <crypto/akcipher.h>
#include <crypto/internal/rsa.h>
/* ------------------------ Macros & Defines ------------------------------- */
#define GET_MOST_SIGNIFICANT_BIT(keySize) (keySize > 0 ? ((keySize - 1) & 7) : 0)
#define GET_ENC_MESSAGE_SIZE_BYTE(keySize) (keySize + 7) >> 3;
#define PKCS1_MGF1_COUNTER_SIZE_BYTE (4)
#define RSA_PSS_PADDING_ZEROS_SIZE_BYTE (8)
#define RSA_PSS_TRAILER_FIELD (0xbc)
#define SHIFT_RIGHT_AND_GET_BYTE(val, x) ((val >> x) & 0xFF)
#define BITS_TO_BYTES(b) (b >> 3)
static const unsigned char zeroes[RSA_PSS_PADDING_ZEROS_SIZE_BYTE] = { 0 };
struct rsa_ctx
{
struct rsa_key key;
bool pub_key_set;
bool priv_key_set;
int size;
};
#endif // #ifdef USE_LKCA
/*!
* Creating and initializing a RSA context.
*
* @return : A void pointer points to a RSA context
*
*/
void *libspdm_rsa_new
(
void
)
{
#ifndef USE_LKCA
return NULL;
#else
struct rsa_ctx *ctx;
ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
if (ctx == NULL)
{
return NULL;
}
memset(ctx, 0, sizeof(*ctx));
ctx->pub_key_set = false;
ctx->priv_key_set = false;
return ctx;
#endif
}
/*!
* To free a RSA context.
*
* @param rsa_context : A RSA context pointer
*
*/
void libspdm_rsa_free
(
void *rsa_context
)
{
#ifdef USE_LKCA
struct rsa_ctx *ctx = rsa_context;
if (ctx != NULL)
{
if (ctx->key.n) kfree(ctx->key.n);
if (ctx->key.e) kfree(ctx->key.e);
if (ctx->key.d) kfree(ctx->key.d);
if (ctx->key.q) kfree(ctx->key.q);
if (ctx->key.p) kfree(ctx->key.p);
if (ctx->key.dq) kfree(ctx->key.dq);
if (ctx->key.dp) kfree(ctx->key.dp);
if (ctx->key.qinv) kfree(ctx->key.qinv);
kfree(ctx);
}
#endif
}
#define rsa_set_key_case(a, a_sz, A) \
case A: \
{ \
if (ctx->key.a) { \
kfree(ctx->key.a); \
} \
ctx->key.a = shadow_num; \
ctx->key.a_sz = bn_size; \
break; \
}
/*!
* To set key into RSA context.
*
* @param rsa_context : A RSA context pointer
* @param key_tag : Indicate key tag for RSA key
* @param big_number : A big nuMber buffer to store rsa KEY
* @param bn_size : The size of bug number
*
* @Return : True if OK; otherwise return False
*/
bool libspdm_rsa_set_key
(
void *rsa_context,
const libspdm_rsa_key_tag_t key_tag,
const uint8_t *big_number,
size_t bn_size
)
{
#ifndef USE_LKCA
return false;
#else
struct rsa_ctx *ctx = rsa_context;
uint8_t *shadow_num;
if (ctx == NULL)
{
return false;
}
// Quick sanity check if tag is valid
switch (key_tag)
{
case LIBSPDM_RSA_KEY_N:
case LIBSPDM_RSA_KEY_E:
case LIBSPDM_RSA_KEY_D:
case LIBSPDM_RSA_KEY_Q:
case LIBSPDM_RSA_KEY_P:
case LIBSPDM_RSA_KEY_DP:
case LIBSPDM_RSA_KEY_DQ:
case LIBSPDM_RSA_KEY_Q_INV:
break;
default:
return false;
break;
}
if (big_number != NULL)
{
shadow_num = kmalloc(bn_size, GFP_KERNEL);
if (shadow_num == NULL)
{
return false;
}
memcpy(shadow_num, big_number, bn_size);
}
else
{
shadow_num = NULL;
bn_size = 0;
}
switch (key_tag)
{
rsa_set_key_case(n, n_sz, LIBSPDM_RSA_KEY_N)
rsa_set_key_case(e, e_sz, LIBSPDM_RSA_KEY_E)
rsa_set_key_case(d, d_sz, LIBSPDM_RSA_KEY_D)
rsa_set_key_case(q, q_sz, LIBSPDM_RSA_KEY_Q)
rsa_set_key_case(p, p_sz, LIBSPDM_RSA_KEY_P)
rsa_set_key_case(dq, dq_sz, LIBSPDM_RSA_KEY_DQ)
rsa_set_key_case(dp, dp_sz, LIBSPDM_RSA_KEY_DP)
rsa_set_key_case(qinv, qinv_sz, LIBSPDM_RSA_KEY_Q_INV)
default:
// We can't get here ever
break;
}
return true;
#endif
}
/*!
* Perform PKCS1 MGF1 operation.
*
* @param mask : A mask pointer to store return data
* @param maskedDB_length : Indicate mask data block length
* @param seed : A seed pointer to store random values
* @param seed_length : The seed length
* @param hash_nid : The hash NID
*
* @Return : True if OK; otherwise return False
*/
static bool NV_PKCS1_MGF1
(
uint8_t *mask,
size_t maskedDB_length,
const uint8_t *seed,
size_t seed_length,
size_t hash_nid
)
{
#ifndef USE_LKCA
return false;
#else
size_t mdLength;
size_t counter;
size_t outLength;
uint8_t counterBuf[4];
void *sha384_ctx = NULL;
uint8_t hash_value[LIBSPDM_SHA384_DIGEST_SIZE];
bool status = false;
if (mask == NULL || seed == NULL)
{
return false;
}
// Only support SHA384 for MGF1 now.
if (hash_nid == LIBSPDM_CRYPTO_NID_SHA384)
{
mdLength = LIBSPDM_SHA384_DIGEST_SIZE;
}
else
{
return false;
}
sha384_ctx = libspdm_sha384_new();
if (sha384_ctx == NULL)
{
pr_err("%s : libspdm_sha384_new() failed \n", __FUNCTION__);
return false;
}
for (counter = 0, outLength = 0; outLength < maskedDB_length; counter++)
{
counterBuf[0] = (uint8_t)SHIFT_RIGHT_AND_GET_BYTE(counter, 24);
counterBuf[1] = (uint8_t)SHIFT_RIGHT_AND_GET_BYTE(counter, 16);
counterBuf[2] = (uint8_t)SHIFT_RIGHT_AND_GET_BYTE(counter, 8);
counterBuf[3] = (uint8_t)SHIFT_RIGHT_AND_GET_BYTE(counter, 0);
status = libspdm_sha384_init(sha384_ctx);
if (!status)
{
pr_err("%s: libspdm_sha384_init() failed !! \n", __FUNCTION__);
goto _error_exit;
}
status = libspdm_sha384_update(sha384_ctx, seed, seed_length);
if (!status)
{
pr_err("%s: libspdm_sha384_update() failed(seed) !! \n", __FUNCTION__);
goto _error_exit;
}
status = libspdm_sha384_update(sha384_ctx, counterBuf, 4);
if (!status)
{
pr_err("%s: libspdm_sha384_update() failed(counterBuf) !! \n", __FUNCTION__);
goto _error_exit;
}
if (outLength + mdLength <= maskedDB_length)
{
status = libspdm_sha384_final(sha384_ctx, mask + outLength);
if (!status)
{
pr_err("%s: libspdm_sha384_final() failed (<= maskedDB_length) !! \n", __FUNCTION__);
goto _error_exit;
}
outLength += mdLength;
}
else
{
status = libspdm_sha384_final(sha384_ctx, hash_value);
if (!status)
{
pr_err("%s: libspdm_sha384_final() failed(> maskedDB_length) !! \n", __FUNCTION__);
goto _error_exit;
}
memcpy(mask + outLength, hash_value, maskedDB_length - outLength);
outLength = maskedDB_length;
}
}
status = true;
_error_exit:
libspdm_sha384_free(sha384_ctx);
return status;
#endif
}
/*
0xbc : Trailer Field
+-----------+
| M |
+-----------+
|
V
Hash
|
V
+--------+----------+----------+
M' = |Padding1| mHash | salt |
+--------+----------+----------+
|--------------|---------------|
|
+--------+----------+ V
DB = |Padding2| salt | Hash
+--------+----------+ |
| |
V |
xor <--- MGF <---|
| |
| |
V V
+-------------------+----------+----+
EM = | maskedDB | H |0xbc|
+-------------------+----------+----+
salt : The random number, we hardcode its size as hash size here.
M' : The concatenation of padding1 + message hash + salt
MGF : Mask generation function.
A mask generation function takes an octet string of variable length
and a desired output length as input, and outputs an octet string of
the desired length
MGF1 is a Mask Generation Function based on a hash function.
Padding1 : 8 zeros
Padding2 : 0x01
The detail spec is at https://datatracker.ietf.org/doc/html/rfc2437
*/
/*!
* Set keys and call PKCS1_MGF1 to generate signature.
*
* @param rsa_context : A RSA context pointer
* @param hash_nid : The hash NID
* @param message_hash : The pointer to message hash
* @param signature : The pointer is used to store generated signature
* @param sig_size : For input, a pointer store signature buffer size.
* For output, a pointer store generate signature size.
* @param salt_Length : The salt length for RSA-PSS algorithm
*
* @Return : True if OK; otherwise return False
*/
static bool nvRsaPaddingAddPkcs1PssMgf1
(
void *rsa_context,
size_t hash_nid,
const uint8_t *message_hash,
size_t hash_size,
uint8_t *signature,
size_t *sig_size,
int salt_length
)
{
#ifndef USE_LKCA
return false;
#else
bool status = false;
struct rsa_ctx *ctx = rsa_context;
void *sha384_ctx = NULL;
uint32_t keySize;
uint32_t msBits;
size_t emLength;
uint8_t saltBuf[64];
size_t maskedDB_length;
size_t i;
uint8_t *tmp_H;
uint8_t *tmp_P;
int rc;
unsigned int ret_data_size;
MPI mpi_n = NULL;
MPI mpi_d = NULL;
MPI mpi_c = mpi_alloc(0);
MPI mpi_p = mpi_alloc(0);
// read modulus to BN struct
mpi_n = mpi_read_raw_data(ctx->key.n, ctx->key.n_sz);
if (mpi_n == NULL)
{
pr_err("%s : mpi_n create failed !! \n", __FUNCTION__);
goto _error_exit;
}
// read private exponent to BN struct
mpi_d = mpi_read_raw_data(ctx->key.d, ctx->key.d_sz);
if (mpi_d == NULL)
{
pr_err("%s : mpi_d create failed !! \n", __FUNCTION__);
goto _error_exit;
}
keySize = mpi_n->nbits;
msBits = GET_MOST_SIGNIFICANT_BIT(keySize);
emLength = BITS_TO_BYTES(keySize);
if (msBits == 0)
{
*signature++ = 0;
emLength--;
}
if (emLength < hash_size + 2)
{
pr_err("%s : emLength < hash_size + 2 !! \n", __FUNCTION__);
goto _error_exit;
}
// Now, we only support salt_length == LIBSPDM_SHA384_DIGEST_SIZE
if (salt_length != LIBSPDM_SHA384_DIGEST_SIZE ||
hash_nid != LIBSPDM_CRYPTO_NID_SHA384)
{
pr_err("%s : Invalid salt_length (%x) \n", __FUNCTION__, salt_length);
goto _error_exit;
}
get_random_bytes(saltBuf, salt_length);
maskedDB_length = emLength - hash_size - 1;
tmp_H = signature + maskedDB_length;
sha384_ctx = libspdm_sha384_new();
if (sha384_ctx == NULL)
{
pr_err("%s : libspdm_sha384_new() failed !! \n", __FUNCTION__);
goto _error_exit;
}
status = libspdm_sha384_init(sha384_ctx);
if (!status)
{
pr_err("%s : libspdm_sha384_init() failed !! \n", __FUNCTION__);
goto _error_exit;
}
status = libspdm_sha384_update(sha384_ctx, zeroes, sizeof(zeroes));
if (!status)
{
pr_err("%s : libspdm_sha384_update() with zeros failed !!\n", __FUNCTION__);
goto _error_exit;
}
status = libspdm_sha384_update(sha384_ctx, message_hash, hash_size);
if (!status)
{
pr_err("%s: libspdm_sha384_update() with message_hash failed !!\n", __FUNCTION__);
goto _error_exit;
}
if (salt_length)
{
status = libspdm_sha384_update(sha384_ctx, saltBuf, salt_length);
if (!status)
{
pr_err("%s : libspdm_sha384_update() with saltBuf failed !!\n", __FUNCTION__);
goto _error_exit;
}
}
status = libspdm_sha384_final(sha384_ctx, tmp_H);
if (!status)
{
pr_err("%s : libspdm_sha384_final() with tmp_H failed !!\n", __FUNCTION__);
goto _error_exit;
}
/* Generate dbMask in place then perform XOR on it */
status = NV_PKCS1_MGF1(signature, maskedDB_length, tmp_H, hash_size, hash_nid);
if (!status)
{
pr_err("%s : NV_PKCS1_MGF1() failed \n", __FUNCTION__);
goto _error_exit;
}
tmp_P = signature;
tmp_P += emLength - salt_length - hash_size - 2;
*tmp_P++ ^= 0x1;
if (salt_length > 0)
{
for (i = 0; i < salt_length; i++)
{
*tmp_P++ ^= saltBuf[i];
}
}
if (msBits)
{
signature[0] &= 0xFF >> (8 - msBits);
}
/* H is already in place so just set final 0xbc */
signature[emLength - 1] = RSA_PSS_TRAILER_FIELD;
// read signature to BN struct
mpi_p = mpi_read_raw_data(signature, emLength);
if (mpi_p == NULL)
{
pr_err("%s : mpi_p() create failed !!\n", __FUNCTION__);
goto _error_exit;
}
// Staring RSA encryption with private key over signature.
rc = mpi_powm(mpi_c, mpi_p, mpi_d, mpi_n);
if (rc != 0)
{
pr_err("%s : mpi_powm() failed \n", __FUNCTION__);
goto _error_exit;
}
rc = mpi_read_buffer(mpi_c, signature, *sig_size, &ret_data_size, NULL);
if (rc != 0)
{
pr_err("%s : mpi_read_buffer() failed \n", __FUNCTION__);
goto _error_exit;
}
if (ret_data_size > *sig_size)
{
goto _error_exit;
}
*sig_size = ret_data_size;
status = true;
_error_exit:
mpi_free(mpi_n);
mpi_free(mpi_d);
mpi_free(mpi_c);
mpi_free(mpi_p);
libspdm_sha384_free(sha384_ctx);
return status;
#endif
}
/*!
* Perform RSA-PSS sigaature sign process with LKCA library.
*
* @param rsa_context : A RSA context pointer
* @param hash_nid : The hash NID
* @param message_hash : The pointer to message hash
* @param signature : The pointer is used to store generated signature
* @param sig_size : For input, a pointer store signature buffer size.
* For output, a pointer store generate signature size.
*
* @Return : True if OK; otherwise return False
*/
bool lkca_rsa_pss_sign
(
void *rsa_context,
size_t hash_nid,
const uint8_t *message_hash,
size_t hash_size,
uint8_t *signature,
size_t *sig_size
)
{
#ifndef USE_LKCA
return true;
#else
return nvRsaPaddingAddPkcs1PssMgf1(rsa_context,
hash_nid,
message_hash,
hash_size,
signature,
sig_size,
LIBSPDM_SHA384_DIGEST_SIZE);
#endif
}

85
kernel-open/nvidia/libspdm_rsa_ext.c Normal file
View File

@@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Comments, prototypes and checks taken from DMTF: Copyright 2021-2022 DMTF. All rights reserved.
* License: BSD 3-Clause License. For full text see link: https://github.com/DMTF/libspdm/blob/main/LICENSE.md
*/
/** @file
* RSA Asymmetric Cipher Wrapper Implementation.
*
* This file implements following APIs which provide more capabilities for RSA:
* 1) rsa_pss_sign
*
* RFC 8017 - PKCS #1: RSA Cryptography Specifications version 2.2
**/
#include "internal_crypt_lib.h"
#include "library/cryptlib.h"
/**
* Carries out the RSA-PSS signature generation with EMSA-PSS encoding scheme.
*
* This function carries out the RSA-PSS signature generation with EMSA-PSS encoding scheme defined in
* RSA PKCS#1 v2.2.
*
* The salt length is same as digest length.
*
* If the signature buffer is too small to hold the contents of signature, false
* is returned and sig_size is set to the required buffer size to obtain the signature.
*
* If rsa_context is NULL, then return false.
* If message_hash is NULL, then return false.
* If hash_size need match the hash_nid. nid could be SHA256, SHA384, SHA512, SHA3_256, SHA3_384, SHA3_512.
* If sig_size is large enough but signature is NULL, then return false.
*
* @param[in] rsa_context Pointer to RSA context for signature generation.
* @param[in] hash_nid hash NID
* @param[in] message_hash Pointer to octet message hash to be signed.
* @param[in] hash_size size of the message hash in bytes.
* @param[out] signature Pointer to buffer to receive RSA-SSA PSS signature.
* @param[in, out] sig_size On input, the size of signature buffer in bytes.
* On output, the size of data returned in signature buffer in bytes.
*
* @retval true signature successfully generated in RSA-SSA PSS.
* @retval false signature generation failed.
* @retval false sig_size is too small.
*
**/
bool libspdm_rsa_pss_sign(void *rsa_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
uint8_t *signature, size_t *sig_size)
{
return lkca_rsa_pss_sign(rsa_context, hash_nid, message_hash, hash_size,
signature, sig_size);
}
//
// In RM, we just need sign process; so we stub verification function.
// Verification function is needed in GSP code only,
//
bool libspdm_rsa_pss_verify(void *rsa_context, size_t hash_nid,
const uint8_t *message_hash, size_t hash_size,
const uint8_t *signature, size_t sig_size)
{
return false;
}

153
kernel-open/nvidia/nv-caps-imex.c Normal file
View File

@@ -0,0 +1,153 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include "nv-linux.h"
extern int NVreg_ImexChannelCount;
static int nv_caps_imex_open(struct inode *inode, struct file *file)
{
return 0;
}
static int nv_caps_imex_release(struct inode *inode, struct file *file)
{
return 0;
}
static struct file_operations g_nv_caps_imex_fops =
{
.owner = THIS_MODULE,
.open = nv_caps_imex_open,
.release = nv_caps_imex_release
};
struct
{
NvBool initialized;
struct cdev cdev;
dev_t devno;
} g_nv_caps_imex;
int NV_API_CALL nv_caps_imex_channel_get(int fd)
{
#if NV_FILESYSTEM_ACCESS_AVAILABLE
struct file *file;
struct inode *inode;
int channel = -1;
file = fget(fd);
if (file == NULL)
{
return channel;
}
inode = NV_FILE_INODE(file);
if (inode == NULL)
{
goto out;
}
/* Make sure the fd belongs to the nv-caps-imex-drv */
if (file->f_op != &g_nv_caps_imex_fops)
{
goto out;
}
/* minor number is same as channel */
channel = MINOR(inode->i_rdev);
out:
fput(file);
return channel;
#else
return -1;
#endif
}
int NV_API_CALL nv_caps_imex_channel_count(void)
{
return NVreg_ImexChannelCount;
}
int NV_API_CALL nv_caps_imex_init(void)
{
int rc;
if (g_nv_caps_imex.initialized)
{
nv_printf(NV_DBG_ERRORS, "nv-caps-imex is already initialized.\n");
return -EBUSY;
}
if (NVreg_ImexChannelCount == 0)
{
nv_printf(NV_DBG_INFO, "nv-caps-imex is disabled.\n");
return 0;
}
rc = alloc_chrdev_region(&g_nv_caps_imex.devno, 0,
NVreg_ImexChannelCount,
"nvidia-caps-imex-channels");
if (rc < 0)
{
nv_printf(NV_DBG_ERRORS, "nv-caps-imex failed to create cdev.\n");
return rc;
}
cdev_init(&g_nv_caps_imex.cdev, &g_nv_caps_imex_fops);
g_nv_caps_imex.cdev.owner = THIS_MODULE;
rc = cdev_add(&g_nv_caps_imex.cdev, g_nv_caps_imex.devno,
NVreg_ImexChannelCount);
if (rc < 0)
{
nv_printf(NV_DBG_ERRORS, "nv-caps-imex failed to add cdev.\n");
goto cdev_add_fail;
}
g_nv_caps_imex.initialized = NV_TRUE;
return 0;
cdev_add_fail:
unregister_chrdev_region(g_nv_caps_imex.devno, NVreg_ImexChannelCount);
return rc;
}
void NV_API_CALL nv_caps_imex_exit(void)
{
if (!g_nv_caps_imex.initialized)
{
return;
}
cdev_del(&g_nv_caps_imex.cdev);
unregister_chrdev_region(g_nv_caps_imex.devno, NVreg_ImexChannelCount);
g_nv_caps_imex.initialized = NV_FALSE;
}

34
kernel-open/nvidia/nv-caps-imex.h Normal file
View File

@@ -0,0 +1,34 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _NV_CAPS_IMEX_H_
#define _NV_CAPS_IMEX_H_
#include <nv-kernel-interface-api.h>
int NV_API_CALL nv_caps_imex_init(void);
void NV_API_CALL nv_caps_imex_exit(void);
int NV_API_CALL nv_caps_imex_channel_get(int fd);
int NV_API_CALL nv_caps_imex_channel_count(void);
#endif /* _NV_CAPS_IMEX_H_ */

7
kernel-open/nvidia/nv-mmap.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -577,12 +577,9 @@ int nvidia_mmap_helper(
//
// This path is similar to the sysmem mapping code.
// TODO: Refactor is needed as part of bug#2001704.
// Use pfn_valid to determine whether the physical address has
// backing struct page. This is used to isolate P8 from P9.
//
if ((nv_get_numa_status(nvl) == NV_NUMA_STATUS_ONLINE) &&
!IS_REG_OFFSET(nv, access_start, access_len) &&
(pfn_valid(PFN_DOWN(mmap_start))))
!IS_REG_OFFSET(nv, access_start, access_len))
{
ret = nvidia_mmap_numa(vma, mmap_context);
if (ret)

41
kernel-open/nvidia/nv-reg.h
View File

@@ -839,6 +839,45 @@
#define __NV_ENABLE_NONBLOCKING_OPEN EnableNonblockingOpen
#define NV_ENABLE_NONBLOCKING_OPEN NV_REG_STRING(__NV_ENABLE_NONBLOCKING_OPEN)
/*
* Option: NVreg_ImexChannelCount
*
* Description:
*
* This option allows users to specify the number of IMEX (import/export)
* channels. Within an IMEX domain, the channels allow sharing memory
* securely in a multi-user environment using the CUDA driver's fabric handle
* based APIs.
*
* An IMEX domain is either an OS instance or a group of securely
* connected OS instances using the NVIDIA IMEX daemon. The option must
* be set to the same value on each OS instance within the IMEX domain.
*
* An IMEX channel is a logical entity that is represented by a /dev node.
* The IMEX channels are global resources within the IMEX domain. When
* exporter and importer CUDA processes have been granted access to the
* same IMEX channel, they can securely share memory.
*
* Note that the NVIDIA driver will not attempt to create the /dev nodes. Thus,
* the related CUDA APIs will fail with an insufficient permission error until
* the /dev nodes are set up. The creation of these /dev nodes,
* /dev/nvidia-caps-imex-channels/channelN, must be handled by the
* administrator, where N is the minor number. The major number can be
* queried from /proc/devices.
*
* nvidia-modprobe CLI support is available to set up the /dev nodes.
* NVreg_ModifyDeviceFiles, NVreg_DeviceFileGID, NVreg_DeviceFileUID
* and NVreg_DeviceFileMode will be honored by nvidia-modprobe.
*
* Possible values:
* 0 - Disable IMEX using CUDA driver's fabric handles.
* N - N IMEX channels will be enabled in the driver to facilitate N
* concurrent users. Default value is 2048 channels, and the current
* maximum value is 20-bit, same as Linux dev_t's minor number limit.
*/
#define __NV_IMEX_CHANNEL_COUNT ImexChannelCount
#define NV_REG_IMEX_CHANNEL_COUNT NV_REG_STRING(__NV_IMEX_CHANNEL_COUNT)
#if defined(NV_DEFINE_REGISTRY_KEY_TABLE)
/*
@@ -887,6 +926,7 @@ NV_DEFINE_REG_STRING_ENTRY(__NV_TEMPORARY_FILE_PATH, NULL);
NV_DEFINE_REG_STRING_ENTRY(__NV_EXCLUDED_GPUS, NULL);
NV_DEFINE_REG_ENTRY(__NV_DMA_REMAP_PEER_MMIO, NV_DMA_REMAP_PEER_MMIO_ENABLE);
NV_DEFINE_REG_STRING_ENTRY(__NV_RM_NVLINK_BW, NULL);
NV_DEFINE_REG_ENTRY_GLOBAL(__NV_IMEX_CHANNEL_COUNT, 2048);
/*
*----------------registry database definition----------------------
@@ -933,6 +973,7 @@ nv_parm_t nv_parms[] = {
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_DBG_BREAKPOINT),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OPENRM_ENABLE_UNSUPPORTED_GPUS),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_DMA_REMAP_PEER_MMIO),
NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_IMEX_CHANNEL_COUNT),
{NULL, NULL}
};

147
kernel-open/nvidia/nv.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -55,6 +55,7 @@
#include "nv-kthread-q.h"
#include "nv-pat.h"
#include "nv-dmabuf.h"
#include "nv-caps-imex.h"
#if !defined(CONFIG_RETPOLINE)
#include "nv-retpoline.h"
@@ -825,11 +826,18 @@ static int __init nvidia_init_module(void)
goto procfs_exit;
}
rc = nv_caps_imex_init();
if (rc < 0)
{
nv_printf(NV_DBG_ERRORS, "NVRM: failed to initialize IMEX channels.\n");
goto caps_root_exit;
}
rc = nv_module_init(&sp);
if (rc < 0)
{
nv_printf(NV_DBG_ERRORS, "NVRM: failed to initialize module.\n");
goto caps_root_exit;
goto caps_imex_exit;
}
count = nvos_count_devices();
@@ -941,6 +949,9 @@ drivers_exit:
module_exit:
nv_module_exit(sp);
caps_imex_exit:
nv_caps_imex_exit();
caps_root_exit:
nv_caps_root_exit();
@@ -967,6 +978,8 @@ static void __exit nvidia_exit_module(void)
nv_module_exit(sp);
nv_caps_imex_exit();
nv_caps_root_exit();
nv_procfs_exit();
@@ -2040,7 +2053,7 @@ nvidia_close_callback(
{
nv_linux_state_t *nvl;
nv_state_t *nv;
nvidia_stack_t *sp;
nvidia_stack_t *sp = nvlfp->sp;
NvBool bRemove = NV_FALSE;
nvl = nvlfp->nvptr;
@@ -2052,12 +2065,11 @@ nvidia_close_callback(
*/
nv_free_file_private(nvlfp);
nv_kmem_cache_free_stack(nvlfp->sp);
nv_kmem_cache_free_stack(sp);
return;
}
nv = NV_STATE_PTR(nvl);
sp = nvlfp->sp;
rm_cleanup_file_private(sp, nv, &nvlfp->nvfp);
@@ -6050,6 +6062,131 @@ failed:
return NV_ERR_NOT_SUPPORTED;
}
void NV_API_CALL nv_get_screen_info(
nv_state_t *nv,
NvU64 *pPhysicalAddress,
NvU32 *pFbWidth,
NvU32 *pFbHeight,
NvU32 *pFbDepth,
NvU32 *pFbPitch,
NvU64 *pFbSize
)
{
*pPhysicalAddress = 0;
*pFbWidth = *pFbHeight = *pFbDepth = *pFbPitch = *pFbSize = 0;
#if defined(CONFIG_FB) && defined(NV_NUM_REGISTERED_FB_PRESENT)
if (num_registered_fb > 0)
{
int i;
for (i = 0; i < num_registered_fb; i++)
{
if (!registered_fb[i])
continue;
/* Make sure base address is mapped to GPU BAR */
if (NV_IS_CONSOLE_MAPPED(nv, registered_fb[i]->fix.smem_start))
{
*pPhysicalAddress = registered_fb[i]->fix.smem_start;
*pFbWidth = registered_fb[i]->var.xres;
*pFbHeight = registered_fb[i]->var.yres;
*pFbDepth = registered_fb[i]->var.bits_per_pixel;
*pFbPitch = registered_fb[i]->fix.line_length;
*pFbSize = (NvU64)(*pFbHeight) * (NvU64)(*pFbPitch);
return;
}
}
}
#endif
/*
* If the screen info is not found in the registered FBs then fallback
* to the screen_info structure.
*
* The SYSFB_SIMPLEFB option, if enabled, marks VGA/VBE/EFI framebuffers as
* generic framebuffers so the new generic system-framebuffer drivers can
* be used instead. DRM_SIMPLEDRM drives the generic system-framebuffers
* device created by SYSFB_SIMPLEFB.
*
* SYSFB_SIMPLEFB registers a dummy framebuffer which does not contain the
* information required by nv_get_screen_info(), therefore you need to
* fall back onto the screen_info structure.
*
* After commit b8466fe82b79 ("efi: move screen_info into efi init code")
* in v6.7, 'screen_info' is exported as GPL licensed symbol for ARM64.
*/
#if NV_CHECK_EXPORT_SYMBOL(screen_info)
/*
* If there is not a framebuffer console, return 0 size.
*
* orig_video_isVGA is set to 1 during early Linux kernel
* initialization, and then will be set to a value, such as
* VIDEO_TYPE_VLFB or VIDEO_TYPE_EFI if an fbdev console is used.
*/
if (screen_info.orig_video_isVGA > 1)
{
NvU64 physAddr = screen_info.lfb_base;
#if defined(VIDEO_CAPABILITY_64BIT_BASE)
physAddr |= (NvU64)screen_info.ext_lfb_base << 32;
#endif
/* Make sure base address is mapped to GPU BAR */
if (NV_IS_CONSOLE_MAPPED(nv, physAddr))
{
*pPhysicalAddress = physAddr;
*pFbWidth = screen_info.lfb_width;
*pFbHeight = screen_info.lfb_height;
*pFbDepth = screen_info.lfb_depth;
*pFbPitch = screen_info.lfb_linelength;
*pFbSize = (NvU64)(*pFbHeight) * (NvU64)(*pFbPitch);
}
}
#else
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
struct pci_dev *pci_dev = nvl->pci_dev;
int i;
if (pci_dev == NULL)
return;
BUILD_BUG_ON(NV_GPU_BAR_INDEX_IMEM != NV_GPU_BAR_INDEX_FB + 1);
for (i = NV_GPU_BAR_INDEX_FB; i <= NV_GPU_BAR_INDEX_IMEM; i++)
{
int bar_index = nv_bar_index_to_os_bar_index(pci_dev, i);
struct resource *gpu_bar_res = &pci_dev->resource[bar_index];
struct resource *res = gpu_bar_res->child;
/*
* Console resource will become child resource of pci-dev resource.
* Check if child resource start address matches with expected
* console start address.
*/
if ((res != NULL) &&
NV_IS_CONSOLE_MAPPED(nv, res->start))
{
NvU32 res_name_len = strlen(res->name);
/*
* The resource name ends with 'fb' (efifb, vesafb, etc.).
* For simple-framebuffer, the resource name is 'BOOTFB'.
* Confirm if the resources name either ends with 'fb' or 'FB'.
*/
if ((res_name_len > 2) &&
!strcasecmp((res->name + res_name_len - 2), "fb"))
{
*pPhysicalAddress = res->start;
*pFbSize = resource_size(res);
return;
}
}
}
}
#endif
}
module_init(nvidia_init_module);
module_exit(nvidia_exit_module);

6
kernel-open/nvidia/nv_gpu_ops.h
View File

@@ -279,9 +279,11 @@ NV_STATUS nvGpuOpsPagingChannelPushStream(UvmGpuPagingChannel *channel,
char *methodStream,
NvU32 methodStreamSize);
NV_STATUS nvGpuOpsFlushReplayableFaultBuffer(struct gpuDevice *device);
NV_STATUS nvGpuOpsFlushReplayableFaultBuffer(gpuFaultInfo *pFaultInfo,
NvBool bCopyAndFlush);
NV_STATUS nvGpuOpsTogglePrefetchFaults(gpuFaultInfo *pFaultInfo, NvBool bEnable);
NV_STATUS nvGpuOpsTogglePrefetchFaults(gpuFaultInfo *pFaultInfo,
NvBool bEnable);
// Interface used for CCSL

14
kernel-open/nvidia/nv_uvm_interface.c
View File

@@ -985,24 +985,30 @@ NV_STATUS nvUvmInterfaceGetNonReplayableFaults(UvmGpuFaultInfo *pFaultInfo,
}
EXPORT_SYMBOL(nvUvmInterfaceGetNonReplayableFaults);
NV_STATUS nvUvmInterfaceFlushReplayableFaultBuffer(uvmGpuDeviceHandle device)
NV_STATUS nvUvmInterfaceFlushReplayableFaultBuffer(UvmGpuFaultInfo *pFaultInfo,
NvBool bCopyAndFlush)
{
nvidia_stack_t *sp = nvUvmGetSafeStack();
NV_STATUS status;
status = rm_gpu_ops_flush_replayable_fault_buffer(sp, (gpuDeviceHandle)device);
status = rm_gpu_ops_flush_replayable_fault_buffer(sp,
pFaultInfo,
bCopyAndFlush);
nvUvmFreeSafeStack(sp);
return status;
}
EXPORT_SYMBOL(nvUvmInterfaceFlushReplayableFaultBuffer);
NV_STATUS nvUvmInterfaceTogglePrefetchFaults(UvmGpuFaultInfo *pFaultInfo, NvBool bEnable)
NV_STATUS nvUvmInterfaceTogglePrefetchFaults(UvmGpuFaultInfo *pFaultInfo,
NvBool bEnable)
{
nvidia_stack_t *sp = nvUvmGetSafeStack();
NV_STATUS status;
status = rm_gpu_ops_toggle_prefetch_faults(sp, pFaultInfo, bEnable);
status = rm_gpu_ops_toggle_prefetch_faults(sp,
pFaultInfo,
bEnable);
nvUvmFreeSafeStack(sp);
return status;

3
kernel-open/nvidia/nvidia-sources.Kbuild
View File

@@ -30,18 +30,21 @@ NVIDIA_SOURCES += nvidia/nv-report-err.c
NVIDIA_SOURCES += nvidia/nv-rsync.c
NVIDIA_SOURCES += nvidia/nv-msi.c
NVIDIA_SOURCES += nvidia/nv-caps.c
NVIDIA_SOURCES += nvidia/nv-caps-imex.c
NVIDIA_SOURCES += nvidia/nv_uvm_interface.c
NVIDIA_SOURCES += nvidia/libspdm_aead.c
NVIDIA_SOURCES += nvidia/libspdm_ecc.c
NVIDIA_SOURCES += nvidia/libspdm_hkdf.c
NVIDIA_SOURCES += nvidia/libspdm_rand.c
NVIDIA_SOURCES += nvidia/libspdm_shash.c
NVIDIA_SOURCES += nvidia/libspdm_rsa.c
NVIDIA_SOURCES += nvidia/libspdm_aead_aes_gcm.c
NVIDIA_SOURCES += nvidia/libspdm_sha.c
NVIDIA_SOURCES += nvidia/libspdm_hmac_sha.c
NVIDIA_SOURCES += nvidia/libspdm_hkdf_sha.c
NVIDIA_SOURCES += nvidia/libspdm_ec.c
NVIDIA_SOURCES += nvidia/libspdm_x509.c
NVIDIA_SOURCES += nvidia/libspdm_rsa_ext.c
NVIDIA_SOURCES += nvidia/nvlink_linux.c
NVIDIA_SOURCES += nvidia/nvlink_caps.c
NVIDIA_SOURCES += nvidia/linux_nvswitch.c

103
kernel-open/nvidia/os-interface.c
View File

@@ -25,6 +25,7 @@
#include "os-interface.h"
#include "nv-linux.h"
#include "nv-caps-imex.h"
#include "nv-time.h"
@@ -59,6 +60,8 @@ NvBool os_dma_buf_enabled = NV_TRUE;
NvBool os_dma_buf_enabled = NV_FALSE;
#endif // CONFIG_DMA_SHARED_BUFFER
NvBool os_imex_channel_is_supported = NV_TRUE;
void NV_API_CALL os_disable_console_access(void)
{
console_lock();
@@ -1231,90 +1234,6 @@ NvBool NV_API_CALL os_is_efi_enabled(void)
return efi_enabled(EFI_BOOT);
}
void NV_API_CALL os_get_screen_info(
NvU64 *pPhysicalAddress,
NvU32 *pFbWidth,
NvU32 *pFbHeight,
NvU32 *pFbDepth,
NvU32 *pFbPitch,
NvU64 consoleBar1Address,
NvU64 consoleBar2Address
)
{
*pPhysicalAddress = 0;
*pFbWidth = *pFbHeight = *pFbDepth = *pFbPitch = 0;
#if defined(CONFIG_FB) && defined(NV_NUM_REGISTERED_FB_PRESENT)
if (num_registered_fb > 0)
{
int i;
for (i = 0; i < num_registered_fb; i++)
{
if (!registered_fb[i])
continue;
/* Make sure base address is mapped to GPU BAR */
if ((registered_fb[i]->fix.smem_start == consoleBar1Address) ||
(registered_fb[i]->fix.smem_start == consoleBar2Address))
{
*pPhysicalAddress = registered_fb[i]->fix.smem_start;
*pFbWidth = registered_fb[i]->var.xres;
*pFbHeight = registered_fb[i]->var.yres;
*pFbDepth = registered_fb[i]->var.bits_per_pixel;
*pFbPitch = registered_fb[i]->fix.line_length;
return;
}
}
}
#endif
/*
* If the screen info is not found in the registered FBs then fallback
* to the screen_info structure.
*
* The SYSFB_SIMPLEFB option, if enabled, marks VGA/VBE/EFI framebuffers as
* generic framebuffers so the new generic system-framebuffer drivers can
* be used instead. DRM_SIMPLEDRM drives the generic system-framebuffers
* device created by SYSFB_SIMPLEFB.
*
* SYSFB_SIMPLEFB registers a dummy framebuffer which does not contain the
* information required by os_get_screen_info(), therefore you need to
* fall back onto the screen_info structure.
*
* After commit b8466fe82b79 ("efi: move screen_info into efi init code")
* in v6.7, 'screen_info' is exported as GPL licensed symbol for ARM64.
*/
#if NV_CHECK_EXPORT_SYMBOL(screen_info)
/*
* If there is not a framebuffer console, return 0 size.
*
* orig_video_isVGA is set to 1 during early Linux kernel
* initialization, and then will be set to a value, such as
* VIDEO_TYPE_VLFB or VIDEO_TYPE_EFI if an fbdev console is used.
*/
if (screen_info.orig_video_isVGA > 1)
{
NvU64 physAddr = screen_info.lfb_base;
#if defined(VIDEO_CAPABILITY_64BIT_BASE)
physAddr |= (NvU64)screen_info.ext_lfb_base << 32;
#endif
/* Make sure base address is mapped to GPU BAR */
if ((physAddr == consoleBar1Address) ||
(physAddr == consoleBar2Address))
{
*pPhysicalAddress = physAddr;
*pFbWidth = screen_info.lfb_width;
*pFbHeight = screen_info.lfb_height;
*pFbDepth = screen_info.lfb_depth;
*pFbPitch = screen_info.lfb_linelength;
}
}
#endif
}
void NV_API_CALL os_dump_stack(void)
{
dump_stack();
@@ -2182,6 +2101,22 @@ void NV_API_CALL os_nv_cap_close_fd
nv_cap_close_fd(fd);
}
NvS32 NV_API_CALL os_imex_channel_count
(
void
)
{
return nv_caps_imex_channel_count();
}
NvS32 NV_API_CALL os_imex_channel_get
(
NvU64 descriptor
)
{
return nv_caps_imex_channel_get((int)descriptor);
}
/*
* Reads the total memory and free memory of a NUMA node from the kernel.
*/

68
kernel-open/nvidia/os-mlock.c
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -87,59 +87,10 @@ static NV_STATUS get_io_ptes(struct vm_area_struct *vma,
return NV_OK;
}
/*!
* @brief Pins user IO pages that have been mapped to the user processes virtual
* address space with remap_pfn_range.
*
* @param[in] vma VMA that contains the virtual address range given by the
* start and the page count.
* @param[in] start Beginning of the virtual address range of the IO pages.
* @param[in] page_count Number of pages to pin from start.
* @param[in,out] page_array Storage array for pointers to the pinned pages.
* Must be large enough to contain at least page_count
* pointers.
*
* @return NV_OK if the pages were pinned successfully, error otherwise.
*/
static NV_STATUS get_io_pages(struct vm_area_struct *vma,
NvUPtr start,
NvU64 page_count,
struct page **page_array)
{
NV_STATUS rmStatus = NV_OK;
NvU64 i, pinned = 0;
unsigned long pfn;
for (i = 0; i < page_count; i++)
{
if ((nv_follow_pfn(vma, (start + (i * PAGE_SIZE)), &pfn) < 0) ||
(!pfn_valid(pfn)))
{
rmStatus = NV_ERR_INVALID_ADDRESS;
break;
}
// Page-backed memory mapped to userspace with remap_pfn_range
page_array[i] = pfn_to_page(pfn);
get_page(page_array[i]);
pinned++;
}
if (pinned < page_count)
{
for (i = 0; i < pinned; i++)
put_page(page_array[i]);
rmStatus = NV_ERR_INVALID_ADDRESS;
}
return rmStatus;
}
NV_STATUS NV_API_CALL os_lookup_user_io_memory(
void *address,
NvU64 page_count,
NvU64 **pte_array,
void **page_array
NvU64 **pte_array
)
{
NV_STATUS rmStatus;
@@ -187,18 +138,9 @@ NV_STATUS NV_API_CALL os_lookup_user_io_memory(
goto done;
}
if (pfn_valid(pfn))
{
rmStatus = get_io_pages(vma, start, page_count, (struct page **)result_array);
if (rmStatus == NV_OK)
*page_array = (void *)result_array;
}
else
{
rmStatus = get_io_ptes(vma, start, page_count, (NvU64 **)result_array);
if (rmStatus == NV_OK)
*pte_array = (NvU64 *)result_array;
}
rmStatus = get_io_ptes(vma, start, page_count, (NvU64 **)result_array);
if (rmStatus == NV_OK)
*pte_array = (NvU64 *)result_array;
done:
nv_mmap_read_unlock(mm);