580.65.06

2026-01-27 11:39:46 +00:00 · 2025-08-04 11:15:02 -07:00
parent d890313300
commit 307159f262
1315 changed files with 477791 additions and 279973 deletions
--- a/kernel-open/nvidia/os-interface.c
+++ b/kernel-open/nvidia/os-interface.c
@@ -25,6 +25,7 @@

 #include "os-interface.h"
 #include "nv-linux.h"
+#include <linux/iommu.h>
 #include "nv-caps-imex.h"

 #include "nv-platform.h"
@@ -34,6 +35,7 @@
 #include <linux/mmzone.h>
 #include <linux/numa.h>
 #include <linux/cpuset.h>
+#include <linux/sys_soc.h>

 #include <linux/pid.h>
 #include <linux/pid_namespace.h>
@@ -51,9 +53,11 @@ extern nv_linux_state_t nv_ctl_device;

 extern nv_kthread_q_t nv_kthread_q;

-NvU32 os_page_size  = PAGE_SIZE;
-NvU64 os_page_mask  = NV_PAGE_MASK;
-NvU8  os_page_shift = PAGE_SHIFT;
+NvU64 os_page_size     = PAGE_SIZE;
+NvU64 os_max_page_size = PAGE_SIZE << NV_MAX_PAGE_ORDER;
+NvU64 os_page_mask     = NV_PAGE_MASK;
+NvU8  os_page_shift    = PAGE_SHIFT;
+
 NvBool os_cc_enabled = 0;
 NvBool os_cc_sev_snp_enabled = 0;
 NvBool os_cc_snp_vtom_enabled = 0;
@@ -474,11 +478,6 @@ void *NV_API_CALL os_mem_copy(
     * When performing memcpy for memory mapped as device, memcpy_[to/from]io
     * must be used. WAR to check the source and destination to determine the
     * correct memcpy_io to use.
-     *
-     * This WAR is limited to just aarch64 for now because the address range used
-     * to map ioremap and vmalloc is different on ppc64le, and is_vmalloc_addr()
-     * does not correctly handle this. is_ioremap_addr() is needed instead. This
-     * will have to be addressed when reorganizing RM to use the new memset model.
     */
    if (is_vmalloc_addr(dst) && !is_vmalloc_addr(src))
    {
@@ -548,11 +547,6 @@ void* NV_API_CALL os_mem_set(
     *
     * WAR to check the destination to determine if the memory is of type Device
     * or Normal, and use the correct memset.
-     *
-     * This WAR is limited to just aarch64 for now because the address range used
-     * to map ioremap and vmalloc is different on ppc64le, and is_vmalloc_addr()
-     * does not correctly handle this. is_ioremap_addr() is needed instead. This
-     * will have to be addressed when reorganizing RM to use the new memset model.
     */
    if (is_vmalloc_addr(dst))
    {
@@ -676,11 +670,11 @@ void NV_API_CALL os_free_mem(void *address)

 /*****************************************************************************
 *
-*   Name: osGetCurrentTime
+*   Name: osGetSystemTime
 *
 *****************************************************************************/

-NV_STATUS NV_API_CALL os_get_current_time(
+NV_STATUS NV_API_CALL os_get_system_time(
    NvU32 *seconds,
    NvU32 *useconds
 )
@@ -698,16 +692,14 @@ NV_STATUS NV_API_CALL os_get_current_time(
 //
 // Get the High resolution tick count of the system uptime
 //
-NvU64 NV_API_CALL os_get_current_tick_hr(void)
+NvU64 NV_API_CALL os_get_monotonic_time_ns_hr(void)
 {
    struct timespec64 tm;
    ktime_get_raw_ts64(&tm);
    return (NvU64) timespec64_to_ns(&tm);
 }

-#if BITS_PER_LONG >= 64
-
-NvU64 NV_API_CALL os_get_current_tick(void)
+NvU64 NV_API_CALL os_get_monotonic_time_ns(void)
 {
 #if defined(NV_JIFFIES_TO_TIMESPEC_PRESENT)
    struct timespec ts;
@@ -720,47 +712,13 @@ NvU64 NV_API_CALL os_get_current_tick(void)
 #endif
 }

-NvU64 NV_API_CALL os_get_tick_resolution(void)
+NvU64 NV_API_CALL os_get_monotonic_tick_resolution_ns(void)
 {
    return (NvU64)jiffies_to_usecs(1) * NSEC_PER_USEC;
 }

-#else
-
-NvU64 NV_API_CALL os_get_current_tick(void)
-{
-    /*
-     * 'jiffies' overflows regularly on 32-bit builds (unsigned long is 4 bytes
-     * instead of 8 bytes), so it's unwise to build a tick counter on it, since
-     * the rest of the Resman assumes the 'tick' returned from this function is
-     * monotonically increasing and never overflows.
-     *
-     * Instead, use the previous implementation that we've lived with since the
-     * beginning, which uses system clock time to calculate the tick. This is
-     * subject to problems if the system clock time changes dramatically
-     * (more than a second or so) while the Resman is actively tracking a
-     * timeout.
-     */
-    NvU32 seconds, useconds;
-
-    (void) os_get_current_time(&seconds, &useconds);
-
-    return ((NvU64)seconds * NSEC_PER_SEC +
-                 (NvU64)useconds * NSEC_PER_USEC);
-}
-
-NvU64 NV_API_CALL os_get_tick_resolution(void)
-{
-    /*
-     * os_get_current_tick() uses os_get_current_time(), which has
-     * microsecond resolution.
-     */
-    return 1000ULL;
-}
-
-#endif
-
 //---------------------------------------------------------------------------
+
 //
 //  Misc services.
 //
@@ -806,6 +764,50 @@ void NV_API_CALL os_get_current_process_name(char *buf, NvU32 len)
    task_unlock(current);
 }

+NV_STATUS NV_API_CALL os_iommu_sva_bind(void *arg, void **handle, NvU32 *pasid)
+{
+    nv_state_t *nv = arg;
+#if defined(CONFIG_IOMMU_SVA) && \
+    (defined(NV_IOASID_GET_PRESENT) || defined(NV_MM_PASID_DROP_PRESENT))
+    nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+    struct iommu_sva *sva_handle;
+
+    if (pasid == NULL || handle == NULL)
+        return NV_ERR_INVALID_ARGUMENT;
+
+    *pasid = 0;
+    *handle = NULL;
+
+    if (nv->ats_support && current && current->mm)
+    {
+#if defined(NV_IOMMU_SVA_BIND_DEVICE_HAS_DRVDATA_ARG)
+        sva_handle = iommu_sva_bind_device(nvl->dev, current->mm, NULL);
+#else
+        sva_handle = iommu_sva_bind_device(nvl->dev, current->mm);
+#endif
+        if (!IS_ERR(sva_handle))
+        {
+            *pasid = iommu_sva_get_pasid(sva_handle);
+            *handle = sva_handle;
+            NV_DEV_PRINTF(NV_DBG_INFO, nv, "PASID: %u\n", *pasid);
+
+            return NV_OK;
+        }
+    }
+#endif
+    NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "IOMMU SVA bind failed\n");
+
+    return NV_ERR_INVALID_STATE;
+}
+
+void NV_API_CALL os_iommu_sva_unbind(void *handle)
+{
+#if defined(CONFIG_IOMMU_SVA) && \
+    (defined(NV_IOASID_GET_PRESENT) || defined(NV_MM_PASID_DROP_PRESENT))
+    iommu_sva_unbind_device(handle);
+#endif
+}
+
 NV_STATUS NV_API_CALL os_get_current_thread(NvU64 *threadId)
 {
    if (in_interrupt())
@@ -1087,8 +1089,6 @@ void NV_API_CALL os_flush_cpu_write_combine_buffer(void)
 {
 #if defined(NVCPU_X86_64)
    asm volatile("sfence" ::: "memory");
-#elif defined(NVCPU_PPC64LE)
-    __asm__ __volatile__ ("sync" : : : "memory");
 #elif defined(NVCPU_AARCH64)
    asm volatile("dsb st" : : : "memory");
 #else
@@ -1247,8 +1247,6 @@ void NV_API_CALL os_dbg_breakpoint(void)
    __asm__ __volatile__ (".word %c0" :: "i" (KGDB_COMPILED_BREAK));
  #elif defined(NVCPU_AARCH64)
    # warning "Need to implement os_dbg_breakpoint() for aarch64"
-  #elif defined(NVCPU_PPC64LE)
-    __asm__ __volatile__ ("trap");
  #endif // NVCPU_*
 #elif defined(CONFIG_KDB)
    KDB_ENTER();
@@ -1515,7 +1513,7 @@ static NV_STATUS os_get_smbios_header_legacy(NvU64 *pSmbsAddr)
 }

 // This function is needed only if "efi" is enabled.
-#if (defined(NV_LINUX_EFI_H_PRESENT) && defined(CONFIG_EFI))
+#if defined(CONFIG_EFI)
 static NV_STATUS os_verify_smbios_header_uefi(NvU64 smbsAddr)
 {
    NV_STATUS status = NV_ERR_OBJECT_NOT_FOUND;
@@ -1558,8 +1556,7 @@ static NV_STATUS os_get_smbios_header_uefi(NvU64 *pSmbsAddr)
 {
    NV_STATUS status = NV_ERR_OPERATING_SYSTEM;

-// Make sure that efi.h is present before using "struct efi".
-#if (defined(NV_LINUX_EFI_H_PRESENT) && defined(CONFIG_EFI))
+#if defined(CONFIG_EFI)

 // Make sure that efi.h has SMBIOS3_TABLE_GUID present.
 #if defined(SMBIOS3_TABLE_GUID)
@@ -1626,9 +1623,7 @@ NV_STATUS NV_API_CALL os_get_acpi_rsdp_from_uefi

    *pRsdpAddr = 0;

-// Make sure that efi.h is present before using "struct efi".
-#if (defined(NV_LINUX_EFI_H_PRESENT) && defined(CONFIG_EFI))
-
+#if defined(CONFIG_EFI)
    if (efi.acpi20 != EFI_INVALID_TABLE_ADDR)
    {
        *pRsdpAddr = efi.acpi20;
@@ -1914,11 +1909,7 @@ NV_STATUS NV_API_CALL os_write_file
    int num_retries = NV_MAX_NUM_FILE_IO_RETRIES;

 retry:
-#if defined(NV_KERNEL_WRITE_HAS_POINTER_POS_ARG)
    num_written = kernel_write(pFile, pBuffer, size, &f_pos);
-#else
-    num_written = kernel_write(pFile, pBuffer, size, f_pos);
-#endif
    if (num_written < 0)
    {
        return NV_ERR_OPERATING_SYSTEM;
@@ -1958,11 +1949,7 @@ NV_STATUS NV_API_CALL os_read_file
    int num_retries = NV_MAX_NUM_FILE_IO_RETRIES;

 retry:
-#if defined(NV_KERNEL_READ_HAS_POINTER_POS_ARG)
    num_read = kernel_read(pFile, pBuffer, size, &f_pos);
-#else
-    num_read = kernel_read(pFile, f_pos, pBuffer, size);
-#endif
    if (num_read < 0)
    {
        return NV_ERR_OPERATING_SYSTEM;
@@ -2067,10 +2054,8 @@ NV_STATUS NV_API_CALL os_get_random_bytes
    NvU16 numBytes
 )
 {
-#if defined NV_WAIT_FOR_RANDOM_BYTES_PRESENT
    if (wait_for_random_bytes() < 0)
        return NV_ERR_NOT_READY;
-#endif

    get_random_bytes(bytes, numBytes);
    return NV_OK;
@@ -2122,17 +2107,45 @@ void NV_API_CALL os_wake_up
    complete_all(&wq->q);
 }

-NV_STATUS NV_API_CALL os_get_tegra_platform
+static bool os_platform_is_fpga(void)
+{
+    const struct soc_device_attribute soc_attrs[] = {
+        { .revision = "*FPGA" },
+        {/* sentinel */}
+    };
+
+    if (soc_device_match(soc_attrs)) {
+        return true;
+    }
+
+    return false;
+}
+
+static bool os_platform_is_vdk(void)
+{
+    const struct soc_device_attribute soc_attrs[] = {
+        { .revision = "VDK" },
+        {/* sentinel */}
+    };
+
+    if (soc_device_match(soc_attrs)) {
+        return true;
+    }
+
+    return false;
+}
+
+    NV_STATUS NV_API_CALL os_get_tegra_platform
 (
    NvU32 *mode
 )
 {
-#if defined(NV_SOC_TEGRA_FUSE_HELPER_H_PRESENT) && NV_SUPPORTS_PLATFORM_DISPLAY_DEVICE
-    if (tegra_platform_is_fpga())
+#if NV_SUPPORTS_PLATFORM_DISPLAY_DEVICE
+    if (os_platform_is_fpga())
    {
        *mode = NV_OS_TEGRA_PLATFORM_FPGA;
    }
-    else if (tegra_platform_is_vdk())
+    else if (os_platform_is_vdk())
    {
        *mode = NV_OS_TEGRA_PLATFORM_SIM;
    }
@@ -2624,7 +2637,6 @@ NV_STATUS NV_API_CALL os_offline_page_at_address
 {
 #if defined(CONFIG_MEMORY_FAILURE)
    int flags = 0;
-    int ret;
    NvU64 pfn;
    struct page *page = NV_GET_PAGE_STRUCT(address);

@@ -2649,22 +2661,18 @@ NV_STATUS NV_API_CALL os_offline_page_at_address
    flags |= MF_SW_SIMULATED;
 #endif

-#ifdef NV_MEMORY_FAILURE_HAS_TRAPNO_ARG
-    ret = memory_failure(pfn, 0, flags);
-#else
-    ret = memory_failure(pfn, flags);
-#endif
+    nv_printf(NV_DBG_INFO, "NVRM: offlining page at address: 0x%llx pfn: 0x%llx\n",
+              address, pfn);

-    if (ret != 0)
-    {
-        nv_printf(NV_DBG_ERRORS, "NVRM: page offlining failed. address: 0x%llx pfn: 0x%llx ret: %d\n",
-                  address, pfn, ret);
-        return NV_ERR_OPERATING_SYSTEM;
-    }
+#ifdef NV_MEMORY_FAILURE_QUEUE_HAS_TRAPNO_ARG
+    memory_failure_queue(pfn, 0, flags);
+#else
+    memory_failure_queue(pfn, flags);
+#endif

    return NV_OK;
 #else // !defined(CONFIG_MEMORY_FAILURE)
-    nv_printf(NV_DBG_ERRORS, "NVRM: memory_failure() not supported by kernel. page offlining failed. address: 0x%llx\n",
+    nv_printf(NV_DBG_ERRORS, "NVRM: memory_failure_queue() not supported by kernel. page offlining failed. address: 0x%llx\n",
              address);
    return NV_ERR_NOT_SUPPORTED;
 #endif