Fixed VRAM for model downloads

2026-04-27 09:42:54 +00:00 · 2024-07-28 13:56:52 -06:00
parent 5b604fffa5
commit 74920cdc1c
2 changed files with 38 additions and 32 deletions
--- a/Launcher.bat
+++ b/Launcher.bat
@@ -388,6 +388,10 @@ if not exist "%st_install_path%" (
    goto :no_st_install_path
 )

+REM Run PowerShell command to retrieve VRAM size and divide by 1GB
+for /f "usebackq tokens=*" %%i in (`powershell -Command "$qwMemorySize = (Get-ItemProperty -Path 'HKLM:\SYSTEM\ControlSet001\Control\Class\{4d36e968-e325-11ce-bfc1-08002be10318}\0*' -Name HardwareInformation.qwMemorySize -ErrorAction SilentlyContinue).'HardwareInformation.qwMemorySize'; if ($null -ne $qwMemorySize -and $qwMemorySize -is [array]) { $qwMemorySize = [double]$qwMemorySize[0] } else { $qwMemorySize = [double]$qwMemorySize }; if ($null -ne $qwMemorySize) { [math]::Round($qwMemorySize/1GB) } else { 'Property not found' }"`) do (
+    set "UVRAM=%%i"
+)

 REM Change the current directory to 'sillytavern' folder
 cd /d "%st_install_path%"
@@ -2024,12 +2028,12 @@ if "%app_installer_tabbyapi_model_choice%"=="1" (
 :install_tabbyapi_model_hathor
 cd /d "%tabbyapi_install_path%\models"
 REM Install model Based on VRAM Size
-if %VRAM% lss 8 (
+if %UVRAM% lss 8 (
 echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Sorry... You need atleast 8GB VRAM or more to run a local LLM%reset%
 pause
 goto :install_tabbyapi_model_menu
-) else if %VRAM% lss 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset%Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% lss 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset%Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2" (
    REM Remove model if it already exists
@@ -2043,8 +2047,8 @@ git clone https://huggingface.co/Nitral-AI/Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2
 echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Successfully installed model: Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2%reset%
 goto :install_tabbyapi_model_hathor_presets

-) else if %VRAM% equ 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% equ 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2" (
    REM Remove model if it already exists
@@ -2058,8 +2062,8 @@ git clone https://huggingface.co/Nitral-AI/Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2
 echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Successfully installed model: Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2%reset%
 goto :install_tabbyapi_model_hathor_presets

-) else if %VRAM% gtr 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% gtr 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Hathor_Tahsin-L3-8B-v0.85-5bpw-exl2" (
    REM Remove model if it already exists
@@ -2083,12 +2087,12 @@ goto :install_tabbyapi_model_hathor_presets
 :install_tabbyapi_model_stheno
 cd /d "%tabbyapi_install_path%\models"
 REM Install model Based on VRAM Size
-if %VRAM% lss 8 (
+if %UVRAM% lss 8 (
 echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Sorry... You need atleast 8GB VRAM or more to run a local LLM%reset%
 pause
 goto :install_tabbyapi_model_menu
-) else if %VRAM% lss 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% lss 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "L3-8B-Stheno-v3.2-exl2-5_0" (
    REM Remove model if it already exists
@@ -2103,8 +2107,8 @@ echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Suc
 pause
 goto :install_tabbyapi_model_menu

-) else if %VRAM% equ 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% equ 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "L3-8B-Stheno-v3.2-exl2-6_5" (
    REM Remove model if it already exists
@@ -2119,8 +2123,8 @@ echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Suc
 pause
 goto :install_tabbyapi_model_menu

-) else if %VRAM% gtr 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% gtr 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "L3-8B-Stheno-v3.2-exl2-6_5" (
    REM Remove model if it already exists
@@ -2145,12 +2149,12 @@ https://huggingface.co/bartowski/Replete-Coder-Instruct-8b-Merged-exl2
 :install_tabbyapi_model_repletecoder
 cd /d "%tabbyapi_install_path%\models"
 REM Install model Based on VRAM Size
-if %VRAM% lss 8 (
+if %UVRAM% lss 8 (
 echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Sorry... You need atleast 8GB VRAM or more to run a local LLM%reset%
 pause
 goto :install_tabbyapi_model_menu
-) else if %VRAM% lss 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% lss 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Replete-Coder-Instruct-8b-Merged-exl2-5_0" (
    REM Remove model if it already exists
@@ -2165,8 +2169,8 @@ echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Suc
 pause
 goto :install_tabbyapi_model_menu

-) else if %VRAM% equ 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% equ 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Replete-Coder-Instruct-8b-Merged-exl2-6_5" (
    REM Remove model if it already exists
@@ -2183,8 +2187,8 @@ echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% %green_fg_strong%Suc
 pause
 goto :install_tabbyapi_model_menu

-) else if %VRAM% gtr 12 (
-echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%VRAM% GB%reset%
+) else if %UVRAM% gtr 12 (
+echo %blue_bg%[%time%]%reset% %blue_fg_strong%[INFO]%reset% Detected GPU VRAM: %cyan_fg_strong%%UVRAM% GB%reset%
 REM Check if model exists
 if exist "Replete-Coder-Instruct-8b-Merged-exl2-6_5" (
    REM Remove model if it already exists
--- a/bin/functions/launch/info_vram.bat
+++ b/bin/functions/launch/info_vram.bat
@@ -8,17 +8,19 @@ REM Get GPU information
 for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do (
    set "gpu_info=!gpu_info! %%i"
 )
+
+
 cls
 echo %blue_fg_strong%^| ^> / Home / VRAM ^& LLM Info                                                                           ^|%reset%
 echo %blue_fg_strong% ======================================================================================================%reset%   
 REM Recommendations Based on VRAM Size
-if %VRAM% lss 8 (
+if %UVRAM% lss 8 (
    echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
-    echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage, 
+    echo %cyan_fg_strong%GPU VRAM: %UVRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage, 
    echo Local models will result in memory error or perform a REAL SLOW output
-) else if %VRAM% lss 12 (
+) else if %UVRAM% lss 12 (
    echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
-    echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B and 8B models. Check info below for BPW
+    echo %cyan_fg_strong%GPU VRAM: %UVRAM% GB%reset% - Great for 7B and 8B models. Check info below for BPW
    endlocal
    echo.
    echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
@@ -39,9 +41,9 @@ if %VRAM% lss 8 (
    echo ║ 3.5    ║ 3.5  ║ 6.0          ║ 6.4 GB    ║ 6.8 GB    ║ 7.8 GB     ║ 9.9 GB     ║ Lower quality, only use if you have to.    ║
    echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
    echo.
-) else if %VRAM% lss 22 (
+) else if %UVRAM% lss 22 (
    echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
-    echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B and 13B models. Check info below for BPW
+    echo %cyan_fg_strong%GPU VRAM: %UVRAM% GB%reset% - Great for 7B, 8B and 13B models. Check info below for BPW
    endlocal
    echo.
    echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
@@ -62,9 +64,9 @@ if %VRAM% lss 8 (
    echo ║ 3.5    ║ 3.5  ║ 6.0          ║ 6.4 GB    ║ 6.8 GB    ║ 7.8 GB     ║ 9.9 GB     ║ Lower quality, only use if you have to.    ║
    echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
    echo.
-) else if %VRAM% lss 25 (
+) else if %UVRAM% lss 25 (
    echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
-    echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B and 30B models, Check info below for BPW
+    echo %cyan_fg_strong%GPU VRAM: %UVRAM% GB%reset% - Great for 7B, 8B, 13B and 30B models, Check info below for BPW
    endlocal
    echo.
    echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
@@ -85,9 +87,9 @@ if %VRAM% lss 8 (
    echo ║ 3.5    ║ 3.5  ║ 6.0          ║ 6.4 GB    ║ 6.8 GB    ║ 7.8 GB     ║ 9.9 GB     ║ Lower quality, only use if you have to.    ║
    echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
    echo.
-) else if %VRAM% gtr 25 (
+) else if %UVRAM% gtr 25 (
    echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
-    echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B, 30B and 70B models. Check info below for BPW
+    echo %cyan_fg_strong%GPU VRAM: %UVRAM% GB%reset% - Great for 7B, 8B, 13B, 30B and 70B models. Check info below for BPW
    endlocal
    echo.
    echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
@@ -115,7 +117,7 @@ if %VRAM% lss 8 (
 set /p "info_vram_choice=Check for compatible models on VRAM calculator website? [Y/N]: "
 if /i "%info_vram_choice%"=="Y" (
    REM Open website in default browser
-    start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM%
+    start https://sillytavernai.com/llm-model-vram-calculator/?vram=%UVRAM%
    goto :home
 ) else if /i "%info_vram_choice%"=="N" (
    goto :home