SillyTavern/SillyTavern-Launcher
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern-Launcher.git synced 2026-05-01 03:31:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

launcher upgrades

Browse Source 
+added new VRAM recommended info about bits per weight
+small text corrections in launcher.bat
This commit is contained in:
deffcolony
2024-07-15 11:05:24 +02:00
parent 4e8d2a5669
commit 4e72c7263d
2 changed files with 102 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Launcher.bat
View File

@@ -32,6 +32,7 @@ REM Normal Background Colors
set "red_bg=" set "red_bg="
set "blue_bg=" set "blue_bg="
set "yellow_bg=" set "yellow_bg="
set "green_bg="
REM Environment Variables (winget) REM Environment Variables (winget)
set "winget_path=%userprofile%\AppData\Local\Microsoft\WindowsApps" set "winget_path=%userprofile%\AppData\Local\Microsoft\WindowsApps"
@@ -221,8 +222,6 @@ if /i "%update_choice%"=="Y" (
goto :startupcheck_no_update goto :startupcheck_no_update
) )
:startupcheck_no_update :startupcheck_no_update
title STL [STARTUP CHECK] title STL [STARTUP CHECK]
REM Check if the folder exists REM Check if the folder exists
@@ -321,7 +320,7 @@ if %errorlevel% neq 0 (
pause pause
exit exit
) else ( ) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%winget%reset% echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%winget%reset% from app: App Installer
) )
REM Check if Git is installed if not then install git REM Check if Git is installed if not then install git
@@ -334,7 +333,7 @@ if %errorlevel% neq 0 (
pause pause
exit exit
) else ( ) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Git%reset% echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%git%reset% from app: Git
) )
REM Get the current PATH value from the registry REM Get the current PATH value from the registry
@@ -377,7 +376,7 @@ if %errorlevel% neq 0 (
pause pause
exit exit
) else ( ) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Miniconda3%reset% echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%conda%reset% from app: Miniconda3
) )
REM Run PowerShell command to retrieve VRAM size and divide by 1GB REM Run PowerShell command to retrieve VRAM size and divide by 1GB

160
bin/functions/launch/info_vram.bat
View File

@@ -3,86 +3,122 @@
:info_vram :info_vram
title STL [VRAM INFO] title STL [VRAM INFO]
chcp 65001 > nul chcp 65001 > nul
setlocal enabledelayedexpansion
REM Get GPU information
for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do (
set "gpu_info=!gpu_info! %%i"
)
cls cls
echo %blue_fg_strong%/ Home / VRAM Info%reset% echo %blue_fg_strong%/ Home / VRAM Info%reset%
echo ------------------------------------------------------------- echo -------------------------------------------------------------
REM Recommendations Based on VRAM Size REM Recommendations Based on VRAM Size
if %VRAM% lss 8 ( if %VRAM% lss 8 (
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage, echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage,
echo Local models will result in memory error or perform a REAL SLOW output echo Local models will result in memory error or perform a REAL SLOW output
) else if %VRAM% lss 12 ( ) else if %VRAM% lss 12 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B and 8B models. Check info below for BPW echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset% echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B and 8B models. Check info below for BPW
echo %blue_bg%║ ║%reset% endlocal
echo %blue_bg%║* EXL2: 5_0 ║%reset% echo.
echo %blue_bg%║* GGUF: Q5_K_M ║%reset% echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset% echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║
echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance. ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. RECOMMENDED ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo. echo.
) else if %VRAM% lss 22 ( ) else if %VRAM% lss 22 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B, 8B and 13B models. Check info below for BPW echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset% echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B and 13B models. Check info below for BPW
echo %blue_bg%║ ║%reset% endlocal
echo %blue_bg%║* EXL2: 6_5 ║%reset% echo.
echo %blue_bg%║* GGUF: Q5_K_M ║%reset% echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset% echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo. echo.
) else if %VRAM% lss 25 ( ) else if %VRAM% lss 25 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Good for 7B, 8B, 13B, 30B, and some efficient 70B models. Powerful local models will run well echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B and 30B models, Check info below for BPW
endlocal
echo. echo.
echo ╔══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╗ echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 can produce, near ║ echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ unquantized performance. ║ echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ Very similar to 8.0, good tradeoff of size vs ║ echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset%
echo ║ ║ ║ ║ ║ ║ ║ ║ performance, recommended. ║ echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset%
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, but usable on 8GB cards. ║ echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, slightly higher ║ echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ ║ ║ ║ ║ ║ ║ ║ quality. ║ echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ╚══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo. echo.
) else if %VRAM% gtr 25 ( ) else if %VRAM% gtr 25 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Suitable for most models, including larger LLMs. echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo You likely have the necessary expertise to pick your own model if you possess more than 25GB of VRAM. echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B, 30B and 70B models. Check info below for BPW
endlocal
echo.
echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║
echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo.
) else ( ) else (
echo An unexpected amount of VRAM detected or unable to detect VRAM. Check your system specifications. echo An unexpected amount of VRAM detected or unable to detect VRAM. Check your system specifications.
) )
echo.
setlocal enabledelayedexpansion set /p "info_vram_choice=Check for compatible models on VRAM calculator website? [Y/N]: "
chcp 65001 > nul if /i "%info_vram_choice%"=="Y" (
REM Get GPU information REM Open website in default browser
for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do ( start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM%
set "gpu_info=!gpu_info! %%i" goto :home
) else if /i "%info_vram_choice%"=="N" (
goto :home
) )
echo. :home
echo %blue_bg%╔════ GPU INFO ═════════════════════════════════╗%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%║* %gpu_info:~1% ║%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%╚═══════════════════════════════════════════════╝%reset%
echo.
endlocal
echo Would you like to open the VRAM calculator website to check compatible models?
set /p uservram_choice=Check compatible models? [Y/N]
REM Check if user input is not empty and is neither "Y" nor "N"
if not "%uservram_choice%"=="" (
if /i not "%uservram_choice%"=="Y" if /i not "%uservram_choice%"=="N" (
echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Invalid input. Please enter Y for yes or N for no%reset%
pause
goto :info_vram
)
)
if /i "%uservram_choice%"=="Y" ( start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM%
)
goto :home