SillyTavern/SillyTavern-Launcher
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern-Launcher.git synced 2026-03-13 23:39:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

launcher upgrades

Browse Source 
+added new VRAM recommended info about bits per weight
+small text corrections in launcher.bat
This commit is contained in:
deffcolony
2024-07-15 11:05:24 +02:00
parent 4e8d2a5669
commit 4e72c7263d
2 changed files with 102 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Launcher.bat
View File

@@ -32,6 +32,7 @@ REM Normal Background Colors
set "red_bg="
set "blue_bg="
set "yellow_bg="
set "green_bg="
REM Environment Variables (winget)
set "winget_path=%userprofile%\AppData\Local\Microsoft\WindowsApps"
@@ -221,8 +222,6 @@ if /i "%update_choice%"=="Y" (
goto :startupcheck_no_update
)
:startupcheck_no_update
title STL [STARTUP CHECK]
REM Check if the folder exists
@@ -321,7 +320,7 @@ if %errorlevel% neq 0 (
pause
exit
) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%winget%reset%
echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%winget%reset% from app: App Installer
)
REM Check if Git is installed if not then install git
@@ -334,7 +333,7 @@ if %errorlevel% neq 0 (
pause
exit
) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Git%reset%
echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%git%reset% from app: Git
)
REM Get the current PATH value from the registry
@@ -377,7 +376,7 @@ if %errorlevel% neq 0 (
pause
exit
) else (
echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Miniconda3%reset%
echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%conda%reset% from app: Miniconda3
)
REM Run PowerShell command to retrieve VRAM size and divide by 1GB

160
bin/functions/launch/info_vram.bat
View File

@@ -3,86 +3,122 @@
:info_vram
title STL [VRAM INFO]
chcp 65001 > nul
setlocal enabledelayedexpansion
REM Get GPU information
for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do (
set "gpu_info=!gpu_info! %%i"
)
cls
echo %blue_fg_strong%/ Home / VRAM Info%reset%
echo -------------------------------------------------------------
REM Recommendations Based on VRAM Size
if %VRAM% lss 8 (
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage,
echo Local models will result in memory error or perform a REAL SLOW output
) else if %VRAM% lss 12 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B and 8B models. Check info below for BPW
echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%║* EXL2: 5_0 ║%reset%
echo %blue_bg%║* GGUF: Q5_K_M ║%reset%
echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset%
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B and 8B models. Check info below for BPW
endlocal
echo.
echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║
echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance. ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. RECOMMENDED ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo.
) else if %VRAM% lss 22 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B, 8B and 13B models. Check info below for BPW
echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%║* EXL2: 6_5 ║%reset%
echo %blue_bg%║* GGUF: Q5_K_M ║%reset%
echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset%
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B and 13B models. Check info below for BPW
endlocal
echo.
echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo.
) else if %VRAM% lss 25 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Good for 7B, 8B, 13B, 30B, and some efficient 70B models. Powerful local models will run well
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B and 30B models, Check info below for BPW
endlocal
echo.
echo ╔══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 can produce, near ║
echo ║ ║ ║ ║ ║ ║ ║ ║ unquantized performance. ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ Very similar to 8.0, good tradeoff of size vs ║
echo ║ ║ ║ ║ ║ ║ ║ ║ performance, recommended. ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, but usable on 8GB cards. ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, slightly higher ║
echo ║ ║ ║ ║ ║ ║ ║ ║ quality. ║
echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║
echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo.
) else if %VRAM% gtr 25 (
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Suitable for most models, including larger LLMs.
echo You likely have the necessary expertise to pick your own model if you possess more than 25GB of VRAM.
echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset%
echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B, 30B and 70B models. Check info below for BPW
endlocal
echo.
echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗
echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo %green_bg%║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║%reset%
echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║%reset%
echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset%
echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║
echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║
echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║
echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║
echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║
echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝
echo.
) else (
echo An unexpected amount of VRAM detected or unable to detect VRAM. Check your system specifications.
)
echo.
setlocal enabledelayedexpansion
chcp 65001 > nul
REM Get GPU information
for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do (
set "gpu_info=!gpu_info! %%i"
set /p "info_vram_choice=Check for compatible models on VRAM calculator website? [Y/N]: "
if /i "%info_vram_choice%"=="Y" (
REM Open website in default browser
start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM%
goto :home
) else if /i "%info_vram_choice%"=="N" (
goto :home
)
echo.
echo %blue_bg%╔════ GPU INFO ═════════════════════════════════╗%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%║* %gpu_info:~1% ║%reset%
echo %blue_bg%║ ║%reset%
echo %blue_bg%╚═══════════════════════════════════════════════╝%reset%
echo.
endlocal
echo Would you like to open the VRAM calculator website to check compatible models?
set /p uservram_choice=Check compatible models? [Y/N]
REM Check if user input is not empty and is neither "Y" nor "N"
if not "%uservram_choice%"=="" (
if /i not "%uservram_choice%"=="Y" if /i not "%uservram_choice%"=="N" (
echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Invalid input. Please enter Y for yes or N for no%reset%
pause
goto :info_vram
)
)
if /i "%uservram_choice%"=="Y" ( start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM%
)
goto :home
:home