diff --git a/Launcher.bat b/Launcher.bat index e01377a..6141752 100644 --- a/Launcher.bat +++ b/Launcher.bat @@ -32,6 +32,7 @@ REM Normal Background Colors set "red_bg=" set "blue_bg=" set "yellow_bg=" +set "green_bg=" REM Environment Variables (winget) set "winget_path=%userprofile%\AppData\Local\Microsoft\WindowsApps" @@ -221,8 +222,6 @@ if /i "%update_choice%"=="Y" ( goto :startupcheck_no_update ) - - :startupcheck_no_update title STL [STARTUP CHECK] REM Check if the folder exists @@ -321,7 +320,7 @@ if %errorlevel% neq 0 ( pause exit ) else ( - echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%winget%reset% + echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%winget%reset% from app: App Installer ) REM Check if Git is installed if not then install git @@ -334,7 +333,7 @@ if %errorlevel% neq 0 ( pause exit ) else ( - echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Git%reset% + echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%git%reset% from app: Git ) REM Get the current PATH value from the registry @@ -377,7 +376,7 @@ if %errorlevel% neq 0 ( pause exit ) else ( - echo [ %green_fg_strong%OK%reset% ] Found app: %cyan_fg_strong%Miniconda3%reset% + echo [ %green_fg_strong%OK%reset% ] Found app command: %cyan_fg_strong%conda%reset% from app: Miniconda3 ) REM Run PowerShell command to retrieve VRAM size and divide by 1GB diff --git a/bin/functions/launch/info_vram.bat b/bin/functions/launch/info_vram.bat index 2f1d7d0..08a9b08 100644 --- a/bin/functions/launch/info_vram.bat +++ b/bin/functions/launch/info_vram.bat @@ -3,86 +3,122 @@ :info_vram title STL [VRAM INFO] chcp 65001 > nul +setlocal enabledelayedexpansion +REM Get GPU information +for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do ( + set "gpu_info=!gpu_info! %%i" +) cls echo %blue_fg_strong%/ Home / VRAM Info%reset% echo ------------------------------------------------------------- REM Recommendations Based on VRAM Size if %VRAM% lss 8 ( + echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset% echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - It's recommended to stick with APIs like OpenAI, Claude or OpenRouter for LLM usage, echo Local models will result in memory error or perform a REAL SLOW output ) else if %VRAM% lss 12 ( - echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B and 8B models. Check info below for BPW - echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset% - echo %blue_bg%║ ║%reset% - echo %blue_bg%║* EXL2: 5_0 ║%reset% - echo %blue_bg%║* GGUF: Q5_K_M ║%reset% - echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset% + echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset% + echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B and 8B models. Check info below for BPW + endlocal + echo. + echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗ + echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance. ║ + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo %green_bg%║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║%reset% + echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. RECOMMENDED ║%reset% + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ + echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ echo. ) else if %VRAM% lss 22 ( - echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Can run 7B, 8B and 13B models. Check info below for BPW - echo %blue_bg%╔════ BPW - Bits Per Weight ═════════════════════════════════╗%reset% - echo %blue_bg%║ ║%reset% - echo %blue_bg%║* EXL2: 6_5 ║%reset% - echo %blue_bg%║* GGUF: Q5_K_M ║%reset% - echo %blue_bg%╚════════════════════════════════════════════════════════════╝%reset% + echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset% + echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B and 13B models. Check info below for BPW + endlocal + echo. + echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗ + echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║ + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset% + echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset% + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ + echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ echo. ) else if %VRAM% lss 25 ( - echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Good for 7B, 8B, 13B, 30B, and some efficient 70B models. Powerful local models will run well + echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset% + echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B and 30B models, Check info below for BPW + endlocal echo. - echo ╔══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╗ - echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ - echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ - echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 can produce, near ║ - echo ║ ║ ║ ║ ║ ║ ║ ║ unquantized performance. ║ - echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ - echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ Very similar to 8.0, good tradeoff of size vs ║ - echo ║ ║ ║ ║ ║ ║ ║ ║ performance, recommended. ║ - echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ - echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, but usable on 8GB cards. ║ - echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ - echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, slightly higher ║ - echo ║ ║ ║ ║ ║ ║ ║ ║ quality. ║ - echo ║══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ - echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ - echo ╚══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ + echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗ + echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║ + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo %green_bg%║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║%reset% + echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║%reset% + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ + echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ echo. ) else if %VRAM% gtr 25 ( - echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Suitable for most models, including larger LLMs. - echo You likely have the necessary expertise to pick your own model if you possess more than 25GB of VRAM. + echo %cyan_fg_strong%GPU: %gpu_info:~1%%reset% + echo %cyan_fg_strong%GPU VRAM: %VRAM% GB%reset% - Great for 7B, 8B, 13B, 30B and 70B models. Check info below for BPW + endlocal + echo. + echo ╔══ EXL2 - RECOMMENDED BPW [Bits Per Weight] ═════════════════════════════════════════════════════════════════════════════════╗ + echo ║ Branch ║ Bits ║ lm_head bits ║ VRAM - 4k ║ VRAM - 8k ║ VRAM - 16k ║ VRAM - 32k ║ Description ║ + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo %green_bg%║ 8.0 ║ 8.0 ║ 8.0 ║ 10.1 GB ║ 10.5 GB ║ 11.5 GB ║ 13.6 GB ║ Maximum quality that ExLlamaV2 ║%reset% + echo %green_bg%║ ║ ║ ║ ║ ║ ║ ║ can produce, near unquantized performance. ║%reset% + echo %green_bg%║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║%reset% + echo ║ 6.5 ║ 6.5 ║ 8.0 ║ 8.9 GB ║ 9.3 GB ║ 10.3 GB ║ 12.4 GB ║ similar to 8.0, good tradeoff of ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ size vs performance, RECOMMENDED. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 5.0 ║ 5.0 ║ 6.0 ║ 7.7 GB ║ 8.1 GB ║ 9.1 GB ║ 11.2 GB ║ Slightly lower quality vs 6.5, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ but usable on 8GB cards. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 4.25 ║ 4.25 ║ 6.0 ║ 7.0 GB ║ 7.4 GB ║ 8.4 GB ║ 10.5 GB ║ GPTQ equivalent bits per weight, ║ + echo ║ ║ ║ ║ ║ ║ ║ ║ slightly higher quality. ║ + echo ║═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════║ + echo ║ 3.5 ║ 3.5 ║ 6.0 ║ 6.4 GB ║ 6.8 GB ║ 7.8 GB ║ 9.9 GB ║ Lower quality, only use if you have to. ║ + echo ╚═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╝ + echo. ) else ( echo An unexpected amount of VRAM detected or unable to detect VRAM. Check your system specifications. ) -echo. -setlocal enabledelayedexpansion -chcp 65001 > nul -REM Get GPU information -for /f "skip=1 delims=" %%i in ('wmic path win32_videocontroller get caption') do ( - set "gpu_info=!gpu_info! %%i" +set /p "info_vram_choice=Check for compatible models on VRAM calculator website? [Y/N]: " +if /i "%info_vram_choice%"=="Y" ( + REM Open website in default browser + start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM% + goto :home +) else if /i "%info_vram_choice%"=="N" ( + goto :home ) -echo. -echo %blue_bg%╔════ GPU INFO ═════════════════════════════════╗%reset% -echo %blue_bg%║ ║%reset% -echo %blue_bg%║* %gpu_info:~1% ║%reset% -echo %blue_bg%║ ║%reset% -echo %blue_bg%╚═══════════════════════════════════════════════╝%reset% -echo. - -endlocal - -echo Would you like to open the VRAM calculator website to check compatible models? -set /p uservram_choice=Check compatible models? [Y/N] - -REM Check if user input is not empty and is neither "Y" nor "N" -if not "%uservram_choice%"=="" ( - if /i not "%uservram_choice%"=="Y" if /i not "%uservram_choice%"=="N" ( - echo %red_bg%[%time%]%reset% %red_fg_strong%[ERROR] Invalid input. Please enter Y for yes or N for no%reset% - pause - goto :info_vram - ) -) - -if /i "%uservram_choice%"=="Y" ( start https://sillytavernai.com/llm-model-vram-calculator/?vram=%VRAM% -) -goto :home \ No newline at end of file +:home \ No newline at end of file