theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #254 from lucyknada/main

Browse Source 
add draft_gpu_split option for spec decoding
This commit is contained in:
Brian
2025-02-11 16:48:03 -05:00
committed by GitHub
parent e290b88568 beb6d8faa5
commit 2e491472d1
3 changed files with 59 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
config_sample.yml
View File

@@ -20,7 +20,7 @@ network:
# Turn on this option if you are ONLY connecting from localhost.
disable_auth: false
# Disable fetching external content in response to requests, such as images from URLs.
# Disable fetching external content in response to requests,such as images from URLs.
disable_fetch_requests: false
# Send tracebacks over the API (default: False).
@@ -166,6 +166,10 @@ draft_model:
# Possible values: 'FP16', 'Q8', 'Q6', 'Q4'.
draft_cache_mode: FP16
# An integer array of GBs of VRAM to split between GPUs (default: []).
# If this isn't filled in, the draft model is autosplit.
draft_gpu_split: []
# Options for Loras
lora:
# Directory to look for LoRAs (default: loras).