turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-04-24 00:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add typical setting to chat example.

Browse Source
This commit is contained in:
turboderp
2023-09-26 19:50:44 +02:00
parent 92a9828450
commit ba5f6191c8
2 changed files with 11 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
tests/test_mmlu.py
View File

@@ -14,22 +14,16 @@ import torch
# Models to test
model_base = "/mnt/str/models/_exl2/llama2-7b-chat-exl2/"
model_base = "/mnt/str/models/_exl2/llama2-70b-chat-exl2/"
variants = [v for v in os.listdir(model_base) if os.path.isdir(os.path.join(model_base, v))]
# variants = [v for v in os.listdir(model_base) if os.path.isdir(os.path.join(model_base, v))]
# variants = \
# [
# "2.5bpw",
# "3.0bpw",
# "3.5bpw",
# "4.0bpw",
# "4.65bpw",
# "5.0bpw",
# "6.0bpw",
# "8.0bpw",
# "fp16",
# ]
variants = \
[
"3.0bpw",
"4.0bpw",
"4.65bpw",
]
gpu_split = (19.5, 24)