turboderp-org/exllamav3
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav3.git synced 2026-04-20 14:29:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

compare_q.py: Fix GGUF VRAM computation when output.weight precedes token_embd.weight

Browse Source
This commit is contained in:
turboderp
2025-06-04 23:34:42 +02:00
parent 9a24e0ce52
commit ab875ba730
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
eval/compare_q_llamacpp.py
View File

@@ -23,11 +23,10 @@ def get_storage_info(model_dir):
head_numel = 0
for tensor_info in tensors:
name = tensor_info.name
if (name == "token_embd.weight" and head_bpw == 0) or \
name == "output.weight":
if (name == "token_embd.weight" and head_bpw == 0) or name == "output.weight":
head_bpw = tensor_info.n_bytes * 8 / tensor_info.n_elements
head_numel = tensor_info.n_elements
elif name.endswith(".weight"):
elif name.endswith(".weight") and name != "token_embd.weight" and not name.endswith("norm.weight"):
sum_bits += tensor_info.n_bytes * 8
sum_numel += tensor_info.n_elements
vram_bits = head_numel * head_bpw + sum_bits