turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-03-15 00:07:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix model_diff script

Browse Source
This commit is contained in:
turboderp
2024-07-06 11:55:37 +02:00
parent adefba1973
commit c294f3436f
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
model_diff.py
View File

@@ -14,7 +14,7 @@ import argparse, os, math, time
import pandas, fastparquet
import torch
import torch.nn.functional as F
from conversion.tokenize import get_tokens
from exllamav2.conversion.tokenize import get_tokens
from exllamav2.util import list_live_tensors
import gc
@@ -47,6 +47,8 @@ config[0].prepare()
config[1].prepare()
config[0].max_batch_size = 1
config[1].max_batch_size = 1
config[0].arch_compat_overrides()
config[1].arch_compat_overrides()
model = (ExLlamaV2(config[0]), ExLlamaV2(config[1]))
model[0].load(lazy = True)