turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-04-30 03:01:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix model_diff script

Browse Source
This commit is contained in:
turboderp
2024-07-06 11:55:37 +02:00
parent adefba1973
commit c294f3436f
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
model_diff.py
View File

@@ -14,7 +14,7 @@ import argparse, os, math, time
import pandas, fastparquet import pandas, fastparquet
import torch import torch
import torch.nn.functional as F import torch.nn.functional as F
from conversion.tokenize import get_tokens from exllamav2.conversion.tokenize import get_tokens
from exllamav2.util import list_live_tensors from exllamav2.util import list_live_tensors
import gc import gc
@@ -47,6 +47,8 @@ config[0].prepare()
config[1].prepare() config[1].prepare()
config[0].max_batch_size = 1 config[0].max_batch_size = 1
config[1].max_batch_size = 1 config[1].max_batch_size = 1
config[0].arch_compat_overrides()
config[1].arch_compat_overrides()
model = (ExLlamaV2(config[0]), ExLlamaV2(config[1])) model = (ExLlamaV2(config[0]), ExLlamaV2(config[1]))
model[0].load(lazy = True) model[0].load(lazy = True)