ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Missing tokenizer.model error during gguf conversion (#6443)

Browse Source 
Co-authored-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
Abhishek Gopinath K
2024-04-03 21:12:52 +05:30
committed by GitHub
parent 17d75e9340
commit 20b9b65e75
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
convert-hf-to-gguf.py
View File

@@ -323,8 +323,7 @@ class Model(ABC):
toktypes: list[int] = [] toktypes: list[int] = []
if not tokenizer_path.is_file(): if not tokenizer_path.is_file():
print(f'Error: Missing {tokenizer_path}', file=sys.stderr) raise FileNotFoundError(f"File not found: {tokenizer_path}")
sys.exit(1)
tokenizer = SentencePieceProcessor(str(tokenizer_path)) tokenizer = SentencePieceProcessor(str(tokenizer_path))
vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size()) vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())