diff --git a/tests/model_test.py b/tests/model_test.py deleted file mode 100644 index 662ccbc..0000000 --- a/tests/model_test.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Test the model container.""" - -from backends.exllamav2.model import ModelContainer - - -def progress(module, modules): - """Wrapper callback for load progress.""" - yield module, modules - - -def test_load_gen(model_path): - """Test loading a model.""" - container = ModelContainer(model_path) - loader = container.load_gen(progress) - for module, modules in loader: - print(module, modules) - container.unload() - del container - - -def test_generate_gen(model_path): - """Test generating from a model.""" - container = ModelContainer(model_path) - generator = container.generate_gen("Once upon a tim", token_healing=True) - for chunk in generator: - print(chunk, end="") - container.unload() - del container - - -def test_generate(model_path): - """Test generating from a model.""" - model_container = ModelContainer(model_path) - model_container.load(progress) - prompt = ( - "All work and no play makes turbo a derpy cat.\n" - "All work and no play makes turbo a derpy cat.\nAll" - ) - response = model_container.generate( - prompt, top_k=1, max_new_tokens=1000, stream_interval=0.5 - ) - print(response) - - -if __name__ == "__main__": - MODEL1 = "/mnt/str/models/_exl2/mistral-7b-instruct-exl2/4.0bpw/" - MODEL2 = "/mnt/str/models/_exl2/mistral-7b-instruct-exl2/4.65bpw/" - test_load_gen(MODEL1) - test_generate_gen(MODEL1) - test_generate(MODEL2)