theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-07-16 16:50:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backends: Remove ExLlamaV2 backend and container abstraction layer

Browse Source 
Remove BaseModelContainer abstraction

LoRA endpoints remain as stubs (supported in exllamav3, but API is undecided)

Fix /v1/lora/unload unloading the
entire model.

The last commit with exllamav2 support is preserved on the
exl2-checkpoint branch.
This commit is contained in:
turboderp
2026-07-14 22:40:33 +02:00
parent 79126f904c
commit c8b9a8f4aa
22 changed files with 110 additions and 2203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@@ -26,6 +26,11 @@
>
> In addition to the README, please read the [Wiki](https://github.com/theroyallab/tabbyAPI/wiki/1.-Getting-Started) page for information about getting started!
> [!NOTE]
>
> ExLlamaV2 models are no longer supported in the `main` branch. The last commit with ExLlamav2
> support is preserved on the `exl2-checkpoint` branch.
> [!NOTE]
>
> Need help? Join the [Discord Server](https://discord.gg/sYQxnuD7Fj) and get the `Tabby` role. Please be nice when asking questions.
@@ -38,9 +43,9 @@
>
> Want to run GGUF models? Take a look at [YALS](https://github.com/theroyallab/YALS), TabbyAPI's sister project.
A FastAPI based application that allows for generating text using an LLM (large language model) using the [Exllamav2](https://github.com/turboderp-org/exllamav2) and [Exllamav3](https://github.com/turboderp-org/exllamav3) backends.
A FastAPI based application that allows for generating text using an LLM (large language model) using the [Exllamav3](https://github.com/turboderp-org/exllamav3) backend.
TabbyAPI is also the official API backend server for ExllamaV2 and V3.
TabbyAPI is also the official API backend server for ExllamaV3.
## Disclaimer
@@ -97,8 +102,6 @@ And much more. If something is missing here, PR it in!
TabbyAPI uses Exllama as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported:
- Exl2/GPTQ (deprecated, will be removed in the near future)
- Exl3 (Highly recommended)
- FP16/BF16

277
backends/base_model_container.py
View File

@@ -1,277 +0,0 @@
import abc
import asyncio
import pathlib
from loguru import logger
from typing import (
Any,
AsyncIterator,
Dict,
List,
Optional,
)
from common.multimodal import MultimodalEmbeddingWrapper
from common.errors import ContextLengthExceededError
from common.sampling import BaseSamplerRequest
from common.templating import PromptTemplate
from common.transformers_utils import HFModel
from common.utils import unwrap
from endpoints.core.types.model import ModelCard
class BaseModelContainer(abc.ABC):
"""Abstract base class for model containers."""
# Exposed model information
model_dir: pathlib.Path = pathlib.Path("models")
prompt_template: Optional[PromptTemplate] = None
tool_format: Optional[str] = None
# HF Model instance
hf_model: HFModel
# Optional features
use_draft_model: bool = False
use_vision: bool = False
# Load synchronization
# The bool is a master switch for accepting requests
# The lock keeps load tasks sequential
# The condition notifies any waiting tasks
active_job_ids: Dict[str, Any] = {}
loaded: bool = False
load_lock: asyncio.Lock
load_condition: asyncio.Condition
reasoning: bool
reasoning_start_token: Optional[str]
reasoning_end_token: Optional[str]
reasoning_suppress_header: Optional[str]
force_enable_thinking: Optional[str]
# Required methods
@classmethod
@abc.abstractmethod
async def create(cls, model_directory: pathlib.Path, hf_model: HFModel, **kwargs):
"""
Asynchronously creates and initializes a model container instance.
Args:
model_directory: Path to the model files.
hf_model: HF config.json wrapper.
**kwargs: Backend-specific configuration options.
Returns:
An instance of the implementing class.
"""
pass
@abc.abstractmethod
async def load(self, progress_callback=None, **kwargs):
"""
Loads the model into memory.
Args:
progress_callback: Optional callback for progress updates.
**kwargs: Additional loading options.
"""
pass
# NOTE: Might be an optional method
@abc.abstractmethod
async def load_gen(self, progress_callback=None, **kwargs):
"""
Loads the model into memory, yielding progress updates.
Args:
progress_callback: Optional callback for progress updates.
**kwargs: Additional loading options.
Yields:
Progress updates
"""
if False:
yield
@abc.abstractmethod
async def unload(self, loras_only: bool = False, **kwargs):
"""
Unloads the model and associated resources from memory.
Args:
loras_only: If True, only unload LoRAs.
**kwargs: Additional unloading options (e.g., shutdown).
"""
pass
@abc.abstractmethod
def encode_tokens(self, text: str, **kwargs) -> List[int]:
"""
Encodes a string of text into a list of token IDs.
Args:
text: The input text string.
**kwargs: Backend-specific encoding options (e.g., add_bos_token).
Returns:
A list of integer token IDs.
"""
pass
@abc.abstractmethod
def decode_tokens(self, ids: List[int], **kwargs) -> str:
"""
Decodes a list of token IDs back into a string.
Args:
ids: A list of integer token IDs.
**kwargs: Backend-specific decoding options (e.g., decode_special_tokens).
Returns:
The decoded text string.
"""
pass
@abc.abstractmethod
def get_special_tokens(self) -> Dict[str, Any]:
"""
Gets special tokens used by the model/tokenizer.
Returns:
A dictionary mapping special token names (e.g., 'bos_token', 'eos_token')
to their string or ID representation.
"""
pass
@abc.abstractmethod
def model_info(self) -> ModelCard:
"""
Returns a dictionary of the current model's configuration parameters.
Returns:
Model parameters provided by the backend
"""
pass
@abc.abstractmethod
async def wait_for_jobs(self, skip_wait: bool = False):
"""
Waits for any active generation jobs to complete.
Args:
skip_wait: If True, cancel jobs immediately instead of waiting.
"""
pass
# Optional methods
async def load_loras(self, lora_directory: pathlib.Path, **kwargs) -> Dict[str, List[str]]:
"""
Loads LoRA adapters. Base implementation does nothing or raises error.
Args:
lora_directory: Path to the directory containing LoRA files.
**kwargs: LoRA configuration (e.g., list of loras, scaling).
Returns:
A dictionary indicating success/failure for each LoRA.
"""
logger.warning("LoRA loading not implemented for this backend.") # type: ignore
return {
"success": [],
"failure": [lora.get("name", "unknown") for lora in kwargs.get("loras", [])],
}
def get_loras(self) -> List[Any]:
"""
Gets the currently loaded LoRA adapters. Base implementation returns empty list.
Returns:
A list representing the loaded LoRAs (backend-specific format).
"""
return []
def validate_context_length(
self,
prompt: str,
params: BaseSamplerRequest,
mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None,
):
"""Validate a prompt before starting a streaming HTTP response."""
context_len = len(
self.encode_tokens(
prompt,
add_bos_token=unwrap(params.add_bos_token, self.hf_model.add_bos_token()),
embeddings=mm_embeddings,
)
)
max_seq_len = self.model_info().parameters.max_seq_len
if context_len > max_seq_len:
raise ContextLengthExceededError(
f"Prompt length {context_len} exceeds the available context size "
f"of {max_seq_len} tokens"
)
@abc.abstractmethod
async def generate(
self,
request_id: str,
prompt: str,
params: BaseSamplerRequest,
abort_event: Optional[asyncio.Event] = None,
mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None,
) -> Dict[str, Any]:
"""
Generates a complete response for a given prompt and parameters.
Args:
request_id: Unique identifier for the generation request.
prompt: The input prompt string.
params: Sampling and generation parameters.
abort_event: An asyncio Event to signal cancellation.
mm_embeddings: Optional multimodal embeddings.
Returns:
A dictionary containing the generation info
"""
pass
@abc.abstractmethod
async def stream_generate(
self,
request_id: str,
prompt: str,
params: BaseSamplerRequest,
abort_event: Optional[asyncio.Event] = None,
mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None,
filter_trigger: str = None,
) -> AsyncIterator[Dict[str, Any]]:
"""
Generates a response iteratively (streaming) for a given prompt.
Args:
request_id: Unique identifier for the generation request.
prompt: The input prompt string.
params: Sampling and generation parameters.
abort_event: An asyncio Event to signal cancellation.
mm_embeddings: Optional multimodal embeddings.
filter_trigger: Delay filters (from params) until trigger text.
Must map to single token.
Yields:
Generation chunks
"""
if False:
yield

158
backends/exllamav2/grammar.py
View File

@@ -1,158 +0,0 @@
import traceback
import typing
from functools import lru_cache
from typing import List
import torch
from exllamav2 import ExLlamaV2, ExLlamaV2Tokenizer
from exllamav2.generator.filters import ExLlamaV2Filter
from formatron.extractor import NonterminalExtractor
from formatron.formatter import FormatterBuilder
from formatron.integrations.exllamav2 import FormatterFilter, create_engine_vocabulary
from formatron.schemas import json_schema
from common.logger import xlogger
class ExLlamaV2Grammar:
"""ExLlamaV2 class for various grammar filters/parsers."""
filters: List[ExLlamaV2Filter]
def __init__(self):
self.filters = []
def add_json_schema_filter(
self,
schema: dict,
model: ExLlamaV2,
tokenizer: ExLlamaV2Tokenizer,
):
"""Adds an ExllamaV2 filter based on a JSON schema."""
# Create the parser
try:
# Add fields required by formatron if not present
if "$id" not in schema:
schema["$id"] = "https://example.com/example.json"
if "$schema" not in schema:
schema["$schema"] = "http://json-schema.org/draft-07/schema#"
# Validate schema and create formatter
schema = json_schema.create_schema(schema)
f = FormatterBuilder()
f.append_line(f"{f.json(schema)}")
except Exception:
traceback.print_exc()
xlogger.error(
"Skipping because the JSON schema couldn't be parsed. "
"Please read the above error for more information.",
{"schema": schema, "exception": traceback.format_exc()},
)
return
lmfilter = _create_formatter_filter(model, tokenizer, f)
# Append the filters
self.filters.append(lmfilter)
def add_regex_filter(
self,
pattern: str,
model: ExLlamaV2,
tokenizer: ExLlamaV2Tokenizer,
):
"""Adds an ExllamaV2 filter based on regular expressions."""
# Create the parser
try:
# Validate regex and create formatter
f = FormatterBuilder()
f.append_line(f"{f.regex(pattern)}")
except Exception:
traceback.print_exc()
xlogger.error(
"Skipping because the regex pattern couldn't be parsed. "
"Please read the above error for more information.",
{"pattern": pattern, "exception": traceback.format_exc()},
)
return
lmfilter = _create_formatter_filter(model, tokenizer, f)
# Append the filters
self.filters.append(lmfilter)
def add_kbnf_filter(
self,
kbnf_string: str,
model: ExLlamaV2,
tokenizer: ExLlamaV2Tokenizer,
):
"""Adds an ExllamaV2 filter based on KBNF grammar."""
# Create the parser
try:
# Validate KBNF and create formatter
f = FormatterBuilder()
f.append_line(
f"""{f.extractor(lambda nonterminal: CFGExtractor(nonterminal, kbnf_string))}"""
)
except Exception:
xlogger.error(
"Skipping because the KBNF string couldn't be parsed. "
"Please read the above error for more information.",
{"kbnf_string": kbnf_string, "exception": traceback.format_exc()},
)
return
lmfilter = _create_formatter_filter(model, tokenizer, f)
# Append the filters
self.filters.append(lmfilter)
class CFGExtractor(NonterminalExtractor):
"""Extractor class for KBNF context-free grammar"""
def __init__(self, nonterminal: str, kbnf_string: str):
super().__init__(nonterminal)
self.kbnf_string = kbnf_string
# Return the entire input string as the extracted string
def extract(self, input_str: str) -> typing.Optional[tuple[str, typing.Any]]:
return "", input_str
@property
def kbnf_definition(self) -> str:
return self.kbnf_string.replace("start", self.nonterminal)
@lru_cache(1)
def _create_cached_engine_vocabulary(tokenizer: ExLlamaV2Tokenizer):
"""Build and cache engine vocabulary on first grammar run"""
return create_engine_vocabulary(tokenizer)
def _create_formatter_filter(
model: ExLlamaV2, tokenizer: ExLlamaV2Tokenizer, formatter_builder: FormatterBuilder
) -> ExLlamaV2Filter:
"""
Create a formatter filter for the ExLlamaV2 engine.
Minimalist clone of formatron.integrations.exllamav2.create_formatter_filter
with lru_cache enabled for engine vocabulary
"""
vocab = _create_cached_engine_vocabulary(tokenizer)
f = formatter_builder.build(vocab, lambda tokens: tokenizer.decode(torch.tensor(tokens)))
return FormatterFilter(model, tokenizer, f)
def clear_grammar_func_cache():
"""Flush tokenizer_data cache to avoid holding references to
tokenizers after unloading a model"""
_create_cached_engine_vocabulary.cache_clear()

1512
backends/exllamav2/model.py
View File

File diff suppressed because it is too large Load Diff

16
backends/exllamav2/utils.py
View File

@@ -1,16 +0,0 @@
from common.logger import xlogger
def exllama_disabled_flash_attn(no_flash_attn: bool):
unsupported_message = (
"ExllamaV2 has disabled Flash Attention. \n"
"Please see the above logs for warnings/errors. \n"
"Switching to compatibility mode. \n"
"This disables parallel batching "
"and features that rely on it (ex. CFG). \n"
)
if no_flash_attn:
xlogger.warning(unsupported_message)
return no_flash_attn

28
backends/exllamav2/vision.py
View File

@@ -1,28 +0,0 @@
"""Vision utilities for ExLlamaV2."""
from async_lru import alru_cache
from common import model
from common.optional_dependencies import dependencies
from common.image_util import get_image
# Since this is used outside the Exl2 backend, the dependency
# may be optional
if dependencies.exllamav2:
from exllamav2.generator import ExLlamaV2MMEmbedding
# Fetch the return type on runtime
@alru_cache(20)
async def get_image_embedding_exl2(url: str) -> "ExLlamaV2MMEmbedding":
image = await get_image(url)
return model.container.vision_model.get_image_embeddings(
model=model.container.model,
tokenizer=model.container.tokenizer,
image=image,
text_alias=None,
)
def clear_image_embedding_cache():
get_image_embedding_exl2.cache_clear()

36
backends/exllamav3/model.py
View File

@@ -24,7 +24,6 @@ from exllamav3 import (
from exllamav3.cache import CacheLayer_quant
from backends.exllamav3.grammar import ExLlamaV3Grammar
from backends.base_model_container import BaseModelContainer
from backends.exllamav3.sampler import ExllamaV3SamplerBuilder
from backends.exllamav3.utils import exllama_supports_nccl
from backends.exllamav3.vision import clear_image_embedding_cache
@@ -52,12 +51,17 @@ from endpoints.OAI.utils.tools import is_supported_format
import inspect
class ExllamaV3Container(BaseModelContainer):
"""Abstract base class for model containers."""
class ExllamaV3Container:
"""Model container for the ExLlamaV3 backend."""
# Exposed model information
model_dir: pathlib.Path = pathlib.Path("models")
prompt_template: Optional[PromptTemplate] = None
tool_format: Optional[str] = None
# Optional features
use_draft_model: bool = False
use_vision: bool = False
# HF Model instance
hf_model: HFModel
@@ -496,17 +500,20 @@ class ExllamaV3Container(BaseModelContainer):
while len(self.active_job_ids) > 0:
await asyncio.sleep(0.01)
async def load(self, progress_callback=None, **kwargs):
"""
Loads the model into memory.
# TODO: Wire up exllamav3's LoRA support once the API surface for it is decided
async def load_loras(self, lora_directory: pathlib.Path, **kwargs) -> Dict[str, List[str]]:
"""Stub. LoRAs aren't hooked up to the ExLlamaV3 backend yet."""
Args:
progress_callback: Optional callback for progress updates.
**kwargs: Additional loading options.
"""
xlogger.error("LoRA loading is not hooked up to the ExLlamaV3 backend yet.")
return {
"success": [],
"failure": [lora.get("name", "unknown") for lora in kwargs.get("loras", [])],
}
async for _ in self.load_gen(progress_callback):
pass
def get_loras(self) -> List[Any]:
"""Stub. LoRAs aren't hooked up to the ExLlamaV3 backend yet."""
return []
async def load_gen(self, progress_callback=None, **kwargs):
"""
@@ -638,6 +645,11 @@ class ExllamaV3Container(BaseModelContainer):
**kwargs: Additional unloading options (e.g., shutdown).
"""
# Nothing to do for LoRA-only unloads until LoRAs are hooked up
if loras_only:
xlogger.error("LoRA unloading is not hooked up to the ExLlamaV3 backend yet.")
return
# Used when shutting down the server
do_shutdown = kwargs.get("shutdown")

11
common/config_models.py
View File

@@ -170,8 +170,7 @@ class ModelConfig(BaseConfigModel):
backend: Optional[str] = Field(
None,
description=(
"Backend to use for this model (auto-detect if not specified)\n"
"Options: exllamav2, exllamav3"
"Backend to use for this model (auto-detect if not specified)\nOptions: exllamav3"
),
)
max_seq_len: Optional[int] = Field(
@@ -195,9 +194,9 @@ class ModelConfig(BaseConfigModel):
"FP16",
description=(
"Enable different cache modes for VRAM savings (default: FP16).\n"
f"Possible values for exllamav2: {str(CACHE_SIZES)[15:-1]}.\n"
"For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits "
"are integers from 2-8 (i.e. 8,8)."
"Specify the pair k_bits,v_bits where k_bits and v_bits "
"are integers from 2-8 (i.e. 8,8).\n"
f"The legacy values {str(CACHE_SIZES)[15:-1]} are also accepted."
),
)
tensor_parallel: Optional[bool] = Field(
@@ -511,7 +510,7 @@ class DeveloperConfig(BaseConfigModel):
unsafe_launch: Optional[bool] = Field(
False,
description=(
"Skip Exllamav2 version check (default: False).\n"
"Skip ExLlamav3 version check (default: False).\n"
"WARNING: It's highly recommended to update your dependencies rather "
"than enabling this flag."
),

63
common/model.py
View File

@@ -10,9 +10,8 @@ from enum import Enum
from fastapi import HTTPException
from common.logger import xlogger
from ruamel.yaml import YAML
from typing import Dict, Optional
from typing import Optional
from backends.base_model_container import BaseModelContainer
from common.errors import ContextLengthExceededError, ContextLengthHTTPException
from common.logger import get_loading_progress_bar
from common.multimodal import MultimodalEmbeddingWrapper
@@ -23,23 +22,12 @@ from common.optional_dependencies import dependencies
from common.transformers_utils import HFModel
from common.utils import deep_merge_dict, unwrap
# Global variables for model container
container: Optional[BaseModelContainer] = None
embeddings_container = None
_BACKEND_REGISTRY: Dict[str, BaseModelContainer] = {}
if dependencies.exllamav2:
from backends.exllamav2.model import ExllamaV2Container
_BACKEND_REGISTRY["exllamav2"] = ExllamaV2Container
if dependencies.exllamav3:
from backends.exllamav3.model import ExllamaV3Container
_BACKEND_REGISTRY["exllamav3"] = ExllamaV3Container
# Global variables for model container
container: Optional["ExllamaV3Container"] = None
embeddings_container = None
if dependencies.extras:
@@ -60,15 +48,25 @@ def load_progress(module, modules):
yield module, modules
def detect_backend(hf_model: HFModel) -> str:
"""Determine the appropriate backend based on model files and configuration."""
def validate_backend(backend: Optional[str], hf_model: HFModel):
"""Check that the requested model can be loaded with the exllamav3 backend."""
if backend == "exllamav2":
raise ValueError("The exllamav2 backend is no longer supported. Please use exllamav3.")
elif backend and backend != "exllamav3":
raise ValueError(f"Invalid backend '{backend}'. Available backends: ['exllamav3']")
quant_method = hf_model.quant_method()
if quant_method in {"exl2", "gptq"}:
raise ValueError(
f"Models quantized with '{quant_method}' require the exllamav2 backend, "
"which is no longer supported. Please use an exl3 or unquantized model."
)
if quant_method == "exl3":
return "exllamav3"
else:
return "exllamav2"
if not dependencies.exllamav3:
raise ValueError(
"The exllamav3 backend is selected, but required dependencies are not installed."
)
async def apply_load_defaults(model_path: pathlib.Path, **kwargs):
@@ -179,25 +177,10 @@ async def load_model_gen(model_path: pathlib.Path, **kwargs):
if max_seq_len == -1:
kwargs["max_seq_len"] = hf_model.hf_config.get_max_position_embeddings()
# Create a new container and check if the right dependencies are installed
backend = unwrap(kwargs.get("backend"), detect_backend(hf_model))
container_class = _BACKEND_REGISTRY.get(backend)
# Check model compatibility and dependencies before creating a container
validate_backend(kwargs.get("backend"), hf_model)
if not container_class:
available_backends = list(_BACKEND_REGISTRY.keys())
if backend in {"exllamav2", "exllamav3"}:
raise ValueError(
f"Backend '{backend}' selected, but required dependencies are not installed."
)
else:
raise ValueError(
f"Invalid backend '{backend}'. Available backends: {available_backends}"
)
xlogger.info(f"Using backend {backend}")
new_container: BaseModelContainer = await container_class.create(
model_path.resolve(), hf_model, **kwargs
)
new_container = await ExllamaV3Container.create(model_path.resolve(), hf_model, **kwargs)
# Add possible types of models that can be loaded
model_type = [ModelType.MODEL]

15
common/multimodal.py
View File

@@ -1,4 +1,3 @@
from backends.exllamav2.vision import get_image_embedding_exl2
from backends.exllamav3.vision import get_image_embedding_exl3
from common import model
from common.logger import xlogger
@@ -7,8 +6,6 @@ from typing import List
from common.optional_dependencies import dependencies
if dependencies.exllamav2:
from exllamav2 import ExLlamaV2VisionTower
if dependencies.exllamav3:
from exllamav3 import Model
@@ -23,19 +20,11 @@ class MultimodalEmbeddingWrapper(BaseModel):
async def add(self, url: str):
# Determine the type of vision embedding to use
if not self.type:
if dependencies.exllamav2 and isinstance(
model.container.vision_model, ExLlamaV2VisionTower
):
self.type = "ExLlamaV2MMEmbedding"
elif dependencies.exllamav3 and isinstance(model.container.vision_model, Model):
if dependencies.exllamav3 and isinstance(model.container.vision_model, Model):
self.type = "MMEmbedding"
# Create the embedding
if self.type == "ExLlamaV2MMEmbedding":
embedding = await get_image_embedding_exl2(url)
self.content.append(embedding)
self.text_alias.append(embedding.text_alias)
elif self.type == "MMEmbedding":
if self.type == "MMEmbedding":
embedding = await get_image_embedding_exl3(url)
self.content.append(embedding)
self.text_alias.append(embedding.text_alias)

19
common/optional_dependencies.py
View File

@@ -1,7 +1,6 @@
"""Construct a model of all optional dependencies"""
import importlib.util
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as package_version
from common.logger import xlogger
from packaging import version
@@ -16,7 +15,6 @@ class DependenciesModel(BaseModel):
"""Model of which optional dependencies are installed."""
torch: bool
exllamav2: bool
exllamav3: bool
flash_attn: bool
infinity_emb: bool
@@ -30,7 +28,7 @@ class DependenciesModel(BaseModel):
@computed_field
@property
def inference(self) -> bool:
return self.torch and (self.exllamav2 or self.exllamav3)
return self.torch and self.exllamav3
def is_installed(package_name: str) -> bool:
@@ -40,18 +38,6 @@ def is_installed(package_name: str) -> bool:
return spec is not None
def is_torch_cuda_13() -> bool:
"""Check whether the installed Torch wheel targets CUDA 13."""
try:
torch_version = package_version("torch")
except PackageNotFoundError:
return False
_, _, local_version = torch_version.partition("+")
return local_version.startswith("cu13")
def get_installed_deps() -> DependenciesModel:
"""Check if optional dependencies are installed by looping over the fields."""
@@ -62,9 +48,6 @@ def get_installed_deps() -> DependenciesModel:
for field_name in fields.keys():
installed_deps[field_name] = is_installed(field_name)
if installed_deps.get("exllamav2") and is_torch_cuda_13():
installed_deps["exllamav2"] = False
return DependenciesModel(**installed_deps)

14
config_sample.yml
View File

@@ -81,7 +81,7 @@ model:
use_as_default: []
# Backend to use for this model (auto-detect if not specified)
# Options: exllamav2, exllamav3
# Options: exllamav3
backend:
# Max sequence length (default: min(max_position_embeddings, cache_size)).
@@ -90,13 +90,11 @@ model:
# Size of the key/value cache to allocate, in tokens (default: 4096).
# Must be a multiple of 256.
# ExllamaV2 note: On AMD GPUs and NVIDIA GPUs older than Ampere, this value
# is ignored. Please use max_seq_len
cache_size:
# Enable different cache modes for VRAM savings (default: FP16).
# Possible values for exllamav2: 'FP16', 'Q8', 'Q6', 'Q4'.
# For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
# Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
# The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
cache_mode: FP16
# Load model with tensor parallelism.
@@ -213,8 +211,8 @@ draft_model:
draft_rope_alpha:
# Cache mode for draft models to save VRAM (default: FP16).
# Possible values for exllamav2: 'FP16', 'Q8', 'Q6', 'Q4'.
# For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
# Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
# The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
draft_cache_mode: FP16
# Array of VRAM sizes to split between GPUs, in GB (default: []).
@@ -281,7 +279,7 @@ memory:
# Options for development and experimentation
developer:
# Skip Exllamav2 version check (default: False).
# Skip Exllamav3 version check (default: False).
# WARNING: It's highly recommended to update your dependencies rather than enabling this flag.
unsafe_launch: false

2
docker/Dockerfile
View File

@@ -26,7 +26,7 @@ WORKDIR /app
# Get requirements
COPY pyproject.toml .
# Install cu12 group first — pins torch+cu128, exllamav2/v3+cu128,
# Install cu12 group first — pins torch+cu128, exllamav3+cu128,
# flash_attn+cu128, and flash-linear-attention.
# The 'extras' group (infinity-emb, sentence-transformers) is installed separately
# with --no-deps so pip cannot resolve xformers transitively and pull a cu130 wheel,

2
docker/Dockerfile.cu13
View File

@@ -27,7 +27,7 @@ WORKDIR /app
COPY pyproject.toml .
# Install cu13 group first. This uses torch 2.11.0+cu130 with exllamav3 cu132
# wheels, flash_attn+cu130, and intentionally does not install exllamav2.
# wheels and flash_attn+cu130.
# The 'extras' group (infinity-emb, sentence-transformers) is installed separately
# with --no-deps so pip cannot resolve xformers transitively and pull incompatible
# CUDA wheels.

37
docs/01.-Getting-Started.md
View File

@@ -12,12 +12,10 @@ To get started, make sure you have the following installed on your system:
> Prefer a video guide? Watch the step-by-step tutorial on [YouTube](https://www.youtube.com/watch?v=03jYz0ijbUU)
> [!WARNING]
> CUDA and ROCm aren't prerequisites because torch can install them for you. However, if this doesn't work (ex. DLL load failed), install the CUDA toolkit or ROCm on your system.
> CUDA isn't a prerequisite because torch can install it for you. However, if this doesn't work (ex. DLL load failed), install the CUDA toolkit on your system.
>
> - [CUDA 12.x](https://developer.nvidia.com/cuda-downloads)
>
> - [ROCm 6.1](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.1.0/how-to/prerequisites.html)
>
> [!WARNING]
> Sometimes there may be an error with Windows that VS build tools needs to be installed. This means that there's a package that isn't supported for your python version.
@@ -48,8 +46,7 @@ To get started, make sure you have the following installed on your system:
2. On Linux: `source venv/bin/activate`
3. Install the pyproject features based on your system:
1. Cuda 12.x: `pip install -U .[cu12]`
2. Cuda 13.x (Exllamav3 only, Python 3.12+): `pip install -U .[cu13]`
3. ROCm 5.6: `pip install -U .[amd]`
2. Cuda 13.x (Python 3.12+): `pip install -U .[cu13]`
4. Start the API by either
1. Run `start.bat/sh`. The script will check if you're in a conda environment and skip venv checks.
2. Run `python main.py` to start the API. This won't automatically upgrade your dependencies.
@@ -62,9 +59,9 @@ TabbyAPI includes a built-in Hugging Face downloader that works via both the API
Example with Turboderp's Llama 3.1 8B quants:
`.\Start.bat download turboderp/Qwen2.5-VL-7B-Instruct-exl2 --revision 4.0bpw`
`.\Start.bat download turboderp/Qwen2.5-VL-7B-Instruct-exl3 --revision 4.0bpw`
If a model is gated, you can provide a HuggingFace access token (most exl2 quants aren't private):
If a model is gated, you can provide a HuggingFace access token (most exl3 quants aren't private):
`.\Start.bat download meta-llama/Llama-3.1-8B --token <token>`
@@ -100,33 +97,31 @@ These scripts exit after running their respective tasks. To start TabbyAPI, run
2. **Manual** - Install the pyproject features and update dependencies depending on your GPU:
1. `pip install -U .[cu12]` = CUDA 12.x
2. `pip install -U .[cu13]` = CUDA 13.x (Exllamav3 only, Python 3.12+)
3. `pip install -U .[amd]` = ROCm 6.0
2. `pip install -U .[cu13]` = CUDA 13.x (Python 3.12+)
If you don't want to update dependencies that come from wheels (torch, exllamav2, exllamav3, and flash attention 2), use `pip install .` or pass the `--nowheel` flag when invoking the start scripts.
If you don't want to update dependencies that come from wheels (torch, exllamav3, and flash attention 2), use `pip install .` or pass the `--nowheel` flag when invoking the start scripts.
### Update Exllamav2
### Update Exllamav3
> [!WARNING]
> These instructions are meant for advanced users.
> [!IMPORTANT]
> If you're installing a custom Exllamav2 wheel, make sure to use `pip install .` when updating! Otherwise, each update will overwrite your custom exllamav2 version.
> If you're installing a custom Exllamav3 wheel, make sure to use `pip install .` when updating! Otherwise, each update will overwrite your custom exllamav3 version.
NOTE:
- TabbyAPI enforces the latest Exllamav2 version for compatibility purposes.
- TabbyAPI enforces the latest Exllamav3 version for compatibility purposes.
- Any upgrades using a pyproject gpu lib feature will result in overwriting your installed wheel.
- To fix this, change the feature in `pyproject.toml` locally, create an issue or PR, or install your version of exllamav2 after upgrades.
- To fix this, change the feature in `pyproject.toml` locally, create an issue or PR, or install your version of exllamav3 after upgrades.
Here are ways to install exllamav2:
Here are ways to install exllamav3:
1. From a [wheel/release](https://github.com/turboderp/exllamav2#method-2-install-from-release-with-prebuilt-extension) (Recommended)
1. Find the version that corresponds with your cuda and python version. For example, a wheel with `cu121` and `cp311` corresponds to CUDA 12.1 and python 3.11
2. From [pip](https://github.com/turboderp/exllamav2#method-3-install-from-pypi): `pip install exllamav2`
2. This is a JIT compiled extension, which means that the initial launch of tabbyAPI will take some time. The build may also not work due to improper environment configuration.
3. From [source](https://github.com/turboderp/exllamav2#method-1-install-from-source)
1. From a [wheel/release](https://github.com/turboderp-org/exllamav3/releases) (Recommended)
1. Find the version that corresponds with your cuda and python version. For example, a wheel with `cu128` and `cp312` corresponds to CUDA 12.8 and python 3.12
2. From [source](https://github.com/turboderp-org/exllamav3#installation)
1. This builds the extension during installation, which may take some time. The build may also not work due to improper environment configuration.
## Other installation methods
@@ -165,7 +160,7 @@ These are short-form instructions for other methods that users can use to instal
### Docker
> [!NOTE]
> If you are planning to use custom versions of dependencies such as dev ExllamaV2, make sure to build the Docker image yourself!
> If you are planning to use custom versions of dependencies such as dev ExllamaV3, make sure to build the Docker image yourself!
1. Install Docker and docker compose from the [docs](https://docs.docker.com/compose/install/)
2. Install the Nvidia container compatibility layer

16
docs/03.-Usage.md
View File

@@ -1,6 +1,6 @@
## Usage
TabbyAPI's main use-case is to be an API server for running ExllamaV2 models.
TabbyAPI's main use-case is to be an API server for running ExllamaV3 models.
### API Server
@@ -17,7 +17,7 @@ Below is an example CURL request using the OpenAI completions endpoint:
curl http://localhost:5000/v1/completions \
-H "Content-Type: application/json" \
-d '{
"model": "Meta-Llama-3-8B-exl2",
"model": "Meta-Llama-3-8B-exl3",
"prompt": "Once upon a time,",
"max_tokens": 400,
"stream": false,
@@ -81,7 +81,7 @@ Below is an example CURL request using the model load endpoint:
curl http://localhost:5000/v1/model/load \
-H "Content-Type: application/json" \
-d '{
"model_name": "Meta-Llama-3-8B-exl2",
"model_name": "Meta-Llama-3-8B-exl3",
"max_seq_len": 8192,
"tensor_parallel": true,
"gpu_split_auto": false,
@@ -96,10 +96,10 @@ A model load request can also include draft model parameters:
curl http://localhost:5000/v1/model/load \
-H "Content-Type: application/json" \
-d '{
"model_name": "Meta-Llama-3-8B-exl2",
"model_name": "Meta-Llama-3-8B-exl3",
... Other parameters
"draft_model": {
draft_model_name: "TinyLlama-1B-32k-exl2",
draft_model_name: "TinyLlama-1B-32k-exl3",
draft_rope_scale: 1.0
}
}'
@@ -116,7 +116,7 @@ An alternative way of switching models is called "inline loading" which hooks in
To get started, set `inline_model_loading` to `true` under the model block of config.yml.
Now to create a tabby config, let's say we have a model in our models directory called `Meta-Llama-3-8B-exl2`. Navigate into that model folder and create a file called `tabby_config.yml`
Now to create a tabby config, let's say we have a model in our models directory called `Meta-Llama-3-8B-exl3`. Navigate into that model folder and create a file called `tabby_config.yml`
Now, you can place any model load parameter from `/v1/model/load` into that file. Here's a simple example which changes the default `max_seq_len` to 8192 and sets a Q6 quantized cache:
@@ -133,7 +133,7 @@ model:
max_seq_len: 8192
cache_mode: Q6
draft_model:
draft_model_name: TinyLlama-1B-32k-exl2
draft_model_name: TinyLlama-1B-32k-exl3
draft_rope_scale: 1.0
```
@@ -145,7 +145,7 @@ Below is an example CURL request for inline loading:
curl http://localhost:5000/v1/completions \
-H "Content-Type: application/json" \
-d '{
"model": "Meta-Llama-3-8B-exl2"
"model": "Meta-Llama-3-8B-exl3"
... Other parameters
}'
```

13
docs/05.-FAQ.md
View File

@@ -9,15 +9,14 @@
- The wiki is meant for user-facing documentation. Devs are recommended to use the autogenerated documentation for [OpenAI](https://theroyallab.github.io/tabbyAPI) and [Kobold](https://theroyallab.github.io/tabbyAPI/kobold) servers
- What does TabbyAPI run?
- TabbyAPI uses Exllamav2 as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported:
- Exl2 (Highly recommended)
- GPTQ
- FP16 (using Exllamav2's loader)
- TabbyAPI uses Exllamav3 as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported:
- Exl3 (Highly recommended)
- FP16/BF16 (using Exllamav3's loader)
- Exllamav2 may error with the following exception: `ImportError: DLL load failed while importing exllamav2_ext: The specified module could not be found.`
- Exllamav3 may error with the following exception: `ImportError: DLL load failed while importing exllamav3_ext: The specified module could not be found.`
- First, make sure to check if the wheel is equivalent to your python version and CUDA version. Also make sure you're in a venv or conda environment.
- If those prerequisites are correct, the torch cache may need to be cleared. This is due to a mismatching exllamav2_ext.
- If those prerequisites are correct, the torch cache may need to be cleared. This is due to a mismatching exllamav3_ext.
- In Windows: Find the cache at `C:\Users\<User>\AppData\Local\torch_extensions\torch_extensions\Cache` where `<User>` is your Windows username
- In Linux: Find the cache at `~/.cache/torch_extensions`
- look for any folder named `exllamav2_ext` in the python subdirectories and delete them.
- look for any folder named `exllamav3_ext` in the python subdirectories and delete them.
- Restart TabbyAPI and launching should work again.

2
endpoints/server.py
View File

@@ -19,7 +19,7 @@ def setup_app(host: Optional[str] = None, port: Optional[int] = None):
app = FastAPI(
title="TabbyAPI",
summary="An OAI compatible exllamav2 API that's both lightweight and fast",
summary="An OAI compatible exllamav3 API that's both lightweight and fast",
description=(
"This docs page is not meant to send requests! Please use a service "
"like Postman or a frontend UI."

6
main.py
View File

@@ -159,7 +159,7 @@ def entrypoint(
# Skip if launching unsafely
if config.developer.unsafe_launch:
logger.warning(
"UNSAFE: Skipping ExllamaV2 version check.\n"
"UNSAFE: Skipping ExllamaV3 version check.\n"
"If you aren't a developer, please keep this off!"
)
elif not dependencies.inference:
@@ -170,12 +170,10 @@ def entrypoint(
f"update_deps.{'bat' if platform.system() == 'Windows' else 'sh'})\n\n"
"Or you can manually run a requirements update "
"using the following command:\n\n"
"For CUDA 12.1:\n"
"For CUDA 12.x:\n"
"pip install --upgrade .[cu12]\n\n"
"For CUDA 13.x:\n"
"pip install --upgrade .[cu13]\n\n"
"For ROCm:\n"
"pip install --upgrade .[amd]\n\n"
)
raise SystemExit(install_message)

32
pyproject.toml
View File

@@ -13,7 +13,7 @@ py-modules = []
[project]
name = "tabbyAPI"
version = "0.0.1"
description = "An OAI compatible exllamav2 API that's both lightweight and fast"
description = "An OAI compatible exllamav3 API that's both lightweight and fast"
requires-python = ">=3.10"
dependencies = [
"fastapi-slim >= 0.115",
@@ -79,16 +79,6 @@ cu12 = [
"xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33-cp39-abi3-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'",
"xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33-cp39-abi3-win_amd64.whl ; platform_system == 'Windows'",
# Exl2
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp313-cp313-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.13'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp312-cp312-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.12'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp311-cp311-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.11'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp310-cp310-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.10'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'",
# Exl3
"exllamav3 @ https://github.com/turboderp-org/exllamav3/releases/download/v0.0.43/exllamav3-0.0.43+cu128.torch2.9.0-cp313-cp313-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.13'",
"exllamav3 @ https://github.com/turboderp-org/exllamav3/releases/download/v0.0.43/exllamav3-0.0.43+cu128.torch2.9.0-cp312-cp312-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.12'",
@@ -145,26 +135,6 @@ cu13 = [
"flash_attn @ https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3%2Bcu130torch2.11-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'",
"flash_attn @ https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3%2Bcu130torch2.11-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'",
]
amd = [
# Torch triton for ROCm
"pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'",
"pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'",
"pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'",
"pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'",
# Torch
"torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp313-cp313-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'",
"torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp312-cp312-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'",
"torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp311-cp311-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'",
"torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp310-cp310-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'",
# Exl2
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'",
"exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'",
]
# MARK: Ruff options
[tool.ruff]

33
start.py
View File

@@ -2,7 +2,6 @@
import argparse
import json
import os
import pathlib
import platform
import subprocess
@@ -44,30 +43,22 @@ def get_user_choice(question: str, options_dict: dict):
def get_install_features(lib_name: str = None):
"""Fetches the appropriate requirements file depending on the GPU"""
install_features = None
possible_features = ["cu12", "cu13", "amd"]
possible_features = ["cu12", "cu13"]
if not lib_name:
has_nvidia = which("nvidia-smi") is not None
has_rocm = which("rocm-smi") is not None
has_amd = which("amd-smi") is not None
has_amd_gpu = has_rocm or has_amd
if has_nvidia and not has_amd_gpu:
if has_nvidia:
lib_name = "cu12"
print("Auto-detected NVIDIA GPU. Using CUDA 12.x backend.")
elif has_amd_gpu and not has_nvidia:
lib_name = "amd"
print("Auto-detected AMD GPU. Using AMD backend.")
else:
gpu_lib_choices = {
"A": {"pretty": "NVIDIA Cuda 12.x", "internal": "cu12"},
"B": {"pretty": "NVIDIA Cuda 13.x", "internal": "cu13"},
"C": {"pretty": "AMD", "internal": "amd"},
}
print(
"WARNING: Auto-detection failed. "
"Please ensure you have either an NVIDIA GPU (with nvidia-smi) "
"or an AMD GPU (with rocm-smi or amd-smi) installed."
"Please ensure you have an NVIDIA GPU (with nvidia-smi) installed."
)
user_input = get_user_choice(
"Select your GPU. If you don't know, select Cuda 12.x (A)",
@@ -92,20 +83,6 @@ def get_install_features(lib_name: str = None):
)
return
if install_features == "amd":
# Exit if using AMD and Windows
if platform.system() == "Windows":
print(
"ERROR: TabbyAPI does not support AMD and Windows. "
"Please use Linux and ROCm 6.4. Exiting."
)
sys.exit(0)
# Override env vars for ROCm support on non-supported GPUs
os.environ["ROCM_PATH"] = "/opt/rocm"
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.3.0"
os.environ["HCC_AMDGPU_TARGET"] = "gfx1030"
return install_features
@@ -148,12 +125,12 @@ def add_start_args(parser: argparse.ArgumentParser):
"-nw",
"--nowheel",
action="store_true",
help="Don't upgrade wheel dependencies (exllamav2, torch)",
help="Don't upgrade wheel dependencies (exllamav3, torch)",
)
start_group.add_argument(
"--gpu-lib",
type=str,
help="Select GPU library. Options: cu12, cu13, amd",
help="Select GPU library. Options: cu12, cu13",
)

10
tests/wheel_test.py
View File

@@ -12,20 +12,12 @@ if find_spec("flash_attn") is not None:
else:
print("Flash attention 2 is not found in your environment.")
if find_spec("exllamav2") is not None:
print(f"Exllamav2 on version {version('exllamav2')} successfully imported")
successful_packages.append("exllamav2")
else:
print("Exllamav2 is not found in your environment.")
if find_spec("exllamav3") is not None:
print(f"Exllamav3 on version {version('exllamav3')} successfully imported")
successful_packages.append("exllamav3")
else:
print("Exllamav3 is not found in your environment.")
if find_spec("exllamav2") is None and find_spec("exllamav3") is None:
errored_packages.append("exllamav2/exllamav3")
errored_packages.append("exllamav3")
if find_spec("torch") is not None:
print(f"Torch on version {version('torch')} successfully imported")