From c8b9a8f4aa86351a665db65dd492589cdee15ac7 Mon Sep 17 00:00:00 2001 From: turboderp <11859846+turboderp@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:40:33 +0200 Subject: [PATCH] Backends: Remove ExLlamaV2 backend and container abstraction layer Remove BaseModelContainer abstraction LoRA endpoints remain as stubs (supported in exllamav3, but API is undecided) Fix /v1/lora/unload unloading the entire model. The last commit with exllamav2 support is preserved on the exl2-checkpoint branch. --- README.md | 11 +- backends/base_model_container.py | 277 ------ backends/exllamav2/grammar.py | 158 ---- backends/exllamav2/model.py | 1512 ------------------------------ backends/exllamav2/utils.py | 16 - backends/exllamav2/vision.py | 28 - backends/exllamav3/model.py | 36 +- common/config_models.py | 11 +- common/model.py | 63 +- common/multimodal.py | 15 +- common/optional_dependencies.py | 19 +- config_sample.yml | 14 +- docker/Dockerfile | 2 +- docker/Dockerfile.cu13 | 2 +- docs/01.-Getting-Started.md | 37 +- docs/03.-Usage.md | 16 +- docs/05.-FAQ.md | 13 +- endpoints/server.py | 2 +- main.py | 6 +- pyproject.toml | 32 +- start.py | 33 +- tests/wheel_test.py | 10 +- 22 files changed, 110 insertions(+), 2203 deletions(-) delete mode 100644 backends/base_model_container.py delete mode 100644 backends/exllamav2/grammar.py delete mode 100644 backends/exllamav2/model.py delete mode 100644 backends/exllamav2/utils.py delete mode 100644 backends/exllamav2/vision.py diff --git a/README.md b/README.md index a637563..7135054 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ > > In addition to the README, please read the [Wiki](https://github.com/theroyallab/tabbyAPI/wiki/1.-Getting-Started) page for information about getting started! +> [!NOTE] +> +> ExLlamaV2 models are no longer supported in the `main` branch. The last commit with ExLlamav2 +> support is preserved on the `exl2-checkpoint` branch. + > [!NOTE] > > Need help? Join the [Discord Server](https://discord.gg/sYQxnuD7Fj) and get the `Tabby` role. Please be nice when asking questions. @@ -38,9 +43,9 @@ > > Want to run GGUF models? Take a look at [YALS](https://github.com/theroyallab/YALS), TabbyAPI's sister project. -A FastAPI based application that allows for generating text using an LLM (large language model) using the [Exllamav2](https://github.com/turboderp-org/exllamav2) and [Exllamav3](https://github.com/turboderp-org/exllamav3) backends. +A FastAPI based application that allows for generating text using an LLM (large language model) using the [Exllamav3](https://github.com/turboderp-org/exllamav3) backend. -TabbyAPI is also the official API backend server for ExllamaV2 and V3. +TabbyAPI is also the official API backend server for ExllamaV3. ## Disclaimer @@ -97,8 +102,6 @@ And much more. If something is missing here, PR it in! TabbyAPI uses Exllama as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported: -- Exl2/GPTQ (deprecated, will be removed in the near future) - - Exl3 (Highly recommended) - FP16/BF16 diff --git a/backends/base_model_container.py b/backends/base_model_container.py deleted file mode 100644 index 320000d..0000000 --- a/backends/base_model_container.py +++ /dev/null @@ -1,277 +0,0 @@ -import abc -import asyncio -import pathlib -from loguru import logger -from typing import ( - Any, - AsyncIterator, - Dict, - List, - Optional, -) -from common.multimodal import MultimodalEmbeddingWrapper -from common.errors import ContextLengthExceededError -from common.sampling import BaseSamplerRequest -from common.templating import PromptTemplate -from common.transformers_utils import HFModel -from common.utils import unwrap -from endpoints.core.types.model import ModelCard - - -class BaseModelContainer(abc.ABC): - """Abstract base class for model containers.""" - - # Exposed model information - model_dir: pathlib.Path = pathlib.Path("models") - prompt_template: Optional[PromptTemplate] = None - tool_format: Optional[str] = None - - # HF Model instance - hf_model: HFModel - - # Optional features - use_draft_model: bool = False - use_vision: bool = False - - # Load synchronization - # The bool is a master switch for accepting requests - # The lock keeps load tasks sequential - # The condition notifies any waiting tasks - active_job_ids: Dict[str, Any] = {} - loaded: bool = False - load_lock: asyncio.Lock - load_condition: asyncio.Condition - - reasoning: bool - reasoning_start_token: Optional[str] - reasoning_end_token: Optional[str] - reasoning_suppress_header: Optional[str] - force_enable_thinking: Optional[str] - - # Required methods - @classmethod - @abc.abstractmethod - async def create(cls, model_directory: pathlib.Path, hf_model: HFModel, **kwargs): - """ - Asynchronously creates and initializes a model container instance. - - Args: - model_directory: Path to the model files. - hf_model: HF config.json wrapper. - **kwargs: Backend-specific configuration options. - - Returns: - An instance of the implementing class. - """ - - pass - - @abc.abstractmethod - async def load(self, progress_callback=None, **kwargs): - """ - Loads the model into memory. - - Args: - progress_callback: Optional callback for progress updates. - **kwargs: Additional loading options. - """ - - pass - - # NOTE: Might be an optional method - @abc.abstractmethod - async def load_gen(self, progress_callback=None, **kwargs): - """ - Loads the model into memory, yielding progress updates. - - Args: - progress_callback: Optional callback for progress updates. - **kwargs: Additional loading options. - - Yields: - Progress updates - """ - - if False: - yield - - @abc.abstractmethod - async def unload(self, loras_only: bool = False, **kwargs): - """ - Unloads the model and associated resources from memory. - - Args: - loras_only: If True, only unload LoRAs. - **kwargs: Additional unloading options (e.g., shutdown). - """ - - pass - - @abc.abstractmethod - def encode_tokens(self, text: str, **kwargs) -> List[int]: - """ - Encodes a string of text into a list of token IDs. - - Args: - text: The input text string. - **kwargs: Backend-specific encoding options (e.g., add_bos_token). - - Returns: - A list of integer token IDs. - """ - - pass - - @abc.abstractmethod - def decode_tokens(self, ids: List[int], **kwargs) -> str: - """ - Decodes a list of token IDs back into a string. - - Args: - ids: A list of integer token IDs. - **kwargs: Backend-specific decoding options (e.g., decode_special_tokens). - - Returns: - The decoded text string. - """ - - pass - - @abc.abstractmethod - def get_special_tokens(self) -> Dict[str, Any]: - """ - Gets special tokens used by the model/tokenizer. - - Returns: - A dictionary mapping special token names (e.g., 'bos_token', 'eos_token') - to their string or ID representation. - """ - - pass - - @abc.abstractmethod - def model_info(self) -> ModelCard: - """ - Returns a dictionary of the current model's configuration parameters. - - Returns: - Model parameters provided by the backend - """ - - pass - - @abc.abstractmethod - async def wait_for_jobs(self, skip_wait: bool = False): - """ - Waits for any active generation jobs to complete. - - Args: - skip_wait: If True, cancel jobs immediately instead of waiting. - """ - - pass - - # Optional methods - async def load_loras(self, lora_directory: pathlib.Path, **kwargs) -> Dict[str, List[str]]: - """ - Loads LoRA adapters. Base implementation does nothing or raises error. - - Args: - lora_directory: Path to the directory containing LoRA files. - **kwargs: LoRA configuration (e.g., list of loras, scaling). - - Returns: - A dictionary indicating success/failure for each LoRA. - """ - - logger.warning("LoRA loading not implemented for this backend.") # type: ignore - return { - "success": [], - "failure": [lora.get("name", "unknown") for lora in kwargs.get("loras", [])], - } - - def get_loras(self) -> List[Any]: - """ - Gets the currently loaded LoRA adapters. Base implementation returns empty list. - - Returns: - A list representing the loaded LoRAs (backend-specific format). - """ - - return [] - - def validate_context_length( - self, - prompt: str, - params: BaseSamplerRequest, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - ): - """Validate a prompt before starting a streaming HTTP response.""" - - context_len = len( - self.encode_tokens( - prompt, - add_bos_token=unwrap(params.add_bos_token, self.hf_model.add_bos_token()), - embeddings=mm_embeddings, - ) - ) - max_seq_len = self.model_info().parameters.max_seq_len - if context_len > max_seq_len: - raise ContextLengthExceededError( - f"Prompt length {context_len} exceeds the available context size " - f"of {max_seq_len} tokens" - ) - - @abc.abstractmethod - async def generate( - self, - request_id: str, - prompt: str, - params: BaseSamplerRequest, - abort_event: Optional[asyncio.Event] = None, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - ) -> Dict[str, Any]: - """ - Generates a complete response for a given prompt and parameters. - - Args: - request_id: Unique identifier for the generation request. - prompt: The input prompt string. - params: Sampling and generation parameters. - abort_event: An asyncio Event to signal cancellation. - mm_embeddings: Optional multimodal embeddings. - - Returns: - A dictionary containing the generation info - """ - - pass - - @abc.abstractmethod - async def stream_generate( - self, - request_id: str, - prompt: str, - params: BaseSamplerRequest, - abort_event: Optional[asyncio.Event] = None, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - filter_trigger: str = None, - ) -> AsyncIterator[Dict[str, Any]]: - """ - Generates a response iteratively (streaming) for a given prompt. - - Args: - request_id: Unique identifier for the generation request. - prompt: The input prompt string. - params: Sampling and generation parameters. - abort_event: An asyncio Event to signal cancellation. - mm_embeddings: Optional multimodal embeddings. - filter_trigger: Delay filters (from params) until trigger text. - Must map to single token. - - Yields: - Generation chunks - """ - - if False: - yield diff --git a/backends/exllamav2/grammar.py b/backends/exllamav2/grammar.py deleted file mode 100644 index 1221b04..0000000 --- a/backends/exllamav2/grammar.py +++ /dev/null @@ -1,158 +0,0 @@ -import traceback -import typing -from functools import lru_cache -from typing import List - -import torch -from exllamav2 import ExLlamaV2, ExLlamaV2Tokenizer -from exllamav2.generator.filters import ExLlamaV2Filter -from formatron.extractor import NonterminalExtractor -from formatron.formatter import FormatterBuilder -from formatron.integrations.exllamav2 import FormatterFilter, create_engine_vocabulary -from formatron.schemas import json_schema -from common.logger import xlogger - - -class ExLlamaV2Grammar: - """ExLlamaV2 class for various grammar filters/parsers.""" - - filters: List[ExLlamaV2Filter] - - def __init__(self): - self.filters = [] - - def add_json_schema_filter( - self, - schema: dict, - model: ExLlamaV2, - tokenizer: ExLlamaV2Tokenizer, - ): - """Adds an ExllamaV2 filter based on a JSON schema.""" - - # Create the parser - try: - # Add fields required by formatron if not present - if "$id" not in schema: - schema["$id"] = "https://example.com/example.json" - if "$schema" not in schema: - schema["$schema"] = "http://json-schema.org/draft-07/schema#" - - # Validate schema and create formatter - schema = json_schema.create_schema(schema) - f = FormatterBuilder() - f.append_line(f"{f.json(schema)}") - except Exception: - traceback.print_exc() - xlogger.error( - "Skipping because the JSON schema couldn't be parsed. " - "Please read the above error for more information.", - {"schema": schema, "exception": traceback.format_exc()}, - ) - - return - - lmfilter = _create_formatter_filter(model, tokenizer, f) - - # Append the filters - self.filters.append(lmfilter) - - def add_regex_filter( - self, - pattern: str, - model: ExLlamaV2, - tokenizer: ExLlamaV2Tokenizer, - ): - """Adds an ExllamaV2 filter based on regular expressions.""" - - # Create the parser - try: - # Validate regex and create formatter - f = FormatterBuilder() - f.append_line(f"{f.regex(pattern)}") - except Exception: - traceback.print_exc() - xlogger.error( - "Skipping because the regex pattern couldn't be parsed. " - "Please read the above error for more information.", - {"pattern": pattern, "exception": traceback.format_exc()}, - ) - - return - - lmfilter = _create_formatter_filter(model, tokenizer, f) - - # Append the filters - self.filters.append(lmfilter) - - def add_kbnf_filter( - self, - kbnf_string: str, - model: ExLlamaV2, - tokenizer: ExLlamaV2Tokenizer, - ): - """Adds an ExllamaV2 filter based on KBNF grammar.""" - - # Create the parser - try: - # Validate KBNF and create formatter - f = FormatterBuilder() - f.append_line( - f"""{f.extractor(lambda nonterminal: CFGExtractor(nonterminal, kbnf_string))}""" - ) - except Exception: - xlogger.error( - "Skipping because the KBNF string couldn't be parsed. " - "Please read the above error for more information.", - {"kbnf_string": kbnf_string, "exception": traceback.format_exc()}, - ) - - return - - lmfilter = _create_formatter_filter(model, tokenizer, f) - - # Append the filters - self.filters.append(lmfilter) - - -class CFGExtractor(NonterminalExtractor): - """Extractor class for KBNF context-free grammar""" - - def __init__(self, nonterminal: str, kbnf_string: str): - super().__init__(nonterminal) - self.kbnf_string = kbnf_string - - # Return the entire input string as the extracted string - def extract(self, input_str: str) -> typing.Optional[tuple[str, typing.Any]]: - return "", input_str - - @property - def kbnf_definition(self) -> str: - return self.kbnf_string.replace("start", self.nonterminal) - - -@lru_cache(1) -def _create_cached_engine_vocabulary(tokenizer: ExLlamaV2Tokenizer): - """Build and cache engine vocabulary on first grammar run""" - - return create_engine_vocabulary(tokenizer) - - -def _create_formatter_filter( - model: ExLlamaV2, tokenizer: ExLlamaV2Tokenizer, formatter_builder: FormatterBuilder -) -> ExLlamaV2Filter: - """ - Create a formatter filter for the ExLlamaV2 engine. - Minimalist clone of formatron.integrations.exllamav2.create_formatter_filter - with lru_cache enabled for engine vocabulary - """ - - vocab = _create_cached_engine_vocabulary(tokenizer) - f = formatter_builder.build(vocab, lambda tokens: tokenizer.decode(torch.tensor(tokens))) - return FormatterFilter(model, tokenizer, f) - - -def clear_grammar_func_cache(): - """Flush tokenizer_data cache to avoid holding references to - tokenizers after unloading a model""" - - _create_cached_engine_vocabulary.cache_clear() diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py deleted file mode 100644 index 56d630b..0000000 --- a/backends/exllamav2/model.py +++ /dev/null @@ -1,1512 +0,0 @@ -"""The model container class for ExLlamaV2 models.""" - -import asyncio -import gc -import math -import pathlib -import torch -from exllamav2 import ( - ExLlamaV2, - ExLlamaV2Config, - ExLlamaV2CacheBase, - ExLlamaV2Cache, - ExLlamaV2Cache_Q4, - ExLlamaV2Cache_Q6, - ExLlamaV2Cache_Q8, - ExLlamaV2Cache_TP, - ExLlamaV2Tokenizer, - ExLlamaV2Lora, - ExLlamaV2VisionTower, -) -from exllamav2.generator import ( - ExLlamaV2Sampler, - ExLlamaV2DynamicGeneratorAsync, - ExLlamaV2DynamicJobAsync, -) -from itertools import zip_longest -from typing import Dict, List, Optional - -from backends.base_model_container import BaseModelContainer -from backends.exllamav2.grammar import ( - ExLlamaV2Grammar, - clear_grammar_func_cache, -) -from backends.exllamav2.utils import exllama_disabled_flash_attn -from backends.exllamav2.vision import clear_image_embedding_cache -from common.concurrency import iterate_in_threadpool -from common.gen_logging import ( - log_generation_params, - log_metrics, - log_prompt, - log_response, -) -from common.errors import ContextLengthExceededError -from common.logger import xlogger -from common.hardware import hardware_supports_flash_attn -from common.health import HealthManager -from common.multimodal import MultimodalEmbeddingWrapper -from common.networking import DisconnectHandler -from common.optional_dependencies import check_package_version -from common.sampling import BaseSamplerRequest -from common.templating import PromptTemplate, find_prompt_template -from common.transformers_utils import HFModel -from common.utils import calculate_rope_alpha, coalesce, unwrap -from endpoints.core.types.model import ModelCard, ModelCardParameters -from endpoints.OAI.utils.tools import is_supported_format - - -class ExllamaV2Container(BaseModelContainer): - """The model container class for ExLlamaV2 models.""" - - # Model directories - model_dir: pathlib.Path = pathlib.Path("models") - draft_model_dir: pathlib.Path = pathlib.Path("models") - prompt_template: Optional[PromptTemplate] = None - - # HF model instance - hf_model: HFModel - - # Exl2 vars - config: Optional[ExLlamaV2Config] = None - model: Optional[ExLlamaV2] = None - cache: Optional[ExLlamaV2Cache] = None - tokenizer: Optional[ExLlamaV2Tokenizer] = None - generator: Optional[ExLlamaV2DynamicGeneratorAsync] = None - prompt_template: Optional[PromptTemplate] = None - paged: bool = True - - # Draft model vars - use_draft_model: bool = False - draft_config: Optional[ExLlamaV2Config] = None - draft_model: Optional[ExLlamaV2] = None - draft_cache: Optional[ExLlamaV2Cache] = None - - # Internal config vars - cache_size: int = None - cache_mode: str = "FP16" - draft_cache_mode: str = "FP16" - max_batch_size: Optional[int] = None - - # GPU split vars - gpu_split: List[float] = [] - draft_gpu_split: List[float] = [] - gpu_split_auto: bool = True - autosplit_reserve: List[float] = [96 * 1024**2] - use_tp: bool = False - - # Vision vars - use_vision: bool = False - vision_model: Optional[ExLlamaV2VisionTower] = None - - # Load synchronization - active_job_ids: Dict[str, Optional[ExLlamaV2DynamicJobAsync]] = {} - loaded: bool = False - load_lock: asyncio.Lock = asyncio.Lock() - load_condition: asyncio.Condition = asyncio.Condition() - - @classmethod - async def create(cls, model_directory: pathlib.Path, hf_model: HFModel, **kwargs): - """ - Primary asynchronous initializer for model container. - - Kwargs are located in config_sample.yml - """ - - _hf = hf_model.hf_config - _tok = hf_model.tokenizer_config - _gen = hf_model.generation_config - xlogger.debug( - "Creating ExLlamaV2 model instance", - { - "kwargs": kwargs, - "hf_config": _hf.model_dump(mode="json") if _hf else {}, - "tokenizer_config": _tok.model_dump(mode="json") if _tok else {}, - "generation_config": _gen.model_dump(mode="json") if _gen else {}, - }, - ) - - # Create a new instance as a "fake self" - self = cls() - - # Make sure ExllamaV2 is up to date - check_package_version("exllamav2", "0.3.1") - - # Initialize config - self.config = ExLlamaV2Config() - self.model_dir = model_directory - self.config.model_dir = str(model_directory.resolve()) - self.hf_model = hf_model - - # Make the max seq len 4096 before preparing the config - # This is a better default than 2048 - self.config.max_seq_len = 4096 - - self.config.prepare() - - # Check if the model arch is compatible with various exl2 features - self.config.arch_compat_overrides() - - # Set vision state and error if vision isn't supported on the current model - self.use_vision = unwrap(kwargs.get("vision"), False) - if self.use_vision and not self.config.vision_model_type: - xlogger.warning( - "The provided model does not have vision capabilities that are " - "supported by ExllamaV2. Vision input is disabled." - ) - self.use_vision = False - - # Prepare the draft model config if necessary - draft_args = unwrap(kwargs.get("draft_model"), {}) - draft_model_name = draft_args.get("draft_model_name") - self.use_draft_model = draft_args and draft_model_name - - # Always disable draft if params are incorrectly configured - if draft_args and draft_model_name is None: - xlogger.warning( - "Draft model is disabled because a model name " - "wasn't provided. Please check your config.yml!" - ) - self.use_draft_model = False - - if self.use_draft_model: - self.draft_config = ExLlamaV2Config() - draft_model_path = pathlib.Path(unwrap(draft_args.get("draft_model_dir"), "models")) - draft_model_path = draft_model_path / draft_model_name - - self.draft_gpu_split = unwrap(draft_args.get("draft_gpu_split"), []) - self.draft_model_dir = draft_model_path - self.draft_config.model_dir = str(draft_model_path.resolve()) - self.draft_config.prepare() - - # MARK: User configuration - - # Get cache mode - self.cache_mode = unwrap(kwargs.get("cache_mode"), "FP16") - - # Catch exllamav3 cache_mode - if self.cache_mode != "FP16" and not self.cache_mode.startswith("Q"): - xlogger.warning( - f"Provided cache mode '{self.cache_mode}' is not a " - "valid choice for exllamav2, please check your settings. " - "Defaulting to FP16." - ) - self.cache_mode = "FP16" - - # Turn off GPU split if the user is using 1 GPU - gpu_count = torch.cuda.device_count() - gpu_split_auto = unwrap(kwargs.get("gpu_split_auto"), True) - use_tp = unwrap(kwargs.get("tensor_parallel"), False) - gpu_split = unwrap(kwargs.get("gpu_split"), []) - gpu_device_list = list(range(0, gpu_count)) - - # Set GPU split options - if gpu_count == 1: - self.gpu_split_auto = False - xlogger.info("Disabling GPU split because one GPU is in use.") - else: - # Set tensor parallel - if use_tp: - self.use_tp = True - - # TP has its own autosplit loader - self.gpu_split_auto = False - - # Enable manual GPU split if provided - if gpu_split: - self.gpu_split_auto = False - self.gpu_split = gpu_split - - gpu_device_list = [ - device_idx for device_idx, memory in enumerate(self.gpu_split) if memory > 0 - ] - elif gpu_split_auto and not self.use_tp: - # Otherwise fallback to autosplit settings - self.gpu_split_auto = gpu_split_auto - - autosplit_reserve_megabytes = unwrap(kwargs.get("autosplit_reserve"), [96]) - - # Reserve VRAM for each GPU - self.autosplit_reserve = [ - int(math.ceil(value * 1024**2)) for value in autosplit_reserve_megabytes - ] - - # Change the GPU device list only if gpu_split's list is too small - # This allows for an uneven list specification - if self.draft_gpu_split and len(self.draft_gpu_split) > len(self.gpu_split): - gpu_device_list = [ - device_idx - for device_idx, memory in enumerate(self.draft_gpu_split) - if memory > 0 - ] - - # Hardcode max output length to 16 - self.config.max_output_len = 16 - - # Set max batch size to the config override - self.max_batch_size = unwrap(kwargs.get("max_batch_size")) - - # Check whether the user's configuration supports flash/paged attention - # Also check if exl2 has disabled flash attention - if exllama_disabled_flash_attn( - self.config.no_flash_attn - ) or not hardware_supports_flash_attn(gpu_device_list): - gpu_unsupported_message = ( - "An unsupported GPU is found in this configuration. " - "Switching to compatibility mode. \n" - "This disables parallel batching " - "and features that rely on it (ex. CFG). \n" - "To disable compatability mode, all GPUs must be ampere " - "(30 series) or newer. AMD GPUs are not supported." - ) - - xlogger.warning(gpu_unsupported_message) - - self.config.no_flash_attn = True - if self.draft_config: - self.draft_config.no_flash_attn = True - self.paged = False - self.max_batch_size = 1 - torch.backends.cuda.enable_flash_sdp(False) - - # Grab user-set max seq len - user_max_seq_len = kwargs.get("max_seq_len") - - # Set k/v cache size - # cache_size is only relevant when paged mode is enabled - if self.paged: - user_cache_size = coalesce(kwargs.get("cache_size"), user_max_seq_len, 4096) - self.cache_size = self.adjust_cache_size(user_cache_size) - self.config.max_seq_len = self.adjust_max_seq_len(user_max_seq_len) - else: - self.config.max_seq_len = unwrap( - user_max_seq_len, - min(hf_model.hf_config.get_max_position_embeddings(), 4096), - ) - self.cache_size = self.config.max_seq_len - - # Set the rope scale - self.config.scale_pos_emb = unwrap(kwargs.get("rope_scale"), self.config.scale_pos_emb) - - # Sets rope alpha value. - # Utilize the model's max_position_embeddings as a base value - # Automatically calculate if unset or defined as an "auto" literal. - rope_alpha = unwrap(kwargs.get("rope_alpha"), "auto") - if rope_alpha == "auto": - self.config.scale_alpha_value = calculate_rope_alpha( - hf_model.hf_config.max_position_embeddings, self.config.max_seq_len - ) - else: - self.config.scale_alpha_value = rope_alpha - - # Try to set prompt template - self.prompt_template = await find_prompt_template( - kwargs.get("prompt_template"), model_directory - ) - - # Tool calling - self.tool_format = kwargs.get("tool_format") - if self.tool_format and not is_supported_format(self.tool_format): - xlogger.warning(f"Unrecognized tool_format in config: {self.tool_format}") - self.tool_format = None - if self.tool_format: - xlogger.info(f"Using tool format: {self.tool_format}") - - # Catch all for template lookup errors - if self.prompt_template: - xlogger.info( - f'Using template "{self.prompt_template.name}" for chat completions.', - {"raw": self.prompt_template.raw_template}, - ) - else: - xlogger.warning( - "Chat completions are disabled because a prompt " - "template wasn't provided or auto-detected." - ) - - # Make sure chunk size is >= 256, keep near or below max seq len - user_chunk_size = unwrap(kwargs.get("chunk_size"), 2048) - chunk_size = sorted((256, user_chunk_size, self.config.max_seq_len))[1] - chunk_remainder = chunk_size % 256 - if chunk_remainder != 0: - rounded_chunk_size = int(256 * ((chunk_size - chunk_remainder) / 256 + 1)) - - xlogger.warning( - f"The given chunk size ({chunk_size}) is " - "not a multiple of 256.\n" - "Overriding chunk_size with an overestimated value of " - f"{rounded_chunk_size} tokens." - ) - chunk_size = rounded_chunk_size - self.config.max_input_len = chunk_size - self.config.max_attention_size = chunk_size**2 - - # Set user-configured draft model values - if self.use_draft_model: - self.draft_config.max_seq_len = self.config.max_seq_len - - self.draft_config.scale_pos_emb = unwrap(draft_args.get("draft_rope_scale"), 1.0) - - # Set draft rope alpha. Follows same behavior as model rope alpha. - # Use the max_position_embeddings of the model - draft_rope_alpha = unwrap(draft_args.get("draft_rope_alpha"), "auto") - if draft_rope_alpha == "auto": - self.draft_config.scale_alpha_value = calculate_rope_alpha( - hf_model.hf_config.max_position_embeddings, - self.draft_config.max_seq_len, - ) - else: - self.draft_config.scale_alpha_value = draft_rope_alpha - - # Set draft cache mode - self.draft_cache_mode = unwrap(draft_args.get("draft_cache_mode"), "FP16") - - # Catch exllamav3 draft_cache_mode - if self.draft_cache_mode != "FP16" and not self.draft_cache_mode.startswith("Q"): - xlogger.warning( - f"Provided draft cache mode '{self.draft_cache_mode}' is not a " - "valid choice for exllamav2, please check your settings. " - "Defaulting to FP16." - ) - self.draft_cache_mode = "FP16" - - # Edit the draft config size - if chunk_size: - self.draft_config.max_input_len = chunk_size - self.draft_config.max_attention_size = chunk_size**2 - - # Reasoning mode - self.reasoning = kwargs.get("reasoning", False) - self.reasoning_start_token = kwargs.get("reasoning_start_token", "") - self.reasoning_end_token = kwargs.get("reasoning_end_token", "") - self.reasoning_suppress_header = kwargs.get("reasoning_suppress_header", None) - self.force_enable_thinking = kwargs.get("force_enable_thinking", False) - - # Return the created instance - return self - - def adjust_cache_size(self, cache_size): - # Enforce a multiple of 256 for cache size - # Overestimate to ensure that the cache isn't below max_seq_len - cache_remainder = cache_size % 256 - if cache_remainder != 0: - rounded_cache_size = int(256 * ((cache_size - cache_remainder) / 256 + 1)) - - xlogger.warning( - f"The given cache size ({cache_size}) is " - "not a multiple of 256.\n" - "Overriding cache_size with an overestimated value of " - f"{rounded_cache_size} tokens." - ) - - cache_size = rounded_cache_size - - if self.config.max_seq_len > cache_size: - xlogger.warning( - f"The given max_seq_len ({self.config.max_seq_len}) is larger than " - f"the cache size and will be limited to {cache_size} tokens." - ) - self.config.max_seq_len = cache_size - - # Warn user if cache size may be inadequate for CFG - if cache_size < 2 * self.config.max_seq_len: - xlogger.warning( - f"The given cache_size ({cache_size}) is less than 2 * max_seq_len " - "and may be too small for requests using CFG. \n" - "Ignore this warning if you do not plan on using CFG." - ) - - return cache_size - - def adjust_max_seq_len(self, max_seq_len): - print(f"User max seq len {max_seq_len}") - if not max_seq_len: - default_max_seq_len = min( - self.hf_model.hf_config.get_max_position_embeddings(), self.cache_size - ) - - xlogger.warning( - f"max_seq_len is undefined. Overriding to {default_max_seq_len} tokens." - ) - max_seq_len = default_max_seq_len - elif max_seq_len > self.cache_size: - xlogger.warning( - f"The given max_seq_len ({max_seq_len}) is larger than the cache size " - f"and will be limited to {self.cache_size} tokens." - ) - max_seq_len = self.cache_size - - return max_seq_len - - def model_info(self): - draft_model_card: ModelCard = None - if self.draft_config: - draft_model_params = ModelCardParameters( - max_seq_len=self.draft_config.max_seq_len, - rope_scale=self.draft_config.scale_pos_emb, - rope_alpha=self.draft_config.scale_alpha_value, - cache_mode=self.draft_cache_mode, - ) - - draft_model_card = ModelCard( - id=self.draft_model_dir.name, - parameters=draft_model_params, - ) - - model_params = ModelCardParameters( - max_seq_len=self.config.max_seq_len, - cache_size=self.cache_size, - rope_scale=self.config.scale_pos_emb, - rope_alpha=self.config.scale_alpha_value, - max_batch_size=self.max_batch_size, - cache_mode=self.cache_mode, - chunk_size=self.config.max_input_len, - use_vision=self.use_vision, - draft=draft_model_card, - ) - - if self.prompt_template: - model_params.prompt_template = self.prompt_template.name - model_params.prompt_template_content = self.prompt_template.raw_template - - model_card = ModelCard( - id=self.model_dir.name, - parameters=model_params, - ) - - return model_card - - async def wait_for_jobs(self, skip_wait: bool = False): - """Polling mechanism to wait for pending generation jobs.""" - - if not self.generator: - return - - # Immediately abort all jobs if asked - if skip_wait: - xlogger.warning( - "Immediately terminating all jobs. Clients will have their requests cancelled.\n" - ) - - for job in self.active_job_ids.values(): - if job: - await job.cancel() - - while len(self.active_job_ids) > 0: - await asyncio.sleep(0.01) - - async def load(self, progress_callback=None): - """ - Load model - - Args: - progress_callback (function, optional): A function to call for each - module loaded. - - Prototype: - def progress(loaded_modules: int, total_modules: int) - """ - - async for _ in self.load_gen(progress_callback): - pass - - async def load_gen(self, progress_callback=None, **kwargs): - """Loads a model and streams progress via a generator.""" - - # Indicate that model load has started - # Do this operation under the load lock's context - try: - await self.load_lock.acquire() - - # Wait for existing generation jobs to finish - await self.wait_for_jobs(kwargs.get("skip_wait")) - - # Streaming gen for model load progress - model_load_generator = self.load_model_sync(progress_callback) - async for value in iterate_in_threadpool(model_load_generator): - yield value - - # Create async generator - await self.create_generator() - - # Clean up any extra vram usage from torch and cuda - # (Helps reduce VRAM bottlenecking on Windows) - gc.collect() - torch.cuda.empty_cache() - - # Cleanup and update model load state - self.loaded = True - xlogger.info("Model successfully loaded.") - finally: - self.load_lock.release() - - async with self.load_condition: - self.load_condition.notify_all() - - @torch.inference_mode() - def load_model_sync(self, progress_callback=None): - """ - Synchronous generator for loading. - - Args: - progress_callback (function, optional): A function to call for each - module loaded. - - Prototype: - def progress(loaded_modules: int, total_modules: int) - - Runs under a shared inference mode context. - """ - - # Reset tokenizer namespace vars and create a tokenizer - ExLlamaV2Tokenizer.unspecial_piece_to_id = {} - ExLlamaV2Tokenizer.unspecial_id_to_piece = {} - ExLlamaV2Tokenizer.extended_id_to_piece = {} - ExLlamaV2Tokenizer.extended_piece_to_id = {} - - self.tokenizer = ExLlamaV2Tokenizer(self.config) - - # Calculate autosplit reserve for all GPUs - gpu_count = torch.cuda.device_count() - autosplit_reserve = self.autosplit_reserve + [0] * (gpu_count - len(self.autosplit_reserve)) - - # Load draft model if a config is present - if self.draft_config: - self.draft_model = ExLlamaV2(self.draft_config) - xlogger.info("Loading draft model: " + self.draft_config.model_dir) - - # Draft uses the autosplit loader, so create a cache that reflects this - draft_cache_class = self.get_cache_class(self.draft_cache_mode) - - if self.draft_gpu_split: - xlogger.info("Loading with a manual GPU split (or a one GPU setup)") - - for value in self.draft_model.load_gen( - self.draft_gpu_split, - callback_gen=progress_callback, - ): - if value: - yield value - - self.draft_cache = self.create_cache( - cache_class=draft_cache_class, - autosplit=False, - use_tp=False, - model=self.draft_model, - ) - else: - xlogger.info("Loading with autosplit") - - self.draft_cache = self.create_cache( - cache_class=draft_cache_class, - autosplit=True, - use_tp=False, - model=self.draft_model, - ) - - for value in self.draft_model.load_autosplit_gen( - self.draft_cache, - reserve_vram=autosplit_reserve, - last_id_only=True, - callback_gen=progress_callback, - ): - if value: - yield value - - # Test VRAM allocation with a full-length forward pass - input_ids = torch.zeros((1, self.config.max_input_len), dtype=torch.long) - self.draft_model.forward(input_ids, cache=self.cache, preprocess_only=True) - - # Load vision tower if it exists - if self.use_vision: - self.vision_model = ExLlamaV2VisionTower(self.config) - - for value in self.vision_model.load_gen(callback_gen=progress_callback): - if value: - yield value - - self.model = ExLlamaV2(self.config) - xlogger.info("Loading model: " + self.config.model_dir) - - # Get class of the model cache - cache_class = self.get_cache_class(self.cache_mode) - - # Load model with manual split - # Entrypoint for single GPU users - if self.use_tp: - xlogger.info("Loading with tensor parallel") - - # GPU split must be None if the array is empty - # Otherwise the TP loader fails - for value in self.model.load_tp_gen( - self.gpu_split or None, - callback_gen=progress_callback, - expect_cache_base=cache_class, - expect_cache_tokens=self.cache_size, - ): - if value: - yield value - elif not self.gpu_split_auto: - xlogger.info("Loading with a manual GPU split (or a one GPU setup)") - - for value in self.model.load_gen( - self.gpu_split, - callback_gen=progress_callback, - ): - if value: - yield value - - # Create the model cache - self.cache = self.create_cache( - cache_class=cache_class, - autosplit=self.gpu_split_auto, - use_tp=self.use_tp, - model=self.model, - ) - - # Load model with autosplit (without TP) - if self.gpu_split_auto and not self.use_tp: - xlogger.info("Loading with autosplit") - - for value in self.model.load_autosplit_gen( - self.cache, - reserve_vram=autosplit_reserve, - last_id_only=True, - callback_gen=progress_callback, - ): - if value: - yield value - - # Test VRAM allocation with a full-length forward pass - input_ids = torch.zeros((1, self.config.max_input_len), dtype=torch.long) - self.model.forward(input_ids, cache=self.cache, preprocess_only=True) - - # TODO: Maybe make a wrapper class with an ID instead of a utility function - def get_cache_class(self, cache_mode: str): - """Utility function to get a cache class based on user preference.""" - - match cache_mode: - case "Q4": - return ExLlamaV2Cache_Q4 - case "Q6": - return ExLlamaV2Cache_Q6 - case "Q8": - return ExLlamaV2Cache_Q8 - case _: - return ExLlamaV2Cache - - def create_cache( - self, - cache_class: ExLlamaV2CacheBase, - autosplit: bool, - use_tp: bool, - model: ExLlamaV2, - ): - """Utility function to create a model cache.""" - - if use_tp: - return ExLlamaV2Cache_TP( - model, - base=cache_class, - max_seq_len=self.cache_size, - batch_size=1, - ) - else: - return cache_class( - model, - max_seq_len=self.cache_size, - lazy=autosplit, - batch_size=1, - ) - - async def create_generator(self): - """Create and save a Exllama generator class.""" - - try: - # Don't acquire locks unless a model is loaded - if self.loaded: - await self.load_lock.acquire() - - # Immediately cancel all jobs - await self.wait_for_jobs(skip_wait=True) - - # Create new generator - self.generator = ExLlamaV2DynamicGeneratorAsync( - model=self.model, - cache=self.cache, - draft_model=self.draft_model, - draft_cache=self.draft_cache, - tokenizer=self.tokenizer, - max_batch_size=self.max_batch_size, - paged=self.paged, - ) - - # Update the state of the container var - if self.max_batch_size is None: - self.max_batch_size = self.generator.generator.max_batch_size - finally: - # This means the generator is being recreated - # The load lock is already released in the load function - if self.loaded: - self.load_lock.release() - - async with self.load_condition: - self.load_condition.notify_all() - - def get_loras(self): - """Convenience function to get all loras.""" - - return unwrap(self.generator.generator.current_loras, []) - - async def load_loras(self, lora_directory: pathlib.Path, **kwargs): - """Load loras.""" - - loras = unwrap(kwargs.get("loras"), []) - - try: - await self.load_lock.acquire() - - # Wait for existing generation jobs to finish - await self.wait_for_jobs(kwargs.get("skip_wait")) - - loras_to_load: List[ExLlamaV2Lora] = [] - success: List[str] = [] - failure: List[str] = [] - - for lora in loras: - lora_name = lora.get("name") - lora_scaling = unwrap(lora.get("scaling"), 1.0) - - if lora_name is None: - xlogger.warning( - "One of your loras does not have a name. Please check your " - "config.yml! Skipping lora load." - ) - failure.append(lora_name) - continue - - xlogger.info(f"Adding lora: {lora_name} at scaling {lora_scaling}") - lora_path = lora_directory / lora_name - - loras_to_load.append( - ExLlamaV2Lora.from_directory(self.model, lora_path, lora_scaling) - ) - xlogger.info(f"Lora successfully added: {lora_name}") - success.append(lora_name) - - self.generator.generator.set_loras(loras_to_load) - xlogger.info("All loras successfully loaded") - - # Return success and failure names - return {"success": success, "failure": failure} - finally: - self.load_lock.release() - - async with self.load_condition: - self.load_condition.notify_all() - - async def unload(self, loras_only: bool = False, **kwargs): - """Free all VRAM resources used by the model (and loras).""" - - # Shutdown immediately unloads and bypasses all locks - do_shutdown = kwargs.get("shutdown") - - try: - if not do_shutdown: - await self.load_lock.acquire() - - # Wait for other jobs to finish - await self.wait_for_jobs(kwargs.get("skip_wait")) - - # Delete references held in the grammar module - clear_grammar_func_cache() - - # Clear the image embedding cache - clear_image_embedding_cache() - - # Unload LoRAs - if self.generator and self.generator.generator.current_loras: - for lora in self.generator.generator.current_loras: - lora.unload() - - self.generator.generator.set_loras([]) - - # Unload the entire model if not just unloading loras - if not loras_only: - if self.model: - self.model.unload() - self.model = None - - if self.vision_model: - self.vision_model.unload() - - self.vision_model = None - - if self.draft_model: - self.draft_model.unload() - self.draft_model = None - - self.config = None - self.cache = None - self.tokenizer = None - - # Cleanup the generator from any pending jobs - if self.generator is not None: - await self.generator.close() - self.generator = None - - # Set all model state variables to False - self.loaded = False - - gc.collect() - torch.cuda.empty_cache() - - xlogger.info("Loras unloaded." if loras_only else "Model unloaded.") - finally: - if not do_shutdown: - self.load_lock.release() - - async with self.load_condition: - self.load_condition.notify_all() - - def encode_tokens(self, text: str, **kwargs): - """Wrapper to encode tokens from a text string.""" - - mm_embeddings: MultimodalEmbeddingWrapper = kwargs.get("embeddings") - mm_embeddings_content = mm_embeddings.content if mm_embeddings else [] - - return ( - self.tokenizer.encode( - text, - add_bos=unwrap(kwargs.get("add_bos_token"), self.hf_model.add_bos_token()), - encode_special_tokens=unwrap(kwargs.get("encode_special_tokens"), True), - embeddings=mm_embeddings_content, - ) - .flatten() - .tolist() - ) - - def decode_tokens(self, ids: List[int], **kwargs): - """Wrapper to decode tokens from a list of IDs""" - - ids = torch.tensor([ids]) - return self.tokenizer.decode( - ids, - decode_special_tokens=unwrap(kwargs.get("decode_special_tokens"), True), - )[0] - - def get_special_tokens(self): - return { - "bos_token": self.tokenizer.bos_token, - "eos_token": self.tokenizer.eos_token, - "pad_token": self.tokenizer.pad_token, - "unk_token": self.tokenizer.unk_token, - } - - def validate_context_length( - self, - prompt: str, - params: BaseSamplerRequest, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - ): - prompts = [prompt] - if params.cfg_scale not in [None, 1.0] and self.paged: - prompts.append(unwrap(params.negative_prompt, self.tokenizer.bos_token)) - - context_lengths = [ - len( - self.encode_tokens( - current_prompt, - add_bos_token=unwrap(params.add_bos_token, self.hf_model.add_bos_token()), - embeddings=mm_embeddings, - ) - ) - for current_prompt in prompts - ] - context_len = max(context_lengths) - if context_len > self.config.max_seq_len: - preamble = "Negative prompt" if context_lengths.index(context_len) == 1 else "Prompt" - raise ContextLengthExceededError( - f"{preamble} length {context_len} exceeds the available context size " - f"of {self.config.max_seq_len} tokens" - ) - - def get_logprobs(self, token_ids: torch.Tensor, token_probs: torch.Tensor): - top_tokens = [ - self.tokenizer.extended_id_to_piece.get( - index, self.tokenizer.get_id_to_piece_list(True)[index] - ) - for index in token_ids.flatten().tolist() - ] - - top_values = torch.log(token_probs).flatten().tolist() - - # Cannot return -inf in JSON - cleaned_values = [-1000 if value == float("-inf") else value for value in top_values] - - return dict(zip_longest(top_tokens, cleaned_values)) - - async def generate( - self, - request_id: str, - prompt: str, - params: BaseSamplerRequest, - disconnect_handler: DisconnectHandler = None, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - ): - """Generate a response to a prompt.""" - generations = [] - async for generation in self.stream_generate( - request_id, - prompt, - params, - disconnect_handler, - mm_embeddings, - ): - if generation is None: - continue - generations.append(generation) - - joined_generation = { - "request_id": "", - "text": "", - "full_text": "", - "prompt_tokens": 0, - "gen_tokens": 0, - "tool_calls": None, - "offset": [], - "token_probs": {}, - "logprobs": [], - } - - if generations: - # Get finish_reason first and then shift where -1 points to - if "finish_reason" in generations[-1]: - finish_chunk = generations.pop() - joined_generation = {**joined_generation, **finish_chunk} - joined_generation["text"] = joined_generation.get("full_text", "") - else: - joined_generation["finish_reason"] = "stop" - - if len(generations) > 0: - for generation in generations: - joined_generation["offset"].append(unwrap(generation.get("offset"), -1)) - joined_generation["token_probs"].update(unwrap(generation.get("token_probs"), {})) - - # Include empty logprob dicts for index preservation - joined_generation["logprobs"].append(unwrap(generation.get("logprobs"), {})) - - joined_generation["prompt_tokens"] = unwrap(generations[-1].get("prompt_tokens"), 0) - joined_generation["generated_tokens"] = unwrap( - generations[-1].get("generated_tokens"), 0 - ) - - return joined_generation - - async def stream_generate( - self, - request_id: str, - prompt: str, - params: BaseSamplerRequest, - disconnect_handler: DisconnectHandler = None, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - filter_trigger: str = None, - ): - try: - # Wait for load lock to be freed before processing - # Mainly used for loras and other operations where the class is available - async with self.load_condition: - await self.load_condition.wait_for(lambda: not self.load_lock.locked()) - - # If the model is being unloaded, don't accept new requests - if not self.loaded: - raise RuntimeError( - "Model is being unloaded. Cannot process new generation requests." - ) - - # Mark that the job is running - self.active_job_ids[request_id] = None - - # Yield from the internal generator - async for generation_chunk in self.generate_gen( - request_id=request_id, - prompt=prompt, - params=params, - disconnect_handler=disconnect_handler, - mm_embeddings=mm_embeddings, - ): - yield generation_chunk - finally: - # Clean up and remove the job from active IDs - del self.active_job_ids[request_id] - - def check_unsupported_settings(self, params: BaseSamplerRequest): - """ - Check and warn the user if a sampler is unsupported. - - Meant for dev wheels! - """ - - return params - - def assign_gen_params( - self, - params: BaseSamplerRequest, - gen_settings: ExLlamaV2Sampler.Settings, - grammar_handler: ExLlamaV2Grammar, - ): - # Apply settings - gen_settings.temperature = params.temperature - gen_settings.temperature_last = params.temperature_last - gen_settings.smoothing_factor = params.smoothing_factor - gen_settings.top_k = params.top_k - gen_settings.top_p = params.top_p - gen_settings.top_a = params.top_a - gen_settings.min_p = params.min_p - gen_settings.tfs = params.tfs - gen_settings.typical = params.typical - gen_settings.mirostat = params.mirostat_mode == 2 - gen_settings.skew = params.skew - - # XTC - if params.xtc_probability > 0.0: - gen_settings.xtc_probability = params.xtc_probability - - # 0.1 is the default for this value - gen_settings.xtc_threshold = params.xtc_threshold - - # DynaTemp settings - max_temp = params.max_temp - min_temp = params.min_temp - - if params.max_temp > params.min_temp: - gen_settings.max_temp = max_temp - gen_settings.min_temp = min_temp - gen_settings.temp_exponent = params.temp_exponent - else: - # Force to default values - gen_settings.max_temp = 1.0 - gen_settings.min_temp = 1.0 - gen_settings.temp_exponent = 1.0 - - # Warn if max/min temp values are > 0 - # and if they're less than or equal to each other - if max_temp < min_temp or (1 not in {min_temp, max_temp} and max_temp == min_temp): - xlogger.warning("Max temp is less than or equal to min temp, skipping DynaTemp.") - - # Default tau and eta fallbacks don't matter if mirostat is off - gen_settings.mirostat_tau = params.mirostat_tau - gen_settings.mirostat_eta = params.mirostat_eta - - # Penalties - gen_settings.token_repetition_penalty = params.repetition_penalty - gen_settings.token_frequency_penalty = params.frequency_penalty - gen_settings.token_presence_penalty = params.presence_penalty - - # Applies for all penalties despite being called token_repetition_range - gen_settings.token_repetition_range = unwrap(params.penalty_range, self.config.max_seq_len) - - # Always make sure the fallback is 0 if range < 0 - # It's technically fine to use -1, but this just validates the passed - # fallback - # Always default to 0 if something goes wrong - if gen_settings.token_repetition_range < 0: - fallback_decay = 0 - else: - fallback_decay = gen_settings.token_repetition_range - gen_settings.token_repetition_decay = coalesce(params.repetition_decay, fallback_decay, 0) - - # DRY options - dry_multiplier = params.dry_multiplier - - # < 0 = disabled - if dry_multiplier > 0: - gen_settings.dry_multiplier = dry_multiplier - gen_settings.dry_allowed_length = params.dry_allowed_length - gen_settings.dry_base = params.dry_base - - # Exl2 has dry_range as 0 for unlimited unlike -1 for penalty_range - # Use max_seq_len as the fallback to stay consistent - gen_settings.dry_range = unwrap(params.dry_range, self.config.max_seq_len) - - # Tokenize sequence breakers - if params.dry_sequence_breakers: - gen_settings.dry_sequence_breakers = { - self.encode_tokens(s)[-1] for s in params.dry_sequence_breakers - } - - # Add JSON schema filter if it exists - if params.json_schema: - grammar_handler.add_json_schema_filter(params.json_schema, self.model, self.tokenizer) - - # Add regex filter if it exists - if params.regex_pattern: - grammar_handler.add_regex_filter(params.regex_pattern, self.model, self.tokenizer) - - # Add EBNF filter if it exists - if params.grammar_string: - grammar_handler.add_kbnf_filter(params.grammar_string, self.model, self.tokenizer) - - # Speculative Ngram - self.generator.speculative_ngram = params.speculative_ngram - - # Override sampler settings for temp = 0 - if gen_settings.temperature == 0: - gen_settings.temperature = 1.0 - gen_settings.top_k = 1 - gen_settings.top_p = 0 - gen_settings.typical = 0 - - xlogger.warning( - "Temperature is set to 0. Overriding temp, " - "top_k, top_p, and typical to 1.0, 1, 0, and 0." - ) - - # Set banned tokens - if params.banned_tokens: - gen_settings.disallow_tokens(self.tokenizer, params.banned_tokens) - - # Set allowed tokens - if params.allowed_tokens: - gen_settings.allow_tokens(self.tokenizer, params.allowed_tokens) - - # Set logit bias - if params.logit_bias: - # Create a vocab tensor if it doesn't exist for token biasing - if gen_settings.token_bias is None: - padding = -self.tokenizer.config.vocab_size % 32 - gen_settings.token_bias = torch.zeros( - (self.tokenizer.config.vocab_size + padding,), - dtype=torch.float, - ) - - # Map logits to the tensor with their biases - for token_id, bias in params.logit_bias.items(): - if 0 <= token_id < len(self.tokenizer.get_id_to_piece_list(True)): - gen_settings.token_bias[token_id] = bias - else: - xlogger.warning( - f"Logit bias: Token {token_id} not present in the model's vocab. Skipping." - ) - - # Adds logprobs to a generation chunk - def handle_logprobs(self, result: dict, generation: dict): - top_tokens = unwrap( - result.get("top_k_tokens"), - torch.empty((1, 0, 1), dtype=torch.long), - ) - - top_probs = unwrap( - result.get("top_k_probs"), - torch.empty((1, 0, 1), dtype=torch.float), - ) - - if top_tokens.numel() > 0 and top_probs.numel() > 0: - logprobs = self.get_logprobs(top_tokens, top_probs) - generation["logprobs"] = logprobs - - # The first logprob is the selected token prob - generation["token_probs"] = { - token: logprobs[token] for token in list(logprobs.keys())[:1] - } - - # Creates and returns a finish chunk - def handle_finish_chunk(self, result: dict, request_id: str, full_text: str): - eos_reason = result.get("eos_reason") - - stop_str = None - if eos_reason == "max_new_tokens": - finish_reason = "length" - else: - finish_reason = "stop" - # Grab stop string if stop was the reason - if eos_reason == "stop_token": - stop_str = result.get("eos_triggering_token_str") - elif eos_reason == "stop_string": - stop_str = result.get("eos_triggering_string") - - # Prompt - prompt_tokens = result.get("prompt_tokens") - cached_tokens = round(result.get("cached_tokens"), 2) - prompt_time = round(result.get("time_prefill"), 2) - prompt_ts = ( - "Indeterminate" - if prompt_time == 0 - else round((prompt_tokens - cached_tokens) / prompt_time, 2) - ) - - # Generated - gen_tokens = result.get("new_tokens") - gen_time = result.get("time_generate") - gen_ts = "Indeterminate" if gen_time == 0 else round(gen_tokens / gen_time, 2) - - # Queue + Total - queue_time = result.get("time_enqueued") - total_time = round(queue_time + prompt_time + gen_time, 2) - - finish_chunk = { - "request_id": request_id, - "prompt_tokens": prompt_tokens, - "prompt_time": round(prompt_time, 2), - "prompt_tokens_per_sec": prompt_ts, - "gen_tokens": gen_tokens, - "gen_time": round(gen_time, 2), - "gen_tokens_per_sec": gen_ts, - "total_time": total_time, - "queue_time": round(queue_time, 2), - "cached_tokens": cached_tokens, - "finish_reason": finish_reason, - "stop_str": stop_str, - "full_text": full_text, - } - - return finish_chunk - - async def generate_gen( - self, - request_id: str, - prompt: str, - params: BaseSamplerRequest, - disconnect_handler: DisconnectHandler = None, - mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None, - ): - """ - Create generator function for prompt completion. - - for kwargs, check common/sampling.py - """ - - prompts = [prompt] - gen_settings = ExLlamaV2Sampler.Settings() - grammar_handler = ExLlamaV2Grammar() - - self.assign_gen_params( - params, - gen_settings, - grammar_handler, - ) - - # Set banned strings - banned_strings = params.banned_strings - if banned_strings and len(grammar_handler.filters) > 0: - xlogger.warning( - "Disabling banned_strings because they cannot be used with grammar filters." - ) - - banned_strings = [] - - # Set CFG scale and negative prompt - cfg_scale = params.cfg_scale - negative_prompt = None - if cfg_scale not in [None, 1.0]: - if self.paged: - gen_settings.cfg_scale = cfg_scale - - # If the negative prompt is empty, use the BOS token - negative_prompt = unwrap(params.negative_prompt, self.tokenizer.bos_token) - - prompts.append(negative_prompt) - else: - xlogger.warning( - "CFG is currently disabled because paged mode is disabled. " - "Please use an ampere (30 series) or higher GPU for CFG support." - ) - - # Dynamically scale penalty range to output tokens - # Only do this if freq/pres pen is enabled - # and the repetition range is -1 - auto_scale_penalty_range = ( - gen_settings.token_frequency_penalty != 0 or gen_settings.token_presence_penalty != 0 - ) and gen_settings.token_repetition_range == -1 - - stop_conditions = params.stop - ban_eos_token = params.ban_eos_token - - # Set add_bos_token for generation - add_bos_token = unwrap(params.add_bos_token, self.hf_model.add_bos_token()) - - # Fetch EOS tokens from the HF model if they exist - eos_tokens = self.hf_model.eos_tokens() or [self.tokenizer.eos_token_id] - - # Ban the EOS token if specified. If not, append to stop conditions - # as well. - # Set this below logging to avoid polluting the stop strings array - if ban_eos_token: - gen_settings.disallow_tokens(self.tokenizer, eos_tokens) - else: - stop_conditions += eos_tokens - - # Get multimodal embeddings if present - mm_embeddings_content = mm_embeddings.content if mm_embeddings else [] - - # Encode both positive and negative prompts - input_ids = [ - self.tokenizer.encode( - prompt, - add_bos=add_bos_token, - encode_special_tokens=True, - embeddings=mm_embeddings_content, - ) - for prompt in prompts - ] - - # The first index will always be the positive prompt - context_len = input_ids[0].size(dim=-1) - - # The second index will be the negative prompt if CFG is enabled - negative_context_len = input_ids[1].size(dim=-1) if negative_prompt else 0 - - # Automatically set max_tokens to fill up the context - max_tokens = unwrap(params.max_tokens, 0) - if max_tokens <= 0: - max_tokens = self.config.max_seq_len - max(context_len, negative_context_len) - - # Determine if the negative context or the context length is bigger - context_to_check = max(negative_context_len, context_len) - - # Check total length of prompt against max context length - if context_to_check > self.config.max_seq_len: - preamble = "Negative prompt" if negative_context_len > context_len else "Prompt" - - raise ContextLengthExceededError( - f"{preamble} length {context_to_check} exceeds the available context size " - f"of {self.config.max_seq_len} tokens" - ) - - # Check total required pages for CFG request to avoid overallocation - if negative_prompt and ( - sum( - 256 * math.ceil((context + max_tokens) / 256) - for context in (context_len, negative_context_len) - ) - > self.cache_size - ): - raise ValueError( - f"Total required page size for request " - f"{context_len} + {negative_context_len} + {max_tokens} * 2 " - f"is greater than cache_size {self.cache_size}" - ) - - # Log prompt to console. Add the BOS token if specified - log_prompt( - f"{self.tokenizer.bos_token if add_bos_token else ''}{prompt}", - request_id, - negative_prompt, - ) - - # Create and add a new job - # Don't use the request ID here as there can be multiple jobs per request - job = ExLlamaV2DynamicJobAsync( - self.generator, - input_ids=input_ids, - max_new_tokens=max_tokens, - min_new_tokens=params.min_tokens, - gen_settings=gen_settings, - stop_conditions=stop_conditions, - decode_special_tokens=True, - filters=grammar_handler.filters, - filter_prefer_eos=bool(grammar_handler.filters), - return_probs=params.logprobs > 0, - return_top_tokens=params.logprobs, - return_logits=params.logprobs > 0, - banned_strings=banned_strings, - token_healing=params.token_healing, - identifier=request_id, - embeddings=mm_embeddings_content, - ) - await disconnect_handler.add_cleanup_task(id(job), job.cancel, ()) - - # Assign the active job to the request ID - self.active_job_ids[request_id] = job - - # Save generated tokens and full response - # Copy over max seq len incase model is unloaded and stored jobs can complete - # Full response is required for offset calculation - max_seq_len = self.config.max_seq_len - generated_tokens = 0 - full_response = "" - metrics_result = {} - - # Get the generation status once it's ready - try: - async for result in job: - # Abort if the event is set while streaming - await disconnect_handler.poll() - - stage = result.get("stage") - result_id = result.get("identifier") - - if stage == "streaming" and result_id == request_id: - chunk = unwrap(result.get("text"), "") - full_response += chunk - - chunk_tokens = result.get("token_ids") - if chunk_tokens is not None: - generated_tokens += chunk_tokens.size(dim=0) - - generation = { - "request_id": request_id, - "text": chunk, - "prompt_tokens": context_len, - "generated_tokens": generated_tokens, - "offset": len(full_response), - } - - # Increase penalty range to generated token amount - if auto_scale_penalty_range: - gen_settings.token_repetition_range = generated_tokens - - # Handle logprobs - if params.logprobs > 0: - self.handle_logprobs(result, generation) - - yield generation - - # Yield a finish chunk when generation is finished - if result.get("eos"): - log_response(request_id, full_response) - finish_chunk = self.handle_finish_chunk(result, request_id, full_response) - await disconnect_handler.finish(id(job)) - - # Save the final result for metrics logging - metrics_result = finish_chunk - - yield finish_chunk - break - except asyncio.CancelledError: - if not job.cancelled: - await job.cancel() - - except Exception as ex: - # Create a new generator since the current state is broken - # No need to wait for this to finish - xlogger.error( - "FATAL ERROR with generation. " - "Attempting to recreate the generator. " - "If this fails, please restart the server.\n", - {"exception": str(ex)}, - ) - asyncio.ensure_future(self.create_generator()) - - await HealthManager.add_unhealthy_event(ex) - - raise ex - finally: - # Log generation options to console - # Some options are too large, so log the args instead - log_generation_params( - request_id=request_id, - bos_token_id=self.tokenizer.bos_token_id, - eos_token_id=eos_tokens, - prompt=prompt, - **params.model_dump(exclude={"prompt"}), - auto_scale_penalty_range=auto_scale_penalty_range, - ) - - # Log the metrics if present - if metrics_result: - log_metrics( - request_id, - metrics_result, - context_len, - max_seq_len, - ) diff --git a/backends/exllamav2/utils.py b/backends/exllamav2/utils.py deleted file mode 100644 index 56330ba..0000000 --- a/backends/exllamav2/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -from common.logger import xlogger - - -def exllama_disabled_flash_attn(no_flash_attn: bool): - unsupported_message = ( - "ExllamaV2 has disabled Flash Attention. \n" - "Please see the above logs for warnings/errors. \n" - "Switching to compatibility mode. \n" - "This disables parallel batching " - "and features that rely on it (ex. CFG). \n" - ) - - if no_flash_attn: - xlogger.warning(unsupported_message) - - return no_flash_attn diff --git a/backends/exllamav2/vision.py b/backends/exllamav2/vision.py deleted file mode 100644 index 90106e3..0000000 --- a/backends/exllamav2/vision.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Vision utilities for ExLlamaV2.""" - -from async_lru import alru_cache - -from common import model -from common.optional_dependencies import dependencies -from common.image_util import get_image - -# Since this is used outside the Exl2 backend, the dependency -# may be optional -if dependencies.exllamav2: - from exllamav2.generator import ExLlamaV2MMEmbedding - - -# Fetch the return type on runtime -@alru_cache(20) -async def get_image_embedding_exl2(url: str) -> "ExLlamaV2MMEmbedding": - image = await get_image(url) - return model.container.vision_model.get_image_embeddings( - model=model.container.model, - tokenizer=model.container.tokenizer, - image=image, - text_alias=None, - ) - - -def clear_image_embedding_cache(): - get_image_embedding_exl2.cache_clear() diff --git a/backends/exllamav3/model.py b/backends/exllamav3/model.py index 55b7e93..7e83c6e 100644 --- a/backends/exllamav3/model.py +++ b/backends/exllamav3/model.py @@ -24,7 +24,6 @@ from exllamav3 import ( from exllamav3.cache import CacheLayer_quant from backends.exllamav3.grammar import ExLlamaV3Grammar -from backends.base_model_container import BaseModelContainer from backends.exllamav3.sampler import ExllamaV3SamplerBuilder from backends.exllamav3.utils import exllama_supports_nccl from backends.exllamav3.vision import clear_image_embedding_cache @@ -52,12 +51,17 @@ from endpoints.OAI.utils.tools import is_supported_format import inspect -class ExllamaV3Container(BaseModelContainer): - """Abstract base class for model containers.""" +class ExllamaV3Container: + """Model container for the ExLlamaV3 backend.""" # Exposed model information model_dir: pathlib.Path = pathlib.Path("models") prompt_template: Optional[PromptTemplate] = None + tool_format: Optional[str] = None + + # Optional features + use_draft_model: bool = False + use_vision: bool = False # HF Model instance hf_model: HFModel @@ -496,17 +500,20 @@ class ExllamaV3Container(BaseModelContainer): while len(self.active_job_ids) > 0: await asyncio.sleep(0.01) - async def load(self, progress_callback=None, **kwargs): - """ - Loads the model into memory. + # TODO: Wire up exllamav3's LoRA support once the API surface for it is decided + async def load_loras(self, lora_directory: pathlib.Path, **kwargs) -> Dict[str, List[str]]: + """Stub. LoRAs aren't hooked up to the ExLlamaV3 backend yet.""" - Args: - progress_callback: Optional callback for progress updates. - **kwargs: Additional loading options. - """ + xlogger.error("LoRA loading is not hooked up to the ExLlamaV3 backend yet.") + return { + "success": [], + "failure": [lora.get("name", "unknown") for lora in kwargs.get("loras", [])], + } - async for _ in self.load_gen(progress_callback): - pass + def get_loras(self) -> List[Any]: + """Stub. LoRAs aren't hooked up to the ExLlamaV3 backend yet.""" + + return [] async def load_gen(self, progress_callback=None, **kwargs): """ @@ -638,6 +645,11 @@ class ExllamaV3Container(BaseModelContainer): **kwargs: Additional unloading options (e.g., shutdown). """ + # Nothing to do for LoRA-only unloads until LoRAs are hooked up + if loras_only: + xlogger.error("LoRA unloading is not hooked up to the ExLlamaV3 backend yet.") + return + # Used when shutting down the server do_shutdown = kwargs.get("shutdown") diff --git a/common/config_models.py b/common/config_models.py index 23de92d..9d9e10d 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -170,8 +170,7 @@ class ModelConfig(BaseConfigModel): backend: Optional[str] = Field( None, description=( - "Backend to use for this model (auto-detect if not specified)\n" - "Options: exllamav2, exllamav3" + "Backend to use for this model (auto-detect if not specified)\nOptions: exllamav3" ), ) max_seq_len: Optional[int] = Field( @@ -195,9 +194,9 @@ class ModelConfig(BaseConfigModel): "FP16", description=( "Enable different cache modes for VRAM savings (default: FP16).\n" - f"Possible values for exllamav2: {str(CACHE_SIZES)[15:-1]}.\n" - "For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits " - "are integers from 2-8 (i.e. 8,8)." + "Specify the pair k_bits,v_bits where k_bits and v_bits " + "are integers from 2-8 (i.e. 8,8).\n" + f"The legacy values {str(CACHE_SIZES)[15:-1]} are also accepted." ), ) tensor_parallel: Optional[bool] = Field( @@ -511,7 +510,7 @@ class DeveloperConfig(BaseConfigModel): unsafe_launch: Optional[bool] = Field( False, description=( - "Skip Exllamav2 version check (default: False).\n" + "Skip ExLlamav3 version check (default: False).\n" "WARNING: It's highly recommended to update your dependencies rather " "than enabling this flag." ), diff --git a/common/model.py b/common/model.py index ed0caca..35da69e 100644 --- a/common/model.py +++ b/common/model.py @@ -10,9 +10,8 @@ from enum import Enum from fastapi import HTTPException from common.logger import xlogger from ruamel.yaml import YAML -from typing import Dict, Optional +from typing import Optional -from backends.base_model_container import BaseModelContainer from common.errors import ContextLengthExceededError, ContextLengthHTTPException from common.logger import get_loading_progress_bar from common.multimodal import MultimodalEmbeddingWrapper @@ -23,23 +22,12 @@ from common.optional_dependencies import dependencies from common.transformers_utils import HFModel from common.utils import deep_merge_dict, unwrap -# Global variables for model container -container: Optional[BaseModelContainer] = None -embeddings_container = None - - -_BACKEND_REGISTRY: Dict[str, BaseModelContainer] = {} - -if dependencies.exllamav2: - from backends.exllamav2.model import ExllamaV2Container - - _BACKEND_REGISTRY["exllamav2"] = ExllamaV2Container - - if dependencies.exllamav3: from backends.exllamav3.model import ExllamaV3Container - _BACKEND_REGISTRY["exllamav3"] = ExllamaV3Container +# Global variables for model container +container: Optional["ExllamaV3Container"] = None +embeddings_container = None if dependencies.extras: @@ -60,15 +48,25 @@ def load_progress(module, modules): yield module, modules -def detect_backend(hf_model: HFModel) -> str: - """Determine the appropriate backend based on model files and configuration.""" +def validate_backend(backend: Optional[str], hf_model: HFModel): + """Check that the requested model can be loaded with the exllamav3 backend.""" + + if backend == "exllamav2": + raise ValueError("The exllamav2 backend is no longer supported. Please use exllamav3.") + elif backend and backend != "exllamav3": + raise ValueError(f"Invalid backend '{backend}'. Available backends: ['exllamav3']") quant_method = hf_model.quant_method() + if quant_method in {"exl2", "gptq"}: + raise ValueError( + f"Models quantized with '{quant_method}' require the exllamav2 backend, " + "which is no longer supported. Please use an exl3 or unquantized model." + ) - if quant_method == "exl3": - return "exllamav3" - else: - return "exllamav2" + if not dependencies.exllamav3: + raise ValueError( + "The exllamav3 backend is selected, but required dependencies are not installed." + ) async def apply_load_defaults(model_path: pathlib.Path, **kwargs): @@ -179,25 +177,10 @@ async def load_model_gen(model_path: pathlib.Path, **kwargs): if max_seq_len == -1: kwargs["max_seq_len"] = hf_model.hf_config.get_max_position_embeddings() - # Create a new container and check if the right dependencies are installed - backend = unwrap(kwargs.get("backend"), detect_backend(hf_model)) - container_class = _BACKEND_REGISTRY.get(backend) + # Check model compatibility and dependencies before creating a container + validate_backend(kwargs.get("backend"), hf_model) - if not container_class: - available_backends = list(_BACKEND_REGISTRY.keys()) - if backend in {"exllamav2", "exllamav3"}: - raise ValueError( - f"Backend '{backend}' selected, but required dependencies are not installed." - ) - else: - raise ValueError( - f"Invalid backend '{backend}'. Available backends: {available_backends}" - ) - - xlogger.info(f"Using backend {backend}") - new_container: BaseModelContainer = await container_class.create( - model_path.resolve(), hf_model, **kwargs - ) + new_container = await ExllamaV3Container.create(model_path.resolve(), hf_model, **kwargs) # Add possible types of models that can be loaded model_type = [ModelType.MODEL] diff --git a/common/multimodal.py b/common/multimodal.py index 85645c6..f222cc9 100644 --- a/common/multimodal.py +++ b/common/multimodal.py @@ -1,4 +1,3 @@ -from backends.exllamav2.vision import get_image_embedding_exl2 from backends.exllamav3.vision import get_image_embedding_exl3 from common import model from common.logger import xlogger @@ -7,8 +6,6 @@ from typing import List from common.optional_dependencies import dependencies -if dependencies.exllamav2: - from exllamav2 import ExLlamaV2VisionTower if dependencies.exllamav3: from exllamav3 import Model @@ -23,19 +20,11 @@ class MultimodalEmbeddingWrapper(BaseModel): async def add(self, url: str): # Determine the type of vision embedding to use if not self.type: - if dependencies.exllamav2 and isinstance( - model.container.vision_model, ExLlamaV2VisionTower - ): - self.type = "ExLlamaV2MMEmbedding" - elif dependencies.exllamav3 and isinstance(model.container.vision_model, Model): + if dependencies.exllamav3 and isinstance(model.container.vision_model, Model): self.type = "MMEmbedding" # Create the embedding - if self.type == "ExLlamaV2MMEmbedding": - embedding = await get_image_embedding_exl2(url) - self.content.append(embedding) - self.text_alias.append(embedding.text_alias) - elif self.type == "MMEmbedding": + if self.type == "MMEmbedding": embedding = await get_image_embedding_exl3(url) self.content.append(embedding) self.text_alias.append(embedding.text_alias) diff --git a/common/optional_dependencies.py b/common/optional_dependencies.py index d0a4355..9e84017 100644 --- a/common/optional_dependencies.py +++ b/common/optional_dependencies.py @@ -1,7 +1,6 @@ """Construct a model of all optional dependencies""" import importlib.util -from importlib.metadata import PackageNotFoundError from importlib.metadata import version as package_version from common.logger import xlogger from packaging import version @@ -16,7 +15,6 @@ class DependenciesModel(BaseModel): """Model of which optional dependencies are installed.""" torch: bool - exllamav2: bool exllamav3: bool flash_attn: bool infinity_emb: bool @@ -30,7 +28,7 @@ class DependenciesModel(BaseModel): @computed_field @property def inference(self) -> bool: - return self.torch and (self.exllamav2 or self.exllamav3) + return self.torch and self.exllamav3 def is_installed(package_name: str) -> bool: @@ -40,18 +38,6 @@ def is_installed(package_name: str) -> bool: return spec is not None -def is_torch_cuda_13() -> bool: - """Check whether the installed Torch wheel targets CUDA 13.""" - - try: - torch_version = package_version("torch") - except PackageNotFoundError: - return False - - _, _, local_version = torch_version.partition("+") - return local_version.startswith("cu13") - - def get_installed_deps() -> DependenciesModel: """Check if optional dependencies are installed by looping over the fields.""" @@ -62,9 +48,6 @@ def get_installed_deps() -> DependenciesModel: for field_name in fields.keys(): installed_deps[field_name] = is_installed(field_name) - if installed_deps.get("exllamav2") and is_torch_cuda_13(): - installed_deps["exllamav2"] = False - return DependenciesModel(**installed_deps) diff --git a/config_sample.yml b/config_sample.yml index ee60d59..389da4e 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -81,7 +81,7 @@ model: use_as_default: [] # Backend to use for this model (auto-detect if not specified) - # Options: exllamav2, exllamav3 + # Options: exllamav3 backend: # Max sequence length (default: min(max_position_embeddings, cache_size)). @@ -90,13 +90,11 @@ model: # Size of the key/value cache to allocate, in tokens (default: 4096). # Must be a multiple of 256. - # ExllamaV2 note: On AMD GPUs and NVIDIA GPUs older than Ampere, this value - # is ignored. Please use max_seq_len cache_size: # Enable different cache modes for VRAM savings (default: FP16). - # Possible values for exllamav2: 'FP16', 'Q8', 'Q6', 'Q4'. - # For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8). + # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8). + # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted. cache_mode: FP16 # Load model with tensor parallelism. @@ -213,8 +211,8 @@ draft_model: draft_rope_alpha: # Cache mode for draft models to save VRAM (default: FP16). - # Possible values for exllamav2: 'FP16', 'Q8', 'Q6', 'Q4'. - # For exllamav3, specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8). + # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8). + # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted. draft_cache_mode: FP16 # Array of VRAM sizes to split between GPUs, in GB (default: []). @@ -281,7 +279,7 @@ memory: # Options for development and experimentation developer: - # Skip Exllamav2 version check (default: False). + # Skip Exllamav3 version check (default: False). # WARNING: It's highly recommended to update your dependencies rather than enabling this flag. unsafe_launch: false diff --git a/docker/Dockerfile b/docker/Dockerfile index 861d444..a31b5db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ WORKDIR /app # Get requirements COPY pyproject.toml . -# Install cu12 group first — pins torch+cu128, exllamav2/v3+cu128, +# Install cu12 group first — pins torch+cu128, exllamav3+cu128, # flash_attn+cu128, and flash-linear-attention. # The 'extras' group (infinity-emb, sentence-transformers) is installed separately # with --no-deps so pip cannot resolve xformers transitively and pull a cu130 wheel, diff --git a/docker/Dockerfile.cu13 b/docker/Dockerfile.cu13 index 342ab3a..4806e46 100644 --- a/docker/Dockerfile.cu13 +++ b/docker/Dockerfile.cu13 @@ -27,7 +27,7 @@ WORKDIR /app COPY pyproject.toml . # Install cu13 group first. This uses torch 2.11.0+cu130 with exllamav3 cu132 -# wheels, flash_attn+cu130, and intentionally does not install exllamav2. +# wheels and flash_attn+cu130. # The 'extras' group (infinity-emb, sentence-transformers) is installed separately # with --no-deps so pip cannot resolve xformers transitively and pull incompatible # CUDA wheels. diff --git a/docs/01.-Getting-Started.md b/docs/01.-Getting-Started.md index c19f15b..e54a8db 100644 --- a/docs/01.-Getting-Started.md +++ b/docs/01.-Getting-Started.md @@ -12,12 +12,10 @@ To get started, make sure you have the following installed on your system: > Prefer a video guide? Watch the step-by-step tutorial on [YouTube](https://www.youtube.com/watch?v=03jYz0ijbUU) > [!WARNING] -> CUDA and ROCm aren't prerequisites because torch can install them for you. However, if this doesn't work (ex. DLL load failed), install the CUDA toolkit or ROCm on your system. +> CUDA isn't a prerequisite because torch can install it for you. However, if this doesn't work (ex. DLL load failed), install the CUDA toolkit on your system. > > - [CUDA 12.x](https://developer.nvidia.com/cuda-downloads) > -> - [ROCm 6.1](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.1.0/how-to/prerequisites.html) -> > [!WARNING] > Sometimes there may be an error with Windows that VS build tools needs to be installed. This means that there's a package that isn't supported for your python version. @@ -48,8 +46,7 @@ To get started, make sure you have the following installed on your system: 2. On Linux: `source venv/bin/activate` 3. Install the pyproject features based on your system: 1. Cuda 12.x: `pip install -U .[cu12]` - 2. Cuda 13.x (Exllamav3 only, Python 3.12+): `pip install -U .[cu13]` - 3. ROCm 5.6: `pip install -U .[amd]` + 2. Cuda 13.x (Python 3.12+): `pip install -U .[cu13]` 4. Start the API by either 1. Run `start.bat/sh`. The script will check if you're in a conda environment and skip venv checks. 2. Run `python main.py` to start the API. This won't automatically upgrade your dependencies. @@ -62,9 +59,9 @@ TabbyAPI includes a built-in Hugging Face downloader that works via both the API Example with Turboderp's Llama 3.1 8B quants: -`.\Start.bat download turboderp/Qwen2.5-VL-7B-Instruct-exl2 --revision 4.0bpw` +`.\Start.bat download turboderp/Qwen2.5-VL-7B-Instruct-exl3 --revision 4.0bpw` -If a model is gated, you can provide a HuggingFace access token (most exl2 quants aren't private): +If a model is gated, you can provide a HuggingFace access token (most exl3 quants aren't private): `.\Start.bat download meta-llama/Llama-3.1-8B --token ` @@ -100,33 +97,31 @@ These scripts exit after running their respective tasks. To start TabbyAPI, run 2. **Manual** - Install the pyproject features and update dependencies depending on your GPU: 1. `pip install -U .[cu12]` = CUDA 12.x - 2. `pip install -U .[cu13]` = CUDA 13.x (Exllamav3 only, Python 3.12+) - 3. `pip install -U .[amd]` = ROCm 6.0 + 2. `pip install -U .[cu13]` = CUDA 13.x (Python 3.12+) -If you don't want to update dependencies that come from wheels (torch, exllamav2, exllamav3, and flash attention 2), use `pip install .` or pass the `--nowheel` flag when invoking the start scripts. +If you don't want to update dependencies that come from wheels (torch, exllamav3, and flash attention 2), use `pip install .` or pass the `--nowheel` flag when invoking the start scripts. -### Update Exllamav2 +### Update Exllamav3 > [!WARNING] > These instructions are meant for advanced users. > [!IMPORTANT] -> If you're installing a custom Exllamav2 wheel, make sure to use `pip install .` when updating! Otherwise, each update will overwrite your custom exllamav2 version. +> If you're installing a custom Exllamav3 wheel, make sure to use `pip install .` when updating! Otherwise, each update will overwrite your custom exllamav3 version. NOTE: -- TabbyAPI enforces the latest Exllamav2 version for compatibility purposes. +- TabbyAPI enforces the latest Exllamav3 version for compatibility purposes. - Any upgrades using a pyproject gpu lib feature will result in overwriting your installed wheel. - - To fix this, change the feature in `pyproject.toml` locally, create an issue or PR, or install your version of exllamav2 after upgrades. + - To fix this, change the feature in `pyproject.toml` locally, create an issue or PR, or install your version of exllamav3 after upgrades. -Here are ways to install exllamav2: +Here are ways to install exllamav3: -1. From a [wheel/release](https://github.com/turboderp/exllamav2#method-2-install-from-release-with-prebuilt-extension) (Recommended) - 1. Find the version that corresponds with your cuda and python version. For example, a wheel with `cu121` and `cp311` corresponds to CUDA 12.1 and python 3.11 -2. From [pip](https://github.com/turboderp/exllamav2#method-3-install-from-pypi): `pip install exllamav2` - 2. This is a JIT compiled extension, which means that the initial launch of tabbyAPI will take some time. The build may also not work due to improper environment configuration. -3. From [source](https://github.com/turboderp/exllamav2#method-1-install-from-source) +1. From a [wheel/release](https://github.com/turboderp-org/exllamav3/releases) (Recommended) + 1. Find the version that corresponds with your cuda and python version. For example, a wheel with `cu128` and `cp312` corresponds to CUDA 12.8 and python 3.12 +2. From [source](https://github.com/turboderp-org/exllamav3#installation) + 1. This builds the extension during installation, which may take some time. The build may also not work due to improper environment configuration. ## Other installation methods @@ -165,7 +160,7 @@ These are short-form instructions for other methods that users can use to instal ### Docker > [!NOTE] -> If you are planning to use custom versions of dependencies such as dev ExllamaV2, make sure to build the Docker image yourself! +> If you are planning to use custom versions of dependencies such as dev ExllamaV3, make sure to build the Docker image yourself! 1. Install Docker and docker compose from the [docs](https://docs.docker.com/compose/install/) 2. Install the Nvidia container compatibility layer diff --git a/docs/03.-Usage.md b/docs/03.-Usage.md index 64285a6..f4a1244 100644 --- a/docs/03.-Usage.md +++ b/docs/03.-Usage.md @@ -1,6 +1,6 @@ ## Usage -TabbyAPI's main use-case is to be an API server for running ExllamaV2 models. +TabbyAPI's main use-case is to be an API server for running ExllamaV3 models. ### API Server @@ -17,7 +17,7 @@ Below is an example CURL request using the OpenAI completions endpoint: curl http://localhost:5000/v1/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "Meta-Llama-3-8B-exl2", + "model": "Meta-Llama-3-8B-exl3", "prompt": "Once upon a time,", "max_tokens": 400, "stream": false, @@ -81,7 +81,7 @@ Below is an example CURL request using the model load endpoint: curl http://localhost:5000/v1/model/load \ -H "Content-Type: application/json" \ -d '{ - "model_name": "Meta-Llama-3-8B-exl2", + "model_name": "Meta-Llama-3-8B-exl3", "max_seq_len": 8192, "tensor_parallel": true, "gpu_split_auto": false, @@ -96,10 +96,10 @@ A model load request can also include draft model parameters: curl http://localhost:5000/v1/model/load \ -H "Content-Type: application/json" \ -d '{ - "model_name": "Meta-Llama-3-8B-exl2", + "model_name": "Meta-Llama-3-8B-exl3", ... Other parameters "draft_model": { - draft_model_name: "TinyLlama-1B-32k-exl2", + draft_model_name: "TinyLlama-1B-32k-exl3", draft_rope_scale: 1.0 } }' @@ -116,7 +116,7 @@ An alternative way of switching models is called "inline loading" which hooks in To get started, set `inline_model_loading` to `true` under the model block of config.yml. -Now to create a tabby config, let's say we have a model in our models directory called `Meta-Llama-3-8B-exl2`. Navigate into that model folder and create a file called `tabby_config.yml` +Now to create a tabby config, let's say we have a model in our models directory called `Meta-Llama-3-8B-exl3`. Navigate into that model folder and create a file called `tabby_config.yml` Now, you can place any model load parameter from `/v1/model/load` into that file. Here's a simple example which changes the default `max_seq_len` to 8192 and sets a Q6 quantized cache: @@ -133,7 +133,7 @@ model: max_seq_len: 8192 cache_mode: Q6 draft_model: - draft_model_name: TinyLlama-1B-32k-exl2 + draft_model_name: TinyLlama-1B-32k-exl3 draft_rope_scale: 1.0 ``` @@ -145,7 +145,7 @@ Below is an example CURL request for inline loading: curl http://localhost:5000/v1/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "Meta-Llama-3-8B-exl2" + "model": "Meta-Llama-3-8B-exl3" ... Other parameters }' ``` diff --git a/docs/05.-FAQ.md b/docs/05.-FAQ.md index e79e504..6c75711 100644 --- a/docs/05.-FAQ.md +++ b/docs/05.-FAQ.md @@ -9,15 +9,14 @@ - The wiki is meant for user-facing documentation. Devs are recommended to use the autogenerated documentation for [OpenAI](https://theroyallab.github.io/tabbyAPI) and [Kobold](https://theroyallab.github.io/tabbyAPI/kobold) servers - What does TabbyAPI run? - - TabbyAPI uses Exllamav2 as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported: - - Exl2 (Highly recommended) - - GPTQ - - FP16 (using Exllamav2's loader) + - TabbyAPI uses Exllamav3 as a powerful and fast backend for model inference, loading, etc. Therefore, the following types of models are supported: + - Exl3 (Highly recommended) + - FP16/BF16 (using Exllamav3's loader) -- Exllamav2 may error with the following exception: `ImportError: DLL load failed while importing exllamav2_ext: The specified module could not be found.` +- Exllamav3 may error with the following exception: `ImportError: DLL load failed while importing exllamav3_ext: The specified module could not be found.` - First, make sure to check if the wheel is equivalent to your python version and CUDA version. Also make sure you're in a venv or conda environment. - - If those prerequisites are correct, the torch cache may need to be cleared. This is due to a mismatching exllamav2_ext. + - If those prerequisites are correct, the torch cache may need to be cleared. This is due to a mismatching exllamav3_ext. - In Windows: Find the cache at `C:\Users\\AppData\Local\torch_extensions\torch_extensions\Cache` where `` is your Windows username - In Linux: Find the cache at `~/.cache/torch_extensions` - - look for any folder named `exllamav2_ext` in the python subdirectories and delete them. + - look for any folder named `exllamav3_ext` in the python subdirectories and delete them. - Restart TabbyAPI and launching should work again. \ No newline at end of file diff --git a/endpoints/server.py b/endpoints/server.py index 72ccacf..a7f979a 100644 --- a/endpoints/server.py +++ b/endpoints/server.py @@ -19,7 +19,7 @@ def setup_app(host: Optional[str] = None, port: Optional[int] = None): app = FastAPI( title="TabbyAPI", - summary="An OAI compatible exllamav2 API that's both lightweight and fast", + summary="An OAI compatible exllamav3 API that's both lightweight and fast", description=( "This docs page is not meant to send requests! Please use a service " "like Postman or a frontend UI." diff --git a/main.py b/main.py index 6dfa380..0dcde35 100644 --- a/main.py +++ b/main.py @@ -159,7 +159,7 @@ def entrypoint( # Skip if launching unsafely if config.developer.unsafe_launch: logger.warning( - "UNSAFE: Skipping ExllamaV2 version check.\n" + "UNSAFE: Skipping ExllamaV3 version check.\n" "If you aren't a developer, please keep this off!" ) elif not dependencies.inference: @@ -170,12 +170,10 @@ def entrypoint( f"update_deps.{'bat' if platform.system() == 'Windows' else 'sh'})\n\n" "Or you can manually run a requirements update " "using the following command:\n\n" - "For CUDA 12.1:\n" + "For CUDA 12.x:\n" "pip install --upgrade .[cu12]\n\n" "For CUDA 13.x:\n" "pip install --upgrade .[cu13]\n\n" - "For ROCm:\n" - "pip install --upgrade .[amd]\n\n" ) raise SystemExit(install_message) diff --git a/pyproject.toml b/pyproject.toml index 568894f..fab7079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ py-modules = [] [project] name = "tabbyAPI" version = "0.0.1" -description = "An OAI compatible exllamav2 API that's both lightweight and fast" +description = "An OAI compatible exllamav3 API that's both lightweight and fast" requires-python = ">=3.10" dependencies = [ "fastapi-slim >= 0.115", @@ -79,16 +79,6 @@ cu12 = [ "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33-cp39-abi3-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'", "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33-cp39-abi3-win_amd64.whl ; platform_system == 'Windows'", - # Exl2 - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp313-cp313-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.13'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp312-cp312-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.12'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp311-cp311-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.11'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp310-cp310-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.10'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+cu128.torch2.9.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'", - # Exl3 "exllamav3 @ https://github.com/turboderp-org/exllamav3/releases/download/v0.0.43/exllamav3-0.0.43+cu128.torch2.9.0-cp313-cp313-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.13'", "exllamav3 @ https://github.com/turboderp-org/exllamav3/releases/download/v0.0.43/exllamav3-0.0.43+cu128.torch2.9.0-cp312-cp312-win_amd64.whl ; platform_system == 'Windows' and python_version == '3.12'", @@ -145,26 +135,6 @@ cu13 = [ "flash_attn @ https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3%2Bcu130torch2.11-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'", "flash_attn @ https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.4/flash_attn-2.8.3%2Bcu130torch2.11-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'", ] -amd = [ - # Torch triton for ROCm - "pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'", - "pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'", - "pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "pytorch_triton_rocm @ https://download.pytorch.org/whl/pytorch_triton_rocm-3.4.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'", - - # Torch - "torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp313-cp313-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'", - "torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp312-cp312-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'", - "torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp311-cp311-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "torch @ https://download.pytorch.org/whl/rocm6.4/torch-2.9.0%2Brocm6.4-cp310-cp310-manylinux_2_28_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'", - - # Exl2 - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.13'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.12'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.11'", - "exllamav2 @ https://github.com/turboderp-org/exllamav2/releases/download/v0.3.2/exllamav2-0.3.2+rocm6.4.torch2.9.0-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64' and python_version == '3.10'", -] - # MARK: Ruff options [tool.ruff] diff --git a/start.py b/start.py index 2df783e..f088aba 100644 --- a/start.py +++ b/start.py @@ -2,7 +2,6 @@ import argparse import json -import os import pathlib import platform import subprocess @@ -44,30 +43,22 @@ def get_user_choice(question: str, options_dict: dict): def get_install_features(lib_name: str = None): """Fetches the appropriate requirements file depending on the GPU""" install_features = None - possible_features = ["cu12", "cu13", "amd"] + possible_features = ["cu12", "cu13"] if not lib_name: has_nvidia = which("nvidia-smi") is not None - has_rocm = which("rocm-smi") is not None - has_amd = which("amd-smi") is not None - has_amd_gpu = has_rocm or has_amd - if has_nvidia and not has_amd_gpu: + if has_nvidia: lib_name = "cu12" print("Auto-detected NVIDIA GPU. Using CUDA 12.x backend.") - elif has_amd_gpu and not has_nvidia: - lib_name = "amd" - print("Auto-detected AMD GPU. Using AMD backend.") else: gpu_lib_choices = { "A": {"pretty": "NVIDIA Cuda 12.x", "internal": "cu12"}, "B": {"pretty": "NVIDIA Cuda 13.x", "internal": "cu13"}, - "C": {"pretty": "AMD", "internal": "amd"}, } print( "WARNING: Auto-detection failed. " - "Please ensure you have either an NVIDIA GPU (with nvidia-smi) " - "or an AMD GPU (with rocm-smi or amd-smi) installed." + "Please ensure you have an NVIDIA GPU (with nvidia-smi) installed." ) user_input = get_user_choice( "Select your GPU. If you don't know, select Cuda 12.x (A)", @@ -92,20 +83,6 @@ def get_install_features(lib_name: str = None): ) return - if install_features == "amd": - # Exit if using AMD and Windows - if platform.system() == "Windows": - print( - "ERROR: TabbyAPI does not support AMD and Windows. " - "Please use Linux and ROCm 6.4. Exiting." - ) - sys.exit(0) - - # Override env vars for ROCm support on non-supported GPUs - os.environ["ROCM_PATH"] = "/opt/rocm" - os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.3.0" - os.environ["HCC_AMDGPU_TARGET"] = "gfx1030" - return install_features @@ -148,12 +125,12 @@ def add_start_args(parser: argparse.ArgumentParser): "-nw", "--nowheel", action="store_true", - help="Don't upgrade wheel dependencies (exllamav2, torch)", + help="Don't upgrade wheel dependencies (exllamav3, torch)", ) start_group.add_argument( "--gpu-lib", type=str, - help="Select GPU library. Options: cu12, cu13, amd", + help="Select GPU library. Options: cu12, cu13", ) diff --git a/tests/wheel_test.py b/tests/wheel_test.py index 5ef0c67..498263e 100644 --- a/tests/wheel_test.py +++ b/tests/wheel_test.py @@ -12,20 +12,12 @@ if find_spec("flash_attn") is not None: else: print("Flash attention 2 is not found in your environment.") -if find_spec("exllamav2") is not None: - print(f"Exllamav2 on version {version('exllamav2')} successfully imported") - successful_packages.append("exllamav2") -else: - print("Exllamav2 is not found in your environment.") - if find_spec("exllamav3") is not None: print(f"Exllamav3 on version {version('exllamav3')} successfully imported") successful_packages.append("exllamav3") else: print("Exllamav3 is not found in your environment.") - -if find_spec("exllamav2") is None and find_spec("exllamav3") is None: - errored_packages.append("exllamav2/exllamav3") + errored_packages.append("exllamav3") if find_spec("torch") is not None: print(f"Torch on version {version('torch')} successfully imported")