mirror of
https://github.com/theroyallab/tabbyAPI.git
synced 2026-04-21 06:48:56 +00:00
Model: Reject model requests when unloading
If a model is being unloaded, that means its being shut down and no requests should be accepted from then on. Also, remove model_is_loaded since we simply check if the container is None now. Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com>
This commit is contained in:
@@ -14,7 +14,6 @@ if dependencies.extras:
|
||||
|
||||
class InfinityContainer:
|
||||
model_dir: pathlib.Path
|
||||
model_is_loading: bool = False
|
||||
model_loaded: bool = False
|
||||
|
||||
# Use a runtime type hint here
|
||||
@@ -24,8 +23,6 @@ class InfinityContainer:
|
||||
self.model_dir = model_directory
|
||||
|
||||
async def load(self, **kwargs):
|
||||
self.model_is_loading = True
|
||||
|
||||
# Use cpu by default
|
||||
device = unwrap(kwargs.get("embeddings_device"), "cpu")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user