mirror of
https://github.com/theroyallab/tabbyAPI.git
synced 2026-03-15 00:07:28 +00:00
Signal: Fix async signal handling
Run unload async functions before exiting the program. Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import gc
|
||||
import pathlib
|
||||
import torch
|
||||
from loguru import logger
|
||||
from typing import List, Optional
|
||||
|
||||
from common.utils import unwrap
|
||||
@@ -50,6 +51,8 @@ class InfinityContainer:
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
logger.info("Embedding model unloaded.")
|
||||
|
||||
async def generate(self, sentence_input: List[str]):
|
||||
result_embeddings, usage = await self.engine.embed(sentence_input)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user