mirror of
https://github.com/theroyallab/tabbyAPI.git
synced 2026-03-15 00:07:28 +00:00
Update exl3 backend model.py: fix for unloading vision models
This change ensures that when unloading vlm their vision part is also unloaded.
This commit is contained in:
@@ -563,6 +563,10 @@ class ExllamaV3Container(BaseModelContainer):
|
||||
self.draft_config = None
|
||||
self.draft_cache = None
|
||||
|
||||
if self.use_vision:
|
||||
self.vision_model.unload()
|
||||
self.vision_model = None
|
||||
|
||||
# Cleanup the generator from any pending jobs
|
||||
if self.generator is not None:
|
||||
await self.generator.close()
|
||||
|
||||
Reference in New Issue
Block a user