mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2026-03-06 05:40:00 +00:00
Decouple orphan pruning from asset seeding
- Remove automatic pruning from scan loop to prevent partial scans from deleting assets belonging to other roots - Add get_all_known_prefixes() helper to get prefixes for all root types - Add prune_orphans() method to AssetSeeder for explicit pruning - Add prune_first parameter to start() for optional pre-scan pruning - Add POST /api/assets/prune endpoint for explicit pruning via API - Update main.py startup to use prune_first=True for full startup scans - Add tests for new prune_orphans functionality Fixes issue where a models-only scan would delete all input/output assets. Amp-Thread-ID: https://ampcode.com/threads/T-019c2ba0-e004-7229-81bf-452b2f7f57a1 Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
@@ -695,3 +695,23 @@ async def cancel_seed(request: web.Request) -> web.Response:
|
||||
if cancelled:
|
||||
return web.json_response({"status": "cancelling"}, status=200)
|
||||
return web.json_response({"status": "idle"}, status=200)
|
||||
|
||||
|
||||
@ROUTES.post("/api/assets/prune")
|
||||
async def prune_orphans(request: web.Request) -> web.Response:
|
||||
"""Prune orphaned assets that no longer exist on the filesystem.
|
||||
|
||||
This removes assets whose cache states point to files outside all known
|
||||
root prefixes (models, input, output).
|
||||
|
||||
Returns:
|
||||
200 OK with count of pruned assets
|
||||
409 Conflict if a scan is currently running
|
||||
"""
|
||||
pruned = asset_seeder.prune_orphans()
|
||||
if pruned == 0 and asset_seeder.get_status().state.value != "IDLE":
|
||||
return web.json_response(
|
||||
{"status": "scan_running", "pruned": 0},
|
||||
status=409,
|
||||
)
|
||||
return web.json_response({"status": "completed", "pruned": pruned}, status=200)
|
||||
|
||||
@@ -61,6 +61,14 @@ def get_prefixes_for_root(root: RootType) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def get_all_known_prefixes() -> list[str]:
|
||||
"""Get all known asset prefixes across all root types."""
|
||||
all_roots: tuple[RootType, ...] = ("models", "input", "output")
|
||||
return [
|
||||
os.path.abspath(p) for root in all_roots for p in get_prefixes_for_root(root)
|
||||
]
|
||||
|
||||
|
||||
def collect_models_files() -> list[str]:
|
||||
out: list[str] = []
|
||||
for folder_name, bases in get_comfy_models_folders():
|
||||
@@ -287,7 +295,11 @@ def _insert_asset_specs(specs: list[SeedAssetSpec], tag_pool: set[str]) -> int:
|
||||
|
||||
|
||||
def seed_assets(roots: tuple[RootType, ...], enable_logging: bool = False) -> None:
|
||||
"""Scan the given roots and seed the assets into the database."""
|
||||
"""Scan the given roots and seed the assets into the database.
|
||||
|
||||
Note: This function does not prune orphaned assets. Call prune_orphaned_assets
|
||||
separately if cleanup is needed.
|
||||
"""
|
||||
if not dependencies_available():
|
||||
if enable_logging:
|
||||
logging.warning("Database dependencies not available, skipping assets scan")
|
||||
@@ -299,20 +311,16 @@ def seed_assets(roots: tuple[RootType, ...], enable_logging: bool = False) -> No
|
||||
for r in roots:
|
||||
existing_paths.update(_sync_root_safely(r))
|
||||
|
||||
all_prefixes = [os.path.abspath(p) for r in roots for p in get_prefixes_for_root(r)]
|
||||
orphans_pruned = _prune_orphans_safely(all_prefixes)
|
||||
|
||||
paths = _collect_paths_for_roots(roots)
|
||||
specs, tag_pool, skipped_existing = _build_asset_specs(paths, existing_paths)
|
||||
created = _insert_asset_specs(specs, tag_pool)
|
||||
|
||||
if enable_logging:
|
||||
logging.info(
|
||||
"Assets scan(roots=%s) completed in %.3fs (created=%d, skipped_existing=%d, orphans_pruned=%d, total_seen=%d)",
|
||||
"Assets scan(roots=%s) completed in %.3fs (created=%d, skipped_existing=%d, total_seen=%d)",
|
||||
roots,
|
||||
time.perf_counter() - t_start,
|
||||
created,
|
||||
skipped_existing,
|
||||
orphans_pruned,
|
||||
len(paths),
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.assets.scanner import (
|
||||
_insert_asset_specs,
|
||||
_prune_orphans_safely,
|
||||
_sync_root_safely,
|
||||
get_all_known_prefixes,
|
||||
get_prefixes_for_root,
|
||||
)
|
||||
from app.database.db import dependencies_available
|
||||
@@ -85,14 +86,16 @@ class AssetSeeder:
|
||||
|
||||
def start(
|
||||
self,
|
||||
roots: tuple[RootType, ...] = ("models",),
|
||||
roots: tuple[RootType, ...] = ("models", "input", "output"),
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
prune_first: bool = False,
|
||||
) -> bool:
|
||||
"""Start a background scan for the given roots.
|
||||
|
||||
Args:
|
||||
roots: Tuple of root types to scan (models, input, output)
|
||||
progress_callback: Optional callback called with progress updates
|
||||
prune_first: If True, prune orphaned assets before scanning
|
||||
|
||||
Returns:
|
||||
True if scan was started, False if already running
|
||||
@@ -104,6 +107,7 @@ class AssetSeeder:
|
||||
self._progress = Progress()
|
||||
self._errors = []
|
||||
self._roots = roots
|
||||
self._prune_first = prune_first
|
||||
self._progress_callback = progress_callback
|
||||
self._cancel_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
@@ -170,6 +174,34 @@ class AssetSeeder:
|
||||
with self._lock:
|
||||
self._thread = None
|
||||
|
||||
def prune_orphans(self) -> int:
|
||||
"""Prune orphaned assets that are outside all known root prefixes.
|
||||
|
||||
This operation is decoupled from scanning to prevent partial scans
|
||||
from accidentally deleting assets belonging to other roots.
|
||||
|
||||
Should be called explicitly when cleanup is desired, typically after
|
||||
a full scan of all roots or during maintenance.
|
||||
|
||||
Returns:
|
||||
Number of orphaned assets pruned, or 0 if dependencies unavailable
|
||||
or a scan is currently running
|
||||
"""
|
||||
with self._lock:
|
||||
if self._state != State.IDLE:
|
||||
logging.warning("Cannot prune orphans while scan is running")
|
||||
return 0
|
||||
|
||||
if not dependencies_available():
|
||||
logging.warning("Database dependencies not available, skipping orphan pruning")
|
||||
return 0
|
||||
|
||||
all_prefixes = get_all_known_prefixes()
|
||||
pruned = _prune_orphans_safely(all_prefixes)
|
||||
if pruned > 0:
|
||||
logging.info("Pruned %d orphaned assets", pruned)
|
||||
return pruned
|
||||
|
||||
def _is_cancelled(self) -> bool:
|
||||
"""Check if cancellation has been requested."""
|
||||
return self._cancel_event.is_set()
|
||||
@@ -254,6 +286,17 @@ class AssetSeeder:
|
||||
)
|
||||
return
|
||||
|
||||
if self._prune_first:
|
||||
all_prefixes = get_all_known_prefixes()
|
||||
pruned = _prune_orphans_safely(all_prefixes)
|
||||
if pruned > 0:
|
||||
logging.info("Pruned %d orphaned assets before scan", pruned)
|
||||
|
||||
if self._is_cancelled():
|
||||
logging.info("Asset scan cancelled after pruning phase")
|
||||
cancelled = True
|
||||
return
|
||||
|
||||
self._log_scan_config(roots)
|
||||
|
||||
existing_paths: set[str] = set()
|
||||
@@ -269,16 +312,6 @@ class AssetSeeder:
|
||||
cancelled = True
|
||||
return
|
||||
|
||||
all_prefixes = [
|
||||
os.path.abspath(p) for r in roots for p in get_prefixes_for_root(r)
|
||||
]
|
||||
orphans_pruned = _prune_orphans_safely(all_prefixes)
|
||||
|
||||
if self._is_cancelled():
|
||||
logging.info("Asset scan cancelled after orphan pruning")
|
||||
cancelled = True
|
||||
return
|
||||
|
||||
paths = _collect_paths_for_roots(roots)
|
||||
total_paths = len(paths)
|
||||
self._update_progress(total=total_paths)
|
||||
@@ -335,12 +368,11 @@ class AssetSeeder:
|
||||
|
||||
elapsed = time.perf_counter() - t_start
|
||||
logging.info(
|
||||
"Asset scan(roots=%s) completed in %.3fs (created=%d, skipped=%d, orphans_pruned=%d, total=%d)",
|
||||
"Asset scan(roots=%s) completed in %.3fs (created=%d, skipped=%d, total=%d)",
|
||||
roots,
|
||||
elapsed,
|
||||
total_created,
|
||||
skipped_existing,
|
||||
orphans_pruned,
|
||||
len(paths),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user