refactor(assets): modular architecture + async two-phase scanner & background seeder (#12621)

app/assets/api/schemas_in.py

@@ -1,6 +1,8 @@
 import json
+from dataclasses import dataclass
 from typing import Any, Literal
 
+from app.assets.helpers import validate_blake3_hash
 from pydantic import (
     BaseModel,
     ConfigDict,
@@ -10,6 +12,41 @@ from pydantic import (
     model_validator,
 )
 
+
+class UploadError(Exception):
+    """Error during upload parsing with HTTP status and code."""
+
+    def __init__(self, status: int, code: str, message: str):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+
+
+class AssetValidationError(Exception):
+    """Validation error in asset processing (invalid tags, metadata, etc.)."""
+
+    def __init__(self, code: str, message: str):
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass
+class ParsedUpload:
+    """Result of parsing a multipart upload request."""
+
+    file_present: bool
+    file_written: int
+    file_client_name: str | None
+    tmp_path: str | None
+    tags_raw: list[str]
+    provided_name: str | None
+    user_metadata_raw: str | None
+    provided_hash: str | None
+    provided_hash_exists: bool | None
+
+
 class ListAssetsQuery(BaseModel):
     include_tags: list[str] = Field(default_factory=list)
     exclude_tags: list[str] = Field(default_factory=list)
@@ -21,7 +58,9 @@ class ListAssetsQuery(BaseModel):
     limit: conint(ge=1, le=500) = 20
     offset: conint(ge=0) = 0
 
-    sort: Literal["name", "created_at", "updated_at", "size", "last_access_time"] = "created_at"
+    sort: Literal["name", "created_at", "updated_at", "size", "last_access_time"] = (
+        "created_at"
+    )
     order: Literal["asc", "desc"] = "desc"
 
     @field_validator("include_tags", "exclude_tags", mode="before")
@@ -61,7 +100,7 @@ class UpdateAssetBody(BaseModel):
     user_metadata: dict[str, Any] | None = None
 
     @model_validator(mode="after")
-    def _at_least_one(self):
+    def _validate_at_least_one_field(self):
         if self.name is None and self.user_metadata is None:
             raise ValueError("Provide at least one of: name, user_metadata.")
         return self
@@ -78,19 +117,11 @@ class CreateFromHashBody(BaseModel):
     @field_validator("hash")
     @classmethod
     def _require_blake3(cls, v):
-        s = (v or "").strip().lower()
-        if ":" not in s:
-            raise ValueError("hash must be 'blake3:<hex>'")
-        algo, digest = s.split(":", 1)
-        if algo != "blake3":
-            raise ValueError("only canonical 'blake3:<hex>' is accepted here")
-        if not digest or any(c for c in digest if c not in "0123456789abcdef"):
-            raise ValueError("hash digest must be lowercase hex")
-        return s
+        return validate_blake3_hash(v or "")
 
     @field_validator("tags", mode="before")
     @classmethod
-    def _tags_norm(cls, v):
+    def _normalize_tags_field(cls, v):
         if v is None:
             return []
         if isinstance(v, list):
@@ -154,15 +185,16 @@ class TagsRemove(TagsAdd):
 
 class UploadAssetSpec(BaseModel):
     """Upload Asset operation.
+
     - tags: ordered; first is root ('models'|'input'|'output');
-            if root == 'models', second must be a valid category from folder_paths.folder_names_and_paths
+            if root == 'models', second must be a valid category
     - name: display name
     - user_metadata: arbitrary JSON object (optional)
-    - hash: optional canonical 'blake3:<hex>' provided by the client for validation / fast-path
+    - hash: optional canonical 'blake3:<hex>' for validation / fast-path
 
-    Files created via this endpoint are stored on disk using the **content hash** as the filename stem
-    and the original extension is preserved when available.
+    Files are stored using the content hash as filename stem.
     """
+
     model_config = ConfigDict(extra="ignore", str_strip_whitespace=True)
 
     tags: list[str] = Field(..., min_length=1)
@@ -175,17 +207,10 @@ class UploadAssetSpec(BaseModel):
     def _parse_hash(cls, v):
         if v is None:
             return None
-        s = str(v).strip().lower()
+        s = str(v).strip()
         if not s:
             return None
-        if ":" not in s:
-            raise ValueError("hash must be 'blake3:<hex>'")
-        algo, digest = s.split(":", 1)
-        if algo != "blake3":
-            raise ValueError("only canonical 'blake3:<hex>' is accepted here")
-        if not digest or any(c for c in digest if c not in "0123456789abcdef"):
-            raise ValueError("hash digest must be lowercase hex")
-        return f"{algo}:{digest}"
+        return validate_blake3_hash(s)
 
     @field_validator("tags", mode="before")
     @classmethod
@@ -260,5 +285,7 @@ class UploadAssetSpec(BaseModel):
             raise ValueError("first tag must be one of: models, input, output")
         if root == "models":
             if len(self.tags) < 2:
-                raise ValueError("models uploads require a category tag as the second tag")
+                raise ValueError(
+                    "models uploads require a category tag as the second tag"
+                )
         return self

app/assets/api/schemas_out.py

@@ -19,7 +19,7 @@ class AssetSummary(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     @field_serializer("created_at", "updated_at", "last_access_time")
-    def _ser_dt(self, v: datetime | None, _info):
+    def _serialize_datetime(self, v: datetime | None, _info):
         return v.isoformat() if v else None
 
 
@@ -40,7 +40,7 @@ class AssetUpdated(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     @field_serializer("updated_at")
-    def _ser_updated(self, v: datetime | None, _info):
+    def _serialize_updated_at(self, v: datetime | None, _info):
         return v.isoformat() if v else None
 
 
@@ -59,7 +59,7 @@ class AssetDetail(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     @field_serializer("created_at", "last_access_time")
-    def _ser_dt(self, v: datetime | None, _info):
+    def _serialize_datetime(self, v: datetime | None, _info):
         return v.isoformat() if v else None
 
 

app/assets/api/upload.py

@@ -0,0 +1,171 @@
+import logging
+import os
+import uuid
+from typing import Callable
+
+from aiohttp import web
+
+import folder_paths
+from app.assets.api.schemas_in import ParsedUpload, UploadError
+from app.assets.helpers import validate_blake3_hash
+
+
+def normalize_and_validate_hash(s: str) -> str:
+    """Validate and normalize a hash string.
+
+    Returns canonical 'blake3:<hex>' or raises UploadError.
+    """
+    try:
+        return validate_blake3_hash(s)
+    except ValueError:
+        raise UploadError(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")
+
+
+async def parse_multipart_upload(
+    request: web.Request,
+    check_hash_exists: Callable[[str], bool],
+) -> ParsedUpload:
+    """
+    Parse a multipart/form-data upload request.
+
+    Args:
+        request: The aiohttp request
+        check_hash_exists: Callable(hash_str) -> bool to check if a hash exists
+
+    Returns:
+        ParsedUpload with parsed fields and temp file path
+
+    Raises:
+        UploadError: On validation or I/O errors
+    """
+    if not (request.content_type or "").lower().startswith("multipart/"):
+        raise UploadError(
+            415, "UNSUPPORTED_MEDIA_TYPE", "Use multipart/form-data for uploads."
+        )
+
+    reader = await request.multipart()
+
+    file_present = False
+    file_client_name: str | None = None
+    tags_raw: list[str] = []
+    provided_name: str | None = None
+    user_metadata_raw: str | None = None
+    provided_hash: str | None = None
+    provided_hash_exists: bool | None = None
+
+    file_written = 0
+    tmp_path: str | None = None
+
+    while True:
+        field = await reader.next()
+        if field is None:
+            break
+
+        fname = getattr(field, "name", "") or ""
+
+        if fname == "hash":
+            try:
+                s = ((await field.text()) or "").strip().lower()
+            except Exception:
+                raise UploadError(
+                    400, "INVALID_HASH", "hash must be like 'blake3:<hex>'"
+                )
+
+            if s:
+                provided_hash = normalize_and_validate_hash(s)
+                try:
+                    provided_hash_exists = check_hash_exists(provided_hash)
+                except Exception as e:
+                    logging.exception(
+                        "check_hash_exists failed for hash=%s: %s", provided_hash, e
+                    )
+                    raise UploadError(
+                        500,
+                        "HASH_CHECK_FAILED",
+                        "Backend error while checking asset hash.",
+                    )
+
+        elif fname == "file":
+            file_present = True
+            file_client_name = (field.filename or "").strip()
+
+            if provided_hash and provided_hash_exists is True:
+                # Hash exists - drain file but don't write to disk
+                try:
+                    while True:
+                        chunk = await field.read_chunk(8 * 1024 * 1024)
+                        if not chunk:
+                            break
+                        file_written += len(chunk)
+                except Exception:
+                    raise UploadError(
+                        500, "UPLOAD_IO_ERROR", "Failed to receive uploaded file."
+                    )
+                continue
+
+            uploads_root = os.path.join(folder_paths.get_temp_directory(), "uploads")
+            unique_dir = os.path.join(uploads_root, uuid.uuid4().hex)
+            os.makedirs(unique_dir, exist_ok=True)
+            tmp_path = os.path.join(unique_dir, ".upload.part")
+
+            try:
+                with open(tmp_path, "wb") as f:
+                    while True:
+                        chunk = await field.read_chunk(8 * 1024 * 1024)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+                        file_written += len(chunk)
+            except Exception:
+                delete_temp_file_if_exists(tmp_path)
+                raise UploadError(
+                    500, "UPLOAD_IO_ERROR", "Failed to receive and store uploaded file."
+                )
+
+        elif fname == "tags":
+            tags_raw.append((await field.text()) or "")
+        elif fname == "name":
+            provided_name = (await field.text()) or None
+        elif fname == "user_metadata":
+            user_metadata_raw = (await field.text()) or None
+
+    if not file_present and not (provided_hash and provided_hash_exists):
+        raise UploadError(
+            400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'."
+        )
+
+    if (
+        file_present
+        and file_written == 0
+        and not (provided_hash and provided_hash_exists)
+    ):
+        delete_temp_file_if_exists(tmp_path)
+        raise UploadError(400, "EMPTY_UPLOAD", "Uploaded file is empty.")
+
+    return ParsedUpload(
+        file_present=file_present,
+        file_written=file_written,
+        file_client_name=file_client_name,
+        tmp_path=tmp_path,
+        tags_raw=tags_raw,
+        provided_name=provided_name,
+        user_metadata_raw=user_metadata_raw,
+        provided_hash=provided_hash,
+        provided_hash_exists=provided_hash_exists,
+    )
+
+
+def delete_temp_file_if_exists(tmp_path: str | None) -> None:
+    """Safely remove a temp file and its parent directory if empty."""
+    if tmp_path:
+        try:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+        except OSError as e:
+            logging.debug("Failed to delete temp file %s: %s", tmp_path, e)
+        try:
+            parent = os.path.dirname(tmp_path)
+            if parent and os.path.isdir(parent):
+                os.rmdir(parent)  # only succeeds if empty
+        except OSError:
+            pass