refactor(assets): modular architecture + async two-phase scanner & background seeder (#12621)

tests-unit/assets_test/test_crud.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 import pytest
 import requests
-from conftest import get_asset_filename, trigger_sync_seed_assets
+from helpers import get_asset_filename, trigger_sync_seed_assets
 
 
 def test_create_from_hash_success(
@@ -24,11 +24,11 @@ def test_create_from_hash_success(
     assert b1["created_new"] is False
     aid = b1["id"]
 
-    # Calling again with the same name should return the same AssetInfo id
+    # Calling again with the same name creates a new AssetReference (duplicates allowed)
     r2 = http.post(f"{api_base}/api/assets/from-hash", json=payload, timeout=120)
     b2 = r2.json()
     assert r2.status_code == 201, b2
-    assert b2["id"] == aid
+    assert b2["id"] != aid  # new reference, not the same one
 
 
 def test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asset: dict):
@@ -42,8 +42,8 @@ def test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asse
     assert "user_metadata" in detail
     assert "filename" in detail["user_metadata"]
 
-    # DELETE
-    rd = http.delete(f"{api_base}/api/assets/{aid}", timeout=120)
+    # DELETE (hard delete to also remove underlying asset and file)
+    rd = http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=120)
     assert rd.status_code == 204
 
     # GET again -> 404
@@ -53,6 +53,35 @@ def test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asse
     assert body["error"]["code"] == "ASSET_NOT_FOUND"
 
 
+def test_soft_delete_hides_from_get(http: requests.Session, api_base: str, seeded_asset: dict):
+    aid = seeded_asset["id"]
+    asset_hash = seeded_asset["asset_hash"]
+
+    # Soft-delete (default, no delete_content param)
+    rd = http.delete(f"{api_base}/api/assets/{aid}", timeout=120)
+    assert rd.status_code == 204
+
+    # GET by reference ID -> 404 (soft-deleted references are hidden)
+    rg = http.get(f"{api_base}/api/assets/{aid}", timeout=120)
+    assert rg.status_code == 404
+
+    # Asset identity is preserved (underlying content still exists)
+    rh = http.head(f"{api_base}/api/assets/hash/{asset_hash}", timeout=120)
+    assert rh.status_code == 200
+
+    # Soft-deleted reference should not appear in listings
+    rl = http.get(
+        f"{api_base}/api/assets",
+        params={"include_tags": "unit-tests", "limit": "500"},
+        timeout=120,
+    )
+    ids = [a["id"] for a in rl.json().get("assets", [])]
+    assert aid not in ids
+
+    # Clean up: hard-delete the soft-deleted reference and orphaned asset
+    http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=120)
+
+
 def test_delete_upon_reference_count(
     http: requests.Session, api_base: str, seeded_asset: dict
 ):
@@ -70,21 +99,32 @@ def test_delete_upon_reference_count(
     assert copy["asset_hash"] == src_hash
     assert copy["created_new"] is False
 
-    # Delete original reference -> asset identity must remain
+    # Soft-delete original reference (default) -> asset identity must remain
     aid1 = seeded_asset["id"]
     rd1 = http.delete(f"{api_base}/api/assets/{aid1}", timeout=120)
     assert rd1.status_code == 204
 
     rh1 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)
-    assert rh1.status_code == 200  # identity still present
+    assert rh1.status_code == 200  # identity still present (second ref exists)
 
-    # Delete the last reference with default semantics -> identity and cached files removed
+    # Soft-delete the last reference -> asset identity preserved (no hard delete)
     aid2 = copy["id"]
     rd2 = http.delete(f"{api_base}/api/assets/{aid2}", timeout=120)
     assert rd2.status_code == 204
 
     rh2 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)
-    assert rh2.status_code == 404  # orphan content removed
+    assert rh2.status_code == 200  # asset identity preserved (soft delete)
+
+    # Re-associate via from-hash, then hard-delete -> orphan content removed
+    r3 = http.post(f"{api_base}/api/assets/from-hash", json=payload, timeout=120)
+    assert r3.status_code == 201, r3.json()
+    aid3 = r3.json()["id"]
+
+    rd3 = http.delete(f"{api_base}/api/assets/{aid3}?delete_content=true", timeout=120)
+    assert rd3.status_code == 204
+
+    rh3 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)
+    assert rh3.status_code == 404  # orphan content removed
 
 
 def test_update_asset_fields(http: requests.Session, api_base: str, seeded_asset: dict):
@@ -126,42 +166,52 @@ def test_head_asset_bad_hash_returns_400_and_no_body(http: requests.Session, api
     assert body == b""
 
 
-def test_delete_nonexistent_returns_404(http: requests.Session, api_base: str):
-    bogus = str(uuid.uuid4())
-    r = http.delete(f"{api_base}/api/assets/{bogus}", timeout=120)
+@pytest.mark.parametrize(
+    "method,endpoint_template,payload,expected_status,error_code",
+    [
+        # Delete nonexistent asset
+        ("delete", "/api/assets/{uuid}", None, 404, "ASSET_NOT_FOUND"),
+        # Bad hash algorithm in from-hash
+        (
+            "post",
+            "/api/assets/from-hash",
+            {"hash": "sha256:" + "0" * 64, "name": "x.bin", "tags": ["models", "checkpoints", "unit-tests"]},
+            400,
+            "INVALID_BODY",
+        ),
+        # Get with bad UUID format
+        ("get", "/api/assets/not-a-uuid", None, 404, None),
+        # Get content with bad UUID format
+        ("get", "/api/assets/not-a-uuid/content", None, 404, None),
+    ],
+    ids=["delete_nonexistent", "bad_hash_algorithm", "get_bad_uuid", "content_bad_uuid"],
+)
+def test_error_responses(
+    http: requests.Session, api_base: str, method, endpoint_template, payload, expected_status, error_code
+):
+    # Replace {uuid} placeholder with a random UUID for delete test
+    endpoint = endpoint_template.replace("{uuid}", str(uuid.uuid4()))
+    url = f"{api_base}{endpoint}"
+
+    if method == "get":
+        r = http.get(url, timeout=120)
+    elif method == "post":
+        r = http.post(url, json=payload, timeout=120)
+    elif method == "delete":
+        r = http.delete(url, timeout=120)
+
+    assert r.status_code == expected_status
+    if error_code:
+        body = r.json()
+        assert body["error"]["code"] == error_code
+
+
+def test_create_from_hash_invalid_json(http: requests.Session, api_base: str):
+    """Invalid JSON body requires special handling (data= instead of json=)."""
+    r = http.post(f"{api_base}/api/assets/from-hash", data=b"{not json}", timeout=120)
     body = r.json()
-    assert r.status_code == 404
-    assert body["error"]["code"] == "ASSET_NOT_FOUND"
-
-
-def test_create_from_hash_invalids(http: requests.Session, api_base: str):
-    # Bad hash algorithm
-    bad = {
-        "hash": "sha256:" + "0" * 64,
-        "name": "x.bin",
-        "tags": ["models", "checkpoints", "unit-tests"],
-    }
-    r1 = http.post(f"{api_base}/api/assets/from-hash", json=bad, timeout=120)
-    b1 = r1.json()
-    assert r1.status_code == 400
-    assert b1["error"]["code"] == "INVALID_BODY"
-
-    # Invalid JSON body
-    r2 = http.post(f"{api_base}/api/assets/from-hash", data=b"{not json}", timeout=120)
-    b2 = r2.json()
-    assert r2.status_code == 400
-    assert b2["error"]["code"] == "INVALID_JSON"
-
-
-def test_get_update_download_bad_ids(http: requests.Session, api_base: str):
-    # All endpoints should be not found, as we UUID regex directly in the route definition.
-    bad_id = "not-a-uuid"
-
-    r1 = http.get(f"{api_base}/api/assets/{bad_id}", timeout=120)
-    assert r1.status_code == 404
-
-    r3 = http.get(f"{api_base}/api/assets/{bad_id}/content", timeout=120)
-    assert r3.status_code == 404
+    assert r.status_code == 400
+    assert body["error"]["code"] == "INVALID_JSON"
 
 
 def test_update_requires_at_least_one_field(http: requests.Session, api_base: str, seeded_asset: dict):