feat: reconcile RF4DB media catalog

rf4_research/media_assets.py

@@ -52,6 +52,16 @@ def _kind(url: str, label: str | None, context: str) -> str:
     return "reference"
 
 
+def _external_id(node: Tag, url: str) -> str | None:
+    link = node.find_parent("a", href=True)
+    if link:
+        return str(link.get("href")).rstrip("/").rsplit("/", 1)[-1]
+    parts = urlsplit(url)
+    if parts.hostname == "oss.rf4db.com" and parts.path.startswith("/game/"):
+        return Path(parts.path).stem
+    return None
+
+
 def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandidate]:
     soup = BeautifulSoup(html, "html.parser")
     result: dict[str, MediaCandidate] = {}
@@ -65,8 +75,7 @@ def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandid
             continue
         label = _label(node)
         context = " ".join(node.parent.get_text(" ", strip=True).split())[:500] if node.parent else ""
-        link = node.find_parent("a", href=True)
-        external_id = str(link.get("href")).rstrip("/").rsplit("/", 1)[-1] if link else None
+        external_id = _external_id(node, url)
         result.setdefault(url, MediaCandidate(source_page, url, _kind(url, label, f"{source_page} {context}"), label, external_id))
     for match in re.finditer(r"url\((['\"]?)(https://[^)'\"]+)\1\)", html, re.I):
         url = match.group(2)
@@ -96,6 +105,37 @@ def merge_manifest(path: Path, candidates: list[MediaCandidate]) -> dict:
     return output
 
 
+def normalize_entity_label(value: str) -> str:
+    """Normalize conservative Russian catalog spelling variants for matching."""
+    return " ".join(value.casefold().replace("ё", "е").split())
+
+
+def reconcile_queued_duplicates(path: Path) -> dict:
+    """Quarantine queued label duplicates while preserving their provenance."""
+    manifest = json.loads(path.read_text(encoding="utf-8"))
+    saved: dict[tuple[str, str], dict] = {}
+    for item in manifest.get("assets", []):
+        parts = urlsplit(str(item.get("asset_url") or ""))
+        if not item.get("external_id") and parts.hostname == "oss.rf4db.com" and parts.path.startswith("/game/"):
+            item["external_id"] = Path(parts.path).stem
+        label = item.get("label")
+        if item.get("status") not in {"stored", "approved"} or not isinstance(label, str):
+            continue
+        saved.setdefault((str(item.get("entity_type")), normalize_entity_label(label)), item)
+    duplicates = 0
+    for item in manifest.get("assets", []):
+        label = item.get("label")
+        if item.get("status") != "queued" or not isinstance(label, str):
+            continue
+        match = saved.get((str(item.get("entity_type")), normalize_entity_label(label)))
+        if match and match.get("asset_url") != item.get("asset_url"):
+            item.update({"status": "duplicate", "duplicate_of": match["asset_url"]})
+            duplicates += 1
+    manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
+    path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    return {"duplicates": duplicates, "queued": sum(item.get("status") == "queued" for item in manifest.get("assets", []))}
+
+
 def reclassify_manifest(path: Path) -> dict:
     manifest = json.loads(path.read_text(encoding="utf-8"))
     for item in manifest.get("assets", []):
@@ -172,7 +212,7 @@ def media_coverage(root: Path) -> dict:
     for entity_type, target in baseline["entities"].items():
         candidates = [item for item in manifest.get("assets", []) if item.get("entity_type") == entity_type]
         candidate_labels = {
-            " ".join(item["label"].split()).casefold()
+            normalize_entity_label(item["label"])
             for item in candidates
             if isinstance(item.get("label"), str) and item["label"].strip()
         }