feat: reconcile RF4DB media catalog

rf4_research/community_cli.py

@@ -35,7 +35,7 @@ USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
 MIN_FETCH_INTERVAL_SECONDS = 30 * 60
 DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json")
 ALLOWED_HOSTS = frozenset({
-    "download.rf4db.com", "rf4db.com",
+    "download.rf4db.com", "rf4db.com", "oss.rf4db.com",
     "rf4-stat.ru",
     "rf4map.ru", "gw.rf4map.ru", "hb.ru-msk.vkcloud-storage.ru",
     "rf4-posts.com",
@@ -99,11 +99,13 @@ def fetch_site_key(url: str) -> str:
     hostname = (urlsplit(url).hostname or "").lower()
     if hostname.startswith("www."):
         hostname = hostname[4:]
-    if hostname.startswith("download."):
+    if hostname.endswith(".rf4db.com"):
+        hostname = "rf4db.com"
+    elif hostname.startswith("download."):
         hostname = hostname[9:]
-    if hostname.startswith("api."):
+    elif hostname.startswith("api."):
         hostname = hostname[4:]
-    if hostname.startswith("cdn."):
+    elif hostname.startswith("cdn."):
         hostname = hostname[4:]
     if hostname == "gw.rf4map.ru":
         hostname = "rf4map.ru"

rf4_research/media_assets.py

@@ -52,6 +52,16 @@ def _kind(url: str, label: str | None, context: str) -> str:
     return "reference"
 
 
+def _external_id(node: Tag, url: str) -> str | None:
+    link = node.find_parent("a", href=True)
+    if link:
+        return str(link.get("href")).rstrip("/").rsplit("/", 1)[-1]
+    parts = urlsplit(url)
+    if parts.hostname == "oss.rf4db.com" and parts.path.startswith("/game/"):
+        return Path(parts.path).stem
+    return None
+
+
 def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandidate]:
     soup = BeautifulSoup(html, "html.parser")
     result: dict[str, MediaCandidate] = {}
@@ -65,8 +75,7 @@ def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandid
             continue
         label = _label(node)
         context = " ".join(node.parent.get_text(" ", strip=True).split())[:500] if node.parent else ""
-        link = node.find_parent("a", href=True)
-        external_id = str(link.get("href")).rstrip("/").rsplit("/", 1)[-1] if link else None
+        external_id = _external_id(node, url)
         result.setdefault(url, MediaCandidate(source_page, url, _kind(url, label, f"{source_page} {context}"), label, external_id))
     for match in re.finditer(r"url\((['\"]?)(https://[^)'\"]+)\1\)", html, re.I):
         url = match.group(2)
@@ -96,6 +105,37 @@ def merge_manifest(path: Path, candidates: list[MediaCandidate]) -> dict:
     return output
 
 
+def normalize_entity_label(value: str) -> str:
+    """Normalize conservative Russian catalog spelling variants for matching."""
+    return " ".join(value.casefold().replace("ё", "е").split())
+
+
+def reconcile_queued_duplicates(path: Path) -> dict:
+    """Quarantine queued label duplicates while preserving their provenance."""
+    manifest = json.loads(path.read_text(encoding="utf-8"))
+    saved: dict[tuple[str, str], dict] = {}
+    for item in manifest.get("assets", []):
+        parts = urlsplit(str(item.get("asset_url") or ""))
+        if not item.get("external_id") and parts.hostname == "oss.rf4db.com" and parts.path.startswith("/game/"):
+            item["external_id"] = Path(parts.path).stem
+        label = item.get("label")
+        if item.get("status") not in {"stored", "approved"} or not isinstance(label, str):
+            continue
+        saved.setdefault((str(item.get("entity_type")), normalize_entity_label(label)), item)
+    duplicates = 0
+    for item in manifest.get("assets", []):
+        label = item.get("label")
+        if item.get("status") != "queued" or not isinstance(label, str):
+            continue
+        match = saved.get((str(item.get("entity_type")), normalize_entity_label(label)))
+        if match and match.get("asset_url") != item.get("asset_url"):
+            item.update({"status": "duplicate", "duplicate_of": match["asset_url"]})
+            duplicates += 1
+    manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
+    path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    return {"duplicates": duplicates, "queued": sum(item.get("status") == "queued" for item in manifest.get("assets", []))}
+
+
 def reclassify_manifest(path: Path) -> dict:
     manifest = json.loads(path.read_text(encoding="utf-8"))
     for item in manifest.get("assets", []):
@@ -172,7 +212,7 @@ def media_coverage(root: Path) -> dict:
     for entity_type, target in baseline["entities"].items():
         candidates = [item for item in manifest.get("assets", []) if item.get("entity_type") == entity_type]
         candidate_labels = {
-            " ".join(item["label"].split()).casefold()
+            normalize_entity_label(item["label"])
             for item in candidates
             if isinstance(item.get("label"), str) and item["label"].strip()
         }

rf4_research/media_cli.py

@@ -8,7 +8,7 @@ import urllib.error
 import urllib.request
 
 from .community_cli import MIN_FETCH_INTERVAL_SECONDS, USER_AGENT, _StrictRedirectHandler, _read_state, _validate_url_before_io, check_and_reserve, fetch_html, fetch_site_key
-from .media_assets import audit_media_catalog, extract_media_candidates, media_coverage, merge_manifest, reclassify_manifest, review_asset, store_asset
+from .media_assets import audit_media_catalog, extract_media_candidates, media_coverage, merge_manifest, reconcile_queued_duplicates, reclassify_manifest, review_asset, store_asset
 
 
 DEFAULT_ROOT = Path("data/media")
@@ -181,6 +181,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--batch-limit", type=int, default=MAX_BATCH_ASSETS_PER_DOMAIN, choices=range(1, MAX_BATCH_ASSETS_PER_DOMAIN + 1), metavar="1..40")
     parser.add_argument("--asset-url", help="Download this exact queued asset instead of the first queue item")
     parser.add_argument("--reclassify", action="store_true", help="Reapply current conservative classifier without network access")
+    parser.add_argument("--dedupe-queue", action="store_true", help="Mark queued label matches to stored assets as duplicates without network access")
     parser.add_argument("--review-url", help="Review an asset already present in the manifest")
     parser.add_argument("--decision", choices=("approved", "rejected"))
     parser.add_argument("--entity-type", choices=("fish", "waterbody", "tackle", "reference"))
@@ -213,6 +214,9 @@ def main(argv: list[str] | None = None) -> int:
             manifest = reclassify_manifest(args.root / "manifest.json")
             print(f"reclassified {len(manifest['assets'])} candidates")
             return 0
+        if args.dedupe_queue:
+            print(json.dumps(reconcile_queued_duplicates(args.root / "manifest.json"), ensure_ascii=False, indent=2))
+            return 0
         if args.download_one:
             print(_download_one(args.root, args.state_file, args.asset_url))
             return 0