feat: validate and unblock media downloads

rf4_research/media_cli.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import argparse
+from datetime import datetime, timezone
 import json
 from pathlib import Path
+import urllib.error
 import urllib.request
 
 from .community_cli import USER_AGENT, _StrictRedirectHandler, _validate_url_before_io, check_and_reserve, fetch_html, fetch_site_key
@@ -22,17 +24,25 @@ def _download_one(root: Path, state_file: Path) -> str:
     url = queued["asset_url"]
     _validate_url_before_io(url)
     check_and_reserve(fetch_site_key(url), state_file=state_file)
-    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "image/*"})
-    opener = urllib.request.build_opener(_StrictRedirectHandler())
-    with opener.open(request, timeout=30) as response:
-        content_type = response.headers.get_content_type()
-        if not content_type.startswith("image/"):
-            raise ValueError(f"expected image, got {content_type}")
-        body = response.read(MAX_ASSET_BYTES + 1)
-        if len(body) > MAX_ASSET_BYTES:
-            raise ValueError("asset exceeded 15MB limit")
-    digest, relative = store_asset(root, body, content_type=content_type, source_url=url)
-    queued.update({"status": "stored", "sha256": digest, "local_path": relative, "content_type": content_type, "bytes": len(body)})
+    attempted_at = datetime.now(timezone.utc).isoformat()
+    try:
+        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "image/*"})
+        opener = urllib.request.build_opener(_StrictRedirectHandler())
+        with opener.open(request, timeout=30) as response:
+            content_type = response.headers.get_content_type()
+            if not content_type.startswith("image/"):
+                raise ValueError(f"expected image, got {content_type}")
+            body = response.read(MAX_ASSET_BYTES + 1)
+            if len(body) > MAX_ASSET_BYTES:
+                raise ValueError("asset exceeded 15MB limit")
+        digest, relative, width, height, detected_mime = store_asset(root, body, content_type=content_type, source_url=url)
+    except Exception as exc:
+        code = exc.code if isinstance(exc, urllib.error.HTTPError) else None
+        status = "missing" if code in {404, 410} else "blocked" if code in {401, 403} else "invalid" if isinstance(exc, ValueError) else "queued"
+        queued.update({"status": status, "last_attempt_at": attempted_at, "last_error": str(exc)[:500]})
+        manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+        raise
+    queued.update({"status": "stored", "sha256": digest, "local_path": relative, "content_type": detected_mime, "bytes": len(body), "width": width, "height": height, "fetched_at": attempted_at})
     manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
     return f"stored {url} as {relative}"