feat: add rate-limited RF4 media collector

rf4_research/media_cli.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import urllib.request
+
+from .community_cli import USER_AGENT, _StrictRedirectHandler, _validate_url_before_io, check_and_reserve, fetch_html, fetch_site_key
+from .media_assets import extract_media_candidates, merge_manifest, reclassify_manifest, store_asset
+
+
+DEFAULT_ROOT = Path("data/media")
+MAX_ASSET_BYTES = 15 * 1024 * 1024
+
+
+def _download_one(root: Path, state_file: Path) -> str:
+    manifest_path = root / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    queued = next((item for item in manifest["assets"] if item.get("status") == "queued"), None)
+    if queued is None:
+        return "queue is empty"
+    url = queued["asset_url"]
+    _validate_url_before_io(url)
+    check_and_reserve(fetch_site_key(url), state_file=state_file)
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "image/*"})
+    opener = urllib.request.build_opener(_StrictRedirectHandler())
+    with opener.open(request, timeout=30) as response:
+        content_type = response.headers.get_content_type()
+        if not content_type.startswith("image/"):
+            raise ValueError(f"expected image, got {content_type}")
+        body = response.read(MAX_ASSET_BYTES + 1)
+        if len(body) > MAX_ASSET_BYTES:
+            raise ValueError("asset exceeded 15MB limit")
+    digest, relative = store_asset(root, body, content_type=content_type, source_url=url)
+    queued.update({"status": "stored", "sha256": digest, "local_path": relative, "content_type": content_type, "bytes": len(body)})
+    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    return f"stored {url} as {relative}"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Index authorized RF4 media without hotlinking")
+    parser.add_argument("url", nargs="?")
+    parser.add_argument("--download-one", action="store_true", help="Store one queued asset while respecting site cooldown")
+    parser.add_argument("--reclassify", action="store_true", help="Reapply current conservative classifier without network access")
+    parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
+    parser.add_argument("--state-file", type=Path, default=Path(".cache/community-fetch-state.json"))
+    args = parser.parse_args(argv)
+    try:
+        if args.reclassify:
+            manifest = reclassify_manifest(args.root / "manifest.json")
+            print(f"reclassified {len(manifest['assets'])} candidates")
+            return 0
+        if args.download_one:
+            print(_download_one(args.root, args.state_file))
+            return 0
+        if not args.url:
+            parser.error("url is required unless --download-one is used")
+        check_and_reserve(fetch_site_key(args.url), state_file=args.state_file)
+        html = fetch_html(args.url)
+        candidates = extract_media_candidates(html, source_page=args.url)
+        manifest = merge_manifest(args.root / "manifest.json", candidates)
+    except Exception as exc:
+        parser.exit(1, f"media index failed: {exc}\n")
+    print(f"indexed {len(candidates)} candidates; manifest contains {len(manifest['assets'])}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())