80 lines
4.0 KiB
Python
80 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from .community_cli import USER_AGENT, _StrictRedirectHandler, _validate_url_before_io, check_and_reserve, fetch_html, fetch_site_key
|
|
from .media_assets import extract_media_candidates, merge_manifest, reclassify_manifest, store_asset
|
|
|
|
|
|
DEFAULT_ROOT = Path("data/media")
|
|
MAX_ASSET_BYTES = 15 * 1024 * 1024
|
|
|
|
|
|
def _download_one(root: Path, state_file: Path) -> str:
|
|
manifest_path = root / "manifest.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
queued = next((item for item in manifest["assets"] if item.get("status") == "queued"), None)
|
|
if queued is None:
|
|
return "queue is empty"
|
|
url = queued["asset_url"]
|
|
_validate_url_before_io(url)
|
|
check_and_reserve(fetch_site_key(url), state_file=state_file)
|
|
attempted_at = datetime.now(timezone.utc).isoformat()
|
|
try:
|
|
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "image/*"})
|
|
opener = urllib.request.build_opener(_StrictRedirectHandler())
|
|
with opener.open(request, timeout=30) as response:
|
|
content_type = response.headers.get_content_type()
|
|
if not content_type.startswith("image/"):
|
|
raise ValueError(f"expected image, got {content_type}")
|
|
body = response.read(MAX_ASSET_BYTES + 1)
|
|
if len(body) > MAX_ASSET_BYTES:
|
|
raise ValueError("asset exceeded 15MB limit")
|
|
digest, relative, width, height, detected_mime = store_asset(root, body, content_type=content_type, source_url=url)
|
|
except Exception as exc:
|
|
code = exc.code if isinstance(exc, urllib.error.HTTPError) else None
|
|
status = "missing" if code in {404, 410} else "blocked" if code in {401, 403} else "invalid" if isinstance(exc, ValueError) else "queued"
|
|
queued.update({"status": status, "last_attempt_at": attempted_at, "last_error": str(exc)[:500]})
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
raise
|
|
queued.update({"status": "stored", "sha256": digest, "local_path": relative, "content_type": detected_mime, "bytes": len(body), "width": width, "height": height, "fetched_at": attempted_at})
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
return f"stored {url} as {relative}"
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Index authorized RF4 media without hotlinking")
|
|
parser.add_argument("url", nargs="?")
|
|
parser.add_argument("--download-one", action="store_true", help="Store one queued asset while respecting site cooldown")
|
|
parser.add_argument("--reclassify", action="store_true", help="Reapply current conservative classifier without network access")
|
|
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
|
parser.add_argument("--state-file", type=Path, default=Path(".cache/community-fetch-state.json"))
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
if args.reclassify:
|
|
manifest = reclassify_manifest(args.root / "manifest.json")
|
|
print(f"reclassified {len(manifest['assets'])} candidates")
|
|
return 0
|
|
if args.download_one:
|
|
print(_download_one(args.root, args.state_file))
|
|
return 0
|
|
if not args.url:
|
|
parser.error("url is required unless --download-one is used")
|
|
check_and_reserve(fetch_site_key(args.url), state_file=args.state_file)
|
|
html = fetch_html(args.url)
|
|
candidates = extract_media_candidates(html, source_page=args.url)
|
|
manifest = merge_manifest(args.root / "manifest.json", candidates)
|
|
except Exception as exc:
|
|
parser.exit(1, f"media index failed: {exc}\n")
|
|
print(f"indexed {len(candidates)} candidates; manifest contains {len(manifest['assets'])}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|