feat: add rate-limited RF4 media collector
This commit is contained in:
@@ -39,6 +39,7 @@ ALLOWED_HOSTS = frozenset({
|
||||
"rf4-stat.ru",
|
||||
"rf4map.ru",
|
||||
"rf4-posts.com",
|
||||
"rf4game.de",
|
||||
})
|
||||
MAX_RESPONSE_BYTES = 5 * 1024 * 1024 # 5 MB
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MediaCandidate:
|
||||
source_page: str
|
||||
asset_url: str
|
||||
entity_type: str
|
||||
label: str | None
|
||||
external_id: str | None
|
||||
|
||||
|
||||
def _label(node: Tag) -> str | None:
|
||||
value = node.get("alt") or node.get("title")
|
||||
if isinstance(value, str) and value.strip():
|
||||
return " ".join(value.split())[:300]
|
||||
parent = node.find_parent(["article", "figure", "a", "section"])
|
||||
if parent:
|
||||
text = " ".join(parent.get_text(" ", strip=True).split())
|
||||
return text[:300] or None
|
||||
return None
|
||||
|
||||
|
||||
def _kind(url: str, label: str | None, context: str) -> str:
|
||||
path = urlsplit(url).path.casefold()
|
||||
value = f"{label or ''} {context}".casefold()
|
||||
if re.search(r"/(?:fish|fishes|fische|species)/", path) or any(word in value for word in ("рыба ", "fish: ", "fisch: ")):
|
||||
return "fish"
|
||||
if re.search(r"/(?:maps?|levels?|lakes?|waterbodies)/", path) or any(word in value for word in ("карта ", "map of ", "gewässerkarte")):
|
||||
return "waterbody"
|
||||
if re.search(r"(?:^|[/_-])(?:bait|lure|rig|rod|reel|hook|tackle|köder|rute|rolle)(?:[/_.-]|$)", path):
|
||||
return "tackle"
|
||||
return "reference"
|
||||
|
||||
|
||||
def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandidate]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
result: dict[str, MediaCandidate] = {}
|
||||
for node in soup.select("img[src], img[data-src], source[srcset]"):
|
||||
raw = node.get("src") or node.get("data-src") or node.get("srcset")
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
raw = raw.split(",", 1)[0].strip().split(" ", 1)[0]
|
||||
url = urljoin(source_page, raw)
|
||||
if urlsplit(url).scheme != "https":
|
||||
continue
|
||||
label = _label(node)
|
||||
context = " ".join(node.parent.get_text(" ", strip=True).split())[:500] if node.parent else ""
|
||||
link = node.find_parent("a", href=True)
|
||||
external_id = str(link.get("href")).rstrip("/").rsplit("/", 1)[-1] if link else None
|
||||
result.setdefault(url, MediaCandidate(source_page, url, _kind(url, label, context), label, external_id))
|
||||
for match in re.finditer(r"url\((['\"]?)(https://[^)'\"]+)\1\)", html, re.I):
|
||||
url = match.group(2)
|
||||
result.setdefault(url, MediaCandidate(source_page, url, _kind(url, None, ""), None, None))
|
||||
return list(result.values())
|
||||
|
||||
|
||||
def merge_manifest(path: Path, candidates: list[MediaCandidate]) -> dict:
|
||||
current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"version": 1, "assets": []}
|
||||
assets = {item["asset_url"]: item for item in current.get("assets", [])}
|
||||
seen_at = datetime.now(timezone.utc).isoformat()
|
||||
for candidate in candidates:
|
||||
item = assets.get(candidate.asset_url, {})
|
||||
item.update(asdict(candidate))
|
||||
item.setdefault("status", "queued")
|
||||
item.setdefault("first_seen_at", seen_at)
|
||||
item["last_seen_at"] = seen_at
|
||||
assets[candidate.asset_url] = item
|
||||
output = {"version": 1, "updated_at": seen_at, "assets": sorted(assets.values(), key=lambda item: item["asset_url"])}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return output
|
||||
|
||||
|
||||
def reclassify_manifest(path: Path) -> dict:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
for item in manifest.get("assets", []):
|
||||
item["entity_type"] = _kind(item["asset_url"], item.get("label"), "")
|
||||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def store_asset(root: Path, body: bytes, *, content_type: str, source_url: str) -> tuple[str, str]:
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
extension = mimetypes.guess_extension(content_type.split(";", 1)[0]) or Path(urlsplit(source_url).path).suffix
|
||||
extension = extension if extension in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"} else ".bin"
|
||||
target = root / "files" / digest[:2] / f"{digest}{extension}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not target.exists():
|
||||
target.write_bytes(body)
|
||||
return digest, str(target.relative_to(root))
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
|
||||
from .community_cli import USER_AGENT, _StrictRedirectHandler, _validate_url_before_io, check_and_reserve, fetch_html, fetch_site_key
|
||||
from .media_assets import extract_media_candidates, merge_manifest, reclassify_manifest, store_asset
|
||||
|
||||
|
||||
DEFAULT_ROOT = Path("data/media")
|
||||
MAX_ASSET_BYTES = 15 * 1024 * 1024
|
||||
|
||||
|
||||
def _download_one(root: Path, state_file: Path) -> str:
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
queued = next((item for item in manifest["assets"] if item.get("status") == "queued"), None)
|
||||
if queued is None:
|
||||
return "queue is empty"
|
||||
url = queued["asset_url"]
|
||||
_validate_url_before_io(url)
|
||||
check_and_reserve(fetch_site_key(url), state_file=state_file)
|
||||
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "image/*"})
|
||||
opener = urllib.request.build_opener(_StrictRedirectHandler())
|
||||
with opener.open(request, timeout=30) as response:
|
||||
content_type = response.headers.get_content_type()
|
||||
if not content_type.startswith("image/"):
|
||||
raise ValueError(f"expected image, got {content_type}")
|
||||
body = response.read(MAX_ASSET_BYTES + 1)
|
||||
if len(body) > MAX_ASSET_BYTES:
|
||||
raise ValueError("asset exceeded 15MB limit")
|
||||
digest, relative = store_asset(root, body, content_type=content_type, source_url=url)
|
||||
queued.update({"status": "stored", "sha256": digest, "local_path": relative, "content_type": content_type, "bytes": len(body)})
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return f"stored {url} as {relative}"
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Index authorized RF4 media without hotlinking")
|
||||
parser.add_argument("url", nargs="?")
|
||||
parser.add_argument("--download-one", action="store_true", help="Store one queued asset while respecting site cooldown")
|
||||
parser.add_argument("--reclassify", action="store_true", help="Reapply current conservative classifier without network access")
|
||||
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--state-file", type=Path, default=Path(".cache/community-fetch-state.json"))
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.reclassify:
|
||||
manifest = reclassify_manifest(args.root / "manifest.json")
|
||||
print(f"reclassified {len(manifest['assets'])} candidates")
|
||||
return 0
|
||||
if args.download_one:
|
||||
print(_download_one(args.root, args.state_file))
|
||||
return 0
|
||||
if not args.url:
|
||||
parser.error("url is required unless --download-one is used")
|
||||
check_and_reserve(fetch_site_key(args.url), state_file=args.state_file)
|
||||
html = fetch_html(args.url)
|
||||
candidates = extract_media_candidates(html, source_page=args.url)
|
||||
manifest = merge_manifest(args.root / "manifest.json", candidates)
|
||||
except Exception as exc:
|
||||
parser.exit(1, f"media index failed: {exc}\n")
|
||||
print(f"indexed {len(candidates)} candidates; manifest contains {len(manifest['assets'])}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user