112 lines
4.3 KiB
Python
112 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .community_sources import (
|
|
parse_rf4db_catches,
|
|
parse_rf4map_point,
|
|
parse_rf4posts_spot,
|
|
parse_rf4stat_fishing,
|
|
parse_rf4stat_posts,
|
|
)
|
|
|
|
|
|
SOURCES = {
|
|
"rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
|
|
"rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
|
|
"rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
|
|
}
|
|
DETAIL_SOURCES = {
|
|
"rf4map-point": parse_rf4map_point,
|
|
"rf4posts-spot": parse_rf4posts_spot,
|
|
}
|
|
USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
|
|
MIN_FETCH_INTERVAL_SECONDS = 30 * 60
|
|
DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json")
|
|
|
|
|
|
def fetch_site_key(url: str) -> str:
|
|
"""Return a stable cooldown key shared by all endpoints of one site."""
|
|
hostname = (urlsplit(url).hostname or "").lower()
|
|
if hostname.startswith("www."):
|
|
hostname = hostname[4:]
|
|
if not hostname:
|
|
raise ValueError("source URL must include a hostname")
|
|
return hostname
|
|
|
|
|
|
def enforce_fetch_interval(
|
|
source: str, *, state_file: Path, now: float | None = None,
|
|
) -> None:
|
|
now = time.time() if now is None else now
|
|
try:
|
|
state = json.loads(state_file.read_text(encoding="utf-8"))
|
|
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
|
state = {}
|
|
last_fetch = state.get(source)
|
|
if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS:
|
|
wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch))
|
|
raise RuntimeError(f"source cooldown is active; retry in {wait} seconds")
|
|
|
|
|
|
def mark_fetch(source: str, *, state_file: Path, now: float | None = None) -> None:
|
|
now = time.time() if now is None else now
|
|
try:
|
|
state = json.loads(state_file.read_text(encoding="utf-8"))
|
|
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
|
state = {}
|
|
state[source] = now
|
|
state_file.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = state_file.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(state, sort_keys=True), encoding="utf-8")
|
|
temporary.replace(state_file)
|
|
|
|
|
|
def fetch_html(url: str, *, timeout: float = 30) -> str:
|
|
request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
|
|
with urlopen(request, timeout=timeout) as response:
|
|
if response.headers.get_content_type() != "text/html":
|
|
raise ValueError(f"expected text/html, got {response.headers.get_content_type()}")
|
|
return response.read().decode(response.headers.get_content_charset() or "utf-8")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
|
|
parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES))
|
|
parser.add_argument("--url", help="Override the configured public page URL")
|
|
parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
|
|
parser.add_argument(
|
|
"--state-file", type=Path,
|
|
default=Path(os.environ.get("RF4_COMMUNITY_FETCH_STATE", DEFAULT_STATE_FILE)),
|
|
help="Persistent per-source cooldown state",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
if args.source in DETAIL_SOURCES and not args.url:
|
|
parser.error(f"--url is required for {args.source}")
|
|
default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source)))
|
|
url = args.url or default_url
|
|
try:
|
|
site_key = fetch_site_key(url)
|
|
enforce_fetch_interval(site_key, state_file=args.state_file)
|
|
# Reserve before network I/O: failed attempts count toward the limit too.
|
|
mark_fetch(site_key, state_file=args.state_file)
|
|
html = fetch_html(url)
|
|
records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit]
|
|
except Exception as exc:
|
|
print(f"community source failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|