47 lines
1.8 KiB
Python
47 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .community_sources import parse_rf4db_catches, parse_rf4stat_fishing, parse_rf4stat_posts
|
|
|
|
|
|
SOURCES = {
|
|
"rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
|
|
"rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
|
|
"rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
|
|
}
|
|
USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
|
|
|
|
|
|
def fetch_html(url: str, *, timeout: float = 30) -> str:
|
|
request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
|
|
with urlopen(request, timeout=timeout) as response:
|
|
if response.headers.get_content_type() != "text/html":
|
|
raise ValueError(f"expected text/html, got {response.headers.get_content_type()}")
|
|
return response.read().decode(response.headers.get_content_charset() or "utf-8")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
|
|
parser.add_argument("source", choices=SOURCES)
|
|
parser.add_argument("--url", help="Override the configured public page URL")
|
|
parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
|
|
args = parser.parse_args(argv)
|
|
default_url, parse = SOURCES[args.source]
|
|
url = args.url or default_url
|
|
try:
|
|
records = parse(fetch_html(url))[:args.limit]
|
|
except Exception as exc:
|
|
print(f"community source failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|