60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .community_sources import (
|
|
parse_rf4db_catches,
|
|
parse_rf4map_point,
|
|
parse_rf4posts_spot,
|
|
parse_rf4stat_fishing,
|
|
parse_rf4stat_posts,
|
|
)
|
|
|
|
|
|
SOURCES = {
|
|
"rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
|
|
"rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
|
|
"rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
|
|
}
|
|
DETAIL_SOURCES = {
|
|
"rf4map-point": parse_rf4map_point,
|
|
"rf4posts-spot": parse_rf4posts_spot,
|
|
}
|
|
USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
|
|
|
|
|
|
def fetch_html(url: str, *, timeout: float = 30) -> str:
|
|
request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
|
|
with urlopen(request, timeout=timeout) as response:
|
|
if response.headers.get_content_type() != "text/html":
|
|
raise ValueError(f"expected text/html, got {response.headers.get_content_type()}")
|
|
return response.read().decode(response.headers.get_content_charset() or "utf-8")
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
|
|
parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES))
|
|
parser.add_argument("--url", help="Override the configured public page URL")
|
|
parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
|
|
args = parser.parse_args(argv)
|
|
if args.source in DETAIL_SOURCES and not args.url:
|
|
parser.error(f"--url is required for {args.source}")
|
|
default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source)))
|
|
url = args.url or default_url
|
|
try:
|
|
html = fetch_html(url)
|
|
records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit]
|
|
except Exception as exc:
|
|
print(f"community source failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|