ik/rf4-spotter
1
0
Fork 0
Code Issues Pull Requests Actions 9 Packages Projects Releases Wiki Activity

feat: add offline waterbody source crosswalk

Browse Source
This commit is contained in:
ik
2026-09-20 19:16:46 +07:00
parent d541443a1a
commit f02cb247a3
11 changed files with 311 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rf4_research/community_cli.py
+20 -8
View File
@@ -18,9 +18,11 @@ from .community_sources import (
parse_rf4db_waterbodies,
parse_rf4db_waterbody_detail,
parse_rf4map_point,
parse_rf4map_waterbodies,
parse_rf4posts_spot,
parse_rf4stat_fishing,
parse_rf4stat_posts,
parse_rf4stat_waterbodies,
)
@@ -29,6 +31,8 @@ SOURCES = {
"rf4db-waterbodies": ("https://rf4db.com/ru/maps", parse_rf4db_waterbodies),
"rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
"rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
"rf4map-waterbodies": ("https://rf4map.ru/lakes", parse_rf4map_waterbodies),
"rf4stat-locations": ("https://en.rf4-stat.ru/locations/", parse_rf4stat_waterbodies),
}
DETAIL_SOURCES = {
"rf4db-waterbody": parse_rf4db_waterbody_detail,
@@ -40,7 +44,7 @@ MIN_FETCH_INTERVAL_SECONDS = 30 * 60
DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json")
ALLOWED_HOSTS = frozenset({
"download.rf4db.com", "rf4db.com", "oss.rf4db.com",
"rf4-stat.ru",
"rf4-stat.ru", "en.rf4-stat.ru",
"rf4map.ru", "gw.rf4map.ru", "hb.ru-msk.vkcloud-storage.ru",
"rf4-posts.com",
"rf4game.de", "rf4game.ru",
@@ -111,6 +115,8 @@ def fetch_site_key(url: str) -> str:
hostname = hostname[4:]
elif hostname.startswith("cdn."):
hostname = hostname[4:]
elif hostname.endswith(".rf4-stat.ru"):
hostname = "rf4-stat.ru"
if hostname == "gw.rf4map.ru":
hostname = "rf4map.ru"
if not hostname:
@@ -304,6 +310,7 @@ def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES))
parser.add_argument("--url", help="Override the configured public page URL")
parser.add_argument("--html", type=Path, help="Parse a previously saved HTML file without network or cooldown")
parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
parser.add_argument(
"--state-file", type=Path,
@@ -315,18 +322,23 @@ def main(argv: list[str] | None = None) -> int:
help="Reserve the shared site cooldown and stop before making an HTTP request",
)
args = parser.parse_args(argv)
if args.html and args.reserve_only:
parser.error("--html cannot be combined with --reserve-only")
if args.source in DETAIL_SOURCES and not args.url:
parser.error(f"--url is required for {args.source}")
default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source)))
url = args.url or default_url
try:
site_key = fetch_site_key(url)
# Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
check_and_reserve(site_key, state_file=args.state_file)
if args.reserve_only:
print(json.dumps({"reserved": True, "source": args.source, "site_key": site_key}, ensure_ascii=False))
return 0
html = fetch_html(url)
if args.html:
html = args.html.read_text(encoding="utf-8")
else:
site_key = fetch_site_key(url)
# Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
check_and_reserve(site_key, state_file=args.state_file)
if args.reserve_only:
print(json.dumps({"reserved": True, "source": args.source, "site_key": site_key}, ensure_ascii=False))
return 0
html = fetch_html(url)
parsed = (
parse(html, source_url=url)
if args.source in DETAIL_SOURCES or args.source == "rf4db-waterbodies"