feat: parse RF4DB waterbody catalog
CI / backend-and-migrations (push) Canceled after 0s
CI / astro-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / compose-e2e (push) Canceled after 0s

This commit is contained in:
ik
2026-09-15 17:19:15 +07:00
parent 75820125aa
commit 4c75db1f74
4 changed files with 107 additions and 1 deletions
+7 -1
View File
@@ -15,6 +15,7 @@ from urllib.request import Request, urlopen, HTTPRedirectHandler, build_opener
from .community_sources import (
parse_rf4db_catches,
parse_rf4db_waterbodies,
parse_rf4map_point,
parse_rf4posts_spot,
parse_rf4stat_fishing,
@@ -24,6 +25,7 @@ from .community_sources import (
SOURCES = {
"rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
"rf4db-waterbodies": ("https://rf4db.com/ru/maps", parse_rf4db_waterbodies),
"rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
"rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
}
@@ -316,7 +318,11 @@ def main(argv: list[str] | None = None) -> int:
# Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
check_and_reserve(site_key, state_file=args.state_file)
html = fetch_html(url)
records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit]
records = (
parse(html, source_url=url)
if args.source in DETAIL_SOURCES or args.source == "rf4db-waterbodies"
else parse(html)
)[:args.limit]
except Exception as exc:
print(f"community source failed: {exc}", file=sys.stderr)
return 1