From fda2e5a7bc5bd749ccb7ca1a3482bc4375df8f1e Mon Sep 17 00:00:00 2001 From: IK Date: Thu, 3 Sep 2026 18:53:03 +0700 Subject: [PATCH] refactor: share official records parser --- .dockerignore | 7 ++ apps/api/Dockerfile | 5 +- apps/api/app/importer.py | 58 +++-------------- compose.yaml | 12 +++- rf4_research/official_parser.py | 105 ++++++++++++++++++++++++++++++ rf4_research/records.py | 110 +++----------------------------- tests/test_parser_contract.py | 4 +- 7 files changed, 144 insertions(+), 157 deletions(-) create mode 100644 .dockerignore create mode 100644 rf4_research/official_parser.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..22f2032 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.venv +**/__pycache__ +**/.pytest_cache +apps/web/node_modules +apps/web/dist +design-reference diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 46c5a17..e888dc4 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,8 +1,9 @@ FROM python:3.12-slim WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 -COPY requirements.txt . +COPY apps/api/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY . . +COPY apps/api . +COPY rf4_research ./rf4_research EXPOSE 8000 CMD ["sh", "-c", "alembic upgrade head && python -m app.seed && uvicorn app.main:app --host 0.0.0.0 --port 8000"] diff --git a/apps/api/app/importer.py b/apps/api/app/importer.py index 7d76f43..f2646da 100644 --- a/apps/api/app/importer.py +++ b/apps/api/app/importer.py @@ -7,9 +7,9 @@ from dataclasses import asdict, dataclass from datetime import date, datetime, time, timezone import httpx -from bs4 import BeautifulSoup, Tag from sqlalchemy import select from sqlalchemy.orm import Session +from rf4_research.official_parser import RecordsContractError, parse_official_records from .models import ( Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus, @@ -60,54 +60,16 @@ def external_id(record: RawRecord) -> str: return hashlib.sha256("|".join(normalize(part) for part in parts).encode()).hexdigest() -def parse_weight(raw: str) -> int: - value = " ".join(raw.replace("\xa0", " ").split()).lower() - match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", value) - if not match: - raise ImportSourceError(f"unsupported weight {raw!r}") - number, unit = match.groups() - number = number.replace(" ", "").replace("'", "").replace("’", "") - if unit == "g": - return int(number.replace(".", "").replace(",", "")) - if "," in number and "." not in number: - number = number.replace(",", ".") - return round(float(number) * 1000) - - -def _text(node: Tag | None) -> str: - return node.get_text(" ", strip=True) if node else "" - - def parse_html(html: str, *, region: str, category: str) -> list[RawRecord]: - soup = BeautifulSoup(html, "html.parser") - table = soup.select_one("div.records.flex_table") - if table is None: - raise ImportSourceError("records table not found") - header = table.select_one(":scope > .row.header") - expected = ["fish", "weight", "location", "bait", "gamername", "data"] - cells = header.find_all("div", recursive=False) if header else [] - actual = [next((key for key in expected if key in cell.get("class", [])), "") for cell in cells] - if actual != expected: - raise ImportSourceError(f"record columns changed: {actual}") - - records: list[RawRecord] = [] - for group in table.select(":scope > .rows > .row > .records_subtable"): - group_header = group.select_one(":scope > .row.header") - if group_header is None: - continue - fish = _text(group_header.select_one(".fish .text")) - rows = [group_header, *group.select(":scope > .rows > .row")] - for row in rows: - bait_node = row.select_one(".bait_icon") - bait = bait_node.get("title", "").strip() if bait_node else "" - try: - record_date = datetime.strptime(_text(row.select_one(".data")), "%d.%m.%y").date() - except ValueError as exc: - raise ImportSourceError("record date format changed") from exc - records.append(RawRecord(region.upper(), category, _text(row.select_one(".gamername")) or None, fish, parse_weight(_text(row.select_one(".weight"))), _text(row.select_one(".location")), bait or None, record_date)) - if not records: - raise ImportSourceError("records table is empty") - return records + try: + rows = parse_official_records(html) + except RecordsContractError as exc: + raise ImportSourceError(str(exc)) from exc + return [RawRecord( + region=region.upper(), category=category, player=row.player, fish=row.fish, + weight_g=row.weight_g, waterbody=row.waterbody, bait=row.bait, + record_date=row.record_date, + ) for row in rows] def fetch_records( diff --git a/compose.yaml b/compose.yaml index 02826be..cd82758 100644 --- a/compose.yaml +++ b/compose.yaml @@ -26,7 +26,9 @@ services: - minio_data:/data api: - build: ./apps/api + build: + context: . + dockerfile: apps/api/Dockerfile environment: DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter ADMIN_TOKEN: ${ADMIN_TOKEN:-change-me-in-production} @@ -64,7 +66,9 @@ services: - "4321:4321" importer: - build: ./apps/api + build: + context: . + dockerfile: apps/api/Dockerfile profiles: ["tools"] environment: DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter @@ -74,7 +78,9 @@ services: command: ["sh", "-c", "alembic upgrade head && python -m app.cli import-records"] scheduler: - build: ./apps/api + build: + context: . + dockerfile: apps/api/Dockerfile profiles: ["scheduler"] restart: unless-stopped environment: diff --git a/rf4_research/official_parser.py b/rf4_research/official_parser.py new file mode 100644 index 0000000..f98eac6 --- /dev/null +++ b/rf4_research/official_parser.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import date, datetime +from typing import Iterable + +from bs4 import BeautifulSoup, Tag + + +class RecordsContractError(ValueError): + """The official records page no longer matches the verified DOM contract.""" + + +@dataclass(frozen=True, slots=True) +class ParsedOfficialRecord: + player: str | None + fish: str + weight_g: int + waterbody: str + bait: str | None + record_date: date + + +def _text(node: Tag | None) -> str: + return node.get_text(" ", strip=True) if node else "" + + +def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None: + wanted = set(classes) + for child in parent.find_all("div", recursive=False): + if wanted.issubset(set(child.get("class", []))): + return child + return None + + +def parse_weight_g(raw: str) -> int: + normalized = " ".join(raw.replace("\xa0", " ").split()).lower() + match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized) + if not match: + raise RecordsContractError(f"unsupported weight: {raw!r}") + number, unit = match.groups() + number = number.replace(" ", "").replace("'", "").replace("’", "") + if unit == "g": + return int(number.replace(".", "").replace(",", "")) + if "," in number and "." not in number: + number = number.replace(",", ".") + return round(float(number) * 1000) + + +def parse_record_date(raw: str, *, today: date | None = None) -> date: + today = today or date.today() + try: + parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date() + except ValueError as exc: + raise RecordsContractError(f"unsupported record date: {raw!r}") from exc + if parsed > today.replace(year=today.year + 1): + raise RecordsContractError(f"record date is implausibly far in the future: {raw!r}") + return parsed + + +def parse_official_records(html: str, *, today: date | None = None) -> list[ParsedOfficialRecord]: + soup = BeautifulSoup(html, "html.parser") + table = soup.select_one("div.records.flex_table") + if table is None: + raise RecordsContractError("records table not found") + top_rows = _direct_child(table, ["rows"]) + header = _direct_child(table, ["row", "header"]) + expected = ["fish", "weight", "location", "bait", "gamername", "data"] + actual = [ + next((name for name in expected if name in cell.get("class", [])), "") + for cell in (header.find_all("div", recursive=False) if header else []) + ] + if actual != expected: + raise RecordsContractError(f"records columns changed: expected {expected}, got {actual}") + if top_rows is None: + raise RecordsContractError("records rows container not found") + + result: list[ParsedOfficialRecord] = [] + for wrapper in top_rows.find_all("div", class_="row", recursive=False): + group = _direct_child(wrapper, ["records_subtable", "flex_table"]) + if group is None: + continue + group_header = _direct_child(group, ["row", "header"]) + more_rows = _direct_child(group, ["rows"]) + if group_header is None: + continue + fish = _text(group_header.select_one(".fish .text")) + rows = [group_header] + if more_rows is not None: + rows.extend(more_rows.find_all("div", class_="row", recursive=False)) + for row in rows: + bait_node = row.select_one(".bait .bait_icon") + bait = bait_node.get("title", "").strip() if bait_node else "" + result.append(ParsedOfficialRecord( + player=_text(row.select_one(".gamername")) or None, + fish=fish, + weight_g=parse_weight_g(_text(row.select_one(".weight"))), + waterbody=_text(row.select_one(".location")), + bait=bait or None, + record_date=parse_record_date(_text(row.select_one(".data")), today=today), + )) + if not result: + raise RecordsContractError("records table is present but contains no records") + return result diff --git a/rf4_research/records.py b/rf4_research/records.py index 83c8c10..0007724 100644 --- a/rf4_research/records.py +++ b/rf4_research/records.py @@ -2,22 +2,19 @@ from __future__ import annotations import argparse import json -import re import sys from dataclasses import asdict, dataclass -from datetime import date, datetime -from typing import Iterable +from datetime import date from urllib.request import Request, urlopen -from bs4 import BeautifulSoup, Tag +from .official_parser import RecordsContractError, parse_official_records, parse_record_date, parse_weight_g DEFAULT_URL = "https://rf4game.de/records/region/RU/" USER_AGENT = "RF4-Spotter-Research/0.1 (+https://github.com/)" -class RecordsParseError(ValueError): - """Raised when the page no longer matches the verified records contract.""" +RecordsParseError = RecordsContractError @dataclass(frozen=True, slots=True) @@ -33,47 +30,6 @@ class OfficialRecord: source_url: str -def _text(node: Tag | None) -> str: - return node.get_text(" ", strip=True) if node else "" - - -def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None: - wanted = set(classes) - for child in parent.find_all("div", recursive=False): - if wanted.issubset(set(child.get("class", []))): - return child - return None - - -def parse_weight_g(raw: str) -> int: - normalized = " ".join(raw.replace("\xa0", " ").split()).lower() - match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized) - if not match: - raise RecordsParseError(f"unsupported weight: {raw!r}") - - number, unit = match.groups() - number = number.replace(" ", "").replace("'", "").replace("’", "") - if unit == "g": - return int(number.replace(".", "").replace(",", "")) - - # The verified pages use a dot as the kg decimal separator and spaces as - # thousands separators (for example, "2 519.264 kg"). - if "," in number and "." not in number: - number = number.replace(",", ".") - return round(float(number) * 1000) - - -def parse_record_date(raw: str, *, today: date | None = None) -> date: - today = today or date.today() - try: - parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date() - except ValueError as exc: - raise RecordsParseError(f"unsupported record date: {raw!r}") from exc - if parsed > today.replace(year=today.year + 1): - raise RecordsParseError(f"record date is implausibly far in the future: {raw!r}") - return parsed - - def parse_records_html( html: str, *, @@ -82,61 +38,11 @@ def parse_records_html( source_url: str, today: date | None = None, ) -> list[OfficialRecord]: - soup = BeautifulSoup(html, "html.parser") - table = soup.select_one("div.records.flex_table") - if table is None: - raise RecordsParseError("records table not found") - - top_rows = _direct_child(table, ["rows"]) - header = _direct_child(table, ["row", "header"]) - expected_classes = ["fish", "weight", "location", "bait", "gamername", "data"] - actual_classes = [ - next((name for name in expected_classes if name in cell.get("class", [])), "") - for cell in (header.find_all("div", recursive=False) if header else []) - ] - if actual_classes != expected_classes: - raise RecordsParseError( - f"records columns changed: expected {expected_classes}, got {actual_classes}" - ) - if top_rows is None: - raise RecordsParseError("records rows container not found") - - result: list[OfficialRecord] = [] - for group_wrapper in top_rows.find_all("div", class_="row", recursive=False): - group = _direct_child(group_wrapper, ["records_subtable", "flex_table"]) - if group is None: - continue - group_header = _direct_child(group, ["row", "header"]) - more_rows = _direct_child(group, ["rows"]) - if group_header is None: - continue - - fish = _text(group_header.select_one(".fish .text")) - rows = [group_header] - if more_rows is not None: - rows.extend(more_rows.find_all("div", class_="row", recursive=False)) - - for row in rows: - bait_node = row.select_one(".bait .bait_icon") - bait = bait_node.get("title", "").strip() if bait_node else "" - player = _text(row.select_one(".gamername")) - result.append( - OfficialRecord( - region=region.upper(), - category=category, - fish=fish, - weight_g=parse_weight_g(_text(row.select_one(".weight"))), - waterbody=_text(row.select_one(".location")), - bait=bait or None, - player=player or None, - record_date=parse_record_date(_text(row.select_one(".data")), today=today), - source_url=source_url, - ) - ) - - if not result: - raise RecordsParseError("records table is present but contains no records") - return result + return [OfficialRecord( + region=region.upper(), category=category, fish=row.fish, weight_g=row.weight_g, + waterbody=row.waterbody, bait=row.bait, player=row.player, + record_date=row.record_date, source_url=source_url, + ) for row in parse_official_records(html, today=today)] def fetch_html(url: str, *, timeout: float = 20.0) -> str: diff --git a/tests/test_parser_contract.py b/tests/test_parser_contract.py index 3f05b85..237c943 100644 --- a/tests/test_parser_contract.py +++ b/tests/test_parser_contract.py @@ -30,7 +30,7 @@ def _production_contract(html: str) -> list[tuple[object, ...]]: ] -def test_research_and_production_parsers_emit_the_same_contract() -> None: +def test_research_and_production_adapters_emit_the_same_shared_contract() -> None: html = FIXTURE.read_text(encoding="utf-8") assert _research_contract(html) == _production_contract(html) @@ -41,5 +41,5 @@ def test_both_parsers_reject_a_changed_column_contract() -> None: with pytest.raises(RecordsParseError, match="records columns changed"): _research_contract(html) - with pytest.raises(ImportSourceError, match="record columns changed"): + with pytest.raises(ImportSourceError, match="records columns changed"): _production_contract(html)