refactor: share official records parser

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.venv
+**/__pycache__
+**/.pytest_cache
+apps/web/node_modules
+apps/web/dist
+design-reference

apps/api/Dockerfile

@@ -1,8 +1,9 @@
 FROM python:3.12-slim
 WORKDIR /app
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
-COPY requirements.txt .
+COPY apps/api/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
-COPY . .
+COPY apps/api .
+COPY rf4_research ./rf4_research
 EXPOSE 8000
 CMD ["sh", "-c", "alembic upgrade head && python -m app.seed && uvicorn app.main:app --host 0.0.0.0 --port 8000"]

apps/api/app/importer.py

@@ -7,9 +7,9 @@ from dataclasses import asdict, dataclass
 from datetime import date, datetime, time, timezone
 
 import httpx
-from bs4 import BeautifulSoup, Tag
 from sqlalchemy import select
 from sqlalchemy.orm import Session
+from rf4_research.official_parser import RecordsContractError, parse_official_records
 
 from .models import (
     Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus,
@@ -60,54 +60,16 @@ def external_id(record: RawRecord) -> str:
     return hashlib.sha256("|".join(normalize(part) for part in parts).encode()).hexdigest()
 
 
-def parse_weight(raw: str) -> int:
-    value = " ".join(raw.replace("\xa0", " ").split()).lower()
-    match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", value)
-    if not match:
-        raise ImportSourceError(f"unsupported weight {raw!r}")
-    number, unit = match.groups()
-    number = number.replace(" ", "").replace("'", "").replace("’", "")
-    if unit == "g":
-        return int(number.replace(".", "").replace(",", ""))
-    if "," in number and "." not in number:
-        number = number.replace(",", ".")
-    return round(float(number) * 1000)
-
-
-def _text(node: Tag | None) -> str:
-    return node.get_text(" ", strip=True) if node else ""
-
-
 def parse_html(html: str, *, region: str, category: str) -> list[RawRecord]:
-    soup = BeautifulSoup(html, "html.parser")
-    table = soup.select_one("div.records.flex_table")
-    if table is None:
-        raise ImportSourceError("records table not found")
-    header = table.select_one(":scope > .row.header")
-    expected = ["fish", "weight", "location", "bait", "gamername", "data"]
-    cells = header.find_all("div", recursive=False) if header else []
-    actual = [next((key for key in expected if key in cell.get("class", [])), "") for cell in cells]
-    if actual != expected:
-        raise ImportSourceError(f"record columns changed: {actual}")
-
-    records: list[RawRecord] = []
-    for group in table.select(":scope > .rows > .row > .records_subtable"):
-        group_header = group.select_one(":scope > .row.header")
-        if group_header is None:
-            continue
-        fish = _text(group_header.select_one(".fish .text"))
-        rows = [group_header, *group.select(":scope > .rows > .row")]
-        for row in rows:
-            bait_node = row.select_one(".bait_icon")
-            bait = bait_node.get("title", "").strip() if bait_node else ""
     try:
-                record_date = datetime.strptime(_text(row.select_one(".data")), "%d.%m.%y").date()
+        rows = parse_official_records(html)
-            except ValueError as exc:
+    except RecordsContractError as exc:
-                raise ImportSourceError("record date format changed") from exc
+        raise ImportSourceError(str(exc)) from exc
-            records.append(RawRecord(region.upper(), category, _text(row.select_one(".gamername")) or None, fish, parse_weight(_text(row.select_one(".weight"))), _text(row.select_one(".location")), bait or None, record_date))
+    return [RawRecord(
-    if not records:
+        region=region.upper(), category=category, player=row.player, fish=row.fish,
-        raise ImportSourceError("records table is empty")
+        weight_g=row.weight_g, waterbody=row.waterbody, bait=row.bait,
-    return records
+        record_date=row.record_date,
+    ) for row in rows]
 
 
 def fetch_records(

compose.yaml

@@ -26,7 +26,9 @@ services:
       - minio_data:/data
 
   api:
-    build: ./apps/api
+    build:
+      context: .
+      dockerfile: apps/api/Dockerfile
     environment:
       DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter
       ADMIN_TOKEN: ${ADMIN_TOKEN:-change-me-in-production}
@@ -64,7 +66,9 @@ services:
       - "4321:4321"
 
   importer:
-    build: ./apps/api
+    build:
+      context: .
+      dockerfile: apps/api/Dockerfile
     profiles: ["tools"]
     environment:
       DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter
@@ -74,7 +78,9 @@ services:
     command: ["sh", "-c", "alembic upgrade head && python -m app.cli import-records"]
 
   scheduler:
-    build: ./apps/api
+    build:
+      context: .
+      dockerfile: apps/api/Dockerfile
     profiles: ["scheduler"]
     restart: unless-stopped
     environment:

rf4_research/official_parser.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import date, datetime
+from typing import Iterable
+
+from bs4 import BeautifulSoup, Tag
+
+
+class RecordsContractError(ValueError):
+    """The official records page no longer matches the verified DOM contract."""
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedOfficialRecord:
+    player: str | None
+    fish: str
+    weight_g: int
+    waterbody: str
+    bait: str | None
+    record_date: date
+
+
+def _text(node: Tag | None) -> str:
+    return node.get_text(" ", strip=True) if node else ""
+
+
+def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None:
+    wanted = set(classes)
+    for child in parent.find_all("div", recursive=False):
+        if wanted.issubset(set(child.get("class", []))):
+            return child
+    return None
+
+
+def parse_weight_g(raw: str) -> int:
+    normalized = " ".join(raw.replace("\xa0", " ").split()).lower()
+    match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized)
+    if not match:
+        raise RecordsContractError(f"unsupported weight: {raw!r}")
+    number, unit = match.groups()
+    number = number.replace(" ", "").replace("'", "").replace("’", "")
+    if unit == "g":
+        return int(number.replace(".", "").replace(",", ""))
+    if "," in number and "." not in number:
+        number = number.replace(",", ".")
+    return round(float(number) * 1000)
+
+
+def parse_record_date(raw: str, *, today: date | None = None) -> date:
+    today = today or date.today()
+    try:
+        parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date()
+    except ValueError as exc:
+        raise RecordsContractError(f"unsupported record date: {raw!r}") from exc
+    if parsed > today.replace(year=today.year + 1):
+        raise RecordsContractError(f"record date is implausibly far in the future: {raw!r}")
+    return parsed
+
+
+def parse_official_records(html: str, *, today: date | None = None) -> list[ParsedOfficialRecord]:
+    soup = BeautifulSoup(html, "html.parser")
+    table = soup.select_one("div.records.flex_table")
+    if table is None:
+        raise RecordsContractError("records table not found")
+    top_rows = _direct_child(table, ["rows"])
+    header = _direct_child(table, ["row", "header"])
+    expected = ["fish", "weight", "location", "bait", "gamername", "data"]
+    actual = [
+        next((name for name in expected if name in cell.get("class", [])), "")
+        for cell in (header.find_all("div", recursive=False) if header else [])
+    ]
+    if actual != expected:
+        raise RecordsContractError(f"records columns changed: expected {expected}, got {actual}")
+    if top_rows is None:
+        raise RecordsContractError("records rows container not found")
+
+    result: list[ParsedOfficialRecord] = []
+    for wrapper in top_rows.find_all("div", class_="row", recursive=False):
+        group = _direct_child(wrapper, ["records_subtable", "flex_table"])
+        if group is None:
+            continue
+        group_header = _direct_child(group, ["row", "header"])
+        more_rows = _direct_child(group, ["rows"])
+        if group_header is None:
+            continue
+        fish = _text(group_header.select_one(".fish .text"))
+        rows = [group_header]
+        if more_rows is not None:
+            rows.extend(more_rows.find_all("div", class_="row", recursive=False))
+        for row in rows:
+            bait_node = row.select_one(".bait .bait_icon")
+            bait = bait_node.get("title", "").strip() if bait_node else ""
+            result.append(ParsedOfficialRecord(
+                player=_text(row.select_one(".gamername")) or None,
+                fish=fish,
+                weight_g=parse_weight_g(_text(row.select_one(".weight"))),
+                waterbody=_text(row.select_one(".location")),
+                bait=bait or None,
+                record_date=parse_record_date(_text(row.select_one(".data")), today=today),
+            ))
+    if not result:
+        raise RecordsContractError("records table is present but contains no records")
+    return result

rf4_research/records.py

@@ -2,22 +2,19 @@ from __future__ import annotations
 
 import argparse
 import json
-import re
 import sys
 from dataclasses import asdict, dataclass
-from datetime import date, datetime
+from datetime import date
-from typing import Iterable
 from urllib.request import Request, urlopen
 
-from bs4 import BeautifulSoup, Tag
+from .official_parser import RecordsContractError, parse_official_records, parse_record_date, parse_weight_g
 
 
 DEFAULT_URL = "https://rf4game.de/records/region/RU/"
 USER_AGENT = "RF4-Spotter-Research/0.1 (+https://github.com/)"
 
 
-class RecordsParseError(ValueError):
+RecordsParseError = RecordsContractError
-    """Raised when the page no longer matches the verified records contract."""
 
 
 @dataclass(frozen=True, slots=True)
@@ -33,47 +30,6 @@ class OfficialRecord:
     source_url: str
 
 
-def _text(node: Tag | None) -> str:
-    return node.get_text(" ", strip=True) if node else ""
-
-
-def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None:
-    wanted = set(classes)
-    for child in parent.find_all("div", recursive=False):
-        if wanted.issubset(set(child.get("class", []))):
-            return child
-    return None
-
-
-def parse_weight_g(raw: str) -> int:
-    normalized = " ".join(raw.replace("\xa0", " ").split()).lower()
-    match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized)
-    if not match:
-        raise RecordsParseError(f"unsupported weight: {raw!r}")
-
-    number, unit = match.groups()
-    number = number.replace(" ", "").replace("'", "").replace("’", "")
-    if unit == "g":
-        return int(number.replace(".", "").replace(",", ""))
-
-    # The verified pages use a dot as the kg decimal separator and spaces as
-    # thousands separators (for example, "2 519.264 kg").
-    if "," in number and "." not in number:
-        number = number.replace(",", ".")
-    return round(float(number) * 1000)
-
-
-def parse_record_date(raw: str, *, today: date | None = None) -> date:
-    today = today or date.today()
-    try:
-        parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date()
-    except ValueError as exc:
-        raise RecordsParseError(f"unsupported record date: {raw!r}") from exc
-    if parsed > today.replace(year=today.year + 1):
-        raise RecordsParseError(f"record date is implausibly far in the future: {raw!r}")
-    return parsed
-
-
 def parse_records_html(
     html: str,
     *,
@@ -82,61 +38,11 @@ def parse_records_html(
     source_url: str,
     today: date | None = None,
 ) -> list[OfficialRecord]:
-    soup = BeautifulSoup(html, "html.parser")
+    return [OfficialRecord(
-    table = soup.select_one("div.records.flex_table")
+        region=region.upper(), category=category, fish=row.fish, weight_g=row.weight_g,
-    if table is None:
+        waterbody=row.waterbody, bait=row.bait, player=row.player,
-        raise RecordsParseError("records table not found")
+        record_date=row.record_date, source_url=source_url,
-
+    ) for row in parse_official_records(html, today=today)]
-    top_rows = _direct_child(table, ["rows"])
-    header = _direct_child(table, ["row", "header"])
-    expected_classes = ["fish", "weight", "location", "bait", "gamername", "data"]
-    actual_classes = [
-        next((name for name in expected_classes if name in cell.get("class", [])), "")
-        for cell in (header.find_all("div", recursive=False) if header else [])
-    ]
-    if actual_classes != expected_classes:
-        raise RecordsParseError(
-            f"records columns changed: expected {expected_classes}, got {actual_classes}"
-        )
-    if top_rows is None:
-        raise RecordsParseError("records rows container not found")
-
-    result: list[OfficialRecord] = []
-    for group_wrapper in top_rows.find_all("div", class_="row", recursive=False):
-        group = _direct_child(group_wrapper, ["records_subtable", "flex_table"])
-        if group is None:
-            continue
-        group_header = _direct_child(group, ["row", "header"])
-        more_rows = _direct_child(group, ["rows"])
-        if group_header is None:
-            continue
-
-        fish = _text(group_header.select_one(".fish .text"))
-        rows = [group_header]
-        if more_rows is not None:
-            rows.extend(more_rows.find_all("div", class_="row", recursive=False))
-
-        for row in rows:
-            bait_node = row.select_one(".bait .bait_icon")
-            bait = bait_node.get("title", "").strip() if bait_node else ""
-            player = _text(row.select_one(".gamername"))
-            result.append(
-                OfficialRecord(
-                    region=region.upper(),
-                    category=category,
-                    fish=fish,
-                    weight_g=parse_weight_g(_text(row.select_one(".weight"))),
-                    waterbody=_text(row.select_one(".location")),
-                    bait=bait or None,
-                    player=player or None,
-                    record_date=parse_record_date(_text(row.select_one(".data")), today=today),
-                    source_url=source_url,
-                )
-            )
-
-    if not result:
-        raise RecordsParseError("records table is present but contains no records")
-    return result
 
 
 def fetch_html(url: str, *, timeout: float = 20.0) -> str:

tests/test_parser_contract.py

@@ -30,7 +30,7 @@ def _production_contract(html: str) -> list[tuple[object, ...]]:
     ]
 
 
-def test_research_and_production_parsers_emit_the_same_contract() -> None:
+def test_research_and_production_adapters_emit_the_same_shared_contract() -> None:
     html = FIXTURE.read_text(encoding="utf-8")
 
     assert _research_contract(html) == _production_contract(html)
@@ -41,5 +41,5 @@ def test_both_parsers_reject_a_changed_column_contract() -> None:
 
     with pytest.raises(RecordsParseError, match="records columns changed"):
         _research_contract(html)
-    with pytest.raises(ImportSourceError, match="record columns changed"):
+    with pytest.raises(ImportSourceError, match="records columns changed"):
         _production_contract(html)