ik/rf4-spotter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: share official records parser

Browse Source
This commit is contained in:
ik
2026-09-03 18:53:03 +07:00
parent c4f4deb24b
commit fda2e5a7bc
7 changed files with 144 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+7
View File
@@ -0,0 +1,7 @@
.git
.venv
**/__pycache__
**/.pytest_cache
apps/web/node_modules
apps/web/dist
design-reference
apps/api/Dockerfile
+3 -2
View File
@@ -1,8 +1,9 @@
FROM python:3.12-slim
WORKDIR /app
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
COPY requirements.txt .
COPY apps/api/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
COPY apps/api .
COPY rf4_research ./rf4_research
EXPOSE 8000
CMD ["sh", "-c", "alembic upgrade head && python -m app.seed && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
apps/api/app/importer.py
+10 -48
View File
@@ -7,9 +7,9 @@ from dataclasses import asdict, dataclass
from datetime import date, datetime, time, timezone
import httpx
from bs4 import BeautifulSoup, Tag
from sqlalchemy import select
from sqlalchemy.orm import Session
from rf4_research.official_parser import RecordsContractError, parse_official_records
from .models import (
Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus,
@@ -60,54 +60,16 @@ def external_id(record: RawRecord) -> str:
return hashlib.sha256("|".join(normalize(part) for part in parts).encode()).hexdigest()
def parse_weight(raw: str) -> int:
value = " ".join(raw.replace("\xa0", " ").split()).lower()
match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", value)
if not match:
raise ImportSourceError(f"unsupported weight {raw!r}")
number, unit = match.groups()
number = number.replace(" ", "").replace("'", "").replace("’", "")
if unit == "g":
return int(number.replace(".", "").replace(",", ""))
if "," in number and "." not in number:
number = number.replace(",", ".")
return round(float(number) * 1000)
def _text(node: Tag | None) -> str:
return node.get_text(" ", strip=True) if node else ""
def parse_html(html: str, *, region: str, category: str) -> list[RawRecord]:
soup = BeautifulSoup(html, "html.parser")
table = soup.select_one("div.records.flex_table")
if table is None:
raise ImportSourceError("records table not found")
header = table.select_one(":scope > .row.header")
expected = ["fish", "weight", "location", "bait", "gamername", "data"]
cells = header.find_all("div", recursive=False) if header else []
actual = [next((key for key in expected if key in cell.get("class", [])), "") for cell in cells]
if actual != expected:
raise ImportSourceError(f"record columns changed: {actual}")
records: list[RawRecord] = []
for group in table.select(":scope > .rows > .row > .records_subtable"):
group_header = group.select_one(":scope > .row.header")
if group_header is None:
continue
fish = _text(group_header.select_one(".fish .text"))
rows = [group_header, *group.select(":scope > .rows > .row")]
for row in rows:
bait_node = row.select_one(".bait_icon")
bait = bait_node.get("title", "").strip() if bait_node else ""
try:
record_date = datetime.strptime(_text(row.select_one(".data")), "%d.%m.%y").date()
except ValueError as exc:
raise ImportSourceError("record date format changed") from exc
records.append(RawRecord(region.upper(), category, _text(row.select_one(".gamername")) or None, fish, parse_weight(_text(row.select_one(".weight"))), _text(row.select_one(".location")), bait or None, record_date))
if not records:
raise ImportSourceError("records table is empty")
return records
try:
rows = parse_official_records(html)
except RecordsContractError as exc:
raise ImportSourceError(str(exc)) from exc
return [RawRecord(
region=region.upper(), category=category, player=row.player, fish=row.fish,
weight_g=row.weight_g, waterbody=row.waterbody, bait=row.bait,
record_date=row.record_date,
) for row in rows]
def fetch_records(
compose.yaml
+9 -3
View File
@@ -26,7 +26,9 @@ services:
- minio_data:/data
api:
build: ./apps/api
build:
context: .
dockerfile: apps/api/Dockerfile
environment:
DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter
ADMIN_TOKEN: ${ADMIN_TOKEN:-change-me-in-production}
@@ -64,7 +66,9 @@ services:
- "4321:4321"
importer:
build: ./apps/api
build:
context: .
dockerfile: apps/api/Dockerfile
profiles: ["tools"]
environment:
DATABASE_URL: postgresql+psycopg://rf4:rf4_local@db:5432/rf4_spotter
@@ -74,7 +78,9 @@ services:
command: ["sh", "-c", "alembic upgrade head && python -m app.cli import-records"]
scheduler:
build: ./apps/api
build:
context: .
dockerfile: apps/api/Dockerfile
profiles: ["scheduler"]
restart: unless-stopped
environment:
rf4_research/official_parser.py
+105
View File
@@ -0,0 +1,105 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from datetime import date, datetime
from typing import Iterable
from bs4 import BeautifulSoup, Tag
class RecordsContractError(ValueError):
"""The official records page no longer matches the verified DOM contract."""
@dataclass(frozen=True, slots=True)
class ParsedOfficialRecord:
player: str | None
fish: str
weight_g: int
waterbody: str
bait: str | None
record_date: date
def _text(node: Tag | None) -> str:
return node.get_text(" ", strip=True) if node else ""
def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None:
wanted = set(classes)
for child in parent.find_all("div", recursive=False):
if wanted.issubset(set(child.get("class", []))):
return child
return None
def parse_weight_g(raw: str) -> int:
normalized = " ".join(raw.replace("\xa0", " ").split()).lower()
match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized)
if not match:
raise RecordsContractError(f"unsupported weight: {raw!r}")
number, unit = match.groups()
number = number.replace(" ", "").replace("'", "").replace("’", "")
if unit == "g":
return int(number.replace(".", "").replace(",", ""))
if "," in number and "." not in number:
number = number.replace(",", ".")
return round(float(number) * 1000)
def parse_record_date(raw: str, *, today: date | None = None) -> date:
today = today or date.today()
try:
parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date()
except ValueError as exc:
raise RecordsContractError(f"unsupported record date: {raw!r}") from exc
if parsed > today.replace(year=today.year + 1):
raise RecordsContractError(f"record date is implausibly far in the future: {raw!r}")
return parsed
def parse_official_records(html: str, *, today: date | None = None) -> list[ParsedOfficialRecord]:
soup = BeautifulSoup(html, "html.parser")
table = soup.select_one("div.records.flex_table")
if table is None:
raise RecordsContractError("records table not found")
top_rows = _direct_child(table, ["rows"])
header = _direct_child(table, ["row", "header"])
expected = ["fish", "weight", "location", "bait", "gamername", "data"]
actual = [
next((name for name in expected if name in cell.get("class", [])), "")
for cell in (header.find_all("div", recursive=False) if header else [])
]
if actual != expected:
raise RecordsContractError(f"records columns changed: expected {expected}, got {actual}")
if top_rows is None:
raise RecordsContractError("records rows container not found")
result: list[ParsedOfficialRecord] = []
for wrapper in top_rows.find_all("div", class_="row", recursive=False):
group = _direct_child(wrapper, ["records_subtable", "flex_table"])
if group is None:
continue
group_header = _direct_child(group, ["row", "header"])
more_rows = _direct_child(group, ["rows"])
if group_header is None:
continue
fish = _text(group_header.select_one(".fish .text"))
rows = [group_header]
if more_rows is not None:
rows.extend(more_rows.find_all("div", class_="row", recursive=False))
for row in rows:
bait_node = row.select_one(".bait .bait_icon")
bait = bait_node.get("title", "").strip() if bait_node else ""
result.append(ParsedOfficialRecord(
player=_text(row.select_one(".gamername")) or None,
fish=fish,
weight_g=parse_weight_g(_text(row.select_one(".weight"))),
waterbody=_text(row.select_one(".location")),
bait=bait or None,
record_date=parse_record_date(_text(row.select_one(".data")), today=today),
))
if not result:
raise RecordsContractError("records table is present but contains no records")
return result
rf4_research/records.py
+8 -102
View File
@@ -2,22 +2,19 @@ from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import asdict, dataclass
from datetime import date, datetime
from typing import Iterable
from datetime import date
from urllib.request import Request, urlopen
from bs4 import BeautifulSoup, Tag
from .official_parser import RecordsContractError, parse_official_records, parse_record_date, parse_weight_g
DEFAULT_URL = "https://rf4game.de/records/region/RU/"
USER_AGENT = "RF4-Spotter-Research/0.1 (+https://github.com/)"
class RecordsParseError(ValueError):
"""Raised when the page no longer matches the verified records contract."""
RecordsParseError = RecordsContractError
@dataclass(frozen=True, slots=True)
@@ -33,47 +30,6 @@ class OfficialRecord:
source_url: str
def _text(node: Tag | None) -> str:
return node.get_text(" ", strip=True) if node else ""
def _direct_child(parent: Tag, classes: Iterable[str]) -> Tag | None:
wanted = set(classes)
for child in parent.find_all("div", recursive=False):
if wanted.issubset(set(child.get("class", []))):
return child
return None
def parse_weight_g(raw: str) -> int:
normalized = " ".join(raw.replace("\xa0", " ").split()).lower()
match = re.fullmatch(r"([\d .,'’]+)\s*(kg|g)", normalized)
if not match:
raise RecordsParseError(f"unsupported weight: {raw!r}")
number, unit = match.groups()
number = number.replace(" ", "").replace("'", "").replace("’", "")
if unit == "g":
return int(number.replace(".", "").replace(",", ""))
# The verified pages use a dot as the kg decimal separator and spaces as
# thousands separators (for example, "2 519.264 kg").
if "," in number and "." not in number:
number = number.replace(",", ".")
return round(float(number) * 1000)
def parse_record_date(raw: str, *, today: date | None = None) -> date:
today = today or date.today()
try:
parsed = datetime.strptime(raw.strip(), "%d.%m.%y").date()
except ValueError as exc:
raise RecordsParseError(f"unsupported record date: {raw!r}") from exc
if parsed > today.replace(year=today.year + 1):
raise RecordsParseError(f"record date is implausibly far in the future: {raw!r}")
return parsed
def parse_records_html(
html: str,
*,
@@ -82,61 +38,11 @@ def parse_records_html(
source_url: str,
today: date | None = None,
) -> list[OfficialRecord]:
soup = BeautifulSoup(html, "html.parser")
table = soup.select_one("div.records.flex_table")
if table is None:
raise RecordsParseError("records table not found")
top_rows = _direct_child(table, ["rows"])
header = _direct_child(table, ["row", "header"])
expected_classes = ["fish", "weight", "location", "bait", "gamername", "data"]
actual_classes = [
next((name for name in expected_classes if name in cell.get("class", [])), "")
for cell in (header.find_all("div", recursive=False) if header else [])
]
if actual_classes != expected_classes:
raise RecordsParseError(
f"records columns changed: expected {expected_classes}, got {actual_classes}"
)
if top_rows is None:
raise RecordsParseError("records rows container not found")
result: list[OfficialRecord] = []
for group_wrapper in top_rows.find_all("div", class_="row", recursive=False):
group = _direct_child(group_wrapper, ["records_subtable", "flex_table"])
if group is None:
continue
group_header = _direct_child(group, ["row", "header"])
more_rows = _direct_child(group, ["rows"])
if group_header is None:
continue
fish = _text(group_header.select_one(".fish .text"))
rows = [group_header]
if more_rows is not None:
rows.extend(more_rows.find_all("div", class_="row", recursive=False))
for row in rows:
bait_node = row.select_one(".bait .bait_icon")
bait = bait_node.get("title", "").strip() if bait_node else ""
player = _text(row.select_one(".gamername"))
result.append(
OfficialRecord(
region=region.upper(),
category=category,
fish=fish,
weight_g=parse_weight_g(_text(row.select_one(".weight"))),
waterbody=_text(row.select_one(".location")),
bait=bait or None,
player=player or None,
record_date=parse_record_date(_text(row.select_one(".data")), today=today),
source_url=source_url,
)
)
if not result:
raise RecordsParseError("records table is present but contains no records")
return result
return [OfficialRecord(
region=region.upper(), category=category, fish=row.fish, weight_g=row.weight_g,
waterbody=row.waterbody, bait=row.bait, player=row.player,
record_date=row.record_date, source_url=source_url,
) for row in parse_official_records(html, today=today)]
def fetch_html(url: str, *, timeout: float = 20.0) -> str:
tests/test_parser_contract.py
+2 -2
View File
@@ -30,7 +30,7 @@ def _production_contract(html: str) -> list[tuple[object, ...]]:
]
def test_research_and_production_parsers_emit_the_same_contract() -> None:
def test_research_and_production_adapters_emit_the_same_shared_contract() -> None:
html = FIXTURE.read_text(encoding="utf-8")
assert _research_contract(html) == _production_contract(html)
@@ -41,5 +41,5 @@ def test_both_parsers_reject_a_changed_column_contract() -> None:
with pytest.raises(RecordsParseError, match="records columns changed"):
_research_contract(html)
with pytest.raises(ImportSourceError, match="record columns changed"):
with pytest.raises(ImportSourceError, match="records columns changed"):
_production_contract(html)