From 722c88d436c22ae6f0ff2f105d30c7d5653ea5ff Mon Sep 17 00:00:00 2001 From: IK Date: Thu, 17 Sep 2026 07:35:04 +0700 Subject: [PATCH] sync --- README.md | 6 +- apps/api/app/activity.py | 17 +- apps/api/app/catalog_audit.py | 25 +++ apps/api/app/cli.py | 68 +++++++- apps/api/app/community_importer.py | 167 ++++++++++++++++++++ apps/api/app/community_review.py | 2 + apps/api/app/models.py | 12 ++ apps/api/app/routers/activity.py | 10 +- apps/api/app/schemas.py | 14 ++ apps/api/tests/test_catalog_audit.py | 23 ++- apps/api/tests/test_community_importer.py | 90 ++++++++++- apps/web/src/components/ActivityCard.astro | 3 +- apps/web/src/components/EntityMedia.astro | 8 +- apps/web/src/lib/api.ts | 8 +- apps/web/src/pages/fish/index.astro | 18 ++- apps/web/src/pages/index.astro | 16 +- apps/web/src/pages/records.astro | 17 +- apps/web/src/pages/waterbodies/[slug].astro | 1 + apps/web/src/pages/waterbodies/index.astro | 18 ++- compose.yaml | 19 +++ docs/RECOVERY_PLAN_2026-09-10.md | 9 +- docs/REGRESSION_FIXES_REPORT.md | 4 +- rf4_research/community_cli.py | 2 + rf4_research/community_sources.py | 90 ++++++++++- tests/test_community_sources.py | 66 ++++++++ 25 files changed, 673 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index f385b69..8841e8f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ RF4 Spotter — неофициальный сервис свежих точек ## Статус разработки -**Проверка 13 сентября 2026 (`787a506`): локальный контур готов к развёртыванию открытой альфы, внешний запуск ждёт сервер и его настройки.** Пакет восстановления A01–A13 закрыт. Python: **157 passed, 1 skipped**; Astro check/build, web unit и API-тесты проходят. Граф миграций имеет единственную голову `0016`; последний полный production bootstrap подтвердил Caddy, scheduler и браузерный сценарий отправки/модерации на предыдущей голове, а актуальная голова проверяется CI на чистой PostgreSQL. Реальные источники во время приёмки не опрашивались. +**Проверка 15 сентября 2026: локальный контур готов к развёртыванию открытой альфы, внешний запуск ждёт сервер и его настройки.** Пакет восстановления A01–A13 закрыт. Python: **178 passed, 1 skipped**; Astro check/build, web unit и API-тесты проходят. Граф миграций имеет единственную голову `0018`; production bootstrap и импорт реальных источников по-прежнему требуют отдельной инфраструктурной/разрешённой приёмки. Актуальные следующие задачи находятся только в [ROADMAP](docs/ROADMAP.md). Старые планы и аудиты сохранены как история и больше не задают порядок работ. До внешнего запуска нужны сервер, DNS/TLS, production-секреты, публичные контакты, внешний backup и канал уведомлений. @@ -148,7 +148,7 @@ docker compose up --build - readiness PostgreSQL, MinIO и импорта с версией/revision сборки: ; - консоль MinIO: . -Контейнер API сам выполняет `alembic upgrade head`, затем идемпотентный seed. PostgreSQL хранит данные в именованном volume `postgres_data`, а MinIO — в `minio_data`. Compose ожидает readiness PostgreSQL и MinIO перед API, а API-контейнер проверяет `/ready`. Версия и commit SHA задаются через `APP_VERSION`/`APP_REVISION`; те же значения доступны администратору в `/api/v1/admin/diagnostics`. Официальный импорт по умолчанию необязателен; при включённом scheduler установите `OFFICIAL_IMPORT_REQUIRED=true`, тогда отсутствующий, неуспешный или просроченный запуск сделает readiness отрицательным. +Одноразовые контейнеры `migrate` и `minio-init` перед запуском API соответственно применяют `alembic upgrade head` и идемпотентно создают локальный `S3_BUCKET`; приложение само не создаёт схему или bucket. PostgreSQL хранит данные в именованном volume `postgres_data`, а MinIO — в `minio_data`. Compose ожидает readiness PostgreSQL, MinIO и успешное завершение обоих init-контейнеров перед API, а API-контейнер проверяет `/ready`. Версия и commit SHA задаются через `APP_VERSION`/`APP_REVISION`; те же значения доступны администратору в `/api/v1/admin/diagnostics`. Здоровье импортов диагностическое и не мешает API или scheduler восстановиться после сбоя. API и scheduler пишут по одной JSON-записи на событие. HTTP-лог содержит только сгенерированный `request_id`, метод, путь без query string, статус и длительность; IP, заголовок авторизации и пользовательский payload не журналируются. `X-Request-ID` возвращается клиенту. Стандартный access-log Uvicorn отключён. Уровень управляется `LOG_LEVEL`. Публичный агрегат активности кэшируется в памяти процесса на 20 секунд (до 128 ключей) и очищается после публикации, модерации или удаления через этот процесс API; изменения scheduler видны после TTL; `X-Cache` показывает `HIT`/`MISS`. Защищённый `/api/v1/admin/diagnostics` скачивает JSON только с идентификатором сборки и агрегированными счётчиками, без имён игроков, исходных URL, payload и ошибок парсеров. @@ -184,7 +184,7 @@ docker compose up --build ## Что реализовано - FastAPI и SQLAlchemy 2; -- PostgreSQL 17 и линейные миграции Alembic до `0016`; +- PostgreSQL 17 и линейные миграции Alembic до `0018`; - идемпотентный seed с двумя точками и свежими демо-уловами; - `GET /api/v1/activity` с фильтрами периода, водоёма, рыбы, способа и сортировки; - `GET /api/v1/spots/{id}` и `/catches`; diff --git a/apps/api/app/activity.py b/apps/api/app/activity.py index 18ba30a..fa0e0b1 100644 --- a/apps/api/app/activity.py +++ b/apps/api/app/activity.py @@ -61,6 +61,9 @@ def activity_rows( confidence = min(confidence, 50) elif len(players) == 2: confidence = min(confidence, 65) + coordinate_precisions = {_coordinate_precision(item) for item in items} + coordinate_precision = max(coordinate_precisions, key=_precision_rank) + coordinate_sources = sorted({_source_system(item) for item in items}) latest = max(_aware(r.reported_at) for r in items) baits = Counter(r.bait.name for r in items if r.bait) freshness_text = _freshness_text(now - latest) @@ -74,7 +77,9 @@ def activity_rows( max_weight_g=max(r.weight_g for r in items), last_confirmed_at=latest, activity_score=activity, confidence_score=confidence, explanation=_explanation(len(items), len(players), freshness_text, activity, confidence), - sources=sorted({_source_system(item) for item in items}), + sources=coordinate_sources, + coordinate_precision=coordinate_precision, + coordinate_sources=coordinate_sources, )) return sorted(result, key=lambda row: (row.activity_score, row.last_confirmed_at), reverse=True) @@ -90,6 +95,16 @@ def _source_system(report: CatchReport) -> str: return "manual-import" +def _coordinate_precision(report: CatchReport) -> str: + provenance = (report.raw_payload or {}).get("provenance", {}) + value = provenance.get("coordinate_precision") if isinstance(provenance, dict) else None + return value if value in {"exact", "approximate", "area", "missing"} else "exact" + + +def _precision_rank(value: str) -> int: + return {"exact": 0, "approximate": 1, "area": 2, "missing": 3}[value] + + def _aware(value: datetime) -> datetime: return value if value.tzinfo else value.replace(tzinfo=timezone.utc) diff --git a/apps/api/app/catalog_audit.py b/apps/api/app/catalog_audit.py index 52d882f..9e7a275 100644 --- a/apps/api/app/catalog_audit.py +++ b/apps/api/app/catalog_audit.py @@ -15,3 +15,28 @@ def audit_catalog(db: Session) -> dict[str, int]: "incomplete_published_staging": db.scalar(select(func.count()).select_from(ExternalObservation).where(ExternalObservation.status == "published", or_(ExternalObservation.fish_id.is_(None), ExternalObservation.waterbody_id.is_(None), ExternalObservation.x.is_(None), ExternalObservation.y.is_(None), ExternalObservation.weight_g.is_(None), ExternalObservation.catch_report_id.is_(None)))) or 0, } return {"fishes": count(Fish), "waterbodies": count(Waterbody), "reports": count(CatchReport), "staging": count(ExternalObservation), **failures, "failures": sum(failures.values())} + + +def audit_waterbody_catalog(db: Session, expected_ids: set[str]) -> dict: + """Check a verified RF4DB snapshot without withdrawing legacy rows.""" + rows = list(db.scalars(select(Waterbody).where(Waterbody.source_system == "rf4db"))) + observed_ids = [str(row.source_external_id) for row in rows if row.source_external_id] + observed = set(observed_ids) + duplicate_ids = sorted({item for item in observed_ids if observed_ids.count(item) > 1}) + missing = sorted(expected_ids - observed) + unexpected = sorted(observed - expected_ids) + provenance_issues = sorted( + str(row.source_external_id) + for row in rows + if not row.source_external_id or not row.source_url or not row.source_checked_at + ) + failures = len(missing) + len(duplicate_ids) + len(provenance_issues) + return { + "expected": len(expected_ids), + "observed": len(observed), + "missing_source_external_ids": missing, + "unexpected_source_external_ids": unexpected, + "duplicate_source_external_ids": duplicate_ids, + "provenance_issues": provenance_issues, + "failures": failures, + } diff --git a/apps/api/app/cli.py b/apps/api/app/cli.py index 103ce83..7ac90b2 100644 --- a/apps/api/app/cli.py +++ b/apps/api/app/cli.py @@ -8,10 +8,10 @@ from dataclasses import asdict from .config import settings from .database import SessionLocal from .importer import import_records -from .community_importer import stage_observations +from .community_importer import stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog from .retention import RetentionPolicy, apply_retention from .storage import delete_screenshot -from .catalog_audit import audit_catalog +from .catalog_audit import audit_catalog, audit_waterbody_catalog from .community_scheduler import run_source, configured_sources # Static registry for argparse choices — no DB required for --help @@ -34,11 +34,20 @@ def main() -> int: community = sub.add_parser("stage-community-json") community.add_argument("--input", default="-", help="JSON array path or - for stdin") community.add_argument("--limit", type=int, default=500) + waterbodies = sub.add_parser("import-waterbody-catalog") + waterbodies.add_argument("--input", required=True, help="JSON snapshot path or - for stdin") + waterbodies.add_argument("--limit", type=int, default=100) + detail = sub.add_parser("import-waterbody-detail") + detail.add_argument("--input", required=True, help="JSON detail snapshot path") + details = sub.add_parser("import-waterbody-details") + details.add_argument("--input", required=True, help="JSON array of detail snapshots") fetch_community = sub.add_parser("fetch-community") fetch_community.add_argument("source", choices=STATIC_SOURCE_CHOICES) cleanup = sub.add_parser("cleanup-retention") cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run") sub.add_parser("audit-catalog") + waterbody_audit = sub.add_parser("audit-waterbody-catalog") + waterbody_audit.add_argument("--input", required=True, help="JSON snapshot path") args = parser.parse_args() with SessionLocal() as session: if args.command == "import-records": @@ -57,6 +66,43 @@ def main() -> int: parser.error("input must be a JSON array") created, updated = stage_observations(session, payload[:args.limit]) print(f"staged: created={created} updated={updated}") + elif args.command == "import-waterbody-catalog": + if not 1 <= args.limit <= 500: + parser.error("--limit must be between 1 and 500") + stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8") + try: + snapshot = json.load(stream) + finally: + if stream is not sys.stdin: + stream.close() + if isinstance(snapshot, dict): + payload = snapshot.get("items") + source_system = snapshot.get("source_system") + if isinstance(payload, list) and isinstance(source_system, str): + payload = [ + {"source_system": source_system, **item} + for item in payload if isinstance(item, dict) + ] + else: + payload = snapshot + if not isinstance(payload, list): + parser.error("input must be a JSON array or an object with an items array") + created, updated = upsert_waterbody_catalog(session, payload[:args.limit]) + print(f"waterbodies: created={created} updated={updated}") + elif args.command == "import-waterbody-detail": + with open(args.input, encoding="utf-8") as stream: + payload = json.load(stream) + if not isinstance(payload, dict): + parser.error("input must be a JSON object") + update_waterbody_detail(session, payload) + print(f"waterbody detail: updated={payload.get('source_external_id', 'unknown')}") + elif args.command == "import-waterbody-details": + with open(args.input, encoding="utf-8") as stream: + payload = json.load(stream) + if not isinstance(payload, list): + parser.error("input must be a JSON array") + created, updated = update_waterbody_details(session, payload) + print(f"waterbody details: created={created} updated={updated}") elif args.command == "fetch-community": # A09: Verify source is enabled at runtime (not just in static choices) enabled = configured_sources() @@ -76,6 +122,24 @@ def main() -> int: ) counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot) print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False)) + elif args.command == "audit-waterbody-catalog": + stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8") + try: + snapshot = json.load(stream) + finally: + if stream is not sys.stdin: + stream.close() + items = snapshot.get("items") if isinstance(snapshot, dict) else snapshot + if not isinstance(items, list): + parser.error("input must be a JSON array or an object with an items array") + expected_ids = { + str(item["source_external_id"]) + for item in items + if isinstance(item, dict) and item.get("source_external_id") + } + result = audit_waterbody_catalog(session, expected_ids) + print(json.dumps(result, ensure_ascii=False)) + return 1 if result["failures"] else 0 else: result = audit_catalog(session) print(json.dumps(result, ensure_ascii=False)) diff --git a/apps/api/app/community_importer.py b/apps/api/app/community_importer.py index 0501188..69b3108 100644 --- a/apps/api/app/community_importer.py +++ b/apps/api/app/community_importer.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import hashlib +import re from datetime import datetime, timezone from typing import Any, Iterable from urllib.parse import urlparse @@ -26,12 +28,153 @@ SOURCE_HOSTS = { "rf4map": {"rf4map.ru"}, "rf4posts-spot": {"rf4-posts.com"}, } +COORDINATE_PRECISIONS = frozenset({"exact", "approximate", "area", "missing"}) class CommunityImportError(ValueError): pass +def upsert_waterbody_catalog( + session: Session, rows: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None, +) -> tuple[int, int]: + """Apply a validated canonical waterbody snapshot without destructive sync. + + Rows are matched by the RF4DB source identity first and by an exact existing + name second. Missing rows are deliberately left untouched: an incomplete + response must never withdraw a previously known waterbody. + """ + fetched_at = fetched_at or datetime.now(timezone.utc) + created = updated = 0 + for raw in rows: + payload = _json_payload(raw) + if payload.get("source_system") != "rf4db": + raise CommunityImportError("waterbody catalog requires source_system=rf4db") + external_id = _required(payload, "source_external_id", 200) + name = _required(payload, "name", 200) + source_url = _required(payload, "source_url", 2000) + parsed_url = urlparse(source_url) + if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}: + raise CommunityImportError("waterbody source_url does not match rf4db") + unlock_level = _integer(payload.get("unlock_level"), minimum=0, maximum=1_000) + unlock_label = _required(payload, "unlock_label", 50) + fish_species_count = _integer(payload.get("fish_species_count"), minimum=0, maximum=10_000) + if fish_species_count is None: + raise CommunityImportError("invalid fish_species_count") + + item = session.scalar(select(Waterbody).where( + Waterbody.source_system == "rf4db", + Waterbody.source_external_id == external_id, + )) + if item is None: + item = session.scalar(select(Waterbody).where(Waterbody.name_ru == name)) + if item is None: + item = Waterbody( + slug=_catalog_slug(session, name, external_id), + name_ru=name, + unlock_level=unlock_level, + ) + session.add(item) + created += 1 + else: + updated += 1 + item.name_ru = name + item.unlock_level = unlock_level + item.fish_species_count = fish_species_count + item.source_system = "rf4db" + item.source_external_id = external_id + item.source_url = source_url + item.source_checked_at = fetched_at + session.commit() + return created, updated + + +def update_waterbody_detail( + session: Session, detail: dict[str, Any], *, fetched_at: datetime | None = None, +) -> bool: + """Persist one complete RF4DB detail snapshot without assigning media roles.""" + fetched_at = fetched_at or datetime.now(timezone.utc) + payload = _validate_waterbody_detail(detail) + _apply_waterbody_detail(session, payload, fetched_at=fetched_at) + session.commit() + return True + + +def update_waterbody_details( + session: Session, details: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None, +) -> tuple[int, int]: + """Validate and apply a detail batch in one transaction.""" + fetched_at = fetched_at or datetime.now(timezone.utc) + payloads = [_validate_waterbody_detail(detail) for detail in details] + external_ids = [str(payload["source_external_id"]) for payload in payloads] + if len(external_ids) != len(set(external_ids)): + raise CommunityImportError("waterbody detail batch contains duplicate source identities") + updated = 0 + for payload in payloads: + _apply_waterbody_detail(session, payload, fetched_at=fetched_at) + updated += 1 + session.commit() + return 0, updated + + +def _validate_waterbody_detail(detail: dict[str, Any]) -> dict[str, Any]: + payload = _json_payload(detail) + if payload.get("source_system") != "rf4db": + raise CommunityImportError("waterbody detail requires source_system=rf4db") + external_id = _required(payload, "source_external_id", 200) + source_url = _required(payload, "source_url", 2000) + parsed_url = urlparse(source_url) + if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}: + raise CommunityImportError("waterbody detail source_url does not match rf4db") + _required(payload, "name", 200) + _optional(payload, "description", 20_000) + _string_list(payload, "aliases", 100, 200) + _string_list(payload, "fish_species", 10_000, 200) + _string_list(payload, "image_urls", 100, 2_000) + _string_list(payload, "point_urls", 10_000, 2_000) + return payload + + +def _apply_waterbody_detail(session: Session, payload: dict[str, Any], *, fetched_at: datetime) -> None: + external_id = str(payload["source_external_id"]) + source_url = str(payload["source_url"]) + item = session.scalar(select(Waterbody).where( + Waterbody.source_system == "rf4db", Waterbody.source_external_id == external_id, + )) + if item is None: + raise CommunityImportError("waterbody detail has no imported catalog identity") + item.description = _optional(payload, "description", 20_000) + item.source_aliases = _string_list(payload, "aliases", 100, 200) + item.source_fish_species = _string_list(payload, "fish_species", 10_000, 200) + item.source_image_urls = _string_list(payload, "image_urls", 100, 2_000) + item.source_point_urls = _string_list(payload, "point_urls", 10_000, 2_000) + item.source_url = source_url + item.source_checked_at = fetched_at + + +def _catalog_slug(session: Session, name: str, external_id: str) -> str: + base = re.sub(r"[^a-z0-9а-яё]+", "-", name.casefold(), flags=re.IGNORECASE).strip("-") + base = base or "waterbody" + candidate = base[:100] + if session.scalar(select(Waterbody.id).where(Waterbody.slug == candidate)) is None: + return candidate + suffix = hashlib.sha256(external_id.encode()).hexdigest()[:10] + return f"{base[:89]}-{suffix}" + + +def _string_list(payload: dict[str, Any], key: str, max_items: int, max_length: int) -> list[str]: + value = payload.get(key) + if not isinstance(value, list) or len(value) > max_items: + raise CommunityImportError(f"invalid {key}") + result = [] + for item in value: + text = str(item).strip() + if not text or len(text) > max_length: + raise CommunityImportError(f"invalid {key}") + result.append(text) + return list(dict.fromkeys(result)) + + def stage_observations( session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None, ) -> tuple[int, int]: @@ -65,6 +208,8 @@ def stage_observations( "waterbody_external_id": _optional(payload, "waterbody_external_id", 200), "x": _integer(payload.get("x"), maximum=10_000), "y": _integer(payload.get("y"), maximum=10_000), + "coordinate_raw": _coordinate_raw(payload), + "coordinate_precision": _coordinate_precision(payload), "weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000), "published_at": _datetime(payload.get("published_at")), "last_seen_at": fetched_at, "payload": payload, @@ -83,6 +228,7 @@ def stage_observations( changed = any(getattr(observation, key) != values[key] for key in ( "source_url", "fish_name", "fish_external_id", "waterbody_name", "waterbody_external_id", "x", "y", "weight_g", + "coordinate_raw", "coordinate_precision", )) or observation.payload != payload if observation.status != "rejected" and changed and observation.catch_report is not None: observation.catch_report.moderation_status = ModerationStatus.pending @@ -186,6 +332,27 @@ def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None: return value or None +def _coordinate_raw(payload: dict[str, Any]) -> str | None: + value = str(payload.get("coordinate_raw") or "").strip() + if len(value) > 200: + raise CommunityImportError("invalid coordinate_raw") + if value: + return value + x, y = payload.get("x"), payload.get("y") + return f"{x}:{y}" if isinstance(x, int) and isinstance(y, int) else None + + +def _coordinate_precision(payload: dict[str, Any]) -> str: + value = str(payload.get("coordinate_precision") or "").strip().casefold() + if not value: + return "exact" if isinstance(payload.get("x"), int) and isinstance(payload.get("y"), int) else "missing" + if value not in COORDINATE_PRECISIONS: + raise CommunityImportError("invalid coordinate_precision") + if value == "exact" and (not isinstance(payload.get("x"), int) or not isinstance(payload.get("y"), int)): + raise CommunityImportError("exact coordinates require x and y") + return value + + def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None: if value is None: return None diff --git a/apps/api/app/community_review.py b/apps/api/app/community_review.py index ab2619f..c1a3b40 100644 --- a/apps/api/app/community_review.py +++ b/apps/api/app/community_review.py @@ -104,6 +104,8 @@ def publish_observation(session: Session, observation: ExternalObservation) -> C "external_observation_id": str(observation.id), "source_system": observation.source_system, "source_external_id": observation.source_external_id, + "coordinate_raw": observation.coordinate_raw, + "coordinate_precision": observation.coordinate_precision, }, "original": observation.payload, }, diff --git a/apps/api/app/models.py b/apps/api/app/models.py index 974d78b..69c8326 100644 --- a/apps/api/app/models.py +++ b/apps/api/app/models.py @@ -49,6 +49,16 @@ class Waterbody(Base): slug: Mapped[str] = mapped_column(String(100), unique=True) name_ru: Mapped[str] = mapped_column(String(200), unique=True) unlock_level: Mapped[int | None] + fish_species_count: Mapped[int | None] + source_system: Mapped[str | None] = mapped_column(String(50)) + source_external_id: Mapped[str | None] = mapped_column(String(200)) + source_url: Mapped[str | None] = mapped_column(Text) + description: Mapped[str | None] = mapped_column(Text) + source_aliases: Mapped[list[str] | None] = mapped_column(JSON) + source_fish_species: Mapped[list[str] | None] = mapped_column(JSON) + source_image_urls: Mapped[list[str] | None] = mapped_column(JSON) + source_point_urls: Mapped[list[str] | None] = mapped_column(JSON) + source_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) class Bait(Base): @@ -187,6 +197,8 @@ class ExternalObservation(Base): waterbody_external_id: Mapped[str | None] = mapped_column(String(200)) x: Mapped[int | None] y: Mapped[int | None] + coordinate_raw: Mapped[str | None] = mapped_column(String(200)) + coordinate_precision: Mapped[str] = mapped_column(String(20), default="missing") weight_g: Mapped[int | None] published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) diff --git a/apps/api/app/routers/activity.py b/apps/api/app/routers/activity.py index edd4ece..175ec75 100644 --- a/apps/api/app/routers/activity.py +++ b/apps/api/app/routers/activity.py @@ -68,7 +68,15 @@ def spot_detail(spot_id: UUID, db: Db) -> SpotOut: def count_since(delta: timedelta) -> int: return sum(aware(report.reported_at) >= now - delta for report in reports) - return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)]) + provenance = [ + (report.raw_payload or {}).get("provenance", {}) + for report in reports + if isinstance((report.raw_payload or {}).get("provenance", {}), dict) + ] + precisions = [item.get("coordinate_precision") for item in provenance] + precision = max((value for value in precisions if value in {"exact", "approximate", "area", "missing"}), key={"exact": 0, "approximate": 1, "area": 2, "missing": 3}.get, default="exact") + sources = sorted({str(item.get("source_system")) for item in provenance if item.get("source_system")}) or ["players"] + return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)], coordinate_precision=precision, coordinate_sources=sources) def _report_source(report: CatchReport) -> str: diff --git a/apps/api/app/schemas.py b/apps/api/app/schemas.py index 615f018..3930cad 100644 --- a/apps/api/app/schemas.py +++ b/apps/api/app/schemas.py @@ -20,6 +20,16 @@ class WaterbodyOut(BaseModel): slug: str name_ru: str unlock_level: int | None + fish_species_count: int | None + source_system: str | None + source_external_id: str | None + source_url: str | None + description: str | None + source_aliases: list[str] | None + source_fish_species: list[str] | None + source_image_urls: list[str] | None + source_point_urls: list[str] | None + source_checked_at: datetime | None class BaitOut(BaseModel): @@ -48,6 +58,8 @@ class ActivityOut(BaseModel): confidence_score: int explanation: str sources: list[str] + coordinate_precision: str + coordinate_sources: list[str] class PaginatedActivityOut(BaseModel): @@ -82,6 +94,8 @@ class SpotOut(BaseModel): catches_3d: int catches_7d: int top_baits: list[str] + coordinate_precision: str + coordinate_sources: list[str] class OfficialRecordOut(BaseModel): diff --git a/apps/api/tests/test_catalog_audit.py b/apps/api/tests/test_catalog_audit.py index 380e963..5540033 100644 --- a/apps/api/tests/test_catalog_audit.py +++ b/apps/api/tests/test_catalog_audit.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from sqlalchemy import create_engine from sqlalchemy.orm import Session -from app.catalog_audit import audit_catalog +from app.catalog_audit import audit_catalog, audit_waterbody_catalog from app.database import Base from app.models import CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody @@ -24,3 +24,24 @@ def test_catalog_audit_checks_the_whole_catalog() -> None: assert result["reports"] == 1 assert result["invalid_coordinates"] == 1 assert result["failures"] == 1 + + +def test_waterbody_catalog_audit_reports_snapshot_gaps_without_legacy_rows() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as db: + now = datetime.now(timezone.utc) + db.add_all([ + Waterbody( + slug="lake", name_ru="Озеро", source_system="rf4db", + source_external_id="level_001_lake", source_url="https://rf4db.com/ru/maps/level_001_lake", + source_checked_at=now, + ), + Waterbody(slug="legacy", name_ru="Старое озеро"), + ]) + db.commit() + result = audit_waterbody_catalog(db, {"level_001_lake", "level_002_river"}) + assert result["expected"] == 2 + assert result["observed"] == 1 + assert result["missing_source_external_ids"] == ["level_002_river"] + assert result["failures"] == 1 diff --git a/apps/api/tests/test_community_importer.py b/apps/api/tests/test_community_importer.py index ff4a309..e2a785d 100644 --- a/apps/api/tests/test_community_importer.py +++ b/apps/api/tests/test_community_importer.py @@ -7,7 +7,7 @@ import pytest from sqlalchemy import create_engine, func, select from sqlalchemy.orm import Session -from app.community_importer import CommunityImportError, stage_observations +from app.community_importer import CommunityImportError, stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog from app.community_review import ExternalReviewError, map_observation, publish_observation, suggest_aliases from app.source_lifecycle import record_scheduled_source_check, record_source_check from app.database import Base @@ -42,6 +42,20 @@ def record(source: str = "rf4db", external_id: str = "catch-1") -> dict[str, obj } +def waterbody_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "source_system": "rf4db", + "source_external_id": "level_001_mosquito", + "source_url": "https://rf4db.com/ru/maps/level_001_mosquito", + "name": "оз. Комариное", + "unlock_level": 1, + "unlock_label": "1", + "fish_species_count": 20, + } + row.update(overrides) + return row + + @pytest.fixture def db() -> Session: engine = create_engine("sqlite://") @@ -69,6 +83,62 @@ def test_staging_is_idempotent_and_preserves_first_seen(db: Session) -> None: assert source is not None and source.enabled is True +def test_waterbody_catalog_upsert_is_idempotent_and_non_destructive(db: Session) -> None: + first = datetime(2026, 9, 16, 10, tzinfo=timezone.utc) + assert upsert_waterbody_catalog(db, [waterbody_row()], fetched_at=first) == (1, 0) + item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito")) + assert item is not None + assert item.slug == "оз-комариное" + assert item.source_checked_at.replace(tzinfo=timezone.utc) == first + assert item.fish_species_count == 20 + + assert upsert_waterbody_catalog(db, [waterbody_row(name="Озеро Комариное", unlock_level=2)], fetched_at=first) == (0, 1) + item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito")) + assert item is not None + assert (item.name_ru, item.unlock_level, item.fish_species_count) == ("Озеро Комариное", 2, 20) + assert db.scalar(select(Waterbody).where(Waterbody.name_ru == "оз. Комариное")) is None + + +def test_waterbody_catalog_rejects_untrusted_source(db: Session) -> None: + with pytest.raises(CommunityImportError, match="source_url"): + upsert_waterbody_catalog(db, [waterbody_row(source_url="https://example.test/map")]) + + +def test_waterbody_detail_updates_only_imported_identity_without_media_roles(db: Session) -> None: + upsert_waterbody_catalog(db, [waterbody_row()]) + assert update_waterbody_detail(db, { + "source_system": "rf4db", + "source_external_id": "level_001_mosquito", + "source_url": "https://rf4db.com/ru/maps/level_001_mosquito", + "name": "оз. Комариное", + "description": "Каменистые берега.", + "aliases": ["Комариное", "Комариное"], + "fish_species": ["Щука", "Окунь"], + "image_urls": ["https://oss.rf4db.com/map.webp"], + "point_urls": ["https://rf4db.com/ru/maps/level_001_mosquito/spots/12-34"], + }) is True + item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito")) + assert item is not None + assert item.source_aliases == ["Комариное"] + assert item.source_fish_species == ["Щука", "Окунь"] + assert item.source_image_urls == ["https://oss.rf4db.com/map.webp"] + + +def test_waterbody_detail_batch_validates_before_writing(db: Session) -> None: + upsert_waterbody_catalog(db, [waterbody_row()]) + valid = { + "source_system": "rf4db", "source_external_id": "level_001_mosquito", + "source_url": "https://rf4db.com/ru/maps/level_001_mosquito", "name": "оз. Комариное", + "description": "Описание", "aliases": [], "fish_species": ["Щука"], + "image_urls": [], "point_urls": [], + } + invalid = valid | {"source_external_id": "unknown", "source_url": "https://example.test/map"} + with pytest.raises(CommunityImportError, match="source_url"): + update_waterbody_details(db, [valid, invalid]) + item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito")) + assert item is not None and item.description is None + + def test_external_ids_are_isolated_by_source(db: Session) -> None: created, updated = stage_observations(db, [record("rf4db"), record("rf4stat-fishing")]) @@ -104,14 +174,28 @@ def test_complete_observation_with_reviewed_aliases_is_published(db: Session) -> assert item.catch_report is not None assert item.catch_report.fish_id == fish.id assert item.catch_report.waterbody_id == waterbody.id - assert db.scalar(select(func.count()).select_from(CatchReport)) == 1 - assert stage_observations(db, [record() | {"weight_g": 5_000}]) == (0, 1) db.refresh(item) assert item.status == "published" assert db.scalar(select(func.count()).select_from(CatchReport)) == 1 +def test_observation_preserves_coordinate_text_and_precision(db: Session) -> None: + stage_observations(db, [record() | { + "source_external_id": "coordinate-area", + "x": None, "y": None, "coordinate_raw": "северная бухта", + "coordinate_precision": "area", "weight_g": None, + }]) + item = db.scalar(select(ExternalObservation).where(ExternalObservation.source_external_id == "coordinate-area")) + assert item is not None + assert (item.coordinate_raw, item.coordinate_precision, item.x, item.y) == ("северная бухта", "area", None, None) + + +def test_coordinate_precision_rejects_unknown_value(db: Session) -> None: + with pytest.raises(CommunityImportError, match="invalid coordinate_precision"): + stage_observations(db, [record() | {"coordinate_precision": "guess"}]) + + def test_changed_published_record_requires_review_and_reuses_report(db: Session) -> None: fish = Fish(slug="pike", name_ru="Щука") water = Waterbody(slug="test-lake", name_ru="Тестовое озеро") diff --git a/apps/web/src/components/ActivityCard.astro b/apps/web/src/components/ActivityCard.astro index 299c5c1..51147f0 100644 --- a/apps/web/src/components/ActivityCard.astro +++ b/apps/web/src/components/ActivityCard.astro @@ -8,6 +8,7 @@ import TackleGlyph from "./TackleGlyph.astro"; const { item } = Astro.props as { item: Activity }; const level = activityLevel(item.activity_score); const limited = item.catches < 3; +const precision = { exact: "точные", approximate: "приблизительные", area: "район", missing: "не указаны" }[item.coordinate_precision]; --- {String(item.activity_score).padStart(2,"0")} @@ -15,7 +16,7 @@ const limited = item.catches < 3;
{item.waterbody}{level.short}{limited && Данных мало}

{item.fish}

-
{item.x}:{item.y} {ago(item.last_confirmed_at)}
+
{item.x}:{item.y} · {precision} {ago(item.last_confirmed_at)}

{item.explanation}

Работает сейчас{item.best_bait ?? "не указана"}
diff --git a/apps/web/src/components/EntityMedia.astro b/apps/web/src/components/EntityMedia.astro index 7764cc6..82d3bdd 100644 --- a/apps/web/src/components/EntityMedia.astro +++ b/apps/web/src/components/EntityMedia.astro @@ -4,6 +4,12 @@ import type { MediaAsset } from "../lib/api"; const { asset, compact = false, sourceLink = false } = Astro.props as { asset: MediaAsset; compact?: boolean; sourceLink?: boolean }; ---
- {asset.label + + {(["avif", "webp"] as const).map(format => { + const variants = (asset.variants ?? []).filter(item => item.format === format); + return variants.length ? `${item.url} ${item.width}w`).join(", ")} sizes={compact ? "180px" : "(max-width: 720px) 100vw, 720px"} /> : null; + })} + {asset.label +
{asset.label ?? "Справочный материал"}
diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 8f69970..bd5951a 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -4,6 +4,7 @@ export type Activity = { unique_players: number; average_weight_g: number; max_weight_g: number; last_confirmed_at: string; activity_score: number; confidence_score: number; explanation: string; sources: string[]; + coordinate_precision: "exact" | "approximate" | "area" | "missing"; coordinate_sources: string[]; }; export type PaginatedActivity = { @@ -13,9 +14,9 @@ export type PaginatedActivity = { offset: number; }; -export type Spot = { id: string; waterbody_slug: string; waterbody: string; x: number; y: number; description: string | null; catches_24h: number; catches_3d: number; catches_7d: number; top_baits: string[] }; +export type Spot = { id: string; waterbody_slug: string; waterbody: string; x: number; y: number; description: string | null; catches_24h: number; catches_3d: number; catches_7d: number; top_baits: string[]; coordinate_precision: "exact" | "approximate" | "area" | "missing"; coordinate_sources: string[] }; export type Catch = { id: string; fish: string; weight_g: number; bait: string | null; player_name: string | null; caught_at: string | null; reported_at: string; retrieve_method: string | null; retrieve_speed: number | null; source_system: string; source_url: string | null }; -export type DictionaryItem = { id: string; slug: string; name_ru: string }; +export type DictionaryItem = { id: string; slug: string; name_ru: string; unlock_level?: number | null; fish_species_count?: number | null; source_system?: string | null; source_external_id?: string | null; source_url?: string | null; description?: string | null; source_aliases?: string[] | null; source_fish_species?: string[] | null; source_image_urls?: string[] | null; source_point_urls?: string[] | null; source_checked_at?: string | null }; export type OfficialRecord = { id: string; fish: string; weight_g: number; waterbody: string; bait: string | null; player_name: string | null; record_date: string | null; category: string | null; region: string | null; source_url: string | null; source_system: string }; export type PaginatedOfficialRecord = { items: OfficialRecord[]; @@ -26,7 +27,8 @@ export type PaginatedOfficialRecord = { export type PublicObservation = { id: string; source_system: string; source_name: string; source_url: string; fish_name: string; waterbody_name: string; x: number | null; y: number | null; weight_g: number | null; last_seen_at: string; missing_fields: string[]; quality: "incomplete" | "unverified" }; export type ImportRun = { id: string; started_at: string; finished_at: string | null; status: string; source_url: string; rows_seen: number; rows_created: number; rows_updated: number; error_summary: string | null }; export type SourceStatus = { source_system: string; name: string; status: "healthy" | "stale" | "temporarily_limited" | "source_changed" | "waiting" | "disabled"; last_started_at: string | null; last_success_at: string | null; observations: number }; -export type MediaAsset = { id: string; entity_type: "fish" | "waterbody" | "tackle" | "reference"; entity_key: string; label: string | null; width: number; height: number; content_type: string; image_url: string; source_system: string; source_url: string }; +export type MediaVariant = { role: "card" | "detail"; format: "webp" | "avif"; width: number; height: number; url: string }; +export type MediaAsset = { id: string; entity_type: "fish" | "waterbody" | "tackle" | "reference"; entity_key: string; label: string | null; width: number; height: number; content_type: string; image_url: string; source_system: string; source_url: string; variants?: MediaVariant[] }; export const spotPath = (item: Pick) => `/spots/${item.waterbody_slug}-${item.x}x${item.y}`; diff --git a/apps/web/src/pages/fish/index.astro b/apps/web/src/pages/fish/index.astro index 3d66009..26fa668 100644 --- a/apps/web/src/pages/fish/index.astro +++ b/apps/web/src/pages/fish/index.astro @@ -4,10 +4,19 @@ import FishSilhouette from "../../components/FishSilhouette.astro"; import EntityMedia from "../../components/EntityMedia.astro"; import PageHero from "../../components/PageHero.astro"; import StatePanel from "../../components/StatePanel.astro"; +import Pagination from "../../components/Pagination.astro"; import { api, type DictionaryItem, type MediaAsset } from "../../lib/api"; import { findMediaByLabel } from "../../lib/media"; -let fishes: DictionaryItem[] = [], media: MediaAsset[] = [], unavailable = false; -try { [fishes, media] = await Promise.all([api("/api/v1/fishes?limit=500"), api("/api/v1/media/catalog?entity_type=fish")]); } catch { unavailable = true; } +const params = Astro.url.searchParams; +const pageLimit = 48; +const requestedOffset = Number(params.get("offset") ?? 0); +let offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; +let fishes: DictionaryItem[] = [], allFishes: DictionaryItem[] = [], media: MediaAsset[] = [], unavailable = false; +try { + [allFishes, media] = await Promise.all([api("/api/v1/fishes?limit=500"), api("/api/v1/media/catalog?entity_type=fish")]); + if (allFishes.length > 0 && offset >= allFishes.length) offset = Math.floor((allFishes.length - 1) / pageLimit) * pageLimit; + fishes = allFishes.slice(offset, offset + pageLimit); +} catch { unavailable = true; } if (unavailable) { Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); @@ -15,6 +24,7 @@ if (unavailable) { } --- - - {unavailable ? : } + + {unavailable ? : } + {!unavailable && } diff --git a/apps/web/src/pages/index.astro b/apps/web/src/pages/index.astro index fe82f0d..cff15dc 100644 --- a/apps/web/src/pages/index.astro +++ b/apps/web/src/pages/index.astro @@ -6,6 +6,7 @@ import SourceBadge from "../components/SourceBadge.astro"; import TackleGlyph from "../components/TackleGlyph.astro"; import SignalFeed from "../components/SignalFeed.astro"; import StatePanel from "../components/StatePanel.astro"; +import Pagination from "../components/Pagination.astro"; import { activityLevel, ago, api, kg, plural, type Activity, type DictionaryItem, type PaginatedActivity, type PublicObservation } from "../lib/api"; const params = Astro.url.searchParams; @@ -13,8 +14,9 @@ const hours = params.get("hours") ?? "24"; const waterbody = params.get("waterbody") ?? ""; const fish = params.get("fish") ?? ""; const sort = params.get("sort") ?? "activity"; +const activityLimit = 20; const requestedOffset = Number(params.get("offset") ?? 0); -const offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; +let offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; const requestedSignalLimit = Number(params.get("signals") ?? 12); const signalLimit = Number.isInteger(requestedSignalLimit) ? Math.min(48, Math.max(12, requestedSignalLimit)) : 12; let items: Activity[] = [], signals: PublicObservation[] = [], fishes: DictionaryItem[] = [], waterbodies: DictionaryItem[] = []; @@ -38,11 +40,16 @@ if (!signalsUnavailable) { signals = signalRows.slice(0, signalLimit); } if (!filterError) { - const query = new URLSearchParams({ hours, waterbody, fish, sort, limit: "20", offset: String(offset) }); + const query = new URLSearchParams({ hours, waterbody, fish, sort, limit: String(activityLimit), offset: String(offset) }); try { const paginated = await api(`/api/v1/activity?${query}`); - items = offset > 0 ? [...items, ...paginated.items] : paginated.items; + items = paginated.items; totalItems = paginated.total; + if (totalItems > 0 && offset >= totalItems) { + offset = Math.floor((totalItems - 1) / activityLimit) * activityLimit; + query.set("offset", String(offset)); + items = (await api(`/api/v1/activity?${query}`)).items; + } } catch { activityUnavailable = true; } } const catalogUnavailable = fishCatalogUnavailable || waterCatalogUnavailable; @@ -86,9 +93,10 @@ const datasetJsonLd = {
{selectedWaterbody}{selectedFish}{periodLabel}{sortLabel}{filtersChanged && Сбросить}
-
За выбранный период

Горячие точки

{items.length} из {totalItems} {plural(totalItems, ["точка", "точки", "точек"])}
{filterError ?

Некорректные фильтры

Выберите период и сортировку из предложенных значений.

Сбросить фильтры
: activityUnavailable ? : items.length ? <>
{items.map(item => )}
{offset + items.length < totalItems && { const p = new URLSearchParams(params); p.delete("offset"); p.set("offset", String(offset + items.length)); return p.toString(); })()}#results`}>Показать ещё {offset + items.length} из {totalItems} ↓} :

Пока нет свежих данных

Для выбранных фильтров нет одобренных наблюдений. Расширьте период или выберите другой водоём.

}
+
За выбранный период

Горячие точки

{items.length} на странице · всего {totalItems}
{filterError ?

Некорректные фильтры

Выберите период и сортировку из предложенных значений.

Сбросить фильтры
: activityUnavailable ? : items.length ?
{items.map(item => )}
:

Пока нет свежих данных

Для выбранных фильтров нет одобренных наблюдений. Расширьте период или выберите другой водоём.

}
{items[0] && leaderLevel && }
+ {!filterError && !activityUnavailable && } {signals.length > 0 && } {signalsUnavailable &&
} {hasMoreSignals && signalLimit < 48 && Показать ещё {signalLimit} из доступных ↓} diff --git a/apps/web/src/pages/records.astro b/apps/web/src/pages/records.astro index 342244e..cf485c5 100644 --- a/apps/web/src/pages/records.astro +++ b/apps/web/src/pages/records.astro @@ -4,19 +4,26 @@ import SourceBadge from "../components/SourceBadge.astro"; import SectionHeading from "../components/SectionHeading.astro"; import StatePanel from "../components/StatePanel.astro"; import TackleGlyph from "../components/TackleGlyph.astro"; +import Pagination from "../components/Pagination.astro"; import { api, kg, type DictionaryItem, type ImportRun, type OfficialRecord, type PaginatedOfficialRecord } from "../lib/api"; const params = Astro.url.searchParams; const fish = params.get("fish") ?? ""; const waterbody = params.get("waterbody") ?? ""; +const recordsLimit = 50; const requestedOffset = Number(params.get("offset") ?? 0); -const offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; +let offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; let items: OfficialRecord[] = [], runs: ImportRun[] = [], fishes: DictionaryItem[] = [], waterbodies: DictionaryItem[] = []; let unavailable = false, showNoIndex = false, totalRecords = 0; try { - const query = new URLSearchParams({ fish, waterbody, limit: "50", offset: String(offset) }); + const query = new URLSearchParams({ fish, waterbody, limit: String(recordsLimit), offset: String(offset) }); const paginated = await api(`/api/v1/records?${query}`); - items = offset > 0 ? [...items, ...paginated.items] : paginated.items; + items = paginated.items; totalRecords = paginated.total; + if (totalRecords > 0 && offset >= totalRecords) { + offset = Math.floor((totalRecords - 1) / recordsLimit) * recordsLimit; + query.set("offset", String(offset)); + items = (await api(`/api/v1/records?${query}`)).items; + } [runs, fishes, waterbodies] = await Promise.all([api("/api/v1/imports?limit=1"), api("/api/v1/fishes"), api("/api/v1/waterbodies")]); } catch { unavailable = true; showNoIndex = true; Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); } const last = runs[0]; @@ -24,10 +31,10 @@ const last = runs[0];
Публичные данные RF4

Официальные
рекорды

{last ? `Импорт: ${last.status}` : "Импорт ещё не запускался"}{last?.finished_at && {new Date(last.finished_at).toLocaleString("ru-RU")} · {last.rows_seen} строк}
{(fish || waterbody) && Сбросить}
-
+
{unavailable && } {!unavailable && items.length &&
РыбаВесВодоёмПриманкаИгрокДата и источник
{items.map(record =>
{record.fish}{kg(record.weight_g)}{record.waterbody}{record.bait ?? "—"}{record.player_name ?? "—"}
)}
} - {!unavailable && items.length && offset + items.length < totalRecords && { const p = new URLSearchParams(params); p.delete("offset"); p.set("offset", String(offset + items.length)); return p.toString(); })()}`}>Показать ещё {offset + items.length} из {totalRecords} ↓} + {!unavailable && } {!unavailable && !items.length && }

Источник: официальный сайт Russian Fishing 4. Координаты в официальных таблицах отсутствуют.

 diff --git a/apps/web/src/pages/waterbodies/[slug].astro b/apps/web/src/pages/waterbodies/[slug].astro index 41c33dd..134dcd7 100644 --- a/apps/web/src/pages/waterbodies/[slug].astro +++ b/apps/web/src/pages/waterbodies/[slug].astro @@ -30,5 +30,6 @@ const schema = water ? { "@context":"https://schema.org", "@type":"CollectionPag {image &&
Карта и образ водоёма

{water!.name_ru}

Материал показан с прямой атрибуцией. Координаты активных точек ниже относятся к данным наблюдений, а не к геометрии изображения.

} + {water && (water.description || water.unlock_level !== undefined || water.source_url || water.source_fish_species) &&
Канонические сведения

{water.name_ru}

{water.description &&

{water.description}

}{water.unlock_level !== undefined && water.unlock_level !== null &&

Открывается с уровня: {water.unlock_level}

}{water.source_fish_species?.length &&

В карточке источника указано видов рыб: {water.source_fish_species.length}.

}{water.source_aliases?.length &&

Алиасы источника: {water.source_aliases.join(", ")}.

}
Источник{water.source_url ?

{water.source_system ?? "Внешний источник"} · исходная карточка

:

Источник для описания ещё не подтверждён.

}{water.source_point_urls?.length &&

Сохранено ссылок на точки: {water.source_point_urls.length}.

}{water.source_image_urls?.length &&

Изображений-кандидатов: {water.source_image_urls.length}; публикация требует отдельной проверки.

}{water.source_checked_at &&

Проверено: {new Date(water.source_checked_at).toLocaleDateString("ru-RU")}

}
} {unavailable ? : !water ? :
{items.length ? items.map(item => ) : }
} diff --git a/apps/web/src/pages/waterbodies/index.astro b/apps/web/src/pages/waterbodies/index.astro index 4f05390..72419c3 100644 --- a/apps/web/src/pages/waterbodies/index.astro +++ b/apps/web/src/pages/waterbodies/index.astro @@ -3,9 +3,18 @@ import Layout from "../../layouts/Layout.astro"; import PageHero from "../../components/PageHero.astro"; import StatePanel from "../../components/StatePanel.astro"; import WaterbodyMark from "../../components/WaterbodyMark.astro"; +import Pagination from "../../components/Pagination.astro"; import { api, type DictionaryItem } from "../../lib/api"; -let waters: DictionaryItem[] = [], unavailable = false; -try { waters = await api("/api/v1/waterbodies?limit=500"); } catch { unavailable = true; } +const params = Astro.url.searchParams; +const pageLimit = 48; +const requestedOffset = Number(params.get("offset") ?? 0); +let offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; +let waters: DictionaryItem[] = [], allWaters: DictionaryItem[] = [], unavailable = false; +try { + allWaters = await api("/api/v1/waterbodies?limit=500"); + if (allWaters.length > 0 && offset >= allWaters.length) offset = Math.floor((allWaters.length - 1) / pageLimit) * pageLimit; + waters = allWaters.slice(offset, offset + pageLimit); +} catch { unavailable = true; } if (unavailable) { Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); @@ -13,6 +22,7 @@ if (unavailable) { } --- - - {unavailable ? : } + + {unavailable ? : } + {!unavailable && } diff --git a/compose.yaml b/compose.yaml index 6f6ca4f..006442a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,6 +30,23 @@ services: volumes: - minio_data:/data + minio-init: + image: minio/mc:RELEASE.2025-07-21T05-28-08Z + restart: "no" + entrypoint: ["/bin/sh", "-c"] + command: + - >- + mc alias set local http://minio:9000 "$$S3_ACCESS_KEY" "$$S3_SECRET_KEY" >/dev/null && + mc mb --ignore-existing "local/$$S3_BUCKET" >/dev/null && + mc stat "local/$$S3_BUCKET" >/dev/null + environment: + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-rf4-local} + S3_SECRET_KEY: ${S3_SECRET_KEY:-rf4-local-secret} + S3_BUCKET: ${S3_BUCKET:-catch-screenshots} + depends_on: + minio: + condition: service_healthy + api: build: context: . @@ -55,6 +72,8 @@ services: condition: service_completed_successfully minio: condition: service_healthy + minio-init: + condition: service_completed_successfully ports: - "8000:8000" healthcheck: diff --git a/docs/RECOVERY_PLAN_2026-09-10.md b/docs/RECOVERY_PLAN_2026-09-10.md index 480cdf5..07df524 100644 --- a/docs/RECOVERY_PLAN_2026-09-10.md +++ b/docs/RECOVERY_PLAN_2026-09-10.md @@ -8,7 +8,14 @@ Проверено: 107 Python-тестов проходят, 1 пропущен; Astro check/build, web unit и Caddy adapt проходят. Это не полная приёмка: проверки не покрывают обнаруженные ниже сценарии. Несовместимость activity envelope исправлена у четырёх потребителей по коду; история site cooldown теперь включает disabled endpoint. Не повторять эти изменения без нового воспроизведения. -`REGRESSION_FIXES_REPORT.md` устарел, содержит противоречивые статусы. Новые задачи ниже пока не выполнены. Источники в сеть для этой проверки не опрашивались. +Ниже сохранён исходный снимок критериев до исправлений; формулировки «не выполнено» и секция «основание» описывают состояние базы `4f68d6b`, а не текущую ветку. По повторной приёмке 11 сентября A01–A13 закрыты в [RECOVERY_FIXES_REPORT.md](RECOVERY_FIXES_REPORT.md). 15 сентября пагинация A04 дополнительно переведена с вводящей в заблуждение кнопки «Показать ещё» на серверные страницы previous/next для activity, records и каталогов; фильтры сохраняются, избыточный offset нормализуется на последнюю страницу, сценарий 45 элементов закреплён unit-тестом. Источники в сеть для этой проверки не опрашивались. + +| Исторический пакет | Текущий статус | Актуальное подтверждение | +|---|---|---| +| A01–A03 | закрыт | `RECOVERY_FIXES_REPORT.md`, тесты readiness/cooldown/redirect | +| A04 | закрыт, повторно усилен 15.09 | серверные previous/next, диапазон и unit-тест 45 элементов | +| A05–A12 | закрыт | `RECOVERY_FIXES_REPORT.md`, адресные regression-тесты | +| A13 | закрыт | итоговый отчёт и единственная активная очередь в `ROADMAP.md` | ## Порядок и критерии приёмки diff --git a/docs/REGRESSION_FIXES_REPORT.md b/docs/REGRESSION_FIXES_REPORT.md index 17fa38e..98e6cd3 100644 --- a/docs/REGRESSION_FIXES_REPORT.md +++ b/docs/REGRESSION_FIXES_REPORT.md @@ -1,6 +1,6 @@ # Отчёт по регрессионному аудиту — 9 сентября 2026 -> **Архивный промежуточный отчёт.** Его раздел «оставшиеся регрессии» закрыт последующим пакетом восстановления. Актуальный итог находится в [RECOVERY_FIXES_REPORT.md](RECOVERY_FIXES_REPORT.md), текущие задачи — в [ROADMAP.md](ROADMAP.md). +> **Архивный промежуточный отчёт.** Его статусы и раздел «Следующие шаги» относятся только к базе `9ae05ef` и не являются текущим backlog. Регрессии закрыты последующим пакетом восстановления; актуальный итог находится в [RECOVERY_FIXES_REPORT.md](RECOVERY_FIXES_REPORT.md), текущие задачи — только в [ROADMAP.md](ROADMAP.md). Пагинация R09 дополнительно усилена 15 сентября полноценными серверными previous/next-страницами. База: `9ae05ef` (после трёх коммитов исправлений предыдущего аудита). Исходный отчёт: [REGRESSION_AUDIT_2026-09-09.md](REGRESSION_AUDIT_2026-09-09.md). @@ -169,7 +169,7 @@ --- -## Следующие шаги +## Исторические следующие шаги (закрыты последующим recovery-пакетом) 1. **R03** — Research CLI cooldown: исправить `fcntl` locking и `encoding`. 2. **R04** — Разделить `configured_sources()` и `site_cooldown_sources()`. diff --git a/rf4_research/community_cli.py b/rf4_research/community_cli.py index 2c770db..246d387 100644 --- a/rf4_research/community_cli.py +++ b/rf4_research/community_cli.py @@ -16,6 +16,7 @@ from urllib.request import Request, urlopen, HTTPRedirectHandler, build_opener from .community_sources import ( parse_rf4db_catches, parse_rf4db_waterbodies, + parse_rf4db_waterbody_detail, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, @@ -30,6 +31,7 @@ SOURCES = { "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts), } DETAIL_SOURCES = { + "rf4db-waterbody": parse_rf4db_waterbody_detail, "rf4map-point": parse_rf4map_point, "rf4posts-spot": parse_rf4posts_spot, } diff --git a/rf4_research/community_sources.py b/rf4_research/community_sources.py index ca11f4d..7c8db78 100644 --- a/rf4_research/community_sources.py +++ b/rf4_research/community_sources.py @@ -35,6 +35,7 @@ class ExternalCatch: clip: str | None fishing_style: str | None evidence_urls: tuple[str, ...] + coordinate_raw: str | None = None @dataclass(frozen=True, slots=True) @@ -67,10 +68,29 @@ class RF4DBWaterbody: image_url: str | None +@dataclass(frozen=True, slots=True) +class RF4DBWaterbodyDetail: + source_system: str + source_external_id: str + source_url: str + name: str + description: str | None + aliases: tuple[str, ...] + fish_species: tuple[str, ...] + fish_external_ids: tuple[str | None, ...] + image_urls: tuple[str, ...] + point_urls: tuple[str, ...] + + def _text(node: Tag | None) -> str: return " ".join(node.get_text(" ", strip=True).split()) if node else "" +def _fish_name(node: Tag) -> str: + """Read a fish label while dropping the optional trophy weight suffix.""" + return re.sub(r"\s*\d+(?:[.,]\d+)?\s*(?:кг|kg|г|g)\s*$", "", _text(node), flags=re.I).strip() + + def _key(href: str | None) -> str | None: if not href: return None @@ -189,6 +209,63 @@ def parse_rf4db_waterbodies( return result +def parse_rf4db_waterbody_detail( + html: str, *, source_url: str, +) -> RF4DBWaterbodyDetail: + """Parse one RF4DB waterbody page without assigning media or coordinates. + + The detail page is accepted only when its localized heading and fish list + are present. Images and point links remain source candidates; review and + canonical crosswalks happen in later pipeline stages. + """ + soup = BeautifulSoup(html, "html.parser") + root = soup.select_one("article.waterbody-detail, main[data-waterbody-detail]") or soup + external_id = _key(source_url) + name = _text(root.select_one("h1")) + fish_nodes = root.select(".waterbody-fish a[href], [data-fish-list] a[href], a[href*='/fishes/']") + if not fish_nodes: + fish_nodes = root.select(".fish-list a[href], ul.fish a[href]") + fish_species: list[str] = [] + fish_external_ids: list[str | None] = [] + seen_fish: set[str] = set() + for node in fish_nodes: + fish_name = _fish_name(node) + fish_id = _key(node.get("href")) + identity = fish_id or fish_name.casefold() + if not fish_name or identity in seen_fish: + continue + fish_species.append(fish_name) + fish_external_ids.append(fish_id) + seen_fish.add(identity) + if not external_id or not name or not fish_species: + raise CommunityParseError("RF4DB waterbody detail not found or incomplete") + + description_node = root.select_one("[data-description], .waterbody-description, .description") + description = _text(description_node) or None + aliases = tuple(dict.fromkeys( + _text(node) for node in root.select("[data-alias], .waterbody-aliases li, .aliases li") + if _text(node) and _text(node) != name + )) + image_urls = tuple(dict.fromkeys( + urljoin(source_url, str(node.get("src") or node.get("data-src"))) + for node in root.select("img[src], img[data-src]") + if (node.get("src") or node.get("data-src")) + and "/fish/" not in str(node.get("src") or node.get("data-src")) + )) + point_urls = tuple(dict.fromkeys( + urljoin(source_url, str(node.get("href"))) + for node in root.select('a[href*="/spots/"], a[href*="/points/"]') + if node.get("href") + )) + return RF4DBWaterbodyDetail( + source_system="rf4db", source_external_id=external_id, + source_url=source_url, name=name, description=description, + aliases=aliases, fish_species=tuple(fish_species), + fish_external_ids=tuple(fish_external_ids), image_urls=image_urls, + point_urls=point_urls, + ) + + def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> list[ExternalCatch]: soup = BeautifulSoup(html, "html.parser") result: list[ExternalCatch] = [] @@ -201,7 +278,8 @@ def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> li external_id = _key(detail.get("href")) if not external_id: continue - x, y = _coordinates(_text(card.select_one(".catch-card__place b"))) + coordinate_raw = _text(card.select_one(".catch-card__place b")) or None + x, y = _coordinates(coordinate_raw or "") bait_link = card.select_one('.catch-card__place a[href*="/wiki/baits/"]') badges = card.select(".catch-badge") weather = next((_text(b.select_one("b")) for b in badges if _text(b).startswith("Погода")), None) @@ -218,7 +296,7 @@ def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> li game_time=_game_time(_text(card.select_one(".catch-card__place"))), published_at=None, player_name=None, weather=weather or None, water_temperature_c=float(temperature_match.group().replace(",", ".")) if temperature_match else None, - clip=None, fishing_style=None, evidence_urls=(), + clip=None, fishing_style=None, evidence_urls=(), coordinate_raw=coordinate_raw, )) if not result: raise CommunityParseError("RF4DB catch cards not found") @@ -288,6 +366,7 @@ def parse_rf4stat_fishing( if not external_id: continue position = row.select_one(".list-position .position:not(.position-locked)") + coordinate_raw = _text(row.select_one(".list-position .position")) or None x, y = _coordinates(_text(position)) style = row.select_one(".post-style-icon[title]") style_text = str(style.get("title", "")) if style else "" @@ -305,6 +384,7 @@ def parse_rf4stat_fishing( clip=_text(row.select_one(".clip")) or None, fishing_style=style_text.removeprefix("Вид ловли:").strip() or None, evidence_urls=(urljoin(base_url, str(row.select_one("a.share.hide-print").get("href"))),) if row.select_one("a.share.hide-print") else (), + coordinate_raw=coordinate_raw, )) if not result: raise CommunityParseError("RF4-STAT fishing rows not found") @@ -323,6 +403,7 @@ def parse_rf4stat_posts( published_raw = str(post.get("data-published-at", "")) published_at = datetime.fromtimestamp(int(published_raw), tz=timezone.utc) if published_raw.isdigit() else None position = post.select_one(".spot-col .position:not(.position-locked)") + coordinate_raw = _text(post.select_one(".spot-col .position")) or None x, y = _coordinates(_text(position)) style = post.select_one(".post-style-icon[title]") style_text = str(style.get("title", "")) if style else "" @@ -344,7 +425,7 @@ def parse_rf4stat_posts( weather=None, water_temperature_c=None, clip=_text(post.select_one(".clip")) or None, fishing_style=style_text.removeprefix("Вид ловли:").strip() or None, - evidence_urls=evidence, + evidence_urls=evidence, coordinate_raw=coordinate_raw, )) if not result: raise CommunityParseError("RF4-STAT posts not found") @@ -386,6 +467,7 @@ def parse_rf4map_point(html: str, *, source_url: str) -> list[ExternalCatch]: clip=str(item["clip"]) if isinstance(item.get("clip"), (int, float)) else None, fishing_style=None, evidence_urls=tuple(url for url in evidence or [] if isinstance(url, str)), + coordinate_raw=f"{item['positionX']}:{item['positionY']}" if isinstance(item.get("positionX"), int) and isinstance(item.get("positionY"), int) else None, )) if not result: raise CommunityParseError("RF4MAP point observations not found") @@ -428,7 +510,7 @@ def parse_rf4posts_spot(html: str, *, source_url: str) -> list[ExternalCatch]: weather=None, water_temperature_c=None, clip=str(spot["clip"]) if isinstance(spot.get("clip"), (int, float)) else None, fishing_style=str(spot["tackleType"]) if isinstance(spot.get("tackleType"), str) else None, - evidence_urls=evidence, + evidence_urls=evidence, coordinate_raw=str(spot["coordinates"]) if isinstance(spot.get("coordinates"), str) else None, )) if not result: raise CommunityParseError("RF4 Posts fish species not found") diff --git a/tests/test_community_sources.py b/tests/test_community_sources.py index c8f860f..a4159c0 100644 --- a/tests/test_community_sources.py +++ b/tests/test_community_sources.py @@ -8,11 +8,16 @@ from rf4_research.community_sources import ( parse_rf4db_catches, parse_rf4db_detail, parse_rf4db_waterbodies, + parse_rf4db_waterbody_detail, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, parse_rf4stat_posts, ) +from rf4_research.waterbody_crosswalk import ( + CanonicalWaterbody, WaterbodyIdentity, missing_external_ids, + suggest_waterbody_crosswalk, +) FIXTURES = Path(__file__).parent / "fixtures" @@ -74,6 +79,65 @@ def test_rf4db_waterbody_catalog_rejects_partial_results() -> None: parse_rf4db_waterbodies(fixture("rf4db_waterbodies_sample.html")) +def test_parses_rf4db_waterbody_detail_without_assigning_media_roles() -> None: + row = parse_rf4db_waterbody_detail( + fixture("rf4db_waterbody_detail_sample.html"), + source_url="https://rf4db.com/ru/maps/level_001_mosquito", + ) + + assert (row.source_external_id, row.name) == ("level_001_mosquito", "оз. Комариное") + assert row.description == "Большое озеро с каменистыми берегами." + assert row.aliases == ("Комариное",) + assert row.fish_species == ("Щука", "Окунь") + assert row.fish_external_ids == ("pike", "perch") + assert row.image_urls == ( + "https://oss.rf4db.com/game/maps/level_001_mosquito.webp", + "https://rf4db.com/game/maps/level_001_mosquito-depth.webp", + ) + assert row.point_urls == ("https://rf4db.com/ru/maps/level_001_mosquito/spots/12-34",) + + +def test_parses_live_waterbody_fish_links_without_fish_media() -> None: + row = parse_rf4db_waterbody_detail( + fixture("rf4db_waterbody_detail_live_sample.html"), + source_url="https://rf4db.com/ru/maps/level_000_home", + ) + assert row.fish_species == ("Лягушка", "Ротан", "Окунь") + assert row.fish_external_ids == ("frog", "a.sleeper", "perch") + assert row.image_urls == ( + "https://oss.rf4db.com/game/maps/level_000_home.png", + "https://oss.rf4db.com/game/paper_maps_webp/map_level_000_home.webp", + ) + + +def test_rf4db_waterbody_detail_rejects_missing_fish_list() -> None: + with pytest.raises(CommunityParseError, match="detail not found or incomplete"): + parse_rf4db_waterbody_detail( + "

Озеро

", + source_url="https://rf4db.com/ru/maps/level_001_mosquito", + ) + + +def test_waterbody_crosswalk_only_suggests_unique_exact_matches() -> None: + canonical = [ + CanonicalWaterbody("mosquito", "оз. Комариное", ("Комариное",)), + CanonicalWaterbody("ambiguous-a", "Большое озеро"), + CanonicalWaterbody("ambiguous-b", "Другое", ("Большое озеро",)), + ] + rows = suggest_waterbody_crosswalk(canonical, [ + WaterbodyIdentity("rf4db", "level_001_mosquito", "КОМАРИНОЕ"), + WaterbodyIdentity("rf4map", "16", "Большое озеро"), + WaterbodyIdentity("rf4-posts", "lake-x", "Норвежское море"), + ]) + assert [(row.status, row.canonical_keys) for row in rows] == [ + ("exact", ("mosquito",)), ("ambiguous", ()), ("unmatched", ()), + ] + + +def test_missing_source_ids_are_diagnostic_not_withdrawals() -> None: + assert missing_external_ids({"16", "17", "18"}, {"16"}) == ("17", "18") + + def test_parses_rf4stat_fishing_rows() -> None: row = parse_rf4stat_fishing( fixture("rf4stat_fishing_sample.html"), @@ -82,6 +146,7 @@ def test_parses_rf4stat_fishing_rows() -> None: assert (row.source_system, row.source_external_id) == ("rf4stat-fishing", "123") assert (row.x, row.y, row.weight_g) == (71, 92, 11_584) + assert row.coordinate_raw == "71:92" assert row.published_at == datetime(2026, 9, 3, 11, 27, tzinfo=timezone.utc) assert (row.player_name, row.clip, row.fishing_style) == ("Игрок", "35", "Донная") @@ -91,6 +156,7 @@ def test_parses_rf4stat_posts_without_using_locked_coordinates() -> None: assert (row.source_system, row.source_external_id) == ("rf4stat-post", "456:0") assert (row.x, row.y, row.weight_g) == (None, None, 4_321) + assert row.coordinate_raw == "XX:XX" assert row.evidence_urls == ("https://img.example.test/proof.jpg",)