From e94d247096bc535d195de452860ba3a51806bbc4 Mon Sep 17 00:00:00 2001 From: IK Date: Sun, 20 Sep 2026 18:26:53 +0700 Subject: [PATCH] feat: add tackle catalog and recommendation analytics --- apps/api/app/main.py | 2 + apps/api/app/routers/activity.py | 11 ++-- apps/api/app/routers/analytics.py | 66 +++++++++++++++++++++++ apps/api/app/routers/catalog.py | 73 ++++++++++++++++++++++++-- apps/api/app/schemas.py | 64 ++++++++++++++++++++++ apps/api/tests/test_analytics.py | 39 ++++++++++++++ apps/api/tests/test_tackle_catalog.py | 32 +++++++++++ apps/web/src/lib/api.ts | 2 + apps/web/src/pages/tackle/index.astro | 40 ++++++++++++++ docs/ROADMAP.md | 4 +- scripts/fetch-waterbodies-chromium.mjs | 28 +++++++++- 11 files changed, 350 insertions(+), 11 deletions(-) create mode 100644 apps/api/app/routers/analytics.py create mode 100644 apps/api/tests/test_analytics.py create mode 100644 apps/api/tests/test_tackle_catalog.py create mode 100644 apps/web/src/pages/tackle/index.astro diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 7fcae13..7c4cac7 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -14,6 +14,7 @@ from .dependencies import Db from .logging_config import configure_logging from .readiness import readiness_report from .routers.activity import router as activity_router +from .routers.analytics import router as analytics_router from .routers.admin import router as admin_router from .routers.catalog import router as catalog_router from .routers.media import router as media_router @@ -90,6 +91,7 @@ def ready(db: Db) -> JSONResponse: app.include_router(catalog_router) app.include_router(media_router) app.include_router(activity_router) +app.include_router(analytics_router) app.include_router(public_data_router) app.include_router(admin_router) app.include_router(submissions_router) diff --git a/apps/api/app/routers/activity.py b/apps/api/app/routers/activity.py index 175ec75..d04b15d 100644 --- a/apps/api/app/routers/activity.py +++ b/apps/api/app/routers/activity.py @@ -5,7 +5,7 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Query, Response from sqlalchemy import func, select -from sqlalchemy.orm import Session, joinedload +from sqlalchemy.orm import Session, joinedload, selectinload from ..activity import activity_rows from ..config import settings @@ -91,8 +91,13 @@ def _report_source(report: CatchReport) -> str: @router.get("/api/v1/spots/{spot_id}/catches", response_model=list[CatchOut]) def spot_catches(spot_id: UUID, db: Db, limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[CatchOut]: _spot_or_404(db, spot_id) - reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc(), CatchReport.id.desc()).offset(offset).limit(limit))) - return [CatchOut(id=report.id, fish=report.fish.name_ru, weight_g=report.weight_g, bait=report.bait.name if report.bait else None, player_name=report.player_name, caught_at=report.caught_at, reported_at=report.reported_at, retrieve_method=report.retrieve_method, retrieve_speed=report.retrieve_speed, source_system=_report_source(report), source_url=report.source_url) for report in reports] + reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait), selectinload(CatchReport.tackle_components)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc(), CatchReport.id.desc()).offset(offset).limit(limit))) + return [CatchOut(id=report.id, fish=report.fish.name_ru, weight_g=report.weight_g, bait=report.bait.name if report.bait else None, player_name=report.player_name, caught_at=report.caught_at, reported_at=report.reported_at, retrieve_method=report.retrieve_method, retrieve_speed=report.retrieve_speed, source_system=_report_source(report), source_url=report.source_url, tackle_components=[{ + "id": component.id, "role": component.role, "position": component.position, + "raw_value": component.raw_value, "tackle_item_id": component.tackle_item_id, + "rig_id": component.rig_id, "source_system": component.source_system, + "source_url": component.source_url, + } for component in sorted(report.tackle_components, key=lambda value: value.position)]) for report in reports] @router.get("/api/v1/spots/{spot_id}/timeline") diff --git a/apps/api/app/routers/analytics.py b/apps/api/app/routers/analytics.py new file mode 100644 index 0000000..32b7ebc --- /dev/null +++ b/apps/api/app/routers/analytics.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +from fastapi import APIRouter, Query +from sqlalchemy import select +from sqlalchemy.orm import Session, joinedload, selectinload + +from ..dependencies import Db +from ..models import CatchReport, Fish, ModerationStatus, Spot, Waterbody +from ..schemas import TackleCombinationOut + +router = APIRouter() + + +@router.get("/api/v1/analytics/tackle", response_model=list[TackleCombinationOut]) +def tackle_combinations( + db: Db, + waterbody: str | None = None, + fish: str | None = None, + method: str | None = None, + hours: int = Query(72, ge=24, le=168), + min_samples: int = Query(3, ge=1, le=100), + min_players: int = Query(2, ge=1, le=100), +) -> list[TackleCombinationOut]: + now = datetime.now(timezone.utc) + query = select(CatchReport).options( + joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), + selectinload(CatchReport.tackle_components), + ).where( + CatchReport.moderation_status == ModerationStatus.approved, + CatchReport.deleted_at.is_(None), + CatchReport.reported_at >= now - timedelta(hours=hours), + ) + if waterbody: + query = query.join(Waterbody, CatchReport.waterbody_id == Waterbody.id).where(Waterbody.slug == waterbody) + if fish: + query = query.join(Fish, CatchReport.fish_id == Fish.id).where(Fish.slug == fish) + if method: + query = query.where(CatchReport.fishing_method == method) + + groups: dict[tuple[str, str], list[CatchReport]] = defaultdict(list) + for report in db.scalars(query): + for component in report.tackle_components: + if component.raw_value.strip(): + groups[(component.role, component.raw_value.strip())].append(report) + + result = [] + for (role, value), reports in groups.items(): + unique_reports = {report.id: report for report in reports} + players = {report.player_name.strip().casefold() for report in unique_reports.values() if report.player_name and report.player_name.strip()} + catches = len(unique_reports) + unique_players = len(players) + last_seen = max(report.reported_at for report in unique_reports.values()) + enough = catches >= min_samples and unique_players >= min_players + result.append(TackleCombinationOut( + role=role, value=value, catches=catches, unique_players=unique_players, + last_seen_at=last_seen, status="recommendation" if enough else "insufficient_data", + explanation=( + "Достаточно независимых наблюдений для рекомендации." + if enough else + f"Данных мало: нужно минимум {min_samples} наблюдения и {min_players} независимых игрока." + ), + )) + return sorted(result, key=lambda item: (item.status != "recommendation", -item.catches, -item.unique_players, item.role, item.value)) diff --git a/apps/api/app/routers/catalog.py b/apps/api/app/routers/catalog.py index ccde394..cb0f62f 100644 --- a/apps/api/app/routers/catalog.py +++ b/apps/api/app/routers/catalog.py @@ -1,9 +1,12 @@ -from fastapi import APIRouter, Query -from sqlalchemy import select +from uuid import UUID + +from fastapi import APIRouter, HTTPException, Query +from sqlalchemy import func, select +from sqlalchemy.orm import selectinload from ..dependencies import Db -from ..models import Bait, CatchReport, Fish, ModerationStatus, Spot, Waterbody -from ..schemas import BaitOut, FishOut, WaterbodyOut +from ..models import Bait, CatchReport, Fish, ModerationStatus, Rig, Spot, TackleItem, Waterbody +from ..schemas import BaitOut, FishOut, PaginatedTackleItemOut, RigOut, TackleItemOut, WaterbodyOut router = APIRouter() @@ -23,6 +26,68 @@ def baits(db: Db, limit: int = Query(200, ge=1, le=500), offset: int = Query(0, return list(db.scalars(select(Bait).order_by(Bait.name, Bait.id).offset(offset).limit(limit))) +def _item_missing_fields(item: TackleItem) -> list[str]: + return [field for field, value in ( + ("subcategory", item.subcategory), ("brand", item.brand), + ("family", item.family), ("unlock_level", item.unlock_level), + ("source_url", item.source_url), ("source_checked_at", item.source_checked_at), + ) if value is None] + + +@router.get("/api/v1/tackle/items", response_model=PaginatedTackleItemOut) +def tackle_items( + db: Db, + category: str | None = Query(None, pattern="^(bait|lure|rod|reel|line|hook|rig|float|sinker|other)$"), + brand: str | None = None, + family: str | None = None, + unlock_level: int | None = Query(None, ge=0), + limit: int = Query(50, ge=1, le=100), + offset: int = Query(0, ge=0), +) -> PaginatedTackleItemOut: + query = select(TackleItem) + if category: + query = query.where(TackleItem.category == category) + if brand: + query = query.where(TackleItem.brand == brand) + if family: + query = query.where(TackleItem.family == family) + if unlock_level is not None: + query = query.where(TackleItem.unlock_level == unlock_level) + total = db.scalar(query.with_only_columns(func.count(TackleItem.id), maintain_column_froms=True).order_by(None)) or 0 + items = list(db.scalars(query.order_by(TackleItem.name, TackleItem.id).offset(offset).limit(limit))) + return PaginatedTackleItemOut( + items=[TackleItemOut.model_validate(item).model_copy(update={"missing_fields": _item_missing_fields(item)}) for item in items], + total=total, limit=limit, offset=offset, + ) + + +@router.get("/api/v1/tackle/items/{item_id}", response_model=TackleItemOut) +def tackle_item(item_id: UUID, db: Db) -> TackleItemOut: + item = db.get(TackleItem, item_id) + if item is None: + raise HTTPException(status_code=404, detail="tackle item not found") + return TackleItemOut.model_validate(item).model_copy(update={"missing_fields": _item_missing_fields(item)}) + + +@router.get("/api/v1/tackle/rigs/{rig_id}", response_model=RigOut) +def rig_detail(rig_id: UUID, db: Db) -> RigOut: + rig = db.scalar(select(Rig).options(selectinload(Rig.components)).where(Rig.id == rig_id)) + if rig is None: + raise HTTPException(status_code=404, detail="rig not found") + missing = [field for field, value in ( + ("source_url", rig.source_url), ("source_checked_at", rig.source_checked_at), + ) if value is None] + return RigOut( + id=rig.id, name=rig.name, source_system=rig.source_system, + source_external_id=rig.source_external_id, source_url=rig.source_url, + source_checked_at=rig.source_checked_at, missing_fields=missing, + components=[{ + "id": component.id, "role": component.role, "position": component.position, + "raw_value": component.raw_value, "tackle_item_id": component.tackle_item_id, + } for component in sorted(rig.components, key=lambda value: value.position)], + ) + + @router.get("/api/v1/public-spot-pages") def public_spot_pages(db: Db, limit: int = Query(500, ge=1, le=500), offset: int = Query(0, ge=0)) -> list[str]: rows = db.execute(select(Waterbody.slug, Spot.x, Spot.y, Fish.slug) diff --git a/apps/api/app/schemas.py b/apps/api/app/schemas.py index 3930cad..8093615 100644 --- a/apps/api/app/schemas.py +++ b/apps/api/app/schemas.py @@ -40,6 +40,48 @@ class BaitOut(BaseModel): kind: str +class TackleItemOut(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: UUID + name: str + category: str + subcategory: str | None + brand: str | None + family: str | None + unlock_level: int | None + source_system: str | None + source_external_id: str | None + source_url: str | None + source_checked_at: datetime | None + missing_fields: list[str] = Field(default_factory=list) + + +class PaginatedTackleItemOut(BaseModel): + items: list[TackleItemOut] + total: int + limit: int + offset: int + + +class RigComponentOut(BaseModel): + id: UUID + role: str + position: int + raw_value: str | None + tackle_item_id: UUID | None + + +class RigOut(BaseModel): + id: UUID + name: str + source_system: str | None + source_external_id: str | None + source_url: str | None + source_checked_at: datetime | None + missing_fields: list[str] = Field(default_factory=list) + components: list[RigComponentOut] + + class ActivityOut(BaseModel): spot_id: UUID waterbody_slug: str @@ -69,6 +111,16 @@ class PaginatedActivityOut(BaseModel): offset: int +class TackleCombinationOut(BaseModel): + role: str + value: str + catches: int + unique_players: int + last_seen_at: datetime + status: str + explanation: str + + class CatchOut(BaseModel): id: UUID fish: str @@ -81,6 +133,18 @@ class CatchOut(BaseModel): retrieve_speed: int | None source_system: str source_url: str | None + tackle_components: list["CatchTackleComponentOut"] + + +class CatchTackleComponentOut(BaseModel): + id: UUID + role: str + position: int + raw_value: str + tackle_item_id: UUID | None + rig_id: UUID | None + source_system: str | None + source_url: str | None class SpotOut(BaseModel): diff --git a/apps/api/tests/test_analytics.py b/apps/api/tests/test_analytics.py new file mode 100644 index 0000000..0754e01 --- /dev/null +++ b/apps/api/tests/test_analytics.py @@ -0,0 +1,39 @@ +from datetime import datetime, timedelta, timezone + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from app.database import Base +from app.models import CatchReport, CatchTackleComponent, Fish, ModerationStatus, SourceType, Spot, Waterbody +from app.routers.analytics import tackle_combinations + + +def test_tackle_recommendation_requires_samples_and_independent_players() -> None: + now = datetime.now(timezone.utc) + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as db: + waterbody = Waterbody(slug="lake", name_ru="Озеро", unlock_level=1) + fish = Fish(slug="pike", name_ru="Щука", trophy_weight_g=10_000) + spot = Spot(waterbody=waterbody, x=10, y=20) + db.add_all([waterbody, fish, spot]) + db.flush() + for index, player in enumerate(("One", "One", "Two")): + report = CatchReport( + fish=fish, waterbody=waterbody, spot=spot, weight_g=1000, + caught_at=now - timedelta(hours=1), reported_at=now - timedelta(hours=1), + player_name=player, source_type=SourceType.user, source_confidence=80, + moderation_status=ModerationStatus.approved, + ) + report.tackle_components.append(CatchTackleComponent(role="lure", position=0, raw_value="Spinner #1")) + db.add(report) + db.commit() + + rows = tackle_combinations(db, waterbody="lake", fish="pike", method=None, hours=72, min_samples=3, min_players=2) + assert len(rows) == 1 + assert (rows[0].status, rows[0].catches, rows[0].unique_players) == ("recommendation", 3, 2) + + rows = tackle_combinations(db, waterbody="lake", fish="pike", method=None, hours=72, min_samples=3, min_players=3) + assert rows[0].status == "insufficient_data" + + engine.dispose() diff --git a/apps/api/tests/test_tackle_catalog.py b/apps/api/tests/test_tackle_catalog.py new file mode 100644 index 0000000..95b97da --- /dev/null +++ b/apps/api/tests/test_tackle_catalog.py @@ -0,0 +1,32 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from app.database import Base +from app.models import Rig, RigComponent, TackleItem +from app.routers.catalog import rig_detail, tackle_item, tackle_items + + +def test_tackle_catalog_filters_details_and_missing_fields() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as db: + item = TackleItem( + name="API тестовая блесна", normalized_name="api тестовая блесна", + category="lure", subcategory="spinner", brand="RF4", family=None, + unlock_level=0, source_system="fixture", source_external_id="api-lure-1", + source_url="https://example.test/lure/api-lure-1", + ) + rig = Rig(name="API тестовый монтаж", normalized_name="api тестовый монтаж", source_system="fixture") + rig.components.append(RigComponent(role="lure", position=0, tackle_item=item, raw_value=item.name)) + db.add(rig) + db.commit() + + page = tackle_items(db, category="lure", brand="RF4", family=None, unlock_level=None, limit=10, offset=0) + assert page.total == 1 + assert page.items[0].missing_fields == ["family", "source_checked_at"] + assert tackle_item(item.id, db).name == item.name + details = rig_detail(rig.id, db) + assert details.components[0].raw_value == item.name + assert details.missing_fields == ["source_url", "source_checked_at"] + + engine.dispose() diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index bd5951a..21b350a 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -17,6 +17,8 @@ export type PaginatedActivity = { export type Spot = { id: string; waterbody_slug: string; waterbody: string; x: number; y: number; description: string | null; catches_24h: number; catches_3d: number; catches_7d: number; top_baits: string[]; coordinate_precision: "exact" | "approximate" | "area" | "missing"; coordinate_sources: string[] }; export type Catch = { id: string; fish: string; weight_g: number; bait: string | null; player_name: string | null; caught_at: string | null; reported_at: string; retrieve_method: string | null; retrieve_speed: number | null; source_system: string; source_url: string | null }; export type DictionaryItem = { id: string; slug: string; name_ru: string; unlock_level?: number | null; fish_species_count?: number | null; source_system?: string | null; source_external_id?: string | null; source_url?: string | null; description?: string | null; source_aliases?: string[] | null; source_fish_species?: string[] | null; source_image_urls?: string[] | null; source_point_urls?: string[] | null; source_checked_at?: string | null }; +export type TackleItem = { id: string; name: string; category: string; subcategory: string | null; brand: string | null; family: string | null; unlock_level: number | null; source_system: string | null; source_external_id: string | null; source_url: string | null; source_checked_at: string | null; missing_fields: string[] }; +export type PaginatedTackleItems = { items: TackleItem[]; total: number; limit: number; offset: number }; export type OfficialRecord = { id: string; fish: string; weight_g: number; waterbody: string; bait: string | null; player_name: string | null; record_date: string | null; category: string | null; region: string | null; source_url: string | null; source_system: string }; export type PaginatedOfficialRecord = { items: OfficialRecord[]; diff --git a/apps/web/src/pages/tackle/index.astro b/apps/web/src/pages/tackle/index.astro new file mode 100644 index 0000000..8efa92b --- /dev/null +++ b/apps/web/src/pages/tackle/index.astro @@ -0,0 +1,40 @@ +--- +import Layout from "../../layouts/Layout.astro"; +import PageHero from "../../components/PageHero.astro"; +import StatePanel from "../../components/StatePanel.astro"; +import Pagination from "../../components/Pagination.astro"; +import TackleGlyph from "../../components/TackleGlyph.astro"; +import { api, type PaginatedTackleItems } from "../../lib/api"; + +const params = Astro.url.searchParams; +const category = params.get("category") ?? ""; +const brand = params.get("brand") ?? ""; +const family = params.get("family") ?? ""; +const requestedOffset = Number(params.get("offset") ?? 0); +const limit = 48; +const offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0; +const query = new URLSearchParams({ limit: String(limit), offset: String(offset) }); +if (category) query.set("category", category); +if (brand) query.set("brand", brand); +if (family) query.set("family", family); +let result: PaginatedTackleItems = { items: [], total: 0, limit, offset }, unavailable = false; +try { result = await api(`/api/v1/tackle/items?${query}`); } catch { unavailable = true; } +if (unavailable) { Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); Astro.response.headers.set("Cache-Control", "no-store"); } +const categoryLabels: Record = { bait: "Наживка", lure: "Приманка", rod: "Удилище", reel: "Катушка", line: "Леска", hook: "Крючок", rig: "Монтаж", float: "Поплавок", sinker: "Груз", other: "Другое" }; +const filterParams = new URLSearchParams(); +if (category) filterParams.set("category", category); +if (brand) filterParams.set("brand", brand); +if (family) filterParams.set("family", family); +--- + + +
+ + + + + {(category || brand || family) && Сбросить} +
+ {unavailable ? : result.items.length ?
{result.items.map(item => {categoryLabels[item.category] ?? item.category}{item.name}{[item.brand, item.family].filter(Boolean).join(" · ") || "Характеристики уточняются"})}
: } + {!unavailable && } + diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 879bb8f..11c551d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -71,8 +71,8 @@ - [ ] **G03 · Модель и provenance.** Добавлены отдельные `tackle_item`, `rig` и `rig_component` с миграцией `0021`; legacy `bait` и `catch_report.bait_id` не изменялись. `tackle_item` хранит `category`, `subcategory`, `brand`, `family`, `unlock_level`, source identity, timestamp и `raw_payload`; `rig_component` хранит роль, порядок, исходное значение и optional canonical item. Связи с catch-потоком добавлены в G05; безопасный backfill остаётся только после реального crosswalk. - [ ] **G04 · Crosswalk и нормализация.** Добавлен offline `gear_crosswalk`: нормализация регистра/пробелов/`е/ё`, точное имя или alias плюс совместимая категория, консервативная проверка brand/family. Неоднозначные, несовместимые, брендовые и unmatched-строки получают review-статус без canonical key; исходное значение сохраняется. Остаётся подать реальные RF4DB/RF4MAP/RF4 Posts identities и вручную подтвердить результаты. - [ ] **G05 · Связи с уловами и источниками.** Добавлены `catch_tackle_component` и offline `gear_components`: можно сохранять несколько unresolved/canonical компонентов с ролью, порядком, исходным значением, source identity и `raw_payload`; legacy `bait_id` не меняется. Parser сохраняет порядок оборудования из detail и разделяет bait/rig в catch-полях. Запись компонентов подключена к community import, официальному импорту и пользовательской форме; повторная обработка идемпотентна. Canonical-привязка и безопасный backfill остаются только после подтверждённого crosswalk. -- [ ] **G06 · API и публичный каталог.** Добавить пагинированные каталоги и detail endpoints с фильтрами по категории, бренду, семейству и уровню, а также безопасные ссылки из улова/точки на использованную приманку, снасть и монтаж. Показывать только подтверждённые характеристики, источник, свежесть и неполноту; не выдавать рейтинг эффективности, если его нельзя объяснить числом наблюдений, игроками, периодом и качеством источников. -- [ ] **G07 · Аналитика сочетаний и рекомендации.** После появления достаточных данных считать отдельно «водоём + рыба + предмет», «точка + рыба + предмет» и «способ ловли + монтаж». Зафиксировать минимальный объём выборки, защиту от одного игрока/дубликатов и decay по свежести; разделить факт использования, частоту и рекомендацию. Пустая или малая выборка должна показывать «данных мало», а не советовать конкретную снасть. +- [ ] **G06 · API и публичный каталог.** Добавлены пагинированный `/api/v1/tackle/items` с фильтрами по категории, бренду, семейству и уровню, detail endpoints для предмета и монтажа, а также ordered `tackle_components` в ответе уловов точки. Ответы показывают только канонические характеристики, provenance, timestamp проверки и `missing_fields`; рейтинг эффективности не добавляется. Остаётся подключить публичные Astro-карточки и ссылки из всех нужных представлений улова. +- [ ] **G07 · Аналитика сочетаний и рекомендации.** Добавлен `/api/v1/analytics/tackle`: approved-наблюдения группируются по роли и исходному компоненту с фильтрами водоёма, рыбы, метода и окна; дубликаты одного улова не увеличивают счётчик, а минимум наблюдений и независимых игроков отделяет факт использования от рекомендации. Пустая или малая выборка получает `insufficient_data`; decay по свежести и отдельные UI-состояния остаются следующим шагом. - [ ] **G08 · Медиа и качество.** Разнести media roles для `tackle_item`, `bait`, `rig` и общего reference; связать варианты через `entity_key`, `duplicate_of`, `supersedes`/`replaced_by`. Проверять dimensions, MIME, SHA-256, прозрачность, aspect ratio, подпись и категорию; не переключать approved-файл автоматически, не считать userguide-скриншот карточкой предмета и не публиковать media-кандидатов без review и разрешённого provenance. - [ ] **G09 · Приёмка и эксплуатация.** Добавить fixture/regression tests, offline catalog/crosswalk/media audits, проверку идемпотентности и сохранения старых данных при сбое, API/UI acceptance для пустых, неоднозначных и многокомпонентных комплектов, browser QA desktop/mobile и query-plan gate для фильтров/сочетаний. Сетевые тесты не выполнять; импорт оставить opt-in, последовательным и под общим cooldown/backoff. Закрывать пакет только после проверяемого счётчика по каждой категории либо явной фиксации `unknown`. diff --git a/scripts/fetch-waterbodies-chromium.mjs b/scripts/fetch-waterbodies-chromium.mjs index 275eb5e..a3fe295 100644 --- a/scripts/fetch-waterbodies-chromium.mjs +++ b/scripts/fetch-waterbodies-chromium.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import fs from "node:fs/promises"; +import fs, { access } from "node:fs/promises"; import readline from "node:readline/promises"; import process from "node:process"; import path from "node:path"; @@ -22,6 +22,7 @@ Options: --html-output PATH retain browser DOM HTML (default: temporary .cache file) --state-file PATH shared cooldown state file --profile PATH persistent Chromium profile directory + --executable PATH Chromium/Chrome executable (default: RF4_CHROMIUM_EXECUTABLE or system Chrome) --headless run Chromium headless; cannot handle manual challenges --wait-seconds N headed wait after load (default: 30) `); @@ -37,6 +38,7 @@ function parseArgs(argv) { const args = { mode, url: mode === "catalog" ? "https://rf4db.com/ru/maps" : null, output: null, htmlOutput: null, stateFile: DEFAULT_STATE, profile: DEFAULT_PROFILE, + executable: process.env.RF4_CHROMIUM_EXECUTABLE || null, headless: false, waitSeconds: 30, }; for (let index = 0; index < argv.length; index += 1) { @@ -46,6 +48,7 @@ function parseArgs(argv) { else if (arg === "--html-output") args.htmlOutput = argv[++index]; else if (arg === "--state-file") args.stateFile = argv[++index]; else if (arg === "--profile") args.profile = argv[++index]; + else if (arg === "--executable") args.executable = argv[++index]; else if (arg === "--headless") args.headless = true; else if (arg === "--wait-seconds") args.waitSeconds = Number(argv[++index]); else throw new Error(`unknown argument: ${arg}`); @@ -61,6 +64,22 @@ function parseArgs(argv) { return args; } +async function resolveExecutable(requested) { + if (requested) { + await access(requested); + return requested; + } + for (const candidate of ["/usr/bin/google-chrome-stable", "/usr/bin/google-chrome", "/usr/bin/chromium"]) { + try { + await access(candidate); + return candidate; + } catch { + // Try the next locally installed browser. + } + } + return undefined; +} + function reserveCooldown(args) { const source = args.mode === "catalog" ? "rf4db-waterbodies" : "rf4db-waterbody"; const command = ["-m", "rf4_research.community_cli", source, "--url", args.url, "--state-file", args.stateFile, "--reserve-only"]; @@ -87,7 +106,12 @@ async function main() { await fs.mkdir(path.dirname(htmlOutput), { recursive: true }); await fs.mkdir(path.dirname(output), { recursive: true }); - const context = await chromium.launchPersistentContext(path.resolve(args.profile), { headless: args.headless }); + const executablePath = await resolveExecutable(args.executable); + if (executablePath) console.log(`Using browser executable: ${executablePath}`); + const context = await chromium.launchPersistentContext(path.resolve(args.profile), { + headless: args.headless, + ...(executablePath ? { executablePath } : {}), + }); try { const page = context.pages()[0] || await context.newPage(); await page.goto(args.url, { waitUntil: "domcontentloaded", timeout: 60_000 });