sync
CI / backend-and-migrations (push) Canceled after 0s
CI / astro-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / compose-e2e (push) Canceled after 0s

This commit is contained in:
ik
2026-09-17 07:35:04 +07:00
parent 5221442aeb
commit 722c88d436
25 changed files with 673 additions and 40 deletions
+16 -1
View File
@@ -61,6 +61,9 @@ def activity_rows(
confidence = min(confidence, 50)
elif len(players) == 2:
confidence = min(confidence, 65)
coordinate_precisions = {_coordinate_precision(item) for item in items}
coordinate_precision = max(coordinate_precisions, key=_precision_rank)
coordinate_sources = sorted({_source_system(item) for item in items})
latest = max(_aware(r.reported_at) for r in items)
baits = Counter(r.bait.name for r in items if r.bait)
freshness_text = _freshness_text(now - latest)
@@ -74,7 +77,9 @@ def activity_rows(
max_weight_g=max(r.weight_g for r in items), last_confirmed_at=latest,
activity_score=activity, confidence_score=confidence,
explanation=_explanation(len(items), len(players), freshness_text, activity, confidence),
sources=sorted({_source_system(item) for item in items}),
sources=coordinate_sources,
coordinate_precision=coordinate_precision,
coordinate_sources=coordinate_sources,
))
return sorted(result, key=lambda row: (row.activity_score, row.last_confirmed_at), reverse=True)
@@ -90,6 +95,16 @@ def _source_system(report: CatchReport) -> str:
return "manual-import"
def _coordinate_precision(report: CatchReport) -> str:
provenance = (report.raw_payload or {}).get("provenance", {})
value = provenance.get("coordinate_precision") if isinstance(provenance, dict) else None
return value if value in {"exact", "approximate", "area", "missing"} else "exact"
def _precision_rank(value: str) -> int:
return {"exact": 0, "approximate": 1, "area": 2, "missing": 3}[value]
def _aware(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
+25
View File
@@ -15,3 +15,28 @@ def audit_catalog(db: Session) -> dict[str, int]:
"incomplete_published_staging": db.scalar(select(func.count()).select_from(ExternalObservation).where(ExternalObservation.status == "published", or_(ExternalObservation.fish_id.is_(None), ExternalObservation.waterbody_id.is_(None), ExternalObservation.x.is_(None), ExternalObservation.y.is_(None), ExternalObservation.weight_g.is_(None), ExternalObservation.catch_report_id.is_(None)))) or 0,
}
return {"fishes": count(Fish), "waterbodies": count(Waterbody), "reports": count(CatchReport), "staging": count(ExternalObservation), **failures, "failures": sum(failures.values())}
def audit_waterbody_catalog(db: Session, expected_ids: set[str]) -> dict:
"""Check a verified RF4DB snapshot without withdrawing legacy rows."""
rows = list(db.scalars(select(Waterbody).where(Waterbody.source_system == "rf4db")))
observed_ids = [str(row.source_external_id) for row in rows if row.source_external_id]
observed = set(observed_ids)
duplicate_ids = sorted({item for item in observed_ids if observed_ids.count(item) > 1})
missing = sorted(expected_ids - observed)
unexpected = sorted(observed - expected_ids)
provenance_issues = sorted(
str(row.source_external_id)
for row in rows
if not row.source_external_id or not row.source_url or not row.source_checked_at
)
failures = len(missing) + len(duplicate_ids) + len(provenance_issues)
return {
"expected": len(expected_ids),
"observed": len(observed),
"missing_source_external_ids": missing,
"unexpected_source_external_ids": unexpected,
"duplicate_source_external_ids": duplicate_ids,
"provenance_issues": provenance_issues,
"failures": failures,
}
+66 -2
View File
@@ -8,10 +8,10 @@ from dataclasses import asdict
from .config import settings
from .database import SessionLocal
from .importer import import_records
from .community_importer import stage_observations
from .community_importer import stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog
from .retention import RetentionPolicy, apply_retention
from .storage import delete_screenshot
from .catalog_audit import audit_catalog
from .catalog_audit import audit_catalog, audit_waterbody_catalog
from .community_scheduler import run_source, configured_sources
# Static registry for argparse choices — no DB required for --help
@@ -34,11 +34,20 @@ def main() -> int:
community = sub.add_parser("stage-community-json")
community.add_argument("--input", default="-", help="JSON array path or - for stdin")
community.add_argument("--limit", type=int, default=500)
waterbodies = sub.add_parser("import-waterbody-catalog")
waterbodies.add_argument("--input", required=True, help="JSON snapshot path or - for stdin")
waterbodies.add_argument("--limit", type=int, default=100)
detail = sub.add_parser("import-waterbody-detail")
detail.add_argument("--input", required=True, help="JSON detail snapshot path")
details = sub.add_parser("import-waterbody-details")
details.add_argument("--input", required=True, help="JSON array of detail snapshots")
fetch_community = sub.add_parser("fetch-community")
fetch_community.add_argument("source", choices=STATIC_SOURCE_CHOICES)
cleanup = sub.add_parser("cleanup-retention")
cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
sub.add_parser("audit-catalog")
waterbody_audit = sub.add_parser("audit-waterbody-catalog")
waterbody_audit.add_argument("--input", required=True, help="JSON snapshot path")
args = parser.parse_args()
with SessionLocal() as session:
if args.command == "import-records":
@@ -57,6 +66,43 @@ def main() -> int:
parser.error("input must be a JSON array")
created, updated = stage_observations(session, payload[:args.limit])
print(f"staged: created={created} updated={updated}")
elif args.command == "import-waterbody-catalog":
if not 1 <= args.limit <= 500:
parser.error("--limit must be between 1 and 500")
stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
try:
snapshot = json.load(stream)
finally:
if stream is not sys.stdin:
stream.close()
if isinstance(snapshot, dict):
payload = snapshot.get("items")
source_system = snapshot.get("source_system")
if isinstance(payload, list) and isinstance(source_system, str):
payload = [
{"source_system": source_system, **item}
for item in payload if isinstance(item, dict)
]
else:
payload = snapshot
if not isinstance(payload, list):
parser.error("input must be a JSON array or an object with an items array")
created, updated = upsert_waterbody_catalog(session, payload[:args.limit])
print(f"waterbodies: created={created} updated={updated}")
elif args.command == "import-waterbody-detail":
with open(args.input, encoding="utf-8") as stream:
payload = json.load(stream)
if not isinstance(payload, dict):
parser.error("input must be a JSON object")
update_waterbody_detail(session, payload)
print(f"waterbody detail: updated={payload.get('source_external_id', 'unknown')}")
elif args.command == "import-waterbody-details":
with open(args.input, encoding="utf-8") as stream:
payload = json.load(stream)
if not isinstance(payload, list):
parser.error("input must be a JSON array")
created, updated = update_waterbody_details(session, payload)
print(f"waterbody details: created={created} updated={updated}")
elif args.command == "fetch-community":
# A09: Verify source is enabled at runtime (not just in static choices)
enabled = configured_sources()
@@ -76,6 +122,24 @@ def main() -> int:
)
counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
elif args.command == "audit-waterbody-catalog":
stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
try:
snapshot = json.load(stream)
finally:
if stream is not sys.stdin:
stream.close()
items = snapshot.get("items") if isinstance(snapshot, dict) else snapshot
if not isinstance(items, list):
parser.error("input must be a JSON array or an object with an items array")
expected_ids = {
str(item["source_external_id"])
for item in items
if isinstance(item, dict) and item.get("source_external_id")
}
result = audit_waterbody_catalog(session, expected_ids)
print(json.dumps(result, ensure_ascii=False))
return 1 if result["failures"] else 0
else:
result = audit_catalog(session)
print(json.dumps(result, ensure_ascii=False))
+167
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import json
import hashlib
import re
from datetime import datetime, timezone
from typing import Any, Iterable
from urllib.parse import urlparse
@@ -26,12 +28,153 @@ SOURCE_HOSTS = {
"rf4map": {"rf4map.ru"},
"rf4posts-spot": {"rf4-posts.com"},
}
COORDINATE_PRECISIONS = frozenset({"exact", "approximate", "area", "missing"})
class CommunityImportError(ValueError):
pass
def upsert_waterbody_catalog(
session: Session, rows: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
) -> tuple[int, int]:
"""Apply a validated canonical waterbody snapshot without destructive sync.
Rows are matched by the RF4DB source identity first and by an exact existing
name second. Missing rows are deliberately left untouched: an incomplete
response must never withdraw a previously known waterbody.
"""
fetched_at = fetched_at or datetime.now(timezone.utc)
created = updated = 0
for raw in rows:
payload = _json_payload(raw)
if payload.get("source_system") != "rf4db":
raise CommunityImportError("waterbody catalog requires source_system=rf4db")
external_id = _required(payload, "source_external_id", 200)
name = _required(payload, "name", 200)
source_url = _required(payload, "source_url", 2000)
parsed_url = urlparse(source_url)
if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
raise CommunityImportError("waterbody source_url does not match rf4db")
unlock_level = _integer(payload.get("unlock_level"), minimum=0, maximum=1_000)
unlock_label = _required(payload, "unlock_label", 50)
fish_species_count = _integer(payload.get("fish_species_count"), minimum=0, maximum=10_000)
if fish_species_count is None:
raise CommunityImportError("invalid fish_species_count")
item = session.scalar(select(Waterbody).where(
Waterbody.source_system == "rf4db",
Waterbody.source_external_id == external_id,
))
if item is None:
item = session.scalar(select(Waterbody).where(Waterbody.name_ru == name))
if item is None:
item = Waterbody(
slug=_catalog_slug(session, name, external_id),
name_ru=name,
unlock_level=unlock_level,
)
session.add(item)
created += 1
else:
updated += 1
item.name_ru = name
item.unlock_level = unlock_level
item.fish_species_count = fish_species_count
item.source_system = "rf4db"
item.source_external_id = external_id
item.source_url = source_url
item.source_checked_at = fetched_at
session.commit()
return created, updated
def update_waterbody_detail(
session: Session, detail: dict[str, Any], *, fetched_at: datetime | None = None,
) -> bool:
"""Persist one complete RF4DB detail snapshot without assigning media roles."""
fetched_at = fetched_at or datetime.now(timezone.utc)
payload = _validate_waterbody_detail(detail)
_apply_waterbody_detail(session, payload, fetched_at=fetched_at)
session.commit()
return True
def update_waterbody_details(
session: Session, details: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
) -> tuple[int, int]:
"""Validate and apply a detail batch in one transaction."""
fetched_at = fetched_at or datetime.now(timezone.utc)
payloads = [_validate_waterbody_detail(detail) for detail in details]
external_ids = [str(payload["source_external_id"]) for payload in payloads]
if len(external_ids) != len(set(external_ids)):
raise CommunityImportError("waterbody detail batch contains duplicate source identities")
updated = 0
for payload in payloads:
_apply_waterbody_detail(session, payload, fetched_at=fetched_at)
updated += 1
session.commit()
return 0, updated
def _validate_waterbody_detail(detail: dict[str, Any]) -> dict[str, Any]:
payload = _json_payload(detail)
if payload.get("source_system") != "rf4db":
raise CommunityImportError("waterbody detail requires source_system=rf4db")
external_id = _required(payload, "source_external_id", 200)
source_url = _required(payload, "source_url", 2000)
parsed_url = urlparse(source_url)
if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
raise CommunityImportError("waterbody detail source_url does not match rf4db")
_required(payload, "name", 200)
_optional(payload, "description", 20_000)
_string_list(payload, "aliases", 100, 200)
_string_list(payload, "fish_species", 10_000, 200)
_string_list(payload, "image_urls", 100, 2_000)
_string_list(payload, "point_urls", 10_000, 2_000)
return payload
def _apply_waterbody_detail(session: Session, payload: dict[str, Any], *, fetched_at: datetime) -> None:
external_id = str(payload["source_external_id"])
source_url = str(payload["source_url"])
item = session.scalar(select(Waterbody).where(
Waterbody.source_system == "rf4db", Waterbody.source_external_id == external_id,
))
if item is None:
raise CommunityImportError("waterbody detail has no imported catalog identity")
item.description = _optional(payload, "description", 20_000)
item.source_aliases = _string_list(payload, "aliases", 100, 200)
item.source_fish_species = _string_list(payload, "fish_species", 10_000, 200)
item.source_image_urls = _string_list(payload, "image_urls", 100, 2_000)
item.source_point_urls = _string_list(payload, "point_urls", 10_000, 2_000)
item.source_url = source_url
item.source_checked_at = fetched_at
def _catalog_slug(session: Session, name: str, external_id: str) -> str:
base = re.sub(r"[^a-z0-9а-яё]+", "-", name.casefold(), flags=re.IGNORECASE).strip("-")
base = base or "waterbody"
candidate = base[:100]
if session.scalar(select(Waterbody.id).where(Waterbody.slug == candidate)) is None:
return candidate
suffix = hashlib.sha256(external_id.encode()).hexdigest()[:10]
return f"{base[:89]}-{suffix}"
def _string_list(payload: dict[str, Any], key: str, max_items: int, max_length: int) -> list[str]:
value = payload.get(key)
if not isinstance(value, list) or len(value) > max_items:
raise CommunityImportError(f"invalid {key}")
result = []
for item in value:
text = str(item).strip()
if not text or len(text) > max_length:
raise CommunityImportError(f"invalid {key}")
result.append(text)
return list(dict.fromkeys(result))
def stage_observations(
session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
) -> tuple[int, int]:
@@ -65,6 +208,8 @@ def stage_observations(
"waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
"x": _integer(payload.get("x"), maximum=10_000),
"y": _integer(payload.get("y"), maximum=10_000),
"coordinate_raw": _coordinate_raw(payload),
"coordinate_precision": _coordinate_precision(payload),
"weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
"published_at": _datetime(payload.get("published_at")),
"last_seen_at": fetched_at, "payload": payload,
@@ -83,6 +228,7 @@ def stage_observations(
changed = any(getattr(observation, key) != values[key] for key in (
"source_url", "fish_name", "fish_external_id", "waterbody_name",
"waterbody_external_id", "x", "y", "weight_g",
"coordinate_raw", "coordinate_precision",
)) or observation.payload != payload
if observation.status != "rejected" and changed and observation.catch_report is not None:
observation.catch_report.moderation_status = ModerationStatus.pending
@@ -186,6 +332,27 @@ def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
return value or None
def _coordinate_raw(payload: dict[str, Any]) -> str | None:
value = str(payload.get("coordinate_raw") or "").strip()
if len(value) > 200:
raise CommunityImportError("invalid coordinate_raw")
if value:
return value
x, y = payload.get("x"), payload.get("y")
return f"{x}:{y}" if isinstance(x, int) and isinstance(y, int) else None
def _coordinate_precision(payload: dict[str, Any]) -> str:
value = str(payload.get("coordinate_precision") or "").strip().casefold()
if not value:
return "exact" if isinstance(payload.get("x"), int) and isinstance(payload.get("y"), int) else "missing"
if value not in COORDINATE_PRECISIONS:
raise CommunityImportError("invalid coordinate_precision")
if value == "exact" and (not isinstance(payload.get("x"), int) or not isinstance(payload.get("y"), int)):
raise CommunityImportError("exact coordinates require x and y")
return value
def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
if value is None:
return None
+2
View File
@@ -104,6 +104,8 @@ def publish_observation(session: Session, observation: ExternalObservation) -> C
"external_observation_id": str(observation.id),
"source_system": observation.source_system,
"source_external_id": observation.source_external_id,
"coordinate_raw": observation.coordinate_raw,
"coordinate_precision": observation.coordinate_precision,
},
"original": observation.payload,
},
+12
View File
@@ -49,6 +49,16 @@ class Waterbody(Base):
slug: Mapped[str] = mapped_column(String(100), unique=True)
name_ru: Mapped[str] = mapped_column(String(200), unique=True)
unlock_level: Mapped[int | None]
fish_species_count: Mapped[int | None]
source_system: Mapped[str | None] = mapped_column(String(50))
source_external_id: Mapped[str | None] = mapped_column(String(200))
source_url: Mapped[str | None] = mapped_column(Text)
description: Mapped[str | None] = mapped_column(Text)
source_aliases: Mapped[list[str] | None] = mapped_column(JSON)
source_fish_species: Mapped[list[str] | None] = mapped_column(JSON)
source_image_urls: Mapped[list[str] | None] = mapped_column(JSON)
source_point_urls: Mapped[list[str] | None] = mapped_column(JSON)
source_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Bait(Base):
@@ -187,6 +197,8 @@ class ExternalObservation(Base):
waterbody_external_id: Mapped[str | None] = mapped_column(String(200))
x: Mapped[int | None]
y: Mapped[int | None]
coordinate_raw: Mapped[str | None] = mapped_column(String(200))
coordinate_precision: Mapped[str] = mapped_column(String(20), default="missing")
weight_g: Mapped[int | None]
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+9 -1
View File
@@ -68,7 +68,15 @@ def spot_detail(spot_id: UUID, db: Db) -> SpotOut:
def count_since(delta: timedelta) -> int:
return sum(aware(report.reported_at) >= now - delta for report in reports)
return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)])
provenance = [
(report.raw_payload or {}).get("provenance", {})
for report in reports
if isinstance((report.raw_payload or {}).get("provenance", {}), dict)
]
precisions = [item.get("coordinate_precision") for item in provenance]
precision = max((value for value in precisions if value in {"exact", "approximate", "area", "missing"}), key={"exact": 0, "approximate": 1, "area": 2, "missing": 3}.get, default="exact")
sources = sorted({str(item.get("source_system")) for item in provenance if item.get("source_system")}) or ["players"]
return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)], coordinate_precision=precision, coordinate_sources=sources)
def _report_source(report: CatchReport) -> str:
+14
View File
@@ -20,6 +20,16 @@ class WaterbodyOut(BaseModel):
slug: str
name_ru: str
unlock_level: int | None
fish_species_count: int | None
source_system: str | None
source_external_id: str | None
source_url: str | None
description: str | None
source_aliases: list[str] | None
source_fish_species: list[str] | None
source_image_urls: list[str] | None
source_point_urls: list[str] | None
source_checked_at: datetime | None
class BaitOut(BaseModel):
@@ -48,6 +58,8 @@ class ActivityOut(BaseModel):
confidence_score: int
explanation: str
sources: list[str]
coordinate_precision: str
coordinate_sources: list[str]
class PaginatedActivityOut(BaseModel):
@@ -82,6 +94,8 @@ class SpotOut(BaseModel):
catches_3d: int
catches_7d: int
top_baits: list[str]
coordinate_precision: str
coordinate_sources: list[str]
class OfficialRecordOut(BaseModel):
+22 -1
View File
@@ -3,7 +3,7 @@ from datetime import datetime, timezone
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from app.catalog_audit import audit_catalog
from app.catalog_audit import audit_catalog, audit_waterbody_catalog
from app.database import Base
from app.models import CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
@@ -24,3 +24,24 @@ def test_catalog_audit_checks_the_whole_catalog() -> None:
assert result["reports"] == 1
assert result["invalid_coordinates"] == 1
assert result["failures"] == 1
def test_waterbody_catalog_audit_reports_snapshot_gaps_without_legacy_rows() -> None:
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
with Session(engine) as db:
now = datetime.now(timezone.utc)
db.add_all([
Waterbody(
slug="lake", name_ru="Озеро", source_system="rf4db",
source_external_id="level_001_lake", source_url="https://rf4db.com/ru/maps/level_001_lake",
source_checked_at=now,
),
Waterbody(slug="legacy", name_ru="Старое озеро"),
])
db.commit()
result = audit_waterbody_catalog(db, {"level_001_lake", "level_002_river"})
assert result["expected"] == 2
assert result["observed"] == 1
assert result["missing_source_external_ids"] == ["level_002_river"]
assert result["failures"] == 1
+87 -3
View File
@@ -7,7 +7,7 @@ import pytest
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session
from app.community_importer import CommunityImportError, stage_observations
from app.community_importer import CommunityImportError, stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog
from app.community_review import ExternalReviewError, map_observation, publish_observation, suggest_aliases
from app.source_lifecycle import record_scheduled_source_check, record_source_check
from app.database import Base
@@ -42,6 +42,20 @@ def record(source: str = "rf4db", external_id: str = "catch-1") -> dict[str, obj
}
def waterbody_row(**overrides: object) -> dict[str, object]:
row: dict[str, object] = {
"source_system": "rf4db",
"source_external_id": "level_001_mosquito",
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito",
"name": "оз. Комариное",
"unlock_level": 1,
"unlock_label": "1",
"fish_species_count": 20,
}
row.update(overrides)
return row
@pytest.fixture
def db() -> Session:
engine = create_engine("sqlite://")
@@ -69,6 +83,62 @@ def test_staging_is_idempotent_and_preserves_first_seen(db: Session) -> None:
assert source is not None and source.enabled is True
def test_waterbody_catalog_upsert_is_idempotent_and_non_destructive(db: Session) -> None:
first = datetime(2026, 9, 16, 10, tzinfo=timezone.utc)
assert upsert_waterbody_catalog(db, [waterbody_row()], fetched_at=first) == (1, 0)
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
assert item is not None
assert item.slug == "оз-комариное"
assert item.source_checked_at.replace(tzinfo=timezone.utc) == first
assert item.fish_species_count == 20
assert upsert_waterbody_catalog(db, [waterbody_row(name="Озеро Комариное", unlock_level=2)], fetched_at=first) == (0, 1)
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
assert item is not None
assert (item.name_ru, item.unlock_level, item.fish_species_count) == ("Озеро Комариное", 2, 20)
assert db.scalar(select(Waterbody).where(Waterbody.name_ru == "оз. Комариное")) is None
def test_waterbody_catalog_rejects_untrusted_source(db: Session) -> None:
with pytest.raises(CommunityImportError, match="source_url"):
upsert_waterbody_catalog(db, [waterbody_row(source_url="https://example.test/map")])
def test_waterbody_detail_updates_only_imported_identity_without_media_roles(db: Session) -> None:
upsert_waterbody_catalog(db, [waterbody_row()])
assert update_waterbody_detail(db, {
"source_system": "rf4db",
"source_external_id": "level_001_mosquito",
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito",
"name": "оз. Комариное",
"description": "Каменистые берега.",
"aliases": ["Комариное", "Комариное"],
"fish_species": ["Щука", "Окунь"],
"image_urls": ["https://oss.rf4db.com/map.webp"],
"point_urls": ["https://rf4db.com/ru/maps/level_001_mosquito/spots/12-34"],
}) is True
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
assert item is not None
assert item.source_aliases == ["Комариное"]
assert item.source_fish_species == ["Щука", "Окунь"]
assert item.source_image_urls == ["https://oss.rf4db.com/map.webp"]
def test_waterbody_detail_batch_validates_before_writing(db: Session) -> None:
upsert_waterbody_catalog(db, [waterbody_row()])
valid = {
"source_system": "rf4db", "source_external_id": "level_001_mosquito",
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito", "name": "оз. Комариное",
"description": "Описание", "aliases": [], "fish_species": ["Щука"],
"image_urls": [], "point_urls": [],
}
invalid = valid | {"source_external_id": "unknown", "source_url": "https://example.test/map"}
with pytest.raises(CommunityImportError, match="source_url"):
update_waterbody_details(db, [valid, invalid])
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
assert item is not None and item.description is None
def test_external_ids_are_isolated_by_source(db: Session) -> None:
created, updated = stage_observations(db, [record("rf4db"), record("rf4stat-fishing")])
@@ -104,14 +174,28 @@ def test_complete_observation_with_reviewed_aliases_is_published(db: Session) ->
assert item.catch_report is not None
assert item.catch_report.fish_id == fish.id
assert item.catch_report.waterbody_id == waterbody.id
assert db.scalar(select(func.count()).select_from(CatchReport)) == 1
assert stage_observations(db, [record() | {"weight_g": 5_000}]) == (0, 1)
db.refresh(item)
assert item.status == "published"
assert db.scalar(select(func.count()).select_from(CatchReport)) == 1
def test_observation_preserves_coordinate_text_and_precision(db: Session) -> None:
stage_observations(db, [record() | {
"source_external_id": "coordinate-area",
"x": None, "y": None, "coordinate_raw": "северная бухта",
"coordinate_precision": "area", "weight_g": None,
}])
item = db.scalar(select(ExternalObservation).where(ExternalObservation.source_external_id == "coordinate-area"))
assert item is not None
assert (item.coordinate_raw, item.coordinate_precision, item.x, item.y) == ("северная бухта", "area", None, None)
def test_coordinate_precision_rejects_unknown_value(db: Session) -> None:
with pytest.raises(CommunityImportError, match="invalid coordinate_precision"):
stage_observations(db, [record() | {"coordinate_precision": "guess"}])
def test_changed_published_record_requires_review_and_reuses_report(db: Session) -> None:
fish = Fish(slug="pike", name_ru="Щука")
water = Waterbody(slug="test-lake", name_ru="Тестовое озеро")