376 lines
16 KiB
Python
376 lines
16 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import hashlib
|
||
import re
|
||
from datetime import datetime, timezone
|
||
from typing import Any, Iterable
|
||
from urllib.parse import urlparse
|
||
|
||
from sqlalchemy import select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from .community_review import publish_observation
|
||
from .models import DataSource, ExternalEntityAlias, ExternalObservation, Fish, ModerationStatus, Waterbody
|
||
|
||
|
||
SOURCE_DEFAULTS = {
|
||
"rf4db": ("RF4DB", "https://rf4db.com", 70),
|
||
"rf4stat-fishing": ("RF4-STAT fishing", "https://rf4-stat.ru/fishing/", 65),
|
||
"rf4stat-post": ("RF4-STAT posts", "https://rf4-stat.ru/posts/", 60),
|
||
"rf4map": ("RF4MAP", "https://rf4map.ru", 55),
|
||
"rf4posts-spot": ("RF4 Posts spots", "https://rf4-posts.com", 50),
|
||
}
|
||
SOURCE_HOSTS = {
|
||
"rf4db": {"rf4db.com", "download.rf4db.com"},
|
||
"rf4stat-fishing": {"rf4-stat.ru"},
|
||
"rf4stat-post": {"rf4-stat.ru"},
|
||
"rf4map": {"rf4map.ru"},
|
||
"rf4posts-spot": {"rf4-posts.com"},
|
||
}
|
||
COORDINATE_PRECISIONS = frozenset({"exact", "approximate", "area", "missing"})
|
||
|
||
|
||
class CommunityImportError(ValueError):
|
||
pass
|
||
|
||
|
||
def upsert_waterbody_catalog(
|
||
session: Session, rows: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
|
||
) -> tuple[int, int]:
|
||
"""Apply a validated canonical waterbody snapshot without destructive sync.
|
||
|
||
Rows are matched by the RF4DB source identity first and by an exact existing
|
||
name second. Missing rows are deliberately left untouched: an incomplete
|
||
response must never withdraw a previously known waterbody.
|
||
"""
|
||
fetched_at = fetched_at or datetime.now(timezone.utc)
|
||
created = updated = 0
|
||
for raw in rows:
|
||
payload = _json_payload(raw)
|
||
if payload.get("source_system") != "rf4db":
|
||
raise CommunityImportError("waterbody catalog requires source_system=rf4db")
|
||
external_id = _required(payload, "source_external_id", 200)
|
||
name = _required(payload, "name", 200)
|
||
source_url = _required(payload, "source_url", 2000)
|
||
parsed_url = urlparse(source_url)
|
||
if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
|
||
raise CommunityImportError("waterbody source_url does not match rf4db")
|
||
unlock_level = _integer(payload.get("unlock_level"), minimum=0, maximum=1_000)
|
||
unlock_label = _required(payload, "unlock_label", 50)
|
||
fish_species_count = _integer(payload.get("fish_species_count"), minimum=0, maximum=10_000)
|
||
if fish_species_count is None:
|
||
raise CommunityImportError("invalid fish_species_count")
|
||
|
||
item = session.scalar(select(Waterbody).where(
|
||
Waterbody.source_system == "rf4db",
|
||
Waterbody.source_external_id == external_id,
|
||
))
|
||
if item is None:
|
||
item = session.scalar(select(Waterbody).where(Waterbody.name_ru == name))
|
||
if item is None:
|
||
item = Waterbody(
|
||
slug=_catalog_slug(session, name, external_id),
|
||
name_ru=name,
|
||
unlock_level=unlock_level,
|
||
)
|
||
session.add(item)
|
||
created += 1
|
||
else:
|
||
updated += 1
|
||
item.name_ru = name
|
||
item.unlock_level = unlock_level
|
||
item.fish_species_count = fish_species_count
|
||
item.source_system = "rf4db"
|
||
item.source_external_id = external_id
|
||
item.source_url = source_url
|
||
item.source_checked_at = fetched_at
|
||
session.commit()
|
||
return created, updated
|
||
|
||
|
||
def update_waterbody_detail(
|
||
session: Session, detail: dict[str, Any], *, fetched_at: datetime | None = None,
|
||
) -> bool:
|
||
"""Persist one complete RF4DB detail snapshot without assigning media roles."""
|
||
fetched_at = fetched_at or datetime.now(timezone.utc)
|
||
payload = _validate_waterbody_detail(detail)
|
||
_apply_waterbody_detail(session, payload, fetched_at=fetched_at)
|
||
session.commit()
|
||
return True
|
||
|
||
|
||
def update_waterbody_details(
|
||
session: Session, details: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
|
||
) -> tuple[int, int]:
|
||
"""Validate and apply a detail batch in one transaction."""
|
||
fetched_at = fetched_at or datetime.now(timezone.utc)
|
||
payloads = [_validate_waterbody_detail(detail) for detail in details]
|
||
external_ids = [str(payload["source_external_id"]) for payload in payloads]
|
||
if len(external_ids) != len(set(external_ids)):
|
||
raise CommunityImportError("waterbody detail batch contains duplicate source identities")
|
||
updated = 0
|
||
for payload in payloads:
|
||
_apply_waterbody_detail(session, payload, fetched_at=fetched_at)
|
||
updated += 1
|
||
session.commit()
|
||
return 0, updated
|
||
|
||
|
||
def _validate_waterbody_detail(detail: dict[str, Any]) -> dict[str, Any]:
|
||
payload = _json_payload(detail)
|
||
if payload.get("source_system") != "rf4db":
|
||
raise CommunityImportError("waterbody detail requires source_system=rf4db")
|
||
external_id = _required(payload, "source_external_id", 200)
|
||
source_url = _required(payload, "source_url", 2000)
|
||
parsed_url = urlparse(source_url)
|
||
if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
|
||
raise CommunityImportError("waterbody detail source_url does not match rf4db")
|
||
_required(payload, "name", 200)
|
||
_optional(payload, "description", 20_000)
|
||
_string_list(payload, "aliases", 100, 200)
|
||
_string_list(payload, "fish_species", 10_000, 200)
|
||
_string_list(payload, "image_urls", 100, 2_000)
|
||
_string_list(payload, "point_urls", 10_000, 2_000)
|
||
return payload
|
||
|
||
|
||
def _apply_waterbody_detail(session: Session, payload: dict[str, Any], *, fetched_at: datetime) -> None:
|
||
external_id = str(payload["source_external_id"])
|
||
source_url = str(payload["source_url"])
|
||
item = session.scalar(select(Waterbody).where(
|
||
Waterbody.source_system == "rf4db", Waterbody.source_external_id == external_id,
|
||
))
|
||
if item is None:
|
||
raise CommunityImportError("waterbody detail has no imported catalog identity")
|
||
item.description = _optional(payload, "description", 20_000)
|
||
item.source_aliases = _string_list(payload, "aliases", 100, 200)
|
||
item.source_fish_species = _string_list(payload, "fish_species", 10_000, 200)
|
||
item.source_image_urls = _string_list(payload, "image_urls", 100, 2_000)
|
||
item.source_point_urls = _string_list(payload, "point_urls", 10_000, 2_000)
|
||
item.source_url = source_url
|
||
item.source_checked_at = fetched_at
|
||
|
||
|
||
def _catalog_slug(session: Session, name: str, external_id: str) -> str:
|
||
base = re.sub(r"[^a-z0-9а-яё]+", "-", name.casefold(), flags=re.IGNORECASE).strip("-")
|
||
base = base or "waterbody"
|
||
candidate = base[:100]
|
||
if session.scalar(select(Waterbody.id).where(Waterbody.slug == candidate)) is None:
|
||
return candidate
|
||
suffix = hashlib.sha256(external_id.encode()).hexdigest()[:10]
|
||
return f"{base[:89]}-{suffix}"
|
||
|
||
|
||
def _string_list(payload: dict[str, Any], key: str, max_items: int, max_length: int) -> list[str]:
|
||
value = payload.get(key)
|
||
if not isinstance(value, list) or len(value) > max_items:
|
||
raise CommunityImportError(f"invalid {key}")
|
||
result = []
|
||
for item in value:
|
||
text = str(item).strip()
|
||
if not text or len(text) > max_length:
|
||
raise CommunityImportError(f"invalid {key}")
|
||
result.append(text)
|
||
return list(dict.fromkeys(result))
|
||
|
||
|
||
def stage_observations(
|
||
session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
|
||
) -> tuple[int, int]:
|
||
fetched_at = fetched_at or datetime.now(timezone.utc)
|
||
created = updated = 0
|
||
touched: list[ExternalObservation] = []
|
||
for raw in records:
|
||
payload = _json_payload(raw)
|
||
source_system = _required(payload, "source_system", 50)
|
||
if source_system not in SOURCE_DEFAULTS:
|
||
raise CommunityImportError(f"unsupported source_system: {source_system}")
|
||
external_id = _required(payload, "source_external_id", 200)
|
||
source = session.get(DataSource, source_system)
|
||
if source is None:
|
||
name, base_url, confidence = SOURCE_DEFAULTS[source_system]
|
||
source = DataSource(key=source_system, name=name, base_url=base_url, default_confidence=confidence, enabled=True)
|
||
session.add(source)
|
||
session.flush()
|
||
observation = session.scalar(select(ExternalObservation).where(
|
||
ExternalObservation.source_system == source_system,
|
||
ExternalObservation.source_external_id == external_id,
|
||
))
|
||
source_url = _required(payload, "source_url", 2000)
|
||
if urlparse(source_url).scheme != "https" or urlparse(source_url).hostname not in SOURCE_HOSTS[source_system]:
|
||
raise CommunityImportError("source_url does not match source_system")
|
||
values = {
|
||
"source_url": source_url,
|
||
"fish_name": _required(payload, "fish", 200),
|
||
"fish_external_id": _optional(payload, "fish_external_id", 200),
|
||
"waterbody_name": _required(payload, "waterbody", 200),
|
||
"waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
|
||
"x": _integer(payload.get("x"), maximum=10_000),
|
||
"y": _integer(payload.get("y"), maximum=10_000),
|
||
"coordinate_raw": _coordinate_raw(payload),
|
||
"coordinate_precision": _coordinate_precision(payload),
|
||
"weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
|
||
"published_at": _datetime(payload.get("published_at")),
|
||
"last_seen_at": fetched_at, "payload": payload,
|
||
"source_check_status": "available", "source_checked_at": fetched_at,
|
||
}
|
||
if observation is None:
|
||
observation = ExternalObservation(
|
||
source_system=source_system, source_external_id=external_id,
|
||
first_seen_at=fetched_at, status="staged", **values,
|
||
)
|
||
session.add(observation)
|
||
created += 1
|
||
else:
|
||
# Preserve the published snapshot, but withdraw it from activity until
|
||
# a moderator confirms the changed source record.
|
||
changed = any(getattr(observation, key) != values[key] for key in (
|
||
"source_url", "fish_name", "fish_external_id", "waterbody_name",
|
||
"waterbody_external_id", "x", "y", "weight_g",
|
||
"coordinate_raw", "coordinate_precision",
|
||
)) or observation.payload != payload
|
||
if observation.status != "rejected" and changed and observation.catch_report is not None:
|
||
observation.catch_report.moderation_status = ModerationStatus.pending
|
||
observation.status = "staged"
|
||
observation.fish = None
|
||
observation.waterbody = None
|
||
observation.review_note = "Source record changed; manual mapping and publication required"
|
||
observation.reviewed_at = fetched_at
|
||
observation.moderation_version += 1
|
||
for key, value in values.items():
|
||
setattr(observation, key, value)
|
||
if observation.status == "withdrawn":
|
||
observation.status = "staged"
|
||
observation.fish = None
|
||
observation.waterbody = None
|
||
observation.review_note = "Source record reappeared; manual confirmation required"
|
||
observation.reviewed_at = fetched_at
|
||
observation.moderation_version += 1
|
||
updated += 1
|
||
touched.append(observation)
|
||
session.commit()
|
||
for observation in touched:
|
||
_auto_publish(session, observation)
|
||
return created, updated
|
||
|
||
|
||
def _auto_publish(session: Session, observation: ExternalObservation) -> bool:
|
||
"""Publish only complete observations covered by previously reviewed aliases."""
|
||
if (
|
||
observation.catch_report_id is not None
|
||
or
|
||
observation.status not in {"staged", "mapped", "ready"}
|
||
or not observation.source.enabled
|
||
or observation.x is None
|
||
or observation.y is None
|
||
or observation.weight_g is None
|
||
):
|
||
return False
|
||
# Fish: prefer external alias, fall back to exact name match
|
||
fish = None
|
||
if observation.fish_external_id is not None:
|
||
fish_alias = session.scalar(select(ExternalEntityAlias).where(
|
||
ExternalEntityAlias.source_system == observation.source_system,
|
||
ExternalEntityAlias.entity_type == "fish",
|
||
ExternalEntityAlias.external_id == observation.fish_external_id,
|
||
))
|
||
if fish_alias and fish_alias.fish:
|
||
fish = fish_alias.fish
|
||
if fish is None:
|
||
# Fallback: exact name match
|
||
fish = session.scalar(
|
||
select(Fish).where(Fish.name_ru == observation.fish_name)
|
||
)
|
||
if fish is None:
|
||
return False
|
||
# Waterbody: prefer external alias, fall back to exact name match
|
||
waterbody = None
|
||
if observation.waterbody_external_id is not None:
|
||
waterbody_alias = session.scalar(select(ExternalEntityAlias).where(
|
||
ExternalEntityAlias.source_system == observation.source_system,
|
||
ExternalEntityAlias.entity_type == "waterbody",
|
||
ExternalEntityAlias.external_id == observation.waterbody_external_id,
|
||
))
|
||
if waterbody_alias and waterbody_alias.waterbody:
|
||
waterbody = waterbody_alias.waterbody
|
||
if waterbody is None:
|
||
# Fallback: exact name match
|
||
waterbody = session.scalar(
|
||
select(Waterbody).where(Waterbody.name_ru == observation.waterbody_name)
|
||
)
|
||
if waterbody is None:
|
||
return False
|
||
observation.fish = fish
|
||
observation.waterbody = waterbody
|
||
observation.status = "ready"
|
||
# A07: Describe actual matching method used
|
||
fish_method = "external_id" if observation.fish_external_id else "name"
|
||
wb_method = "external_id" if observation.waterbody_external_id else "name"
|
||
observation.review_note = f"Auto-matched: fish via {fish_method}, waterbody via {wb_method}"
|
||
publish_observation(session, observation)
|
||
return True
|
||
|
||
|
||
def _json_payload(raw: dict[str, Any]) -> dict[str, Any]:
|
||
if not isinstance(raw, dict):
|
||
raise CommunityImportError("each observation must be an object")
|
||
return json.loads(json.dumps(raw, default=str))
|
||
|
||
|
||
def _required(payload: dict[str, Any], key: str, limit: int) -> str:
|
||
value = str(payload.get(key) or "").strip()
|
||
if not value or len(value) > limit:
|
||
raise CommunityImportError(f"invalid {key}")
|
||
return value
|
||
|
||
|
||
def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
|
||
value = str(payload.get(key) or "").strip()
|
||
if len(value) > limit:
|
||
raise CommunityImportError(f"invalid {key}")
|
||
return value or None
|
||
|
||
|
||
def _coordinate_raw(payload: dict[str, Any]) -> str | None:
|
||
value = str(payload.get("coordinate_raw") or "").strip()
|
||
if len(value) > 200:
|
||
raise CommunityImportError("invalid coordinate_raw")
|
||
if value:
|
||
return value
|
||
x, y = payload.get("x"), payload.get("y")
|
||
return f"{x}:{y}" if isinstance(x, int) and isinstance(y, int) else None
|
||
|
||
|
||
def _coordinate_precision(payload: dict[str, Any]) -> str:
|
||
value = str(payload.get("coordinate_precision") or "").strip().casefold()
|
||
if not value:
|
||
return "exact" if isinstance(payload.get("x"), int) and isinstance(payload.get("y"), int) else "missing"
|
||
if value not in COORDINATE_PRECISIONS:
|
||
raise CommunityImportError("invalid coordinate_precision")
|
||
if value == "exact" and (not isinstance(payload.get("x"), int) or not isinstance(payload.get("y"), int)):
|
||
raise CommunityImportError("exact coordinates require x and y")
|
||
return value
|
||
|
||
|
||
def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
|
||
if value is None:
|
||
return None
|
||
try:
|
||
parsed = int(value)
|
||
except (TypeError, ValueError) as exc:
|
||
raise CommunityImportError("invalid integer field") from exc
|
||
if not minimum <= parsed <= maximum:
|
||
raise CommunityImportError("integer field outside allowed range")
|
||
return parsed
|
||
|
||
|
||
def _datetime(value: Any) -> datetime | None:
|
||
if value in {None, ""}:
|
||
return None
|
||
try:
|
||
parsed = datetime.fromisoformat(str(value))
|
||
except ValueError as exc:
|
||
raise CommunityImportError("invalid published_at") from exc
|
||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|