393 lines
17 KiB
Python
393 lines
17 KiB
Python
import json
|
|
from dataclasses import asdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.community_importer import CommunityImportError, stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog
|
|
from app.community_review import ExternalReviewError, map_observation, publish_observation, suggest_aliases
|
|
from app.source_lifecycle import record_scheduled_source_check, record_source_check
|
|
from app.database import Base
|
|
from app.models import CatchReport, DataSource, ExternalEntityAlias, ExternalObservation, Fish, Waterbody
|
|
from rf4_research.community_sources import parse_rf4db_catches, parse_rf4map_point, parse_rf4posts_spot
|
|
|
|
|
|
FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "rf4db_catches_sample.html"
|
|
FIXTURES = FIXTURE.parent
|
|
|
|
|
|
def record(source: str = "rf4db", external_id: str = "catch-1") -> dict[str, object]:
|
|
urls = {
|
|
"rf4db": f"https://rf4db.com/ru/catches/{external_id}",
|
|
"rf4stat-fishing": f"https://rf4-stat.ru/fishing/{external_id}",
|
|
"rf4map": f"https://rf4map.ru/points/{external_id}",
|
|
"rf4posts-spot": f"https://rf4-posts.com/ru/spots/{external_id}",
|
|
}
|
|
return {
|
|
"source_system": source,
|
|
"source_external_id": external_id,
|
|
"source_url": urls.get(source, f"https://rf4-stat.ru/fishing/{external_id}"),
|
|
"fish": "Щука",
|
|
"fish_external_id": "pike",
|
|
"waterbody": "Тестовое озеро",
|
|
"waterbody_external_id": "test-lake",
|
|
"x": 71,
|
|
"y": 92,
|
|
"weight_g": None,
|
|
"published_at": None,
|
|
"bait": "Приманка",
|
|
}
|
|
|
|
|
|
def waterbody_row(**overrides: object) -> dict[str, object]:
|
|
row: dict[str, object] = {
|
|
"source_system": "rf4db",
|
|
"source_external_id": "level_001_mosquito",
|
|
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito",
|
|
"name": "оз. Комариное",
|
|
"unlock_level": 1,
|
|
"unlock_label": "1",
|
|
"fish_species_count": 20,
|
|
}
|
|
row.update(overrides)
|
|
return row
|
|
|
|
|
|
@pytest.fixture
|
|
def db() -> Session:
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
yield session
|
|
engine.dispose()
|
|
|
|
|
|
def test_staging_is_idempotent_and_preserves_first_seen(db: Session) -> None:
|
|
first = datetime(2026, 9, 3, 10, tzinfo=timezone.utc)
|
|
second = datetime(2026, 9, 3, 11, tzinfo=timezone.utc)
|
|
|
|
assert stage_observations(db, [record()], fetched_at=first) == (1, 0)
|
|
changed = record() | {"x": 73, "weight_g": 5_000}
|
|
assert stage_observations(db, [changed], fetched_at=second) == (0, 1)
|
|
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None
|
|
assert (item.x, item.weight_g, item.status) == (73, 5_000, "staged")
|
|
assert item.first_seen_at.replace(tzinfo=timezone.utc) == first
|
|
assert item.last_seen_at.replace(tzinfo=timezone.utc) == second
|
|
assert db.scalar(select(func.count()).select_from(ExternalObservation)) == 1
|
|
source = db.get(DataSource, "rf4db")
|
|
assert source is not None and source.enabled is True
|
|
|
|
|
|
def test_waterbody_catalog_upsert_is_idempotent_and_non_destructive(db: Session) -> None:
|
|
first = datetime(2026, 9, 16, 10, tzinfo=timezone.utc)
|
|
assert upsert_waterbody_catalog(db, [waterbody_row()], fetched_at=first) == (1, 0)
|
|
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
|
|
assert item is not None
|
|
assert item.slug == "оз-комариное"
|
|
assert item.source_checked_at.replace(tzinfo=timezone.utc) == first
|
|
assert item.fish_species_count == 20
|
|
|
|
assert upsert_waterbody_catalog(db, [waterbody_row(name="Озеро Комариное", unlock_level=2)], fetched_at=first) == (0, 1)
|
|
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
|
|
assert item is not None
|
|
assert (item.name_ru, item.unlock_level, item.fish_species_count) == ("Озеро Комариное", 2, 20)
|
|
assert db.scalar(select(Waterbody).where(Waterbody.name_ru == "оз. Комариное")) is None
|
|
|
|
|
|
def test_waterbody_catalog_rejects_untrusted_source(db: Session) -> None:
|
|
with pytest.raises(CommunityImportError, match="source_url"):
|
|
upsert_waterbody_catalog(db, [waterbody_row(source_url="https://example.test/map")])
|
|
|
|
|
|
def test_waterbody_detail_updates_only_imported_identity_without_media_roles(db: Session) -> None:
|
|
upsert_waterbody_catalog(db, [waterbody_row()])
|
|
assert update_waterbody_detail(db, {
|
|
"source_system": "rf4db",
|
|
"source_external_id": "level_001_mosquito",
|
|
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito",
|
|
"name": "оз. Комариное",
|
|
"description": "Каменистые берега.",
|
|
"aliases": ["Комариное", "Комариное"],
|
|
"fish_species": ["Щука", "Окунь"],
|
|
"image_urls": ["https://oss.rf4db.com/map.webp"],
|
|
"point_urls": ["https://rf4db.com/ru/maps/level_001_mosquito/spots/12-34"],
|
|
}) is True
|
|
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
|
|
assert item is not None
|
|
assert item.source_aliases == ["Комариное"]
|
|
assert item.source_fish_species == ["Щука", "Окунь"]
|
|
assert item.source_image_urls == ["https://oss.rf4db.com/map.webp"]
|
|
|
|
|
|
def test_waterbody_detail_batch_validates_before_writing(db: Session) -> None:
|
|
upsert_waterbody_catalog(db, [waterbody_row()])
|
|
valid = {
|
|
"source_system": "rf4db", "source_external_id": "level_001_mosquito",
|
|
"source_url": "https://rf4db.com/ru/maps/level_001_mosquito", "name": "оз. Комариное",
|
|
"description": "Описание", "aliases": [], "fish_species": ["Щука"],
|
|
"image_urls": [], "point_urls": [],
|
|
}
|
|
invalid = valid | {"source_external_id": "unknown", "source_url": "https://example.test/map"}
|
|
with pytest.raises(CommunityImportError, match="source_url"):
|
|
update_waterbody_details(db, [valid, invalid])
|
|
item = db.scalar(select(Waterbody).where(Waterbody.source_external_id == "level_001_mosquito"))
|
|
assert item is not None and item.description is None
|
|
|
|
|
|
def test_external_ids_are_isolated_by_source(db: Session) -> None:
|
|
created, updated = stage_observations(db, [record("rf4db"), record("rf4stat-fishing")])
|
|
|
|
assert (created, updated) == (2, 0)
|
|
|
|
|
|
def test_research_sources_enter_enabled_staging(db: Session) -> None:
|
|
created, updated = stage_observations(db, [record("rf4map"), record("rf4posts-spot")])
|
|
|
|
assert (created, updated) == (2, 0)
|
|
assert db.get(DataSource, "rf4map").enabled is True
|
|
assert db.get(DataSource, "rf4posts-spot").enabled is True
|
|
|
|
|
|
def test_complete_observation_with_reviewed_aliases_is_published(db: Session) -> None:
|
|
source = DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)
|
|
fish = Fish(slug="pike", name_ru="Щука")
|
|
waterbody = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([source, fish, waterbody])
|
|
db.flush()
|
|
now = datetime.now(timezone.utc)
|
|
db.add_all([
|
|
ExternalEntityAlias(source_system="rf4db", entity_type="fish", external_id="pike", external_name="Щука", fish=fish, updated_at=now),
|
|
ExternalEntityAlias(source_system="rf4db", entity_type="waterbody", external_id="test-lake", external_name="Тестовое озеро", waterbody=waterbody, updated_at=now),
|
|
])
|
|
db.commit()
|
|
|
|
assert stage_observations(db, [record() | {"weight_g": 5_000}]) == (1, 0)
|
|
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None
|
|
assert item.status == "published"
|
|
assert item.catch_report is not None
|
|
assert item.catch_report.fish_id == fish.id
|
|
assert item.catch_report.waterbody_id == waterbody.id
|
|
assert stage_observations(db, [record() | {"weight_g": 5_000}]) == (0, 1)
|
|
db.refresh(item)
|
|
assert item.status == "published"
|
|
assert db.scalar(select(func.count()).select_from(CatchReport)) == 1
|
|
|
|
|
|
def test_observation_preserves_coordinate_text_and_precision(db: Session) -> None:
|
|
stage_observations(db, [record() | {
|
|
"source_external_id": "coordinate-area",
|
|
"x": None, "y": None, "coordinate_raw": "северная бухта",
|
|
"coordinate_precision": "area", "weight_g": None,
|
|
}])
|
|
item = db.scalar(select(ExternalObservation).where(ExternalObservation.source_external_id == "coordinate-area"))
|
|
assert item is not None
|
|
assert (item.coordinate_raw, item.coordinate_precision, item.x, item.y) == ("северная бухта", "area", None, None)
|
|
|
|
|
|
def test_coordinate_precision_rejects_unknown_value(db: Session) -> None:
|
|
with pytest.raises(CommunityImportError, match="invalid coordinate_precision"):
|
|
stage_observations(db, [record() | {"coordinate_precision": "guess"}])
|
|
|
|
|
|
def test_changed_published_record_requires_review_and_reuses_report(db: Session) -> None:
|
|
fish = Fish(slug="pike", name_ru="Щука")
|
|
water = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([fish, water])
|
|
db.commit()
|
|
# D04: auto-publish now works with name match fallback
|
|
stage_observations(db, [record() | {"weight_g": 5000}])
|
|
item = db.scalar(select(ExternalObservation))
|
|
# Item auto-published via name match fallback (D04)
|
|
assert item.status == "published"
|
|
report = publish_observation(db, item)
|
|
report_id = report.id
|
|
stage_observations(db, [record() | {"weight_g": 6000}])
|
|
assert item.status == "staged"
|
|
assert report.moderation_status.value == "pending"
|
|
assert report.weight_g == 5000
|
|
assert item.weight_g == 6000
|
|
assert item.moderation_version == 1
|
|
assert item.reviewed_at is not None
|
|
stage_observations(db, [record() | {"weight_g": 6000}])
|
|
assert item.status == "staged"
|
|
with pytest.raises(ExternalReviewError):
|
|
publish_observation(db, item)
|
|
map_observation(db, item, fish, water)
|
|
updated = publish_observation(db, item)
|
|
assert updated.id == report_id
|
|
assert updated.weight_g == 6000
|
|
assert updated.moderation_status.value == "approved"
|
|
assert db.scalar(select(func.count()).select_from(CatchReport)) == 1
|
|
|
|
|
|
def test_missing_source_withdraws_published_record_until_manual_review(db: Session) -> None:
|
|
fish = Fish(slug="pike", name_ru="Щука")
|
|
water = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([fish, water])
|
|
db.commit()
|
|
seen = datetime(2026, 9, 13, 8, tzinfo=timezone.utc)
|
|
checked = datetime(2026, 9, 13, 9, tzinfo=timezone.utc)
|
|
stage_observations(db, [record() | {"weight_g": 5000}], fetched_at=seen)
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None and item.catch_report is not None
|
|
|
|
record_source_check(db, item, "missing", checked_at=checked)
|
|
|
|
assert item.status == "withdrawn"
|
|
assert item.source_check_status == "missing"
|
|
assert item.source_checked_at.replace(tzinfo=timezone.utc) == checked
|
|
assert item.catch_report.moderation_status.value == "pending"
|
|
assert item.moderation_version == 1
|
|
|
|
stage_observations(db, [record() | {"weight_g": 5000}], fetched_at=checked)
|
|
assert item.status == "staged"
|
|
assert item.source_check_status == "available"
|
|
assert item.catch_report.moderation_status.value == "pending"
|
|
assert "reappeared" in (item.review_note or "")
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["temporary_error", "blocked"])
|
|
def test_non_authoritative_source_failures_do_not_withdraw(db: Session, status: str) -> None:
|
|
fish = Fish(slug="pike", name_ru="Щука")
|
|
water = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([fish, water])
|
|
db.commit()
|
|
stage_observations(db, [record() | {"weight_g": 5000}])
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None and item.catch_report is not None
|
|
|
|
record_source_check(db, item, status) # type: ignore[arg-type]
|
|
|
|
assert item.status == "published"
|
|
assert item.catch_report.moderation_status.value == "approved"
|
|
assert item.source_check_status == status
|
|
|
|
|
|
def test_scheduled_failure_only_affects_exact_source_url(db: Session) -> None:
|
|
stage_observations(db, [
|
|
record(external_id="matching"),
|
|
record(external_id="other") | {"source_url": "https://rf4db.com/catches/other"},
|
|
])
|
|
|
|
affected = record_scheduled_source_check(
|
|
db,
|
|
source_system="rf4db",
|
|
source_url="https://rf4db.com/ru/catches/matching",
|
|
status="missing",
|
|
)
|
|
items = {item.source_external_id: item for item in db.scalars(select(ExternalObservation))}
|
|
|
|
assert affected == 1
|
|
assert items["matching"].status == "withdrawn"
|
|
assert items["other"].status != "withdrawn"
|
|
assert items["other"].source_check_status == "available"
|
|
|
|
record_scheduled_source_check(
|
|
db,
|
|
source_system="rf4db",
|
|
source_url="https://rf4db.com/ru/catches/matching",
|
|
status="available",
|
|
)
|
|
assert items["matching"].source_check_status == "available"
|
|
assert items["matching"].status == "withdrawn"
|
|
|
|
|
|
def test_auto_publication_requires_enabled_source(db: Session) -> None:
|
|
source = DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=False)
|
|
fish = Fish(slug="pike", name_ru="Щука")
|
|
waterbody = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([source, fish, waterbody])
|
|
db.flush()
|
|
now = datetime.now(timezone.utc)
|
|
db.add_all([
|
|
ExternalEntityAlias(source_system="rf4db", entity_type="fish", external_id="pike", external_name="Щука", fish=fish, updated_at=now),
|
|
ExternalEntityAlias(source_system="rf4db", entity_type="waterbody", external_id="test-lake", external_name="Тестовое озеро", waterbody=waterbody, updated_at=now),
|
|
])
|
|
db.commit()
|
|
|
|
stage_observations(db, [record() | {"weight_g": 5_000}])
|
|
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None and item.status == "staged" and item.catch_report is None
|
|
|
|
|
|
def test_confirmed_aliases_are_suggestions_and_cannot_be_retargeted(db: Session) -> None:
|
|
source = DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)
|
|
pike = Fish(slug="pike", name_ru="Щука")
|
|
perch = Fish(slug="perch", name_ru="Окунь")
|
|
waterbody = Waterbody(slug="test-lake", name_ru="Тестовое озеро")
|
|
db.add_all([source, pike, perch, waterbody])
|
|
db.commit()
|
|
stage_observations(db, [record(external_id="first")])
|
|
first = db.scalar(select(ExternalObservation).where(ExternalObservation.source_external_id == "first"))
|
|
assert first is not None
|
|
map_observation(db, first, pike, waterbody)
|
|
|
|
stage_observations(db, [record(external_id="second")])
|
|
second = db.scalar(select(ExternalObservation).where(ExternalObservation.source_external_id == "second"))
|
|
assert second is not None
|
|
suggested_fish, suggested_waterbody = suggest_aliases(db, second)
|
|
assert (suggested_fish.slug, suggested_waterbody.slug) == ("pike", "test-lake")
|
|
|
|
with pytest.raises(ExternalReviewError, match="confirmed fish alias"):
|
|
map_observation(db, second, perch, waterbody)
|
|
db.rollback()
|
|
alias = db.scalar(select(ExternalEntityAlias).where(ExternalEntityAlias.entity_type == "fish"))
|
|
assert alias is not None and alias.fish_id == pike.id
|
|
|
|
|
|
def test_parser_json_can_be_staged_without_losing_provenance(db: Session) -> None:
|
|
parsed = parse_rf4db_catches(FIXTURE.read_text(encoding="utf-8"))
|
|
payload = json.loads(json.dumps([asdict(item) for item in parsed], default=str))
|
|
|
|
assert stage_observations(db, payload) == (1, 0)
|
|
item = db.scalar(select(ExternalObservation))
|
|
assert item is not None
|
|
assert (item.source_system, item.fish_external_id, item.x, item.y) == ("rf4db", "pike", 71, 92)
|
|
|
|
|
|
@pytest.mark.parametrize(("fixture_name", "source_url", "parser"), [
|
|
("rf4map_point_sample.html", "https://rf4map.ru/points/275", parse_rf4map_point),
|
|
(
|
|
"rf4posts_spot_sample.html",
|
|
"https://rf4-posts.com/ru/spots/d0c6d9c6-4ebf-49a7-98a8-9a562553a8ee",
|
|
parse_rf4posts_spot,
|
|
),
|
|
])
|
|
def test_new_parser_json_can_be_staged(
|
|
db: Session, fixture_name: str, source_url: str, parser,
|
|
) -> None:
|
|
parsed = parser((FIXTURES / fixture_name).read_text(encoding="utf-8"), source_url=source_url)
|
|
payload = json.loads(json.dumps([asdict(item) for item in parsed], default=str))
|
|
|
|
created, updated = stage_observations(db, payload)
|
|
|
|
assert (created, updated) == (len(parsed), 0)
|
|
assert db.scalar(select(func.count()).select_from(ExternalObservation)) == len(parsed)
|
|
|
|
|
|
def test_invalid_source_rolls_back_caller_transaction(db: Session) -> None:
|
|
with pytest.raises(CommunityImportError, match="unsupported source_system"):
|
|
stage_observations(db, [record("unknown")])
|
|
db.rollback()
|
|
|
|
assert db.scalar(select(func.count()).select_from(ExternalObservation)) == 0
|
|
|
|
|
|
@pytest.mark.parametrize("change", [
|
|
{"source_url": "https://attacker.example/catch-1"},
|
|
{"x": 10_001},
|
|
{"weight_g": 3_000_001},
|
|
])
|
|
def test_invalid_provenance_and_ranges_are_rejected(db: Session, change: dict[str, object]) -> None:
|
|
with pytest.raises(CommunityImportError):
|
|
stage_observations(db, [record() | change])
|
|
db.rollback()
|