feat: serialize official record imports
CI / backend-and-migrations (push) Canceled after 0s
CI / astro-build (push) Canceled after 0s
CI / compose-e2e (push) Canceled after 0s

This commit is contained in:
ik
2026-09-07 07:54:07 +07:00
parent 687b4c9cb5
commit c45b4511b7
9 changed files with 115 additions and 13 deletions
+7
View File
@@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool
from app.database import Base, get_session
from app.community_importer import stage_observations
from app.importer import ImportAlreadyRunning
from app.main import app
from app.models import Bait, BaitKind, CatchReport, ExternalEntityAlias, ExternalObservation, Fish, ImportStatus, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
@@ -186,6 +187,12 @@ def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
listed = client.get("/api/v1/admin/imports?limit=1&offset=0", headers=headers)
assert listed.status_code == 200
assert listed.json()[0]["id"] == started.json()["id"]
def busy_import(*args, **kwargs):
raise ImportAlreadyRunning("official import is already running")
monkeypatch.setattr("app.main.import_records", busy_import)
conflict = client.post("/api/v1/admin/imports/official-records", headers=headers)
assert conflict.status_code == 409
def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
@@ -0,0 +1,27 @@
import os
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from app.importer import ImportAlreadyRunning, _official_import_lock
@pytest.mark.skipif(not os.environ.get("DATABASE_URL", "").startswith("postgresql"), reason="requires PostgreSQL")
def test_postgresql_import_lock_blocks_only_same_source_category() -> None:
engine = create_engine(os.environ["DATABASE_URL"])
first = Session(engine)
second = Session(engine)
try:
with _official_import_lock(first, url="https://example.test/records", region="RU", category="records"):
with pytest.raises(ImportAlreadyRunning):
with _official_import_lock(second, url="https://example.test/records", region="RU", category="records"):
pass
with _official_import_lock(second, url="https://example.test/records", region="RU", category="weekly"):
pass
with _official_import_lock(second, url="https://example.test/records", region="RU", category="records"):
pass
finally:
first.close()
second.close()
engine.dispose()
+21 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session
from app.database import Base
from app.importer import FetchResult, ImportSourceError, import_records, parse_html
from app.importer import FetchResult, ImportAlreadyRunning, ImportSourceError, _lock_key, _official_import_lock, import_records, parse_html
from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
@@ -15,6 +15,26 @@ FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "records_ru_sample.
WEEKLY_FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "weekly_records_sample.html"
def test_import_lock_is_stable_and_fails_closed_when_busy() -> None:
class Connection:
def scalar(self, statement, parameters):
assert "pg_try_advisory_lock" in str(statement)
assert parameters == {"key": _lock_key("https://example.test", "RU", "records")}
return False
def close(self):
self.closed = True
connection = Connection()
bind = type("Bind", (), {"dialect": type("Dialect", (), {"name": "postgresql"})(), "connect": lambda self: connection})()
session = type("Session", (), {"get_bind": lambda self: bind})()
assert _lock_key("https://example.test", "ru", "records") == _lock_key("https://example.test", "RU", "records")
with pytest.raises(ImportAlreadyRunning, match="already running"):
with _official_import_lock(session, url="https://example.test", region="RU", category="records"):
raise AssertionError("busy lock must not enter import")
assert connection.closed is True
def test_parser_and_import_are_idempotent() -> None:
html = FIXTURE.read_text(encoding="utf-8")
parsed = parse_html(html, region="RU", category="records")