feat: serialize official record imports
This commit is contained in:
@@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.database import Base, get_session
|
||||
from app.community_importer import stage_observations
|
||||
from app.importer import ImportAlreadyRunning
|
||||
from app.main import app
|
||||
from app.models import Bait, BaitKind, CatchReport, ExternalEntityAlias, ExternalObservation, Fish, ImportStatus, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
|
||||
|
||||
@@ -186,6 +187,12 @@ def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
|
||||
listed = client.get("/api/v1/admin/imports?limit=1&offset=0", headers=headers)
|
||||
assert listed.status_code == 200
|
||||
assert listed.json()[0]["id"] == started.json()["id"]
|
||||
def busy_import(*args, **kwargs):
|
||||
raise ImportAlreadyRunning("official import is already running")
|
||||
|
||||
monkeypatch.setattr("app.main.import_records", busy_import)
|
||||
conflict = client.post("/api/v1/admin/imports/official-records", headers=headers)
|
||||
assert conflict.status_code == 409
|
||||
|
||||
|
||||
def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.importer import ImportAlreadyRunning, _official_import_lock
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.environ.get("DATABASE_URL", "").startswith("postgresql"), reason="requires PostgreSQL")
|
||||
def test_postgresql_import_lock_blocks_only_same_source_category() -> None:
|
||||
engine = create_engine(os.environ["DATABASE_URL"])
|
||||
first = Session(engine)
|
||||
second = Session(engine)
|
||||
try:
|
||||
with _official_import_lock(first, url="https://example.test/records", region="RU", category="records"):
|
||||
with pytest.raises(ImportAlreadyRunning):
|
||||
with _official_import_lock(second, url="https://example.test/records", region="RU", category="records"):
|
||||
pass
|
||||
with _official_import_lock(second, url="https://example.test/records", region="RU", category="weekly"):
|
||||
pass
|
||||
with _official_import_lock(second, url="https://example.test/records", region="RU", category="records"):
|
||||
pass
|
||||
finally:
|
||||
first.close()
|
||||
second.close()
|
||||
engine.dispose()
|
||||
@@ -7,7 +7,7 @@ from sqlalchemy import create_engine, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import Base
|
||||
from app.importer import FetchResult, ImportSourceError, import_records, parse_html
|
||||
from app.importer import FetchResult, ImportAlreadyRunning, ImportSourceError, _lock_key, _official_import_lock, import_records, parse_html
|
||||
from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
|
||||
|
||||
|
||||
@@ -15,6 +15,26 @@ FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "records_ru_sample.
|
||||
WEEKLY_FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "weekly_records_sample.html"
|
||||
|
||||
|
||||
def test_import_lock_is_stable_and_fails_closed_when_busy() -> None:
|
||||
class Connection:
|
||||
def scalar(self, statement, parameters):
|
||||
assert "pg_try_advisory_lock" in str(statement)
|
||||
assert parameters == {"key": _lock_key("https://example.test", "RU", "records")}
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
connection = Connection()
|
||||
bind = type("Bind", (), {"dialect": type("Dialect", (), {"name": "postgresql"})(), "connect": lambda self: connection})()
|
||||
session = type("Session", (), {"get_bind": lambda self: bind})()
|
||||
assert _lock_key("https://example.test", "ru", "records") == _lock_key("https://example.test", "RU", "records")
|
||||
with pytest.raises(ImportAlreadyRunning, match="already running"):
|
||||
with _official_import_lock(session, url="https://example.test", region="RU", category="records"):
|
||||
raise AssertionError("busy lock must not enter import")
|
||||
assert connection.closed is True
|
||||
|
||||
|
||||
def test_parser_and_import_are_idempotent() -> None:
|
||||
html = FIXTURE.read_text(encoding="utf-8")
|
||||
parsed = parse_html(html, region="RU", category="records")
|
||||
|
||||
Reference in New Issue
Block a user