Add conditional caching and opt-in import scheduler
This commit is contained in:
@@ -7,7 +7,7 @@ from sqlalchemy import create_engine, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import Base
|
||||
from app.importer import ImportSourceError, import_records, parse_html
|
||||
from app.importer import FetchResult, ImportSourceError, import_records, parse_html
|
||||
from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
|
||||
|
||||
|
||||
@@ -55,3 +55,28 @@ def test_import_rejects_changed_column_contract() -> None:
|
||||
)
|
||||
with pytest.raises(ImportSourceError, match="record columns changed"):
|
||||
parse_html(html, region="RU", category="records")
|
||||
|
||||
|
||||
def test_import_reuses_http_validators_and_handles_not_modified(monkeypatch) -> None:
|
||||
html = FIXTURE.read_text(encoding="utf-8")
|
||||
parsed = parse_html(html, region="RU", category="records")
|
||||
calls: list[tuple[str | None, str | None]] = []
|
||||
|
||||
def fake_fetch(url: str, *, region: str, category: str, etag: str | None, last_modified: str | None) -> FetchResult:
|
||||
calls.append((etag, last_modified))
|
||||
if len(calls) == 1:
|
||||
return FetchResult(parsed, 200, '"fixture-v1"', "Wed, 02 Sep 2026 00:00:00 GMT", "text/html", len(html))
|
||||
return FetchResult(None, 304, None, None, None, 0)
|
||||
|
||||
monkeypatch.setattr("app.importer.fetch_records", fake_fetch)
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as db:
|
||||
first = import_records(db, url="https://example.test/records", region="RU", category="records")
|
||||
second = import_records(db, url="https://example.test/records", region="RU", category="records")
|
||||
assert calls == [(None, None), ('"fixture-v1"', "Wed, 02 Sep 2026 00:00:00 GMT")]
|
||||
assert first.response_status == 200
|
||||
assert second.response_status == 304
|
||||
assert second.not_modified is True
|
||||
assert second.response_etag == '"fixture-v1"'
|
||||
assert second.rows_seen == 0
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import Base
|
||||
from app.models import ImportStatus, OfficialRecordImport
|
||||
from app.scheduler import import_is_due
|
||||
|
||||
|
||||
def test_import_is_due_respects_minimum_interval() -> None:
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
now = datetime.now(timezone.utc)
|
||||
with Session(engine) as db:
|
||||
assert import_is_due(db, now=now) is True
|
||||
db.add(OfficialRecordImport(
|
||||
started_at=now - timedelta(seconds=settings.import_interval_seconds - 1),
|
||||
finished_at=now,
|
||||
status=ImportStatus.success,
|
||||
source_url=settings.official_records_url,
|
||||
rows_seen=0,
|
||||
rows_created=0,
|
||||
rows_updated=0,
|
||||
))
|
||||
db.commit()
|
||||
assert import_is_due(db, now=now) is False
|
||||
assert import_is_due(db, now=now + timedelta(seconds=settings.import_interval_seconds)) is True
|
||||
Reference in New Issue
Block a user