Add conditional caching and opt-in import scheduler

This commit is contained in:
ik
2026-09-03 08:04:57 +07:00
parent ddd8b84055
commit a528fc707f
13 changed files with 248 additions and 9 deletions
+26 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session
from app.database import Base
from app.importer import ImportSourceError, import_records, parse_html
from app.importer import FetchResult, ImportSourceError, import_records, parse_html
from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
@@ -55,3 +55,28 @@ def test_import_rejects_changed_column_contract() -> None:
)
with pytest.raises(ImportSourceError, match="record columns changed"):
parse_html(html, region="RU", category="records")
def test_import_reuses_http_validators_and_handles_not_modified(monkeypatch) -> None:
html = FIXTURE.read_text(encoding="utf-8")
parsed = parse_html(html, region="RU", category="records")
calls: list[tuple[str | None, str | None]] = []
def fake_fetch(url: str, *, region: str, category: str, etag: str | None, last_modified: str | None) -> FetchResult:
calls.append((etag, last_modified))
if len(calls) == 1:
return FetchResult(parsed, 200, '"fixture-v1"', "Wed, 02 Sep 2026 00:00:00 GMT", "text/html", len(html))
return FetchResult(None, 304, None, None, None, 0)
monkeypatch.setattr("app.importer.fetch_records", fake_fetch)
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
with Session(engine) as db:
first = import_records(db, url="https://example.test/records", region="RU", category="records")
second = import_records(db, url="https://example.test/records", region="RU", category="records")
assert calls == [(None, None), ('"fixture-v1"', "Wed, 02 Sep 2026 00:00:00 GMT")]
assert first.response_status == 200
assert second.response_status == 304
assert second.not_modified is True
assert second.response_etag == '"fixture-v1"'
assert second.rows_seen == 0
+31
View File
@@ -0,0 +1,31 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from app.config import settings
from app.database import Base
from app.models import ImportStatus, OfficialRecordImport
from app.scheduler import import_is_due
def test_import_is_due_respects_minimum_interval() -> None:
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
with Session(engine) as db:
assert import_is_due(db, now=now) is True
db.add(OfficialRecordImport(
started_at=now - timedelta(seconds=settings.import_interval_seconds - 1),
finished_at=now,
status=ImportStatus.success,
source_url=settings.official_records_url,
rows_seen=0,
rows_created=0,
rows_updated=0,
))
db.commit()
assert import_is_due(db, now=now) is False
assert import_is_due(db, now=now + timedelta(seconds=settings.import_interval_seconds)) is True