Add conditional caching and opt-in import scheduler

apps/api/app/config.py

@@ -1,3 +1,4 @@
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -13,6 +14,7 @@ class Settings(BaseSettings):
     official_records_url: str = "https://rf4game.de/records/region/RU/"
     official_records_region: str = "RU"
     official_records_category: str = "records"
+    import_interval_seconds: int = Field(default=3600, ge=3600)
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 

apps/api/app/importer.py

@@ -36,6 +36,16 @@ class RawRecord:
     record_date: date
 
 
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    records: list[RawRecord] | None
+    status_code: int
+    etag: str | None
+    last_modified: str | None
+    content_type: str | None
+    response_bytes: int
+
+
 def normalize(value: str) -> str:
     return " ".join(value.replace("\xa0", " ").replace("–", "-").replace("—", "-").split()).casefold()
 
@@ -100,15 +110,32 @@ def parse_html(html: str, *, region: str, category: str) -> list[RawRecord]:
     return records
 
 
-def fetch_records(url: str, *, region: str, category: str) -> list[RawRecord]:
-    with httpx.Client(timeout=20, follow_redirects=True, headers={"User-Agent": USER_AGENT, "Accept": "text/html"}) as client:
+def fetch_records(
+    url: str, *, region: str, category: str,
+    etag: str | None = None, last_modified: str | None = None,
+) -> FetchResult:
+    headers = {"User-Agent": USER_AGENT, "Accept": "text/html"}
+    if etag:
+        headers["If-None-Match"] = etag
+    if last_modified:
+        headers["If-Modified-Since"] = last_modified
+    with httpx.Client(timeout=20, follow_redirects=True, headers=headers) as client:
         for attempt in range(3):
             try:
                 response = client.get(url)
+                metadata = {
+                    "status_code": response.status_code,
+                    "etag": response.headers.get("etag"),
+                    "last_modified": response.headers.get("last-modified"),
+                    "content_type": response.headers.get("content-type"),
+                    "response_bytes": len(response.content),
+                }
+                if response.status_code == 304:
+                    return FetchResult(records=None, **metadata)
                 response.raise_for_status()
                 if "text/html" not in response.headers.get("content-type", ""):
                     raise ImportSourceError("source did not return HTML")
-                return parse_html(response.text, region=region, category=category)
+                return FetchResult(records=parse_html(response.text, region=region, category=category), **metadata)
             except (httpx.HTTPError, ImportSourceError):
                 if attempt == 2:
                     raise
@@ -121,7 +148,32 @@ def import_records(session: Session, *, url: str, region: str, category: str, ht
     session.add(run)
     session.commit()
     try:
-        records = parse_html(html, region=region, category=category) if html is not None else fetch_records(url, region=region, category=category)
+        if html is not None:
+            records = parse_html(html, region=region, category=category)
+        else:
+            previous = session.scalar(
+                select(OfficialRecordImport).where(
+                    OfficialRecordImport.source_url == url,
+                    OfficialRecordImport.status == ImportStatus.success,
+                ).order_by(OfficialRecordImport.started_at.desc()).limit(1)
+            )
+            fetched = fetch_records(
+                url, region=region, category=category,
+                etag=previous.response_etag if previous else None,
+                last_modified=previous.response_last_modified if previous else None,
+            )
+            run.response_status = fetched.status_code
+            run.response_etag = fetched.etag or (previous.response_etag if previous else None)
+            run.response_last_modified = fetched.last_modified or (previous.response_last_modified if previous else None)
+            run.response_content_type = fetched.content_type
+            run.response_bytes = fetched.response_bytes
+            if fetched.records is None:
+                run.not_modified = True
+                run.status = ImportStatus.success
+                run.finished_at = datetime.now(timezone.utc)
+                session.commit()
+                return run
+            records = fetched.records
         run.rows_seen = len(records)
         for raw in records:
             key = external_id(raw)

apps/api/app/models.py

@@ -110,6 +110,12 @@ class OfficialRecordImport(Base):
     rows_created: Mapped[int] = mapped_column(default=0)
     rows_updated: Mapped[int] = mapped_column(default=0)
     error_summary: Mapped[str | None] = mapped_column(Text)
+    response_status: Mapped[int | None]
+    response_etag: Mapped[str | None] = mapped_column(String(500))
+    response_last_modified: Mapped[str | None] = mapped_column(String(500))
+    response_content_type: Mapped[str | None] = mapped_column(String(200))
+    response_bytes: Mapped[int | None]
+    not_modified: Mapped[bool] = mapped_column(default=False)
 
 
 class ModerationEvent(Base):

apps/api/app/scheduler.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import logging
+import time
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .config import settings
+from .database import SessionLocal
+from .importer import import_records
+from .models import OfficialRecordImport
+
+
+logger = logging.getLogger("rf4.import_scheduler")
+
+
+def import_is_due(session: Session, *, now: datetime | None = None) -> bool:
+    current = now or datetime.now(timezone.utc)
+    latest = session.scalar(
+        select(OfficialRecordImport.started_at)
+        .where(OfficialRecordImport.source_url == settings.official_records_url)
+        .order_by(OfficialRecordImport.started_at.desc())
+        .limit(1)
+    )
+    if latest is None:
+        return True
+    if latest.tzinfo is None:
+        latest = latest.replace(tzinfo=timezone.utc)
+    return latest <= current - timedelta(seconds=settings.import_interval_seconds)
+
+
+def run_due_import() -> bool:
+    with SessionLocal() as session:
+        if not import_is_due(session):
+            return False
+        run = import_records(
+            session,
+            url=settings.official_records_url,
+            region=settings.official_records_region,
+            category=settings.official_records_category,
+        )
+        logger.info(
+            "official import completed status=%s seen=%d created=%d updated=%d not_modified=%s",
+            run.status.value, run.rows_seen, run.rows_created, run.rows_updated, run.not_modified,
+        )
+        return True
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
+    logger.info("scheduler started interval_seconds=%d", settings.import_interval_seconds)
+    while True:
+        try:
+            run_due_import()
+        except Exception:
+            logger.exception("scheduled official import failed")
+        time.sleep(settings.import_interval_seconds)
+
+
+if __name__ == "__main__":
+    main()

apps/api/app/schemas.py

@@ -98,6 +98,12 @@ class ImportRunOut(BaseModel):
     rows_created: int
     rows_updated: int
     error_summary: str | None
+    response_status: int | None
+    response_etag: str | None
+    response_last_modified: str | None
+    response_content_type: str | None
+    response_bytes: int | None
+    not_modified: bool
 
 
 class CatchReportCreate(BaseModel):