Add conditional caching and opt-in import scheduler

apps/api/app/importer.py

@@ -36,6 +36,16 @@ class RawRecord:
     record_date: date
 
 
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    records: list[RawRecord] | None
+    status_code: int
+    etag: str | None
+    last_modified: str | None
+    content_type: str | None
+    response_bytes: int
+
+
 def normalize(value: str) -> str:
     return " ".join(value.replace("\xa0", " ").replace("–", "-").replace("—", "-").split()).casefold()
 
@@ -100,15 +110,32 @@ def parse_html(html: str, *, region: str, category: str) -> list[RawRecord]:
     return records
 
 
-def fetch_records(url: str, *, region: str, category: str) -> list[RawRecord]:
-    with httpx.Client(timeout=20, follow_redirects=True, headers={"User-Agent": USER_AGENT, "Accept": "text/html"}) as client:
+def fetch_records(
+    url: str, *, region: str, category: str,
+    etag: str | None = None, last_modified: str | None = None,
+) -> FetchResult:
+    headers = {"User-Agent": USER_AGENT, "Accept": "text/html"}
+    if etag:
+        headers["If-None-Match"] = etag
+    if last_modified:
+        headers["If-Modified-Since"] = last_modified
+    with httpx.Client(timeout=20, follow_redirects=True, headers=headers) as client:
         for attempt in range(3):
             try:
                 response = client.get(url)
+                metadata = {
+                    "status_code": response.status_code,
+                    "etag": response.headers.get("etag"),
+                    "last_modified": response.headers.get("last-modified"),
+                    "content_type": response.headers.get("content-type"),
+                    "response_bytes": len(response.content),
+                }
+                if response.status_code == 304:
+                    return FetchResult(records=None, **metadata)
                 response.raise_for_status()
                 if "text/html" not in response.headers.get("content-type", ""):
                     raise ImportSourceError("source did not return HTML")
-                return parse_html(response.text, region=region, category=category)
+                return FetchResult(records=parse_html(response.text, region=region, category=category), **metadata)
             except (httpx.HTTPError, ImportSourceError):
                 if attempt == 2:
                     raise
@@ -121,7 +148,32 @@ def import_records(session: Session, *, url: str, region: str, category: str, ht
     session.add(run)
     session.commit()
     try:
-        records = parse_html(html, region=region, category=category) if html is not None else fetch_records(url, region=region, category=category)
+        if html is not None:
+            records = parse_html(html, region=region, category=category)
+        else:
+            previous = session.scalar(
+                select(OfficialRecordImport).where(
+                    OfficialRecordImport.source_url == url,
+                    OfficialRecordImport.status == ImportStatus.success,
+                ).order_by(OfficialRecordImport.started_at.desc()).limit(1)
+            )
+            fetched = fetch_records(
+                url, region=region, category=category,
+                etag=previous.response_etag if previous else None,
+                last_modified=previous.response_last_modified if previous else None,
+            )
+            run.response_status = fetched.status_code
+            run.response_etag = fetched.etag or (previous.response_etag if previous else None)
+            run.response_last_modified = fetched.last_modified or (previous.response_last_modified if previous else None)
+            run.response_content_type = fetched.content_type
+            run.response_bytes = fetched.response_bytes
+            if fetched.records is None:
+                run.not_modified = True
+                run.status = ImportStatus.success
+                run.finished_at = datetime.now(timezone.utc)
+                session.commit()
+                return run
+            records = fetched.records
         run.rows_seen = len(records)
         for raw in records:
             key = external_id(raw)