feat: add privacy-safe structured logging

apps/api/app/config.py

@@ -17,6 +17,7 @@ class Settings(BaseSettings):
     official_import_required: bool = False
     import_interval_seconds: int = Field(default=3600, ge=3600)
     rate_limit_secret: str = "change-rate-limit-secret"
+    log_level: str = "INFO"
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 

apps/api/app/logging_config.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+import logging
+import re
+from datetime import datetime, timezone
+
+
+SAFE_FIELDS = (
+    "request_id", "method", "path", "status_code", "duration_ms",
+    "component", "event", "result", "rows_seen", "rows_created",
+    "rows_updated", "not_modified", "interval_seconds", "error_type",
+)
+SENSITIVE = re.compile(
+    r"(?i)(bearer\s+)[^\s]+|((?:token|password|secret|authorization)\s*[=:]\s*)[^\s,;]+"
+)
+
+
+def redact(value: str) -> str:
+    return SENSITIVE.sub(lambda match: f"{match.group(1) or match.group(2)}[REDACTED]", value)
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, object] = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "message": redact(record.getMessage()),
+        }
+        for field in SAFE_FIELDS:
+            value = getattr(record, field, None)
+            if value is not None:
+                payload[field] = value
+        if record.exc_info and "error_type" not in payload:
+            payload["error_type"] = record.exc_info[0].__name__
+        return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
+
+
+def configure_logging(level: str = "INFO") -> None:
+    root = logging.getLogger()
+    if not any(getattr(handler, "_rf4_json", False) for handler in root.handlers):
+        handler = logging.StreamHandler()
+        handler.setFormatter(JsonFormatter())
+        handler._rf4_json = True  # type: ignore[attr-defined]
+        root.handlers.clear()
+        root.addHandler(handler)
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))

apps/api/app/main.py

@@ -4,8 +4,11 @@ from collections import Counter
 from datetime import datetime, timedelta, timezone
 import hashlib
 import hmac
+import logging
+import time as time_module
 from typing import Annotated, Literal
 from uuid import UUID
+import uuid
 
 import httpx
 from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile
@@ -19,12 +22,15 @@ from .database import get_session
 from .config import settings
 from .community_review import ExternalReviewError, map_observation, publish_observation, reject_observation
 from .importer import ImportSourceError, import_records, normalize
+from .logging_config import configure_logging
 from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
 from .readiness import readiness_report
 from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
 
 
+configure_logging(settings.log_level)
+logger = logging.getLogger("rf4.api")
 app = FastAPI(title="RF4 Spotter API", version="0.1.0")
 app.add_middleware(
     CORSMiddleware,
@@ -35,6 +41,31 @@ app.add_middleware(
 Db = Annotated[Session, Depends(get_session)]
 
 
+@app.middleware("http")
+async def structured_request_log(request: Request, call_next):
+    request_id = uuid.uuid4().hex
+    started = time_module.perf_counter()
+    status_code = 500
+    try:
+        response = await call_next(request)
+        status_code = response.status_code
+        response.headers["X-Request-ID"] = request_id
+        return response
+    except Exception as exc:
+        logger.error("request failed", extra={"request_id": request_id, "error_type": type(exc).__name__})
+        raise
+    finally:
+        logger.log(
+            logging.DEBUG if request.url.path in {"/health", "/ready"} else logging.INFO,
+            "request completed",
+            extra={
+                "request_id": request_id, "method": request.method,
+                "path": request.url.path, "status_code": status_code,
+                "duration_ms": round((time_module.perf_counter() - started) * 1000, 2),
+            },
+        )
+
+
 @app.get("/health")
 def health() -> dict[str, str]:
     return {"status": "ok"}

apps/api/app/scheduler.py

@@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
 from .config import settings
 from .database import SessionLocal
 from .importer import import_records
+from .logging_config import configure_logging
 from .models import OfficialRecordImport
 
 
@@ -42,20 +43,24 @@ def run_due_import() -> bool:
             category=settings.official_records_category,
         )
         logger.info(
-            "official import completed status=%s seen=%d created=%d updated=%d not_modified=%s",
-            run.status.value, run.rows_seen, run.rows_created, run.rows_updated, run.not_modified,
+            "official import completed",
+            extra={
+                "event": "official_import_completed", "result": run.status.value,
+                "rows_seen": run.rows_seen, "rows_created": run.rows_created,
+                "rows_updated": run.rows_updated, "not_modified": run.not_modified,
+            },
         )
         return True
 
 
 def main() -> None:
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
-    logger.info("scheduler started interval_seconds=%d", settings.import_interval_seconds)
+    configure_logging(settings.log_level)
+    logger.info("scheduler started", extra={"event": "scheduler_started", "interval_seconds": settings.import_interval_seconds})
     while True:
         try:
             run_due_import()
         except Exception:
-            logger.exception("scheduled official import failed")
+            logger.exception("scheduled official import failed", extra={"event": "official_import_failed"})
         time.sleep(settings.import_interval_seconds)