feat: schedule authorized community imports

apps/api/alembic/versions/0013_community_import_runs.py

@@ -0,0 +1,28 @@
+"""Add persistent community scheduler journal."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0013"
+down_revision = "0012"
+branch_labels = None
+depends_on = None
+
+def upgrade() -> None:
+    op.create_table("community_import_run",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("source_system", sa.String(50), sa.ForeignKey("data_source.key"), nullable=False),
+        sa.Column("source_url", sa.Text(), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True)),
+        sa.Column("status", sa.String(30), nullable=False),
+        sa.Column("rows_seen", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("rows_created", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("rows_updated", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("error_summary", sa.Text()),
+    )
+    op.create_index("ix_community_import_run_source_system", "community_import_run", ["source_system"])
+    op.create_index("ix_community_import_run_started_at", "community_import_run", ["started_at"])
+    op.create_index("ix_community_import_run_status", "community_import_run", ["status"])
+
+def downgrade() -> None:
+    op.drop_table("community_import_run")

apps/api/app/community_scheduler.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import asdict
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select, text
+
+from rf4_research.community_cli import SOURCES, fetch_html
+from rf4_research.community_sources import parse_rf4map_point, parse_rf4posts_spot
+from .community_importer import stage_observations
+from .config import settings
+from .database import SessionLocal
+from .logging_config import configure_logging
+from .models import CommunityImportRun, DataSource
+
+logger = logging.getLogger("rf4.community_scheduler")
+
+def configured_sources():
+    return {
+        "rf4db": SOURCES["rf4db"],
+        "rf4stat-fishing": SOURCES["rf4stat-fishing"],
+        "rf4stat-post": (SOURCES["rf4stat-posts"][0], SOURCES["rf4stat-posts"][1]),
+        "rf4map": (settings.rf4map_point_url, parse_rf4map_point),
+        "rf4posts-spot": (settings.rf4posts_spot_url, parse_rf4posts_spot),
+    }
+
+def run_source(source_system: str, *, now: datetime | None = None) -> bool:
+    current = now or datetime.now(timezone.utc)
+    url, parser = configured_sources()[source_system]
+    with SessionLocal() as session:
+        source = session.get(DataSource, source_system)
+        if source is None or not source.enabled:
+            return False
+        latest = session.scalar(select(CommunityImportRun.started_at).where(CommunityImportRun.source_system == source_system).order_by(CommunityImportRun.started_at.desc()).limit(1))
+        if latest and (latest if latest.tzinfo else latest.replace(tzinfo=timezone.utc)) > current - timedelta(seconds=settings.community_import_interval_seconds):
+            return False
+        if session.bind and session.bind.dialect.name == "postgresql" and not session.scalar(text("select pg_try_advisory_xact_lock(hashtext(:key))"), {"key": f"community:{source_system}"}):
+            return False
+        run = CommunityImportRun(source_system=source_system, source_url=url, started_at=current, status="running")
+        session.add(run); session.commit()
+        try:
+            html = fetch_html(url)
+            records = parser(html, source_url=url) if source_system in {"rf4map", "rf4posts-spot"} else parser(html)
+            created, updated = stage_observations(session, [asdict(item) for item in records])
+            run.status, run.rows_seen, run.rows_created, run.rows_updated = "success", len(records), created, updated
+        except Exception as exc:
+            run.status, run.error_summary = "failed", f"{type(exc).__name__}: {str(exc)[:500]}"
+            logger.exception("community import failed", extra={"event":"community_import_failed", "source_system":source_system})
+        run.finished_at = datetime.now(timezone.utc); session.commit()
+        return True
+
+def main() -> None:
+    configure_logging(settings.log_level)
+    while True:
+        for source_system in configured_sources():
+            run_source(source_system)
+        time.sleep(60)
+
+if __name__ == "__main__":
+    main()

apps/api/app/config.py

@@ -24,6 +24,9 @@ class Settings(BaseSettings):
     retention_audit_days: int = Field(default=365, ge=90)
     retention_published_payload_days: int = Field(default=365, ge=90)
     import_interval_seconds: int = Field(default=3600, ge=3600)
+    community_import_interval_seconds: int = Field(default=1800, ge=1800)
+    rf4map_point_url: str = "https://rf4map.ru/points/275"
+    rf4posts_spot_url: str = "https://rf4-posts.com/ru/spots/d0c6d9c6-4ebf-49a7-98a8-9a562553a8ee"
     rate_limit_secret: str = "change-rate-limit-secret"
     log_level: str = "INFO"
     cors_origins: list[str] = Field(default_factory=lambda: ["http://localhost:4321", "http://127.0.0.1:4321"])

apps/api/app/models.py

@@ -148,6 +148,20 @@ class DataSource(Base):
     enabled: Mapped[bool] = mapped_column(default=True)
 
 
+class CommunityImportRun(Base):
+    __tablename__ = "community_import_run"
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+    source_system: Mapped[str] = mapped_column(ForeignKey("data_source.key"), index=True)
+    source_url: Mapped[str] = mapped_column(Text)
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    status: Mapped[str] = mapped_column(String(30), index=True)
+    rows_seen: Mapped[int] = mapped_column(default=0)
+    rows_created: Mapped[int] = mapped_column(default=0)
+    rows_updated: Mapped[int] = mapped_column(default=0)
+    error_summary: Mapped[str | None] = mapped_column(Text)
+
+
 class ExternalObservation(Base):
     __tablename__ = "external_observation"
     __table_args__ = (UniqueConstraint("source_system", "source_external_id"),)

apps/api/tests/test_community_scheduler.py

@@ -0,0 +1,14 @@
+import pytest
+from pydantic import ValidationError
+
+from app.community_scheduler import configured_sources
+from app.config import Settings
+
+
+def test_all_authorized_sources_are_scheduled() -> None:
+    assert set(configured_sources()) == {"rf4db", "rf4stat-fishing", "rf4stat-post", "rf4map", "rf4posts-spot"}
+
+
+def test_community_interval_cannot_be_less_than_30_minutes() -> None:
+    with pytest.raises(ValidationError):
+        Settings(community_import_interval_seconds=1799)