feat: check source links during scheduled fetches

This commit is contained in:
ik
2026-09-13 16:32:10 +07:00
parent bc3ceb5dfe
commit 1305cccfa5
6 changed files with 110 additions and 7 deletions
+18 -1
View File
@@ -14,6 +14,7 @@ from .config import settings
from .database import SessionLocal
from .logging_config import configure_logging
from .models import CommunityImportRun, DataSource
from .source_lifecycle import classify_source_failure, record_scheduled_source_check
logger = logging.getLogger("rf4.community_scheduler")
MAX_BACKOFF_SECONDS = 24 * 60 * 60
@@ -102,11 +103,27 @@ def run_source(source_system: str, *, now: datetime | None = None) -> bool:
html = fetch_html(url)
records = parser(html, source_url=url) if source_system in {"rf4map", "rf4posts-spot"} else parser(html)
created, updated = stage_observations(session, [asdict(item) for item in records])
record_scheduled_source_check(
session, source_system=source_system, source_url=url,
status="available", checked_at=current,
)
run.status, run.rows_seen, run.rows_created, run.rows_updated = "success", len(records), created, updated
except Exception as exc:
session.rollback()
source_status = classify_source_failure(exc)
checked = datetime.now(timezone.utc)
affected = record_scheduled_source_check(
session,
source_system=source_system,
source_url=url,
status=source_status,
checked_at=checked,
)
run.status, run.error_summary = "failed", f"{type(exc).__name__}: {str(exc)[:500]}"
logger.exception("community import failed", extra={"event":"community_import_failed", "source_system":source_system})
logger.exception("community import failed", extra={
"event":"community_import_failed", "source_system":source_system,
"source_check_status": source_status, "affected_observations": affected,
})
run.finished_at = datetime.now(timezone.utc); session.commit()
return True