feat: check source links during scheduled fetches
This commit is contained in:
@@ -14,6 +14,7 @@ from .config import settings
|
||||
from .database import SessionLocal
|
||||
from .logging_config import configure_logging
|
||||
from .models import CommunityImportRun, DataSource
|
||||
from .source_lifecycle import classify_source_failure, record_scheduled_source_check
|
||||
|
||||
logger = logging.getLogger("rf4.community_scheduler")
|
||||
MAX_BACKOFF_SECONDS = 24 * 60 * 60
|
||||
@@ -102,11 +103,27 @@ def run_source(source_system: str, *, now: datetime | None = None) -> bool:
|
||||
html = fetch_html(url)
|
||||
records = parser(html, source_url=url) if source_system in {"rf4map", "rf4posts-spot"} else parser(html)
|
||||
created, updated = stage_observations(session, [asdict(item) for item in records])
|
||||
record_scheduled_source_check(
|
||||
session, source_system=source_system, source_url=url,
|
||||
status="available", checked_at=current,
|
||||
)
|
||||
run.status, run.rows_seen, run.rows_created, run.rows_updated = "success", len(records), created, updated
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
source_status = classify_source_failure(exc)
|
||||
checked = datetime.now(timezone.utc)
|
||||
affected = record_scheduled_source_check(
|
||||
session,
|
||||
source_system=source_system,
|
||||
source_url=url,
|
||||
status=source_status,
|
||||
checked_at=checked,
|
||||
)
|
||||
run.status, run.error_summary = "failed", f"{type(exc).__name__}: {str(exc)[:500]}"
|
||||
logger.exception("community import failed", extra={"event":"community_import_failed", "source_system":source_system})
|
||||
logger.exception("community import failed", extra={
|
||||
"event":"community_import_failed", "source_system":source_system,
|
||||
"source_check_status": source_status, "affected_observations": affected,
|
||||
})
|
||||
run.finished_at = datetime.now(timezone.utc); session.commit()
|
||||
return True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user