ik/rf4-spotter
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix: enforce community cooldown per site

Browse Source
This commit is contained in:
ik
2026-09-08 16:32:42 +07:00
parent 731eade7b3
commit 486b4e9645
8 changed files with 72 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/community_scheduler.py
+17 -3
View File
@@ -7,7 +7,7 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import select, text
from rf4_research.community_cli import SOURCES, fetch_html
from rf4_research.community_cli import SOURCES, fetch_html, fetch_site_key
from rf4_research.community_sources import parse_rf4map_point, parse_rf4posts_spot
from .community_importer import stage_observations
from .config import settings
@@ -35,18 +35,32 @@ def configured_sources():
"rf4posts-spot": (settings.rf4posts_spot_url, parse_rf4posts_spot),
}
def oldest_site_source(source_system: str, latest_by_source: dict[str, datetime]) -> str:
sources = configured_sources()
site = fetch_site_key(sources[source_system][0])
candidates = [key for key, (url, _) in sources.items() if fetch_site_key(url) == site]
order = {key: index for index, key in enumerate(candidates)}
return min(candidates, key=lambda key: (latest_by_source.get(key, datetime.min.replace(tzinfo=timezone.utc)), order[key]))
def run_source(source_system: str, *, now: datetime | None = None) -> bool:
current = now or datetime.now(timezone.utc)
url, parser = configured_sources()[source_system]
site_key = fetch_site_key(url)
site_sources = [key for key, (candidate_url, _) in configured_sources().items() if fetch_site_key(candidate_url) == site_key]
with SessionLocal() as session:
source = session.get(DataSource, source_system)
if source is None or not source.enabled:
return False
# Lock before reading cooldown: committing the reservation makes it visible
# to the next contender before releasing this transaction lock.
if session.bind and session.bind.dialect.name == "postgresql" and not session.scalar(text("select pg_try_advisory_xact_lock(hashtext(:key))"), {"key": f"community:{source_system}"}):
if session.bind and session.bind.dialect.name == "postgresql" and not session.scalar(text("select pg_try_advisory_xact_lock(hashtext(:key))"), {"key": f"community-site:{site_key}"}):
return False
recent = list(session.scalars(select(CommunityImportRun).where(CommunityImportRun.source_system.in_(site_sources)).order_by(CommunityImportRun.started_at.desc()).limit(32)))
latest_by_source: dict[str, datetime] = {}
for previous in recent:
latest_by_source.setdefault(previous.source_system, previous.started_at if previous.started_at.tzinfo else previous.started_at.replace(tzinfo=timezone.utc))
if oldest_site_source(source_system, latest_by_source) != source_system:
return False
recent = list(session.scalars(select(CommunityImportRun).where(CommunityImportRun.source_system == source_system).order_by(CommunityImportRun.started_at.desc()).limit(8)))
latest = recent[0].started_at if recent else None
delay = retry_delay([run.status for run in recent])
if latest and (latest if latest.tzinfo else latest.replace(tzinfo=timezone.utc)) > current - timedelta(seconds=delay):