A02: Atomic check-and-reserve with lockfile for cross-process coordination
- Single exclusive lock covers read-check-write in one critical section - Lockfile pattern ensures cross-process mutual exclusion - Atomic write via temp file + rename after unlock - Flush + fsync before unlock to prevent data loss - Real multi-process test: 3 concurrent processes get exactly 1 reservation - 111 Python tests pass (+2 new tests)
This commit is contained in:
@@ -87,19 +87,61 @@ def fetch_site_key(url: str) -> str:
|
||||
def enforce_fetch_interval(
|
||||
source: str, *, state_file: Path, now: float | None = None,
|
||||
) -> None:
|
||||
now = time.time() if now is None else now
|
||||
state = _read_state(state_file)
|
||||
last_fetch = state.get(source)
|
||||
if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS:
|
||||
wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch))
|
||||
raise RuntimeError(f"source cooldown is active; retry in {wait} seconds")
|
||||
"""Legacy: use check_and_reserve instead for atomic check-and-reserve."""
|
||||
check_and_reserve(source, state_file=state_file, now=now)
|
||||
|
||||
|
||||
def mark_fetch(source: str, *, state_file: Path, now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
state = _read_state(state_file)
|
||||
"""Legacy: use check_and_reserve instead for atomic check-and-reserve."""
|
||||
check_and_reserve(source, state_file=state_file, now=now)
|
||||
|
||||
|
||||
def check_and_reserve(
|
||||
source: str, *, state_file: Path, now: float | None = None,
|
||||
) -> None:
|
||||
"""A02: Atomic check-and-reserve under a single exclusive lock.
|
||||
|
||||
Opens state file with exclusive lock, reads state, checks cooldown,
|
||||
reserves if allowed — all in one critical section. Uses lockfile
|
||||
pattern for cross-process coordination.
|
||||
"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock_file = state_file.with_suffix(".lock")
|
||||
# Create lock file if not exists
|
||||
lock_file.touch(exist_ok=True)
|
||||
with open(lock_file, "w") as lf:
|
||||
fcntl.flock(lf, fcntl.LOCK_EX)
|
||||
try:
|
||||
# Read state under lock
|
||||
try:
|
||||
with open(state_file, "r") as sf:
|
||||
state = json.loads(sf.read()) or {}
|
||||
except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError):
|
||||
state = {}
|
||||
# Check cooldown
|
||||
last_fetch = state.get(source)
|
||||
if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS:
|
||||
wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch))
|
||||
raise RuntimeError(f"source cooldown is active; retry in {wait} seconds")
|
||||
# Reserve
|
||||
state[source] = now
|
||||
# Write atomically via temp file
|
||||
temp_file = state_file.with_suffix(".tmp")
|
||||
with open(temp_file, "w") as sf:
|
||||
sf.write(json.dumps(state, sort_keys=True))
|
||||
sf.flush()
|
||||
os.fsync(sf.fileno())
|
||||
temp_file.replace(state_file)
|
||||
finally:
|
||||
fcntl.flock(lf, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _mark_fetch_only(source: str, state: dict, now: float) -> dict:
|
||||
"""Internal: update state without cooldown check (for internal use)."""
|
||||
state[source] = now
|
||||
_write_state(state_file, state)
|
||||
return state
|
||||
|
||||
|
||||
MAX_REDIRECT_HOPS = 5
|
||||
|
||||
Reference in New Issue
Block a user