fix: R03 research CLI cooldown — _read_state/_write_state helpers, handle missing file (#1788956171115)
This commit is contained in:
@@ -41,6 +41,30 @@ ALLOWED_HOSTS = frozenset({
|
||||
MAX_RESPONSE_BYTES = 5 * 1024 * 1024 # 5 MB
|
||||
|
||||
|
||||
def _read_state(state_file: Path) -> dict:
|
||||
"""Read state file with shared lock; return empty dict if missing/corrupt."""
|
||||
try:
|
||||
with open(state_file, "r") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_SH)
|
||||
try:
|
||||
return json.loads(f.read())
|
||||
finally:
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_state(state_file: Path, state: dict) -> None:
|
||||
"""Write state file atomically with exclusive lock."""
|
||||
state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(state_file, "w") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_EX)
|
||||
try:
|
||||
f.write(json.dumps(state, sort_keys=True))
|
||||
finally:
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def fetch_site_key(url: str) -> str:
|
||||
"""Return a stable cooldown key shared by all endpoints of one site."""
|
||||
hostname = (urlsplit(url).hostname or "").lower()
|
||||
@@ -55,12 +79,7 @@ def enforce_fetch_interval(
|
||||
source: str, *, state_file: Path, now: float | None = None,
|
||||
) -> None:
|
||||
now = time.time() if now is None else now
|
||||
with open(state_file, "r") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_SH)
|
||||
try:
|
||||
state = json.loads(f.read(encoding="utf-8"))
|
||||
finally:
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
state = _read_state(state_file)
|
||||
last_fetch = state.get(source)
|
||||
if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS:
|
||||
wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch))
|
||||
@@ -69,20 +88,9 @@ def enforce_fetch_interval(
|
||||
|
||||
def mark_fetch(source: str, *, state_file: Path, now: float | None = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(state_file, "r+") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_EX)
|
||||
try:
|
||||
try:
|
||||
state = json.loads(f.read(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
state = {}
|
||||
state[source] = now
|
||||
f.seek(0)
|
||||
f.truncate()
|
||||
f.write(json.dumps(state, sort_keys=True))
|
||||
finally:
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
state = _read_state(state_file)
|
||||
state[source] = now
|
||||
_write_state(state_file, state)
|
||||
|
||||
|
||||
def fetch_html(url: str, *, timeout: float = 30) -> str:
|
||||
|
||||
Reference in New Issue
Block a user