From 57aa3ffafb4af6f17281d36d4f1d50ca8e146abd Mon Sep 17 00:00:00 2001 From: IK Date: Thu, 10 Sep 2026 20:34:05 +0700 Subject: [PATCH] A02/A03: Add state validation and normalize subdomain keys for shared cooldown --- rf4_research/community_cli.py | 43 +++++++++++++++++++++--- tests/test_community_cli.py | 61 ++++++++++++++++++++++++++++++++--- 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/rf4_research/community_cli.py b/rf4_research/community_cli.py index 76a61e3..b2ecb44 100644 --- a/rf4_research/community_cli.py +++ b/rf4_research/community_cli.py @@ -44,12 +44,25 @@ MAX_RESPONSE_BYTES = 5 * 1024 * 1024 # 5 MB def _read_state(state_file: Path) -> dict: - """Read state file with shared lock; return empty dict if missing/corrupt.""" + """Read state file with shared lock; return empty dict if missing/corrupt. + + Validates that the parsed JSON is a flat dict of string->number pairs. + Returns {} for missing, corrupt, or structurally invalid files. + """ try: with open(state_file, "r") as f: fcntl.flock(f, fcntl.LOCK_SH) try: - return json.loads(f.read()) + data = json.loads(f.read()) + if not isinstance(data, dict): + return {} + # Validate structure: flat dict of string->number + for key, value in data.items(): + if not isinstance(key, str): + return {} + if not isinstance(value, (int, float)): + return {} + return data finally: fcntl.flock(f, fcntl.LOCK_UN) except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError): @@ -76,10 +89,21 @@ def _write_state(state_file: Path, state: dict) -> None: def fetch_site_key(url: str) -> str: - """Return a stable cooldown key shared by all endpoints of one site.""" + """Return a stable cooldown key shared by all endpoints of one site. + + Normalizes hostname to a common key for related domains: + - Strips common subdomains (www, download, api, cdn) + - Keeps the base domain as the key + """ hostname = (urlsplit(url).hostname or "").lower() if hostname.startswith("www."): hostname = hostname[4:] + if hostname.startswith("download."): + hostname = hostname[9:] + if hostname.startswith("api."): + hostname = hostname[4:] + if hostname.startswith("cdn."): + hostname = hostname[4:] if not hostname: raise ValueError("source URL must include a hostname") return hostname @@ -105,6 +129,8 @@ def check_and_reserve( Opens state file with exclusive lock, reads state, checks cooldown, reserves if allowed — all in one critical section. Uses lockfile pattern for cross-process coordination. + + Handles corrupt/missing state files gracefully by treating them as empty. """ if now is None: now = time.time() @@ -115,10 +141,17 @@ def check_and_reserve( with open(lock_file, "w") as lf: fcntl.flock(lf, fcntl.LOCK_EX) try: - # Read state under lock + # Read state under lock; validate structure try: with open(state_file, "r") as sf: - state = json.loads(sf.read()) or {} + raw = sf.read() + state = json.loads(raw) or {} + if not isinstance(state, dict): + state = {} + for key, value in state.items(): + if not isinstance(key, str) or not isinstance(value, (int, float)): + state = {} + break except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError): state = {} # Check cooldown diff --git a/tests/test_community_cli.py b/tests/test_community_cli.py index 8603aab..c085341 100644 --- a/tests/test_community_cli.py +++ b/tests/test_community_cli.py @@ -28,6 +28,19 @@ def test_fetch_site_key_groups_endpoints_and_normalizes_www() -> None: assert fetch_site_key("https://www.rf4-stat.ru/posts/") == "rf4-stat.ru" +def test_fetch_site_key_normalizes_common_subdomains() -> None: + """A02: download/api/cdn subdomains share the base domain key.""" + assert fetch_site_key("https://download.rf4db.com/ru/catches") == "rf4db.com" + assert fetch_site_key("https://api.rf4db.com/v1/catches") == "rf4db.com" + assert fetch_site_key("https://cdn.rf4db.com/assets/img.jpg") == "rf4db.com" + assert fetch_site_key("https://www.rf4db.com/") == "rf4db.com" + # Base domain stays the same + assert fetch_site_key("https://rf4db.com/") == "rf4db.com" + # Non-normalized subdomains stay as-is + assert fetch_site_key("https://rf4map.ru/point/1") == "rf4map.ru" + assert fetch_site_key("https://rf4-posts.com/spots/") == "rf4-posts.com" + + def test_failed_fetch_still_reserves_site_cooldown(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: state_file = tmp_path / "fetch-state.json" @@ -36,7 +49,8 @@ def test_failed_fetch_still_reserves_site_cooldown(tmp_path: Path, monkeypatch: monkeypatch.setattr(community_cli, "fetch_html", fail) assert community_cli.main(["rf4db", "--state-file", str(state_file)]) == 1 - assert "download.rf4db.com" in json.loads(state_file.read_text(encoding="utf-8")) + # A02: download.rf4db.com normalized to rf4db.com + assert "rf4db.com" in json.loads(state_file.read_text(encoding="utf-8")) def test_validate_url_host_rejects_disallowed_hosts() -> None: @@ -73,15 +87,15 @@ def test_write_state_is_atomic_with_flush(tmp_path: Path) -> None: from rf4_research.community_cli import _read_state, _write_state state_file = tmp_path / "state.json" - _write_state(state_file, {"key1": "value1"}) + _write_state(state_file, {"key1": 1000.0}) assert state_file.exists() assert not (state_file.with_suffix(".tmp")).exists() state = _read_state(state_file) - assert state == {"key1": "value1"} + assert state == {"key1": 1000.0} - _write_state(state_file, {"key1": "value2", "key2": "value3"}) + _write_state(state_file, {"key1": 2000.0, "key2": 3000.0}) state = _read_state(state_file) - assert state == {"key1": "value2", "key2": "value3"} + assert state == {"key1": 2000.0, "key2": 3000.0} assert not (state_file.with_suffix(".tmp")).exists() @@ -273,3 +287,40 @@ def test_urljoin_resolves_relative_redirects() -> None: assert urljoin("https://rf4-stat.ru/old/path", "new") == "https://rf4-stat.ru/old/new" # Absolute URL assert urljoin("https://rf4-stat.ru/old", "https://rf4-stat.ru/absolute") == "https://rf4-stat.ru/absolute" + + +# A02: State validation tests + +def test_read_state_rejects_corrupt_json(tmp_path: Path) -> None: + """A02: Corrupt JSON returns empty dict.""" + from rf4_research.community_cli import _read_state + + state_file = tmp_path / "corrupt.json" + state_file.write_text("not valid json {{{") + assert _read_state(state_file) == {} + + +def test_read_state_rejects_invalid_structure(tmp_path: Path) -> None: + """A02: Non-dict or non-flat structures return empty dict.""" + from rf4_research.community_cli import _read_state + + state_file = tmp_path / "invalid.json" + # List instead of dict + state_file.write_text("[1, 2, 3]") + assert _read_state(state_file) == {} + # Dict with non-number values + state_file.write_text('{"key": "value"}') + assert _read_state(state_file) == {} + # Dict with nested dict + state_file.write_text('{"key": {"nested": true}}') + assert _read_state(state_file) == {} + # Dict with non-string keys (JSON always has string keys, but validate anyway) + state_file.write_text('{"valid": 123}') + assert _read_state(state_file) == {"valid": 123} + + +def test_read_state_handles_missing_file() -> None: + """A02: Missing state file returns empty dict.""" + from rf4_research.community_cli import _read_state + + assert _read_state(Path("/nonexistent/state.json")) == {}