"""v2.2.18: one test (or a small group) per finding of the 2026-11-19 review, each reproducing the reviewer's case offline (LocalStorage, the ChangingStore website double from test_scraper.py, no network, no Azure, no Oxylabs). F1 a five-record Dairy run replaced a 145-record full delivery -> TestF1PartialRunNeverReplacesFull F2 missing totals and overlapping category counts could look complete -> TestF2CompletenessIsProven F3 a checkpoint could resume under another store -> TestF3CheckpointBinding F4 a failed coupon write left new products with the old report -> TestF4OneWriterAtomicDelivery F5 requests were not reserved durably before dispatch -> TestF5DurableReservation F6 normal collection had no deadline (only the probe had one) -> TestF6CollectionDeadline """ from __future__ import annotations import dataclasses import json import threading import time import pytest from test_scraper import CFG, COUPON, ChangingStore, FakeApi, latest_text # noqa: E402 (same tests folder) from marianos_scraper.delivery import Delivery, PublishFailed from marianos_scraper.runner import CheckpointMismatch, Runner, deadline_seconds from marianos_scraper.site_runner import SiteRunner from marianos_scraper.storage import LocalStorage, PreconditionFailed LOC_KEY = "marianos-lakeshore-east" @pytest.fixture(autouse=True) def site_thresholds(monkeypatch, tmp_path): """The thresholds the ChangingStore catalogue (five products per taxonomy root) can meet.""" for k, val in {"min_records_per_category": 1, "min_ppm_share": 0.0, "min_total_records": 1}.items(): monkeypatch.setitem(CFG.settings["validation"], k, val) monkeypatch.setitem(CFG.settings["site_source"], "api_cross_check_sample", 0) CFG.storage_root = str(tmp_path) def site_runner(storage, site, loc=None, **kw): return SiteRunner(CFG, loc or CFG.location(LOC_KEY), site=site, api=FakeApi(), storage=storage, **kw) def delivered(storage): d = Delivery(storage, LOC_KEY) ptr = d.current() return ptr, {n: d.read(ptr, n) for n in ("products.json", "coupons.json", "report.json", "catalog.json")} def failed_checks(report): return {c["check"] for c in report["validation"]["checks"] if not c["passed"]} # --- F1 --------------------------------------------------------------------------------------------------------- class TestF1PartialRunNeverReplacesFull: def test_five_record_dairy_run_is_a_sample_and_the_145_record_delivery_stays(self, tmp_path): storage = LocalStorage(str(tmp_path)) full = site_runner(storage, ChangingStore()).run() assert full["status"] == "published" and full["unique_records"] == 145 # the reviewer's full delivery ptr_before, files_before = delivered(storage) dairy = site_runner(storage, ChangingStore(), only_categories=["Dairy"], fresh=True).run() assert dairy["unique_records"] == 5 # the reviewer's Dairy run assert dairy["status"] == "sample" and dairy["delivery"]["kind"] == "sample" assert dairy["delivery"]["release"] == f"samples/{LOC_KEY}/{dairy['run_id']}/" assert len(json.loads(storage.read_text(dairy["delivery"]["release"] + "products.json"))) == 5 ptr_after, files_after = delivered(storage) assert ptr_after == ptr_before and files_after == files_before # latest/ untouched assert len(json.loads(files_after["products.json"])) == 145 assert json.loads(files_after["report.json"])["run_id"] == full["run_id"] def test_refinish_judges_a_run_by_its_own_scope(self, tmp_path): storage = LocalStorage(str(tmp_path)) site_runner(storage, ChangingStore()).run() ptr_before, _ = delivered(storage) dairy = site_runner(storage, ChangingStore(), only_categories=["Dairy"], fresh=True).run() # refinish without --categories must not turn the Dairy sample into a full delivery again = site_runner(storage, ChangingStore()).refinish(dairy["run_id"]) assert again["status"] == "sample" and delivered(storage)[0] == ptr_before def test_api_runner_category_subset_is_a_sample_too(self, tmp_path): storage = LocalStorage(str(tmp_path)) report = Runner(CFG, CFG.location(LOC_KEY), api=FakeApi(), storage=storage, only_categories=["Dairy"]).run() assert report["status"] == "sample" and latest_text(tmp_path) is None # --- F2 --------------------------------------------------------------------------------------------------------- class NoTotalStore(ChangingStore): """The site answers one Dairy listing without its totalCount.""" def search(self, location_id, taxonomy, offset=0, size=100, brand="", facets=False, brand_facet=False): items, total, f = super().search(location_id, taxonomy, offset, size, brand, facets, brand_facet) if not facets and taxonomy == self.leaf("02"): return items, None, f return items, total, f class OverlapStore(ChangingStore): """Dairy root '02' has two aisles of five products each (site count 10), but both aisles list the SAME five products: summed per unit that is 10/10 'complete', distinct it is 5 of 10.""" def search(self, location_id, taxonomy, offset=0, size=100, brand="", facets=False, brand_facet=False): if taxonomy == "02" and facets: return [], 10, [{"id": "02", "name": "Dairy", "count": 10, "children": [ {"id": "02001", "name": "Aisle A", "count": 5, "children": []}, {"id": "02002", "name": "Aisle B", "count": 5, "children": []}]}] if taxonomy in ("02001", "02002") and not facets: ids = self.upcs_of("02001") return [{"upc": u} for u in ids[offset:offset + size]], 5, {} return super().search(location_id, taxonomy, offset, size, brand, facets, brand_facet) class ShortCouponStore(ChangingStore): """The site says 3 digital coupons; the paged list keeps returning only one of them.""" def __init__(self, total=3): super().__init__() self.total, self.passes = total, 0 def all_coupons(self, kind): if kind == "standard": self.passes += 1 return [COUPON], ({"totalCount": self.total} if self.total is not None else {}) return [], {"totalCount": 0} class TestF2CompletenessIsProven: def test_a_listing_without_the_site_total_is_not_complete(self, tmp_path): storage = LocalStorage(str(tmp_path)) report = site_runner(storage, NoTotalStore()).run() assert report["status"] == "held_back" assert "site_count_reported_for_every_unit" in failed_checks(report) assert latest_text(tmp_path) is None def test_overlapping_units_are_counted_once(self, tmp_path): storage = LocalStorage(str(tmp_path)) report = site_runner(storage, OverlapStore()).run() dairy = report["category_completeness"]["Dairy"] assert dairy["ids_listed_incl_overlap"] == 10 and dairy["ids_collected"] == 5 and dairy["coverage"] == 0.5 assert report["status"] == "held_back" and "category_completeness_vs_site_counts" in failed_checks(report) def test_coupons_short_of_the_site_total_are_read_again_then_held_back(self, tmp_path): storage = LocalStorage(str(tmp_path)) site = ShortCouponStore(total=3) report = site_runner(storage, site).run() assert site.passes == 3 # three passes over the re-sorting list assert report["coupons"]["site_totals"]["standard"] == 3 assert report["status"] == "held_back" and "coupons_complete_vs_site_totals" in failed_checks(report) def test_coupons_without_a_site_total_are_not_complete(self, tmp_path): report = site_runner(LocalStorage(str(tmp_path)), ShortCouponStore(total=None)).run() assert report["status"] == "held_back" and "coupons_complete_vs_site_totals" in failed_checks(report) def test_every_product_must_have_been_asked_for_its_coupons(self, tmp_path): class SkipsLookup(ChangingStore): def coupons_for(self, upcs): return [] storage = LocalStorage(str(tmp_path)) runner = site_runner(storage, SkipsLookup()) real = runner._link_coupons def lookup_half(unit, t0): # as if the lookup had covered only part of the catalogue res = real(unit, t0) res["candidate_upcs"] = res["candidate_upcs"][: len(res["candidate_upcs"]) // 2] return res runner._link_coupons = lookup_half report = runner.run() assert report["status"] == "held_back" and "coupon_lookup_covered_every_product" in failed_checks(report) # --- F3 --------------------------------------------------------------------------------------------------------- class OtherStore(ChangingStore): STORE = {**ChangingStore.STORE, "locationId": "53100502", "facilityId": "15092", "storeNumber": "00502", "locale": {"address": {"name": "Vernon Hills", "addressLines": ["275 Hawthorn Village Commons"], "cityTown": "VERNON HILLS", "stateProvince": "IL", "postalCode": "60061"}}} class CountingStore(ChangingStore): def __init__(self): super().__init__() self.sent = 0 def store(self, zipcode, location_id): self.sent += 1 return super().store(zipcode, location_id) def search(self, *a, **kw): self.sent += 1 return super().search(*a, **kw) class TestF3CheckpointBinding: def interrupted(self, storage): first = site_runner(storage, ChangingStore(), stop_after_units=3).run() assert first["status"] == "interrupted" return first, storage.read_text(f"work/{LOC_KEY}/current.json") def test_an_unfinished_run_never_resumes_under_another_store(self, tmp_path): storage = LocalStorage(str(tmp_path)) first, state_before = self.interrupted(storage) other = dataclasses.replace(CFG.location(LOC_KEY), location_id="53100502", zipcode="60061", facility_id="15092") site = CountingStore() with pytest.raises(CheckpointMismatch, match="not resumed"): site_runner(storage, site, loc=other).run() assert site.sent == 0 # nothing sent assert storage.read_text(f"work/{LOC_KEY}/current.json") == state_before # run state untouched def test_a_store_that_answers_as_another_store_on_resume_is_refused(self, tmp_path): storage = LocalStorage(str(tmp_path)) self.interrupted(storage) loose = dataclasses.replace(CFG.location(LOC_KEY), facility_id="") # facility not pinned in config # binding differs (facility) -> refused before anything is sent with pytest.raises(CheckpointMismatch): site_runner(storage, OtherStore(), loc=loose).run() def test_another_source_never_resumes_a_website_run(self, tmp_path): storage = LocalStorage(str(tmp_path)) self.interrupted(storage) api = FakeApi() with pytest.raises(CheckpointMismatch): Runner(CFG, CFG.location(LOC_KEY), api=api, storage=storage).run() assert api.calls == 0 def test_a_foreign_checkpoint_file_is_not_used(self, tmp_path): storage = LocalStorage(str(tmp_path)) first, _ = self.interrupted(storage) units = f"work/{LOC_KEY}/{first['run_id']}/units/" key = sorted(k for k in storage.list(units))[1] res = json.loads(storage.read_text(key)) storage.write_text(key, json.dumps({**res, "binding_digest": "0123456789abcdef"})) # from another binding with pytest.raises(CheckpointMismatch, match="not used"): site_runner(storage, ChangingStore()).run() assert latest_text(tmp_path) is None # --- F4 --------------------------------------------------------------------------------------------------------- class FailingStorage(LocalStorage): """Fails every write of one file name (e.g. the coupon file of a new delivery).""" def __init__(self, root, fail_name): super().__init__(root) self.fail_name = fail_name def write_text(self, key, text): if key.startswith("releases/") and key.endswith(self.fail_name): raise OSError(f"simulated write failure: {key}") super().write_text(key, text) class TestF4OneWriterAtomicDelivery: @pytest.mark.parametrize("fail_name", ["coupons.json", "report.json", "catalog.json"]) def test_a_failed_write_leaves_the_previous_delivery_whole(self, tmp_path, fail_name): good = site_runner(LocalStorage(str(tmp_path)), ChangingStore()).run() ptr_before, before = delivered(LocalStorage(str(tmp_path))) site = ChangingStore() site.absent = {site.upcs_of(site.leaf("02"))[0]} # the new run differs from the old delivery new = site_runner(FailingStorage(str(tmp_path), fail_name), site, fresh=True).run() assert new["status"] == "publish_failed" and fail_name in new["delivery"]["error"] ptr_after, after = delivered(LocalStorage(str(tmp_path))) assert ptr_after == ptr_before and after == before # same release, same bytes: nothing mixed report = json.loads(after["report.json"]) assert report["run_id"] == good["run_id"] == json.loads(after["catalog.json"])["run_id"] from marianos_scraper.delivery import sha256_text assert report["evidence"]["products_sha256"] == sha256_text(after["products.json"]) assert json.loads(LocalStorage(str(tmp_path)).read_text(f"runs/{LOC_KEY}/{new['run_id']}/report.json"))["status"] \ == "publish_failed" def test_products_coupons_report_and_history_move_together(self, tmp_path): storage = LocalStorage(str(tmp_path)) a = site_runner(storage, ChangingStore()).run() b = site_runner(storage, ChangingStore(), fresh=True).run() ptr, files = delivered(storage) assert ptr["run_id"] == b["run_id"] and ptr["previous"]["run_id"] == a["run_id"] assert [h["run_id"] for h in ptr["history"]] == [a["run_id"], b["run_id"]] assert json.loads(files["report.json"])["run_id"] == json.loads(files["catalog.json"])["run_id"] == b["run_id"] assert set(ptr["files"]) == {"products.json", "coupons.json", "report.json", "catalog.json"} def test_a_second_writer_cannot_overwrite_the_pointer(self, tmp_path): storage = LocalStorage(str(tmp_path)) d = Delivery(storage, LOC_KEY) files = {n: "{}" for n in ("products.json", "coupons.json", "report.json", "catalog.json")} d.publish("RUN-A", files, {}) _, etag = storage.read_with_etag(d.pointer_key) d.publish("RUN-B", files, {}) # another writer got in between with pytest.raises(PreconditionFailed): storage.write_if_match(d.pointer_key, "{}", etag) assert d.current()["run_id"] == "RUN-B" def test_a_changed_release_file_is_never_read(self, tmp_path): storage = LocalStorage(str(tmp_path)) site_runner(storage, ChangingStore()).run() ptr, _ = delivered(storage) storage.write_text(ptr["release"] + "coupons.json", "[]") from marianos_scraper.delivery import DeliveryDamaged with pytest.raises(DeliveryDamaged): Delivery(storage, LOC_KEY).read(ptr, "coupons.json") def test_publish_refuses_a_partial_file_set(self, tmp_path): with pytest.raises(PublishFailed): Delivery(LocalStorage(str(tmp_path)), LOC_KEY).publish("R", {"products.json": "[]"}, {}) # --- F5 --------------------------------------------------------------------------------------------------------- class Killed(BaseException): """The process dies (power loss, kill -9): nothing below run() gets to save anything.""" class ReservingStore(ChangingStore): """Every request goes through the transport's reserve hook first, exactly as SiteSession._begin_attempt does; at each dispatch the test reads the DURABLE run state and checks the request is already reserved there.""" def __init__(self, storage, kill_after=None): super().__init__() self.storage, self.kill_after, self.reserve, self.dispatched = storage, kill_after, None, 0 self.unreserved_dispatches = [] def _send(self, what): if self.kill_after is not None and self.dispatched >= self.kill_after: raise Killed() if self.reserve is not None: self.reserve(what) self.stats.requests += 1 self.dispatched += 1 state = json.loads(self.storage.read_text(f"work/{LOC_KEY}/current.json")) if state["site_requests_reserved"] < state["site_requests_base"] + self.stats.requests: self.unreserved_dispatches.append(what) def store(self, zipcode, location_id): self._send("store") return super().store(zipcode, location_id) def search(self, *a, **kw): self._send("search") return super().search(*a, **kw) def products(self, upcs, verified=True): self._send("products") return super().products(upcs, verified) class TestF5DurableReservation: def test_every_request_is_reserved_in_the_run_state_before_it_is_sent(self, tmp_path): storage = LocalStorage(str(tmp_path)) site = ReservingStore(storage) report = site_runner(storage, site).run() assert report["status"] == "published" assert site.dispatched > 0 and site.unreserved_dispatches == [] state = json.loads(storage.read_text(f"work/{LOC_KEY}/current.json")) assert state["site_requests_reserved"] == state["site_requests_used"] == site.dispatched # exact at the end def test_a_killed_process_is_counted_at_its_full_reservation_and_the_budget_holds(self, tmp_path, monkeypatch): monkeypatch.setitem(CFG.settings["site_source"], "request_reservation_block", 10) storage = LocalStorage(str(tmp_path)) budget = 60 first = ReservingStore(storage, kill_after=23) with pytest.raises(Killed): site_runner(storage, first, max_requests_per_run=budget).run() storage.release_lock(f"work/{LOC_KEY}/run.lock") # a dead process's lock is stale on_disk = json.loads(storage.read_text(f"work/{LOC_KEY}/current.json")) assert on_disk["site_requests_used"] < first.dispatched <= on_disk["site_requests_reserved"] second = ReservingStore(storage) report = site_runner(storage, second, max_requests_per_run=budget).run() state = json.loads(storage.read_text(f"work/{LOC_KEY}/current.json")) assert state["requests_counted_from_reservation"] == on_disk["site_requests_reserved"] - on_disk["site_requests_used"] assert state["site_requests_used"] >= first.dispatched + second.dispatched # never under-counted assert first.dispatched + second.dispatched <= budget # the run budget held assert report["status"] in ("interrupted", "published") def test_no_request_is_sent_when_the_reservation_cannot_be_written(self, tmp_path): class NoStateWrites(LocalStorage): armed = False def write_text(self, key, text): if self.armed and key.endswith("current.json"): raise OSError("simulated storage outage") super().write_text(key, text) storage = NoStateWrites(str(tmp_path)) site = ReservingStore(storage) runner = site_runner(storage, site) runner._load_or_start_state() runner._save_state(runner._state) storage.armed = True from marianos_scraper.site_source import RequestCapReached with pytest.raises(RequestCapReached, match="nothing sent"): site.store("60601", "53100503") assert site.dispatched == 0 def test_the_real_transport_reserves_before_it_counts_and_sends(self): from marianos_scraper.site_source import SiteSession order = [] s = SiteSession("", {**CFG.settings["site_source"], "min_interval_seconds": 0}) s.reserve = lambda path: order.append(("reserve", s.stats.requests)) class Page: def evaluate(self, js, args): order.append(("send", s.stats.requests)) return {"status": 200, "type": "application/json", "body": json.dumps({"data": {"stores": []}}), "ms": 1} s._page = Page() s.get("/atlas/v1/stores/v2/locator?filter.query=60601", expect="stores") assert order == [("reserve", 0), ("send", 1)] def test_oxylabs_collection_without_the_allowance_ledger_sends_nothing(self, monkeypatch, tmp_path): from marianos_scraper import __main__ as cli monkeypatch.setenv("SCRAPER_DATA_DIR", str(tmp_path)) monkeypatch.setattr(cli, "build_site", lambda *a, **k: pytest.fail("no transport may be built")) assert cli.main(["run", "--location", LOC_KEY, "--transport", "oxylabs", "--source", "site"]) == 2 def test_oxylabs_collection_reserves_jobs_in_the_ledger_before_the_first_one(self, monkeypatch, tmp_path): from marianos_scraper import __main__ as cli ledger = tmp_path / "ledger.json" seen = {} class Ledgered(ChangingStore): transport_name = "oxylabs" def store(self, zipcode, location_id): if "at_first_job" not in seen: seen["at_first_job"] = json.loads(ledger.read_text(encoding="utf-8")) self.stats.requests += 1 return super().store(zipcode, location_id) site = Ledgered() monkeypatch.setenv("SCRAPER_DATA_DIR", str(tmp_path / "data")) monkeypatch.setenv("KROGER_CLIENT_ID", "") monkeypatch.setattr(cli, "build_site", lambda *a, **k: site) code = cli.main(["run", "--location", LOC_KEY, "--transport", "oxylabs", "--source", "site", "--max-requests", "7", "--ledger", str(ledger), "--ledger-ceiling", "15", "--categories", "Dairy"]) entry = seen["at_first_job"]["executions"][0] assert entry["status"] == "reserved" and entry["reserved_jobs"] == 7 and site.max_requests == 7 final = json.loads(ledger.read_text(encoding="utf-8"))["executions"][0] assert final["status"] == "final" and final["oxylabs_jobs_submitted"] == site.stats.requests assert code in (0, 1, 4) # --- F6 --------------------------------------------------------------------------------------------------------- class SlowStore(ChangingStore): def search(self, location_id, taxonomy, offset=0, size=100, brand="", facets=False, brand_facet=False): if not facets: time.sleep(0.05) # each unit's listing is slow; planning is not return super().search(location_id, taxonomy, offset, size, brand, facets, brand_facet) class TestF6CollectionDeadline: def test_normal_collection_gets_the_deadline_without_any_option(self): from marianos_scraper.site_source import SiteSession s = SiteSession("", dict(CFG.settings["site_source"])) assert s.max_seconds is None # nothing on the command line runner = SiteRunner(CFG, CFG.location(LOC_KEY), site=s, api=FakeApi(), storage=object()) assert s.max_seconds == runner.deadline == deadline_seconds(CFG.settings) == 18000 def test_every_request_has_a_timeout_even_without_a_runtime_cap(self): from marianos_scraper.site_source import SiteSession s = SiteSession("", {**CFG.settings["site_source"], "min_interval_seconds": 0}) sent = [] class Page: def evaluate(self, js, args): sent.append(args[2]) return {"status": 200, "type": "application/json", "body": json.dumps({"data": {"stores": []}}), "ms": 1} s._page = Page() s.get("/atlas/v1/stores/v2/locator?filter.query=60601", expect="stores") assert sent == [90000] # v2.2.17 sent 0 = no limit when no --max-seconds was given def test_a_site_run_stops_resumably_at_the_deadline(self, tmp_path, monkeypatch): monkeypatch.setitem(CFG.settings, "collection", {"deadline_seconds": 0.5}) storage = LocalStorage(str(tmp_path)) report = site_runner(storage, SlowStore()).run() assert report["status"] == "interrupted" done = storage.list(f"work/{LOC_KEY}/{report['run_id']}/units/") assert 0 < len(done) < 30 and latest_text(tmp_path) is None monkeypatch.setitem(CFG.settings, "collection", {"deadline_seconds": 600}) resumed = site_runner(storage, SlowStore()).run() # same transport: the run is resumed assert resumed["run_id"] == report["run_id"] and resumed["status"] == "published" def test_an_api_run_stops_at_the_deadline_between_units(self, tmp_path, monkeypatch): import dataclasses as dc class SlowApi(FakeApi): def search_page(self, term, location_id, start): time.sleep(0.05) return super().search_page(term, location_id, start) monkeypatch.setitem(CFG.settings, "collection", {"deadline_seconds": 0.2}) cfg = dc.replace(CFG, categories=[c for c in CFG.categories if c.name == "Dairy"]) report = Runner(cfg, CFG.location(LOC_KEY), api=SlowApi(), storage=LocalStorage(str(tmp_path))).run() assert report["status"] == "interrupted" def test_the_retry_wait_stops_at_the_deadline(self, monkeypatch): monkeypatch.setitem(CFG.settings, "collection", {"deadline_seconds": 10}) runner = Runner(CFG, CFG.location(LOC_KEY), api=FakeApi(), storage=object()) from marianos_scraper.runner import Interrupted with pytest.raises(Interrupted): runner._wait(30) # v2.2.17 slept 30 s regardless def test_a_stuck_call_is_ended_by_the_watchdog(self): from marianos_scraper.__main__ import start_watchdog fired = threading.Event() codes = [] start_watchdog(0.2, exit=lambda code: (codes.append(code), fired.set())) assert fired.wait(3) and codes == [4] def test_the_deadline_stays_below_the_stale_lock_age(self): with pytest.raises(ValueError): deadline_seconds({"collection": {"deadline_seconds": 6 * 3600}})