"""Full run against marianos.com (primary source). Same lock / checkpoint / resume / validate / publish cycle as the API runner; what changes is where products come from and what is checked. Plan: every CSV category maps to retailer taxonomy roots (config/site_taxonomy.json). At the start of a run the current subcategory tree and the site's own product counts are read for each root; nodes larger than one search window (350 products) are split into their children. One unit = one taxonomy node. The plan is stored with the run, so a resumed run works through exactly the same units. Category placement: a product reachable from several CSV categories (e.g. an organic yogurt under Dairy and Natural & Organic) is written once, in the category that matches the retailer's own taxonomy of that product; every other placement is listed in placements.json. """ from __future__ import annotations import gzip import hashlib import json import random import time from collections import Counter, defaultdict from datetime import datetime, timezone from pathlib import Path from .kroger_api import KrogerApi, SourceError from .log import log from .mapper import FIELD_ORDER, SiteMapper from .delivery import Delivery from .runner import CheckpointMismatch, Runner, code_fingerprint, slug, utc_now from .site_source import SEARCH_MAX_OFFSET, PAGE_SIZE, RequestCapReached, RunBudgetSpent, SiteSession from .validate import completeness, record_key, validate_run COUPON_LINK_UNIT = {"id": "99-coupon-links", "category": "Coupons", "kind": "coupon_links", "sub_category": "", "term": "coupon-links", "taxonomy": "", "expected": None} def _find_node(nodes: list[dict], node_id: str) -> dict | None: for n in nodes or []: if n.get("id") == node_id: return n hit = _find_node(n.get("children"), node_id) if hit: return hit return None class SiteRunner(Runner): def __init__(self, cfg, location, *, site: SiteSession | None = None, max_requests_per_run: int | None = None, **kw): api = kw.pop("api", None) if api is None and cfg.client_id: api = KrogerApi(cfg.client_id, cfg.client_secret, cfg.settings["api"]) # used only for the price cross-check super().__init__(cfg, location, api=api or _NoApi(), **kw) s = cfg.settings["site_source"] self.site = site or SiteSession(s["cdp_url"], s) # Request budget of the whole run, across every resume (None = unlimited, the default). The per-process cap # (max_requests) starts from zero at every start; this one does not, so retries by a wrapper cannot exceed it. self.max_requests_per_run = max_requests_per_run if max_requests_per_run is not None else s.get("max_requests_per_run") self.max_node = s.get("max_node_size", SEARCH_MAX_OFFSET + PAGE_SIZE) self.mapper = SiteMapper(cfg.settings) self.taxonomy = json.loads((Path(cfg.config_dir) / "site_taxonomy.json").read_text(encoding="utf-8"))["categories"] self.run_calls: list[dict] = [] self._run_id = "" # v2.2.18: normal collection always has a runtime deadline (settings collection.deadline_seconds unless # --max-seconds sets a shorter one); every request and wait of the transport stops at it if getattr(self.site, "max_seconds", "absent") is None: self.site.max_seconds = self.deadline def binding(self) -> dict: b = super().binding() b.update(source="marianos.com", transport=getattr(self.site, "transport_name", type(self.site).__name__), facility_id=self.loc.facility_id, coupon_link_scope=self.cfg.settings["site_source"].get("coupon_link_scope", "all")) return b # --- run-level request accounting -------------------------------------------------------------------- def _drain_calls(self) -> list[dict]: calls = [{k: v for k, v in c.items() if k != "raw"} for c in self.site.calls] self.site.calls = [] return calls def run(self) -> dict: self.site.open() try: return super().run() finally: self.site.close() def _load_or_start_state(self) -> dict: state = super()._load_or_start_state() self._run_id = state["run_id"] # requests this run already used in earlier segments. v2.2.18: requests are reserved in the run state before # they are sent; when the previous process died, everything it had reserved counts as sent used = state.get("site_requests_used", 0) reserved = state.get("site_requests_reserved", used) if state.get("_resumed_after_crash") and reserved > used: state["requests_counted_from_reservation"] = state.get("requests_counted_from_reservation", 0) + reserved - used log("warning", "reservation_counted_in_full", run_id=state["run_id"], recorded_used=used, reserved=reserved, note="the previous process stopped without a clean save; its whole reservation counts as sent") used = reserved state["site_requests_base"] = state["site_requests_used"] = state["site_requests_reserved"] = used self.site.stats.requests = 0 self._state = state self.site.reserve = self._reserve_request if self.max_requests_per_run is not None: left = self.max_requests_per_run - state["site_requests_base"] if left <= 0: log("error", "run_request_budget_spent", run_id=state["run_id"], used=state["site_requests_base"], budget=self.max_requests_per_run, note="nothing sent; not retried; raise the budget or start with --fresh") raise RunBudgetSpent(f"run {state['run_id']} used {state['site_requests_base']} of its " f"{self.max_requests_per_run} requests") cap = self.site.max_requests self.site.max_requests = left if cap is None else min(cap, left) log("info", "run_request_budget", run_id=state["run_id"], budget=self.max_requests_per_run, used_before=state["site_requests_base"], this_process_cap=self.site.max_requests) return state def _save_state(self, state: dict) -> None: state["site_requests_used"] = state.get("site_requests_base", 0) + self.site.stats.requests if state.get("segments") and state["segments"][-1].get("end") is not None: state["site_requests_reserved"] = state["site_requests_used"] # segment closed: the exact count is known else: state["site_requests_reserved"] = max(state.get("site_requests_reserved", 0), state["site_requests_used"]) super()._save_state(state) def _reserve_request(self, path: str) -> None: """Called by the transport immediately before it counts and sends a request: makes sure the request lies inside a reservation already written to the run state (and read back). Reservations are taken in blocks (site_source.request_reservation_block), never beyond --max-requests-per-run. If the reservation cannot be written, nothing is sent and the run stops resumably.""" st = self._state need = st["site_requests_base"] + self.site.stats.requests + 1 if need <= st.get("site_requests_reserved", 0): return block = max(1, int(self.cfg.settings["site_source"].get("request_reservation_block", 25))) new = need - 1 + block if self.max_requests_per_run is not None: new = min(new, self.max_requests_per_run) if new < need: raise RequestCapReached(f"run request budget {self.max_requests_per_run} fully reserved before {path[:80]}") before = st.get("site_requests_reserved", 0) st["site_requests_reserved"], st["site_requests_used"] = new, need - 1 key = f"work/{self.work_key}/current.json" text = json.dumps({k: v for k, v in st.items() if not k.startswith("_")}, indent=1) try: self.storage.write_text(key, text) ok = self.storage.read_text(key) == text except Exception as exc: # noqa: BLE001 - any storage failure: the request is not sent st["site_requests_reserved"] = before raise RequestCapReached(f"request reservation could not be written ({type(exc).__name__}: {exc}); " f"nothing sent for {path[:80]}") from exc if not ok: st["site_requests_reserved"] = before raise RequestCapReached(f"request reservation read back differently; nothing sent for {path[:80]}") log("info", "requests_reserved", reserved_up_to=new, block=new - before) # --- store --------------------------------------------------------------------------------------------- def verify_location(self) -> tuple[dict, dict]: s = self.site.store(self.loc.zipcode, self.loc.location_id) addr = (s.get("locale") or {}).get("address") or {} problems = [] if str(s.get("brand", "")).upper() != self.loc.expected_chain.upper(): problems.append(f"brand {s.get('brand')} != {self.loc.expected_chain}") if str(addr.get("postalCode", ""))[:5] != self.loc.zipcode: problems.append(f"zip {addr.get('postalCode')} != {self.loc.zipcode}") if self.loc.facility_id and str(s.get("facilityId")) != self.loc.facility_id: problems.append(f"facility {s.get('facilityId')} != {self.loc.facility_id}") self.site.set_store(self.loc.location_id, str(s.get("facilityId")), { "addressLines": addr.get("addressLines", []), "cityTown": addr.get("cityTown", ""), "countryCode": "US", "postalCode": addr.get("postalCode", ""), "stateProvince": addr.get("stateProvince", "")}) line = ", ".join(x for x in [" ".join(addr.get("addressLines", [])), str(addr.get("cityTown", "")).title(), f"{addr.get('stateProvince', '')} {addr.get('postalCode', '')}".strip()] if x) result = {"ok": not problems, "checked_at": utc_now(), "detail": f"{addr.get('name') or s.get('vanityName')} | {line} | brand {s.get('brand')} | store {s.get('locationId')} " f"| facility {s.get('facilityId')}" + (f" | MISMATCH: {problems}" if problems else "")} location = {"zipcode": str(addr.get("postalCode", ""))[:5], "store_name": self.loc.store_name, "store_location": line, "store_logo": self.loc.store_logo, "location_id": self.loc.location_id, "api_name": s.get("vanityName", ""), "chain": s.get("brand", ""), "facility_id": str(s.get("facilityId", "")), "store_number": s.get("storeNumber", "")} self._save_check_raw("store-check") self.run_calls += self._drain_calls() return result, location def check_canaries(self) -> dict: key = f"state/{self.loc.key}/canaries.json" upcs = list(self.loc.canary_upcs) if not upcs and self.storage.exists(key): upcs = json.loads(self.storage.read_text(key))["upcs"] if not upcs: return {"configured": 0, "found": 0, "missing": []} found = self.site.products(upcs) ok = {p.get("id") for p in found if ((p.get("price") or {}).get("storePrices") or {}).get("sourceLocationId") == self.loc.location_id} self._save_check_raw("canaries") self.run_calls += self._drain_calls() return {"configured": len(upcs), "found": len(ok), "missing": [u for u in upcs if u not in ok]} # --- plan ------------------------------------------------------------------------------------------------ def plan_units(self) -> list[dict]: plan_key = f"work/{self.work_key}/{self._run_id}/plan.json" if self.storage.exists(plan_key): plan = json.loads(self.storage.read_text(plan_key)) if plan.get("binding_digest") != self._state.get("binding_digest"): raise CheckpointMismatch(f"plan of run {self._run_id} belongs to binding {plan.get('binding_digest')}, " f"the run is {self._state.get('binding_digest')}: not used") self.expected = plan["expected"] plan["units"] = list({u["id"]: u for u in plan["units"]}.values()) if not any(u.get("kind") == "coupon_links" for u in plan["units"]): plan["units"].append(COUPON_LINK_UNIT) self.storage.write_text(plan_key, json.dumps(plan, indent=1)) return plan["units"] units, expected = [], {} for ci, cat in enumerate(self.cfg.categories): if self.only and cat.name.casefold() not in self.only: continue spec = self.taxonomy.get(cat.name) if spec is None: raise SourceError(f"Category '{cat.name}' has no entry in site_taxonomy.json") if spec.get("coupons"): continue # the Coupons page is always planned first (below): every unit links products to it excl = set(spec.get("exclude", [])) cat_expected = 0 for root in spec["roots"]: _, total, facets = self.site.search(self.loc.location_id, root, size=1, facets=True) if total is None: raise SourceError(f"the site gave no product count for taxonomy root {root}: completeness " f"cannot be proven, nothing planned") node = _find_node(facets, root) or {"id": root, "count": total, "children": []} cat_expected += total - sum((_find_node(facets, e) or {}).get("count", 0) for e in excl if e.startswith(root)) level2 = [node] if len(root) >= 5 else [c for c in node.get("children", []) if c["id"] not in excl] for child in level2: for leaf in self._split(child): tax, brand = leaf.get("taxonomy", leaf["id"]), leaf.get("brand", "") uid = f"{ci:02d}-{slug(cat.name)}-{tax}" + (f"-brand-{slug(brand)}" if brand else "") units.append({"id": uid, "category": cat.name, "kind": "taxonomy", "root": root, "sub_category": child["name"], "term": tax + (f" | brand {brand}" if brand else ""), "taxonomy": tax, "brand": brand, "leaf_name": leaf["name"], "expected": leaf["count"], "capped": leaf["count"] > self.max_node}) expected[cat.name] = {"site_total": cat_expected, "roots": spec["roots"], "exclude": sorted(excl)} units.insert(0, {"id": "00-coupons-all-coupons", "category": "Coupons", "kind": "coupons", "sub_category": "", "term": "coupons", "taxonomy": "", "expected": None}) units.append(COUPON_LINK_UNIT) units = list({u["id"]: u for u in units}.values()) # the site's facet tree can list a node twice self.expected = expected self.storage.write_text(plan_key, json.dumps({"units": units, "expected": expected, "planned_at": utc_now(), "binding": self._state.get("binding"), "binding_digest": self._state.get("binding_digest")}, indent=1)) self.run_calls += self._drain_calls() return units def _split(self, node: dict) -> list[dict]: if node.get("count") is None: raise SourceError(f"the site gave no product count for taxonomy node {node.get('id')}: completeness " f"cannot be proven, nothing planned") if node["count"] <= self.max_node: return [node] children = node.get("children") if not children: _, _, facets = self.site.search(self.loc.location_id, node["id"], size=1, facets=True) children = (_find_node(facets, node["id"]) or {}).get("children") or [] if not children: return self._brand_split(node) out = [] for c in children: out.extend(self._split(c)) covered = sum(c.get("count", 0) for c in children) if covered < node["count"]: # products tagged only at this level: read the node itself too (deduplicated later). A node larger than the # 350-id window is read brand by brand, so nothing tagged only here can fall outside the window. out.extend([{**node, "children": []}] if node["count"] <= self.max_node else self._brand_split(node)) return out def _brand_split(self, node: dict) -> list[dict]: """A node larger than the 350-id search window, split by the site's brand facet (the brand counts add up to it).""" _, total, brands = self.site.search(self.loc.location_id, node["id"], size=1, brand_facet=True) if brands and total is not None and all(b.get("count") is not None for b in brands) \ and sum(b["count"] for b in brands) >= total: return [{"id": node["id"], "taxonomy": node["id"], "name": node["name"], "brand": b["name"], "count": b["count"]} for b in brands] return [{**node, "children": []}] # cannot be split further; collected up to the window and reported as capped # --- collection ----------------------------------------------------------------------------------------- def collect_unit(self, unit: dict, location: dict) -> dict: self.run_calls += self._drain_calls() t0 = time.time() crawl_date = datetime.now(timezone.utc).strftime("%Y-%m-%d") if unit.get("kind") == "coupon_links": return self._link_coupons(unit, t0) if unit.get("kind") == "coupons": # v2.2.18: the site's own total (meta totalCount) of each coupon type is kept; the paged list re-sorts # between pages, so it is read again (at most three passes, union by coupon id) until every coupon the # site counts has been seen. A missing total is recorded as missing and holds the run back. got, totals, entries, counts = {}, {}, {}, {} for kind in ("standard", "cash_back"): seen, listed, total, meta = {}, 0, None, {} for _ in range(3): batch, meta = self.site.all_coupons(kind) listed += len(batch) total = int(meta["totalCount"]) if meta.get("totalCount") is not None else None for c in batch: seen.setdefault(str(c.get("id")), c) if total is None or len(seen) >= total: break got[kind], totals[kind], entries[kind] = list(seen.values()), total, listed counts.update(meta.get("userSavingsInfoByType") or {}) raw = [c.get("raw") for c in self.site.calls if c.get("raw") is not None] self._write_raw(unit, raw) calls = self._drain_calls() return {"unit": unit, "records": [], "coupons": got, "site_totals": totals, "coupons_page_entries": entries, "site_counts": counts, "excluded": {}, "pages": len(calls), "api_total": len(got["standard"]) + len(got["cash_back"]), "products_seen": 0, "products_priced": 0, "requests": calls, "seconds": round(time.time() - t0, 2), "finished_at": utc_now()} upcs, total, pages, total_missing = [], None, 0, False for _ in range(3): offset = 0 while True: items, total, _ = self.site.search(self.loc.location_id, unit["taxonomy"], offset=offset, brand=unit.get("brand", "")) pages += 1 for it in items: if it.get("upc") and it["upc"] not in upcs: upcs.append(it["upc"]) if total is None: total_missing = True # v2.2.18: no site count = completeness unknown, never "complete" break last_offset, offset = offset, offset + PAGE_SIZE if not items or offset >= total: break if offset > SEARCH_MAX_OFFSET: # the site accepts offsets up to 250: ask that last page too (it overlaps the previous one; ids are # de-duplicated), so a node of up to 350 products is read completely if last_offset >= SEARCH_MAX_OFFSET: break offset = SEARCH_MAX_OFFSET # the listing order can shift between pages (one id shown twice, another skipped): when fewer ids than the # site's count came back, read the listing again and keep the union (at most three passes) if total_missing or not total or len(upcs) >= min(total, SEARCH_MAX_OFFSET + PAGE_SIZE): break products = self.site.products(upcs) if upcs else [] returned = {p.get("id") for p in products} missing = [u for u in upcs if u not in returned] if missing: # ask once more without the 'verified' filter before treating an id as detail-less products += [p for p in self.site.products(missing, verified=False) if p.get("id") in missing] returned = {p.get("id") for p in products} # every product answer must echo the store + fulfilment context that was sent; otherwise the unit is # rejected (retried at the end of the run, then the run is held back), whatever the overall success rate self.site.check_context(products, unit["id"]) by_upc = self._coupons_by_upc() records, metas, excluded, status, excluded_items = [], {}, Counter(), Counter(), [] priced = mismatched = 0 for p in products: rec, reason, meta = self.mapper.map_site(p, by_upc.get(p.get("id"), []), location=location, category=unit["category"], sub_category=unit["sub_category"], crawl_date=crawl_date) if meta["source_location"]: priced += 1 if meta["source_location"] and meta["source_location"] != self.loc.location_id: mismatched += 1 if rec is None: excluded[reason] += 1 # only approximate-weight products are left out item = p.get("item") or {} regular = (((p.get("price") or {}).get("storePrices") or {}).get("regular") or {}) excluded_items.append({"upc": item.get("upc") or p.get("id"), "title": item.get("description", ""), "size": item.get("customerFacingSize", ""), "reason": reason, "price_text": regular.get("defaultDescription", ""), "sell_by": (p.get("price") or {}).get("sellBy", ""), "unit": unit["id"], "raw_file": f"raw/{unit['id']}.json.gz"}) continue status[meta["price_status"]] += 1 records.append(rec) metas[rec["upc"]] = meta for u in upcs: if u not in returned: # listed by the category search but no product details returned: kept with its UPC, other fields empty rec = {k: "" for k in FIELD_ORDER} rec.update({"zipcode": location["zipcode"], "store_name": location["store_name"], "store_location": location["store_location"], "store_logo": location.get("store_logo", ""), "store_brand": "no", "category": unit["category"], "sub_category": unit["sub_category"], "upc": u, "crawl_date": crawl_date}) records.append(rec) metas[u] = {"source_location": "", "fulfillment": [], "fulfillment_listed": [], "stock_level": "", "departments": [], "price_status": "listed_without_details"} status["listed_without_details"] += 1 raw = [c.get("raw") for c in self.site.calls if c.get("raw") is not None] self._write_raw(unit, raw) calls = self._drain_calls() return {"unit": unit, "records": records, "meta": metas, "excluded": {k: v for k, v in excluded.items() if v}, "excluded_items": excluded_items, "pages": pages, "api_total": total, "site_total_reported": not total_missing, "listed_upcs": upcs, "found_upcs": len(upcs), "products_seen": len(products), "products_priced": priced, "store_mismatch": mismatched, "price_status": dict(status), "coupons_linked": sum(1 for r in records if r["digital_deals_description"]), "requests": calls, "seconds": round(time.time() - t0, 2), "finished_at": utc_now()} def _link_coupons(self, unit: dict, t0: float) -> dict: """The Coupons page lists coupons without their UPCs. Every collected product is asked for its coupons (100 UPCs per request, about 320 requests for this store); the answer names each coupon's UPCs, which links coupon to product. It also finds coupons the paged Coupons list missed (that list re-sorts between pages). settings site_source.coupon_link_scope = "brand" limits it to products whose title carries a coupon brand.""" units_dir = f"work/{self.work_key}/{self._run_id}/units" coupon_key = next((k for k in self.storage.list(units_dir) if "all-coupons" in k), None) coupons = json.loads(self.storage.read_text(coupon_key))["coupons"]["standard"] if coupon_key else [] brands = {str(c.get("brand", "")).casefold() for c in coupons} - {"", "other", "general", "kroger"} every = self.cfg.settings["site_source"].get("coupon_link_scope", "all") == "all" candidates, seen = [], set() for key in sorted(self.storage.list(units_dir)): if "all-coupons" in key or "coupon-links" in key: continue for r in json.loads(self.storage.read_text(key)).get("records", []): title = r["product_title"].casefold() if r["upc"] not in seen and (every or any(b in title for b in brands)): seen.add(r["upc"]) candidates.append(r["upc"]) found = self.site.coupons_for(candidates) if candidates else [] links = defaultdict(list) for c in found: for u in c.get("upcs") or []: if c not in links[u]: links[u].append(c) raw = [c.get("raw") for c in self.site.calls if c.get("raw") is not None] self._write_raw(unit, raw) calls = self._drain_calls() return {"unit": unit, "records": [], "links": links, "candidates": len(candidates), "candidate_upcs": candidates, "coupons_answered": len(found), "excluded": {}, "pages": len(calls), "api_total": len(candidates), "products_seen": 0, "products_priced": 0, "requests": calls, "seconds": round(time.time() - t0, 2), "finished_at": utc_now()} def _coupons_by_upc(self) -> dict: """Digital coupons from the Coupons page unit (collected first), indexed by the UPCs each coupon applies to.""" if getattr(self, "_coupon_index", None) is None: key = next((k for k in self.storage.list(f"work/{self.work_key}/{self._run_id}/units") if "all-coupons" in k), None) coupons = json.loads(self.storage.read_text(key))["coupons"]["standard"] if key else [] index = defaultdict(list) for c in coupons: for u in c.get("upcs") or []: index[u].append(c) self._coupon_index = index return self._coupon_index def _save_check_raw(self, kind: str) -> None: """Store the raw store-locator / canary responses of every check (before and after each run segment), so the store identity can be audited from the site's own answer, not only from the log line.""" raw = [c.get("raw") for c in self.site.calls if c.get("raw") is not None] if raw and self._run_id: stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") self._write_raw({"id": f"_{kind}-{stamp}"}, raw) def _write_raw(self, unit: dict, raw: list) -> None: if hasattr(self.storage, "write_bytes"): self.storage.write_bytes(f"runs/{self.loc.key}/{self._run_id}/raw/{unit['id']}.json.gz", gzip.compress(json.dumps(raw).encode("utf-8"))) def refinish(self, run_id: str, previous_run: str = "") -> dict: """Rebuild output, report and validation of a finished run from its checkpoints, without contacting the site (used after a mapping fix). Store checks and canaries are taken from the run's own report.""" self._run_id = run_id state = json.loads(self.storage.read_text(f"work/{self.work_key}/current.json")) if state["run_id"] != run_id: raise SourceError(f"current run is {state['run_id']}, not {run_id}") if not state.get("binding"): raise CheckpointMismatch(f"run {run_id} has no source/store/scope binding (written before v2.2.18): not refinished") self._state = state self.only = set(state.get("only") or []) or None # the run's own scope, whatever this command line says old = json.loads(self.storage.read_text(f"runs/{self.loc.key}/{run_id}/report.json")) if "finished_epoch" not in state: state["finished_epoch"] = datetime.strptime(old["finished"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() checks = {c["check"]: c for c in old["validation"]["checks"]} loc_checks = {p: {"ok": checks[f"location_verified_{p}_run"]["passed"], "detail": checks[f"location_verified_{p}_run"]["detail"]} for p in ("before", "after")} state["run_calls_carry"] = state.get("run_calls_carry", []) if previous_run: state["compared_with"] = previous_run log("info", "refinish", run_id=run_id, note="rebuilt from checkpoints; no site requests") return self._finish(state, self.plan_units(), [], loc_checks, old.get("canaries", {})) def _load_catalog(self, state: dict) -> tuple[dict, str | None]: """Every product of the store ever published, with its last observation, as it stood BEFORE this run (a refinish of an already published run uses the release that publish replaced). v2.2.18: read from the published delivery; a store last written by v2.2.17 is read from its legacy files once.""" base = self.delivery.baseline(state["run_id"]) if base and not base.get("legacy"): text = self.delivery.read(base, "catalog.json") if text: data = json.loads(text) return data["products"], data.get("run_id") or base["run_id"] return self._catalog_from_run(base["run_id"], base["release"] + "products.json") for key in (f"state/{self.loc.key}/catalog.json", f"state/{self.loc.key}/catalog_previous.json"): if self.storage.exists(key): data = json.loads(self.storage.read_text(key)) if data.get("run_id") != state["run_id"]: return data["products"], data.get("run_id") if state.get("compared_with") and self.storage.exists(f"runs/{self.loc.key}/{state['compared_with']}/products.json"): return self._catalog_from_run(state["compared_with"], f"runs/{self.loc.key}/{state['compared_with']}/products.json") if not self.storage.exists(f"latest/{self.loc.key}/products.json"): return {}, None prev = json.loads(self.storage.read_text(f"latest/{self.loc.key}/report.json")) if prev.get("run_id") == state["run_id"]: return {}, None return self._catalog_from_run(prev.get("run_id"), f"latest/{self.loc.key}/products.json") def _catalog_from_run(self, run: str, products_key: str) -> tuple[dict, str | None]: ev = {} ev_key = f"runs/{self.loc.key}/{run}/store_evidence.jsonl" if self.storage.exists(ev_key): for line in self.storage.read_text(ev_key).splitlines(): if line.strip(): e = json.loads(line) ev[e["upc"]] = {"source_location": e.get("sourceLocationId", ""), "fulfillment": e.get("fulfillment_sellable", []), "stock_level": e.get("stock_level", "")} products = json.loads(self.storage.read_text(products_key)) return {r["upc"]: {"record": r, "last_observed_run": run, "meta": ev.get(r["upc"], {})} for r in products}, run def _catalog_of(self, loc_key: str) -> dict | None: """Another configured store's published catalog (identity fill only), or None.""" d = Delivery(self.storage, loc_key) ptr = d.current() if ptr is not None: text = d.read(ptr, "catalog.json") return json.loads(text)["products"] if text else None key = f"state/{loc_key}/catalog.json" return json.loads(self.storage.read_text(key))["products"] if self.storage.exists(key) else None def _carry_api_stats(self, state: dict) -> None: state["run_calls_carry"] = state.get("run_calls_carry", []) + self.run_calls + self._drain_calls() self.run_calls = [] # --- finish ---------------------------------------------------------------------------------------------- def _owner_category(self, meta: dict, candidates: list[str]) -> str: """The CSV category whose taxonomy covers the product's own retailer taxonomy (dept+commodity codes).""" codes = meta.get("departments") or [] for cat in candidates: spec = self.taxonomy.get(cat, {}) for code in codes: if any(code.startswith(r) for r in spec.get("roots", [])) and not any(code.startswith(e) for e in spec.get("exclude", [])): return cat return candidates[0] def _finish(self, state, units, failed, location_checks, canaries) -> dict: run_id = state["run_id"] unit_prefix = f"work/{self.work_key}/{run_id}/units/" by_id = {Path(k).stem: k for k in self.storage.list(unit_prefix)} found_in = defaultdict(list) # upc -> [(category, record, meta)] in plan (CSV) order coupon_links, coupon_candidates = {}, 0 excluded, excluded_items = Counter(), {} coupons = {"standard": [], "cash_back": []} coupon_site_counts, listed_entries = {}, {} cat_ids, cat_listed = defaultdict(set), Counter() # v2.2.18: DISTINCT ids per category coupon_site_totals, coupon_candidate_upcs = {}, None unit_requests, unit_seconds, pages, capped = [], 0.0, 0, [] products_seen = products_priced = store_mismatch = 0 unit_cover, site_drift = [], Counter() for unit in units: if unit["id"] not in by_id: continue res = json.loads(self.storage.read_text(by_id[unit["id"]])) self._check_unit(state, unit["id"], res) unit_requests += res.get("requests", []) unit_seconds += res.get("seconds", 0) pages += res.get("pages", 0) if unit.get("kind") == "coupons": coupons = res["coupons"] coupon_site_counts = res.get("site_counts", {}) coupon_site_totals = res.get("site_totals", {}) listed_entries = res.get("coupons_page_entries") or {t: len(v) for t, v in coupons.items()} # the paged list can return one coupon twice (it re-sorts between pages): one entry per coupon id coupons = {t: list({str(c.get("id")): c for c in reversed(v)}.values())[::-1] for t, v in coupons.items()} continue if unit.get("kind") == "coupon_links": coupon_links = res.get("links", {}) coupon_candidates = res.get("candidates", 0) coupon_candidate_upcs = set(res.get("candidate_upcs") or []) continue excluded.update(res.get("excluded", {})) for x in res.get("excluded_items", []): excluded_items.setdefault(x["upc"], x) products_seen += res.get("products_seen", 0) products_priced += res.get("products_priced", 0) store_mismatch += res.get("store_mismatch", 0) cat_ids[unit["category"]].update(res.get("listed_upcs") or []) cat_listed[unit["category"]] += res.get("found_upcs", 0) unit_cover.append((unit["id"], res.get("found_upcs", 0), res.get("api_total", 0))) # the site's own count for this node can fall between planning and collecting (products leave the listing); # when every id the site listed at collection time was collected, that drop is the site's change, not a gap if unit.get("expected") is not None and res.get("api_total") is not None and res.get("found_upcs", 0) >= res["api_total"]: site_drift[unit["category"]] += max(0, unit["expected"] - res["api_total"]) if unit.get("capped"): capped.append(unit["id"]) for rec in res["records"]: found_in[rec["upc"]].append((unit["category"], rec, res["meta"].get(rec["upc"], {}), unit["id"])) records, placements, moved, output_meta, evidence = [], {}, 0, {}, [] for upc, hits in found_in.items(): cats = list(dict.fromkeys(h[0] for h in hits)) owner = self._owner_category(hits[0][2], cats) if len(cats) > 1 else cats[0] rec = dict(next(h[1] for h in hits if h[0] == owner)) if upc in coupon_links: rec["digital_deals_description"] = self.mapper.digital_text(coupon_links[upc]) rec["digital_deals_price"] = self.mapper.digital_price(coupon_links[upc]) own = next(h for h in hits if h[0] == owner) output_meta[upc] = own[2] evidence.append({"upc": upc, "sourceLocationId": own[2].get("source_location", ""), "fulfillment_sellable": own[2].get("fulfillment", []), "unit": own[3], "raw_file": f"raw/{own[3]}.json.gz"}) if owner != hits[0][0]: moved += 1 records.append(rec) if len(hits) > 1: placements[upc] = [f"{h[1]['category']} > {h[1]['sub_category']}" for h in hits] duplicates_removed = sum(len(h) - 1 for h in found_in.values()) fresh_records = list(records) # Availability rule: nothing is dropped for being unavailable. A product observed in an earlier published run # but not listed by the site this time is carried forward with its last observation (its crawl_date stays the # date it was last seen), and availability.jsonl marks it carried_forward, so an old price never reads as new. limited = self.limited_reasons() catalog, prev_run = ({}, None) if limited else self._load_catalog(state) availability, changes = [], [] for upc, m in output_meta.items(): before = catalog.get(upc, {}).get("meta", {}) now_unavailable = not m.get("fulfillment") or m.get("stock_level") == "TEMPORARILY_OUT_OF_STOCK" was_available = bool(before.get("fulfillment")) and before.get("stock_level") != "TEMPORARILY_OUT_OF_STOCK" if was_available and now_unavailable: changes.append({"upc": upc, "before": {"run": catalog[upc]["last_observed_run"], "sellable": before.get("fulfillment"), "stock_level": before.get("stock_level", "")}, "now": {"run": run_id, "sellable": m.get("fulfillment"), "stock_level": m.get("stock_level", ""), "price_status": m.get("price_status")}, "kept_as": "fresh record"}) # identity-only records (listed by the category search, no product details returned for the store): keep the # UPC and fill title / size / image / url known from the last observation of that UPC; prices and deals stay # empty, nothing is invented, and availability.jsonl names the run the identity came from identity_from = {} others = None for r in records: if output_meta.get(r["upc"], {}).get("price_status") != "listed_without_details": continue source = (catalog.get(r["upc"]), self.loc.key) if source[0] is None: # not seen at this store before: a product's identity (title, size, image, url) is the same at every # store, so another configured store's last observation may supply it; its prices never do if others is None: others = {} for loc in self.cfg.locations: other = self._catalog_of(loc.key) if loc.key != self.loc.key else None if other is not None: others[loc.key] = other source = next(((c[r["upc"]], k) for k, c in others.items() if r["upc"] in c and c[r["upc"]]["record"].get("product_title")), (None, None)) if source[0] is None: continue # never described by the site at any configured store (the official API has no record either) old, origin = source[0]["record"], f"{source[1]}/{source[0]['last_observed_run']}" filled = [k for k in ("product_title", "weight", "image_url", "url") if not r[k] and (old or {}).get(k)] for k in filled: r[k] = old[k] if filled: identity_from[r["upc"]] = origin rec_by_upc = {r["upc"]: r for r in records} carried_origin = Counter() for upc, entry in catalog.items(): if upc in output_meta or upc in excluded_items: continue # seen this run (fresh), or seen and excluded by a rule: never carried forward records.append(dict(entry["record"])) carried_origin[entry["last_observed_run"]] += 1 availability.append({"upc": upc, "observation": "carried_forward", "last_observed_run": entry["last_observed_run"], "crawl_date": entry["record"]["crawl_date"], "reason": "not listed by the site in this run", "price_status": entry.get("meta", {}).get("price_status", ""), "stock_level": "", "fulfillment_sellable": []}) changes.append({"upc": upc, "before": {"run": entry["last_observed_run"], "sellable": entry.get("meta", {}).get("fulfillment", []), "stock_level": entry.get("meta", {}).get("stock_level", "")}, "now": {"run": run_id, "listed": False}, "kept_as": "carried_forward (last observation)"}) for upc, m in output_meta.items(): availability.append({"upc": upc, "observation": "fresh", "last_observed_run": run_id, "crawl_date": rec_by_upc[upc]["crawl_date"], "price_status": m.get("price_status", ""), "stock_level": m.get("stock_level", ""), "fulfillment_sellable": m.get("fulfillment", []), "fulfillment_listed": m.get("fulfillment_listed", []), **({"identity_from_run": identity_from[upc]} if upc in identity_from else {})}) order = {c.name: i for i, c in enumerate(self.cfg.categories)} records.sort(key=lambda r: (order.get(r["category"], 99), r["sub_category"], r["product_title"], r["upc"])) # timing and request reconciliation for seg in state["segments"]: seg["end"] = seg["end"] or time.time() active = sum(max(0.0, seg["end"] - seg["start"]) for seg in state["segments"]) state.setdefault("finished_epoch", time.time()) # kept on a later refinish, so elapsed stays the run's own elapsed = state["finished_epoch"] - state["started_epoch"] run_level = state.get("run_calls_carry", []) + self.run_calls + self._drain_calls() state["run_calls_carry"], self.run_calls = run_level, [] # stored with the run, so a refinish counts them too all_requests = run_level + unit_requests kinds = Counter("search" if "products-search" in r["path"] else "product" if "/product/v2/" in r["path"] else "coupons" if "savings-coupons" in r["path"] else "store" if "locator" in r["path"] else "other" for r in all_requests) # completeness against the site's own counts completeness_by_cat = {} counts_cat = Counter(r["category"] for r in fresh_records) for cat, exp in getattr(self, "expected", {}).items(): site_total = exp["site_total"] drift = min(site_drift.get(cat, 0), site_total) current = site_total - drift found = len(cat_ids.get(cat, ())) completeness_by_cat[cat] = {"site_count": site_total, "ids_collected": found, "ids_listed_incl_overlap": cat_listed.get(cat, 0), "records_written": counts_cat.get(cat, 0), "site_count_drop_during_run": drift, "coverage_vs_planned_count": round(min(found, site_total) / site_total, 4) if site_total else 1.0, "coverage": round(min(found, current) / current, 4) if current else 1.0} v = self.cfg.settings["validation"] coupon_planned = any(u.get("kind") == "coupons" for u in units) low = {c: x["coverage"] for c, x in completeness_by_cat.items() if x["coverage"] < v["min_category_coverage"]} store_priced = [r for r in fresh_records if output_meta[r["upc"]].get("price_status") == "store_price"] ppm_share = sum(1 for r in store_priced if r["ppm"]) / len(store_priced) if store_priced else 0 foreign = [r["upc"] for r in fresh_records if r["regular_price"] and output_meta[r["upc"]].get("source_location") != self.loc.location_id] price_status = Counter(m.get("price_status", "") for m in output_meta.values()) output_upcs = {r["upc"] for r in records} coupon_upcs = set(coupon_links) | {u for c in coupons["standard"] for u in c.get("upcs") or []} # coupons the product lookup returned that the paged Coupons list did not: added to coupons.json, marked listed_unique = {t: len(v) for t, v in coupons.items()} known, found_only = {str(c.get("id")) for v in coupons.values() for c in v}, Counter() for upc in sorted(coupon_links): for c in coupon_links[upc]: cid = str(c.get("id")) if cid in known: continue known.add(cid) kind = "cash_back" if str(c.get("type", "")).upper() == "CASH_BACK" else "standard" coupons.setdefault(kind, []).append({**c, "found_via": "product coupon lookup (not returned by the paged Coupons list)"}) found_only[kind] += 1 # no percentage may hide a short node: every unit must hold every id the site listed for it when it was read short_units = [f"{uid} {found}/{total}" for uid, found, total in unit_cover if total is not None and found < total] no_total = [uid for uid, _, total in unit_cover if total is None] coupon_short = [] if coupon_planned: for kind in ("standard", "cash_back"): site_total, have = coupon_site_totals.get(kind), len(coupons.get(kind, [])) if site_total is None or have < site_total: coupon_short.append(f"{kind}: {have} distinct, site total " f"{site_total if site_total is not None else 'not reported'}") fresh_upcs = {r["upc"] for r in fresh_records} link_scope_all = self.cfg.settings["site_source"].get("coupon_link_scope", "all") == "all" not_looked_up = sorted(fresh_upcs - (coupon_candidate_upcs or set())) if coupon_planned and link_scope_all else [] extra = [ {"check": "site_count_reported_for_every_unit", "passed": not no_total, "detail": "the site reported its own product count for every taxonomy unit" if not no_total else f"{len(no_total)} units without a site count (completeness unknown): {no_total[:20]}"}, {"check": "coupons_complete_vs_site_totals", "passed": not coupon_short, "detail": "every coupon type: distinct coupons collected >= the site's own total" if not coupon_short else f"incomplete: {coupon_short}"}, {"check": "coupon_lookup_covered_every_product", "passed": not not_looked_up, "detail": f"all {len(fresh_upcs)} distinct products were asked for their coupons" if not not_looked_up else f"{len(not_looked_up)} products never asked for their coupons: {not_looked_up[:10]}"}, {"check": "every_unit_paginated_completely", "passed": not short_units, "detail": "every taxonomy unit collected every id the site listed for it" if not short_units else f"{len(short_units)} units short of the site's own count: {short_units[:20]}"}, {"check": "category_completeness_vs_site_counts", "passed": not low, "detail": f"every category collected at least {v['min_category_coverage']:.0%} of the product count the site reports " f"(distinct products; an id listed by several units of a category counts once)" if not low else f"below {v['min_category_coverage']:.0%}: {low}"}, {"check": "store_on_every_price", "passed": not foreign, "detail": f"every price in the output came back with sourceLocationId {self.loc.location_id}; " f"{price_status.get('other_location_price_not_used', 0)} products priced only for another location are kept with empty price fields" if not foreign else f"{len(foreign)} records with a foreign price"}, {"check": "ppm_present", "passed": ppm_share >= v["min_ppm_share"], "detail": f"{ppm_share:.1%} of records carry the displayed price per measure (min {v['min_ppm_share']:.0%})"}, {"check": "coupons_page_collected", "passed": not coupon_planned or len(coupons["standard"]) > 0, "detail": f"{len(coupons['standard'])} digital coupons + {len(coupons['cash_back'])} cash-back offers read from the Coupons page"}, ] priced_share = products_priced / products_seen if products_seen else 0.0 previous = None if not limited: prev_key = f"runs/{self.loc.key}/{state['compared_with']}/report.json" if state.get("compared_with") else None prev = (json.loads(self.storage.read_text(prev_key)) if prev_key and self.storage.exists(prev_key) else self._baseline_report(run_id)) if prev and prev.get("source", "").startswith("marianos.com") and prev.get("run_id") != run_id: previous = prev if previous: state["compared_with"] = previous["run_id"] # kept, so a later refinish compares with the same run supported = [c for c in completeness_by_cat] validation = validate_run(records=records, category_counts=counts_cat, supported_categories=supported, failed_units=failed, location_checks=location_checks, priced_share=priced_share, canaries=canaries, previous=previous, settings=self.cfg.settings, extra_checks=extra, min_priced_share=v["site_min_priced_share"]) cross = self._api_cross_check(records) products_json = json.dumps(records, ensure_ascii=False, indent=1) deal_examples = { "ppm": [r for r in records if r["ppm"]][:3], "volume_deal": [r for r in records if r["volume_deals_description"]][:3], "volume_deal_with_price": [r for r in records if r["volume_deals_price"]][:3], "digital_deal": [r for r in records if r["digital_deals_description"] and not r["digital_deals_price"]][:3], "digital_deal_with_price": [r for r in records if r["digital_deals_price"]][:3], "sale_price": [r for r in records if r["sale_price"]][:3], "no_deal": [r for r in records if not (r["sale_price"] or r["volume_deals_description"] or r["digital_deals_description"])][:3], } report = { "run_id": run_id, "status": ("sample" if limited else "published") if validation["passed"] else "held_back", "delivery": {"kind": "sample" if limited else "full", "limited_by": limited, "rule": "only a full run that passes every check replaces latest/; a limited run is a sample release"}, "source": "marianos.com (site JSON endpoints, transport %s)" % (state.get("binding") or {}).get("transport", "unknown"), "location": state["location"], "started": state["started"], "finished": datetime.fromtimestamp(state["finished_epoch"], timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "runtime": {"active_seconds": round(active, 1), "elapsed_seconds": round(elapsed, 1), "unit_collection_seconds": round(unit_seconds, 1), "segments": len(state["segments"]), "resumed_times": state.get("resumed", 0), "crashed_segments": state.get("crashed_segments", 0), "stopped_segments": state.get("stopped_segments", 0), "note": "active = sum of process run segments; elapsed = first start to finish, including any time " "between an interruption and the resume"}, "runtime_seconds": round(active, 1), "resumed_times": state.get("resumed", 0), "scope": "partial (categories: %s)" % ", ".join(sorted(self.only)) if self.only else "full catalog", "binding": state.get("binding"), "requests_reserved": {"counted_from_reservation_after_a_crash": state.get("requests_counted_from_reservation", 0), "run_total_counted": state.get("site_requests_base", 0) + self.site.stats.requests}, "unique_records": len(records), "duplicates_removed": duplicates_removed, "placed_by_retailer_taxonomy": moved, "availability": {"fresh_records": len(fresh_records), "carried_forward_records": len(records) - len(fresh_records), "carried_forward_from_run": prev_run, "price_status": dict(price_status), "stock_level": dict(Counter(m.get("stock_level") or "not_reported" for m in output_meta.values())), "not_sellable_for_any_fulfilment": sum(1 for m in output_meta.values() if not m.get("fulfillment")), "rule": "no product is removed for availability; prices only from this store, otherwise empty; carried-forward " "records keep their last crawl_date and are listed in availability.jsonl"}, "availability_changes_vs_previous": { "previous_catalog_run": prev_run, "previous_catalog_products": len(catalog), "new_upcs": sum(1 for u in output_meta if u not in catalog), "seen_again": sum(1 for u in output_meta if u in catalog), "carried_forward": len(records) - len(fresh_records), "carried_forward_last_observed_in": dict(sorted(carried_origin.items())), "identity_only_restored_from_earlier_runs": len(identity_from), "available_before_unavailable_now": sum(1 for c in changes if c["kept_as"] == "fresh record"), "note": "available = sellable for at least one fulfilment at this store and not TEMPORARILY_OUT_OF_STOCK; " "unavailable = no sellable fulfilment or TEMPORARILY_OUT_OF_STOCK; full list in availability_changes.json"}, "available_to_unavailable_examples": [c for c in changes if c["kept_as"] == "fresh record"][:15] + [c for c in changes if c["kept_as"] != "fresh record"][:5], "excluded": dict(excluded), "excluded_products": dict(Counter(x["reason"] for x in excluded_items.values())), "products_seen": products_seen, "priced_share": round(priced_share, 4), "api_pages": pages, "requests": {"total": len(all_requests), "by_endpoint": dict(kinds), "units": len(unit_requests), "run_level": len(run_level), "retries": self.site.stats.retries, "errors": self.site.stats.errors, "non_200": sum(1 for r in all_requests if r["status"] != 200)}, "category_completeness": completeness_by_cat, "capped_units": capped, "counts_by_category": dict(sorted(counts_cat.items(), key=lambda kv: -kv[1])), "counts_by_subcategory": dict(sorted(Counter(f"{r['category']} > {r['sub_category']}" for r in records).items())), "coupons": {"digital_standard": len(coupons["standard"]), "cash_back": len(coupons["cash_back"]), "coupons_page_entries": listed_entries, "coupons_page_unique": listed_unique, "found_only_via_product_lookup": dict(found_only), "site_reported": coupon_site_counts, "site_totals": coupon_site_totals, "digital_coupon_upcs": len(coupon_upcs), "of_which_in_output": len(coupon_upcs & output_upcs), "records_with_digital_deal": sum(1 for r in records if r["digital_deals_description"]), "link_candidates_checked": coupon_candidates, "products_linked_to_coupons": len(coupon_links)}, "coupon_links": None, "field_completeness_pct": completeness(records), "records_with_sale_price": sum(1 for r in records if r["sale_price"]), "records_with_volume_deal": sum(1 for r in records if r["volume_deals_description"]), "records_with_digital_deal": sum(1 for r in records if r["digital_deals_description"]), "records_with_digital_price": sum(1 for r in records if r["digital_deals_price"]), "records_with_ppm": sum(1 for r in records if r["ppm"]), "store_brand_records": sum(1 for r in records if r["store_brand"] == "yes"), "deal_examples": deal_examples, "api_cross_check": cross, "canaries": canaries, "validation": validation, "evidence": {"products_sha256": hashlib.sha256(products_json.encode()).hexdigest(), "code_fingerprint": code_fingerprint(), "units_completed": len(by_id), "units_planned": len(units)}, } # CSV row -> taxonomy -> units -> site count -> collected -> written, one line per row of categories.csv units_by_cat = defaultdict(list) for u in units: units_by_cat[u["category"]].append(u["id"]) reconciliation = [] for c in self.cfg.categories: spec = self.taxonomy.get(c.name, {}) comp = completeness_by_cat.get(c.name, {}) reconciliation.append({ "csv_category": c.name, "csv_url": c.url, "taxonomy_roots": spec.get("roots", []), "excluded_subtrees": spec.get("exclude", []), "units": len(units_by_cat.get(c.name, [])), "unit_ids": units_by_cat.get(c.name, []), "site_count": comp.get("site_count"), "ids_collected": comp.get("ids_collected"), "records_written": comp.get("records_written"), "note": "Coupons page: all digital coupons + cash-back offers, then linked to products by UPC" if spec.get("coupons") else ""}) report["csv_reconciliation"] = [{k: v for k, v in r.items() if k != "unit_ids"} for r in reconciliation] by_coupon = {} for upc, cs in coupon_links.items(): if upc not in output_meta: continue for cpn in cs: entry = by_coupon.setdefault(str(cpn.get("id")), {"id": cpn.get("id"), "type": cpn.get("type"), "title": cpn.get("title"), "shortDescription": cpn.get("shortDescription"), "requirementDescription": cpn.get("requirementDescription"), "expirationDate": cpn.get("expirationDate"), "upcs_in_output": []}) entry["upcs_in_output"].append(upc) for e in by_coupon.values(): e["upcs_in_output"].sort() is_cb = lambda c: str(c.get("type", "")).upper() == "CASH_BACK" linked = sorted(by_coupon.values(), key=lambda e: (-len(e["upcs_in_output"]), str(e["id"]))) not_linked = lambda kind: [{"id": c.get("id"), "title": c.get("title"), "shortDescription": c.get("shortDescription"), "reason": "no product collected for this store answered with this offer"} for c in coupons.get(kind, []) if str(c.get("id")) not in by_coupon] out = f"runs/{self.loc.key}/{run_id}/" self.storage.write_text(out + "reconciliation.json", json.dumps(reconciliation, indent=1, ensure_ascii=False)) self.storage.write_text(out + "coupon_links.json", json.dumps({ "note": "digital coupons (clip to card) feed digital_deals_*; cash-back offers are linked separately and never " "written to the output fields", "digital_coupons_linked": [e for e in linked if not is_cb(e)], "digital_coupons_not_linked": not_linked("standard"), "cash_back_offers_linked": [e for e in linked if is_cb(e)], "cash_back_offers_not_linked": not_linked("cash_back")}, indent=1, ensure_ascii=False)) report["coupon_links"] = {"digital_coupons_linked": sum(1 for e in linked if not is_cb(e)), "digital_coupons_not_linked": len(not_linked("standard")), "cash_back_offers_linked": sum(1 for e in linked if is_cb(e)), "cash_back_offers_not_linked": len(not_linked("cash_back")), "products_checked": coupon_candidates} self.storage.write_text(out + "store_evidence.jsonl", "\n".join(json.dumps(e) for e in evidence)) self.storage.write_text(out + "availability.jsonl", "\n".join(json.dumps(a) for a in availability)) self.storage.write_text(out + "availability_changes.json", json.dumps(changes, indent=1)) self.storage.write_text(out + "excluded.jsonl", "\n".join(json.dumps(x, ensure_ascii=False) for x in sorted(excluded_items.values(), key=lambda x: (x["reason"], x["upc"])))) coupons_json = json.dumps(coupons, indent=1, ensure_ascii=False) self.storage.write_text(out + "products.json", products_json) self.storage.write_text(out + "placements.json", json.dumps(placements, indent=1)) self.storage.write_text(out + "coupons.json", coupons_json) self.storage.write_text(out + "report.json", json.dumps(report, indent=1, ensure_ascii=False)) self.storage.write_text(out + "requests.jsonl", "\n".join(json.dumps(r) for r in all_requests)) if validation["passed"]: # v2.2.18: products, coupons, report and the store's observation history (catalog) are published together # by the one writer (delivery.py); a limited run becomes a sample release and never touches latest/ new_catalog = dict(catalog) for r in fresh_records: new_catalog[r["upc"]] = {"record": r, "last_observed_run": run_id, "meta": output_meta[r["upc"]]} for upc in excluded_items: new_catalog.pop(upc, None) self._deliver(state, report, {"products.json": products_json, "coupons.json": coupons_json, "report.json": json.dumps(report, indent=1, ensure_ascii=False), "catalog.json": json.dumps({"run_id": run_id, "products": new_catalog})}) if state["status"] == "published" and not self.storage.exists(f"state/{self.loc.key}/canaries.json"): self._seed_canaries(fresh_records) else: state["status"] = "incomplete" if failed else "held_back" log("error", "run_held_back", reason="validation failed; last good dataset left untouched", failed_checks=[c for c in validation["checks"] if not c["passed"]]) self._save_state(state) log_path = Path(self.cfg.storage_root) / "_runlogs" / f"{self.loc.key}-{run_id}.log" if log_path.is_file(): self.storage.write_text(out + "run.log", log_path.read_text(encoding="utf-8")) return report def _api_cross_check(self, records: list[dict]) -> dict: """Random records re-priced through the official Kroger API for the same store (independent second source).""" n = self.cfg.settings["site_source"].get("api_cross_check_sample", 0) if not n or isinstance(self.api, _NoApi) or not records: return {"skipped": True} sample = random.Random(7).sample(records, min(n, len(records))) try: found = {p["productId"]: p for p in self.api.products_by_id([r["upc"] for r in sample], self.loc.location_id)} except SourceError as exc: return {"skipped": True, "error": str(exc)} same = diff = missing = 0 diffs = [] for r in sample: p = found.get(r["upc"]) price = ((p or {}).get("items") or [{}])[0].get("price") or {} if not price.get("regular"): missing += 1 continue if f"{float(price['regular']):.2f}" == r["regular_price"]: same += 1 else: diff += 1 diffs.append({"upc": r["upc"], "site": r["regular_price"], "api": f"{float(price['regular']):.2f}"}) return {"sample": len(sample), "same_regular_price": same, "different": diff, "not_in_api": missing, "differences": diffs[:10]} class _NoApi: class _S: requests = retries = errors = slow_responses = token_refreshes = 0 request_log: list = [] stats = _S()