"""One scheduled run for one store: verify store -> collect units with checkpoints -> validate -> publish or hold. Storage layout (local folder or Azure Blob container): work//current.json run state; an unfinished run is resumed by the next start work///units/.json one finished category/subcategory/term unit (the checkpoint) runs///... every run's output, report, request log and run log (good or failed) latest//CURRENT.json pointer to the last FULL run that passed validation (v2.2.18: delivery.py) releases//// that delivery: products, coupons, report, catalog, published together samples/// a passed run that is not a full delivery (subset, fallback): never latest/ state//canaries.json canary products seeded from the first good run """ from __future__ import annotations import hashlib import json import re import secrets import signal import time from collections import Counter, defaultdict from datetime import datetime, timezone from pathlib import Path from .config import Config, Location from .delivery import Delivery, PublishFailed from .kroger_api import KrogerApi, SourceBlocked, SourceError from .site_source import RequestCapReached from .log import add_sink, log, remove_sink, set_context from .mapper import FIELD_ORDER, Mapper from .storage import open_storage from .validate import completeness, record_key, validate_run RESUME_MAX_AGE_HOURS = 48 LOCK_STALE_SECONDS = 6 * 3600 DEFAULT_DEADLINE_SECONDS = 5 * 3600 # one collection process; below LOCK_STALE_SECONDS, so a live lock never looks stale CANARY_COUNT = 20 GENERIC_HEADS = {"food", "foods", "care", "supplies", "products", "items", "meals", "mix", "drinks", "fresh", "relief", "aid", "aids"} def slug(text: str) -> str: return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") def keyword_pattern(words: list[str]) -> re.Pattern | None: """Match a word and its plural/-y forms at the end of a word: 'berries' -> berr(y|ies), 'nuts' -> nut(s). Long stems may sit inside a compound ('strawberries' matches 'berries'); short stems must start a word so that 'pens' does not match 'open' and 'ham' does not match 'graham'.""" alternatives = set() for word in words: w = word.casefold().strip() if not w or w in GENERIC_HEADS: continue if w.endswith("ies"): w = w[:-3] elif w.endswith("oes"): w = w[:-2] elif w.endswith("s") and not w.endswith("ss"): w = w[:-1] alternatives.add((r"\b" if len(w) <= 3 else "") + re.escape(w)) if not alternatives: return None return re.compile(r"(?:%s)(?:s|es|y|ies)?\b" % "|".join(sorted(alternatives, key=len, reverse=True)), re.IGNORECASE) def subcategory_matchers(category) -> list[tuple[str, re.Pattern | None]]: """Per subcategory: the title words from config 'match', or else the head word of each search term.""" match = getattr(category, "match", {}) or {} out = [] for sub, terms in category.subcategories.items(): words = match.get(sub) or [t.split()[-1] for t in terms] out.append((sub, keyword_pattern(words))) return out def utc_now() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def code_fingerprint() -> str: h = hashlib.sha256() for path in sorted(Path(__file__).parent.glob("*.py")): h.update(path.read_bytes()) return h.hexdigest()[:16] class Interrupted(Exception): pass class CheckpointMismatch(SourceError): """An unfinished run (or one of its checkpoints) belongs to another source, store or scope: never resumed or used, nothing is sent (exit 6, not retried; resume it with the same settings or start over with --fresh).""" def deadline_seconds(settings: dict) -> float: """Runtime deadline of one collection process (settings collection.deadline_seconds), always finite.""" value = float((settings.get("collection") or {}).get("deadline_seconds") or DEFAULT_DEADLINE_SECONDS) if not 0 < value < LOCK_STALE_SECONDS: raise ValueError(f"collection.deadline_seconds must be between 0 and {LOCK_STALE_SECONDS}, got {value}") return value def digest(data) -> str: return hashlib.sha256(json.dumps(data, sort_keys=True).encode("utf-8")).hexdigest()[:16] class Runner: def __init__(self, cfg: Config, location: Location, *, fresh: bool = False, only_categories: list[str] | None = None, stop_after_units: int | None = None, api: KrogerApi | None = None, storage=None, fallback_reason: str = ""): self.cfg = cfg self.fallback_reason = fallback_reason # A fallback run keeps its own state and checkpoints, so it can never replace an unfinished website run. self.work_key = location.key + ("-api-fallback" if fallback_reason else "") self.loc = location self.fresh = fresh self.only = {c.casefold() for c in only_categories} if only_categories else None self.stop_after_units = stop_after_units self.api = api or KrogerApi(cfg.client_id, cfg.client_secret, cfg.settings["api"]) self.storage = storage or open_storage(cfg) self.mapper = Mapper(cfg.settings) self.page_size = cfg.settings["api"]["page_size"] self.max_start = cfg.settings["api"]["max_start"] self._interrupted = False self.matchers: dict[str, list] = {} self.excluders: dict[str, re.Pattern | None] = {} self.delivery = Delivery(self.storage, location.key) self.deadline = deadline_seconds(cfg.settings) self._t0 = time.monotonic() # --- what this run is (v2.2.18) --------------------------------------------------------------------- def binding(self) -> dict: """Source, store and scope of this run. Stored with the run state, the plan and every unit checkpoint; a run is resumed, and a checkpoint used, only under exactly the same binding.""" return {"source": "kroger_public_api" + (" (fallback)" if self.fallback_reason else ""), "location_key": self.loc.key, "location_id": self.loc.location_id, "zipcode": self.loc.zipcode, "chain": self.loc.expected_chain, "scope": sorted(self.only) if self.only else "full"} def limited_reasons(self) -> list[str]: """Why this run is not a full delivery (empty = full). A limited run is released as a sample only. Read from the run's own binding, so a refinish judges the run by how it was collected, not by today's options.""" b = (getattr(self, "_state", None) or {}).get("binding") or self.binding() out = [] if b.get("scope") != "full": out.append("categories limited to: " + ", ".join(b.get("scope") or ["(unknown)"])) if "fallback" in str(b.get("source", "")): out.append("fallback source: " + (self.fallback_reason or "Kroger Public API")) if b.get("coupon_link_scope", "all") != "all": out.append(f"coupon lookup limited (coupon_link_scope={b['coupon_link_scope']})") return out def time_left(self) -> float: return self.deadline - (time.monotonic() - self._t0) def _wait(self, seconds: float) -> None: """A wait between units that stops at the deadline instead of running past it.""" if seconds >= self.time_left(): raise Interrupted("cap") time.sleep(seconds) def _check_unit(self, state: dict, unit_id: str, result: dict) -> None: if result.get("binding_digest") != state.get("binding_digest"): raise CheckpointMismatch(f"checkpoint {unit_id} of run {state['run_id']} was written for binding " f"{result.get('binding_digest') or 'none (before v2.2.18)'}, the run is " f"{state.get('binding_digest')}: not used") def _baseline_report(self, run_id: str) -> dict | None: text = self.delivery.read(self.delivery.baseline(run_id), "report.json") return json.loads(text) if text else None def _deliver(self, state: dict, report: dict, files: dict) -> None: """A passed run: a full delivery through the one writer, or a sample release. On any failure the previous delivery stays as it was.""" run_id, out = state["run_id"], f"runs/{self.loc.key}/{state['run_id']}/" try: if report["delivery"]["kind"] == "sample": report["delivery"]["release"] = self.delivery.release_sample(run_id, files) state["status"] = "sample" log("info", "run_released_as_sample", release=report["delivery"]["release"], limited_by=report["delivery"]["limited_by"], note="latest/ untouched") else: ptr = self.delivery.publish(run_id, files, {"records": report["unique_records"], "products_sha256": report["evidence"]["products_sha256"]}) report["delivery"]["release"] = ptr["release"] report["delivery"]["replaced"] = (ptr.get("previous") or {}).get("run_id") state["status"] = "published" log("info", "run_published", records=report["unique_records"], release=ptr["release"], runtime_seconds=report["runtime_seconds"]) except PublishFailed as exc: report["status"] = "publish_failed" report["delivery"]["error"] = str(exc) state["status"] = "publish_failed" log("error", "publish_failed", error=str(exc), note="the previous delivery is unchanged") self.storage.write_text(out + "report.json", json.dumps(report, indent=1, ensure_ascii=False)) def classify(self, unit: dict, title: str) -> tuple[str, str]: """Pick the subcategory whose words appear in the title, preferring the one being searched. A search for 'eggs' also returns cheeses and creamers from the Dairy department; those must not be labelled 'Eggs'. Returns (sub_category, note) where sub_category is "" if nothing matches.""" matchers = self.matchers.get(unit["category"], []) for sub, pattern in matchers: if sub == unit["sub_category"] and pattern and pattern.search(title): return sub, "" for sub, pattern in matchers: if pattern and pattern.search(title): return sub, "placed_in_other_subcategory" return "", "no_subcategory_match" # --- plan --------------------------------------------------------------------------------------- def plan_units(self) -> list[dict]: units = [] for ci, cat in enumerate(self.cfg.categories): if not cat.supported or (self.only and cat.name.casefold() not in self.only): continue self.matchers[cat.name] = subcategory_matchers(cat) self.excluders[cat.name] = keyword_pattern(cat.exclude) if cat.exclude else None for si, (sub, terms) in enumerate(cat.subcategories.items()): for ti, term in enumerate(terms): units.append({ "id": f"{ci:02d}{si:02d}{ti:02d}-{slug(cat.name)}-{slug(sub)}-{slug(term)}", "category": cat.name, "sub_category": sub, "term": term, "departments": cat.departments, }) return units # --- store verification --------------------------------------------------------------------------- def verify_location(self) -> tuple[dict, dict]: data = self.api.location(self.loc.location_id) addr = data.get("address", {}) store_location = ", ".join(x for x in [addr.get("addressLine1", ""), addr.get("city", "").title(), f"{addr.get('state', '')} {addr.get('zipCode', '')}".strip()] if x) problems = [] if str(data.get("locationId")) != self.loc.location_id: problems.append(f"locationId {data.get('locationId')} != {self.loc.location_id}") if str(data.get("chain", "")).upper() != self.loc.expected_chain.upper(): problems.append(f"chain {data.get('chain')} != {self.loc.expected_chain}") if str(addr.get("zipCode", ""))[:5] != self.loc.zipcode: problems.append(f"zip {addr.get('zipCode')} != {self.loc.zipcode}") result = { "ok": not problems, "detail": f"{data.get('name')} | {store_location} | chain {data.get('chain')} | id {data.get('locationId')}" + (f" | MISMATCH: {problems}" if problems else ""), "checked_at": utc_now(), } location = {"zipcode": str(addr.get("zipCode", ""))[:5], "store_name": self.loc.store_name, "store_location": store_location, "store_logo": self.loc.store_logo, "location_id": self.loc.location_id, "api_name": data.get("name", ""), "chain": data.get("chain", "")} return result, location def check_canaries(self) -> dict: key = f"state/{self.loc.key}/canaries.json" upcs = list(self.loc.canary_upcs) if not upcs and self.storage.exists(key): upcs = json.loads(self.storage.read_text(key))["upcs"] if not upcs: return {"configured": 0, "found": 0, "missing": []} found = self.api.products_by_id(upcs, self.loc.location_id) priced = {p["productId"] for p in found if any((i.get("price") or {}).get("regular") for i in p.get("items", []))} return {"configured": len(upcs), "found": len(priced), "missing": [u for u in upcs if u not in priced]} # --- collection --------------------------------------------------------------------------------------- def collect_unit(self, unit: dict, location: dict) -> dict: departments = set(unit["departments"]) records, excluded, labelled = [], Counter(), Counter() seen, pages, products_seen, priced, api_total = set(), 0, 0, 0, 0 start = 0 log_from = len(self.api.stats.request_log) crawl_date = datetime.now(timezone.utc).strftime("%Y-%m-%d") while True: data, api_total = self.api.search_page(unit["term"], self.loc.location_id, start) pages += 1 for product in data: products_seen += 1 pid = product["productId"] if pid in seen: excluded["repeated_on_later_page"] += 1 continue seen.add(pid) if any((i.get("price") or {}).get("regular") for i in product.get("items", [])): priced += 1 if departments and not departments.intersection(product.get("categories") or []): excluded["outside_category"] += 1 continue record, reason = self.mapper.map(product, location=location, category=unit["category"], sub_category=unit["sub_category"], crawl_date=crawl_date) if record is None: excluded[reason] += 1 continue excluder = self.excluders.get(unit["category"]) if excluder and excluder.search(record["product_title"]): excluded["excluded_by_title_rule"] += 1 continue record["sub_category"], note = self.classify(unit, record["product_title"]) if not record["sub_category"]: excluded["no_subcategory_match"] += 1 # department matched but nothing in the title fits this category continue if note: labelled[note] += 1 # kept, but labelled by product title rather than by the search term records.append(record) next_start = start + self.page_size if len(data) < self.page_size or next_start >= api_total or next_start > self.max_start: break start = next_start # the unit's own request log is stored with its checkpoint, so request counts survive a killed process return {"unit": unit, "records": records, "excluded": dict(excluded), "labelled": dict(labelled), "pages": pages, "api_total": api_total, "products_seen": products_seen, "products_priced": priced, "requests": self.api.stats.request_log[log_from:], "finished_at": utc_now()} # --- main -------------------------------------------------------------------------------------------- def _on_signal(self, signum, _frame): self._interrupted = True log("warning", "interrupt_requested", signal=signum) def run(self) -> dict: lock_key = f"work/{self.loc.key}/run.lock" # one lock per store, whatever the source self.storage.acquire_lock(lock_key, LOCK_STALE_SECONDS) for sig in (signal.SIGINT, signal.SIGTERM): try: signal.signal(sig, self._on_signal) except ValueError: pass try: return self._run_locked() finally: self.storage.release_lock(lock_key) def _load_or_start_state(self) -> dict: key = f"work/{self.work_key}/current.json" if not self.fresh and self.storage.exists(key): state = json.loads(self.storage.read_text(key)) age_h = (time.time() - state["started_epoch"]) / 3600 if state["status"] in ("running", "incomplete", "interrupted") and age_h < RESUME_MAX_AGE_HOURS: if state.get("binding") != self.binding(): raise CheckpointMismatch( f"unfinished run {state['run_id']} was started for " f"{state.get('binding') or 'an unrecorded source/store/scope (before v2.2.18)'}; this start is " f"for {self.binding()}: not resumed, nothing sent; resume it with the same settings or start " f"over with --fresh") state["resumed"] = state.get("resumed", 0) + 1 state["_resumed_after_crash"] = False if state["segments"][-1]["end"] is None: # previous process died without a clean stop seg = state["segments"][-1] seg["end"] = max(seg["start"], state.get("last_progress", seg["start"])) state["crashed_segments"] = state.get("crashed_segments", 0) + 1 state["_resumed_after_crash"] = True state["segments"].append({"start": time.time(), "end": None}) state["_event"] = ("resume_run", {"age_hours": round(age_h, 1), "resumed": state["resumed"]}) return state # Timestamp for humans + random suffix so two starts in the same second never share checkpoints. run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(2) state = {"run_id": run_id, "started": utc_now(), "started_epoch": time.time(), "status": "running", "resumed": 0, "segments": [{"start": time.time(), "end": None}], "only": sorted(self.only or []), "binding": self.binding(), "binding_digest": digest(self.binding()), "_resumed_after_crash": False} state["_event"] = ("start_run", {}) return state def _save_state(self, state: dict) -> None: self.storage.write_text(f"work/{self.work_key}/current.json", json.dumps({k: v for k, v in state.items() if not k.startswith("_")}, indent=1)) def _run_locked(self) -> dict: state = self._load_or_start_state() self._state = state run_id = state["run_id"] set_context(run_id=run_id, location=self.loc.key) log_path = Path(self.cfg.storage_root) / "_runlogs" / f"{self.loc.key}-{run_id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) log_fh = open(log_path, "a", encoding="utf-8") add_sink(log_fh) try: # Logged after the run log is attached so every segment's start is in runs//run.log too. event, fields = state.pop("_event") log("info", event, **fields, crashed_segments=state.get("crashed_segments", 0)) self._save_state(state) before, location = self.verify_location() log("info" if before["ok"] else "error", "location_verified", phase="before", **before) if not before["ok"]: state["status"] = "failed_location" self._save_state(state) raise SourceError(f"Store verification failed: {before['detail']}") prior = state.get("location") if prior and any(str(prior.get(k, "")) != str(location.get(k, "")) for k in ("location_id", "facility_id", "zipcode")): raise CheckpointMismatch(f"run {run_id} was collected for store {prior.get('location_id')} facility " f"{prior.get('facility_id')} zip {prior.get('zipcode')}; the store answering now " f"is {location.get('location_id')} / {location.get('facility_id')} / " f"{location.get('zipcode')}: not resumed") state["location"] = location canaries = self.check_canaries() log("info", "canaries_checked", **{k: v for k, v in canaries.items() if k != "missing"}, missing=canaries["missing"][:10]) if canaries["configured"] and canaries["found"] / canaries["configured"] < self.cfg.settings["validation"]["required_canaries_found_share"]: state["status"] = "failed_canaries" self._save_state(state) raise SourceError(f"Canary check failed before collection ({canaries['found']}/{canaries['configured']}) - source degraded or blocked") units = self.plan_units() unit_prefix = f"work/{self.work_key}/{run_id}/units/" done = {Path(k).stem for k in self.storage.list(unit_prefix)} todo = [u for u in units if u["id"] not in done] log("info", "plan", units_total=len(units), units_done=len(done), units_todo=len(todo)) failed: list[str] = [] processed = 0 for attempt in (1, 2): retry_later = [] for unit in todo: if self._interrupted or (self.stop_after_units is not None and processed >= self.stop_after_units): raise Interrupted() if self.time_left() <= 0: log("warning", "deadline_reached", deadline_seconds=self.deadline, unit=unit["id"]) raise Interrupted("cap") try: result = self.collect_unit(unit, location) except SourceBlocked as exc: log("error", "source_blocked", unit=unit["id"], error=str(exc)) raise Interrupted("blocked") from exc except RequestCapReached as exc: # the process's own request/runtime budget is spent: stop like a requested stop (resumable) log("warning", "request_cap_reached", unit=unit["id"], error=str(exc)) raise Interrupted("cap") from exc except SourceError as exc: log("error", "unit_failed", unit=unit["id"], attempt=attempt, error=str(exc)) retry_later.append(unit) continue result["binding_digest"] = state["binding_digest"] self.storage.write_text(unit_prefix + unit["id"] + ".json", json.dumps(result)) processed += 1 state["last_progress"] = time.time() self._save_state(state) log("info", "unit_done", unit=unit["id"], category=unit["category"], sub_category=unit["sub_category"], term=unit["term"], pages=result["pages"], kept=len(result["records"]), excluded=result["excluded"]) todo = retry_later if not todo: break self._wait(30) failed = [u["id"] for u in todo] after, _ = self.verify_location() log("info" if after["ok"] else "error", "location_verified", phase="after", **after) return self._finish(state, units, failed, {"before": before, "after": after}, canaries) except SourceError: # stopped at a store/canary check or by a failing source: close this segment properly so the # runtime and request counts stay exact, keep the run resumable, then let the caller report it self._carry_api_stats(state) state["segments"][-1]["end"] = time.time() state["stopped_segments"] = state.get("stopped_segments", 0) + 1 self._save_state(state) raise except Interrupted as stop: self._carry_api_stats(state) state["status"] = "interrupted" state["segments"][-1]["end"] = time.time() self._save_state(state) blocked = str(stop) == "blocked" reason = "source blocked" if blocked else "request/runtime cap reached" if str(stop) == "cap" else "stop requested" log("warning", "run_interrupted", reason=reason, note="finished units are checkpointed; the next start resumes this run; latest/ is untouched") return {"status": "blocked" if blocked else "interrupted", "run_id": run_id} finally: remove_sink(log_fh) log_fh.close() def _finish(self, state: dict, units: list[dict], failed: list[str], location_checks: dict, canaries: dict) -> dict: run_id = state["run_id"] unit_prefix = f"work/{self.work_key}/{run_id}/units/" by_id = {Path(k).stem: k for k in self.storage.list(unit_prefix)} records, placements = [], defaultdict(list) excluded, labelled, seen_index = Counter(), Counter(), {} cross_duplicates = 0 empty_terms = [] products_seen = products_priced = pages = unit_requests = 0 for unit in units: # plan order = categories.csv order, so the first placement wins deterministically if unit["id"] not in by_id: continue result = json.loads(self.storage.read_text(by_id[unit["id"]])) self._check_unit(state, unit["id"], result) excluded.update(result["excluded"]) labelled.update(result.get("labelled", {})) products_seen += result["products_seen"] products_priced += result["products_priced"] pages += result["pages"] unit_requests += len(result.get("requests", [])) if result["api_total"] == 0: empty_terms.append(f"{unit['category']} > {unit['sub_category']}: {unit['term']}") for rec in result["records"]: key = record_key(rec) placements[key].append(f"{rec['category']} > {rec['sub_category']}") if key in seen_index: cross_duplicates += 1 continue seen_index[key] = len(records) records.append(rec) for seg in state["segments"]: seg["end"] = seg["end"] or time.time() runtime = sum(seg["end"] - seg["start"] for seg in state["segments"]) counts_cat = Counter(r["category"] for r in records) counts_sub = Counter(f"{r['category']} > {r['sub_category']}" for r in records) supported = [c.name for c in self.cfg.categories if c.supported and (not self.only or c.name.casefold() in self.only)] limited = self.limited_reasons() previous = None if limited else self._baseline_report(run_id) priced_share = products_priced / products_seen if products_seen else 0.0 extra = [] if self.fallback_reason: # Fallback = the official API because the website session was unavailable. It has no PPM, deal text or # coupons, so by default it is kept for diagnosis and does not replace the last full dataset. extra.append({"check": "primary_source_available", "passed": bool(self.cfg.settings["validation"].get("publish_fallback_runs")), "detail": f"fallback run on the Kroger Public API: {self.fallback_reason}"}) validation = validate_run(records=records, category_counts=counts_cat, supported_categories=supported, failed_units=failed, location_checks=location_checks, priced_share=priced_share, canaries=canaries, previous=previous, settings=self.cfg.settings, extra_checks=extra) stats = self.api.stats products_json = json.dumps(records, ensure_ascii=False, indent=1) report = { "run_id": run_id, "status": ("sample" if limited else "published") if validation["passed"] else "held_back", "delivery": {"kind": "sample" if limited else "full", "limited_by": limited, "rule": "only a full run that passes every check replaces latest/; a limited run is a sample release"}, "source": "kroger_public_api" + (" (fallback)" if self.fallback_reason else ""), "location": state["location"], "started": state["started"], "finished": utc_now(), "runtime_seconds": round(runtime, 1), "resumed_times": state.get("resumed", 0), "runtime": {"active_seconds": round(runtime, 1), "elapsed_seconds": round(time.time() - state["started_epoch"], 1)}, "requests_in_unit_checkpoints": unit_requests, "scope": "partial (categories: %s)" % ", ".join(sorted(self.only)) if self.only else "full catalog", "unique_records": len(records), "duplicates_removed": cross_duplicates, "excluded": dict(excluded), "subcategory_labelling": dict(labelled), "records_without_subcategory": sum(1 for r in records if not r["sub_category"]), "search_terms_with_no_results": empty_terms, "products_seen": products_seen, "priced_share": round(priced_share, 4), "api_pages": pages, "counts_by_category": dict(sorted(counts_cat.items(), key=lambda kv: -kv[1])), "counts_by_subcategory": dict(sorted(counts_sub.items())), "unsupported_categories": {c.name: c.reason for c in self.cfg.categories if not c.supported}, "field_completeness_pct": completeness(records), "records_with_sale_price": sum(1 for r in records if r["sale_price"]), "store_brand_records": sum(1 for r in records if r["store_brand"] == "yes"), "api": {k: getattr(stats, k) + state.get("api_carry", {}).get(k, 0) for k in ("requests", "retries", "errors", "slow_responses", "token_refreshes")}, "canaries": canaries, "validation": validation, "evidence": {"products_sha256": hashlib.sha256(products_json.encode()).hexdigest(), "code_fingerprint": code_fingerprint(), "units_completed": len(by_id), "units_planned": len(units)}, } out = f"runs/{self.loc.key}/{run_id}/" self.storage.write_text(out + "products.json", products_json) self.storage.write_text(out + "placements.json", json.dumps({k: v for k, v in placements.items() if len(v) > 1}, indent=1)) self.storage.write_text(out + "report.json", json.dumps(report, indent=1, ensure_ascii=False)) earlier = [self.storage.read_text(k) for k in self.storage.list(f"work/{self.work_key}/{run_id}") if "/requests-" in k] self.storage.write_text(out + "requests.jsonl", "\n".join(earlier + [json.dumps(r) for r in stats.request_log])) if validation["passed"]: # Publish: only now is latest/ touched, and only through the one writer. A failed run never gets here. base = None if limited else self.delivery.baseline(run_id) files = {"products.json": products_json, "coupons.json": json.dumps({"standard": [], "cash_back": [], "note": "the Kroger Public API has no coupons"}), "report.json": json.dumps(report, indent=1, ensure_ascii=False), # the API has no availability data: the store's observation history is carried over unchanged "catalog.json": self.delivery.read(base, "catalog.json") or json.dumps({"run_id": None, "products": {}})} self._deliver(state, report, files) if state["status"] == "published" and not self.storage.exists(f"state/{self.loc.key}/canaries.json"): self._seed_canaries(records) else: state["status"] = "incomplete" if failed else "held_back" failed_checks = [c for c in validation["checks"] if not c["passed"]] log("error", "run_held_back", reason="validation failed; last good dataset left untouched", failed_checks=failed_checks) self._save_state(state) log_path = Path(self.cfg.storage_root) / "_runlogs" / f"{self.loc.key}-{run_id}.log" if log_path.is_file(): self.storage.write_text(out + "run.log", log_path.read_text(encoding="utf-8")) return report def _carry_api_stats(self, state: dict) -> None: """Keep request counts and the request log of a stopped segment so the final report covers the whole run.""" st = self.api.stats carry = state.setdefault("api_carry", {"requests": 0, "retries": 0, "errors": 0, "slow_responses": 0, "token_refreshes": 0}) for k in carry: carry[k] += getattr(st, k) self.storage.write_text(f"work/{self.work_key}/{state['run_id']}/requests-{len(state['segments'])}.jsonl", "\n".join(json.dumps(r) for r in st.request_log)) def _seed_canaries(self, records: list[dict]) -> None: stable = [r for r in records if r["store_brand"] == "yes" and r["upc"]] step = max(1, len(stable) // CANARY_COUNT) upcs = [r["upc"] for r in stable[::step]][:CANARY_COUNT] self.storage.write_text(f"state/{self.loc.key}/canaries.json", json.dumps({"upcs": upcs, "seeded": utc_now()}, indent=1)) log("info", "canaries_seeded", count=len(upcs))