"""Mariano's website as the primary source, read through a real Chrome session (Chrome DevTools Protocol). The scraper does not drive the page like a user. It opens marianos.com once in a real Chrome, then calls the same JSON endpoints the site's own pages call, from inside that page, so the requests carry the browser's normal session: /atlas/v1/stores/v2/locator store identity (address, facility) for the configured location /atlas/v1/search/v1/products-search product ids per retailer taxonomy node, with the site's own counts /atlas/v1/product/v2/products product details + store prices, displayed price per measure, promotions /atlas/v1/savings-coupons/v2/coupons digital (clip) coupons and cash-back offers, per UPC or as a full list Every call is paced, retried on 429/5xx, logged, and validated (status, JSON, expected keys). A challenge page, an access-denied page or an unexpected shape raises SourceError, so a blocked source can never pass as "no products". """ from __future__ import annotations import json import random import time from urllib.parse import quote from .kroger_api import ApiStats, SourceBlocked, SourceError from .log import log BASE = "https://www.marianos.com" SEARCH_MAX_OFFSET = 250 # the site rejects page.offset > 250, so one taxonomy node yields at most 350 products PAGE_SIZE = 100 BATCH = 100 _FETCH_JS = """async ([url, headers, limitMs]) => { const t0 = performance.now(); const ctl = new AbortController(); const timer = limitMs > 0 ? setTimeout(() => ctl.abort(), limitMs) : null; try { const r = await fetch(url, {headers, credentials: 'include', signal: ctl.signal}); const body = await r.text(); return {status: r.status, type: r.headers.get('content-type') || '', body, ms: Math.round(performance.now() - t0)}; } catch (e) { return {status: 0, type: '', body: String(e), ms: Math.round(performance.now() - t0)}; } finally { if (timer) clearTimeout(timer); } }""" class SiteUnavailable(SourceBlocked): """The browser session could not reach marianos.com (Chrome not running, blocked, or challenged).""" class RequestCapReached(SourceError): """The configured request or runtime budget of this process is used up (nothing more is sent).""" class RunBudgetSpent(Exception): """The run's request budget across all its resumes is used up: not retried, a person decides (exit 5).""" def echoed_context(product: dict) -> tuple[str, str, str]: """The fulfilment context the site says it answered for (product.laf[0].modality): type, storeId, facilityId.""" laf = product.get("laf") or [{}] modality = (laf[0] or {}).get("modality") or {} handoff = modality.get("handoffLocation") or {} return str(modality.get("type", "")), str(handoff.get("storeId", "")), str(handoff.get("facilityId", "")) class SiteSession: transport_name = "chrome-cdp" # part of a run's checkpoint binding: a run never resumes on another transport def __init__(self, cdp_url: str, settings: dict): self.cdp_url = cdp_url self.min_interval = settings.get("min_interval_seconds", 3.0) self.block_wait = settings.get("block_wait_seconds", 300) self.max_retries = settings.get("max_retries", 4) self.assortment_key = settings["assortment_key"] # Budget of this process: counted before every attempt (retries included), so a cap of N means at most # N requests leave this machine. None = unlimited (the production default). self.max_requests: int | None = settings.get("max_requests") self.max_seconds: float | None = settings.get("max_seconds") # Longest one request may take even without a runtime cap (v2.2.18: normal collection never waits unbounded). self.request_timeout: float = float(settings.get("request_timeout_seconds", 90)) # Called immediately before an attempt is counted and sent; the run sets it to reserve the request durably in # its state (v2.2.18). It raises to stop the request; None = no reservation (probe, tests). self.reserve = None self.started = time.time() # wall clock, for evidence timestamps only self._t0 = time.monotonic() # the runtime cap is measured on this clock (immune to clock changes) # Evidence capture (the probe turns it on): every attempt with its exact answer, kept before any parse or raise. self.capture = bool(settings.get("capture_attempts", False)) self.attempts: list[dict] = [] self.session_generation = 0 # bumped when the transport replaces its session (Oxylabs); never by Chrome self.stats = ApiStats() self.calls: list[dict] = [] # request log of the current unit; the runner stores and clears it per unit self._pw = None self._page = None self._last = float("-inf") # no request yet: the first one never waits (monotonic time can start near 0) self.laf = "" # --- budget --------------------------------------------------------------------------------------- def elapsed(self) -> float: """Seconds since this session was created, on the monotonic clock.""" return time.monotonic() - self._t0 def check_caps(self, path: str) -> None: """Raise before a request would exceed the request count or the runtime deadline.""" if self.max_requests is not None and self.stats.requests >= self.max_requests: raise RequestCapReached(f"request cap {self.max_requests} reached before {path[:80]}") if self.max_seconds is not None and self.elapsed() >= self.max_seconds: raise RequestCapReached(f"runtime cap {int(self.max_seconds)}s reached before {path[:80]}") def time_left(self) -> float | None: """Seconds left before the runtime deadline; None without one.""" if self.max_seconds is None: return None return self.max_seconds - self.elapsed() def _sleep(self, seconds: float, path: str) -> None: """Every wait (pacing, backoff, refusal wait) stops at the runtime cap instead of running past it.""" left = self.time_left() if left is not None and seconds >= left: raise RequestCapReached(f"runtime cap {int(self.max_seconds)}s would pass during a {seconds:.0f}s wait before {path[:80]}") if seconds > 0: time.sleep(seconds) def _begin_attempt(self, path: str, attempt: int) -> dict: """The last step before a request leaves: both caps checked again, then the attempt is counted. A cap of N therefore means at most N dispatched attempts, whatever happens to them (answer, error, timeout, lost page).""" self.check_caps(path) if self.reserve is not None: self.reserve(path) # durable reservation first; nothing is sent if it cannot be written self.stats.requests += 1 rec = {"seq": self.stats.requests, "t": round(time.time(), 3), "elapsed_s": round(self.elapsed(), 3), "path": path[:500], "attempt": attempt + 1} if self.capture: self.attempts.append(rec) return rec def _request_limit(self, default: float | None) -> float | None: """Longest one request may take: its own timeout, never beyond the runtime cap.""" left = self.time_left() if left is None: return default return max(0.5, left if default is None else min(default, left)) # --- browser ------------------------------------------------------------------------------------ def open(self) -> None: from playwright.sync_api import sync_playwright try: self._pw = sync_playwright().start() browser = self._pw.chromium.connect_over_cdp(self.cdp_url, timeout=15000) except Exception as exc: raise SiteUnavailable(f"Chrome is not reachable at {self.cdp_url}: {exc}") from exc ctx = browser.contexts[0] pages = [p for p in ctx.pages if p.url.startswith(BASE)] self._page = pages[0] if pages else ctx.new_page() if not self._page.url.startswith(BASE): # browser start-up page load: outside the JSON-request counter, but inside the runtime deadline limit = self._request_limit(60.0) or 60.0 self._page.goto(BASE + "/", wait_until="domcontentloaded", timeout=int(limit * 1000)) self._sleep(6, "/ (start-up page)") html_head = self._page.content()[:20000] if "Access Denied" in html_head or "errors.edgesuite.net" in html_head: raise SiteUnavailable("marianos.com answered Access Denied in the browser session") def close(self) -> None: if self._pw: self._pw.stop() def set_store(self, location_id: str, facility_id: str, address: dict) -> None: """LAF = the site's location/assortment/fulfilment context. Every product and coupon call carries it.""" self.laf = json.dumps([{ "assortmentKeys": [self.assortment_key], "listingKeys": [location_id], "modality": {"handoffAddress": {"address": address}, "handoffLocation": {"facilityId": facility_id, "storeId": location_id}, "type": "PICKUP"}, "priceAvailabilities": [{"id": location_id, "lookup": "LISTING_KEY", "storeId": location_id}], "sources": [{"facilityId": facility_id, "storeId": location_id}], }]) def sent_context(self) -> tuple[str, str, str]: """The fulfilment context this session sends (x-laf-object): modality type, storeId, facilityId.""" modality = ((json.loads(self.laf) if self.laf else [{}])[0] or {}).get("modality") or {} handoff = modality.get("handoffLocation") or {} return str(modality.get("type", "")), str(handoff.get("storeId", "")), str(handoff.get("facilityId", "")) def check_context(self, products: list[dict], path: str) -> None: """Every product answer must echo the fulfilment context that was sent; one that does not rejects the call.""" wanted = self.sent_context() for p in products: echoed = echoed_context(p) if echoed != wanted: raise SourceError(f"fulfilment context lost: product {p.get('id')} answered for {echoed}, sent {wanted} ({path[:60]})") # --- transport ---------------------------------------------------------------------------------- def request_headers(self) -> dict: """The headers every site call carries; the store/fulfilment context (LAF) is one of them.""" headers = {"accept": "application/json", "x-kroger-channel": "WEB"} if self.laf: headers["x-laf-object"] = self.laf return headers def parse_body(self, path: str, text: str, expect: str) -> dict: """Validate a 200 answer: JSON with data.; anything else is a source problem, never 'no products'.""" try: body = json.loads(text) if isinstance(text, str) else text except ValueError as exc: raise SourceError(f"Malformed JSON from {path[:80]}") from exc if not isinstance(body, dict) or not isinstance(body.get("data"), dict) or expect not in body["data"]: raise SourceError(f"Unexpected shape from {path[:80]}: no data.{expect}") return body def get(self, path: str, *, expect: str) -> dict: headers = self.request_headers() last = None for attempt in range(self.max_retries + 1): self.check_caps(path) wait = self.min_interval + random.uniform(0, self.min_interval / 2) - (time.monotonic() - self._last) self._sleep(wait, path) att = self._begin_attempt(path, attempt) # rechecked and counted immediately before dispatch self._last = time.monotonic() limit = self._request_limit(self.request_timeout) try: res = self._page.evaluate(_FETCH_JS, [path, headers, int(limit * 1000) if limit else 0]) except Exception as exc: att.update(status=None, error=f"{type(exc).__name__}: {str(exc)[:300]}") msg = str(exc).lower() if ("destroyed" in msg or "navigation" in msg) and attempt < self.max_retries: # the page reloaded itself (the site does this after a refusal): let it settle, then ask again log("warning", "site_page_reloaded", path=path[:120], attempt=attempt + 1) try: self._page.wait_for_load_state("domcontentloaded", timeout=int((self._request_limit(60.0) or 60.0) * 1000)) except Exception: pass self._sleep(10, path) continue # the Chrome window/tab was closed, crashed or kept reloading: stop like a refusal (resumable) raise SiteUnavailable(f"browser session lost during {path[:80]}: {str(exc)[:160]}") from exc status, ms = res["status"], res["ms"] att.update(status=status, content_type=res["type"], ms=ms, body=res["body"]) # kept before any parse or raise self.stats.total_seconds += ms / 1000 entry = {"t": round(time.time(), 3), "path": path[:300], "status": status, "ms": ms, "bytes": len(res["body"])} self.calls.append(entry) if status == 200: if "json" not in res["type"]: raise SiteUnavailable(f"Non-JSON answer from {path[:80]} (challenge page?)") body = self.parse_body(path, res["body"], expect) entry["raw"] = body return body if status in (401, 403) or "Access Denied" in res["body"][:500]: raise SiteUnavailable(f"HTTP {status} from {path[:80]}: blocked") if status == 0: left = self.time_left() if left is not None and left <= 0: raise RequestCapReached(f"runtime cap {int(self.max_seconds)}s reached during {path[:80]} (request aborted)") # fetch() itself failed: the site dropped the connection (bot protection). Wait once, then stop. if attempt == 0: log("warning", "site_connection_refused", path=path[:120], wait_seconds=self.block_wait) self._sleep(self.block_wait, path) continue raise SiteUnavailable(f"connection to marianos.com refused for {path[:80]} after waiting {self.block_wait}s: {res['body'][:80]}") if status == 400: raise SourceError(f"HTTP 400 from {path[:80]}: {res['body'][:200]}") last = status if attempt < self.max_retries: self.stats.retries += 1 delay = min(60.0, 2 ** attempt + random.uniform(0, 1)) log("warning", "site_retry", path=path[:120], status=status, attempt=attempt + 1, sleep=round(delay, 1)) self._sleep(delay, path) self.stats.errors += 1 raise SourceError(f"{path[:80]} still failing after {self.max_retries} retries (last status {last})") # --- endpoints ---------------------------------------------------------------------------------- def store(self, zipcode: str, location_id: str) -> dict: body = self.get(f"/atlas/v1/stores/v2/locator?filter.query={zipcode}&projections=full", expect="stores") for s in body["data"]["stores"]: if str(s.get("locationId")) == location_id: return s raise SourceError(f"Store {location_id} not returned by the site's store locator for {zipcode}") def search(self, location_id: str, taxonomy: str, offset: int = 0, size: int = PAGE_SIZE, facets: bool = False, brand: str = "", brand_facet: bool = False) -> tuple[list[dict], int, list]: # No fulfilment filter: products unavailable for pickup, delivery or purchase are listed too. q = (f"/atlas/v1/search/v1/products-search?filter.locationId={location_id}&filter.taxonomies={taxonomy}" f"&page.offset={offset}&page.size={size}") if facets: q += "&option.facets=TAXONOMY" if brand: q += "&filter.brands=" + quote(brand, safe="") if brand_facet: body = self.get(q + "&option.facets=BRAND", expect="productsSearch") meta = (body.get("meta") or {}).get("productsSearch") or {} total = meta.get("totalCount") return [], (int(total) if total is not None else None), (meta.get("facets") or {}).get("brands", []) body = self.get(q, expect="productsSearch") meta = (body.get("meta") or {}).get("productsSearch") or {} items = body["data"]["productsSearch"] # v2.2.18: a missing count is reported as None, never replaced by the number of items on the page, so a # listing without the site's own total can never look complete total = meta.get("totalCount") return items, (int(total) if total is not None else None), (meta.get("facets") or {}).get("category", []) def products(self, upcs: list[str], verified: bool = True) -> list[dict]: out = [] flag = "&filter.verified=true" if verified else "" for i in range(0, len(upcs), BATCH): q = "&".join("filter.gtin13s=" + u for u in upcs[i:i + BATCH]) body = self.get(f"/atlas/v1/product/v2/products?{q}{flag}&projections=items.full,offers.compact", expect="products") out.extend(body["data"]["products"]) return out def coupons_for(self, upcs: list[str]) -> list[dict]: out = [] for i in range(0, len(upcs), BATCH): q = "&".join("filter.upc=" + u for u in upcs[i:i + BATCH]) offset = 0 while True: page_q = f"&page.offset={offset}" if offset else "" body = self.get(f"/atlas/v1/savings-coupons/v2/coupons?{q}&filter.type=standard&filter.type=cash_back{page_q}", expect="coupons") batch = body["data"]["coupons"] out.extend(batch) if not batch or not ((body.get("meta") or {}).get("coupons") or {}).get("page", {}).get("hasMore"): break offset += len(batch) return out def all_coupons(self, coupon_type: str) -> tuple[list[dict], dict]: """The Coupons page: every coupon of one type, paged until the site says there are no more.""" out, offset, meta = [], 0, {} while True: body = self.get(f"/atlas/v1/savings-coupons/v2/coupons?filter.type={coupon_type}&page.offset={offset}&page.size=100", expect="coupons") batch = body["data"]["coupons"] out.extend(batch) meta = (body.get("meta") or {}).get("coupons") or {} page = meta.get("page") or {} if not batch or not page.get("hasMore"): break offset += len(batch) return out, meta