"""Command line entry point. python -m marianos_scraper run --location marianos-lakeshore-east one store (resumes an unfinished run) python -m marianos_scraper run --all every enabled store, one after another python -m marianos_scraper run --location ... --fresh ignore an unfinished run and start over python -m marianos_scraper run --location ... --source site|api|auto website (default: auto = website, API fallback) python -m marianos_scraper verify-location --location ... store check only, no collection python -m marianos_scraper find-stores --zip 60601 list Mariano's stores near a ZIP code python -m marianos_scraper run ... --transport oxylabs same run, requests via Oxylabs Web Scraper API python -m marianos_scraper run ... --max-requests 40 --max-seconds 1800 stop (resumable) when a budget is spent python -m marianos_scraper run ... --max-requests-per-run 2500 budget of the whole run across resumes (exit 5 when spent) python -m marianos_scraper run ... --transport oxylabs --ledger blob: Oxylabs collection: jobs reserved in the shared allowance ledger before any is sent (required) Exit codes of run: 0 published (or a passed sample release), 1 held back / publish failed, 2 aborted (resumable), 3 another run active, 4 interrupted or deadline reached (resumable), 5 budget/allowance spent or ledger refused, 6 checkpoint belongs to another source/store/scope (not resumed; same settings or --fresh) python -m marianos_scraper probe --location ... --transport oxylabs --upcs A,B --roles ppm-volume,unavailable --max-requests 15 --max-seconds 1200 --out DIR capped store+product+coupon test, evidence to DIR; exit 0 accepted, 1 completed but not accepted, 2/4 stopped """ from __future__ import annotations import argparse import json import os import sys import threading import time from .config import load_config from .kroger_api import KrogerApi, SourceError from .log import log from .runner import CheckpointMismatch, Runner, deadline_seconds from .site_runner import SiteRunner from .site_source import RunBudgetSpent, SiteSession, SiteUnavailable from .storage import LockHeld, open_storage def build_site(cfg, transport: str, max_requests: int | None, max_seconds: float | None) -> SiteSession: """The website transport for this process. 'site' = local Chrome over CDP (default, unchanged); 'oxylabs' = the same endpoints and headers through the Oxylabs Web Scraper API (credentials from env only).""" settings = dict(cfg.settings["site_source"]) if max_requests is not None: settings["max_requests"] = max_requests if max_seconds is not None: settings["max_seconds"] = max_seconds if transport == "oxylabs": from .oxylabs_source import OxylabsSession return OxylabsSession(settings) return SiteSession(settings["cdp_url"], settings) WATCHDOG_GRACE_SECONDS = 300 def start_watchdog(seconds: float, exit=os._exit) -> threading.Timer: """Hard stop of a collection process still running `seconds` after it started: a request or browser call that never returns cannot hold the run, its lock or its reservation past the deadline. Exit 4 (resumable): finished units are checkpointed and the stuck request is already counted in the run's durable reservation.""" def fire(): def say(): try: sys.stderr.write(json.dumps({"level": "error", "event": "deadline_watchdog", "after_seconds": seconds, "note": "process stopped hard; the next start resumes the run"}) + "\n") sys.stderr.flush() except Exception: # noqa: BLE001 pass t = threading.Thread(target=say, daemon=True) # a blocked stderr cannot keep the process alive t.start() t.join(0.5) exit(4) timer = threading.Timer(seconds, fire) timer.daemon = True timer.start() return timer def reserve_allowance(args, loc, site): """Oxylabs collection: reserve this process's jobs in the shared ledger before anything is sent; cap the transport at the reservation. Returns the Allowance (finalized by the caller).""" from . import __version__ from .allowance import Allowance, open_store allowance = Allowance(open_store(args.ledger), ceiling=args.ledger_ceiling) stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) wanted = args.max_requests if args.max_requests is not None else site.max_requests info = allowance.reserve(f"collect/{loc.key}/{stamp}", f"v{__version__}", wanted) log("info", "allowance_reserved", **{k: v for k, v in info.items() if k != "ledger"}) if info["cap_this_run"] <= 0: raise RunBudgetSpent(f"the shared allowance has no jobs left ({info['counted_before']} of {info['ceiling']} counted)") site.max_requests = info["cap_this_run"] return allowance def resumable_site_run(cfg, loc) -> bool: storage = open_storage(cfg) key = f"work/{loc.key}/current.json" if not storage.exists(key): return False state = json.loads(storage.read_text(key)) return state.get("status") in ("running", "interrupted", "incomplete") and storage.exists(f"work/{loc.key}/{state['run_id']}/plan.json") def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="marianos_scraper") sub = parser.add_subparsers(dest="cmd", required=True) run = sub.add_parser("run") target = run.add_mutually_exclusive_group(required=True) target.add_argument("--location") target.add_argument("--all", action="store_true") run.add_argument("--fresh", action="store_true") run.add_argument("--categories", help="comma-separated subset, for testing") run.add_argument("--stop-after-units", type=int, help="stop cleanly after N units (interruption test)") run.add_argument("--source", choices=["auto", "site", "api"], help="override settings.json 'source'") for cmd in (run,): cmd.add_argument("--transport", choices=["site", "oxylabs"], help="website transport (default: settings.json 'transport' or 'site')") cmd.add_argument("--max-requests", type=int, help="stop (resumable) once this many requests have been sent by this process") cmd.add_argument("--max-seconds", type=float, help="stop (resumable) once this process has run this long") cmd.add_argument("--max-requests-per-run", type=int, help="request budget of the whole run, counted across resumes; when spent: exit 5, not retried") cmd.add_argument("--ledger", help="shared allowance ledger (blob: or a local file); required with --transport oxylabs") cmd.add_argument("--ledger-ceiling", type=int, default=15, help="total Oxylabs jobs allowed across all executions") probe = sub.add_parser("probe", help="capped store + product + coupon test of one transport; writes an evidence folder") probe.add_argument("--location", required=True) probe.add_argument("--transport", choices=["site", "oxylabs"], default="site") probe.add_argument("--upcs", help="comma-separated 13-digit UPCs") probe.add_argument("--upcs-file", help="text file, one UPC per line") probe.add_argument("--max-requests", type=int, default=15) probe.add_argument("--max-seconds", type=float, default=1200) probe.add_argument("--out", required=True, help="evidence folder (must not exist yet)") probe.add_argument("--egress-ip", default="", help="this machine's public IP as the operator read it (recorded only)") probe.add_argument("--roles", help="comma-separated role per UPC, same order: ppm-volume, ppm-digital, regular, unavailable, any") probe.add_argument("--azure-imds", action="store_true", help="record which Azure VM ran the probe (Instance Metadata Service)") probe.add_argument("--ledger", help="shared cumulative-usage ledger: blob: (leased, ETag-checked, read back; managed identity) or a local file; this run is capped at what is left") probe.add_argument("--ledger-ceiling", type=int, default=15, help="total jobs allowed across all executions") ver = sub.add_parser("verify-location") ver.add_argument("--location", required=True) ref = sub.add_parser("refinish", help="rebuild a finished run's output and report from its checkpoints (no site requests)") ref.add_argument("--location", required=True) ref.add_argument("--run-id", required=True) ref.add_argument("--previous-run", default="", help="run to compare yields with (default: the latest published run)") find = sub.add_parser("find-stores") find.add_argument("--zip", required=True) args = parser.parse_args(argv) cfg = load_config() if args.cmd == "find-stores": api = KrogerApi(cfg.client_id, cfg.client_secret, cfg.settings["api"]) body = api.get("/locations", {"filter.chain": "MARIANOS", "filter.zipCode.near": args.zip, "filter.limit": 20}) for s in body.get("data", []): a = s["address"] print(f"{s['locationId']} {s['name']:40s} {a['addressLine1']}, {a['city']} {a['zipCode']}") return 0 if args.cmd == "probe": from pathlib import Path from .probe import Probe upcs = [u.strip() for u in (args.upcs or "").split(",") if u.strip()] if args.upcs_file: upcs += [l.strip() for l in Path(args.upcs_file).read_text(encoding="utf-8").splitlines() if l.strip() and not l.startswith("#")] if not upcs: parser.error("probe needs --upcs or --upcs-file") out = Path(args.out) if out.exists(): parser.error(f"--out {out} already exists; every probe writes a new folder so earlier evidence is never touched") loc = cfg.location(args.location) roles = [r.strip() for r in (args.roles or "").split(",") if r.strip()] try: site = build_site(cfg, args.transport, args.max_requests, args.max_seconds) except SourceError as exc: # e.g. OXYLABS_USERNAME / OXYLABS_PASSWORD not set: nothing was sent log("error", "probe_not_started", error=str(exc)) return 2 try: probe_run = Probe(cfg, loc, site, upcs, out, transport=args.transport, egress_ip=args.egress_ip, roles=roles, azure_imds=args.azure_imds, ledger=args.ledger or None, ledger_ceiling=args.ledger_ceiling) except ValueError as exc: parser.error(str(exc)) return probe_run.run() # 0 completed + accepted, 1 completed but not accepted, 2 stopped, 4 site refused if args.cmd == "refinish": report = SiteRunner(cfg, cfg.location(args.location)).refinish(args.run_id, args.previous_run) print(json.dumps({k: report[k] for k in ("run_id", "status", "unique_records")})) return 0 if report["status"] == "published" else 1 if args.cmd == "verify-location": result, location = Runner(cfg, cfg.location(args.location)).verify_location() print(json.dumps({"check": result, "location": location}, indent=1)) return 0 if result["ok"] else 2 locations = [loc for loc in cfg.locations if loc.enabled] if args.all else [cfg.location(args.location)] cats = [c.strip() for c in args.categories.split(",")] if args.categories else None source = args.source or cfg.settings.get("source", "auto") transport = args.transport or cfg.settings.get("transport", "site") if transport == "oxylabs" and source != "api" and not args.ledger: log("error", "run_not_started", error="--transport oxylabs needs --ledger: every Oxylabs job is reserved in the " "shared allowance ledger before it is sent; nothing was sent") return 2 deadline = deadline_seconds(cfg.settings) if args.max_seconds is not None: deadline = min(deadline, args.max_seconds) exit_code = 0 for loc in locations: opts = dict(fresh=args.fresh, only_categories=cats, stop_after_units=args.stop_after_units) watchdog = start_watchdog(deadline + WATCHDOG_GRACE_SECONDS) allowance, site = None, None try: if source == "api": report = Runner(cfg, loc, **opts).run() else: try: site = build_site(cfg, transport, args.max_requests, args.max_seconds if args.max_seconds is not None else deadline) if transport == "oxylabs": allowance = reserve_allowance(args, loc, site) report = SiteRunner(cfg, loc, site=site, max_requests_per_run=args.max_requests_per_run, **opts).run() except SiteUnavailable as exc: if source == "site": raise if not args.fresh and resumable_site_run(cfg, loc): # an unfinished website run is waiting: keep it, stop, and let the next start resume it log("warning", "site_unavailable_resume_later", location=loc.key, error=str(exc)) exit_code = max(exit_code, 4) continue # Safe fallback: collect what the official API offers; validation keeps it out of latest/ # unless settings allow publishing fallback runs. log("error", "site_unavailable_fallback", location=loc.key, error=str(exc)) report = Runner(cfg, loc, fallback_reason=str(exc), **{**opts, "fresh": True}).run() except CheckpointMismatch as exc: log("error", "checkpoint_mismatch", location=loc.key, error=str(exc)) exit_code = max(exit_code, 6) # not retried: a person resumes with the same settings or starts --fresh continue except LockHeld as exc: log("warning", "skipped_overlapping_run", location=loc.key, detail=str(exc)) exit_code = max(exit_code, 3) continue except RunBudgetSpent as exc: log("error", "run_budget_spent", location=loc.key, detail=str(exc)) exit_code = max(exit_code, 5) # wrappers do not retry 5: a person raises the budget or starts fresh continue except SourceError as exc: log("error", "run_aborted", location=loc.key, error=str(exc)) exit_code = max(exit_code, 2) continue except Exception as exc: # noqa: BLE001 - a ledger that cannot be read/written/verified: a person decides from .allowance import LedgerError if not isinstance(exc, LedgerError): raise log("error", "allowance_refused", location=loc.key, error=str(exc), note="nothing more is sent") exit_code = max(exit_code, 5) continue finally: watchdog.cancel() if allowance is not None and allowance.reservation_id: try: allowance.finalize(site.stats.requests, exit_code, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) log("info", "allowance_finalized", jobs=site.stats.requests) except Exception as exc: # noqa: BLE001 - the full reservation stays counted log("error", "allowance_not_finalized", error=str(exc), note="the full reservation stays counted") status = report.get("status") if status in ("blocked", "interrupted"): exit_code = max(exit_code, 4) # resumable: the next start continues the same run elif status not in ("published", "sample"): exit_code = max(exit_code, 1) log("info", "location_finished", location=loc.key, status=status, records=report.get("unique_records")) return exit_code if __name__ == "__main__": sys.exit(main())