"""Offline tests: mapping rules, validation gate, publish protection, resume and locking. No network needed.""" from __future__ import annotations import json import sys from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from marianos_scraper.config import load_config # noqa: E402 from marianos_scraper.kroger_api import ApiStats, SourceError # noqa: E402 from marianos_scraper.mapper import FIELD_ORDER, Mapper, fix_text # noqa: E402 from marianos_scraper.runner import Runner # noqa: E402 from marianos_scraper.storage import LocalStorage, LockHeld # noqa: E402 CFG = load_config() def product(pid, title="Roundy's® Select Milk", brand="Roundy's Select", size="1 gal", regular=2.99, promo=None, cats=("Dairy",)): price = {"regular": regular} if regular is not None else {} if promo is not None: price["promo"] = promo return { "productId": pid, "upc": pid, "brand": brand, "description": title, "categories": list(cats), "productPageURI": f"/p/item/{pid}?cid=dis.api.tpi_products-api", "images": [{"perspective": "front", "featured": True, "sizes": [{"size": "xlarge", "url": f"https://img/{pid}"}]}], "items": [{"size": size, "price": price}], } class FakeApi: """Returns a fixed catalogue for every term; can be told to fail or to lose store context.""" def __init__(self, n=40, fail_terms=(), unpriced=False, zip_code="60601", chain="MARIANOS"): self.catalogue = [product(f"{i:013d}", title=f"Whole Milk {i}", promo=1.99 if i % 5 == 0 else None) for i in range(n)] self.fail_terms = set(fail_terms) self.unpriced = unpriced self.zip_code, self.chain = zip_code, chain self.stats = ApiStats() self.calls = 0 def location(self, location_id): return {"locationId": location_id, "chain": self.chain, "name": "Marianos - Lakeshore East", "address": {"addressLine1": "333 E Benton Pl", "city": "Chicago", "state": "IL", "zipCode": self.zip_code}} def search_page(self, term, location_id, start): self.calls += 1 self.stats.requests += 1 if term in self.fail_terms: raise SourceError("HTTP 403 blocked") page = [dict(p) for p in self.catalogue[start:start + 50]] if self.unpriced: page = [{**p, "items": [{"size": "1 gal", "price": {}}]} for p in page] return page, len(self.catalogue) def products_by_id(self, ids, location_id): return [p for p in self.catalogue if p["productId"] in ids] def make_runner(tmp_path, api, **kw): # v2.2.18: a run limited to some categories is only a sample release, so these API-runner tests use a # configuration whose whole catalogue is Dairy: a full run of that configuration import dataclasses loc = CFG.location("marianos-lakeshore-east") CFG.storage_root = str(tmp_path) cfg = dataclasses.replace(CFG, categories=[c for c in CFG.categories if c.name == "Dairy"]) runner = Runner(cfg, loc, api=api, storage=LocalStorage(str(tmp_path)), **kw) return runner def latest_text(tmp_path, name="products.json"): """A file of the published delivery, read through latest//CURRENT.json (None when nothing is published).""" from marianos_scraper.delivery import Delivery d = Delivery(LocalStorage(str(tmp_path)), "marianos-lakeshore-east") return d.read(d.current(), name) @pytest.fixture(autouse=True) def small_thresholds(monkeypatch): monkeypatch.setitem(CFG.settings["validation"], "min_total_records", 10) import time as _t monkeypatch.setattr("marianos_scraper.runner.time.sleep", lambda s: None) yield _t # --- mapping ------------------------------------------------------------------------------------------ def test_field_order_and_empty_strings(): m = Mapper(CFG.settings) loc = {"zipcode": "60601", "store_name": "Mariano's", "store_location": "333 E Benton Pl, Chicago, IL 60601", "store_logo": ""} rec, _ = m.map(product("0001111042850", regular=4.49, promo=3.99), location=loc, category="Dairy", sub_category="Milk", crawl_date="2026-10-17") assert list(rec) == FIELD_ORDER and list(rec)[-1] == "crawl_date" assert all(isinstance(v, str) for v in rec.values()) assert rec["regular_price"] == "4.49" and rec["sale_price"] == "3.99" assert rec["gupc"] == "" and rec["unit_price"] == "" and rec["ppm"] == "" assert rec["upc"] == "0001111042850" assert rec["url"] == "https://www.marianos.com/p/item/0001111042850" assert rec["store_brand"] == "yes" def test_approximate_weight_excluded_and_unpriced_excluded(): m = Mapper(CFG.settings) loc = {"zipcode": "", "store_name": "", "store_location": "", "store_logo": ""} assert m.map(product("1", size="approx. 2 lb"), location=loc, category="Meat", sub_category="Beef", crawl_date="d") == (None, "approximate_weight") assert m.map(product("2", size="Approximately 1.5 lb"), location=loc, category="Meat", sub_category="Beef", crawl_date="d")[1] == "approximate_weight" assert m.map(product("3", regular=None), location=loc, category="Meat", sub_category="Beef", crawl_date="d")[1] == "not_priced_at_location" def test_promo_not_lower_is_not_sale_and_national_brand_is_no(): m = Mapper(CFG.settings) loc = {"zipcode": "", "store_name": "", "store_location": "", "store_logo": ""} rec, _ = m.map(product("4", brand="Horizon Organic", title="Horizon Organic Milk", regular=5.0, promo=5.0), location=loc, category="Dairy", sub_category="Milk", crawl_date="d") assert rec["sale_price"] == "" and rec["regular_price"] == "5.00" and rec["store_brand"] == "no" def test_mojibake_repaired(): assert fix_text("Simple Truth Organic® Milk") == "Simple Truth Organic® Milk" # --- runs -------------------------------------------------------------------------------------------- def test_good_run_publishes_and_dedupes(tmp_path): report = make_runner(tmp_path, FakeApi()).run() assert report["status"] == "published" latest = json.loads(latest_text(tmp_path)) assert len(latest) == 40 # every Dairy term returns the same 40 products -> deduped assert report["duplicates_removed"] > 0 assert report["validation"]["duplicates_in_output"] == 0 def test_failed_run_never_replaces_last_good_data(tmp_path): make_runner(tmp_path, FakeApi()).run() good = latest_text(tmp_path) report = make_runner(tmp_path, FakeApi(fail_terms={"milk", "cheese", "yogurt"}), fresh=True).run() assert report["status"] == "held_back" assert latest_text(tmp_path) == good def test_lost_store_context_is_blocked(tmp_path): report = make_runner(tmp_path, FakeApi(unpriced=True)).run() assert report["status"] == "held_back" assert latest_text(tmp_path) is None def test_wrong_store_aborts_before_collection(tmp_path): api = FakeApi(zip_code="60661") with pytest.raises(SourceError): make_runner(tmp_path, api).run() assert api.calls == 0 def test_interrupted_run_resumes_without_redoing_units(tmp_path): api1 = FakeApi() first = make_runner(tmp_path, api1, stop_after_units=5).run() assert first["status"] == "interrupted" api2 = FakeApi() report = make_runner(tmp_path, api2).run() assert report["status"] == "published" and report["run_id"] == first["run_id"] and report["resumed_times"] == 1 planned = len(make_runner(tmp_path, FakeApi()).plan_units()) assert api1.calls + api2.calls == planned # each unit fetched exactly once across both processes assert report["api"]["requests"] == planned # request counts of the stopped segment are carried over def test_overlapping_run_is_refused(tmp_path): storage = LocalStorage(str(tmp_path)) storage.acquire_lock("work/marianos-lakeshore-east/run.lock", 3600) with pytest.raises(LockHeld): make_runner(tmp_path, FakeApi()).run() # --- subcategory labelling ------------------------------------------------------------------------------- def test_subcategory_follows_title_not_search_term(tmp_path): runner = make_runner(tmp_path, FakeApi()) runner.plan_units() eggs_unit = {"category": "Dairy", "sub_category": "Eggs"} assert runner.classify(eggs_unit, "Kroger Large Grade A Eggs 12 ct") == ("Eggs", "") assert runner.classify(eggs_unit, "Sargento Shredded Mozzarella Cheese")[0] == "Cheese" assert runner.classify(eggs_unit, "Lunchables Turkey Crackers")[0] == "" def test_title_exclusion_rule_keeps_sauce_out_of_meat(): runner = Runner(CFG, CFG.location("marianos-lakeshore-east"), api=FakeApi(), storage=LocalStorage(CFG.storage_root), only_categories=["Meat"]) runner.plan_units() assert runner.excluders["Meat"].search("Kroger The Universal Steak Sauce") assert not runner.excluders["Meat"].search("Butcher's Choice Ground Beef 80% Lean") assert runner.classify({"category": "Meat", "sub_category": "Hot Dogs"}, "Oscar Mayer Bun-Length Beef Franks Hot Dogs")[0] == "Hot Dogs" # --- website mapping (real marianos.com payloads, trimmed) -------------------------------------------------- from marianos_scraper.mapper import SiteMapper # noqa: E402 SITE = json.loads((Path(__file__).parent / "fixtures" / "site_products.json").read_text(encoding="utf-8")) COUPON = json.loads((Path(__file__).parent / "fixtures" / "site_coupon.json").read_text(encoding="utf-8")) LOC = {"zipcode": "60601", "store_name": "Mariano's", "store_location": "333 E Benton Pl, Chicago, IL 60601", "store_logo": ""} def site_map(key, coupons=()): rec, reason, meta = SiteMapper(CFG.settings).map_site(SITE[key], list(coupons), location=LOC, category="Batteries", sub_category="Batteries", crawl_date="2026-10-19") return rec, reason, meta def test_site_regular_price_and_displayed_ppm(): rec, _, meta = site_map("regular") assert list(rec) == FIELD_ORDER assert rec["regular_price"] == "12.99" and rec["sale_price"] == "" and rec["ppm"] == "$1.62/each" assert rec["volume_deals_description"] == "" and rec["digital_deals_description"] == "" assert rec["url"].startswith("https://www.marianos.com/p/") and rec["upc"] == "0004133382501" assert meta["source_location"] and "26003" in meta["departments"] def test_site_quantity_offer_is_a_volume_deal_with_its_shown_price(): rec, _, _ = site_map("volume_offer") assert rec["volume_deals_description"].startswith("Save $1 each when you buy 5 or more. Valid 2026-10-07 to 2026-10-27.") assert rec["volume_deals_price"] == "4.99" and rec["sale_price"] == "" assert rec["ppm"] == "$1.25/each" def test_site_plain_sale_goes_to_sale_price(): rec, _, _ = site_map("plain_sale") assert rec["sale_price"] == "14.99" and rec["regular_price"] == "17.99" and rec["volume_deals_description"] == "" assert rec["ppm"] == "$0.94/each" def test_site_digital_coupon_keeps_wording_and_stated_price(): rec, _, _ = site_map("regular", [COUPON, {**COUPON, "type": "CASH_BACK", "shortDescription": "Cash back"}]) assert rec["digital_deals_description"].startswith("Digital Coupon (clip to card): $1.49 Post Small Size Cereal.") assert "Use up to 5 times per transaction" in rec["digital_deals_description"] assert "Cash back" not in rec["digital_deals_description"] assert rec["digital_deals_price"] == "1.49" # the coupon title states the price; nothing is computed def test_coupon_price_only_when_the_coupon_states_a_price(): priced = {**COUPON, "title": "$2.99", "shortDescription": "$2.99 Dawn Ez-Squeeze"} savings = {**COUPON, "title": "Save $2.00", "shortDescription": "Save $2.00 on Suavitel"} assert SiteMapper.digital_price([priced]) == "2.99" assert SiteMapper.digital_price([savings]) == "" assert SiteMapper.digital_price([{**priced, "type": "CASH_BACK"}]) == "" def test_points_offers_are_not_digital_deals(): points = {**COUPON, "title": "4X", "shortDescription": "Earn 4X Points on Frozen Food and Dairy Purchases on 10/14 -10/20"} assert SiteMapper.digital_text([points]) == "" # --- availability rule: nothing is dropped for availability; foreign or missing prices stay empty ----------------- def test_product_priced_only_for_another_store_is_kept_with_empty_prices(): p = json.loads(json.dumps(SITE["volume_offer"])) loc = {**LOC, "location_id": "53100503"} # the fixture was priced for store 53100526 rec, reason, meta = SiteMapper(CFG.settings).map_site(p, [], location=loc, category="Batteries", sub_category="Batteries", crawl_date="d") assert rec is not None and reason == "" and meta["price_status"] == "other_location_price_not_used" assert rec["regular_price"] == rec["sale_price"] == rec["volume_deals_price"] == rec["volume_deals_description"] == rec["ppm"] == "" assert rec["upc"] and rec["product_title"] def test_unpriced_and_out_of_stock_product_is_kept(): p = json.loads(json.dumps(SITE["regular"])) p["price"] = {} p["inventory"] = {"locations": [{"locationId": "53100526", "stockLevel": "TEMPORARILY_OUT_OF_STOCK"}]} p["fulfillmentSummaries"] = [{"type": "PICKUP", "availability": {"sellable": False}}] rec, reason, meta = SiteMapper(CFG.settings).map_site(p, [], location=LOC, category="Batteries", sub_category="Batteries", crawl_date="d") assert rec is not None and meta["price_status"] == "no_price_returned" and meta["stock_level"] == "TEMPORARILY_OUT_OF_STOCK" assert rec["regular_price"] == "" and rec["upc"] == "0004133382501" and list(rec) == FIELD_ORDER def test_fallback_run_never_touches_an_unfinished_website_run(tmp_path): storage = LocalStorage(str(tmp_path)) site_state = {"run_id": "SITE-RUN", "status": "interrupted", "started_epoch": 0, "segments": []} storage.write_text("work/marianos-lakeshore-east/current.json", json.dumps(site_state)) CFG.storage_root = str(tmp_path) report = Runner(CFG, CFG.location("marianos-lakeshore-east"), api=FakeApi(), storage=storage, only_categories=["Dairy"], fallback_reason="site refused", fresh=True).run() assert report["status"] == "held_back" and report["source"].endswith("(fallback)") assert json.loads(storage.read_text("work/marianos-lakeshore-east/current.json"))["run_id"] == "SITE-RUN" assert latest_text(tmp_path) is None def test_refinish_compares_with_the_catalog_from_before_the_run(tmp_path): from marianos_scraper.site_runner import SiteRunner storage = LocalStorage(str(tmp_path)) runner = SiteRunner(CFG, CFG.location("marianos-lakeshore-east"), site=object(), api=FakeApi(), storage=storage) old = {"run_id": "OLD", "products": {"1": {"record": {}, "last_observed_run": "OLD", "meta": {}}}} storage.write_text("state/marianos-lakeshore-east/catalog_previous.json", json.dumps(old)) storage.write_text("state/marianos-lakeshore-east/catalog.json", json.dumps({**old, "run_id": "NEW"})) catalog, prev_run = runner._load_catalog({"run_id": "NEW"}) assert prev_run == "OLD" # never compared with itself assert runner._load_catalog({"run_id": "NEXT"})[1] == "NEW" # a new run uses the current catalog def test_every_collected_product_is_asked_for_its_coupons(tmp_path): from marianos_scraper.site_runner import COUPON_LINK_UNIT, SiteRunner class FakeSite: calls: list = [] def coupons_for(self, upcs): self.asked = list(upcs) return [{"id": "7", "type": "STANDARD", "upcs": ["0000000000001"]}] storage = LocalStorage(str(tmp_path)) site = FakeSite() runner = SiteRunner(CFG, CFG.location("marianos-lakeshore-east"), site=site, api=FakeApi(), storage=storage) runner._run_id = "RUN" units = "work/marianos-lakeshore-east/RUN/units/" storage.write_text(units + "00-coupons-all-coupons.json", json.dumps({"coupons": {"standard": [{"id": "7", "brand": "Kroger Brand"}]}})) storage.write_text(units + "01-dairy.json", json.dumps({"records": [{"upc": "0000000000001", "product_title": "Kroger String Cheese"}, {"upc": "0000000000002", "product_title": "Store Cheese"}]})) res = runner._link_coupons(COUPON_LINK_UNIT, 0.0) assert site.asked == ["0000000000001", "0000000000002"] # not only products whose title carries a coupon brand assert [c["id"] for c in res["links"]["0000000000001"]] == ["7"] def test_about_each_is_approximate_weight_but_per_pound_pricing_is_kept(): import copy base = copy.deepcopy(next(iter(SITE.values()))) about = copy.deepcopy(base) about["price"] = {"sellBy": "Weight", "orderBy": "Unit", "storePrices": {"sourceLocationId": "53100503", "regular": {"defaultDescription": "about $15.78 each", "secondaryDescription": "$9.99/lb", "price": "USD 15.78"}}} loose = copy.deepcopy(base) loose["price"] = {"sellBy": "Weight", "orderBy": "Weight", "storePrices": {"sourceLocationId": "53100503", "regular": {"defaultDescription": "$2.79/lb", "price": "USD 2.79"}}} loose["fulfillmentSummaries"] = [{"type": "PICKUP", "availability": {"sellable": True}, "regular": {"price": "USD 2.79"}}] # as the site sends it: no PPM there m = SiteMapper(CFG.settings) assert m.map_site(about, [], location=LOC, category="Seafood", sub_category="", crawl_date="2026-10-21")[:2] == (None, "approximate_weight_price") rec, reason, _ = m.map_site(loose, [], location=LOC, category="Produce", sub_category="", crawl_date="2026-10-21") assert reason == "" and rec["regular_price"] == "2.79" and rec["ppm"] == "$2.79/lb" # displayed price per lb, kept def test_multi_buy_keeps_its_end_date(): import copy p = copy.deepcopy(next(iter(SITE.values()))) p["offers"] = [] p["price"] = {"sellBy": "Unit", "orderBy": "Unit", "storePrices": {"sourceLocationId": "53100503", "regular": {"defaultDescription": "$4.29", "price": "USD 4.29"}, "promo": {"defaultDescription": "2 For $7.00", "nfor": 2, "nforPrice": "USD 7.00", "price": "USD 3.50", "expirationDate": {"value": "2026-10-27T03:59:59Z", "timezone": "UTC"}}}} rec, _, _ = SiteMapper(CFG.settings).map_site(p, [], location=LOC, category="Produce", sub_category="", crawl_date="2026-10-22") assert rec["volume_deals_description"].startswith("2 for $7.00. Ends 2026-10-27.") and rec["volume_deals_price"] == "7.00" def test_a_node_of_340_products_is_read_to_the_end(tmp_path): import copy from marianos_scraper.site_runner import SiteRunner ids = [f"{i:013d}" for i in range(1, 341)] base = next(iter(SITE.values())) class FakeSite: calls: list = [] def search(self, location_id, taxonomy, offset=0, size=100, brand="", **kw): assert offset <= 250, "the site rejects offsets above 250" return [{"upc": u} for u in ids[offset:offset + 100]], len(ids), {} def check_context(self, products, path): pass # the echoed-context check itself is tested in test_oxylabs_probe.py def products(self, upcs, verified=True): out = [] for u in upcs: p = copy.deepcopy(base) p["id"] = u p["item"] = {**p.get("item", {}), "upc": u} out.append(p) return out runner = SiteRunner(CFG, CFG.location("marianos-lakeshore-east"), site=FakeSite(), api=FakeApi(), storage=LocalStorage(str(tmp_path))) runner._run_id = "RUN" runner._coupon_index = {} unit = {"id": "10-natural-organic-18007", "category": "Natural Organic", "sub_category": "x", "taxonomy": "18007", "expected": 340} res = runner.collect_unit(unit, {"zipcode": "60601", "store_name": "Mariano's", "store_location": "x", "store_logo": "", "location_id": "53100503"}) assert res["found_upcs"] == 340 and res["api_total"] == 340 # --- v2.2.16: full-run retention across repeat runs ---------------------------------------------------------- class ChangingStore: """The whole website for SiteRunner.run(), offline: every taxonomy root has one leaf of five products. Between runs the test changes `unavailable` (listed, not sellable) and `absent` (no longer listed at all).""" STORE = {"locationId": "53100503", "facilityId": "15096", "brand": "MARIANOS", "storeNumber": "00503", "vanityName": "Mariano's Lakeshore East", "locale": {"address": {"name": "Lakeshore East", "addressLines": ["333 E Benton Pl"], "cityTown": "CHICAGO", "stateProvince": "IL", "postalCode": "60601"}}} def __init__(self): import types self.unavailable, self.absent = set(), set() self.calls, self.max_requests, self.max_seconds = [], None, None self.stats = types.SimpleNamespace(requests=0, retries=0, errors=0, total_seconds=0.0) @staticmethod def leaf(root): return root if len(root) >= 5 else root + "001" @staticmethod def upcs_of(leaf): return [f"{leaf}{i:02d}".rjust(13, "0") for i in range(1, 6)] def listed(self, leaf): return [u for u in self.upcs_of(leaf) if u not in self.absent] def open(self): pass def close(self): pass def store(self, zipcode, location_id): return self.STORE def set_store(self, *a, **kw): pass def check_context(self, products, path): pass def search(self, location_id, taxonomy, offset=0, size=100, brand="", facets=False, brand_facet=False): if facets: if len(taxonomy) >= 5: n = len(self.listed(taxonomy)) if taxonomy in self._leaves() else 0 return [], n, [{"id": taxonomy, "name": taxonomy, "count": n, "children": []}] leaf = self.leaf(taxonomy) n = len(self.listed(leaf)) return [], n, [{"id": taxonomy, "name": taxonomy, "count": n, "children": [{"id": leaf, "name": f"Aisle {leaf}", "count": n, "children": []}]}] ids = self.listed(taxonomy) return [{"upc": u} for u in ids[offset:offset + size]], len(ids), {} def _leaves(self): import json as _j tax = _j.loads((Path(CFG.config_dir) / "site_taxonomy.json").read_text(encoding="utf-8"))["categories"] return {r for spec in tax.values() for r in spec.get("roots", []) if len(r) >= 5} def products(self, upcs, verified=True): import copy out = [] for u in upcs: p = copy.deepcopy(SITE["regular"]) p["id"] = u p["item"] = {**p.get("item", {}), "upc": u, "description": f"Roundy's Product {u}"} p["price"]["storePrices"]["sourceLocationId"] = "53100503" if u in self.unavailable: for s in p.get("fulfillmentSummaries") or []: s.setdefault("availability", {})["sellable"] = False out.append(p) return out def all_coupons(self, kind): found = [COUPON] if kind == "standard" else [] return found, {"totalCount": len(found)} # the site states its own total, as the real one does def coupons_for(self, upcs): return [] def test_unavailable_and_later_absent_products_survive_repeat_runs_with_their_real_last_observation(tmp_path, monkeypatch): """Three full runs of SiteRunner.run() on one store (LocalStorage, no network): run 1 (Oct 01) everything listed; run 2 (Oct 02) GONE is no longer listed and OOS is listed but not sellable; run 3 (Oct 03) GONE is still absent and OOS is absent too. Every product stays in every published output; a carried record keeps the crawl_date of its last observation (never today's), availability.jsonl tells fresh from carried, and nothing is invented.""" from datetime import datetime as _dt from marianos_scraper import site_runner as sr from marianos_scraper.site_runner import SiteRunner for k, val in {"min_records_per_category": 1, "min_ppm_share": 0.0, "min_total_records": 10}.items(): monkeypatch.setitem(CFG.settings["validation"], k, val) CFG.storage_root = str(tmp_path) storage = LocalStorage(str(tmp_path)) site = ChangingStore() leaf = site.leaf("02") # Dairy GONE, OOS = site.upcs_of(leaf)[0], site.upcs_of(leaf)[1] day = {"v": "2026-10-01"} class FixedDay(_dt): @classmethod def now(cls, tz=None): return _dt.fromisoformat(day["v"] + "T12:00:00+00:00") monkeypatch.setattr(sr, "datetime", FixedDay) def run(d): day["v"] = d runner = SiteRunner(CFG, CFG.location("marianos-lakeshore-east"), site=site, api=FakeApi(), storage=storage) report = runner.run() assert report["status"] == "published", [c for c in report["validation"]["checks"] if not c["passed"]] out = f"runs/marianos-lakeshore-east/{report['run_id']}/" products = {r["upc"]: r for r in json.loads(storage.read_text(out + "products.json"))} avail = {a["upc"]: a for a in map(json.loads, storage.read_text(out + "availability.jsonl").splitlines())} latest = {r["upc"] for r in json.loads(latest_text(tmp_path))} assert set(products) == latest return report["run_id"], products, avail run1, p1, a1 = run("2026-10-01") assert {GONE, OOS} <= set(p1) and a1[GONE]["observation"] == a1[OOS]["observation"] == "fresh" total = len(p1) site.absent, site.unavailable = {GONE}, {OOS} run2, p2, a2 = run("2026-10-02") assert len(p2) == total # nothing dropped assert p2[GONE]["crawl_date"] == "2026-10-01" # carried: its real last observation assert a2[GONE] == {**a2[GONE], "observation": "carried_forward", "last_observed_run": run1, "crawl_date": "2026-10-01"} assert p2[OOS]["crawl_date"] == "2026-10-02" and a2[OOS]["observation"] == "fresh" # listed: a fresh record assert a2[OOS]["fulfillment_sellable"] == [] and a2[OOS]["last_observed_run"] == run2 site.absent = {GONE, OOS} run3, p3, a3 = run("2026-10-03") assert len(p3) == total assert p3[GONE]["crawl_date"] == "2026-10-01" and a3[GONE]["last_observed_run"] == run1 # not reset by run 2 assert p3[OOS]["crawl_date"] == "2026-10-02" and a3[OOS]["last_observed_run"] == run2 assert a3[GONE]["observation"] == a3[OOS]["observation"] == "carried_forward" fresh = [u for u, a in a3.items() if a["observation"] == "fresh"] assert len(fresh) == total - 2 and all(p3[u]["crawl_date"] == "2026-10-03" for u in fresh) assert p3[GONE] == p1[GONE] and p3[OOS] == p2[OOS] # the record itself is the last observation, unchanged changes = json.loads(storage.read_text(f"runs/marianos-lakeshore-east/{run3}/availability_changes.json")) assert {c["upc"] for c in changes if c["kept_as"].startswith("carried_forward")} == {GONE, OOS}