"""Storage backends: local folder (development) and Azure Blob Storage (production). Both expose the same small interface so the runner never knows where it writes. Writes are whole-object and atomic (temp file + rename locally, single PUT on Blob), so a crash can never leave a half-written file that a later step mistakes for good data. """ from __future__ import annotations import json import os import socket import time from pathlib import Path class LockHeld(Exception): pass class PreconditionFailed(Exception): """A conditional write found the object changed since it was read (or present when it had to be absent).""" def _etag_of(data: bytes) -> str: import hashlib return hashlib.sha256(data).hexdigest() class LocalStorage: def __init__(self, root: str): self.root = Path(root) self.root.mkdir(parents=True, exist_ok=True) def _p(self, key: str) -> Path: return self.root / key def exists(self, key: str) -> bool: return self._p(key).is_file() def read_text(self, key: str) -> str: return self._p(key).read_text(encoding="utf-8") def write_text(self, key: str, text: str) -> None: path = self._p(key) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(path.name + f".tmp{os.getpid()}") tmp.write_text(text, encoding="utf-8") os.replace(tmp, path) def write_bytes(self, key: str, data: bytes) -> None: path = self._p(key) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(path.name + f".tmp{os.getpid()}") tmp.write_bytes(data) os.replace(tmp, path) def delete(self, key: str) -> None: self._p(key).unlink(missing_ok=True) def read_with_etag(self, key: str) -> tuple[str | None, str | None]: """Text and ETag (SHA-256 of the bytes) of an object, or (None, None) when it does not exist.""" path = self._p(key) if not path.is_file(): return None, None data = path.read_bytes() # text exactly as read_text() returns it (text mode: Windows line endings folded back), ETag over the bytes return data.decode("utf-8").replace("\r\n", "\n"), _etag_of(data) def write_if_match(self, key: str, text: str, etag: str | None) -> None: """Replace the object only if it is still the one read (etag None: only if it does not exist yet). The run lock makes the run the only writer; this check also refuses a write over anything changed meanwhile.""" _, current = self.read_with_etag(key) if current != etag: raise PreconditionFailed(f"{key} changed since it was read") self.write_text(key, text) def list(self, prefix: str) -> list[str]: base = self._p(prefix) if not base.is_dir(): return [] return sorted(str(p.relative_to(self.root)).replace("\\", "/") for p in base.rglob("*") if p.is_file() and ".tmp" not in p.name) def acquire_lock(self, key: str, stale_after_seconds: int) -> None: path = self._p(key) path.parent.mkdir(parents=True, exist_ok=True) info = json.dumps({"host": socket.gethostname(), "pid": os.getpid(), "since": time.time()}) try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) except FileExistsError: held = json.loads(path.read_text(encoding="utf-8") or "{}") if time.time() - held.get("since", 0) < stale_after_seconds: raise LockHeld(f"Another run holds {key} since {time.ctime(held.get('since', 0))} ({held.get('host')})") path.unlink(missing_ok=True) # stale lock from a crashed run fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) with os.fdopen(fd, "w") as fh: fh.write(info) def release_lock(self, key: str) -> None: self._p(key).unlink(missing_ok=True) class BlobStorage: """Azure Blob Storage backend. Auth: connection string, or managed identity via DefaultAzureCredential.""" def __init__(self, container: str, connection_string: str = "", account_url: str = ""): from azure.storage.blob import BlobServiceClient if connection_string: service = BlobServiceClient.from_connection_string(connection_string) else: from azure.identity import DefaultAzureCredential service = BlobServiceClient(account_url=account_url, credential=DefaultAzureCredential()) self.container = service.get_container_client(container) if not self.container.exists(): self.container.create_container() self._leases: dict = {} def exists(self, key: str) -> bool: return self.container.get_blob_client(key).exists() def read_text(self, key: str) -> str: return self.container.download_blob(key).readall().decode("utf-8") def write_text(self, key: str, text: str) -> None: self.container.upload_blob(key, text.encode("utf-8"), overwrite=True) def write_bytes(self, key: str, data: bytes) -> None: self.container.upload_blob(key, data, overwrite=True) def delete(self, key: str) -> None: blob = self.container.get_blob_client(key) if blob.exists(): blob.delete_blob(delete_snapshots="include") def list(self, prefix: str) -> list[str]: return sorted(b.name for b in self.container.list_blobs(name_starts_with=prefix.rstrip("/") + "/")) def read_with_etag(self, key: str) -> tuple[str | None, str | None]: from azure.core.exceptions import ResourceNotFoundError try: d = self.container.download_blob(key) except ResourceNotFoundError: return None, None return d.readall().decode("utf-8"), d.properties.etag def write_if_match(self, key: str, text: str, etag: str | None) -> None: """One PUT with If-Match on the ETag read (etag None: If-None-Match *, i.e. create only).""" from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceExistsError blob = self.container.get_blob_client(key) try: if etag is None: blob.upload_blob(text.encode("utf-8"), overwrite=False) else: blob.upload_blob(text.encode("utf-8"), overwrite=True, etag=etag, match_condition=MatchConditions.IfNotModified) except ResourceExistsError as exc: raise PreconditionFailed(f"{key} exists already") from exc except HttpResponseError as exc: if getattr(exc, "status_code", None) == 412: raise PreconditionFailed(f"{key} changed since it was read") from exc raise def acquire_lock(self, key: str, stale_after_seconds: int) -> None: """Infinite blob lease = cluster-wide mutex. A lease older than stale_after_seconds is broken (crashed run).""" from azure.core.exceptions import HttpResponseError, ResourceExistsError blob = self.container.get_blob_client(key) if not blob.exists(): try: blob.upload_blob(b"{}", overwrite=False) except ResourceExistsError: pass try: lease = blob.acquire_lease(lease_duration=-1) except HttpResponseError: since = float((blob.get_blob_properties().metadata or {}).get("since", "0")) if time.time() - since < stale_after_seconds: raise LockHeld(f"Another run holds {key}") blob.break_lease(lease_break_period=0) lease = blob.acquire_lease(lease_duration=-1) blob.set_blob_metadata({"since": str(time.time()), "host": socket.gethostname()}, lease=lease) self._leases[key] = lease def release_lock(self, key: str) -> None: lease = self._leases.pop(key, None) if lease is not None: lease.release() def open_storage(cfg) -> LocalStorage | BlobStorage: if cfg.storage_mode == "azure": return BlobStorage(cfg.azure_container, cfg.azure_connection_string, cfg.azure_account_url) return LocalStorage(cfg.storage_root)