Nearle/routesapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Suriya
2026-06-22 17:40:08 +05:30
commit c742ef0e53
308 changed files with 68519 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

864
app/services/routing/delivery_history_service.py Normal file
View File

@@ -0,0 +1,864 @@
"""
Delivery History Service — Empirical ETA from ground truth
==========================================================
WHY THIS EXISTS
---------------
Until now every ETA in the system was a *formula guess*
(`RealisticETACalculator`: distance / configured_speed + fixed buffers) that was
never checked against what actually happened in the field. Senior feedback was
that manual riders deliver faster than our estimates — i.e. the guess is wrong.
This service closes the loop. The external `nearledb` Postgres already records,
for every completed delivery, when the rider picked up (`pickuptime`) and when
the order was delivered (`deliverytime`). From those we reconstruct **actual
per-leg travel times** and learn empirical medians, so ETAs reflect reality
instead of a hand-tuned formula.
GROUND-TRUTH RECONSTRUCTION (per-leg, matches the optimizer)
-----------------------------------------------------------
The optimizer feeds *aerial* (straight-line) leg distance to `calculate_eta`
(see route_optimizer.py — `step_dist` comes from `aerial_matrix`, no road
factor). To learn a model that plugs in behind the same call, we reconstruct
observations the same way:
* Group a rider's completed deliveries by day, ordered by `deliverytime`.
* For each pair of consecutive deliveries:
leg_min = deliverytime[i] - deliverytime[i-1] (real door-to-door time)
leg_km = aerial haversine(drop[i-1], drop[i]) (same metric as optimizer)
* The first delivery of a group is the kitchen→first-drop leg:
leg_min = deliverytime[0] - pickuptime[0] (distance unknown w/o
kitchen coords, so it only feeds the non-distance keys).
Only `droplat/droplon`, `pickuptime`, `deliverytime`, `pickupcustomer`, `userid`
are needed — all confirmed-present columns (same set batch_analytics reads).
AGGREGATION + LOOKUP (hierarchical, cold-start safe)
----------------------------------------------------
Each observation feeds several keys at decreasing specificity. At prediction
time we walk the same hierarchy and use the first key with enough samples,
falling back to the formula when history is too thin:
kzd kitchen | drop_zone | traffic | dist_bucket (most specific)
kz kitchen | drop_zone | traffic
zd drop_zone | traffic | dist_bucket
z drop_zone | traffic
dt dist_bucket | traffic
t traffic (least specific)
→ RealisticETACalculator formula (no data)
`rkz` (rider | kitchen | zone | traffic) is also stored for Phase-2 learned
rider affinity; it is not used by the default lookup yet.
"""
import logging
import math
import os
import sqlite3
import threading
import statistics
import time
from datetime import datetime, timedelta
from collections import defaultdict
from typing import Any, Dict, List, Optional, Tuple
from app.services.routing.zone_service import ZoneService
logger = logging.getLogger(__name__)
_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
_WRITE_LOCK = threading.Lock()
# Aerial leg-distance buckets (km) — must match the metric the optimizer feeds
# to calculate_eta (pure haversine, no road factor).
_DIST_BUCKETS: List[Tuple[float, float]] = [
(0.0, 1.0), (1.0, 2.0), (2.0, 3.0), (3.0, 5.0), (5.0, 8.0), (8.0, 12.0), (12.0, 1e9)
]
# Sanity filters for reconstructed legs.
_MAX_LEG_MIN = 60.0 # gaps longer than this are batch boundaries / idle, not a leg
_MAX_LEG_KM = 40.0 # implausible single hop
# ---------------------------------------------------------------------------
# Pure helpers
# ---------------------------------------------------------------------------
def dist_bucket(km: Optional[float]) -> Optional[str]:
"""Aerial leg distance -> discrete bucket label, or None if unknown."""
if km is None:
return None
for lo, hi in _DIST_BUCKETS:
if km <= hi:
return f"{lo:g}-{hi:g}" if hi < 1e9 else f"{lo:g}+"
return f"{_DIST_BUCKETS[-1][0]:g}+"
def hour_to_traffic(hour: int) -> str:
"""Time-of-day -> traffic category, mirroring get_time_of_day_category()."""
if (8 <= hour < 10) or (12 <= hour < 14) or (17 <= hour < 20):
return "peak"
if hour < 7 or hour >= 22:
return "light"
return "normal"
def normalize_kitchen(name: Any) -> str:
"""Normalize a kitchen / pickup-customer name to a stable key token."""
if not name:
return "?"
return " ".join(str(name).strip().lower().split())
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
"""Great-circle (aerial) distance in km."""
try:
rlat1, rlon1, rlat2, rlon2 = map(math.radians, (lat1, lon1, lat2, lon2))
dlat = rlat2 - rlat1
dlon = rlon2 - rlon1
a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2) ** 2
return 6371.0 * 2 * math.asin(min(1.0, math.sqrt(a)))
except Exception:
return 0.0
def _parse_dt(val: Any) -> Optional[datetime]:
if val in (None, "", 0):
return None
if isinstance(val, datetime):
return val
s = str(val).strip()
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S.%f",
"%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M"):
try:
return datetime.strptime(s.split("+")[0].strip(), fmt)
except Exception:
continue
try:
from dateutil.parser import parse as _du
return _du(s)
except Exception:
return None
# ---------------------------------------------------------------------------
# Shared Postgres connector (single source of truth for nearledb creds)
# ---------------------------------------------------------------------------
def connect_nearledb(connect_timeout: int = 10):
"""
Open a connection to the external nearledb Postgres.
Credentials come from DB_* env vars (same defaults batch_analytics used).
Raises on failure — callers convert to their own error type.
"""
import psycopg2 # imported lazily so the app boots without it
conn = psycopg2.connect(
host=os.getenv("DB_HOST", "66.116.207.225"),
port=int(os.getenv("DB_PORT", "6432")),
dbname=os.getenv("DB_NAME", "nearledb"),
user=os.getenv("DB_USER", "admin"),
password=os.getenv("DB_PASSWORD", "Package@123#"),
connect_timeout=connect_timeout,
)
# Best-effort read-only session. (Port 6432 is pgbouncer, which rejects the
# `options` startup param, so we set it post-connect; our code only issues
# SELECTs regardless — we never write to or alter the source DB.)
try:
conn.set_session(readonly=True)
except Exception:
pass
return conn
# ---------------------------------------------------------------------------
# Key construction (build-side and lookup-side share this so they stay in sync)
# ---------------------------------------------------------------------------
def _keys_for_observation(kitchen: str, zone: str, traffic: str,
bucket: Optional[str], rider_id: Any) -> List[str]:
"""All aggregation keys an observation contributes to."""
keys = [
f"kz|{kitchen}|{zone}|{traffic}",
f"z|{zone}|{traffic}",
f"t|{traffic}",
f"rkz|{rider_id}|{kitchen}|{zone}|{traffic}",
]
if bucket is not None:
keys += [
f"kzd|{kitchen}|{zone}|{traffic}|{bucket}",
f"zd|{zone}|{traffic}|{bucket}",
f"dt|{bucket}|{traffic}",
]
return keys
def _lookup_keys(kitchen: Optional[str], zone: str, traffic: str,
bucket: Optional[str]) -> List[str]:
"""Ordered candidate keys, most specific first, for prediction-time lookup."""
ordered: List[str] = []
if kitchen and bucket is not None:
ordered.append(f"kzd|{kitchen}|{zone}|{traffic}|{bucket}")
if kitchen:
ordered.append(f"kz|{kitchen}|{zone}|{traffic}")
if bucket is not None:
ordered.append(f"zd|{zone}|{traffic}|{bucket}")
ordered.append(f"z|{zone}|{traffic}")
if bucket is not None:
ordered.append(f"dt|{bucket}|{traffic}")
ordered.append(f"t|{traffic}")
return ordered
# ---------------------------------------------------------------------------
# Service
# ---------------------------------------------------------------------------
class DeliveryHistoryService:
"""Ingests completed deliveries, learns empirical leg-time medians, serves lookups."""
def __init__(self):
self._db_path = _DB_PATH
self._zone = ZoneService()
# cache: full_key -> (count, median_min, p75_min)
self._cache: Dict[str, Tuple[int, float, float]] = {}
self._last_refreshed: Optional[datetime] = None
self._last_summary: Dict[str, Any] = {}
self._refresh_lock = threading.Lock()
self._refresh_attempt_at: Optional[datetime] = None
self._scheduler_started = False
self._ensure_db()
self._load_cache()
# -- schema -------------------------------------------------------------
def _ensure_db(self) -> None:
try:
os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
conn = sqlite3.connect(self._db_path)
conn.execute("""
CREATE TABLE IF NOT EXISTS delivery_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
delivery_ts TEXT,
rider_id TEXT,
kitchen TEXT,
drop_zone TEXT,
traffic TEXT,
dist_bucket TEXT,
leg_km REAL,
leg_min REAL,
is_first INTEGER DEFAULT 0
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS eta_stats (
key TEXT PRIMARY KEY,
key_type TEXT,
sample_count INTEGER,
median_min REAL,
p75_min REAL,
updated_at TEXT
)
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_eta_stats_type ON eta_stats(key_type)")
# Local mirror of the nearledb `deliveries` rows we care about.
# The request path NEVER touches Postgres — only this table.
# Deduped by deliveryid so incremental syncs are idempotent.
conn.execute("""
CREATE TABLE IF NOT EXISTS delivery_raw (
deliveryid TEXT PRIMARY KEY,
userid TEXT,
pickupcustomer TEXT,
pickuptime TEXT,
deliverytime TEXT,
dlat REAL,
dlon REAL,
plat REAL,
plon REAL
)
""")
# Migration for stores created before pickup coords were added.
for _ddl in (
"ALTER TABLE delivery_raw ADD COLUMN plat REAL",
"ALTER TABLE delivery_raw ADD COLUMN plon REAL",
):
try:
conn.execute(_ddl)
except Exception:
pass
conn.execute("CREATE INDEX IF NOT EXISTS idx_raw_dtime ON delivery_raw(deliverytime)")
# Single-row sync watermark / bookkeeping.
conn.execute("""
CREATE TABLE IF NOT EXISTS eta_sync_state (
id INTEGER PRIMARY KEY CHECK (id = 1),
last_delivery_ts TEXT,
last_synced_at TEXT,
total_rows INTEGER DEFAULT 0
)
""")
conn.execute("INSERT OR IGNORE INTO eta_sync_state (id, total_rows) VALUES (1, 0)")
conn.commit()
conn.close()
except Exception as e:
logger.error(f"[DeliveryHistory] DB init failed: {e}")
# -- ingestion ----------------------------------------------------------
def fetch_completed_deliveries(
self, days: int, tenant_id: int = 916, since: Optional[str] = None
) -> List[Dict[str, Any]]:
"""
READ-ONLY pull from nearledb `deliveries`.
* `since` set -> incremental: only rows newer than the watermark.
* `since` None -> initial backfill of the last `days`.
Only the `deliveries` table is read; the session is read-only (no writes,
no schema changes — enforced at the connection level).
"""
conn = connect_nearledb()
try:
cur = conn.cursor()
if since:
where_time = "d.deliverytime::timestamp > %s"
time_param = since
else:
where_time = "d.deliverytime::timestamp >= %s"
time_param = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
cur.execute(
f"""
SELECT
d.deliveryid,
d.userid,
d.pickupcustomer,
d.pickuptime,
d.deliverytime,
COALESCE(d.droplat, d.deliverylat) AS dlat,
COALESCE(d.droplon, d.deliverylong) AS dlon,
d.pickuplat AS plat,
d.pickuplon AS plon
FROM deliveries d
WHERE d.tenantid = %s
AND d.deliverytime IS NOT NULL
AND d.pickuptime IS NOT NULL
AND {where_time}
AND COALESCE(d.droplat, d.deliverylat) IS NOT NULL
AND d.userid IS NOT NULL
ORDER BY d.deliverytime
""",
(tenant_id, time_param),
)
cols = [c.name for c in cur.description]
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
cur.close()
return rows
finally:
conn.close()
# -- watermark / local raw store ---------------------------------------
def _get_watermark(self) -> Optional[str]:
try:
conn = sqlite3.connect(self._db_path)
row = conn.execute(
"SELECT last_delivery_ts FROM eta_sync_state WHERE id = 1"
).fetchone()
conn.close()
return row[0] if row and row[0] else None
except Exception:
return None
def sync_from_db(self, days: int = 14, tenant_id: int = 916,
full: bool = False) -> Dict[str, Any]:
"""
Pull NEW completed deliveries from nearledb into the local mirror.
Idempotent (INSERT OR IGNORE on deliveryid). On the first run (empty
watermark) or `full=True`, backfills the last `days`; afterwards only
rows newer than the watermark are fetched — minimal DB load.
"""
watermark = None if full else self._get_watermark()
rows = self.fetch_completed_deliveries(days, tenant_id, since=watermark)
inserted = 0
max_ts = watermark
conn = sqlite3.connect(self._db_path)
try:
if full:
# A full sync truly rebuilds the local mirror (also backfills any
# newly-added columns that INSERT OR IGNORE would otherwise skip).
conn.execute("DELETE FROM delivery_raw")
conn.execute("UPDATE eta_sync_state SET last_delivery_ts = NULL WHERE id = 1")
max_ts = None
for r in rows:
dt = str(r.get("deliverytime") or "")
try:
cur = conn.execute(
"INSERT OR IGNORE INTO delivery_raw "
"(deliveryid, userid, pickupcustomer, pickuptime, deliverytime, dlat, dlon, plat, plon) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(str(r.get("deliveryid")), str(r.get("userid")),
r.get("pickupcustomer"), str(r.get("pickuptime") or ""),
dt, r.get("dlat"), r.get("dlon"), r.get("plat"), r.get("plon")),
)
inserted += cur.rowcount
except Exception:
continue
if dt and (max_ts is None or dt > max_ts):
max_ts = dt
total = conn.execute("SELECT COUNT(*) FROM delivery_raw").fetchone()[0]
conn.execute(
"UPDATE eta_sync_state SET last_delivery_ts = ?, last_synced_at = ?, total_rows = ? WHERE id = 1",
(max_ts, datetime.utcnow().isoformat(), total),
)
conn.commit()
finally:
conn.close()
return {"fetched": len(rows), "inserted": inserted, "local_total": total,
"watermark": max_ts, "mode": "full" if full else "incremental"}
def _prune_raw(self, days: int) -> int:
"""Drop local rows older than the rolling window to bound the store."""
cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
try:
conn = sqlite3.connect(self._db_path)
cur = conn.execute("DELETE FROM delivery_raw WHERE deliverytime < ?", (cutoff,))
deleted = cur.rowcount
conn.commit()
conn.close()
return deleted
except Exception:
return 0
def _load_raw_rows(self, days: int) -> List[Dict[str, Any]]:
"""Read the local mirror within the rolling window (no DB hit)."""
cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"SELECT deliveryid, userid, pickupcustomer, pickuptime, deliverytime, dlat, dlon, plat, plon "
"FROM delivery_raw WHERE deliverytime >= ? ORDER BY deliverytime",
(cutoff,),
).fetchall()
conn.close()
cols = ["deliveryid", "userid", "pickupcustomer", "pickuptime", "deliverytime", "dlat", "dlon", "plat", "plon"]
return [dict(zip(cols, r)) for r in rows]
def sample_batches(self, days: int = 14, min_drops: int = 4, max_drops: int = 15,
limit: int = 10) -> List[Dict[str, Any]]:
"""
Real recent delivery batches from the local mirror for the road-sequencing
decision agent. Grouped by (rider, day, kitchen); origin = that kitchen's
pickup coords (centroid fallback). Most-recent batches first.
Returns [{origin, drops, rider, day, kitchen}].
"""
rows = self._load_raw_rows(days)
groups: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = defaultdict(list)
for r in rows:
dt = _parse_dt(r.get("deliverytime"))
if dt is None:
continue
try:
if not (float(r["dlat"]) and float(r["dlon"])):
continue
except (TypeError, ValueError):
continue
key = (str(r.get("userid")), dt.strftime("%Y-%m-%d"),
normalize_kitchen(r.get("pickupcustomer")))
groups[key].append(r)
batches: List[Dict[str, Any]] = []
for key in sorted(groups, key=lambda k: k[1], reverse=True):
items = groups[key]
if not (min_drops <= len(items) <= max_drops):
continue
drops = [(float(i["dlat"]), float(i["dlon"])) for i in items]
origin = None
for i in items:
try:
pla, plo = float(i["plat"]), float(i["plon"])
if pla and plo:
origin = (pla, plo)
break
except (TypeError, ValueError):
continue
if origin is None:
origin = (sum(d[0] for d in drops) / len(drops),
sum(d[1] for d in drops) / len(drops))
batches.append({"origin": origin, "drops": drops,
"rider": key[0], "day": key[1], "kitchen": key[2]})
if len(batches) >= limit:
break
return batches
def _build_observations(self, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Reconstruct per-leg observations from ordered delivery rows."""
# Group by (rider, calendar day of delivery)
groups: Dict[Tuple[str, str], List[Dict[str, Any]]] = defaultdict(list)
for r in rows:
dt = _parse_dt(r.get("deliverytime"))
pt = _parse_dt(r.get("pickuptime"))
if dt is None or pt is None:
continue
try:
dlat = float(r.get("dlat"))
dlon = float(r.get("dlon"))
except (TypeError, ValueError):
continue
if dlat == 0 or dlon == 0:
continue
groups[(str(r.get("userid")), dt.strftime("%Y-%m-%d"))].append({
"dt": dt, "pt": pt, "lat": dlat, "lon": dlon,
"kitchen": normalize_kitchen(r.get("pickupcustomer")),
"rider": str(r.get("userid")),
})
obs: List[Dict[str, Any]] = []
for _key, items in groups.items():
items.sort(key=lambda x: x["dt"])
prev = None
for i, it in enumerate(items):
zone = self._zone.determine_zone(it["lat"], it["lon"])
traffic = hour_to_traffic(it["dt"].hour)
if i == 0:
# kitchen -> first drop; distance unknown without kitchen coords
leg_min = (it["dt"] - it["pt"]).total_seconds() / 60.0
leg_km = None
is_first = 1
else:
leg_min = (it["dt"] - prev["dt"]).total_seconds() / 60.0
leg_km = _haversine_km(prev["lat"], prev["lon"], it["lat"], it["lon"])
is_first = 0
prev = it
# filter implausible legs
if leg_min <= 0 or leg_min > _MAX_LEG_MIN:
continue
if leg_km is not None and (leg_km <= 0 or leg_km > _MAX_LEG_KM):
continue
obs.append({
"delivery_ts": it["dt"].strftime("%Y-%m-%d %H:%M:%S"),
"rider_id": it["rider"],
"kitchen": it["kitchen"],
"drop_zone": zone,
"traffic": traffic,
"dist_bucket": dist_bucket(leg_km),
"leg_km": leg_km,
"leg_min": round(leg_min, 2),
"is_first": is_first,
})
return obs
@staticmethod
def _aggregate(observations: List[Dict[str, Any]]) -> Dict[str, Tuple[int, float, float]]:
"""Group observations by every key and compute (count, median, p75)."""
buckets: Dict[str, List[float]] = defaultdict(list)
for o in observations:
for k in _keys_for_observation(
o["kitchen"], o["drop_zone"], o["traffic"], o["dist_bucket"], o["rider_id"]
):
buckets[k].append(o["leg_min"])
stats: Dict[str, Tuple[int, float, float]] = {}
for k, vals in buckets.items():
vals.sort()
n = len(vals)
median = statistics.median(vals)
p75 = vals[min(n - 1, int(math.ceil(0.75 * n)) - 1)] if n else median
stats[k] = (n, round(median, 2), round(p75, 2))
return stats
def rebuild_aggregates(self, days: int = 14) -> Dict[str, Any]:
"""
Rebuild empirical medians from the LOCAL mirror only (no DB hit).
Reconstructs per-leg observations, aggregates, persists, reloads cache.
"""
rows = self._load_raw_rows(days)
observations = self._build_observations(rows)
stats = self._aggregate(observations)
now = datetime.utcnow().isoformat()
try:
conn = sqlite3.connect(self._db_path)
conn.execute("DELETE FROM delivery_history")
conn.execute("DELETE FROM eta_stats")
conn.executemany(
"INSERT INTO delivery_history "
"(delivery_ts, rider_id, kitchen, drop_zone, traffic, dist_bucket, leg_km, leg_min, is_first) "
"VALUES (?,?,?,?,?,?,?,?,?)",
[(o["delivery_ts"], o["rider_id"], o["kitchen"], o["drop_zone"],
o["traffic"], o["dist_bucket"], o["leg_km"], o["leg_min"], o["is_first"])
for o in observations],
)
conn.executemany(
"INSERT INTO eta_stats (key, key_type, sample_count, median_min, p75_min, updated_at) "
"VALUES (?,?,?,?,?,?)",
[(k, k.split("|", 1)[0], n, med, p75, now) for k, (n, med, p75) in stats.items()],
)
conn.commit()
conn.close()
except Exception as e:
logger.error(f"[DeliveryHistory] persist failed: {e}", exc_info=True)
return {"status": "persist_failed", "error": str(e)}
self._load_cache()
self._last_refreshed = datetime.utcnow()
n_by_type: Dict[str, int] = defaultdict(int)
for k in stats:
n_by_type[k.split("|", 1)[0]] += 1
summary = {
"status": "ok",
"local_rows": len(rows),
"observations": len(observations),
"stat_keys": len(stats),
"keys_by_type": dict(n_by_type),
"history_days": days,
"rebuilt_at": self._last_refreshed.isoformat(),
}
logger.info(
f"[DeliveryHistory] aggregates rebuilt: local_rows={len(rows)} "
f"obs={len(observations)} keys={len(stats)} ({dict(n_by_type)})"
)
return summary
def refresh_eta_stats(self, days: int = 14, tenant_id: int = 916,
full: bool = False) -> Dict[str, Any]:
"""
Full pipeline: incremental DB sync -> prune local store -> rebuild
aggregates locally. This is what the scheduler and the admin endpoint
call. The DB is touched only by the sync step (new rows only).
"""
with _WRITE_LOCK:
try:
sync = self.sync_from_db(days=days, tenant_id=tenant_id, full=full)
except Exception as e:
logger.error(f"[DeliveryHistory] sync failed: {e}", exc_info=True)
# Still try to serve whatever is already local.
rebuilt = self.rebuild_aggregates(days)
self._last_summary = {"status": "sync_failed", "error": str(e), "rebuild": rebuilt}
return self._last_summary
self._prune_raw(days)
rebuilt = self.rebuild_aggregates(days)
self._last_summary = {"status": "ok", "sync": sync, "rebuild": rebuilt}
return self._last_summary
# -- cache + lookup -----------------------------------------------------
def _load_cache(self) -> None:
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"SELECT key, sample_count, median_min, p75_min FROM eta_stats"
).fetchall()
conn.close()
self._cache = {k: (int(n), float(med), float(p75)) for k, n, med, p75 in rows}
if self._cache:
logger.info(f"[DeliveryHistory] loaded {len(self._cache)} eta_stats keys into cache")
except Exception as e:
logger.warning(f"[DeliveryHistory] cache load failed: {e}")
self._cache = {}
def has_data(self) -> bool:
return bool(self._cache)
def maybe_background_refresh(self, days: int = 14, tenant_id: int = 916,
cooldown_s: int = 600) -> bool:
"""
If the cache is empty, kick off a one-shot background refresh (at most
once per cooldown). Keeps the prediction hot path non-blocking — the
current call falls back to the formula; later calls use empirical data.
Returns True if a refresh thread was started.
"""
if self._cache:
return False
with self._refresh_lock:
if self._cache:
return False
now = datetime.utcnow()
if (self._refresh_attempt_at is not None
and (now - self._refresh_attempt_at).total_seconds() < cooldown_s):
return False
self._refresh_attempt_at = now
def _run():
try:
self.refresh_eta_stats(days, tenant_id)
except Exception as e:
logger.warning(f"[DeliveryHistory] background refresh failed: {e}")
threading.Thread(target=_run, daemon=True, name="eta-refresh").start()
return True
def local_count(self) -> int:
try:
conn = sqlite3.connect(self._db_path)
n = conn.execute("SELECT COUNT(*) FROM delivery_raw").fetchone()[0]
conn.close()
return int(n)
except Exception:
return 0
def ensure_background_sync(self, interval_hours: int = 6, days: int = 14,
tenant_id: int = 916) -> bool:
"""
Start the autonomous sync agent (once per process). It refreshes
immediately on startup (full backfill if the local store is empty,
otherwise incremental), then re-syncs every `interval_hours`.
Runs in a daemon thread so it never blocks the app; all request-path
ETA lookups read the local SQLite mirror, never Postgres.
"""
with self._refresh_lock:
if self._scheduler_started:
return False
self._scheduler_started = True
def _loop():
logger.info(
f"[ETA-Agent] autonomous sync started — interval={interval_hours}h, window={days}d"
)
first = True
while True:
try:
full = first and self.local_count() == 0
result = self.refresh_eta_stats(days=days, tenant_id=tenant_id, full=full)
logger.info(f"[ETA-Agent] sync cycle done: {result.get('status')}")
except Exception as e:
logger.warning(f"[ETA-Agent] sync cycle failed (will retry): {e}")
# Recompute learned rider affinity from the freshly-synced mirror.
try:
from app.services.routing.rider_affinity_service import get_rider_affinity
get_rider_affinity().refresh(days=days)
except Exception as e:
logger.debug(f"[ETA-Agent] affinity refresh skipped: {e}")
first = False
time.sleep(max(1, int(interval_hours)) * 3600)
threading.Thread(target=_loop, daemon=True, name="eta-sync-agent").start()
return True
def lookup(
self,
distance_km: float,
traffic_cat: str,
kitchen: Optional[str] = None,
drop_coords: Optional[Tuple[float, float]] = None,
min_samples: int = 20,
stat: str = "median",
) -> Optional[Dict[str, Any]]:
"""
Return the empirical leg time (minutes) for a context, or None if no key
has >= min_samples. Walks the specificity hierarchy.
"""
if not self._cache:
return None
zone = "Unknown"
if drop_coords and drop_coords[0] and drop_coords[1]:
zone = self._zone.determine_zone(float(drop_coords[0]), float(drop_coords[1]))
bucket = dist_bucket(distance_km) if distance_km and distance_km > 0 else None
k_norm = normalize_kitchen(kitchen) if kitchen else None
for full_key in _lookup_keys(k_norm, zone, traffic_cat, bucket):
hit = self._cache.get(full_key)
if hit and hit[0] >= min_samples:
count, median, p75 = hit
value = p75 if stat == "p75" else median
return {
"value_min": value,
"sample_count": count,
"source_key": full_key,
"key_type": full_key.split("|", 1)[0],
}
return None
# -- diagnostics / backtest --------------------------------------------
def get_summary(self) -> Dict[str, Any]:
return {
"has_data": self.has_data(),
"cache_keys": len(self._cache),
"last_summary": self._last_summary,
}
def backtest(self, days: int = 14, tenant_id: int = 916,
min_samples: int = 20, stat: str = "median") -> Dict[str, Any]:
"""
Honest time-split backtest: build empirical stats on the older 80% of
observations, then compare formula vs empirical MAE on the most recent
20% (held out). Proves whether empirical beats the formula before trust.
"""
from app.services.routing.realistic_eta_calculator import RealisticETACalculator
formula = RealisticETACalculator()
# Backtest runs entirely on the local mirror — no DB hit.
rows = self._load_raw_rows(days)
obs = self._build_observations(rows)
obs = [o for o in obs if o["is_first"] == 0] # need leg_km for both predictors
if len(obs) < 50:
return {"status": "insufficient_data", "observations": len(obs)}
obs.sort(key=lambda o: o["delivery_ts"])
split = int(len(obs) * 0.8)
train, test = obs[:split], obs[split:]
train_stats = self._aggregate(train)
def _empirical(o) -> Optional[float]:
for full_key in _lookup_keys(o["kitchen"], o["drop_zone"], o["traffic"], o["dist_bucket"]):
hit = train_stats.get(full_key)
if hit and hit[0] >= min_samples:
return hit[2] if stat == "p75" else hit[1]
return None
f_err, e_err, e_err_fallback, covered = [], [], [], 0
for o in test:
actual = o["leg_min"]
f_pred = formula.calculate_eta(
distance_km=o["leg_km"], is_first_order=False,
order_type="Economy", time_of_day=o["traffic"],
)
f_err.append(abs(f_pred - actual))
emp = _empirical(o)
if emp is not None:
covered += 1
e_err.append(abs(emp - actual))
e_err_fallback.append(abs(emp - actual))
else:
e_err_fallback.append(abs(f_pred - actual)) # fallback to formula
def _mae(xs):
return round(sum(xs) / len(xs), 2) if xs else None
return {
"status": "ok",
"history_days": days,
"observations": len(obs),
"test_size": len(test),
"empirical_coverage_pct": round(100.0 * covered / len(test), 1),
"formula_mae_min": _mae(f_err),
"empirical_mae_min_covered": _mae(e_err),
"empirical_mae_min_with_fallback": _mae(e_err_fallback),
"min_samples": min_samples,
"stat": stat,
"interpretation": (
"empirical_better"
if (_mae(e_err_fallback) is not None and _mae(f_err) is not None
and _mae(e_err_fallback) < _mae(f_err))
else "no_improvement"
),
}
# ---------------------------------------------------------------------------
# Singleton
# ---------------------------------------------------------------------------
_service: Optional[DeliveryHistoryService] = None
_service_lock = threading.Lock()
def get_delivery_history_service() -> DeliveryHistoryService:
"""Get (and lazily build) the DeliveryHistoryService singleton."""
global _service
with _service_lock:
if _service is None:
_service = DeliveryHistoryService()
return _service