Initial commit

.dockerignore

@@ -0,0 +1,58 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Documentation
+*.md
+!README.md
+
+# Environment
+.env
+.env.local
+
+# Logs
+*.log
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+
+# Docker
+Dockerfile
+docker-compose.yml
+.dockerignore
+
+# Test files
+test_*.py
+*_test.py
+
+# Temporary files
+*.tmp
+*.bak
+

.env

@@ -0,0 +1,9 @@
+REDIS_PASSWORD=Package@324969#
+GOOGLE_MAPS_API_KEY=AIzaSyBhkGfnq27sN0wV5y_S-M2KojpFTk_by-Q
+
+# nearledb (read-only source for empirical ETA — `deliveries` table only)
+DB_HOST=66.116.207.225
+DB_PORT=6432
+DB_NAME=nearledb
+DB_USER=admin
+DB_PASSWORD=Package@123#

Dockerfile

@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+# Install dependencies first
+COPY requirements.txt ./
+RUN pip install --upgrade pip \
+    && pip install -r requirements.txt
+
+# Copy application code
+COPY app ./app
+COPY start.py ./start.py
+COPY docker-entrypoint.sh ./docker-entrypoint.sh
+
+# Make entrypoint executable
+RUN chmod +x docker-entrypoint.sh
+
+EXPOSE 8002
+
+ENTRYPOINT ["./docker-entrypoint.sh"]

app/__init__.py

@@ -0,0 +1 @@
+# Delivery Route Optimization API

app/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration package for mobile delivery optimization."""

app/config/dynamic_config.py

@@ -0,0 +1,408 @@
+"""
+Dynamic Configuration - rider-api
+
+Replaces all hardcoded hyperparameters with DB-backed values.
+The ML hypertuner writes optimal values here; services read from here.
+
+Fallback: If DB is unavailable or no tuned values exist, defaults are used.
+This means zero risk - the system works day 1 with no data.
+"""
+
+import contextvars
+import json
+import logging
+import os
+import sqlite3
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# --- DB Path ------------------------------------------------------------------
+_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
+
+# ---------------------------------------------------------------------------
+# Per-request strategy override (async-safe via contextvars).
+# Each FastAPI request/asyncio task gets its own copy — no cross-request leaks.
+# Usage:
+#   set_request_strategy("fuel_saver")   → override active for this request
+#   clear_request_strategy()             → restore to DB value
+# ---------------------------------------------------------------------------
+_strategy_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
+    "strategy_override", default=None
+)
+
+
+def set_request_strategy(strategy: Optional[str]) -> None:
+    """Override ml_strategy for the current async task only (thread-safe)."""
+    _strategy_override.set(strategy)
+
+
+def clear_request_strategy() -> None:
+    """Remove the per-request strategy override for the current async task."""
+    _strategy_override.set(None)
+
+
+# --- Hard Defaults (What the system used before ML) ---------------------------
+DEFAULTS: Dict[str, Any] = {
+    # System Strategy / Prompt
+    "ml_strategy": "balanced",
+    # AssignmentService
+    "max_pickup_distance_km": 10.0,
+    "max_kitchen_distance_km": 3.0,
+    "max_orders_per_rider": 12,
+    "ideal_load": 6,
+    "workload_balance_threshold": 0.7,
+    "workload_penalty_weight": 100.0,
+    "distance_penalty_weight": 2.0,
+    "preference_bonus": -15.0,
+    "home_zone_bonus_4km": -3.0,
+    "home_zone_bonus_2km": -5.0,
+    "emergency_load_penalty": 3.0,  # km penalty per order in emergency assign
+    # RouteOptimizer
+    "search_time_limit_seconds": 5,
+    "avg_speed_kmh": 18.0,
+    "road_factor": 1.3,
+    # ClusteringService
+    "cluster_radius_km": 3.0,
+    # KalmanFilter
+    "kalman_process_noise": 1e-4,
+    "kalman_measurement_noise": 0.01,
+    # RealisticETACalculator
+    "eta_pickup_time_min": 3.0,
+    "eta_delivery_time_min": 4.0,
+    "eta_navigation_buffer_min": 1.5,
+    "eta_short_trip_factor": 0.8,  # speed multiplier for dist < 2km
+    "eta_long_trip_factor": 1.1,  # speed multiplier for dist > 8km
+    # EmpiricalETACalculator (learned ETAs from actual delivery times)
+    "eta_empirical_enabled": True,  # False -> instantly revert to the formula
+    "eta_min_samples": 10,          # min history samples a key needs before trust
+                                    # (backtest on live 14d data: 10 -> MAE 4.85 vs
+                                    #  formula 5.73; 20 -> 5.64. 10 wins on held-out.)
+    "eta_history_days": 14,         # rolling window pulled from nearledb
+    "eta_stat": "median",           # "median" or "p75" (p75 = more conservative)
+    "eta_sync_interval_hours": 6,   # autonomous background sync cadence
+    # Road-aware sequencing (Phase 2). OFF by default: enabling adds a Google
+    # Directions call (cost + latency) to the route hot path. Results are cached.
+    # Only the *visiting order* changes; step/ETA metrics stay aerial-based.
+    "routing_use_road_distance": False,       # AGENT-MANAGED (see routing_auto_manage)
+    "routing_road_cache_ttl_seconds": 86400,  # road geometry is stable; cache 24h
+    "routing_road_max_stops": 25,             # Google distance-matrix practical cap
+    # Autonomous road-sequencing decision agent: measures road-vs-aerial travel
+    # time on real batches and flips routing_use_road_distance on its own.
+    "routing_auto_manage": True,              # False -> humans own the flag
+    "routing_auto_enable_gain_pct": 3.0,      # enable when mean gain >= this
+    "routing_auto_disable_gain_pct": 1.0,     # disable when mean gain < this (hysteresis)
+    "routing_eval_sample_batches": 8,         # batches measured per cycle (cost bound)
+    "routing_eval_min_batches": 3,            # need >= this evaluated to decide
+    "routing_eval_interval_hours": 24,        # decision cadence
+    "routing_eval_days": 14,                  # window sampled from the local mirror
+    # Learned rider->kitchen affinity (soft steering only; union with curated config).
+    "rider_affinity_enabled": True,           # False -> pure curated config
+    "rider_affinity_min_deliveries": 10,      # learned owner needs >= this many deliveries
+    "rider_affinity_refresh_hours": 6,        # recompute cadence (piggybacks the agent)
+}
+
+
+class DynamicConfig:
+    """
+    Thread-safe, DB-backed configuration store.
+
+    Usage:
+        cfg = DynamicConfig()
+        max_dist = cfg.get("max_pickup_distance_km")
+        all_params = cfg.get_all()
+    """
+
+    _instance: Optional["DynamicConfig"] = None
+
+    def __new__(cls) -> "DynamicConfig":
+        """Singleton - one config per process."""
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        self._initialized = True
+        self._cache: Dict[str, Any] = {}
+        self._last_loaded: Optional[datetime] = None
+        self._ensure_db()
+        self._load()
+
+    # --------------------------------------------------------------------------
+    # Public API
+    # --------------------------------------------------------------------------
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get a config value. Returns ML-tuned value if available, else default.
+
+        For 'ml_strategy' specifically, a per-request ContextVar override takes
+        precedence so that hypertuning_params requests don't mutate the shared
+        singleton (thread-safe for concurrent FastAPI requests).
+        """
+        self._maybe_reload()
+        # Per-request strategy override (async-safe, no cross-request leaks)
+        if key == "ml_strategy":
+            override = _strategy_override.get()
+            if override is not None:
+                return override
+        val = self._cache.get(key)
+        if val is not None:
+            return val
+        fallback = default if default is not None else DEFAULTS.get(key)
+        return fallback
+
+    def get_all(self) -> Dict[str, Any]:
+        """Return all current config values (ML-tuned + defaults for missing keys)."""
+        self._maybe_reload()
+        result = dict(DEFAULTS)
+        result.update(self._cache)
+        return result
+
+    def set(self, key: str, value: Any, source: str = "manual") -> None:
+        """Write a config value to DB (used by hypertuner)."""
+        try:
+            os.makedirs(os.path.dirname(_DB_PATH) or ".", exist_ok=True)
+            conn = sqlite3.connect(_DB_PATH)
+            conn.execute(
+                """
+                INSERT INTO dynamic_config (key, value, source, updated_at)
+                VALUES (?, ?, ?, ?)
+                ON CONFLICT(key) DO UPDATE SET
+                    value=excluded.value,
+                    source=excluded.source,
+                    updated_at=excluded.updated_at
+            """,
+                (key, json.dumps(value), source, datetime.utcnow().isoformat()),
+            )
+            conn.commit()
+            conn.close()
+            self._cache[key] = value
+            logger.info(f"[DynamicConfig] Set {key}={value} (source={source})")
+        except Exception as e:
+            logger.error(f"[DynamicConfig] Failed to set {key}: {e}")
+
+    def set_bulk(self, params: Dict[str, Any], source: str = "ml_hypertuner") -> None:
+        """Write multiple config values at once (called after each Optuna study)."""
+        for key, value in params.items():
+            self.set(key, value, source=source)
+        logger.info(f"[DynamicConfig] Bulk update: {len(params)} params from {source}")
+
+    def reset_to_defaults(self) -> None:
+        """Wipe all ML-tuned values, revert to hardcoded defaults."""
+        try:
+            conn = sqlite3.connect(_DB_PATH)
+            conn.execute("DELETE FROM dynamic_config")
+            conn.commit()
+            conn.close()
+            self._cache.clear()
+            logger.warning("[DynamicConfig] Reset to factory defaults.")
+        except Exception as e:
+            logger.error(f"[DynamicConfig] Reset failed: {e}")
+
+    # --------------------------------------------------------------------------
+    # Internal
+    # --------------------------------------------------------------------------
+
+    def _ensure_db(self) -> None:
+        try:
+            os.makedirs(os.path.dirname(_DB_PATH) or ".", exist_ok=True)
+            conn = sqlite3.connect(_DB_PATH)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS dynamic_config (
+                    key        TEXT PRIMARY KEY,
+                    value      TEXT NOT NULL,
+                    source     TEXT DEFAULT 'manual',
+                    updated_at TEXT
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS kitchen_encoding (
+                    kitchen_name TEXT PRIMARY KEY,
+                    label_id     INTEGER NOT NULL,
+                    frequency    REAL DEFAULT 0.0,
+                    avg_profit   REAL DEFAULT 0.0,
+                    order_count  INTEGER DEFAULT 0,
+                    updated_at   TEXT
+                )
+            """)
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            logger.error(f"[DynamicConfig] DB init failed: {e}")
+
+    def _load(self) -> None:
+        try:
+            conn = sqlite3.connect(_DB_PATH)
+            rows = conn.execute("SELECT key, value FROM dynamic_config").fetchall()
+            conn.close()
+            self._cache = {}
+            for key, raw in rows:
+                try:
+                    self._cache[key] = json.loads(raw)
+                except Exception:
+                    self._cache[key] = raw
+            self._last_loaded = datetime.utcnow()
+            if self._cache:
+                logger.info(
+                    f"[DynamicConfig] Loaded {len(self._cache)} ML-tuned params from DB"
+                )
+        except Exception as e:
+            logger.warning(
+                f"[DynamicConfig] Could not load from DB (using defaults): {e}"
+            )
+            self._cache = {}
+
+    def _maybe_reload(self, interval_seconds: int = 300) -> None:
+        """Reload from DB every 5 minutes - picks up new tuned params without restart."""
+        if self._last_loaded is None:
+            self._load()
+            return
+        delta = (datetime.utcnow() - self._last_loaded).total_seconds()
+        if delta > interval_seconds:
+            self._load()
+
+
+# --- Module-level convenience singleton ---------------------------------------
+_cfg = DynamicConfig()
+
+
+def get_config() -> DynamicConfig:
+    """Get the global DynamicConfig singleton."""
+    return _cfg
+
+
+__all__ = [
+    "DynamicConfig",
+    "get_config",
+    "set_request_strategy",
+    "clear_request_strategy",
+    "get_kitchen_label_id",
+    "get_kitchen_frequency",
+    "update_kitchen_stats",
+    "get_kitchen_avg_profit_smoothed",
+]
+
+
+# --- Kitchen Encoding Persistence ---------------------------------------------
+def get_kitchen_label_id(kitchen_name: str) -> int:
+    """Get or create persistent label ID for a kitchen."""
+    try:
+        conn = sqlite3.connect(_DB_PATH)
+        row = conn.execute(
+            "SELECT label_id FROM kitchen_encoding WHERE kitchen_name = ?",
+            (kitchen_name,),
+        ).fetchone()
+        if row:
+            conn.close()
+            return row[0]
+
+        max_id = conn.execute(
+            "SELECT COALESCE(MAX(label_id), -1) FROM kitchen_encoding"
+        ).fetchone()[0]
+        new_id = max_id + 1
+        conn.close()
+        return new_id
+    except Exception as e:
+        logger.warning(f"[KitchenEncoding] Failed to get label_id: {e}")
+        return hash(kitchen_name.lower().strip()) % 10000
+
+
+def get_kitchen_frequency(kitchen_name: str) -> float:
+    """Get frequency ratio for a kitchen from DB."""
+    try:
+        conn = sqlite3.connect(_DB_PATH)
+        row = conn.execute(
+            "SELECT frequency FROM kitchen_encoding WHERE kitchen_name = ?",
+            (kitchen_name,),
+        ).fetchone()
+        conn.close()
+        return row[0] if row else 0.0
+    except Exception:
+        return 0.0
+
+
+def update_kitchen_stats(kitchen_name: str, profit: float):
+    """Update kitchen stats after order completion."""
+    try:
+        conn = sqlite3.connect(_DB_PATH)
+        row = conn.execute(
+            "SELECT order_count, avg_profit FROM kitchen_encoding WHERE kitchen_name = ?",
+            (kitchen_name,),
+        ).fetchone()
+
+        if row:
+            count, avg = row
+            new_count = count + 1
+            new_avg = ((avg * count) + profit) / new_count
+            new_freq = new_count / (
+                conn.execute(
+                    "SELECT SUM(order_count) FROM kitchen_encoding"
+                ).fetchone()[0]
+                or 1
+            )
+            conn.execute(
+                "UPDATE kitchen_encoding SET order_count = ?, avg_profit = ?, frequency = ?, updated_at = ? WHERE kitchen_name = ?",
+                (
+                    new_count,
+                    new_avg,
+                    new_freq,
+                    datetime.utcnow().isoformat(),
+                    kitchen_name,
+                ),
+            )
+        else:
+            conn.execute(
+                "INSERT INTO kitchen_encoding (kitchen_name, label_id, frequency, avg_profit, order_count, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
+                (
+                    kitchen_name,
+                    get_kitchen_label_id(kitchen_name),
+                    1.0,
+                    profit,
+                    1,
+                    datetime.utcnow().isoformat(),
+                ),
+            )
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        logger.warning(f"[KitchenEncoding] Failed to update stats: {e}")
+
+
+def get_kitchen_avg_profit_smoothed(
+    kitchen_name: str, global_avg: float = 40.0, min_samples: int = 5
+) -> float:
+    """
+    Get smoothed average profit for a kitchen using Bayesian smoothing.
+    Reduces noise for kitchens with few orders.
+
+    Formula: smoothed = (kitchen_count * kitchen_avg + min_samples * global_avg) / (kitchen_count + min_samples)
+
+    This means:
+    - Kitchen with many samples -> uses its own avg
+    - Kitchen with few samples -> pulls toward global avg
+    """
+    try:
+        conn = sqlite3.connect(_DB_PATH)
+        row = conn.execute(
+            "SELECT order_count, avg_profit FROM kitchen_encoding WHERE kitchen_name = ?",
+            (kitchen_name,),
+        ).fetchone()
+        conn.close()
+
+        if row and row[0] > 0:
+            count, avg = row
+            if count >= min_samples:
+                return avg
+            # Bayesian smoothing
+            smoothed = ((count * avg) + (min_samples * global_avg)) / (
+                count + min_samples
+            )
+            return smoothed
+
+        return global_avg  # Unknown kitchen defaults to global avg
+    except Exception:
+        return global_avg

app/config/mobile_config.py

@@ -0,0 +1,33 @@
+"""Mobile-specific configuration for delivery route optimization."""
+
+# Mobile optimization settings
+MOBILE_CONFIG = {
+    "default_algorithm": "greedy",
+    "max_deliveries": 100,
+    "timeout_seconds": 5,
+    "response_compression": True,
+    "performance_monitoring": True,
+    "mobile_headers": True
+}
+
+# Performance targets for mobile
+PERFORMANCE_TARGETS = {
+    "greedy_algorithm": {
+        "max_response_time": 0.1,  # 100ms
+        "max_deliveries": 50,
+        "description": "Ultra-fast for real-time mobile apps"
+    },
+    "tsp_algorithm": {
+        "max_response_time": 3.0,  # 3 seconds
+        "max_deliveries": 30,
+        "description": "Optimal but slower, good for planning"
+    }
+}
+
+# Mobile app recommendations
+MOBILE_RECOMMENDATIONS = {
+    "real_time_delivery": "greedy",
+    "route_planning": "tsp",
+    "large_batches": "greedy",
+    "cost_optimization": "tsp"
+}

app/config/rider_preferences.py

@@ -0,0 +1,51 @@
+"""
+Rider Preferred Kitchens Configuration
+Mapping of Rider ID (int) to list of preferred Kitchen names (str).
+Updated based on Deployment Plan.
+"""
+
+RIDER_PREFERRED_KITCHENS = {
+    # 1. VIGNESH - RS PURAM, TOWNHALL, PODANUR, KUNIAMUTHUR
+    1036: ["Bhuvaneshwari kitchen"],
+
+    # 2. VARUN EDWARD - KAVUNDAMPALAYAM, THUDIYALUR
+    897: ["Daily grubs(jayanthi kitchen)"],
+
+    # 3. JAYASABESH - GANAPATHY, SARAVANAMPATTI
+    950: ["Daily grubs nandhini"],
+
+    # 4. TAMILAZHAGAN - GANDHIMA NAGAR, NEHRU NAGAR, SITRA
+    1114: ["Daily grubs nandhini"],
+
+    # 5. RAJAN - PEELAMEDU, P N PALAYAM
+    883: ["Daily grubs nandhini", "Vidhya kitchen"],
+
+    # 6. MANIKANDAN - SINGANALLUR
+    753: ["Vidhya kitchen"],
+
+    # 7. NAGALAKSHMI - SAIBABA COLONY, R S PURAM, GANDHIPURAM
+    1062: ["Daily grubs(jayanthi kitchen)"],
+
+    # 8. MURALI - RAMANATHAPURAM, PODANUR, RACE COURSE
+    1111: ["Vidhya kitchen"],
+}
+
+# ---------------------------------------------------------------------------
+# Blocked / test rider IDs — SINGLE source of truth.
+# Import this set wherever blocked-rider filtering is needed instead of
+# hard-coding the list in multiple files.
+# ---------------------------------------------------------------------------
+BLOCKED_RIDERS: frozenset = frozenset({1242, 1266, 1245, 1232, 1240, 1007})
+
+# Anchor Coordinates for Riders (Based on Area Name)
+# Used as fallback if GPS is missing, or to bias assignment to their Home Zone.
+RIDER_HOME_LOCATIONS = {
+    1036: (11.0067, 76.9558), # VIGNESH: RS PURAM, TOWNHALL, PODANUR, KUNIAMUTHUR
+    897:  (11.0430, 76.9380), # VARUN EDWARD: KAVUNDAMPALAYAM, THUDIYALUR
+    950:  (11.0330, 76.9800), # JAYASABESH: GANAPATHY, SARAVANAMPATTI
+    1114: (11.0450, 77.0000), # TAMILAZHAGAN: GANDHIMA NAGAR, NEHRU NAGAR, SITRA
+    883:  (11.0200, 77.0000), # RAJAN: PEELAMEDU, P N PALAYAM
+    753:  (11.0000, 77.0300), # MANIKANDAN: SINGANALLUR
+    1062: (11.0250, 76.9450), # NAGALAKSHMI: SAIBABA COLONY, R S PURAM, GANDHIPURAM
+    1111: (10.9950, 77.0000), # MURALI: RAMANATHAPURAM, PODANUR, RACE COURSE
+}

app/controllers/__init__.py

@@ -0,0 +1,5 @@
+"""Controllers package."""
+
+from .route_controller import RouteController
+
+__all__ = ["RouteController"]

app/controllers/route_controller.py

@@ -0,0 +1,104 @@
+"""Controller for provider payload optimization and forwarding."""
+
+import logging
+import hashlib
+import json
+from typing import Dict, Any
+import httpx
+from fastapi import HTTPException
+
+from app.core.exceptions import ValidationError, APIException
+from app.services.routing.route_optimizer import RouteOptimizer
+from app.services import cache
+
+logger = logging.getLogger(__name__)
+
+
+class RouteController:
+    """Controller for optimizing provider payloads and forwarding upstream."""
+    
+    def __init__(self):
+        self.route_optimizer = RouteOptimizer()
+    
+    def _hash_key(self, prefix: str, payload: Dict[str, Any]) -> str:
+        """Create a stable cache key from a dict payload."""
+        # ensure deterministic json by sorting keys
+        serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+        digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest()
+        return f"routes:{prefix}:{digest}"
+    
+    async def optimize_and_forward_provider_payload(self, orders: list[dict], forward_url: str) -> dict:
+        """Optimize provider payload and return it (forwarding paused).
+        
+        - Input: list of provider orders (dicts)
+        - Output: {code, details, message, status} where details is the optimized array
+        """
+        try:
+            if not isinstance(orders, list) or not orders:
+                raise ValidationError("Orders array is required", field="body")
+
+            # Create a cache key from the orders payload
+            cache_key = self._hash_key("provider_payload", {"orders": orders})
+            
+            # Check cache
+            cached_result = cache.get_json(cache_key)
+            if cached_result:
+                logger.info(f"Cache hit for key: {cache_key}")
+                return cached_result
+
+            optimized = await self.route_optimizer.optimize_provider_payload(orders)
+
+            # Debug sample of optimized payload (first 3 items, select keys)
+            try:
+                sample = [
+                    {
+                        k: item.get(k)
+                        for k in ("orderheaderid", "orderid", "deliverycustomerid", "step", "previouskms", "cumulativekms", "eta")
+                    }
+                    for item in optimized[:3]
+                ]
+                logger.debug(f"Optimized payload sample: {sample}")
+                trace = [
+                    {
+                        "orderid": item.get("orderid"),
+                        "step": item.get("step"),
+                        "prev": item.get("previouskms"),
+                        "cum": item.get("cumulativekms"),
+                    }
+                    for item in optimized
+                ]
+                logger.debug(f"Optimized order trace: {trace}")
+            except Exception:
+                logger.debug("Optimized payload sample logging failed")
+
+            # Forwarding paused: return optimized payload directly
+            result = {
+                "code": 200,
+                "details": optimized,
+                "message": "Success",
+                "status": True,
+            }
+            
+            # Store in cache
+            try:
+                cache.set_json(cache_key, result)
+            except Exception as e:
+                logger.warning(f"Failed to cache optimization result: {e}")
+
+            return result
+        except ValidationError:
+            raise
+        except httpx.HTTPStatusError as e:
+            status_code = e.response.status_code
+            body_text = e.response.text
+            logger.error(f"Forwarding failed: {status_code} - {body_text}")
+            # Surface upstream details to the client for faster debugging
+            raise APIException(
+                status_code=502,
+                message=f"Upstream service error (status {status_code}): {body_text}",
+                code="UPSTREAM_ERROR"
+            )
+        except Exception as e:
+            logger.error(f"Error optimizing/forwarding provider payload: {e}", exc_info=True)
+            raise APIException(status_code=500, message="Internal server error", code="INTERNAL_ERROR")
+# Batch routes removed - use single-route optimization for each pickup location

app/core/__init__.py

@@ -0,0 +1,2 @@
+"""Core application components."""
+

app/core/arrow_utils.py

@@ -0,0 +1,63 @@
+"""
+High-performance utilities using Apache Arrow and NumPy for geographic data.
+Provides vectorized operations for distances and coordinate processing.
+"""
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+import logging
+from typing import List, Dict, Any, Tuple
+
+logger = logging.getLogger(__name__)
+
+def calculate_haversine_matrix_vectorized(lats: np.ndarray, lons: np.ndarray) -> np.ndarray:
+    """
+    Calculate an N x N distance matrix using the Haversine formula.
+    Fully vectorized using NumPy for O(N^2) speed improvement over Python loops.
+    """
+    # Earth's radius in kilometers
+    R = 6371.0
+    
+    # Convert degrees to radians
+    lats_rad = np.radians(lats)
+    lons_rad = np.radians(lons)
+    
+    # Create meshgrids for pairwise differences
+    # lats.reshape(-1, 1) creates a column vector
+    # lats.reshape(1, -1) creates a row vector
+    # Subtracting them creates an N x N matrix of differences
+    dlat = lats_rad.reshape(-1, 1) - lats_rad.reshape(1, -1)
+    dlon = lons_rad.reshape(-1, 1) - lons_rad.reshape(1, -1)
+    
+    # Haversine formula
+    a = np.sin(dlat / 2)**2 + np.cos(lats_rad.reshape(-1, 1)) * np.cos(lats_rad.reshape(1, -1)) * np.sin(dlon / 2)**2
+    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
+    
+    return R * c
+
+def orders_to_arrow_table(orders: List[Dict[str, Any]]) -> pa.Table:
+    """
+    Convert a list of order dictionaries to an Apache Arrow Table.
+    This enables zero-copy operations and efficient columnar storage.
+    """
+    return pa.Table.from_pylist(orders)
+
+def save_optimized_route_parquet(orders: List[Dict[str, Any]], filename: str):
+    """
+    Save optimized route data to a Parquet file for high-speed analysis.
+    Useful for logging and historical simulation replays.
+    """
+    try:
+        table = orders_to_arrow_table(orders)
+        pq.write_table(table, filename)
+        logger.info(f" Saved route data to Parquet: {filename}")
+    except Exception as e:
+        logger.error(f" Failed to save Parquet: {e}")
+
+def load_route_parquet(filename: str) -> List[Dict[str, Any]]:
+    """
+    Load route data from a Parquet file and return as a list of dicts.
+    """
+    table = pq.read_table(filename)
+    return table.to_pylist()

app/core/constants.py

@@ -0,0 +1,26 @@
+"""API constants and configuration."""
+
+# API Configuration
+API_VERSION = "2.0.0"
+API_TITLE = "Route Optimization API"
+API_DESCRIPTION = "Professional API for delivery route optimization"
+
+# Route Optimization Limits
+MAX_DELIVERIES = 50
+MIN_DELIVERIES = 1
+
+# Coordinate Validation
+MIN_LATITUDE = -90
+MAX_LATITUDE = 90
+MIN_LONGITUDE = -180
+MAX_LONGITUDE = 180
+
+# Algorithm Types
+ALGORITHM_GREEDY = "greedy"
+ALGORITHM_TSP = "tsp"
+
+# Response Messages
+MESSAGE_SUCCESS = "Route optimized successfully"
+MESSAGE_VALIDATION_ERROR = "Request validation failed"
+MESSAGE_INTERNAL_ERROR = "An unexpected error occurred"
+

app/core/exception_handlers.py

@@ -0,0 +1,112 @@
+"""Professional exception handlers for the API."""
+
+import logging
+from fastapi import Request, status
+from fastapi.responses import JSONResponse
+from fastapi.exceptions import RequestValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from app.core.exceptions import APIException
+from app.models.errors import ErrorResponse, ErrorDetail
+
+logger = logging.getLogger(__name__)
+
+
+async def api_exception_handler(request: Request, exc: APIException) -> JSONResponse:
+    """Handle custom API exceptions."""
+    request_id = getattr(request.state, "request_id", None)
+    
+    error_response = ErrorResponse(
+        success=False,
+        error=ErrorDetail(
+            field=exc.field,
+            message=exc.message,
+            code=exc.code
+        ),
+        path=request.url.path,
+        request_id=request_id
+    )
+    
+    logger.warning(f"API Exception: {exc.code} - {exc.message} (Request ID: {request_id})")
+    
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=error_response.model_dump(exclude_none=True)
+    )
+
+
+async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
+    """Handle HTTP exceptions."""
+    request_id = getattr(request.state, "request_id", None)
+    
+    error_response = ErrorResponse(
+        success=False,
+        error=ErrorDetail(
+            message=exc.detail,
+            code="HTTP_ERROR"
+        ),
+        path=request.url.path,
+        request_id=request_id
+    )
+    
+    logger.warning(f"HTTP Exception: {exc.status_code} - {exc.detail} (Request ID: {request_id})")
+    
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=error_response.model_dump(exclude_none=True)
+    )
+
+
+async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+    """Handle validation errors with detailed field information."""
+    request_id = getattr(request.state, "request_id", None)
+    
+    errors = exc.errors()
+    if errors:
+        first_error = errors[0]
+        field = ".".join(str(loc) for loc in first_error.get("loc", []))
+        message = first_error.get("msg", "Validation error")
+    else:
+        field = None
+        message = "Validation error"
+    
+    error_response = ErrorResponse(
+        success=False,
+        error=ErrorDetail(
+            field=field,
+            message=message,
+            code="VALIDATION_ERROR"
+        ),
+        path=request.url.path,
+        request_id=request_id
+    )
+    
+    logger.warning(f"Validation Error: {message} (Field: {field}, Request ID: {request_id})")
+    
+    return JSONResponse(
+        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content=error_response.model_dump(exclude_none=True)
+    )
+
+
+async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Handle unexpected exceptions."""
+    request_id = getattr(request.state, "request_id", None)
+    
+    error_response = ErrorResponse(
+        success=False,
+        error=ErrorDetail(
+            message="An unexpected error occurred. Please try again later.",
+            code="INTERNAL_SERVER_ERROR"
+        ),
+        path=request.url.path,
+        request_id=request_id
+    )
+    
+    logger.error(f"Unexpected Error: {str(exc)} (Request ID: {request_id})", exc_info=True)
+    
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content=error_response.model_dump(exclude_none=True)
+    )
+

app/core/exceptions.py

@@ -0,0 +1,70 @@
+"""Custom exceptions for the API."""
+
+from fastapi import HTTPException, status
+
+
+class APIException(HTTPException):
+    """Base API exception with structured error format."""
+    
+    def __init__(
+        self,
+        status_code: int,
+        message: str,
+        field: str = None,
+        code: str = None,
+        detail: str = None
+    ):
+        self.message = message
+        self.field = field
+        self.code = code or self._get_default_code(status_code)
+        super().__init__(status_code=status_code, detail=detail or message)
+    
+    def _get_default_code(self, status_code: int) -> str:
+        """Get default error code based on status code."""
+        codes = {
+            400: "BAD_REQUEST",
+            401: "UNAUTHORIZED",
+            403: "FORBIDDEN",
+            404: "NOT_FOUND",
+            409: "CONFLICT",
+            422: "VALIDATION_ERROR",
+            429: "RATE_LIMIT_EXCEEDED",
+            500: "INTERNAL_SERVER_ERROR",
+            503: "SERVICE_UNAVAILABLE"
+        }
+        return codes.get(status_code, "UNKNOWN_ERROR")
+
+
+class ValidationError(APIException):
+    """Validation error exception."""
+    
+    def __init__(self, message: str, field: str = None):
+        super().__init__(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            message=message,
+            field=field,
+            code="VALIDATION_ERROR"
+        )
+
+
+class NotFoundError(APIException):
+    """Resource not found exception."""
+    
+    def __init__(self, message: str = "Resource not found"):
+        super().__init__(
+            status_code=status.HTTP_404_NOT_FOUND,
+            message=message,
+            code="NOT_FOUND"
+        )
+
+
+class RateLimitError(APIException):
+    """Rate limit exceeded exception."""
+    
+    def __init__(self, message: str = "Rate limit exceeded"):
+        super().__init__(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            message=message,
+            code="RATE_LIMIT_EXCEEDED"
+        )
+

app/core/log_context.py

@@ -0,0 +1,30 @@
+"""
+Per-request logging context.
+
+Carries the request id (the same one `RequestIDMiddleware` puts on the
+`X-Request-ID` header) into every log record via a ContextVar + logging.Filter,
+so a whole request's log lines can be grepped by one id — essential when
+watching the live API.
+"""
+
+import contextvars
+import logging
+
+# Default "-" so non-request logs (startup, background agents) still format cleanly.
+_request_id: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="-")
+
+
+def set_request_id(request_id: str) -> None:
+    _request_id.set(request_id or "-")
+
+
+def get_request_id() -> str:
+    return _request_id.get()
+
+
+class RequestIdFilter(logging.Filter):
+    """Injects `request_id` onto every LogRecord so the format string can use it."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.request_id = _request_id.get()
+        return True

app/main.py

@@ -0,0 +1,192 @@
+"""FastAPI application — Daily Grubs Rider Dispatch API."""
+
+import logging
+import os
+import sys
+import time
+from contextlib import asynccontextmanager
+
+# Load .env (DB_*, REDIS_*, GOOGLE_MAPS_API_KEY) into the environment early,
+# before any service reads os.getenv at import time.
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except Exception:
+    pass
+
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from app.core.exception_handlers import (
+    api_exception_handler,
+    general_exception_handler,
+    http_exception_handler,
+    validation_exception_handler,
+)
+from app.core.exceptions import APIException
+from app.middleware.request_id import RequestIDMiddleware
+from app.routes import cache_router, health_router, ml_router, ml_web_router, optimization_router, batch_analytics_router, riders_router
+
+# ---------------------------------------------------------------------------
+# Logging
+# ---------------------------------------------------------------------------
+from app.core.log_context import RequestIdFilter
+
+_log_level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
+_handler = logging.StreamHandler(sys.stdout)
+_handler.addFilter(RequestIdFilter())
+logging.basicConfig(
+    level=_log_level,
+    format="%(asctime)s - %(name)s - %(levelname)s - [req:%(request_id)s] - [%(filename)s:%(lineno)d] - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    handlers=[_handler],
+)
+logger = logging.getLogger(__name__)
+for _lib in ("httpx", "uvicorn", "uvicorn.error", "uvicorn.access"):
+    logging.getLogger(_lib).setLevel(_log_level)
+
+
+# ---------------------------------------------------------------------------
+# Lifespan (startup / shutdown)
+# ---------------------------------------------------------------------------
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logger.info("[START] Route Optimization API starting up...")
+
+    # Warm up the delivery-history FAISS store in the background
+    try:
+        from app.services.vector.delivery_history_store import get_delivery_history_store
+        store = get_delivery_history_store()
+        logger.info(
+            f"[DeliveryHistory] FAISS store ready — {store.record_count()} historical records loaded."
+        )
+    except Exception as e:
+        logger.warning(f"[DeliveryHistory] Store warm-up failed (non-fatal): {e}")
+
+    # Warm up the customer-coord FAISS store
+    try:
+        from app.services.vector.faiss_customer_store import get_faiss_store
+        cs = get_faiss_store()
+        logger.info(f"[FAISSCoords] Customer coord store ready — {cs.record_count()} records.")
+    except Exception as e:
+        logger.warning(f"[FAISSCoords] Store warm-up failed (non-fatal): {e}")
+
+    # Log how many historical assignment events we have for analytics
+    try:
+        from app.services.ml.ml_data_collector import get_collector
+        n = get_collector().count_records()
+        logger.info(f"[Analytics] {n} assignment events in the analytics DB.")
+    except Exception as e:
+        logger.warning(f"[Analytics] DB check failed (non-fatal): {e}")
+
+    # Warm up the Thompson Sampling bandit (bootstraps from historical DB)
+    try:
+        from app.services.ml.strategy_bandit import get_bandit
+        bandit = get_bandit()
+        stats  = bandit.get_stats()
+        logger.info(
+            f"[Bandit] RL strategy bandit ready — "
+            f"{stats['context_count']} contexts, "
+            f"{stats['total_updates']} historical updates loaded."
+        )
+    except Exception as e:
+        logger.warning(f"[Bandit] Warm-up failed (non-fatal): {e}")
+
+    # Start the autonomous empirical-ETA sync agent (daemon thread).
+    # It mirrors completed deliveries locally and rebuilds learned ETAs on a
+    # schedule, so the request path never touches Postgres.
+    try:
+        from app.services.routing.delivery_history_service import get_delivery_history_service
+        from app.config.dynamic_config import get_config
+        cfg = get_config()
+        get_delivery_history_service().ensure_background_sync(
+            interval_hours=int(cfg.get("eta_sync_interval_hours", 6)),
+            days=int(cfg.get("eta_history_days", 14)),
+        )
+        logger.info("[ETA-Agent] autonomous empirical-ETA sync scheduled.")
+    except Exception as e:
+        logger.warning(f"[ETA-Agent] Sync agent start failed (non-fatal): {e}")
+
+    # Start the autonomous road-sequencing decision agent. It measures road vs
+    # aerial sequencing on real batches and turns routing_use_road_distance on/off
+    # by itself — no manual flag flip needed.
+    try:
+        from app.services.routing.road_sequencing_agent import get_road_agent
+        from app.config.dynamic_config import get_config
+        get_road_agent().ensure_background_agent(
+            interval_hours=int(get_config().get("routing_eval_interval_hours", 24)),
+        )
+        logger.info("[RoadAgent] autonomous road-sequencing decision agent scheduled.")
+    except Exception as e:
+        logger.warning(f"[RoadAgent] Decision agent start failed (non-fatal): {e}")
+
+    logger.info("[OK] Application ready.")
+    yield
+    logger.info("[STOP] Route Optimization API shutting down.")
+
+
+# ---------------------------------------------------------------------------
+# App
+# ---------------------------------------------------------------------------
+
+app = FastAPI(
+    title="Route Optimization API",
+    version="2.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    openapi_url="/api/v1/openapi.json",
+    lifespan=lifespan,
+)
+
+app.add_middleware(RequestIDMiddleware)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allow_headers=["*"],
+    expose_headers=["X-Request-ID", "X-Process-Time"],
+)
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+
+
+@app.middleware("http")
+async def add_process_time_header(request: Request, call_next):
+    start = time.time()
+    response = await call_next(request)
+    response.headers["X-Process-Time"] = str(round(time.time() - start, 4))
+    response.headers["X-API-Version"]  = "2.0.0"
+    return response
+
+
+app.add_exception_handler(APIException,            api_exception_handler)
+app.add_exception_handler(StarletteHTTPException,  http_exception_handler)
+app.add_exception_handler(RequestValidationError,  validation_exception_handler)
+app.add_exception_handler(Exception,               general_exception_handler)
+
+app.include_router(optimization_router)
+app.include_router(health_router)
+app.include_router(cache_router)
+app.include_router(ml_router)
+app.include_router(ml_web_router)
+app.include_router(batch_analytics_router)
+app.include_router(riders_router)
+
+
+@app.get("/", tags=["Root"])
+async def root(request: Request):
+    return {
+        "service": "Route Optimization API",
+        "version": "2.0.0",
+        "status": "operational",
+        "docs": "/docs",
+    }
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("app.main:app", host="0.0.0.0", port=8002, reload=True)

app/middleware/__init__.py

@@ -0,0 +1,2 @@
+"""Middleware components."""
+

app/middleware/request_id.py

@@ -0,0 +1,30 @@
+"""Request ID middleware for request tracing."""
+
+import uuid
+from fastapi import Request
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import Response
+
+from app.core.log_context import set_request_id
+
+
+class RequestIDMiddleware(BaseHTTPMiddleware):
+    """Middleware to add unique request ID to each request."""
+
+    async def dispatch(self, request: Request, call_next):
+        # Generate or retrieve request ID (short form keeps logs readable)
+        request_id = request.headers.get("X-Request-ID") or uuid.uuid4().hex[:8]
+
+        # Add request ID to request state + logging context (so every log line
+        # for this request carries it — grep one id to follow the whole request).
+        request.state.request_id = request_id
+        set_request_id(request_id)
+
+        # Process request
+        response = await call_next(request)
+
+        # Add request ID to response headers
+        response.headers["X-Request-ID"] = request_id
+
+        return response
+

app/models/__init__.py

@@ -0,0 +1,21 @@
+"""Models package."""
+
+from .schemas import (
+    Location,
+    Delivery,
+    RouteOptimizationRequest,
+    RouteStep,
+    OptimizedRoute,
+    PickupLocation,
+    DeliveryLocation
+)
+
+__all__ = [
+    "Location",
+    "Delivery",
+    "RouteOptimizationRequest",
+    "RouteStep",
+    "OptimizedRoute",
+    "PickupLocation",
+    "DeliveryLocation"
+]

app/models/errors.py

@@ -0,0 +1,45 @@
+"""Professional error response models for API."""
+
+from typing import Optional, Any, Dict
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class ErrorDetail(BaseModel):
+    """Detailed error information."""
+    field: Optional[str] = Field(None, description="Field name that caused the error")
+    message: str = Field(..., description="Error message")
+    code: Optional[str] = Field(None, description="Error code")
+
+
+class ErrorResponse(BaseModel):
+    """Standardized error response model."""
+    success: bool = Field(False, description="Request success status")
+    error: ErrorDetail = Field(..., description="Error details")
+    timestamp: str = Field(default_factory=lambda: datetime.utcnow().isoformat(), description="Error timestamp")
+    path: Optional[str] = Field(None, description="Request path")
+    request_id: Optional[str] = Field(None, description="Request ID for tracing")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "success": False,
+                "error": {
+                    "field": "pickup_location",
+                    "message": "Pickup location is required",
+                    "code": "VALIDATION_ERROR"
+                },
+                "timestamp": "2024-01-15T10:30:00.000Z",
+                "path": "/api/v1/optimization/single-route",
+                "request_id": "req-123456"
+            }
+        }
+
+
+class SuccessResponse(BaseModel):
+    """Standardized success response wrapper."""
+    success: bool = Field(True, description="Request success status")
+    data: Any = Field(..., description="Response data")
+    timestamp: str = Field(default_factory=lambda: datetime.utcnow().isoformat(), description="Response timestamp")
+    request_id: Optional[str] = Field(None, description="Request ID for tracing")
+

app/models/schemas.py

@@ -0,0 +1,167 @@
+"""Professional Pydantic models for request/response validation."""
+
+from typing import List, Optional
+from pydantic import BaseModel, Field, field_validator
+from datetime import datetime
+
+
+class Location(BaseModel):
+    """Location model with latitude and longitude."""
+    lat: float = Field(..., description="Latitude")
+    lng: float = Field(..., description="Longitude")
+
+
+class PickupLocation(BaseModel):
+    """Pickup location model with latitude and longitude."""
+    pickuplat: float = Field(
+        ..., 
+        description="Pickup latitude",
+        ge=-90, 
+        le=90,
+        examples=[11.0050534]
+    )
+    pickuplon: float = Field(
+        ..., 
+        description="Pickup longitude",
+        ge=-180, 
+        le=180,
+        examples=[76.9508991]
+    )
+    
+    @field_validator("pickuplat", "pickuplon")
+    @classmethod
+    def validate_coordinates(cls, v):
+        """Validate coordinate values."""
+        if v is None:
+            raise ValueError("Coordinate cannot be None")
+        return float(v)
+
+
+class DeliveryLocation(BaseModel):
+    """Delivery location model with latitude and longitude."""
+    deliverylat: float = Field(
+        ..., 
+        description="Delivery latitude",
+        ge=-90, 
+        le=90,
+        examples=[11.0309723]
+    )
+    deliverylong: float = Field(
+        ..., 
+        description="Delivery longitude",
+        ge=-180, 
+        le=180,
+        examples=[77.0004574]
+    )
+    
+    @field_validator("deliverylat", "deliverylong")
+    @classmethod
+    def validate_coordinates(cls, v):
+        """Validate coordinate values."""
+        if v is None:
+            raise ValueError("Coordinate cannot be None")
+        return float(v)
+
+
+class Delivery(BaseModel):
+    """Delivery order model."""
+    deliveryid: str = Field(..., description="Unique delivery identifier")
+    deliverycustomerid: int = Field(..., description="Customer ID for this delivery")
+    location: DeliveryLocation = Field(..., description="Delivery location coordinates")
+
+
+class RouteOptimizationRequest(BaseModel):
+    """
+    Request model for route optimization.
+    
+    Optimizes delivery routes starting from a pickup location (warehouse/store) to multiple delivery locations.
+    Uses greedy nearest-neighbor algorithm for fast, efficient route calculation.
+    """
+    pickup_location: PickupLocation = Field(
+        ..., 
+        description="Pickup location (warehouse/store) coordinates - starting point for optimization"
+    )
+    pickup_location_id: Optional[int] = Field(
+        None, 
+        description="Optional pickup location ID for tracking purposes"
+    )
+    deliveries: List[Delivery] = Field(
+        ..., 
+        min_items=1,
+        max_items=50,
+        description="List of delivery locations to optimize (1-50 deliveries supported)"
+    )
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "pickup_location": {
+                    "pickuplat": 11.0050534,
+                    "pickuplon": 76.9508991
+                },
+                "pickup_location_id": 1,
+                "deliveries": [
+                    {
+                        "deliveryid": "90465",
+                        "deliverycustomerid": 1,
+                        "location": {
+                            "deliverylat": 11.0309723,
+                            "deliverylong": 77.0004574
+                        }
+                    }
+                ]
+            }
+        }
+
+
+class RouteStep(BaseModel):
+    """Single step in the optimized route."""
+    step_number: int = Field(..., description="Step number in the route")
+    delivery_id: str = Field(..., description="Delivery ID for this step")
+    delivery_customer_id: int = Field(..., description="Customer ID for this delivery")
+    location: DeliveryLocation = Field(..., description="Delivery location coordinates")
+    distance_from_previous_km: float = Field(..., description="Distance from previous step in kilometers")
+    cumulative_distance_km: float = Field(..., description="Total distance traveled so far in kilometers")
+
+
+class OptimizedRoute(BaseModel):
+    """
+    Optimized route response with step-by-step delivery sequence.
+    
+    Contains the optimized route starting from pickup location, with each step showing:
+    - Delivery order (Step 1, Step 2, etc.)
+    - Distance from previous step
+    - Cumulative distance traveled
+    """
+    route_id: str = Field(..., description="Unique route identifier (UUID)")
+    pickup_location_id: Optional[int] = Field(None, description="Pickup location ID")
+    pickup_location: PickupLocation = Field(..., description="Pickup location (warehouse/store) coordinates")
+    total_distance_km: float = Field(
+        ..., 
+        ge=0,
+        description="Total route distance in kilometers",
+        examples=[12.45]
+    )
+    total_deliveries: int = Field(
+        ..., 
+        ge=1,
+        description="Total number of deliveries in the route",
+        examples=[5]
+    )
+    optimization_algorithm: str = Field(
+        "greedy",
+        description="Algorithm used for optimization",
+        examples=["greedy"]
+    )
+    steps: List[RouteStep] = Field(
+        ..., 
+        description="Ordered list of route steps (Step 1 = nearest from pickup, Step 2 = nearest from Step 1, etc.)"
+    )
+    created_at: str = Field(
+        default_factory=lambda: datetime.utcnow().isoformat(),
+        description="Route creation timestamp (ISO 8601)"
+    )
+
+
+# Batch optimization removed - no rider support needed
+# Use single-route optimization for each pickup location

app/routes/__init__.py

@@ -0,0 +1,18 @@
+"""Routes package."""
+
+from .optimization import router as optimization_router
+from .health import router as health_router
+from .cache import router as cache_router
+from .ml_admin import router as ml_router, web_router as ml_web_router
+from .batch_analytics import router as batch_analytics_router
+from .riders import router as riders_router
+
+__all__ = [
+    "optimization_router",
+    "health_router",
+    "cache_router",
+    "ml_router",
+    "ml_web_router",
+    "batch_analytics_router",
+    "riders_router",
+]

app/routes/batch_analytics.py

@@ -0,0 +1,263 @@
+"""
+Batch Efficiency Analytics Endpoint
+=====================================
+POST /api/v1/batch/efficiency
+
+Analyses a delivery batch for idle-rider substitution opportunities.
+Supports named batch windows (morning / afternoon / evening) or custom
+time ranges so you can run the same analysis for any shift.
+
+Request body:
+  {
+    "batch":     "morning",       // "morning" | "afternoon" | "evening" | "custom"
+    "date":      "2026-05-28",    // defaults to today
+    "tenant_id": 916,             // defaults to 916
+    "from_time": "06:00",         // only for batch="custom"
+    "to_time":   "09:00",         // only for batch="custom"
+    "deliveries": [...],          // supply inline instead of DB fetch
+    "rider_names": {"1036": "Vignesh S", ...},
+    "config": {
+      "idle_threshold_minutes": 30,
+      "road_kmh": 13.0,
+      "max_transfer_orders": 4
+    }
+  }
+
+Batch windows (assigntime range, inclusive start / exclusive end):
+  morning   : 06:00 – 09:00   (breakfast + early lunch prep)
+  afternoon : 11:00 – 15:00   (lunch)
+  evening   : 17:00 – 21:30   (dinner)
+  custom    : caller provides from_time / to_time
+
+Header shorthand (all equivalent to body.batch):
+  X-Batch-Window: morning | afternoon | evening
+"""
+
+import logging
+import os
+from datetime import date as _date
+from typing import Any
+
+from fastapi import APIRouter, Body, Header, HTTPException, status
+
+from app.services.routing.batch_efficiency import analyse_batch
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/api/v1/batch",
+    tags=["Batch Analytics"],
+    responses={500: {"description": "Internal server error"}},
+)
+
+# ---------------------------------------------------------------------------
+# Named batch windows  {name: (from_time, to_time)}  — 24-h "HH:MM" strings
+# ---------------------------------------------------------------------------
+BATCH_WINDOWS: dict[str, tuple[str, str]] = {
+    "morning":   ("06:00", "09:00"),
+    "afternoon": ("11:00", "15:00"),
+    "evening":   ("17:00", "21:30"),
+}
+
+DEFAULT_BATCH = "morning"
+
+
+# ---------------------------------------------------------------------------
+# DB fetch
+# ---------------------------------------------------------------------------
+
+def _fetch_from_db(
+    target_date: str,
+    tenant_id: int,
+    from_time: str,
+    to_time: str,
+) -> tuple[list[dict], dict[int, str]]:
+    """
+    Fetch deliveries assigned within [from_time, to_time) on target_date.
+    Times are 24-h "HH:MM" strings, e.g. "06:00", "09:00".
+    Returns (deliveries, rider_names) where rider_names maps userid → username.
+    """
+    # Shared nearledb connector (single source of truth for DB_* creds).
+    from app.services.routing.delivery_history_service import connect_nearledb
+
+    try:
+        conn = connect_nearledb()
+    except ImportError:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="psycopg2 not installed — cannot fetch from DB.",
+        )
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail=f"DB connection failed: {exc}",
+        )
+
+    try:
+        cur = conn.cursor()
+        cur.execute(
+            """
+            SELECT
+                d.deliveryid,
+                d.userid,
+                d.pickupcustomer,
+                d.assigntime,
+                d.arrivaltime,
+                d.pickuptime,
+                d.deliverytime,
+                COALESCE(d.droplat,  d.deliverylat)  AS dlat,
+                COALESCE(d.droplon, d.deliverylong)  AS dlon
+            FROM deliveries d
+            WHERE d.tenantid = %s
+              AND DATE(d.assigntime::timestamp) = %s
+              AND CAST(SPLIT_PART(d.assigntime, ' ', 2) AS TIME) >= %s
+              AND CAST(SPLIT_PART(d.assigntime, ' ', 2) AS TIME) <  %s
+              AND COALESCE(d.droplat, d.deliverylat) IS NOT NULL
+              AND d.userid IS NOT NULL
+            ORDER BY d.userid, d.assigntime
+            """,
+            (tenant_id, target_date, from_time + ":00", to_time + ":00"),
+        )
+        cols = [c.name for c in cur.description]
+        rows = cur.fetchall()
+        cur.close()
+        deliveries = [dict(zip(cols, r)) for r in rows]
+
+        # Fetch rider names for the returned userids
+        rider_names_db: dict[int, str] = {}
+        try:
+            unique_uids = list({int(r["userid"]) for r in deliveries if r.get("userid") is not None})
+            if unique_uids:
+                cur2 = conn.cursor()
+                cur2.execute(
+                    "SELECT userid, username FROM users WHERE userid = ANY(%s)",
+                    (unique_uids,)
+                )
+                for uid, uname in cur2.fetchall():
+                    if uname:
+                        rider_names_db[int(uid)] = str(uname)
+                cur2.close()
+        except Exception:
+            pass  # names are non-critical; callers fall back to "Rider {uid}"
+
+        return deliveries, rider_names_db
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"DB query failed: {exc}",
+        )
+    finally:
+        conn.close()
+
+
+# ---------------------------------------------------------------------------
+# Endpoint
+# ---------------------------------------------------------------------------
+
+@router.post(
+    "/efficiency",
+    summary="Batch efficiency analysis",
+    description=(
+        "Analyse a delivery batch for idle-rider substitution opportunities. "
+        "Pass `batch` as 'morning', 'afternoon', or 'evening' to select the "
+        "time window automatically, or use `batch='custom'` with `from_time`/`to_time`. "
+        "Supply `deliveries` inline to skip the DB fetch entirely."
+    ),
+)
+async def batch_efficiency(
+    body: Any = Body(default=None),
+    x_batch_window: str | None = Header(default=None),
+):
+    if body is None:
+        body = {}
+
+    # ---- Parse inputs -------------------------------------------------------
+    deliveries: list[dict] | None = None
+    target_date: str | None = None
+    tenant_id: int = 916
+    rider_names: dict[int, str] = {}
+    cfg: dict = {}
+    batch_name: str = DEFAULT_BATCH
+    from_time: str | None = None
+    to_time: str | None = None
+
+    if isinstance(body, dict):
+        deliveries  = body.get("deliveries")
+        target_date = body.get("date")
+        tenant_id   = int(body.get("tenant_id", 916))
+        batch_name  = (body.get("batch") or x_batch_window or DEFAULT_BATCH).lower()
+        from_time   = body.get("from_time")
+        to_time     = body.get("to_time")
+        rider_names_raw = body.get("rider_names") or {}
+        rider_names = {int(k): v for k, v in rider_names_raw.items()}
+        cfg = body.get("config") or {}
+    elif isinstance(body, list):
+        deliveries = body
+        batch_name = (x_batch_window or DEFAULT_BATCH).lower()
+
+    # ---- Resolve time window ------------------------------------------------
+    if batch_name == "custom":
+        if not from_time or not to_time:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="batch='custom' requires from_time and to_time (e.g. '11:00', '15:00').",
+            )
+    elif batch_name in BATCH_WINDOWS:
+        from_time, to_time = BATCH_WINDOWS[batch_name]
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Unknown batch '{batch_name}'. "
+                f"Valid values: {list(BATCH_WINDOWS.keys())} or 'custom'."
+            ),
+        )
+
+    # ---- Fetch from DB if no inline deliveries ------------------------------
+    if not deliveries:
+        if not target_date:
+            target_date = str(_date.today())
+
+        logger.info(
+            f"[BatchEfficiency] Fetching batch={batch_name} "
+            f"date={target_date} window={from_time}-{to_time} tenant={tenant_id}"
+        )
+        deliveries, db_rider_names = _fetch_from_db(target_date, tenant_id, from_time, to_time)
+
+        if not deliveries:
+            return {
+                "batch": batch_name,
+                "window": {"from": from_time, "to": to_time},
+                "date": target_date,
+                "fleet_summary": {},
+                "rider_timelines": [],
+                "substitution_opportunities": [],
+                "top_recommendation": None,
+                "message": (
+                    f"No {batch_name}-batch orders found for {target_date} "
+                    f"between {from_time} and {to_time}."
+                ),
+            }
+
+        # Merge: DB-fetched names as base, request-provided names take precedence
+        rider_names = {**db_rider_names, **rider_names}
+
+    logger.info(
+        f"[BatchEfficiency] Analysing {len(deliveries)} deliveries — "
+        f"batch={batch_name} date={target_date or 'inline'}"
+    )
+
+    # ---- Run analysis -------------------------------------------------------
+    result = analyse_batch(
+        deliveries=deliveries,
+        rider_names=rider_names,
+        road_kmh=float(cfg.get("road_kmh", 13.0)),
+        idle_threshold_min=float(cfg.get("idle_threshold_minutes", 30.0)),
+        max_transfer=int(cfg.get("max_transfer_orders", 4)),
+    )
+
+    result["batch"]  = batch_name
+    result["window"] = {"from": from_time, "to": to_time}
+    result["date"]   = target_date or "inline"
+    result["input_delivery_count"] = len(deliveries)
+    return result

app/routes/cache.py

@@ -0,0 +1,79 @@
+"""Cache management API endpoints."""
+
+import logging
+from fastapi import APIRouter, HTTPException
+from typing import Dict, Any
+
+from app.services import cache
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/cache", tags=["Cache Management"])
+
+
+@router.get("/stats", response_model=Dict[str, Any])
+async def get_cache_stats():
+    """
+    Get cache statistics.
+    
+    Returns:
+    - hits: Number of cache hits
+    - misses: Number of cache misses  
+    - sets: Number of cache writes
+    - total_keys: Current number of cached route keys
+    - enabled: Whether Redis cache is enabled
+    """
+    try:
+        stats = cache.get_stats()
+        # Calculate hit rate
+        total_requests = stats.get("hits", 0) + stats.get("misses", 0)
+        if total_requests > 0:
+            stats["hit_rate"] = round(stats.get("hits", 0) / total_requests * 100, 2)
+        else:
+            stats["hit_rate"] = 0.0
+        return stats
+    except Exception as e:
+        logger.error(f"Error getting cache stats: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+
+@router.get("/keys")
+async def list_cache_keys(pattern: str = "routes:*"):
+    """
+    List cache keys matching pattern.
+    
+    - **pattern**: Redis key pattern (default: "routes:*")
+    """
+    try:
+        keys = cache.get_keys(pattern)
+        return {
+            "pattern": pattern,
+            "count": len(keys),
+            "keys": keys[:100]  # Limit to first 100 for response size
+        }
+    except Exception as e:
+        logger.error(f"Error listing cache keys: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+
+@router.delete("/clear")
+async def clear_cache(pattern: str = "routes:*"):
+    """
+    Clear cache keys matching pattern.
+    
+    - **pattern**: Redis key pattern to delete (default: "routes:*")
+    
+    [WARN] **Warning**: This will delete cached route optimizations!
+    """
+    try:
+        deleted_count = cache.delete(pattern)
+        logger.info(f"Cleared {deleted_count} cache keys matching pattern: {pattern}")
+        return {
+            "pattern": pattern,
+            "deleted_count": deleted_count,
+            "message": f"Cleared {deleted_count} cache keys"
+        }
+    except Exception as e:
+        logger.error(f"Error clearing cache: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+

app/routes/health.py

@@ -0,0 +1,98 @@
+"""Professional health check endpoints."""
+
+import time
+import logging
+import sys
+from typing import Optional
+from datetime import datetime
+from fastapi import APIRouter, Request
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/health", tags=["Health"])
+
+start_time = time.time()
+
+
+class HealthResponse(BaseModel):
+    """Health check response model."""
+    status: str = Field(..., description="Service status")
+    uptime_seconds: float = Field(..., description="Service uptime in seconds")
+    version: str = Field("2.0.0", description="API version")
+    timestamp: str = Field(..., description="Health check timestamp (ISO 8601)")
+    request_id: Optional[str] = Field(None, description="Request ID for tracing")
+
+
+@router.get("/", response_model=HealthResponse)
+async def health_check(request: Request):
+    """
+    Health check endpoint.
+    
+    Returns the current health status of the API service including:
+    - Service status (healthy/unhealthy)
+    - Uptime in seconds
+    - API version
+    - Timestamp
+    """
+    try:
+        uptime = time.time() - start_time
+        request_id = getattr(request.state, "request_id", None)
+        
+        return HealthResponse(
+            status="healthy",
+            uptime_seconds=round(uptime, 2),
+            version="2.0.0",
+            timestamp=datetime.utcnow().isoformat() + "Z",
+            request_id=request_id
+        )
+    except Exception as e:
+        logger.error(f"Health check failed: {e}", exc_info=True)
+        request_id = getattr(request.state, "request_id", None)
+        
+        return HealthResponse(
+            status="unhealthy",
+            uptime_seconds=0.0,
+            version="2.0.0",
+            timestamp=datetime.utcnow().isoformat() + "Z",
+            request_id=request_id
+        )
+
+
+@router.get("/ready")
+async def readiness_check(request: Request):
+    """
+    Readiness check endpoint for load balancers.
+    
+    Returns 200 if the service is ready to accept requests.
+    """
+    try:
+        # Check if critical services are available
+        # Add your service health checks here
+        
+        return {
+            "status": "ready",
+            "timestamp": datetime.utcnow().isoformat() + "Z",
+            "request_id": getattr(request.state, "request_id", None)
+        }
+    except Exception as e:
+        logger.error(f"Readiness check failed: {e}")
+        return {
+            "status": "not_ready",
+            "timestamp": datetime.utcnow().isoformat() + "Z",
+            "request_id": getattr(request.state, "request_id", None)
+        }
+
+
+@router.get("/live")
+async def liveness_check(request: Request):
+    """
+    Liveness check endpoint for container orchestration.
+    
+    Returns 200 if the service is alive.
+    """
+    return {
+        "status": "alive",
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "request_id": getattr(request.state, "request_id", None)
+    }

app/routes/ml_admin.py

@@ -0,0 +1,586 @@
+"""
+Analytics & ML Admin API
+=========================
+Exposes historical assignment quality data, the empirical-ETA pipeline, and the
+autonomous road-sequencing / rider-affinity agents. (The XGBoost/Optuna hypertuner,
+ID3 risk tree, and profit predictor have all been retired — they never affected
+assignment.)
+
+Endpoints:
+  GET  /api/v1/ml/status          – quality trend, analytics DB + history stats
+  GET  /api/v1/ml/analytics       – hourly stats, zone stats, histogram
+  GET  /api/v1/ml/config          – active config values
+  PATCH /api/v1/ml/config         – manual config override
+  POST /api/v1/ml/reset           – reset config to defaults
+  POST /api/v1/ml/strategy        – change optimization strategy
+  POST /api/v1/ml/refresh-eta     – sync nearledb + rebuild empirical ETA stats
+  GET  /api/v1/ml/eta-accuracy    – formula vs empirical ETA backtest
+  GET/POST /api/v1/ml/road-eval   – autonomous road-sequencing decision
+  GET  /api/v1/ml/rider-affinity  – learned vs configured rider→kitchen affinity
+  GET  /api/v1/ml/export          – download assignment log as CSV
+  GET  /api/v1/ml/history         – delivery-history FAISS store stats
+"""
+
+import json
+import logging
+import os
+
+from fastapi import APIRouter, Body, HTTPException
+from fastapi.responses import PlainTextResponse, FileResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/api/v1/ml",
+    tags=["Analytics & ML"],
+    responses={500: {"description": "Internal server error"}},
+)
+
+web_router = APIRouter(tags=["ML Monitor Web Dashboard"])
+
+
+# ---------------------------------------------------------------------------
+# Dashboard (HTML)
+# ---------------------------------------------------------------------------
+
+@web_router.get("/ml-ops", summary="Visual ML monitoring dashboard")
+def ml_dashboard():
+    path = os.path.join(os.getcwd(), "app/templates/ml_dashboard.html")
+    if not os.path.isfile(path):
+        raise HTTPException(status_code=404, detail=f"Dashboard template not found at {path}")
+    return FileResponse(path)
+
+
+# ---------------------------------------------------------------------------
+# GET /status
+# ---------------------------------------------------------------------------
+
+@router.get("/status", summary="Assignment quality trend & store stats")
+def ml_status():
+    """
+    Returns:
+    - How many assignment events are logged
+    - Recent quality score trend (last 50 calls)
+    - Delivery history FAISS store record count
+    - Active config values
+    """
+    try:
+        from app.services.ml.ml_data_collector import get_collector
+        from app.services.vector.delivery_history_store import get_delivery_history_store
+        from app.config.dynamic_config import get_config
+
+        collector = get_collector()
+        history   = get_delivery_history_store()
+        cfg       = get_config()
+
+        return {
+            "status": "ok",
+            "db_records":       collector.count_records(),
+            "quality_trend":    collector.get_recent_quality_trend(last_n=50),
+            "delivery_history": {
+                "record_count": history.record_count(),
+                "status": "ready" if history.record_count() > 0 else "empty",
+            },
+            "config": cfg.get_all(),
+        }
+    except Exception as e:
+        logger.error(f"[ML API] status: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /analytics
+# ---------------------------------------------------------------------------
+
+@router.get("/analytics", summary="Hourly stats, zone stats, quality histogram")
+def ml_analytics():
+    """Operational analytics from historical assignment logs."""
+    try:
+        from app.services.ml.ml_data_collector import get_collector
+        collector = get_collector()
+        return {
+            "status":             "ok",
+            "hourly_stats":       collector.get_hourly_stats(),
+            "zone_stats":         collector.get_zone_stats(),
+            "quality_histogram":  collector.get_quality_histogram(),
+            "strategy_comparison": collector.get_strategy_comparison(),
+        }
+    except Exception as e:
+        logger.error(f"[ML API] analytics: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /config
+# ---------------------------------------------------------------------------
+
+@router.get("/config", summary="Current active configuration values")
+def ml_config():
+    from app.config.dynamic_config import get_config, DEFAULTS
+    try:
+        cfg         = get_config()
+        all_values  = cfg.get_all()
+        cached_keys = set(cfg._cache.keys())
+        annotated   = {
+            k: {"value": v, "source": "override" if k in cached_keys else "default"}
+            for k, v in all_values.items()
+        }
+        return {
+            "status":          "ok",
+            "hyperparameters": annotated,
+            "total_params":    len(annotated),
+            "override_count":  sum(1 for x in annotated.values() if x["source"] == "override"),
+        }
+    except Exception as e:
+        logger.error(f"[ML API] config: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.patch("/config", summary="Override specific config values")
+def ml_config_patch(payload: dict = Body(...)):
+    """Manually set any config key, e.g. {\"road_factor\": 1.4}"""
+    from app.config.dynamic_config import get_config
+    try:
+        get_config().set_bulk(payload, source="ml_admin")
+        return {"status": "ok", "updated": list(payload.keys())}
+    except Exception as e:
+        logger.error(f"[ML API] config patch: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# POST /reset
+# ---------------------------------------------------------------------------
+
+@router.post("/reset", summary="Reset all config overrides to factory defaults")
+def ml_reset():
+    from app.config.dynamic_config import get_config
+    try:
+        get_config().reset_to_defaults()
+        return {"status": "ok", "message": "All config values reset to factory defaults."}
+    except Exception as e:
+        logger.error(f"[ML API] reset: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# POST /strategy
+# ---------------------------------------------------------------------------
+
+@router.post("/strategy", summary="Change the optimization strategy")
+def ml_strategy(strategy: str = Body(default="balanced", embed=True)):
+    """
+    Choices: balanced | fuel_saver | aggressive_speed | zone_strict
+    Affects how the quality score is computed in analytics only.
+    """
+    valid = ["balanced", "fuel_saver", "aggressive_speed", "zone_strict"]
+    if strategy not in valid:
+        raise HTTPException(400, f"Invalid strategy. Choose from {valid}")
+    from app.config.dynamic_config import get_config
+    try:
+        get_config().set("ml_strategy", strategy)
+        return {"status": "ok", "strategy": strategy}
+    except Exception as e:
+        logger.error(f"[ML API] strategy: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# POST /auto-tune
+# ---------------------------------------------------------------------------
+
+@router.post("/auto-tune", summary="Run SQL-based strategy auto-tuner")
+def ml_auto_tune():
+    """
+    Analyses the assignment log and picks the best-performing ml_strategy
+    based on average quality score across all recorded calls.
+
+    Rules:
+    - A strategy needs ≥ 10 calls to be considered.
+    - At least 2 strategies must have enough data to compare.
+    - If a better strategy is found it is written to DynamicConfig
+      immediately and takes effect on the next /riderassign call.
+    - Also returns per-hour breakdown so you can see peak-hour patterns.
+
+    Safe to call any time.  Runs in < 100ms.
+    """
+    from app.services.ml.ml_data_collector import get_collector
+    from app.config.dynamic_config import get_config, DEFAULTS
+    try:
+        collector  = get_collector()
+        cfg        = get_config()
+        comparison = collector.get_strategy_comparison()
+        hourly     = collector.get_hourly_stats()
+        total_records = collector.count_records()
+
+        if total_records == 0:
+            return {
+                "status": "no_data",
+                "message": "No assignment events logged yet. "
+                           "Call /riderassign a few times first.",
+                "total_records": 0,
+            }
+
+        qualified = [s for s in comparison if s["call_count"] >= 10]
+        current_strategy = cfg.get("ml_strategy", "balanced")
+        action = "no_change"
+        recommendation = None
+
+        if len(qualified) >= 2:
+            best = max(qualified, key=lambda x: x["avg_quality"])
+            recommendation = best["strategy"]
+            if best["strategy"] != current_strategy:
+                cfg.set("ml_strategy", best["strategy"], source="auto_tuner")
+                action = "updated"
+                logger.info(
+                    f"[AutoTune API] Strategy: '{current_strategy}' → "
+                    f"'{best['strategy']}' (quality={best['avg_quality']:.1f})"
+                )
+        elif len(comparison) > 0:
+            action = "insufficient_data"
+            recommendation = comparison[0]["strategy"]  # best so far even if < 10 calls
+
+        # Per-hour best strategy (informational — not auto-applied)
+        # Shows which strategy logged the highest quality at each hour
+        hour_best: list = []
+        if hourly:
+            for h in hourly:
+                # Find which strategy performed best in this hour block
+                # (simple: use the dominant strategy for that hour from comparison)
+                hour_best.append({
+                    "hour": h["hour"],
+                    "avg_quality": h["avg_quality"],
+                    "call_count":  h["call_count"],
+                    "sla_breaches": h["sla_breaches"],
+                })
+
+        return {
+            "status":            "ok",
+            "action":            action,
+            "current_strategy":  cfg.get("ml_strategy", "balanced"),
+            "recommendation":    recommendation,
+            "total_records":     total_records,
+            "strategy_comparison": comparison,
+            "hourly_quality":    hour_best,
+            "message": (
+                f"Strategy updated to '{recommendation}'."
+                if action == "updated"
+                else "Current strategy is already optimal."
+                if action == "no_change"
+                else "More data needed (≥ 10 calls per strategy to compare)."
+            ),
+        }
+    except Exception as e:
+        logger.error(f"[ML API] auto-tune: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /bandit
+# ---------------------------------------------------------------------------
+
+@router.get("/bandit", summary="Thompson Sampling bandit — posterior stats per context")
+def ml_bandit():
+    """
+    Shows the current state of the RL strategy bandit.
+
+    Each context (time_band|load_band) has 4 arms (strategies).
+    For each arm:
+      mean_reward   — expected quality / 100 based on posterior mean
+      observations  — number of observed calls (excluding prior)
+      alpha / beta  — Beta distribution parameters
+
+    The bandit uses Thompson Sampling to select strategies automatically
+    on every /riderassign call, balancing exploration vs exploitation.
+    """
+    try:
+        from app.services.ml.strategy_bandit import get_bandit
+        return {"status": "ok", **get_bandit().get_stats()}
+    except Exception as e:
+        logger.error(f"[ML API] bandit: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /rider-efficiency
+# ---------------------------------------------------------------------------
+
+@router.get("/rider-efficiency", summary="Per-rider efficiency scores from 30-day CSV history")
+def ml_rider_efficiency():
+    """
+    Computes efficiency scores for each rider from the delivery_details.csv.
+
+    Metrics:
+      delivery_count  — total deliveries in the 30-day window
+      avg_km          — average km per delivery
+      unique_zones    — number of distinct 1.1 km delivery cells served
+      efficiency_score — normalised 0..1 composite (high = efficient)
+
+    Used internally as tiebreaker during solo rider consolidation
+    and Phase-0 pattern pre-assignment host selection.
+    """
+    try:
+        from app.services.vector.delivery_history_store import get_delivery_history_store
+        scores = get_delivery_history_store().get_rider_efficiency_scores()
+        ranked = sorted(scores.items(), key=lambda x: x[1]["efficiency_score"], reverse=True)
+        return {
+            "status": "ok",
+            "rider_count": len(scores),
+            "riders": [{"rider_id": rid, **data} for rid, data in ranked],
+        }
+    except Exception as e:
+        logger.error(f"[ML API] rider-efficiency: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /history
+# ---------------------------------------------------------------------------
+
+@router.get("/history", summary="Delivery-history FAISS store stats + pattern table")
+def ml_history(patterns: bool = False):
+    """
+    Shows the delivery history store status.
+
+    Add ?patterns=true to include the full pattern table
+    (all clear dominant-rider zones, sorted by pattern score).
+    """
+    from app.services.vector.delivery_history_store import (
+        get_delivery_history_store, _INDEX_PATH, _META_PATH, CSV_PATH
+    )
+    try:
+        store = get_delivery_history_store()
+        meta  = {}
+        if os.path.isfile(_META_PATH):
+            with open(_META_PATH, "r", encoding="utf-8") as f:
+                meta = json.load(f)
+
+        resp = {
+            "status":         "ok",
+            "record_count":   store.record_count(),
+            "pattern_count":  store.pattern_count(),
+            "index_ready":    store.record_count() > 0,
+            "disk_index":     os.path.isfile(_INDEX_PATH),
+            "csv_path":       CSV_PATH,
+            "csv_exists":     os.path.isfile(CSV_PATH),
+            "saved_meta":     meta,
+        }
+        if patterns:
+            resp["patterns"] = store.get_pattern_stats()
+        return resp
+    except Exception as e:
+        logger.error(f"[ML API] history: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# POST /inject-corrections
+# ---------------------------------------------------------------------------
+
+@router.post("/inject-corrections", summary="Inject manually-corrected delivery CSV as high-weight priors")
+def ml_inject_corrections(
+    corrections_path: str = Body(default="delivery_corrections.csv", embed=True),
+    weight: int = Body(default=5, embed=True),
+):
+    """
+    Loads a manually-corrected delivery CSV and injects it into the live
+    delivery-history pattern table as high-weight prior evidence.
+
+    Each record in the corrections file is counted `weight` times (default 5),
+    so 5 correction votes easily override 1-2 noise votes from regular history.
+
+    The correction CSV must have the same columns as delivery_details.csv:
+      pickupcustomer, pickuplat, pickuplon, deliverylat, deliverylong,
+      userid, ridername
+
+    After injection the pattern table is rebuilt in-memory and saved to disk.
+    No server restart needed.
+
+    Parameters:
+      corrections_path  Path inside the container (default: delivery_corrections.csv)
+      weight            How many times each correction record is counted (default: 5)
+    """
+    from app.services.vector.delivery_history_store import get_delivery_history_store
+    try:
+        store  = get_delivery_history_store()
+        result = store.inject_corrections(corrections_path, weight=weight)
+        return {"status": "ok", **result}
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except Exception as e:
+        logger.error(f"[ML API] inject-corrections: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# POST /reload-history
+# ---------------------------------------------------------------------------
+
+@router.post("/reload-history", summary="Rebuild FAISS delivery-history index from CSV")
+def ml_reload_history():
+    """
+    Forces a full rebuild of the delivery-history FAISS index from
+    delivery_details.csv, then saves the new index to disk.
+
+    Call this whenever you update the CSV with a fresh 30-day export.
+    No server restart needed — the in-memory store is hot-swapped.
+
+    Returns the number of records now loaded.
+    """
+    from app.services.vector.delivery_history_store import get_delivery_history_store
+    try:
+        store = get_delivery_history_store()
+        n     = store.reload_from_csv()
+        return {
+            "status":       "ok",
+            "record_count": n,
+            "message":      f"FAISS history index rebuilt from CSV — {n} records loaded and saved to disk.",
+        }
+    except Exception as e:
+        logger.error(f"[ML API] reload-history: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+
+# ---------------------------------------------------------------------------
+# POST /refresh-eta
+# ---------------------------------------------------------------------------
+
+@router.post("/refresh-eta", summary="Sync nearledb -> local mirror and rebuild empirical ETA stats")
+def ml_refresh_eta(
+    days: int = Body(default=None, embed=True),
+    tenant_id: int = Body(default=916, embed=True),
+    full: bool = Body(default=False, embed=True),
+):
+    """
+    Manually trigger the empirical-ETA pipeline (the autonomous agent does this
+    on a schedule too): incremental READ-ONLY pull of new completed deliveries
+    from nearledb into the local mirror, then rebuild the learned medians.
+
+    `days` defaults to `eta_history_days`. Set `full=true` to force a full
+    backfill of the window instead of an incremental sync.
+    """
+    from app.services.routing.delivery_history_service import get_delivery_history_service
+    from app.config.dynamic_config import get_config
+    try:
+        window = int(days if days is not None else get_config().get("eta_history_days", 14))
+        result = get_delivery_history_service().refresh_eta_stats(
+            days=window, tenant_id=tenant_id, full=full
+        )
+        return {"status": "ok", "result": result}
+    except Exception as e:
+        logger.error(f"[ML API] refresh-eta: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /eta-accuracy
+# ---------------------------------------------------------------------------
+
+@router.get("/eta-accuracy", summary="Backtest: formula ETA MAE vs empirical ETA MAE")
+def ml_eta_accuracy(days: int = None, tenant_id: int = 916):
+    """
+    Honest time-split backtest. Trains empirical medians on the older 80% of the
+    history window and reports mean-absolute-error (minutes) of the formula vs
+    the empirical model on the held-out recent 20%.
+
+    Use this as the gate before trusting empirical ETAs: if `interpretation` is
+    not "empirical_better", leave `eta_empirical_enabled=false` and investigate.
+    """
+    from app.services.routing.delivery_history_service import get_delivery_history_service
+    from app.config.dynamic_config import get_config
+    try:
+        cfg = get_config()
+        window = int(days if days is not None else cfg.get("eta_history_days", 14))
+        result = get_delivery_history_service().backtest(
+            days=window,
+            tenant_id=tenant_id,
+            min_samples=int(cfg.get("eta_min_samples", 20)),
+            stat=str(cfg.get("eta_stat", "median")),
+        )
+        return {"status": "ok", "backtest": result}
+    except Exception as e:
+        logger.error(f"[ML API] eta-accuracy: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET/POST /road-eval  (autonomous road-sequencing decision agent)
+# ---------------------------------------------------------------------------
+
+@router.get("/road-eval", summary="Last autonomous road-sequencing decision")
+def ml_road_eval_get():
+    """
+    Show the road-sequencing agent's most recent decision: whether road-aware
+    ordering beat straight-line ordering on real batches, the measured mean
+    travel-time gain, and whether it auto-enabled/disabled `routing_use_road_distance`.
+    """
+    from app.services.routing.road_sequencing_agent import get_road_agent
+    from app.config.dynamic_config import get_config
+    try:
+        agent = get_road_agent()
+        return {
+            "status": "ok",
+            "road_distance_enabled": bool(get_config().get("routing_use_road_distance", False)),
+            "auto_manage": bool(get_config().get("routing_auto_manage", True)),
+            "last_decision": agent.last_decision or get_config().get("routing_road_eval", {}),
+        }
+    except Exception as e:
+        logger.error(f"[ML API] road-eval get: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/road-eval", summary="Run the road-sequencing decision now")
+def ml_road_eval_run():
+    """
+    Trigger an immediate evaluation + autonomous decision (the agent also does
+    this on a schedule). Measures road vs aerial sequencing on sampled real
+    batches and may flip `routing_use_road_distance` based on the measured gain.
+    Also reports whether we beat the riders' actual delivered order.
+    """
+    from app.services.routing.road_sequencing_agent import get_road_agent
+    try:
+        return {"status": "ok", "decision": get_road_agent().decide_and_apply()}
+    except Exception as e:
+        logger.error(f"[ML API] road-eval run: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /rider-affinity  (learned vs configured rider→kitchen affinity)
+# ---------------------------------------------------------------------------
+
+@router.get("/rider-affinity", summary="Learned vs configured rider→kitchen affinity")
+def ml_rider_affinity(refresh: bool = False):
+    """
+    Show the learned rider→kitchen affinity (from real delivery history) merged
+    with the curated config. `?refresh=true` recomputes from the local mirror
+    first. Learned data only augments SOFT steering — hard kitchen locks and
+    BLOCKED_RIDERS stay sourced from the curated config.
+    """
+    from app.services.routing.rider_affinity_service import get_rider_affinity
+    try:
+        aff = get_rider_affinity()
+        if refresh:
+            aff.refresh()
+        return {"status": "ok", "affinity": aff.get_summary()}
+    except Exception as e:
+        logger.error(f"[ML API] rider-affinity: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------------------------------------------------------------------------
+# GET /export
+# ---------------------------------------------------------------------------
+
+@router.get("/export", summary="Download all assignment logs as CSV")
+def ml_export():
+    from app.services.ml.ml_data_collector import get_collector
+    try:
+        csv_data = get_collector().export_csv()
+        response = PlainTextResponse(content=csv_data, media_type="text/csv")
+        response.headers["Content-Disposition"] = 'attachment; filename="assignment_log.csv"'
+        return response
+    except Exception as e:
+        logger.error(f"[ML API] export: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))

app/routes/riders.py

@@ -0,0 +1,149 @@
+"""
+Riders Admin API
+================
+Endpoints for the operations team to manage rider substitutions.
+
+  POST   /api/v1/riders/substitution                              – register one or many substitutions
+  GET    /api/v1/riders/substitution                              – list upcoming/active subs
+  DELETE /api/v1/riders/substitution/{sub_date}/{absent_rider_id} – cancel one
+"""
+
+import logging
+from datetime import date
+from typing import List
+
+from fastapi import APIRouter, Body, HTTPException, Path
+from pydantic import BaseModel, model_validator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/api/v1/riders",
+    tags=["Riders & Substitutions"],
+    responses={500: {"description": "Internal server error"}},
+)
+
+
+class SubstitutionEntry(BaseModel):
+    sub_date: str
+    absent_rider_id: int
+    sub_rider_id: int
+
+    @model_validator(mode="after")
+    def validate_entry(self):
+        try:
+            date.fromisoformat(self.sub_date)
+        except ValueError:
+            raise ValueError(f"Invalid date format '{self.sub_date}'. Use YYYY-MM-DD.")
+        if self.absent_rider_id == self.sub_rider_id:
+            raise ValueError("absent_rider_id and sub_rider_id must be different.")
+        return self
+
+
+@router.post("/substitution", summary="Register one or multiple rider substitutions")
+def register_substitution(entries: List[SubstitutionEntry] = Body(...)):
+    """
+    Register rider substitutions for the operations team. Accepts a list so
+    you can submit all absent riders for a day in one call.
+
+    Example — single:
+    ```json
+    [{"sub_date": "2026-06-20", "absent_rider_id": 101, "sub_rider_id": 205}]
+    ```
+
+    Example — batch (3 riders absent same day):
+    ```json
+    [
+      {"sub_date": "2026-06-20", "absent_rider_id": 101, "sub_rider_id": 205},
+      {"sub_date": "2026-06-20", "absent_rider_id": 102, "sub_rider_id": 206},
+      {"sub_date": "2026-06-21", "absent_rider_id": 103, "sub_rider_id": 207}
+    ]
+    ```
+
+    Each sub rider **must** appear in that day's getriderlogs response.
+    The assignment engine copies the absent rider's kitchen ownership, soft
+    preferences, and home location onto the sub rider automatically — reverts
+    the next day with no action needed.
+
+    Posting the same (sub_date, absent_rider_id) again updates sub_rider_id.
+    """
+    if not entries:
+        raise HTTPException(status_code=400, detail="Request body must be a non-empty list.")
+
+    try:
+        from app.services.rider.substitution_service import get_substitution_service
+        svc = get_substitution_service()
+        results = [svc.register(e.sub_date, e.absent_rider_id, e.sub_rider_id) for e in entries]
+        return {
+            "status": "ok",
+            "registered": len(results),
+            "substitutions": results,
+        }
+    except Exception as e:
+        logger.error(f"[Riders API] register_substitution: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/substitution", summary="List upcoming and active rider substitutions")
+def list_substitutions(from_date: str = None):
+    """
+    Returns all substitution records on or after `from_date` (default: today).
+    Use `?from_date=2026-06-01` to look back further.
+    """
+    if from_date is not None:
+        try:
+            date.fromisoformat(from_date)
+        except ValueError:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid date format '{from_date}'. Use YYYY-MM-DD.",
+            )
+    try:
+        from app.services.rider.substitution_service import get_substitution_service
+        records = get_substitution_service().list_all(from_date)
+        return {
+            "status": "ok",
+            "count": len(records),
+            "substitutions": records,
+        }
+    except Exception as e:
+        logger.error(f"[Riders API] list_substitutions: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete(
+    "/substitution/{sub_date}/{absent_rider_id}",
+    summary="Cancel a rider substitution",
+)
+def cancel_substitution(
+    sub_date: str = Path(..., example="2026-06-20"),
+    absent_rider_id: int = Path(..., example=101),
+):
+    """
+    Remove a substitution record. The sub rider will no longer inherit the
+    absent rider's profile on that date.
+    """
+    try:
+        date.fromisoformat(sub_date)
+    except ValueError:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid date format '{sub_date}'. Use YYYY-MM-DD.",
+        )
+    try:
+        from app.services.rider.substitution_service import get_substitution_service
+        removed = get_substitution_service().cancel(sub_date, absent_rider_id)
+        if not removed:
+            raise HTTPException(
+                status_code=404,
+                detail=f"No substitution found for absent_rider_id={absent_rider_id} on {sub_date}.",
+            )
+        return {
+            "status": "ok",
+            "message": f"Substitution cancelled: rider {absent_rider_id} on {sub_date}.",
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"[Riders API] cancel_substitution: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))

app/services/__init__.py

@@ -0,0 +1,175 @@
+"""Services package."""
+
+from __future__ import annotations
+
+import json
+import os
+import logging
+from typing import Any, Optional, Dict
+
+try:
+    import redis  # type: ignore
+except Exception:  # pragma: no cover
+    redis = None  # type: ignore
+
+logger = logging.getLogger(__name__)
+
+
+class RedisCache:
+    """Lightweight Redis cache wrapper with graceful in-memory fallback."""
+
+    def __init__(self, url_env: str = "REDIS_URL", default_ttl_seconds: Optional[int] = None) -> None:
+        import threading
+        self._lock = threading.Lock()
+        self._memory_cache: Dict[str, tuple[float, str]] = {}  # key -> (expire_time, serialized_value)
+        
+        # Allow TTL to be configurable via env var (default 300s = 5 min, or 86400 = 24h)
+        ttl_env = os.getenv("REDIS_CACHE_TTL_SECONDS")
+        if default_ttl_seconds is None:
+            default_ttl_seconds = int(ttl_env) if ttl_env else 300
+        
+        self.default_ttl_seconds = default_ttl_seconds
+        self._enabled = False
+        self._client = None
+        self._stats = {"hits": 0, "misses": 0, "sets": 0}
+
+        url = os.getenv(url_env)
+        if not url or redis is None:
+            logger.warning("Redis not configured or client unavailable; falling back to local thread-safe in-memory cache")
+            return
+        try:
+            self._client = redis.Redis.from_url(url, decode_responses=True)
+            self._client.ping()
+            self._enabled = True
+            logger.info(f"Redis cache connected (TTL: {self.default_ttl_seconds}s)")
+        except Exception as exc:
+            logger.warning(f"Redis connection failed: {exc}; falling back to local thread-safe in-memory cache")
+            self._enabled = False
+            self._client = None
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled and self._client is not None
+
+    def get_json(self, key: str) -> Optional[Any]:
+        if self.enabled:
+            try:
+                raw = self._client.get(key)  # type: ignore[union-attr]
+                if raw:
+                    self._stats["hits"] += 1
+                    return json.loads(raw)
+                else:
+                    self._stats["misses"] += 1
+                    return None
+            except Exception as exc:
+                logger.debug(f"Redis get_json error for key={key}: {exc}")
+                self._stats["misses"] += 1
+                return None
+        else:
+            import time
+            with self._lock:
+                if key in self._memory_cache:
+                    expire_time, raw = self._memory_cache[key]
+                    if expire_time < 0 or expire_time > time.time():
+                        self._stats["hits"] += 1
+                        return json.loads(raw)
+                    else:
+                        del self._memory_cache[key]
+                self._stats["misses"] += 1
+                return None
+
+    def set_json(self, key: str, value: Any, ttl_seconds: Optional[int] = None) -> None:
+        payload = json.dumps(value, default=lambda o: getattr(o, "model_dump", lambda: o)())
+        ttl = ttl_seconds if ttl_seconds is not None else self.default_ttl_seconds
+        
+        if self.enabled:
+            try:
+                if ttl > 0:
+                    self._client.setex(key, ttl, payload)  # type: ignore[union-attr]
+                else:
+                    self._client.set(key, payload)  # type: ignore[union-attr]
+                self._stats["sets"] += 1
+            except Exception as exc:
+                logger.debug(f"Redis set_json error for key={key}: {exc}")
+        else:
+            import time
+            expire_time = (time.time() + ttl) if ttl > 0 else -1.0
+            with self._lock:
+                # Evict oldest keys if cache grows too large to prevent leak
+                if len(self._memory_cache) >= 2000:
+                    now = time.time()
+                    expired_keys = [k for k, (exp, _) in self._memory_cache.items() if exp > 0 and exp < now]
+                    for k in expired_keys:
+                        del self._memory_cache[k]
+                    if len(self._memory_cache) >= 2000:
+                        first_key = next(iter(self._memory_cache))
+                        del self._memory_cache[first_key]
+                self._memory_cache[key] = (expire_time, payload)
+                self._stats["sets"] += 1
+
+    def delete(self, pattern: str) -> int:
+        """Delete keys matching pattern (e.g., 'routes:*'). Returns count deleted."""
+        if self.enabled:
+            try:
+                keys = list(self._client.scan_iter(match=pattern))  # type: ignore[union-attr]
+                if keys:
+                    return self._client.delete(*keys)  # type: ignore[union-attr]
+                return 0
+            except Exception as exc:
+                logger.error(f"Redis delete error for pattern={pattern}: {exc}")
+                return 0
+        else:
+            import fnmatch
+            deleted = 0
+            with self._lock:
+                keys_to_del = [k for k in self._memory_cache.keys() if fnmatch.fnmatchcase(k, pattern)]
+                for k in keys_to_del:
+                    del self._memory_cache[k]
+                    deleted += 1
+            return deleted
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        stats = self._stats.copy()
+        if self.enabled:
+            try:
+                # Count cache keys
+                route_keys = list(self._client.scan_iter(match="routes:*"))  # type: ignore[union-attr]
+                stats["total_keys"] = len(route_keys)
+                stats["enabled"] = True
+                stats["type"] = "Redis"
+            except Exception:
+                stats["total_keys"] = 0
+                stats["enabled"] = True
+                stats["type"] = "Redis"
+        else:
+            import time
+            now = time.time()
+            with self._lock:
+                active_keys = [k for k, (exp, _) in self._memory_cache.items() if exp < 0 or exp > now]
+                stats["total_keys"] = len(active_keys)
+                stats["enabled"] = True
+                stats["type"] = "In-Memory Fallback"
+        return stats
+
+    def get_keys(self, pattern: str = "routes:*") -> list[str]:
+        """Get list of cache keys matching pattern."""
+        if self.enabled:
+            try:
+                return list(self._client.scan_iter(match=pattern))  # type: ignore[union-attr]
+            except Exception as exc:
+                logger.error(f"Redis get_keys error for pattern={pattern}: {exc}")
+                return []
+        else:
+            import fnmatch
+            import time
+            now = time.time()
+            with self._lock:
+                return [
+                    k for k, (exp, _) in self._memory_cache.items()
+                    if (exp < 0 or exp > now) and fnmatch.fnmatchcase(k, pattern)
+                ]
+
+
+# Singleton cache instance for app
+cache = RedisCache()

app/services/ml/behavior_analyzer.py

@@ -0,0 +1,55 @@
+"""
+Feature band encoders
+======================
+Discrete bucketers for assignment-context features (distance, time-of-day, load,
+order density). These are small pure helpers used by the Thompson-sampling
+strategy bandit (`strategy_bandit.py`) and the /riderassign bandit context.
+
+NOTE: The ID3 SUCCESS/RISK decision tree that used to live here has been retired —
+it only ever produced response metadata and never affected any assignment. Only
+the generic feature-band encoders remain.
+"""
+
+from datetime import datetime
+
+
+def distance_band(km: float) -> str:
+    """Total route distance -> discrete band."""
+    if km <= 5.0:   return "SHORT"
+    if km <= 15.0:  return "MID"
+    if km <= 30.0:  return "LONG"
+    return "VERY_LONG"
+
+
+def time_band(ts_str: str) -> str:
+    """ISO timestamp -> time-of-day band."""
+    try:
+        hour = datetime.fromisoformat(ts_str).hour
+        if 6  <= hour < 10: return "MORNING_RUSH"
+        if 10 <= hour < 12: return "LATE_MORNING"
+        if 12 <= hour < 14: return "LUNCH_RUSH"
+        if 14 <= hour < 17: return "AFTERNOON"
+        if 17 <= hour < 20: return "EVENING_RUSH"
+        if 20 <= hour < 23: return "NIGHT"
+        return "LATE_NIGHT"
+    except Exception:
+        return "UNKNOWN"
+
+
+def load_band(avg_load: float) -> str:
+    """Average orders-per-rider -> load band."""
+    if avg_load <= 2.0:  return "LIGHT"
+    if avg_load <= 5.0:  return "MODERATE"
+    if avg_load <= 8.0:  return "HEAVY"
+    return "OVERLOADED"
+
+
+def order_density_band(num_orders: int, num_riders: int) -> str:
+    """Orders per available rider -> density band."""
+    if num_riders == 0:
+        return "NO_RIDERS"
+    ratio = num_orders / num_riders
+    if ratio <= 2.0:  return "SPARSE"
+    if ratio <= 5.0:  return "NORMAL"
+    if ratio <= 9.0:  return "DENSE"
+    return "OVERLOADED"

app/services/ml/ml_data_collector.py

@@ -0,0 +1,611 @@
+"""
+ML Data Collector - Production Grade
+======================================
+Logs every assignment call (inputs + outcomes) to SQLite.
+
+Key upgrades over the original
+--------------------------------
+1. FROZEN historical scores  - quality_score is written ONCE at log time.
+   get_training_data() returns scores as-is from the DB (no retroactive mutation).
+2. Rich schema              - zone_id, city_id, is_peak, weather_code,
+   sla_breached, avg_delivery_time_min for richer features.
+3. SLA tracking             - logs whether delivery SLA was breached.
+4. Analytics API            - get_hourly_stats(), get_strategy_comparison(),
+   get_quality_histogram(), get_zone_stats() for dashboard consumption.
+5. Thread-safe writes       - connection-per-write pattern for FastAPI workers.
+6. Indexed columns          - timestamp, ml_strategy, zone_id for fast queries.
+"""
+
+import csv
+import io
+import logging
+import os
+import sqlite3
+import threading
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
+_WRITE_LOCK = threading.Lock()
+
+
+def _std(values: List[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((v - mean) ** 2 for v in values) / len(values)) ** 0.5
+
+
+class MLDataCollector:
+    """
+    Event logger for assignment service calls.
+
+    Each log_assignment_event() call writes one row capturing:
+      - Operating context   (time, orders, riders, zone, city)
+      - Active hyperparams  (exact config snapshot for this call)
+      - Measured outcomes   (quality score, SLA, latency, distances)
+
+    quality_score is computed once and FROZEN - never retroactively changed.
+    """
+
+    def __init__(self):
+        self._db_path = _DB_PATH
+        self._ensure_db()
+
+    # ------------------------------------------------------------------
+    # Main logging API
+    # ------------------------------------------------------------------
+
+    def log_assignment_event(
+        self,
+        *,
+        num_orders: int,
+        num_riders: int,
+        hyperparams: Dict[str, Any],
+        assignments: Dict[int, List[Any]],
+        unassigned_count: int,
+        elapsed_ms: float,
+        zone_id: str = "default",
+        city_id: str = "default",
+        weather_code: str = "CLEAR",
+        sla_minutes: Optional[float] = None,
+        avg_delivery_time_min: Optional[float] = None,
+    ) -> None:
+        """
+        Log one assignment event.
+
+        Call this at the END of AssignmentService.assign_orders() once
+        outcomes are known.
+        """
+        try:
+            now = datetime.utcnow()
+            hour = now.hour
+            day_of_week = now.weekday()
+            is_peak = int(hour in (7, 8, 9, 12, 13, 18, 19, 20))
+
+            rider_loads = [len(orders) for orders in assignments.values() if orders]
+            riders_used = len(rider_loads)
+            total_assigned = sum(rider_loads)
+            avg_load = total_assigned / riders_used if riders_used else 0.0
+            load_std = _std(rider_loads) if rider_loads else 0.0
+
+            all_orders = [
+                o for orders in assignments.values() if orders for o in orders
+            ]
+            total_distance_km = sum(self._get_km(o) for o in all_orders)
+            ml_strategy = hyperparams.get("ml_strategy", "balanced")
+            max_opr = hyperparams.get("max_orders_per_rider", 12)
+
+            sla_breached = 0
+            if sla_minutes and avg_delivery_time_min:
+                sla_breached = int(avg_delivery_time_min > sla_minutes)
+
+            # Quality score - FROZEN at log time
+            quality_score = self._compute_quality_score(
+                num_orders=num_orders,
+                unassigned_count=unassigned_count,
+                load_std=load_std,
+                riders_used=riders_used,
+                num_riders=num_riders,
+                total_distance_km=total_distance_km,
+                max_orders_per_rider=max_opr,
+                ml_strategy=ml_strategy,
+            )
+
+            row = {
+                "timestamp": now.isoformat(),
+                "hour": hour,
+                "day_of_week": day_of_week,
+                "is_peak": is_peak,
+                "zone_id": zone_id,
+                "city_id": city_id,
+                "weather_code": weather_code,
+                "num_orders": num_orders,
+                "num_riders": num_riders,
+                "max_pickup_distance_km": hyperparams.get(
+                    "max_pickup_distance_km", 10.0
+                ),
+                "max_kitchen_distance_km": hyperparams.get(
+                    "max_kitchen_distance_km", 3.0
+                ),
+                "max_orders_per_rider": max_opr,
+                "ideal_load": hyperparams.get("ideal_load", 6),
+                "workload_balance_threshold": hyperparams.get(
+                    "workload_balance_threshold", 0.7
+                ),
+                "workload_penalty_weight": hyperparams.get(
+                    "workload_penalty_weight", 100.0
+                ),
+                "distance_penalty_weight": hyperparams.get(
+                    "distance_penalty_weight", 2.0
+                ),
+                "cluster_radius_km": hyperparams.get("cluster_radius_km", 3.0),
+                "search_time_limit_seconds": hyperparams.get(
+                    "search_time_limit_seconds", 5
+                ),
+                "road_factor": hyperparams.get("road_factor", 1.3),
+                "ml_strategy": ml_strategy,
+                "riders_used": riders_used,
+                "total_assigned": total_assigned,
+                "unassigned_count": unassigned_count,
+                "avg_load": round(avg_load, 3),
+                "load_std": round(load_std, 3),
+                "total_distance_km": round(total_distance_km, 2),
+                "elapsed_ms": round(elapsed_ms, 1),
+                "sla_breached": sla_breached,
+                "avg_delivery_time_min": round(avg_delivery_time_min or 0.0, 2),
+                "quality_score": round(quality_score, 2),
+            }
+
+            with _WRITE_LOCK:
+                self._insert(row)
+
+            logger.info(
+                f"[MLCollector] zone={zone_id} orders={num_orders} "
+                f"assigned={total_assigned} unassigned={unassigned_count} "
+                f"quality={quality_score:.1f} elapsed={elapsed_ms:.0f}ms"
+            )
+            return round(quality_score, 2)
+
+        except Exception as e:
+            logger.warning(f"[MLCollector] Logging failed (non-fatal): {e}")
+            return 50.0   # neutral fallback so bandit update still fires
+
+    # ------------------------------------------------------------------
+    # Data retrieval for training
+    # ------------------------------------------------------------------
+
+    def get_training_data(
+        self,
+        min_records: int = 30,
+        strategy_filter: Optional[str] = None,
+        since_hours: Optional[int] = None,
+    ) -> Optional[List[Dict[str, Any]]]:
+        """
+        Return logged rows for model training.
+        quality_score is returned AS-IS (frozen at log time - no re-scoring).
+        """
+        try:
+            conn = sqlite3.connect(self._db_path)
+            conn.row_factory = sqlite3.Row
+
+            query = "SELECT * FROM assignment_ml_log"
+            params: list = []
+            clauses: list = []
+
+            if strategy_filter:
+                clauses.append("ml_strategy = ?")
+                params.append(strategy_filter)
+            if since_hours:
+                cutoff = (datetime.utcnow() - timedelta(hours=since_hours)).isoformat()
+                clauses.append("timestamp >= ?")
+                params.append(cutoff)
+
+            if clauses:
+                query += " WHERE " + " AND ".join(clauses)
+            query += " ORDER BY id ASC"
+
+            rows = conn.execute(query, params).fetchall()
+            conn.close()
+
+            if len(rows) < min_records:
+                logger.info(
+                    f"[MLCollector] {len(rows)} records < {min_records} minimum."
+                )
+                return None
+
+            return [dict(r) for r in rows]
+
+        except Exception as e:
+            logger.error(f"[MLCollector] get_training_data failed: {e}")
+            return None
+
+    # ------------------------------------------------------------------
+    # Analytics API
+    # ------------------------------------------------------------------
+
+    def get_recent_quality_trend(self, last_n: int = 50) -> Dict[str, Any]:
+        """Recent quality scores + series for sparkline charts."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                "SELECT quality_score, timestamp, unassigned_count, elapsed_ms "
+                "FROM assignment_ml_log ORDER BY id DESC LIMIT ?",
+                (last_n,),
+            ).fetchall()
+            conn.close()
+            if not rows:
+                return {"avg_quality": 0.0, "sample_size": 0, "history": []}
+            scores = [r[0] for r in rows]
+            return {
+                "avg_quality": round(sum(scores) / len(scores), 2),
+                "min_quality": round(min(scores), 2),
+                "max_quality": round(max(scores), 2),
+                "sample_size": len(scores),
+                "history": list(reversed(scores)),
+                "timestamps": list(reversed([r[1] for r in rows])),
+                "unassigned_series": list(reversed([r[2] for r in rows])),
+                "latency_series": list(reversed([r[3] for r in rows])),
+            }
+        except Exception:
+            return {"avg_quality": 0.0, "sample_size": 0, "history": []}
+
+    def get_hourly_stats(self, last_days: int = 7) -> List[Dict[str, Any]]:
+        """Quality, SLA, and call volume aggregated by hour-of-day."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            cutoff = (datetime.utcnow() - timedelta(days=last_days)).isoformat()
+            rows = conn.execute(
+                """
+                SELECT hour,
+                       COUNT(*)                              AS call_count,
+                       AVG(quality_score)                    AS avg_quality,
+                       AVG(unassigned_count)                 AS avg_unassigned,
+                       AVG(elapsed_ms)                       AS avg_latency_ms,
+                       SUM(CASE WHEN sla_breached=1 THEN 1 ELSE 0 END) AS sla_breaches
+                FROM assignment_ml_log WHERE timestamp >= ?
+                GROUP BY hour ORDER BY hour
+                """,
+                (cutoff,),
+            ).fetchall()
+            conn.close()
+            return [
+                {
+                    "hour": r[0],
+                    "call_count": r[1],
+                    "avg_quality": round(r[2] or 0.0, 2),
+                    "avg_unassigned": round(r[3] or 0.0, 2),
+                    "avg_latency_ms": round(r[4] or 0.0, 1),
+                    "sla_breaches": r[5],
+                }
+                for r in rows
+            ]
+        except Exception as e:
+            logger.error(f"[MLCollector] get_hourly_stats: {e}")
+            return []
+
+    def get_strategy_comparison(self) -> List[Dict[str, Any]]:
+        """Compare quality metrics across ml_strategy values."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                """
+                SELECT ml_strategy,
+                       COUNT(*)                  AS call_count,
+                       AVG(quality_score)         AS avg_quality,
+                       MIN(quality_score)         AS min_quality,
+                       MAX(quality_score)         AS max_quality,
+                       AVG(unassigned_count)      AS avg_unassigned,
+                       AVG(total_distance_km)     AS avg_distance_km,
+                       AVG(elapsed_ms)            AS avg_latency_ms
+                FROM assignment_ml_log
+                GROUP BY ml_strategy ORDER BY avg_quality DESC
+                """
+            ).fetchall()
+            conn.close()
+            return [
+                {
+                    "strategy": r[0],
+                    "call_count": r[1],
+                    "avg_quality": round(r[2] or 0.0, 2),
+                    "min_quality": round(r[3] or 0.0, 2),
+                    "max_quality": round(r[4] or 0.0, 2),
+                    "avg_unassigned": round(r[5] or 0.0, 2),
+                    "avg_distance_km": round(r[6] or 0.0, 2),
+                    "avg_latency_ms": round(r[7] or 0.0, 1),
+                }
+                for r in rows
+            ]
+        except Exception as e:
+            logger.error(f"[MLCollector] get_strategy_comparison: {e}")
+            return []
+
+    def get_quality_histogram(self, bins: int = 10) -> List[Dict[str, Any]]:
+        """Quality score distribution for histogram chart."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                "SELECT quality_score FROM assignment_ml_log"
+            ).fetchall()
+            conn.close()
+            scores = [r[0] for r in rows if r[0] is not None]
+            if not scores:
+                return []
+            bin_width = 100.0 / bins
+            return [
+                {
+                    "range": f"{i * bin_width:.0f}-{(i + 1) * bin_width:.0f}",
+                    "count": sum(
+                        1 for s in scores if i * bin_width <= s < (i + 1) * bin_width
+                    ),
+                }
+                for i in range(bins)
+            ]
+        except Exception as e:
+            logger.error(f"[MLCollector] get_quality_histogram: {e}")
+            return []
+
+    def get_zone_stats(self) -> List[Dict[str, Any]]:
+        """Quality and SLA stats grouped by zone."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                """
+                SELECT zone_id, COUNT(*) AS call_count,
+                       AVG(quality_score) AS avg_quality,
+                       SUM(sla_breached)  AS sla_breaches,
+                       AVG(total_distance_km) AS avg_distance_km
+                FROM assignment_ml_log
+                GROUP BY zone_id ORDER BY avg_quality DESC
+                """
+            ).fetchall()
+            conn.close()
+            return [
+                {
+                    "zone_id": r[0],
+                    "call_count": r[1],
+                    "avg_quality": round(r[2] or 0.0, 2),
+                    "sla_breaches": r[3],
+                    "avg_distance_km": round(r[4] or 0.0, 2),
+                }
+                for r in rows
+            ]
+        except Exception as e:
+            logger.error(f"[MLCollector] get_zone_stats: {e}")
+            return []
+
+    def count_records(self) -> int:
+        try:
+            conn = sqlite3.connect(self._db_path)
+            count = conn.execute("SELECT COUNT(*) FROM assignment_ml_log").fetchone()[0]
+            conn.close()
+            return count
+        except Exception:
+            return 0
+
+    def count_by_strategy(self) -> Dict[str, int]:
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                "SELECT ml_strategy, COUNT(*) FROM assignment_ml_log GROUP BY ml_strategy"
+            ).fetchall()
+            conn.close()
+            return {r[0]: r[1] for r in rows}
+        except Exception:
+            return {}
+
+    def export_csv(self) -> str:
+        """Export all records as CSV string."""
+        try:
+            conn = sqlite3.connect(self._db_path)
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT * FROM assignment_ml_log ORDER BY id ASC"
+            ).fetchall()
+            conn.close()
+            if not rows:
+                return ""
+            buf = io.StringIO()
+            writer = csv.DictWriter(buf, fieldnames=rows[0].keys())
+            writer.writeheader()
+            writer.writerows([dict(r) for r in rows])
+            return buf.getvalue()
+        except Exception as e:
+            logger.error(f"[MLCollector] export_csv failed: {e}")
+            return ""
+
+    def purge_old_records(self, keep_days: int = 90) -> int:
+        """Delete records older than keep_days. Returns count deleted."""
+        try:
+            cutoff = (datetime.utcnow() - timedelta(days=keep_days)).isoformat()
+            conn = sqlite3.connect(self._db_path)
+            cursor = conn.execute(
+                "DELETE FROM assignment_ml_log WHERE timestamp < ?", (cutoff,)
+            )
+            deleted = cursor.rowcount
+            conn.commit()
+            conn.close()
+            logger.info(
+                f"[MLCollector] Purged {deleted} records older than {keep_days} days."
+            )
+            return deleted
+        except Exception as e:
+            logger.error(f"[MLCollector] purge failed: {e}")
+            return 0
+
+    # ------------------------------------------------------------------
+    # Quality Score Formula (frozen at log time - do not change behavior)
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _compute_quality_score(
+        num_orders: int,
+        unassigned_count: int,
+        load_std: float,
+        riders_used: int,
+        num_riders: int,
+        total_distance_km: float,
+        max_orders_per_rider: int,
+        ml_strategy: str = "balanced",
+    ) -> float:
+        """
+        Multi-dimensional quality score (0–100, higher = better).
+
+        Components:
+        ┌──────────────────────┬────────────────────────────────────────────────┐
+        │ assigned_ratio       │ % of orders successfully assigned              │
+        │ distance_ratio       │ inverse of total km (shorter routes = better)  │
+        │ balance_ratio        │ load spread across riders (lower std = better) │
+        │ rider_efficiency     │ reward using minimal riders for the batch size  │
+        └──────────────────────┴────────────────────────────────────────────────┘
+
+        Strategy weights (w_assign, w_dist, w_balance, w_efficiency):
+        - balanced        : (45, 20, 20, 15)
+        - aggressive_speed: (70, 15,  0, 15)  — care about assignment + efficiency
+        - fuel_saver      : (25, 60,  0, 15)  — heavily penalise long routes
+        - zone_strict     : (35, 25, 25, 15)  — balanced with zone awareness
+        """
+        import math
+        if num_orders == 0:
+            return 0.0
+
+        assigned = num_orders - unassigned_count
+        assigned_ratio = assigned / num_orders
+
+        max_std = max(1.0, max_orders_per_rider / 2.0)
+        if riders_used <= 1:
+            balance_ratio = 0.5  # spread is undefined for a single rider; use neutral
+        else:
+            balance_ratio = max(0.0, 1.0 - (load_std / max_std))
+
+        max_dist = max(1.0, float(assigned * 8.0))
+        distance_ratio = max(0.0, 1.0 - (total_distance_km / max_dist))
+
+        # Rider efficiency: 1.0 = used the theoretical minimum; drops as we
+        # use more riders than needed.
+        min_riders_needed = max(1, math.ceil(num_orders / max_orders_per_rider))
+        rider_efficiency = min(1.0, min_riders_needed / max(1, riders_used))
+
+        weights = {
+            #                  assign  dist  balance  efficiency
+            "aggressive_speed": (70.0, 15.0,   0.0,    15.0),
+            "fuel_saver":       (25.0, 60.0,   0.0,    15.0),
+            "zone_strict":      (35.0, 25.0,  25.0,    15.0),
+            "balanced":         (45.0, 20.0,  20.0,    15.0),
+        }
+        w_comp, w_dist, w_bal, w_eff = weights.get(ml_strategy, (45.0, 20.0, 20.0, 15.0))
+
+        return min(
+            assigned_ratio   * w_comp
+            + distance_ratio * w_dist
+            + balance_ratio  * w_bal
+            + rider_efficiency * w_eff,
+            100.0,
+        )
+
+    @staticmethod
+    def _get_km(order: Any) -> float:
+        try:
+            return float(order.get("kms") or order.get("calculationDistanceKm") or 0.0)
+        except Exception:
+            return 0.0
+
+    # ------------------------------------------------------------------
+    # DB Bootstrap
+    # ------------------------------------------------------------------
+
+    def _ensure_db(self) -> None:
+        try:
+            os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
+            conn = sqlite3.connect(self._db_path)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS assignment_ml_log (
+                    id                          INTEGER PRIMARY KEY AUTOINCREMENT,
+                    timestamp                   TEXT    NOT NULL,
+                    hour                        INTEGER,
+                    day_of_week                 INTEGER,
+                    is_peak                     INTEGER DEFAULT 0,
+                    zone_id                     TEXT    DEFAULT 'default',
+                    city_id                     TEXT    DEFAULT 'default',
+                    weather_code                TEXT    DEFAULT 'CLEAR',
+                    num_orders                  INTEGER,
+                    num_riders                  INTEGER,
+                    max_pickup_distance_km      REAL,
+                    max_kitchen_distance_km     REAL,
+                    max_orders_per_rider        INTEGER,
+                    ideal_load                  INTEGER,
+                    workload_balance_threshold  REAL,
+                    workload_penalty_weight     REAL,
+                    distance_penalty_weight     REAL,
+                    cluster_radius_km           REAL,
+                    search_time_limit_seconds   INTEGER,
+                    road_factor                 REAL,
+                    ml_strategy                 TEXT    DEFAULT 'balanced',
+                    riders_used                 INTEGER,
+                    total_assigned              INTEGER,
+                    unassigned_count            INTEGER,
+                    avg_load                    REAL,
+                    load_std                    REAL,
+                    total_distance_km           REAL    DEFAULT 0.0,
+                    elapsed_ms                  REAL,
+                    sla_breached                INTEGER DEFAULT 0,
+                    avg_delivery_time_min       REAL    DEFAULT 0.0,
+                    quality_score               REAL
+                )
+            """)
+            migrations = [
+                "ALTER TABLE assignment_ml_log ADD COLUMN is_peak INTEGER DEFAULT 0",
+                "ALTER TABLE assignment_ml_log ADD COLUMN zone_id TEXT DEFAULT 'default'",
+                "ALTER TABLE assignment_ml_log ADD COLUMN city_id TEXT DEFAULT 'default'",
+                "ALTER TABLE assignment_ml_log ADD COLUMN weather_code TEXT DEFAULT 'CLEAR'",
+                "ALTER TABLE assignment_ml_log ADD COLUMN sla_breached INTEGER DEFAULT 0",
+                "ALTER TABLE assignment_ml_log ADD COLUMN avg_delivery_time_min REAL DEFAULT 0.0",
+                "ALTER TABLE assignment_ml_log ADD COLUMN ml_strategy TEXT DEFAULT 'balanced'",
+                "ALTER TABLE assignment_ml_log ADD COLUMN total_distance_km REAL DEFAULT 0.0",
+            ]
+            for ddl in migrations:
+                try:
+                    conn.execute(ddl)
+                except Exception:
+                    pass
+            for idx in [
+                "CREATE INDEX IF NOT EXISTS idx_timestamp ON assignment_ml_log(timestamp)",
+                "CREATE INDEX IF NOT EXISTS idx_strategy  ON assignment_ml_log(ml_strategy)",
+                "CREATE INDEX IF NOT EXISTS idx_zone      ON assignment_ml_log(zone_id)",
+            ]:
+                conn.execute(idx)
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            logger.error(f"[MLCollector] DB init failed: {e}")
+
+    def _insert(self, row: Dict[str, Any]) -> None:
+        os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
+        conn = sqlite3.connect(self._db_path)
+        cols = ", ".join(row.keys())
+        placeholders = ", ".join(["?"] * len(row))
+        conn.execute(
+            f"INSERT INTO assignment_ml_log ({cols}) VALUES ({placeholders})",
+            list(row.values()),
+        )
+        conn.commit()
+        conn.close()
+
+
+# ---------------------------------------------------------------------------
+# Module-level singleton
+# ---------------------------------------------------------------------------
+
+_collector: Optional[MLDataCollector] = None
+_collector_lock = threading.Lock()
+
+
+def get_collector() -> MLDataCollector:
+    global _collector
+    if _collector is None:
+        with _collector_lock:
+            if _collector is None:
+                _collector = MLDataCollector()
+    return _collector

app/services/ml/strategy_bandit.py

@@ -0,0 +1,277 @@
+"""
+Thompson Sampling Contextual Bandit — Strategy Selector
+=========================================================
+Replaces the greedy SQL auto-tuner with proper online RL.
+
+Problem
+-------
+The old greedy tuner picks the strategy with the highest *average* quality
+across all history. It never explores alternatives once one strategy leads,
+and it ignores context (the same strategy isn't best at all times of day
+and all load levels).
+
+Solution
+--------
+A contextual multi-armed bandit with Thompson Sampling:
+
+  Context = (time_band, load_band)   — up to 7 × 4 = 28 states
+  Arms    = 4 strategies             — balanced / fuel_saver / aggressive_speed / zone_strict
+  Reward  = quality_score / 100      — 0..1 float, already logged by MLDataCollector
+
+How Thompson Sampling works
+---------------------------
+Each (context, arm) pair has a Beta(α, β) posterior where:
+  α = sum of rewards seen so far  (high quality calls push α up)
+  β = sum of "anti-rewards"       (low quality calls push β up)
+
+To SELECT a strategy:
+  1. For each arm, sample θ ~ Beta(α, β)
+  2. Pick arm with highest θ
+  → Naturally balances exploration (uncertain arms get sampled often)
+    with exploitation (well-known good arms dominate when confident)
+
+To UPDATE after an assignment:
+  reward = quality_score / 100
+  α += reward
+  β += (1 - reward)
+
+Bootstrap
+---------
+On first startup the entire historical SQLite DB is replayed to warm up
+posteriors so the bandit starts informed, not blank.
+If saved state already exists it is loaded from disk instead.
+
+Persistence
+-----------
+ml_data/strategy_bandit.json — saved every 10 updates.
+"""
+
+import json
+import logging
+import os
+import threading
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+_SAVE_PATH = os.getenv("BANDIT_PATH", "ml_data/strategy_bandit.json")
+_DB_PATH   = os.getenv("ML_DB_PATH",  "ml_data/ml_store.db")
+
+ARMS = ["balanced", "fuel_saver", "aggressive_speed", "zone_strict"]
+
+
+class ContextualBandit:
+    """
+    Thompson Sampling bandit for ml_strategy selection.
+
+    Context key : "{time_band}|{load_band}"
+    Arms        : ARMS list (4 strategies)
+    Prior       : Beta(1, 1) — uniform, no initial preference
+    """
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        # _posteriors[ctx_key][arm] = [alpha, beta]
+        self._posteriors: Dict[str, Dict[str, List[float]]] = {}
+        self._update_count = 0
+        self._total_pulls  = 0
+        self._load()
+        self._bootstrap_from_db()
+
+    # ── Public API ───────────────────────────────────────────────────────────
+
+    def select(self, time_band: str, load_band: str) -> str:
+        """
+        Sample from Beta posteriors and return the strategy with the highest
+        sample. Explores uncertain arms naturally; exploits known-good arms
+        as confidence grows.
+        """
+        ctx = self._ctx(time_band, load_band)
+        with self._lock:
+            samples = {
+                arm: float(np.random.beta(*self._ab(ctx, arm)))
+                for arm in ARMS
+            }
+            self._total_pulls += 1
+        chosen = max(samples, key=samples.__getitem__)
+        logger.debug(
+            f"[Bandit] ctx={ctx} "
+            f"samples={{{', '.join(f'{k}:{v:.3f}' for k, v in samples.items())}}} "
+            f"→ {chosen}"
+        )
+        return chosen
+
+    def update(self, time_band: str, load_band: str,
+               strategy: str, quality_score: float) -> None:
+        """Update the Beta posterior for (context, arm) after observing quality."""
+        if strategy not in ARMS:
+            return
+        ctx    = self._ctx(time_band, load_band)
+        reward = min(1.0, max(0.0, float(quality_score) / 100.0))
+        with self._lock:
+            ab     = self._ab(ctx, strategy)
+            ab[0] += reward           # alpha  ← quality adds to success mass
+            ab[1] += (1.0 - reward)   # beta   ← (1-quality) adds to failure mass
+            self._update_count += 1
+            should_save = self._update_count % 10 == 0
+        if should_save:
+            self._save()
+
+    def best_arm(self, time_band: str, load_band: str) -> Tuple[str, float]:
+        """
+        Return the arm with the highest posterior mean (pure exploitation, no
+        sampling noise). Used for logging and dashboard, not for live selection.
+        """
+        ctx = self._ctx(time_band, load_band)
+        with self._lock:
+            means = {
+                arm: self._ab(ctx, arm)[0] / sum(self._ab(ctx, arm))
+                for arm in ARMS
+            }
+        best = max(means, key=means.__getitem__)
+        return best, round(means[best], 4)
+
+    def get_stats(self) -> Dict:
+        """Return per-context arm statistics for the ML admin dashboard."""
+        with self._lock:
+            stats: Dict[str, Dict] = {}
+            for ctx, arms in sorted(self._posteriors.items()):
+                stats[ctx] = {}
+                for arm in ARMS:
+                    if arm not in arms:
+                        stats[ctx][arm] = {"mean_reward": 0.5, "observations": 0,
+                                           "alpha": 1.0, "beta": 1.0}
+                        continue
+                    alpha, beta = arms[arm]
+                    n    = alpha + beta - 2.0   # subtract the Beta(1,1) prior mass
+                    mean = alpha / (alpha + beta)
+                    stats[ctx][arm] = {
+                        "mean_reward":  round(mean, 4),
+                        "observations": round(max(0, n), 1),
+                        "alpha":        round(alpha, 2),
+                        "beta":         round(beta, 2),
+                    }
+        return {
+            "total_pulls":  self._total_pulls,
+            "total_updates": self._update_count,
+            "context_count": len(self._posteriors),
+            "arms":          ARMS,
+            "contexts":      stats,
+        }
+
+    # ── Persistence ──────────────────────────────────────────────────────────
+
+    def _save(self) -> None:
+        try:
+            with self._lock:
+                snapshot = {
+                    "posteriors": {
+                        ctx: {arm: list(ab) for arm, ab in arms.items()}
+                        for ctx, arms in self._posteriors.items()
+                    },
+                    "update_count": self._update_count,
+                    "total_pulls":  self._total_pulls,
+                }
+            os.makedirs(os.path.dirname(_SAVE_PATH) or ".", exist_ok=True)
+            with open(_SAVE_PATH, "w", encoding="utf-8") as f:
+                json.dump(snapshot, f, indent=2)
+        except Exception as e:
+            logger.warning(f"[Bandit] Save failed: {e}")
+
+    def _load(self) -> None:
+        try:
+            if not os.path.exists(_SAVE_PATH):
+                logger.info("[Bandit] No saved state — will bootstrap from DB.")
+                return
+            with open(_SAVE_PATH, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            self._posteriors   = data.get("posteriors", {})
+            self._update_count = data.get("update_count", 0)
+            self._total_pulls  = data.get("total_pulls", 0)
+            logger.info(
+                f"[Bandit] Loaded from disk — "
+                f"{len(self._posteriors)} contexts, {self._update_count} updates"
+            )
+        except Exception as e:
+            logger.warning(f"[Bandit] Load failed (starting fresh): {e}")
+
+    # ── Bootstrap ────────────────────────────────────────────────────────────
+
+    def _bootstrap_from_db(self) -> None:
+        """
+        Warm-up posteriors by replaying every historical assignment event.
+        Skipped if the saved JSON already reflects current DB data.
+        """
+        if self._update_count > 0:
+            logger.info(
+                f"[Bandit] Already warmed ({self._update_count} updates) — "
+                "skipping DB bootstrap."
+            )
+            return
+        try:
+            import sqlite3
+            from app.services.ml.behavior_analyzer import (
+                time_band as _tb,
+                load_band as _lb,
+            )
+            conn = sqlite3.connect(_DB_PATH)
+            rows = conn.execute(
+                "SELECT timestamp, avg_load, ml_strategy, quality_score "
+                "FROM assignment_ml_log "
+                "WHERE ml_strategy IS NOT NULL AND quality_score IS NOT NULL "
+                "ORDER BY id ASC"
+            ).fetchall()
+            conn.close()
+
+            if not rows:
+                logger.info("[Bandit] No historical data — starting with uniform priors.")
+                return
+
+            for ts, avg_load, strategy, quality in rows:
+                t_band = _tb(str(ts)) if ts else "UNKNOWN"
+                l_band = _lb(float(avg_load or 0))
+                self.update(t_band, l_band, strategy, float(quality or 50))
+
+            # Reset counter so we don't confuse bootstrap updates with live ones
+            with self._lock:
+                self._update_count = len(rows)
+
+            logger.info(
+                f"[Bandit] Bootstrapped from {len(rows)} historical events — "
+                f"{len(self._posteriors)} contexts, {len(ARMS)} arms."
+            )
+            self._save()
+        except Exception as e:
+            logger.warning(f"[Bandit] Bootstrap failed (non-fatal): {e}")
+
+    # ── Helpers ──────────────────────────────────────────────────────────────
+
+    @staticmethod
+    def _ctx(time_band: str, load_band: str) -> str:
+        return f"{time_band}|{load_band}"
+
+    def _ab(self, ctx: str, arm: str) -> List[float]:
+        """Get or create Beta(1, 1) uniform prior for (ctx, arm). NOT thread-safe alone."""
+        if ctx not in self._posteriors:
+            self._posteriors[ctx] = {}
+        if arm not in self._posteriors[ctx]:
+            self._posteriors[ctx][arm] = [1.0, 1.0]
+        return self._posteriors[ctx][arm]
+
+
+# ── Singleton ────────────────────────────────────────────────────────────────
+
+_bandit_instance: Optional[ContextualBandit] = None
+_bandit_lock = threading.Lock()
+
+
+def get_bandit() -> ContextualBandit:
+    """Return the process-level singleton ContextualBandit."""
+    global _bandit_instance
+    if _bandit_instance is None:
+        with _bandit_lock:
+            if _bandit_instance is None:
+                _bandit_instance = ContextualBandit()
+    return _bandit_instance

app/services/rider/get_active_riders.py

@@ -0,0 +1,78 @@
+
+import httpx
+import logging
+from datetime import datetime
+from typing import List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+async def fetch_active_riders() -> List[Dict[str, Any]]:
+    """
+    Fetch active rider logs from the external API for the current date.
+    Returns a list of rider log dictionaries.
+    """
+    try:
+        today_str = datetime.now().strftime("%Y-%m-%d")
+        url = "https://jupiter.nearle.app/live/api/v2/partners/getriderlogs/"
+        params = {
+            "applocationid": 1,
+            "partnerid": 44,
+            "fromdate": today_str,
+            "todate": today_str,
+            "keyword": ""
+        }
+        
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(url, params=params)
+            response.raise_for_status()
+            data = response.json()
+            
+            if data and data.get("code") == 200 and data.get("details"):
+                 # Filter riders who are in our preferences list and are 'active' or 'idle' (assuming we want online riders)
+                 # The user's example showed "onduty": 1. We might want to filter by that.
+                 # For now, returning all logs, filtering can happen in assignment logic or here.
+                 # Let's return the raw list as requested, filtering logic will be applied during assignment.
+                 return data.get("details", [])
+            
+            logger.warning(f"Fetch active riders returned no details: {data}")
+            return []
+
+    except Exception as e:
+        logger.error(f"Error fetching active riders: {e}", exc_info=True)
+        return []
+
+async def fetch_created_orders() -> List[Dict[str, Any]]:
+    """
+    Fetch all orders in 'created' state for the current date.
+    """
+    try:
+        today_str = datetime.now().strftime("%Y-%m-%d")
+        url = "https://jupiter.nearle.app/live/api/v1/orders/tenant/getorders/"
+        # Removed pagesize as per user request to fetch all
+        params = {
+            "applocationid": 0,
+            "tenantid": 0,
+            "locationid": 0,
+            "status": "created",
+            "fromdate": today_str,
+            "todate": today_str,
+            "keyword": "",
+            "pageno": 1
+            # "pagesize" intentionally omitted to fetch all
+        }
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            response = await client.get(url, params=params)
+            response.raise_for_status()
+            data = response.json()
+            
+            if data and data.get("code") == 200 and data.get("details"):
+                return data.get("details", [])
+            
+            logger.warning(f"Fetch created orders returned no details: {data}")
+            return []
+
+    except Exception as e:
+        logger.error(f"Error fetching created orders: {e}", exc_info=True)
+        return []
+

app/services/rider/rider_history_service.py

@@ -0,0 +1,68 @@
+
+import os
+import pickle
+import logging
+from datetime import datetime
+from typing import Dict, Any
+
+logger = logging.getLogger(__name__)
+
+# Absolute path so this pickle is written to the same location regardless of
+# which directory uvicorn is launched from.
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)
+))))
+HISTORY_FILE = os.path.join(_PROJECT_ROOT, "data", "rider_history.pkl")
+
+class RiderHistoryService:
+    def __init__(self, history_file: str = HISTORY_FILE):
+        self.history_file = history_file
+        self.history = self._load_history()
+
+    def _load_history(self) -> Dict[int, Dict[str, float]]:
+        """Load history from pickle file."""
+        os.makedirs(os.path.dirname(self.history_file), exist_ok=True)
+        if not os.path.exists(self.history_file):
+            return {}
+        
+        try:
+            with open(self.history_file, 'rb') as f:
+                return pickle.load(f)
+        except Exception as e:
+            logger.error(f"Failed to load rider history: {e}")
+            return {}
+
+    def _save_history(self):
+        """Save history to pickle file."""
+        try:
+            with open(self.history_file, 'wb') as f:
+                pickle.dump(self.history, f)
+        except Exception as e:
+            logger.error(f"Failed to save rider history: {e}")
+
+    def update_rider_stats(self, rider_id: int, distance_km: float, order_count: int):
+        """Update cumulative stats for a rider."""
+        rider_id = int(rider_id)
+        if rider_id not in self.history:
+            self.history[rider_id] = {
+                "total_km": 0.0,
+                "total_orders": 0,
+                "last_updated": datetime.now().isoformat()
+            }
+        
+        self.history[rider_id]["total_km"] += distance_km
+        self.history[rider_id]["total_orders"] += order_count
+        self.history[rider_id]["last_updated"] = datetime.now().isoformat()
+        
+        # Auto-save on update
+        self._save_history()
+
+    def get_rider_score(self, rider_id: int) -> float:
+        """
+        Get a score representing the rider's historical 'load' (KMs).
+        Higher Score = More KMs driven recently.
+        """
+        rider_id = int(rider_id)
+        stats = self.history.get(rider_id, {})
+        return stats.get("total_km", 0.0)
+

app/services/rider/rider_state_manager.py

@@ -0,0 +1,120 @@
+import os
+import pickle
+import logging
+import threading
+import time
+from datetime import datetime
+from typing import Dict, Any, List, Set
+
+logger = logging.getLogger(__name__)
+
+# Use an absolute path anchored to the project root (two levels up from this file)
+# so the pickle is always written to the same place regardless of the working
+# directory at startup time.
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)
+))))
+STATE_FILE = os.path.join(_PROJECT_ROOT, "data", "rider_active_state.pkl")
+_FILE_LOCK = threading.Lock()
+
+
+class RiderStateManager:
+    """
+    Manages the 'Short-Term' Active State of Riders for session persistence.
+    Tracks:
+    - Minutes Committed (Remaining Workload)
+    - Active Kitchens (Unique Pickups in current queue)
+    - Last Planned Drop Location (for Daisy Chaining)
+    - Timestamp of last update (for Time Decay)
+    """
+    def __init__(self, state_file: str = STATE_FILE):
+        self.state_file = state_file
+        self.states = self._load_states()
+
+    def _load_states(self) -> Dict[str, Any]:
+        """Load states from pickle."""
+        os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
+        if not os.path.exists(self.state_file):
+            return {}
+        try:
+            with _FILE_LOCK:
+                with open(self.state_file, 'rb') as f:
+                    return pickle.load(f)
+        except Exception as e:
+            logger.error(f"Failed to load rider active states: {e}")
+            return {}
+
+    def _save_states(self):
+        """Save states to pickle."""
+        try:
+            with _FILE_LOCK:
+                with open(self.state_file, 'wb') as f:
+                    pickle.dump(self.states, f)
+        except Exception as e:
+            logger.error(f"Failed to save rider active states: {e}")
+
+    def get_rider_state(self, rider_id: int) -> Dict[str, Any]:
+        """
+        Get the current active state of a rider with TIME DECAY applied.
+        If the server restarts after 30 mins, the 'minutes_committed' should reduce by 30.
+        """
+        rider_id = int(rider_id)
+        raw_state = self.states.get(rider_id)
+        
+        if not raw_state:
+            return {
+                'minutes_remaining': 0.0,
+                'last_drop_lat': None,
+                'last_drop_lon': None,
+                'active_kitchens': set(),
+                'last_updated_ts': time.time()
+            }
+        
+        # Apply Time Decay
+        last_ts = raw_state.get('last_updated_ts', time.time())
+        current_ts = time.time()
+        elapsed_mins = (current_ts - last_ts) / 60.0
+        
+        remaining = max(0.0, raw_state.get('minutes_remaining', 0.0) - elapsed_mins)
+        
+        # If queue is empty, kitchens are cleared
+        kitchens = raw_state.get('active_kitchens', set())
+        if remaining <= 5.0: # Buffer: if almost done, free up kitchens
+            kitchens = set()
+            
+        return {
+            'minutes_remaining': remaining,
+            'last_drop_lat': raw_state.get('last_drop_lat'),
+            'last_drop_lon': raw_state.get('last_drop_lon'),
+            'active_kitchens': kitchens,
+            'last_updated_ts': current_ts
+        }
+
+    def update_rider_state(self, rider_id: int, added_minutes: float, new_kitchens: Set[str], last_lat: float, last_lon: float):
+        """
+        Update the state after a new assignment.
+        """
+        rider_id = int(rider_id)
+        
+        # Get current state (decayed)
+        current = self.get_rider_state(rider_id)
+        
+        # Accumulate
+        updated_minutes = current['minutes_remaining'] + added_minutes
+        updated_kitchens = current['active_kitchens'].union(new_kitchens)
+        
+        self.states[rider_id] = {
+            'minutes_remaining': updated_minutes,
+            'last_drop_lat': last_lat,
+            'last_drop_lon': last_lon,
+            'active_kitchens': updated_kitchens,
+            'last_updated_ts': time.time()
+        }
+        
+        self._save_states()
+
+    def clear_state(self, rider_id: int):
+        rider_id = int(rider_id)
+        if rider_id in self.states:
+            del self.states[rider_id]
+            self._save_states()

app/services/rider/substitution_service.py

@@ -0,0 +1,163 @@
+"""
+Rider Substitution Service
+==========================
+Stores ops-team substitution records: when rider X is absent, rider Y
+(the sub) covers X's kitchens and zone for that day.
+
+On each dispatch call, AssignmentService loads today's map and merges the
+absent rider's kitchen + zone affinity into the sub rider's profile for
+that request only.  No server restart needed.  Automatically reverts the
+next day because the lookup is date-keyed.
+
+Storage: SQLite at data/substitutions.db
+"""
+
+import logging
+import os
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+from zoneinfo import ZoneInfo
+
+logger = logging.getLogger(__name__)
+
+# All date comparisons use IST so the substitution fires on the correct
+# calendar day regardless of what timezone the server (Docker/UTC) runs in.
+_IST = ZoneInfo("Asia/Kolkata")
+
+
+def _today_ist() -> str:
+    """Current date in IST as YYYY-MM-DD string."""
+    return datetime.now(_IST).date().isoformat()
+
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)
+))))
+_DB_PATH = os.path.join(_PROJECT_ROOT, "data", "substitutions.db")
+
+_DDL = """
+CREATE TABLE IF NOT EXISTS rider_substitutions (
+    sub_date        TEXT    NOT NULL,
+    absent_rider_id INTEGER NOT NULL,
+    sub_rider_id    INTEGER NOT NULL,
+    created_at      TEXT    NOT NULL,
+    PRIMARY KEY (sub_date, absent_rider_id)
+);
+"""
+
+
+class SubstitutionService:
+    def __init__(self, db_path: str = _DB_PATH):
+        self._db_path = db_path
+        self._lock = threading.Lock()
+        # In-memory cache for today's substitution map.
+        # Keyed by IST date string so it auto-invalidates at midnight IST.
+        self._cache: Dict[int, int] = {}
+        self._cache_date: str = ""
+        self._init_db()
+
+    def _init_db(self):
+        os.makedirs(os.path.dirname(self._db_path), exist_ok=True)
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute(_DDL)
+            conn.commit()
+
+    def _conn(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _invalidate_cache(self):
+        """Clear the in-memory cache so the next get_today_map() re-reads from DB."""
+        self._cache_date = ""
+
+    def register(self, sub_date: str, absent_rider_id: int, sub_rider_id: int) -> Dict:
+        """Register or update a substitution for a given date."""
+        created_at = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            with self._conn() as conn:
+                conn.execute(
+                    """
+                    INSERT INTO rider_substitutions
+                        (sub_date, absent_rider_id, sub_rider_id, created_at)
+                    VALUES (?, ?, ?, ?)
+                    ON CONFLICT(sub_date, absent_rider_id)
+                        DO UPDATE SET sub_rider_id = excluded.sub_rider_id,
+                                      created_at   = excluded.created_at
+                    """,
+                    (sub_date, int(absent_rider_id), int(sub_rider_id), created_at),
+                )
+                conn.commit()
+            self._invalidate_cache()
+        logger.info(
+            f"[Substitution] Registered: {sub_date} — absent={absent_rider_id} sub={sub_rider_id}"
+        )
+        return {
+            "sub_date": sub_date,
+            "absent_rider_id": absent_rider_id,
+            "sub_rider_id": sub_rider_id,
+            "created_at": created_at,
+        }
+
+    def cancel(self, sub_date: str, absent_rider_id: int) -> bool:
+        """Remove a substitution record. Returns True if a row was deleted."""
+        with self._lock:
+            with self._conn() as conn:
+                cur = conn.execute(
+                    "DELETE FROM rider_substitutions WHERE sub_date=? AND absent_rider_id=?",
+                    (sub_date, int(absent_rider_id)),
+                )
+                conn.commit()
+            self._invalidate_cache()
+            return cur.rowcount > 0
+
+    def get_today_map(self) -> Dict[int, int]:
+        """
+        Return {absent_rider_id: sub_rider_id} for today (IST).
+
+        Result is cached in memory — one SQLite read per IST day (or after any
+        register/cancel call).  Every dispatch call after the first is a plain
+        dict lookup with no DB I/O.
+        """
+        today = _today_ist()
+        with self._lock:
+            if self._cache_date == today:
+                return dict(self._cache)
+            # Cache miss: query DB once, store result
+            with self._conn() as conn:
+                rows = conn.execute(
+                    "SELECT absent_rider_id, sub_rider_id FROM rider_substitutions WHERE sub_date=?",
+                    (today,),
+                ).fetchall()
+            self._cache = {int(r["absent_rider_id"]): int(r["sub_rider_id"]) for r in rows}
+            self._cache_date = today
+            if self._cache:
+                logger.info(f"[Substitution] Cache loaded for {today}: {self._cache}")
+            return dict(self._cache)
+
+    def list_all(self, from_date: Optional[str] = None) -> List[Dict]:
+        """Return all substitutions on or after from_date (default: today), ordered by date."""
+        cutoff = from_date or _today_ist()
+        with self._conn() as conn:
+            rows = conn.execute(
+                """SELECT sub_date, absent_rider_id, sub_rider_id, created_at
+                   FROM rider_substitutions
+                   WHERE sub_date >= ?
+                   ORDER BY sub_date, absent_rider_id""",
+                (cutoff,),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+
+_service: Optional[SubstitutionService] = None
+_service_lock = threading.Lock()
+
+
+def get_substitution_service() -> SubstitutionService:
+    global _service
+    if _service is None:
+        with _service_lock:
+            if _service is None:
+                _service = SubstitutionService()
+    return _service

app/services/routing/batch_efficiency.py

@@ -0,0 +1,860 @@
+"""
+Batch Efficiency Analyser
+=========================
+Pure-function service that analyses a delivery batch and returns:
+
+  - fleet_summary        : aggregate metrics + load-balance scores
+  - rider_timelines      : per-rider start/finish/pace/utilisation
+  - substitution_opportunities : ranked, scored transfer plans
+  - top_recommendation   : best action with confidence, root-cause & risk factors
+
+No DB access here.  The route handler owns fetching; this layer is
+fully testable with any list of dicts.
+
+Expected delivery dict keys (all optional except userid):
+  userid          : int   rider id
+  pickupcustomer  : str   e.g. "Daily Grubs Bhuvaneshwari"
+  assigntime      : str   "YYYY-MM-DD HH:MM:SS"
+  pickuptime      : str   "YYYY-MM-DD HH:MM:SS" or None
+  deliverytime    : str   "YYYY-MM-DD HH:MM:SS" or None
+  dlat / droplat  : float delivery lat (either key accepted)
+  dlon / droplon  : float delivery lon (either key accepted)
+  deliveryid      : int   order id (for transfer manifests)
+"""
+
+from __future__ import annotations
+
+import math
+import statistics
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Defaults
+# ---------------------------------------------------------------------------
+
+DEFAULT_KITCHEN_COORDS: dict[str, tuple[float, float]] = {
+    "vidhya":        (11.01633, 77.01478),
+    "jayanthi":      (11.03887, 76.93008),
+    "nandhini":      (11.04324, 77.00068),
+    "bhuvaneshwari": (11.00352, 76.95455),
+    "selvarani":     (10.99274, 77.00535),
+}
+
+DEFAULT_KITCHEN_FRAGMENTS: list[str] = [
+    "bhuvaneshwari", "jayanthi", "nandhini", "vidhya", "selvarani"
+]
+
+DEFAULT_ROAD_KMH: float = 13.0
+DEFAULT_IDLE_THRESHOLD_MIN: float = 30.0
+DEFAULT_MAX_TRANSFER: int = 4
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+def _hav(la1: float, lo1: float, la2: float, lo2: float) -> float:
+    R = 6371.0
+    la1, lo1, la2, lo2 = map(math.radians, [la1, lo1, la2, lo2])
+    a = (math.sin((la2 - la1) / 2) ** 2
+         + math.cos(la1) * math.cos(la2) * math.sin((lo2 - lo1) / 2) ** 2)
+    return R * 2 * math.asin(math.sqrt(max(0.0, min(1.0, a))))
+
+
+def _travel_min(km: float, kmh: float = DEFAULT_ROAD_KMH) -> float:
+    return km / kmh * 60.0
+
+
+def _parse_ts(s: Any) -> datetime | None:
+    if not s:
+        return None
+    s = str(s).strip()
+    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M"):
+        try:
+            return datetime.strptime(s, fmt)
+        except ValueError:
+            continue
+    return None
+
+
+def _fmt(dt: datetime | None) -> str | None:
+    return dt.strftime("%H:%M:%S") if dt else None
+
+
+def _get_coord(order: dict) -> tuple[float, float] | None:
+    lat = order.get("dlat") or order.get("droplat") or order.get("deliverylat")
+    lon = order.get("dlon") or order.get("droplon") or order.get("deliverylong")
+    try:
+        return float(lat), float(lon)
+    except (TypeError, ValueError):
+        return None
+
+
+def _detect_kitchen(
+    pickupcustomer: str | None,
+    fragments: list[str],
+) -> str | None:
+    kl = (pickupcustomer or "").lower()
+    for frag in fragments:
+        if frag in kl:
+            return frag
+    return None
+
+
+def _stdev(values: list[float]) -> float:
+    """Population stdev; returns 0 for fewer than 2 values."""
+    if len(values) < 2:
+        return 0.0
+    try:
+        return statistics.stdev(values)
+    except Exception:
+        return 0.0
+
+
+def _score_candidate(
+    o: dict,
+    arrive_at_kitchen: datetime,
+    k_coord: tuple[float, float],
+    road_kmh: float,
+) -> float:
+    """
+    Score a candidate order for transfer to the idle rider.
+    Higher = better candidate.
+
+    Two components:
+      time_gain  : minutes saved vs the original delivery time.
+                   Positive means idle rider genuinely arrives earlier.
+      geo_penalty: haversine distance from kitchen to the order drop point.
+                   Penalises far orders that inflate the idle rider's extra km.
+
+    Orders with negative time_gain (idle rider would be slower) still get a
+    score, allowing the caller to filter them out with a feasibility check.
+    """
+    d_ts = _parse_ts(o.get("deliverytime"))
+    coord = _get_coord(o)
+    if not d_ts or not coord:
+        return -9999.0
+    dist_km = _hav(k_coord[0], k_coord[1], coord[0], coord[1])
+    est_deliver = arrive_at_kitchen + timedelta(minutes=_travel_min(dist_km, road_kmh))
+    time_gain_min = (d_ts - est_deliver).total_seconds() / 60.0
+    # Weight: time gain matters more than geography (70/30 split).
+    # Penalise 2 min per km of extra distance so nearby clusters float up.
+    return time_gain_min * 0.7 - dist_km * 2.0 * 0.3
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def analyse_batch(
+    deliveries: list[dict],
+    rider_names: dict[int, str] | None = None,
+    kitchen_coords: dict[str, tuple[float, float]] | None = None,
+    kitchen_fragments: list[str] | None = None,
+    road_kmh: float = DEFAULT_ROAD_KMH,
+    idle_threshold_min: float = DEFAULT_IDLE_THRESHOLD_MIN,
+    max_transfer: int = DEFAULT_MAX_TRANSFER,
+) -> dict:
+    """
+    Analyse a delivery batch and return substitution opportunities.
+
+    Parameters
+    ----------
+    deliveries        : list of order dicts (see module docstring)
+    rider_names       : optional {userid: name} override
+    kitchen_coords    : optional kitchen pickup coordinates override
+    kitchen_fragments : optional kitchen detection strings override
+    road_kmh          : estimated loaded-rider road speed
+    idle_threshold_min: minimum idle window to flag as opportunity
+    max_transfer      : maximum orders to suggest transferring to one rider
+    """
+    if kitchen_coords is None:
+        kitchen_coords = DEFAULT_KITCHEN_COORDS
+    if kitchen_fragments is None:
+        kitchen_fragments = DEFAULT_KITCHEN_FRAGMENTS
+    if rider_names is None:
+        rider_names = {}
+
+    # ------------------------------------------------------------------
+    # 1. Group deliveries by rider
+    # ------------------------------------------------------------------
+    by_rider: dict[int, list[dict]] = defaultdict(list)
+    for o in deliveries:
+        uid = o.get("userid")
+        if uid is not None:
+            by_rider[int(uid)].append(o)
+
+    if not by_rider:
+        return {
+            "fleet_summary": {},
+            "rider_timelines": [],
+            "substitution_opportunities": [],
+            "top_recommendation": None,
+            "error": "No deliveries with valid userid found.",
+        }
+
+    # ------------------------------------------------------------------
+    # 2. Per-rider timeline
+    # ------------------------------------------------------------------
+    timelines: list[dict] = []
+
+    for uid, orders in by_rider.items():
+        # Detect primary kitchen by majority vote
+        kitchen_votes: dict[str, int] = defaultdict(int)
+        for o in orders:
+            k = _detect_kitchen(o.get("pickupcustomer"), kitchen_fragments)
+            if k:
+                kitchen_votes[k] += 1
+        primary_kitchen = (
+            max(kitchen_votes, key=kitchen_votes.get) if kitchen_votes else None
+        )
+        kitchen_confidence = (
+            round(kitchen_votes[primary_kitchen] / len(orders), 2)
+            if primary_kitchen else 0.0
+        )
+
+        # Timestamps
+        finish_ts = None
+        start_ts = None
+        last_coord: tuple[float, float] | None = None
+        completed_orders = 0
+
+        for o in orders:
+            a = _parse_ts(o.get("assigntime"))
+            d = _parse_ts(o.get("deliverytime"))
+            if a and (start_ts is None or a < start_ts):
+                start_ts = a
+            if d:
+                completed_orders += 1
+                if finish_ts is None or d > finish_ts:
+                    finish_ts = d
+                    coord = _get_coord(o)
+                    if coord:
+                        last_coord = coord
+
+        # Active duration and pace
+        active_minutes: float | None = None
+        pace_orders_per_hour: float | None = None
+        if start_ts and finish_ts and finish_ts > start_ts:
+            active_minutes = round((finish_ts - start_ts).total_seconds() / 60, 1)
+            pace_orders_per_hour = round(completed_orders / (active_minutes / 60), 1) if active_minutes else None
+
+        timelines.append({
+            "userid": uid,
+            "name": rider_names.get(uid, f"Rider {uid}"),
+            "kitchen": primary_kitchen,
+            "kitchen_confidence": kitchen_confidence,
+            "order_count": len(orders),
+            "completed_orders": completed_orders,
+            "pending_orders": len(orders) - completed_orders,
+            "started_at": _fmt(start_ts),
+            "finished_at": _fmt(finish_ts),
+            "active_minutes": active_minutes,
+            "pace_orders_per_hour": pace_orders_per_hour,
+            "_finish_dt": finish_ts,
+            "_start_dt": start_ts,
+            "last_position": (
+                {"lat": round(last_coord[0], 6), "lon": round(last_coord[1], 6)}
+                if last_coord else None
+            ),
+            "_last_coord": last_coord,
+        })
+
+    # Sort by finish time
+    timelines.sort(key=lambda t: t["_finish_dt"] or datetime.min)
+
+    # ------------------------------------------------------------------
+    # 3. Fleet summary
+    # ------------------------------------------------------------------
+    valid_start = [t["_start_dt"] for t in timelines if t["_start_dt"]]
+    valid_finish = [t["_finish_dt"] for t in timelines if t["_finish_dt"]]
+
+    fleet_start = min(valid_start) if valid_start else None
+    fleet_done = max(valid_finish) if valid_finish else None
+
+    # Load balance: stdev of order counts and finish-time spread
+    order_counts = [t["order_count"] for t in timelines]
+    finish_offsets_min = (
+        [(f - fleet_start).total_seconds() / 60 for f in valid_finish]
+        if fleet_start and valid_finish else []
+    )
+    finish_spread_min = (
+        round((max(valid_finish) - min(valid_finish)).total_seconds() / 60)
+        if len(valid_finish) >= 2 else 0
+    )
+    load_balance_stdev = round(_stdev([float(c) for c in order_counts]), 2)
+    finish_time_stdev = round(_stdev(finish_offsets_min), 1)
+
+    # Utilisation: how much of the batch window each rider was actively delivering
+    batch_duration_min = (
+        round((fleet_done - fleet_start).total_seconds() / 60)
+        if fleet_start and fleet_done else None
+    )
+    avg_active_minutes = (
+        round(
+            sum(t["active_minutes"] for t in timelines if t["active_minutes"])
+            / max(1, sum(1 for t in timelines if t["active_minutes"])),
+            1,
+        )
+        if any(t["active_minutes"] for t in timelines) else None
+    )
+    avg_utilisation_pct = (
+        round(avg_active_minutes / batch_duration_min * 100, 1)
+        if avg_active_minutes and batch_duration_min else None
+    )
+
+    fleet_summary = {
+        "total_orders": len(deliveries),
+        "total_riders": len(by_rider),
+        "fleet_start": _fmt(fleet_start),
+        "fleet_done": _fmt(fleet_done),
+        "total_duration_minutes": batch_duration_min,
+        "orders_per_rider_avg": round(len(deliveries) / len(by_rider), 1),
+        "load_balance_stdev": load_balance_stdev,
+        "finish_time_spread_minutes": finish_spread_min,
+        "finish_time_stdev_minutes": finish_time_stdev,
+        "avg_utilisation_pct": avg_utilisation_pct,
+        "avg_active_minutes": avg_active_minutes,
+    }
+
+    if not fleet_done:
+        return {
+            "fleet_summary": fleet_summary,
+            "rider_timelines": _clean_timelines(timelines, fleet_done),
+            "substitution_opportunities": [],
+            "top_recommendation": None,
+            "error": "No completed deliveries found.",
+        }
+
+    # Mark each rider's idle status and free window
+    for t in timelines:
+        fd = t["_finish_dt"]
+        if fd:
+            idle_min = (fleet_done - fd).total_seconds() / 60.0
+            t["idle_minutes"] = round(idle_min)
+            t["free_window_minutes"] = round(idle_min)  # time available for substitution
+            t["status"] = "idle" if idle_min >= idle_threshold_min else "active"
+        else:
+            t["idle_minutes"] = 0
+            t["free_window_minutes"] = 0
+            t["status"] = "unknown"
+
+    # ------------------------------------------------------------------
+    # 4. Substitution opportunities
+    # ------------------------------------------------------------------
+    opportunities: list[dict] = []
+
+    idle_riders = [t for t in timelines if t["status"] == "idle" and t["_last_coord"]]
+
+    for idle in idle_riders:
+        idle_uid = idle["userid"]
+        idle_finish = idle["_finish_dt"]
+        idle_coord = idle["_last_coord"]
+        free_window_min = idle["free_window_minutes"]
+
+        if idle["kitchen"] is None:
+            continue  # can't determine origin kitchen
+
+        for target_kitchen, k_coord in kitchen_coords.items():
+            if target_kitchen == idle["kitchen"]:
+                continue  # skip own kitchen
+
+            # Travel from idle rider's last drop to the target kitchen
+            travel_km = _hav(idle_coord[0], idle_coord[1], k_coord[0], k_coord[1])
+            travel_min = _travel_min(travel_km, road_kmh)
+
+            # Skip if idle rider can't even reach the kitchen before fleet is done
+            if travel_min >= free_window_min:
+                continue
+
+            arrive_at_kitchen = idle_finish + timedelta(minutes=travel_min)
+
+            # ----------------------------------------------------------
+            # Candidate selection: score every order from this kitchen
+            # that was delivered after the idle rider could arrive.
+            # Score = time_gain (70%) + geo proximity (30%).
+            # This prefers orders where idle rider is genuinely faster
+            # AND that are close to the kitchen (less detour).
+            # ----------------------------------------------------------
+            candidate_orders: list[tuple[int, dict, float]] = []  # (rid, order, score)
+            for rid, r_orders in by_rider.items():
+                if rid == idle_uid:
+                    continue
+                r_kitchen = next(
+                    (t["kitchen"] for t in timelines if t["userid"] == rid), None
+                )
+                if r_kitchen != target_kitchen:
+                    continue
+                for o in r_orders:
+                    d_ts = _parse_ts(o.get("deliverytime"))
+                    if not d_ts or d_ts <= arrive_at_kitchen:
+                        continue
+                    score = _score_candidate(o, arrive_at_kitchen, k_coord, road_kmh)
+                    candidate_orders.append((rid, o, score))
+
+            if not candidate_orders:
+                continue
+
+            # Sort best candidates first (highest score = most time gained, closest)
+            candidate_orders.sort(key=lambda x: x[2], reverse=True)
+            take_pool = candidate_orders[:max_transfer]
+
+            # Greedy nearest-neighbour route from the kitchen through selected orders
+            unvisited = list(range(len(take_pool)))
+            curr_nn = k_coord
+            greedy_take: list[tuple[int, dict]] = []
+            while unvisited:
+                ni = min(
+                    unvisited,
+                    key=lambda i: (
+                        _hav(curr_nn[0], curr_nn[1], *c)
+                        if (c := _get_coord(take_pool[i][1])) else 999.0
+                    ),
+                )
+                unvisited.remove(ni)
+                greedy_take.append((take_pool[ni][0], take_pool[ni][1]))
+                coord = _get_coord(take_pool[ni][1])
+                if coord:
+                    curr_nn = coord
+
+            # Simulate idle rider executing the greedy route
+            curr_pos = k_coord
+            est_time = arrive_at_kitchen
+            transfer_manifests: list[dict] = []
+            total_delivery_leg_min = 0.0
+
+            for orig_rid, o in greedy_take:
+                coord = _get_coord(o)
+                if not coord:
+                    continue
+                d_km = _hav(curr_pos[0], curr_pos[1], coord[0], coord[1])
+                d_min = _travel_min(d_km, road_kmh)
+                total_delivery_leg_min += d_min
+                est_deliver = est_time + timedelta(minutes=d_min)
+                orig_deliver = _parse_ts(o.get("deliverytime"))
+
+                improvement = (
+                    round((orig_deliver - est_deliver).total_seconds() / 60)
+                    if orig_deliver and est_deliver else None
+                )
+                is_feasible = improvement is not None and improvement > 0
+
+                transfer_manifests.append({
+                    "deliveryid": o.get("deliveryid"),
+                    "from_rider_id": orig_rid,
+                    "from_rider_name": rider_names.get(orig_rid, f"Rider {orig_rid}"),
+                    "original_delivery_time": _fmt(orig_deliver),
+                    "estimated_delivery_time": _fmt(est_deliver),
+                    "improvement_minutes": improvement,
+                    "is_feasible": is_feasible,
+                    "location": {"lat": round(coord[0], 6), "lon": round(coord[1], 6)},
+                })
+                curr_pos = coord
+                est_time = est_deliver
+
+            # Only keep orders where idle rider is actually faster
+            feasible_manifests = [m for m in transfer_manifests if m["is_feasible"]]
+            if not feasible_manifests:
+                continue
+
+            idle_new_finish = est_time
+
+            # Check idle rider can complete within their free window
+            total_obligation_min = travel_min + total_delivery_leg_min
+            if total_obligation_min > free_window_min:
+                # Idle rider would finish after fleet, extending rather than helping
+                continue
+
+            # New fleet done after the transfer
+            taken_order_objs = {id(o) for (_, o) in greedy_take}
+
+            new_finish_by_rider: dict[int, datetime | None] = {}
+            for rid, r_orders in by_rider.items():
+                if rid == idle_uid:
+                    new_finish_by_rider[rid] = idle_new_finish
+                    continue
+                remaining = [o for o in r_orders if id(o) not in taken_order_objs]
+                finishes = [_parse_ts(o.get("deliverytime")) for o in remaining]
+                valid = [f for f in finishes if f]
+                new_finish_by_rider[rid] = max(valid) if valid else None
+
+            new_fleet_done = max(
+                (v for v in new_finish_by_rider.values() if v),
+                default=fleet_done,
+            )
+            fleet_improvement = round(
+                (fleet_done - new_fleet_done).total_seconds() / 60
+            )
+
+            # Most relieved rider
+            orig_last_by_rider = {
+                rid: max(
+                    (f for f in [_parse_ts(o.get("deliverytime")) for o in r_o] if f),
+                    default=None,
+                )
+                for rid, r_o in by_rider.items()
+            }
+            most_impacted_rid = max(
+                (rid for rid in {r for r, _ in greedy_take}),
+                key=lambda r: (orig_last_by_rider.get(r) or datetime.min),
+                default=None,
+            )
+            orig_overloaded_finish = orig_last_by_rider.get(most_impacted_rid)
+            new_overloaded_finish = new_finish_by_rider.get(most_impacted_rid)
+            time_saved = (
+                round((orig_overloaded_finish - new_overloaded_finish).total_seconds() / 60)
+                if orig_overloaded_finish and new_overloaded_finish else 0
+            )
+
+            # Total extra km for idle rider (idle→kitchen + all delivery legs)
+            total_extra_km = travel_km + sum(
+                _hav(
+                    (k_coord if i == 0 else (_get_coord(greedy_take[i-1][1]) or k_coord))[0],
+                    (k_coord if i == 0 else (_get_coord(greedy_take[i-1][1]) or k_coord))[1],
+                    *(_get_coord(o) or k_coord),
+                )
+                for i, (_, o) in enumerate(greedy_take)
+                if _get_coord(o)
+            )
+
+            # ----------------------------------------------------------
+            # Confidence score (0-100)
+            # Measures how comfortable this transfer is given real constraints.
+            #
+            # Component 1 – Slack ratio: how much free time the idle rider
+            #   has beyond the time they'll spend doing the transfer.
+            #   (free_window - total_obligation) / free_window → 0..1
+            #
+            # Component 2 – Feasibility ratio: what fraction of the
+            #   transferred orders actually deliver earlier than original.
+            #   feasible_count / total_transferred → 0..1
+            #
+            # Component 3 – Fleet gain ratio: minutes saved as a fraction
+            #   of total batch duration.  Capped at 20 min improvement for
+            #   full score so small batches don't produce inflated scores.
+            # ----------------------------------------------------------
+            slack_ratio = max(0.0, (free_window_min - total_obligation_min) / free_window_min)
+            feasibility_ratio = len(feasible_manifests) / max(1, len(transfer_manifests))
+            fleet_gain_ratio = min(1.0, fleet_improvement / 20.0) if fleet_improvement > 0 else 0.0
+
+            confidence_score = round(
+                (slack_ratio * 0.4 + feasibility_ratio * 0.35 + fleet_gain_ratio * 0.25) * 100
+            )
+
+            efficiency_ratio = (
+                round(fleet_improvement / max(0.1, total_extra_km), 2)
+                if total_extra_km > 0 else 0.0
+            )
+            if efficiency_ratio >= 5:
+                efficiency_rating = "high"
+            elif efficiency_ratio >= 2:
+                efficiency_rating = "medium"
+            else:
+                efficiency_rating = "low"
+
+            opportunities.append({
+                "idle_rider": {
+                    "userid": idle_uid,
+                    "name": idle["name"],
+                    "primary_kitchen": idle["kitchen"],
+                    "order_count": idle["order_count"],
+                    "finished_at": _fmt(idle_finish),
+                    "idle_minutes": idle["idle_minutes"],
+                    "free_window_minutes": free_window_min,
+                    "last_position": idle["last_position"],
+                },
+                "target_kitchen": target_kitchen,
+                "travel_to_kitchen_km": round(travel_km, 1),
+                "travel_to_kitchen_minutes": round(travel_min),
+                "arrive_at_kitchen": _fmt(arrive_at_kitchen),
+                "orders_to_transfer": transfer_manifests,
+                "total_orders_transferred": len(feasible_manifests),
+                "feasible_orders_count": len(feasible_manifests),
+                "most_relieved_rider": {
+                    "userid": most_impacted_rid,
+                    "name": rider_names.get(most_impacted_rid, f"Rider {most_impacted_rid}"),
+                    "original_finish": _fmt(orig_overloaded_finish),
+                    "new_finish": _fmt(new_overloaded_finish),
+                    "time_saved_minutes": time_saved,
+                },
+                "extra_km_for_idle_rider": round(total_extra_km, 1),
+                "total_obligation_minutes": round(total_obligation_min),
+                "idle_rider_new_finish": _fmt(idle_new_finish),
+                "original_fleet_done": _fmt(fleet_done),
+                "new_fleet_done": _fmt(new_fleet_done),
+                "fleet_improvement_minutes": fleet_improvement,
+                "confidence_score": confidence_score,
+                "efficiency_ratio": efficiency_ratio,
+                "efficiency_rating": efficiency_rating,
+            })
+
+    # Keep only net-positive opportunities
+    opportunities = [o for o in opportunities if o["fleet_improvement_minutes"] > 0]
+
+    # Sort: confidence first (overall quality), then fleet improvement, then rider time saved
+    opportunities.sort(
+        key=lambda x: (
+            -x["confidence_score"],
+            -x["fleet_improvement_minutes"],
+            -x["most_relieved_rider"]["time_saved_minutes"],
+        )
+    )
+
+    # ------------------------------------------------------------------
+    # 5. Top recommendation
+    # ------------------------------------------------------------------
+    top_recommendation = _build_recommendation(
+        opportunities, idle_threshold_min, fleet_done, timelines, fleet_summary
+    )
+
+    return {
+        "fleet_summary": fleet_summary,
+        "rider_timelines": _clean_timelines(timelines, fleet_done),
+        "substitution_opportunities": opportunities,
+        "top_recommendation": top_recommendation,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Helpers for clean output
+# ---------------------------------------------------------------------------
+
+def _clean_timelines(
+    timelines: list[dict],
+    fleet_done: datetime | None,
+) -> list[dict]:
+    out = []
+    for t in timelines:
+        out.append({
+            "userid": t["userid"],
+            "name": t["name"],
+            "kitchen": t["kitchen"],
+            "kitchen_confidence": t.get("kitchen_confidence", 0.0),
+            "order_count": t["order_count"],
+            "completed_orders": t.get("completed_orders", t["order_count"]),
+            "pending_orders": t.get("pending_orders", 0),
+            "started_at": t["started_at"],
+            "finished_at": t["finished_at"],
+            "active_minutes": t.get("active_minutes"),
+            "pace_orders_per_hour": t.get("pace_orders_per_hour"),
+            "idle_minutes": t.get("idle_minutes", 0),
+            "free_window_minutes": t.get("free_window_minutes", 0),
+            "status": t.get("status", "unknown"),
+            "last_position": t.get("last_position"),
+        })
+    return out
+
+
+def _build_recommendation(
+    opportunities: list[dict],
+    idle_threshold: float,
+    fleet_done: datetime | None,
+    timelines: list[dict],
+    fleet_summary: dict,
+) -> dict | None:
+    if not opportunities:
+        # Diagnose WHY there are no opportunities even if some riders were idle
+        idle_count = sum(1 for t in timelines if t.get("status") == "idle")
+        if idle_count == 0:
+            reason = "All riders finished within the idle threshold window — batch was well balanced."
+        else:
+            reason = (
+                f"{idle_count} rider(s) finished early but no feasible substitution found: "
+                "either travel time exceeds the idle window, or all candidate orders "
+                "would be delivered later by the idle rider than the original."
+            )
+        return {
+            "action": "none",
+            "reason": reason,
+            "fleet_balance_assessment": _balance_assessment(fleet_summary),
+        }
+
+    best = opportunities[0]
+    idle = best["idle_rider"]
+    target = best["target_kitchen"]
+    relieved = best["most_relieved_rider"]
+    primary_kitchen = idle["primary_kitchen"] or "unknown"
+    confidence = best["confidence_score"]
+
+    # Root cause: why was this rider idle?
+    root_cause = _diagnose_root_cause(idle, timelines, fleet_summary)
+
+    # Risk factors
+    risk_factors = _identify_risks(best, idle_threshold)
+
+    description = (
+        f"{idle['name']} ({primary_kitchen}) finished all {idle['order_count']} orders "
+        f"at {idle['finished_at']} — {idle['idle_minutes']} min before the fleet finished. "
+        f"Assigning {best['feasible_orders_count']} {target} orders: "
+        f"travel {best['travel_to_kitchen_km']} km ({best['travel_to_kitchen_minutes']} min), "
+        f"arrive at {target} kitchen at {best['arrive_at_kitchen']}. "
+        f"Relieves {relieved['name']} by {relieved['time_saved_minutes']} min "
+        f"({relieved['original_finish']} → {relieved['new_finish']}). "
+        f"Fleet finishes {best['fleet_improvement_minutes']} min earlier "
+        f"({best['original_fleet_done']} → {best['new_fleet_done']}). "
+        f"Confidence: {confidence}/100."
+    )
+
+    # Dynamic thresholds derived from the actual batch data
+    idle_rider_loads = [t["order_count"] for t in timelines if t.get("kitchen") == primary_kitchen]
+    target_rider_loads = [t["order_count"] for t in timelines if t.get("kitchen") == target]
+    activate_idle_threshold = max(6, idle.get("order_count", 6) + 2)
+    activate_target_threshold = max(8, round(sum(target_rider_loads) / max(1, len(target_rider_loads)) * 1.1))
+
+    activate_rule = {
+        "condition": "AND",
+        "rules": [
+            {
+                "field": f"{primary_kitchen}_order_count",
+                "operator": "<=",
+                "value": activate_idle_threshold,
+                "reason": (
+                    f"{idle['name']} had {idle['order_count']} orders today and was idle "
+                    f"{idle['idle_minutes']} min. Dual-kitchen kicks in when their load "
+                    f"stays at or below {activate_idle_threshold}."
+                ),
+            },
+            {
+                "field": f"{target}_order_count",
+                "operator": ">=",
+                "value": activate_target_threshold,
+                "reason": (
+                    f"{target.capitalize()} had enough orders today to justify the detour "
+                    f"({sum(target_rider_loads)} total across {len(target_rider_loads)} rider(s)). "
+                    f"Activate when {target} load is ≥ {activate_target_threshold}."
+                ),
+            },
+        ],
+    }
+
+    return {
+        "action": "dual_kitchen_assignment",
+        "idle_rider_id": idle["userid"],
+        "idle_rider_name": idle["name"],
+        "primary_kitchen": primary_kitchen,
+        "second_kitchen": target,
+        "second_kitchen_dispatch_after": best["arrive_at_kitchen"],
+        "description": description,
+        "fleet_improvement_minutes": best["fleet_improvement_minutes"],
+        "confidence_score": confidence,
+        "efficiency_rating": best["efficiency_rating"],
+        "root_cause": root_cause,
+        "risk_factors": risk_factors,
+        "activate_when": activate_rule,
+        "fleet_balance_assessment": _balance_assessment(fleet_summary),
+        "api_hint": {
+            "endpoint": "/api/v1/optimize",
+            "note": (
+                f"In the next batch, pre-assign the last "
+                f"{best['feasible_orders_count']} {target} orders to "
+                f"rider {idle['userid']} ({idle['name']}) with a "
+                f"dispatch-after time of {best['arrive_at_kitchen']}."
+            ),
+        },
+    }
+
+
+def _diagnose_root_cause(
+    idle: dict,
+    timelines: list[dict],
+    fleet_summary: dict,
+) -> str:
+    """
+    Explain WHY this rider finished early by comparing their load
+    against the fleet average and their kitchen's order volume.
+    """
+    avg_orders = fleet_summary.get("orders_per_rider_avg", 0)
+    rider_orders = idle["order_count"]
+    kitchen = idle["primary_kitchen"] or "their kitchen"
+    name = idle["name"]
+
+    if avg_orders > 0 and rider_orders < avg_orders * 0.7:
+        shortfall = round(avg_orders - rider_orders, 1)
+        return (
+            f"{name} received {rider_orders} orders vs fleet average of {avg_orders:.1f} "
+            f"(−{shortfall:.1f}). {kitchen.capitalize()} kitchen generated fewer orders "
+            f"than the fleet needed to keep this rider fully utilised. "
+            f"This is a systematic under-loading of the {kitchen} kitchen in this batch."
+        )
+    elif rider_orders <= 4:
+        return (
+            f"{name} had only {rider_orders} orders — a very light load regardless of fleet average. "
+            f"Likely a short-demand window at {kitchen} kitchen. "
+            f"Dual-kitchen assignment is especially effective when primary kitchen load is ≤ 4 orders."
+        )
+    else:
+        spread = fleet_summary.get("finish_time_spread_minutes", 0)
+        return (
+            f"{name} is simply faster than peers — finished {idle['idle_minutes']} min ahead "
+            f"despite a normal load of {rider_orders} orders. "
+            f"Fleet finish-time spread is {spread} min, indicating uneven workload distribution."
+        )
+
+
+def _identify_risks(best: dict, idle_threshold: float) -> list[str]:
+    """
+    Enumerate operational risks for the recommended substitution.
+    """
+    risks: list[str] = []
+    travel_min = best["travel_to_kitchen_minutes"]
+    obligation = best["total_obligation_minutes"]
+    free_window = best["idle_rider"]["free_window_minutes"]
+    confidence = best["confidence_score"]
+    extra_km = best["extra_km_for_idle_rider"]
+
+    slack_min = free_window - obligation
+    if slack_min < 10:
+        risks.append(
+            f"Tight schedule: only {slack_min} min of slack between idle rider's "
+            f"estimated finish and fleet completion. Any delay (traffic, kitchen wait) "
+            f"would eliminate the benefit."
+        )
+    if travel_min > 15:
+        risks.append(
+            f"Long commute to target kitchen ({travel_min} min). "
+            f"Kitchen departure time must be precise — a late start erodes time savings."
+        )
+    if extra_km > 8:
+        risks.append(
+            f"Extra {extra_km} km for the idle rider adds fuel cost and rider fatigue. "
+            f"Verify this is worthwhile if fleet improvement is marginal."
+        )
+    if confidence < 50:
+        risks.append(
+            f"Low confidence ({confidence}/100): limited slack or few feasible transfers. "
+            f"Consider this as a contingency plan rather than a guaranteed improvement."
+        )
+    if best.get("feasible_orders_count", 0) < best.get("total_orders_transferred", 1):
+        risks.append(
+            "Not all proposed transfers save time — some orders are included to fill "
+            "the idle rider's route but don't improve individual delivery times."
+        )
+    if not risks:
+        risks.append("No significant risks identified. Transfer looks operationally sound.")
+
+    return risks
+
+
+def _balance_assessment(fleet_summary: dict) -> str:
+    """Short human-readable verdict on batch balance quality."""
+    spread = fleet_summary.get("finish_time_spread_minutes", 0)
+    stdev = fleet_summary.get("load_balance_stdev", 0)
+    util = fleet_summary.get("avg_utilisation_pct")
+
+    if spread <= 10 and stdev <= 1:
+        verdict = "Excellent — riders finished close together with balanced loads."
+    elif spread <= 20 and stdev <= 2:
+        verdict = "Good — minor imbalance, acceptable for this fleet size."
+    elif spread <= 35:
+        verdict = f"Moderate imbalance — {spread} min spread between earliest and latest finish."
+    else:
+        verdict = (
+            f"High imbalance — {spread} min spread. Some riders sat idle while others overran. "
+            f"Pre-planning dual-kitchen assignments is strongly recommended."
+        )
+
+    if util is not None:
+        verdict += f" Average rider utilisation: {util}% of batch window."
+    return verdict

app/services/routing/clustering_service.py

@@ -0,0 +1,300 @@
+"""
+Geographic Clustering Service for Order Assignment
+Uses K-means clustering to group orders by kitchen location.
+Enhanced with Geohash encoding for spatial learning (idea.txt data encoding).
+"""
+
+import logging
+import numpy as np
+from typing import List, Dict, Any, Tuple
+from collections import defaultdict
+from math import radians, cos, sin, asin, sqrt
+
+logger = logging.getLogger(__name__)
+
+
+class GeohashEncoder:
+    """
+    Geohash Encoding for spatial data.
+    Converts lat/lon coordinates to grid cell strings for locality capture.
+
+    Precision levels:
+    - 4 chars: ~156km x 156km (regional)
+    - 5 chars: ~39km x 19km (city-level)
+    - 6 chars: ~4.9km x 4.9km (neighborhood)
+    - 7 chars: ~1.2km x 609m (local zone)
+    """
+
+    BASE32 = "0123456789bcdefghjkmnpqrstuvwxyz"
+
+    @classmethod
+    def encode(cls, lat: float, lon: float, precision: int = 6) -> str:
+        """
+        Encode lat/lon to geohash string.
+
+        Args:
+            lat: Latitude (-90 to 90)
+            lon: Longitude (-180 to 180)
+            precision: Number of characters (4-12)
+
+        Returns:
+            Geohash string
+        """
+        if lat == 0 and lon == 0:
+            return "unknown"
+
+        lat_min, lat_max = -90.0, 90.0
+        lon_min, lon_max = -180.0, 180.0
+
+        hash_chars = []
+        is_lon = True
+
+        for _ in range(precision):
+            char_bits = 0
+            for _ in range(5):
+                if is_lon:
+                    mid = (lon_min + lon_max) / 2
+                    if lon >= mid:
+                        char_bits = (char_bits << 1) | 1
+                        lon_min = mid
+                    else:
+                        char_bits = char_bits << 1
+                        lon_max = mid
+                else:
+                    mid = (lat_min + lat_max) / 2
+                    if lat >= mid:
+                        char_bits = (char_bits << 1) | 1
+                        lat_min = mid
+                    else:
+                        char_bits = char_bits << 1
+                        lat_max = mid
+                is_lon = not is_lon
+            hash_chars.append(cls.BASE32[char_bits])
+
+        return "".join(hash_chars)
+
+    @classmethod
+    def get_zone_from_geohash(cls, geohash: str) -> Dict[str, Any]:
+        """
+        Extract zone metadata from geohash for ML features.
+
+        Returns:
+            Dict with zone info: zone_id, precision, cell_size, etc.
+        """
+        if not geohash or geohash == "unknown":
+            return {"zone_id": "unknown", "precision": 0}
+
+        precision = len(geohash)
+
+        # Approximate cell sizes (in km)
+        lat_error = 180.0 / (2 ** (precision * 5 // 2)) / 2
+        lon_error = 360.0 / (2 ** (precision * 5 // 2 + 1)) / 2
+
+        # Cell size approximation
+        cell_width_km = lat_error * 111  # 1 degree lat ≈ 111km
+        cell_height_km = (
+            lon_error * 111 * cos(11 * 3.14159 / 180)
+        )  # Adjust for Coimbatore lat
+
+        return {
+            "zone_id": geohash,
+            "precision": precision,
+            "cell_width_km": round(cell_width_km, 3),
+            "cell_height_km": round(cell_height_km, 3),
+            "prefix_4": geohash[:4] if len(geohash) >= 4 else geohash,
+            "prefix_5": geohash[:5] if len(geohash) >= 5 else geohash,
+            "prefix_6": geohash[:6] if len(geohash) >= 6 else geohash,
+        }
+
+
+class ClusteringService:
+    """Clusters orders geographically to enable balanced rider assignment."""
+
+    def __init__(self):
+        self.earth_radius_km = 6371
+        self.geohash_encoder = GeohashEncoder()
+
+    def haversine(self, lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+        """Calculate distance between two points in km."""
+        lon1, lat1, lon2, lat2 = map(
+            radians, [float(lon1), float(lat1), float(lon2), float(lat2)]
+        )
+        dlon = lon2 - lon1
+        dlat = lat2 - lat1
+        a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
+        c = 2 * asin(min(1.0, sqrt(a)))
+        return c * self.earth_radius_km
+
+    def get_kitchen_location(self, order: Dict[str, Any]) -> Tuple[float, float]:
+        """Extract kitchen coordinates from order."""
+        try:
+            lat = float(order.get("pickuplat", 0))
+            lon = float(order.get("pickuplon") or order.get("pickuplong", 0))
+            if lat != 0 and lon != 0:
+                return lat, lon
+        except (ValueError, TypeError):
+            pass
+        return 0.0, 0.0
+
+    def _encode_location_features(self, lat: float, lon: float) -> Dict[str, Any]:
+        """
+        Encode location with multiple spatial features (Data Encoding: Geohash + Distance).
+
+        Returns:
+            Dict with geohash, zone info, distance features
+        """
+        features = {}
+
+        # Primary geohash encoding (6 chars = ~5km zone for Coimbatore)
+        geohash_6 = self.geohash_encoder.encode(lat, lon, 6)
+        features["geohash_6"] = geohash_6
+
+        # Fine-grained geohash (7 chars = ~1.2km zone)
+        geohash_7 = self.geohash_encoder.encode(lat, lon, 7)
+        features["geohash_7"] = geohash_7
+
+        # Coarse geohash for regional grouping (4 chars = ~156km)
+        geohash_4 = self.geohash_encoder.encode(lat, lon, 4)
+        features["geohash_4"] = geohash_4
+
+        # Zone metadata
+        features["zone_info"] = self.geohash_encoder.get_zone_from_geohash(geohash_6)
+
+        return features
+
+    def cluster_orders_by_kitchen(
+        self, orders: List[Dict[str, Any]], max_cluster_radius_km: float = 3.0
+    ) -> List[Dict[str, Any]]:
+        """
+        Cluster orders by kitchen proximity.
+
+        Returns list of clusters, each containing:
+        - centroid: (lat, lon) of cluster center
+        - orders: list of orders in this cluster
+        - kitchen_names: set of kitchen names in cluster
+        - total_orders: count
+        - geohash_6: geohash encoding of centroid (NEW)
+        - zone_info: zone metadata (NEW)
+        - spatial_features: all location encodings (NEW)
+        """
+        if not orders:
+            return []
+
+        # Group by kitchen location
+        kitchen_groups = defaultdict(list)
+        kitchen_coords = {}
+
+        for order in orders:
+            k_name = self._get_kitchen_name(order)
+            k_lat, k_lon = self.get_kitchen_location(order)
+
+            if k_lat == 0:
+                # Fallback: use delivery location if pickup missing
+                k_lat = float(order.get("deliverylat", 0))
+                k_lon = float(order.get("deliverylong", 0))
+
+            if k_lat != 0:
+                kitchen_groups[k_name].append(order)
+                kitchen_coords[k_name] = (k_lat, k_lon)
+
+        # Now cluster kitchens that are close together
+        clusters = []
+        processed_kitchens = set()
+
+        for k_name, k_orders in kitchen_groups.items():
+            if k_name in processed_kitchens:
+                continue
+
+            # Start a new cluster with this kitchen
+            cluster_kitchens = [k_name]
+            cluster_orders = k_orders[:]
+            processed_kitchens.add(k_name)
+
+            k_lat, k_lon = kitchen_coords[k_name]
+
+            # ── CHAIN-MERGE FIX ──────────────────────────────────────────
+            # Original bug: only checked against the SEED kitchen.
+            # Example: A→B = 2km, B→C = 2km, A→C = 4km (max_radius = 3km).
+            # Old code: C is NOT merged (A→C > 3km).
+            # New code: iteratively re-check remaining kitchens against the
+            # CURRENT centroid after each merge, so chains like A→B→C
+            # are correctly collapsed into one cluster.
+            # ─────────────────────────────────────────────────────────────
+            changed = True
+            while changed:
+                changed = False
+                # Recompute centroid of current cluster
+                c_lats = [kitchen_coords[n][0] for n in cluster_kitchens if n in kitchen_coords]
+                c_lons = [kitchen_coords[n][1] for n in cluster_kitchens if n in kitchen_coords]
+                c_lat = sum(c_lats) / len(c_lats) if c_lats else k_lat
+                c_lon = sum(c_lons) / len(c_lons) if c_lons else k_lon
+
+                for other_name, other_coords in kitchen_coords.items():
+                    if other_name in processed_kitchens:
+                        continue
+                    other_lat, other_lon = other_coords
+                    dist = self.haversine(c_lat, c_lon, other_lat, other_lon)
+                    if dist <= max_cluster_radius_km:
+                        cluster_kitchens.append(other_name)
+                        cluster_orders.extend(kitchen_groups[other_name])
+                        processed_kitchens.add(other_name)
+                        changed = True  # centroid shifted — re-scan remaining
+
+            # Calculate cluster centroid
+            lats = []
+            lons = []
+            for order in cluster_orders:
+                lat, lon = self.get_kitchen_location(order)
+                if lat != 0:
+                    lats.append(lat)
+                    lons.append(lon)
+
+            if lats:
+                centroid_lat = sum(lats) / len(lats)
+                centroid_lon = sum(lons) / len(lons)
+            else:
+                centroid_lat, centroid_lon = k_lat, k_lon
+
+            # ENHANCED: Add geohash encoding features
+            spatial_features = self._encode_location_features(
+                centroid_lat, centroid_lon
+            )
+
+            clusters.append(
+                {
+                    "centroid": (centroid_lat, centroid_lon),
+                    "orders": cluster_orders,
+                    "kitchen_names": set(cluster_kitchens),
+                    "total_orders": len(cluster_orders),
+                    # NEW: Geohash encoding features
+                    "geohash_6": spatial_features["geohash_6"],
+                    "geohash_7": spatial_features["geohash_7"],
+                    "geohash_4": spatial_features["geohash_4"],
+                    "zone_info": spatial_features["zone_info"],
+                    "spatial_features": spatial_features,
+                }
+            )
+
+        # Sort clusters by order count (largest first)
+        clusters.sort(key=lambda x: x["total_orders"], reverse=True)
+
+        logger.info(
+            f"Created {len(clusters)} clusters from {len(kitchen_groups)} kitchens with geohash encoding"
+        )
+        return clusters
+
+    def _get_kitchen_name(self, order: Dict[str, Any]) -> str:
+        """Extract kitchen name from order."""
+        possible_keys = [
+            "pickupcustomer",   # confirmed primary field in production orders
+            "locationname",     # confirmed backup field in production orders
+            "storename", "store_name",
+            "restaurantname", "restaurant_name",
+            "kitchenname", "kitchen_name",
+            "partnername", "partner_name",
+            "tenantname",
+        ]
+        for key in possible_keys:
+            if key in order and order[key]:
+                return str(order[key]).strip()
+        return "Unknown"

app/services/routing/delivery_history_service.py

@@ -0,0 +1,864 @@
+"""
+Delivery History Service — Empirical ETA from ground truth
+==========================================================
+
+WHY THIS EXISTS
+---------------
+Until now every ETA in the system was a *formula guess*
+(`RealisticETACalculator`: distance / configured_speed + fixed buffers) that was
+never checked against what actually happened in the field. Senior feedback was
+that manual riders deliver faster than our estimates — i.e. the guess is wrong.
+
+This service closes the loop. The external `nearledb` Postgres already records,
+for every completed delivery, when the rider picked up (`pickuptime`) and when
+the order was delivered (`deliverytime`). From those we reconstruct **actual
+per-leg travel times** and learn empirical medians, so ETAs reflect reality
+instead of a hand-tuned formula.
+
+GROUND-TRUTH RECONSTRUCTION (per-leg, matches the optimizer)
+-----------------------------------------------------------
+The optimizer feeds *aerial* (straight-line) leg distance to `calculate_eta`
+(see route_optimizer.py — `step_dist` comes from `aerial_matrix`, no road
+factor). To learn a model that plugs in behind the same call, we reconstruct
+observations the same way:
+
+  * Group a rider's completed deliveries by day, ordered by `deliverytime`.
+  * For each pair of consecutive deliveries:
+        leg_min = deliverytime[i] - deliverytime[i-1]      (real door-to-door time)
+        leg_km  = aerial haversine(drop[i-1], drop[i])     (same metric as optimizer)
+  * The first delivery of a group is the kitchen→first-drop leg:
+        leg_min = deliverytime[0] - pickuptime[0]          (distance unknown w/o
+        kitchen coords, so it only feeds the non-distance keys).
+
+Only `droplat/droplon`, `pickuptime`, `deliverytime`, `pickupcustomer`, `userid`
+are needed — all confirmed-present columns (same set batch_analytics reads).
+
+AGGREGATION + LOOKUP (hierarchical, cold-start safe)
+----------------------------------------------------
+Each observation feeds several keys at decreasing specificity. At prediction
+time we walk the same hierarchy and use the first key with enough samples,
+falling back to the formula when history is too thin:
+
+    kzd  kitchen | drop_zone | traffic | dist_bucket   (most specific)
+    kz   kitchen | drop_zone | traffic
+    zd   drop_zone | traffic | dist_bucket
+    z    drop_zone | traffic
+    dt   dist_bucket | traffic
+    t    traffic                                        (least specific)
+    →    RealisticETACalculator formula                 (no data)
+
+`rkz` (rider | kitchen | zone | traffic) is also stored for Phase-2 learned
+rider affinity; it is not used by the default lookup yet.
+"""
+
+import logging
+import math
+import os
+import sqlite3
+import threading
+import statistics
+import time
+from datetime import datetime, timedelta
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
+
+from app.services.routing.zone_service import ZoneService
+
+logger = logging.getLogger(__name__)
+
+_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
+_WRITE_LOCK = threading.Lock()
+
+# Aerial leg-distance buckets (km) — must match the metric the optimizer feeds
+# to calculate_eta (pure haversine, no road factor).
+_DIST_BUCKETS: List[Tuple[float, float]] = [
+    (0.0, 1.0), (1.0, 2.0), (2.0, 3.0), (3.0, 5.0), (5.0, 8.0), (8.0, 12.0), (12.0, 1e9)
+]
+
+# Sanity filters for reconstructed legs.
+_MAX_LEG_MIN = 60.0    # gaps longer than this are batch boundaries / idle, not a leg
+_MAX_LEG_KM = 40.0     # implausible single hop
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+def dist_bucket(km: Optional[float]) -> Optional[str]:
+    """Aerial leg distance -> discrete bucket label, or None if unknown."""
+    if km is None:
+        return None
+    for lo, hi in _DIST_BUCKETS:
+        if km <= hi:
+            return f"{lo:g}-{hi:g}" if hi < 1e9 else f"{lo:g}+"
+    return f"{_DIST_BUCKETS[-1][0]:g}+"
+
+
+def hour_to_traffic(hour: int) -> str:
+    """Time-of-day -> traffic category, mirroring get_time_of_day_category()."""
+    if (8 <= hour < 10) or (12 <= hour < 14) or (17 <= hour < 20):
+        return "peak"
+    if hour < 7 or hour >= 22:
+        return "light"
+    return "normal"
+
+
+def normalize_kitchen(name: Any) -> str:
+    """Normalize a kitchen / pickup-customer name to a stable key token."""
+    if not name:
+        return "?"
+    return " ".join(str(name).strip().lower().split())
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle (aerial) distance in km."""
+    try:
+        rlat1, rlon1, rlat2, rlon2 = map(math.radians, (lat1, lon1, lat2, lon2))
+        dlat = rlat2 - rlat1
+        dlon = rlon2 - rlon1
+        a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2) ** 2
+        return 6371.0 * 2 * math.asin(min(1.0, math.sqrt(a)))
+    except Exception:
+        return 0.0
+
+
+def _parse_dt(val: Any) -> Optional[datetime]:
+    if val in (None, "", 0):
+        return None
+    if isinstance(val, datetime):
+        return val
+    s = str(val).strip()
+    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S.%f",
+                "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M"):
+        try:
+            return datetime.strptime(s.split("+")[0].strip(), fmt)
+        except Exception:
+            continue
+    try:
+        from dateutil.parser import parse as _du
+        return _du(s)
+    except Exception:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Shared Postgres connector (single source of truth for nearledb creds)
+# ---------------------------------------------------------------------------
+
+def connect_nearledb(connect_timeout: int = 10):
+    """
+    Open a connection to the external nearledb Postgres.
+
+    Credentials come from DB_* env vars (same defaults batch_analytics used).
+    Raises on failure — callers convert to their own error type.
+    """
+    import psycopg2  # imported lazily so the app boots without it
+
+    conn = psycopg2.connect(
+        host=os.getenv("DB_HOST", "66.116.207.225"),
+        port=int(os.getenv("DB_PORT", "6432")),
+        dbname=os.getenv("DB_NAME", "nearledb"),
+        user=os.getenv("DB_USER", "admin"),
+        password=os.getenv("DB_PASSWORD", "Package@123#"),
+        connect_timeout=connect_timeout,
+    )
+    # Best-effort read-only session. (Port 6432 is pgbouncer, which rejects the
+    # `options` startup param, so we set it post-connect; our code only issues
+    # SELECTs regardless — we never write to or alter the source DB.)
+    try:
+        conn.set_session(readonly=True)
+    except Exception:
+        pass
+    return conn
+
+
+# ---------------------------------------------------------------------------
+# Key construction (build-side and lookup-side share this so they stay in sync)
+# ---------------------------------------------------------------------------
+
+def _keys_for_observation(kitchen: str, zone: str, traffic: str,
+                          bucket: Optional[str], rider_id: Any) -> List[str]:
+    """All aggregation keys an observation contributes to."""
+    keys = [
+        f"kz|{kitchen}|{zone}|{traffic}",
+        f"z|{zone}|{traffic}",
+        f"t|{traffic}",
+        f"rkz|{rider_id}|{kitchen}|{zone}|{traffic}",
+    ]
+    if bucket is not None:
+        keys += [
+            f"kzd|{kitchen}|{zone}|{traffic}|{bucket}",
+            f"zd|{zone}|{traffic}|{bucket}",
+            f"dt|{bucket}|{traffic}",
+        ]
+    return keys
+
+
+def _lookup_keys(kitchen: Optional[str], zone: str, traffic: str,
+                 bucket: Optional[str]) -> List[str]:
+    """Ordered candidate keys, most specific first, for prediction-time lookup."""
+    ordered: List[str] = []
+    if kitchen and bucket is not None:
+        ordered.append(f"kzd|{kitchen}|{zone}|{traffic}|{bucket}")
+    if kitchen:
+        ordered.append(f"kz|{kitchen}|{zone}|{traffic}")
+    if bucket is not None:
+        ordered.append(f"zd|{zone}|{traffic}|{bucket}")
+    ordered.append(f"z|{zone}|{traffic}")
+    if bucket is not None:
+        ordered.append(f"dt|{bucket}|{traffic}")
+    ordered.append(f"t|{traffic}")
+    return ordered
+
+
+# ---------------------------------------------------------------------------
+# Service
+# ---------------------------------------------------------------------------
+
+class DeliveryHistoryService:
+    """Ingests completed deliveries, learns empirical leg-time medians, serves lookups."""
+
+    def __init__(self):
+        self._db_path = _DB_PATH
+        self._zone = ZoneService()
+        # cache: full_key -> (count, median_min, p75_min)
+        self._cache: Dict[str, Tuple[int, float, float]] = {}
+        self._last_refreshed: Optional[datetime] = None
+        self._last_summary: Dict[str, Any] = {}
+        self._refresh_lock = threading.Lock()
+        self._refresh_attempt_at: Optional[datetime] = None
+        self._scheduler_started = False
+        self._ensure_db()
+        self._load_cache()
+
+    # -- schema -------------------------------------------------------------
+
+    def _ensure_db(self) -> None:
+        try:
+            os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
+            conn = sqlite3.connect(self._db_path)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS delivery_history (
+                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+                    delivery_ts   TEXT,
+                    rider_id      TEXT,
+                    kitchen       TEXT,
+                    drop_zone     TEXT,
+                    traffic       TEXT,
+                    dist_bucket   TEXT,
+                    leg_km        REAL,
+                    leg_min       REAL,
+                    is_first      INTEGER DEFAULT 0
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS eta_stats (
+                    key          TEXT PRIMARY KEY,
+                    key_type     TEXT,
+                    sample_count INTEGER,
+                    median_min   REAL,
+                    p75_min      REAL,
+                    updated_at   TEXT
+                )
+            """)
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_eta_stats_type ON eta_stats(key_type)")
+            # Local mirror of the nearledb `deliveries` rows we care about.
+            # The request path NEVER touches Postgres — only this table.
+            # Deduped by deliveryid so incremental syncs are idempotent.
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS delivery_raw (
+                    deliveryid    TEXT PRIMARY KEY,
+                    userid        TEXT,
+                    pickupcustomer TEXT,
+                    pickuptime    TEXT,
+                    deliverytime  TEXT,
+                    dlat          REAL,
+                    dlon          REAL,
+                    plat          REAL,
+                    plon          REAL
+                )
+            """)
+            # Migration for stores created before pickup coords were added.
+            for _ddl in (
+                "ALTER TABLE delivery_raw ADD COLUMN plat REAL",
+                "ALTER TABLE delivery_raw ADD COLUMN plon REAL",
+            ):
+                try:
+                    conn.execute(_ddl)
+                except Exception:
+                    pass
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_raw_dtime ON delivery_raw(deliverytime)")
+            # Single-row sync watermark / bookkeeping.
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS eta_sync_state (
+                    id             INTEGER PRIMARY KEY CHECK (id = 1),
+                    last_delivery_ts TEXT,
+                    last_synced_at TEXT,
+                    total_rows     INTEGER DEFAULT 0
+                )
+            """)
+            conn.execute("INSERT OR IGNORE INTO eta_sync_state (id, total_rows) VALUES (1, 0)")
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            logger.error(f"[DeliveryHistory] DB init failed: {e}")
+
+    # -- ingestion ----------------------------------------------------------
+
+    def fetch_completed_deliveries(
+        self, days: int, tenant_id: int = 916, since: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        READ-ONLY pull from nearledb `deliveries`.
+
+        * `since` set      -> incremental: only rows newer than the watermark.
+        * `since` None     -> initial backfill of the last `days`.
+
+        Only the `deliveries` table is read; the session is read-only (no writes,
+        no schema changes — enforced at the connection level).
+        """
+        conn = connect_nearledb()
+        try:
+            cur = conn.cursor()
+            if since:
+                where_time = "d.deliverytime::timestamp > %s"
+                time_param = since
+            else:
+                where_time = "d.deliverytime::timestamp >= %s"
+                time_param = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
+            cur.execute(
+                f"""
+                SELECT
+                    d.deliveryid,
+                    d.userid,
+                    d.pickupcustomer,
+                    d.pickuptime,
+                    d.deliverytime,
+                    COALESCE(d.droplat,  d.deliverylat)  AS dlat,
+                    COALESCE(d.droplon, d.deliverylong)  AS dlon,
+                    d.pickuplat                          AS plat,
+                    d.pickuplon                          AS plon
+                FROM deliveries d
+                WHERE d.tenantid = %s
+                  AND d.deliverytime IS NOT NULL
+                  AND d.pickuptime  IS NOT NULL
+                  AND {where_time}
+                  AND COALESCE(d.droplat, d.deliverylat) IS NOT NULL
+                  AND d.userid IS NOT NULL
+                ORDER BY d.deliverytime
+                """,
+                (tenant_id, time_param),
+            )
+            cols = [c.name for c in cur.description]
+            rows = [dict(zip(cols, r)) for r in cur.fetchall()]
+            cur.close()
+            return rows
+        finally:
+            conn.close()
+
+    # -- watermark / local raw store ---------------------------------------
+
+    def _get_watermark(self) -> Optional[str]:
+        try:
+            conn = sqlite3.connect(self._db_path)
+            row = conn.execute(
+                "SELECT last_delivery_ts FROM eta_sync_state WHERE id = 1"
+            ).fetchone()
+            conn.close()
+            return row[0] if row and row[0] else None
+        except Exception:
+            return None
+
+    def sync_from_db(self, days: int = 14, tenant_id: int = 916,
+                     full: bool = False) -> Dict[str, Any]:
+        """
+        Pull NEW completed deliveries from nearledb into the local mirror.
+
+        Idempotent (INSERT OR IGNORE on deliveryid). On the first run (empty
+        watermark) or `full=True`, backfills the last `days`; afterwards only
+        rows newer than the watermark are fetched — minimal DB load.
+        """
+        watermark = None if full else self._get_watermark()
+        rows = self.fetch_completed_deliveries(days, tenant_id, since=watermark)
+
+        inserted = 0
+        max_ts = watermark
+        conn = sqlite3.connect(self._db_path)
+        try:
+            if full:
+                # A full sync truly rebuilds the local mirror (also backfills any
+                # newly-added columns that INSERT OR IGNORE would otherwise skip).
+                conn.execute("DELETE FROM delivery_raw")
+                conn.execute("UPDATE eta_sync_state SET last_delivery_ts = NULL WHERE id = 1")
+                max_ts = None
+            for r in rows:
+                dt = str(r.get("deliverytime") or "")
+                try:
+                    cur = conn.execute(
+                        "INSERT OR IGNORE INTO delivery_raw "
+                        "(deliveryid, userid, pickupcustomer, pickuptime, deliverytime, dlat, dlon, plat, plon) "
+                        "VALUES (?,?,?,?,?,?,?,?,?)",
+                        (str(r.get("deliveryid")), str(r.get("userid")),
+                         r.get("pickupcustomer"), str(r.get("pickuptime") or ""),
+                         dt, r.get("dlat"), r.get("dlon"), r.get("plat"), r.get("plon")),
+                    )
+                    inserted += cur.rowcount
+                except Exception:
+                    continue
+                if dt and (max_ts is None or dt > max_ts):
+                    max_ts = dt
+            total = conn.execute("SELECT COUNT(*) FROM delivery_raw").fetchone()[0]
+            conn.execute(
+                "UPDATE eta_sync_state SET last_delivery_ts = ?, last_synced_at = ?, total_rows = ? WHERE id = 1",
+                (max_ts, datetime.utcnow().isoformat(), total),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+        return {"fetched": len(rows), "inserted": inserted, "local_total": total,
+                "watermark": max_ts, "mode": "full" if full else "incremental"}
+
+    def _prune_raw(self, days: int) -> int:
+        """Drop local rows older than the rolling window to bound the store."""
+        cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
+        try:
+            conn = sqlite3.connect(self._db_path)
+            cur = conn.execute("DELETE FROM delivery_raw WHERE deliverytime < ?", (cutoff,))
+            deleted = cur.rowcount
+            conn.commit()
+            conn.close()
+            return deleted
+        except Exception:
+            return 0
+
+    def _load_raw_rows(self, days: int) -> List[Dict[str, Any]]:
+        """Read the local mirror within the rolling window (no DB hit)."""
+        cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S")
+        conn = sqlite3.connect(self._db_path)
+        rows = conn.execute(
+            "SELECT deliveryid, userid, pickupcustomer, pickuptime, deliverytime, dlat, dlon, plat, plon "
+            "FROM delivery_raw WHERE deliverytime >= ? ORDER BY deliverytime",
+            (cutoff,),
+        ).fetchall()
+        conn.close()
+        cols = ["deliveryid", "userid", "pickupcustomer", "pickuptime", "deliverytime", "dlat", "dlon", "plat", "plon"]
+        return [dict(zip(cols, r)) for r in rows]
+
+    def sample_batches(self, days: int = 14, min_drops: int = 4, max_drops: int = 15,
+                       limit: int = 10) -> List[Dict[str, Any]]:
+        """
+        Real recent delivery batches from the local mirror for the road-sequencing
+        decision agent. Grouped by (rider, day, kitchen); origin = that kitchen's
+        pickup coords (centroid fallback). Most-recent batches first.
+        Returns [{origin, drops, rider, day, kitchen}].
+        """
+        rows = self._load_raw_rows(days)
+        groups: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = defaultdict(list)
+        for r in rows:
+            dt = _parse_dt(r.get("deliverytime"))
+            if dt is None:
+                continue
+            try:
+                if not (float(r["dlat"]) and float(r["dlon"])):
+                    continue
+            except (TypeError, ValueError):
+                continue
+            key = (str(r.get("userid")), dt.strftime("%Y-%m-%d"),
+                   normalize_kitchen(r.get("pickupcustomer")))
+            groups[key].append(r)
+
+        batches: List[Dict[str, Any]] = []
+        for key in sorted(groups, key=lambda k: k[1], reverse=True):
+            items = groups[key]
+            if not (min_drops <= len(items) <= max_drops):
+                continue
+            drops = [(float(i["dlat"]), float(i["dlon"])) for i in items]
+            origin = None
+            for i in items:
+                try:
+                    pla, plo = float(i["plat"]), float(i["plon"])
+                    if pla and plo:
+                        origin = (pla, plo)
+                        break
+                except (TypeError, ValueError):
+                    continue
+            if origin is None:
+                origin = (sum(d[0] for d in drops) / len(drops),
+                          sum(d[1] for d in drops) / len(drops))
+            batches.append({"origin": origin, "drops": drops,
+                            "rider": key[0], "day": key[1], "kitchen": key[2]})
+            if len(batches) >= limit:
+                break
+        return batches
+
+    def _build_observations(self, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Reconstruct per-leg observations from ordered delivery rows."""
+        # Group by (rider, calendar day of delivery)
+        groups: Dict[Tuple[str, str], List[Dict[str, Any]]] = defaultdict(list)
+        for r in rows:
+            dt = _parse_dt(r.get("deliverytime"))
+            pt = _parse_dt(r.get("pickuptime"))
+            if dt is None or pt is None:
+                continue
+            try:
+                dlat = float(r.get("dlat"))
+                dlon = float(r.get("dlon"))
+            except (TypeError, ValueError):
+                continue
+            if dlat == 0 or dlon == 0:
+                continue
+            groups[(str(r.get("userid")), dt.strftime("%Y-%m-%d"))].append({
+                "dt": dt, "pt": pt, "lat": dlat, "lon": dlon,
+                "kitchen": normalize_kitchen(r.get("pickupcustomer")),
+                "rider": str(r.get("userid")),
+            })
+
+        obs: List[Dict[str, Any]] = []
+        for _key, items in groups.items():
+            items.sort(key=lambda x: x["dt"])
+            prev = None
+            for i, it in enumerate(items):
+                zone = self._zone.determine_zone(it["lat"], it["lon"])
+                traffic = hour_to_traffic(it["dt"].hour)
+                if i == 0:
+                    # kitchen -> first drop; distance unknown without kitchen coords
+                    leg_min = (it["dt"] - it["pt"]).total_seconds() / 60.0
+                    leg_km = None
+                    is_first = 1
+                else:
+                    leg_min = (it["dt"] - prev["dt"]).total_seconds() / 60.0
+                    leg_km = _haversine_km(prev["lat"], prev["lon"], it["lat"], it["lon"])
+                    is_first = 0
+                prev = it
+                # filter implausible legs
+                if leg_min <= 0 or leg_min > _MAX_LEG_MIN:
+                    continue
+                if leg_km is not None and (leg_km <= 0 or leg_km > _MAX_LEG_KM):
+                    continue
+                obs.append({
+                    "delivery_ts": it["dt"].strftime("%Y-%m-%d %H:%M:%S"),
+                    "rider_id": it["rider"],
+                    "kitchen": it["kitchen"],
+                    "drop_zone": zone,
+                    "traffic": traffic,
+                    "dist_bucket": dist_bucket(leg_km),
+                    "leg_km": leg_km,
+                    "leg_min": round(leg_min, 2),
+                    "is_first": is_first,
+                })
+        return obs
+
+    @staticmethod
+    def _aggregate(observations: List[Dict[str, Any]]) -> Dict[str, Tuple[int, float, float]]:
+        """Group observations by every key and compute (count, median, p75)."""
+        buckets: Dict[str, List[float]] = defaultdict(list)
+        for o in observations:
+            for k in _keys_for_observation(
+                o["kitchen"], o["drop_zone"], o["traffic"], o["dist_bucket"], o["rider_id"]
+            ):
+                buckets[k].append(o["leg_min"])
+        stats: Dict[str, Tuple[int, float, float]] = {}
+        for k, vals in buckets.items():
+            vals.sort()
+            n = len(vals)
+            median = statistics.median(vals)
+            p75 = vals[min(n - 1, int(math.ceil(0.75 * n)) - 1)] if n else median
+            stats[k] = (n, round(median, 2), round(p75, 2))
+        return stats
+
+    def rebuild_aggregates(self, days: int = 14) -> Dict[str, Any]:
+        """
+        Rebuild empirical medians from the LOCAL mirror only (no DB hit).
+        Reconstructs per-leg observations, aggregates, persists, reloads cache.
+        """
+        rows = self._load_raw_rows(days)
+        observations = self._build_observations(rows)
+        stats = self._aggregate(observations)
+
+        now = datetime.utcnow().isoformat()
+        try:
+            conn = sqlite3.connect(self._db_path)
+            conn.execute("DELETE FROM delivery_history")
+            conn.execute("DELETE FROM eta_stats")
+            conn.executemany(
+                "INSERT INTO delivery_history "
+                "(delivery_ts, rider_id, kitchen, drop_zone, traffic, dist_bucket, leg_km, leg_min, is_first) "
+                "VALUES (?,?,?,?,?,?,?,?,?)",
+                [(o["delivery_ts"], o["rider_id"], o["kitchen"], o["drop_zone"],
+                  o["traffic"], o["dist_bucket"], o["leg_km"], o["leg_min"], o["is_first"])
+                 for o in observations],
+            )
+            conn.executemany(
+                "INSERT INTO eta_stats (key, key_type, sample_count, median_min, p75_min, updated_at) "
+                "VALUES (?,?,?,?,?,?)",
+                [(k, k.split("|", 1)[0], n, med, p75, now) for k, (n, med, p75) in stats.items()],
+            )
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            logger.error(f"[DeliveryHistory] persist failed: {e}", exc_info=True)
+            return {"status": "persist_failed", "error": str(e)}
+
+        self._load_cache()
+        self._last_refreshed = datetime.utcnow()
+        n_by_type: Dict[str, int] = defaultdict(int)
+        for k in stats:
+            n_by_type[k.split("|", 1)[0]] += 1
+        summary = {
+            "status": "ok",
+            "local_rows": len(rows),
+            "observations": len(observations),
+            "stat_keys": len(stats),
+            "keys_by_type": dict(n_by_type),
+            "history_days": days,
+            "rebuilt_at": self._last_refreshed.isoformat(),
+        }
+        logger.info(
+            f"[DeliveryHistory] aggregates rebuilt: local_rows={len(rows)} "
+            f"obs={len(observations)} keys={len(stats)} ({dict(n_by_type)})"
+        )
+        return summary
+
+    def refresh_eta_stats(self, days: int = 14, tenant_id: int = 916,
+                          full: bool = False) -> Dict[str, Any]:
+        """
+        Full pipeline: incremental DB sync -> prune local store -> rebuild
+        aggregates locally. This is what the scheduler and the admin endpoint
+        call. The DB is touched only by the sync step (new rows only).
+        """
+        with _WRITE_LOCK:
+            try:
+                sync = self.sync_from_db(days=days, tenant_id=tenant_id, full=full)
+            except Exception as e:
+                logger.error(f"[DeliveryHistory] sync failed: {e}", exc_info=True)
+                # Still try to serve whatever is already local.
+                rebuilt = self.rebuild_aggregates(days)
+                self._last_summary = {"status": "sync_failed", "error": str(e), "rebuild": rebuilt}
+                return self._last_summary
+
+            self._prune_raw(days)
+            rebuilt = self.rebuild_aggregates(days)
+            self._last_summary = {"status": "ok", "sync": sync, "rebuild": rebuilt}
+            return self._last_summary
+
+    # -- cache + lookup -----------------------------------------------------
+
+    def _load_cache(self) -> None:
+        try:
+            conn = sqlite3.connect(self._db_path)
+            rows = conn.execute(
+                "SELECT key, sample_count, median_min, p75_min FROM eta_stats"
+            ).fetchall()
+            conn.close()
+            self._cache = {k: (int(n), float(med), float(p75)) for k, n, med, p75 in rows}
+            if self._cache:
+                logger.info(f"[DeliveryHistory] loaded {len(self._cache)} eta_stats keys into cache")
+        except Exception as e:
+            logger.warning(f"[DeliveryHistory] cache load failed: {e}")
+            self._cache = {}
+
+    def has_data(self) -> bool:
+        return bool(self._cache)
+
+    def maybe_background_refresh(self, days: int = 14, tenant_id: int = 916,
+                                 cooldown_s: int = 600) -> bool:
+        """
+        If the cache is empty, kick off a one-shot background refresh (at most
+        once per cooldown). Keeps the prediction hot path non-blocking — the
+        current call falls back to the formula; later calls use empirical data.
+        Returns True if a refresh thread was started.
+        """
+        if self._cache:
+            return False
+        with self._refresh_lock:
+            if self._cache:
+                return False
+            now = datetime.utcnow()
+            if (self._refresh_attempt_at is not None
+                    and (now - self._refresh_attempt_at).total_seconds() < cooldown_s):
+                return False
+            self._refresh_attempt_at = now
+
+        def _run():
+            try:
+                self.refresh_eta_stats(days, tenant_id)
+            except Exception as e:
+                logger.warning(f"[DeliveryHistory] background refresh failed: {e}")
+
+        threading.Thread(target=_run, daemon=True, name="eta-refresh").start()
+        return True
+
+    def local_count(self) -> int:
+        try:
+            conn = sqlite3.connect(self._db_path)
+            n = conn.execute("SELECT COUNT(*) FROM delivery_raw").fetchone()[0]
+            conn.close()
+            return int(n)
+        except Exception:
+            return 0
+
+    def ensure_background_sync(self, interval_hours: int = 6, days: int = 14,
+                               tenant_id: int = 916) -> bool:
+        """
+        Start the autonomous sync agent (once per process). It refreshes
+        immediately on startup (full backfill if the local store is empty,
+        otherwise incremental), then re-syncs every `interval_hours`.
+
+        Runs in a daemon thread so it never blocks the app; all request-path
+        ETA lookups read the local SQLite mirror, never Postgres.
+        """
+        with self._refresh_lock:
+            if self._scheduler_started:
+                return False
+            self._scheduler_started = True
+
+        def _loop():
+            logger.info(
+                f"[ETA-Agent] autonomous sync started — interval={interval_hours}h, window={days}d"
+            )
+            first = True
+            while True:
+                try:
+                    full = first and self.local_count() == 0
+                    result = self.refresh_eta_stats(days=days, tenant_id=tenant_id, full=full)
+                    logger.info(f"[ETA-Agent] sync cycle done: {result.get('status')}")
+                except Exception as e:
+                    logger.warning(f"[ETA-Agent] sync cycle failed (will retry): {e}")
+                # Recompute learned rider affinity from the freshly-synced mirror.
+                try:
+                    from app.services.routing.rider_affinity_service import get_rider_affinity
+                    get_rider_affinity().refresh(days=days)
+                except Exception as e:
+                    logger.debug(f"[ETA-Agent] affinity refresh skipped: {e}")
+                first = False
+                time.sleep(max(1, int(interval_hours)) * 3600)
+
+        threading.Thread(target=_loop, daemon=True, name="eta-sync-agent").start()
+        return True
+
+    def lookup(
+        self,
+        distance_km: float,
+        traffic_cat: str,
+        kitchen: Optional[str] = None,
+        drop_coords: Optional[Tuple[float, float]] = None,
+        min_samples: int = 20,
+        stat: str = "median",
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Return the empirical leg time (minutes) for a context, or None if no key
+        has >= min_samples. Walks the specificity hierarchy.
+        """
+        if not self._cache:
+            return None
+        zone = "Unknown"
+        if drop_coords and drop_coords[0] and drop_coords[1]:
+            zone = self._zone.determine_zone(float(drop_coords[0]), float(drop_coords[1]))
+        bucket = dist_bucket(distance_km) if distance_km and distance_km > 0 else None
+        k_norm = normalize_kitchen(kitchen) if kitchen else None
+
+        for full_key in _lookup_keys(k_norm, zone, traffic_cat, bucket):
+            hit = self._cache.get(full_key)
+            if hit and hit[0] >= min_samples:
+                count, median, p75 = hit
+                value = p75 if stat == "p75" else median
+                return {
+                    "value_min": value,
+                    "sample_count": count,
+                    "source_key": full_key,
+                    "key_type": full_key.split("|", 1)[0],
+                }
+        return None
+
+    # -- diagnostics / backtest --------------------------------------------
+
+    def get_summary(self) -> Dict[str, Any]:
+        return {
+            "has_data": self.has_data(),
+            "cache_keys": len(self._cache),
+            "last_summary": self._last_summary,
+        }
+
+    def backtest(self, days: int = 14, tenant_id: int = 916,
+                 min_samples: int = 20, stat: str = "median") -> Dict[str, Any]:
+        """
+        Honest time-split backtest: build empirical stats on the older 80% of
+        observations, then compare formula vs empirical MAE on the most recent
+        20% (held out). Proves whether empirical beats the formula before trust.
+        """
+        from app.services.routing.realistic_eta_calculator import RealisticETACalculator
+        formula = RealisticETACalculator()
+
+        # Backtest runs entirely on the local mirror — no DB hit.
+        rows = self._load_raw_rows(days)
+        obs = self._build_observations(rows)
+        obs = [o for o in obs if o["is_first"] == 0]  # need leg_km for both predictors
+        if len(obs) < 50:
+            return {"status": "insufficient_data", "observations": len(obs)}
+
+        obs.sort(key=lambda o: o["delivery_ts"])
+        split = int(len(obs) * 0.8)
+        train, test = obs[:split], obs[split:]
+        train_stats = self._aggregate(train)
+
+        def _empirical(o) -> Optional[float]:
+            for full_key in _lookup_keys(o["kitchen"], o["drop_zone"], o["traffic"], o["dist_bucket"]):
+                hit = train_stats.get(full_key)
+                if hit and hit[0] >= min_samples:
+                    return hit[2] if stat == "p75" else hit[1]
+            return None
+
+        f_err, e_err, e_err_fallback, covered = [], [], [], 0
+        for o in test:
+            actual = o["leg_min"]
+            f_pred = formula.calculate_eta(
+                distance_km=o["leg_km"], is_first_order=False,
+                order_type="Economy", time_of_day=o["traffic"],
+            )
+            f_err.append(abs(f_pred - actual))
+            emp = _empirical(o)
+            if emp is not None:
+                covered += 1
+                e_err.append(abs(emp - actual))
+                e_err_fallback.append(abs(emp - actual))
+            else:
+                e_err_fallback.append(abs(f_pred - actual))  # fallback to formula
+
+        def _mae(xs):
+            return round(sum(xs) / len(xs), 2) if xs else None
+
+        return {
+            "status": "ok",
+            "history_days": days,
+            "observations": len(obs),
+            "test_size": len(test),
+            "empirical_coverage_pct": round(100.0 * covered / len(test), 1),
+            "formula_mae_min": _mae(f_err),
+            "empirical_mae_min_covered": _mae(e_err),
+            "empirical_mae_min_with_fallback": _mae(e_err_fallback),
+            "min_samples": min_samples,
+            "stat": stat,
+            "interpretation": (
+                "empirical_better"
+                if (_mae(e_err_fallback) is not None and _mae(f_err) is not None
+                    and _mae(e_err_fallback) < _mae(f_err))
+                else "no_improvement"
+            ),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Singleton
+# ---------------------------------------------------------------------------
+
+_service: Optional[DeliveryHistoryService] = None
+_service_lock = threading.Lock()
+
+
+def get_delivery_history_service() -> DeliveryHistoryService:
+    """Get (and lazily build) the DeliveryHistoryService singleton."""
+    global _service
+    with _service_lock:
+        if _service is None:
+            _service = DeliveryHistoryService()
+        return _service

app/services/routing/empirical_eta_calculator.py

@@ -0,0 +1,143 @@
+"""
+Empirical ETA Calculator
+========================
+
+Drop-in replacement for `RealisticETACalculator` that prefers ETAs *learned
+from actual delivery times* (see delivery_history_service.py) and falls back to
+the original formula whenever history is too thin.
+
+Design goals
+------------
+* **Interface-compatible**: `calculate_eta(...)` keeps the same positional args
+  and `int`-minutes return as the formula calculator, so existing call sites
+  work unchanged. Extra args (`kitchen`, `drop_coords`, `rider_id`) are optional
+  and let callers that *have* that context (the optimizer does) get sharper,
+  zone-aware estimates.
+* **Safe by default**: if empirical data is missing for a context, or the
+  feature is disabled via `eta_empirical_enabled`, it returns exactly what the
+  formula would — zero behavior change until real data exists.
+* **Non-blocking**: never calls Postgres on the hot path. If the stats cache is
+  empty it kicks off a one-shot background refresh and serves the formula
+  meanwhile.
+
+Empirical values are real door-to-door leg times (gap between consecutive
+deliveries), so they already include travel + drop service time. We only add the
+kitchen pickup buffer for the first leg, mirroring the formula's semantics.
+"""
+
+import logging
+from typing import Any, List, Optional, Tuple
+
+from app.services.routing.realistic_eta_calculator import (
+    RealisticETACalculator,
+    get_time_of_day_category,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class EmpiricalETACalculator:
+    """ETA calculator backed by empirical history, with formula fallback."""
+
+    def __init__(self):
+        # Composed formula calculator — the fallback and the source of buffers.
+        self.formula = RealisticETACalculator()
+
+    # ------------------------------------------------------------------
+    # Main entry point (signature is a superset of RealisticETACalculator)
+    # ------------------------------------------------------------------
+
+    def calculate_eta(
+        self,
+        distance_km: float,
+        is_first_order: bool = False,
+        order_type: str = "Economy",
+        time_of_day: str = "peak",
+        kitchen: Optional[str] = None,
+        drop_coords: Optional[Tuple[float, float]] = None,
+        rider_id: Optional[Any] = None,
+    ) -> int:
+        """Return ETA in minutes — empirical if available, else the formula."""
+        if distance_km is not None and distance_km <= 0 and not is_first_order:
+            return 0
+
+        from app.config.dynamic_config import get_config
+        cfg = get_config()
+
+        if not bool(cfg.get("eta_empirical_enabled", True)):
+            return self._formula_eta(distance_km, is_first_order, order_type, time_of_day)
+
+        try:
+            from app.services.routing.delivery_history_service import (
+                get_delivery_history_service,
+            )
+            svc = get_delivery_history_service()
+
+            if not svc.has_data():
+                # Populate in the background; serve the formula for now.
+                svc.maybe_background_refresh(
+                    days=int(cfg.get("eta_history_days", 14)),
+                )
+                return self._formula_eta(distance_km, is_first_order, order_type, time_of_day)
+
+            hit = svc.lookup(
+                distance_km=float(distance_km or 0.0),
+                traffic_cat=time_of_day,
+                kitchen=kitchen,
+                drop_coords=drop_coords,
+                min_samples=int(cfg.get("eta_min_samples", 20)),
+                stat=str(cfg.get("eta_stat", "median")),
+            )
+        except Exception as e:
+            logger.debug(f"[EmpiricalETA] lookup failed, using formula: {e}")
+            hit = None
+
+        if not hit:
+            return self._formula_eta(distance_km, is_first_order, order_type, time_of_day)
+
+        value = float(hit["value_min"])
+        # First leg includes time spent picking up at the kitchen; the empirical
+        # leg gap starts at pickup completion, so add the same buffer the formula uses.
+        if is_first_order:
+            value += float(cfg.get("eta_pickup_time_min", 3.0))
+        return int(value) + 1  # round up for safety, matching the formula
+
+    def _formula_eta(self, distance_km, is_first_order, order_type, time_of_day) -> int:
+        return self.formula.calculate_eta(
+            distance_km=distance_km,
+            is_first_order=is_first_order,
+            order_type=order_type,
+            time_of_day=time_of_day,
+        )
+
+    # ------------------------------------------------------------------
+    # Batch helper (kept compatible with RealisticETACalculator)
+    # ------------------------------------------------------------------
+
+    def calculate_batch_eta(self, orders: List[dict]) -> List[dict]:
+        """Calculate ETAs for a batch in sequence (formula-parity batch path)."""
+        traffic = get_time_of_day_category()
+        for order in orders:
+            distance_km = float(order.get("previouskms", 0) or 0)
+            step = order.get("step", 1)
+            order_type = order.get("ordertype", "Economy")
+            drop = None
+            try:
+                dlat = float(order.get("deliverylat") or order.get("droplat") or 0)
+                dlon = float(order.get("deliverylong") or order.get("droplon") or 0)
+                if dlat and dlon:
+                    drop = (dlat, dlon)
+            except (TypeError, ValueError):
+                drop = None
+            eta = self.calculate_eta(
+                distance_km=distance_km,
+                is_first_order=(step == 1),
+                order_type=order_type,
+                time_of_day=traffic,
+                kitchen=order.get("pickupcustomer") or order.get("locationname"),
+                drop_coords=drop,
+                rider_id=order.get("userid"),
+            )
+            order["eta"] = str(eta)
+            order["eta_empirical"] = True
+        return orders

app/services/routing/kalman_filter.py

@@ -0,0 +1,327 @@
+"""
+GPS Kalman Filter \u2014 rider-api
+
+A 1D Kalman filter applied independently to latitude and longitude
+to smooth noisy GPS coordinates from riders and delivery points.
+
+Why Kalman for GPS?
+- GPS readings contain measurement noise (\u00b15\u201315m typical, \u00b150m poor signal)
+- Rider location pings can "jump" due to bad signal or device error
+- Kalman filter gives an optimal estimate by balancing:
+    (1) Previous predicted position (process model)
+    (2) New GPS measurement (observation model)
+
+Design:
+- Separate filter instance per rider (stateful \u2014 preserves history)
+- `CoordinateKalmanFilter` \u2014 single lat/lon smoother
+- `GPSKalmanFilter`          \u2014 wraps two CoordinateKalmanFilters (lat + lon)
+- `RiderKalmanRegistry`      \u2014 manages per-rider filter instances
+- `smooth_coordinates()`     \u2014 stateless single-shot smoother for delivery coords
+
+Usage:
+    # Stateless (one-shot, no history \u2014 for delivery coords):
+    smooth_lat, smooth_lon = smooth_coordinates(raw_lat, raw_lon)
+
+    # Stateful (per-rider, preserves motion history):
+    registry = RiderKalmanRegistry()
+    lat, lon = registry.update(rider_id=1116, lat=11.0067, lon=76.9558)
+"""
+
+import logging
+import time
+from typing import Dict, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# CORE 1D KALMAN FILTER
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+class CoordinateKalmanFilter:
+    """
+    1-dimensional Kalman filter for a single GPS coordinate (lat or lon).
+
+    State model: position only (constant position with random walk).
+
+    Equations:
+        Prediction:   x\u0302\u2096\u207b  = x\u0302\u2096\u208b\u2081          (no movement assumed between pings)
+                      P\u0302\u2096\u207b   = P\u2096\u208b\u2081 + Q       (uncertainty grows over time)
+
+        Update:       K\u2096    = P\u0302\u2096\u207b / (P\u0302\u2096\u207b + R)   (Kalman gain)
+                      x\u0302\u2096    = x\u0302\u2096\u207b + K\u2096\u00b7(z\u2096 - x\u0302\u2096\u207b)  (weighted fusion)
+                      P\u2096    = (1 - K\u2096)\u00b7P\u0302\u2096\u207b          (update uncertainty)
+
+    Parameters:
+        process_noise (Q): How much position can change between measurements.
+                           Higher = filter trusts new measurements more (less smoothing).
+        measurement_noise (R): GPS measurement uncertainty.
+                               Higher = filter trusts history more (more smoothing).
+    """
+
+    def __init__(
+        self,
+        process_noise: float = 1e-4,
+        measurement_noise: float = 0.01,
+        initial_uncertainty: float = 1.0,
+    ):
+        self.Q = process_noise
+        self.R = measurement_noise
+        self._x: Optional[float] = None
+        self._P: float = initial_uncertainty
+
+    @property
+    def initialized(self) -> bool:
+        return self._x is not None
+
+    def update(self, measurement: float) -> float:
+        """Process one new measurement and return the filtered estimate."""
+        if not self.initialized:
+            self._x = measurement
+            return self._x
+
+        # Predict
+        x_prior = self._x
+        P_prior = self._P + self.Q
+
+        # Update
+        K = P_prior / (P_prior + self.R)
+        self._x = x_prior + K * (measurement - x_prior)
+        self._P = (1.0 - K) * P_prior
+
+        return self._x
+
+    def reset(self):
+        self._x = None
+        self._P = 1.0
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# 2D GPS KALMAN FILTER (lat + lon)
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+class GPSKalmanFilter:
+    """
+    Two-dimensional GPS smoother using independent 1D Kalman filters
+    for latitude and longitude.
+    """
+
+    def __init__(
+        self,
+        process_noise: float = 1e-4,
+        measurement_noise: float = 0.01,
+    ):
+        self.lat_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
+        self.lon_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
+        self.last_updated: float = time.time()
+        self.update_count: int = 0
+
+    def update(self, lat: float, lon: float) -> Tuple[float, float]:
+        """Feed a new GPS reading and get the smoothed (lat, lon)."""
+        if not self._is_valid_coord(lat, lon):
+            if self.lat_filter.initialized:
+                return self.lat_filter._x, self.lon_filter._x
+            return lat, lon
+
+        smooth_lat = self.lat_filter.update(lat)
+        smooth_lon = self.lon_filter.update(lon)
+        self.last_updated = time.time()
+        self.update_count += 1
+
+        return smooth_lat, smooth_lon
+
+    def get_estimate(self) -> Optional[Tuple[float, float]]:
+        if self.lat_filter.initialized:
+            return self.lat_filter._x, self.lon_filter._x
+        return None
+
+    def reset(self):
+        self.lat_filter.reset()
+        self.lon_filter.reset()
+        self.update_count = 0
+
+    @staticmethod
+    def _is_valid_coord(lat: float, lon: float) -> bool:
+        try:
+            lat, lon = float(lat), float(lon)
+            return (
+                -90.0 <= lat <= 90.0
+                and -180.0 <= lon <= 180.0
+                and not (lat == 0.0 and lon == 0.0)
+            )
+        except (TypeError, ValueError):
+            return False
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# PER-RIDER FILTER REGISTRY
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+class RiderKalmanRegistry:
+    """
+    Maintains per-rider Kalman filter instances across calls.
+    Stale filters (> 30 min silence) are automatically reset.
+    """
+
+    def __init__(
+        self,
+        process_noise: float = 1e-4,
+        measurement_noise: float = 0.01,
+        stale_seconds: float = 1800.0,
+    ):
+        self._filters: Dict[str, GPSKalmanFilter] = {}
+        self._process_noise = process_noise
+        self._measurement_noise = measurement_noise
+        self._stale_seconds = stale_seconds
+
+    def _get_or_create(self, rider_id) -> GPSKalmanFilter:
+        key = str(rider_id)
+        now = time.time()
+        if key in self._filters:
+            f = self._filters[key]
+            if now - f.last_updated > self._stale_seconds:
+                f.reset()
+            return f
+        self._filters[key] = GPSKalmanFilter(
+            process_noise=self._process_noise,
+            measurement_noise=self._measurement_noise,
+        )
+        return self._filters[key]
+
+    def update(self, rider_id, lat: float, lon: float) -> Tuple[float, float]:
+        return self._get_or_create(rider_id).update(lat, lon)
+
+    def get_estimate(self, rider_id) -> Optional[Tuple[float, float]]:
+        key = str(rider_id)
+        if key in self._filters:
+            return self._filters[key].get_estimate()
+        return None
+
+    def reset_rider(self, rider_id):
+        key = str(rider_id)
+        if key in self._filters:
+            self._filters[key].reset()
+
+    def clear_all(self):
+        self._filters.clear()
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# GLOBAL REGISTRY (process-level singleton)
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+_global_registry = RiderKalmanRegistry()
+
+
+def get_registry() -> RiderKalmanRegistry:
+    """Get the process-level rider Kalman filter registry."""
+    return _global_registry
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# STATELESS COORDINATE SMOOTHER
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+def smooth_coordinates(
+    lat: float,
+    lon: float,
+    *,
+    prior_lat: Optional[float] = None,
+    prior_lon: Optional[float] = None,
+    process_noise: float = 1e-4,
+    measurement_noise: float = 0.01,
+) -> Tuple[float, float]:
+    """
+    Stateless single-shot GPS smoother.
+    If a prior is provided, blends the new reading towards it.
+    """
+    f = GPSKalmanFilter(process_noise=process_noise, measurement_noise=measurement_noise)
+    if prior_lat is not None and prior_lon is not None:
+        try:
+            _flat = float(prior_lat)
+            _flon = float(prior_lon)
+            if GPSKalmanFilter._is_valid_coord(_flat, _flon):
+                f.update(_flat, _flon)
+        except (TypeError, ValueError):
+            pass
+    return f.update(lat, lon)
+
+
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+# BATCH SMOOTHERS
+# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+
+def smooth_rider_locations(riders: list) -> list:
+    """
+    Apply Kalman smoothing to a list of rider dicts in-place using
+    the global per-rider registry (history preserved across calls).
+
+    Reads/writes: latitude, longitude (and currentlat/currentlong if present).
+    Adds: _kalman_smoothed = True on each processed rider.
+    """
+    registry = get_registry()
+    for rider in riders:
+        try:
+            rider_id = (
+                rider.get("userid") or rider.get("riderid") or
+                rider.get("id") or "unknown"
+            )
+            raw_lat = float(rider.get("latitude") or rider.get("currentlat") or 0)
+            raw_lon = float(rider.get("longitude") or rider.get("currentlong") or 0)
+            if raw_lat == 0.0 and raw_lon == 0.0:
+                continue
+            smooth_lat, smooth_lon = registry.update(rider_id, raw_lat, raw_lon)
+            # Cast back to string for Go compatibility
+            s_lat, s_lon = str(round(smooth_lat, 8)), str(round(smooth_lon, 8))
+            rider["latitude"] = s_lat
+            rider["longitude"] = s_lon
+            if "currentlat" in rider:
+                rider["currentlat"] = s_lat
+            if "currentlong" in rider:
+                rider["currentlong"] = s_lon
+            rider["_kalman_smoothed"] = True
+        except Exception as e:
+            logger.debug(f"Kalman rider smoothing skipped: {e}")
+    return riders
+
+
+def smooth_order_coordinates(orders: list) -> list:
+    """
+    Validate and lightly normalise delivery coordinates in a list of order dicts.
+
+    DESIGN NOTE — why we do NOT use Kalman filtering here:
+    ─────────────────────────────────────────────────────
+    Kalman filtering is a *temporal* smoother: it blends successive measurements
+    from the same sensor over time. Delivery coordinates are a single static
+    point (one measurement). Feeding the kitchen location as a "prior" would
+    pull the customer's address toward the kitchen — exactly wrong.
+
+    Per-customer GPS accuracy is handled upstream by the FAISS coordinate store
+    (verified historical rider-confirmed delivery points). This function only
+    normalises the coordinate fields to floats so downstream code never sees
+    raw strings or None values.
+
+    Modifies orders in-place. Returns the same list.
+    """
+    for order in orders:
+        try:
+            dlat_raw = order.get("deliverylat") or order.get("droplat")
+            dlon_raw = order.get("deliverylong") or order.get("droplon")
+            if dlat_raw is None or dlon_raw is None:
+                continue
+            dlat = float(dlat_raw)
+            dlon = float(dlon_raw)
+            if not GPSKalmanFilter._is_valid_coord(dlat, dlon):
+                continue
+            # Normalise to string (Go service expects string coordinates)
+            s_lat = str(round(dlat, 8))
+            s_lon = str(round(dlon, 8))
+            order["deliverylat"] = s_lat
+            order["deliverylong"] = s_lon
+            if "droplat" in order:
+                order["droplat"] = s_lat
+            if "droplon" in order:
+                order["droplon"] = s_lon
+        except Exception as e:
+            logger.debug(f"Coordinate normalisation skipped: {e}")
+    return orders

app/services/routing/realistic_eta_calculator.py

@@ -0,0 +1,127 @@
+"""
+Realistic ETA Calculator for Delivery Operations
+
+Accounts for:
+- City traffic conditions
+- Stop time at pickup/delivery
+- Navigation time
+- Parking/finding address time
+- Different speeds for different order types
+"""
+
+import logging
+from typing import Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class RealisticETACalculator:
+    """
+    Calculates realistic ETAs accounting for real-world delivery conditions.
+    """
+    
+    def __init__(self):
+        from app.config.dynamic_config import get_config
+        cfg = get_config()
+
+        # BASE SPEED (km/h) - Driven by the DB configuration
+        base_speed = cfg.get("avg_speed_kmh", 18.0)
+        
+        # REALISTIC SPEEDS based on time of day
+        self.CITY_SPEED_HEAVY_TRAFFIC = base_speed * 0.7  # Usually ~12 km/h
+        self.CITY_SPEED_MODERATE      = base_speed        # Usually ~18 km/h
+        self.CITY_SPEED_LIGHT         = base_speed * 1.2  # Usually ~21.6 km/h
+        
+        # TIME BUFFERS (minutes)
+        self.PICKUP_TIME = cfg.get("eta_pickup_time_min", 3.0)
+        self.DELIVERY_TIME = cfg.get("eta_delivery_time_min", 4.0)
+        self.NAVIGATION_BUFFER = cfg.get("eta_navigation_buffer_min", 1.5)
+        
+        # DISTANCE-BASED SPEED SELECTION
+        # Short distances (<2km) are slower due to more stops/starts
+        # Long distances (>8km) might have highway portions
+        self.SHORT_TRIP_FACTOR = cfg.get("eta_short_trip_factor", 0.8)
+        self.LONG_TRIP_FACTOR  = cfg.get("eta_long_trip_factor", 1.1)
+        
+    def calculate_eta(self, 
+                     distance_km: float, 
+                     is_first_order: bool = False,
+                     order_type: str = "Economy",
+                     time_of_day: str = "peak") -> int:
+        """
+        Calculate realistic ETA in minutes.
+        
+        Args:
+            distance_km: Distance to travel in kilometers
+            is_first_order: If True, includes pickup time
+            order_type: "Economy", "Premium", or "Risky"
+            time_of_day: "peak", "normal", or "light" traffic
+            
+        Returns:
+            ETA in minutes (rounded up for safety)
+        """
+        
+        if distance_km <= 0:
+            return 0
+        
+        # 1. SELECT SPEED BASED ON CONDITIONS
+        if time_of_day == "peak":
+            base_speed = self.CITY_SPEED_HEAVY_TRAFFIC
+        elif time_of_day == "light":
+            base_speed = self.CITY_SPEED_LIGHT
+        else:
+            base_speed = self.CITY_SPEED_MODERATE
+        
+        # 2. ADJUST SPEED BASED ON DISTANCE
+        # Short trips are slower (more intersections, traffic lights)
+        if distance_km < 2.0:
+            effective_speed = base_speed * self.SHORT_TRIP_FACTOR
+        elif distance_km > 8.0:
+            effective_speed = base_speed * self.LONG_TRIP_FACTOR
+        else:
+            effective_speed = base_speed
+        
+        # 3. CALCULATE TRAVEL TIME
+        travel_time = (distance_km / effective_speed) * 60  # Convert to minutes
+        
+        # 4. ADD BUFFERS
+        total_time = travel_time
+        
+        # Pickup time (only for first order in sequence)
+        if is_first_order:
+            total_time += self.PICKUP_TIME
+        
+        # Delivery time (always)
+        total_time += self.DELIVERY_TIME
+        
+        # Navigation buffer (proportional to distance)
+        if distance_km > 3.0:
+            total_time += self.NAVIGATION_BUFFER
+        
+        # 5. SAFETY MARGIN (Round up to next minute)
+        # Riders prefer to arrive early than late
+        eta_minutes = int(total_time) + 1
+        
+        return eta_minutes
+    
+
+def get_time_of_day_category() -> str:
+    """
+    Determine current traffic conditions based on time.
+    
+    Returns:
+        "peak", "normal", or "light"
+    """
+    from datetime import datetime
+    
+    
+    current_hour = datetime.now().hour
+    
+    # Peak hours: 8-10 AM, 12-2 PM, 5-8 PM
+    if (8 <= current_hour < 10) or (12 <= current_hour < 14) or (17 <= current_hour < 20):
+        return "peak"
+    # Light traffic: Late night/early morning
+    elif current_hour < 7 or current_hour >= 22:
+        return "light"
+    else:
+        return "normal"

app/services/routing/rider_affinity_service.py

@@ -0,0 +1,171 @@
+"""
+Rider Affinity Service (learned, agentic)
+=========================================
+
+Learns each rider's real kitchen affinity and operating area from the local
+delivery mirror (`delivery_raw`) and exposes them MERGED with the curated config
+in `app/config/rider_preferences.py`:
+
+  * get_preferred_kitchens() = curated RIDER_PREFERRED_KITCHENS  ∪  learned
+    (rider served a kitchen >= `rider_affinity_min_deliveries` times). Union only —
+    never drops a curated owner.
+  * get_home_locations()     = curated RIDER_HOME_LOCATIONS, with a learned drop
+    centroid filled in ONLY for riders missing from the config.
+
+These feed SOFT steering (preference discount / home bonus / distance bypass) in
+the optimizer + assignment service. HARD kitchen locks and BLOCKED_RIDERS remain
+sourced from the curated config — learned data can only broaden preference, never
+change who is *eligible* for a kitchen. Recomputed by the autonomous agent loop.
+"""
+
+import logging
+import threading
+from collections import defaultdict
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+from app.config.rider_preferences import RIDER_PREFERRED_KITCHENS, RIDER_HOME_LOCATIONS
+
+logger = logging.getLogger(__name__)
+
+
+def _to_int_rid(v: Any) -> Optional[int]:
+    try:
+        return int(v)
+    except (TypeError, ValueError):
+        return None
+
+
+class RiderAffinityService:
+    def __init__(self):
+        self._learned_pref: Dict[int, List[str]] = {}
+        self._learned_home: Dict[int, Tuple[float, float]] = {}
+        self._last_refreshed: Optional[datetime] = None
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+
+    def refresh(self, days: Optional[int] = None) -> Dict[str, Any]:
+        """Recompute learned affinity from the local delivery mirror (no DB hit)."""
+        from app.config.dynamic_config import get_config
+        from app.services.routing.delivery_history_service import (
+            get_delivery_history_service, normalize_kitchen,
+        )
+        cfg = get_config()
+        window = int(days if days is not None else cfg.get("eta_history_days", 14))
+        min_n = int(cfg.get("rider_affinity_min_deliveries", 10))
+
+        rows = get_delivery_history_service()._load_raw_rows(window)
+        counts: Dict[int, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+        coords: Dict[int, List[Tuple[float, float]]] = defaultdict(list)
+        for r in rows:
+            rid = _to_int_rid(r.get("userid"))
+            if rid is None:
+                continue
+            kitchen = normalize_kitchen(r.get("pickupcustomer"))
+            if kitchen and kitchen != "?":
+                counts[rid][kitchen] += 1
+            try:
+                la, lo = float(r.get("dlat")), float(r.get("dlon"))
+                if la and lo:
+                    coords[rid].append((la, lo))
+            except (TypeError, ValueError):
+                continue
+
+        learned_pref = {
+            rid: [k for k, n in kc.items() if n >= min_n]
+            for rid, kc in counts.items()
+        }
+        learned_pref = {rid: ks for rid, ks in learned_pref.items() if ks}
+        learned_home = {
+            rid: (round(sum(p[0] for p in pts) / len(pts), 6),
+                  round(sum(p[1] for p in pts) / len(pts), 6))
+            for rid, pts in coords.items() if pts
+        }
+
+        with self._lock:
+            self._learned_pref = learned_pref
+            self._learned_home = learned_home
+            self._last_refreshed = datetime.utcnow()
+
+        summary = {
+            "status": "ok",
+            "riders_with_learned_kitchens": len(learned_pref),
+            "riders_with_learned_home": len(learned_home),
+            "min_deliveries": min_n,
+            "window_days": window,
+            "refreshed_at": self._last_refreshed.isoformat(),
+        }
+        logger.info(
+            f"[Affinity] learned kitchens for {len(learned_pref)} riders, "
+            f"home for {len(learned_home)} (min_deliveries={min_n})"
+        )
+        return summary
+
+    def _ensure_loaded(self) -> None:
+        if self._last_refreshed is None:
+            try:
+                self.refresh()
+            except Exception as e:
+                logger.debug(f"[Affinity] lazy refresh failed: {e}")
+
+    # ------------------------------------------------------------------
+
+    def get_preferred_kitchens(self) -> Dict[int, List[str]]:
+        """Curated config ∪ learned (union — never drops a curated owner)."""
+        from app.config.dynamic_config import get_config
+        if not bool(get_config().get("rider_affinity_enabled", True)):
+            return {rid: list(v) for rid, v in RIDER_PREFERRED_KITCHENS.items()}
+        self._ensure_loaded()
+        merged: Dict[int, List[str]] = {rid: list(v) for rid, v in RIDER_PREFERRED_KITCHENS.items()}
+        with self._lock:
+            learned = {rid: list(v) for rid, v in self._learned_pref.items()}
+        for rid, kitchens in learned.items():
+            base = merged.setdefault(rid, [])
+            base_lower = {b.lower() for b in base}
+            for k in kitchens:
+                if k.lower() not in base_lower:
+                    base.append(k)
+        return merged
+
+    def get_home_locations(self) -> Dict[int, Tuple[float, float]]:
+        """Curated home primary; learned drop centroid only for riders absent from config."""
+        from app.config.dynamic_config import get_config
+        if not bool(get_config().get("rider_affinity_enabled", True)):
+            return dict(RIDER_HOME_LOCATIONS)
+        self._ensure_loaded()
+        merged: Dict[int, Tuple[float, float]] = dict(RIDER_HOME_LOCATIONS)
+        with self._lock:
+            learned = dict(self._learned_home)
+        for rid, home in learned.items():
+            if rid not in merged or merged.get(rid) in (None, (0.0, 0.0)):
+                merged[rid] = home
+        return merged
+
+    def get_summary(self) -> Dict[str, Any]:
+        """Learned-vs-config diff for the admin endpoint."""
+        self._ensure_loaded()
+        with self._lock:
+            learned_pref = {rid: list(v) for rid, v in self._learned_pref.items()}
+            learned_home = dict(self._learned_home)
+        config_riders = set(RIDER_PREFERRED_KITCHENS)
+        new_pref_riders = sorted(set(learned_pref) - config_riders)
+        return {
+            "last_refreshed": self._last_refreshed.isoformat() if self._last_refreshed else None,
+            "config_preferred_riders": sorted(config_riders),
+            "learned_preferred": {str(k): v for k, v in sorted(learned_pref.items())},
+            "riders_newly_learned_not_in_config": new_pref_riders,
+            "learned_home_count": len(learned_home),
+        }
+
+
+_affinity: Optional[RiderAffinityService] = None
+_affinity_lock = threading.Lock()
+
+
+def get_rider_affinity() -> RiderAffinityService:
+    global _affinity
+    with _affinity_lock:
+        if _affinity is None:
+            _affinity = RiderAffinityService()
+        return _affinity

app/services/routing/road_sequencing_agent.py

@@ -0,0 +1,233 @@
+"""
+Road-Sequencing Decision Agent
+==============================
+
+Autonomous controller for the `routing_use_road_distance` feature. Instead of a
+human flipping a config flag, this agent periodically *measures* whether road-aware
+sequencing (road travel-time matrix + OR-Tools open-TSP) actually beats the default
+straight-line ordering on REAL recent batches, and turns the feature on or off by
+itself — re-checking every cycle so it self-corrects if the gain ever disappears.
+
+Decision (with hysteresis, so it doesn't flap):
+    mean travel-time gain >= routing_auto_enable_gain_pct  -> enable
+    mean travel-time gain <  routing_auto_disable_gain_pct -> disable
+    in between                                             -> leave as-is
+
+Every decision is stored (auditable) in DynamicConfig under `routing_road_eval`
+and exposed via GET /api/v1/ml/road-eval. Cost is bounded: it evaluates only
+`routing_eval_sample_batches` batches once per `routing_eval_interval_hours`.
+"""
+
+import asyncio
+import json
+import logging
+import threading
+import time
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+def _total(order_idx: List[int], matrix: List[List[float]]) -> float:
+    """Total open-route cost: origin(0) -> drops in `order_idx` (1-based into matrix)."""
+    seq = [0] + [i + 1 for i in order_idx]
+    return sum(matrix[seq[k]][seq[k + 1]] for k in range(len(seq) - 1))
+
+
+class RoadSequencingAgent:
+    """Measures road-vs-aerial sequencing on real batches and auto-toggles the flag."""
+
+    def __init__(self):
+        self._opt = None  # lazy RouteOptimizer
+        self._scheduler_started = False
+        self._lock = threading.Lock()
+        self.last_decision: Dict[str, Any] = {}
+
+    def _optimizer(self):
+        if self._opt is None:
+            from app.services.routing.route_optimizer import RouteOptimizer
+            self._opt = RouteOptimizer()
+        return self._opt
+
+    async def evaluate(self, sample_batches: int = 8, days: int = 14) -> Dict[str, Any]:
+        """
+        Compare aerial-order vs road-order travel time on sampled real batches.
+        Returns mean gain % and per-batch detail. No flag changes here.
+        """
+        from app.services.routing.delivery_history_service import get_delivery_history_service
+        from app.core.arrow_utils import calculate_haversine_matrix_vectorized
+
+        opt = self._optimizer()
+        if not opt.use_google_maps:
+            return {"evaluated": 0, "reason": "no_google_key", "mean_gain_pct": 0.0}
+
+        batches = get_delivery_history_service().sample_batches(
+            days=days, limit=sample_batches
+        )
+        if not batches:
+            return {"evaluated": 0, "reason": "no_local_batches", "mean_gain_pct": 0.0}
+
+        per_batch: List[Dict[str, Any]] = []
+        for b in batches:
+            origin, drops = b["origin"], b["drops"]
+            locs = [origin] + drops
+            matrix = await opt._road_duration_matrix(locs)
+            if matrix is None:
+                continue
+            # aerial order (current production behaviour)
+            lats = np.array([p[0] for p in locs])
+            lons = np.array([p[1] for p in locs])
+            aerial = calculate_haversine_matrix_vectorized(lats, lons)
+            aer = [i - 1 for i in opt._two_opt_improve(opt._solve_greedy(locs, aerial), aerial) if i != 0]
+            # road order (OR-Tools open-TSP on the road travel-time matrix)
+            road = [i - 1 for i in opt._solve_tsp_ortools(locs, matrix) if i != 0]
+
+            # Human's ACTUAL route: drops are already in delivered order
+            # (_load_raw_rows is ORDER BY deliverytime), so identity = what the
+            # rider really drove. This is the "do we beat the humans?" baseline.
+            human = list(range(len(drops)))
+
+            t_aer = _total(aer, matrix)
+            t_road = _total(road, matrix)
+            t_human = _total(human, matrix)
+            gain = (100.0 * (t_aer - t_road) / t_aer) if t_aer > 0 else 0.0
+            gain_vs_human = (100.0 * (t_human - t_road) / t_human) if t_human > 0 else 0.0
+            per_batch.append({
+                "rider": b["rider"], "day": b["day"], "drops": len(drops),
+                "aerial_min": round(t_aer, 1), "road_min": round(t_road, 1),
+                "human_min": round(t_human, 1),
+                "gain_pct": round(gain, 1),
+                "gain_vs_human_pct": round(gain_vs_human, 1),
+                "beat_human": t_road <= t_human + 1e-9,
+            })
+
+        if not per_batch:
+            return {"evaluated": 0, "reason": "matrix_unavailable", "mean_gain_pct": 0.0}
+
+        n = len(per_batch)
+        mean_gain = sum(x["gain_pct"] for x in per_batch) / n
+        mean_vs_human = sum(x["gain_vs_human_pct"] for x in per_batch) / n
+        beats = sum(1 for x in per_batch if x["beat_human"])
+        return {
+            "evaluated": n,
+            "mean_gain_pct": round(mean_gain, 2),
+            "median_gain_pct": round(sorted(x["gain_pct"] for x in per_batch)[n // 2], 2),
+            "human_beat_rate_pct": round(100.0 * beats / n, 1),
+            "human_beat_count": f"{beats}/{n}",
+            "mean_gain_vs_human_pct": round(mean_vs_human, 2),
+            "per_batch": per_batch,
+        }
+
+    def decide_and_apply(self) -> Dict[str, Any]:
+        """Run an evaluation and autonomously enable/disable road sequencing."""
+        from app.config.dynamic_config import get_config
+        cfg = get_config()
+
+        sample = int(cfg.get("routing_eval_sample_batches", 8))
+        days = int(cfg.get("routing_eval_days", 14))
+        enable_thr = float(cfg.get("routing_auto_enable_gain_pct", 3.0))
+        disable_thr = float(cfg.get("routing_auto_disable_gain_pct", 1.0))
+        min_batches = int(cfg.get("routing_eval_min_batches", 3))
+        auto_manage = bool(cfg.get("routing_auto_manage", True))
+
+        try:
+            result = asyncio.run(self.evaluate(sample_batches=sample, days=days))
+        except RuntimeError:
+            # An event loop is already running in this thread — use a fresh one.
+            loop = asyncio.new_event_loop()
+            try:
+                result = loop.run_until_complete(self.evaluate(sample_batches=sample, days=days))
+            finally:
+                loop.close()
+
+        current = bool(cfg.get("routing_use_road_distance", False))
+        evaluated = result.get("evaluated", 0)
+        gain = result.get("mean_gain_pct", 0.0)
+        action = "kept"
+        new_state = current
+
+        if not auto_manage:
+            action = "auto_manage_off"
+        elif evaluated < min_batches:
+            action = "insufficient_data"
+        else:
+            if gain >= enable_thr and not current:
+                new_state = True
+                cfg.set("routing_use_road_distance", True, source="road_agent")
+                action = "enabled"
+            elif gain < disable_thr and current:
+                new_state = False
+                cfg.set("routing_use_road_distance", False, source="road_agent")
+                action = "disabled"
+
+        decision = {
+            "decided_at": datetime.utcnow().isoformat(),
+            "action": action,
+            "flag_before": current,
+            "flag_after": new_state,
+            "mean_gain_pct": gain,
+            "enable_threshold_pct": enable_thr,
+            "disable_threshold_pct": disable_thr,
+            "evaluation": result,
+        }
+        self.last_decision = decision
+        try:
+            # Persist a compact copy for audit (without the full per-batch list).
+            compact = {k: v for k, v in decision.items() if k != "evaluation"}
+            compact["evaluated"] = evaluated
+            compact["human_beat_rate_pct"] = result.get("human_beat_rate_pct")
+            compact["mean_gain_vs_human_pct"] = result.get("mean_gain_vs_human_pct")
+            cfg.set("routing_road_eval", compact, source="road_agent")
+        except Exception:
+            pass
+
+        logger.info(
+            f"[RoadAgent] action={action} gain_vs_aerial={gain}% evaluated={evaluated} "
+            f"flag {current}->{new_state}"
+        )
+        if evaluated:
+            logger.info(
+                f"[RoadAgent] beat humans on {result.get('human_beat_count', '?')} batches "
+                f"(avg {result.get('mean_gain_vs_human_pct', 0)}% faster than actual delivered order)"
+            )
+        return decision
+
+    def ensure_background_agent(self, interval_hours: int = 24,
+                               warmup_seconds: int = 120) -> bool:
+        """Start the autonomous decision loop once (daemon thread)."""
+        with self._lock:
+            if self._scheduler_started:
+                return False
+            self._scheduler_started = True
+
+        def _loop():
+            # Let the ETA sync agent populate the local mirror first.
+            time.sleep(max(0, warmup_seconds))
+            logger.info(f"[RoadAgent] autonomous road-sequencing decision loop started "
+                        f"(interval={interval_hours}h)")
+            while True:
+                try:
+                    self.decide_and_apply()
+                except Exception as e:
+                    logger.warning(f"[RoadAgent] decision cycle failed (will retry): {e}")
+                from app.config.dynamic_config import get_config
+                hrs = int(get_config().get("routing_eval_interval_hours", interval_hours))
+                time.sleep(max(1, hrs) * 3600)
+
+        threading.Thread(target=_loop, daemon=True, name="road-seq-agent").start()
+        return True
+
+
+_agent: Optional[RoadSequencingAgent] = None
+_agent_lock = threading.Lock()
+
+
+def get_road_agent() -> RoadSequencingAgent:
+    global _agent
+    with _agent_lock:
+        if _agent is None:
+            _agent = RoadSequencingAgent()
+        return _agent

app/services/routing/zone_service.py

@@ -0,0 +1,201 @@
+
+import logging
+from typing import List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+class ZoneService:
+    """
+    Service to classify orders and riders into geographic zones.
+    Defaulting to Coimbatore logic as per user context.
+    """
+    
+    # Approximate Center of Coimbatore (Gandhipuram/Bus Stand area)
+    CENTER_LAT = 11.0168
+    CENTER_LON = 76.9558
+    
+    def __init__(self):
+        pass
+
+    def determine_zone(self, lat: float, lon: float) -> str:
+        """
+        Determine the zone (North, South, East, West, etc.) based on coordinates.
+        """
+        if lat == 0 or lon == 0:
+            return "Unknown"
+
+        lat_diff = lat - self.CENTER_LAT
+        lon_diff = lon - self.CENTER_LON
+        
+        # Simple Quadrant Logic
+        # North: +Lat
+        # South: -Lat
+        # East: +Lon
+        # West: -Lon
+        
+        # Define a small central buffer (0.01 degrees ~ 1.1km)
+        buffer = 0.010
+        
+        is_north = lat_diff > buffer
+        is_south = lat_diff < -buffer
+        is_east = lon_diff > buffer
+        is_west = lon_diff < -buffer
+        
+        zone_parts = []
+        
+        if is_north: zone_parts.append("North")
+        elif is_south: zone_parts.append("South")
+        
+        if is_east: zone_parts.append("East")
+        elif is_west: zone_parts.append("West")
+        
+        if not zone_parts:
+            return "Central"
+            
+        return " ".join(zone_parts)
+
+    def group_by_zones(self, flat_orders: List[Dict[str, Any]], unassigned_orders: List[Dict[str, Any]] = None, fuel_charge: float = 2.5, base_pay: float = 0.0) -> Dict[str, Any]:
+        """
+        Group a flat list of optimized orders into Zones -> Riders -> Orders.
+        Calculates profit per order and per zone.
+        """
+        zones_map = {} # "North East": { "riders": { rider_id: [orders] } }
+        unassigned_orders = unassigned_orders or []
+        
+        # Merge both for initial processing if you want everything zoned
+        all_to_process = []
+        for o in flat_orders:
+            all_to_process.append((o, True))
+        for o in unassigned_orders:
+            all_to_process.append((o, False))
+
+        for order, is_assigned in all_to_process:
+            # 1. Extract Coords
+            try:
+                # Prefer Delivery location for zoning (where the customer is)
+                lat = float(order.get("deliverylat") or order.get("droplat") or 0)
+                lon = float(order.get("deliverylong") or order.get("droplon") or 0)
+                pincode = str(order.get("deliveryzip") or "")
+            except:
+                lat, lon, pincode = 0, 0, ""
+            
+            # 2. Get Zone
+            zone_name = self.determine_zone(lat, lon)
+            order["zone_name"] = zone_name
+            
+            # 3. Initialize Zone Bucket
+            if zone_name not in zones_map:
+                zones_map[zone_name] = {
+                    "riders_map": {}, 
+                    "total_orders": 0,
+                    "assigned_orders": 0,
+                    "unassigned_orders": [],
+                    "total_kms": 0.0,
+                    "total_profit": 0.0
+                }
+            
+            # 4. Add to Rider bucket within Zone
+            rider_id = order.get("userid") or order.get("_id")
+            
+            # Track kms and profit for this zone
+            try:
+                # 'actualkms' is preferred for delivery distance
+                dist = float(order.get("actualkms", order.get("previouskms", 0)))
+                zones_map[zone_name]["total_kms"] += dist
+                
+                # Individual charge for this order: Fixed Base + Variable Distance
+                order_amount = float(order.get("orderamount") or order.get("deliveryamount") or 0)
+                rider_payment = dist * fuel_charge
+                profit = order_amount - rider_payment
+                
+                order["rider_charge"] = round(rider_payment, 2)
+                order["profit"] = round(profit, 2)
+
+                # ── PROFIT CLASS (separate from ordertype) ───────────────
+                # `ordertype` is set by route_optimizer based on delivery
+                # DISTANCE (Economy ≤5km, Premium ≤12km, Risky >12km).
+                # Overwriting it here with a profit-based label broke ETA
+                # calculations and downstream routing logic.
+                # Use `profit_class` for profit-based analytics instead.
+                if profit <= 0:
+                    order["profit_class"] = "Loss"
+                elif profit <= 5:
+                    order["profit_class"] = "Marginal"
+                elif profit <= 10:
+                    order["profit_class"] = "Profitable"
+                else:
+                    order["profit_class"] = "HighMargin"
+
+                zones_map[zone_name]["total_profit"] += profit
+            except:
+                pass
+
+            # If strictly unassigned order (no rider), put in unassigned
+            if not is_assigned:
+                zones_map[zone_name]["unassigned_orders"].append(order)
+            else:
+                str_rid = str(rider_id)
+                if str_rid not in zones_map[zone_name]["riders_map"]:
+                    zones_map[zone_name]["riders_map"][str_rid] = {
+                        "rider_details": {
+                            "id": str_rid,
+                            "name": order.get("username", "Unknown")
+                        },
+                        "orders": []
+                    }
+                zones_map[zone_name]["riders_map"][str_rid]["orders"].append(order)
+                zones_map[zone_name]["assigned_orders"] += 1
+            
+            zones_map[zone_name]["total_orders"] += 1
+
+        # 5. Restructure for API Response
+        output_zones = []
+        zone_metrics = []
+        
+        sorted_zone_names = sorted(zones_map.keys())
+        
+        for z_name in sorted_zone_names:
+            z_data = zones_map[z_name]
+            
+            # Flatten riders map
+            riders_list = []
+            for r_id, r_data in z_data["riders_map"].items():
+                riders_list.append({
+                    "rider_id": r_data["rider_details"]["id"],
+                    "rider_name": r_data["rider_details"]["name"],
+                    "orders_count": len(r_data["orders"]),
+                    "orders": r_data["orders"]
+                })
+            
+            # Create the flat metric summary
+            metrics = {
+                "zone_name": z_name,
+                "total_orders": z_data["total_orders"],
+                "assigned_orders": z_data["assigned_orders"],
+                "unassigned_orders_count": len(z_data["unassigned_orders"]),
+                "active_riders_count": len(riders_list),
+                "total_delivery_kms": round(z_data["total_kms"], 2),
+                "total_profit": round(z_data["total_profit"], 2)
+            }
+            
+            zone_metrics.append(metrics)
+            
+            # Create the detailed zone object with flattened metrics
+            zone_obj = {
+                "zone_name": z_name,
+                "total_orders": metrics["total_orders"],
+                "active_riders_count": metrics["active_riders_count"],
+                "assigned_orders": metrics["assigned_orders"],
+                "unassigned_orders_count": metrics["unassigned_orders_count"],
+                "total_delivery_kms": metrics["total_delivery_kms"],
+                "total_profit": metrics["total_profit"],
+                "riders": riders_list,
+                "unassigned_orders": z_data["unassigned_orders"]
+            }
+            
+            output_zones.append(zone_obj)
+            
+        return {
+            "detailed_zones": output_zones,
+            "zone_analysis": zone_metrics
+        }