new changes in the api
This commit is contained in:
Binary file not shown.
Binary file not shown.
BIN
app/services/routing/__pycache__/gps_smoother.cpython-312.pyc
Normal file
BIN
app/services/routing/__pycache__/gps_smoother.cpython-312.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -444,6 +444,37 @@ class DeliveryHistoryService:
|
||||
cols = ["deliveryid", "userid", "pickupcustomer", "pickuptime", "deliverytime", "dlat", "dlon", "plat", "plon"]
|
||||
return [dict(zip(cols, r)) for r in rows]
|
||||
|
||||
def get_pattern_records(self, days: int) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Shape the local nearledb mirror into the record format the delivery-
|
||||
history pattern store (Phase-0 kitchen+zone rider lookup) expects:
|
||||
kitchen/pickuplat/pickuplon/deliverylat/deliverylong/userid/ridername.
|
||||
|
||||
`ridername` is always "" — delivery_raw doesn't store it (it's
|
||||
display-only in the /ml/history debug endpoint; every actual matching
|
||||
decision is keyed on userid, so this doesn't affect assignment).
|
||||
"""
|
||||
records: List[Dict[str, Any]] = []
|
||||
for r in self._load_raw_rows(days):
|
||||
try:
|
||||
plat = float(r.get("plat") or 0)
|
||||
plon = float(r.get("plon") or 0)
|
||||
dlat = float(r.get("dlat") or 0)
|
||||
dlon = float(r.get("dlon") or 0)
|
||||
uid = int(float(r.get("userid") or 0))
|
||||
if not plat or not dlat or uid == 0:
|
||||
continue
|
||||
records.append({
|
||||
"kitchen": (r.get("pickupcustomer") or "").strip().lower(),
|
||||
"pickuplat": plat, "pickuplon": plon,
|
||||
"deliverylat": dlat, "deliverylong": dlon,
|
||||
"userid": uid,
|
||||
"ridername": "",
|
||||
})
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return records
|
||||
|
||||
def sample_batches(self, days: int = 14, min_drops: int = 4, max_drops: int = 15,
|
||||
limit: int = 10) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
@@ -623,12 +654,24 @@ class DeliveryHistoryService:
|
||||
full: bool = False) -> Dict[str, Any]:
|
||||
"""
|
||||
Full pipeline: incremental DB sync -> prune local store -> rebuild
|
||||
aggregates locally. This is what the scheduler and the admin endpoint
|
||||
call. The DB is touched only by the sync step (new rows only).
|
||||
aggregates locally -> (if pattern_source=db) rebuild the Phase-0
|
||||
pattern store from the same synced rows. This is what the scheduler
|
||||
and the admin endpoint call. The DB is touched only by the sync step
|
||||
(new rows only).
|
||||
"""
|
||||
from app.config.dynamic_config import get_config
|
||||
cfg = get_config()
|
||||
pattern_on = str(cfg.get("pattern_source", "csv")) == "db"
|
||||
pattern_days = int(cfg.get("pattern_history_days", 30))
|
||||
# Local retention must cover whichever consumer needs more history
|
||||
# (the ETA window vs. the pattern-store window) without changing the
|
||||
# ETA computation's own `days` window below — that backtest result is
|
||||
# already validated at 14 days and this must not perturb it.
|
||||
retain_days = max(days, pattern_days) if pattern_on else days
|
||||
|
||||
with _WRITE_LOCK:
|
||||
try:
|
||||
sync = self.sync_from_db(days=days, tenant_id=tenant_id, full=full)
|
||||
sync = self.sync_from_db(days=retain_days, tenant_id=tenant_id, full=full)
|
||||
except Exception as e:
|
||||
logger.error(f"[DeliveryHistory] sync failed: {e}", exc_info=True)
|
||||
# Still try to serve whatever is already local.
|
||||
@@ -636,9 +679,23 @@ class DeliveryHistoryService:
|
||||
self._last_summary = {"status": "sync_failed", "error": str(e), "rebuild": rebuilt}
|
||||
return self._last_summary
|
||||
|
||||
self._prune_raw(days)
|
||||
self._prune_raw(retain_days)
|
||||
rebuilt = self.rebuild_aggregates(days)
|
||||
self._last_summary = {"status": "ok", "sync": sync, "rebuild": rebuilt}
|
||||
|
||||
pattern_rebuild = None
|
||||
if pattern_on:
|
||||
try:
|
||||
from app.services.vector.delivery_history_store import get_delivery_history_store
|
||||
records = self.get_pattern_records(pattern_days)
|
||||
n = get_delivery_history_store().rebuild_from_records(records)
|
||||
pattern_rebuild = {"records": n, "days": pattern_days}
|
||||
except Exception as e:
|
||||
logger.warning(f"[DeliveryHistory] pattern-store rebuild skipped: {e}")
|
||||
|
||||
self._last_summary = {
|
||||
"status": "ok", "sync": sync, "rebuild": rebuilt,
|
||||
"pattern_rebuild": pattern_rebuild,
|
||||
}
|
||||
return self._last_summary
|
||||
|
||||
# -- cache + lookup -----------------------------------------------------
|
||||
|
||||
150
app/services/routing/gps_smoother.py
Normal file
150
app/services/routing/gps_smoother.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
GPS location smoothing — rider-api
|
||||
|
||||
Smooths noisy rider GPS pings (typical error +-5-15m, worse on poor signal,
|
||||
occasional bad-fix "jumps") using a per-rider exponential moving average
|
||||
(EMA): each new reading is blended with the running estimate so a single bad
|
||||
ping can't yank the rider's position, while the estimate still tracks real
|
||||
movement.
|
||||
|
||||
This used to be implemented as a full Kalman filter (per-coordinate process/
|
||||
measurement covariance, gain computed every update). For a "constant
|
||||
position" state model with no velocity term — which is what this is, since
|
||||
we're smoothing noisy pings, not tracking motion — the Kalman update reduces
|
||||
mathematically to an EMA once the gain reaches steady state, which happens
|
||||
within the first couple of updates. The EMA below is the same behavior with
|
||||
one constant instead of two, and no covariance bookkeeping to explain to the
|
||||
next person reading this file.
|
||||
|
||||
Only two things are actually used elsewhere in the app:
|
||||
smooth_rider_locations(riders) — per-rider EMA, stateful across calls
|
||||
smooth_order_coordinates(orders) — validates/normalises delivery coords
|
||||
(NOT smoothed — see its docstring)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Smoothing factor: how much weight a new GPS reading gets against the
|
||||
# running estimate. Lower = smoother/slower to react, higher = trusts each
|
||||
# new ping more. 0.1 matches the steady-state behavior of the Kalman filter
|
||||
# this replaced (process_noise=1e-4, measurement_noise=0.01).
|
||||
_ALPHA = 0.1
|
||||
_STALE_SECONDS = 1800.0 # reset a rider's running estimate after 30 min silence
|
||||
|
||||
|
||||
def _is_valid_coord(lat: float, lon: float) -> bool:
|
||||
try:
|
||||
lat, lon = float(lat), float(lon)
|
||||
return (
|
||||
-90.0 <= lat <= 90.0
|
||||
and -180.0 <= lon <= 180.0
|
||||
and not (lat == 0.0 and lon == 0.0)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
class _RiderEstimate:
|
||||
"""Running EMA estimate of one rider's position."""
|
||||
|
||||
def __init__(self):
|
||||
self.lat: Optional[float] = None
|
||||
self.lon: Optional[float] = None
|
||||
self.last_updated: float = time.time()
|
||||
|
||||
def update(self, lat: float, lon: float) -> Tuple[float, float]:
|
||||
if not _is_valid_coord(lat, lon):
|
||||
return (self.lat, self.lon) if self.lat is not None else (lat, lon)
|
||||
|
||||
if time.time() - self.last_updated > _STALE_SECONDS:
|
||||
self.lat = self.lon = None # stale — start fresh
|
||||
|
||||
if self.lat is None:
|
||||
self.lat, self.lon = lat, lon
|
||||
else:
|
||||
self.lat += _ALPHA * (lat - self.lat)
|
||||
self.lon += _ALPHA * (lon - self.lon)
|
||||
|
||||
self.last_updated = time.time()
|
||||
return self.lat, self.lon
|
||||
|
||||
|
||||
_rider_estimates: Dict[str, _RiderEstimate] = {}
|
||||
|
||||
|
||||
def smooth_rider_locations(riders: list) -> list:
|
||||
"""
|
||||
Apply EMA smoothing to a list of rider dicts in-place, keyed by rider id
|
||||
(history preserved across calls via a process-level registry).
|
||||
|
||||
Reads/writes: latitude, longitude (and currentlat/currentlong if present).
|
||||
Adds: _location_smoothed = True on each processed rider.
|
||||
"""
|
||||
for rider in riders:
|
||||
try:
|
||||
rider_id = str(
|
||||
rider.get("userid") or rider.get("riderid") or
|
||||
rider.get("id") or "unknown"
|
||||
)
|
||||
raw_lat = float(rider.get("latitude") or rider.get("currentlat") or 0)
|
||||
raw_lon = float(rider.get("longitude") or rider.get("currentlong") or 0)
|
||||
if raw_lat == 0.0 and raw_lon == 0.0:
|
||||
continue
|
||||
|
||||
estimate = _rider_estimates.setdefault(rider_id, _RiderEstimate())
|
||||
smooth_lat, smooth_lon = estimate.update(raw_lat, raw_lon)
|
||||
|
||||
# Cast back to string for Go compatibility
|
||||
s_lat, s_lon = str(round(smooth_lat, 8)), str(round(smooth_lon, 8))
|
||||
rider["latitude"] = s_lat
|
||||
rider["longitude"] = s_lon
|
||||
if "currentlat" in rider:
|
||||
rider["currentlat"] = s_lat
|
||||
if "currentlong" in rider:
|
||||
rider["currentlong"] = s_lon
|
||||
rider["_location_smoothed"] = True
|
||||
except Exception as e:
|
||||
logger.debug(f"Rider location smoothing skipped: {e}")
|
||||
return riders
|
||||
|
||||
|
||||
def smooth_order_coordinates(orders: list) -> list:
|
||||
"""
|
||||
Validate and lightly normalise delivery coordinates in a list of order dicts.
|
||||
|
||||
DESIGN NOTE — why these are NOT smoothed:
|
||||
Smoothing blends successive measurements from the same source over time.
|
||||
Delivery coordinates are a single static point (one measurement) — there
|
||||
is nothing to blend. Per-customer GPS accuracy is handled upstream by the
|
||||
FAISS coordinate store (verified historical rider-confirmed delivery
|
||||
points). This function only normalises the coordinate fields to floats
|
||||
so downstream code never sees raw strings or None values.
|
||||
|
||||
Modifies orders in-place. Returns the same list.
|
||||
"""
|
||||
for order in orders:
|
||||
try:
|
||||
dlat_raw = order.get("deliverylat") or order.get("droplat")
|
||||
dlon_raw = order.get("deliverylong") or order.get("droplon")
|
||||
if dlat_raw is None or dlon_raw is None:
|
||||
continue
|
||||
dlat = float(dlat_raw)
|
||||
dlon = float(dlon_raw)
|
||||
if not _is_valid_coord(dlat, dlon):
|
||||
continue
|
||||
# Normalise to string (Go service expects string coordinates)
|
||||
s_lat = str(round(dlat, 8))
|
||||
s_lon = str(round(dlon, 8))
|
||||
order["deliverylat"] = s_lat
|
||||
order["deliverylong"] = s_lon
|
||||
if "droplat" in order:
|
||||
order["droplat"] = s_lat
|
||||
if "droplon" in order:
|
||||
order["droplon"] = s_lon
|
||||
except Exception as e:
|
||||
logger.debug(f"Coordinate normalisation skipped: {e}")
|
||||
return orders
|
||||
@@ -1,327 +0,0 @@
|
||||
"""
|
||||
GPS Kalman Filter \u2014 rider-api
|
||||
|
||||
A 1D Kalman filter applied independently to latitude and longitude
|
||||
to smooth noisy GPS coordinates from riders and delivery points.
|
||||
|
||||
Why Kalman for GPS?
|
||||
- GPS readings contain measurement noise (\u00b15\u201315m typical, \u00b150m poor signal)
|
||||
- Rider location pings can "jump" due to bad signal or device error
|
||||
- Kalman filter gives an optimal estimate by balancing:
|
||||
(1) Previous predicted position (process model)
|
||||
(2) New GPS measurement (observation model)
|
||||
|
||||
Design:
|
||||
- Separate filter instance per rider (stateful \u2014 preserves history)
|
||||
- `CoordinateKalmanFilter` \u2014 single lat/lon smoother
|
||||
- `GPSKalmanFilter` \u2014 wraps two CoordinateKalmanFilters (lat + lon)
|
||||
- `RiderKalmanRegistry` \u2014 manages per-rider filter instances
|
||||
- `smooth_coordinates()` \u2014 stateless single-shot smoother for delivery coords
|
||||
|
||||
Usage:
|
||||
# Stateless (one-shot, no history \u2014 for delivery coords):
|
||||
smooth_lat, smooth_lon = smooth_coordinates(raw_lat, raw_lon)
|
||||
|
||||
# Stateful (per-rider, preserves motion history):
|
||||
registry = RiderKalmanRegistry()
|
||||
lat, lon = registry.update(rider_id=1116, lat=11.0067, lon=76.9558)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# CORE 1D KALMAN FILTER
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
class CoordinateKalmanFilter:
|
||||
"""
|
||||
1-dimensional Kalman filter for a single GPS coordinate (lat or lon).
|
||||
|
||||
State model: position only (constant position with random walk).
|
||||
|
||||
Equations:
|
||||
Prediction: x\u0302\u2096\u207b = x\u0302\u2096\u208b\u2081 (no movement assumed between pings)
|
||||
P\u0302\u2096\u207b = P\u2096\u208b\u2081 + Q (uncertainty grows over time)
|
||||
|
||||
Update: K\u2096 = P\u0302\u2096\u207b / (P\u0302\u2096\u207b + R) (Kalman gain)
|
||||
x\u0302\u2096 = x\u0302\u2096\u207b + K\u2096\u00b7(z\u2096 - x\u0302\u2096\u207b) (weighted fusion)
|
||||
P\u2096 = (1 - K\u2096)\u00b7P\u0302\u2096\u207b (update uncertainty)
|
||||
|
||||
Parameters:
|
||||
process_noise (Q): How much position can change between measurements.
|
||||
Higher = filter trusts new measurements more (less smoothing).
|
||||
measurement_noise (R): GPS measurement uncertainty.
|
||||
Higher = filter trusts history more (more smoothing).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
process_noise: float = 1e-4,
|
||||
measurement_noise: float = 0.01,
|
||||
initial_uncertainty: float = 1.0,
|
||||
):
|
||||
self.Q = process_noise
|
||||
self.R = measurement_noise
|
||||
self._x: Optional[float] = None
|
||||
self._P: float = initial_uncertainty
|
||||
|
||||
@property
|
||||
def initialized(self) -> bool:
|
||||
return self._x is not None
|
||||
|
||||
def update(self, measurement: float) -> float:
|
||||
"""Process one new measurement and return the filtered estimate."""
|
||||
if not self.initialized:
|
||||
self._x = measurement
|
||||
return self._x
|
||||
|
||||
# Predict
|
||||
x_prior = self._x
|
||||
P_prior = self._P + self.Q
|
||||
|
||||
# Update
|
||||
K = P_prior / (P_prior + self.R)
|
||||
self._x = x_prior + K * (measurement - x_prior)
|
||||
self._P = (1.0 - K) * P_prior
|
||||
|
||||
return self._x
|
||||
|
||||
def reset(self):
|
||||
self._x = None
|
||||
self._P = 1.0
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# 2D GPS KALMAN FILTER (lat + lon)
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
class GPSKalmanFilter:
|
||||
"""
|
||||
Two-dimensional GPS smoother using independent 1D Kalman filters
|
||||
for latitude and longitude.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
process_noise: float = 1e-4,
|
||||
measurement_noise: float = 0.01,
|
||||
):
|
||||
self.lat_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
|
||||
self.lon_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
|
||||
self.last_updated: float = time.time()
|
||||
self.update_count: int = 0
|
||||
|
||||
def update(self, lat: float, lon: float) -> Tuple[float, float]:
|
||||
"""Feed a new GPS reading and get the smoothed (lat, lon)."""
|
||||
if not self._is_valid_coord(lat, lon):
|
||||
if self.lat_filter.initialized:
|
||||
return self.lat_filter._x, self.lon_filter._x
|
||||
return lat, lon
|
||||
|
||||
smooth_lat = self.lat_filter.update(lat)
|
||||
smooth_lon = self.lon_filter.update(lon)
|
||||
self.last_updated = time.time()
|
||||
self.update_count += 1
|
||||
|
||||
return smooth_lat, smooth_lon
|
||||
|
||||
def get_estimate(self) -> Optional[Tuple[float, float]]:
|
||||
if self.lat_filter.initialized:
|
||||
return self.lat_filter._x, self.lon_filter._x
|
||||
return None
|
||||
|
||||
def reset(self):
|
||||
self.lat_filter.reset()
|
||||
self.lon_filter.reset()
|
||||
self.update_count = 0
|
||||
|
||||
@staticmethod
|
||||
def _is_valid_coord(lat: float, lon: float) -> bool:
|
||||
try:
|
||||
lat, lon = float(lat), float(lon)
|
||||
return (
|
||||
-90.0 <= lat <= 90.0
|
||||
and -180.0 <= lon <= 180.0
|
||||
and not (lat == 0.0 and lon == 0.0)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# PER-RIDER FILTER REGISTRY
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
class RiderKalmanRegistry:
|
||||
"""
|
||||
Maintains per-rider Kalman filter instances across calls.
|
||||
Stale filters (> 30 min silence) are automatically reset.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
process_noise: float = 1e-4,
|
||||
measurement_noise: float = 0.01,
|
||||
stale_seconds: float = 1800.0,
|
||||
):
|
||||
self._filters: Dict[str, GPSKalmanFilter] = {}
|
||||
self._process_noise = process_noise
|
||||
self._measurement_noise = measurement_noise
|
||||
self._stale_seconds = stale_seconds
|
||||
|
||||
def _get_or_create(self, rider_id) -> GPSKalmanFilter:
|
||||
key = str(rider_id)
|
||||
now = time.time()
|
||||
if key in self._filters:
|
||||
f = self._filters[key]
|
||||
if now - f.last_updated > self._stale_seconds:
|
||||
f.reset()
|
||||
return f
|
||||
self._filters[key] = GPSKalmanFilter(
|
||||
process_noise=self._process_noise,
|
||||
measurement_noise=self._measurement_noise,
|
||||
)
|
||||
return self._filters[key]
|
||||
|
||||
def update(self, rider_id, lat: float, lon: float) -> Tuple[float, float]:
|
||||
return self._get_or_create(rider_id).update(lat, lon)
|
||||
|
||||
def get_estimate(self, rider_id) -> Optional[Tuple[float, float]]:
|
||||
key = str(rider_id)
|
||||
if key in self._filters:
|
||||
return self._filters[key].get_estimate()
|
||||
return None
|
||||
|
||||
def reset_rider(self, rider_id):
|
||||
key = str(rider_id)
|
||||
if key in self._filters:
|
||||
self._filters[key].reset()
|
||||
|
||||
def clear_all(self):
|
||||
self._filters.clear()
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# GLOBAL REGISTRY (process-level singleton)
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
_global_registry = RiderKalmanRegistry()
|
||||
|
||||
|
||||
def get_registry() -> RiderKalmanRegistry:
|
||||
"""Get the process-level rider Kalman filter registry."""
|
||||
return _global_registry
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# STATELESS COORDINATE SMOOTHER
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
def smooth_coordinates(
|
||||
lat: float,
|
||||
lon: float,
|
||||
*,
|
||||
prior_lat: Optional[float] = None,
|
||||
prior_lon: Optional[float] = None,
|
||||
process_noise: float = 1e-4,
|
||||
measurement_noise: float = 0.01,
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Stateless single-shot GPS smoother.
|
||||
If a prior is provided, blends the new reading towards it.
|
||||
"""
|
||||
f = GPSKalmanFilter(process_noise=process_noise, measurement_noise=measurement_noise)
|
||||
if prior_lat is not None and prior_lon is not None:
|
||||
try:
|
||||
_flat = float(prior_lat)
|
||||
_flon = float(prior_lon)
|
||||
if GPSKalmanFilter._is_valid_coord(_flat, _flon):
|
||||
f.update(_flat, _flon)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return f.update(lat, lon)
|
||||
|
||||
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
# BATCH SMOOTHERS
|
||||
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
||||
|
||||
def smooth_rider_locations(riders: list) -> list:
|
||||
"""
|
||||
Apply Kalman smoothing to a list of rider dicts in-place using
|
||||
the global per-rider registry (history preserved across calls).
|
||||
|
||||
Reads/writes: latitude, longitude (and currentlat/currentlong if present).
|
||||
Adds: _kalman_smoothed = True on each processed rider.
|
||||
"""
|
||||
registry = get_registry()
|
||||
for rider in riders:
|
||||
try:
|
||||
rider_id = (
|
||||
rider.get("userid") or rider.get("riderid") or
|
||||
rider.get("id") or "unknown"
|
||||
)
|
||||
raw_lat = float(rider.get("latitude") or rider.get("currentlat") or 0)
|
||||
raw_lon = float(rider.get("longitude") or rider.get("currentlong") or 0)
|
||||
if raw_lat == 0.0 and raw_lon == 0.0:
|
||||
continue
|
||||
smooth_lat, smooth_lon = registry.update(rider_id, raw_lat, raw_lon)
|
||||
# Cast back to string for Go compatibility
|
||||
s_lat, s_lon = str(round(smooth_lat, 8)), str(round(smooth_lon, 8))
|
||||
rider["latitude"] = s_lat
|
||||
rider["longitude"] = s_lon
|
||||
if "currentlat" in rider:
|
||||
rider["currentlat"] = s_lat
|
||||
if "currentlong" in rider:
|
||||
rider["currentlong"] = s_lon
|
||||
rider["_kalman_smoothed"] = True
|
||||
except Exception as e:
|
||||
logger.debug(f"Kalman rider smoothing skipped: {e}")
|
||||
return riders
|
||||
|
||||
|
||||
def smooth_order_coordinates(orders: list) -> list:
|
||||
"""
|
||||
Validate and lightly normalise delivery coordinates in a list of order dicts.
|
||||
|
||||
DESIGN NOTE — why we do NOT use Kalman filtering here:
|
||||
─────────────────────────────────────────────────────
|
||||
Kalman filtering is a *temporal* smoother: it blends successive measurements
|
||||
from the same sensor over time. Delivery coordinates are a single static
|
||||
point (one measurement). Feeding the kitchen location as a "prior" would
|
||||
pull the customer's address toward the kitchen — exactly wrong.
|
||||
|
||||
Per-customer GPS accuracy is handled upstream by the FAISS coordinate store
|
||||
(verified historical rider-confirmed delivery points). This function only
|
||||
normalises the coordinate fields to floats so downstream code never sees
|
||||
raw strings or None values.
|
||||
|
||||
Modifies orders in-place. Returns the same list.
|
||||
"""
|
||||
for order in orders:
|
||||
try:
|
||||
dlat_raw = order.get("deliverylat") or order.get("droplat")
|
||||
dlon_raw = order.get("deliverylong") or order.get("droplon")
|
||||
if dlat_raw is None or dlon_raw is None:
|
||||
continue
|
||||
dlat = float(dlat_raw)
|
||||
dlon = float(dlon_raw)
|
||||
if not GPSKalmanFilter._is_valid_coord(dlat, dlon):
|
||||
continue
|
||||
# Normalise to string (Go service expects string coordinates)
|
||||
s_lat = str(round(dlat, 8))
|
||||
s_lon = str(round(dlon, 8))
|
||||
order["deliverylat"] = s_lat
|
||||
order["deliverylong"] = s_lon
|
||||
if "droplat" in order:
|
||||
order["droplat"] = s_lat
|
||||
if "droplon" in order:
|
||||
order["droplon"] = s_lon
|
||||
except Exception as e:
|
||||
logger.debug(f"Coordinate normalisation skipped: {e}")
|
||||
return orders
|
||||
@@ -10,7 +10,6 @@ FEATURES:
|
||||
- Automatic outlier detection and coordinate correction
|
||||
- Hybrid distance calculation (Google Maps + Haversine fallback)
|
||||
- Robust error handling for invalid inputs
|
||||
- Composite cost function (idea.txt: distance + profit - margin)
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -21,7 +20,7 @@ import asyncio
|
||||
from typing import Dict, Any, List as _List, Optional, Tuple, Union
|
||||
from datetime import datetime, timedelta
|
||||
import httpx
|
||||
from app.services.routing.kalman_filter import smooth_order_coordinates
|
||||
from app.services.routing.gps_smoother import smooth_order_coordinates
|
||||
import numpy as np
|
||||
from app.core.arrow_utils import calculate_haversine_matrix_vectorized
|
||||
from app.config.dynamic_config import get_config
|
||||
@@ -38,191 +37,6 @@ except ImportError:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CompositeCostCalculator:
|
||||
"""
|
||||
Composite Cost Function for Profit-Aware Route Optimization.
|
||||
|
||||
Based on idea.txt data encoding techniques:
|
||||
- Total Cost = Distance Cost + Rider Cost - Merchant Profit
|
||||
- Edge Cost = (distance * fuel_rate) + rider_cost - merchant_margin
|
||||
- route_score = profit - composite_cost (what we want to MAXIMIZE)
|
||||
|
||||
This transforms the problem from:
|
||||
"Find shortest route" -> "Find most profitable route"
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Cost parameters (can be ML-tuned via DynamicConfig)
|
||||
self.fuel_rate_per_km = 2.5
|
||||
self.base_rider_cost = 0.0
|
||||
self.opportunity_cost_per_km = 0.5 # Cost of rider's time per km
|
||||
self.profit_weight = 0.3 # How much profit influences routing (0-1)
|
||||
self.traffic_multiplier_peak = 1.5 # Peak hour traffic penalty
|
||||
self.traffic_multiplier_normal = 1.2 # Normal traffic multiplier
|
||||
|
||||
# Defaults for orders without profit data
|
||||
self.default_order_amount = 80.0
|
||||
self.default_merchant_margin = 5.0
|
||||
|
||||
def calculate_composite_cost(
|
||||
self,
|
||||
distance_km: float,
|
||||
order_amount: float = None,
|
||||
merchant_margin: float = None,
|
||||
traffic_factor: float = 1.0,
|
||||
time_of_day: str = "NORMAL",
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
Calculate composite cost for a route edge.
|
||||
|
||||
Args:
|
||||
distance_km: Distance for this leg
|
||||
order_amount: Revenue from this order (if known)
|
||||
merchant_margin: Merchant's margin for this order (if known)
|
||||
traffic_factor: Traffic multiplier (1.0 = normal)
|
||||
time_of_day: Traffic time category ("PEAK", "NORMAL", "OFF_PEAK")
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- distance_cost: Raw distance cost
|
||||
- rider_cost: Total rider cost for this leg
|
||||
- gross_profit: Revenue - rider cost
|
||||
- net_cost: Cost after profit adjustment (MINIMIZE THIS)
|
||||
- route_score: Profitability score (MAXIMIZE THIS)
|
||||
"""
|
||||
# Distance cost = fuel + opportunity cost
|
||||
distance_cost = distance_km * self.fuel_rate_per_km
|
||||
|
||||
# Rider cost = base + distance cost
|
||||
rider_cost = self.base_rider_cost + distance_cost
|
||||
|
||||
# Apply traffic penalty
|
||||
if time_of_day == "PEAK":
|
||||
rider_cost *= self.traffic_multiplier_peak
|
||||
elif time_of_day == "NORMAL":
|
||||
rider_cost *= self.traffic_multiplier_normal
|
||||
|
||||
# Apply custom traffic factor
|
||||
rider_cost *= traffic_factor
|
||||
|
||||
# Profit calculation (target encoding: use order data if available)
|
||||
if order_amount is None:
|
||||
order_amount = self.default_order_amount
|
||||
if merchant_margin is None:
|
||||
merchant_margin = self.default_merchant_margin
|
||||
|
||||
gross_profit = order_amount - rider_cost
|
||||
|
||||
# Net cost = rider cost - profit contribution
|
||||
# This means high-profit orders have LOWER cost (more desirable)
|
||||
profit_adjustment = gross_profit * self.profit_weight
|
||||
net_cost = rider_cost - profit_adjustment
|
||||
|
||||
# Route score = profit - opportunity cost (for route planning)
|
||||
route_score = gross_profit - (distance_km * self.opportunity_cost_per_km)
|
||||
|
||||
# Ensure net_cost is never negative (minimum cost for any delivery)
|
||||
net_cost = max(net_cost, 5.0) # Minimum 5 km equivalent cost
|
||||
|
||||
return {
|
||||
"distance_cost": round(distance_cost, 2),
|
||||
"rider_cost": round(rider_cost, 2),
|
||||
"gross_profit": round(gross_profit, 2),
|
||||
"net_cost": round(net_cost, 2),
|
||||
"route_score": round(route_score, 2),
|
||||
}
|
||||
|
||||
def calculate_cost_matrix(
|
||||
self,
|
||||
dist_matrix: np.ndarray,
|
||||
orders: _List[Dict[str, Any]] = None,
|
||||
traffic_condition: str = "NORMAL",
|
||||
) -> Tuple[np.ndarray, Dict[str, Any]]:
|
||||
"""
|
||||
Calculate composite cost matrix for all node pairs.
|
||||
|
||||
Args:
|
||||
dist_matrix: Distance matrix (N x N)
|
||||
orders: List of orders (for profit data)
|
||||
traffic_condition: Traffic condition ("PEAK", "NORMAL", "OFF_PEAK")
|
||||
|
||||
Returns:
|
||||
Tuple of (cost_matrix, summary_stats)
|
||||
"""
|
||||
n = len(dist_matrix)
|
||||
cost_matrix = np.zeros((n, n))
|
||||
|
||||
# Extract order data for profit encoding
|
||||
order_amounts = []
|
||||
merchant_margins = []
|
||||
|
||||
if orders:
|
||||
for o in orders:
|
||||
try:
|
||||
amount = float(
|
||||
o.get("orderamount")
|
||||
or o.get("deliveryamount")
|
||||
or self.default_order_amount
|
||||
)
|
||||
margin = float(
|
||||
o.get("merchant_margin") or self.default_merchant_margin
|
||||
)
|
||||
except:
|
||||
amount = self.default_order_amount
|
||||
margin = self.default_merchant_margin
|
||||
order_amounts.append(amount)
|
||||
merchant_margins.append(margin)
|
||||
else:
|
||||
order_amounts = [self.default_order_amount] * n
|
||||
merchant_margins = [self.default_merchant_margin] * n
|
||||
|
||||
# Calculate costs for each pair
|
||||
total_cost = 0.0
|
||||
total_profit = 0.0
|
||||
high_cost_count = 0
|
||||
|
||||
for i in range(n):
|
||||
for j in range(n):
|
||||
if i == j:
|
||||
cost_matrix[i][j] = 0
|
||||
continue
|
||||
|
||||
dist = dist_matrix[i][j]
|
||||
|
||||
# Use order j's profit data (destination)
|
||||
order_amount = (
|
||||
order_amounts[j - 1] if j > 0 else self.default_order_amount
|
||||
)
|
||||
merchant_margin = (
|
||||
merchant_margins[j - 1] if j > 0 else self.default_merchant_margin
|
||||
)
|
||||
|
||||
cost_data = self.calculate_composite_cost(
|
||||
distance_km=dist,
|
||||
order_amount=order_amount,
|
||||
merchant_margin=merchant_margin,
|
||||
time_of_day=traffic_condition,
|
||||
)
|
||||
|
||||
cost_matrix[i][j] = cost_data["net_cost"]
|
||||
total_cost += cost_data["net_cost"]
|
||||
total_profit += cost_data["gross_profit"]
|
||||
|
||||
if cost_data["net_cost"] > 50:
|
||||
high_cost_count += 1
|
||||
|
||||
summary = {
|
||||
"total_cost": round(total_cost, 2),
|
||||
"total_profit": round(total_profit, 2),
|
||||
"avg_cost": round(total_cost / (n * n) if n > 0 else 0, 2),
|
||||
"avg_profit": round(total_profit / (n * n) if n > 0 else 0, 2),
|
||||
"high_cost_legs": high_cost_count,
|
||||
"traffic_condition": traffic_condition,
|
||||
}
|
||||
|
||||
return cost_matrix, summary
|
||||
|
||||
|
||||
class RouteOptimizer:
|
||||
"""Route optimization using Google OR-Tools (Async)."""
|
||||
|
||||
@@ -253,8 +67,6 @@ class RouteOptimizer:
|
||||
# Solver time limit (ML-tuned)
|
||||
self.search_time_limit_seconds = int(_cfg.get("search_time_limit_seconds"))
|
||||
|
||||
self.cost_calculator = CompositeCostCalculator()
|
||||
|
||||
def haversine_distance(
|
||||
self, lat1: float, lon1: float, lat2: float, lon2: float
|
||||
) -> float:
|
||||
@@ -274,153 +86,6 @@ class RouteOptimizer:
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
async def _get_google_maps_distances_batch(
|
||||
self, origin_lat: float, origin_lon: float, destinations: _List[tuple]
|
||||
) -> Dict[tuple, float]:
|
||||
"""Get road distances for multiple destinations from Google Maps API. (Async, Parallel)"""
|
||||
if not self.use_google_maps or not destinations:
|
||||
return {}
|
||||
|
||||
results = {}
|
||||
batch_size = 25
|
||||
chunks = [
|
||||
destinations[i : i + batch_size]
|
||||
for i in range(0, len(destinations), batch_size)
|
||||
]
|
||||
|
||||
async def process_batch(batch):
|
||||
batch_result = {}
|
||||
try:
|
||||
dest_str = "|".join([f"{lat},{lon}" for lat, lon in batch])
|
||||
url = "https://maps.googleapis.com/maps/api/distancematrix/json"
|
||||
params = {
|
||||
"origins": f"{origin_lat},{origin_lon}",
|
||||
"destinations": dest_str,
|
||||
"key": self.google_maps_api_key,
|
||||
"units": "metric",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
response = await client.get(url, params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
if data.get("status") == "OK":
|
||||
rows = data.get("rows", [])
|
||||
if rows:
|
||||
elements = rows[0].get("elements", [])
|
||||
for idx, element in enumerate(elements):
|
||||
if idx < len(batch):
|
||||
dest_coord = batch[idx]
|
||||
if element.get("status") == "OK":
|
||||
dist = element.get("distance", {}).get("value")
|
||||
dur = element.get("duration", {}).get("value")
|
||||
if dist is not None:
|
||||
batch_result[dest_coord] = {
|
||||
"distance": dist / 1000.0,
|
||||
"duration": dur / 60.0 if dur else None,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Google Maps batch call failed: {e}")
|
||||
return batch_result
|
||||
|
||||
batch_results_list = await asyncio.gather(
|
||||
*[process_batch(chunk) for chunk in chunks]
|
||||
)
|
||||
for res in batch_results_list:
|
||||
results.update(res)
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# GOOGLE DIRECTIONS - WAYPOINT OPTIMISATION
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _optimize_waypoints_google(
|
||||
self,
|
||||
origin_lat: float,
|
||||
origin_lon: float,
|
||||
waypoints: _List[Tuple[float, float]],
|
||||
) -> Tuple[Optional[_List[int]], Optional[_List[float]]]:
|
||||
"""
|
||||
Ask Google Directions API to find the optimal visiting order for a set
|
||||
of delivery points starting from a kitchen/pickup location.
|
||||
|
||||
Uses `optimize:true` in the waypoints parameter - Google solves the TSP
|
||||
internally using actual road geometry (turn restrictions, one-way
|
||||
streets, real distances) rather than Haversine approximation.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(waypoint_order, leg_km)
|
||||
waypoint_order 0-based indices into `waypoints` in optimal order.
|
||||
e.g. [2, 0, 1] means visit wp[2] -> wp[0] -> wp[1].
|
||||
leg_km Actual road distance (km) for each leg in order:
|
||||
leg_km[0] = kitchen -> wp[order[0]],
|
||||
leg_km[1] = wp[order[0]] -> wp[order[1]], etc.
|
||||
Both are None on any failure - caller falls back to OR-Tools.
|
||||
|
||||
Notes
|
||||
-----
|
||||
- Supports up to 25 intermediate waypoints (Google's standard limit).
|
||||
- destination = origin (closed-loop TSP); the return leg is discarded.
|
||||
- One API call per rider per assignment - cheap at delivery scale.
|
||||
"""
|
||||
if not self.use_google_maps or not waypoints or len(waypoints) < 2:
|
||||
return None, None
|
||||
if len(waypoints) > 25:
|
||||
return None, None # fall back to OR-Tools for unusually large routes
|
||||
|
||||
try:
|
||||
wp_str = "optimize:true|" + "|".join(
|
||||
f"{lat},{lon}" for lat, lon in waypoints
|
||||
)
|
||||
params = {
|
||||
"origin": f"{origin_lat},{origin_lon}",
|
||||
"destination": f"{origin_lat},{origin_lon}", # closed loop
|
||||
"waypoints": wp_str,
|
||||
"key": self.google_maps_api_key,
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
resp = await client.get(
|
||||
"https://maps.googleapis.com/maps/api/directions/json",
|
||||
params=params,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
status_code = data.get("status")
|
||||
if status_code != "OK":
|
||||
logger.debug(
|
||||
f"[GoogleWaypoints] status={status_code} "
|
||||
f"error='{data.get('error_message', '')}'"
|
||||
)
|
||||
return None, None
|
||||
|
||||
routes = data.get("routes", [])
|
||||
if not routes:
|
||||
return None, None
|
||||
|
||||
route = routes[0]
|
||||
wp_order = route.get("waypoint_order")
|
||||
legs = route.get("legs", [])
|
||||
|
||||
if wp_order is None or len(wp_order) != len(waypoints):
|
||||
return None, None
|
||||
|
||||
# Extract leg distances (metres -> km), skip the return-to-origin leg
|
||||
leg_km: _List[float] = []
|
||||
for leg in legs[: len(waypoints)]: # first N legs only
|
||||
dist_m = leg.get("distance", {}).get("value")
|
||||
leg_km.append(dist_m / 1000.0 if dist_m is not None else 0.0)
|
||||
|
||||
logger.debug(
|
||||
f"[GoogleWaypoints] Optimised {len(waypoints)} stops -> order={wp_order}"
|
||||
)
|
||||
return wp_order, leg_km
|
||||
|
||||
except Exception as _e:
|
||||
logger.debug(f"[GoogleWaypoints] Failed (non-fatal): {_e}")
|
||||
return None, None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ROAD-AWARE VISITING ORDER (Phase 2 - opt-in, cached)
|
||||
# ------------------------------------------------------------------
|
||||
@@ -698,8 +363,8 @@ class RouteOptimizer:
|
||||
routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH
|
||||
)
|
||||
# TSP time limit hard-capped at 2 seconds per kitchen.
|
||||
# The ML hypertuner may push search_time_limit_seconds up to 8-10s
|
||||
# chasing marginally better routes, but at delivery scale (< 15 stops)
|
||||
# search_time_limit_seconds can be tuned up to 8-10s via config, but at
|
||||
# delivery scale (< 15 stops)
|
||||
# OR-Tools finds a near-optimal solution in < 200ms. Waiting 8-10s
|
||||
# per kitchen x 3 kitchens x 4 riders = 96s of unnecessary waiting.
|
||||
# The VRP already has its own 3s cap. This cap applies to per-rider
|
||||
|
||||
Reference in New Issue
Block a user