Nearle/routesapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new changes in the api

Browse Source
This commit is contained in:
Suriya
2026-07-06 15:15:51 +05:30
parent c742ef0e53
commit 871981035a
43 changed files with 414 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/services/core/__pycache__/assignment_service.cpython-312.pyc
View File

Binary file not shown.

227
app/services/core/assignment_service.py
View File

@@ -6,110 +6,15 @@ from math import radians, cos, sin, asin, sqrt, ceil
from typing import List, Dict, Any, Optional, Set
from collections import defaultdict
from app.config.rider_preferences import RIDER_PREFERRED_KITCHENS, BLOCKED_RIDERS as _BLOCKED_RIDERS_SET
from app.services.routing.kalman_filter import (
from app.services.routing.gps_smoother import (
smooth_rider_locations,
smooth_order_coordinates,
)
from app.config.dynamic_config import (
get_config,
get_kitchen_label_id as _get_kitchen_label_id,
get_kitchen_frequency as _get_kitchen_frequency,
update_kitchen_stats,
)
from app.services.ml.ml_data_collector import get_collector
from app.config.dynamic_config import get_config
logger = logging.getLogger(__name__)
class DataEncoder:
"""
Data Encoding Utilities for ML-ready feature engineering.
Implements techniques from idea.txt for categorical, spatial, and temporal data.
"""
EARTH_RADIUS_KM = 6371
@staticmethod
def haversine(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
"""Calculate great circle distance between two points."""
try:
lon1, lat1, lon2, lat2 = map(
radians, [float(lon1), float(lat1), float(lon2), float(lat2)]
)
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
c = 2 * asin(min(1.0, sqrt(a)))
return c * DataEncoder.EARTH_RADIUS_KM
except:
return 0.0
@staticmethod
def cyclic_encode_hour(hour: int) -> tuple[float, float]:
"""
Cyclic time encoding - captures traffic patterns better than discrete buckets.
hour_sin = sin(2π * hour / 24)
hour_cos = cos(2π * hour / 24)
"""
hour_sin = math.sin(2 * math.pi * hour / 24)
hour_cos = math.cos(2 * math.pi * hour / 24)
return hour_sin, hour_cos
@staticmethod
def cyclic_encode_day(day_of_week: int) -> tuple[float, float]:
"""
Cyclic day encoding for weekly patterns.
"""
day_sin = math.sin(2 * math.pi * day_of_week / 7)
day_cos = math.cos(2 * math.pi * day_of_week / 7)
return day_sin, day_cos
@staticmethod
def geohash_encode(lat: float, lon: float, precision: int = 7) -> str:
"""
Geohash encoding for spatial data.
Converts lat/lon to grid cell string for locality capture.
precision=7 gives ~153m x 153m cells (good for delivery zones)
"""
try:
return _simple_geohash(lat, lon, precision)
except:
return "unknown"
def _simple_geohash(lat: float, lon: float, precision: int = 7) -> str:
"""Standard geohash encoding — 5 bits per character, BASE32 output."""
if lat == 0 or lon == 0:
return "unknown"
BASE32 = "0123456789bcdefghjkmnpqrstuvwxyz"
lat_min, lat_max = -90.0, 90.0
lon_min, lon_max = -180.0, 180.0
hash_chars = []
is_lon = True
for _ in range(precision):
char_bits = 0
for _ in range(5):
if is_lon:
mid = (lon_min + lon_max) / 2
if lon >= mid:
char_bits = (char_bits << 1) | 1
lon_min = mid
else:
char_bits = char_bits << 1
lon_max = mid
else:
mid = (lat_min + lat_max) / 2
if lat >= mid:
char_bits = (char_bits << 1) | 1
lat_min = mid
else:
char_bits = char_bits << 1
lat_max = mid
is_lon = not is_lon
hash_chars.append(BASE32[char_bits])
return "".join(hash_chars)
def _zone_key(lat: float, lon: float) -> str:
"""O(1) ~5 km grid cell key for zone proximity matching."""
if lat == 0 or lon == 0:
@@ -117,26 +22,6 @@ def _zone_key(lat: float, lon: float) -> str:
return f"{int(lat / 0.044)},{int(lon / 0.044)}"
# Kitchen encoding now uses persistent DB storage from dynamic_config
def update_kitchen_encoding(kitchen_name: str, profit: float = None):
"""Update kitchen stats in DB when orders are processed."""
if kitchen_name and kitchen_name != "Unknown":
if profit is not None:
update_kitchen_stats(kitchen_name, profit)
def get_kitchen_label_id(kitchen_name: str) -> int:
"""Get persistent label ID for a kitchen."""
return _get_kitchen_label_id(kitchen_name)
def get_kitchen_frequency(kitchen_name: str) -> float:
"""Get persistent frequency ratio for a kitchen."""
return _get_kitchen_frequency(kitchen_name)
class AssignmentService:
def __init__(self):
# Curated config drives HARD kitchen ownership. Copy so substitution
@@ -196,35 +81,19 @@ class AssignmentService:
self.earth_radius_km = 6371
self._cfg = get_config()
self._encoder = DataEncoder()
# Cost parameters for composite cost function
self._fuel_rate = 2.5 # Per km
self._base_rider_cost = 0.0
self._merchant_margin_avg = 5.0 # Default average margin
# Profit encoding cache
self._kitchen_profit_cache: Dict[str, List[float]] = defaultdict(list)
self._profit_mean = 0.0
self._profit_std = 1.0
def calculate_order_profit_features(
self, order: Dict[str, Any], distance_km: float
) -> Dict[str, float]:
"""
Calculate engineered profit features for ML-ready data.
Features:
- profit: order amount - rider cost
- profit_density: profit per kilometer (key signal!)
- cost_efficiency: rider cost per estimated time
- route_score: profit - composite cost (maximize this!)
- encoded_geohash: spatial encoding
- cyclic_time: hour_sin, hour_cos
Calculate profit and profit-density for one order, used as a scoring
signal (profit_bonus) when picking which rider gets a cluster.
"""
features = {}
# Extract order values
try:
order_amount = float(
order.get("orderamount") or order.get("deliveryamount") or 0
@@ -232,38 +101,17 @@ class AssignmentService:
except:
order_amount = 0.0
# Calculate rider cost: base + (distance * fuel_rate)
# Rider cost: base + (distance * fuel_rate)
rider_cost = self._base_rider_cost + (distance_km * self._fuel_rate)
features["rider_cost"] = rider_cost
# Profit = revenue - cost
profit = order_amount - rider_cost
features["profit"] = profit
# Profit density = profit / distance (HIGH SIGNAL feature!)
# High density = profitable short deliveries
if distance_km > 0:
features["profit_density"] = profit / distance_km
else:
features["profit_density"] = 0.0
# Profit density = profit / distance — high density means profitable
# short deliveries; used to prioritise clusters worth serving.
profit_density = profit / distance_km if distance_km > 0 else 0.0
# Cost efficiency = cost / time estimate (assuming 15 min per order average)
estimated_time_min = max(15, distance_km * 4) # Rough estimate
features["cost_efficiency"] = (
rider_cost / estimated_time_min if estimated_time_min > 0 else 0
)
# Route score = profit - distance_cost (what we want to MAXIMIZE)
# This is the core optimization target
features["route_score"] = profit - (
distance_km * 0.5
) # 0.5 = opportunity cost per km
# Composite edge weight for optimizer (MINIMIZE this)
# weight = cost - profit_margin_bonus
features["composite_weight"] = rider_cost - (profit * 0.3) # 30% profit bonus
return features
return {"profit": profit, "profit_density": profit_density}
def _load_config(self):
"""Load ML-tuned hyperparams fresh on every assignment call."""
@@ -478,7 +326,6 @@ class AssignmentService:
# Use caller-supplied pricing so dynamic API rates flow into scoring
self._fuel_rate = fuel_charge
self._base_rider_cost = base_pay
_call_start = time.time()
# 0. Prep
assignments: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
@@ -588,36 +435,14 @@ class AssignmentService:
orders, max_cluster_radius_km=self.MAX_KITCHEN_DISTANCE_KM
)
# 2b. ENRICH CLUSTERS WITH PROFIT FEATURES (Data Encoding: Target + Frequency)
# 2b. Tag each cluster with the set of kitchen names it covers (used
# below for hard kitchen-ownership matching).
for cluster in clusters:
cluster["kitchen_names"] = set()
for order in cluster["orders"]:
k_name = self.get_order_kitchen(order)
cluster["kitchen_names"].add(k_name)
update_kitchen_encoding(k_name)
cluster["kitchen_names"].add(self.get_order_kitchen(order))
# Update kitchen profit cache for target encoding
for order in cluster["orders"]:
k_name = self.get_order_kitchen(order)
profit = (
float(order.get("orderamount") or order.get("deliveryamount") or 50)
- 40
)
self._kitchen_profit_cache[k_name].append(profit)
# Calculate profit statistics for normalization
all_profits = [
p for profits in self._kitchen_profit_cache.values() for p in profits
]
if all_profits:
self._profit_mean = sum(all_profits) / len(all_profits)
if len(all_profits) > 1:
variance = sum((p - self._profit_mean) ** 2 for p in all_profits) / len(
all_profits
)
self._profit_std = variance**0.5
logger.info(f"Created {len(clusters)} order clusters with profit encoding")
logger.info(f"Created {len(clusters)} order clusters")
# 2c. MINIMAL RIDER PRE-SELECTION
# Calculate the theoretical minimum number of riders needed so we don't
@@ -659,15 +484,6 @@ class AssignmentService:
cluster_geohash = _zone_key(centroid_lat, centroid_lon)
for order in cluster_orders:
k_name = self.get_order_kitchen(order)
# Target encoding: use average profit for this kitchen
kitchen_profits = self._kitchen_profit_cache.get(k_name, [0])
avg_kitchen_profit = (
sum(kitchen_profits) / len(kitchen_profits)
if kitchen_profits
else 0
)
o_lat = float(order.get("pickuplat", 0))
o_lon = float(order.get("pickuplon", 0))
dist = (
@@ -1004,19 +820,10 @@ class AssignmentService:
# 6. Commit State and History
self._post_process(assignments, rider_states, state_mgr)
# 7. -- ML DATA COLLECTION -----------------------------------------
try:
elapsed_ms = (time.time() - _call_start) * 1000
get_collector().log_assignment_event(
num_orders=len(orders),
num_riders=len(riders),
hyperparams=self._cfg.get_all(),
assignments=assignments,
unassigned_count=len(unassigned_orders),
elapsed_ms=elapsed_ms,
)
except Exception as _ml_err:
logger.debug(f"ML logging skipped: {_ml_err}")
# ML event logging happens once, in the /riderassign endpoint after
# Phase-0 history merge + solo consolidation — not here — so every
# request produces exactly one assignment_ml_log row (see
# optimization.py::_bg_log_assignment).
# Log final distribution (use r_orders to avoid shadowing the outer `orders` list)
logger.info("=" * 50)

BIN
app/services/ml/__pycache__/ml_data_collector.cpython-312.pyc
View File

Binary file not shown.

55
app/services/ml/behavior_analyzer.py
View File

@@ -1,55 +0,0 @@
"""
Feature band encoders
======================
Discrete bucketers for assignment-context features (distance, time-of-day, load,
order density). These are small pure helpers used by the Thompson-sampling
strategy bandit (`strategy_bandit.py`) and the /riderassign bandit context.
NOTE: The ID3 SUCCESS/RISK decision tree that used to live here has been retired —
it only ever produced response metadata and never affected any assignment. Only
the generic feature-band encoders remain.
"""
from datetime import datetime
def distance_band(km: float) -> str:
"""Total route distance -> discrete band."""
if km <= 5.0: return "SHORT"
if km <= 15.0: return "MID"
if km <= 30.0: return "LONG"
return "VERY_LONG"
def time_band(ts_str: str) -> str:
"""ISO timestamp -> time-of-day band."""
try:
hour = datetime.fromisoformat(ts_str).hour
if 6 <= hour < 10: return "MORNING_RUSH"
if 10 <= hour < 12: return "LATE_MORNING"
if 12 <= hour < 14: return "LUNCH_RUSH"
if 14 <= hour < 17: return "AFTERNOON"
if 17 <= hour < 20: return "EVENING_RUSH"
if 20 <= hour < 23: return "NIGHT"
return "LATE_NIGHT"
except Exception:
return "UNKNOWN"
def load_band(avg_load: float) -> str:
"""Average orders-per-rider -> load band."""
if avg_load <= 2.0: return "LIGHT"
if avg_load <= 5.0: return "MODERATE"
if avg_load <= 8.0: return "HEAVY"
return "OVERLOADED"
def order_density_band(num_orders: int, num_riders: int) -> str:
"""Orders per available rider -> density band."""
if num_riders == 0:
return "NO_RIDERS"
ratio = num_orders / num_riders
if ratio <= 2.0: return "SPARSE"
if ratio <= 5.0: return "NORMAL"
if ratio <= 9.0: return "DENSE"
return "OVERLOADED"

114
app/services/ml/ml_data_collector.py
View File

@@ -7,13 +7,12 @@ Key upgrades over the original
--------------------------------
1. FROZEN historical scores - quality_score is written ONCE at log time.
get_training_data() returns scores as-is from the DB (no retroactive mutation).
2. Rich schema - zone_id, city_id, is_peak, weather_code,
sla_breached, avg_delivery_time_min for richer features.
3. SLA tracking - logs whether delivery SLA was breached.
4. Analytics API - get_hourly_stats(), get_strategy_comparison(),
get_quality_histogram(), get_zone_stats() for dashboard consumption.
5. Thread-safe writes - connection-per-write pattern for FastAPI workers.
6. Indexed columns - timestamp, ml_strategy, zone_id for fast queries.
2. Rich schema - zone_id, city_id, is_peak, weather_code for
richer features.
3. Analytics API - get_hourly_stats(), get_quality_histogram(),
get_zone_stats() for dashboard consumption.
4. Thread-safe writes - connection-per-write pattern for FastAPI workers.
5. Indexed columns - timestamp, zone_id for fast queries.
"""
import csv
@@ -45,7 +44,7 @@ class MLDataCollector:
Each log_assignment_event() call writes one row capturing:
- Operating context (time, orders, riders, zone, city)
- Active hyperparams (exact config snapshot for this call)
- Measured outcomes (quality score, SLA, latency, distances)
- Measured outcomes (quality score, latency, distances)
quality_score is computed once and FROZEN - never retroactively changed.
"""
@@ -70,8 +69,6 @@ class MLDataCollector:
zone_id: str = "default",
city_id: str = "default",
weather_code: str = "CLEAR",
sla_minutes: Optional[float] = None,
avg_delivery_time_min: Optional[float] = None,
) -> None:
"""
Log one assignment event.
@@ -95,13 +92,8 @@ class MLDataCollector:
o for orders in assignments.values() if orders for o in orders
]
total_distance_km = sum(self._get_km(o) for o in all_orders)
ml_strategy = hyperparams.get("ml_strategy", "balanced")
max_opr = hyperparams.get("max_orders_per_rider", 12)
sla_breached = 0
if sla_minutes and avg_delivery_time_min:
sla_breached = int(avg_delivery_time_min > sla_minutes)
# Quality score - FROZEN at log time
quality_score = self._compute_quality_score(
num_orders=num_orders,
@@ -111,7 +103,6 @@ class MLDataCollector:
num_riders=num_riders,
total_distance_km=total_distance_km,
max_orders_per_rider=max_opr,
ml_strategy=ml_strategy,
)
row = {
@@ -146,7 +137,6 @@ class MLDataCollector:
"search_time_limit_seconds", 5
),
"road_factor": hyperparams.get("road_factor", 1.3),
"ml_strategy": ml_strategy,
"riders_used": riders_used,
"total_assigned": total_assigned,
"unassigned_count": unassigned_count,
@@ -154,8 +144,6 @@ class MLDataCollector:
"load_std": round(load_std, 3),
"total_distance_km": round(total_distance_km, 2),
"elapsed_ms": round(elapsed_ms, 1),
"sla_breached": sla_breached,
"avg_delivery_time_min": round(avg_delivery_time_min or 0.0, 2),
"quality_score": round(quality_score, 2),
}
@@ -171,7 +159,7 @@ class MLDataCollector:
except Exception as e:
logger.warning(f"[MLCollector] Logging failed (non-fatal): {e}")
return 50.0 # neutral fallback so bandit update still fires
return 50.0 # neutral fallback
# ------------------------------------------------------------------
# Data retrieval for training
@@ -180,7 +168,6 @@ class MLDataCollector:
def get_training_data(
self,
min_records: int = 30,
strategy_filter: Optional[str] = None,
since_hours: Optional[int] = None,
) -> Optional[List[Dict[str, Any]]]:
"""
@@ -195,9 +182,6 @@ class MLDataCollector:
params: list = []
clauses: list = []
if strategy_filter:
clauses.append("ml_strategy = ?")
params.append(strategy_filter)
if since_hours:
cutoff = (datetime.utcnow() - timedelta(hours=since_hours)).isoformat()
clauses.append("timestamp >= ?")
@@ -253,7 +237,7 @@ class MLDataCollector:
return {"avg_quality": 0.0, "sample_size": 0, "history": []}
def get_hourly_stats(self, last_days: int = 7) -> List[Dict[str, Any]]:
"""Quality, SLA, and call volume aggregated by hour-of-day."""
"""Quality and call volume aggregated by hour-of-day."""
try:
conn = sqlite3.connect(self._db_path)
cutoff = (datetime.utcnow() - timedelta(days=last_days)).isoformat()
@@ -263,8 +247,7 @@ class MLDataCollector:
COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
AVG(unassigned_count) AS avg_unassigned,
AVG(elapsed_ms) AS avg_latency_ms,
SUM(CASE WHEN sla_breached=1 THEN 1 ELSE 0 END) AS sla_breaches
AVG(elapsed_ms) AS avg_latency_ms
FROM assignment_ml_log WHERE timestamp >= ?
GROUP BY hour ORDER BY hour
""",
@@ -278,7 +261,6 @@ class MLDataCollector:
"avg_quality": round(r[2] or 0.0, 2),
"avg_unassigned": round(r[3] or 0.0, 2),
"avg_latency_ms": round(r[4] or 0.0, 1),
"sla_breaches": r[5],
}
for r in rows
]
@@ -286,42 +268,6 @@ class MLDataCollector:
logger.error(f"[MLCollector] get_hourly_stats: {e}")
return []
def get_strategy_comparison(self) -> List[Dict[str, Any]]:
"""Compare quality metrics across ml_strategy values."""
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"""
SELECT ml_strategy,
COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
MIN(quality_score) AS min_quality,
MAX(quality_score) AS max_quality,
AVG(unassigned_count) AS avg_unassigned,
AVG(total_distance_km) AS avg_distance_km,
AVG(elapsed_ms) AS avg_latency_ms
FROM assignment_ml_log
GROUP BY ml_strategy ORDER BY avg_quality DESC
"""
).fetchall()
conn.close()
return [
{
"strategy": r[0],
"call_count": r[1],
"avg_quality": round(r[2] or 0.0, 2),
"min_quality": round(r[3] or 0.0, 2),
"max_quality": round(r[4] or 0.0, 2),
"avg_unassigned": round(r[5] or 0.0, 2),
"avg_distance_km": round(r[6] or 0.0, 2),
"avg_latency_ms": round(r[7] or 0.0, 1),
}
for r in rows
]
except Exception as e:
logger.error(f"[MLCollector] get_strategy_comparison: {e}")
return []
def get_quality_histogram(self, bins: int = 10) -> List[Dict[str, Any]]:
"""Quality score distribution for histogram chart."""
try:
@@ -348,14 +294,13 @@ class MLDataCollector:
return []
def get_zone_stats(self) -> List[Dict[str, Any]]:
"""Quality and SLA stats grouped by zone."""
"""Quality stats grouped by zone."""
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"""
SELECT zone_id, COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
SUM(sla_breached) AS sla_breaches,
AVG(total_distance_km) AS avg_distance_km
FROM assignment_ml_log
GROUP BY zone_id ORDER BY avg_quality DESC
@@ -367,8 +312,7 @@ class MLDataCollector:
"zone_id": r[0],
"call_count": r[1],
"avg_quality": round(r[2] or 0.0, 2),
"sla_breaches": r[3],
"avg_distance_km": round(r[4] or 0.0, 2),
"avg_distance_km": round(r[3] or 0.0, 2),
}
for r in rows
]
@@ -385,17 +329,6 @@ class MLDataCollector:
except Exception:
return 0
def count_by_strategy(self) -> Dict[str, int]:
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"SELECT ml_strategy, COUNT(*) FROM assignment_ml_log GROUP BY ml_strategy"
).fetchall()
conn.close()
return {r[0]: r[1] for r in rows}
except Exception:
return {}
def export_csv(self) -> str:
"""Export all records as CSV string."""
try:
@@ -448,7 +381,6 @@ class MLDataCollector:
num_riders: int,
total_distance_km: float,
max_orders_per_rider: int,
ml_strategy: str = "balanced",
) -> float:
"""
Multi-dimensional quality score (0–100, higher = better).
@@ -461,11 +393,7 @@ class MLDataCollector:
│ rider_efficiency │ reward using minimal riders for the batch size │
└──────────────────────┴────────────────────────────────────────────────┘
Strategy weights (w_assign, w_dist, w_balance, w_efficiency):
- balanced : (45, 20, 20, 15)
- aggressive_speed: (70, 15, 0, 15) — care about assignment + efficiency
- fuel_saver : (25, 60, 0, 15) — heavily penalise long routes
- zone_strict : (35, 25, 25, 15) — balanced with zone awareness
One fixed weighting (45, 20, 20, 15) is used for every call.
"""
import math
if num_orders == 0:
@@ -488,14 +416,7 @@ class MLDataCollector:
min_riders_needed = max(1, math.ceil(num_orders / max_orders_per_rider))
rider_efficiency = min(1.0, min_riders_needed / max(1, riders_used))
weights = {
# assign dist balance efficiency
"aggressive_speed": (70.0, 15.0, 0.0, 15.0),
"fuel_saver": (25.0, 60.0, 0.0, 15.0),
"zone_strict": (35.0, 25.0, 25.0, 15.0),
"balanced": (45.0, 20.0, 20.0, 15.0),
}
w_comp, w_dist, w_bal, w_eff = weights.get(ml_strategy, (45.0, 20.0, 20.0, 15.0))
w_comp, w_dist, w_bal, w_eff = (45.0, 20.0, 20.0, 15.0)
return min(
assigned_ratio * w_comp
@@ -542,7 +463,6 @@ class MLDataCollector:
cluster_radius_km REAL,
search_time_limit_seconds INTEGER,
road_factor REAL,
ml_strategy TEXT DEFAULT 'balanced',
riders_used INTEGER,
total_assigned INTEGER,
unassigned_count INTEGER,
@@ -550,8 +470,6 @@ class MLDataCollector:
load_std REAL,
total_distance_km REAL DEFAULT 0.0,
elapsed_ms REAL,
sla_breached INTEGER DEFAULT 0,
avg_delivery_time_min REAL DEFAULT 0.0,
quality_score REAL
)
""")
@@ -560,9 +478,6 @@ class MLDataCollector:
"ALTER TABLE assignment_ml_log ADD COLUMN zone_id TEXT DEFAULT 'default'",
"ALTER TABLE assignment_ml_log ADD COLUMN city_id TEXT DEFAULT 'default'",
"ALTER TABLE assignment_ml_log ADD COLUMN weather_code TEXT DEFAULT 'CLEAR'",
"ALTER TABLE assignment_ml_log ADD COLUMN sla_breached INTEGER DEFAULT 0",
"ALTER TABLE assignment_ml_log ADD COLUMN avg_delivery_time_min REAL DEFAULT 0.0",
"ALTER TABLE assignment_ml_log ADD COLUMN ml_strategy TEXT DEFAULT 'balanced'",
"ALTER TABLE assignment_ml_log ADD COLUMN total_distance_km REAL DEFAULT 0.0",
]
for ddl in migrations:
@@ -572,7 +487,6 @@ class MLDataCollector:
pass
for idx in [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON assignment_ml_log(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_strategy ON assignment_ml_log(ml_strategy)",
"CREATE INDEX IF NOT EXISTS idx_zone ON assignment_ml_log(zone_id)",
]:
conn.execute(idx)

277
app/services/ml/strategy_bandit.py
View File

@@ -1,277 +0,0 @@
"""
Thompson Sampling Contextual Bandit — Strategy Selector
=========================================================
Replaces the greedy SQL auto-tuner with proper online RL.
Problem
-------
The old greedy tuner picks the strategy with the highest *average* quality
across all history. It never explores alternatives once one strategy leads,
and it ignores context (the same strategy isn't best at all times of day
and all load levels).
Solution
--------
A contextual multi-armed bandit with Thompson Sampling:
Context = (time_band, load_band) — up to 7 × 4 = 28 states
Arms = 4 strategies — balanced / fuel_saver / aggressive_speed / zone_strict
Reward = quality_score / 100 — 0..1 float, already logged by MLDataCollector
How Thompson Sampling works
---------------------------
Each (context, arm) pair has a Beta(α, β) posterior where:
α = sum of rewards seen so far (high quality calls push α up)
β = sum of "anti-rewards" (low quality calls push β up)
To SELECT a strategy:
1. For each arm, sample θ ~ Beta(α, β)
2. Pick arm with highest θ
→ Naturally balances exploration (uncertain arms get sampled often)
with exploitation (well-known good arms dominate when confident)
To UPDATE after an assignment:
reward = quality_score / 100
α += reward
β += (1 - reward)
Bootstrap
---------
On first startup the entire historical SQLite DB is replayed to warm up
posteriors so the bandit starts informed, not blank.
If saved state already exists it is loaded from disk instead.
Persistence
-----------
ml_data/strategy_bandit.json — saved every 10 updates.
"""
import json
import logging
import os
import threading
from typing import Dict, List, Optional, Tuple
import numpy as np
logger = logging.getLogger(__name__)
_SAVE_PATH = os.getenv("BANDIT_PATH", "ml_data/strategy_bandit.json")
_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
ARMS = ["balanced", "fuel_saver", "aggressive_speed", "zone_strict"]
class ContextualBandit:
"""
Thompson Sampling bandit for ml_strategy selection.
Context key : "{time_band}|{load_band}"
Arms : ARMS list (4 strategies)
Prior : Beta(1, 1) — uniform, no initial preference
"""
def __init__(self):
self._lock = threading.Lock()
# _posteriors[ctx_key][arm] = [alpha, beta]
self._posteriors: Dict[str, Dict[str, List[float]]] = {}
self._update_count = 0
self._total_pulls = 0
self._load()
self._bootstrap_from_db()
# ── Public API ───────────────────────────────────────────────────────────
def select(self, time_band: str, load_band: str) -> str:
"""
Sample from Beta posteriors and return the strategy with the highest
sample. Explores uncertain arms naturally; exploits known-good arms
as confidence grows.
"""
ctx = self._ctx(time_band, load_band)
with self._lock:
samples = {
arm: float(np.random.beta(*self._ab(ctx, arm)))
for arm in ARMS
}
self._total_pulls += 1
chosen = max(samples, key=samples.__getitem__)
logger.debug(
f"[Bandit] ctx={ctx} "
f"samples={{{', '.join(f'{k}:{v:.3f}' for k, v in samples.items())}}} "
f"→ {chosen}"
)
return chosen
def update(self, time_band: str, load_band: str,
strategy: str, quality_score: float) -> None:
"""Update the Beta posterior for (context, arm) after observing quality."""
if strategy not in ARMS:
return
ctx = self._ctx(time_band, load_band)
reward = min(1.0, max(0.0, float(quality_score) / 100.0))
with self._lock:
ab = self._ab(ctx, strategy)
ab[0] += reward # alpha ← quality adds to success mass
ab[1] += (1.0 - reward) # beta ← (1-quality) adds to failure mass
self._update_count += 1
should_save = self._update_count % 10 == 0
if should_save:
self._save()
def best_arm(self, time_band: str, load_band: str) -> Tuple[str, float]:
"""
Return the arm with the highest posterior mean (pure exploitation, no
sampling noise). Used for logging and dashboard, not for live selection.
"""
ctx = self._ctx(time_band, load_band)
with self._lock:
means = {
arm: self._ab(ctx, arm)[0] / sum(self._ab(ctx, arm))
for arm in ARMS
}
best = max(means, key=means.__getitem__)
return best, round(means[best], 4)
def get_stats(self) -> Dict:
"""Return per-context arm statistics for the ML admin dashboard."""
with self._lock:
stats: Dict[str, Dict] = {}
for ctx, arms in sorted(self._posteriors.items()):
stats[ctx] = {}
for arm in ARMS:
if arm not in arms:
stats[ctx][arm] = {"mean_reward": 0.5, "observations": 0,
"alpha": 1.0, "beta": 1.0}
continue
alpha, beta = arms[arm]
n = alpha + beta - 2.0 # subtract the Beta(1,1) prior mass
mean = alpha / (alpha + beta)
stats[ctx][arm] = {
"mean_reward": round(mean, 4),
"observations": round(max(0, n), 1),
"alpha": round(alpha, 2),
"beta": round(beta, 2),
}
return {
"total_pulls": self._total_pulls,
"total_updates": self._update_count,
"context_count": len(self._posteriors),
"arms": ARMS,
"contexts": stats,
}
# ── Persistence ──────────────────────────────────────────────────────────
def _save(self) -> None:
try:
with self._lock:
snapshot = {
"posteriors": {
ctx: {arm: list(ab) for arm, ab in arms.items()}
for ctx, arms in self._posteriors.items()
},
"update_count": self._update_count,
"total_pulls": self._total_pulls,
}
os.makedirs(os.path.dirname(_SAVE_PATH) or ".", exist_ok=True)
with open(_SAVE_PATH, "w", encoding="utf-8") as f:
json.dump(snapshot, f, indent=2)
except Exception as e:
logger.warning(f"[Bandit] Save failed: {e}")
def _load(self) -> None:
try:
if not os.path.exists(_SAVE_PATH):
logger.info("[Bandit] No saved state — will bootstrap from DB.")
return
with open(_SAVE_PATH, "r", encoding="utf-8") as f:
data = json.load(f)
self._posteriors = data.get("posteriors", {})
self._update_count = data.get("update_count", 0)
self._total_pulls = data.get("total_pulls", 0)
logger.info(
f"[Bandit] Loaded from disk — "
f"{len(self._posteriors)} contexts, {self._update_count} updates"
)
except Exception as e:
logger.warning(f"[Bandit] Load failed (starting fresh): {e}")
# ── Bootstrap ────────────────────────────────────────────────────────────
def _bootstrap_from_db(self) -> None:
"""
Warm-up posteriors by replaying every historical assignment event.
Skipped if the saved JSON already reflects current DB data.
"""
if self._update_count > 0:
logger.info(
f"[Bandit] Already warmed ({self._update_count} updates) — "
"skipping DB bootstrap."
)
return
try:
import sqlite3
from app.services.ml.behavior_analyzer import (
time_band as _tb,
load_band as _lb,
)
conn = sqlite3.connect(_DB_PATH)
rows = conn.execute(
"SELECT timestamp, avg_load, ml_strategy, quality_score "
"FROM assignment_ml_log "
"WHERE ml_strategy IS NOT NULL AND quality_score IS NOT NULL "
"ORDER BY id ASC"
).fetchall()
conn.close()
if not rows:
logger.info("[Bandit] No historical data — starting with uniform priors.")
return
for ts, avg_load, strategy, quality in rows:
t_band = _tb(str(ts)) if ts else "UNKNOWN"
l_band = _lb(float(avg_load or 0))
self.update(t_band, l_band, strategy, float(quality or 50))
# Reset counter so we don't confuse bootstrap updates with live ones
with self._lock:
self._update_count = len(rows)
logger.info(
f"[Bandit] Bootstrapped from {len(rows)} historical events — "
f"{len(self._posteriors)} contexts, {len(ARMS)} arms."
)
self._save()
except Exception as e:
logger.warning(f"[Bandit] Bootstrap failed (non-fatal): {e}")
# ── Helpers ──────────────────────────────────────────────────────────────
@staticmethod
def _ctx(time_band: str, load_band: str) -> str:
return f"{time_band}|{load_band}"
def _ab(self, ctx: str, arm: str) -> List[float]:
"""Get or create Beta(1, 1) uniform prior for (ctx, arm). NOT thread-safe alone."""
if ctx not in self._posteriors:
self._posteriors[ctx] = {}
if arm not in self._posteriors[ctx]:
self._posteriors[ctx][arm] = [1.0, 1.0]
return self._posteriors[ctx][arm]
# ── Singleton ────────────────────────────────────────────────────────────────
_bandit_instance: Optional[ContextualBandit] = None
_bandit_lock = threading.Lock()
def get_bandit() -> ContextualBandit:
"""Return the process-level singleton ContextualBandit."""
global _bandit_instance
if _bandit_instance is None:
with _bandit_lock:
if _bandit_instance is None:
_bandit_instance = ContextualBandit()
return _bandit_instance

BIN
app/services/rider/__pycache__/get_active_riders.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
app/services/rider/__pycache__/rider_history_service.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
app/services/rider/__pycache__/rider_state_manager.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
app/services/rider/__pycache__/substitution_service.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/clustering_service.cpython-312.pyc
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/delivery_history_service.cpython-312.pyc
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/gps_smoother.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/realistic_eta_calculator.cpython-312.pyc
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/route_optimizer.cpython-312.pyc
View File

Binary file not shown.

BIN
app/services/routing/__pycache__/zone_service.cpython-312.pyc
View File

Binary file not shown.

67
app/services/routing/delivery_history_service.py
View File

@@ -444,6 +444,37 @@ class DeliveryHistoryService:
cols = ["deliveryid", "userid", "pickupcustomer", "pickuptime", "deliverytime", "dlat", "dlon", "plat", "plon"]
return [dict(zip(cols, r)) for r in rows]
def get_pattern_records(self, days: int) -> List[Dict[str, Any]]:
"""
Shape the local nearledb mirror into the record format the delivery-
history pattern store (Phase-0 kitchen+zone rider lookup) expects:
kitchen/pickuplat/pickuplon/deliverylat/deliverylong/userid/ridername.
`ridername` is always "" — delivery_raw doesn't store it (it's
display-only in the /ml/history debug endpoint; every actual matching
decision is keyed on userid, so this doesn't affect assignment).
"""
records: List[Dict[str, Any]] = []
for r in self._load_raw_rows(days):
try:
plat = float(r.get("plat") or 0)
plon = float(r.get("plon") or 0)
dlat = float(r.get("dlat") or 0)
dlon = float(r.get("dlon") or 0)
uid = int(float(r.get("userid") or 0))
if not plat or not dlat or uid == 0:
continue
records.append({
"kitchen": (r.get("pickupcustomer") or "").strip().lower(),
"pickuplat": plat, "pickuplon": plon,
"deliverylat": dlat, "deliverylong": dlon,
"userid": uid,
"ridername": "",
})
except (TypeError, ValueError):
continue
return records
def sample_batches(self, days: int = 14, min_drops: int = 4, max_drops: int = 15,
limit: int = 10) -> List[Dict[str, Any]]:
"""
@@ -623,12 +654,24 @@ class DeliveryHistoryService:
full: bool = False) -> Dict[str, Any]:
"""
Full pipeline: incremental DB sync -> prune local store -> rebuild
aggregates locally. This is what the scheduler and the admin endpoint
call. The DB is touched only by the sync step (new rows only).
aggregates locally -> (if pattern_source=db) rebuild the Phase-0
pattern store from the same synced rows. This is what the scheduler
and the admin endpoint call. The DB is touched only by the sync step
(new rows only).
"""
from app.config.dynamic_config import get_config
cfg = get_config()
pattern_on = str(cfg.get("pattern_source", "csv")) == "db"
pattern_days = int(cfg.get("pattern_history_days", 30))
# Local retention must cover whichever consumer needs more history
# (the ETA window vs. the pattern-store window) without changing the
# ETA computation's own `days` window below — that backtest result is
# already validated at 14 days and this must not perturb it.
retain_days = max(days, pattern_days) if pattern_on else days
with _WRITE_LOCK:
try:
sync = self.sync_from_db(days=days, tenant_id=tenant_id, full=full)
sync = self.sync_from_db(days=retain_days, tenant_id=tenant_id, full=full)
except Exception as e:
logger.error(f"[DeliveryHistory] sync failed: {e}", exc_info=True)
# Still try to serve whatever is already local.
@@ -636,9 +679,23 @@ class DeliveryHistoryService:
self._last_summary = {"status": "sync_failed", "error": str(e), "rebuild": rebuilt}
return self._last_summary
self._prune_raw(days)
self._prune_raw(retain_days)
rebuilt = self.rebuild_aggregates(days)
self._last_summary = {"status": "ok", "sync": sync, "rebuild": rebuilt}
pattern_rebuild = None
if pattern_on:
try:
from app.services.vector.delivery_history_store import get_delivery_history_store
records = self.get_pattern_records(pattern_days)
n = get_delivery_history_store().rebuild_from_records(records)
pattern_rebuild = {"records": n, "days": pattern_days}
except Exception as e:
logger.warning(f"[DeliveryHistory] pattern-store rebuild skipped: {e}")
self._last_summary = {
"status": "ok", "sync": sync, "rebuild": rebuilt,
"pattern_rebuild": pattern_rebuild,
}
return self._last_summary
# -- cache + lookup -----------------------------------------------------

150
app/services/routing/gps_smoother.py Normal file
View File

@@ -0,0 +1,150 @@
"""
GPS location smoothing — rider-api
Smooths noisy rider GPS pings (typical error +-5-15m, worse on poor signal,
occasional bad-fix "jumps") using a per-rider exponential moving average
(EMA): each new reading is blended with the running estimate so a single bad
ping can't yank the rider's position, while the estimate still tracks real
movement.
This used to be implemented as a full Kalman filter (per-coordinate process/
measurement covariance, gain computed every update). For a "constant
position" state model with no velocity term — which is what this is, since
we're smoothing noisy pings, not tracking motion — the Kalman update reduces
mathematically to an EMA once the gain reaches steady state, which happens
within the first couple of updates. The EMA below is the same behavior with
one constant instead of two, and no covariance bookkeeping to explain to the
next person reading this file.
Only two things are actually used elsewhere in the app:
smooth_rider_locations(riders) — per-rider EMA, stateful across calls
smooth_order_coordinates(orders) — validates/normalises delivery coords
(NOT smoothed — see its docstring)
"""
import logging
import time
from typing import Dict, Optional, Tuple
logger = logging.getLogger(__name__)
# Smoothing factor: how much weight a new GPS reading gets against the
# running estimate. Lower = smoother/slower to react, higher = trusts each
# new ping more. 0.1 matches the steady-state behavior of the Kalman filter
# this replaced (process_noise=1e-4, measurement_noise=0.01).
_ALPHA = 0.1
_STALE_SECONDS = 1800.0 # reset a rider's running estimate after 30 min silence
def _is_valid_coord(lat: float, lon: float) -> bool:
try:
lat, lon = float(lat), float(lon)
return (
-90.0 <= lat <= 90.0
and -180.0 <= lon <= 180.0
and not (lat == 0.0 and lon == 0.0)
)
except (TypeError, ValueError):
return False
class _RiderEstimate:
"""Running EMA estimate of one rider's position."""
def __init__(self):
self.lat: Optional[float] = None
self.lon: Optional[float] = None
self.last_updated: float = time.time()
def update(self, lat: float, lon: float) -> Tuple[float, float]:
if not _is_valid_coord(lat, lon):
return (self.lat, self.lon) if self.lat is not None else (lat, lon)
if time.time() - self.last_updated > _STALE_SECONDS:
self.lat = self.lon = None # stale — start fresh
if self.lat is None:
self.lat, self.lon = lat, lon
else:
self.lat += _ALPHA * (lat - self.lat)
self.lon += _ALPHA * (lon - self.lon)
self.last_updated = time.time()
return self.lat, self.lon
_rider_estimates: Dict[str, _RiderEstimate] = {}
def smooth_rider_locations(riders: list) -> list:
"""
Apply EMA smoothing to a list of rider dicts in-place, keyed by rider id
(history preserved across calls via a process-level registry).
Reads/writes: latitude, longitude (and currentlat/currentlong if present).
Adds: _location_smoothed = True on each processed rider.
"""
for rider in riders:
try:
rider_id = str(
rider.get("userid") or rider.get("riderid") or
rider.get("id") or "unknown"
)
raw_lat = float(rider.get("latitude") or rider.get("currentlat") or 0)
raw_lon = float(rider.get("longitude") or rider.get("currentlong") or 0)
if raw_lat == 0.0 and raw_lon == 0.0:
continue
estimate = _rider_estimates.setdefault(rider_id, _RiderEstimate())
smooth_lat, smooth_lon = estimate.update(raw_lat, raw_lon)
# Cast back to string for Go compatibility
s_lat, s_lon = str(round(smooth_lat, 8)), str(round(smooth_lon, 8))
rider["latitude"] = s_lat
rider["longitude"] = s_lon
if "currentlat" in rider:
rider["currentlat"] = s_lat
if "currentlong" in rider:
rider["currentlong"] = s_lon
rider["_location_smoothed"] = True
except Exception as e:
logger.debug(f"Rider location smoothing skipped: {e}")
return riders
def smooth_order_coordinates(orders: list) -> list:
"""
Validate and lightly normalise delivery coordinates in a list of order dicts.
DESIGN NOTE — why these are NOT smoothed:
Smoothing blends successive measurements from the same source over time.
Delivery coordinates are a single static point (one measurement) — there
is nothing to blend. Per-customer GPS accuracy is handled upstream by the
FAISS coordinate store (verified historical rider-confirmed delivery
points). This function only normalises the coordinate fields to floats
so downstream code never sees raw strings or None values.
Modifies orders in-place. Returns the same list.
"""
for order in orders:
try:
dlat_raw = order.get("deliverylat") or order.get("droplat")
dlon_raw = order.get("deliverylong") or order.get("droplon")
if dlat_raw is None or dlon_raw is None:
continue
dlat = float(dlat_raw)
dlon = float(dlon_raw)
if not _is_valid_coord(dlat, dlon):
continue
# Normalise to string (Go service expects string coordinates)
s_lat = str(round(dlat, 8))
s_lon = str(round(dlon, 8))
order["deliverylat"] = s_lat
order["deliverylong"] = s_lon
if "droplat" in order:
order["droplat"] = s_lat
if "droplon" in order:
order["droplon"] = s_lon
except Exception as e:
logger.debug(f"Coordinate normalisation skipped: {e}")
return orders

327
app/services/routing/kalman_filter.py
View File

@@ -1,327 +0,0 @@
"""
GPS Kalman Filter \u2014 rider-api
A 1D Kalman filter applied independently to latitude and longitude
to smooth noisy GPS coordinates from riders and delivery points.
Why Kalman for GPS?
- GPS readings contain measurement noise (\u00b15\u201315m typical, \u00b150m poor signal)
- Rider location pings can "jump" due to bad signal or device error
- Kalman filter gives an optimal estimate by balancing:
(1) Previous predicted position (process model)
(2) New GPS measurement (observation model)
Design:
- Separate filter instance per rider (stateful \u2014 preserves history)
- `CoordinateKalmanFilter` \u2014 single lat/lon smoother
- `GPSKalmanFilter` \u2014 wraps two CoordinateKalmanFilters (lat + lon)
- `RiderKalmanRegistry` \u2014 manages per-rider filter instances
- `smooth_coordinates()` \u2014 stateless single-shot smoother for delivery coords
Usage:
# Stateless (one-shot, no history \u2014 for delivery coords):
smooth_lat, smooth_lon = smooth_coordinates(raw_lat, raw_lon)
# Stateful (per-rider, preserves motion history):
registry = RiderKalmanRegistry()
lat, lon = registry.update(rider_id=1116, lat=11.0067, lon=76.9558)
"""
import logging
import time
from typing import Dict, Optional, Tuple
logger = logging.getLogger(__name__)
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# CORE 1D KALMAN FILTER
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
class CoordinateKalmanFilter:
"""
1-dimensional Kalman filter for a single GPS coordinate (lat or lon).
State model: position only (constant position with random walk).
Equations:
Prediction: x\u0302\u2096\u207b = x\u0302\u2096\u208b\u2081 (no movement assumed between pings)
P\u0302\u2096\u207b = P\u2096\u208b\u2081 + Q (uncertainty grows over time)
Update: K\u2096 = P\u0302\u2096\u207b / (P\u0302\u2096\u207b + R) (Kalman gain)
x\u0302\u2096 = x\u0302\u2096\u207b + K\u2096\u00b7(z\u2096 - x\u0302\u2096\u207b) (weighted fusion)
P\u2096 = (1 - K\u2096)\u00b7P\u0302\u2096\u207b (update uncertainty)
Parameters:
process_noise (Q): How much position can change between measurements.
Higher = filter trusts new measurements more (less smoothing).
measurement_noise (R): GPS measurement uncertainty.
Higher = filter trusts history more (more smoothing).
"""
def __init__(
self,
process_noise: float = 1e-4,
measurement_noise: float = 0.01,
initial_uncertainty: float = 1.0,
):
self.Q = process_noise
self.R = measurement_noise
self._x: Optional[float] = None
self._P: float = initial_uncertainty
@property
def initialized(self) -> bool:
return self._x is not None
def update(self, measurement: float) -> float:
"""Process one new measurement and return the filtered estimate."""
if not self.initialized:
self._x = measurement
return self._x
# Predict
x_prior = self._x
P_prior = self._P + self.Q
# Update
K = P_prior / (P_prior + self.R)
self._x = x_prior + K * (measurement - x_prior)
self._P = (1.0 - K) * P_prior
return self._x
def reset(self):
self._x = None
self._P = 1.0
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# 2D GPS KALMAN FILTER (lat + lon)
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
class GPSKalmanFilter:
"""
Two-dimensional GPS smoother using independent 1D Kalman filters
for latitude and longitude.
"""
def __init__(
self,
process_noise: float = 1e-4,
measurement_noise: float = 0.01,
):
self.lat_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
self.lon_filter = CoordinateKalmanFilter(process_noise, measurement_noise)
self.last_updated: float = time.time()
self.update_count: int = 0
def update(self, lat: float, lon: float) -> Tuple[float, float]:
"""Feed a new GPS reading and get the smoothed (lat, lon)."""
if not self._is_valid_coord(lat, lon):
if self.lat_filter.initialized:
return self.lat_filter._x, self.lon_filter._x
return lat, lon
smooth_lat = self.lat_filter.update(lat)
smooth_lon = self.lon_filter.update(lon)
self.last_updated = time.time()
self.update_count += 1
return smooth_lat, smooth_lon
def get_estimate(self) -> Optional[Tuple[float, float]]:
if self.lat_filter.initialized:
return self.lat_filter._x, self.lon_filter._x
return None
def reset(self):
self.lat_filter.reset()
self.lon_filter.reset()
self.update_count = 0
@staticmethod
def _is_valid_coord(lat: float, lon: float) -> bool:
try:
lat, lon = float(lat), float(lon)
return (
-90.0 <= lat <= 90.0
and -180.0 <= lon <= 180.0
and not (lat == 0.0 and lon == 0.0)
)
except (TypeError, ValueError):
return False
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# PER-RIDER FILTER REGISTRY
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
class RiderKalmanRegistry:
"""
Maintains per-rider Kalman filter instances across calls.
Stale filters (> 30 min silence) are automatically reset.
"""
def __init__(
self,
process_noise: float = 1e-4,
measurement_noise: float = 0.01,
stale_seconds: float = 1800.0,
):
self._filters: Dict[str, GPSKalmanFilter] = {}
self._process_noise = process_noise
self._measurement_noise = measurement_noise
self._stale_seconds = stale_seconds
def _get_or_create(self, rider_id) -> GPSKalmanFilter:
key = str(rider_id)
now = time.time()
if key in self._filters:
f = self._filters[key]
if now - f.last_updated > self._stale_seconds:
f.reset()
return f
self._filters[key] = GPSKalmanFilter(
process_noise=self._process_noise,
measurement_noise=self._measurement_noise,
)
return self._filters[key]
def update(self, rider_id, lat: float, lon: float) -> Tuple[float, float]:
return self._get_or_create(rider_id).update(lat, lon)
def get_estimate(self, rider_id) -> Optional[Tuple[float, float]]:
key = str(rider_id)
if key in self._filters:
return self._filters[key].get_estimate()
return None
def reset_rider(self, rider_id):
key = str(rider_id)
if key in self._filters:
self._filters[key].reset()
def clear_all(self):
self._filters.clear()
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# GLOBAL REGISTRY (process-level singleton)
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
_global_registry = RiderKalmanRegistry()
def get_registry() -> RiderKalmanRegistry:
"""Get the process-level rider Kalman filter registry."""
return _global_registry
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# STATELESS COORDINATE SMOOTHER
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
def smooth_coordinates(
lat: float,
lon: float,
*,
prior_lat: Optional[float] = None,
prior_lon: Optional[float] = None,
process_noise: float = 1e-4,
measurement_noise: float = 0.01,
) -> Tuple[float, float]:
"""
Stateless single-shot GPS smoother.
If a prior is provided, blends the new reading towards it.
"""
f = GPSKalmanFilter(process_noise=process_noise, measurement_noise=measurement_noise)
if prior_lat is not None and prior_lon is not None:
try:
_flat = float(prior_lat)
_flon = float(prior_lon)
if GPSKalmanFilter._is_valid_coord(_flat, _flon):
f.update(_flat, _flon)
except (TypeError, ValueError):
pass
return f.update(lat, lon)
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
# BATCH SMOOTHERS
# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
def smooth_rider_locations(riders: list) -> list:
"""
Apply Kalman smoothing to a list of rider dicts in-place using
the global per-rider registry (history preserved across calls).
Reads/writes: latitude, longitude (and currentlat/currentlong if present).
Adds: _kalman_smoothed = True on each processed rider.
"""
registry = get_registry()
for rider in riders:
try:
rider_id = (
rider.get("userid") or rider.get("riderid") or
rider.get("id") or "unknown"
)
raw_lat = float(rider.get("latitude") or rider.get("currentlat") or 0)
raw_lon = float(rider.get("longitude") or rider.get("currentlong") or 0)
if raw_lat == 0.0 and raw_lon == 0.0:
continue
smooth_lat, smooth_lon = registry.update(rider_id, raw_lat, raw_lon)
# Cast back to string for Go compatibility
s_lat, s_lon = str(round(smooth_lat, 8)), str(round(smooth_lon, 8))
rider["latitude"] = s_lat
rider["longitude"] = s_lon
if "currentlat" in rider:
rider["currentlat"] = s_lat
if "currentlong" in rider:
rider["currentlong"] = s_lon
rider["_kalman_smoothed"] = True
except Exception as e:
logger.debug(f"Kalman rider smoothing skipped: {e}")
return riders
def smooth_order_coordinates(orders: list) -> list:
"""
Validate and lightly normalise delivery coordinates in a list of order dicts.
DESIGN NOTE — why we do NOT use Kalman filtering here:
─────────────────────────────────────────────────────
Kalman filtering is a *temporal* smoother: it blends successive measurements
from the same sensor over time. Delivery coordinates are a single static
point (one measurement). Feeding the kitchen location as a "prior" would
pull the customer's address toward the kitchen — exactly wrong.
Per-customer GPS accuracy is handled upstream by the FAISS coordinate store
(verified historical rider-confirmed delivery points). This function only
normalises the coordinate fields to floats so downstream code never sees
raw strings or None values.
Modifies orders in-place. Returns the same list.
"""
for order in orders:
try:
dlat_raw = order.get("deliverylat") or order.get("droplat")
dlon_raw = order.get("deliverylong") or order.get("droplon")
if dlat_raw is None or dlon_raw is None:
continue
dlat = float(dlat_raw)
dlon = float(dlon_raw)
if not GPSKalmanFilter._is_valid_coord(dlat, dlon):
continue
# Normalise to string (Go service expects string coordinates)
s_lat = str(round(dlat, 8))
s_lon = str(round(dlon, 8))
order["deliverylat"] = s_lat
order["deliverylong"] = s_lon
if "droplat" in order:
order["droplat"] = s_lat
if "droplon" in order:
order["droplon"] = s_lon
except Exception as e:
logger.debug(f"Coordinate normalisation skipped: {e}")
return orders

341
app/services/routing/route_optimizer.py
View File

@@ -10,7 +10,6 @@ FEATURES:
- Automatic outlier detection and coordinate correction
- Hybrid distance calculation (Google Maps + Haversine fallback)
- Robust error handling for invalid inputs
- Composite cost function (idea.txt: distance + profit - margin)
"""
import math
@@ -21,7 +20,7 @@ import asyncio
from typing import Dict, Any, List as _List, Optional, Tuple, Union
from datetime import datetime, timedelta
import httpx
from app.services.routing.kalman_filter import smooth_order_coordinates
from app.services.routing.gps_smoother import smooth_order_coordinates
import numpy as np
from app.core.arrow_utils import calculate_haversine_matrix_vectorized
from app.config.dynamic_config import get_config
@@ -38,191 +37,6 @@ except ImportError:
logger = logging.getLogger(__name__)
class CompositeCostCalculator:
"""
Composite Cost Function for Profit-Aware Route Optimization.
Based on idea.txt data encoding techniques:
- Total Cost = Distance Cost + Rider Cost - Merchant Profit
- Edge Cost = (distance * fuel_rate) + rider_cost - merchant_margin
- route_score = profit - composite_cost (what we want to MAXIMIZE)
This transforms the problem from:
"Find shortest route" -> "Find most profitable route"
"""
def __init__(self):
# Cost parameters (can be ML-tuned via DynamicConfig)
self.fuel_rate_per_km = 2.5
self.base_rider_cost = 0.0
self.opportunity_cost_per_km = 0.5 # Cost of rider's time per km
self.profit_weight = 0.3 # How much profit influences routing (0-1)
self.traffic_multiplier_peak = 1.5 # Peak hour traffic penalty
self.traffic_multiplier_normal = 1.2 # Normal traffic multiplier
# Defaults for orders without profit data
self.default_order_amount = 80.0
self.default_merchant_margin = 5.0
def calculate_composite_cost(
self,
distance_km: float,
order_amount: float = None,
merchant_margin: float = None,
traffic_factor: float = 1.0,
time_of_day: str = "NORMAL",
) -> Dict[str, float]:
"""
Calculate composite cost for a route edge.
Args:
distance_km: Distance for this leg
order_amount: Revenue from this order (if known)
merchant_margin: Merchant's margin for this order (if known)
traffic_factor: Traffic multiplier (1.0 = normal)
time_of_day: Traffic time category ("PEAK", "NORMAL", "OFF_PEAK")
Returns:
Dict with:
- distance_cost: Raw distance cost
- rider_cost: Total rider cost for this leg
- gross_profit: Revenue - rider cost
- net_cost: Cost after profit adjustment (MINIMIZE THIS)
- route_score: Profitability score (MAXIMIZE THIS)
"""
# Distance cost = fuel + opportunity cost
distance_cost = distance_km * self.fuel_rate_per_km
# Rider cost = base + distance cost
rider_cost = self.base_rider_cost + distance_cost
# Apply traffic penalty
if time_of_day == "PEAK":
rider_cost *= self.traffic_multiplier_peak
elif time_of_day == "NORMAL":
rider_cost *= self.traffic_multiplier_normal
# Apply custom traffic factor
rider_cost *= traffic_factor
# Profit calculation (target encoding: use order data if available)
if order_amount is None:
order_amount = self.default_order_amount
if merchant_margin is None:
merchant_margin = self.default_merchant_margin
gross_profit = order_amount - rider_cost
# Net cost = rider cost - profit contribution
# This means high-profit orders have LOWER cost (more desirable)
profit_adjustment = gross_profit * self.profit_weight
net_cost = rider_cost - profit_adjustment
# Route score = profit - opportunity cost (for route planning)
route_score = gross_profit - (distance_km * self.opportunity_cost_per_km)
# Ensure net_cost is never negative (minimum cost for any delivery)
net_cost = max(net_cost, 5.0) # Minimum 5 km equivalent cost
return {
"distance_cost": round(distance_cost, 2),
"rider_cost": round(rider_cost, 2),
"gross_profit": round(gross_profit, 2),
"net_cost": round(net_cost, 2),
"route_score": round(route_score, 2),
}
def calculate_cost_matrix(
self,
dist_matrix: np.ndarray,
orders: _List[Dict[str, Any]] = None,
traffic_condition: str = "NORMAL",
) -> Tuple[np.ndarray, Dict[str, Any]]:
"""
Calculate composite cost matrix for all node pairs.
Args:
dist_matrix: Distance matrix (N x N)
orders: List of orders (for profit data)
traffic_condition: Traffic condition ("PEAK", "NORMAL", "OFF_PEAK")
Returns:
Tuple of (cost_matrix, summary_stats)
"""
n = len(dist_matrix)
cost_matrix = np.zeros((n, n))
# Extract order data for profit encoding
order_amounts = []
merchant_margins = []
if orders:
for o in orders:
try:
amount = float(
o.get("orderamount")
or o.get("deliveryamount")
or self.default_order_amount
)
margin = float(
o.get("merchant_margin") or self.default_merchant_margin
)
except:
amount = self.default_order_amount
margin = self.default_merchant_margin
order_amounts.append(amount)
merchant_margins.append(margin)
else:
order_amounts = [self.default_order_amount] * n
merchant_margins = [self.default_merchant_margin] * n
# Calculate costs for each pair
total_cost = 0.0
total_profit = 0.0
high_cost_count = 0
for i in range(n):
for j in range(n):
if i == j:
cost_matrix[i][j] = 0
continue
dist = dist_matrix[i][j]
# Use order j's profit data (destination)
order_amount = (
order_amounts[j - 1] if j > 0 else self.default_order_amount
)
merchant_margin = (
merchant_margins[j - 1] if j > 0 else self.default_merchant_margin
)
cost_data = self.calculate_composite_cost(
distance_km=dist,
order_amount=order_amount,
merchant_margin=merchant_margin,
time_of_day=traffic_condition,
)
cost_matrix[i][j] = cost_data["net_cost"]
total_cost += cost_data["net_cost"]
total_profit += cost_data["gross_profit"]
if cost_data["net_cost"] > 50:
high_cost_count += 1
summary = {
"total_cost": round(total_cost, 2),
"total_profit": round(total_profit, 2),
"avg_cost": round(total_cost / (n * n) if n > 0 else 0, 2),
"avg_profit": round(total_profit / (n * n) if n > 0 else 0, 2),
"high_cost_legs": high_cost_count,
"traffic_condition": traffic_condition,
}
return cost_matrix, summary
class RouteOptimizer:
"""Route optimization using Google OR-Tools (Async)."""
@@ -253,8 +67,6 @@ class RouteOptimizer:
# Solver time limit (ML-tuned)
self.search_time_limit_seconds = int(_cfg.get("search_time_limit_seconds"))
self.cost_calculator = CompositeCostCalculator()
def haversine_distance(
self, lat1: float, lon1: float, lat2: float, lon2: float
) -> float:
@@ -274,153 +86,6 @@ class RouteOptimizer:
except Exception:
return 0.0
async def _get_google_maps_distances_batch(
self, origin_lat: float, origin_lon: float, destinations: _List[tuple]
) -> Dict[tuple, float]:
"""Get road distances for multiple destinations from Google Maps API. (Async, Parallel)"""
if not self.use_google_maps or not destinations:
return {}
results = {}
batch_size = 25
chunks = [
destinations[i : i + batch_size]
for i in range(0, len(destinations), batch_size)
]
async def process_batch(batch):
batch_result = {}
try:
dest_str = "|".join([f"{lat},{lon}" for lat, lon in batch])
url = "https://maps.googleapis.com/maps/api/distancematrix/json"
params = {
"origins": f"{origin_lat},{origin_lon}",
"destinations": dest_str,
"key": self.google_maps_api_key,
"units": "metric",
}
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(url, params=params)
response.raise_for_status()
data = response.json()
if data.get("status") == "OK":
rows = data.get("rows", [])
if rows:
elements = rows[0].get("elements", [])
for idx, element in enumerate(elements):
if idx < len(batch):
dest_coord = batch[idx]
if element.get("status") == "OK":
dist = element.get("distance", {}).get("value")
dur = element.get("duration", {}).get("value")
if dist is not None:
batch_result[dest_coord] = {
"distance": dist / 1000.0,
"duration": dur / 60.0 if dur else None,
}
except Exception as e:
logger.warning(f"Google Maps batch call failed: {e}")
return batch_result
batch_results_list = await asyncio.gather(
*[process_batch(chunk) for chunk in chunks]
)
for res in batch_results_list:
results.update(res)
return results
# ------------------------------------------------------------------
# GOOGLE DIRECTIONS - WAYPOINT OPTIMISATION
# ------------------------------------------------------------------
async def _optimize_waypoints_google(
self,
origin_lat: float,
origin_lon: float,
waypoints: _List[Tuple[float, float]],
) -> Tuple[Optional[_List[int]], Optional[_List[float]]]:
"""
Ask Google Directions API to find the optimal visiting order for a set
of delivery points starting from a kitchen/pickup location.
Uses `optimize:true` in the waypoints parameter - Google solves the TSP
internally using actual road geometry (turn restrictions, one-way
streets, real distances) rather than Haversine approximation.
Returns
-------
(waypoint_order, leg_km)
waypoint_order 0-based indices into `waypoints` in optimal order.
e.g. [2, 0, 1] means visit wp[2] -> wp[0] -> wp[1].
leg_km Actual road distance (km) for each leg in order:
leg_km[0] = kitchen -> wp[order[0]],
leg_km[1] = wp[order[0]] -> wp[order[1]], etc.
Both are None on any failure - caller falls back to OR-Tools.
Notes
-----
- Supports up to 25 intermediate waypoints (Google's standard limit).
- destination = origin (closed-loop TSP); the return leg is discarded.
- One API call per rider per assignment - cheap at delivery scale.
"""
if not self.use_google_maps or not waypoints or len(waypoints) < 2:
return None, None
if len(waypoints) > 25:
return None, None # fall back to OR-Tools for unusually large routes
try:
wp_str = "optimize:true|" + "|".join(
f"{lat},{lon}" for lat, lon in waypoints
)
params = {
"origin": f"{origin_lat},{origin_lon}",
"destination": f"{origin_lat},{origin_lon}", # closed loop
"waypoints": wp_str,
"key": self.google_maps_api_key,
}
async with httpx.AsyncClient(timeout=8.0) as client:
resp = await client.get(
"https://maps.googleapis.com/maps/api/directions/json",
params=params,
)
resp.raise_for_status()
data = resp.json()
status_code = data.get("status")
if status_code != "OK":
logger.debug(
f"[GoogleWaypoints] status={status_code} "
f"error='{data.get('error_message', '')}'"
)
return None, None
routes = data.get("routes", [])
if not routes:
return None, None
route = routes[0]
wp_order = route.get("waypoint_order")
legs = route.get("legs", [])
if wp_order is None or len(wp_order) != len(waypoints):
return None, None
# Extract leg distances (metres -> km), skip the return-to-origin leg
leg_km: _List[float] = []
for leg in legs[: len(waypoints)]: # first N legs only
dist_m = leg.get("distance", {}).get("value")
leg_km.append(dist_m / 1000.0 if dist_m is not None else 0.0)
logger.debug(
f"[GoogleWaypoints] Optimised {len(waypoints)} stops -> order={wp_order}"
)
return wp_order, leg_km
except Exception as _e:
logger.debug(f"[GoogleWaypoints] Failed (non-fatal): {_e}")
return None, None
# ------------------------------------------------------------------
# ROAD-AWARE VISITING ORDER (Phase 2 - opt-in, cached)
# ------------------------------------------------------------------
@@ -698,8 +363,8 @@ class RouteOptimizer:
routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH
)
# TSP time limit hard-capped at 2 seconds per kitchen.
# The ML hypertuner may push search_time_limit_seconds up to 8-10s
# chasing marginally better routes, but at delivery scale (< 15 stops)
# search_time_limit_seconds can be tuned up to 8-10s via config, but at
# delivery scale (< 15 stops)
# OR-Tools finds a near-optimal solution in < 200ms. Waiting 8-10s
# per kitchen x 3 kitchens x 4 riders = 96s of unnecessary waiting.
# The VRP already has its own 3s cap. This cap applies to per-rider

BIN
app/services/vector/__pycache__/delivery_history_store.cpython-312.pyc
View File

Binary file not shown.

140
app/services/vector/delivery_history_store.py
View File

@@ -1,12 +1,21 @@
"""
Delivery History Store
======================
Pattern-first vector rider lookup using 30-day delivery history.
Pattern-first vector rider lookup using recent delivery history.
Works on any platform — no faiss-cpu dependency.
Data source (config `pattern_source`)
--------------------------------------
"csv" (legacy) — built from delivery_details.csv, only refreshed when a
human re-exports it and calls POST /ml/reload-history.
"db" (default going forward) — built from the nearledb mirror the ETA-sync
agent already maintains (delivery_raw), rebuilt
automatically every sync cycle via rebuild_from_records().
No manual step, no extra database load.
How it works
------------
At startup the CSV is parsed once and two structures are built:
Records (however sourced) are turned into two structures:
1. PATTERN TABLE (primary, O(1) lookup)
City divided into ~1.1 km grid cells (round coords to 2 d.p.).
@@ -65,9 +74,21 @@ CORRECTIONS_PATH = os.getenv("DELIVERY_CORRECTIONS_CSV", "delivery_corrections.c
_CORRECTION_WEIGHT = int(os.getenv("CORRECTION_WEIGHT", "1"))
_STORE_DIR = os.getenv("FAISS_HISTORY_DIR", "ml_data/faiss_history")
_VECTORS_PATH = os.path.join(_STORE_DIR, "delivery_history_vectors.npy")
_RECORDS_PATH = os.path.join(_STORE_DIR, "delivery_history_records.pkl")
_META_PATH = os.path.join(_STORE_DIR, "delivery_history.meta")
def _paths_for(source: str) -> Tuple[str, str, str]:
"""
Disk cache paths, namespaced by source ("csv" or "db"). Namespaced so that
flipping `pattern_source` at runtime (e.g. to debug, or to roll back)
never clobbers the other mode's cached snapshot — each mode keeps its own
independent copy on disk.
"""
suffix = "" if source == "csv" else f".{source}"
return (
os.path.join(_STORE_DIR, f"delivery_history_vectors{suffix}.npy"),
os.path.join(_STORE_DIR, f"delivery_history_records{suffix}.pkl"),
os.path.join(_STORE_DIR, f"delivery_history{suffix}.meta"),
)
# ---------------------------------------------------------------------------
# Thresholds
@@ -78,6 +99,15 @@ _MIN_PATTERN_DOMINANCE = 0.60 # pattern table: rider owns ≥ 60 % of zone
_MIN_PATTERN_VOLUME = 3 # pattern table: at least 3 deliveries in zone
def _pattern_source() -> str:
"""'csv' (legacy, manual refresh) or 'db' (auto-refreshed from nearledb mirror)."""
try:
from app.config.dynamic_config import get_config
return str(get_config().get("pattern_source", "csv"))
except Exception:
return "csv"
# ---------------------------------------------------------------------------
# Pure-NumPy L2 index — drop-in replacement for faiss.IndexFlatL2
# ---------------------------------------------------------------------------
@@ -152,6 +182,7 @@ class DeliveryHistoryStore:
store.pattern_count() -> int
store.get_pattern_stats() -> list
store.reload_from_csv() -> int
store.rebuild_from_records(records) -> int
"""
def __init__(self):
@@ -174,8 +205,21 @@ class DeliveryHistoryStore:
def _load(self) -> None:
os.makedirs(_STORE_DIR, exist_ok=True)
if _pattern_source() == "db":
# DB-driven: the ETA-sync agent keeps this fresh on its own cadence
# by calling rebuild_from_records() after every sync cycle. At
# startup we just load whatever was last persisted — no CSV
# freshness check applies in this mode.
if self._load_from_disk("db"):
return
logger.info(
"[DeliveryHistory] pattern_source=db, no persisted snapshot yet "
"— will populate on the next ETA-sync cycle."
)
return
if self._saved_files_are_current():
if self._load_from_disk():
if self._load_from_disk("csv"):
return
logger.warning(
"[DeliveryHistory] Saved files corrupt — rebuilding from CSV."
@@ -184,14 +228,15 @@ class DeliveryHistoryStore:
records = self._parse_csv()
if not records:
return
self._build_and_save(records)
self._build_and_save(records, source="csv")
def _saved_files_are_current(self) -> bool:
for path in (_VECTORS_PATH, _RECORDS_PATH, _META_PATH):
vectors_path, records_path, meta_path = _paths_for("csv")
for path in (vectors_path, records_path, meta_path):
if not os.path.isfile(path):
return False
try:
with open(_META_PATH, "r", encoding="utf-8") as f:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f)
if not os.path.isfile(CSV_PATH):
return True
@@ -199,10 +244,11 @@ class DeliveryHistoryStore:
except Exception:
return False
def _load_from_disk(self) -> bool:
def _load_from_disk(self, source: str) -> bool:
try:
vectors = np.load(_VECTORS_PATH) # (N, 4) float32
with open(_RECORDS_PATH, "rb") as f:
vectors_path, records_path, _ = _paths_for(source)
vectors = np.load(vectors_path) # (N, 4) float32
with open(records_path, "rb") as f:
records = pickle.load(f)
if not records or len(vectors) != len(records):
@@ -342,7 +388,7 @@ class DeliveryHistoryStore:
return patterns, zone_index
def _build_and_save(self, records: List[Dict]) -> None:
def _build_and_save(self, records: List[Dict], source: str = "csv") -> None:
vectors = np.array(
[[r["pickuplat"], r["pickuplon"], r["deliverylat"], r["deliverylong"]]
for r in records],
@@ -373,19 +419,28 @@ class DeliveryHistoryStore:
)
try:
np.save(_VECTORS_PATH, vectors)
with open(_RECORDS_PATH, "wb") as f:
vectors_path, records_path, meta_path = _paths_for(source)
np.save(vectors_path, vectors)
with open(records_path, "wb") as f:
pickle.dump(records, f, protocol=pickle.HIGHEST_PROTOCOL)
csv_mtime = os.path.getmtime(CSV_PATH) if os.path.isfile(CSV_PATH) else 0
with open(_META_PATH, "w", encoding="utf-8") as f:
# csv_mtime only means something for the CSV path's freshness check
# (_saved_files_are_current). DB-built snapshots are refreshed by the
# ETA-sync agent's own schedule, not a file-mtime comparison.
csv_mtime = (
os.path.getmtime(CSV_PATH)
if source == "csv" and os.path.isfile(CSV_PATH)
else 0
)
with open(meta_path, "w", encoding="utf-8") as f:
json.dump({
"source": source,
"csv_mtime": csv_mtime,
"record_count": len(records),
"pattern_zones": len(patterns),
"clear_patterns": clear,
}, f, indent=2)
logger.info(
f"[DeliveryHistory] Saved to '{_STORE_DIR}'. "
f"[DeliveryHistory] Saved to '{_STORE_DIR}' (source={source}). "
"Next startup loads from disk."
)
except Exception as e:
@@ -665,7 +720,34 @@ class DeliveryHistoryStore:
records = self._parse_csv() # already merges corrections internally
if not records:
return 0
self._build_and_save(records)
self._build_and_save(records, source="csv")
return len(records)
def rebuild_from_records(self, records: List[Dict]) -> int:
"""
Rebuild the pattern table + vector index directly from pre-shaped
records (kitchen/pickuplat/pickuplon/deliverylat/deliverylong/userid/
ridername), bypassing CSV parsing entirely.
Called by the ETA-sync agent (delivery_history_service.py) after each
sync cycle when pattern_source=db, so this store stays as fresh as the
nearledb mirror — no manual CSV re-export/reload needed.
"""
if not records:
logger.warning(
"[DeliveryHistory] rebuild_from_records got 0 records — "
"keeping the existing store as-is."
)
return 0
# Merge manually-verified corrections on top, same as the CSV path,
# so the human-override mechanism still works in DB mode.
if os.path.isfile(CORRECTIONS_PATH):
corr = self._parse_csv_file(CORRECTIONS_PATH, label="Corrections CSV")
if corr:
records = records + (corr * _CORRECTION_WEIGHT)
self._build_and_save(records, source="db")
return len(records)
def inject_corrections(self, corrections_path: str = CORRECTIONS_PATH,
@@ -720,14 +802,22 @@ class DeliveryHistoryStore:
self._patterns = patterns
self._zone_index = zone_index
# Save to disk so next restart includes corrections
# Save to disk (under whichever source is currently active) so next
# restart includes corrections
try:
np.save(_VECTORS_PATH, vectors)
with open(_RECORDS_PATH, "wb") as f:
active_source = _pattern_source()
vectors_path, records_path, meta_path = _paths_for(active_source)
np.save(vectors_path, vectors)
with open(records_path, "wb") as f:
pickle.dump(merged, f)
csv_mtime = os.path.getmtime(CSV_PATH) if os.path.isfile(CSV_PATH) else 0
with open(_META_PATH, "w", encoding="utf-8") as f:
json.dump({"csv_mtime": csv_mtime, "record_count": len(merged),
csv_mtime = (
os.path.getmtime(CSV_PATH)
if active_source == "csv" and os.path.isfile(CSV_PATH)
else 0
)
with open(meta_path, "w", encoding="utf-8") as f:
json.dump({"source": active_source, "csv_mtime": csv_mtime,
"record_count": len(merged),
"pattern_count": len(patterns)}, f)
logger.info(
f"[DeliveryHistory] Corrections injected and saved — "