Nearle/routesapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new changes in the api

Browse Source
This commit is contained in:
Suriya
2026-07-06 15:15:51 +05:30
parent c742ef0e53
commit 871981035a
43 changed files with 414 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
app/services/ml/ml_data_collector.py
View File

@@ -7,13 +7,12 @@ Key upgrades over the original
--------------------------------
1. FROZEN historical scores - quality_score is written ONCE at log time.
get_training_data() returns scores as-is from the DB (no retroactive mutation).
2. Rich schema - zone_id, city_id, is_peak, weather_code,
sla_breached, avg_delivery_time_min for richer features.
3. SLA tracking - logs whether delivery SLA was breached.
4. Analytics API - get_hourly_stats(), get_strategy_comparison(),
get_quality_histogram(), get_zone_stats() for dashboard consumption.
5. Thread-safe writes - connection-per-write pattern for FastAPI workers.
6. Indexed columns - timestamp, ml_strategy, zone_id for fast queries.
2. Rich schema - zone_id, city_id, is_peak, weather_code for
richer features.
3. Analytics API - get_hourly_stats(), get_quality_histogram(),
get_zone_stats() for dashboard consumption.
4. Thread-safe writes - connection-per-write pattern for FastAPI workers.
5. Indexed columns - timestamp, zone_id for fast queries.
"""
import csv
@@ -45,7 +44,7 @@ class MLDataCollector:
Each log_assignment_event() call writes one row capturing:
- Operating context (time, orders, riders, zone, city)
- Active hyperparams (exact config snapshot for this call)
- Measured outcomes (quality score, SLA, latency, distances)
- Measured outcomes (quality score, latency, distances)
quality_score is computed once and FROZEN - never retroactively changed.
"""
@@ -70,8 +69,6 @@ class MLDataCollector:
zone_id: str = "default",
city_id: str = "default",
weather_code: str = "CLEAR",
sla_minutes: Optional[float] = None,
avg_delivery_time_min: Optional[float] = None,
) -> None:
"""
Log one assignment event.
@@ -95,13 +92,8 @@ class MLDataCollector:
o for orders in assignments.values() if orders for o in orders
]
total_distance_km = sum(self._get_km(o) for o in all_orders)
ml_strategy = hyperparams.get("ml_strategy", "balanced")
max_opr = hyperparams.get("max_orders_per_rider", 12)
sla_breached = 0
if sla_minutes and avg_delivery_time_min:
sla_breached = int(avg_delivery_time_min > sla_minutes)
# Quality score - FROZEN at log time
quality_score = self._compute_quality_score(
num_orders=num_orders,
@@ -111,7 +103,6 @@ class MLDataCollector:
num_riders=num_riders,
total_distance_km=total_distance_km,
max_orders_per_rider=max_opr,
ml_strategy=ml_strategy,
)
row = {
@@ -146,7 +137,6 @@ class MLDataCollector:
"search_time_limit_seconds", 5
),
"road_factor": hyperparams.get("road_factor", 1.3),
"ml_strategy": ml_strategy,
"riders_used": riders_used,
"total_assigned": total_assigned,
"unassigned_count": unassigned_count,
@@ -154,8 +144,6 @@ class MLDataCollector:
"load_std": round(load_std, 3),
"total_distance_km": round(total_distance_km, 2),
"elapsed_ms": round(elapsed_ms, 1),
"sla_breached": sla_breached,
"avg_delivery_time_min": round(avg_delivery_time_min or 0.0, 2),
"quality_score": round(quality_score, 2),
}
@@ -171,7 +159,7 @@ class MLDataCollector:
except Exception as e:
logger.warning(f"[MLCollector] Logging failed (non-fatal): {e}")
return 50.0 # neutral fallback so bandit update still fires
return 50.0 # neutral fallback
# ------------------------------------------------------------------
# Data retrieval for training
@@ -180,7 +168,6 @@ class MLDataCollector:
def get_training_data(
self,
min_records: int = 30,
strategy_filter: Optional[str] = None,
since_hours: Optional[int] = None,
) -> Optional[List[Dict[str, Any]]]:
"""
@@ -195,9 +182,6 @@ class MLDataCollector:
params: list = []
clauses: list = []
if strategy_filter:
clauses.append("ml_strategy = ?")
params.append(strategy_filter)
if since_hours:
cutoff = (datetime.utcnow() - timedelta(hours=since_hours)).isoformat()
clauses.append("timestamp >= ?")
@@ -253,7 +237,7 @@ class MLDataCollector:
return {"avg_quality": 0.0, "sample_size": 0, "history": []}
def get_hourly_stats(self, last_days: int = 7) -> List[Dict[str, Any]]:
"""Quality, SLA, and call volume aggregated by hour-of-day."""
"""Quality and call volume aggregated by hour-of-day."""
try:
conn = sqlite3.connect(self._db_path)
cutoff = (datetime.utcnow() - timedelta(days=last_days)).isoformat()
@@ -263,8 +247,7 @@ class MLDataCollector:
COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
AVG(unassigned_count) AS avg_unassigned,
AVG(elapsed_ms) AS avg_latency_ms,
SUM(CASE WHEN sla_breached=1 THEN 1 ELSE 0 END) AS sla_breaches
AVG(elapsed_ms) AS avg_latency_ms
FROM assignment_ml_log WHERE timestamp >= ?
GROUP BY hour ORDER BY hour
""",
@@ -278,7 +261,6 @@ class MLDataCollector:
"avg_quality": round(r[2] or 0.0, 2),
"avg_unassigned": round(r[3] or 0.0, 2),
"avg_latency_ms": round(r[4] or 0.0, 1),
"sla_breaches": r[5],
}
for r in rows
]
@@ -286,42 +268,6 @@ class MLDataCollector:
logger.error(f"[MLCollector] get_hourly_stats: {e}")
return []
def get_strategy_comparison(self) -> List[Dict[str, Any]]:
"""Compare quality metrics across ml_strategy values."""
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"""
SELECT ml_strategy,
COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
MIN(quality_score) AS min_quality,
MAX(quality_score) AS max_quality,
AVG(unassigned_count) AS avg_unassigned,
AVG(total_distance_km) AS avg_distance_km,
AVG(elapsed_ms) AS avg_latency_ms
FROM assignment_ml_log
GROUP BY ml_strategy ORDER BY avg_quality DESC
"""
).fetchall()
conn.close()
return [
{
"strategy": r[0],
"call_count": r[1],
"avg_quality": round(r[2] or 0.0, 2),
"min_quality": round(r[3] or 0.0, 2),
"max_quality": round(r[4] or 0.0, 2),
"avg_unassigned": round(r[5] or 0.0, 2),
"avg_distance_km": round(r[6] or 0.0, 2),
"avg_latency_ms": round(r[7] or 0.0, 1),
}
for r in rows
]
except Exception as e:
logger.error(f"[MLCollector] get_strategy_comparison: {e}")
return []
def get_quality_histogram(self, bins: int = 10) -> List[Dict[str, Any]]:
"""Quality score distribution for histogram chart."""
try:
@@ -348,14 +294,13 @@ class MLDataCollector:
return []
def get_zone_stats(self) -> List[Dict[str, Any]]:
"""Quality and SLA stats grouped by zone."""
"""Quality stats grouped by zone."""
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"""
SELECT zone_id, COUNT(*) AS call_count,
AVG(quality_score) AS avg_quality,
SUM(sla_breached) AS sla_breaches,
AVG(total_distance_km) AS avg_distance_km
FROM assignment_ml_log
GROUP BY zone_id ORDER BY avg_quality DESC
@@ -367,8 +312,7 @@ class MLDataCollector:
"zone_id": r[0],
"call_count": r[1],
"avg_quality": round(r[2] or 0.0, 2),
"sla_breaches": r[3],
"avg_distance_km": round(r[4] or 0.0, 2),
"avg_distance_km": round(r[3] or 0.0, 2),
}
for r in rows
]
@@ -385,17 +329,6 @@ class MLDataCollector:
except Exception:
return 0
def count_by_strategy(self) -> Dict[str, int]:
try:
conn = sqlite3.connect(self._db_path)
rows = conn.execute(
"SELECT ml_strategy, COUNT(*) FROM assignment_ml_log GROUP BY ml_strategy"
).fetchall()
conn.close()
return {r[0]: r[1] for r in rows}
except Exception:
return {}
def export_csv(self) -> str:
"""Export all records as CSV string."""
try:
@@ -448,7 +381,6 @@ class MLDataCollector:
num_riders: int,
total_distance_km: float,
max_orders_per_rider: int,
ml_strategy: str = "balanced",
) -> float:
"""
Multi-dimensional quality score (0–100, higher = better).
@@ -461,11 +393,7 @@ class MLDataCollector:
│ rider_efficiency │ reward using minimal riders for the batch size │
└──────────────────────┴────────────────────────────────────────────────┘
Strategy weights (w_assign, w_dist, w_balance, w_efficiency):
- balanced : (45, 20, 20, 15)
- aggressive_speed: (70, 15, 0, 15) — care about assignment + efficiency
- fuel_saver : (25, 60, 0, 15) — heavily penalise long routes
- zone_strict : (35, 25, 25, 15) — balanced with zone awareness
One fixed weighting (45, 20, 20, 15) is used for every call.
"""
import math
if num_orders == 0:
@@ -488,14 +416,7 @@ class MLDataCollector:
min_riders_needed = max(1, math.ceil(num_orders / max_orders_per_rider))
rider_efficiency = min(1.0, min_riders_needed / max(1, riders_used))
weights = {
# assign dist balance efficiency
"aggressive_speed": (70.0, 15.0, 0.0, 15.0),
"fuel_saver": (25.0, 60.0, 0.0, 15.0),
"zone_strict": (35.0, 25.0, 25.0, 15.0),
"balanced": (45.0, 20.0, 20.0, 15.0),
}
w_comp, w_dist, w_bal, w_eff = weights.get(ml_strategy, (45.0, 20.0, 20.0, 15.0))
w_comp, w_dist, w_bal, w_eff = (45.0, 20.0, 20.0, 15.0)
return min(
assigned_ratio * w_comp
@@ -542,7 +463,6 @@ class MLDataCollector:
cluster_radius_km REAL,
search_time_limit_seconds INTEGER,
road_factor REAL,
ml_strategy TEXT DEFAULT 'balanced',
riders_used INTEGER,
total_assigned INTEGER,
unassigned_count INTEGER,
@@ -550,8 +470,6 @@ class MLDataCollector:
load_std REAL,
total_distance_km REAL DEFAULT 0.0,
elapsed_ms REAL,
sla_breached INTEGER DEFAULT 0,
avg_delivery_time_min REAL DEFAULT 0.0,
quality_score REAL
)
""")
@@ -560,9 +478,6 @@ class MLDataCollector:
"ALTER TABLE assignment_ml_log ADD COLUMN zone_id TEXT DEFAULT 'default'",
"ALTER TABLE assignment_ml_log ADD COLUMN city_id TEXT DEFAULT 'default'",
"ALTER TABLE assignment_ml_log ADD COLUMN weather_code TEXT DEFAULT 'CLEAR'",
"ALTER TABLE assignment_ml_log ADD COLUMN sla_breached INTEGER DEFAULT 0",
"ALTER TABLE assignment_ml_log ADD COLUMN avg_delivery_time_min REAL DEFAULT 0.0",
"ALTER TABLE assignment_ml_log ADD COLUMN ml_strategy TEXT DEFAULT 'balanced'",
"ALTER TABLE assignment_ml_log ADD COLUMN total_distance_km REAL DEFAULT 0.0",
]
for ddl in migrations:
@@ -572,7 +487,6 @@ class MLDataCollector:
pass
for idx in [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON assignment_ml_log(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_strategy ON assignment_ml_log(ml_strategy)",
"CREATE INDEX IF NOT EXISTS idx_zone ON assignment_ml_log(zone_id)",
]:
conn.execute(idx)