"""Provider payload optimization endpoints.""" import asyncio as _asyncio import logging import os import time from concurrent.futures import ThreadPoolExecutor from math import radians, cos, sin, asin, sqrt as _math_sqrt from typing import Any from fastapi import APIRouter, Body, Request, Depends, status, HTTPException, Query from app.controllers.route_controller import RouteController from app.core.exceptions import APIException from app.core.arrow_utils import save_optimized_route_parquet logger = logging.getLogger(__name__) # ── Thread pool for parallel per-rider TSP route optimization ──────────────── # # Problem: asyncio.gather on async coroutines that call OR-Tools internally # does NOT give true CPU parallelism. Python's GIL means only one thread # runs Python bytecode at a time, so "parallel" async TSP calls actually # execute sequentially on a single core. # # Solution: run each rider's optimize_provider_payload in a ThreadPoolExecutor. # OR-Tools is a C++ extension that RELEASES the GIL during solving, so threads # genuinely run on separate CPU cores simultaneously. # # Result: 8 riders' routes are solved in parallel instead of serially. # Typical gain: 8 × 200ms serial → ~200ms parallel (4-8× speedup). # # Pool size: capped at 12 workers (enough for max expected simultaneous riders; # more workers than cores still helps because OR-Tools releases the GIL). _route_executor = ThreadPoolExecutor( max_workers=min(12, (os.cpu_count() or 4)), thread_name_prefix="route-tsp", ) def _sync_optimize_route(rider_orders: list) -> list: """ Thread-pool worker: run optimize_provider_payload inside an isolated event loop. Design decisions: - Fresh RouteOptimizer per call: the class is cheap to construct (just config reads) and creates no shared mutable state, so this is safe and avoids any future thread-safety risk from shared instances. - Fresh asyncio event loop per thread: event loops are NOT thread-safe; each OS thread must own its own loop. asyncio.new_event_loop() + run_until_complete is the documented way to call async code from a synchronous thread context. """ from app.services.routing.route_optimizer import RouteOptimizer as _RO _opt = _RO() _loop = _asyncio.new_event_loop() try: return _loop.run_until_complete( _opt.optimize_provider_payload(rider_orders, start_coords=None) ) except Exception: logger.exception("[ThreadTSP] Route optimization failed in worker thread") return [] finally: _loop.close() # ── Background thread pool for ML tasks (logging, retraining, auto-tuning) ─── # Kept deliberately small (2 workers) — these jobs are lightweight SQLite # writes and in-memory ID3 fits that finish in < 1 s each. They must never # compete with the route-tsp pool for CPU during a live request. _ml_executor = ThreadPoolExecutor( max_workers=2, thread_name_prefix="ml-bg", ) def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: """Great-circle distance in km (inline, no external dep).""" try: la1, lo1, la2, lo2 = map(radians, [float(lat1), float(lon1), float(lat2), float(lon2)]) dlat = la2 - la1 dlon = lo2 - lo1 a = sin(dlat / 2) ** 2 + cos(la1) * cos(la2) * sin(dlon / 2) ** 2 return 2 * asin(min(1.0, _math_sqrt(a))) * 6371.0 except Exception: return float("inf") def _auto_tune_strategy(collector) -> None: """ SQL-based strategy auto-tuner. Reads quality_score statistics from the assignment log grouped by ml_strategy. If one strategy clearly outperforms the rest (minimum 10 calls, ≥ 2 strategies compared) it is written to DynamicConfig so all future requests benefit automatically. Safe to call at any time — DynamicConfig write is idempotent. """ try: comparison = collector.get_strategy_comparison() if not comparison or len(comparison) < 2: return # not enough variety to compare # Only consider strategies with at least 10 real observations qualified = [s for s in comparison if s["call_count"] >= 10] if not qualified: return best = max(qualified, key=lambda x: x["avg_quality"]) from app.config.dynamic_config import get_config as _gc cfg = _gc() current = cfg.get("ml_strategy", "balanced") if best["strategy"] != current: cfg.set("ml_strategy", best["strategy"], source="auto_tuner") logger.info( f"[AutoTune] Strategy updated: '{current}' → '{best['strategy']}' " f"(avg_quality={best['avg_quality']:.1f}, n={best['call_count']})" ) else: logger.debug( f"[AutoTune] Strategy '{current}' already optimal " f"(avg_quality={best['avg_quality']:.1f})" ) except Exception as _ate: logger.warning(f"[AutoTune] Failed (non-fatal): {_ate}") def _bg_log_and_maybe_retrain( num_orders: int, num_riders: int, hyperparams: dict, assignments: dict, unassigned_count: int, elapsed_ms: float, bandit_context: "tuple | None" = None, # (time_band, load_band) ) -> None: """ Background worker: log event → update RL bandit → auto-retrain ID3 → auto-tune strategy. Runs in _ml_executor (never blocks the response). Thresholds: every call → bandit posterior update (Thompson Sampling RL) every 100 evts → greedy SQL auto-tuner (backup check, rarely fires now) """ try: from app.services.ml.ml_data_collector import get_collector as _gc collector = _gc() # ── 1. Log event, get back quality_score ────────────────────────── quality_score = collector.log_assignment_event( num_orders=num_orders, num_riders=num_riders, hyperparams=hyperparams, assignments=assignments, unassigned_count=unassigned_count, elapsed_ms=elapsed_ms, ) or 50.0 count = collector.count_records() logger.debug(f"[ML BG] Logged event #{count}, quality={quality_score:.1f}") # ── 2. Thompson Sampling bandit update (RL feedback) ────────────── if bandit_context: try: from app.services.ml.strategy_bandit import get_bandit as _gb _t_band, _l_band = bandit_context _strategy = hyperparams.get("ml_strategy", "balanced") get_bandit_fn = _gb get_bandit_fn().update(_t_band, _l_band, _strategy, quality_score) logger.debug( f"[Bandit] Updated ctx={_t_band}|{_l_band} " f"arm={_strategy} reward={quality_score/100:.3f}" ) except Exception as _be: logger.debug(f"[Bandit] Update failed (non-fatal): {_be}") # ── 3. Greedy SQL auto-tune every 100 events (backup) ──────────── if count >= 20 and count % 100 == 0: _auto_tune_strategy(collector) except Exception as _e: logger.warning(f"[ML BG] Background task failed (non-fatal): {_e}") router = APIRouter( prefix="/api/v1/optimization", tags=["Route Optimization"], responses={ 400: {"description": "Bad request - Invalid input parameters"}, 422: {"description": "Validation error - Request validation failed"}, 500: {"description": "Internal server error"}, }, ) def get_route_controller() -> RouteController: """Dependency injection for route controller.""" return RouteController() # Legacy single-route endpoint removed; provider flow only. @router.post( "/createdeliveries", status_code=status.HTTP_200_OK, summary="Optimize provider payload (forwarding paused)", description=""" Accepts the provider's orders array, reorders it using greedy nearest-neighbor, adds only: - step (1..N) - previouskms (distance from previous stop in km) - cumulativekms (total distance so far in km) - actualkms (direct pickup-to-delivery distance) Forwarding is temporarily paused: returns the optimized array in the response. """, responses={ 200: { "description": "Upstream response", "content": { "application/json": { "example": { "code": 200, "details": [], "message": "Success", "status": True, } } }, } }, ) async def provider_optimize_forward( body: list[dict], controller: RouteController = Depends(get_route_controller) ): """ Accept provider JSON array, reorder by greedy nearest-neighbor, annotate each item with: - step (1..N) - previouskms (km from previous point) - cumulativekms (km so far) - actualkms (pickup to delivery distance) Then forward the optimized array to the external API and return only its response. """ try: url = "https://jupiter.nearle.app/live/api/v1/deliveries/createdeliveries" result = await controller.optimize_and_forward_provider_payload(body, url) # Parquet snapshot — offloaded so it never blocks the response def _snap(): try: os.makedirs("data/snapshots", exist_ok=True) snapshot_path = f"data/snapshots/route_{int(time.time())}.parquet" save_optimized_route_parquet(body, snapshot_path) logger.info(f"Apache Arrow: Snapshot saved to {snapshot_path}") except Exception as e: logger.warning(f"Could not save Arrow snapshot: {e}") _ml_executor.submit(_snap) return result except APIException: raise except Exception as e: logger.error( f"Unexpected error in provider_optimize_forward: {e}", exc_info=True ) raise HTTPException(status_code=500, detail="Internal server error") @router.get("/createdeliveries", summary="Usage info for provider optimize forward") async def provider_optimize_forward_info(): """Return usage info; this endpoint accepts POST only for processing.""" return { "message": "Use POST with a JSON array of orders to optimize and forward.", "method": "POST", "path": "/api/v1/optimization/provider-optimize-forward", } # --------------------------------------------------------------------------- # STEP RECONCILIATION # --------------------------------------------------------------------------- def _detect_step_anomaly(orders: list[dict]) -> tuple[bool, str]: """ Check whether a rider's order list has a clean sequential step sequence. Returns (anomaly_found, reason_string). Anomaly cases: - duplicate step numbers - gap in step numbers (e.g. 1,2,3,5,6 — missing 4) - step numbers don't start at 1 for pending-only list - an order has no "step" field at all """ steps = [] for o in orders: s = o.get("step") if s is None: return True, "missing_step" try: steps.append(int(s)) except (ValueError, TypeError): return True, "invalid_step" if not steps: return False, "empty" steps_sorted = sorted(steps) # Duplicate check if len(steps_sorted) != len(set(steps_sorted)): return True, "duplicate" # Gap / non-sequential check expected = list(range(steps_sorted[0], steps_sorted[0] + len(steps_sorted))) if steps_sorted != expected: return True, "gap" # Should start from 1 (not from some arbitrary offset) if steps_sorted[0] != 1: return True, "wrong_start" return False, "ok" @router.post( "/reconcile-steps", status_code=status.HTTP_200_OK, summary="Reconcile step numbers after manual rider reassignment", description=""" When the operations team manually moves an order from one rider to another, step numbers become inconsistent: - Donor rider gets a gap (steps 1,2,3,5,6 — step 4 was transferred) - Recipient rider gets a new order with a conflicting or missing step This endpoint: 1. Detects anomalies per rider (gap, duplicate, missing, wrong order) 2. Separates each rider's already-delivered orders from pending ones 3. Re-runs the route optimizer on the pending orders to get the best sequence 4. Returns clean sequential steps starting from 1 for each rider Input: { "riders": [ { "rider_id": 883, "orders": [ ...full order objects with current step values... ] } ] } The "orders" array for each rider should include ALL current orders (delivered + pending). The endpoint determines delivered status from the "deliverytime" field (non-null/non-empty = delivered). """, ) async def reconcile_steps(body: Any = Body(default=None)): if not body or not isinstance(body, dict): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail='Body must be {"riders": [{"rider_id": X, "orders": [...]}]}', ) riders_input: list = body.get("riders") or [] if not riders_input: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="No riders provided.", ) from app.services.routing.route_optimizer import RouteOptimizer results = [] for rider_block in riders_input: rider_id = rider_block.get("rider_id") or rider_block.get("userid") all_orders: list[dict] = rider_block.get("orders") or [] # Resolve rider name: explicit input > order username field > fallback rider_name = ( rider_block.get("rider_name") or rider_block.get("username") or next( (o.get("username") or o.get("rider") for o in all_orders if o.get("username") or o.get("rider")), None, ) or f"Rider {rider_id}" ) if not all_orders: results.append({ "rider_id": rider_id, "rider_name": rider_name, "anomaly_detected": False, "anomaly_type": "empty", "delivered_count": 0, "pending_count": 0, "resequenced": False, "orders": [], }) continue # ── Split delivered vs pending ────────────────────────────────────── # Delivered = deliverytime is non-null and non-empty string. # Pending = everything else (in-transit or not yet started). delivered: list[dict] = [] pending: list[dict] = [] for o in all_orders: dt = o.get("deliverytime") if dt and str(dt).strip() not in ("", "0", "null", "None"): delivered.append(dict(o)) else: pending.append(dict(o)) # ── Detect anomaly on the FULL order list ─────────────────────────── anomaly, reason = _detect_step_anomaly(all_orders) if not anomaly and not pending: # Everything delivered, steps clean — nothing to do results.append({ "rider_id": rider_id, "rider_name": rider_name, "anomaly_detected": False, "anomaly_type": "ok", "delivered_count": len(delivered), "pending_count": 0, "resequenced": False, "orders": all_orders, }) continue # ── Step-value helper (used by both fallback sort and delivered sort) ─ def _step_val(o: dict) -> int: try: return int(o.get("step") or 0) except (ValueError, TypeError): return 0 # ── Coordinate normalizer ─────────────────────────────────────────── # optimize_provider_payload reads only deliverylat / deliverylong. # Orders from reconcile callers may carry coords under droplat/droplon # or dlat/dlon. Normalise so the optimizer always gets valid coords. def _norm_coords(o: dict) -> dict: o = dict(o) def _fv(v): try: return float(v or 0) except: return 0.0 if _fv(o.get("deliverylat")) == 0: o["deliverylat"] = o.get("droplat") or o.get("dlat") or o.get("deliverylat") or "" if _fv(o.get("deliverylong")) == 0: o["deliverylong"] = ( o.get("droplon") or o.get("dlon") or o.get("deliverylong") or "" ) return o # ── Re-optimise pending orders ────────────────────────────────────── # Run the same route optimizer used at assignment time so step order # matches the greedy + 2-opt algorithm (now with crossings eliminated). # Fallback: sort by existing step value so at least the sequence is # deterministic when the optimizer cannot run. resequenced_pending: list[dict] = sorted(pending, key=_step_val) if pending: try: _opt = RouteOptimizer() _optimized = await _opt.optimize_provider_payload( [_norm_coords(o) for o in pending], start_coords=None, ) if _optimized: # Optimizer returns list in route order (index 0 = first stop). # Use it; otherwise keep the sorted fallback. resequenced_pending = _optimized else: logger.warning( f"[Reconcile] Optimizer returned empty result for rider " f"{rider_id} — keeping step-sorted fallback." ) except Exception as _oe: logger.warning( f"[Reconcile] Route re-optimize failed for rider {rider_id} " f"(non-fatal, keeping step-sorted fallback): {_oe}" ) # ── Renumber: delivered keep positions 1..N, pending continue after ─ # Sort delivered by their existing step so the sequence is stable. delivered_sorted = sorted(delivered, key=_step_val) # Reassign delivered steps cleanly (fills any gap in delivered portion) for i, o in enumerate(delivered_sorted, start=1): o["step"] = i max_delivered_step = len(delivered_sorted) # Pending steps start immediately after the last delivered step. # resequenced_pending is already in route order (optimizer output or # step-sorted fallback), so enumerate order == correct delivery order. for i, o in enumerate(resequenced_pending, start=max_delivered_step + 1): o["step"] = i final_orders = delivered_sorted + resequenced_pending results.append({ "rider_id": rider_id, "rider_name": rider_name, "anomaly_detected": anomaly, "anomaly_type": reason, "delivered_count": len(delivered), "pending_count": len(pending), "resequenced": True, "orders": final_orders, }) logger.info( f"[Reconcile] Rider {rider_id}: anomaly={reason} " f"delivered={len(delivered)} pending={len(pending)} " f"resequenced={len(resequenced_pending)}" ) return { "status": True, "reconciled_riders": len(results), "riders": results, } @router.post( "/riderassign", status_code=status.HTTP_200_OK, summary="Assign created orders to active riders", description=""" Assigns orders to riders based on kitchen preferences, proximity, and load. - If a payload of orders is provided, processes those. - If payload is empty, fetches all 'created' orders from the external API. - Fetches active riders and matches them. """, responses={ 200: { "description": "Assignment Result", "content": { "application/json": { "example": { "code": 200, "details": {"1234": [{"orderid": "..."}]}, "message": "Success", "status": True, } } }, } }, ) async def assign_orders_to_riders( request: Request, body: Any = Body(default=None), reshuffle: bool = Query(False, alias="reshuffle"), hypertuning_params: str = None, ): """ Smart assignment of orders to riders. Accepts two payload formats: - Legacy: flat JSON array of order objects - New: {"deliveries": [...], "absent_riders": [{"userid": 123, "username": "..."}]} Riders listed in absent_riders are excluded from all assignment phases for this call — even if they are the best pattern match. """ from app.services.rider.get_active_riders import ( fetch_active_riders, fetch_created_orders, ) from app.services.core.assignment_service import AssignmentService from app.services.routing.route_optimizer import RouteOptimizer from app.services.routing.empirical_eta_calculator import EmpiricalETACalculator from datetime import datetime, timedelta from dateutil.parser import parse as parse_date import asyncio eta_calculator = EmpiricalETACalculator() try: _t0 = time.time() # wall-clock start for elapsed_ms logging # ── Parse new payload format ───────────────────────────────────────── # {"deliveries": [...], "absent_riders": [{"userid": X}, ...]} # vs legacy flat list [...]. absent_rider_ids: set = set() if isinstance(body, dict): _absent_list = body.get("absent_riders") or [] for _ar in _absent_list: try: _arid = int(_ar.get("userid") or _ar.get("id") or 0) if _arid: absent_rider_ids.add(_arid) except (ValueError, TypeError): pass body = body.get("deliveries") or [] if absent_rider_ids: logger.info( f"[Absent] Excluding {len(absent_rider_ids)} absent riders " f"from this batch: {sorted(absent_rider_ids)}" ) # Accept both ?reshuffle and legacy typo variants in URL for backwards compat q_params = request.query_params do_reshuffle = reshuffle or any( k in q_params for k in ["resuffle", "rehuffle"] ) # 1. Fetch riders and (if body is empty) orders in parallel. if body: riders = await fetch_active_riders() orders = body logger.info(f"[PROCESS] Received {len(orders)} orders from payload.") else: logger.info( "[PROCESS] No payload — fetching riders and orders in parallel." ) riders, orders = await asyncio.gather( fetch_active_riders(), fetch_created_orders(), ) if orders: logger.info(f"[PROCESS] Fetched {len(orders)} created orders from external API.") else: logger.info("[PROCESS] No created orders returned from external API.") fuel_charge = 2.5 base_pay = 0.0 # 2. Validate absent_rider_ids against the live roster _live_ids: set = { int(r.get("userid") or r.get("riderid") or r.get("id") or 0) for r in riders } - {0} if absent_rider_ids: _unknown_absent = absent_rider_ids - _live_ids _confirmed_absent = absent_rider_ids & _live_ids if _unknown_absent: logger.warning( f"[Absent] Rider ID(s) not in active roster (typo?): " f"{sorted(_unknown_absent)} — these will have no effect." ) if _confirmed_absent: logger.info( f"[Absent] Confirmed absent (in roster, now excluded): " f"{sorted(_confirmed_absent)}" ) else: _unknown_absent = set() _confirmed_absent = set() # 3. Log summary after all data is in hand mode_str = "reshuffle" if do_reshuffle else "normal" tuning_str = hypertuning_params if hypertuning_params else "null" logger.info( f"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" f"[API HIT] POST /api/v1/optimization/riderassign\n" f"[CONFIG] Mode: {mode_str.upper()} | Hypertuning: {tuning_str} | Active Riders: {len(riders)}\n" f"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" ) logger.info(f"[API] riderassign ▶ orders={len(orders)} riders={len(riders)} mode={mode_str}") if not orders: return { "code": 200, "details": {}, "message": "No orders found to assign.", "status": True, "meta": {"active_riders_count": len(riders)}, } _faiss_corrected = 0 _faiss_store = None # 2b. FAISS coordinate verification — correct noisy delivery coords # using historically verified rider delivery positions. try: from app.services.vector.faiss_customer_store import get_faiss_store _faiss_store = get_faiss_store() _faiss_corrected = 0 for _order in orders: # Resolve phone number across common field name variants _phone = ( _order.get("customerphonenumber") or _order.get("customerphone") or _order.get("phonenumber") or _order.get("mobile") or _order.get("phone") or _order.get("customer_phone") ) _name = ( _order.get("customername") or _order.get("customer_name") or _order.get("deliveryname") or "" ) try: _raw_lat = float( _order.get("deliverylat") or _order.get("droplat") or 0 ) _raw_lon = float( _order.get("deliverylong") or _order.get("droplon") or 0 ) except (ValueError, TypeError): _raw_lat, _raw_lon = 0.0, 0.0 if not _phone or _raw_lat == 0.0 or _raw_lon == 0.0: continue _v_lat, _v_lon, _corrected = _faiss_store.get_verified_coords( _phone, _name, _raw_lat, _raw_lon ) if _corrected: # Replace delivery coords with verified historical coords _order["deliverylat"] = _v_lat _order["deliverylong"] = _v_lon if "droplat" in _order: _order["droplat"] = _v_lat if "droplon" in _order: _order["droplon"] = _v_lon _order["_coord_source"] = "faiss_verified" _faiss_corrected += 1 else: _order["_coord_source"] = "input" logger.info( f"[FAISS] Verified {_faiss_corrected} out of {len(orders)} orders " f"using historical coordinates (from {_faiss_store.record_count()} stored records)." ) except Exception as _fe: logger.warning(f"[FAISS] Coord verification failed (non-fatal): {_fe}") # ── PHASE 0: FAISS DELIVERY HISTORY — pre-assign well-known routes ──── # For each order look up the rider who covered that kitchen → delivery # area most in the last 30-day CSV. Only pre-assigns when: # (a) that rider wins ≥ 50 % of filtered historical neighbours # (b) that rider is currently active # (c) that rider is not blocked # Remaining orders go to the VRP solver as normal. history_assignments: dict = {} history_pre_assigned_ids: set = set() _history_hits = 0 _KITCHEN_KEYS_PH0 = [ "pickupcustomer", "locationname", "storename", "store_name", "restaurantname", "restaurant_name", "kitchenname", "kitchen_name", "partnername", "partner_name", "tenantname", ] try: from app.services.vector.delivery_history_store import get_delivery_history_store from app.config.rider_preferences import BLOCKED_RIDERS as _BLOCKED_H _hist_store = get_delivery_history_store() if _hist_store.record_count() > 0: # Build set of active, non-blocked, non-absent rider IDs for validation _active_rider_ids: set = set() for _r in riders: try: _rid0 = int(_r.get("userid") or _r.get("riderid") or _r.get("id") or 0) if _rid0 and _rid0 not in _BLOCKED_H and _rid0 not in absent_rider_ids: _active_rider_ids.add(_rid0) except (ValueError, TypeError): pass for _order in orders: # Extract kitchen name _kitchen_h = "" for _kk in _KITCHEN_KEYS_PH0: _kv = _order.get(_kk) if _kv and str(_kv).strip(): _kitchen_h = str(_kv).strip() break try: _h_plat = float(_order.get("pickuplat") or 0) _h_plon = float(_order.get("pickuplon") or _order.get("pickuplong") or 0) _h_dlat = float(_order.get("deliverylat") or _order.get("droplat") or 0) _h_dlon = float(_order.get("deliverylong") or _order.get("droplon") or 0) except (TypeError, ValueError): continue if not _h_dlat: continue # no delivery coords, skip _hit = _hist_store.find_rider( _kitchen_h, _h_plat, _h_plon, _h_dlat, _h_dlon ) if _hit and _hit["userid"] in _active_rider_ids: _h_rid = _hit["userid"] history_assignments.setdefault(_h_rid, []).append(_order) history_pre_assigned_ids.add(id(_order)) _history_hits += 1 logger.info( f"[FAISS History] Pre-assigned {_history_hits}/{len(orders)} orders " f"from delivery history ({_hist_store.record_count()} records). " f"Riders: {list(history_assignments.keys())}" ) else: logger.info("[FAISS History] Store empty — skipping history pre-assignment.") except Exception as _he: logger.warning(f"[FAISS History] Pre-assignment failed (non-fatal): {_he}") # Orders not covered by FAISS history go to the VRP / 2-phase solver. vrp_orders = [o for o in orders if id(o) not in history_pre_assigned_ids] # 3. Run Assignment (AssignmentService) # -- Per-request strategy override (thread-safe via contextvars) -- # Old approach mutated _cfg._cache directly — a race condition when two # concurrent requests set different strategies simultaneously. # New approach: each async task gets its OWN copy of the ContextVar, # so overrides never leak between concurrent requests. from app.config.dynamic_config import get_config, set_request_strategy from app.services.ml.behavior_analyzer import time_band as _tb, load_band as _lb from datetime import datetime as _DT _cfg = get_config() # ── Bandit context (used both for selection + background update) ── _bandit_t_band = _tb(_DT.now().isoformat()) _bandit_l_band = _lb(len(vrp_orders) / max(1, len(riders))) _bandit_context = (_bandit_t_band, _bandit_l_band) valid_strategies = ["balanced", "fuel_saver", "aggressive_speed", "zone_strict"] if hypertuning_params and hypertuning_params in valid_strategies: # Explicit caller override always wins set_request_strategy(hypertuning_params) logger.info(f"[HYPERTUNE] Per-request strategy override: {hypertuning_params}") else: # ── Thompson Sampling: let the bandit choose ────────────────── # The bandit samples from Beta posteriors for this (time, load) # context and picks the arm with highest sample. It naturally # explores when uncertain and exploits known-good strategies as # data accumulates. Replaces the one-size-fits-all greedy picker. try: from app.services.ml.strategy_bandit import get_bandit as _gb _bandit_strategy = _gb().select(_bandit_t_band, _bandit_l_band) set_request_strategy(_bandit_strategy) logger.info( f"[Bandit] Strategy='{_bandit_strategy}' " f"ctx={_bandit_t_band}|{_bandit_l_band}" ) except Exception as _bse: logger.debug(f"[Bandit] Selection failed (non-fatal): {_bse}") optimizer = RouteOptimizer() # ── PHASE 3a: TRUE VRP (primary solver) ────────────────────────────── # Attempt to solve assignment + routing simultaneously with OR-Tools VRP. # This is globally optimal — one model for all riders and all orders. # Falls back to 2-phase if OR-Tools unavailable or solver finds no solution. # ───────────────────────────────────────────────────────────────────── vrp_assignments: dict = {} unassigned_orders: list = [] used_vrp = False if not do_reshuffle: # VRP not meaningful during reshuffle (intentional exploration) try: # Build minimal rider info for VRP from app.services.routing.kalman_filter import smooth_rider_locations _smooth_riders = smooth_rider_locations(list(riders)) from app.services.core.assignment_service import AssignmentService as _AS _svc_tmp = _AS() # Import the single source-of-truth blocked-rider set. from app.config.rider_preferences import BLOCKED_RIDERS as _BLOCKED_RIDERS rider_data_for_vrp = [] for r in _smooth_riders: rid_raw = r.get("userid") or r.get("riderid") or r.get("id") try: rid = int(rid_raw) except (ValueError, TypeError): continue if rid in _BLOCKED_RIDERS or rid in absent_rider_ids: continue lat, lon = _svc_tmp.get_lat_lon(r) if lat == 0 or lon == 0: continue rider_data_for_vrp.append({"id": rid, "lat": lat, "lon": lon}) if rider_data_for_vrp: import math as _math # Hard cap at 12 orders/rider regardless of what ML tunes. # This gives ceil(35/12) = 3 riders for a typical 35-order load, # which matches the real-world target for this fleet. # Preference discounts (below) steer each of those 3 riders # toward their own kitchen's orders — so cap=12 is safe. _max_opr = int(_cfg.get("max_orders_per_rider", 12)) _vrp_cap = min(_max_opr, 12) # never let ML push this above 12 logger.info( f"[VRP] Capacity: {_vrp_cap}/rider | " f"{len(vrp_orders)} orders (of {len(orders)} total, " f"{_history_hits} pre-assigned by FAISS history) / " f"{len(rider_data_for_vrp)} riders → " f"expect ≈{_math.ceil(max(1, len(vrp_orders)) / _vrp_cap)} active riders" ) vrp_result = optimizer.solve_vrp_multi_rider( orders=vrp_orders, # excludes Phase-0 pre-assigned orders rider_data=rider_data_for_vrp, max_orders_per_rider=_vrp_cap, soft_prefs=_svc_tmp.soft_preferences, soft_home=_svc_tmp.home_locations, ) if vrp_result: vrp_assignments = vrp_result # Orders not in any VRP route AND not in Phase-0 → unassigned _vrp_assigned_ids = { id(o) for rider_ords in vrp_assignments.values() for o in rider_ords } unassigned_orders = [ o for o in orders if id(o) not in _vrp_assigned_ids and id(o) not in history_pre_assigned_ids ] used_vrp = True # Removed logging from here, moved to after both solvers run except Exception as _vrp_err: logger.warning(f"[VRP] Primary solver skipped: {_vrp_err}") # ── PHASE 3b: 2-PHASE FALLBACK (cluster → score → assign) ──────────── if not used_vrp: service = AssignmentService() _available_riders = [ r for r in riders if int(r.get("userid") or r.get("riderid") or r.get("id") or 0) not in absent_rider_ids ] if absent_rider_ids else riders vrp_assignments, _fb_unassigned = await service.assign_orders( riders=_available_riders, orders=vrp_orders, # excludes Phase-0 pre-assigned orders fuel_charge=fuel_charge, base_pay=base_pay, reshuffle=do_reshuffle, ) # Unassigned = 2-phase leftovers (history orders are always assigned) unassigned_orders = list(_fb_unassigned) # Merge Phase-0 FAISS history pre-assignments into VRP/fallback result for _h_rid, _h_ords in history_assignments.items(): if _h_rid in vrp_assignments: # Prepend history orders (rider knows these routes best) vrp_assignments[_h_rid] = list(_h_ords) + vrp_assignments[_h_rid] else: vrp_assignments[_h_rid] = list(_h_ords) assignments = vrp_assignments # ── SOLO RIDER CONSOLIDATION ───────────────────────────────────────── # A rider deployed for a single order is almost always a net loss. # Rule: if a rider has exactly 1 order and another rider is already # running a route from the same (or nearby) kitchen, transfer the # order there instead of deploying a solo trip. # # Priority for matching a host: # 0 → same pickuplocationid (same kitchen counter, zero extra pickup) # 1 → same kitchen name (logical same kitchen, different ID edge-case) # 2 → pickup within 1.5 km (very close kitchen cluster) # # Among equal-priority hosts: pick the one whose delivery centroid is # nearest to the solo order's drop point (minimises route extension). # # Pass 1: merge solos into multi-order riders (≥ 2 orders). # Pass 2: merge remaining solos with each other (same kitchen only). # # Hard cap: never pile > 10 orders onto one host. # Safety: if no suitable host found, keep the solo assignment as-is # (unassigned is worse than a solo trip). # ───────────────────────────────────────────────────────────────────── _SOLO_MAX_PICKUP_KM = 1.5 _SOLO_ORDER_CAP = 10 _consolidation_moves = 0 def _ord_pickup_locid(o: dict) -> int: try: return int(o.get("pickuplocationid") or 0) except: return 0 def _ord_kitchen(o: dict) -> str: for _k in ("pickupcustomer", "locationname", "tenantname"): _v = o.get(_k, "") if _v: return str(_v).strip().lower() return "" def _ord_pickup_coords(o: dict): try: return (float(o.get("pickuplat") or 0), float(o.get("pickuplon") or o.get("pickuplong") or 0)) except: return (0.0, 0.0) def _delivery_centroid(ords: list): lats = [float(o.get("deliverylat") or o.get("droplat") or 0) for o in ords] lons = [float(o.get("deliverylong") or o.get("droplon") or 0) for o in ords] lats = [x for x in lats if x]; lons = [x for x in lons if x] if not lats: return (0.0, 0.0) return (sum(lats) / len(lats), sum(lons) / len(lons)) # Rider efficiency scores (from 30-day CSV history) used as tiebreaker _rider_efficiency: dict = {} try: from app.services.vector.delivery_history_store import get_delivery_history_store as _gds _rider_efficiency = _gds().get_rider_efficiency_scores() except Exception: pass def _best_host(solo_order: dict, host_pool: dict, min_host_orders: int) -> "tuple | None": """Return (host_rider_id, score) or None if no match. Score tuple: (priority, drop_dist, -efficiency) lower priority = better kitchen match lower drop_dist = less route extension higher efficiency = prefer that rider as tiebreaker """ _s_locid = _ord_pickup_locid(solo_order) _s_kname = _ord_kitchen(solo_order) _s_plat, _s_plon = _ord_pickup_coords(solo_order) try: _s_dlat = float(solo_order.get("deliverylat") or solo_order.get("droplat") or 0) _s_dlon = float(solo_order.get("deliverylong") or solo_order.get("droplon") or 0) except: _s_dlat = _s_dlon = 0.0 _best_rid, _best_score = None, None for _hrid, _hords in host_pool.items(): if len(_hords) < min_host_orders or len(_hords) >= _SOLO_ORDER_CAP: continue _h0 = _hords[0] _h_locid = _ord_pickup_locid(_h0) _h_kname = _ord_kitchen(_h0) _h_plat, _h_plon = _ord_pickup_coords(_h0) _pickup_dist = ( _haversine_km(_s_plat, _s_plon, _h_plat, _h_plon) if _s_plat and _h_plat else 999.0 ) if _s_locid and _h_locid and _s_locid == _h_locid: _pri = 0 elif _s_kname and _h_kname and _s_kname == _h_kname: _pri = 1 elif _pickup_dist <= _SOLO_MAX_PICKUP_KM: _pri = 2 else: continue _centroid = _delivery_centroid(_hords) _drop_dist = ( _haversine_km(_s_dlat, _s_dlon, _centroid[0], _centroid[1]) if _s_dlat and _centroid[0] else 999.0 ) # Efficiency tiebreaker: higher score = better host (negate for min-sort) _eff = _rider_efficiency.get(_hrid, {}).get("efficiency_score", 0.5) _score = (_pri, round(_drop_dist, 2), round(1.0 - _eff, 4)) if _best_score is None or _score < _best_score: _best_score = _score; _best_rid = _hrid return (_best_rid, _best_score) if _best_rid is not None else None # --- Pass 1: solo → multi-order host (≥ 2 existing orders) ---------- _solo_rids = [rid for rid, ords in assignments.items() if len(ords) == 1] _active_pool = {rid: ords for rid, ords in assignments.items() if len(ords) >= 2} for _srid in _solo_rids: if not assignments.get(_srid): continue # already cleared _solo_ord = assignments[_srid][0] _match = _best_host(_solo_ord, _active_pool, min_host_orders=2) if _match: _hrid, _hscore = _match _active_pool[_hrid].append(_solo_ord) assignments[_hrid] = _active_pool[_hrid] assignments[_srid] = [] _consolidation_moves += 1 logger.info( f"[Consolidate] Rider {_srid} (1 order) → merged into rider " f"{_hrid} (now {len(_active_pool[_hrid])} orders, " f"kitchen-priority={_hscore[0]}, drop-dist={_hscore[1]:.1f}km)" ) # --- Pass 2: remaining solos → each other (same kitchen only) -------- _still_solo = [rid for rid in _solo_rids if len(assignments.get(rid, [])) == 1] _solo_pool = {rid: assignments[rid] for rid in _still_solo} _merged_in_p2: set = set() for _srid in _still_solo: if _srid in _merged_in_p2: continue _solo_ord = assignments[_srid][0] # Only same-kitchen merges in pass 2 (pickup_locid or name match) _candidates = { rid: ords for rid, ords in _solo_pool.items() if rid != _srid and rid not in _merged_in_p2 } _match = _best_host(_solo_ord, _candidates, min_host_orders=1) if _match: _hrid, _hscore = _match if _hscore[0] <= 1: # only locid/name matches in pass 2 assignments[_hrid].append(_solo_ord) assignments[_srid] = [] _merged_in_p2.add(_srid) _consolidation_moves += 1 logger.info( f"[Consolidate-P2] Solo {_srid} merged into solo " f"{_hrid} (now {len(assignments[_hrid])} orders, " f"kitchen-priority={_hscore[0]})" ) if _consolidation_moves: logger.info( f"[Consolidate] {_consolidation_moves} solo order(s) merged into " f"existing routes — {len([r for r in _solo_rids if not assignments.get(r)])} " f"rider(s) freed up." ) # Log outcome for whichever solver was used assigned_count = sum(len(v) for v in assignments.values()) solver_name = "VRP Optimal" if used_vrp else "2-Phase Heuristic" logger.info( f"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" f"[SUMMARY] Solver Used: {solver_name}\n" f"[OUTCOME] Assigned {assigned_count} out of {len(orders)} orders.\n" f"[RIDERS] Utilized {len([v for v in assignments.values() if v])} out of {len(riders)} active riders.\n" f"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" ) # No explicit restore needed — contextvars are scoped to this async task. # When the task ends the ContextVar value is automatically discarded. if do_reshuffle: logger.info( "[RESHUFFLE] Retry mode active - exploring alternative rider assignments." ) # 4. Optimize Routes for Each Rider and Flatten Response # (OR-Tools TSP + 2-opt per rider, now with time windows if pickupSlot present) flat_orders_list = [] # Build an ordered list of (rider_id, orders) pairs so we can align # task_contexts with results after the parallel gather. task_items = [ (rider_id, rider_orders) for rider_id, rider_orders in assignments.items() if rider_orders ] task_contexts = [rider_id for rider_id, _ in task_items] total_assigned = 0 if task_items: # ── TRUE PARALLEL TSP via ThreadPoolExecutor ───────────────────── # OR-Tools (C++ extension) releases the GIL during solving, so # threads genuinely run on separate CPU cores at the same time. # asyncio.get_running_loop().run_in_executor schedules each worker # on the thread pool and yields control to the event loop while # the threads compute, keeping FastAPI fully responsive. loop = asyncio.get_running_loop() results = await asyncio.gather(*[ loop.run_in_executor( _route_executor, _sync_optimize_route, rider_orders, ) for _, rider_orders in task_items ]) # Create a lookup for rider details rider_info_map = {} for r in riders: # Use string conversion for robust ID matching r_id = str(r.get("userid") or r.get("_id", "")) if r_id: rider_info_map[r_id] = { "name": r.get("username", ""), "contactno": r.get("contactno", ""), } # Process results matching them back to riders for stored_rider_id, optimized_route in zip(task_contexts, results): r_id_str = str(stored_rider_id) r_info = rider_info_map.get(r_id_str, {}) rider_name = r_info.get("name", "") rider_contact = r_info.get("contactno", "") # Calculate total distance for this rider total_rider_kms = 0 if optimized_route: # Usually the last order has the max cumulative kms if steps are 1..N try: total_rider_kms = max( [float(o.get("cumulativekms", 0)) for o in optimized_route] ) except: total_rider_kms = sum( [ float(o.get("actualkms", o.get("kms", 0))) for o in optimized_route ] ) for order in optimized_route: order["userid"] = stored_rider_id order["username"] = rider_name # Populate the specific fields requested by the user order["rider"] = rider_name order["ridercontactno"] = rider_contact order["riderkms"] = str(round(total_rider_kms, 2)) # --- DYNAMIC ETA COMPUTATION ----------------------------- # Try various cases and names for pickup slot pickup_slot_str = ( order.get("pickupSlot") or order.get("pickupslot") or order.get("pickup_slot") or order.get("pickuptime") ) if pickup_slot_str: try: # Robust date parsing (handles almost any format magically) pickup_time = parse_date(str(pickup_slot_str)) # Use cumulative_eta (total time from kitchen → this stop) # if the route optimizer produced it; fall back to # a fresh per-leg calculation using cumulativekms. if order.get("cumulative_eta"): eta_mins = int(order["cumulative_eta"]) else: dist_km = float( order.get("cumulativekms") or order.get("actualkms", order.get("kms", 0)) ) step = int(order.get("step", 1)) order_type = order.get("ordertype", "Economy") from app.services.routing.realistic_eta_calculator import get_time_of_day_category _dcoords = None try: _dlat = float(order.get("deliverylat") or order.get("droplat") or 0) _dlon = float(order.get("deliverylong") or order.get("droplon") or 0) if _dlat and _dlon: _dcoords = (_dlat, _dlon) except (TypeError, ValueError): _dcoords = None eta_mins = eta_calculator.calculate_eta( distance_km=dist_km, is_first_order=(step == 1), order_type=order_type, time_of_day=get_time_of_day_category(), kitchen=order.get("pickupcustomer") or order.get("locationname"), drop_coords=_dcoords, rider_id=order.get("userid"), ) expected_time = pickup_time + timedelta(minutes=eta_mins) # Format output as requested: "2026-03-24 08:25 AM" order["expectedDeliveryTime"] = expected_time.strftime( "%Y-%m-%d %I:%M %p" ) order["transitMinutes"] = eta_mins order["calculationDistanceKm"] = round( float(order.get("cumulativekms") or order.get("actualkms", 0)), 2 ) except Exception as e: logger.warning( f"Could not calculate ETA from pickupSlot '{pickup_slot_str}': {e}" ) # --------------------------------------------------------- flat_orders_list.append(order) total_assigned += len(optimized_route) # The ID3 "risk" tree was retired (it never affected assignment). Key kept # static so existing response consumers don't break. risk_meta: dict = {"label": "n/a", "model_trained": False, "deprecated": True} # ── BACKGROUND ML LOGGING ──────────────────────────────────────────── # Fire-and-forget: log this assignment event to SQLite, auto-retrain # ID3 every 50 events, auto-tune strategy every 100 events. # Uses _ml_executor so the API response is never delayed. try: _elapsed_ms = (time.time() - _t0) * 1000 _hyp_snapshot = get_config().get_all() # frozen copy for this call _ml_executor.submit( _bg_log_and_maybe_retrain, len(orders), len(riders), _hyp_snapshot, {rid: list(ords) for rid, ords in assignments.items()}, len(unassigned_orders), round(_elapsed_ms, 1), _bandit_context, # (time_band, load_band) for RL update ) except Exception as _mle: logger.debug(f"[ML BG] Submit failed (non-fatal): {_mle}") # ── Compact exit trace (grep one request_id to see the whole request) ── try: from app.services.routing.delivery_history_service import get_delivery_history_service _road_on = bool(get_config().get("routing_use_road_distance", False)) _eta_src = ( "empirical" if (get_config().get("eta_empirical_enabled", True) and get_delivery_history_service().has_data()) else "formula" ) logger.info( f"[API] riderassign ◀ solver={'vrp_optimal' if used_vrp else '2phase'} " f"assigned={total_assigned}/{len(orders)} unassigned={len(unassigned_orders)} " f"riders_used={len([v for v in assignments.values() if v])}/{len(riders)} " f"road_seq={'on' if _road_on else 'off'} eta={_eta_src} " f"elapsed={round(_elapsed_ms)}ms" ) except Exception: pass # 5. Zone Processing from app.services.routing.zone_service import ZoneService zone_service = ZoneService() zone_data = zone_service.group_by_zones( flat_orders_list, unassigned_orders, fuel_charge=fuel_charge, base_pay=base_pay, ) zones_structure = zone_data["detailed_zones"] zone_analysis = zone_data["zone_analysis"] return { "code": 200, "zone_summary": zone_analysis, # High-level zone metrics "zones": zones_structure, # Detailed data "details": flat_orders_list, # Flat list "message": "Success", "status": True, "meta": { "total_orders": len(orders), "utilized_riders": len([rid for rid, rl in assignments.items() if rl]), "active_riders_pool": len(riders), "assigned_orders": total_assigned, "unassigned_orders": len(unassigned_orders), "total_profit": round(sum(z["total_profit"] for z in zone_analysis), 2), "fuel_charge_base": fuel_charge, "unassigned_details": [ { "orderid": o.get("orderid") or o.get("_id"), "reason": o.get( "unassigned_reason", "Unknown capacity/proximity issue" ), } for o in unassigned_orders ], "distribution_summary": { rid: len(rl) for rid, rl in assignments.items() if rl }, "reshuffle_mode": do_reshuffle, "solver_mode": "vrp_optimal" if used_vrp else "2phase_heuristic", "hypertuning_params": hypertuning_params or "default", "faiss_coord_corrections": _faiss_corrected, "faiss_customer_records": _faiss_store.record_count() if _faiss_store else 0, "faiss_history_preassigned": _history_hits, "solo_consolidations": _consolidation_moves, "absent_riders_excluded": sorted(_confirmed_absent), "absent_riders_unknown": sorted(_unknown_absent), "risk_assessment": risk_meta, }, } except Exception as e: logger.error(f"Error in rider assignment: {e}", exc_info=True) raise HTTPException( status_code=500, detail="Internal server error during assignment" )