Initial commit
This commit is contained in:
408
app/config/dynamic_config.py
Normal file
408
app/config/dynamic_config.py
Normal file
@@ -0,0 +1,408 @@
|
||||
"""
|
||||
Dynamic Configuration - rider-api
|
||||
|
||||
Replaces all hardcoded hyperparameters with DB-backed values.
|
||||
The ML hypertuner writes optimal values here; services read from here.
|
||||
|
||||
Fallback: If DB is unavailable or no tuned values exist, defaults are used.
|
||||
This means zero risk - the system works day 1 with no data.
|
||||
"""
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- DB Path ------------------------------------------------------------------
|
||||
_DB_PATH = os.getenv("ML_DB_PATH", "ml_data/ml_store.db")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-request strategy override (async-safe via contextvars).
|
||||
# Each FastAPI request/asyncio task gets its own copy — no cross-request leaks.
|
||||
# Usage:
|
||||
# set_request_strategy("fuel_saver") → override active for this request
|
||||
# clear_request_strategy() → restore to DB value
|
||||
# ---------------------------------------------------------------------------
|
||||
_strategy_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
|
||||
"strategy_override", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_request_strategy(strategy: Optional[str]) -> None:
|
||||
"""Override ml_strategy for the current async task only (thread-safe)."""
|
||||
_strategy_override.set(strategy)
|
||||
|
||||
|
||||
def clear_request_strategy() -> None:
|
||||
"""Remove the per-request strategy override for the current async task."""
|
||||
_strategy_override.set(None)
|
||||
|
||||
|
||||
# --- Hard Defaults (What the system used before ML) ---------------------------
|
||||
DEFAULTS: Dict[str, Any] = {
|
||||
# System Strategy / Prompt
|
||||
"ml_strategy": "balanced",
|
||||
# AssignmentService
|
||||
"max_pickup_distance_km": 10.0,
|
||||
"max_kitchen_distance_km": 3.0,
|
||||
"max_orders_per_rider": 12,
|
||||
"ideal_load": 6,
|
||||
"workload_balance_threshold": 0.7,
|
||||
"workload_penalty_weight": 100.0,
|
||||
"distance_penalty_weight": 2.0,
|
||||
"preference_bonus": -15.0,
|
||||
"home_zone_bonus_4km": -3.0,
|
||||
"home_zone_bonus_2km": -5.0,
|
||||
"emergency_load_penalty": 3.0, # km penalty per order in emergency assign
|
||||
# RouteOptimizer
|
||||
"search_time_limit_seconds": 5,
|
||||
"avg_speed_kmh": 18.0,
|
||||
"road_factor": 1.3,
|
||||
# ClusteringService
|
||||
"cluster_radius_km": 3.0,
|
||||
# KalmanFilter
|
||||
"kalman_process_noise": 1e-4,
|
||||
"kalman_measurement_noise": 0.01,
|
||||
# RealisticETACalculator
|
||||
"eta_pickup_time_min": 3.0,
|
||||
"eta_delivery_time_min": 4.0,
|
||||
"eta_navigation_buffer_min": 1.5,
|
||||
"eta_short_trip_factor": 0.8, # speed multiplier for dist < 2km
|
||||
"eta_long_trip_factor": 1.1, # speed multiplier for dist > 8km
|
||||
# EmpiricalETACalculator (learned ETAs from actual delivery times)
|
||||
"eta_empirical_enabled": True, # False -> instantly revert to the formula
|
||||
"eta_min_samples": 10, # min history samples a key needs before trust
|
||||
# (backtest on live 14d data: 10 -> MAE 4.85 vs
|
||||
# formula 5.73; 20 -> 5.64. 10 wins on held-out.)
|
||||
"eta_history_days": 14, # rolling window pulled from nearledb
|
||||
"eta_stat": "median", # "median" or "p75" (p75 = more conservative)
|
||||
"eta_sync_interval_hours": 6, # autonomous background sync cadence
|
||||
# Road-aware sequencing (Phase 2). OFF by default: enabling adds a Google
|
||||
# Directions call (cost + latency) to the route hot path. Results are cached.
|
||||
# Only the *visiting order* changes; step/ETA metrics stay aerial-based.
|
||||
"routing_use_road_distance": False, # AGENT-MANAGED (see routing_auto_manage)
|
||||
"routing_road_cache_ttl_seconds": 86400, # road geometry is stable; cache 24h
|
||||
"routing_road_max_stops": 25, # Google distance-matrix practical cap
|
||||
# Autonomous road-sequencing decision agent: measures road-vs-aerial travel
|
||||
# time on real batches and flips routing_use_road_distance on its own.
|
||||
"routing_auto_manage": True, # False -> humans own the flag
|
||||
"routing_auto_enable_gain_pct": 3.0, # enable when mean gain >= this
|
||||
"routing_auto_disable_gain_pct": 1.0, # disable when mean gain < this (hysteresis)
|
||||
"routing_eval_sample_batches": 8, # batches measured per cycle (cost bound)
|
||||
"routing_eval_min_batches": 3, # need >= this evaluated to decide
|
||||
"routing_eval_interval_hours": 24, # decision cadence
|
||||
"routing_eval_days": 14, # window sampled from the local mirror
|
||||
# Learned rider->kitchen affinity (soft steering only; union with curated config).
|
||||
"rider_affinity_enabled": True, # False -> pure curated config
|
||||
"rider_affinity_min_deliveries": 10, # learned owner needs >= this many deliveries
|
||||
"rider_affinity_refresh_hours": 6, # recompute cadence (piggybacks the agent)
|
||||
}
|
||||
|
||||
|
||||
class DynamicConfig:
|
||||
"""
|
||||
Thread-safe, DB-backed configuration store.
|
||||
|
||||
Usage:
|
||||
cfg = DynamicConfig()
|
||||
max_dist = cfg.get("max_pickup_distance_km")
|
||||
all_params = cfg.get_all()
|
||||
"""
|
||||
|
||||
_instance: Optional["DynamicConfig"] = None
|
||||
|
||||
def __new__(cls) -> "DynamicConfig":
|
||||
"""Singleton - one config per process."""
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self._initialized = True
|
||||
self._cache: Dict[str, Any] = {}
|
||||
self._last_loaded: Optional[datetime] = None
|
||||
self._ensure_db()
|
||||
self._load()
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Public API
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""Get a config value. Returns ML-tuned value if available, else default.
|
||||
|
||||
For 'ml_strategy' specifically, a per-request ContextVar override takes
|
||||
precedence so that hypertuning_params requests don't mutate the shared
|
||||
singleton (thread-safe for concurrent FastAPI requests).
|
||||
"""
|
||||
self._maybe_reload()
|
||||
# Per-request strategy override (async-safe, no cross-request leaks)
|
||||
if key == "ml_strategy":
|
||||
override = _strategy_override.get()
|
||||
if override is not None:
|
||||
return override
|
||||
val = self._cache.get(key)
|
||||
if val is not None:
|
||||
return val
|
||||
fallback = default if default is not None else DEFAULTS.get(key)
|
||||
return fallback
|
||||
|
||||
def get_all(self) -> Dict[str, Any]:
|
||||
"""Return all current config values (ML-tuned + defaults for missing keys)."""
|
||||
self._maybe_reload()
|
||||
result = dict(DEFAULTS)
|
||||
result.update(self._cache)
|
||||
return result
|
||||
|
||||
def set(self, key: str, value: Any, source: str = "manual") -> None:
|
||||
"""Write a config value to DB (used by hypertuner)."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(_DB_PATH) or ".", exist_ok=True)
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO dynamic_config (key, value, source, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value=excluded.value,
|
||||
source=excluded.source,
|
||||
updated_at=excluded.updated_at
|
||||
""",
|
||||
(key, json.dumps(value), source, datetime.utcnow().isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
self._cache[key] = value
|
||||
logger.info(f"[DynamicConfig] Set {key}={value} (source={source})")
|
||||
except Exception as e:
|
||||
logger.error(f"[DynamicConfig] Failed to set {key}: {e}")
|
||||
|
||||
def set_bulk(self, params: Dict[str, Any], source: str = "ml_hypertuner") -> None:
|
||||
"""Write multiple config values at once (called after each Optuna study)."""
|
||||
for key, value in params.items():
|
||||
self.set(key, value, source=source)
|
||||
logger.info(f"[DynamicConfig] Bulk update: {len(params)} params from {source}")
|
||||
|
||||
def reset_to_defaults(self) -> None:
|
||||
"""Wipe all ML-tuned values, revert to hardcoded defaults."""
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
conn.execute("DELETE FROM dynamic_config")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
self._cache.clear()
|
||||
logger.warning("[DynamicConfig] Reset to factory defaults.")
|
||||
except Exception as e:
|
||||
logger.error(f"[DynamicConfig] Reset failed: {e}")
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Internal
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _ensure_db(self) -> None:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(_DB_PATH) or ".", exist_ok=True)
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS dynamic_config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
source TEXT DEFAULT 'manual',
|
||||
updated_at TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS kitchen_encoding (
|
||||
kitchen_name TEXT PRIMARY KEY,
|
||||
label_id INTEGER NOT NULL,
|
||||
frequency REAL DEFAULT 0.0,
|
||||
avg_profit REAL DEFAULT 0.0,
|
||||
order_count INTEGER DEFAULT 0,
|
||||
updated_at TEXT
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"[DynamicConfig] DB init failed: {e}")
|
||||
|
||||
def _load(self) -> None:
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
rows = conn.execute("SELECT key, value FROM dynamic_config").fetchall()
|
||||
conn.close()
|
||||
self._cache = {}
|
||||
for key, raw in rows:
|
||||
try:
|
||||
self._cache[key] = json.loads(raw)
|
||||
except Exception:
|
||||
self._cache[key] = raw
|
||||
self._last_loaded = datetime.utcnow()
|
||||
if self._cache:
|
||||
logger.info(
|
||||
f"[DynamicConfig] Loaded {len(self._cache)} ML-tuned params from DB"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"[DynamicConfig] Could not load from DB (using defaults): {e}"
|
||||
)
|
||||
self._cache = {}
|
||||
|
||||
def _maybe_reload(self, interval_seconds: int = 300) -> None:
|
||||
"""Reload from DB every 5 minutes - picks up new tuned params without restart."""
|
||||
if self._last_loaded is None:
|
||||
self._load()
|
||||
return
|
||||
delta = (datetime.utcnow() - self._last_loaded).total_seconds()
|
||||
if delta > interval_seconds:
|
||||
self._load()
|
||||
|
||||
|
||||
# --- Module-level convenience singleton ---------------------------------------
|
||||
_cfg = DynamicConfig()
|
||||
|
||||
|
||||
def get_config() -> DynamicConfig:
|
||||
"""Get the global DynamicConfig singleton."""
|
||||
return _cfg
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DynamicConfig",
|
||||
"get_config",
|
||||
"set_request_strategy",
|
||||
"clear_request_strategy",
|
||||
"get_kitchen_label_id",
|
||||
"get_kitchen_frequency",
|
||||
"update_kitchen_stats",
|
||||
"get_kitchen_avg_profit_smoothed",
|
||||
]
|
||||
|
||||
|
||||
# --- Kitchen Encoding Persistence ---------------------------------------------
|
||||
def get_kitchen_label_id(kitchen_name: str) -> int:
|
||||
"""Get or create persistent label ID for a kitchen."""
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
row = conn.execute(
|
||||
"SELECT label_id FROM kitchen_encoding WHERE kitchen_name = ?",
|
||||
(kitchen_name,),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.close()
|
||||
return row[0]
|
||||
|
||||
max_id = conn.execute(
|
||||
"SELECT COALESCE(MAX(label_id), -1) FROM kitchen_encoding"
|
||||
).fetchone()[0]
|
||||
new_id = max_id + 1
|
||||
conn.close()
|
||||
return new_id
|
||||
except Exception as e:
|
||||
logger.warning(f"[KitchenEncoding] Failed to get label_id: {e}")
|
||||
return hash(kitchen_name.lower().strip()) % 10000
|
||||
|
||||
|
||||
def get_kitchen_frequency(kitchen_name: str) -> float:
|
||||
"""Get frequency ratio for a kitchen from DB."""
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
row = conn.execute(
|
||||
"SELECT frequency FROM kitchen_encoding WHERE kitchen_name = ?",
|
||||
(kitchen_name,),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
return row[0] if row else 0.0
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def update_kitchen_stats(kitchen_name: str, profit: float):
|
||||
"""Update kitchen stats after order completion."""
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
row = conn.execute(
|
||||
"SELECT order_count, avg_profit FROM kitchen_encoding WHERE kitchen_name = ?",
|
||||
(kitchen_name,),
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
count, avg = row
|
||||
new_count = count + 1
|
||||
new_avg = ((avg * count) + profit) / new_count
|
||||
new_freq = new_count / (
|
||||
conn.execute(
|
||||
"SELECT SUM(order_count) FROM kitchen_encoding"
|
||||
).fetchone()[0]
|
||||
or 1
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE kitchen_encoding SET order_count = ?, avg_profit = ?, frequency = ?, updated_at = ? WHERE kitchen_name = ?",
|
||||
(
|
||||
new_count,
|
||||
new_avg,
|
||||
new_freq,
|
||||
datetime.utcnow().isoformat(),
|
||||
kitchen_name,
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"INSERT INTO kitchen_encoding (kitchen_name, label_id, frequency, avg_profit, order_count, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
kitchen_name,
|
||||
get_kitchen_label_id(kitchen_name),
|
||||
1.0,
|
||||
profit,
|
||||
1,
|
||||
datetime.utcnow().isoformat(),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"[KitchenEncoding] Failed to update stats: {e}")
|
||||
|
||||
|
||||
def get_kitchen_avg_profit_smoothed(
|
||||
kitchen_name: str, global_avg: float = 40.0, min_samples: int = 5
|
||||
) -> float:
|
||||
"""
|
||||
Get smoothed average profit for a kitchen using Bayesian smoothing.
|
||||
Reduces noise for kitchens with few orders.
|
||||
|
||||
Formula: smoothed = (kitchen_count * kitchen_avg + min_samples * global_avg) / (kitchen_count + min_samples)
|
||||
|
||||
This means:
|
||||
- Kitchen with many samples -> uses its own avg
|
||||
- Kitchen with few samples -> pulls toward global avg
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(_DB_PATH)
|
||||
row = conn.execute(
|
||||
"SELECT order_count, avg_profit FROM kitchen_encoding WHERE kitchen_name = ?",
|
||||
(kitchen_name,),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
if row and row[0] > 0:
|
||||
count, avg = row
|
||||
if count >= min_samples:
|
||||
return avg
|
||||
# Bayesian smoothing
|
||||
smoothed = ((count * avg) + (min_samples * global_avg)) / (
|
||||
count + min_samples
|
||||
)
|
||||
return smoothed
|
||||
|
||||
return global_avg # Unknown kitchen defaults to global avg
|
||||
except Exception:
|
||||
return global_avg
|
||||
Reference in New Issue
Block a user