Nearle/routesapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new changes in the api

Browse Source
This commit is contained in:
Suriya
2026-07-06 15:15:51 +05:30
parent c742ef0e53
commit 871981035a
43 changed files with 414 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/services/vector/__pycache__/delivery_history_store.cpython-312.pyc
View File

Binary file not shown.

140
app/services/vector/delivery_history_store.py
View File

@@ -1,12 +1,21 @@
"""
Delivery History Store
======================
Pattern-first vector rider lookup using 30-day delivery history.
Pattern-first vector rider lookup using recent delivery history.
Works on any platform — no faiss-cpu dependency.
Data source (config `pattern_source`)
--------------------------------------
"csv" (legacy) — built from delivery_details.csv, only refreshed when a
human re-exports it and calls POST /ml/reload-history.
"db" (default going forward) — built from the nearledb mirror the ETA-sync
agent already maintains (delivery_raw), rebuilt
automatically every sync cycle via rebuild_from_records().
No manual step, no extra database load.
How it works
------------
At startup the CSV is parsed once and two structures are built:
Records (however sourced) are turned into two structures:
1. PATTERN TABLE (primary, O(1) lookup)
City divided into ~1.1 km grid cells (round coords to 2 d.p.).
@@ -65,9 +74,21 @@ CORRECTIONS_PATH = os.getenv("DELIVERY_CORRECTIONS_CSV", "delivery_corrections.c
_CORRECTION_WEIGHT = int(os.getenv("CORRECTION_WEIGHT", "1"))
_STORE_DIR = os.getenv("FAISS_HISTORY_DIR", "ml_data/faiss_history")
_VECTORS_PATH = os.path.join(_STORE_DIR, "delivery_history_vectors.npy")
_RECORDS_PATH = os.path.join(_STORE_DIR, "delivery_history_records.pkl")
_META_PATH = os.path.join(_STORE_DIR, "delivery_history.meta")
def _paths_for(source: str) -> Tuple[str, str, str]:
"""
Disk cache paths, namespaced by source ("csv" or "db"). Namespaced so that
flipping `pattern_source` at runtime (e.g. to debug, or to roll back)
never clobbers the other mode's cached snapshot — each mode keeps its own
independent copy on disk.
"""
suffix = "" if source == "csv" else f".{source}"
return (
os.path.join(_STORE_DIR, f"delivery_history_vectors{suffix}.npy"),
os.path.join(_STORE_DIR, f"delivery_history_records{suffix}.pkl"),
os.path.join(_STORE_DIR, f"delivery_history{suffix}.meta"),
)
# ---------------------------------------------------------------------------
# Thresholds
@@ -78,6 +99,15 @@ _MIN_PATTERN_DOMINANCE = 0.60 # pattern table: rider owns ≥ 60 % of zone
_MIN_PATTERN_VOLUME = 3 # pattern table: at least 3 deliveries in zone
def _pattern_source() -> str:
"""'csv' (legacy, manual refresh) or 'db' (auto-refreshed from nearledb mirror)."""
try:
from app.config.dynamic_config import get_config
return str(get_config().get("pattern_source", "csv"))
except Exception:
return "csv"
# ---------------------------------------------------------------------------
# Pure-NumPy L2 index — drop-in replacement for faiss.IndexFlatL2
# ---------------------------------------------------------------------------
@@ -152,6 +182,7 @@ class DeliveryHistoryStore:
store.pattern_count() -> int
store.get_pattern_stats() -> list
store.reload_from_csv() -> int
store.rebuild_from_records(records) -> int
"""
def __init__(self):
@@ -174,8 +205,21 @@ class DeliveryHistoryStore:
def _load(self) -> None:
os.makedirs(_STORE_DIR, exist_ok=True)
if _pattern_source() == "db":
# DB-driven: the ETA-sync agent keeps this fresh on its own cadence
# by calling rebuild_from_records() after every sync cycle. At
# startup we just load whatever was last persisted — no CSV
# freshness check applies in this mode.
if self._load_from_disk("db"):
return
logger.info(
"[DeliveryHistory] pattern_source=db, no persisted snapshot yet "
"— will populate on the next ETA-sync cycle."
)
return
if self._saved_files_are_current():
if self._load_from_disk():
if self._load_from_disk("csv"):
return
logger.warning(
"[DeliveryHistory] Saved files corrupt — rebuilding from CSV."
@@ -184,14 +228,15 @@ class DeliveryHistoryStore:
records = self._parse_csv()
if not records:
return
self._build_and_save(records)
self._build_and_save(records, source="csv")
def _saved_files_are_current(self) -> bool:
for path in (_VECTORS_PATH, _RECORDS_PATH, _META_PATH):
vectors_path, records_path, meta_path = _paths_for("csv")
for path in (vectors_path, records_path, meta_path):
if not os.path.isfile(path):
return False
try:
with open(_META_PATH, "r", encoding="utf-8") as f:
with open(meta_path, "r", encoding="utf-8") as f:
meta = json.load(f)
if not os.path.isfile(CSV_PATH):
return True
@@ -199,10 +244,11 @@ class DeliveryHistoryStore:
except Exception:
return False
def _load_from_disk(self) -> bool:
def _load_from_disk(self, source: str) -> bool:
try:
vectors = np.load(_VECTORS_PATH) # (N, 4) float32
with open(_RECORDS_PATH, "rb") as f:
vectors_path, records_path, _ = _paths_for(source)
vectors = np.load(vectors_path) # (N, 4) float32
with open(records_path, "rb") as f:
records = pickle.load(f)
if not records or len(vectors) != len(records):
@@ -342,7 +388,7 @@ class DeliveryHistoryStore:
return patterns, zone_index
def _build_and_save(self, records: List[Dict]) -> None:
def _build_and_save(self, records: List[Dict], source: str = "csv") -> None:
vectors = np.array(
[[r["pickuplat"], r["pickuplon"], r["deliverylat"], r["deliverylong"]]
for r in records],
@@ -373,19 +419,28 @@ class DeliveryHistoryStore:
)
try:
np.save(_VECTORS_PATH, vectors)
with open(_RECORDS_PATH, "wb") as f:
vectors_path, records_path, meta_path = _paths_for(source)
np.save(vectors_path, vectors)
with open(records_path, "wb") as f:
pickle.dump(records, f, protocol=pickle.HIGHEST_PROTOCOL)
csv_mtime = os.path.getmtime(CSV_PATH) if os.path.isfile(CSV_PATH) else 0
with open(_META_PATH, "w", encoding="utf-8") as f:
# csv_mtime only means something for the CSV path's freshness check
# (_saved_files_are_current). DB-built snapshots are refreshed by the
# ETA-sync agent's own schedule, not a file-mtime comparison.
csv_mtime = (
os.path.getmtime(CSV_PATH)
if source == "csv" and os.path.isfile(CSV_PATH)
else 0
)
with open(meta_path, "w", encoding="utf-8") as f:
json.dump({
"source": source,
"csv_mtime": csv_mtime,
"record_count": len(records),
"pattern_zones": len(patterns),
"clear_patterns": clear,
}, f, indent=2)
logger.info(
f"[DeliveryHistory] Saved to '{_STORE_DIR}'. "
f"[DeliveryHistory] Saved to '{_STORE_DIR}' (source={source}). "
"Next startup loads from disk."
)
except Exception as e:
@@ -665,7 +720,34 @@ class DeliveryHistoryStore:
records = self._parse_csv() # already merges corrections internally
if not records:
return 0
self._build_and_save(records)
self._build_and_save(records, source="csv")
return len(records)
def rebuild_from_records(self, records: List[Dict]) -> int:
"""
Rebuild the pattern table + vector index directly from pre-shaped
records (kitchen/pickuplat/pickuplon/deliverylat/deliverylong/userid/
ridername), bypassing CSV parsing entirely.
Called by the ETA-sync agent (delivery_history_service.py) after each
sync cycle when pattern_source=db, so this store stays as fresh as the
nearledb mirror — no manual CSV re-export/reload needed.
"""
if not records:
logger.warning(
"[DeliveryHistory] rebuild_from_records got 0 records — "
"keeping the existing store as-is."
)
return 0
# Merge manually-verified corrections on top, same as the CSV path,
# so the human-override mechanism still works in DB mode.
if os.path.isfile(CORRECTIONS_PATH):
corr = self._parse_csv_file(CORRECTIONS_PATH, label="Corrections CSV")
if corr:
records = records + (corr * _CORRECTION_WEIGHT)
self._build_and_save(records, source="db")
return len(records)
def inject_corrections(self, corrections_path: str = CORRECTIONS_PATH,
@@ -720,14 +802,22 @@ class DeliveryHistoryStore:
self._patterns = patterns
self._zone_index = zone_index
# Save to disk so next restart includes corrections
# Save to disk (under whichever source is currently active) so next
# restart includes corrections
try:
np.save(_VECTORS_PATH, vectors)
with open(_RECORDS_PATH, "wb") as f:
active_source = _pattern_source()
vectors_path, records_path, meta_path = _paths_for(active_source)
np.save(vectors_path, vectors)
with open(records_path, "wb") as f:
pickle.dump(merged, f)
csv_mtime = os.path.getmtime(CSV_PATH) if os.path.isfile(CSV_PATH) else 0
with open(_META_PATH, "w", encoding="utf-8") as f:
json.dump({"csv_mtime": csv_mtime, "record_count": len(merged),
csv_mtime = (
os.path.getmtime(CSV_PATH)
if active_source == "csv" and os.path.isfile(CSV_PATH)
else 0
)
with open(meta_path, "w", encoding="utf-8") as f:
json.dump({"source": active_source, "csv_mtime": csv_mtime,
"record_count": len(merged),
"pattern_count": len(patterns)}, f)
logger.info(
f"[DeliveryHistory] Corrections injected and saved — "