Add MQTT webhook bridge with resilient MQTT connection handling
FastAPI service that subscribes to nearle/# on the MQTT broker and forwards rider status, periodic logs, and profile updates to the Jupiter webhook API. Includes fixes for a failure mode that took the bridge down silently for ~3.8 days in June 2026, during which it served HTTP 200 on / while forwarding zero events: - Retry MQTT connection with backoff instead of letting a single failed connect kill the background thread permanently. - Subscribe inside on_connect so subscriptions are restored after an automatic reconnect; the session is clean, so a reconnect previously came back subscribed to nothing. - Add /healthz returning 503 when MQTT is disconnected, so a monitor can distinguish a live bridge from a dead one. - Replace thread-per-message forwarding with a bounded worker pool, which keeps memory flat when the upstream API stalls on its 5s timeout. - Pin requirements.txt to the versions verified in production; it was unpinned and had already drifted to paho-mqtt 2.1.0. - Set PYTHONUNBUFFERED so container logs are not block-buffered. - Name the compose image nearle-api-image to match the deployed tag. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
3
.gitattributes
vendored
Normal file
3
.gitattributes
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# This code is built and run in a Linux container; keep LF everywhere so a
|
||||
# Windows checkout does not push CRLF back to the server.
|
||||
* text=auto eol=lf
|
||||
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.env
|
||||
*.bak
|
||||
16
Dockerfile
Normal file
16
Dockerfile
Normal file
@@ -0,0 +1,16 @@
|
||||
FROM python:3.9-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Without this, stdout is block-buffered when not a TTY, so `docker logs` lags far
|
||||
# behind reality — during the June outage the "MQTT Connection failed" line did not
|
||||
# surface until long after the failure.
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
31
docker-compose.yml
Normal file
31
docker-compose.yml
Normal file
@@ -0,0 +1,31 @@
|
||||
services:
|
||||
nearle-api:
|
||||
build: .
|
||||
# Pinned so `docker compose up -d --build` rebuilds the SAME tag the running
|
||||
# container was created from (it was started with `docker run`, not compose).
|
||||
# Without this, compose would build `nearleapi-nearle-api` and leave a
|
||||
# confusing second image behind.
|
||||
image: nearle-api-image
|
||||
container_name: nearle_api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- MQTT_BROKER=66.116.225.226
|
||||
- API_STATUS_URL=https://jupiter.nearle.app/live/api/v1/utils/createriderstatus
|
||||
- API_LOGS_URL=https://jupiter.nearle.app/live/api/v1/utils/createriderperiodiclog
|
||||
- API_PROFILE_URL=https://jupiter.nearle.app/live/api/v1/utils/createuserredis
|
||||
# /healthz returns 503 whenever the MQTT side is down, so `docker ps` shows
|
||||
# "unhealthy" instead of the container looking fine while forwarding nothing.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=3).status==200 else 1)\""]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
networks:
|
||||
- nearle-net
|
||||
|
||||
networks:
|
||||
nearle-net:
|
||||
external: true
|
||||
206
main.py
Normal file
206
main.py
Normal file
@@ -0,0 +1,206 @@
|
||||
from fastapi import FastAPI
|
||||
import paho.mqtt.client as mqtt
|
||||
from paho.mqtt.enums import CallbackAPIVersion
|
||||
import json
|
||||
import threading
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
app = FastAPI(title="Nearle MQTT Webhook Bridge API")
|
||||
|
||||
# Configuration
|
||||
MQTT_BROKER = os.getenv("MQTT_BROKER", "66.116.225.226")
|
||||
MQTT_PORT = int(os.getenv("MQTT_PORT", "1883"))
|
||||
MQTT_USER = os.getenv("MQTT_USER", "admin")
|
||||
MQTT_PASS = os.getenv("MQTT_PASS", "Package@321#")
|
||||
MQTT_TOPIC = os.getenv("MQTT_TOPIC", "nearle/#")
|
||||
|
||||
# Define your specific external POST APIs here
|
||||
API_STATUS_URL = os.getenv("API_STATUS_URL", "https://jupiter.nearle.app/live/api/v1/utils/createriderstatus")
|
||||
API_LOGS_URL = os.getenv("API_LOGS_URL", "https://jupiter.nearle.app/live/api/v1/utils/createriderperiodiclog")
|
||||
API_PROFILE_URL = os.getenv("API_PROFILE_URL", "https://jupiter.nearle.app/live/api/v1/utils/createuserredis")
|
||||
|
||||
# Bounded worker pool. Previously every MQTT message spawned its own thread, so a
|
||||
# stall on the Jupiter API (the 5s read timeouts seen in the logs) let threads pile
|
||||
# up without limit. A fixed pool keeps memory flat and applies natural backpressure.
|
||||
FORWARD_WORKERS = int(os.getenv("FORWARD_WORKERS", "16"))
|
||||
_executor = ThreadPoolExecutor(max_workers=FORWARD_WORKERS, thread_name_prefix="forward")
|
||||
|
||||
# Liveness state, exposed on / so a monitor can tell a live bridge from a dead one.
|
||||
_state = {
|
||||
"mqtt_connected": False,
|
||||
"last_connect_at": None,
|
||||
"last_disconnect_at": None,
|
||||
"last_message_at": None,
|
||||
"connect_attempts": 0,
|
||||
"messages_received": 0,
|
||||
}
|
||||
_state_lock = threading.Lock()
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
|
||||
|
||||
# Function to forward data to a specific URL
|
||||
def forward_to_api(target_url: str, payload_str: str):
|
||||
if not target_url:
|
||||
return
|
||||
|
||||
# Determine the content type and prepare data
|
||||
headers = {}
|
||||
|
||||
# Check if the payload is JSON or plain string
|
||||
try:
|
||||
payload_data = json.loads(payload_str)
|
||||
headers['Content-Type'] = 'application/json'
|
||||
|
||||
# We send the exact parsed JSON so the receiving API sees it cleanly
|
||||
data_kwargs = {"json": payload_data}
|
||||
except:
|
||||
# If it's a plain string like "Offline", send as text/plain
|
||||
headers['Content-Type'] = 'text/plain'
|
||||
data_kwargs = {"data": payload_str}
|
||||
|
||||
try:
|
||||
response = requests.post(target_url, headers=headers, timeout=5, **data_kwargs)
|
||||
if response.status_code in (200, 201):
|
||||
print(f"SUCCESS ({response.status_code}) -> {target_url} | Sent: {data_kwargs}", flush=True)
|
||||
else:
|
||||
print(f"WARNING: HTTP {response.status_code} from {target_url} | Response: {response.text}", flush=True)
|
||||
except Exception as e:
|
||||
print(f"ERROR: Failed to send to {target_url} | Exception: {e}", flush=True)
|
||||
|
||||
|
||||
def route_payload(data_type: str, rider_id: str, payload_str: str):
|
||||
payload_str = payload_str.strip()
|
||||
if data_type == "status":
|
||||
# Ensure userid is an integer if possible so Jupiter API doesn't reject it
|
||||
r_id = int(rider_id) if rider_id.isdigit() else rider_id
|
||||
status_json = json.dumps({"userid": r_id, "status": payload_str})
|
||||
forward_to_api(API_STATUS_URL, status_json)
|
||||
elif data_type == "logs":
|
||||
forward_to_api(API_LOGS_URL, payload_str)
|
||||
elif data_type == "profile":
|
||||
forward_to_api(API_PROFILE_URL, payload_str)
|
||||
else:
|
||||
# Ignore other data types or add additional routes here
|
||||
pass
|
||||
|
||||
|
||||
# MQTT Callbacks
|
||||
def on_connect(client, userdata, connect_flags, reason_code, properties=None):
|
||||
if reason_code == 0:
|
||||
with _state_lock:
|
||||
_state["mqtt_connected"] = True
|
||||
_state["last_connect_at"] = _now()
|
||||
# Subscribing here (not once after connect) is the important part: the
|
||||
# session is clean, so every reconnect starts with no subscriptions. Doing
|
||||
# it in on_connect means the bridge resubscribes automatically after any
|
||||
# broker restart or network blip instead of sitting connected and deaf.
|
||||
client.subscribe(MQTT_TOPIC)
|
||||
print(f"MQTT connected to {MQTT_BROKER}:{MQTT_PORT}, subscribed to {MQTT_TOPIC}", flush=True)
|
||||
else:
|
||||
print(f"MQTT connect refused: {reason_code}", flush=True)
|
||||
|
||||
|
||||
def on_disconnect(client, userdata, disconnect_flags, reason_code, properties=None):
|
||||
with _state_lock:
|
||||
_state["mqtt_connected"] = False
|
||||
_state["last_disconnect_at"] = _now()
|
||||
print(f"MQTT disconnected: {reason_code} (will auto-reconnect)", flush=True)
|
||||
|
||||
|
||||
def on_message(client, userdata, msg):
|
||||
try:
|
||||
topic = msg.topic
|
||||
payload = msg.payload.decode()
|
||||
|
||||
with _state_lock:
|
||||
_state["last_message_at"] = _now()
|
||||
_state["messages_received"] += 1
|
||||
|
||||
# Topic structure: nearle/riders/{id}/{data_type}/...
|
||||
parts = topic.split('/')
|
||||
if len(parts) >= 4 and parts[0] == "nearle" and parts[1] == "riders":
|
||||
rider_id = parts[2]
|
||||
data_type = parts[3]
|
||||
|
||||
# Hand off to the pool so the MQTT network loop is never blocked
|
||||
_executor.submit(route_payload, data_type, rider_id, payload)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing MQTT message: {e}", flush=True)
|
||||
|
||||
|
||||
def start_mqtt():
|
||||
# This loop is why the bridge no longer dies silently. The old version called
|
||||
# connect() inside a try/except: one failure ("No route to host" on 2026-06-10)
|
||||
# killed the thread for good while uvicorn kept answering 200 OK on /, so the
|
||||
# bridge looked healthy while forwarding nothing until someone restarted it.
|
||||
backoff = 5
|
||||
while True:
|
||||
try:
|
||||
with _state_lock:
|
||||
_state["connect_attempts"] += 1
|
||||
|
||||
client = mqtt.Client(CallbackAPIVersion.VERSION2)
|
||||
client.username_pw_set(MQTT_USER, MQTT_PASS)
|
||||
client.on_connect = on_connect
|
||||
client.on_disconnect = on_disconnect
|
||||
client.on_message = on_message
|
||||
client.reconnect_delay_set(min_delay=1, max_delay=60)
|
||||
|
||||
print(f"Connecting to MQTT Broker at {MQTT_BROKER}...", flush=True)
|
||||
|
||||
# connect_async + retry_first_connection means a broker that is down at
|
||||
# startup is retried instead of being a fatal error.
|
||||
client.connect_async(MQTT_BROKER, MQTT_PORT, keepalive=60)
|
||||
backoff = 5
|
||||
client.loop_forever(retry_first_connection=True)
|
||||
|
||||
print("MQTT loop returned unexpectedly; restarting.", flush=True)
|
||||
except Exception as e:
|
||||
print(f"MQTT loop crashed: {e} | retrying in {backoff}s", flush=True)
|
||||
finally:
|
||||
with _state_lock:
|
||||
_state["mqtt_connected"] = False
|
||||
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 60)
|
||||
|
||||
|
||||
# Start MQTT in a background thread
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
thread = threading.Thread(target=start_mqtt, daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
# simple health check endpoint
|
||||
@app.get("/")
|
||||
def read_root():
|
||||
with _state_lock:
|
||||
snapshot = dict(_state)
|
||||
return {
|
||||
"status": "Nearle Webhook Bridge is running",
|
||||
"description": "Listening to MQTT and forwarding to external POST APIs.",
|
||||
"mqtt": snapshot,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz():
|
||||
"""Returns 200 only when the MQTT side is actually connected, so an external
|
||||
monitor can detect a bridge that is up but not receiving anything."""
|
||||
with _state_lock:
|
||||
connected = _state["mqtt_connected"]
|
||||
snapshot = dict(_state)
|
||||
from fastapi.responses import JSONResponse
|
||||
return JSONResponse(
|
||||
status_code=200 if connected else 503,
|
||||
content={"healthy": connected, "mqtt": snapshot},
|
||||
)
|
||||
7
requirements.txt
Normal file
7
requirements.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
# Pinned to the versions verified running in production on 2026-07-27.
|
||||
# Previously unpinned, so every rebuild silently pulled whatever was latest.
|
||||
fastapi==0.128.8
|
||||
uvicorn==0.39.0
|
||||
paho-mqtt==2.1.0
|
||||
requests==2.32.5
|
||||
pydantic==2.12.5
|
||||
Reference in New Issue
Block a user