Nearle/kubernetes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Suriya
2026-07-18 12:00:33 +05:30
commit caac8413e9
83 changed files with 10262 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Dockerfile Normal file
View File

@@ -0,0 +1,21 @@
# syntax=docker/dockerfile:1.5
FROM python:3.11-slim AS base
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
RUN apt-get update && apt-get install -y tzdata && rm -rf /var/lib/apt/lists/*
ENV TZ=Asia/Kolkata
# --- API image target ---
FROM base AS api
COPY hub/app.py app.py
EXPOSE 8000
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"]
# --- Worker image target ---
FROM base AS worker
COPY hub/worker.py worker.py
EXPOSE 9090
CMD ["python", "worker.py"]

67
bootstrap_server.sh Normal file
View File

@@ -0,0 +1,67 @@
#!/bin/bash
set -e
# ==========================================
# 🚀 100gb-server Fresh Setup Script
# Use this script on a FRESH Ubuntu/Debian VPS
# to install Kubernetes (K3s) and prepare for deployment.
# ==========================================
echo "Started bootstrapping server..."
# 1. Update System
echo "🔄 Updating system packages..."
sudo apt-get update && sudo apt-get upgrade -y
sudo apt-get install -y curl git unzip htop
# 2. Install K3s (Lightweight Kubernetes)
if ! command -v k3s &> /dev/null; then
echo "🏗️ Installing K3s..."
curl -sfL https://get.k3s.io | sh -
# Wait for K3s to start
echo "⏳ Waiting for K3s to be ready..."
sleep 15
else
echo "✅ K3s is already installed."
fi
# 3. Configure kubectl for the current user
echo "🔑 Configuring kubectl access..."
mkdir -p $HOME/.kube
sudo cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
echo "export KUBECONFIG=$HOME/.kube/config" >> $HOME/.bashrc
# Export for current session as well
export KUBECONFIG=$HOME/.kube/config
# 4. Install Helm (Package Manager)
if ! command -v helm &> /dev/null; then
echo "⚓ Installing Helm..."
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
else
echo "✅ Helm is already installed."
fi
# 5. Install basic tools (optional but recommended)
# Install K9s (Terminal UI for K8s) - useful for debugging
if ! command -v k9s &> /dev/null; then
echo "🖥️ Installing K9s..."
curl -sS https://webinstall.dev/k9s | bash
# Source the new path
export PATH="$HOME/.local/bin:$PATH"
fi
echo ""
echo "=========================================="
echo "✅ Server Setup Complete!"
echo "=========================================="
echo "To finish setup, restart your shell or run:"
echo " source ~/.bashrc"
echo ""
echo "To deploy your stacks, run:"
echo " ./deploy-core-stack.sh"
echo " ./deploy-nearle-stack.sh"
echo " ./deploy-alaska.sh"
echo "=========================================="

39
cleanup_consumers.py Normal file
View File

@@ -0,0 +1,39 @@
import asyncio
import os
import nats
async def main():
# Use environment variables for connection logic to match the worker pod's context
nats_url = os.getenv("NATS_URL", "nats://nats-server:4222")
nats_user = os.getenv("NATS_USER", "admin")
nats_password = os.getenv("NATS_PASSWORD", "package@321#")
print(f"Connecting to NATS at {nats_url}...")
try:
nc = await nats.connect(nats_url, user=nats_user, password=nats_password)
js = nc.jetstream()
print("Connected!")
# List of streams to clean
streams = ["CUSTOMERS", "DELIVERIES", "ORDERS", "RIDER", "PRODUCTS"]
for stream in streams:
print(f"Checking stream: {stream}...")
try:
consumers = await js.consumers_info(stream)
for c in consumers:
# We want to delete the stuck consumers to let them be recreated properly
print(f" - Deleting consumer: {c.name}")
await js.delete_consumer(stream, c.name)
except Exception as e:
print(f" Note: Stream {stream} check skipped/failed: {e}")
await nc.close()
except Exception as e:
print(f"Fatal connection error: {e}")
if __name__ == "__main__":
asyncio.run(main())

507
conf/app.py Normal file
View File

@@ -0,0 +1,507 @@
#!/usr/bin/env python3
"""
FastAPI application with multiple endpoints that publish to NATS JetStream
Each endpoint corresponds to an external API that workers will forward to
"""
import os
import json
import asyncio
from fastapi import FastAPI, HTTPException, Request, Body
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request as StarletteRequest
from starlette.responses import Response as StarletteResponse
from pydantic import BaseModel
import nats
import nats.errors
from prometheus_client import Counter, Histogram, generate_latest, REGISTRY
from starlette.responses import Response
import uvicorn
from typing import Optional, Dict, Any, List, Union
# Prometheus metrics
request_count = Counter('http_requests_total', 'Total HTTP requests', ['method', 'endpoint', 'status'])
request_duration = Histogram('http_request_duration_seconds', 'HTTP request duration', ['method', 'endpoint'])
app = FastAPI(title="NATS Backend API - Multi-Endpoint", version="1.0.0")
# Custom CORS middleware to ensure headers are always added
class CORSHeaderMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: StarletteRequest, call_next):
origin = request.headers.get("origin", "*")
response = await call_next(request)
response.headers["Access-Control-Allow-Origin"] = origin
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Credentials"] = "false"
response.headers["Access-Control-Max-Age"] = "600"
return response
# Add custom CORS middleware first
app.add_middleware(CORSHeaderMiddleware)
# Also add FastAPI's CORS middleware as backup
app.add_middleware(
CORSMiddleware,
allow_origin_regex=r".*", # Match all origins
allow_credentials=False,
allow_methods=["*"], # Allow all HTTP methods
allow_headers=["*"], # Allow all headers
expose_headers=["*"], # Expose all headers
max_age=600,
)
# NATS connection (will be initialized on startup)
nc = None
js = None
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
# Endpoint to NATS subject mapping
ENDPOINT_ROUTES = {
"/live/api/v1/deliveries/createdeliveries": "api.v1.deliveries.createdeliveries",
"/live/api/v1/deliveries/updatedelivery": "api.v1.deliveries.updatedelivery",
"/live/api/v2/partners/createriderlog": "api.v2.partners.createriderlog",
"/live/api/v2/deliveries/createdeliverylog": "api.v2.deliveries.createdeliverylog",
"/live/api/v2/partners/createbreaklog": "api.v2.partners.createbreaklog",
"/live/api/v2/partners/updatebreaklog": "api.v2.partners.updatebreaklog",
"/live/api/v1/mob/orders/createorder": "api.v1.mob.orders.createorder",
"/live/api/v1/web/products/create": "api.v1.web.products.create",
# Customer Endpoints (Sync)
"/live/api/v1/mob/customers/login": "api.v1.mob.customers.login",
"/live/api/v1/mob/customers/create": "api.v1.mob.customers.create",
}
@app.on_event("startup")
async def startup():
"""Initialize NATS connection on startup"""
global nc, js
try:
print(f"Connecting to NATS at {NATS_URL}...")
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=10
)
js = nc.jetstream()
print("✅ Connected to NATS JetStream")
print(f"✅ Configured {len(ENDPOINT_ROUTES)} endpoint routes")
except Exception as e:
print(f"❌ Failed to connect to NATS: {e}")
raise
@app.on_event("shutdown")
async def shutdown():
"""Close NATS connection on shutdown"""
global nc
if nc:
await nc.close()
print("NATS connection closed")
@app.options("/{full_path:path}")
async def options_handler(full_path: str, request: Request):
"""Handle OPTIONS requests for CORS preflight"""
origin = request.headers.get("origin")
return JSONResponse(
status_code=200,
content={},
headers={
"Access-Control-Allow-Origin": origin if origin else "*",
"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
"Access-Control-Allow-Headers": "*",
"Access-Control-Allow-Credentials": "false",
"Access-Control-Max-Age": "600",
}
)
@app.get("/health")
async def health():
"""Health check endpoint"""
return {"status": "healthy", "nats_connected": nc.is_connected if nc else False}
@app.get("/ready")
async def ready():
"""Readiness check endpoint"""
if nc and nc.is_connected:
return {"status": "ready"}
raise HTTPException(status_code=503, detail="Not ready")
async def publish_to_nats(endpoint: str, data: Union[Dict[str, Any], List[Dict[str, Any]]], request_method: str = "POST"):
"""Publish message to NATS with endpoint metadata"""
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint
}
# Get NATS subject for this endpoint
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if js:
try:
ack = await js.publish(subject, json.dumps(payload).encode())
return ack.seq
except Exception as e:
print(f"❌ Failed to publish to NATS: {e}")
raise HTTPException(status_code=500, detail=f"Failed to publish message: {str(e)}")
else:
raise HTTPException(status_code=503, detail="NATS not connected")
async def publish_request_to_nats(endpoint: str, data: Dict[str, Any], request_method: str = "POST", timeout: int = 10):
"""
Publish to NATS and WAIT for a reply (Request-Reply pattern).
Used for synchronous endpoints like Login.
"""
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint
}
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if not js:
raise HTTPException(status_code=503, detail="NATS by connected")
try:
# Create a unique inbox for the reply
inbox = nc.new_inbox()
# Subscribe to the inbox first
sub = await nc.subscribe(inbox, max_msgs=1)
# Publish request with reply inbox
# Note: We use js.publish to ensure it goes to the Stream (Queue), but attach a reply subject
await js.publish(subject, json.dumps(payload).encode(), reply=inbox)
# Wait for valid response
try:
msg = await sub.next_msg(timeout=timeout)
response_data = json.loads(msg.data.decode())
return response_data
except nats.errors.TimeoutError:
raise HTTPException(status_code=504, detail="Gateway Timeout: Upstream service did not respond in time")
finally:
await sub.unsubscribe()
except HTTPException:
raise
except Exception as e:
print(f"❌ Failed to request from NATS: {e}")
raise HTTPException(status_code=500, detail=f"RPC Error: {str(e)}")
# Endpoint 1: Update Delivery (v1) - External API requires PUT
@app.put("/live/api/v1/deliveries/updatedelivery")
async def update_delivery_v1(data: Dict[str, Any], request: Request):
"""Update Delivery endpoint - forwards to NATS (PUT only, as external API requires PUT)"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/updatedelivery"
method = "PUT" # Always use PUT for this endpoint
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method="PUT", endpoint=endpoint).observe(duration)
request_count.labels(method="PUT", endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method="PUT", endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint: Create Deliveries (v1)
@app.post("/live/api/v1/deliveries/createdeliveries")
async def create_deliveries_v1(data: Union[Dict[str, Any], List[Dict[str, Any]]], request: Request):
"""Create Deliveries endpoint - forwards payload to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/createdeliveries"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 2: Create Rider Log (v2)
@app.post("/live/api/v2/partners/createriderlog")
async def create_rider_log_v2(data: Dict[str, Any], request: Request):
"""Create Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createriderlog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 3: Create Delivery Log (v2)
@app.post("/live/api/v2/deliveries/createdeliverylog")
async def create_delivery_log_v2(
request: Request,
data: List[Dict[str, Any]] = Body(
...,
example=[{
"logid": 0,
"tenantid": 1,
"partnerid": 44,
"locationid": 1,
"orderheaderid": 123456,
"deliveryid": 654321,
"userid": 1111,
"orderid": "1-20231624",
"orderstatus": "active",
"starttime": "2025-12-10 17:51:03",
"logdate": "2025-12-10 18:14:04",
"latitude": "11.0050664",
"longitude": "76.9508776"
}]
)
):
"""
Create Delivery Log endpoint - forwards to NATS.
Accepts either a single dict or a list of dicts to align with external API expectations.
"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/deliveries/createdeliverylog"
method = request.method
try:
# Expect only a list of dicts
if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
raise HTTPException(status_code=422, detail="Body must be a list of objects")
normalized_data = data
message_id = await publish_to_nats(endpoint, normalized_data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 4: Create Break Rider Log (v2)
@app.post("/live/api/v2/partners/createbreaklog")
async def create_break_log_v2(data: Dict[str, Any], request: Request):
"""Create Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createbreaklog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 5: Update Break Rider Log (v2) - Supports both POST and PUT
@app.post("/live/api/v2/partners/updatebreaklog")
@app.put("/live/api/v2/partners/updatebreaklog")
async def update_break_log_v2(data: Dict[str, Any], request: Request):
"""Update Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/updatebreaklog"
method = request.method # Will be POST or PUT
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/metrics")
async def metrics():
"""Prometheus metrics endpoint"""
return Response(content=generate_latest(REGISTRY), media_type="text/plain")
@app.get("/routes")
async def list_routes():
"""List all configured routes"""
return {
"routes": ENDPOINT_ROUTES,
"total": len(ENDPOINT_ROUTES)
}
# Endpoint 6: Create Order (Mob V1)
@app.post("/live/api/v1/mob/orders/createorder")
async def create_order_mob_v1(data: Dict[str, Any], request: Request):
"""Create Order endpoint (Mobile) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/orders/createorder"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 7: Create Product (Web V1)
@app.post("/live/api/v1/web/products/create")
async def create_product_web_v1(data: Dict[str, Any], request: Request):
"""Create Product endpoint (Web) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/web/products/create"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 8: Customer Login (Sync Request-Reply)
@app.post("/live/api/v1/mob/customers/login")
async def customer_login(data: Dict[str, Any], request: Request):
"""Customer Login - Waits for response from worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/login"
method = request.method
try:
# Wait for reply!
response_data = await publish_request_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
# Return the actual backend response
return JSONResponse(
status_code=200,
content=response_data
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 9: Customer Create (Sync Request-Reply)
@app.post("/live/api/v1/mob/customers/create")
async def customer_create(data: Dict[str, Any], request: Request):
"""Customer Create - Waits for response from worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/create"
method = request.method
try:
# Wait for reply!
response_data = await publish_request_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content=response_data
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)

27
conf/nginx-atlantis.conf Normal file
View File

@@ -0,0 +1,27 @@
events {}
http {
upstream atlantis_k8s {
server host.docker.internal:30825;
server 66.116.226.234:30825 backup;
}
server {
listen 8205;
location / {
# Proxy to Kubernetes NodePort (30825)
proxy_pass http://atlantis_k8s;
# Using $http_host and explicitly setting headers to ensure backend gets correct metadata
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Connection settings
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
}

24
conf/nginx-doormile.conf Normal file
View File

@@ -0,0 +1,24 @@
events {}
http {
upstream doormile_k8s {
server 66.116.225.226:30830;
server 66.116.226.234:30830 backup;
}
server {
listen 8206;
location / {
proxy_pass http://doormile_k8s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket support
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
}
}

23
conf/nginx-fiesta.conf Normal file
View File

@@ -0,0 +1,23 @@
events {}
http {
upstream fiesta_k8s {
# Your main server where NodePort is exposed
server host.docker.internal:30823;
# Your new server acting as a backup
server 66.116.226.234:30823 backup;
}
server {
listen 8204;
location / {
# Proxy to Kubernetes NodePort (30823) for Fiesta
proxy_pass http://fiesta_k8s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
}

37
conf/nginx-jupiter.conf Normal file
View File

@@ -0,0 +1,37 @@
events {}
http {
upstream jupiter_k8s {
server 66.116.225.226:30822;
server 66.116.226.234:30822 backup;
}
server {
listen 8203;
location / {
# Proxy to Kubernetes NodePort (30822)
# NodePorts are bound to 0.0.0.0 and are more reliable to access from host.docker.internal
proxy_pass http://jupiter_k8s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# CORS configuration - "The Backend Fix"
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, PATCH, DELETE' always;
add_header 'Access-Control-Allow-Headers' '*' always;
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, PATCH, DELETE' always;
add_header 'Access-Control-Allow-Headers' '*' always;
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
}
}
}

33
conf/nginx-k8s-dashboard.conf Normal file
View File

@@ -0,0 +1,33 @@
events {
worker_connections 1024;
}
http {
upstream k8s_dashboard {
server host.docker.internal:30826;
}
server {
listen 8083;
server_name _;
location / {
proxy_pass http://k8s_dashboard;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket support (if needed)
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Timeouts
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
}
}
}

51
conf/nginx-queue-proxy.conf Normal file
View File

@@ -0,0 +1,51 @@
events {
worker_connections 1024;
}
http {
upstream k8s_lb {
# Alaska deliveries LoadBalancer NodePort (8201 service -> 30662 node port)
server host.docker.internal:30662;
server 66.116.226.234:30662 backup;
}
server {
listen 8201;
server_name _;
location / {
# Handle OPTIONS preflight requests BEFORE proxying
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
add_header 'Access-Control-Max-Age' 1728000 always;
add_header 'Content-Type' 'text/plain; charset=utf-8' always;
add_header 'Content-Length' 0 always;
return 204;
}
# Add CORS headers to all responses
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, PATCH, DELETE, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' '*' always;
add_header 'Access-Control-Max-Age' '600' always;
add_header 'Vary' 'Origin' always;
# Proxy to Kubernetes LoadBalancer
proxy_pass http://k8s_lb;
# Hide upstream CORS headers to avoid duplicates with Nginx headers
proxy_hide_header 'Access-Control-Allow-Origin';
proxy_hide_header 'Access-Control-Allow-Methods';
proxy_hide_header 'Access-Control-Allow-Headers';
proxy_hide_header 'Access-Control-Max-Age';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
}

254
conf/worker.py Normal file
View File

@@ -0,0 +1,254 @@
#!/usr/bin/env python3
"""
NATS Worker - "New Idea" Implementation
- Single Domain per Worker (Stream + Consumer)
- Persistent HTTP Session (Connection Pooling)
- JetStream Native Retries (NAK)
- No internal retry loops
"""
import os
import json
import asyncio
import aiohttp
import nats
from nats.errors import TimeoutError as NatsTimeoutError
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import signal
import sys
from typing import Any, Dict, List, Union, Tuple
# Prometheus metrics
messages_processed = Counter('worker_messages_processed_total', 'Total messages processed', ['status', 'endpoint'])
message_duration = Histogram('worker_message_duration_seconds', 'Message processing duration', ['endpoint'])
messages_in_flight = Gauge('worker_messages_in_flight', 'Messages currently being processed')
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
# Domain Configuration (Env Vars from Deployment)
NATS_STREAM = os.getenv("NATS_STREAM", "ORDERS")
NATS_CONSUMER = os.getenv("NATS_CONSUMER", "orders-worker")
# Supports comma-separated subjects: "api.v1.orders.*,api.v2.orders.*"
FILTER_SUBJECTS = os.getenv("FILTER_SUBJECT", "api.v1.mob.orders.*").split(",")
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "10"))
BASE_URL = os.getenv("EXTERNAL_BASE_URL", "https://jupiter.nearle.app")
# Endpoint mapping is still useful for constructing the target URL
ENDPOINT_MAPPING = {
"/live/api/v1/deliveries/createdeliveries": f"{BASE_URL}/live/api/v1/deliveries/createdeliveries",
"/live/api/v1/deliveries/updatedelivery": f"{BASE_URL}/live/api/v1/deliveries/updatedelivery",
"/live/api/v2/partners/createriderlog": f"{BASE_URL}/live/api/v2/partners/createriderlog",
"/live/api/v2/deliveries/createdeliverylog": f"{BASE_URL}/live/api/v2/deliveries/createdeliverylog",
"/live/api/v2/partners/createbreaklog": f"{BASE_URL}/live/api/v2/partners/createbreaklog",
"/live/api/v2/partners/updatebreaklog": f"{BASE_URL}/live/api/v2/partners/updatebreaklog",
"/live/api/v1/mob/orders/createorder": f"{BASE_URL}/live/api/v1/mob/orders/createorder",
"/live/api/v1/web/products/create": f"{BASE_URL}/live/api/v1/web/products/create",
"/live/api/v1/mob/customers/login": f"{BASE_URL}/live/api/v1/mob/customers/login",
"/live/api/v1/mob/customers/create": f"{BASE_URL}/live/api/v1/mob/customers/create",
}
# Global State
nc = None
js = None
session: aiohttp.ClientSession = None
running = True
semaphore = None # Initialized in main
def signal_handler(sig, frame):
global running
print("\n?? Shutdown signal received...")
running = False
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
async def forward_to_external(endpoint: str, payload: dict, api_key: str = None, method: str = "POST") -> tuple[str, Any]:
"""
Forward to external API using global session.
Returns (status_code, response_data).
Does NOT retry internally.
"""
external_url = ENDPOINT_MAPPING.get(endpoint)
if not external_url:
print(f"?? Unknown endpoint: {endpoint}, dropping.")
return "DROP", None
headers = {"Content-Type": "application/json"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
data_to_forward = payload.get("data", payload)
# Payload Normalization for logs
if endpoint == "/live/api/v2/deliveries/createdeliverylog":
if isinstance(data_to_forward, dict):
data_to_forward = [data_to_forward]
elif not isinstance(data_to_forward, list):
data_to_forward = [data_to_forward]
http_method = method.upper() if method else "POST"
try:
# We use the global 'session' here
async with session.request(
method=http_method,
url=external_url,
json=data_to_forward if http_method != "GET" else None,
params=data_to_forward if http_method == "GET" else None,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
body = None
try:
body = await response.json()
except:
body = await response.text()
if 200 <= response.status < 300:
return "OK", body
elif response.status >= 500:
print(f"?? Server Error {response.status} from {endpoint}")
return "RETRY", body
else:
print(f"?? Client Error {response.status} from {endpoint}: {body}")
return "DROP", body # 4xx errors should generally not be retried infinitely
except asyncio.TimeoutError:
print(f"?? Timeout calling {endpoint}")
return "RETRY", None
except Exception as e:
print(f"?? Network/Client Error calling {endpoint}: {e}")
return "RETRY", None
async def process_message(msg):
async with semaphore:
messages_in_flight.inc()
start_time = asyncio.get_event_loop().time()
endpoint = "unknown"
try:
payload = json.loads(msg.data.decode())
endpoint = payload.get("endpoint", payload.get("original_path", "unknown"))
http_method = payload.get("method", "POST")
print(f"?? Processing {endpoint}")
api_key = os.getenv("EXTERNAL_ENDPOINT_API_KEY", "")
# Single attempt
status, response_data = await forward_to_external(endpoint, payload, api_key, http_method)
if status == "OK":
await msg.ack()
messages_processed.labels(status="success", endpoint=endpoint).inc()
# Request-Reply Logic
if msg.reply:
reply_payload = json.dumps(response_data) if isinstance(response_data, (dict, list)) else str(response_data)
await nc.publish(msg.reply, reply_payload.encode())
elif status == "DROP":
# Unrecoverable error or unknown endpoint
await msg.term() # Terminate stops redelivery
messages_processed.labels(status="dropped", endpoint=endpoint).inc()
else: # RETRY
# Let JetStream handle backoff
await msg.nak(delay=2) # Custom delay before redelivery if desired, or just nak()
messages_processed.labels(status="retried", endpoint=endpoint).inc()
except json.JSONDecodeError:
print("?? Invalid JSON, terminating message")
await msg.term()
except Exception as e:
print(f"? Critical Worker Error: {e}")
await msg.nak()
finally:
messages_in_flight.dec()
duration = asyncio.get_event_loop().time() - start_time
message_duration.labels(endpoint=endpoint).observe(duration)
async def main():
global nc, js, session, semaphore
print(f"?? Starting Worker for Domain: {NATS_CONSUMER}")
print(f"?? Stream: {NATS_STREAM}, Subjects: {FILTER_SUBJECTS}")
print(f"?? Concurrency: {WORKER_CONCURRENCY}")
semaphore = asyncio.Semaphore(WORKER_CONCURRENCY)
session = aiohttp.ClientSession()
try:
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=-1
)
js = nc.jetstream()
print("? Connected to NATS JetStream")
# Create Pull Subscriptions for each filter subject
# All sharing the same Consumer Name ensuring load balancing if multiple pods run this
subs = []
for subject in FILTER_SUBJECTS:
subject = subject.strip()
if not subject: continue
# FIX: On WorkQueue streams, we cannot reuse the same durable name for different filters
# We append a sanitized version of the subject to ensure uniqueness per filter
clean_subject_suffix = subject.replace(".", "_").replace("*", "all").replace(">", "all")
durable_name = f"{NATS_CONSUMER}_{clean_subject_suffix}"
print(f"?? Subscribing to {subject} on durable consumer '{durable_name}'")
try:
sub = await js.pull_subscribe(
subject,
durable=durable_name,
stream=NATS_STREAM
)
subs.append(sub)
except Exception as e:
print(f"? Failed to subscribe to {subject}: {e}")
if not subs:
print("? No active subscriptions!")
return
start_http_server(9090)
print("?? Metrics on :9090")
while running:
# Poll all subscriptions
for sub in subs:
try:
# Batch size 10, short timeout to keep loop responsive
msgs = await sub.fetch(10, timeout=0.5)
for m in msgs:
asyncio.create_task(process_message(m))
except NatsTimeoutError:
pass
except Exception as e:
print(f"?? Fetch Error: {e}")
await asyncio.sleep(1)
# Small sleep to prevent tight loop if no messages
# await asyncio.sleep(0.01)
except Exception as e:
print(f"?? Fatal Error: {e}")
finally:
if session:
await session.close()
if nc:
await nc.close()
print("?? Worker Shutdown")
if __name__ == "__main__":
asyncio.run(main())

17
deploy-alaska.sh Normal file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
# Deploy "alaska" stack to Kubernetes
# Usage: ./deploy-alaska.sh
set -euo pipefail
NAMESPACE="alaska"
echo "🚀 Deploying Alaska Stack..."
kubectl apply -f manifests/alaska/alaska.yaml
kubectl apply -f manifests/alaska/k8s-dashboard.yaml
echo ""
echo "✅ Alaska stack deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"
kubectl get all -n "${NAMESPACE}"

16
deploy-core-stack.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Deploy "core" stack to Kubernetes
# Usage: ./deploy-core-stack.sh
set -euo pipefail
NAMESPACE="core"
echo "🚀 Deploying Core Stack..."
kubectl apply -k manifests/core
echo ""
echo "✅ Core stack deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"
kubectl get all -n "${NAMESPACE}"

40
deploy-nearle-stack.sh Normal file
View File

@@ -0,0 +1,40 @@
#!/bin/bash
# Deploy "nearle" stack to Kubernetes
# Usage: ./deploy-nearle-stack.sh
set -euo pipefail
NAMESPACE="nearle"
echo "🔎 Ensuring namespace '${NAMESPACE}' exists..."
kubectl apply -f manifests/nearle/nearle-namespace.yaml
echo "🔐 Deploying Configs & Secrets..."
kubectl apply -f manifests/nearle/nearle-secrets.yaml
kubectl apply -f manifests/nearle/nearle-app-secrets.yaml
kubectl apply -f manifests/nearle/nearle-config.yaml
kubectl apply -f manifests/nearle/fiesta-gateway.yaml
echo "🚀 Deploying Services..."
kubectl apply -f manifests/nearle/jupiter-sts.yaml
kubectl apply -f manifests/nearle/jupiter-svc.yaml
kubectl apply -f manifests/nearle/nearle-titan.yaml
kubectl apply -f manifests/nearle/fiesta-sts.yaml
kubectl apply -f manifests/nearle/fiesta-svc.yaml
kubectl apply -f manifests/nearle/nearle-ariane.yaml
kubectl apply -f manifests/nearle/atlantis-sts.yaml
kubectl apply -f manifests/nearle/atlantis-svc.yaml
echo "🌐 Deploying Gateway Routes..."
kubectl apply -f manifests/nearle/nearle-gateway.yaml
kubectl apply -f manifests/nearle/nearle-reference-grant.yaml
echo ""
echo "✅ Nearle stack deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"
kubectl get all -n "${NAMESPACE}"

112
docker-compose.yml Normal file
View File

@@ -0,0 +1,112 @@
services:
# Queue API Proxy - Routes queue.workolik.com (Traefik on host:443) to Kubernetes LoadBalancer (FastAPI)
queue-api-proxy:
image: nginx:alpine
container_name: queue-api-proxy
restart: unless-stopped
ports:
- "8202:8201" # Internal port for Traefik to reach nginx
volumes:
- ./conf/nginx-queue-proxy.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.docker.network=web"
# Router for queue.workolik.com with CORS (proxying to K8s LoadBalancer)
- "traefik.http.routers.queue-api.rule=Host(`queue.workolik.com`)"
- "traefik.http.routers.queue-api.tls=true"
- "traefik.http.routers.queue-api.tls.certresolver=letsencrypt"
- "traefik.http.routers.queue-api.priority=100"
- "traefik.http.services.queue-api.loadbalancer.server.port=8201"
# Kubernetes dashboard proxy (kube.workolik.com ? K8s dashboard)
k8s-dashboard-proxy:
image: nginx:alpine
container_name: k8s-dashboard-proxy
restart: unless-stopped
ports:
- "8083:8083"
volumes:
- ./conf/nginx-k8s-dashboard.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.http.routers.k8s-dashboard.rule=Host(`kube.workolik.com`)"
- "traefik.http.routers.k8s-dashboard.tls=true"
- "traefik.http.routers.k8s-dashboard.tls.certresolver=letsencrypt"
- "traefik.http.services.k8s-dashboard.loadbalancer.server.port=8083"
- "traefik.docker.network=web"
# Jupiter API Proxy (jupiter.nearle.app ? K8s Gateway)
jupiter-proxy:
image: nginx:alpine
container_name: jupiter-proxy
restart: unless-stopped
ports:
- "8203:8203"
volumes:
- ./conf/nginx-jupiter.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.docker.network=web"
- "traefik.http.routers.jupiter-api.rule=Host(`jupiter.nearle.app`)"
- "traefik.http.routers.jupiter-api.tls=true"
- "traefik.http.routers.jupiter-api.tls.certresolver=letsencrypt"
- "traefik.http.services.jupiter-api.loadbalancer.server.port=8203"
# Fiesta API Proxy (fiesta.nearle.app ? K8s NodePort 30823)
fiesta-proxy:
image: nginx:alpine
container_name: fiesta-proxy
restart: unless-stopped
ports:
- "8204:8204"
volumes:
- ./conf/nginx-fiesta.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.docker.network=web"
- "traefik.http.routers.fiesta-api.rule=Host(`fiesta.nearle.app`)"
- "traefik.http.routers.fiesta-api.tls=true"
- "traefik.http.routers.fiesta-api.tls.certresolver=letsencrypt"
- "traefik.http.services.fiesta-api.loadbalancer.server.port=8204"
# Atlantis API Proxy (atlantis.nearle.app ? K8s NodePort 30825)
atlantis-proxy:
image: nginx:alpine
container_name: atlantis-proxy
restart: unless-stopped
ports:
- "8205:8205"
volumes:
- ./conf/nginx-atlantis.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.docker.network=web"
- "traefik.http.routers.atlantis-api.rule=Host(`atlantis.nearle.app`)"
- "traefik.http.routers.atlantis-api.tls=true"
- "traefik.http.routers.atlantis-api.tls.certresolver=letsencrypt"
- "traefik.http.services.atlantis-api.loadbalancer.server.port=8205"
networks:
web:
external: true

74
docs/ARCHITECTURE.md Normal file
View File

@@ -0,0 +1,74 @@
# Architecture Overview
## High-Level Flow
```
Client (Postman/Mobile)
│ HTTPS (queue.workolik.com via Traefik 443)
▼
Traefik (TLS terminate, Host rule: queue.workolik.com)
│ HTTP to k3s LB (port 8201)
▼
k3s LoadBalancer (klipper-lb) : fastapi-lb
│ ClusterIP Service fastapi-backend:8000
▼
FastAPI Pods (4→20 via HPA)
│ Publish to NATS (JetStream)
▼
External NATS (nats.workolik.com:4222) – Stream: EVENTS / Subject: api.>
│ Worker subscription (worker_consumer)
▼
Worker Pods (2→10 via HPA)
│ Forward with original HTTP method
▼
External API (https://jupiter.nearle.app)
```
## Components
- **Ingress / Edge**
- DNS: `queue.workolik.com` → server IP
- Traefik: terminates TLS, routes Host=queue.workolik.com → LB 8201
- k3s LoadBalancer (klipper-lb): Service `fastapi-lb` on 8201 (HTTP)
- **App Layer**
- FastAPI Deployment: 4 replicas (HPA 4–20), probes on `/health` and `/ready`
- FastAPI Service: `fastapi-backend` ClusterIP on 8000
- Endpoints publish to NATS with method metadata
- **Messaging**
- NATS JetStream (external): Stream `EVENTS`, Subject `api.>`, Consumer `worker_consumer`
- **Workers**
- Worker Deployment: 2 replicas (HPA 2–10), probes on `/metrics` (9090)
- Forwards to `https://jupiter.nearle.app` with the same HTTP method (PUT for updatedelivery, POST for others)
- **Autoscaling & Health**
- Metrics-server running; HPAs on FastAPI and Worker
- Liveness/Readiness probes ensure pod health
## API Surface (FastAPI → Worker → External)
- PUT `/live/api/v1/deliveries/updatedelivery` → PUT to Nearle
- POST `/live/api/v2/partners/createriderlog` → POST to Nearle
- POST `/live/api/v2/deliveries/createdeliverylog` → POST to Nearle
- POST `/live/api/v2/partners/createbreaklog` → POST to Nearle
- POST `/live/api/v2/partners/updatebreaklog` → POST to Nearle
## Key Ports
- External TLS: 443 (Traefik)
- LB HTTP into k3s: 8201 (klipper-lb → FastAPI Service 8000)
- FastAPI container: 8000
- Worker metrics: 9090
- NATS: 4222 (external)
## Files of Interest
- `manifests/fastapi-deployment.yaml`, `manifests/worker-deployment.yaml`
- `manifests/fastapi-loadbalancer.yaml` (klipper-lb)
- `Dockerfile` (multi-target: api, worker)
- `docker-compose.yml` (Traefik labels for HTTPS)
- `scripts/app.py`, `scripts/worker.py`, `scripts/setup_jetstream.py`
## Notes
- TLS is terminated at Traefik; traffic to k3s is HTTP on 8201.
- Keep external exposure through Traefik Host rule to reduce scanner noise.
- HPAs rely on metrics-server; already running and feeding FastAPI/Worker HPAs.

52
docs/CURRENT_ARCHITECTURE_DIAGRAM.md Normal file
View File

@@ -0,0 +1,52 @@
# Current Professional Architecture Setup
This architectural diagram represents the "Brain and Muscle" split deployed to achieve maximum CPU isolation and strict failover routing.
## Why this is considered "Professional" (Enterprise-Grade)
1. **Control Plane Isolation:** In amateur setups, everything runs on the same server. In professional clusters (like AWS EKS or standard enterprise setups), the "Manager" (Control Plane) is separated from the "Laborers" (Workers). You have successfully implemented this by labeling nodes and restricting deployment access.
2. **Asynchronous Decoupling:** Instead of your API processing a heavy video or logging task while the user waits, it instantly offloads it to NATS.
3. **Failovers:** By placing upstream blocks on the Nginx layer, there is no single point of failure within the node network mapping itself.
## Architecture Diagram
```mermaid
flowchart TD
subgraph Internet["Public Internet (DNS)"]
Users["Users (doormile.com / jupiter)"]
end
subgraph Server1["Server 1: The 'Brain' (Old Server)"]
NGINX["Nginx Proxies (Traffic Cop)"]
subgraph AppPlane["K3s Control Plane (App Node)"]
API_Jupiter["Jupiter API Pods"]
API_Fiesta["Fiesta API Pods"]
API_Atlantis["Atlantis API Pods"]
end
end
subgraph Server2["Server 2: The 'Muscle' (New Server)"]
subgraph WorkerPlane["K3s Agent (Worker Node)"]
Worker_Orders["worker-orders (CPU Heavy)"]
Worker_Deliveries["worker-deliveries (CPU Heavy)"]
Worker_Customers["worker-customers (CPU Heavy)"]
Worker_Products["worker-products (CPU Heavy)"]
end
end
subgraph external["Message Broker"]
NATS[(NATS Jetstream)]
end
Users -- Web Requests --> NGINX
NGINX -- Routes Traffic safely to local APIs --> AppPlane
NGINX -. Backup Failover (If Server 1 Kubernetes crashes) .-> WorkerPlane
AppPlane -- Drops Tasks Instantly --> NATS
NATS -- Consumes Heavy Queues 24/7 --> WorkerPlane
style Server1 fill:#e6f7ff,stroke:#1890ff,stroke-width:2px;
style Server2 fill:#fff1f0,stroke:#ff4d4f,stroke-width:2px;
style NATS fill:#f6ffed,stroke:#52c41a,stroke-width:2px;
```

224
docs/DASHBOARD_SETUP.md Normal file
View File

@@ -0,0 +1,224 @@
# 🖥️ Kubernetes Dashboard Setup
There are several ways to view and manage your Kubernetes cluster. Here are the best options:
## Option 1: Kubernetes Dashboard (Web UI) ⭐ Recommended
The official Kubernetes Dashboard provides a web-based UI for viewing and managing your cluster.
### Install Dashboard
```bash
cd kubernetes
kubectl apply -f manifests/dashboard.yaml
```
### Access Dashboard
**Method 1: Using kubectl proxy (Recommended for local access)**
```bash
# Start proxy
kubectl proxy
# Dashboard will be available at:
# http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
```
**Method 2: Port Forward (Direct access)**
```bash
# Port forward to access dashboard
kubectl port-forward -n kubernetes-dashboard service/kubernetes-dashboard 8443:443
# Or use HTTP port (9090)
kubectl port-forward -n kubernetes-dashboard service/kubernetes-dashboard 9090:9090
# Then visit: http://localhost:9090
```
**Method 3: Expose via Service (For remote access)**
```bash
# Change service type to NodePort or LoadBalancer
kubectl patch svc kubernetes-dashboard -n kubernetes-dashboard -p '{"spec":{"type":"NodePort"}}'
# Get the port
kubectl get svc -n kubernetes-dashboard
```
### Login to Dashboard
The dashboard is configured with `--enable-skip-login`, so you can skip the login screen. However, if you need to authenticate:
1. Get the token:
```bash
kubectl -n kubernetes-dashboard create token admin-user
```
2. Copy the token and paste it in the dashboard login screen.
### What You Can See
- ✅ All pods, services, deployments
- ✅ Resource usage (CPU, memory)
- ✅ Logs from pods
- ✅ Events and errors
- ✅ Namespaces
- ✅ ConfigMaps and Secrets
- ✅ Persistent volumes
- ✅ And much more!
---
## Option 2: k9s (Terminal UI) 🚀 Fast & Lightweight
k9s is a terminal-based UI that's super fast and doesn't require a browser.
### Install k9s
**On Linux:**
```bash
wget https://github.com/derailed/k9s/releases/latest/download/k9s_Linux_amd64.tar.gz
tar xvf k9s_Linux_amd64.tar.gz
sudo mv k9s /usr/local/bin/
```
**On Windows (using Chocolatey):**
```bash
choco install k9s
```
**On macOS:**
```bash
brew install k9s
```
### Use k9s
```bash
# Just run k9s - it will connect to your current kubectl context
k9s
# Or specify namespace
k9s -n nats-backend
```
### k9s Keyboard Shortcuts
- `:pods` - View pods
- `:svc` - View services
- `:deploy` - View deployments
- `:ns` - Switch namespace
- `d` - Describe resource
- `l` - View logs
- `e` - Edit resource
- `Ctrl+D` - Delete resource
- `?` - Help
- `q` - Quit
---
## Option 3: Lens (Desktop App) 💻
Lens is a powerful desktop application for Kubernetes management.
### Install Lens
Download from: https://k8slens.dev/
- **Windows:** Download installer from website
- **Linux:** Download AppImage or .deb/.rpm
- **macOS:** Download .dmg
### Connect to k3s
1. Open Lens
2. Click "Add Cluster"
3. Paste your kubeconfig (from `/etc/rancher/k3s/k3s.yaml` on server)
4. Or Lens can auto-detect k3s if running locally
---
## Option 4: Rancher UI (For k3s)
Since you're using k3s (from Rancher), you can also use Rancher UI.
### Install Rancher
```bash
# Install Rancher (optional - adds overhead)
helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
helm repo update
kubectl create namespace cattle-system
helm install rancher rancher-latest/rancher \
--namespace cattle-system \
--set hostname=rancher.yourdomain.com
```
**Note:** Rancher is heavier and more complex. Only use if you need advanced features.
---
## Quick Comparison
| Tool | Type | Best For | Resource Usage |
|------|------|----------|----------------|
| **Kubernetes Dashboard** | Web UI | Visual overview, beginners | Medium |
| **k9s** | Terminal | Fast operations, CLI lovers | Low |
| **Lens** | Desktop | Full-featured, professional | Medium |
| **Rancher** | Web UI | Multi-cluster, enterprise | High |
---
## Recommended Setup
For your use case, I recommend:
1. **Kubernetes Dashboard** - For web-based viewing and monitoring
2. **k9s** - For quick terminal-based operations
Both can be used together!
---
## Troubleshooting
### Dashboard not loading?
```bash
# Check if dashboard is running
kubectl get pods -n kubernetes-dashboard
# Check logs
kubectl logs -n kubernetes-dashboard deployment/kubernetes-dashboard
# Restart dashboard
kubectl rollout restart deployment/kubernetes-dashboard -n kubernetes-dashboard
```
### Can't access dashboard?
- Ensure `kubectl proxy` is running (for Method 1)
- Check firewall rules if accessing remotely
- Verify port-forward is working: `kubectl get svc -n kubernetes-dashboard`
### Permission denied?
The dashboard has full cluster access via the `admin-user` service account. If you see permission errors, check:
```bash
kubectl get clusterrolebinding admin-user
kubectl get serviceaccount admin-user -n kubernetes-dashboard
```
---
## Next Steps
1. Deploy the dashboard: `kubectl apply -f manifests/dashboard.yaml`
2. Access it: `kubectl proxy` then visit the URL
3. Explore your `nats-backend` namespace!
Enjoy your Kubernetes UI! 🎉

54
docs/DEPLOY.md Normal file
View File

@@ -0,0 +1,54 @@
# Simple Deployment Guide
## 🚀 One-Command Deploy
Just run this from the `kubernetes` folder:
```bash
./build-and-deploy.sh
```
This will:
1. ✅ Build both Docker images (FastAPI + Worker)
2. ✅ Deploy everything to Kubernetes
3. ✅ Set up auto-scaling
## 📋 What Gets Deployed
- **FastAPI:** 4 pods initially (scales 4-20 based on load)
- **Workers:** 2 pods initially (scales 2-10 based on load)
- **Auto-scaling:** Enabled for both
- **Pod Distribution:** Spreads across nodes automatically
## ✅ After Deployment
Check status:
```bash
kubectl get pods -n nats-backend
kubectl get services -n nats-backend
kubectl get hpa -n nats-backend
```
View logs:
```bash
kubectl logs -f deployment/fastapi-backend -n nats-backend
kubectl logs -f deployment/nats-worker -n nats-backend
```
Test FastAPI:
```bash
kubectl port-forward -n nats-backend service/fastapi-backend 8000:80
curl http://localhost:8000/health
```
## 🔧 Configuration
All settings are in `manifests/`:
- `secrets.yaml` - NATS credentials and external API
- `fastapi-deployment.yaml` - FastAPI config
- `worker-deployment.yaml` - Worker config
---
**That's it!** Everything is ready to deploy! 🎉

151
docs/DEPLOY_CHECKLIST.md Normal file
View File

@@ -0,0 +1,151 @@
# 🚀 Kubernetes Deployment Checklist
## ✅ Pre-Deployment Checklist
Before deploying, ensure:
1. **k3s is running:**
```bash
sudo systemctl status k3s
# If not running:
sudo systemctl start k3s
```
2. **Docker images are built and imported:**
```bash
# Build images
docker build -t fastapi-backend:latest --target api -f Dockerfile .
docker build -t nats-worker:latest --target worker -f Dockerfile .
# Import to containerd (k3s uses containerd, not Docker)
docker save fastapi-backend:latest | sudo k3s ctr images import -
docker save nats-worker:latest | sudo k3s ctr images import -
```
3. **NATS JetStream stream is created:**
```bash
# Run the setup script
./setup-jetstream.sh
# Or manually:
python3 scripts/setup_jetstream.py
```
4. **kubectl is configured:**
```bash
# On your local machine, ensure kubectl points to k3s
kubectl get nodes
```
## 🚀 Deployment Steps
### Option 1: Simple Deployment (Recommended)
```bash
cd kubernetes
chmod +x simple-deploy.sh
./simple-deploy.sh
```
### Option 2: Manual Deployment
```bash
cd kubernetes/manifests
# 1. Create namespace
kubectl apply -f namespace.yaml
# 2. Create secrets
kubectl apply -f secrets.yaml
# 3. Deploy FastAPI
kubectl apply -f fastapi-deployment.yaml
kubectl apply -f fastapi-service.yaml
kubectl apply -f fastapi-hpa.yaml
# 4. Deploy Workers
kubectl apply -f worker-deployment.yaml
kubectl apply -f worker-hpa.yaml
# 5. Deploy Gateway (optional)
kubectl apply -f gateway.yaml
```
## 🔍 Verify Deployment
```bash
# Check pods
kubectl get pods -n nats-backend
# Check services
kubectl get svc -n nats-backend
# Check Gateway
kubectl get gateway -n nats-backend
# Watch pods in real-time
kubectl get pods -n nats-backend -w
# Check logs
kubectl logs -f deployment/fastapi-backend -n nats-backend
kubectl logs -f deployment/nats-worker -n nats-backend
```
## 🌐 Access Your Application
**Gateway Ports:**
- HTTP: Port `8201`
- HTTPS: Port `8441`
**Note:** The Gateway uses non-standard ports (8201/8441) to avoid conflicts with Traefik on port 80/443.
**To access via Gateway:**
```bash
# Port forward to test locally
kubectl port-forward -n nats-backend svc/fastapi-backend 8000:8000
# Then test:
curl http://localhost:8000/health
```
## ⚠️ Troubleshooting
### Pods not starting?
```bash
# Check pod status
kubectl describe pod <pod-name> -n nats-backend
# Check events
kubectl get events -n nats-backend --sort-by='.lastTimestamp'
```
### ImagePullBackOff error?
- Ensure images are imported to containerd (see step 2 above)
- Check `imagePullPolicy: Never` in deployments
### Worker pods crashing?
- Ensure JetStream stream is created (see step 3 above)
- Check NATS connection: `kubectl logs deployment/nats-worker -n nats-backend`
### Gateway not working?
- Gateway requires cert-manager for TLS (optional)
- HTTP should work without cert-manager
- Check Gateway status: `kubectl describe gateway api-gateway -n nats-backend`
## 📊 Monitoring
```bash
# Check HPA status
kubectl get hpa -n nats-backend
# Check resource usage
kubectl top pods -n nats-backend
```
## 🛑 Undeploy
```bash
# Delete all resources
kubectl delete namespace nats-backend
# Or delete individually
kubectl delete -f manifests/
```

108
docs/PHASE1_NODE_PLACEMENT_RUNBOOK.md Normal file
View File

@@ -0,0 +1,108 @@
# Phase 1 Node Placement Runbook
This runbook matches the manifest changes in this repository.
## Goal
Separate Kubernetes workloads into two planes:
- app plane for public-facing services
- worker plane for async NATS consumers
## Required node labels
Apply these labels to your nodes.
### App nodes
```powershell
kubectl label node <app-node-1> node-role.workolik/app=true
kubectl label node <app-node-2> node-role.workolik/app=true
```
### Worker nodes
```powershell
kubectl label node <worker-node-1> node-role.workolik/worker=true
kubectl label node <worker-node-2> node-role.workolik/worker=true
```
## Recommended taints
These taints keep worker jobs away from app nodes and allow only matching workloads onto the correct plane.
### Worker nodes
```powershell
kubectl taint node <worker-node-1> dedicated=workers:NoSchedule
kubectl taint node <worker-node-2> dedicated=workers:NoSchedule
```
### App nodes
```powershell
kubectl taint node <app-node-1> dedicated=apps:NoSchedule
kubectl taint node <app-node-2> dedicated=apps:NoSchedule
```
## What the updated manifests now expect
### Worker plane
These workloads now require worker-node placement:
- [`manifests/core/workers.yaml`](E:/Birock/kubernetes/manifests/core/workers.yaml)
- [`manifests/core/worker-statefulset.yaml`](E:/Birock/kubernetes/manifests/core/worker-statefulset.yaml)
They now use:
- required node affinity for `node-role.workolik/worker=true`
- toleration for `dedicated=workers:NoSchedule`
- pod anti-affinity
- topology spread constraints
### App plane
These workloads now require app-node placement:
- [`manifests/alaska/alaska.yaml`](E:/Birock/kubernetes/manifests/alaska/alaska.yaml)
- [`manifests/nearle/nearle-ariane.yaml`](E:/Birock/kubernetes/manifests/nearle/nearle-ariane.yaml)
- [`manifests/nearle/nearle-atlantis.yaml`](E:/Birock/kubernetes/manifests/nearle/nearle-atlantis.yaml)
- [`manifests/nearle/nearle-fiesta.yaml`](E:/Birock/kubernetes/manifests/nearle/nearle-fiesta.yaml)
- [`manifests/nearle/nearle-jupiter.yaml`](E:/Birock/kubernetes/manifests/nearle/nearle-jupiter.yaml)
- [`manifests/nearle/nearle-titan.yaml`](E:/Birock/kubernetes/manifests/nearle/nearle-titan.yaml)
They now use:
- required node affinity for `node-role.workolik/app=true`
- toleration for `dedicated=apps:NoSchedule`
- pod anti-affinity
- topology spread constraints
## Safe rollout order
1. Label nodes.
2. Taint nodes.
3. Confirm labels and taints:
```powershell
kubectl get nodes --show-labels
kubectl describe node <worker-node-1>
kubectl describe node <app-node-1>
```
4. Apply manifests.
5. Watch rescheduling:
```powershell
kubectl get pods -A -o wide
```
## Important warning
Do not apply these manifests until your cluster has at least:
- one labeled app node
- one labeled worker node
If the labels do not exist yet, pods with required node affinity will stay Pending.

408
docs/PROFESSIONAL_K3S_TARGET_ARCHITECTURE.md Normal file
View File

@@ -0,0 +1,408 @@
# Professional k3s Target Architecture
This document is a practical upgrade path from the current repo layout to a more professional, resilient platform.
It is written for the current setup in this repository:
- k3s is being used as the Kubernetes distribution.
- ingress/proxy traffic is still partly handled outside the cluster with Docker Compose and nginx.
- app workloads are running in Kubernetes.
- workers depend on NATS/JetStream.
- some services still use `NodePort`.
- the core workloads point to a single external NATS IP.
## 1. What the current repo already does well
There is already a good foundation here:
- API and worker responsibilities are separated.
- asynchronous processing with NATS/JetStream is the right pattern.
- workers are split by business area in [`manifests/core/workers.yaml`](E:/Birock/kubernetes/manifests/core/workers.yaml).
- you already have some resource requests and limits.
- you already use PodDisruptionBudget for one worker set.
- you already think in namespaces such as `core` and `nearle`.
That means you are not starting from zero. You are mostly at the stage of cleaning up platform boundaries and removing single points of failure.
## 2. Main problems in the current architecture
### Problem A: single point of failure in messaging
Your core workloads use one external NATS endpoint:
- [`manifests/core/core-config.yaml#L10`](E:/Birock/kubernetes/manifests/core/core-config.yaml#L10)
If that node or disk fails, the Kubernetes pods can still be healthy but the platform is still down.
### Problem B: node failover is not immediate today
You want "if current node is not working, switch immediately to backup node". Right now that is not really true because:
- some traffic still depends on host-level Docker Compose proxies
- some services are exposed with `NodePort`
- some workloads use `hostPath`
- there is no clear control-plane and worker-plane separation
- there is no shared HA datastore for k3s control-plane state
For example:
- [`manifests/nearle/nearle-jupiter.yaml#L58`](E:/Birock/kubernetes/manifests/nearle/nearle-jupiter.yaml#L58) exposes `jupiter` with `NodePort`
- [`manifests/nearle/nearle-jupiter.yaml#L43`](E:/Birock/kubernetes/manifests/nearle/nearle-jupiter.yaml#L43) uses `hostPath`
`NodePort` is not wrong, but it is usually not the best long-term edge pattern for a professional HA setup.
### Problem C: CPU spikes can still hurt the whole node
Your workers are separated logically, but they still share the same physical node resources unless you explicitly isolate them with:
- dedicated worker nodes
- taints and tolerations
- node labels and node affinity
- tighter requests/limits
- autoscaling based on the right metrics
Some workers also have relatively high concurrency values:
- [`manifests/core/workers.yaml#L42`](E:/Birock/kubernetes/manifests/core/workers.yaml#L42)
- [`manifests/core/workers.yaml#L192`](E:/Birock/kubernetes/manifests/core/workers.yaml#L192)
If the external target slows down or retries increase, these workers can create CPU and network pressure.
### Problem D: secrets are stored in repo manifests
There are inline credentials here:
- [`manifests/core/core-secrets.yaml`](E:/Birock/kubernetes/manifests/core/core-secrets.yaml)
For a professional setup, secrets should move to a proper secret manager or at least be injected outside git.
## 3. What "professional architecture" should look like here
For your use case, the clean target is not "one main node and one cold backup node".
The cleaner target is:
1. Highly available k3s control plane
2. Separate worker nodes for workloads
3. NATS deployed as an HA cluster
4. Ingress handled inside Kubernetes
5. Backups for both cluster state and NATS data
6. Workload placement rules so noisy workers do not affect all services
## 4. Recommended target layout
### Minimum professional layout
- `3` k3s server nodes
- `2` workload worker nodes
- `3` NATS pods with JetStream replication
- `1` in-cluster ingress controller
- `1` backup system for cluster resources and persistent volumes
### Example node roles
- `cp-1`: k3s server
- `cp-2`: k3s server
- `cp-3`: k3s server
- `app-1`: app services and ingress
- `wrk-1`: NATS workers and heavy async jobs
Better:
- `app-1`, `app-2`: app/service nodes
- `wrk-1`, `wrk-2`: dedicated worker nodes
### Planes
Control plane:
- k3s server nodes only
- no application workloads if possible
Service plane:
- FastAPI / business services
- ingress controller
- gateway
- observability stack
Worker plane:
- NATS worker consumers
- batch jobs
- CPU-heavy or retry-heavy services
Messaging plane:
- NATS cluster with JetStream
- dedicated storage
## 5. Best failover model for your requirement
You asked for immediate switch to a backup node if one node fails.
There are two ways to think about that:
### Option 1: active-passive node failover
This means one main node and one backup node waiting.
This is simpler to understand, but it is still not the best Kubernetes design because:
- one active node is still a bottleneck
- failover is not truly instant
- stateful components are harder to fail over cleanly
- you will still need shared storage or replicated state
### Option 2: active-active cluster with multiple nodes
This is the better professional design.
Instead of "switching to backup", Kubernetes simply reschedules workloads onto healthy nodes because:
- services already run across more than one node
- ingress already points at the cluster, not one host
- NATS data is replicated
- pods have anti-affinity and replicas across nodes
This is the model I recommend for you.
## 6. Recommended production architecture for this repo
### Edge and ingress
Move ingress fully into Kubernetes:
- install Traefik or nginx ingress inside k3s
- stop depending on host-level Docker Compose nginx proxies for production routing
- expose ingress through a proper load balancer or a floating IP/VIP
Good choices:
- `kube-vip` for virtual IP failover on bare metal
- `MetalLB` for service load balancers on bare metal
This is the clean replacement for current host-side proxying in [`docker-compose.yml`](E:/Birock/kubernetes/docker-compose.yml).
### Kubernetes control plane
Use HA k3s server nodes:
- `3` k3s server nodes
- external datastore or embedded etcd in HA mode
For small-to-medium production, HA k3s with embedded etcd is often enough and simpler than trying to maintain a single-node server plus a backup.
### App services
Run FastAPI and business services on app nodes:
- Deployment instead of StatefulSet unless stable pod identity is required
- `replicas >= 2`
- topology spread constraints
- anti-affinity across nodes
- proper readiness and liveness probes
### Worker services
Run workers only on worker nodes:
- label worker nodes, for example `workload-type=async`
- taint worker nodes
- add tolerations to worker pods
- add node affinity so worker pods land only there
This gives you the "separate worker plane" you asked for.
### NATS
Deploy NATS inside Kubernetes as an HA cluster, not as a single external endpoint.
Use:
- `3` NATS pods
- JetStream replication factor `3`
- persistent volumes
- pod anti-affinity
- dedicated node pool if possible
This is the biggest architecture improvement you can make for service survival.
### Backups
Use two backup layers:
1. Kubernetes resource/state backup
2. JetStream or persistent volume backup
Recommended tools:
- Velero for cluster resource backup and restore
- CSI snapshots or storage-level snapshots for persistent volumes
- scheduled export/backup for critical NATS data if needed
## 7. How to separate service plane and worker plane
This is a very good idea for your stack.
### Service plane should host
- ingress controller
- FastAPI/API gateway
- frontend-facing services
- dashboard/observability tools
### Worker plane should host
- NATS consumers
- retry-heavy jobs
- long-running async processors
- any CPU-heavy integration jobs
### Basic implementation pattern
On nodes:
- label app nodes: `node-role.workolik/app=true`
- label worker nodes: `node-role.workolik/worker=true`
Optionally taint worker nodes:
- `dedicated=workers:NoSchedule`
Then:
- app manifests use node affinity for `app=true`
- worker manifests use node affinity and toleration for worker nodes
## 8. CPU spike reduction strategy
The CPU spike problem is usually not solved by "adding one backup node". It is solved by isolation, limits, and scaling.
### Do these first
1. Put workers on separate nodes.
2. Tighten worker CPU limits and requests based on real usage.
3. Reduce high default concurrency for the noisiest workers.
4. Add HPA or KEDA scaling from queue depth, not only CPU.
5. Ensure retries do not cause synchronized storms.
### Very likely spike sources in this repo
- high worker concurrency
- many worker StatefulSets sharing the same node
- retries against slow external APIs
- host-level proxying plus cluster-level routing mix
- single-node k3s carrying ingress, apps, workers, and maybe NATS responsibilities together
### Better autoscaling choice
For queue workers, KEDA is often better than plain HPA because it can scale on:
- NATS lag
- queue depth
- custom Prometheus metrics
That is usually more useful than only scaling from CPU percentage.
## 9. Concrete migration path
### Phase 1: stabilize current cluster
Do this before any big redesign:
- move secrets out of git
- standardize on ingress instead of many host nginx proxies
- remove unnecessary `NodePort` exposure where possible
- add resource dashboards and alerting
- capture actual CPU and memory usage for each worker
### Phase 2: separate worker nodes
- add a second or third node
- label and taint worker nodes
- move worker workloads there
- keep app services on separate nodes
This alone will already reduce blast radius from worker CPU spikes.
### Phase 3: make k3s highly available
- create `3` k3s server nodes
- use HA embedded etcd
- put ingress behind `kube-vip` or `MetalLB`
Now loss of one server does not mean cluster loss.
### Phase 4: make NATS highly available
- deploy NATS cluster inside Kubernetes
- enable JetStream replication
- use persistent volumes
- update workloads to connect to in-cluster NATS service
This replaces the current single external NATS dependency from [`manifests/core/core-config.yaml#L10`](E:/Birock/kubernetes/manifests/core/core-config.yaml#L10).
### Phase 5: add backup and restore
- install Velero
- schedule cluster backups
- schedule volume snapshots
- test restore into a fresh environment
If restore has never been tested, backup is not yet reliable.
## 10. What I would choose for you
Because you said you are a beginner, I would not jump straight into a very large platform.
I would choose this as the practical target:
- `3` k3s server nodes
- `2` worker nodes
- in-cluster Traefik
- `MetalLB` or `kube-vip`
- NATS HA cluster with JetStream replication
- Velero backups
- node separation for app plane and worker plane
This is modern, realistic, and still manageable.
## 11. What I would not recommend
I would avoid these patterns for your next version:
- one main Kubernetes node plus one cold backup node as the final design
- storing secrets directly in yaml in git
- relying on `NodePort` as the main production exposure pattern
- mixing host Docker Compose production routing with cluster routing long-term
- keeping NATS as a single external IP with no clear HA story
## 12. Immediate next actions for this repo
If we continue from this document, the best implementation order is:
1. Add node placement rules for workers and apps.
2. Convert external exposure to a single in-cluster ingress pattern.
3. Remove hard dependence on the single external NATS IP.
4. Add observability for CPU, memory, restart count, and queue lag.
5. Introduce backup tooling and test restore.
## 13. Summary in simple words
The clean professional version of your stack is:
- Kubernetes control plane on multiple HA server nodes
- app services on one node group
- NATS workers on another node group
- NATS itself running as a replicated cluster
- ingress and failover handled at cluster level, not manually by switching servers
- backups for both Kubernetes state and message data
That gives you what you want:
- less CPU blast radius
- better reliability
- faster failover
- cleaner separation of responsibilities
- a more modern production setup

43
docs/QUICK_START.md Normal file
View File

@@ -0,0 +1,43 @@
# Quick Start - Kubernetes in 5 Minutes
## 🎯 Goal
Run Kubernetes locally in containers and deploy your apps.
## ⚡ Fast Track
### 1. Install k3d (if not using Docker Desktop)
```bash
# Windows: Download from https://k3d.io/
# Or use Docker Desktop Kubernetes (easier!)
```
### 2. Create Cluster
```bash
k3d cluster create mycluster --servers 1 --agents 2
```
### 3. Build Images
```bash
cd E:\nats\kubernetes
docker build -f Dockerfile --target api -t fastapi-backend:latest .
docker build -f Dockerfile --target worker -t nats-worker:latest .
```
### 4. Update Manifests
- `manifests/fastapi-deployment.yaml`: Change image to `fastapi-backend:latest`
- `manifests/worker-deployment.yaml`: Change image to `nats-worker:latest`
### 5. Deploy
```bash
./deploy.sh
```
### 6. Check
```bash
kubectl get pods -n nats-backend
```
## 🎉 Done!
Your apps are running in Kubernetes!

138
docs/READY_TO_DEPLOY.md Normal file
View File

@@ -0,0 +1,138 @@
# ✅ Kubernetes Setup - Ready to Deploy!
## 🎯 Everything is Configured and Ready
### ✅ What's Set Up
1. **Kubernetes Manifests** (`kubernetes/manifests/`)
- ✅ `namespace.yaml` - Namespace for your app
- ✅ `secrets.yaml` - NATS credentials and external API config
- ✅ `fastapi-deployment.yaml` - FastAPI backend (4 replicas, auto-scaling)
- ✅ `fastapi-service.yaml` - Service on port 8000
- ✅ `fastapi-hpa.yaml` - Horizontal Pod Autoscaler (4-20 pods)
- ✅ `worker-deployment.yaml` - NATS workers (2 replicas, auto-scaling)
- ✅ `worker-hpa.yaml` - Worker autoscaler (2-10 pods)
- ✅ `gateway.yaml` - Gateway API (ports 8201/8441 - no conflict with Traefik)
- ✅ `dashboard.yaml` - Kubernetes Dashboard UI
2. **Deployment Scripts**
- ✅ `simple-deploy.sh` - One-command deployment
- ✅ `deploy-dashboard.sh` - Dashboard deployment
- ✅ `setup-jetstream.sh` - JetStream setup
3. **Configuration**
- ✅ Port conflicts resolved (8201/8441 instead of 80/443)
- ✅ Image pull policy set to `Never` (uses local containerd images)
- ✅ NATS connection configured (external NATS server)
- ✅ External API endpoints configured
- ✅ HTTP methods correctly forwarded (PUT for update delivery)
4. **Documentation**
- ✅ `DEPLOY_CHECKLIST.md` - Step-by-step deployment guide
- ✅ `DASHBOARD_SETUP.md` - Dashboard access guide
- ✅ `README.md` - Main documentation
## 🚀 Quick Deployment Steps
### Step 1: Start k3s (if not running)
```bash
sudo systemctl start k3s
sudo systemctl status k3s
```
### Step 2: Build and Import Docker Images
```bash
cd ~/kubernetes
# Build images
docker build -t fastapi-backend:latest --target api -f Dockerfile .
docker build -t nats-worker:latest --target worker -f Dockerfile .
# Import to containerd (k3s uses containerd)
docker save fastapi-backend:latest | sudo k3s ctr images import -
docker save nats-worker:latest | sudo k3s ctr images import -
```
### Step 3: Setup JetStream
```bash
chmod +x setup-jetstream.sh
./setup-jetstream.sh
```
### Step 4: Deploy Everything
```bash
chmod +x simple-deploy.sh
./simple-deploy.sh
```
### Step 5: Deploy Dashboard (Optional)
```bash
chmod +x deploy-dashboard.sh
./deploy-dashboard.sh
```
## 🔍 Verify Deployment
```bash
# Check pods
kubectl get pods -n nats-backend
# Check services
kubectl get svc -n nats-backend
# Check Gateway
kubectl get gateway -n nats-backend
# Check HPA
kubectl get hpa -n nats-backend
```
## 🖥️ Access Dashboard
```bash
# Start proxy
kubectl proxy
# Visit in browser:
# http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
```
Or use port-forward:
```bash
kubectl port-forward -n kubernetes-dashboard service/kubernetes-dashboard 9090:9090
# Visit: http://localhost:9090
```
## 📊 What You'll See in Dashboard
- **Namespaces:** `nats-backend`, `kubernetes-dashboard`
- **Pods:** FastAPI pods, Worker pods, Dashboard pod
- **Services:** FastAPI service, Dashboard service
- **Deployments:** All your deployments
- **HPA:** Autoscaling configurations
- **Resources:** CPU, Memory usage
- **Logs:** View logs from any pod
- **Events:** Cluster events and errors
## ⚙️ Configuration Summary
| Component | Configuration |
|-----------|--------------|
| **FastAPI** | 4 initial pods, scales 4-20, port 8000 |
| **Workers** | 2 initial pods, scales 2-10 |
| **Gateway** | Ports 8201 (HTTP), 8441 (HTTPS) |
| **NATS** | External: `nats://nats.workolik.com:4222` |
| **Domain** | `queue.workolik.com` |
| **External API** | `https://jupiter.nearle.app` |
## ✅ All Systems Ready!
Everything is configured and ready to deploy. Just follow the steps above!
**Need help?** Check:
- `DEPLOY_CHECKLIST.md` - Detailed deployment guide
- `DASHBOARD_SETUP.md` - Dashboard access options
- `README.md` - Full documentation
🎉 **You're all set!**

152
docs/SETUP_LOCAL_K8S.md Normal file
View File

@@ -0,0 +1,152 @@
# Simple Kubernetes Setup Guide
## 🎯 What We're Doing
We'll run Kubernetes **inside containers** on your computer, then deploy your FastAPI + Worker apps to it.
Think of it like:
- **Kubernetes** = A manager that runs your apps
- **k3d** = Tool that runs Kubernetes in Docker containers
- **Your Apps** = FastAPI and Worker that we deploy
## 📦 Step 1: Install k3d (Kubernetes in Docker)
k3d runs Kubernetes in Docker containers - perfect for learning!
**Windows (PowerShell):**
```powershell
# Install via Chocolatey (if you have it)
choco install k3d
# OR download from: https://k3d.io/
```
**Or use Docker Desktop with Kubernetes:**
- Open Docker Desktop
- Go to Settings → Kubernetes
- Enable Kubernetes
- Click "Apply & Restart"
## 🚀 Step 2: Create a 3-Node Kubernetes Cluster
```bash
# Create cluster with 3 nodes (1 master + 2 workers)
k3d cluster create mycluster --servers 1 --agents 2
# OR if using Docker Desktop Kubernetes, skip this step
```
**What this does:**
- Creates 3 containers running Kubernetes
- 1 master node (controls everything)
- 2 worker nodes (run your apps)
## ✅ Step 3: Verify Cluster is Running
```bash
# Check if cluster is ready
kubectl get nodes
# You should see 3 nodes:
# NAME STATUS ROLES
# k3d-mycluster-0 Ready control-plane
# k3d-mycluster-1 Ready <none>
# k3d-mycluster-2 Ready <none>
```
## 🐳 Step 4: Build Your Docker Images
```bash
# Go to your project folder
cd E:\nats\kubernetes
# Build API image
docker build -f Dockerfile --target api -t fastapi-backend:latest .
# Build Worker image
docker build -f Dockerfile --target worker -t nats-worker:latest .
```
**What this does:**
- Builds your FastAPI app into a Docker image
- Builds your Worker app into a Docker image
- Stores them locally (k3d can use local images)
## 📝 Step 5: Update Image Names
Edit `manifests/fastapi-deployment.yaml`:
- Change `your-registry/fastapi-backend:latest` → `fastapi-backend:latest`
Edit `manifests/worker-deployment.yaml`:
- Change `your-registry/nats-worker:latest` → `nats-worker:latest`
## 🚀 Step 6: Deploy Everything
```bash
# Make script executable (if on Linux/Mac)
chmod +x deploy.sh
# Run deployment
./deploy.sh
# OR manually:
kubectl apply -f manifests/namespace.yaml
kubectl apply -f manifests/secrets.yaml
kubectl apply -f manifests/fastapi-deployment.yaml
kubectl apply -f manifests/fastapi-service.yaml
kubectl apply -f manifests/fastapi-hpa.yaml
kubectl apply -f manifests/worker-deployment.yaml
kubectl apply -f manifests/worker-hpa.yaml
```
## ✅ Step 7: Check Everything is Running
```bash
# See all your pods
kubectl get pods -n nats-backend
# See pods spread across nodes
kubectl get pods -n nats-backend -o wide
# Check if pods are running
# You should see:
# - 4 fastapi-backend pods
# - 2 nats-worker pods
```
## 🧪 Step 8: Test Your App
```bash
# Forward port to access FastAPI
kubectl port-forward -n nats-backend service/fastapi-backend 8000:80
# In another terminal, test:
curl http://localhost:8000/health
```
## 📊 Useful Commands
```bash
# See everything
kubectl get all -n nats-backend
# See logs
kubectl logs -f deployment/fastapi-backend -n nats-backend
# See which node a pod is on
kubectl get pods -n nats-backend -o wide
# Delete everything (if needed)
kubectl delete namespace nats-backend
```
## 🎯 Summary
1. **Install k3d** → Creates Kubernetes in containers
2. **Create cluster** → 3 nodes ready
3. **Build images** → Your apps as Docker images
4. **Deploy** → Run `./deploy.sh`
5. **Test** → Access your app
**That's it!** Your apps are now running in Kubernetes! 🎉

70
docs/SIMPLIFIED_FAILOVER_SETUP.md Normal file
View File

@@ -0,0 +1,70 @@
# Simplified k3s Failover & Performance Setup
To fix your **CPU Spikes** and achieve **Resilient Failover** without building a massive cluster, follow this simple 2-node or 3-node plan.
## 1. The Strategy
Instead of making one massive node do everything, we split the work:
- **App Node:** Only runs your public APIs (`jupiter`, `fiesta`, `atlantis`, etc.).
- **Worker Node:** Only runs the heavy background tasks (`worker-orders`, `worker-deliveries`, etc.).
This ensures that if a background worker spikes to 100% CPU, your **main website remains fast and healthy**.
---
## 2. Setting Up Your Planes (Placing Nodes Into Groups)
You need to "tell" Kubernetes which of your servers is which. Run these commands:
### Identify your Node names
```bash
kubectl get nodes
```
### Label your nodes by their role
Replace `<node-name>` with your actual server names (e.g., `server-1`, `server-2`).
#### A. Assign Node 1 as the "App Host"
```bash
kubectl label node <node-1> node-role.workolik/app=true
kubectl taint node <node-1> dedicated=apps:NoSchedule
```
#### B. Assign Node 2 as the "Worker Host"
*(This node will take all the CPU spikes)*
```bash
kubectl label node <node-2> node-role.workolik/worker=true
kubectl taint node <node-2> dedicated=workers:NoSchedule
```
---
## 3. Simplified Ingress (Replaces Docker Compose)
You no longer need the complex Nginx proxies in your `docker-compose.yml`. I have created a single "Unified Ingress" that replaces all of them.
### Why this is better:
- **Failover:** If an "App Node" fails, Kubernetes automatically moves your APIs to another node, and k3s's built-in Traefik handles the routing instantly.
- **Simplicity:** One file manages all your domains (`queue.workolik.com`, `jupiter.nearle.app`, etc.).
### How to apply it:
1. Apply the Middleware (CORS settings):
```bash
kubectl apply -f manifests/core/traefik-middlewares.yaml
```
2. Apply the Unified Ingress:
```bash
kubectl apply -f manifests/core/ingress-unified.yaml
```
---
## 4. Summary of Improvements
1. **CPU Isolation:** Workers in `manifests/core/workers.yaml` now have strict limits and are "pushed" to the Worker Plane.
2. **Professional Routing:** Moved from Host-side Nginx (manual failover) to K8s-side Traefik (automatic failover).
3. **No NATS Changes:** We are still using your external NATS, but with the new Ingress, your services are much safer.
## 5. Next Practical Steps (When you are ready)
- **High Availability (HA):** When you have 3 control-plane servers, k3s will survive even if a "Master" node reboots.
- **Persistence:** Ensure any databases or disks aren't tied to a single machine's filesystem (`hostPath`).
**You can now stop using the `docker-compose.yml` for traffic routing once you point your DNS/Load Balancer to your k3s cluster IP.**

1350
docs/TECH_STACK_ARCHITECTURE.txt Normal file
View File

File diff suppressed because it is too large Load Diff

985
docs/TECH_STACK_ARCHITECTURE_INTERMEDIATE.txt Normal file
View File

@@ -0,0 +1,985 @@
================================================================================
NATS MESSAGE QUEUE SYSTEM
Intermediate-Level Guide
================================================================================
TABLE OF CONTENTS
-----------------
1. What This System Does (Simple Explanation)
2. The Big Picture - How Everything Works Together
3. Each Component Explained Simply
4. How Messages Flow Through the System
5. Why We Use Each Technology
6. How to Deploy and Use
7. Common Questions Answered
8. Troubleshooting Made Easy
================================================================================
1. WHAT THIS SYSTEM DOES (SIMPLE EXPLANATION)
================================================================================
Imagine you have a restaurant:
- Customers place orders (HTTP requests)
- Orders go to the kitchen queue (NATS message queue)
- Chefs process orders (Workers)
- Food gets delivered (Forwarded to external API)
OUR SYSTEM:
-----------
1. Your mobile app or website sends a request (like "update delivery status")
2. FastAPI receives it and puts it in a message queue (NATS)
3. Workers pick up messages from the queue
4. Workers forward the request to the actual external API
5. Everything is monitored and can scale automatically
WHY USE A QUEUE?
---------------
✓ If the external API is slow, your app doesn't wait
✓ If the external API is down, messages wait in queue (won't be lost)
✓ You can process many requests without overloading anything
✓ Easy to add more workers if you have lots of messages
================================================================================
2. THE BIG PICTURE - HOW EVERYTHING WORKS TOGETHER
================================================================================
┌─────────────────────────────────────────────────────────────────────┐
│ SIMPLE FLOW DIAGRAM │
└─────────────────────────────────────────────────────────────────────┘
[Your App]
│
│ Sends HTTP request: "Update delivery #123"
▼
[FastAPI Server]
│
│ Puts message in queue
▼
[NATS Message Queue]
│
│ Stores message safely
│ (like a mailbox)
▼
[Worker Process]
│
│ Picks up message
│ Forwards to external API
▼
[External API]
│
│ Processes the request
▼
[Done!]
COMPONENTS BREAKDOWN:
---------------------
1. NATS = The Message Queue (like a post office)
- Receives messages from FastAPI
- Stores them safely
- Gives them to workers when ready
2. FastAPI = The API Server (like a receptionist)
- Receives requests from your app
- Quickly puts them in the queue
- Returns "OK, got it!" immediately
3. Workers = The Processors (like workers in a factory)
- Take messages from queue
- Forward to external API
- Handle retries if something fails
4. Nginx = The Helper (like a translator)
- Helps the dashboard talk to NATS
- Adds security headers (CORS)
5. Dashboard = The Monitor (like a control panel)
- Shows you what's happening
- See how many messages are waiting
- Monitor if everything is working
================================================================================
3. EACH COMPONENT EXPLAINED SIMPLY
================================================================================
┌─────────────────────────────────────────────────────────────────────┐
│ 3.1 NATS - THE MESSAGE QUEUE │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Think of NATS like a smart mailbox system:
- FastAPI puts messages in
- Workers take messages out
- Messages are stored safely (won't be lost)
- Can handle millions of messages
WHY WE USE IT:
-------------
✓ Super fast (messages delivered in less than 1 millisecond)
✓ Reliable (messages won't disappear)
✓ Simple to use (no complicated setup)
✓ Lightweight (doesn't need much resources)
HOW IT WORKS:
-------------
1. FastAPI publishes a message: "Here's a delivery update"
2. NATS stores it in a "stream" (like a folder)
3. Worker asks: "Any messages for me?"
4. NATS gives worker the message
5. Worker processes it and says "Done!" (ACK)
6. NATS removes the message from queue
REAL EXAMPLE:
-------------
Message looks like this:
{
"endpoint": "/live/api/v1/deliveries/updatedelivery",
"method": "PUT",
"data": {"delivery_id": 123, "status": "delivered"},
"received_at": 1234567890
}
This gets stored in NATS and workers pick it up.
┌─────────────────────────────────────────────────────────────────────┐
│ 3.2 FASTAPI - THE API SERVER │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
FastAPI is a Python web server that:
- Listens for HTTP requests
- Validates the data
- Puts messages in NATS queue
- Returns response immediately
WHY WE USE IT:
-------------
✓ Fast (handles many requests per second)
✓ Easy to write (Python is simple)
✓ Automatic documentation (shows all endpoints)
✓ Built-in validation (catches bad data)
HOW IT WORKS:
-------------
1. Your app sends: PUT /live/api/v1/deliveries/updatedelivery
2. FastAPI receives it
3. Checks if data is valid
4. Creates a message with all the info
5. Publishes to NATS (super fast, < 1ms)
6. Returns: {"status": "accepted", "message_id": 12345}
ENDPOINTS WE HAVE:
------------------
- PUT /live/api/v1/deliveries/updatedelivery
- POST /live/api/v1/deliveries/createdeliveries
- POST /live/api/v2/partners/createriderlog
- POST /live/api/v2/deliveries/createdeliverylog
- POST /live/api/v2/partners/createbreaklog
- POST /live/api/v2/partners/updatebreaklog
Each endpoint does the same thing: receives request → puts in queue → returns OK
HEALTH CHECKS:
--------------
- GET /health → Is the server running? (Yes/No)
- GET /ready → Is NATS connected? (Yes/No)
- GET /metrics → Statistics (for monitoring)
┌─────────────────────────────────────────────────────────────────────┐
│ 3.3 WORKERS - THE MESSAGE PROCESSORS │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Workers are background processes that:
- Watch the NATS queue for new messages
- Take messages out
- Forward them to the external API
- Handle errors and retries
WHY WE USE THEM:
---------------
✓ Keeps your API fast (doesn't wait for external API)
✓ Handles failures automatically (retries if API is down)
✓ Can scale independently (add more workers if needed)
✓ Separates concerns (API vs processing)
HOW IT WORKS:
-------------
1. Worker asks NATS: "Any messages?"
2. NATS gives worker a batch (up to 10 messages)
3. For each message:
a. Reads the endpoint and data
b. Maps to external API URL
c. Sends HTTP request (PUT or POST)
d. Waits for response
4. If success: Tells NATS "Done!" (ACK) → message removed
5. If failure: Tells NATS "Try again" (NAK) → message stays in queue
RETRY LOGIC:
------------
If external API fails:
- Wait 5 seconds, try again
- If fails, wait 10 seconds, try again
- If fails, wait 15 seconds, try again
- Up to 5 attempts total
- After 5 failures, message stays in queue for manual review
EXAMPLE:
--------
Message says: "Update delivery #123"
Worker:
1. Reads message
2. Maps to: https://jupiter.nearle.app/live/api/v1/deliveries/updatedelivery
3. Sends PUT request with the data
4. External API responds: "OK"
5. Worker tells NATS: "Done!" → message removed
┌─────────────────────────────────────────────────────────────────────┐
│ 3.4 NGINX - THE HELPER PROXY │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Nginx is a web server that acts as a middleman:
- Sits between Dashboard and NATS monitoring
- Adds security headers (CORS)
- Handles browser security requirements
WHY WE NEED IT:
---------------
The NATS Dashboard runs in your browser (JavaScript).
Browsers have security rules (CORS) that prevent websites from
talking to other websites unless they have permission.
Nginx adds the permission headers so the dashboard can work.
HOW IT WORKS:
-------------
1. Dashboard (in browser) wants to check NATS status
2. Browser blocks it (security rule)
3. Nginx intercepts the request
4. Adds headers: "Yes, dashboard is allowed to talk to NATS"
5. Browser says "OK" and allows the request
6. Dashboard gets the data it needs
SIMPLE ANALOGY:
---------------
Like a bouncer at a club:
- Dashboard wants to enter (access NATS)
- Bouncer (Nginx) checks the list
- Adds your name to the VIP list (CORS headers)
- You get in!
┌─────────────────────────────────────────────────────────────────────┐
│ 3.5 NATS DASHBOARD - THE MONITORING SCREEN │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
A web page that shows you:
- How many messages are in the queue
- How fast messages are being processed
- If workers are running
- Statistics and graphs
WHY WE USE IT:
-------------
✓ See what's happening in real-time
✓ Know if something is wrong
✓ Monitor performance
✓ Debug issues
WHAT YOU CAN SEE:
-----------------
- Stream Statistics: How many messages total
- Consumer Status: Are workers processing?
- Message Rate: Messages per second
- Pending Messages: How many waiting to be processed
- Consumer Lag: How far behind are workers?
HOW TO ACCESS:
--------------
URL: https://natsadmin.workolik.com
Username: admin
Password: package@321#
┌─────────────────────────────────────────────────────────────────────┐
│ 3.6 DOCKER - CONTAINERIZATION │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Docker packages everything into containers:
- Like shipping containers for software
- Each service runs in its own container
- All containers work together
WHY WE USE IT:
-------------
✓ Same environment everywhere (dev, staging, production)
✓ Easy to deploy (just run docker-compose up)
✓ Isolated services (if one crashes, others keep running)
✓ Easy to update (just rebuild container)
HOW IT WORKS:
-------------
docker-compose.yml defines:
- NATS container (the message queue)
- Nginx container (the proxy)
- Dashboard container (the monitoring)
All containers talk to each other via a network.
SIMPLE ANALOGY:
---------------
Like apartments in a building:
- Each apartment (container) is separate
- But they share utilities (network)
- You can move apartments (containers) easily
- If one apartment has issues, others are fine
┌─────────────────────────────────────────────────────────────────────┐
│ 3.7 KUBERNETES - CONTAINER ORCHESTRATION │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Kubernetes manages containers across multiple servers:
- Like a manager for a team of workers
- Automatically starts/stops containers
- Spreads work across multiple servers
- Auto-scales when busy
WHY WE USE IT:
-------------
✓ High availability (if one server dies, others take over)
✓ Auto-scaling (adds more workers when busy)
✓ Self-healing (restarts crashed containers)
✓ Load balancing (spreads requests evenly)
HOW IT WORKS:
-------------
1. You define: "I want 4 FastAPI servers and 2 workers"
2. Kubernetes creates them across different servers
3. If one crashes, Kubernetes restarts it
4. If load increases, Kubernetes adds more
5. If load decreases, Kubernetes removes some
AUTO-SCALING EXAMPLE:
---------------------
Normal load: 4 FastAPI pods, 2 worker pods
High load: Kubernetes sees CPU at 80%
→ Adds more pods automatically
→ Now: 10 FastAPI pods, 5 worker pods
Low load: Kubernetes sees CPU at 20%
→ Removes some pods
→ Back to: 4 FastAPI pods, 2 worker pods
┌─────────────────────────────────────────────────────────────────────┐
│ 3.8 TRAEFIK - THE ROUTER │
└─────────────────────────────────────────────────────────────────────┘
WHAT IT IS:
-----------
Traefik is like a smart receptionist:
- Receives all incoming requests
- Routes them to the right service
- Handles HTTPS certificates automatically
- Adds security (like password protection)
WHY WE USE IT:
-------------
✓ Automatic HTTPS (free SSL certificates)
✓ Easy routing (just add labels to containers)
✓ Handles authentication
✓ One entry point for everything
HOW IT WORKS:
-------------
1. Request comes in: https://queue.workolik.com
2. Traefik receives it
3. Checks: "This goes to FastAPI"
4. Routes to FastAPI service
5. Returns response
HTTPS CERTIFICATES:
-------------------
Traefik automatically gets free SSL certificates from Let's Encrypt.
You don't need to do anything - it just works!
AUTHENTICATION:
---------------
For the dashboard (natsadmin.workolik.com):
- Traefik adds password protection
- Username: admin
- Password: package@321#
================================================================================
4. HOW MESSAGES FLOW THROUGH THE SYSTEM
================================================================================
STEP-BY-STEP EXAMPLE:
---------------------
Let's say your app wants to update a delivery status.
STEP 1: YOUR APP SENDS REQUEST
--------------------------------
Your mobile app sends:
PUT https://queue.workolik.com/live/api/v1/deliveries/updatedelivery
Body: {"delivery_id": 123, "status": "delivered"}
STEP 2: TRAEFIK RECEIVES IT
----------------------------
Traefik sees: "This is for queue.workolik.com"
Routes to: FastAPI service
STEP 3: FASTAPI PROCESSES IT
-----------------------------
FastAPI:
- Receives the request
- Validates the data (checks it's valid JSON)
- Creates a message:
{
"endpoint": "/live/api/v1/deliveries/updatedelivery",
"method": "PUT",
"data": {"delivery_id": 123, "status": "delivered"},
"received_at": 1234567890
}
- Publishes to NATS (super fast!)
- Returns: {"status": "accepted", "message_id": 12345}
STEP 4: YOUR APP GETS RESPONSE
-------------------------------
Your app receives: "OK, got it!" (takes < 1 second)
Your app doesn't wait for the external API - it's done!
STEP 5: NATS STORES MESSAGE
---------------------------
NATS:
- Receives message
- Stores in stream "EVENTS"
- Subject: "api.v1.deliveries.updatedelivery"
- Message is safe and won't be lost
STEP 6: WORKER PICKS IT UP
---------------------------
Worker:
- Asks NATS: "Any messages?"
- NATS gives worker the message
- Worker reads: "Update delivery #123"
STEP 7: WORKER FORWARDS TO EXTERNAL API
----------------------------------------
Worker:
- Maps endpoint to: https://jupiter.nearle.app/live/api/v1/deliveries/updatedelivery
- Sends PUT request with the data
- Waits for response
STEP 8: EXTERNAL API RESPONDS
------------------------------
External API:
- Processes the request
- Updates delivery status
- Returns: 200 OK
STEP 9: WORKER ACKNOWLEDGES
----------------------------
Worker:
- Sees success (200 OK)
- Tells NATS: "Done!" (ACK)
- NATS removes message from queue
- Done!
IF SOMETHING GOES WRONG:
------------------------
If external API is down:
- Worker gets error
- Tells NATS: "Try again later" (NAK)
- NATS keeps message in queue
- Worker waits 5 seconds
- Tries again
- Repeats up to 5 times
- If still fails, message stays for manual review
================================================================================
5. WHY WE USE EACH TECHNOLOGY
================================================================================
NATS (Message Queue):
---------------------
✓ Super fast (messages in < 1ms)
✓ Reliable (messages won't disappear)
✓ Simple (easy to set up and use)
✓ Lightweight (doesn't need much resources)
✓ Perfect for this use case
Why not RabbitMQ? Too heavy, more complex
Why not Kafka? Overkill, too complex for this
Why not Redis? Less mature, fewer features
FASTAPI (API Server):
---------------------
✓ Fast (handles many requests)
✓ Easy (Python is simple to write)
✓ Modern (async/await support)
✓ Automatic docs (shows all endpoints)
✓ Type validation (catches errors early)
Why not Flask? Slower, no async support
Why not Django? Too heavy, overkill
Why not Node.js? Team knows Python better
WORKERS (Message Processors):
------------------------------
✓ Decouples API from external service
✓ Handles retries automatically
✓ Can scale independently
✓ Easy to monitor separately
Why separate workers? Keeps API fast, handles failures better
NGINX (Proxy):
--------------
✓ Needed for CORS (browser security)
✓ Lightweight (small container)
✓ Fast (high performance)
✓ Simple config
Why needed? Browsers block cross-origin requests without CORS headers
DOCKER (Containers):
--------------------
✓ Same environment everywhere
✓ Easy deployment
✓ Isolated services
✓ Easy updates
Why containers? Consistency, easy deployment, isolation
KUBERNETES (Orchestration):
----------------------------
✓ High availability (multiple servers)
✓ Auto-scaling (adds workers when busy)
✓ Self-healing (restarts crashed services)
✓ Load balancing
Why Kubernetes? Production needs reliability and scaling
TRAEFIK (Router):
-----------------
✓ Automatic HTTPS (free certificates)
✓ Easy routing (just labels)
✓ Handles authentication
✓ One entry point
Why Traefik? Simplifies HTTPS and routing
================================================================================
6. HOW TO DEPLOY AND USE
================================================================================
┌─────────────────────────────────────────────────────────────────────┐
│ 6.1 DOCKER COMPOSE (SIMPLE DEPLOYMENT) │
└─────────────────────────────────────────────────────────────────────┘
FOR: Local development, small deployments, single server
STEPS:
------
1. Make sure Docker is installed
2. Go to the kubernetes folder
3. Run: docker-compose up -d
4. That's it! Everything starts automatically
WHAT GETS STARTED:
-----------------
- NATS server (message queue)
- Nginx proxy (CORS helper)
- NATS Dashboard (monitoring)
ACCESS:
-------
- Dashboard: https://natsadmin.workolik.com
- NATS Monitoring: https://nats.workolik.com
CHECK STATUS:
-------------
docker-compose ps # See what's running
docker-compose logs -f # See logs
docker-compose restart # Restart everything
┌─────────────────────────────────────────────────────────────────────┐
│ 6.2 KUBERNETES (PRODUCTION DEPLOYMENT) │
└─────────────────────────────────────────────────────────────────────┘
FOR: Production, high availability, auto-scaling
STEPS:
------
1. Build Docker images
2. Push to registry
3. Update secrets.yaml with your credentials
4. Run: ./deploy.sh
5. Wait for pods to start
WHAT GETS DEPLOYED:
-------------------
- FastAPI pods (4-20, auto-scales)
- Worker pods (2-10, auto-scales)
- Services (load balancers)
- Auto-scaling rules
CHECK STATUS:
-------------
kubectl get pods -n nats-backend # See pods
kubectl logs -f deployment/fastapi-backend # See logs
kubectl get hpa -n nats-backend # See auto-scaling
TEST ENDPOINTS:
---------------
curl -X PUT https://queue.workolik.com/live/api/v1/deliveries/updatedelivery \
-H "Content-Type: application/json" \
-d '{"delivery_id": 123, "status": "delivered"}'
Expected response:
{"status": "accepted", "message_id": 12345, "endpoint": "..."}
================================================================================
7. COMMON QUESTIONS ANSWERED
================================================================================
Q: Why use a message queue instead of calling the API directly?
A:
- Your app doesn't have to wait for slow external API
- If external API is down, messages wait (won't be lost)
- Can handle traffic spikes better
- Easy to add more workers if needed
Q: What happens if NATS goes down?
A:
- Messages are stored on disk (persistent)
- When NATS restarts, messages are still there
- Workers reconnect automatically
- No data loss
Q: What happens if a worker crashes?
A:
- Kubernetes restarts it automatically
- Messages stay in queue
- Other workers keep processing
- No data loss
Q: How fast is it?
A:
- FastAPI responds in < 1 second (usually < 100ms)
- NATS stores message in < 1 millisecond
- Worker processes in seconds (depends on external API)
- Total: Usually under 2 seconds end-to-end
Q: Can I add more endpoints?
A:
- Yes! Add endpoint in FastAPI (app.py)
- Add mapping in worker (worker.py)
- Deploy and it works
Q: How do I monitor if everything is working?
A:
- Check NATS Dashboard: https://natsadmin.workolik.com
- Check FastAPI metrics: GET /metrics
- Check worker metrics: Port 9090
- Check Kubernetes: kubectl get pods
Q: What if external API is slow?
A:
- Messages wait in queue
- Workers retry automatically
- Your API still responds fast (doesn't wait)
- No impact on your users
Q: How many messages can it handle?
A:
- NATS: Millions per second
- FastAPI: Thousands per second (depends on hardware)
- Workers: Depends on external API speed
- Can scale workers automatically if needed
Q: Is it secure?
A:
- HTTPS for all external access (Traefik)
- Password protection for dashboard
- NATS authentication required
- CORS properly configured
Q: Can I test locally?
A:
- Yes! Use Docker Compose
- Everything runs on your machine
- Same as production (just smaller scale)
================================================================================
8. TROUBLESHOOTING MADE EASY
================================================================================
PROBLEM: FastAPI returns 503 "NATS not connected"
-----------------------------------------------
WHAT IT MEANS: FastAPI can't talk to NATS
HOW TO FIX:
1. Check if NATS is running: docker ps | grep nats
2. Check NATS logs: docker logs nats
3. Check network: Can FastAPI reach NATS?
4. Check credentials: Are username/password correct?
PROBLEM: Messages not being processed
-------------------------------------
WHAT IT MEANS: Workers aren't picking up messages
HOW TO FIX:
1. Check if workers are running: kubectl get pods | grep worker
2. Check worker logs: kubectl logs -f deployment/nats-worker
3. Check NATS Dashboard: Are messages in queue?
4. Check consumer: Is worker_consumer active?
PROBLEM: CORS errors in browser
--------------------------------
WHAT IT MEANS: Dashboard can't access NATS monitoring
HOW TO FIX:
1. Check Nginx proxy is running: docker ps | grep nats-proxy
2. Check nginx-nats.conf: Are CORS headers correct?
3. Check NATS_MONITORING_URL in dashboard config
4. Restart nginx-proxy: docker-compose restart nats-proxy
PROBLEM: External API not receiving requests
--------------------------------------------
WHAT IT MEANS: Workers aren't forwarding messages
HOW TO FIX:
1. Check worker logs: Look for forwarding errors
2. Check EXTERNAL_BASE_URL: Is it correct?
3. Test connectivity: Can workers reach external API?
4. Check endpoint mapping: Is endpoint in worker.py?
PROBLEM: Auto-scaling not working
---------------------------------
WHAT IT MEANS: Pods aren't scaling up/down
HOW TO FIX:
1. Check HPA: kubectl get hpa -n nats-backend
2. Check metrics-server: kubectl top pods
3. Check resource limits: Are they set correctly?
4. Check if max replicas reached
QUICK HEALTH CHECK:
-------------------
1. Are all pods running? → kubectl get pods
2. Are messages processing? → Check dashboard
3. Are there errors? → Check logs
4. Is external API reachable? → Test from worker pod
COMMON COMMANDS:
----------------
# Docker Compose
docker-compose ps # Status
docker-compose logs -f # Logs
docker-compose restart # Restart
# Kubernetes
kubectl get pods # Pod status
kubectl logs -f [pod-name] # Pod logs
kubectl describe pod [pod] # Pod details
kubectl get hpa # Auto-scaling status
# NATS
# Check dashboard: https://natsadmin.workolik.com
================================================================================
9. KEY CONCEPTS EXPLAINED SIMPLY
================================================================================
MESSAGE QUEUE:
--------------
Like a post office:
- You drop off a letter (message)
- Post office stores it safely
- Mail carrier picks it up
- Delivers to destination
In our system:
- FastAPI drops off message
- NATS stores it
- Worker picks it up
- Delivers to external API
PUBLISH/SUBSCRIBE:
------------------
Like a radio station:
- Radio station broadcasts (publishes)
- Radios listen (subscribe)
- Many radios can listen to same station
In our system:
- FastAPI publishes messages
- Workers subscribe to messages
- Many workers can process same queue
ACKNOWLEDGMENT (ACK/NAK):
-------------------------
Like a receipt:
- You send a package
- Recipient signs for it (ACK)
- If rejected, you get it back (NAK)
In our system:
- Worker processes message
- If success: ACK → message removed
- If failure: NAK → message stays for retry
STREAM:
-------
Like a folder:
- All related messages go in one stream
- Easy to organize
- Can have multiple consumers
In our system:
- Stream: "EVENTS"
- All API messages go here
- Workers read from this stream
CONSUMER:
---------
Like a reader:
- Reads messages from stream
- Processes them
- Acknowledges when done
In our system:
- Consumer: "worker_consumer"
- Workers use this to get messages
- Tracks which messages are processed
AUTO-SCALING:
-------------
Like a restaurant:
- Few customers → Few waiters
- Many customers → More waiters
- Automatically adjusts
In our system:
- Low load → Few pods
- High load → More pods
- Kubernetes does it automatically
LOAD BALANCING:
---------------
Like distributing work:
- Manager gives tasks to available workers
- Spreads work evenly
- No one gets overloaded
In our system:
- Requests go to available FastAPI pod
- Spreads load evenly
- Kubernetes handles it
================================================================================
10. REAL-WORLD EXAMPLE
================================================================================
SCENARIO: Your delivery app needs to update delivery status
WITHOUT MESSAGE QUEUE:
----------------------
1. App sends request to your API
2. Your API calls external API directly
3. External API is slow (takes 5 seconds)
4. Your app waits 5 seconds
5. User sees loading spinner
6. If external API is down, your app fails
WITH MESSAGE QUEUE (OUR SYSTEM):
---------------------------------
1. App sends request to FastAPI
2. FastAPI puts message in queue (< 100ms)
3. FastAPI returns "OK" immediately
4. User sees success right away
5. Worker processes message in background
6. If external API is slow/down, message waits
7. Worker retries automatically
8. No impact on your users
BENEFITS:
---------
✓ Users get instant response
✓ System handles failures gracefully
✓ Can handle traffic spikes
✓ Easy to scale
✓ Messages never lost
================================================================================
11. QUICK REFERENCE
================================================================================
ENDPOINTS:
----------
PUT /live/api/v1/deliveries/updatedelivery
POST /live/api/v1/deliveries/createdeliveries
POST /live/api/v2/partners/createriderlog
POST /live/api/v2/deliveries/createdeliverylog
POST /live/api/v2/partners/createbreaklog
POST /live/api/v2/partners/updatebreaklog
URLS:
-----
FastAPI: https://queue.workolik.com
Dashboard: https://natsadmin.workolik.com
NATS Monitoring: https://nats.workolik.com
CREDENTIALS:
------------
NATS:
- Username: admin
- Password: package@321#
Dashboard:
- Username: admin
- Password: package@321#
PORTS:
------
NATS: 4222 (client), 8222 (monitoring)
FastAPI: 8000 (internal), 8201 (external)
Worker: 9090 (metrics)
Nginx: 80 (internal), 8082 (external)
FILES:
------
docker-compose.yml # Docker deployment
scripts/app.py # FastAPI code
scripts/worker.py # Worker code
nginx-nats.conf # Nginx config
manifests/*.yaml # Kubernetes configs
================================================================================
END OF GUIDE
================================================================================
This guide explains the system in simple terms. For more technical details,
see TECH_STACK_ARCHITECTURE.txt.
Remember:
- FastAPI receives requests and puts them in queue
- NATS stores messages safely
- Workers process messages and forward to external API
- Everything is monitored and can scale automatically
Questions? Check Section 7 (Common Questions) or Section 8 (Troubleshooting).
Last Updated: 2025-01-XX
Version: Intermediate Level

157
docs/test-delivery-logs.ps1 Normal file
View File

@@ -0,0 +1,157 @@
# PowerShell script to verify delivery logs are working
# Usage: .\test-delivery-logs.ps1
Write-Host "==================================================================" -ForegroundColor Cyan
Write-Host " DELIVERY LOG VERIFICATION TEST" -ForegroundColor Cyan
Write-Host "==================================================================" -ForegroundColor Cyan
Write-Host ""
# Configuration
$FastAPIUrl = if ($env:FASTAPI_URL) { $env:FASTAPI_URL } else { "https://queue.workolik.com" }
$Endpoint = "/live/api/v2/deliveries/createdeliverylog"
# Generate unique test data
$Timestamp = [int][double]::Parse((Get-Date -UFormat %s))
$OrderId = "TEST-$Timestamp"
$CurrentTime = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
Write-Host "📋 Test Configuration:" -ForegroundColor Yellow
Write-Host " FastAPI URL: $FastAPIUrl"
Write-Host " Endpoint: $Endpoint"
Write-Host " Test Order ID: $OrderId"
Write-Host ""
# Test payload
$Payload = @(
@{
logid = 0
tenantid = 1
partnerid = 44
locationid = 1
orderheaderid = $Timestamp
deliveryid = $Timestamp + 1000
userid = 1111
orderid = $OrderId
orderstatus = "active"
starttime = $CurrentTime
logdate = $CurrentTime
latitude = "11.0050664"
longitude = "76.9508776"
}
) | ConvertTo-Json
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host "STEP 1: Testing FastAPI Endpoint" -ForegroundColor Cyan
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host ""
Write-Host "Sending POST request to: $FastAPIUrl$Endpoint" -ForegroundColor Yellow
Write-Host ""
try {
$Response = Invoke-RestMethod -Uri "$FastAPIUrl$Endpoint" `
-Method Post `
-ContentType "application/json" `
-Body $Payload `
-ErrorAction Stop
Write-Host "HTTP Status Code: 200" -ForegroundColor Green
Write-Host "Response Body:" -ForegroundColor Yellow
$Response | ConvertTo-Json -Depth 10
Write-Host ""
if ($Response.status -eq "accepted") {
Write-Host "✅ SUCCESS: FastAPI accepted the request" -ForegroundColor Green
if ($Response.message_id) {
Write-Host " Message ID: $($Response.message_id)" -ForegroundColor Green
}
Write-Host " Status: Accepted" -ForegroundColor Green
} else {
Write-Host "⚠️ WARNING: Response status is not 'accepted'" -ForegroundColor Yellow
}
} catch {
$StatusCode = $_.Exception.Response.StatusCode.value__
Write-Host "❌ FAILED: FastAPI returned status $StatusCode" -ForegroundColor Red
Write-Host ""
Write-Host "Error Details:" -ForegroundColor Red
Write-Host $_.Exception.Message -ForegroundColor Red
Write-Host ""
Write-Host "Troubleshooting:" -ForegroundColor Yellow
Write-Host "1. Check if FastAPI is running"
Write-Host "2. Check FastAPI logs for errors"
Write-Host "3. Verify the endpoint URL is correct"
exit 1
}
Write-Host ""
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host "STEP 2: Waiting for Processing" -ForegroundColor Cyan
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host ""
Write-Host "⏳ Waiting 5 seconds for message to be processed by worker..." -ForegroundColor Yellow
Start-Sleep -Seconds 5
Write-Host ""
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host "STEP 3: Next Steps for Verification" -ForegroundColor Cyan
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host ""
Write-Host "To complete verification, check the following:" -ForegroundColor Yellow
Write-Host ""
Write-Host "1. 📊 NATS Dashboard:" -ForegroundColor Cyan
Write-Host " - URL: https://natsadmin.workolik.com"
Write-Host " - Login: admin / package@321#"
Write-Host " - Check Stream 'EVENTS' → Look for message count"
Write-Host " - Check Consumer 'worker_consumer' → Look for pending messages"
Write-Host ""
Write-Host "2. 📋 Worker Logs:" -ForegroundColor Cyan
Write-Host " Docker Compose:" -ForegroundColor Yellow
Write-Host " docker-compose logs -f nats-worker | grep deliverylog"
Write-Host ""
Write-Host " Kubernetes:" -ForegroundColor Yellow
Write-Host " kubectl logs -f deployment/nats-worker -n nats-backend | grep deliverylog"
Write-Host ""
Write-Host " Look for:" -ForegroundColor Yellow
Write-Host " ✅ '📨 Processing message for endpoint: /live/api/v2/deliveries/createdeliverylog'"
Write-Host " ✅ '➡️ Forwarding createdeliverylog payload: ...'"
Write-Host " ✅ '✅ Message processed and forwarded successfully'"
Write-Host ""
Write-Host "3. 📈 FastAPI Metrics:" -ForegroundColor Cyan
Write-Host " curl $FastAPIUrl/metrics | grep deliverylog"
Write-Host ""
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host "QUICK CHECK COMMANDS" -ForegroundColor Cyan
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host ""
Write-Host "# Check if workers are running (Docker Compose):" -ForegroundColor Yellow
Write-Host "docker-compose ps | grep worker"
Write-Host ""
Write-Host "# Check if workers are running (Kubernetes):" -ForegroundColor Yellow
Write-Host "kubectl get pods -n nats-backend | grep worker"
Write-Host ""
Write-Host "# View recent worker logs (Docker Compose):" -ForegroundColor Yellow
Write-Host "docker-compose logs --tail=50 nats-worker"
Write-Host ""
Write-Host "# View recent worker logs (Kubernetes):" -ForegroundColor Yellow
Write-Host "kubectl logs --tail=50 deployment/nats-worker -n nats-backend"
Write-Host ""
Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Gray
Write-Host ""
Write-Host "✅ Test request sent successfully!" -ForegroundColor Green
Write-Host ""
Write-Host "Test Order ID: $OrderId" -ForegroundColor Cyan
Write-Host "Use this ID to track the message through the system."
Write-Host ""
Write-Host "For detailed verification steps, see: HOW_TO_VERIFY_DELIVERY_LOGS.txt" -ForegroundColor Yellow

20
hub/Dockerfile Normal file
View File

@@ -0,0 +1,20 @@
# syntax=docker/dockerfile:1.5
FROM python:3.11-slim AS base
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# --- API image target ---
FROM base AS api
COPY app.py app.py
EXPOSE 8000
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"]
# --- Worker image target ---
FROM base AS worker
COPY worker.py worker.py
EXPOSE 9090
CMD ["python", "worker.py"]

9
hub/requirements.txt Normal file
View File

@@ -0,0 +1,9 @@
fastapi==0.104.1
uvicorn[standard]==0.24.0
nats-py==2.6.0
pydantic==2.5.0
prometheus-client==0.19.0
aiohttp==3.9.1
requests==2.31.0

362
manifests/alaska/alaska.yaml Normal file
View File

@@ -0,0 +1,362 @@
apiVersion: v1
kind: Namespace
metadata:
name: alaska
labels:
name: alaska
environment: production
app.kubernetes.io/name: alaska
app.kubernetes.io/managed-by: manuals
---
apiVersion: v1
kind: ConfigMap
metadata:
name: alaska-config
namespace: alaska
labels:
app.kubernetes.io/name: alaska-config
app.kubernetes.io/part-of: alaska
data:
NATS_URL: "nats://nats.workolik.com:4222"
LOG_LEVEL: "info"
ALLOWED_ORIGINS: "http://localhost:3001,http://localhost:3000,https://queue.workolik.com,https://console.nearlexpress.com"
EXTERNAL_BASE_URL: "https://jupiter.nearle.app"
WORKER_CONCURRENCY: "10"
RETRY_ATTEMPTS: "5"
RETRY_DELAY_SECONDS: "5"
NATS_STREAM: "EVENTS"
NATS_SUBJECT: "api.>"
NATS_CONSUMER: "worker_consumer"
---
apiVersion: v1
kind: Secret
metadata:
name: nats-credentials
namespace: alaska
labels:
app.kubernetes.io/name: nats-credentials
app.kubernetes.io/part-of: alaska
type: Opaque
stringData:
username: admin
password: package@321#
---
apiVersion: v1
kind: Secret
metadata:
name: external-endpoint-secrets
namespace: alaska
labels:
app.kubernetes.io/name: external-endpoint-secrets
app.kubernetes.io/part-of: alaska
type: Opaque
stringData:
api_key: "" # Add your API key securely
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: deliveries
namespace: alaska
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/instance: deliveries-primary
app.kubernetes.io/part-of: alaska
app.kubernetes.io/component: backend
spec:
serviceName: "deliveries" # Required for StatefulSet
replicas: 4
selector:
matchLabels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/instance: deliveries-primary
template:
metadata:
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/instance: deliveries-primary
app.kubernetes.io/part-of: alaska
app.kubernetes.io/component: backend
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8000"
prometheus.io/path: "/metrics"
spec:
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 2000
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- deliveries
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: deliveries
containers:
- name: deliveries
image: workolik360/alaska:v1.2.0
imagePullPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop:
- ALL
ports:
- containerPort: 8000
name: http
envFrom:
- configMapRef:
name: alaska-config
env:
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
resources:
requests:
memory: "256Mi"
cpu: "200m"
limits:
memory: "1Gi"
cpu: "1000m"
livenessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
readinessProbe:
httpGet:
path: /ready
port: 8000
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3
---
apiVersion: v1
kind: Service
metadata:
name: deliveries-service
namespace: alaska
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/part-of: alaska
spec:
type: ClusterIP
ports:
- port: 8000
targetPort: 8000
protocol: TCP
name: http
selector:
app.kubernetes.io/name: deliveries
app.kubernetes.io/instance: deliveries-primary
sessionAffinity: None
---
apiVersion: v1
kind: Service
metadata:
name: deliveries-loadbalancer
namespace: alaska
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/part-of: alaska
annotations:
# Service health check port for LoadBalancer
# service.kubernetes.io/klipper-lb.healthcheck-port: "8201"
spec:
type: NodePort
externalTrafficPolicy: Cluster
selector:
app.kubernetes.io/name: deliveries
app.kubernetes.io/instance: deliveries-primary
ports:
- name: http
port: 8201 # external LB port
targetPort: 8000 # API container port
nodePort: 30662
protocol: TCP
- name: https
port: 8441 # optional HTTPS passthrough
targetPort: 8000
protocol: TCP
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: deliveries-pdb
namespace: alaska
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/part-of: alaska
spec:
minAvailable: 50%
selector:
matchLabels:
app.kubernetes.io/name: deliveries
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: deliveries-hpa
namespace: alaska
labels:
app.kubernetes.io/name: deliveries
app.kubernetes.io/part-of: alaska
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: StatefulSet
name: deliveries
minReplicas: 4
maxReplicas: 20
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 60
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 70
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: gateway
namespace: alaska
labels:
app.kubernetes.io/name: gateway
app.kubernetes.io/part-of: alaska
spec:
gatewayClassName: standard
listeners:
- name: http
protocol: HTTP
port: 8201
allowedRoutes:
namespaces:
from: All
- name: https
protocol: HTTPS
port: 8441
allowedRoutes:
namespaces:
from: All
tls:
mode: Terminate
certificateRefs:
- name: deliveries-tls-cert
- name: nearle-http
protocol: HTTP
port: 8202
allowedRoutes:
namespaces:
from: All
- name: nearle-https
protocol: HTTPS
port: 8442
allowedRoutes:
namespaces:
from: All
tls:
mode: Terminate
certificateRefs:
- name: nearle-tls-cert
namespace: nearle # Must copy secret to alaska or use ReferenceGrant. For now assume secret is in Alaska or copied.
# Actually, simpler: Use 'nearle-tls-cert' but putting secret in alaska namespace is required for cross-namespace ref usually unless ReferenceGrant used.
# Let's keep it simple: We will COPY the secret to 'alaska' namespace.
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: deliveries-route
namespace: alaska
labels:
app.kubernetes.io/name: deliveries-route
app.kubernetes.io/part-of: alaska
spec:
parentRefs:
- name: gateway
namespace: alaska
hostnames:
- "queue.workolik.com"
rules:
- matches:
- path:
type: PathPrefix
value: /live
backendRefs:
- name: deliveries-service
port: 8000
weight: 100
backendRefs:
- name: deliveries-service
port: 8000
weight: 100
- matches:
- path:
type: PathPrefix
value: /live/api/v1/mob/orders
backendRefs:
- name: fiesta
namespace: nearle
port: 80
weight: 100
- matches:
- path:
type: PathPrefix
value: /live/api/v1/web/products
backendRefs:
- name: fiesta
namespace: nearle
port: 80
weight: 100
- matches:
- path:
type: PathPrefix
value: /health
backendRefs:
- name: deliveries-service
port: 8000
weight: 100

328
manifests/alaska/k8s-dashboard.yaml Normal file
View File

@@ -0,0 +1,328 @@
# Kubernetes Dashboard - Official Web UI
# Deploy with: kubectl apply -f manifests/alaska/k8s-dashboard.yaml
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
name: admin-user
namespace: kubernetes-dashboard
annotations:
kubernetes.io/service-account.name: "admin-user"
type: kubernetes.io/service-account-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
labels:
k8s-app: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: kubernetesui/dashboard:v2.7.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
- --enable-skip-login
- --enable-insecure-login
- --insecure-port=9090
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
- name: tmp-volume
mountPath: /tmp
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
periodSeconds: 10
failureThreshold: 3
resources:
limits:
cpu: 200m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
---
apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
labels:
k8s-app: kubernetes-dashboard
spec:
type: ClusterIP
ports:
- port: 443
targetPort: 8443
protocol: TCP
name: https
- port: 9090
targetPort: 9090
protocol: TCP
name: http
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kubernetes-dashboard
rules:
- apiGroups: [""]
resources: ["*"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources: ["*"]
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources: ["*"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kubernetes-dashboard-secrets
namespace: kubernetes-dashboard
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "update", "create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubernetes-dashboard-secrets
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard-secrets
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: "" # Will be auto-generated by dashboard
---
apiVersion: v1
kind: ConfigMap
metadata:
name: dashboard-proxy-config
namespace: kubernetes-dashboard
data:
nginx.conf: |
events {
worker_connections 1024;
}
http {
upstream k8s_dashboard {
server kubernetes-dashboard:443;
}
server {
listen 8083;
server_name _;
location / {
proxy_pass https://k8s_dashboard;
proxy_ssl_verify off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
}
}
}
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: dashboard-proxy
namespace: kubernetes-dashboard
labels:
app.kubernetes.io/name: dashboard-proxy
app.kubernetes.io/part-of: kubernetes-dashboard
spec:
serviceName: "dashboard-proxy"
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dashboard-proxy
template:
metadata:
labels:
app.kubernetes.io/name: dashboard-proxy
app.kubernetes.io/part-of: kubernetes-dashboard
spec:
containers:
- name: nginx
image: nginx:alpine
ports:
- containerPort: 8083
volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
volumes:
- name: nginx-config
configMap:
name: dashboard-proxy-config
---
apiVersion: v1
kind: Service
metadata:
name: dashboard-proxy
namespace: kubernetes-dashboard
labels:
app.kubernetes.io/name: dashboard-proxy
spec:
type: NodePort
selector:
app.kubernetes.io/name: dashboard-proxy
ports:
- port: 8083
targetPort: 8083
nodePort: 30826
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: dashboard-loadbalancer
namespace: kubernetes-dashboard
labels:
app.kubernetes.io/name: kubernetes-dashboard
app.kubernetes.io/component: loadbalancer
spec:
type: NodePort
selector:
k8s-app: kubernetes-dashboard
ports:
- name: http
port: 9090
targetPort: 9090 # Dashboard HTTP port
nodePort: 30827 # Fixed NodePort for nginx proxy
protocol: TCP

19
manifests/core/core-config.yaml Normal file
View File

@@ -0,0 +1,19 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: core-config
namespace: core
labels:
app.kubernetes.io/name: core-config
app.kubernetes.io/part-of: core
data:
NATS_URL: "nats://66.116.226.161:4222"
LOG_LEVEL: "info"
ALLOWED_ORIGINS: "http://localhost:3001,http://localhost:3000,https://queue.workolik.com,https://console.nearlexpress.com"
EXTERNAL_BASE_URL: "https://jupiter.nearle.app"
WORKER_CONCURRENCY: "10"
RETRY_ATTEMPTS: "5"
RETRY_DELAY_SECONDS: "5"
NATS_STREAM: "EVENTS"
NATS_SUBJECT: "api.>"
NATS_CONSUMER: "worker_consumer"

6
manifests/core/core-namespace.yaml Normal file
View File

@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: core
labels:
name: core

25
manifests/core/core-secrets.yaml Normal file
View File

@@ -0,0 +1,25 @@
apiVersion: v1
kind: Secret
metadata:
name: nats-credentials
namespace: core
labels:
app.kubernetes.io/name: nats-credentials
app.kubernetes.io/part-of: core
type: Opaque
stringData:
username: admin
password: package@321#
---
apiVersion: v1
kind: Secret
metadata:
name: external-endpoint-secrets
namespace: core
labels:
app.kubernetes.io/name: external-endpoint-secrets
app.kubernetes.io/part-of: core
type: Opaque
stringData:
api_key: "" # Add your API key securely

76
manifests/core/ingress-unified.yaml Normal file
View File

@@ -0,0 +1,76 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: queue-ingress
namespace: alaska
annotations:
traefik.ingress.kubernetes.io/router.middlewares: alaska-common-cors@kubernetescrd
traefik.ingress.kubernetes.io/router.tls: "true"
# Cert resolver if you have it configured globally in Traefik (usually k3s-provided)
# traefik.ingress.kubernetes.io/router.tls.certresolver: "letsencrypt"
spec:
rules:
- host: queue.workolik.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: deliveries-service
port:
number: 8000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nearle-ingress
namespace: nearle
annotations:
traefik.ingress.kubernetes.io/router.tls: "true"
spec:
rules:
- host: jupiter.nearle.app
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jupiter-cors-proxy
port:
number: 80
- host: fiesta.nearle.app
http:
paths:
- path: /live/api/v1/mob/orders/createorder
pathType: Prefix
backend:
service:
name: fiesta
port:
number: 8000
- path: /live/api/v1/web/products/create
pathType: Prefix
backend:
service:
name: fiesta
port:
number: 8000
- path: /
pathType: Prefix
backend:
service:
name: fiesta
port:
number: 80
- host: atlantis.nearle.app
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: atlantis
port:
number: 80

12
manifests/core/kustomization.yaml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: core
resources:
- core-namespace.yaml
- core-secrets.yaml
- core-config.yaml
- worker-script.yaml
- workers.yaml
- worker-pdb.yaml

41
manifests/core/traefik-middlewares.yaml Normal file
View File

@@ -0,0 +1,41 @@
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: common-cors
namespace: alaska
spec:
headers:
accessControlAllowMethods:
- GET
- POST
- PUT
- PATCH
- DELETE
- OPTIONS
accessControlAllowOriginList:
- "*"
accessControlAllowHeaders:
- "*"
accessControlMaxAge: 600
addVaryHeader: true
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: common-cors
namespace: nearle
spec:
headers:
accessControlAllowMethods:
- GET
- POST
- PUT
- PATCH
- DELETE
- OPTIONS
accessControlAllowOriginList:
- "*"
accessControlAllowHeaders:
- "*"
accessControlMaxAge: 600
addVaryHeader: true

13
manifests/core/worker-pdb.yaml Normal file
View File

@@ -0,0 +1,13 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: worker-pdb
namespace: core
labels:
app.kubernetes.io/name: worker
app.kubernetes.io/part-of: core
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: worker

261
manifests/core/worker-script.yaml Normal file
View File

@@ -0,0 +1,261 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: worker-script
namespace: core
data:
worker.py: |
#!/usr/bin/env python3
"""
NATS Worker - "New Idea" Implementation
- Single Domain per Worker (Stream + Consumer)
- Persistent HTTP Session (Connection Pooling)
- JetStream Native Retries (NAK)
- No internal retry loops
"""
import os
import json
import asyncio
import aiohttp
import nats
from nats.errors import TimeoutError as NatsTimeoutError
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import signal
import sys
from typing import Any, Dict, List, Union, Tuple
# Prometheus metrics
messages_processed = Counter('worker_messages_processed_total', 'Total messages processed', ['status', 'endpoint'])
message_duration = Histogram('worker_message_duration_seconds', 'Message processing duration', ['endpoint'])
messages_in_flight = Gauge('worker_messages_in_flight', 'Messages currently being processed')
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
# Domain Configuration (Env Vars from Deployment)
NATS_STREAM = os.getenv("NATS_STREAM", "ORDERS")
NATS_CONSUMER = os.getenv("NATS_CONSUMER", "orders-worker")
# Supports comma-separated subjects: "api.v1.orders.*,api.v2.orders.*"
FILTER_SUBJECTS = os.getenv("FILTER_SUBJECT", "api.v1.mob.orders.*").split(",")
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "10"))
BASE_URL = os.getenv("EXTERNAL_BASE_URL", "https://jupiter.nearle.app")
# Endpoint mapping is still useful for constructing the target URL
ENDPOINT_MAPPING = {
"/live/api/v1/deliveries/createdeliveries": f"{BASE_URL}/live/api/v1/deliveries/createdeliveries",
"/live/api/v1/deliveries/updatedelivery": f"{BASE_URL}/live/api/v1/deliveries/updatedelivery",
"/live/api/v2/partners/createriderlog": f"{BASE_URL}/live/api/v2/partners/createriderlog",
"/live/api/v2/deliveries/createdeliverylog": f"{BASE_URL}/live/api/v2/deliveries/createdeliverylog",
"/live/api/v2/partners/createbreaklog": f"{BASE_URL}/live/api/v2/partners/createbreaklog",
"/live/api/v2/partners/updatebreaklog": f"{BASE_URL}/live/api/v2/partners/updatebreaklog",
"/live/api/v1/mob/orders/createorder": f"{BASE_URL}/live/api/v1/mob/orders/createorder",
"/live/api/v1/web/products/create": f"{BASE_URL}/live/api/v1/web/products/create",
"/live/api/v1/mob/customers/login": f"{BASE_URL}/live/api/v1/mob/customers/login",
"/live/api/v1/mob/customers/create": f"{BASE_URL}/live/api/v1/mob/customers/create",
}
# Global State
nc = None
js = None
session: aiohttp.ClientSession = None
running = True
semaphore = None # Initialized in main
def signal_handler(sig, frame):
global running
print("\n?? Shutdown signal received...")
running = False
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
async def forward_to_external(endpoint: str, payload: dict, api_key: str = None, method: str = "POST") -> tuple[str, Any]:
"""
Forward to external API using global session.
Returns (status_code, response_data).
Does NOT retry internally.
"""
external_url = ENDPOINT_MAPPING.get(endpoint)
if not external_url:
print(f"?? Unknown endpoint: {endpoint}, dropping.")
return "DROP", None
headers = {"Content-Type": "application/json"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
data_to_forward = payload.get("data", payload)
# Payload Normalization for logs
if endpoint == "/live/api/v2/deliveries/createdeliverylog":
if isinstance(data_to_forward, dict):
data_to_forward = [data_to_forward]
elif not isinstance(data_to_forward, list):
data_to_forward = [data_to_forward]
http_method = method.upper() if method else "POST"
try:
# We use the global 'session' here
async with session.request(
method=http_method,
url=external_url,
json=data_to_forward if http_method != "GET" else None,
params=data_to_forward if http_method == "GET" else None,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
body = None
try:
body = await response.json()
except:
body = await response.text()
if 200 <= response.status < 300:
return "OK", body
elif response.status >= 500:
print(f"?? Server Error {response.status} from {endpoint}: {body}")
return "RETRY", body
else:
print(f"?? Client Error {response.status} from {endpoint}: {body}")
return "DROP", body # 4xx errors should generally not be retried infinitely
except asyncio.TimeoutError:
print(f"?? Timeout calling {endpoint}")
return "RETRY", None
except Exception as e:
print(f"?? Network/Client Error calling {endpoint}: {e}")
return "RETRY", None
async def process_message(msg):
async with semaphore:
messages_in_flight.inc()
start_time = asyncio.get_event_loop().time()
endpoint = "unknown"
try:
payload = json.loads(msg.data.decode())
endpoint = payload.get("endpoint", payload.get("original_path", "unknown"))
http_method = payload.get("method", "POST")
print(f"?? Processing {endpoint}")
api_key = os.getenv("EXTERNAL_ENDPOINT_API_KEY", "")
# Single attempt
status, response_data = await forward_to_external(endpoint, payload, api_key, http_method)
if status == "OK":
await msg.ack()
messages_processed.labels(status="success", endpoint=endpoint).inc()
# Request-Reply Logic
if msg.reply:
reply_payload = json.dumps(response_data) if isinstance(response_data, (dict, list)) else str(response_data)
await nc.publish(msg.reply, reply_payload.encode())
elif status == "DROP":
# Unrecoverable error or unknown endpoint
await msg.term() # Terminate stops redelivery
messages_processed.labels(status="dropped", endpoint=endpoint).inc()
else: # RETRY
# Let JetStream handle backoff
await msg.nak(delay=2) # Custom delay before redelivery if desired, or just nak()
messages_processed.labels(status="retried", endpoint=endpoint).inc()
except json.JSONDecodeError:
print("?? Invalid JSON, terminating message")
await msg.term()
except Exception as e:
print(f"? Critical Worker Error: {e}")
await msg.nak()
finally:
messages_in_flight.dec()
duration = asyncio.get_event_loop().time() - start_time
message_duration.labels(endpoint=endpoint).observe(duration)
async def main():
global nc, js, session, semaphore
print(f"?? Starting Worker for Domain: {NATS_CONSUMER}")
print(f"?? Stream: {NATS_STREAM}, Subjects: {FILTER_SUBJECTS}")
print(f"?? Concurrency: {WORKER_CONCURRENCY}")
semaphore = asyncio.Semaphore(WORKER_CONCURRENCY)
session = aiohttp.ClientSession()
try:
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=-1
)
js = nc.jetstream()
print("? Connected to NATS JetStream")
# Create Pull Subscriptions for each filter subject
# All sharing the same Consumer Name ensuring load balancing if multiple pods run this
subs = []
for subject in FILTER_SUBJECTS:
subject = subject.strip()
if not subject: continue
# FIX: On WorkQueue streams, we cannot reuse the same durable name for different filters
# We append a sanitized version of the subject to ensure uniqueness per filter
clean_subject_suffix = subject.replace(".", "_").replace("*", "all").replace(">", "all")
durable_name = f"{NATS_CONSUMER}_{clean_subject_suffix}"
print(f"?? Subscribing to {subject} on durable consumer '{durable_name}'")
try:
sub = await js.pull_subscribe(
subject,
durable=durable_name,
stream=NATS_STREAM
)
subs.append(sub)
except Exception as e:
print(f"? Failed to subscribe to {subject}: {e}")
if not subs:
print("? No active subscriptions!")
return
start_http_server(9090)
print("?? Metrics on :9090")
while running:
# Poll all subscriptions
for sub in subs:
try:
# Batch size 10, short timeout to keep loop responsive
msgs = await sub.fetch(10, timeout=0.5)
for m in msgs:
asyncio.create_task(process_message(m))
except NatsTimeoutError:
pass
except Exception as e:
print(f"?? Fetch Error: {e}")
await asyncio.sleep(1)
# Small sleep to prevent tight loop if no messages
# await asyncio.sleep(0.01)
except Exception as e:
print(f"?? Fatal Error: {e}")
finally:
if session:
await session.close()
if nc:
await nc.close()
print("?? Worker Shutdown")
if __name__ == "__main__":
asyncio.run(main())

129
manifests/core/worker-statefulset.yaml Normal file
View File

@@ -0,0 +1,129 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker
namespace: core
labels:
app.kubernetes.io/name: worker
app.kubernetes.io/instance: worker-primary
app.kubernetes.io/part-of: core
app.kubernetes.io/component: worker
spec:
serviceName: "worker" # Required for StatefulSet
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: worker
app.kubernetes.io/instance: worker-primary
template:
metadata:
labels:
app.kubernetes.io/name: worker
app.kubernetes.io/instance: worker-primary
app.kubernetes.io/part-of: core
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 2000
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
capabilities:
drop:
- ALL
ports:
- containerPort: 9090
name: metrics
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_ENDPOINT_API_KEY
valueFrom:
secretKeyRef:
name: external-endpoint-secrets
key: api_key
optional: true
resources:
requests:
memory: "256Mi"
cpu: "200m"
limits:
memory: "512Mi"
cpu: "500m"
livenessProbe:
httpGet:
path: /metrics
port: 9090
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 3
readinessProbe:
httpGet:
path: /metrics
port: 9090
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
volumes:
- name: worker-script-vol
configMap:
name: worker-script

542
manifests/core/workers.yaml Normal file
View File

@@ -0,0 +1,542 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker-orders
namespace: core
labels:
app.kubernetes.io/name: worker-orders
app.kubernetes.io/component: worker
spec:
serviceName: "worker-orders"
replicas: 3
selector:
matchLabels:
app.kubernetes.io/name: worker-orders
template:
metadata:
labels:
app.kubernetes.io/name: worker-orders
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker-orders
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker-orders
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_STREAM
value: "ORDERS"
- name: NATS_CONSUMER
value: "orders-worker"
- name: FILTER_SUBJECT
value: "api.v1.mob.orders.createorder"
- name: WORKER_CONCURRENCY
value: "20"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_ENDPOINT_API_KEY
valueFrom:
secretKeyRef:
name: external-endpoint-secrets
key: api_key
optional: true
- name: EXTERNAL_BASE_URL
value: "http://10.43.229.168"
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "500m"
ports:
- containerPort: 9090
name: metrics
volumes:
- name: worker-script-vol
configMap:
name: worker-script
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker-deliveries
namespace: core
labels:
app.kubernetes.io/name: worker-deliveries
app.kubernetes.io/component: worker
spec:
serviceName: "worker-deliveries"
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: worker-deliveries
template:
metadata:
labels:
app.kubernetes.io/name: worker-deliveries
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker-deliveries
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker-deliveries
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_STREAM
value: "DELIVERIES"
- name: NATS_CONSUMER
value: "deliveries-worker"
- name: FILTER_SUBJECT
value: "api.v1.deliveries.createdeliveries,api.v1.deliveries.updatedelivery,api.v2.deliveries.createdeliverylog"
- name: WORKER_CONCURRENCY
value: "10"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_BASE_URL
value: "http://10.43.224.63"
resources:
requests:
memory: "128Mi"
cpu: "80m"
limits:
memory: "256Mi"
cpu: "400m"
ports:
- containerPort: 9090
name: metrics
volumes:
- name: worker-script-vol
configMap:
name: worker-script
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker-customers
namespace: core
labels:
app.kubernetes.io/name: worker-customers
app.kubernetes.io/component: worker
spec:
serviceName: "worker-customers"
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: worker-customers
template:
metadata:
labels:
app.kubernetes.io/name: worker-customers
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker-customers
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker-customers
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_STREAM
value: "CUSTOMERS"
- name: NATS_CONSUMER
value: "customers-worker"
- name: FILTER_SUBJECT
value: "api.v1.mob.customers.login,api.v1.mob.customers.create"
- name: WORKER_CONCURRENCY
value: "30"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_ENDPOINT_API_KEY
valueFrom:
secretKeyRef:
name: external-endpoint-secrets
key: api_key
optional: true
- name: EXTERNAL_BASE_URL
value: "http://10.43.229.168"
resources:
requests:
memory: "128Mi"
cpu: "60m"
limits:
memory: "256Mi"
cpu: "300m"
ports:
- containerPort: 9090
name: metrics
volumes:
- name: worker-script-vol
configMap:
name: worker-script
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker-rider-logs
namespace: core
labels:
app.kubernetes.io/name: worker-rider-logs
app.kubernetes.io/component: worker
spec:
serviceName: "worker-rider-logs"
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: worker-rider-logs
template:
metadata:
labels:
app.kubernetes.io/name: worker-rider-logs
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker-rider-logs
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker-rider-logs
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_STREAM
value: "RIDER"
- name: NATS_CONSUMER
value: "rider-logs-worker"
- name: FILTER_SUBJECT
value: "api.v2.partners.createriderlog,api.v2.partners.createbreaklog,api.v2.partners.updatebreaklog"
- name: WORKER_CONCURRENCY
value: "10"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_ENDPOINT_API_KEY
valueFrom:
secretKeyRef:
name: external-endpoint-secrets
key: api_key
optional: true
- name: EXTERNAL_BASE_URL
value: "http://10.43.224.63"
resources:
requests:
memory: "128Mi"
cpu: "40m"
limits:
memory: "128Mi"
cpu: "200m"
ports:
- containerPort: 9090
name: metrics
volumes:
- name: worker-script-vol
configMap:
name: worker-script
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: worker-products
namespace: core
labels:
app.kubernetes.io/name: worker-products
app.kubernetes.io/component: worker
spec:
serviceName: "worker-products"
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: worker-products
template:
metadata:
labels:
app.kubernetes.io/name: worker-products
app.kubernetes.io/component: worker
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
prometheus.io/path: "/metrics"
spec:
tolerations:
- key: dedicated
operator: Equal
value: workers
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/worker
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- worker-products
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app.kubernetes.io/name: worker-products
containers:
- name: worker
image: workolik360/nats-worker:v1.1.0
imagePullPolicy: IfNotPresent
command: ["python3", "-u", "/scripts/worker.py"]
volumeMounts:
- name: worker-script-vol
mountPath: /scripts
envFrom:
- configMapRef:
name: core-config
env:
- name: NATS_STREAM
value: "PRODUCTS"
- name: NATS_CONSUMER
value: "products-worker"
- name: FILTER_SUBJECT
value: "api.v1.web.products.create"
- name: WORKER_CONCURRENCY
value: "5"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: EXTERNAL_ENDPOINT_API_KEY
valueFrom:
secretKeyRef:
name: external-endpoint-secrets
key: api_key
optional: true
- name: EXTERNAL_BASE_URL
value: "http://10.43.229.168"
resources:
requests:
memory: "128Mi"
cpu: "40m"
limits:
memory: "256Mi"
cpu: "200m"
ports:
- containerPort: 9090
name: metrics
volumes:
- name: worker-script-vol
configMap:
name: worker-script

13
manifests/doormile/firebase-service-account.json Normal file
View File

@@ -0,0 +1,13 @@
{
"type": "service_account",
"project_id": "doormile-abee7",
"private_key_id": "66e9b8b66fb40961271f095dd924935eff19452b",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDLBD8DlrQwvwo8\ngkW9j/z1+eLPZYAT2TA4xWT1l+i8LV5he8wZ2TjxJT385OGS+zjIVUHQKtvStdDB\no9/0NXnSC6n88puNhQp9xCRjxEU+FLiSnqJGRUzSO4pu+0TuukGsouDII6QhJFNf\ndmtTOMmjV4V+wAOLfbE3qI5kZQ5trqf8wNBPZKxp9fhrqHdP080HUbOs14jiSrHl\n3OYRj0ZhDTsavboV6QTbdJn5ivH19nboFfdzuJrS5ApMhLqIGdoSEqjjQj8t6f6I\nm1RgQMfztIZLRPliA5MyJieEDZZdy5mmDQDnqCP6zFtH+WRVBe9Lo2A7F+Rf1vv4\n8/5FRd3HAgMBAAECggEACmEEsbPGMYnKxa4pT5gpaA/m6xB23EzpvLVGxJGIGfq9\nzQENvbKPyTBMu32eFKwYSpGlRDW0uFCIRCYTIIKNYFItVhu3HSSUlTpuW9VgbtyT\nVRecFziaxVK68JKTAxttmRxYnpLH5NPdGU/OC9qm4F1smz7Iz5xU75ID+Zj7BFtm\nkjZsgU9ALT7ikm205KhjYepc16ZQODIvvGW8vdNwxaxNl+5+RpkXdq+NK3KiAjK4\noqsuvDRRcchHttXU/qhri7f4nT0VBIuMnPDtRWGEaWtgCt4urLFAD4fZE91i1SHe\n7vV0iYdS9lzjrwl3Umzcf4k0pSPN8Sky1ZjN9hvF5QKBgQD+p2NMePwHY4uksA98\nVWT6y6haM+rIGxSsBL06UnCGO4E1gEWd++G5osDNc9hQk29xFg64pyIPKovw/Qez\nZ+gyCySC8KJoC4q740mxqn1EmANHXAvxGUMgPHmeiX+j3i1MwJWG1w7uyLwQ2216\nEgHy6Mqzk3jLUnD8vXf08CAUxQKBgQDMFvrRwRwpqsLR2yHWiu8uWykU276m1mKI\n8saR/iQ+jXQ9NZsgVdOjaV9EIv7jLwzSwEcXHVh9+SB5pnlCOhovGmmiSIdLVr5f\n1XOyWY6n2NzM6tm2RXjWOnwpVviOtdAwVVWXQNwaNbRLMZZkv+0bpOwk7i/ONv2z\n1bNP5pHJGwKBgQCo8LeJxkG9TScZTMwZOjXRxEaeAQ9jTcP7EvHOvV51Twi7S6vj\n0XTAyN6L7qy1V/p41Z9SlV6dHSbV66euN+LtQEMqsyWrTOzvFSyQt4B/fzG9p60s\neAv2WOh+as3JEt6Oqr0IRahcdcN+k3Qfpc6xLgs78zwR91GJ+tCYKK2/2QKBgBNX\nfi/Q0F12jH4c+Wj+w8ot44fRh5ECMlZuQ7lkT4UKHOEMKoZ2+9a5zsiDVIqtU9bJ\nWDNtmYxexuAgkc9f+ElMhIRXwVK3htVIQm29pRJtMAfPnhBzJw+OfWPhlS7ZNkDO\nZY0vHWzyeALEUU99DPiYubSKSkeN2J9pelPxMxHPAoGBAOOTd5XJRQOnFpd0eVwE\nlJ65zeQyT82V1OX/OPLKOcFwzNNKYF0Toto98nLK5rqqAGNUPp0fBMdUi7b76i79\nwve7aEZtCQWbhQsOUXsWocZyr8w5EexYtbxWa6q7jzmJaL/UfagE8pZWUI2k59CN\nRajHiJG83UxGMUN8KbSgv7h7\n-----END PRIVATE KEY-----\n",
"client_email": "firebase-adminsdk-fbsvc@doormile-abee7.iam.gserviceaccount.com",
"client_id": "108510619961572391178",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-fbsvc%40doormile-abee7.iam.gserviceaccount.com",
"universe_domain": "googleapis.com"
}

89
manifests/doormile/miletruth.yaml Normal file
View File

@@ -0,0 +1,89 @@
apiVersion: v1
kind: Namespace
metadata:
name: doormile
Read 1 file
Found the critical issue — INTERNAL_API_KEY not set in the manifest means
all /internal/* endpoode explicitly rejects empty keys). That plus the NATS_URL duplicate.
Here's the corrected miletruth.yaml with all missing env vars added. Run
this on the server:
cat > /root/kuberneteh.yaml << 'EOF'
apiVersion: v1
kind: Namespace
metadata:
name: doormile
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: doormile
namespace: doormile
spec:
serviceName: "doormile-service"
replicas: 3
selector:
matchLabels:
app: doormile
app-group: doormile-api
template:
metadata:
labels:
app: doormile
app-group: do
spec:
containers:
- name: doormile
image: doormi
imagePullPolicy: Always
env:
- name: ENV
value: "production"
- name: APP_PORT
value: "8081"
- name: DB_HO
value: "31.97.228.132"
- name: DB_PO
value: "5433"
- name: DB_NA
value: "logistics"
- name: DB_US
value: "admin"
- name: DB_PASSWORD
value: "Pac
- name: REDIS_HOST
value: "31.97.228.132"
- name: REDIS
value: "6379"
- name: REDIS_USER
value: "adm
- name: REDIS_PASSWORD
value: "Package@321#"
- name: JWT_S
value: "DoormileSuperSecretJWTKey2026!"
- name: NATS_
value: "nats://66.116.226.161:4223"
- name: NATS_
value: "doormile"
- name: NATS_PASSWORD
value: "Pac
- name: INTERNAL_API_KEY
value: "doormile-internal-2024"
---
apiVersion: v1
kind: Service
metadata:
name: doormile-service
namespace: doormile
spec:
type: NodePort
selector:
app-group: doormile-api
ports:
- protocol: TCP
port: 8081
targetPort: 808
nodePort: 30830

61
manifests/nearle/atlantis-sts.yaml Normal file
View File

@@ -0,0 +1,61 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: atlantis
namespace: nearle
labels:
app: atlantis
spec:
serviceName: "atlantis"
replicas: 2
selector:
matchLabels:
app: atlantis
template:
metadata:
labels:
app: atlantis
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: atlantis
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: atlantis
containers:
- name: backend
image: nearlecommerce/atlantis:v0.0.41
imagePullPolicy: Always
ports:
- containerPort: 3000
env:
- name: PORT
value: "3000"
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets

16
manifests/nearle/atlantis-svc.yaml Normal file
View File

@@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: atlantis
namespace: nearle
labels:
app: atlantis
spec:
type: NodePort
ports:
- port: 80
targetPort: 3000
nodePort: 30825
protocol: TCP
selector:
app: atlantis

514
manifests/nearle/fiesta-gateway.yaml Normal file
View File

@@ -0,0 +1,514 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: fiesta-gateway-script
namespace: nearle
data:
app.py: |
#!/usr/bin/env python3
"""
FastAPI application with multiple endpoints that publish to NATS JetStream
Each endpoint corresponds to an external API that workers will forward to
"""
import os
import json
import asyncio
from fastapi import FastAPI, HTTPException, Request, Body
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request as StarletteRequest
from starlette.responses import Response as StarletteResponse
from pydantic import BaseModel
import nats
import nats.errors
from prometheus_client import Counter, Histogram, generate_latest, REGISTRY
from starlette.responses import Response
import uvicorn
from typing import Optional, Dict, Any, List, Union
# Prometheus metrics
request_count = Counter('http_requests_total', 'Total HTTP requests', ['method', 'endpoint', 'status'])
request_duration = Histogram('http_request_duration_seconds', 'HTTP request duration', ['method', 'endpoint'])
app = FastAPI(title="NATS Backend API - Multi-Endpoint", version="1.0.0")
# Custom CORS middleware to ensure headers are always added
class CORSHeaderMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: StarletteRequest, call_next):
origin = request.headers.get("origin", "*")
response = await call_next(request)
response.headers["Access-Control-Allow-Origin"] = origin
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Credentials"] = "false"
response.headers["Access-Control-Max-Age"] = "600"
return response
# Add custom CORS middleware first
app.add_middleware(CORSHeaderMiddleware)
# Also add FastAPI's CORS middleware as backup
app.add_middleware(
CORSMiddleware,
allow_origin_regex=r".*", # Match all origins
allow_credentials=False,
allow_methods=["*"], # Allow all HTTP methods
allow_headers=["*"], # Allow all headers
expose_headers=["*"], # Expose all headers
max_age=600,
)
# NATS connection (will be initialized on startup)
nc = None
js = None
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
# Endpoint to NATS subject mapping
ENDPOINT_ROUTES = {
"/live/api/v1/deliveries/createdeliveries": "api.v1.deliveries.createdeliveries",
"/live/api/v1/deliveries/updatedelivery": "api.v1.deliveries.updatedelivery",
"/live/api/v2/partners/createriderlog": "api.v2.partners.createriderlog",
"/live/api/v2/deliveries/createdeliverylog": "api.v2.deliveries.createdeliverylog",
"/live/api/v2/partners/createbreaklog": "api.v2.partners.createbreaklog",
"/live/api/v2/partners/updatebreaklog": "api.v2.partners.updatebreaklog",
"/live/api/v1/mob/orders/createorder": "api.v1.mob.orders.createorder",
"/live/api/v1/web/products/create": "api.v1.web.products.create",
# Customer Endpoints (Sync)
"/live/api/v1/mob/customers/login": "api.v1.mob.customers.login",
"/live/api/v1/mob/customers/create": "api.v1.mob.customers.create",
}
@app.on_event("startup")
async def startup():
"""Initialize NATS connection on startup"""
global nc, js
try:
print(f"Connecting to NATS at {NATS_URL}...")
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=10
)
js = nc.jetstream()
print("✅ Connected to NATS JetStream")
print(f"✅ Configured {len(ENDPOINT_ROUTES)} endpoint routes")
except Exception as e:
print(f"❌ Failed to connect to NATS: {e}")
raise
@app.on_event("shutdown")
async def shutdown():
"""Close NATS connection on shutdown"""
global nc
if nc:
await nc.close()
print("NATS connection closed")
@app.options("/{full_path:path}")
async def options_handler(full_path: str, request: Request):
"""Handle OPTIONS requests for CORS preflight"""
origin = request.headers.get("origin")
return JSONResponse(
status_code=200,
content={},
headers={
"Access-Control-Allow-Origin": origin if origin else "*",
"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
"Access-Control-Allow-Headers": "*",
"Access-Control-Allow-Credentials": "false",
"Access-Control-Max-Age": "600",
}
)
@app.get("/health")
async def health():
"""Health check endpoint"""
return {"status": "healthy", "nats_connected": nc.is_connected if nc else False}
@app.get("/ready")
async def ready():
"""Readiness check endpoint"""
if nc and nc.is_connected:
return {"status": "ready"}
raise HTTPException(status_code=503, detail="Not ready")
async def publish_to_nats(endpoint: str, data: Union[Dict[str, Any], List[Dict[str, Any]]], request_method: str = "POST"):
"""Publish message to NATS with endpoint metadata"""
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint
}
# Get NATS subject for this endpoint
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if js:
try:
ack = await js.publish(subject, json.dumps(payload).encode())
return ack.seq
except Exception as e:
print(f"❌ Failed to publish to NATS: {e}")
raise HTTPException(status_code=500, detail=f"Failed to publish message: {str(e)}")
else:
raise HTTPException(status_code=503, detail="NATS not connected")
async def publish_request_to_nats(endpoint: str, data: Dict[str, Any], request_method: str = "POST", timeout: int = 10):
"""
Publish to NATS and WAIT for a reply (Request-Reply pattern).
Used for synchronous endpoints like Login.
"""
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint
}
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if not js:
raise HTTPException(status_code=503, detail="NATS by connected")
try:
# Create a unique inbox for the reply
inbox = nc.new_inbox()
# Subscribe to the inbox first
sub = await nc.subscribe(inbox, max_msgs=1)
# Publish request with reply inbox
# Note: We use js.publish to ensure it goes to the Stream (Queue), but attach a reply subject
await js.publish(subject, json.dumps(payload).encode(), reply=inbox)
# Wait for valid response
try:
msg = await sub.next_msg(timeout=timeout)
response_data = json.loads(msg.data.decode())
return response_data
except nats.errors.TimeoutError:
raise HTTPException(status_code=504, detail="Gateway Timeout: Upstream service did not respond in time")
finally:
await sub.unsubscribe()
except HTTPException:
raise
except Exception as e:
print(f"❌ Failed to request from NATS: {e}")
raise HTTPException(status_code=500, detail=f"RPC Error: {str(e)}")
# Endpoint 1: Update Delivery (v1) - External API requires PUT
@app.put("/live/api/v1/deliveries/updatedelivery")
async def update_delivery_v1(data: Dict[str, Any], request: Request):
"""Update Delivery endpoint - forwards to NATS (PUT only, as external API requires PUT)"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/updatedelivery"
method = "PUT" # Always use PUT for this endpoint
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method="PUT", endpoint=endpoint).observe(duration)
request_count.labels(method="PUT", endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method="PUT", endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint: Create Deliveries (v1)
@app.post("/live/api/v1/deliveries/createdeliveries")
async def create_deliveries_v1(data: Union[Dict[str, Any], List[Dict[str, Any]]], request: Request):
"""Create Deliveries endpoint - forwards payload to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/createdeliveries"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 2: Create Rider Log (v2)
@app.post("/live/api/v2/partners/createriderlog")
async def create_rider_log_v2(data: Dict[str, Any], request: Request):
"""Create Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createriderlog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 3: Create Delivery Log (v2)
@app.post("/live/api/v2/deliveries/createdeliverylog")
async def create_delivery_log_v2(
request: Request,
data: List[Dict[str, Any]] = Body(
...,
example=[{
"logid": 0,
"tenantid": 1,
"partnerid": 44,
"locationid": 1,
"orderheaderid": 123456,
"deliveryid": 654321,
"userid": 1111,
"orderid": "1-20231624",
"orderstatus": "active",
"starttime": "2025-12-10 17:51:03",
"logdate": "2025-12-10 18:14:04",
"latitude": "11.0050664",
"longitude": "76.9508776"
}]
)
):
"""
Create Delivery Log endpoint - forwards to NATS.
Accepts either a single dict or a list of dicts to align with external API expectations.
"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/deliveries/createdeliverylog"
method = request.method
try:
# Expect only a list of dicts
if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
raise HTTPException(status_code=422, detail="Body must be a list of objects")
normalized_data = data
message_id = await publish_to_nats(endpoint, normalized_data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 4: Create Break Rider Log (v2)
@app.post("/live/api/v2/partners/createbreaklog")
async def create_break_log_v2(data: Dict[str, Any], request: Request):
"""Create Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createbreaklog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 5: Update Break Rider Log (v2) - Supports both POST and PUT
@app.post("/live/api/v2/partners/updatebreaklog")
@app.put("/live/api/v2/partners/updatebreaklog")
async def update_break_log_v2(data: Dict[str, Any], request: Request):
"""Update Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/updatebreaklog"
method = request.method # Will be POST or PUT
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/metrics")
async def metrics():
"""Prometheus metrics endpoint"""
return Response(content=generate_latest(REGISTRY), media_type="text/plain")
@app.get("/routes")
async def list_routes():
"""List all configured routes"""
return {
"routes": ENDPOINT_ROUTES,
"total": len(ENDPOINT_ROUTES)
}
# Endpoint 6: Create Order (Mob V1)
@app.post("/live/api/v1/mob/orders/createorder")
async def create_order_mob_v1(data: Dict[str, Any], request: Request):
"""Create Order endpoint (Mobile) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/orders/createorder"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 7: Create Product (Web V1)
@app.post("/live/api/v1/web/products/create")
async def create_product_web_v1(data: Dict[str, Any], request: Request):
"""Create Product endpoint (Web) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/web/products/create"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 8: Customer Login (Sync Request-Reply)
@app.post("/live/api/v1/mob/customers/login")
async def customer_login(data: Dict[str, Any], request: Request):
"""Customer Login - Waits for response from worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/login"
method = request.method
try:
# Wait for reply!
response_data = await publish_request_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
# Return the actual backend response
return JSONResponse(
status_code=200,
content=response_data
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 9: Customer Create (Sync Request-Reply)
@app.post("/live/api/v1/mob/customers/create")
async def customer_create(data: Dict[str, Any], request: Request):
"""Customer Create - Waits for response from worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/create"
method = request.method
try:
# Wait for reply!
response_data = await publish_request_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content=response_data
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)

99
manifests/nearle/fiesta-sts.yaml Normal file
View File

@@ -0,0 +1,99 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: fiesta
namespace: nearle
labels:
app: fiesta
spec:
serviceName: "fiesta"
replicas: 3
selector:
matchLabels:
app: fiesta
template:
metadata:
labels:
app: fiesta
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: fiesta
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: fiesta
containers:
- name: backend
image: nearlecommerce/fiesta:v1.3.67
imagePullPolicy: Always
ports:
- containerPort: 1122
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
env:
- name: PORT
value: "1122"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: gateway
image: workolik360/alaska:v1.2.0
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
volumeMounts:
- name: gateway-script
mountPath: /app/app.py
subPath: app.py
envFrom:
- configMapRef:
name: nearle-config
env:
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
volumes:
- name: gateway-script
configMap:
name: fiesta-gateway-script

21
manifests/nearle/fiesta-svc.yaml Normal file
View File

@@ -0,0 +1,21 @@
apiVersion: v1
kind: Service
metadata:
name: fiesta
namespace: nearle
labels:
app: fiesta
spec:
type: NodePort
ports:
- port: 80
targetPort: 1122
nodePort: 30823
protocol: TCP
name: main
- port: 8000
targetPort: 8000
name: gateway
protocol: TCP
selector:
app: fiesta

97
manifests/nearle/jupiter-cors-proxy.yaml Normal file
View File

@@ -0,0 +1,97 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: jupiter-cors-config
namespace: nearle
data:
nginx.conf: |
events {}
http {
upstream jupiter_backend {
server jupiter:80;
}
server {
listen 80;
location / {
proxy_pass http://jupiter_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, PATCH, DELETE' always;
add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization, X-Requested-With, Accept, Origin, X-Auth-Token' always;
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, PATCH, DELETE';
add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization, X-Requested-With, Accept, Origin, X-Auth-Token';
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
}
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: jupiter-cors-proxy
namespace: nearle
labels:
app: jupiter-cors-proxy
spec:
replicas: 1
selector:
matchLabels:
app: jupiter-cors-proxy
template:
metadata:
labels:
app: jupiter-cors-proxy
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
containers:
- name: nginx
image: nginx:alpine
ports:
- containerPort: 80
volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
resources:
requests:
memory: "32Mi"
cpu: "10m"
limits:
memory: "64Mi"
cpu: "100m"
volumes:
- name: nginx-config
configMap:
name: jupiter-cors-config
---
apiVersion: v1
kind: Service
metadata:
name: jupiter-cors-proxy
namespace: nearle
labels:
app: jupiter-cors-proxy
spec:
selector:
app: jupiter-cors-proxy
ports:
- port: 80
targetPort: 80
protocol: TCP

77
manifests/nearle/jupiter-sts.yaml Normal file
View File

@@ -0,0 +1,77 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jupiter
namespace: nearle
labels:
app: jupiter
spec:
serviceName: "jupiter"
replicas: 3
selector:
matchLabels:
app: jupiter
template:
metadata:
labels:
app: jupiter
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: jupiter
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: jupiter
containers:
- name: jupiter
image: nearlecommerce/jupiter:v2.7.53
imagePullPolicy: Always
ports:
- containerPort: 1009
env:
- name: PORT
value: "1009"
- name: TZ
value: "Asia/Kolkata"
volumeMounts:
- name: tz-config
mountPath: /etc/localtime
readOnly: true
- name: tz-data
mountPath: /usr/share/zoneinfo
readOnly: true
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
volumes:
- name: tz-config
hostPath:
path: /usr/share/zoneinfo/Asia/Kolkata
- name: tz-data
hostPath:
path: /usr/share/zoneinfo

16
manifests/nearle/jupiter-svc.yaml Normal file
View File

@@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: jupiter
namespace: nearle
labels:
app: jupiter
spec:
type: NodePort
ports:
- port: 80
targetPort: 1009
nodePort: 30822
protocol: TCP
selector:
app: jupiter

23
manifests/nearle/nearle-app-secrets.yaml Normal file
View File

@@ -0,0 +1,23 @@
apiVersion: v1
kind: Secret
metadata:
name: app-secrets
namespace: nearle
labels:
app: fiesta
type: Opaque
stringData:
# The IP of your BigRock Server
DATABASE_HOST: "66.116.207.225"
DB_HOST: "66.116.207.225"
# The user we confirmed works
DATABASE_USERNAME: "admin"
DB_USER: "admin"
# The password we confirmed works
DATABASE_PASSWORD: "Package@123#"
DB_PASSWORD: "Package@123#"
# The rest...
JWT_SECRET_KEY: "nearle"

99
manifests/nearle/nearle-ariane.yaml Normal file
View File

@@ -0,0 +1,99 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: ariane
namespace: nearle
labels:
app: ariane
spec:
serviceName: "ariane"
replicas: 3
selector:
matchLabels:
app: ariane
template:
metadata:
labels:
app: ariane
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: ariane
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: ariane
containers:
- name: backend
image: nearlecommerce/ariane:v1.0.22
imagePullPolicy: Always
ports:
- containerPort: 1000
env:
- name: PORT
value: "1000"
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
---
apiVersion: v1
kind: Service
metadata:
name: ariane
namespace: nearle
labels:
app: ariane
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 1000
protocol: TCP
selector:
app: ariane
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: ariane-route
namespace: nearle
labels:
app: ariane
spec:
parentRefs:
- name: gateway
namespace: alaska
hostnames:
- "ariane.nearle.app"
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: ariane
port: 80

100
manifests/nearle/nearle-atlantis.yaml Normal file
View File

@@ -0,0 +1,100 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: atlantis
namespace: nearle
labels:
app: atlantis
spec:
serviceName: "atlantis"
replicas: 2
selector:
matchLabels:
app: atlantis
template:
metadata:
labels:
app: atlantis
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: atlantis
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: atlantis
containers:
- name: backend
image: nearlecommerce/atlantis:v0.0.41
imagePullPolicy: Always
ports:
- containerPort: 3000
env:
- name: PORT
value: "3000"
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
---
apiVersion: v1
kind: Service
metadata:
name: atlantis
namespace: nearle
labels:
app: atlantis
spec:
type: NodePort
ports:
- port: 80
targetPort: 3000
nodePort: 30825
protocol: TCP
selector:
app: atlantis
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: atlantis-route
namespace: nearle
labels:
app: atlantis
spec:
parentRefs:
- name: gateway
namespace: alaska
hostnames:
- "atlantis.nearle.app"
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: atlantis
port: 80

23
manifests/nearle/nearle-config.yaml Normal file
View File

@@ -0,0 +1,23 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: nearle-config
namespace: nearle
labels:
app: fiesta
data:
NATS_URL: "nats://66.116.226.161:4222"
LOG_LEVEL: "info"
# The stream is set to ORDERS as per your request
NATS_STREAM: "ORDERS"
# Base subject pattern - the app likely appends the path to this or uses it as a listener filter
NATS_SUBJECT: "api.>"
ALLOWED_ORIGINS: "*"
ENV: "production"
DATABASE_NAME: "nearledb"
DB_NAME: "nearledb"
DATABASE_PORT: "5432"
DB_PORT: "5432"
DATABASE_SERVER_HOST: "66.116.207.225"
DB_HOST: "66.116.207.225"
USER_CONTEXT_KEY: "nearle"

157
manifests/nearle/nearle-fiesta.yaml Normal file
View File

@@ -0,0 +1,157 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: fiesta
namespace: nearle
labels:
app: fiesta
spec:
serviceName: "fiesta"
replicas: 3
selector:
matchLabels:
app: fiesta
template:
metadata:
labels:
app: fiesta
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: fiesta
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: fiesta
containers:
- name: backend
image: nearlecommerce/fiesta:v1.3.50
imagePullPolicy: Always
ports:
- containerPort: 1122
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
env:
- name: PORT
value: "1122"
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
- name: gateway
image: workolik360/alaska:v1.2.0
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
volumeMounts:
- name: gateway-script
mountPath: /app/app.py
subPath: app.py
envFrom:
- configMapRef:
name: nearle-config
env:
- name: NATS_USER
valueFrom:
secretKeyRef:
name: nats-credentials
key: username
- name: NATS_PASSWORD
valueFrom:
secretKeyRef:
name: nats-credentials
key: password
volumes:
- name: gateway-script
configMap:
name: fiesta-gateway-script
---
apiVersion: v1
kind: Service
metadata:
name: fiesta
namespace: nearle
labels:
app: fiesta
spec:
type: NodePort
ports:
- port: 80
targetPort: 1122
nodePort: 30823
protocol: TCP
name: main
- port: 8000
targetPort: 8000
name: gateway
protocol: TCP
selector:
app: fiesta
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: fiesta-route
namespace: nearle
labels:
app: fiesta
spec:
parentRefs:
- name: gateway
namespace: alaska
hostnames:
- "fiesta.nearle.app"
rules:
- matches:
- path:
type: PathPrefix
value: /live/api/v1/mob/orders/createorder
backendRefs:
- name: fiesta
port: 8000
- matches:
- path:
type: PathPrefix
value: /live/api/v1/web/products/create
backendRefs:
- name: fiesta
port: 8000
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: fiesta
port: 80

27
manifests/nearle/nearle-gateway.yaml Normal file
View File

@@ -0,0 +1,27 @@
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: gateway
namespace: nearle
labels:
app.kubernetes.io/name: gateway
app.kubernetes.io/part-of: nearle
spec:
gatewayClassName: standard
listeners:
- name: http
protocol: HTTP
port: 8202
allowedRoutes:
namespaces:
from: Same
- name: https
protocol: HTTPS
port: 8442
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- name: nearle-tls-cert

94
manifests/nearle/nearle-jupiter.yaml Normal file
View File

@@ -0,0 +1,94 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: jupiter
namespace: nearle
labels:
app: jupiter
spec:
serviceName: "jupiter"
replicas: 3
selector:
matchLabels:
app: jupiter
template:
metadata:
labels:
app: jupiter
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: jupiter
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: jupiter
containers:
- name: jupiter
image: nearlecommerce/jupiter:v2.7.31
imagePullPolicy: Always
ports:
- containerPort: 1009
env:
- name: PORT
value: "1009"
- name: TZ
value: "Asia/Kolkata"
volumeMounts:
- name: tz-config
mountPath: /etc/localtime
readOnly: true
- name: tz-data
mountPath: /usr/share/zoneinfo
readOnly: true
envFrom:
- configMapRef:
name: nearle-config
- secretRef:
name: app-secrets
volumes:
- name: tz-config
hostPath:
path: /usr/share/zoneinfo/Asia/Kolkata
- name: tz-data
hostPath:
path: /usr/share/zoneinfo
---
apiVersion: v1
kind: Service
metadata:
name: jupiter
namespace: nearle
labels:
app: jupiter
spec:
type: NodePort
ports:
- port: 80
targetPort: 1009
nodePort: 30822
protocol: TCP
selector:
app: jupiter

6
manifests/nearle/nearle-namespace.yaml Normal file
View File

@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: nearle
labels:
name: nearle

14
manifests/nearle/nearle-reference-grant.yaml Normal file
View File

@@ -0,0 +1,14 @@
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
name: allow-alaska-gateway
namespace: nearle
spec:
from:
- group: gateway.networking.k8s.io
kind: HTTPRoute
namespace: alaska
to:
- group: ""
kind: Service
name: fiesta

11
manifests/nearle/nearle-secrets.yaml Normal file
View File

@@ -0,0 +1,11 @@
apiVersion: v1
kind: Secret
metadata:
name: nats-credentials
namespace: nearle
labels:
app: fiesta
type: Opaque
stringData:
username: admin
password: package@321#

91
manifests/nearle/nearle-titan.yaml Normal file
View File

@@ -0,0 +1,91 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: titan
namespace: nearle
labels:
app: titan
spec:
serviceName: "titan"
replicas: 3
selector:
matchLabels:
app: titan
template:
metadata:
labels:
app: titan
spec:
tolerations:
- key: dedicated
operator: Equal
value: apps
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.workolik/app
operator: In
values:
- "true"
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: titan
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: titan
containers:
- name: backend
image: groomgear/groomgear:v1.0.41
imagePullPolicy: Always
ports:
- containerPort: 1006
---
apiVersion: v1
kind: Service
metadata:
name: titan
namespace: nearle
labels:
app: titan
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 1006
protocol: TCP
selector:
app: titan
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: titan-route
namespace: nearle
labels:
app: titan
spec:
parentRefs:
- name: gateway
namespace: alaska
hostnames:
- "titan.nearle.app"
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: titan
port: 80

8
requirements.txt Normal file
View File

@@ -0,0 +1,8 @@
fastapi==0.104.1
uvicorn[standard]==0.24.0
nats-py==2.6.0
pydantic==2.5.0
prometheus-client==0.19.0
aiohttp==3.9.1
requests==2.31.0

514
scripts/app.py Normal file
View File

@@ -0,0 +1,514 @@
#!/usr/bin/env python3
"""
FastAPI application with multiple endpoints that publish to NATS JetStream
Each endpoint corresponds to an external API that workers will forward to
"""
import os
import json
import asyncio
from fastapi import FastAPI, HTTPException, Request, Body
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request as StarletteRequest
from starlette.responses import Response as StarletteResponse
from pydantic import BaseModel
import nats
from prometheus_client import Counter, Histogram, generate_latest, REGISTRY
from starlette.responses import Response
import uvicorn
from typing import Optional, Dict, Any, List, Union
# Prometheus metrics
request_count = Counter('http_requests_total', 'Total HTTP requests', ['method', 'endpoint', 'status'])
request_duration = Histogram('http_request_duration_seconds', 'HTTP request duration', ['method', 'endpoint'])
app = FastAPI(title="NATS Backend API - Multi-Endpoint", version="1.0.0")
# Custom CORS middleware to ensure headers are always added
class CORSHeaderMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: StarletteRequest, call_next):
origin = request.headers.get("origin", "*")
response = await call_next(request)
response.headers["Access-Control-Allow-Origin"] = origin
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Credentials"] = "false"
response.headers["Access-Control-Max-Age"] = "600"
return response
# Add custom CORS middleware first
app.add_middleware(CORSHeaderMiddleware)
# Also add FastAPI's CORS middleware as backup
app.add_middleware(
CORSMiddleware,
allow_origin_regex=r".*", # Match all origins
allow_credentials=False,
allow_methods=["*"], # Allow all HTTP methods
allow_headers=["*"], # Allow all headers
expose_headers=["*"], # Expose all headers
max_age=600,
)
# NATS connection (will be initialized on startup)
nc = None
js = None
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
# Endpoint to NATS subject mapping
ENDPOINT_ROUTES = {
"/live/api/v1/deliveries/createdeliveries": "api.v1.deliveries.createdeliveries",
"/live/api/v1/deliveries/updatedelivery": "api.v1.deliveries.updatedelivery",
"/live/api/v2/partners/createriderlog": "api.v2.partners.createriderlog",
"/live/api/v2/deliveries/createdeliverylog": "api.v2.deliveries.createdeliverylog",
"/live/api/v2/partners/createbreaklog": "api.v2.partners.createbreaklog",
"/live/api/v2/partners/updatebreaklog": "api.v2.partners.updatebreaklog",
"/live/api/v1/mob/orders/createorder": "api.v1.mob.orders.createorder",
"/live/api/v1/web/products/create": "api.v1.web.products.create",
"/live/api/v1/mob/customers/create": "api.v1.mob.customers.create",
"/live/api/v1/mob/customers/login": "api.v1.mob.customers.login",
}
@app.on_event("startup")
async def startup():
"""Initialize NATS connection on startup"""
global nc, js
try:
print(f"Connecting to NATS at {NATS_URL}...")
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=10
)
js = nc.jetstream()
print("✅ Connected to NATS JetStream")
print(f"✅ Configured {len(ENDPOINT_ROUTES)} endpoint routes")
except Exception as e:
print(f"❌ Failed to connect to NATS: {e}")
raise
@app.on_event("shutdown")
async def shutdown():
"""Close NATS connection on shutdown"""
global nc
if nc:
await nc.close()
print("NATS connection closed")
@app.options("/{full_path:path}")
async def options_handler(full_path: str, request: Request):
"""Handle OPTIONS requests for CORS preflight"""
origin = request.headers.get("origin")
return JSONResponse(
status_code=200,
content={},
headers={
"Access-Control-Allow-Origin": origin if origin else "*",
"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
"Access-Control-Allow-Headers": "*",
"Access-Control-Allow-Credentials": "false",
"Access-Control-Max-Age": "600",
}
)
@app.get("/health")
async def health():
"""Health check endpoint"""
return {"status": "healthy", "nats_connected": nc.is_connected if nc else False}
@app.get("/ready")
async def ready():
"""Readiness check endpoint"""
if nc and nc.is_connected:
return {"status": "ready"}
raise HTTPException(status_code=503, detail="Not ready")
async def publish_to_nats(endpoint: str, data: Union[Dict[str, Any], List[Dict[str, Any]]], request_method: str = "POST"):
"""Publish message to NATS with endpoint metadata"""
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint
}
# Get NATS subject for this endpoint
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if js:
try:
ack = await js.publish(subject, json.dumps(payload).encode())
return ack.seq
except Exception as e:
print(f"❌ Failed to publish to NATS: {e}")
raise HTTPException(status_code=500, detail=f"Failed to publish message: {str(e)}")
else:
raise HTTPException(status_code=503, detail="NATS not connected")
async def publish_to_nats_sync(endpoint: str, data: Union[Dict[str, Any], List[Dict[str, Any]]], request_method: str = "POST", timeout: int = 15):
"""
Publish message to NATS and wait for a dedicated reply from the worker (Synchronous)
"""
if not nc or not js:
raise HTTPException(status_code=503, detail="NATS not connected")
# Create a unique inbox for the reply
reply_subject = nc.new_inbox()
# Subscribe to the inbox matching the unique reply subject
sub = await nc.subscribe(reply_subject)
payload = {
"endpoint": endpoint,
"method": request_method,
"data": data,
"received_at": int(asyncio.get_event_loop().time() * 1000),
"original_path": endpoint,
"reply_subject": reply_subject # Tell worker where to reply
}
# Get NATS subject for this endpoint
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
try:
# Publish to the Stream
ack = await js.publish(subject, json.dumps(payload).encode())
print(f"Sync message published to {subject}, waiting for reply on {reply_subject}...")
# Wait for the worker to process and reply
# We assume the worker will send a single JSON message: {"status": <int>, "body": <str|json>}
msg = await sub.next_msg(timeout=timeout)
# Parse the reply
response_data = json.loads(msg.data.decode())
status_code = response_data.get("status", 500)
body_content = response_data.get("body", "")
# Attempt to parse body as JSON if it's a string, to avoid double-serialization
# If the body is already a dict, use it directly.
final_content = body_content
if isinstance(body_content, str):
try:
final_content = json.loads(body_content)
except:
pass # Keep as string if not valid JSON
await sub.unsubscribe()
return status_code, final_content
except asyncio.TimeoutError:
await sub.unsubscribe()
print(f"❌ Timeout waiting for sync response on {endpoint}")
raise HTTPException(status_code=504, detail="Gateway Timeout: Worker took too long to respond")
except Exception as e:
await sub.unsubscribe()
print(f"❌ Failed during sync processing: {e}")
raise HTTPException(status_code=500, detail=f"Failed to process synchronous request: {str(e)}")
# Endpoint 1: Update Delivery (v1) - External API requires PUT
@app.put("/live/api/v1/deliveries/updatedelivery")
async def update_delivery_v1(data: Dict[str, Any], request: Request):
"""Update Delivery endpoint - forwards to NATS (PUT only, as external API requires PUT)"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/updatedelivery"
method = "PUT" # Always use PUT for this endpoint
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method="PUT", endpoint=endpoint).observe(duration)
request_count.labels(method="PUT", endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method="PUT", endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint: Create Deliveries (v1)
@app.post("/live/api/v1/deliveries/createdeliveries")
async def create_deliveries_v1(data: Union[Dict[str, Any], List[Dict[str, Any]]], request: Request):
"""Create Deliveries endpoint - forwards payload to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/deliveries/createdeliveries"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 2: Create Rider Log (v2)
@app.post("/live/api/v2/partners/createriderlog")
async def create_rider_log_v2(data: Dict[str, Any], request: Request):
"""Create Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createriderlog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 3: Create Delivery Log (v2)
@app.post("/live/api/v2/deliveries/createdeliverylog")
async def create_delivery_log_v2(
request: Request,
data: List[Dict[str, Any]] = Body(
...,
example=[{
"logid": 0,
"tenantid": 1,
"partnerid": 44,
"locationid": 1,
"orderheaderid": 123456,
"deliveryid": 654321,
"userid": 1111,
"orderid": "1-20231624",
"orderstatus": "active",
"starttime": "2025-12-10 17:51:03",
"logdate": "2025-12-10 18:14:04",
"latitude": "11.0050664",
"longitude": "76.9508776"
}]
)
):
"""
Create Delivery Log endpoint - forwards to NATS.
Accepts either a single dict or a list of dicts to align with external API expectations.
"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/deliveries/createdeliverylog"
method = request.method
try:
# Expect only a list of dicts
if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
raise HTTPException(status_code=422, detail="Body must be a list of objects")
normalized_data = data
message_id = await publish_to_nats(endpoint, normalized_data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 4: Create Break Rider Log (v2)
@app.post("/live/api/v2/partners/createbreaklog")
async def create_break_log_v2(data: Dict[str, Any], request: Request):
"""Create Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/createbreaklog"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 5: Update Break Rider Log (v2) - Supports both POST and PUT
@app.post("/live/api/v2/partners/updatebreaklog")
@app.put("/live/api/v2/partners/updatebreaklog")
async def update_break_log_v2(data: Dict[str, Any], request: Request):
"""Update Break Rider Log endpoint - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v2/partners/updatebreaklog"
method = request.method # Will be POST or PUT
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/metrics")
async def metrics():
"""Prometheus metrics endpoint"""
return Response(content=generate_latest(REGISTRY), media_type="text/plain")
@app.get("/routes")
async def list_routes():
"""List all configured routes"""
return {
"routes": ENDPOINT_ROUTES,
"total": len(ENDPOINT_ROUTES)
}
# Endpoint 6: Create Order (Mob V1)
@app.post("/live/api/v1/mob/orders/createorder")
async def create_order_mob_v1(data: Dict[str, Any], request: Request):
"""Create Order endpoint (Mobile) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/orders/createorder"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 7: Create Product (Web V1)
@app.post("/live/api/v1/web/products/create")
async def create_product_web_v1(data: Dict[str, Any], request: Request):
"""Create Product endpoint (Web) - forwards to NATS"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/web/products/create"
method = request.method
try:
message_id = await publish_to_nats(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status="200").inc()
return JSONResponse(
status_code=200,
content={"status": "accepted", "message_id": message_id, "endpoint": endpoint}
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 8: Create Customer (Mob V1)
@app.post("/live/api/v1/mob/customers/create")
async def create_customer_v1(data: Dict[str, Any], request: Request):
"""Create Customer endpoint (Mobile) - synchronous wait for NATS worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/create"
method = request.method
try:
status_code, content = await publish_to_nats_sync(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status=str(status_code)).inc()
return JSONResponse(
status_code=status_code,
content=content
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Endpoint 9: Login Customer (Mob V1)
@app.post("/live/api/v1/mob/customers/login")
async def login_customer_v1(data: Dict[str, Any], request: Request):
"""Login Customer endpoint (Mobile) - synchronous wait for NATS worker"""
start_time = asyncio.get_event_loop().time()
endpoint = "/live/api/v1/mob/customers/login"
method = request.method
try:
status_code, content = await publish_to_nats_sync(endpoint, data, method)
duration = asyncio.get_event_loop().time() - start_time
request_duration.labels(method=method, endpoint=endpoint).observe(duration)
request_count.labels(method=method, endpoint=endpoint, status=str(status_code)).inc()
return JSONResponse(
status_code=status_code,
content=content
)
except HTTPException:
raise
except Exception as e:
request_count.labels(method=method, endpoint=endpoint, status="500").inc()
print(f"❌ Error processing {endpoint}: {e}")
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)

40
scripts/purge-old-messages.py Normal file
View File

@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""
Purge old failed messages from NATS JetStream
"""
import asyncio
import nats
import os
async def purge_messages():
nats_url = "nats://nats.workolik.com:4222"
nats_user = "admin"
nats_password = "package@321#"
try:
print(f"Connecting to NATS at {nats_url}...")
nc = await nats.connect(
servers=[nats_url],
user=nats_user,
password=nats_password
)
print("✅ Connected to NATS")
js = nc.jetstream()
# Purge the stream to remove all old messages
print("Purging old messages from stream 'EVENTS'...")
await js.purge_stream("EVENTS")
print("✅ Stream purged - all old messages removed")
await nc.close()
print("✅ Done!")
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(purge_messages())

166
scripts/setup_jetstream.py Normal file
View File

@@ -0,0 +1,166 @@
#!/usr/bin/env python3
"""
Setup JetStream stream and consumer for NATS
Run this after NATS is deployed and running
"""
import asyncio
import nats
import os
import sys
async def setup_jetstream():
"""Configure JetStream with two streams (DELIVERIES, RIDER) and per-subject consumers."""
nats_url = os.getenv("NATS_URL", "nats://nats.workolik.com:4222")
nats_user = os.getenv("NATS_USER", "admin")
nats_password = os.getenv("NATS_PASSWORD", "package@321#")
# Stream definitions
streams = {
"DELIVERIES": {
"subjects": [
"api.v1.deliveries.createdeliveries",
"api.v1.deliveries.updatedelivery",
"api.v2.deliveries.createdeliverylog",
],
},
"RIDER": {
"subjects": [
"api.v2.partners.createriderlog",
"api.v2.partners.createbreaklog",
"api.v2.partners.updatebreaklog",
],
},
"ORDERS": {
"subjects": [
"api.v1.mob.orders.createorder",
],
},
"PRODUCTS": {
"subjects": [
"api.v1.web.products.create",
],
},
"CUSTOMERS": {
"subjects": [
"api.v1.mob.customers.login",
"api.v1.mob.customers.create",
],
"retention": "work" # Special handling for Login queue: delete immediately after ack
},
}
# Per-subject durable consumers
consumers = {
"DELIVERIES": {
"api.v1.deliveries.createdeliveries": "deliveries_createdeliveries",
"api.v1.deliveries.updatedelivery": "deliveries_updatedelivery",
"api.v2.deliveries.createdeliverylog": "deliveries_createdeliverylog",
},
"RIDER": {
"api.v2.partners.createriderlog": "rider_createriderlog",
"api.v2.partners.createbreaklog": "rider_createbreaklog",
"api.v2.partners.updatebreaklog": "rider_updatebreaklog",
},
"ORDERS": {
"api.v1.mob.orders.createorder": "orders_createorder",
},
"PRODUCTS": {
"api.v1.web.products.create": "products_create",
},
"CUSTOMERS": {
"api.v1.mob.customers.login": "customers_login",
"api.v1.mob.customers.create": "customers_create",
},
}
try:
print(f"Connecting to NATS at {nats_url}...")
nc = await nats.connect(
servers=[nats_url],
user=nats_user,
password=nats_password,
)
print("✅ Connected to NATS")
js = nc.jetstream()
# Create / recreate streams
for stream_name, cfg in streams.items():
try:
info = await js.stream_info(stream_name)
print(f"⚠️ Stream '{stream_name}' already exists with subjects={info.config.subjects}, updating...")
# Determine retention policy
retention_policy = cfg.get("retention", "limits")
await js.update_stream(
name=stream_name,
subjects=cfg["subjects"],
storage="memory",
retention=retention_policy,
max_age=24 * 60 * 60,
max_msgs=50000,
max_bytes=512 * 1024 * 1024,
)
print(f"✅ Stream '{stream_name}' updated")
except Exception as e:
if "not found" in str(e).lower() or "404" in str(e).lower():
print(f"Creating stream '{stream_name}'...")
# Determine retention policy (use 'limits' by default, 'work' for queues)
retention_policy = cfg.get("retention", "limits")
await js.add_stream(
name=stream_name,
subjects=cfg["subjects"],
storage="memory",
retention=retention_policy,
max_age=24 * 60 * 60,
max_msgs=50000,
max_bytes=512 * 1024 * 1024,
)
print(f"✅ Stream '{stream_name}' created")
else:
print(f"⚠️ Could not inspect stream '{stream_name}': {e}")
# Create durable consumers per subject
print("\nConfiguring consumers...")
for stream_name, subject_map in consumers.items():
for subject, durable in subject_map.items():
try:
print(f"Creating consumer '{durable}' on stream '{stream_name}' for subject '{subject}'...")
await js.add_consumer(
stream_name,
durable_name=durable,
filter_subject=subject,
ack_policy="explicit",
deliver_policy="all",
max_deliver=5,
ack_wait=30,
)
print(f"✅ Consumer '{durable}' created")
except Exception as e:
if "already in use" in str(e).lower() or "already exists" in str(e).lower():
print(f"⚠️ Consumer '{durable}' already exists, skipping...")
else:
raise
print("\n✅ JetStream setup complete!")
print(" Streams:")
for name, cfg in streams.items():
print(f" - {name}: {', '.join(cfg['subjects'])}")
print(" Consumers:")
for stream_name, subject_map in consumers.items():
for subject, durable in subject_map.items():
print(f" - {durable}: stream={stream_name}, subject={subject}")
await nc.close()
sys.exit(0)
except Exception as e:
print(f"❌ Error: {e}")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(setup_jetstream())

66
scripts/sync_manifests.py Normal file
View File

@@ -0,0 +1,66 @@
import os
def update_yaml_with_script(yaml_path, script_path, key_line_start):
with open(script_path, 'r', encoding='utf-8') as f:
script_content = f.read()
# Indent script content by 4 spaces
indented_script = '\n'.join(' ' + line if line.strip() else line for line in script_content.splitlines())
with open(yaml_path, 'r', encoding='utf-8') as f:
yaml_lines = f.readlines()
# Find the key line (e.g., " app.py: |")
start_index = -1
for i, line in enumerate(yaml_lines):
if key_line_start in line:
start_index = i + 1
break
if start_index == -1:
print(f"Error: Could not find '{key_line_start}' in {yaml_path}")
return
# Find where the script block ends (next line that is NOT indented by at least 4 spaces, or EOF)
# Actually, the Data block might be the last thing.
# We assume the script goes until the end of the file or next unindented key.
# In these files, the script is usually the main data.
# Let's just truncate and append if it looks like the script is the last/main thing.
# But usually, it's safer to just replace the lines that look like script.
# Simple heuristic: The script block ends when indentation drops to 2 spaces or 0?
# In worker-script.yaml:
# 6: data:
# 7: worker.py: |
# 8: ...script...
# The script is indented by 4 spaces.
pre_script = yaml_lines[:start_index]
# We will just write the pre_script + indented_script
# WARNING: If there are other keys after worker.py, this deletes them.
# Let's check the files.
# worker-script.yaml: 378 lines. Script ends at 378. Nothing follows.
# fiesta-gateway.yaml: 407 lines. Script ends at 407. Nothing follows.
# So appending is SAFE.
with open(yaml_path, 'w', encoding='utf-8') as f:
f.writelines(pre_script)
f.write(indented_script)
f.write('\n') # Ensure newline at EOF
print(f"Successfully updated {yaml_path}")
# Update Fiesta Gateway
update_yaml_with_script(
r'e:\nats\kubernetes\manifests\nearle\fiesta-gateway.yaml',
r'e:\nats\kubernetes\conf\app.py',
' app.py: |'
)
# Update Worker Script
update_yaml_with_script(
r'e:\nats\kubernetes\manifests\core\worker-script.yaml',
r'e:\nats\kubernetes\conf\worker.py',
' worker.py: |'
)

285
scripts/worker.py Normal file
View File

@@ -0,0 +1,285 @@
#!/usr/bin/env python3
"""
NATS Worker that consumes messages and routes them to different external endpoints
Based on the endpoint in the message, forwards to the corresponding external API
"""
import os
import json
import asyncio
import aiohttp
import nats
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import signal
import sys
# Prometheus metrics
messages_processed = Counter('worker_messages_processed_total', 'Total messages processed', ['status', 'endpoint'])
message_duration = Histogram('worker_message_duration_seconds', 'Message processing duration', ['endpoint'])
messages_in_flight = Gauge('worker_messages_in_flight', 'Messages currently being processed')
queue_depth = Gauge('worker_queue_depth', 'Approximate queue depth')
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
NATS_STREAM = os.getenv("NATS_STREAM", "EVENTS")
NATS_SUBJECT = os.getenv("NATS_SUBJECT", "api.>") # Subscribe to all API subjects
NATS_CONSUMER = os.getenv("NATS_CONSUMER", "worker_consumer")
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "10"))
RETRY_ATTEMPTS = int(os.getenv("RETRY_ATTEMPTS", "5"))
RETRY_DELAY = int(os.getenv("RETRY_DELAY_SECONDS", "5"))
# Base URL for external APIs
BASE_URL = os.getenv("EXTERNAL_BASE_URL", "https://jupiter.nearle.app")
# Endpoint mapping: FastAPI endpoint -> External API endpoint
ENDPOINT_MAPPING = {
"/live/api/v1/deliveries/createdeliveries": f"{BASE_URL}/live/api/v1/deliveries/createdeliveries",
"/live/api/v1/deliveries/updatedelivery": f"{BASE_URL}/live/api/v1/deliveries/updatedelivery",
"/live/api/v2/partners/createriderlog": f"{BASE_URL}/live/api/v2/partners/createriderlog",
"/live/api/v2/deliveries/createdeliverylog": f"{BASE_URL}/live/api/v2/deliveries/createdeliverylog",
"/live/api/v2/partners/createbreaklog": f"{BASE_URL}/live/api/v2/partners/createbreaklog",
"/live/api/v2/partners/updatebreaklog": f"{BASE_URL}/live/api/v2/partners/updatebreaklog",
}
# Global state
nc = None
js = None
running = True
semaphore = asyncio.Semaphore(WORKER_CONCURRENCY)
def signal_handler(sig, frame):
"""Handle shutdown signals"""
global running
print("\n🛑 Shutdown signal received, stopping worker...")
running = False
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
async def forward_to_external(endpoint: str, payload: dict, api_key: str = None, method: str = "POST") -> bool:
"""Forward message to external endpoint with retries"""
external_url = ENDPOINT_MAPPING.get(endpoint)
if not external_url:
print(f"⚠️ Unknown endpoint: {endpoint}, skipping...")
return False
headers = {
"Content-Type": "application/json"
}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
# Get the actual data to forward
data_to_forward = payload.get("data", payload)
# Normalize payload shape for endpoints that expect an array
if endpoint == "/live/api/v2/deliveries/createdeliverylog":
if isinstance(data_to_forward, dict):
data_to_forward = [data_to_forward]
elif not isinstance(data_to_forward, list):
# Fallback to wrapping anything that's not already list-like
data_to_forward = [data_to_forward]
# Debug logging for delivery log payload shape
print(f"➡️ Forwarding createdeliverylog payload: {data_to_forward}")
# Normalize method to uppercase
http_method = method.upper() if method else "POST"
for attempt in range(RETRY_ATTEMPTS):
try:
async with aiohttp.ClientSession() as session:
# Use the correct HTTP method
if http_method == "GET":
async with session.get(
external_url,
params=data_to_forward if isinstance(data_to_forward, dict) else {},
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
return await _handle_response(response, endpoint, attempt)
elif http_method == "PUT":
async with session.put(
external_url,
json=data_to_forward,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
return await _handle_response(response, endpoint, attempt)
elif http_method == "PATCH":
async with session.patch(
external_url,
json=data_to_forward,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
return await _handle_response(response, endpoint, attempt)
elif http_method == "DELETE":
async with session.delete(
external_url,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
return await _handle_response(response, endpoint, attempt)
else: # Default to POST
async with session.post(
external_url,
json=data_to_forward,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
return await _handle_response(response, endpoint, attempt)
except asyncio.TimeoutError:
print(f"⚠️ Timeout forwarding {endpoint} to external API (attempt {attempt + 1}/{RETRY_ATTEMPTS})")
if attempt < RETRY_ATTEMPTS - 1:
await asyncio.sleep(RETRY_DELAY * (attempt + 1))
except Exception as e:
print(f"⚠️ Error forwarding {endpoint} to external API: {e} (attempt {attempt + 1}/{RETRY_ATTEMPTS})")
if attempt < RETRY_ATTEMPTS - 1:
await asyncio.sleep(RETRY_DELAY * (attempt + 1))
return False
async def _handle_response(response, endpoint: str, attempt: int) -> bool:
"""Handle HTTP response and determine if we should retry"""
body_text = None
try:
body_text = await response.text()
except Exception:
body_text = "<no body>"
if response.status == 200 or response.status == 201:
return True
elif response.status >= 500:
# Server error - retry
print(f"⚠️ External API returned {response.status} for {endpoint}, retrying... (attempt {attempt + 1}/{RETRY_ATTEMPTS}) body={body_text}")
if attempt < RETRY_ATTEMPTS - 1:
await asyncio.sleep(RETRY_DELAY * (attempt + 1)) # Exponential backoff
return False
else:
# Client error - don't retry
print(f"❌ External API returned {response.status} for {endpoint}, not retrying; body={body_text}")
return False
async def process_message(msg):
"""Process a single message"""
async with semaphore:
messages_in_flight.inc()
start_time = asyncio.get_event_loop().time()
endpoint = "unknown"
try:
# Decode message
payload = json.loads(msg.data.decode())
endpoint = payload.get("endpoint", payload.get("original_path", "unknown"))
http_method = payload.get("method", "POST") # Get HTTP method from payload
print(f"📨 Processing message for endpoint: {endpoint} (method: {http_method})")
# Forward to external endpoint with correct HTTP method
api_key = os.getenv("EXTERNAL_ENDPOINT_API_KEY", "")
success = await forward_to_external(endpoint, payload, api_key if api_key else None, http_method)
duration = asyncio.get_event_loop().time() - start_time
message_duration.labels(endpoint=endpoint).observe(duration)
if success:
# Acknowledge message (remove from queue)
await msg.ack()
messages_processed.labels(status="success", endpoint=endpoint).inc()
print(f"✅ Message processed and forwarded successfully for {endpoint}")
else:
# Negative acknowledge (requeue for retry)
await msg.nak()
messages_processed.labels(status="failed", endpoint=endpoint).inc()
print(f"❌ Failed to forward message for {endpoint}, requeued for retry")
except json.JSONDecodeError as e:
print(f"❌ Invalid JSON in message: {e}")
await msg.term() # Terminate (don't retry)
messages_processed.labels(status="invalid", endpoint=endpoint).inc()
except Exception as e:
print(f"❌ Error processing message for {endpoint}: {e}")
await msg.nak() # Requeue for retry
messages_processed.labels(status="error", endpoint=endpoint).inc()
finally:
messages_in_flight.dec()
async def consume_messages():
"""Consume messages from NATS JetStream"""
global nc, js
try:
print(f"Connecting to NATS at {NATS_URL}...")
nc = await nats.connect(
servers=[NATS_URL],
user=NATS_USER,
password=NATS_PASSWORD,
reconnect_time_wait=2,
max_reconnect_attempts=-1 # Retry forever
)
js = nc.jetstream()
print("✅ Connected to NATS JetStream")
# Subscribe to messages (using wildcard to catch all API subjects)
print(f"Subscribing to stream '{NATS_STREAM}', consumer '{NATS_CONSUMER}'...")
print(f" Subject pattern: {NATS_SUBJECT}")
print(f" Endpoints configured: {len(ENDPOINT_MAPPING)}")
psub = await js.pull_subscribe(
NATS_SUBJECT,
NATS_CONSUMER,
stream=NATS_STREAM
)
print(f"✅ Subscribed, waiting for messages...")
# Start metrics server
start_http_server(9090)
print("✅ Metrics server started on port 9090")
# Process messages in a loop
while running:
try:
# Fetch messages (batch of up to 10)
msgs = await psub.fetch(10, timeout=5)
for msg in msgs:
# Process asynchronously
asyncio.create_task(process_message(msg))
except asyncio.TimeoutError:
# No messages available, continue
continue
except Exception as e:
print(f"⚠️ Error fetching messages: {e}")
await asyncio.sleep(1)
except Exception as e:
print(f"❌ Fatal error: {e}")
raise
finally:
if nc:
await nc.close()
print("NATS connection closed")
async def main():
"""Main entry point"""
print("🚀 Starting NATS Worker (Multi-Endpoint)...")
print(f" External Base URL: {BASE_URL}")
print(f" Concurrency: {WORKER_CONCURRENCY}")
print(f" Retry Attempts: {RETRY_ATTEMPTS}")
print(f" Configured Endpoints:")
for fastapi_endpoint, external_url in ENDPOINT_MAPPING.items():
print(f" {fastapi_endpoint} → {external_url}")
try:
await consume_messages()
except KeyboardInterrupt:
print("\n🛑 Worker stopped by user")
except Exception as e:
print(f"❌ Worker crashed: {e}")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(main())

93
shfiles/check-k8s-status.sh Normal file
View File

@@ -0,0 +1,93 @@
#!/bin/bash
echo "=========================================="
echo "🔍 KUBERNETES DEPLOYMENT STATUS"
echo "=========================================="
echo ""
echo "📦 1. PODS STATUS (All Namespaces)"
echo "-----------------------------------"
kubectl get pods -A -o wide
echo ""
echo "📦 2. NATS-BACKEND NAMESPACE - PODS"
echo "-----------------------------------"
kubectl get pods -n nats-backend -o wide
echo ""
echo "🌐 3. SERVICES & LOAD BALANCER"
echo "-----------------------------------"
kubectl get svc -n nats-backend -o wide
echo ""
echo "⚖️ 4. LOAD BALANCER DETAILS"
echo "-----------------------------------"
kubectl get svc fastapi-lb -n nats-backend -o yaml | grep -A 10 "spec:"
echo ""
echo "🚀 5. K3S KUBERNETES LOAD BALANCER PODS (klipper-lb)"
echo "-----------------------------------"
kubectl get pods -n kube-system -l "svccontroller.k3s.cattle.io/svcname=fastapi-lb" -o wide
echo ""
echo "📊 6. DEPLOYMENTS & REPLICAS"
echo "-----------------------------------"
kubectl get deployments -n nats-backend -o wide
echo ""
echo "📈 7. HORIZONTAL POD AUTOSCALER (HPA)"
echo "-----------------------------------"
kubectl get hpa -n nats-backend
echo ""
echo "🔗 8. ENDPOINTS (Service Backends)"
echo "-----------------------------------"
kubectl get endpoints -n nats-backend
echo ""
echo "🌍 9. INGRESS/GATEWAY STATUS"
echo "-----------------------------------"
kubectl get gateway -A 2>/dev/null || echo "No Gateway API resources found"
kubectl get ingress -A 2>/dev/null || echo "No Ingress resources found"
echo ""
echo "📋 10. RECENT EVENTS (Last 20)"
echo "-----------------------------------"
kubectl get events -n nats-backend --sort-by='.lastTimestamp' | tail -20
echo ""
echo "💾 11. RESOURCE USAGE (CPU/Memory)"
echo "-----------------------------------"
kubectl top pods -n nats-backend 2>/dev/null || echo "Metrics server not available"
kubectl top nodes 2>/dev/null || echo "Metrics server not available"
echo ""
echo "🔍 12. FASTAPI POD LOGS (Last 30 lines)"
echo "-----------------------------------"
kubectl logs -n nats-backend -l app=fastapi-backend --tail=30 2>/dev/null || echo "No FastAPI pods found"
echo ""
echo "🔍 13. WORKER POD LOGS (Last 30 lines)"
echo "-----------------------------------"
kubectl logs -n nats-backend -l app=nats-worker --tail=30 2>/dev/null || echo "No worker pods found"
echo ""
echo "🌐 14. NETWORK FLOW CHECK"
echo "-----------------------------------"
echo "Load Balancer External IP/Port:"
kubectl get svc fastapi-lb -n nats-backend -o jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}' 2>/dev/null || echo "Checking NodePort..."
kubectl get svc fastapi-lb -n nats-backend -o jsonpath='NodePort: {.spec.ports[0].nodePort}' 2>/dev/null
echo ""
echo "FastAPI Service ClusterIP:"
kubectl get svc fastapi-backend -n nats-backend -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}' 2>/dev/null
echo ""
echo "✅ 15. HEALTH CHECK"
echo "-----------------------------------"
kubectl run health-check --rm -i --restart=Never --image=curlimages/curl -- curl -s http://fastapi-backend.nats-backend:8000/health 2>/dev/null || echo "Health check failed"
echo ""
echo "=========================================="
echo "✅ Status Check Complete!"
echo "=========================================="

19
shfiles/deploy-alaska.sh Normal file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
# Deploy updated "alaska" stack to Kubernetes
# Usage: ./deploy-alaska.sh
set -euo pipefail
NAMESPACE="alaska"
echo "🚀 Deploying Alaska Stack (Application & Gateway)..."
kubectl apply -f manifests/alaska/alaska.yaml
echo "📊 Deploying Kubernetes Dashboard & Proxy..."
kubectl apply -f manifests/alaska/k8s-dashboard.yaml
echo ""
echo "✅ Alaska deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"
kubectl get all -n "${NAMESPACE}" || true

16
shfiles/deploy-core-stack.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Deploy "core" stack to Kubernetes
# Usage: ./deploy-core-stack.sh
set -euo pipefail
NAMESPACE="core"
echo "🚀 Deploying Core Stack..."
kubectl apply -k manifests/core
echo ""
echo "✅ Core stack deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"
kubectl get all -n "${NAMESPACE}"

24
shfiles/deploy-nearle-stack.sh Normal file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
# Deploy "nearle" stack to Kubernetes
# Usage: ./deploy-nearle-stack.sh
set -euo pipefail
NAMESPACE="nearle"
echo "🔎 Ensuring namespace '${NAMESPACE}' exists..."
kubectl apply -f manifests/nearle/nearle-namespace.yaml
echo "🚀 Deploying Services..."
kubectl apply -f manifests/nearle/nearle-jupiter.yaml
kubectl apply -f manifests/nearle/nearle-titan.yaml
kubectl apply -f manifests/nearle/nearle-fiesta.yaml
kubectl apply -f manifests/nearle/nearle-ariane.yaml
echo ""
echo "✅ Nearle stack deployment applied."
echo "📋 Current status:"
echo " kubectl get all -n ${NAMESPACE}"

47
shfiles/deploy-queue-proxy.sh Normal file
View File

@@ -0,0 +1,47 @@
#!/bin/bash
# Deploy queue.workolik.com proxy with CORS support
echo "🚀 Deploying Queue API Proxy with CORS..."
# 1. Start the proxy service
echo "📦 Starting queue-api-proxy..."
docker compose up -d queue-api-proxy
# 2. Wait a moment for it to start
sleep 2
# 3. Check if it's running
if docker ps | grep -q queue-api-proxy; then
echo "✅ queue-api-proxy is running"
else
echo "❌ queue-api-proxy failed to start"
docker logs queue-api-proxy
exit 1
fi
# 4. Test the proxy directly (bypassing Traefik)
echo ""
echo "🧪 Testing proxy directly..."
curl -X OPTIONS http://localhost:8202/live/api/v1/deliveries/createdeliveries \
-H 'Origin: https://console.nearlexpress.com' \
-H 'Access-Control-Request-Method: POST' \
-v 2>&1 | grep -E "access-control-allow-origin|HTTP/" | head -5
# 5. Test via Traefik (if accessible)
echo ""
echo "🧪 Testing via Traefik (HTTPS)..."
echo " Run this command to test:"
echo " curl -X OPTIONS https://queue.workolik.com/live/api/v1/deliveries/createdeliveries \\"
echo " -H 'Origin: https://console.nearlexpress.com' \\"
echo " -H 'Access-Control-Request-Method: POST' \\"
echo " -v"
echo ""
echo "✅ Deployment complete!"
echo ""
echo "📋 If queue.workolik.com was previously routed elsewhere, you may need to:"
echo " 1. Remove the old router configuration"
echo " 2. Or ensure this new router takes precedence"
echo ""
echo "💡 The proxy adds CORS headers and forwards to Kubernetes LoadBalancer (port 8201)"

68
shfiles/deploy.sh Normal file
View File

@@ -0,0 +1,68 @@
#!/bin/bash
# Kubernetes Deployment Script for 3-Node Cluster
# Usage: ./deploy.sh
set -e
echo "🚀 Deploying to Kubernetes (3-Node Cluster)..."
# Check if kubectl is available
if ! command -v kubectl &> /dev/null; then
echo "❌ kubectl not found. Please install kubectl first."
exit 1
fi
# Check cluster connection
echo "📡 Checking Kubernetes cluster connection..."
kubectl cluster-info || {
echo "❌ Cannot connect to Kubernetes cluster. Please configure kubectl."
exit 1
}
# Check node count
NODE_COUNT=$(kubectl get nodes --no-headers | wc -l)
echo "📊 Cluster has $NODE_COUNT node(s)"
# Apply namespace
echo "📦 Creating namespace..."
kubectl apply -f manifests/namespace.yaml
# Apply secrets
echo "🔐 Creating secrets..."
kubectl apply -f manifests/secrets.yaml
# Apply FastAPI
echo "🐍 Deploying FastAPI backend..."
kubectl apply -f manifests/fastapi-deployment.yaml
kubectl apply -f manifests/fastapi-service.yaml
kubectl apply -f manifests/fastapi-hpa.yaml
# Apply Workers
echo "👷 Deploying NATS workers..."
kubectl apply -f manifests/worker-deployment.yaml
kubectl apply -f manifests/worker-hpa.yaml
# Apply Gateway (optional - only if Gateway API is installed)
read -p "Deploy Gateway API? (requires Gateway API controller) [y/N] " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "🚪 Deploying Gateway API..."
kubectl apply -f manifests/gateway.yaml
fi
echo ""
echo "✅ Deployment complete!"
echo ""
echo "📋 Check status:"
echo " kubectl get pods -n nats-backend -o wide"
echo " kubectl get nodes"
echo " kubectl get hpa -n nats-backend"
echo ""
echo "📊 View pod distribution across nodes:"
echo " kubectl get pods -n nats-backend -o wide | grep -E 'NAME|fastapi|worker'"
echo ""
echo "📝 View logs:"
echo " kubectl logs -f deployment/fastapi-backend -n nats-backend"
echo " kubectl logs -f deployment/nats-worker -n nats-backend"
echo ""

27
shfiles/setup-jetstream.sh Normal file
View File

@@ -0,0 +1,27 @@
#!/bin/bash
# Setup JetStream stream and consumer for NATS
echo "🚀 Setting up NATS JetStream..."
cd "$(dirname "$0")"
# Check if Python is available
if ! command -v python3 &> /dev/null; then
echo "❌ Python3 not found. Please install Python3."
exit 1
fi
# Check if nats-py is installed
if ! python3 -c "import nats" 2>/dev/null; then
echo "📦 Installing nats-py..."
pip3 install nats-py
fi
# Set environment variables
export NATS_URL="nats://nats.workolik.com:4222"
export NATS_USER="admin"
export NATS_PASSWORD="package@321#"
# Run the setup script
python3 scripts/setup_jetstream.py

31
terraform/README.md Normal file
View File

@@ -0,0 +1,31 @@
# Terraform Infrastructure as Code (IaC)
This directory contains the "Recipe Book" for your k3s environment.
## 1. What's here?
Instead of manually running `kubectl apply`, you use **Terraform** to manage your entire cluster.
- **Provider:** It uses the standard `hashicorp/kubernetes` provider.
- **Resources:** It manages your Namespaces, Workers, and Services.
## 2. How to run it:
1. **Install Terraform:** Download and install the Terraform CLI on your computer.
2. **Initialize the project:**
```bash
terraform init
```
3. **Check for changes:** (The "Safety Preview")
```bash
terraform plan
```
4. **Apply the changes:** (This actually updates the cluster)
```bash
terraform apply
```
## 3. Why this is good for your boss:
- **Reproducibility:** If you get a new server, run `terraform apply` and the whole setup is recreated in seconds.
- **Versioning:** You can see every change ever made to your cluster.
- **No drifting:** Terraform ensures that what's in your `.tf` code is EXACTLY what's running in your cluster.
## 4. Next Step:
We can add a **BigRock Provider** (if available) or a generic **SSH/Shell** provider to even automate the creation of the VPS itself.

34
terraform/main.tf Normal file
View File

@@ -0,0 +1,34 @@
terraform {
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.0.0"
}
}
}
provider "kubernetes" {
# This tells Terraform how to connect to your k3s cluster.
# Usually it looks for the file in ~/.kube/config.
config_path = "~/.kube/config"
}
# Example: Manage a Kubernetes Namespace using Terraform
resource "kubernetes_namespace" "core" {
metadata {
name = "core"
labels = {
name = "core"
environment = "production"
}
}
}
# Practical: Point Terraform to your updated .yaml files
# Instead of you running 'kubectl apply', Terraform will do it.
resource "kubernetes_manifest" "worker_orders" {
manifest = yamldecode(file("${path.module}/../manifests/core/workers.yaml"))
}
# (Add more resources here for nearle, alaska, etc.)

20
terraform/namespaces.tf Normal file
View File

@@ -0,0 +1,20 @@
# Create namespaces using Terraform
# This makes sure the zones 'core', 'nearle', 'alaska' are always present and correctly labeled.
resource "kubernetes_namespace" "core" {
metadata {
name = "core"
}
}
resource "kubernetes_namespace" "nearle" {
metadata {
name = "nearle"
}
}
resource "kubernetes_namespace" "alaska" {
metadata {
name = "alaska"
}
}

14
terraform/providers.tf Normal file
View File

@@ -0,0 +1,14 @@
terraform {
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.10.0"
}
}
}
provider "kubernetes" {
# This points to your k3s config file.
# When you get your new server, we will update this to point to the new IP.
config_path = "~/.kube/config"
}

45
terraform/workloads.tf Normal file
View File

@@ -0,0 +1,45 @@
# Manage the Nearle Stack (Jupiter, Atlantis, Fiesta)
# This is the "All-in-one" Terraform control for your major services
# 1. Jupiter Service
resource "kubernetes_manifest" "nearle_jupiter_sts" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/jupiter-sts.yaml"))
}
resource "kubernetes_manifest" "nearle_jupiter_svc" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/jupiter-svc.yaml"))
}
# 2. Atlantis Service
resource "kubernetes_manifest" "nearle_atlantis_sts" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/atlantis-sts.yaml"))
}
resource "kubernetes_manifest" "nearle_atlantis_svc" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/atlantis-svc.yaml"))
}
# 3. Fiesta Service
resource "kubernetes_manifest" "nearle_fiesta_sts" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/fiesta-sts.yaml"))
}
resource "kubernetes_manifest" "nearle_fiesta_svc" {
manifest = yamldecode(file("${path.module}/../manifests/nearle/fiesta-svc.yaml"))
}
# 4. Workers (CPU-heavy isolated nodes)
resource "kubernetes_manifest" "core_workers" {
# This uses your existing workers.yaml file
# Note: Since this file has MANY documents, I recommend splitting it the same way as above.
manifest = yamldecode(file("${path.module}/../manifests/core/workers.yaml"))
}
# 5. Ingress (Unified routing that replaces Docker-side Nginx)
resource "kubernetes_manifest" "core_ingress" {
manifest = yamldecode(file("${path.module}/../manifests/core/ingress-unified.yaml"))
}
resource "kubernetes_manifest" "traefik_middlewares" {
manifest = yamldecode(file("${path.module}/../manifests/core/traefik-middlewares.yaml"))
}

20
traefik-cors-middleware.yml Normal file
View File

@@ -0,0 +1,20 @@
# Traefik CORS Middleware Configuration
# This file can be used if Traefik is configured with file-based dynamic config
# Place this in Traefik's dynamic config directory (usually /etc/traefik/dynamic/ or /traefik/dynamic/)
http:
middlewares:
cors-headers:
headers:
accessControlAllowMethods:
- GET
- POST
- PUT
- PATCH
- DELETE
- OPTIONS
accessControlAllowOrigin: "*"
accessControlAllowHeaders: "*"
accessControlMaxAge: 600
addVaryHeader: true