Nearle/kubernetes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix worker/gateway logic bugs and duplicate CORS headers

Browse Source 
worker.py (both the ConfigMap copy and conf/worker.py):
- Set an explicit ack_wait=60s on the JetStream pull consumer. It was
  previously left at the implicit default (~30s), the same ballpark
  as the outbound HTTP timeout - a slow-but-legitimate external call
  could cause JetStream to redeliver the message to another worker
  while the first was still mid-request, double-processing a
  non-idempotent call (e.g. duplicate order creation).
- Track in-flight tasks and drain them (bounded wait) before closing
  the NATS/HTTP connections on shutdown, instead of cutting them off
  immediately - avoids dropped/duplicated messages on pod restarts.
- Generic exception handler now does nak(delay=5) instead of an
  undelayed nak(), avoiding a tight redelivery loop on a persistent
  bug.
- Missing 'data' field in a message now explicitly drops with a log
  line instead of silently forwarding the entire internal envelope.
- Removed the hardcoded NATS password fallback baked into the source
  (every deployment already supplies it via a Secret at runtime, so
  this was a redundant plaintext copy sitting in a ConfigMap).

app.py (both the ConfigMap copy and conf/app.py):
- Fixed "NATS by connected" typo -> "NATS not connected".
- Same hardcoded-password-fallback removal as worker.py.

CORS:
- conf/nginx-jupiter.conf and the in-cluster jupiter-cors-proxy nginx
  config both add their own CORS headers without stripping any the
  upstream might set, unlike nginx-queue-proxy.conf which does this
  correctly. Added proxy_hide_header for the ACA-* headers in both -
  browsers reject a response with duplicate Access-Control-* values.

docker-compose.yml:
- Added the missing doormile-proxy service (doormile.com -> :8206 ->
  NodePort 30830). nginx-doormile.conf existed but had no service
  wiring it into Traefik, unlike every other app.
This commit is contained in:
Suriya
2026-07-18 16:07:49 +05:30
parent 836c079a05
commit 91dd240431
7 changed files with 101 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
conf/app.py
View File

@@ -60,7 +60,7 @@ js = None
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "")
# Endpoint to NATS subject mapping
ENDPOINT_ROUTES = {
@@ -172,7 +172,7 @@ async def publish_request_to_nats(endpoint: str, data: Dict[str, Any], request_m
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if not js:
raise HTTPException(status_code=503, detail="NATS by connected")
raise HTTPException(status_code=503, detail="NATS not connected")
try:
# Create a unique inbox for the reply

8
conf/nginx-jupiter.conf
View File

@@ -13,6 +13,14 @@ http {
# NodePorts are bound to 0.0.0.0 and are more reliable to access from host.docker.internal
proxy_pass http://jupiter_k8s;
# Strip any CORS headers the backend may set itself, so we don't
# end up sending duplicate Access-Control-* headers (browsers
# reject a response that has more than one value for these).
proxy_hide_header 'Access-Control-Allow-Origin';
proxy_hide_header 'Access-Control-Allow-Methods';
proxy_hide_header 'Access-Control-Allow-Headers';
proxy_hide_header 'Access-Control-Max-Age';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

26
conf/worker.py
View File

@@ -12,6 +12,7 @@ import asyncio
import aiohttp
import nats
from nats.errors import TimeoutError as NatsTimeoutError
from nats.js.api import ConsumerConfig
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import signal
import sys
@@ -25,7 +26,7 @@ messages_in_flight = Gauge('worker_messages_in_flight', 'Messages currently bein
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "")
# Domain Configuration (Env Vars from Deployment)
NATS_STREAM = os.getenv("NATS_STREAM", "ORDERS")
@@ -35,6 +36,10 @@ FILTER_SUBJECTS = os.getenv("FILTER_SUBJECT", "api.v1.mob.orders.*").split(",")
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "10"))
BASE_URL = os.getenv("EXTERNAL_BASE_URL", "https://jupiter.nearle.app")
# Must stay comfortably above the outbound HTTP timeout below (30s), otherwise
# JetStream can redeliver a message to another worker while this one is still
# waiting on a slow-but-legitimate response, causing a duplicate forward.
ACK_WAIT_SECONDS = int(os.getenv("ACK_WAIT_SECONDS", "60"))
# Endpoint mapping is still useful for constructing the target URL
ENDPOINT_MAPPING = {
@@ -56,6 +61,7 @@ js = None
session: aiohttp.ClientSession = None
running = True
semaphore = None # Initialized in main
active_tasks: set = set() # In-flight process_message() tasks, drained on shutdown
def signal_handler(sig, frame):
global running
@@ -80,7 +86,10 @@ async def forward_to_external(endpoint: str, payload: dict, api_key: str = None,
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
data_to_forward = payload.get("data", payload)
if "data" not in payload:
print(f"?? Message for {endpoint} missing 'data' field, dropping.")
return "DROP", None
data_to_forward = payload["data"]
# Payload Normalization for logs
if endpoint == "/live/api/v2/deliveries/createdeliverylog":
@@ -166,7 +175,8 @@ async def process_message(msg):
await msg.term()
except Exception as e:
print(f"? Critical Worker Error: {e}")
await msg.nak()
# Delay avoids a tight redelivery loop if this is a persistent bug
await msg.nak(delay=5)
finally:
messages_in_flight.dec()
duration = asyncio.get_event_loop().time() - start_time
@@ -211,7 +221,8 @@ async def main():
sub = await js.pull_subscribe(
subject,
durable=durable_name,
stream=NATS_STREAM
stream=NATS_STREAM,
config=ConsumerConfig(ack_wait=ACK_WAIT_SECONDS)
)
subs.append(sub)
except Exception as e:
@@ -231,7 +242,9 @@ async def main():
# Batch size 10, short timeout to keep loop responsive
msgs = await sub.fetch(10, timeout=0.5)
for m in msgs:
asyncio.create_task(process_message(m))
task = asyncio.create_task(process_message(m))
active_tasks.add(task)
task.add_done_callback(active_tasks.discard)
except NatsTimeoutError:
pass
except Exception as e:
@@ -244,6 +257,9 @@ async def main():
except Exception as e:
print(f"?? Fatal Error: {e}")
finally:
if active_tasks:
print(f"?? Draining {len(active_tasks)} in-flight message(s) before shutdown...")
await asyncio.wait(active_tasks, timeout=25)
if session:
await session.close()
if nc:

21
docker-compose.yml
View File

@@ -106,6 +106,27 @@ services:
- "traefik.http.routers.atlantis-api.tls.certresolver=letsencrypt"
- "traefik.http.services.atlantis-api.loadbalancer.server.port=8205"
# Doormile API Proxy (doormile.com -> K8s NodePort 30830)
doormile-proxy:
image: nginx:alpine
container_name: doormile-proxy
restart: unless-stopped
ports:
- "8206:8206"
volumes:
- ./conf/nginx-doormile.conf:/etc/nginx/nginx.conf:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- web
labels:
- "traefik.enable=true"
- "traefik.docker.network=web"
- "traefik.http.routers.doormile-api.rule=Host(`doormile.com`)"
- "traefik.http.routers.doormile-api.tls=true"
- "traefik.http.routers.doormile-api.tls.certresolver=letsencrypt"
- "traefik.http.services.doormile-api.loadbalancer.server.port=8206"
networks:
web:
external: true

28
manifests/core/worker-script.yaml
View File

@@ -19,6 +19,7 @@ data:
import aiohttp
import nats
from nats.errors import TimeoutError as NatsTimeoutError
from nats.js.api import ConsumerConfig
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import signal
import sys
@@ -32,7 +33,7 @@ data:
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "")
# Domain Configuration (Env Vars from Deployment)
NATS_STREAM = os.getenv("NATS_STREAM", "ORDERS")
@@ -42,6 +43,10 @@ data:
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "10"))
BASE_URL = os.getenv("EXTERNAL_BASE_URL", "https://jupiter.nearle.app")
# Must stay comfortably above the outbound HTTP timeout below (30s), otherwise
# JetStream can redeliver a message to another worker while this one is still
# waiting on a slow-but-legitimate response, causing a duplicate forward.
ACK_WAIT_SECONDS = int(os.getenv("ACK_WAIT_SECONDS", "60"))
# Endpoint mapping is still useful for constructing the target URL
ENDPOINT_MAPPING = {
@@ -63,6 +68,7 @@ data:
session: aiohttp.ClientSession = None
running = True
semaphore = None # Initialized in main
active_tasks: set = set() # In-flight process_message() tasks, drained on shutdown
def signal_handler(sig, frame):
global running
@@ -87,7 +93,10 @@ data:
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
data_to_forward = payload.get("data", payload)
if "data" not in payload:
print(f"?? Message for {endpoint} missing 'data' field, dropping.")
return "DROP", None
data_to_forward = payload["data"]
# Payload Normalization for logs
if endpoint == "/live/api/v2/deliveries/createdeliverylog":
@@ -118,7 +127,7 @@ data:
if 200 <= response.status < 300:
return "OK", body
elif response.status >= 500:
print(f"?? Server Error {response.status} from {endpoint}: {body}")
print(f"?? Server Error {response.status} from {endpoint}")
return "RETRY", body
else:
print(f"?? Client Error {response.status} from {endpoint}: {body}")
@@ -173,7 +182,8 @@ data:
await msg.term()
except Exception as e:
print(f"? Critical Worker Error: {e}")
await msg.nak()
# Delay avoids a tight redelivery loop if this is a persistent bug
await msg.nak(delay=5)
finally:
messages_in_flight.dec()
duration = asyncio.get_event_loop().time() - start_time
@@ -218,7 +228,8 @@ data:
sub = await js.pull_subscribe(
subject,
durable=durable_name,
stream=NATS_STREAM
stream=NATS_STREAM,
config=ConsumerConfig(ack_wait=ACK_WAIT_SECONDS)
)
subs.append(sub)
except Exception as e:
@@ -238,7 +249,9 @@ data:
# Batch size 10, short timeout to keep loop responsive
msgs = await sub.fetch(10, timeout=0.5)
for m in msgs:
asyncio.create_task(process_message(m))
task = asyncio.create_task(process_message(m))
active_tasks.add(task)
task.add_done_callback(active_tasks.discard)
except NatsTimeoutError:
pass
except Exception as e:
@@ -251,6 +264,9 @@ data:
except Exception as e:
print(f"?? Fatal Error: {e}")
finally:
if active_tasks:
print(f"?? Draining {len(active_tasks)} in-flight message(s) before shutdown...")
await asyncio.wait(active_tasks, timeout=25)
if session:
await session.close()
if nc:

4
manifests/nearle/fiesta-gateway.yaml
View File

@@ -67,7 +67,7 @@ data:
# Configuration
NATS_URL = os.getenv("NATS_URL", "nats://nats-server:4222")
NATS_USER = os.getenv("NATS_USER", "admin")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "package@321#")
NATS_PASSWORD = os.getenv("NATS_PASSWORD", "")
# Endpoint to NATS subject mapping
ENDPOINT_ROUTES = {
@@ -179,7 +179,7 @@ data:
subject = ENDPOINT_ROUTES.get(endpoint, "api.unknown")
if not js:
raise HTTPException(status_code=503, detail="NATS by connected")
raise HTTPException(status_code=503, detail="NATS not connected")
try:
# Create a unique inbox for the reply

9
manifests/nearle/jupiter-cors-proxy.yaml
View File

@@ -16,6 +16,15 @@ data:
location / {
proxy_pass http://jupiter_backend;
# Strip any CORS headers the backend may set itself, so we don't
# end up sending duplicate Access-Control-* headers (browsers
# reject a response that has more than one value for these).
proxy_hide_header 'Access-Control-Allow-Origin';
proxy_hide_header 'Access-Control-Allow-Methods';
proxy_hide_header 'Access-Control-Allow-Headers';
proxy_hide_header 'Access-Control-Max-Age';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;