From c38a36709be1e7b3c91f3b9077f0ee8c5f5227c4 Mon Sep 17 00:00:00 2001
From: Suriya <tenext@Suriyas-MacBook-Air.local>
Date: Sat, 18 Jul 2026 16:17:18 +0530
Subject: [PATCH] Fix queue.workolik.com routing conflict and give deliveries
 HPA burst headroom

queue.workolik.com was served by three separate routing definitions that
didn't agree: nginx-queue-proxy.conf and the classic queue-ingress both sent
everything to the deliveries app, but the Gateway API HTTPRoute
(deliveries-route) had a carve-out sending /live/api/v1/mob/orders and
/live/api/v1/web/products to fiesta's raw backend in the nearle namespace
instead (3 fixed replicas, no autoscaling, no resource limits) - a
completely different capacity profile from deliveries (HPA'd, 4-20
replicas). Depending on which router won for a given request, orders could
land on two backends with very different ability to absorb a burst,
plausibly explaining partial order loss / 429s under concurrent load.
Removed the carve-out so all three routing paths agree: everything goes to
deliveries-service.

Also raised deliveries-hpa minReplicas 4->8 and added an explicit
aggressive scaleUp behavior (no stabilization delay, up to 4 pods or 100%
every 15s). Autoscaling reacts to sustained load over roughly 30-60s
(metric polling + pod scheduling + readiness delay), so it does very
little for a burst that's over in seconds - minReplicas is the actual
defense; the behavior block just makes any further scaling land as fast as
possible.
---
 manifests/alaska/alaska.yaml | 41 +++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/manifests/alaska/alaska.yaml b/manifests/alaska/alaska.yaml
index 9bb8060..9e25b1a 100644
--- a/manifests/alaska/alaska.yaml
+++ b/manifests/alaska/alaska.yaml
@@ -244,8 +244,29 @@ spec:
     apiVersion: apps/v1
     kind: StatefulSet
     name: deliveries
-  minReplicas: 4
+  minReplicas: 8
   maxReplicas: 20
+  # Kubernetes autoscaling reacts to sustained load over ~30-60s (metric
+  # polling + pod scheduling + readiness delay) - it does very little for a
+  # burst that's over in seconds. This behavior block removes the scale-up
+  # stabilization delay and allows adding pods aggressively, so the cluster
+  # reacts as fast as the metrics pipeline allows rather than waiting extra
+  # cycles. minReplicas above is the main defense for bursts (baseline
+  # capacity that's already there before a burst starts); this just makes
+  # whatever additional scaling happens land as quickly as possible.
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 0
+      policies:
+      - type: Percent
+        value: 100
+        periodSeconds: 15
+      - type: Pods
+        value: 4
+        periodSeconds: 15
+      selectPolicy: Max
+    scaleDown:
+      stabilizationWindowSeconds: 300
   metrics:
   - type: Resource
     resource:
@@ -330,24 +351,6 @@ spec:
     - name: deliveries-service
       port: 8000
       weight: 100
-  - matches:
-    - path:
-        type: PathPrefix
-        value: /live/api/v1/mob/orders
-    backendRefs:
-    - name: fiesta
-      namespace: nearle
-      port: 80
-      weight: 100
-  - matches:
-    - path:
-        type: PathPrefix
-        value: /live/api/v1/web/products
-    backendRefs:
-    - name: fiesta
-      namespace: nearle
-      port: 80
-      weight: 100
   - matches:
     - path:
         type: PathPrefix