c38a36709b
queue.workolik.com was served by three separate routing definitions that didn't agree: nginx-queue-proxy.conf and the classic queue-ingress both sent everything to the deliveries app, but the Gateway API HTTPRoute (deliveries-route) had a carve-out sending /live/api/v1/mob/orders and /live/api/v1/web/products to fiesta's raw backend in the nearle namespace instead (3 fixed replicas, no autoscaling, no resource limits) - a completely different capacity profile from deliveries (HPA'd, 4-20 replicas). Depending on which router won for a given request, orders could land on two backends with very different ability to absorb a burst, plausibly explaining partial order loss / 429s under concurrent load. Removed the carve-out so all three routing paths agree: everything goes to deliveries-service. Also raised deliveries-hpa minReplicas 4->8 and added an explicit aggressive scaleUp behavior (no stabilization delay, up to 4 pods or 100% every 15s). Autoscaling reacts to sustained load over roughly 30-60s (metric polling + pod scheduling + readiness delay), so it does very little for a burst that's over in seconds - minReplicas is the actual defense; the behavior block just makes any further scaling land as fast as possible.
362 lines
9.0 KiB
YAML
362 lines
9.0 KiB
YAML
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: alaska
|
|
labels:
|
|
name: alaska
|
|
environment: production
|
|
app.kubernetes.io/name: alaska
|
|
app.kubernetes.io/managed-by: manuals
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: alaska-config
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: alaska-config
|
|
app.kubernetes.io/part-of: alaska
|
|
data:
|
|
NATS_URL: "nats://nats.workolik.com:4222"
|
|
LOG_LEVEL: "info"
|
|
ALLOWED_ORIGINS: "http://localhost:3001,http://localhost:3000,https://queue.workolik.com,https://console.nearlexpress.com"
|
|
EXTERNAL_BASE_URL: "https://jupiter.nearle.app"
|
|
WORKER_CONCURRENCY: "10"
|
|
RETRY_ATTEMPTS: "5"
|
|
RETRY_DELAY_SECONDS: "5"
|
|
NATS_STREAM: "EVENTS"
|
|
NATS_SUBJECT: "api.>"
|
|
NATS_CONSUMER: "worker_consumer"
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: nats-credentials
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: nats-credentials
|
|
app.kubernetes.io/part-of: alaska
|
|
type: Opaque
|
|
stringData:
|
|
username: admin
|
|
password: package@321#
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: external-endpoint-secrets
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: external-endpoint-secrets
|
|
app.kubernetes.io/part-of: alaska
|
|
type: Opaque
|
|
stringData:
|
|
api_key: "" # Add your API key securely
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: deliveries
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/instance: deliveries-primary
|
|
app.kubernetes.io/part-of: alaska
|
|
app.kubernetes.io/component: backend
|
|
spec:
|
|
serviceName: "deliveries" # Required for StatefulSet
|
|
replicas: 4
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/instance: deliveries-primary
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/instance: deliveries-primary
|
|
app.kubernetes.io/part-of: alaska
|
|
app.kubernetes.io/component: backend
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "8000"
|
|
prometheus.io/path: "/metrics"
|
|
spec:
|
|
securityContext:
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
fsGroup: 2000
|
|
tolerations:
|
|
- key: dedicated
|
|
operator: Equal
|
|
value: apps
|
|
effect: NoSchedule
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: node-role.workolik/app
|
|
operator: In
|
|
values:
|
|
- "true"
|
|
podAntiAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
podAffinityTerm:
|
|
labelSelector:
|
|
matchExpressions:
|
|
- key: app.kubernetes.io/name
|
|
operator: In
|
|
values:
|
|
- deliveries
|
|
topologyKey: kubernetes.io/hostname
|
|
topologySpreadConstraints:
|
|
- maxSkew: 1
|
|
topologyKey: kubernetes.io/hostname
|
|
whenUnsatisfiable: ScheduleAnyway
|
|
labelSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: deliveries
|
|
containers:
|
|
- name: deliveries
|
|
image: workolik360/alaska:v1.2.0
|
|
imagePullPolicy: Always
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
ports:
|
|
- containerPort: 8000
|
|
name: http
|
|
envFrom:
|
|
- configMapRef:
|
|
name: alaska-config
|
|
env:
|
|
- name: NATS_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: nats-credentials
|
|
key: username
|
|
- name: NATS_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: nats-credentials
|
|
key: password
|
|
resources:
|
|
requests:
|
|
memory: "256Mi"
|
|
cpu: "200m"
|
|
limits:
|
|
memory: "1Gi"
|
|
cpu: "1000m"
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /ready
|
|
port: 8000
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 5
|
|
timeoutSeconds: 3
|
|
failureThreshold: 3
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: deliveries-service
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/part-of: alaska
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- port: 8000
|
|
targetPort: 8000
|
|
protocol: TCP
|
|
name: http
|
|
selector:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/instance: deliveries-primary
|
|
sessionAffinity: None
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: deliveries-loadbalancer
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/part-of: alaska
|
|
annotations:
|
|
# Service health check port for LoadBalancer
|
|
# service.kubernetes.io/klipper-lb.healthcheck-port: "8201"
|
|
spec:
|
|
type: NodePort
|
|
externalTrafficPolicy: Cluster
|
|
selector:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/instance: deliveries-primary
|
|
ports:
|
|
- name: http
|
|
port: 8201 # external LB port
|
|
targetPort: 8000 # API container port
|
|
nodePort: 30662
|
|
protocol: TCP
|
|
- name: https
|
|
port: 8441 # optional HTTPS passthrough
|
|
targetPort: 8000
|
|
protocol: TCP
|
|
---
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata:
|
|
name: deliveries-pdb
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/part-of: alaska
|
|
spec:
|
|
minAvailable: 50%
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: deliveries
|
|
---
|
|
apiVersion: autoscaling/v2
|
|
kind: HorizontalPodAutoscaler
|
|
metadata:
|
|
name: deliveries-hpa
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries
|
|
app.kubernetes.io/part-of: alaska
|
|
spec:
|
|
scaleTargetRef:
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
name: deliveries
|
|
minReplicas: 8
|
|
maxReplicas: 20
|
|
# Kubernetes autoscaling reacts to sustained load over ~30-60s (metric
|
|
# polling + pod scheduling + readiness delay) - it does very little for a
|
|
# burst that's over in seconds. This behavior block removes the scale-up
|
|
# stabilization delay and allows adding pods aggressively, so the cluster
|
|
# reacts as fast as the metrics pipeline allows rather than waiting extra
|
|
# cycles. minReplicas above is the main defense for bursts (baseline
|
|
# capacity that's already there before a burst starts); this just makes
|
|
# whatever additional scaling happens land as quickly as possible.
|
|
behavior:
|
|
scaleUp:
|
|
stabilizationWindowSeconds: 0
|
|
policies:
|
|
- type: Percent
|
|
value: 100
|
|
periodSeconds: 15
|
|
- type: Pods
|
|
value: 4
|
|
periodSeconds: 15
|
|
selectPolicy: Max
|
|
scaleDown:
|
|
stabilizationWindowSeconds: 300
|
|
metrics:
|
|
- type: Resource
|
|
resource:
|
|
name: cpu
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 60
|
|
- type: Resource
|
|
resource:
|
|
name: memory
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 70
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: gateway
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: gateway
|
|
app.kubernetes.io/part-of: alaska
|
|
spec:
|
|
gatewayClassName: standard
|
|
listeners:
|
|
- name: http
|
|
protocol: HTTP
|
|
port: 8201
|
|
allowedRoutes:
|
|
namespaces:
|
|
from: All
|
|
- name: https
|
|
protocol: HTTPS
|
|
port: 8441
|
|
allowedRoutes:
|
|
namespaces:
|
|
from: All
|
|
tls:
|
|
mode: Terminate
|
|
certificateRefs:
|
|
- name: deliveries-tls-cert
|
|
- name: nearle-http
|
|
protocol: HTTP
|
|
port: 8202
|
|
allowedRoutes:
|
|
namespaces:
|
|
from: All
|
|
- name: nearle-https
|
|
protocol: HTTPS
|
|
port: 8442
|
|
allowedRoutes:
|
|
namespaces:
|
|
from: All
|
|
tls:
|
|
mode: Terminate
|
|
certificateRefs:
|
|
- name: nearle-tls-cert
|
|
namespace: nearle # Must copy secret to alaska or use ReferenceGrant. For now assume secret is in Alaska or copied.
|
|
# Actually, simpler: Use 'nearle-tls-cert' but putting secret in alaska namespace is required for cross-namespace ref usually unless ReferenceGrant used.
|
|
# Let's keep it simple: We will COPY the secret to 'alaska' namespace.
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: deliveries-route
|
|
namespace: alaska
|
|
labels:
|
|
app.kubernetes.io/name: deliveries-route
|
|
app.kubernetes.io/part-of: alaska
|
|
spec:
|
|
parentRefs:
|
|
- name: gateway
|
|
namespace: alaska
|
|
hostnames:
|
|
- "queue.workolik.com"
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /live
|
|
backendRefs:
|
|
- name: deliveries-service
|
|
port: 8000
|
|
weight: 100
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /health
|
|
backendRefs:
|
|
- name: deliveries-service
|
|
port: 8000
|
|
weight: 100
|