feat: add daily retention trim for riderlogs Redis list
riderlogs is an unbounded RPUSH-only list (929K+ entries as of writing, ~650MB), which is why getriderlogs needed a longer timeout in v2.7.58. Adds a CronJob that runs daily at 3am IST, binary-searches for the 90-day retention cutoff (the list is append-ordered so roughly chronological), and LTRIMs anything older. Binary search avoids scanning the full list - ~20 LINDEX calls instead of pulling 900K+ entries. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
144
manifests/nearle/riderlogs-retention-cronjob.yaml
Normal file
144
manifests/nearle/riderlogs-retention-cronjob.yaml
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: nearle-redis-secrets
|
||||||
|
namespace: nearle
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
REDIS_HOST: "66.116.226.255"
|
||||||
|
REDIS_PORT: "6379"
|
||||||
|
REDIS_USER: "default"
|
||||||
|
REDIS_PASSWORD: "Package@324969#"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: riderlogs-retention-script
|
||||||
|
namespace: nearle
|
||||||
|
data:
|
||||||
|
trim.py: |
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Trims the riderlogs Redis list down to a rolling retention window.
|
||||||
|
riderlogs is append-only (RPUSH), so it's roughly chronologically
|
||||||
|
ordered; we binary-search for the first entry within the retention
|
||||||
|
window and LTRIM everything before it, rather than scanning the
|
||||||
|
whole (900K+ entry) list.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import redis
|
||||||
|
|
||||||
|
RETENTION_DAYS = int(os.getenv("RETENTION_DAYS", "90"))
|
||||||
|
KEY = os.getenv("REDIS_KEY", "riderlogs")
|
||||||
|
|
||||||
|
r = redis.Redis(
|
||||||
|
host=os.environ["REDIS_HOST"],
|
||||||
|
port=int(os.environ.get("REDIS_PORT", "6379")),
|
||||||
|
username=os.environ.get("REDIS_USER", "default"),
|
||||||
|
password=os.environ["REDIS_PASSWORD"],
|
||||||
|
socket_timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
cutoff = datetime.utcnow() - timedelta(days=RETENTION_DAYS)
|
||||||
|
|
||||||
|
def get_logdate(idx):
|
||||||
|
v = r.lindex(KEY, idx)
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
d = json.loads(v)
|
||||||
|
ld = d.get("logdate")
|
||||||
|
if not ld:
|
||||||
|
return None
|
||||||
|
return datetime.strptime(ld, "%Y-%m-%d %H:%M:%S")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
n = r.llen(KEY)
|
||||||
|
if n == 0:
|
||||||
|
print(f"{KEY}: empty, nothing to do")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
oldest = get_logdate(0)
|
||||||
|
if oldest is None or oldest >= cutoff:
|
||||||
|
print(f"{KEY}: oldest entry ({oldest}) already within the {RETENTION_DAYS}-day window, nothing to trim")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
lo, hi = 0, n - 1
|
||||||
|
while lo < hi:
|
||||||
|
mid = (lo + hi) // 2
|
||||||
|
d = get_logdate(mid)
|
||||||
|
if d is None or d < cutoff:
|
||||||
|
lo = mid + 1
|
||||||
|
else:
|
||||||
|
hi = mid
|
||||||
|
|
||||||
|
before = n
|
||||||
|
r.ltrim(KEY, lo, -1)
|
||||||
|
after = r.llen(KEY)
|
||||||
|
print(f"{KEY}: cutoff={cutoff.isoformat()} trimmed {before - after} entries ({before} -> {after})")
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: riderlogs-retention
|
||||||
|
namespace: nearle
|
||||||
|
spec:
|
||||||
|
schedule: "0 3 * * *"
|
||||||
|
timeZone: "Asia/Kolkata"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
successfulJobsHistoryLimit: 3
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
activeDeadlineSeconds: 300
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: trim
|
||||||
|
image: python:3.11-slim
|
||||||
|
command: ["sh", "-c", "pip install -q redis && python3 /scripts/trim.py"]
|
||||||
|
env:
|
||||||
|
- name: RETENTION_DAYS
|
||||||
|
value: "90"
|
||||||
|
- name: REDIS_KEY
|
||||||
|
value: "riderlogs"
|
||||||
|
- name: REDIS_HOST
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: nearle-redis-secrets
|
||||||
|
key: REDIS_HOST
|
||||||
|
- name: REDIS_PORT
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: nearle-redis-secrets
|
||||||
|
key: REDIS_PORT
|
||||||
|
- name: REDIS_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: nearle-redis-secrets
|
||||||
|
key: REDIS_USER
|
||||||
|
- name: REDIS_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: nearle-redis-secrets
|
||||||
|
key: REDIS_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "64Mi"
|
||||||
|
cpu: "50m"
|
||||||
|
limits:
|
||||||
|
memory: "128Mi"
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /scripts
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: riderlogs-retention-script
|
||||||
Reference in New Issue
Block a user