feat: add daily retention trim for riderlogs Redis list
riderlogs is an unbounded RPUSH-only list (929K+ entries as of writing, ~650MB), which is why getriderlogs needed a longer timeout in v2.7.58. Adds a CronJob that runs daily at 3am IST, binary-searches for the 90-day retention cutoff (the list is append-ordered so roughly chronological), and LTRIMs anything older. Binary search avoids scanning the full list - ~20 LINDEX calls instead of pulling 900K+ entries. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
144
manifests/nearle/riderlogs-retention-cronjob.yaml
Normal file
144
manifests/nearle/riderlogs-retention-cronjob.yaml
Normal file
@@ -0,0 +1,144 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: nearle-redis-secrets
|
||||
namespace: nearle
|
||||
type: Opaque
|
||||
stringData:
|
||||
REDIS_HOST: "66.116.226.255"
|
||||
REDIS_PORT: "6379"
|
||||
REDIS_USER: "default"
|
||||
REDIS_PASSWORD: "Package@324969#"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: riderlogs-retention-script
|
||||
namespace: nearle
|
||||
data:
|
||||
trim.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Trims the riderlogs Redis list down to a rolling retention window.
|
||||
riderlogs is append-only (RPUSH), so it's roughly chronologically
|
||||
ordered; we binary-search for the first entry within the retention
|
||||
window and LTRIM everything before it, rather than scanning the
|
||||
whole (900K+ entry) list.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import redis
|
||||
|
||||
RETENTION_DAYS = int(os.getenv("RETENTION_DAYS", "90"))
|
||||
KEY = os.getenv("REDIS_KEY", "riderlogs")
|
||||
|
||||
r = redis.Redis(
|
||||
host=os.environ["REDIS_HOST"],
|
||||
port=int(os.environ.get("REDIS_PORT", "6379")),
|
||||
username=os.environ.get("REDIS_USER", "default"),
|
||||
password=os.environ["REDIS_PASSWORD"],
|
||||
socket_timeout=30,
|
||||
)
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=RETENTION_DAYS)
|
||||
|
||||
def get_logdate(idx):
|
||||
v = r.lindex(KEY, idx)
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
d = json.loads(v)
|
||||
ld = d.get("logdate")
|
||||
if not ld:
|
||||
return None
|
||||
return datetime.strptime(ld, "%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
n = r.llen(KEY)
|
||||
if n == 0:
|
||||
print(f"{KEY}: empty, nothing to do")
|
||||
sys.exit(0)
|
||||
|
||||
oldest = get_logdate(0)
|
||||
if oldest is None or oldest >= cutoff:
|
||||
print(f"{KEY}: oldest entry ({oldest}) already within the {RETENTION_DAYS}-day window, nothing to trim")
|
||||
sys.exit(0)
|
||||
|
||||
lo, hi = 0, n - 1
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
d = get_logdate(mid)
|
||||
if d is None or d < cutoff:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
|
||||
before = n
|
||||
r.ltrim(KEY, lo, -1)
|
||||
after = r.llen(KEY)
|
||||
print(f"{KEY}: cutoff={cutoff.isoformat()} trimmed {before - after} entries ({before} -> {after})")
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: riderlogs-retention
|
||||
namespace: nearle
|
||||
spec:
|
||||
schedule: "0 3 * * *"
|
||||
timeZone: "Asia/Kolkata"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
activeDeadlineSeconds: 300
|
||||
backoffLimit: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: trim
|
||||
image: python:3.11-slim
|
||||
command: ["sh", "-c", "pip install -q redis && python3 /scripts/trim.py"]
|
||||
env:
|
||||
- name: RETENTION_DAYS
|
||||
value: "90"
|
||||
- name: REDIS_KEY
|
||||
value: "riderlogs"
|
||||
- name: REDIS_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nearle-redis-secrets
|
||||
key: REDIS_HOST
|
||||
- name: REDIS_PORT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nearle-redis-secrets
|
||||
key: REDIS_PORT
|
||||
- name: REDIS_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nearle-redis-secrets
|
||||
key: REDIS_USER
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nearle-redis-secrets
|
||||
key: REDIS_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
memory: "64Mi"
|
||||
cpu: "50m"
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: riderlogs-retention-script
|
||||
Reference in New Issue
Block a user