diff --git a/manifests/nearle/riderlogs-retention-cronjob.yaml b/manifests/nearle/riderlogs-retention-cronjob.yaml new file mode 100644 index 0000000..fff06dc --- /dev/null +++ b/manifests/nearle/riderlogs-retention-cronjob.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nearle-redis-secrets + namespace: nearle +type: Opaque +stringData: + REDIS_HOST: "66.116.226.255" + REDIS_PORT: "6379" + REDIS_USER: "default" + REDIS_PASSWORD: "Package@324969#" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: riderlogs-retention-script + namespace: nearle +data: + trim.py: | + #!/usr/bin/env python3 + """ + Trims the riderlogs Redis list down to a rolling retention window. + riderlogs is append-only (RPUSH), so it's roughly chronologically + ordered; we binary-search for the first entry within the retention + window and LTRIM everything before it, rather than scanning the + whole (900K+ entry) list. + """ + import os + import json + import sys + from datetime import datetime, timedelta + + import redis + + RETENTION_DAYS = int(os.getenv("RETENTION_DAYS", "90")) + KEY = os.getenv("REDIS_KEY", "riderlogs") + + r = redis.Redis( + host=os.environ["REDIS_HOST"], + port=int(os.environ.get("REDIS_PORT", "6379")), + username=os.environ.get("REDIS_USER", "default"), + password=os.environ["REDIS_PASSWORD"], + socket_timeout=30, + ) + + cutoff = datetime.utcnow() - timedelta(days=RETENTION_DAYS) + + def get_logdate(idx): + v = r.lindex(KEY, idx) + if v is None: + return None + try: + d = json.loads(v) + ld = d.get("logdate") + if not ld: + return None + return datetime.strptime(ld, "%Y-%m-%d %H:%M:%S") + except Exception: + return None + + n = r.llen(KEY) + if n == 0: + print(f"{KEY}: empty, nothing to do") + sys.exit(0) + + oldest = get_logdate(0) + if oldest is None or oldest >= cutoff: + print(f"{KEY}: oldest entry ({oldest}) already within the {RETENTION_DAYS}-day window, nothing to trim") + sys.exit(0) + + lo, hi = 0, n - 1 + while lo < hi: + mid = (lo + hi) // 2 + d = get_logdate(mid) + if d is None or d < cutoff: + lo = mid + 1 + else: + hi = mid + + before = n + r.ltrim(KEY, lo, -1) + after = r.llen(KEY) + print(f"{KEY}: cutoff={cutoff.isoformat()} trimmed {before - after} entries ({before} -> {after})") +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: riderlogs-retention + namespace: nearle +spec: + schedule: "0 3 * * *" + timeZone: "Asia/Kolkata" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 300 + backoffLimit: 1 + template: + spec: + restartPolicy: Never + containers: + - name: trim + image: python:3.11-slim + command: ["sh", "-c", "pip install -q redis && python3 /scripts/trim.py"] + env: + - name: RETENTION_DAYS + value: "90" + - name: REDIS_KEY + value: "riderlogs" + - name: REDIS_HOST + valueFrom: + secretKeyRef: + name: nearle-redis-secrets + key: REDIS_HOST + - name: REDIS_PORT + valueFrom: + secretKeyRef: + name: nearle-redis-secrets + key: REDIS_PORT + - name: REDIS_USER + valueFrom: + secretKeyRef: + name: nearle-redis-secrets + key: REDIS_USER + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: nearle-redis-secrets + key: REDIS_PASSWORD + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + volumeMounts: + - name: script + mountPath: /scripts + volumes: + - name: script + configMap: + name: riderlogs-retention-script