apiVersion: v1 kind: Secret metadata: name: nearle-redis-secrets namespace: nearle type: Opaque stringData: REDIS_HOST: "66.116.226.255" REDIS_PORT: "6379" REDIS_USER: "default" REDIS_PASSWORD: "Package@324969#" --- apiVersion: v1 kind: ConfigMap metadata: name: riderlogs-retention-script namespace: nearle data: trim.py: | #!/usr/bin/env python3 """ Trims the riderlogs Redis list down to a rolling retention window. riderlogs is append-only (RPUSH), so it's roughly chronologically ordered; we binary-search for the first entry within the retention window and LTRIM everything before it, rather than scanning the whole (900K+ entry) list. """ import os import json import sys from datetime import datetime, timedelta import redis RETENTION_DAYS = int(os.getenv("RETENTION_DAYS", "90")) KEY = os.getenv("REDIS_KEY", "riderlogs") r = redis.Redis( host=os.environ["REDIS_HOST"], port=int(os.environ.get("REDIS_PORT", "6379")), username=os.environ.get("REDIS_USER", "default"), password=os.environ["REDIS_PASSWORD"], socket_timeout=30, ) cutoff = datetime.utcnow() - timedelta(days=RETENTION_DAYS) def get_logdate(idx): v = r.lindex(KEY, idx) if v is None: return None try: d = json.loads(v) ld = d.get("logdate") if not ld: return None return datetime.strptime(ld, "%Y-%m-%d %H:%M:%S") except Exception: return None n = r.llen(KEY) if n == 0: print(f"{KEY}: empty, nothing to do") sys.exit(0) oldest = get_logdate(0) if oldest is None or oldest >= cutoff: print(f"{KEY}: oldest entry ({oldest}) already within the {RETENTION_DAYS}-day window, nothing to trim") sys.exit(0) lo, hi = 0, n - 1 while lo < hi: mid = (lo + hi) // 2 d = get_logdate(mid) if d is None or d < cutoff: lo = mid + 1 else: hi = mid before = n r.ltrim(KEY, lo, -1) after = r.llen(KEY) print(f"{KEY}: cutoff={cutoff.isoformat()} trimmed {before - after} entries ({before} -> {after})") --- apiVersion: batch/v1 kind: CronJob metadata: name: riderlogs-retention namespace: nearle spec: schedule: "0 3 * * *" timeZone: "Asia/Kolkata" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: spec: activeDeadlineSeconds: 300 backoffLimit: 1 template: spec: restartPolicy: Never containers: - name: trim image: python:3.11-slim command: ["sh", "-c", "pip install -q redis && python3 /scripts/trim.py"] env: - name: RETENTION_DAYS value: "90" - name: REDIS_KEY value: "riderlogs" - name: REDIS_HOST valueFrom: secretKeyRef: name: nearle-redis-secrets key: REDIS_HOST - name: REDIS_PORT valueFrom: secretKeyRef: name: nearle-redis-secrets key: REDIS_PORT - name: REDIS_USER valueFrom: secretKeyRef: name: nearle-redis-secrets key: REDIS_USER - name: REDIS_PASSWORD valueFrom: secretKeyRef: name: nearle-redis-secrets key: REDIS_PASSWORD resources: requests: memory: "64Mi" cpu: "50m" limits: memory: "128Mi" volumeMounts: - name: script mountPath: /scripts volumes: - name: script configMap: name: riderlogs-retention-script