From a5ceedf4da775e92cda00a7e74b77dfa12862f13 Mon Sep 17 00:00:00 2001
From: CTO Agent <cto@basicstack.de>
Date: Sun, 9 Aug 2026 17:40:50 +0000
Subject: [PATCH] fix(monitoring): prune Prometheus S3 backups by prefix-date,
 not file mtime (DEV-465)

The original `rclone delete --min-age 7d` step ate the freshly uploaded
backup: rclone preserves each source file's mtime, and Prometheus TSDB
chunk/block files retain very old mtimes (weeks-old immutable blocks),
so filtering by file age deleted ~72% of the objects immediately after
sync (verified: 8.764 GiB destination reduced to 2.310 GiB / 37 objects
before we noticed).

Replaced the mtime prune with a prefix-name-based prune: every top-level
prefix is `prometheus-YYYYMMDD-HHMMSS`, so we parse the encoded date and
`rclone purge` whole prefixes older than 7 days. This keeps the latest
7 daily snapshots intact regardless of the Prometheus block ages.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 .../monitoring/prometheus-backup-cronjob.yaml | 43 +++++++++++++++++--
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/apps/monitoring/prometheus-backup-cronjob.yaml b/apps/monitoring/prometheus-backup-cronjob.yaml
index 80f8ee2..62e1a44 100644
--- a/apps/monitoring/prometheus-backup-cronjob.yaml
+++ b/apps/monitoring/prometheus-backup-cronjob.yaml
@@ -129,10 +129,45 @@ spec:
                   fi
                   echo "OK: destination size acceptable."
 
-                  echo "Pruning prometheus backups older than 7 days..."
-                  rclone delete "hetzner-s3:${S3_BUCKET}/prometheus/" \
-                    --min-age 7d || echo "prune step reported errors (continuing)"
-                  rclone rmdirs "hetzner-s3:${S3_BUCKET}/prometheus/" --leave-root || true
+                  # Prune by prefix name, NOT by file mtime. rclone preserves
+                  # each source file's mtime on upload, and Prometheus TSDB
+                  # chunk files retain very old mtimes (weeks-old blocks),
+                  # so `rclone delete --min-age 7d` would eat the just-
+                  # uploaded backup. Every top-level prefix is named
+                  # `prometheus-YYYYMMDD-HHMMSS`, so we compare the encoded
+                  # date to a 7-day threshold and purge whole prefixes.
+                  echo "Pruning prometheus backup prefixes older than 7 days..."
+                  # BusyBox date lacks GNU's `-d "7 days ago"` and BSD's `-v -7d`,
+                  # so compute the cutoff via @epoch which BusyBox does support.
+                  CUTOFF_EPOCH=$(( $(date +%s) - 7 * 86400 ))
+                  CUTOFF=$(date -d "@${CUTOFF_EPOCH}" +%Y%m%d)
+                  echo "Cutoff (delete prefixes with date < ${CUTOFF}):"
+                  KEEP=0
+                  PRUNE=0
+                  for PREFIX in $(rclone lsf --dirs-only "hetzner-s3:${S3_BUCKET}/prometheus/" 2>/dev/null | sed 's:/$::'); do
+                    case "${PREFIX}" in
+                      prometheus-*)
+                        PDATE=$(echo "${PREFIX}" | sed -n 's/^prometheus-\([0-9]\{8\}\)-.*/\1/p')
+                        if [ -z "${PDATE}" ]; then
+                          echo "  SKIP ${PREFIX} (unparseable name)"
+                          continue
+                        fi
+                        if [ "${PDATE}" -lt "${CUTOFF}" ]; then
+                          echo "  PURGE ${PREFIX} (date ${PDATE} < ${CUTOFF})"
+                          rclone purge "hetzner-s3:${S3_BUCKET}/prometheus/${PREFIX}" \
+                            || echo "  WARN purge failed for ${PREFIX} (continuing)"
+                          PRUNE=$((PRUNE + 1))
+                        else
+                          echo "  KEEP  ${PREFIX} (date ${PDATE})"
+                          KEEP=$((KEEP + 1))
+                        fi
+                        ;;
+                      *)
+                        echo "  SKIP ${PREFIX} (not a prometheus- prefix)"
+                        ;;
+                    esac
+                  done
+                  echo "Prune summary: kept ${KEEP}, purged ${PRUNE}."
 
                   echo "Post-run inventory (prometheus/ prefixes):"
                   rclone lsd "hetzner-s3:${S3_BUCKET}/prometheus/" || true