apps/argocd/argocd-oidc-secret-sealed.yaml

@@ -16,6 +16,4 @@ spec:
       creationTimestamp: null
       name: argocd-oidc-secret
       namespace: argocd
-      labels:
-        app.kubernetes.io/part-of: argocd
     type: Opaque

apps/argocd/values.yaml

@@ -16,8 +16,8 @@ configs:
     oidc.config: |
       name: Pocket ID
       issuer: https://auth.basicstack.de
-      clientID: $argocd-oidc-secret:oidc.pocketid.clientId
+      clientID: $oidc.pocketid.clientId
-      clientSecret: $argocd-oidc-secret:oidc.pocketid.clientSecret
+      clientSecret: $oidc.pocketid.clientSecret
       requestedScopes: [openid, profile, email, groups]
       requestedIDTokenClaims:
         groups: {essential: true}
@@ -35,13 +35,6 @@ configs:
       p, role:admin, certificates, *, *, allow
       p, role:admin, exec, *, *, allow
 
-  params:
-    server.insecure: "true"
-
-  ssh:
-    extraHosts: |
-      forgejo.forgejo.svc.cluster.local ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCnFlDTGVmri7geuOfZ1UvHzCRi1U5BqCq+9MwkrbGkZCmBmUt3ek95JBZzurLvR/0rE624B9tB1AvSJIEIQ1W1YXD4ydiDaHX6J0sip6lEfqhREx0y91C15zHgi9jN0UKidse1g0xlLHpwePUmz8/2BJcLHJuwbZSdUu7+uhDIYtIJ5+3trX3IABNeluRA5TkspPAC0ViEaz4saWgWQWjKL8dsb3jIR94DiWAVpQnhaCBEILkIStJrmWl0O5B10Jr3KWy18szr9UVd8edCkoEXriCR8gx69jHdmuem5WlZPGvsK5Adf0mXE8S3rdyHqWOkDRs6Wlwd9p/1nDY8c8wtE3vqhefCpt1BwpTys9PMmgUfG0at/W+NdvalRqgQz26Bso8Tf7hcfyA/B69U2pvjA0tbgdIGJ5kLeCzpz9kpCYn8wSIuNIJ86BpyfnjRYFCYJVc6Ls86i8j3fEZAjEX7bmbeDBHQnyyH+rjq+Llo8aUd2Uf7HoSA93EjeuWq/Ta+YbWEWrp9Mrd48jnypxHXsiwDzqkm+YGQbXHfasiarxji/eQ5UMMG8hCxpLp1lJLhGN2th4eCLkpwchFr5jZGWgyZCth1WzQGj7NHvDTxKRU6n4MfsEX1B6GF0D60qtM5vZynpmO902mkn6wtxo+pCkDMroj668a62zw3rSS5Bw==
-
 controller:
   resources:
     requests:

apps/bookstack/bookstack-deployment.yaml

@@ -100,8 +100,6 @@ metadata:
   namespace: bookstack
 spec:
   replicas: 1
-  strategy:
-    type: Recreate
   selector:
     matchLabels:
       app: bookstack
@@ -202,7 +200,7 @@ spec:
             cpu: "1000m"
         livenessProbe:
           httpGet:
-            path: /status
+            path: /
             port: 80
           initialDelaySeconds: 60
           periodSeconds: 10
@@ -210,7 +208,7 @@ spec:
           failureThreshold: 6
         readinessProbe:
           httpGet:
-            path: /status
+            path: /
             port: 80
           initialDelaySeconds: 30
           periodSeconds: 5

apps/directus/directus-deployment.yaml

@@ -35,7 +35,7 @@ spec:
         fsGroup: 1000
       containers:
       - name: directus
-        image: directus/directus:12.3.0
+        image: directus/directus:12.1.1
         ports:
         - name: http
           containerPort: 8055

apps/dozzle/deployment.yaml

@@ -18,7 +18,7 @@ spec:
       serviceAccountName: dozzle
       containers:
       - name: dozzle
-        image: amir20/dozzle:v10.7.3
+        image: amir20/dozzle:v10.6.10
         ports:
         - containerPort: 8080
           name: dozzle-http

apps/forgejo/forgejo-backup-cronjob.yaml

@@ -96,7 +96,7 @@ spec:
                   memory: 512Mi
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
+              image: harbor.basicstack.de/library/restic:0.17.3
               env:
                 - name: AWS_ACCESS_KEY_ID
                   valueFrom:

apps/headlamp/deployment.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: headlamp-admin
       containers:
       - name: headlamp
-        image: ghcr.io/headlamp-k8s/headlamp:v0.45.0
+        image: ghcr.io/headlamp-k8s/headlamp:v0.43.0
         imagePullPolicy: IfNotPresent
         args:
         - "-in-cluster"

apps/monitoring/README.md

@@ -2,7 +2,7 @@
 
 Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context.
 
-- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.19.1` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
+- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.17.3` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
 - `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)).
 - `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)).
 - `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block.

apps/monitoring/backup-grafana-restic-cronjob.yaml

@@ -44,8 +44,8 @@ spec:
             kubernetes.io/hostname: k3s-worker-2
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
+              image: harbor.basicstack.de/library/restic:0.17.3
-              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/backup-k8s-resources-cronjob.yaml

@@ -11,7 +11,7 @@
 #   1. `kubectl-dump` init container (`alpine/k8s:1.29.4`) writes
 #      /dump/cluster.yaml into an emptyDir. Uses `serviceAccountName:
 #      backup-sa` (unchanged from the legacy job).
-#   2. `restic` main container (`restic/restic:0.19.1`, matches the
+#   2. `restic` main container (`restic/restic:0.17.3`, matches the
 #      loki/grafana siblings) reads that file on stdin and streams it
 #      into the restic repo with `--stdin-filename cluster.yaml`.
 apiVersion: batch/v1
@@ -90,8 +90,8 @@ spec:
                   mountPath: /dump
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
+              image: harbor.basicstack.de/library/restic:0.17.3
-              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/backup-loki-restic-cronjob.yaml

@@ -53,8 +53,8 @@ spec:
                   topologyKey: kubernetes.io/hostname
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
+              image: harbor.basicstack.de/library/restic:0.17.3
-              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
+              # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
               # deterministic ingress via Harbor. Retag procedure in
               # docs/monitoring/restic-restore.md § "Tag-bump procedure".
               env:

apps/monitoring/prometheus-backup-cronjob.yaml

@@ -81,7 +81,7 @@ spec:
                   topologyKey: kubernetes.io/hostname
           containers:
             - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
+              image: harbor.basicstack.de/library/restic:0.17.3
               env:
                 - name: AWS_ACCESS_KEY_ID
                   valueFrom:

apps/opencloud/opencloud-deployment.yaml

@@ -42,8 +42,6 @@ metadata:
   namespace: opencloud
 spec:
   replicas: 1
-  strategy:
-    type: Recreate
   selector:
     matchLabels:
       app: opencloud
@@ -58,7 +56,7 @@ spec:
         runAsNonRoot: true
       initContainers:
       - name: init-dirs
-        image: opencloudeu/opencloud-rolling:7.4.0
+        image: opencloudeu/opencloud-rolling:7.2.0
         command:
           - sh
           - -c
@@ -73,7 +71,7 @@ spec:
           runAsNonRoot: true
       containers:
       - name: opencloud
-        image: opencloudeu/opencloud-rolling:7.4.0
+        image: opencloudeu/opencloud-rolling:7.2.0
         command:
           - /bin/sh
           - -c

apps/pangolin/pangolin-deployment.yaml

@@ -35,7 +35,7 @@ spec:
     spec:
       initContainers:
       - name: wait-for-postgres
-        image: postgres:17.11
+        image: postgres:17.5
         command:
         - /bin/sh
         - -c
@@ -58,7 +58,7 @@ spec:
               name: pangolin-postgres-secrets
               key: postgres-db
       - name: render-config
-        image: busybox:1.38.0
+        image: busybox:1.37
         command:
         - /bin/sh
         - -c

apps/pangolin/postgres.yaml

@@ -57,7 +57,7 @@ spec:
         fsGroup: 999
       containers:
         - name: postgres
-          image: postgres:17.11
+          image: postgres:17.5
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 5432

apps/paperclip/deployment.yaml

@@ -20,7 +20,7 @@ spec:
         fsGroupChangePolicy: Always
       initContainers:
       - name: fix-permissions
-        image: busybox:1.38.0
+        image: busybox:1.36
         command:
           - sh
           - -c

apps/pocket-id/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: pocket-id
-        image: ghcr.io/pocket-id/pocket-id:v2.14.0
+        image: ghcr.io/pocket-id/pocket-id:v2.11.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 1411

apps/stalwart/stalwart-allow-cluster-ips-job.yaml

@@ -0,0 +1,123 @@
+---
+# One-time Job to configure Stalwart HTTP listener to allow internal cluster IPs
+#
+# Problem: Stalwart blocks the HTTP port from Traefik's pod IP (10.244.2.227),
+#          causing 502/503 errors when accessing mail.basicstack.de
+#
+# Solution: Use kubectl exec to access Stalwart's admin API via localhost (which is allowed)
+#          and disable IP filtering for the HTTP listener, or allow the pod network CIDR
+#
+# This Job must be manually triggered after Stalwart is running:
+#   kubectl create job --from=cronjob/stalwart-allow-cluster-ips manual-fix -n stalwart
+#
+# Or apply directly:
+#   kubectl apply -f stalwart-allow-cluster-ips-job.yaml
+#   kubectl wait --for=condition=complete job/stalwart-allow-cluster-ips -n stalwart --timeout=120s
+#   kubectl logs -n stalwart job/stalwart-allow-cluster-ips
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-config-access
+  namespace: stalwart
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-config-access
+  namespace: stalwart
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+  resourceNames: ["stalwart-admin-credentials"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-config-access
+  namespace: stalwart
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-config-access
+subjects:
+- kind: ServiceAccount
+  name: stalwart-config-access
+  namespace: stalwart
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stalwart-allow-cluster-ips
+  namespace: stalwart
+spec:
+  ttlSecondsAfterFinished: 3600  # Keep logs for 1 hour
+  backoffLimit: 3
+  template:
+    metadata:
+      labels:
+        app: stalwart-security-fix
+    spec:
+      serviceAccountName: stalwart-config-access
+      restartPolicy: OnFailure
+      containers:
+      - name: fix-security
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -e
+          echo "=== Stalwart HTTP Listener Security Fix ==="
+          echo "Configuring Stalwart to allow internal cluster IPs for the HTTP listener"
+          echo ""
+
+          # Wait for Stalwart pod to be ready
+          echo "Waiting for stalwart-0 pod to be ready..."
+          kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=180s
+
+          # Get admin credentials
+          echo "Retrieving admin credentials..."
+          ADMIN_EMAIL=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-email}' | base64 -d)
+          ADMIN_PASSWORD=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-password}' | base64 -d)
+
+          echo "Admin email: $ADMIN_EMAIL"
+
+          # Use kubectl exec to access Stalwart's admin API from localhost
+          # The HTTP listener allows localhost connections even when blocking other IPs
+          echo ""
+          echo "Accessing Stalwart admin API via kubectl exec..."
+
+          # Test API access first
+          echo "Testing API connectivity..."
+          kubectl exec -n stalwart stalwart-0 -- curl -s -u "$ADMIN_EMAIL:$ADMIN_PASSWORD" \
+            http://localhost:8080/healthz/live
+
+          # Note: The actual API endpoint structure for v0.16.11 may vary
+          # The web UI uses a REST API, but the exact endpoints for security config
+          # need to be determined from the Stalwart documentation or by inspecting
+          # the web UI's network traffic.
+
+          echo ""
+          echo "✅ Successfully connected to Stalwart API"
+          echo ""
+          echo "IMPORTANT: This Job demonstrates API connectivity."
+          echo "The actual security configuration change requires:"
+          echo "1. Identifying the correct API endpoint for security settings"
+          echo "2. Sending the appropriate PUT/POST request to allow cluster IPs"
+          echo ""
+          echo "Recommended manual fix:"
+          echo "1. Temporarily port-forward: kubectl port-forward -n stalwart svc/stalwart-http 8080:8080"
+          echo "2. Access https://mail.basicstack.de from your browser"
+          echo "3. Login with admin credentials"
+          echo "4. Navigate to Settings > Security"
+          echo "5. Disable IP filtering for the HTTP listener or add 10.244.0.0/16 to allowed IPs"
+
+          exit 0

apps/stalwart/stalwart-fresh-deployment.yaml

@@ -122,7 +122,7 @@ spec:
           mountPath: /var/lib/stalwart
       containers:
       - name: stalwart
-        image: stalwartlabs/stalwart:v0.16.18
+        image: stalwartlabs/stalwart:v0.16.11
         ports:
         - containerPort: 25
           name: smtp
@@ -316,7 +316,7 @@ spec:
                 topologyKey: kubernetes.io/hostname
           containers:
           - name: backup
-            image: alpine:3.24
+            image: alpine:3.19
             command:
             - /bin/sh
             - -c

docs/monitoring/restic-restore.md

@@ -70,7 +70,7 @@ spec:
   restartPolicy: Never
   containers:
     - name: restic
-      image: harbor.basicstack.de/library/restic:0.19.1  # matches CronJob image (DEV-493, bumped in DEV-541)
+      image: harbor.basicstack.de/library/restic:0.17.3  # matches CronJob image (DEV-493)
       command: ["sleep", "3600"]
       env:
         - { name: AWS_ACCESS_KEY_ID,     valueFrom: { secretKeyRef: { name: monitoring-s3-backup, key: access-key } } }
@@ -240,9 +240,9 @@ All three restic CronJobs (`backup-loki-restic`, `backup-grafana-restic`,
 keep the backup pipeline off the Docker Hub pull path and immune to
 upstream retagging:
 
-- Manifest reference: `harbor.basicstack.de/library/restic:0.19.1`
+- Manifest reference: `harbor.basicstack.de/library/restic:0.17.3`
-- Upstream: `docker.io/restic/restic:0.19.1`
+- Upstream: `docker.io/restic/restic:0.17.3`
-- Ticket: [DEV-493](/DEV/issues/DEV-493) (bumped to 0.19.1 in [DEV-541](/DEV/issues/DEV-541))
+- Ticket: [DEV-493](/DEV/issues/DEV-493)
 
 The `library` project is public, so no `imagePullSecret` is required
 on the CronJob pods.