apps/outline/redis.yaml aktualisiert

Scale down outline-redis

apps/app-grist.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: grist
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/grist
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: grist
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/app-kaneo.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kaneo
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/kaneo
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kaneo
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/app-openwebui.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: openwebui
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/openwebui
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: openwebui
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/app-outline.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: outline
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/outline
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: outline
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/app-velero-monitoring.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero-monitoring
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: velero
+    directory:
+      recurse: false
+      include: '{servicemonitor.yaml,prometheusrule.yaml}'
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: velero
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true

apps/app-velero-restore-drill.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero-restore-drill
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: velero/restore-drill
+    directory:
+      recurse: false
+      include: '*.yaml'
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: velero
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true

apps/app-velero-schedules.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero-schedules
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: velero/schedules
+    directory:
+      recurse: false
+      include: '*.yaml'
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: velero
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true

apps/app-velero.yaml

@@ -0,0 +1,21 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/velero
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: velero
+  syncPolicy:
+    automated:
+      prune: false
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true

apps/app-vikunja.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: vikunja
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/vikunja
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: vikunja
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/argocd/README.md

@@ -27,7 +27,7 @@ Memory limits were added after DEV-281 (resource exhaustion incident on 2026-07-
 
 | Component | Memory Limit | Memory Request |
 |-----------|--------------|----------------|
-| application-controller | 512Mi | 256Mi |
+| application-controller | 1Gi | 512Mi |
 | repo-server | 512Mi | 256Mi |
 | redis | 256Mi | 128Mi |
 | server | 256Mi | 128Mi |

apps/argocd/values.yaml

@@ -45,9 +45,9 @@ configs:
 controller:
   resources:
     requests:
-      memory: 256Mi
-    limits:
       memory: 512Mi
+    limits:
+      memory: 1Gi
 
 repoServer:
   resources:

apps/bookstack/bookstack-deployment.yaml

@@ -125,6 +125,7 @@ spec:
       containers:
       - name: bookstack
         image: lscr.io/linuxserver/bookstack:latest
+        imagePullPolicy: Always
         ports:
         - containerPort: 80
           name: http

apps/grist/README.md

@@ -0,0 +1,94 @@
+# Grist Deployment
+
+This directory contains the Kubernetes manifests for the Grist deployment at https://grist.basicstack.de
+
+Grist is an open-source, self-hostable spreadsheet / database hybrid ([grist-core](https://github.com/gristlabs/grist-core)).
+
+## Components
+
+- **grist-deployment.yaml**: Main Grist application deployment
+  - Image: `gristlabs/grist:1.7.7`
+  - Port: 8484
+  - OIDC login via Pocket ID (`https://auth.basicstack.de`)
+  - Data persisted at `/persist`
+
+- **grist-service.yaml**: ClusterIP service exposing port 80 -> 8484
+
+- **grist-data-pvc.yaml**: PVC for Grist app data (`/persist`), 20 GiB on `hcloud-volumes-encrypted`
+
+- **grist-postgres-deployment.yaml** / **grist-postgres-service.yaml**: In-namespace PostgreSQL 16 for the Grist home database (TypeORM connection)
+
+- **grist-db-pvc.yaml**: PVC for PostgreSQL data, 20 GiB on `hcloud-volumes-encrypted`
+
+- **grist-ingress.yaml**: Traefik ingress with TLS
+  - Host: `grist.basicstack.de`
+  - TLS via cert-manager (`letsencrypt-prod`, DNS-01 via Hetzner webhook)
+
+- **grist-db-credentials-sealed.yaml**: SealedSecret for PostgreSQL user password
+- **grist-app-secrets-sealed.yaml**: SealedSecret with `GRIST_SESSION_SECRET`
+- **grist-oidc-secret-sealed.yaml**: SealedSecret with Pocket ID OIDC client credentials
+
+## OIDC Configuration (Pocket ID)
+
+- Pocket ID group: `grist`
+- Client name in Pocket ID: `Grist`
+- Callback URL: `https://grist.basicstack.de/oauth2/callback`
+- Client is restricted to members of the `grist` group
+
+Grist environment variables driving OIDC (from https://pocket-id.org/docs/client-examples/grist):
+
+```
+GRIST_OIDC_IDP_ISSUER=https://auth.basicstack.de
+GRIST_OIDC_IDP_CLIENT_ID=<from secret>
+GRIST_OIDC_IDP_CLIENT_SECRET=<from secret>
+GRIST_OIDC_SP_HOST=https://grist.basicstack.de
+GRIST_OIDC_IDP_SCOPES="openid email profile"
+GRIST_OIDC_IDP_END_SESSION_ENDPOINT=https://auth.basicstack.de/api/oidc/end-session
+GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED=true
+GRIST_DEFAULT_EMAIL=andreas.leinen@basicstack.de
+GRIST_FORCE_LOGIN=true
+```
+
+`GRIST_DEFAULT_EMAIL` is used by Grist as the primary admin identity - the value must be a valid Pocket ID user.
+
+## Storage
+
+Both PVCs (`grist-data`, `grist-db`) use `hcloud-volumes-encrypted` (Retain reclaim policy) as required for
+sensitive workload data on Hetzner.
+
+## GitOps
+
+Managed by Argo CD via `apps/app-grist.yaml`:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: grist
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
+    targetRevision: main
+    path: apps/grist
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: grist
+  syncPolicy:
+    syncOptions:
+    - CreateNamespace=true
+```
+
+## Verification
+
+```
+kubectl -n grist get pods
+kubectl -n grist get pvc
+kubectl -n grist get ingress
+curl -I https://grist.basicstack.de
+```
+
+## Backup
+
+The PostgreSQL PVC uses reclaim policy `Retain`. For scheduled logical backups add a `CronJob` like the one in `apps/directus/directus-backup-cronjob.yaml`.

apps/grist/grist-app-secrets-sealed.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: grist-app-secrets
+  namespace: grist
+spec:
+  encryptedData:
+    session-secret: AgBN2KDg6QQgvhD2WvTG/r4+y6FrY7EXc8kU8N4hUDIh/VJDoyK+xiEHFvuOXMH3ZpNw4qTEpL+a+6LVWM1/c6mdcTIW5J52+hOOoXEtSXJhoBCjljgddlTTxHy9sIdE0694dMgeBJwmnvT9vm1TNQWMbiBaD05Xh4VCX3D908L4l/CiM11M9CLkoMGK7cTz9OCuGtd36k4OUxmw3/mQnafCDyzKf07NRW70Fr/bMGCdKE4Jt1rqWBfIaTb2mq96FCHosFq37keTCh1gu2nKf3iWesfv40rsVNN3wEhATaG2ghmjP2ul21zMOIO5GuCnMcawZv8VR1gyb5f98WOinL1zf5NMPCPRdTl/3G2gpqPc9VRNxOGguup3NLCaV2J+vbH5dgEj/h6zeTXw5bRARRQHAaGhUgFS3PALzubTzaUyNyFOyaMoXbAnE79HU+7QEd3Yjpdujup4Pe6OioK2k7jGWJj/jxZokffCjUsG9+hvNlg2E4ETtXxmHXIkwgTx7nHON6UApnprCtAUatRuMT3H+4VzfgkjV6y2aIHcp21EbH+qDBLo83+sBeUeWYyRtA/W2BGoAhx1CXeBysmEjD/Cls2h2PVhIvUdfNBY9fDgLemuK7MxdKbR8hRb3zHeedLXM4wevg2zCTyyGtizeC3JmpnqO1UHsKBZMGyUh43r+7sh3SyBKVUJxZt0q2JOWibaZNbIsrqcAyb8Te/QZlayZ2s2BxvMvvNrNpcOTYs/KzwwaHva8OUl
+  template:
+    metadata:
+      creationTimestamp: null
+      name: grist-app-secrets
+      namespace: grist

apps/grist/grist-data-pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grist-data
+  namespace: grist
+  labels:
+    app: grist
+    component: data
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 20Gi

apps/grist/grist-db-credentials-sealed.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: grist-db-credentials
+  namespace: grist
+spec:
+  encryptedData:
+    postgres-password: AgA7MFNJ8//EOg4CdfHlSe2yGW6zcJyGsoiWVYlbVkb2GgMZmGQDH/ImYOTfeZXkEQhAWhA0braRB7Prx8fwpON2+6PsOczg+S1N1ykuDMJ+KDuXHhJtm5LNqbgibddX9y6YYInorgxM5mh/ORt2D1IdTwf1AXACkEkp8Ws4IGIfuUCym7kNM77mVD9jW4xyf/c7RE/J3va7L/6SRdx2rdoil8loMzMTHH+lvc738UdFA67hoLkKuseiYgA6NZap5Ppb48GTQul47ji3wH24F/u8DTKiHWbMBxwWO48gQJvAISgXSCW4eiKhkAcm28JACCOd1jT7Asen/yd5qEISB8Fr8q2qwxQwM3kXFIJRf9VVN7BWv6hEMnDO3AhBmcRr+gCPhKial17Varakh0ruBXB4eQDlxGQZ09trwtRPZkOsQnNf38iMQBpDH5U12tASmi1t5ddGKzQQh3EWx1vJyap8xNFf5zK6MuoDEMekHrjcclqgoMWppMH26Daghp2XZl740bV0UhLQBO1TOFEORHBs2DA2A3msmN25selVxjFBHvwLZHQJOHoVHbcqlbF7e2bFSLnij1TBMlPre/llBzoiVYrjHQoHw3fg7xVMrPBT+U1wn4ItDFZTP+nCxnWejtQ/dxzBfMrvBcHVhACsw7VhVkM5nhNFJBda65czyud0BWCSMYJYZEBZjDs220/mhQWIh2/RLANN6pcwJ2ONGyNJs3uo7Q245DlmDTjH5Qppmg==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: grist-db-credentials
+      namespace: grist

apps/grist/grist-db-pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grist-db
+  namespace: grist
+  labels:
+    app: grist
+    component: database
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 20Gi

apps/grist/grist-deployment.yaml

@@ -0,0 +1,127 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grist
+  namespace: grist
+  labels:
+    app: grist
+    component: app
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: grist
+      component: app
+  template:
+    metadata:
+      labels:
+        app: grist
+        component: app
+    spec:
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until pg_isready -h grist-postgres -U grist; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+      securityContext:
+        fsGroup: 1000
+      containers:
+      - name: grist
+        image: gristlabs/grist:1.7.19
+        ports:
+        - name: http
+          containerPort: 8484
+        env:
+        - name: APP_HOME_URL
+          value: https://grist.basicstack.de
+        - name: GRIST_DOMAIN
+          value: grist.basicstack.de
+        - name: GRIST_SINGLE_ORG
+          value: docs
+        - name: GRIST_ORG_IN_PATH
+          value: "false"
+        - name: GRIST_DEFAULT_PRODUCT
+          value: Free
+        - name: GRIST_HIDE_UI_ELEMENTS
+          value: billing
+        - name: PORT
+          value: "8484"
+        - name: TYPEORM_TYPE
+          value: postgres
+        - name: TYPEORM_HOST
+          value: grist-postgres
+        - name: TYPEORM_PORT
+          value: "5432"
+        - name: TYPEORM_DATABASE
+          value: grist
+        - name: TYPEORM_USERNAME
+          value: grist
+        - name: TYPEORM_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: grist-db-credentials
+              key: postgres-password
+        - name: GRIST_SESSION_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: grist-app-secrets
+              key: session-secret
+        - name: GRIST_OIDC_IDP_ISSUER
+          value: https://auth.basicstack.de
+        - name: GRIST_OIDC_IDP_CLIENT_ID
+          valueFrom:
+            secretKeyRef:
+              name: grist-oidc-secret
+              key: client-id
+        - name: GRIST_OIDC_IDP_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: grist-oidc-secret
+              key: client-secret
+        - name: GRIST_OIDC_SP_HOST
+          value: https://grist.basicstack.de
+        - name: GRIST_OIDC_IDP_SCOPES
+          value: openid email profile
+        - name: GRIST_OIDC_IDP_END_SESSION_ENDPOINT
+          value: https://auth.basicstack.de/api/oidc/end-session
+        - name: GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED
+          value: "true"
+        - name: GRIST_DEFAULT_EMAIL
+          value: andreas.leinen@basicstack.de
+        - name: GRIST_FORCE_LOGIN
+          value: "true"
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: "1"
+            memory: 2Gi
+        livenessProbe:
+          tcpSocket:
+            port: 8484
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /status
+            port: 8484
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          timeoutSeconds: 3
+        volumeMounts:
+        - name: data
+          mountPath: /persist
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: grist-data

apps/grist/grist-ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: grist
+  namespace: grist
+  labels:
+    app: grist
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  tls:
+  - hosts:
+    - grist.basicstack.de
+    secretName: grist-tls
+  rules:
+  - host: grist.basicstack.de
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: grist
+            port:
+              number: 80

apps/grist/grist-oidc-secret-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: grist-oidc-secret
+  namespace: grist
+spec:
+  encryptedData:
+    client-id: AgAAQEIKFFtfc3pm1yp3HKu7uIiTsYtcer6CYJbrUAY6bwNBE+Cj2NF3wvDVeW1M2padhXqyuuhuOIaYbbQ8NOgR3kVg6UIKgn5oNimpXgkwef4xCuGNbNN7kfRfVTJfwrJm8jON0nzmmvxnmsNsz7hJgRszQ2Tp1qJlCI/KIPsIZ0vcXehy+VWV5xBOQOnP+b/wxR+tbt9BnDX/acRi95JAJLXYabqkD7FDc0rLGiWmEsyl4prhhRcT2Ux7O6CJuyegAOOodPDsylkV2iyaI6L3EEY9IrrV5w4ta2tDDPEcnuLkUo/1LipfJUSGzO/tqYJMAbd6klTtfI3cIjimmQXUP/0uiy4YXT7LJud/yEwjSajRIMAdIMQNhAcXPM3EnW85nck4NYSO8qUgu04+lpPiQKiE1eSjCa/LWBIqOMG69nE/hZCCceNBa2krvJIhjrPewBEMxBvg4aCUUS9GO3YanoDN3Y5T9HcpjvbhtLvLMGu20uAKxw63m8DTyNnzruYkL4Qr/5KxyhquCGv9YuwJMtaiUlWc36mh6YKEyyiYSe56cMegUnIZKBWWjsJwJm+ApPKZSI3u9/AGiHodF9tKmqt8vCzuqyzhsAskQXpewdn3iHe2IY2KwuWPPoObUaaSYWcKdoEZYVgaY9IofXf/x4GbJ5GVqbrwCGFpXJGk3GFFtvlyFICINU+CCp5PXgoMiqvpNvUVEHVclCmcTMi8lWufWQi/6Xu4gH+08QHVMkzf55o=
+    client-secret: AgBrF2e/M48zFC0PdXm08IavK+qa3Sl3a0qmO0+eH/z5WYnRpgos5dFdCLxcrdDcJUgKy//pU+I23qoTXqgybxRnfVzsOdYb4UksGgWh9640BQ/NRuxqhdnfXjQN7PrJj+D+fumwRdTQU1PqyzEePM/6CVf6Ro9vdP6iTMDzbAnPnHoA8L8Hd+f3YKo43OKMR2GMwjNKAhpD20wMCYt1vTiEQX4drR9JW4mjcKfSCdaIrEFwIYn4EBkr1SwuC4Ek/KTn4CNQO82BQH+jjWO5CHRvLP6b4v2bVkgdQEqLR4+HpNjMm0JBKOjHvhfKv952T5mwROamWcpNlN4KW02YzIZMBU+dh7WhouZkDkQCalP4G7AGoLrjhWmLIgS56JqTuoiobFf90Uvz6ewcDmHAzPuYZMfn2mKndPYvoi/KSGzM3ceeNrFxpHCPhsG+UHbOfSCBmg7jqIeE9ial6fMsRRLeEqc3ay4lAN7PBXwJ/o3XMrezUy8RQsXLex9oS0IS9rktimwQNWEpY1adwWRERdBKqGAIEbVofVwai/SuC/dcgQsoX2r9kJOiA8ahM+eDxUg1534hb21v3tFWykp3y/EbFBxbPPtCsUY4ghZZ8O/lLtXsZdHxaq68RKbVqDbRGa/eLqCqStWS8Bk1XzwkJFBBHFcN9hDvNwoT+33QSSsuuiw8gLiztlSrewo33rVKEc9Enqb+f/uUX6HY9j3Iz7hvKZ2aYX/U8ayNN7hQaJryYQ==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: grist-oidc-secret
+      namespace: grist

apps/grist/grist-postgres-deployment.yaml

@@ -0,0 +1,70 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grist-postgres
+  namespace: grist
+  labels:
+    app: grist
+    component: database
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: grist
+      component: database
+  template:
+    metadata:
+      labels:
+        app: grist
+        component: database
+    spec:
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        ports:
+        - name: postgres
+          containerPort: 5432
+        env:
+        - name: POSTGRES_DB
+          value: grist
+        - name: POSTGRES_USER
+          value: grist
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: grist-db-credentials
+              key: postgres-password
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - grist
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - grist
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: grist-db

apps/grist/grist-postgres-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grist-postgres
+  namespace: grist
+  labels:
+    app: grist
+    component: database
+spec:
+  type: ClusterIP
+  selector:
+    app: grist
+    component: database
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432

apps/grist/grist-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grist
+  namespace: grist
+  labels:
+    app: grist
+    component: app
+spec:
+  type: ClusterIP
+  selector:
+    app: grist
+    component: app
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8484

apps/harbor/values.yaml

@@ -95,6 +95,9 @@ harbor:
         cpu: 500m
 
   registry:
+    # Skip Kopia fs-backup of the registry blob PVC (DEV-573 safety net).
+    podAnnotations:
+      backup.velero.io/backup-volumes-excludes: registry-data
     registry:
       resources:
         requests:
@@ -106,6 +109,9 @@ harbor:
 
   trivy:
     enabled: true
+    # Skip Kopia fs-backup of the trivy scan cache PVC (DEV-573 safety net).
+    podAnnotations:
+      backup.velero.io/backup-volumes-excludes: data
     resources:
       requests:
         memory: 512Mi

apps/kaneo/README.md

@@ -0,0 +1,61 @@
+# Kaneo
+
+Self-hosted [Kaneo](https://github.com/usekaneo/kaneo) — open-source project
+management — deployed on the BasicStack Kubernetes cluster.
+
+- **URL:** https://kaneo.basicstack.de
+- **Namespace:** `kaneo`
+- **Distribution:** hand-rolled manifests, GitOps-managed by Argo CD
+  (see `apps/app-kaneo.yaml`).
+- **Auth:** OIDC via Pocket ID (`https://auth.basicstack.de`) with
+  `CUSTOM_OAUTH_*` env vars. Client id + secret live in
+  `kaneo-oidc-secret` (SealedSecret). Access is restricted to
+  Pocket ID groups `kaneo-admins` / `kaneo-members`.
+- **Database:** dedicated PostgreSQL in-namespace on
+  `hcloud-volumes-encrypted` (20Gi). Both the Postgres and Kaneo
+  pods pin to the `fsn1` datacenter via
+  `csi.hetzner.cloud/location=fsn1` node affinity — the Hetzner
+  volume is DC-local.
+- **TLS:** Traefik ingress, cert issued by `letsencrypt-prod`
+  (DNS-01 via the Hetzner cert-manager webhook).
+- **Image:** `ghcr.io/usekaneo/kaneo:2.22.0` (single container
+  bundling API + web on port 5173, health at `/api/health`).
+
+## Files
+
+| File | Purpose |
+| --- | --- |
+| `kaneo-postgres-pvc.yaml` | 20 Gi encrypted Hetzner volume for Postgres data. |
+| `kaneo-postgres-deployment.yaml` | Postgres 16-alpine, pinned to `fsn1`. |
+| `kaneo-postgres-service.yaml` | ClusterIP service `kaneo-postgres:5432`. |
+| `kaneo-db-credentials-sealed.yaml` | Postgres user password (SealedSecret). |
+| `kaneo-app-secrets-sealed.yaml` | `AUTH_SECRET` (Better Auth JWT secret, ≥32 chars). |
+| `kaneo-oidc-secret-sealed.yaml` | OIDC client id + secret from Pocket ID. |
+| `kaneo-smtp-sealed.yaml` | SMTP submission credentials for `kaneo@basicstack.de` (SealedSecret). |
+| `kaneo-deployment.yaml` | Kaneo app deployment (pinned to `fsn1`). |
+| `kaneo-service.yaml` | ClusterIP service on port 80 → container 5173. |
+| `kaneo-ingress.yaml` | Traefik ingress + `letsencrypt-prod` cert. |
+
+## Post-deploy setup
+
+1. Point browser at https://kaneo.basicstack.de and sign in via the
+   "Sign in with Pocket ID" (custom OAuth) button.
+2. The first admin (`andreas.leinen@basicstack.de`, member of
+   `kaneo-admins`) must be promoted inside Kaneo — the current
+   upstream release does not map OIDC groups to roles.
+
+## SMTP
+
+Outbound SMTP is wired to the Stalwart submission relay at
+`mail.basicstack.de:587` (STARTTLS) using the `kaneo@basicstack.de`
+mailbox. Credentials live in the `kaneo-smtp` SealedSecret
+(`kaneo-smtp-sealed.yaml`) with keys `smtp-user` and `smtp-password`.
+
+SMTP is used only for workspace invites and notifications;
+`DISABLE_EMAIL_OTP_SIGN_IN=true` remains set so users still sign in via
+Pocket ID SSO, not email codes.
+
+To rotate the mailbox password: update it in Stalwart, re-seal
+`kaneo-smtp` with the new value, commit, and let Argo apply. The
+existing pod picks up the new secret on next restart
+(`kubectl -n kaneo rollout restart deploy/kaneo`).

apps/kaneo/kaneo-app-secrets-sealed.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: kaneo-app-secrets
+  namespace: kaneo
+spec:
+  encryptedData:
+    auth-secret: AgArCARyFt7NinaeQOfDDEpTKKWVba8IlKh7NN/vflgdOkE5bjz/P/ZzGUVhUOTyqCs+3dU+hX6sqc7IYbYckXdprwKJSXy47HLEsZ/cuJ0FRIBShRyXm+I62Yu8WWLHNJLTnlWyfirBC6Pg65kXckoKiYPwcVLSD5I+YHzo3Dt3Hl6MeW/iVocM9xgjBD/5yRt1ag+S4JYv0t1y39Cz80XxbFvo55RzYWr3OlYNI325b9W/d5uylqlDemln8NymL5iCdfnLDZoc+1r+eHAf1HDP6Uch1dG+c/xsrlvoWpzyMeSEbEY1tGeYFZ1oKm4B1pyUH9dfdxWEaNThheL5Xw2QX/klpZdTq5Jl6Lfxme5aLyqNtP9J6ovd5b1oVYW1DWc+RAqBPhFlHnkfc+8ZeP6vrmnUULq0ssSVqpXOz4zLgDHU61UzSllNdpZWXokigranj3dq5CM7WkYLEj9kfdDIGS3gDUD+EmIkIj/N96URNpOINSwFVAcnE73k3ij9mZWECLl++lzh1fqHrdYQiTI6A7UpxPudc7BoujBQFqGhUDXGdzBWsrqr11tldD+kb/kcKQYxs6sFHyw0pDgCLVZukPv7TWRNfVRYUv68h/1ZKAwmj8KkuGlNtAQ+NyvUWyct3SA+pQYWabr33pIPekWPIXMIty6/zfKu92hLFglwkauewxqLtEBZ6FFukKMyyfowV+iwDIMyY3//MvxiRzxD7sc5sLwcfvPCBzd9XGaJO1ZLX6r43gfOu6HFHiq4FqdjYJ9fqG3zUKeo5bnav7E6
+  template:
+    metadata:
+      creationTimestamp: null
+      name: kaneo-app-secrets
+      namespace: kaneo

apps/kaneo/kaneo-db-credentials-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: kaneo-db-credentials
+  namespace: kaneo
+spec:
+  encryptedData:
+    database-url: AgDEiqPdexD58tc4Sv63mk1erIHAeweJzGpKgoOcro6ioW4eX9pWnfVnVfmtsqQK8WXkRuEP5pAnj1LZAhuo6hfwk/lOpoYItqZKl6KGK0EQZ3u/hcmUt0yeEkTJuDPGBbgk1RuT1G2u2uDDYYJYHmAQtIIFbAoeJS2iDKnKkl6UHZdEJhO10w6IHGiNCT6Z9NovLJau5cw9tTv7xnbGO8F1QqwLh/dL0kI2oYNTYg1RxpegXRFeKGz+QSJ3rbxlH2P/E01Fevma2/ZhOBjALS1bpW9R6pgQm4n/FDu/fu72T62yw2oNvj6nIeRtIsm9ze02jQqYwF7mgct6IJOdqY7RoYnTZJQ+/Xl/iiBIGpzXan9bU/JZw6sEeyxS+tyeaolz4oiTeNfRXp8QNdX3qbXM9OE5Q+bjqx1q+cZcnFcBIvGK8vGqlWwDjwNicucuNBZuv4JWEOp2N8axoFVRUsUn/Frv7McqKT1Q2mRXf0aiGYh9m7EIdnFAx3JeiBJv392jBWcwvJpQ+J0LIsh/bJEBtdC4c16Y+38nbqfjYEUJWVIoj9SvJEo3QZPitUz99cwdtDn+cOVVYp+lv5BiunM7OgvAgBbX7F4Ri/QlL8rt3/8lO0Augwp933lSTuGGnMCrXPz2QCagI6B0l8F5q5OTnljgR7JXslyGFaemtj9jZEnCiSsw9KAiDdOUyk36CJSfHmRedJJ6o2nalb3zdFDF7iMXxz6fpTQK0QQGAccO518zUHOpb3Jb6XbOBKDgTBOWPEGlroiavrOtoRDJ+B5nR/N4c3Indpn7d0TTDU/4rz4V1uNkRrHnA26ya9Q=
+    postgres-password: AgBo5f/NzqZ/9jsOzXmoivPVVeLA3xk9GWocQQlLv34Rc1K5fzhSoCYdy8o7k06R83VeNsJylxfAsOI7HNIo0xWUPqC8HnbW482ajtREh93wOFCaZzjKu8ClbFIkvTb19GRCPLSmzBMFsTCJzjSa5rWGqK2Ezj7LFeTTiehsqS/PbPUYi6w4X6ZnChS3VuMXJVJAvbCF8fFS3x/YuuJVjjd+TmVsPmGOa6J+r/6RG7l4DsR+SmV3HQ7M6bLqTsUPjzg5Foz11yc8ER8xQQTgbbfrBPwgJQoTX622yHf0Q313ns7vyiKGMqge9DSQvDmGmMDmAxqtYInycVBH1CwmfKVbw3CyFGt0D5LJXiBaTE89uLErsNCQCK6NKAODqffKlOWpZ1KVF+ZIQ6nqJETtj49uqdKGowauCs3tDl/Hi82+hxycEVMWUwgu81CJ6GT7+vSZfz4+9FRbLoXlbdfR/duG14f+AkPirikeDcy3bFy5XiBgPVQXL0EbAY8TaPkYu+TS1Roo2F6Gqj3D1+i4GapVodrBntl1gMPvMnd3dcGGl/VzREGZvRsZgdRprFi+nGPmotRhIO3eGcPautlZ5m0WMqKCpA4jW0sG1sgkVwRpn2Zdq4Uw2+IKNHTkJKclIXCaw2/b6WNCmiwbe1ssFoR/QgikzshshoB5XpIYFe7Jr5Sj9mmNH6L4PpCIZ/sCe8MF5eMD8IuyWSVmflID07sfBD6GL3XB9TFLzSyviHE/VJdGY7l+ZeR64UZZ1ti4bwI=
+  template:
+    metadata:
+      creationTimestamp: null
+      name: kaneo-db-credentials
+      namespace: kaneo

apps/kaneo/kaneo-deployment.yaml

@@ -0,0 +1,157 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kaneo
+  namespace: kaneo
+  labels:
+    app: kaneo
+    component: app
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: kaneo
+      component: app
+  template:
+    metadata:
+      labels:
+        app: kaneo
+        component: app
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until pg_isready -h kaneo-postgres -U kaneo -d kaneo; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+      containers:
+      - name: kaneo
+        image: ghcr.io/usekaneo/kaneo:2.22.0
+        ports:
+        - name: http
+          containerPort: 5173
+        env:
+        # Public URL of this Kaneo instance.
+        - name: KANEO_CLIENT_URL
+          value: https://kaneo.basicstack.de
+        - name: CORS_ORIGINS
+          value: https://kaneo.basicstack.de
+        # Better Auth JWT secret (>=32 chars).
+        - name: AUTH_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-app-secrets
+              key: auth-secret
+        # Database credentials. DATABASE_URL is preferred; POSTGRES_* are
+        # accepted as a fallback and useful for the built-in migrations.
+        - name: POSTGRES_DB
+          value: kaneo
+        - name: POSTGRES_USER
+          value: kaneo
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-db-credentials
+              key: postgres-password
+        - name: POSTGRES_HOST
+          value: kaneo-postgres
+        - name: POSTGRES_PORT
+          value: "5432"
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-db-credentials
+              key: database-url
+        # Self-hosted: keep public registration off, log in via OIDC only.
+        - name: DISABLE_REGISTRATION
+          value: "true"
+        - name: DISABLE_PASSWORD_REGISTRATION
+          value: "true"
+        - name: DISABLE_EMAIL_OTP_SIGN_IN
+          value: "true"
+        - name: KANEO_CLOUD
+          value: "false"
+        # Custom OAuth / OIDC via Pocket ID.
+        - name: CUSTOM_OAUTH_CLIENT_ID
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-oidc-secret
+              key: client-id
+        - name: CUSTOM_OAUTH_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-oidc-secret
+              key: client-secret
+        - name: CUSTOM_OAUTH_DISCOVERY_URL
+          value: https://auth.basicstack.de/.well-known/openid-configuration
+        - name: CUSTOM_OAUTH_AUTHORIZATION_URL
+          value: https://auth.basicstack.de/authorize
+        - name: CUSTOM_OAUTH_TOKEN_URL
+          value: https://auth.basicstack.de/api/oidc/token
+        - name: CUSTOM_OAUTH_USER_INFO_URL
+          value: https://auth.basicstack.de/api/oidc/userinfo
+        - name: CUSTOM_OAUTH_LOGOUT_URL
+          value: https://auth.basicstack.de/api/oidc/end-session
+        - name: CUSTOM_OAUTH_SCOPES
+          value: openid,profile,email,groups
+        - name: CUSTOM_AUTH_PKCE
+          value: "true"
+        # SMTP: outbound only, used for workspace invites / notifications.
+        # Sign-in still requires OIDC (DISABLE_EMAIL_OTP_SIGN_IN=true above).
+        - name: SMTP_HOST
+          value: mail.basicstack.de
+        - name: SMTP_PORT
+          value: "587"
+        - name: SMTP_SECURE
+          value: "false"
+        - name: SMTP_REQUIRE_TLS
+          value: "true"
+        - name: SMTP_USER
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-smtp
+              key: smtp-user
+        - name: SMTP_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-smtp
+              key: smtp-password
+        - name: SMTP_FROM
+          value: "Kaneo <kaneo@basicstack.de>"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 1Gi
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: http
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: http
+          initialDelaySeconds: 15
+          periodSeconds: 5
+          timeoutSeconds: 3

apps/kaneo/kaneo-ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: kaneo
+  namespace: kaneo
+  labels:
+    app: kaneo
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  tls:
+  - hosts:
+    - kaneo.basicstack.de
+    secretName: kaneo-tls
+  rules:
+  - host: kaneo.basicstack.de
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: kaneo
+            port:
+              number: 80

apps/kaneo/kaneo-oidc-secret-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: kaneo-oidc-secret
+  namespace: kaneo
+spec:
+  encryptedData:
+    client-id: AgBGmlu+mGYlHcWPCRHZ3IbevGB5kGntqPasFBnny6ZydTeCmN8NWpy0bmoorecJSbvdeP38UDgfEjtmusfkipnrlfsZd7BugZLiijq5+T+9al5JTl/UChMl6u/gReaVPgLi9iketDN6oB2XC/56mV+MdMb/bO4d1u3CRFUJ/Pq1znWeQpVsXO1yOLoOge4+PYDYjgd4O6mJsbAQcmGxQ7pwcByw//+50Mvuw0gJBqI5ZSGKd0jSNnaVcECtvjY6v3xN1NCeVF1FoWKl4s7o/Nrq5mDltW/y6g0h3N1Fzan/D47UVQjGLYu6cTmM7ReqToYk65cznttGkyXfqrlVL/oEiYENtaqp/BZvwPBMKEAjJsufwGvbNgsxa3eGs+p3L+HwzdgTQbVGu2wVp+TXW8zwZvkDyprUU91Fu6QFsspV/GRYgqVmp4RxrJ0PzZInp4ZhLt/zOyWzbEkILMPV1A5fzPjrPmwpmhln/K0VTdpRN62XQRgH0P7EP6Ao0sP2/ae39eTkGlhb5fwzhSqK3LSmBYmoXNcffA4nBLFohSndmwgoxzZXM1FJ1kXl4wuevU8Eh8BD3dVIZkuYacKiIUFoNn74NbPtJpl3/MWfXef9Eb7IBR8KyurbdwSmKSAZDqjmeGTa271+KUiGfST+FYQgSDfFfQQEmh+K49WqqbOiod4Mh5RTGWJx3WNwEONsTRKVjiGgLw4A+ENNDB39PoRChoRoRRogg+ZJZE27EjxbmrKIjrw=
+    client-secret: AgBJQi6fIlTkOIZa1zr9jjyNOFAyZUjHWy0ZBjfcNh+iY2V3qIpG8kNPBntmvtVtemYI+UxEvvlalq0iEue+AtVmUoJeeO/lYuEDv+BrJGUWeZITbItuyoG+xGZM4x41X6FPgfRcnLCfii3dmwOeDZh1asR2Ahy2szdXrd2QaDeZMCCXNR+IeuP9U+IqJncfm+KjCqPABVbsZuGfDrT7r4XRv4mHQGnILD4IyA9N0/XyfY/cZRJryWeF5xbrOAP/ZWJduqYGIm2ahJfRZWIzk1Ujknq35iJ4bOZiw5eWb+aYhD9cxJj2UMBwWdqnXnpH+p55lrBqWlCzI0yDOfk4UQCI2RzQBJc/RvVS0z16k7El4dl9lOFTcgwBg13C/JC8Y0wo4M1jEap2ty/2ifwVZBZdflT8Uy5sk8dKBOv9wCupA/N60ACwlsavCUlgtQetVZUHIlI1U1rKCzArpGZ6PuREZokjdHw01u/j9ANEt1AaMe3G5jLcGohyvpcgL+zqI2S0rCqbSdRqWzZvr2kM8uFoeZSJLG3y4wdwZMgvRGMfoAKD1YlUGuhWeeZTda77ScM6ii8iEZEjvIgJXrfSCUf1vJ4QIAdMnNi05d3pTTDVHNTRjWBrklVb6oTpe/rFF1BBP6zhwGylWtb71xszbQyX8lzD+7v1O4iXr7ZWo86L9mGKz6FMnj0bO4LLVoqeVMy5DHo2fCuplix03P3WQCyjGKranAWdLalEV59QdnEhaQ==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: kaneo-oidc-secret
+      namespace: kaneo

apps/kaneo/kaneo-postgres-deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kaneo-postgres
+  namespace: kaneo
+  labels:
+    app: kaneo
+    component: database
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: kaneo
+      component: database
+  template:
+    metadata:
+      labels:
+        app: kaneo
+        component: database
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        ports:
+        - name: postgres
+          containerPort: 5432
+        env:
+        - name: POSTGRES_DB
+          value: kaneo
+        - name: POSTGRES_USER
+          value: kaneo
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: kaneo-db-credentials
+              key: postgres-password
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - kaneo
+            - -d
+            - kaneo
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - kaneo
+            - -d
+            - kaneo
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: kaneo-postgres-data

apps/kaneo/kaneo-postgres-pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: kaneo-postgres-data
+  namespace: kaneo
+  labels:
+    app: kaneo
+    component: database
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 20Gi

apps/kaneo/kaneo-postgres-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kaneo-postgres
+  namespace: kaneo
+  labels:
+    app: kaneo
+    component: database
+spec:
+  type: ClusterIP
+  selector:
+    app: kaneo
+    component: database
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432

apps/kaneo/kaneo-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kaneo
+  namespace: kaneo
+  labels:
+    app: kaneo
+    component: app
+spec:
+  type: ClusterIP
+  selector:
+    app: kaneo
+    component: app
+  ports:
+  - name: http
+    port: 80
+    targetPort: 5173

apps/kaneo/kaneo-smtp-sealed.yaml

@@ -0,0 +1,16 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: kaneo-smtp
+  namespace: kaneo
+spec:
+  encryptedData:
+    smtp-password: AgByMIhrSeQbPJezoqPf4jtTCx/b/R+TGZPHjjvUcdZJimgZY8XO72BA2nI/419UWqY3rBmWyquYMnEFJT0/48Rn0Nk4aFxBwBIxVuiG9y1+f5yXIhT2JuOqGoBnImYezCCtYOcgn4Byo6Twldce1ElRCqwH6grsVOLKtogZvLJC0sz6EWbqPfLrDuGyOFz2wg5P3Cjmy8W9ZgcTDTpVI4JxZh2JKbIJpq4kGDQD0miW8Av0BOmSi75uvT2PvaV/uinJ72VJFLP/GTHWjf/9kpOiIC2TL4B+Kk0HtMeWSzNF79itsD53ow7bya/bq//zUiwdaIeSaAcSJL8XqL9q38LIEL0Vr9ccXN9xsPZttLw836NDeLsryE+RhF3pIK2vEs76YIeO03AEnXNjEYOcwy7ePGYh6Y/Uf/B75QNzbzrq8H4B5lKFVmQgTBLJnvxTzOc6Sd/Yssgl1wJw90jgw2HYcLHXt/eQrB4cGKWSwrh2x6fnU79UpSGatdTqIyP2Q8vbgKmxqqmg6NDzjgeatT6ADKOqJKAjnXfrr8Bm3NjvAVWOGWuIfWdS0wHiEOhAoUNNAQ+bD7f/SbJE24q8W6juhzTmRLuU1BM2K51YPHA3BqyEpzygaboSi/bgHVOuoIAEAt7jAM6mbW5RQBwiNglF/C2Oem8J3LOdDlndJoSMaWKm5p1lKc3M59jr47jaDXm9HKM4ypC58cqty3varqxBwwdZHirZ6No0Ar4Sd78I4jo4kXk9zu33UjCcR/X2NX1abIYZ8eNoUga+I9AOkpM=
+    smtp-user: AgAq1mT9fzt9/VHmcbn0Yzr/8HWadEBe1XjJOTonao7Tdc3EO1PvCgEE9nBkrqOK5x2uDP1jZuP98XxJXr6mHuCnEOn8SHMN93TqK5rLhsKir1ImdNWKC+ruVI3TPQfTkeEz5m3u0DjHWSymva7BjmTIIW5gE4SDt/7WOFrsWbX1fLqQKfuB/k1qIBd+rFL7rVSe3XnRzxxwwrIbYfczSCknMbOot65FRcR+pC6ox7hOYC7pMIU76WMrVYx0cmDfAZAidcOQP9+K/H4LY0vkwPnaNk7+Is5gJifWR37lTnCkjRknMWB+yyDZF3BkmmdHNvXE6OlMcBF8l/jrO7S0rTtkCDaXllpRt1tWoSY2Kyr2DTaLt9QQvd1v8Rz2mw1qeDZg0mO/BswPF10SEKwjD3RlGFkA6mK9hZE5G+eDpgMH9u5HEKftI6jD/wdzElY+GdEFhlmj/ATZuNk+oC+eVnHGHUnY39lQYX+7PEtOujsKAjsuCPBaMDMkTP+f+LOUguuaRaanpLctmRAnT5BK19dmNuJtZPZAPgzCLu44884BH1qpKpVXkaUZSUbYA1RpKflcf1+uWJaqp7oBOZLSVoVSLiVG8ypGv7IPeTiILQcWQK27z6H0VxiAZ0S8yzNcCmeTWg8/wUGVzk0VJ9imPIShpiq515X6GuU1v1LsMiuJtFfL3EpNdDln04dLFWVb0ghIdt4Hn4U3IpJRIcP+XEEpv4+6
+  template:
+    metadata:
+      creationTimestamp: null
+      name: kaneo-smtp
+      namespace: kaneo
+---

apps/monitoring/README.md

@@ -2,11 +2,10 @@
 
 Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context.
 
-- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.19.1` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
 - `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)).
 - `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)).
 - `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block.
-- `backup-restic-alerts.yaml` + `backup-restic-alerts.test.yaml` — `PrometheusRule` with freshness, integrity, and repo-size alerts covering the four restic repos (loki/grafana/k8s-resources/prometheus), plus a `promtool test rules` unit test proving each alert fires against synthetic samples ([DEV-490](/DEV/issues/DEV-490), extended in [DEV-492](/DEV/issues/DEV-492)).
+- `backup-restic-alerts.yaml` + `backup-restic-alerts.test.yaml` — `PrometheusRule` with freshness, integrity, and repo-size alerts covering the three restic repos (loki/grafana/prometheus), plus a `promtool test rules` unit test proving each alert fires against synthetic samples ([DEV-490](/DEV/issues/DEV-490), extended in [DEV-492](/DEV/issues/DEV-492)). The `k8s-resources` restic repo was superseded by Velero cluster-reconstruction backups and retired in [DEV-576](/DEV/issues/DEV-576).
 
 ## Shared SealedSecret
 
@@ -18,7 +17,7 @@ All four restic CronJobs read Hetzner S3 credentials + the restic repository pas
 | `secret-key`     | Hetzner Object Storage secret access key                                                                         |
 | `endpoint`       | S3 endpoint hostname (e.g. `fsn1.your-objectstorage.com`)                                                        |
 | `bucket`         | Bucket name (single bucket, per-prefix repos)                                                                    |
-| `restic-password`| 32-byte random string sealed at [DEV-484](/DEV/issues/DEV-484); plaintext copy in Passbolt entry `restic / monitoring backups`. Rotation: `restic key add` → seal new value → `restic key remove` old id. Same key protects all four repos (loki/grafana/k8s-resources/prometheus) — rotating rewrites the key file on every repo. |
+| `restic-password`| 32-byte random string sealed at [DEV-484](/DEV/issues/DEV-484); plaintext copy in Passbolt entry `restic / monitoring backups`. Rotation: `restic key add` → seal new value → `restic key remove` old id. Same key protects all three remaining repos (loki/grafana/prometheus) — rotating rewrites the key file on every repo. |
 
 ## Restore
 
@@ -30,20 +29,20 @@ Each restic CronJob writes `/metrics/backup_<kind>.prom` (atomic — `.prom.tmp`
 
 | Metric                                     | Emitted by                                                                       |
 |--------------------------------------------|----------------------------------------------------------------------------------|
-| `backup_<kind>_success` (0/1)              | `backup-{loki,grafana,k8s-resources}-*-cronjob.yaml`, `prometheus-backup-cronjob.yaml` |
+| `backup_<kind>_success` (0/1)              | `backup-{loki,grafana}-*-cronjob.yaml`, `prometheus-backup-cronjob.yaml`         |
 | `backup_<kind>_timestamp_seconds`          | same                                                                             |
 | `backup_<kind>_check_status` (exit code)   | same — from `restic check --read-data-subset=5%`                                 |
 | `backup_prometheus_backup_status`          | `prometheus-backup-cronjob.yaml` — `restic backup` exit code (3 = accepted compaction race) |
 | `restic_repo_size_bytes{repo="<kind>"}`    | same — from `restic stats --json --mode raw-data` (added in DEV-490)             |
 
-**Cross-node staleness note.** Because a backup CronJob may run on a different worker across days (loki/prometheus follow their app pods, `k8s-resources` is unpinned), a `.prom` file can linger on a node the job has since left and node-exporter keeps exposing it. The freshness alerts collapse the per-node samples with `max()` so the freshest sample wins; check/size alerts fire when *any* node reports a bad value, which is intentional — a recent failure is still a signal until the file is manually cleaned or the job returns to that node.
+**Cross-node staleness note.** Because a backup CronJob may run on a different worker across days (loki/prometheus follow their app pods), a `.prom` file can linger on a node the job has since left and node-exporter keeps exposing it. The freshness alerts collapse the per-node samples with `max()` so the freshest sample wins; check/size alerts fire when *any* node reports a bad value, which is intentional — a recent failure is still a signal until the file is manually cleaned or the job returns to that node.
 
 ## Alerts
 
-`backup-restic-alerts.yaml` defines nine alerts (all `severity: warning`):
+`backup-restic-alerts.yaml` defines seven alerts (all `severity: warning`):
 
-- `BackupLokiStale` / `BackupGrafanaStale` / `BackupK8sResourcesStale` / `BackupPrometheusStale` — `time() - max(backup_<kind>_timestamp_seconds) > 28h`. Daily schedule + 4 h grace. `max()` collapses per-node samples so a stale `.prom` on a node the job has left does not fire.
-- `BackupLokiCheckFailed` / `BackupGrafanaCheckFailed` / `BackupK8sResourcesCheckFailed` / `BackupPrometheusCheckFailed` — `backup_<kind>_check_status != 0`.
+- `BackupLokiStale` / `BackupGrafanaStale` / `BackupPrometheusStale` — `time() - max(backup_<kind>_timestamp_seconds) > 28h`. Daily schedule + 4 h grace. `max()` collapses per-node samples so a stale `.prom` on a node the job has left does not fire.
+- `BackupLokiCheckFailed` / `BackupGrafanaCheckFailed` / `BackupPrometheusCheckFailed` — `backup_<kind>_check_status != 0`.
 - `ResticRepoOversize` — `restic_repo_size_bytes > 20 GiB`. Baseline expected < 5 GiB; catches retention/prune regressions.
 
 All alerts carry a `Runbook: docs/monitoring/restic-restore.md` annotation. To iterate on the rule file locally:
@@ -55,4 +54,4 @@ promtool check rules /tmp/backup-restic-rules.yaml
 promtool test rules  apps/monitoring/backup-restic-alerts.test.yaml
 ```
 
-The legacy `backup-volumes` CronJob and its 100 Gi local-path `backup-storage` PVC were retired in [DEV-489](/DEV/issues/DEV-489) once the restic pipeline was proven end-to-end. With the shared destination PVC gone, the DEV-483 bridge `nodeSelector` pinning Loki to `k3s-worker-2` was also removed — `backup-loki-restic` follows the Loki pod via `podAffinity` regardless of which node the RWO CSI volume lands on. `backup-grafana-restic` still nodeSelects `k3s-worker-2` because its source PV (`grafana-storage`, local-path) is anchored there. `backup-k8s-resources` has no PVC dep and stays unpinned. `prometheus-backup` uses `podAffinity` on `app=prometheus` (RWO PVC on Hetzner CSI, single-node attach) and follows the Prometheus pod between workers.
+The legacy `backup-volumes` CronJob and its 100 Gi local-path `backup-storage` PVC were retired in [DEV-489](/DEV/issues/DEV-489) once the restic pipeline was proven end-to-end. With the shared destination PVC gone, the DEV-483 bridge `nodeSelector` pinning Loki to `k3s-worker-2` was also removed — `backup-loki-restic` follows the Loki pod via `podAffinity` regardless of which node the RWO CSI volume lands on. `backup-grafana-restic` still nodeSelects `k3s-worker-2` because its source PV (`grafana-storage`, local-path) is anchored there. `prometheus-backup` uses `podAffinity` on `app=prometheus` (RWO PVC on Hetzner CSI, single-node attach) and follows the Prometheus pod between workers. The `backup-k8s-resources` CronJob and its restic repo were retired in [DEV-576](/DEV/issues/DEV-576) after Velero `cluster-all` took over cluster-manifest coverage.

apps/monitoring/backup-k8s-resources-cronjob.yaml

@@ -1,205 +0,0 @@
----
-# Kubernetes-resource backup via restic to Hetzner Object Storage
-# (DEV-487, DEV-482 Option 4). Step 4 of the Option 4 rollout.
-#
-# Streams a concatenated YAML dump of cluster-scoped and per-namespace
-# resources through `restic backup --stdin` into
-# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/k8s-resources`. No PVC mount
-# (drops the local-path `backup-storage` dependency), no node pin.
-#
-# Two-container pattern:
-#   1. `kubectl-dump` init container (`alpine/k8s:1.29.4`) writes
-#      /dump/cluster.yaml into an emptyDir. Uses `serviceAccountName:
-#      backup-sa` (unchanged from the legacy job).
-#   2. `restic` main container (`restic/restic:0.19.1`, matches the
-#      loki/grafana siblings) reads that file on stdin and streams it
-#      into the restic repo with `--stdin-filename cluster.yaml`.
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: backup-k8s-resources
-  namespace: monitoring
-  labels:
-    app: backup
-    type: k8s-resources
-    backend: restic
-spec:
-  schedule: "0 2 * * *"
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 3
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    metadata:
-      labels:
-        app: backup
-        type: k8s-resources
-        backend: restic
-    spec:
-      backoffLimit: 2
-      activeDeadlineSeconds: 3600
-      template:
-        metadata:
-          labels:
-            app: backup
-            type: k8s-resources
-            backend: restic
-        spec:
-          restartPolicy: OnFailure
-          serviceAccountName: backup-sa
-          initContainers:
-            - name: kubectl-dump
-              image: alpine/k8s:1.29.4
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -eu
-                  echo "=== kubectl-dump started at $(date -u +%FT%TZ) ==="
-                  DUMP=/dump/cluster.yaml
-                  : > "${DUMP}"
-
-                  echo "--- namespaces ---"
-                  kubectl get namespaces -o yaml >> "${DUMP}"
-                  echo "---" >> "${DUMP}"
-
-                  echo "--- cluster-scoped resources ---"
-                  kubectl get persistentvolumes,storageclasses,clusterroles,clusterrolebindings \
-                    -o yaml >> "${DUMP}"
-                  echo "---" >> "${DUMP}"
-
-                  echo "--- namespaced resources ---"
-                  for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do
-                    echo "  ns=${ns}"
-                    kubectl get \
-                      configmaps,secrets,services,deployments,statefulsets,daemonsets,jobs,cronjobs,ingresses,persistentvolumeclaims \
-                      -n "${ns}" -o yaml >> "${DUMP}" 2>/dev/null || true
-                    echo "---" >> "${DUMP}"
-                  done
-
-                  echo "dump size: $(wc -c < ${DUMP}) bytes"
-                  echo "=== kubectl-dump finished at $(date -u +%FT%TZ) ==="
-              resources:
-                requests:
-                  cpu: 50m
-                  memory: 128Mi
-                limits:
-                  cpu: 500m
-                  memory: 512Mi
-              volumeMounts:
-                - name: dump
-                  mountPath: /dump
-          containers:
-            - name: restic
-              image: harbor.basicstack.de/library/restic:0.19.1
-              # Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
-              # deterministic ingress via Harbor. Retag procedure in
-              # docs/monitoring/restic-restore.md § "Tag-bump procedure".
-              env:
-                - name: AWS_ACCESS_KEY_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: monitoring-s3-backup
-                      key: access-key
-                - name: AWS_SECRET_ACCESS_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: monitoring-s3-backup
-                      key: secret-key
-                - name: RESTIC_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: monitoring-s3-backup
-                      key: restic-password
-                - name: S3_ENDPOINT
-                  valueFrom:
-                    secretKeyRef:
-                      name: monitoring-s3-backup
-                      key: endpoint
-                - name: S3_BUCKET
-                  valueFrom:
-                    secretKeyRef:
-                      name: monitoring-s3-backup
-                      key: bucket
-                - name: RESTIC_REPOSITORY
-                  value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/k8s-resources"
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -eu
-                  echo "=== backup-k8s-resources-restic started at $(date -u +%FT%TZ) ==="
-                  echo "Repository: ${RESTIC_REPOSITORY}"
-
-                  # First-run tolerance: init if the repo isn't there yet.
-                  if restic snapshots >/dev/null 2>&1; then
-                    echo "Repo exists, skipping init."
-                  else
-                    echo "Repo missing, initialising..."
-                    restic init
-                  fi
-
-                  echo "--- restic backup --stdin cluster.yaml ---"
-                  restic backup --stdin \
-                    --stdin-filename cluster.yaml \
-                    --tag k8s-resources \
-                    --host k3s < /dump/cluster.yaml
-
-                  echo "--- restic forget/prune ---"
-                  restic forget --tag k8s-resources \
-                    --keep-daily 7 \
-                    --keep-weekly 4 \
-                    --keep-monthly 6 \
-                    --prune
-
-                  echo "--- restic check --read-data-subset=5% ---"
-                  CHECK_STATUS=0
-                  restic check --read-data-subset=5% || CHECK_STATUS=$?
-                  echo "restic check exit: ${CHECK_STATUS}"
-
-                  echo "--- restic stats (repo size) ---"
-                  REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
-                    | grep -oE '"total_size":[0-9]+' \
-                    | head -1 \
-                    | cut -d: -f2)
-                  REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
-                  echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
-
-                  # Textfile-collector metrics; identical wiring to the
-                  # loki/grafana siblings. See loki cronjob for the
-                  # atomic-write rationale (DEV-494).
-                  {
-                    echo "backup_k8s_resources_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
-                    echo "backup_k8s_resources_timestamp_seconds $(date +%s)"
-                    echo "backup_k8s_resources_check_status ${CHECK_STATUS}"
-                    echo "restic_repo_size_bytes{repo=\"k8s-resources\"} ${REPO_SIZE_BYTES}"
-                  } > /metrics/backup_k8s_resources.prom.tmp
-                  mv /metrics/backup_k8s_resources.prom.tmp /metrics/backup_k8s_resources.prom
-
-                  echo "=== backup-k8s-resources-restic finished at $(date -u +%FT%TZ) ==="
-                  exit ${CHECK_STATUS}
-              volumeMounts:
-                - name: dump
-                  mountPath: /dump
-                  readOnly: true
-                - name: metrics
-                  mountPath: /metrics
-                - name: cache
-                  mountPath: /root/.cache/restic
-              resources:
-                requests:
-                  cpu: 100m
-                  memory: 128Mi
-                limits:
-                  cpu: 1500m
-                  memory: 1Gi
-          volumes:
-            - name: dump
-              emptyDir: {}
-            - name: metrics
-              hostPath:
-                # node-exporter's textfile-collector directory
-                # (DEV-494). See sibling loki cronjob for detail.
-                path: /var/lib/node_exporter/textfile_collector
-                type: DirectoryOrCreate
-            - name: cache
-              emptyDir: {}

apps/monitoring/backup-restic-alerts.test.yaml

@@ -35,8 +35,6 @@ tests:
         values: '-104400x60'
       - series: 'backup_grafana_timestamp_seconds'
         values: '-104400x60'
-      - series: 'backup_k8s_resources_timestamp_seconds'
-        values: '-104400x60'
       - series: 'backup_prometheus_timestamp_seconds'
         values: '-104400x60'
     alert_rule_test:
@@ -70,21 +68,6 @@ tests:
                 `backup_grafana_timestamp_seconds` sample in more
                 than 28 hours. Expected daily at 03:15 UTC.
                 Runbook: docs/monitoring/restic-restore.md
-      - eval_time: 30m
-        alertname: BackupK8sResourcesStale
-        exp_alerts:
-          - exp_labels:
-              severity: warning
-              service: monitoring
-              component: backup
-              repo: k8s-resources
-            exp_annotations:
-              summary: "K8s-resources restic backup is stale (>28h)"
-              description: |
-                `backup-k8s-resources` has not written a fresh
-                `backup_k8s_resources_timestamp_seconds` sample in
-                more than 28 hours. Expected daily at 02:00 UTC.
-                Runbook: docs/monitoring/restic-restore.md
       - eval_time: 30m
         alertname: BackupPrometheusStale
         exp_alerts:
@@ -112,8 +95,6 @@ tests:
         values: '-3600x60'
       - series: 'backup_grafana_timestamp_seconds'
         values: '-3600x60'
-      - series: 'backup_k8s_resources_timestamp_seconds'
-        values: '-3600x60'
       - series: 'backup_prometheus_timestamp_seconds'
         values: '-3600x60'
     alert_rule_test:
@@ -123,9 +104,6 @@ tests:
       - eval_time: 30m
         alertname: BackupGrafanaStale
         exp_alerts: []
-      - eval_time: 30m
-        alertname: BackupK8sResourcesStale
-        exp_alerts: []
       - eval_time: 30m
         alertname: BackupPrometheusStale
         exp_alerts: []
@@ -139,8 +117,6 @@ tests:
         values: '1x30'
       - series: 'backup_grafana_check_status'
         values: '2x30'
-      - series: 'backup_k8s_resources_check_status'
-        values: '1x30'
       - series: 'backup_prometheus_check_status'
         values: '1x30'
     alert_rule_test:
@@ -172,21 +148,6 @@ tests:
                 `restic check --read-data-subset=5%` returned
                 exit code 2 on the grafana repository.
                 Runbook: docs/monitoring/restic-restore.md
-      - eval_time: 15m
-        alertname: BackupK8sResourcesCheckFailed
-        exp_alerts:
-          - exp_labels:
-              severity: warning
-              service: monitoring
-              component: backup
-              repo: k8s-resources
-            exp_annotations:
-              summary: "restic check failed on k8s-resources repo"
-              description: |
-                `restic check --read-data-subset=5%` returned
-                exit code 1 on the k8s-resources
-                repository.
-                Runbook: docs/monitoring/restic-restore.md
       - eval_time: 15m
         alertname: BackupPrometheusCheckFailed
         exp_alerts:
@@ -212,8 +173,6 @@ tests:
         values: '0x30'
       - series: 'backup_grafana_check_status'
         values: '0x30'
-      - series: 'backup_k8s_resources_check_status'
-        values: '0x30'
       - series: 'backup_prometheus_check_status'
         values: '0x30'
     alert_rule_test:
@@ -223,9 +182,6 @@ tests:
       - eval_time: 15m
         alertname: BackupGrafanaCheckFailed
         exp_alerts: []
-      - eval_time: 15m
-        alertname: BackupK8sResourcesCheckFailed
-        exp_alerts: []
       - eval_time: 15m
         alertname: BackupPrometheusCheckFailed
         exp_alerts: []

apps/monitoring/backup-restic-alerts.yaml

@@ -19,7 +19,7 @@
 # kube-prometheus-stack node-exporter DaemonSet (DEV-494).
 #
 # Because a CronJob may run on a different worker across days (loki
-# backup follows the loki pod; k8s-resources is unpinned), stale
+# backup follows the loki pod), stale
 # `.prom` files can linger on nodes the job has since left. That
 # would leave a per-node series with an old timestamp/check-status
 # indefinitely. The alerts below aggregate across instances so a
@@ -74,21 +74,6 @@ spec:
               `backup_grafana_timestamp_seconds` sample in more
               than 28 hours. Expected daily at 03:15 UTC.
               Runbook: docs/monitoring/restic-restore.md
-        - alert: BackupK8sResourcesStale
-          expr: time() - max(backup_k8s_resources_timestamp_seconds) > 28 * 3600
-          for: 15m
-          labels:
-            severity: warning
-            service: monitoring
-            component: backup
-            repo: k8s-resources
-          annotations:
-            summary: "K8s-resources restic backup is stale (>28h)"
-            description: |
-              `backup-k8s-resources` has not written a fresh
-              `backup_k8s_resources_timestamp_seconds` sample in
-              more than 28 hours. Expected daily at 02:00 UTC.
-              Runbook: docs/monitoring/restic-restore.md
         - alert: BackupPrometheusStale
           expr: time() - max(backup_prometheus_timestamp_seconds) > 28 * 3600
           for: 15m
@@ -150,21 +135,6 @@ spec:
               `restic check --read-data-subset=5%` returned
               exit code {{ $value }} on the grafana repository.
               Runbook: docs/monitoring/restic-restore.md
-        - alert: BackupK8sResourcesCheckFailed
-          expr: backup_k8s_resources_check_status != 0
-          for: 5m
-          labels:
-            severity: warning
-            service: monitoring
-            component: backup
-            repo: k8s-resources
-          annotations:
-            summary: "restic check failed on k8s-resources repo"
-            description: |
-              `restic check --read-data-subset=5%` returned
-              exit code {{ $value }} on the k8s-resources
-              repository.
-              Runbook: docs/monitoring/restic-restore.md
         - alert: BackupPrometheusCheckFailed
           expr: backup_prometheus_check_status != 0
           for: 5m

apps/monitoring/prometheus-backup-cronjob.yaml

@@ -48,7 +48,7 @@ metadata:
     type: prometheus
     backend: restic
 spec:
-  schedule: "30 3 * * *"  # daily 03:30, offset from loki/grafana/k8s-resources
+  schedule: "30 3 * * *"  # daily 03:30, offset from loki/grafana
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
@@ -173,7 +173,7 @@ spec:
                   echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
 
                   # Textfile-collector metrics. Same hostPath pattern as
-                  # the loki/grafana/k8s-resources siblings (DEV-494) —
+                  # the loki/grafana siblings (DEV-494) —
                   # written atomically via `.tmp` + rename so a mid-write
                   # read never surfaces a truncated sample.
                   # backup_prometheus_success rolls in both stages: the

apps/monitoring/prometheus-deployment.yaml

@@ -0,0 +1,110 @@
+---
+# Prometheus data volume — RWO on hcloud-volumes-encrypted (fsn1). Because the
+# volume is RWO, the Deployment below must use Recreate; a rolling update would
+# spin up a surge pod that cannot mount the volume while the old pod holds it
+# and the rollout would stall at Init:0/1. See DEV-583 / mirrors DEV-538.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheus-data-encrypted
+  namespace: monitoring
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: monitoring
+  labels:
+    app: prometheus
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9090
+      targetPort: 9090
+      name: web
+  selector:
+    app: prometheus
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: monitoring
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  # prometheus-data-encrypted is RWO on hcloud-volumes-encrypted, so a rolling
+  # update deadlocks (new pod cannot attach the PVC while the old pod holds
+  # it). Recreate drops the old pod first so the CSI detaches the volume
+  # before the new pod tries to attach it. Same pattern as loki/opencloud.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+      annotations:
+        # Exclude the prometheus TSDB volume from Velero fs-backup — TSDB is
+        # backed up separately by prometheus-backup-cronjob and streaming it
+        # through Kopia would double-copy tens of GiB on every backup.
+        backup.velero.io/backup-volumes-excludes: prometheus-storage
+    spec:
+      serviceAccountName: prometheus
+      initContainers:
+        - name: fix-permissions
+          image: busybox:latest
+          command:
+            - sh
+            - -c
+            - chown -R 65534:65534 /prometheus && chmod -R 755 /prometheus
+          volumeMounts:
+            - name: prometheus-storage
+              mountPath: /prometheus
+      containers:
+        - name: prometheus
+          image: prom/prometheus:v2.47.0
+          args:
+            - '--config.file=/etc/prometheus/prometheus.yml'
+            - '--storage.tsdb.path=/prometheus'
+            - '--storage.tsdb.retention.time=30d'
+            - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+            - '--web.console.templates=/usr/share/prometheus/consoles'
+            - '--web.enable-lifecycle'
+          ports:
+            - containerPort: 9090
+              name: web
+          volumeMounts:
+            - name: prometheus-config
+              mountPath: /etc/prometheus
+            - name: prometheus-rules
+              mountPath: /etc/prometheus/rules
+            - name: prometheus-storage
+              mountPath: /prometheus
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: 1000m
+              memory: 2Gi
+      volumes:
+        - name: prometheus-config
+          configMap:
+            name: prometheus-config
+        - name: prometheus-rules
+          configMap:
+            name: prometheus-rules
+        - name: prometheus-storage
+          persistentVolumeClaim:
+            claimName: prometheus-data-encrypted

apps/opencloud/backup-cronjob.yaml

@@ -6,6 +6,7 @@ metadata:
   name: opencloud-backup
   namespace: opencloud
 spec:
+  suspend: true  # CronJob deaktiviert
   schedule: "0 2 * * *"  # Daily at 2 AM UTC
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
@@ -90,8 +91,8 @@ spec:
               readOnly: true
             resources:
               requests:
-                memory: "256Mi"
-                cpu: "100m"
+                memory: "16Mi"
+                cpu: "20m"
               limits:
                 memory: "512Mi"
                 cpu: "500m"

apps/opencloud/opencloud-deployment.yaml

@@ -41,7 +41,7 @@ metadata:
   name: opencloud
   namespace: opencloud
 spec:
-  replicas: 1
+  replicas: 0
   strategy:
     type: Recreate
   selector:
@@ -51,6 +51,12 @@ spec:
     metadata:
       labels:
         app: opencloud
+      annotations:
+        # Skip Kopia fs-backup of the 100 GiB opencloud object-store PVC — this
+        # blob store is out of scope for the DEV-573 data schedules; the
+        # exclusion is a safety net so an accidental defaultVolumesToFsBackup
+        # run never streams it into the Kopia repo.
+        backup.velero.io/backup-volumes-excludes: opencloud-data
     spec:
       securityContext:
         fsGroup: 1000
@@ -448,7 +454,7 @@ spec:
         resources:
           requests:
             memory: "512Mi"
-            cpu: "250m"
+            cpu: "50m"
           limits:
             memory: "2Gi"
             cpu: "1000m"

apps/opencloud/tika-deployment.yaml

@@ -22,7 +22,7 @@ metadata:
   name: tika
   namespace: opencloud
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       app: tika

apps/openwebui/README.md

@@ -0,0 +1,46 @@
+# Open WebUI (openwebui.basicstack.de)
+
+Self-hosted LLM front-end — [Open WebUI](https://openwebui.com/).
+
+## Stack
+
+- **App:** `ghcr.io/open-webui/open-webui:v0.11.3` (Deployment, 1 replica, `Recreate` strategy)
+- **Database:** Postgres 16 + pgvector (`pgvector/pgvector:pg16`, StatefulSet, PVC 20Gi on `hcloud-volumes-encrypted`)
+- **Vector DB:** `pgvector` in the same Postgres instance (`VECTOR_DB=pgvector`, extension auto-created on first start)
+- **Cache / websocket manager:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`, `WEBSOCKET_MANAGER=redis` on `db=1`)
+- **Object storage:** Hetzner Object Storage bucket `openwebui-basicstack` (region `hel1`, path-style)
+- **Auth:** Pocket ID OIDC — `Open WebUI` client, restricted to `openwebui_admins` / `openwebui_users`. Sign-up via OAuth only (`ENABLE_LOGIN_FORM=false`, `ENABLE_SIGNUP=false`)
+- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
+- **Node placement:** all stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
+
+## Files
+
+| File | Purpose |
+| ---- | ------- |
+| `namespace.yaml` | Namespace declaration |
+| `postgres.yaml` | Postgres+pgvector StatefulSet + Service (fsn1-pinned) |
+| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
+| `openwebui.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
+| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
+| `app-secrets-sealed.yaml` | Sealed `WEBUI_SECRET_KEY`, `DATABASE_URL`, `PGVECTOR_DB_URL`, `REDIS_URL`, `WEBSOCKET_REDIS_URL` |
+| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
+| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
+
+## Deployment references
+
+- Argo Application: `apps/app-openwebui.yaml`
+- Pocket ID OIDC client id: `3ffbd06b-12cc-4562-bbe5-da4433962870` (PKCE **enabled**, group restriction on, allowed groups: `openwebui_admins`, `openwebui_users`)
+- Pocket ID groups: `openwebui_admins` (`737f4b7f-63f2-42b5-98bb-4e96c4a80546`), `openwebui_users` (`74b0af52-0942-4cb6-b37a-83d87d203e32`)
+- OIDC redirect URI: `https://openwebui.basicstack.de/oauth/oidc/callback`
+- Role mapping: OIDC `groups` claim → Open WebUI roles. `openwebui_admins` → admin, `openwebui_users` → user.
+- Initial admin: `andreas.leinen@basicstack.de` (member of both groups; `OAUTH_ADMIN_ROLES=openwebui_admins`)
+- DNS: `openwebui.basicstack.de` A → `138.199.128.63` (cluster ingress)
+- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `openwebui-basicstack` (CORS whitelisted to `https://openwebui.basicstack.de`). S3 credentials are shared with the Hetzner project (same key used for `outline-basicstack`, `basicstack-backup`).
+
+## Deferred / follow-ups
+
+- **Model providers (Ollama / OpenAI):** none wired yet. Add `OLLAMA_BASE_URL` or `OPENAI_API_BASE_URL`/`OPENAI_API_KEY` via a sealed secret when a backend is chosen.
+- **RAG embeddings:** `RAG_EMBEDDING_ENGINE=default` (local). Switch to `openai` + `RAG_OPENAI_*` for hosted embeddings when a provider is picked.
+- **Content extraction (Tika):** not deployed. Add `CONTENT_EXTRACTION_ENGINE=tika` + `TIKA_SERVER_URL` once a Tika service is available if PDF/Docx ingestion is required.
+- **SMTP:** Open WebUI has no built-in mailer; not wired.
+- **HA:** single replica with `Recreate` strategy; Redis websocket manager already in place, so scaling to N replicas only requires bumping `spec.replicas` (Postgres/Redis remain single-instance).

apps/openwebui/app-secrets-sealed.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: openwebui-app-secrets
+  namespace: openwebui
+spec:
+  encryptedData:
+    DATABASE_URL: AgBZ3lRcF8KC9OApWa7g3oUoJG8DT8KG0/mgN6O1VwfnC5JZ9V5Lsp5sFtvYg5OXkFW4iBhzFu67n4zAR34EXwCSs977nt2uH7ShgZdrQSOlBTyaKO33ixOcY6WtGQVyJCzl0vjVHBKLwbsVkqEhBDLtqey1CZ9nAW/RZh8MahMSHhRNgJ4XtTyNK/5z2nmJyjW1oVwZab/cWeChlQUtJgt8qWxiQenlgu319P9plXrBDAPmDN/UKDbwu0ivH1s3sBQikuBYpCc9EJS+7NXBNnphsyJ90cF1oyqA5zt+0Q00fZnqE0qLdSOYbVCW058lK7NkeyecxeQwpFgPZ2XaLmU1rIl1wayYpYWCjZ9sLsImnIpFRz+0ouoWoKCXnJvgQ0inGjO5tCmMfHt0H6xTm/Etb3455raEujMcYbRD+mnC2UFmLHshuEb+X5JVUx+wfabVIFps8Vmf34OOf9yODvM127e8DdBtXQ5LAw9JP43tHT38+5XK9koCJIUtv0OQcsp9AAN5Y/BIPEloyt9iFWSQC/vNufYprzdeRGo5DkRvaJOij+s/nw3irOFw1oMF+V9+xTOfsgbxgn+gKJcNSyjHL2Z0gtm8oMLWHIxbjZHxpcY427cvqxITkb1am0QgNDLsTpO+Ie76qGtDiDdEZfgLNeGXaWbJEvIO6L71STj5RmdqSptMxquNIZkhGRz5XRaTupqcmyXAuIUCLIdejXMLBLXNjkOIPut6p8S3keGj3PdIO3AUFnLlpZt+idWMpn7sk07HKSj+qs3DdZcOpkHL0EH5A/qiNVXi9M1NoQgDqvEzF2KMYkvRyJJGwkyg+4snhb+rkV+ioQA=
+    PGVECTOR_DB_URL: AgAF2jka+S+kBgNuEGKupNdThqJesdGRTtcgo760wCMOza4yXJ/ivHMr8onLmykeZLV87Yv3rBKEYvssUciT/RD7QxCVEW31aKKPdi+MHg24/Fi3zXwQKoop05ksXizGVxWia5vrtG93j/XxWKi7JEb9glHe0Taj6TX0Hl+JsEBJELSTPrdngeOKzkYRFDvHP43AadUhvOFwxFoidbLGLulU9t3AdPpXb/T0wLNOcukZFpQxaB36MIIipaJl9leZ1sQ6C0e/MeimUYw3W5+JNf1P2lrL75dH/lqkkJZ0rLetSB7qRmpPyH8dMJumkv+16fkMJMt+LV64KdE2KdqtcxuXFO+RjYkMfzf8lFEtWPNA3qNuPXtwXZAfmLWEcjIJaTXv33Ed+3kAwiHil5Veh1q1eBhGQ0HKvkRGSnxxSyvVfoznLlLvNQVlNLgNfhd407K8ZOutushxaf/NJFW0RFdQcsq69SaQUFpIbv6IGaAM0TRN2O+qNatP3709U6dYQPYdGTJN4LOoKmfIIXaWhiqeYzOiP01QMhML0GvUVbwi3b3fK8twDx5O85Q8QSUgEOYkKnarSciUE/XYhQKi+E7Czv6vGbKmB3G5Oo9Bw6ONDvxd29kpkO2Ko9zwSFaqUooONy3PzkIwPiQ7Vp/ygsoFaVJNU1vsUWtDT8wJMVNcLPLCnd4ERhsMk/r+gY0SXZe3Slo4KIrqoAInf0Y1RBvKZ+XLQks1HxHmarA7uRe3Px/lMXm1kFRtlLwsNxwMUU5u65CbgehGphqq8UZRpByT8dFGhfnJK+rN4ORRX0Fd3fxgxtxyb5VgCe40rgQrH9LtdYculZDpRVc=
+    REDIS_URL: AgBuATUR1mMRwov35kA8dxOUqgl8O480/qxvsbBj3vt51Zo1s5cAzWBG0LlDLtAVh2NEAowSn+egj0mmLsAXJrndreQfAw+UpHDVHi4DbNbPeg7IUCivw/rMjLHyTMO4zsxqreKspwSNE38JU9vzgDBqv62HnQB7hzeGgpwUpsrWXiHNBji8xK/WJtIAGikbPEEzccDC0aCX9Ut/+Yb9ynv1AOtq/xMFIhsWTQ5r2vmbuMS/f59MNWMkWCZBApJL510+QUGzKXC/m553Mk9wHF4G+wX8dd2T4dCQxZ+A68gonvVeFiEYkSHzd5c7s+RYoSBGABrhJ7SziQOG5KWQSbXjjlu2o7JPmsxbgj9tpS0ErmCF/wzgEzd89DjtVNqJKV08nT8X4x3yb2L7oDtFPvEUZ9Ro+IpXYIR/ZnpXUcUgzIoaMJR9RsWBCrAKMFRlAtRsxWnHF0Mx+P6Di92mn2zyJ729FiB8QH6V2lmmko3xRhEZ9l7houxeoTG+37APzpD7U34VaO0W92ybvpcjfrL5BHZEXhpFqE3DXOXD21rf6fmxdOkNqJVJ45CXmweCbKzS3L/0l+GsTNrb8EZALj3RThcNMDgBTzpeTt4iHsodDXb3ziaVufbtw9BVCCrVNb01e2hemAgor5Z/TIkOxK7bLQhNTrpwvs2r7skhOUhzRNuusFi4tz1oFX0JO1Rj96X7Q37fQn5/fKkG6FeOtTIlLJ0mM8FoNijGiRkVgoU=
+    WEBSOCKET_REDIS_URL: AgBjDV7t/UTomB4Ml+7iX5pL1menP1c5kPZOSMlTBPaee6tXnPa7cdEs9/vu7PxfFtP03BczSoobXy4Q/4wvLe4sRAUHRQrcEk+hkJ7PbDT5n/K5zayaqlQJm5MQukytYe5EhlinsjR3EAphq5oCJB8NUAtdC0S+KQHckbxhUa96V7YEeHGu2U1oUJrZCCfQgdfHbFKY8FKm3lEd//uNxmmNovWcC+N4fwWmwj2Z9I68oNK3/H4E0GZwnL/1xxEDgaty68xB6wa7Cg3ZfeX9NLqZk0Xz5cyTziAPQ9rPwo/sShBnplDd7JZRR1podSASUEMhr904LRpbwHv+bfoEt6hUlq3g7gilF4gBVuMM0AwjWkdepFjLGSW++N7C16rei8l1uFthvVvMwdExd9dI8y2h+8ghkWfqwnFuomIxY9NgyAW4ijW7GmVfhKsnD0K9khUe8eiJtR4aqyuQWYsB7QkTMjuFPGZLe3qEg2q3Cl3h7H9tlQxUPqfvaN1Sb56rzQJV3dEMl55SAhoHrPzCjaTibZsuSkRW9djqEyc8lJ/PfTtln7aI0CG9h17WCc8cIAHHIhRIcGpNednIfmkDZjtrKVh415rdMMHZsX5W0/VRnKIkZK/nl0G2HDHw3MJVKaIk05mp0nPb7IFjE3i3t8p/jRgEhgdtIh/s0wma8OijzaXIbnKb8cpQ+AVYAh2TfEzU1bZuWRYzUck/lAzJ4xR4e/yWpsl9L/0tFN+TWco=
+    WEBUI_SECRET_KEY: AgBqvo0l+vmqUBvOtln8zf7UVDaVM739x2/xY3lCf/i1/GlmCK/3aGDLj6tqODFSXuKTKxMS3liFVDleR/7EdYvu9/7H0ThSfO2xhheLzXdooxpIGHnfriKL58UpyWcieo3Y6VIOAQzrSyjBSd6S3GHxjtBdqyqvuGSUXkojf+jOjOKC2yOf+0qySklm/LAww8MigPrM55rlUcqRwb0D/KdpAIxw/wplkAMdFz0RGGfF2Xr03K0AKqJ2egUb9exI0JDUyGQBDJv9isVQPr+2xZHvoSb1QcPCExL5FA4YLN/EhwKhlp56itlrGT44aT3Cx6e8k8cEeqUyP0/4Fjz6xZnQLnTt1dvFAmEAusT9riqTkcAqhHR87MmkPIYmlJLkud1aT8G3LpXUmghrQgYaam5G4DKbQbQ1SBJ5eNC19VNW7hcL5zOvdB+Vm41/OzM4n2AW+Oa0UjFMgWO27IVIGVNjwn2VKapUWsRHLkidGfeVQIYyWznXRFPiBhSwvHhO767Bt1TASst3R3f3bZDcjASLr+0VfRm0MbJ0iXZkK8IrQk0un8zHEZRwCWk5Jl/LM0AnS5NeRgO9XAPlXm8xv5Xy8D2DKsIZGMkXZWeoEJ57/l2i/mn9JMWhq+va5+jyHKQwrqAH9ZqZpWvmip+LeIrl/AWiKGSzarhSIw3DLin+HTJs0rNswVdU4FgA7CDi1Dg1rTW1bbDMrebSB8KNLYqrWlokW4MQ2d9FuzrN+UhMYKbiq/TOS26/oBlWvMtdyO2qVTHjWJPWUDpYny1yAVHJ
+  template:
+    metadata:
+      creationTimestamp: null
+      name: openwebui-app-secrets
+      namespace: openwebui

apps/openwebui/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openwebui
+  labels:
+    name: openwebui

apps/openwebui/oidc-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: openwebui-oidc
+  namespace: openwebui
+spec:
+  encryptedData:
+    client-id: AgBPXc6HeT9gaqtXK9W/Bo0Jgt7UA/2Aygb7bHmwXSJHjv00Ja/ciISsmHQD7fovmG6eoM/fi3fRdJZqJkekRX+3Od+09I3BYQGzXjNwBkplydPLPEuzqDcCLdRTqZCpDFIz3If08++23hyeH233Vp6BwDxcZXunqzzLMNY+i907QhGm0ULb+T54zmZ49NBElXv8/moD9cag807ySCDw+I5pcWQMIZRv1FKtso8aeA/KA8Mzgfmbhxe+3HpPZ5svuOhFlzHED8bZ39c1l8cakjPDuRvrrIXzwogNhuIUArmXIyBhgq9G5W0yTsvqrsep1gMWEtYnZSNgVkK3vzodH2oOwstsOMgKThd1c5bnIPbN3bkLxwd/TbyHmQdLSCMNwOCVo8XGbeRhSABMxVacLfDI7hR+WIyRdkiMwKKQnQMqdBynzcWWXeSlnzIBpS3N+oodO7gS+l+6G6KX2ZXyUyRlxdDo7m9BJhc+HhxJ66L8vVj1P8QKczrixWJUqrUIFxM3jKY1A7XOI/x20/gQeodCJWIHetKvIHwwQh+w63mwLP8gRFglBnjNFok2zeYNBo3eSv7zhXEYve0y83al86/TrLggSfI7juJmLcy3bCy98ZExXdf818JslqmDutavUkUeYn8wIvdkeZbRtWgYLCu7izJZ1UDN8zkP6VQuhzLHO2OrhUuKnwpiI7Jp4/gIrwuGI9Ahal1OX+8T6R8JrAd4lfz5MfyoNzFZ7qrp7xe5zlTZZxE=
+    client-secret: AgABqIXEErUfaN5XW7SxGr/yJgs4GoPufPmNZF4d34STcCi7s+kE181h985ubaL12mwt6zNTDxUlGdz5p7ABLePRf8gEMUtzFNom6BOqIiJjhQuWLJRR1//DIfu+9G1KjmSFKS4iLkoHV7bh2+b+q+u3NXFVfClV8bVMICN1Nzp+doSdwTAh2CyQTK5JXmu6jXDJxuOkJClQdEH+rwdE5/It8ZHhC3y568m5H1JO4dxgvBvZv7jLxX8b+hGGLD8N9P6zMRwiKUlup+UPCv1V/SjppAP5kLmPKumqhtWqEY0j4XIu6xsIaZz5WeIAzlpQoRLshd1VGk9ebfXN5SfXcJdHyZlH5zy2W9HL/iBmr5kv2RkzpH2Sludewk2pWPatXtEcE4N1sat1cR2TnfW4wxJySZ2ni2d7H5jPFNu6Md7MFVVneYhanSdt++CtWol5bikEkzuSk2kjPT7Bf1wRhkz68VLo3Faf6ltktvpHV6eGfAegW8HgTSYjzwA55MdqWNrLlN/iZfcfhR6+8A2xblpUFXBgvCkTzLQzM2d7gK9mr1j+6osWpUDzD+Os8NSsnPBM4ywm2xfW5ZSEHdX2TmF9s7hDLCffQfd+ILSre/uyxNXuSNesE4pC0bFF9OndE2IvIQkuMPy306NnCEBzZsp+P8J0NHIr7EKWg+IMFRsYMQrIl5Gv+opyTssr9fTMAUN9ns0LdQPxBnjce+0egHV6g27zg3QVIc6K6vsZNciYyQ==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: openwebui-oidc
+      namespace: openwebui

apps/openwebui/openwebui.yaml

@@ -0,0 +1,279 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: openwebui-data
+  namespace: openwebui
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openwebui
+  namespace: openwebui
+  labels:
+    app: openwebui
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: openwebui
+  template:
+    metadata:
+      labels:
+        app: openwebui
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until pg_isready -h openwebui-postgres -p 5432 -U "$POSTGRES_USER"; do
+            echo "Waiting for openwebui-postgres..."; sleep 2;
+          done
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-postgres-credentials
+              key: POSTGRES_USER
+      - name: wait-for-redis
+        image: redis:7-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until redis-cli -h openwebui-redis -p 6379 ping | grep -q PONG; do
+            echo "Waiting for openwebui-redis..."; sleep 2;
+          done
+      containers:
+      - name: openwebui
+        image: ghcr.io/open-webui/open-webui:v0.11.3
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 8080
+        env:
+        # Core
+        - name: PORT
+          value: "8080"
+        - name: WEBUI_URL
+          value: "https://openwebui.basicstack.de"
+        - name: WEBUI_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-app-secrets
+              key: WEBUI_SECRET_KEY
+        - name: ENV
+          value: "prod"
+        - name: WEBUI_AUTH
+          value: "true"
+        - name: ENABLE_SIGNUP
+          value: "false"
+        - name: ENABLE_LOGIN_FORM
+          value: "false"
+        # Database (Postgres)
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-app-secrets
+              key: DATABASE_URL
+        # Vector DB (pgvector)
+        - name: VECTOR_DB
+          value: "pgvector"
+        - name: PGVECTOR_DB_URL
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-app-secrets
+              key: PGVECTOR_DB_URL
+        - name: PGVECTOR_CREATE_EXTENSION
+          value: "true"
+        # Redis (main + websocket manager)
+        - name: REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-app-secrets
+              key: REDIS_URL
+        - name: ENABLE_WEBSOCKET_SUPPORT
+          value: "true"
+        - name: WEBSOCKET_MANAGER
+          value: "redis"
+        - name: WEBSOCKET_REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-app-secrets
+              key: WEBSOCKET_REDIS_URL
+        # Object Storage (Hetzner S3-compatible)
+        - name: STORAGE_PROVIDER
+          value: "s3"
+        - name: S3_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-s3
+              key: bucket
+        - name: S3_REGION_NAME
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-s3
+              key: region
+        - name: S3_ENDPOINT_URL
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-s3
+              key: endpoint
+        - name: S3_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-s3
+              key: access-key
+        - name: S3_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-s3
+              key: secret-key
+        - name: S3_ADDRESSING_STYLE
+          value: "path"
+        - name: S3_USE_ACCELERATE_ENDPOINT
+          value: "false"
+        # OIDC (Pocket ID)
+        - name: ENABLE_OAUTH_SIGNUP
+          value: "true"
+        - name: OAUTH_MERGE_ACCOUNTS_BY_EMAIL
+          value: "true"
+        - name: OAUTH_UPDATE_PICTURE_ON_LOGIN
+          value: "true"
+        - name: DEFAULT_USER_ROLE
+          value: "user"
+        - name: OAUTH_PROVIDER_NAME
+          value: "Pocket ID"
+        - name: OPENID_PROVIDER_URL
+          value: "https://auth.basicstack.de/.well-known/openid-configuration"
+        - name: OAUTH_SCOPES
+          value: "openid email profile groups"
+        - name: OAUTH_CLIENT_ID
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-oidc
+              key: client-id
+        - name: OAUTH_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: openwebui-oidc
+              key: client-secret
+        - name: ENABLE_OAUTH_ROLE_MANAGEMENT
+          value: "true"
+        - name: ENABLE_OAUTH_GROUP_MANAGEMENT
+          value: "true"
+        - name: ENABLE_OAUTH_GROUP_CREATION
+          value: "true"
+        - name: OAUTH_ROLES_CLAIM
+          value: "groups"
+        - name: OAUTH_ALLOWED_ROLES
+          value: "openwebui_users,openwebui_admins"
+        - name: OAUTH_ADMIN_ROLES
+          value: "openwebui_admins"
+        - name: OAUTH_GROUPS_CLAIM
+          value: "groups"
+        # Runtime
+        - name: UVICORN_WORKERS
+          value: "1"
+        volumeMounts:
+        - name: openwebui-data
+          mountPath: /app/backend/data
+        resources:
+          requests:
+            cpu: 250m
+            memory: 1Gi
+          limits:
+            cpu: 2000m
+            memory: 4Gi
+        startupProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 60
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 60
+          periodSeconds: 30
+          timeoutSeconds: 10
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /health/db
+            port: 8080
+          initialDelaySeconds: 20
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+      volumes:
+      - name: openwebui-data
+        persistentVolumeClaim:
+          claimName: openwebui-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openwebui
+  namespace: openwebui
+  labels:
+    app: openwebui
+spec:
+  type: ClusterIP
+  selector:
+    app: openwebui
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8080
+    protocol: TCP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: openwebui
+  namespace: openwebui
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  tls:
+  - hosts:
+    - openwebui.basicstack.de
+    secretName: openwebui-tls
+  rules:
+  - host: openwebui.basicstack.de
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: openwebui
+            port:
+              number: 80

apps/openwebui/postgres-credentials-sealed.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: openwebui-postgres-credentials
+  namespace: openwebui
+spec:
+  encryptedData:
+    POSTGRES_DB: AgC27bglkH6N5Ule5F23XbirvuqWgiQlwd8yyqEqpcZYsvT8605y70g2mS4zZBtvVqdYFcORJF5abdqWdVhmPhZAV71tesJBTOynMQb9Zd39EbBcEb/6Shresa7FdaLQgaQGLxIOOt1xWLmh/eRv9bLiIGNtz1ezKCa9fXYuCrD1Eg0EGmUjdioXbkuVoVNON5rBwUTWWpfVq7/d/UNQGUvAuhOX2RORD4hy/5IRZG5LvAF7C9oK1LaCbi5B65kbC9DOtlCPfB499tt8fSQDYic8YBLkmPKQgHd3+8NlNrH+TtTZMTO6UOR53IqJW/xxoJZapAl/40XkFj7Q59X+xDzG+p81DfwwOu9U+Fu5a2zpdduCAVQCIRSVZBL/ziYkRjFR08/5ToqHF9+p0C6khIgt1EkyvyUTxZfKPlcq68soY9cl4UzV3UPe6DyyEXwKsPmaUXPG4ckGEAWIIIoEd4M2cFN6jyWTRyYiWfe9uaPSIkjBGi8TIXk+Tk2ZTcZG5UQd48MOtc9gnXQwoFo2JzAqqBYa4HcATzPPiDzQCKhMXjGelINzExqvjOm26L8Zi5yCsaAoP3Ev/Kedl3ohdWW7CpwTxcffnd+CTYo5oz9K8HD0CvS2gc46HL2zCzZSF8Zpap/0bsM4FWFcEEI8ru9mdkhXZ9EygI97fcxtAi5eDtYm2UjKRDzCLLb3a+eorUEKfpXPUTG5E8A=
+    POSTGRES_PASSWORD: AgAislLVvgaIxtm+U9LzD1lK0Rz05MaiphMWHuHVpGayAqTorWy7jz+iSI4aQH7QQFzClv6WbSgzcwSjYiTZhH9+GxpRHpIAcyYbjHVG8qU3n99A8WXXCd5V8K4alQK4z5kA6OxZGOuYRQ2mfAbHGdAUBNIVTqNv7DtpNuATKzZbkLd/tMX3ezIcpJqOEtb0AqgMvcH0gsw2j7I04WBlCdH7qQ6e8EUqHjxWBc1YMF7INzAxlkVgIcaNkpfcErEL4y7X1E1yo0KRzAg0O18rzUqs+hOi1XVSNaEuA0yno+a1ZXzGnmiwJFK+BboL3BZUtev6Xlbe3DqcCwUjmw3v18suO84gXGfLEJa/OYsys1vgHQ4qt0tP9guSWaqnTfdkc2ZVilw7M+d2TS5PtQC0U2/MW+SNmS8EdC+/q6FvTgaMr2uptqgC6hrrLFemQ1KFzJibut1s5o7HuLKqD915gSbHTPJ0vtPOstIJo11ggmvLw37JGqhbHkME7SwkJzZXWAFPvAWfsJTRqL8fzdk1RLlVYQeak7VnX/iL5q9bUomsGvYXMV90/DH3vdkhFuxOUx1kdjnROtx8PYg3rnzXPvfR49+x4EhUWRqaF1XDXWdWhCKrZFrSskvcm7h01gHvLQBhbKK/X0nOz6CM/Ht8zzIsnsmjssoZaz7iFrvYYyhSWyp/U/N9GdFkcDA6qzRsZSM8WGnRnEeQIHklzIL1fZlvsPl57rFjHSYZzdXD1Iolzn9v8KU4vt7kGxOiPh6Abfg=
+    POSTGRES_USER: AgC5pjM5LU5zyzqvfsrf1u5koa/IwO0wTnDUv4GcEzRLKD/d0Z5iDlT6bdyCS1bFr159NWUyz0FyqfMDqsGFzJiGwEd6Bq2PHaaa+h5wfhY/KguVPndb2SCRJWDcxddH85GGRo6iTkVLeNBO5mOI3imQnDNyKaBpowRINoLNnWDoTEZ7COrTrLTWBlJz8ar4JyIWAToS08cBIhQqIBCWJkUpXSS52Frju3yYKBCeR0lDBrL97WghqnM7cD38luPXajLZeysck8iAwZm2Wt57ZmLUCLSo9cxkhL7Wx/mV2A/s43KzL0m7w+YkQsk6x8anug7+hr2lKAzb58ytKkyPsJ/OirGB9fRfIujEyU6B75AFDePtvv5W6Nf3ngWn4hH25K9dDA+LyMKo0KVXQi5uMnVFitwMUMRFLFlnyreydt/aQyG/G+XjE1TZtBgjF2OHdPKSaxqOh9F2vZ8GLRNV70EVVOO54/4lqeLChNRBbuxgn5v1MtD8lOUMtoS4Ho+82995MARCzZeukfP61tqMbKvTn9HtbbQq7jBfoWCUgCY4KeiQ9sf5+Ks9jfmgDvdgBckiO0IJCq0oaiVvUGW5fkLwa273sypVo6sglLoa1I+YCkUJ+Ym7nOw51r+zATrNmzzAG8W04ynz5Jx6iF6mS0K29giqAx0POxFpw72CPI67COYLQdXr4b/UoSAlWWwCH0eGCJUfcA/LYIE=
+  template:
+    metadata:
+      creationTimestamp: null
+      name: openwebui-postgres-credentials
+      namespace: openwebui

apps/openwebui/postgres.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openwebui-postgres
+  namespace: openwebui
+  labels:
+    app: openwebui-postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: openwebui-postgres
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: openwebui-postgres
+  namespace: openwebui
+  labels:
+    app: openwebui-postgres
+spec:
+  serviceName: openwebui-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openwebui-postgres
+  template:
+    metadata:
+      labels:
+        app: openwebui-postgres
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: postgres
+        image: pgvector/pgvector:pg16
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: postgres
+          containerPort: 5432
+          protocol: TCP
+        envFrom:
+        - secretRef:
+            name: openwebui-postgres-credentials
+        env:
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 1000m
+            memory: 1Gi
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 5
+          failureThreshold: 6
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+          subPath: pgdata
+  volumeClaimTemplates:
+  - metadata:
+      name: postgres-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: hcloud-volumes-encrypted
+      resources:
+        requests:
+          storage: 20Gi

apps/openwebui/redis.yaml

@@ -0,0 +1,89 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openwebui-redis
+  namespace: openwebui
+  labels:
+    app: openwebui-redis
+spec:
+  type: ClusterIP
+  selector:
+    app: openwebui-redis
+  ports:
+  - name: redis
+    port: 6379
+    targetPort: 6379
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: openwebui-redis
+  namespace: openwebui
+  labels:
+    app: openwebui-redis
+spec:
+  serviceName: openwebui-redis
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openwebui-redis
+  template:
+    metadata:
+      labels:
+        app: openwebui-redis
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: redis
+        image: redis:7-alpine
+        imagePullPolicy: IfNotPresent
+        args:
+        - --appendonly
+        - "yes"
+        - --save
+        - "60"
+        - "1000"
+        ports:
+        - name: redis
+          containerPort: 6379
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            cpu: 200m
+            memory: 256Mi
+        readinessProbe:
+          exec:
+            command: ["redis-cli", "ping"]
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        livenessProbe:
+          exec:
+            command: ["redis-cli", "ping"]
+          initialDelaySeconds: 15
+          periodSeconds: 30
+        volumeMounts:
+        - name: redis-data
+          mountPath: /data
+  volumeClaimTemplates:
+  - metadata:
+      name: redis-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: hcloud-volumes-encrypted
+      resources:
+        requests:
+          storage: 2Gi

apps/openwebui/s3-sealed.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: openwebui-s3
+  namespace: openwebui
+spec:
+  encryptedData:
+    access-key: AgDBruwsmBhUuqOg34A8jWTT+9Ko9K4erxigpVtcQRxfvoBu0Jn9O/W80XDPO4+HD6krMlxZyPb0uP/qwsLU99Ll2gcxsUDVOvhJwkqzI1ohZbl+iueJ7qG7Ty5HO8CKCe8AiDgXnEW5/hWUxYT+0lX23aIv3SaJdhOHaScMCw5DSGlR4/JA43uEfK+Rbx+FDYokw17pi60LAhuQhZpbzoy0sgd4A5nKaSyNRis/YPnIZzN5gb8yyuGYa5fHSLGnowsuAlfj4x1MzNsw8pzuygnti6KsoMJyWnqW1TsaWb8WkhWDRKEN2AO4tTD+2BuvixV6LBynomkLfwZ8DSmN0Di5Db3bKGEkECiRAloeIc4efr5L5W4ucAxPE1YFOM7VINt5k/7pdymQapGRyQymzEsC6dWZNl54SjN65zzZIerkM2N2II0oqyzhtYW9WZ1ihPYvXSSp2JoaiKny1ljzqyXeizX/Xkl4ZjUZETYV8KIIFMx3SwTGBvShGwmv3T/v7XnMDxG7lIvjtHvCmcaiW3fegz3reckuczuG/c4rtKQo+bzPnkWznKlXVrHNg7Hks9DisIQPngn/u1bJPK8ncEYaX0MRGIIqb5yn8eoT9Ggxk9eY7QHym9Y/bjFMbR5MMR7I5fSQSP2cUI1K6OQ8eqkteCUB2i0wCfDXHqnkE44NJM38hWUl8b3rNH2yo2gnLyYhkaeqdetqNbW992quRUp7UXshrA==
+    bucket: AgAm7zODlaJ10GoHumXsgXJC4An3GNegi/Gm3aNhKlyNV3ItVU8q43/UPMck3ed14XZ1gbMUIkkIU8L9IIIofwEam4YrfQYfoQomtaYF2EHnRjH56+xrnRP+WKzTbqDpiZIcRfr6E1+ajBM70H7A0TfztLiPmFNJaGv8YPLSdWh8E5m9vgiJJ4ErEDwMxMrhLpfjZ6iNp/H9vchPLOtSP3CtAJvrrdWO8My1xZX3bnMpoU9YLlSSfPAGpglZ5dsxeRBA+HSSRo4PouhLKMm7PULk4eFFwtetNgPh0DwHXhD7hboRcZJECmZMPz2xdWwu8+KqKmMywTaN+gy6IDOw7pZV0hHwVd2AWhlE5dXnV5pVEm0vUKbC5ceS4Swq27RphO5g5/7lfnkV1I9JqTPETFXCydxQ+vhW8UxbEunTWp7PxE/XCNMvYBvecksWEmLix3iVFGj2chH2tnDLx/m/K2HxQkqNo25c1PnDO3aL5WTSjcK9VWSwrv4ucoROtDA4MiD33FCSF4eBwzUm6AUPuWIIsoobjGOKZcwNiRDDtom6O52h6+0/xdplrNdj5X/5GHe7jnEM/IwaYdNsYBu0/Hi9eohA453brJfGzShe9mcKw0TWktq9xrmfk57PuWAKny6dg6VWixS0YHNPd7v1EkANmnm1nagNMcurScolwgY/MXYEbtQbbHPk5Zn/yQGrDCFoHSkE83gC7go9dTvnmEi4anDfYQ==
+    endpoint: AgAlRlwvCwNLdPXgWgpSYTz7MhtX+wsK9TNCep0C6oGJi6HMlEzT8s1YW+58HTGmc5wFkoB8P48JDwcfdb7BjAgFMVgkiBz3WrbtYYgUda8EeiD+5Kt04lTcetPibi/R/sbe1Hvxp2SIsJ4Qgt6x6qElR0GM3hfiavP95mazIL6zqX0MImXtKFEqGBwhSiFIQ2w0ldbbBxpl2B0w9JpOO0f9TxEZW4do4JyKkTSfUnBpJdpN1tqDP+j18PV/H7s7O4zdpVWmwlp8l96xSVZtg0VZvVvdC201mPuAKF7EV4bIy6/a6jIgKeUv0k0aeZqqTgeNkGmCzxI43ujShMxVEuNaDDVt1PAzcTl2r8IImyJlf/B3AipgyGpwXUzn2+A0DiBIqjOXC1dezkezS0+iq5yekO20d3N73KswIsWJu25Dvc5j+c97nPSQmOYGcoYtZmoIac2GNf+UTApbLiJdzSaP0hQZ4c0p6g+xtCMjesLl1xI1ihMOJtetW/mDsKokGWfACWcNdkN4mKbgBHtOS8GvfK13NXpDRDfgl95avhVgj0hdGANre3fSk7cmEX4fcFCyyj3wUqHOBK98oNL8zaNVmHjEDoajw5RI6EwHOM1lnnUfMyEuJT+Ye6vRCmJTLoMrVqCOUkcX0rfIA7+VRJjSoeeoz7+iFJcsNGH6HIdE6rjOH9HdC4v/L5wNH5CUbiPLG8tFcFiFF2k9EUg6V2YymkrgCM8WZb2PN0QQloohe1VkRQ==
+    region: AgBWLZOWPXCo8H5D9fKTqK/UWSOGsB32Swr0H3mhoOvZntLkqtxR0WsToH1hYzLnsVz41O5ntBRiw4nx00SvKs8MdKT3rW2gcuJcJMznUMXN8WoWLfUIrRHO5xRR4bh3BDeqaJo80iGL0Ql1/460BYwf3F5mBnKDEBdbaJCNKraRGLUVSOJlFFkebzwTsgOszQgOOZFFA3oRUNgZugHFzeOq9HYYqf+XQ1WSA5AyKjgR2xYK55vOiw/qag1keX31kWGgB039M5IcNhdI1kVUrQWgBeDWU4zp30983FFBaxJENKqTXEhtkfJGgi33+kPuMGdz7KQfNShd6bQNVpfGeX9ZsTPY+ZIwlsiljQzuJ2FTu84Xnmf3jNsoDuOnq4BmpYHV7+t9iGPFmiiTHCYN5hTlNWptLvBa0ZYE1dOwEz41ZEEisj8RAMWBNBiHZDBPyVEvWjkmld0Lro71F6XGb50jlaQlOV31b3zVksatzgnkQmtNfeZucHY6AyxIrwlbw+UxTIJ1dfzFBRyrPAdGZamUgyqZLqXWkJmwy6ktD/tRJrfpso3QjkPjAlCXYWNIp34Qx7GHAf4t8jaCJx+qjlpSuQ4iv28xFL5xug0I02cfvZwV441ANVP5i4ATnVpJ/TIb1zbUtl5ffuGrD5fY58DQ74ntsHSQ8h8JIpQDAu7WL+ZSWjGpV5tBUz3vIP4wsDJnHN7p
+    secret-key: AgAHkBSjTxORjuS5i75zd5Sk3FytIYwfQTgRs64LUHl2SAD86znxUh+4T+AGCO6dZHoKp+madkx84ghUn31LdA/2FRfXrrujxHLlom4LKHkv/jDc4hSRtJPMBqpi0KUXUtXk83nJ1s1BN5er93S60IiaYZOBcS0XojTfh+TbwlKJJsKstJcOB1PNVb0V4/OC+fSWpOJNkSI5PAEkzVVcSFsEaIuIRpAazwm7hBbnvWCGwd/WlSzZEHNVnf/fpI2QstVYGe42fg0NzEqnFk/I5QGRcKhUvM3wQ5smNidb2mors8FE0UzdvFcP4cytYb7tDf0GKclX4p2kmc7Js7EWTRmin1N31EVdGPGsJ2sV5u5JQoAg0b381H4ZqQckInB8NIoAx5L2p1gqnPKPt2yTV192jes7COzNhmODTVaSKY0z3v//u6tVpsanfiDvkSl4y47BKeJfxp4Ihd4wJNgN3vkaxi4mDYBiNiY2QXSczZ+2qyacZeGowl1ZpGH13KSg6bOV/hQ/T0tr0lbwiZ04V7v5pORAw8SnvHT6rqzHLrE6PxDiLQsY36bIpMvqPLNke0ErEfqXMwGeYClij0fpU/7qZ5t9Ag2PvClroxY3wuLdpKE9ffeHtH50BgyabOS3DsMXcB1pODyXf2QaEzB5NaGqMp/PEbmoy3JLYwM9We5NCQObxfjEI6gkgBitBY1PDBbU6DGfTM3Smf8y6qJ6Y4kHx31VUU2eVHP7+O912OH9yzguyq1DpkiI
+  template:
+    metadata:
+      creationTimestamp: null
+      name: openwebui-s3
+      namespace: openwebui

apps/outline/README.md

@@ -0,0 +1,39 @@
+# Outline (outline.basicstack.de)
+
+Self-hosted knowledge base — [Outline](https://www.getoutline.com/).
+
+## Stack
+
+- **App:** `outlinewiki/outline:1.10.0` (Deployment, 1 replica, `Recreate` strategy)
+- **Database:** Postgres 16 (StatefulSet, PVC 10Gi on `hcloud-volumes-encrypted`)
+- **Cache/queue:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`)
+- **Object storage:** Hetzner Object Storage bucket `outline-basicstack` (region `hel1`)
+- **Auth:** Pocket ID OIDC (`Outline` client, restricted to `outline_admins` / `outline_members`)
+- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
+- **Node placement:** stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
+
+## Files
+
+| File | Purpose |
+| ---- | ------- |
+| `namespace.yaml` | Namespace declaration |
+| `postgres.yaml` | Postgres StatefulSet + Service (fsn1-pinned) |
+| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
+| `outline.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
+| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
+| `app-secrets-sealed.yaml` | Sealed `SECRET_KEY`, `UTILS_SECRET`, `DATABASE_URL`, `REDIS_URL` |
+| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
+| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
+
+## Deployment references
+
+- Argo Application: `apps/app-outline.yaml`
+- Pocket ID OIDC client id: `b4a84bbd-fae4-445e-868e-5c1197e5d155` (PKCE **disabled** — Outline's OIDC plugin does not send `code_challenge`; group restriction on, allowed groups: `outline_admins`, `outline_members`)
+- Pocket ID groups: `outline_admins` (`2b41fc67-eb7b-43d8-8954-85426214f1a8`), `outline_members` (`aa6df56f-b868-4de1-adf7-8696e990aa53`)
+- DNS: `outline.basicstack.de` A → `138.199.128.63` (cluster ingress)
+- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `outline-basicstack` (CORS whitelisted to `https://outline.basicstack.de`)
+
+## Deferred items
+
+- **SMTP:** Outline runs without email notifications. To enable, create/reuse a Stalwart mailbox (e.g. `outline@basicstack.de`) and add a sealed `outline-smtp` secret with `smtp-host`, `smtp-port`, `smtp-username`, `smtp-password`, then extend `outline.yaml` with the `SMTP_*` env vars.
+- **Admin promotion:** first Pocket ID user in `outline_admins` becomes the initial team owner via first-login. If the admin role does not stick automatically, promote in Settings → Members after login.

apps/outline/app-secrets-sealed.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: outline-app-secrets
+  namespace: outline
+spec:
+  encryptedData:
+    DATABASE_URL: AgA0XHjlu1iWM7AJaStmEtMdVtYMwKzXxk7/nMCtIaNxaXTa7XH3K40277Wc5a/mfM4KLiF6iLExquNIa7YI6R3EROpUC23cTJbstpQNP6tffzZwr3FeTeod4qWYJ3HfIHaBlocNc0p1ntM9H/9k+Vy3rQCn9T5kLJ1c6LqUGhPinWwQxwiDgNU6JQyEfILIAX35th5fe24GH4Ka10k56wq5Fnx9CkYYxsVQ21NNKCdtqZecR+C2RqfYhXharvtio1ojWP4KNQZuXEXGDAH/nb7QPtaNPCV8hhsS66FAjm3hHTOmhJ4XXbJQpWYCiPOWBa9MVHxMg3+IDfdR2OWLDOpgFzN/vMKx+iqa3aVHXfdrXcFUqEC48d0h3MXjdCge4c8wY469PUTar+74d4BcljjG6R1FL5CJ1Wmxygt6rG2Qk+pWihZ00Frc0gttsxowS7c9s8iPL6VF1ShyaTXdYq0W7pHlX/Yq927C7h8E6TNQ+Rxw3UI4qPXN7xkU85FTQvWZjqoyyej5GaPbGy7Qp/SO52wsq4Fh5qPrZKoGl5xQVcmuF4LZKbb8WQDQcFDatHdWZiHZqsX1wKxRP0jR8RkT3Q7Z/M2itMN9lODtddCv/hjrwd1ih950kaZoVW3cdZCVb+S/M4pbkuB0BAasRbntJx/hOy/9zp21MAWfTGTyscBetIjHeC/oy4S1WQyo5TwkMSeugqXJ/mn/zTEpDYrxuqso7Aeu9H658WeenwgUVh2VB9SVWFWxZnHe0rB+keMPDwJ7Dx7NsdydybsRmORtTIc4zj+OsqDhmg0wwCq6pSslp+/h9MwfUB8YkYv9ioPP
+    REDIS_URL: AgCKT7MxyQJV4Yk+I8TO52x8x5nfXd0ECB9H2w8p2gRFMyiTbrTMvdzaF9fqrgTiGRuB3VgDeeQpsnFGvemoYWqbLZ+WI1UfQKlyar5y3tHh88GMoI53dmkaBc6V7Fr353rkcy9NmRJC+g37V5PWasiX5aWFSdYoslmaUOC/o4pQwyDZEwo3xFHysbm9tD5jGVSsCgnUGeiG0Urva8pNrM3aTfvSoVOhts312eKk3+x+8mda7ZchH1DtQ9EZ1V8emEEjOv1Jh6gh5wLjl710cHAS0ch/9RTj8ooCH//pX4V5gj0ue5EWL1vk0xgV+kv5tGG2pwR4aUZWa1d1eQzbFDshHwkbAWKm0pjCq6eIMg6QLQRdagGXe4ZKsRAahUJ+7Qo1MqDXthoJHDEDDhB+/SbbPKTOrgm+eh3oA3EuWDcp+iKAqboBc1WkzUWh7kRVRt4bkVBHkeK3UjFvjGtuwC8cI2yOQ3DyRh7Ija+Fp1o8ust86FKYS1/VUQRqBX2HTBamognfS+EN4PyrHJ1XZzoL85njGm7v6nOG+EQN5QTlRkmLTLqLVWzUk9tuQbpm57r0ULqbkMKYzfnaj6opWo6EsCH2uQg2KGMVgxAhY6jFuFc8cLXdz44Fb7tjTlSgq1Z5S3hqHRrP2fd5upjFICP/ECa+JRb6ZGpTSvIJ2MpH/0oCTFkgl5+A2KUUP7PDNkqCoVhujRLS05XuSize+owTbjeEz7fJCRy66A==
+    SECRET_KEY: AgBYcN97oeke6wlfUel6cki6sYIUyM2zJUuQfBSSFoZRawSurDhL7MboCbVXXtgold8QMp1aF541442TO0DdR1eMvOvNqGHIQog3sNDlbjr3LOk3vV7FcZFoME866tsQ0uzfIR5Z7SBvkD6EqdnrIe+sNfSQHt1Jn3/QCPhgiYbsN4tPD1t7HFAuuPmI/gs91wnv40r7PGxK2cYYcPgGrT1P+prjSsca5sbVvy2e/jkW8FWLoRxrNw4Q7FImgDPP180Adt2V0n+7HfY8u5gSS94oY+5CBmWuR7W6GhStuI/LfKDDVVgWdKtnP90Eq9NiGRg1Xk+EwjNLWSm9aTLZ/2h1jzDGqmrJ75W+Z5OHV2bdgxCFb7lZJRU1mDTCjqMNf1jKEzSQ2pFYHHmyRhz0dQ7FVD7wUS3SSTkC8epxxRf+s8Jb2O67x5sM87V/QPyMGlCPs/634OOwj4+7V8owcc+NaC9iiGTjwd+6AurFCF3O2t2YcYgsJXoDp7Rd0hf/kk3cEKAhKMqO2DDAew9Og3fpJrF2mgYnMJb5fDAaEq7FrA8y7P9a1vyL6nZMmfXZcZ/a9g+r7uoFzoE2wxsaCpmlYvgep/X+a1iQgE2+LwjB+lquYFH+Ztam5HOneY7wNMD8EpLbyUq6RO54+f/1COipeLZxyUc8tI1zMlsWlBKNSl0qljZqA3HAE3a49xQBPTiEHlfsLk9me55zkR2t/H9+HBlJ5LQvGb43sYhVssSfv7mYTAzEOJPiAjUiLfpg47cu+dt+c5x4qenFmqUDh78J
+    UTILS_SECRET: AgCAxEUaZS1TqU9HA2a0Ze1lxsP97DYgyUnWFaVAWc2qSM3sqp4BffN9ZJ8mVUXkZdZXmY5Yoirui/p4z/Ie1BpZBH5dbcuVGWu1DopCcs0UROpftJTWLKLZjoYGxp6RbazUNSVc500KvLuwMX+E0wG7OUSAt+nRN65/ebHRWl4kg1DceFLVQVmies/uHHz1++1jkVTGrr2ClLdmj0z/VYc5dYifxouYUEgjKWzB9ERkso/J5CCjG7hRRTbAeEaRH/SeyV6qw9kvhgCK8Kd0wicJatu8EcbHBDZs+0IiQ6vkqkdQE47MlFqfwem/lLWM9ce9rGBMkJ/rcMTGgW+pP6GKaQVKrhP3+0G6lz7IHneFvEF+W5tEaRjf7VbQdbgoX4lfykjPQ8pk3cIGPWLOrFQTxdl5EOhwS3dQ/WoAQs/GLgDsNUO8TlQziyURGCJT/LIfXvEVZsbOTcrbayd3ywwkL5aGN0TAcnWb0TPXgZVhw91N7Z7e2gRoGx2CoSpZzjv0k3liBrffCRYvvc85HrQnKtd8l0uutpCrtVOStc1wqRA5g0s9j0bgqspvQp7CGfyxs7vnGP6PMBoOtgH8mR14oI41SXTUn30uRWzGR3yebGLRXZ6ORgtzB8lQxDZAo6VUcc4afzyL8RXdhtJNmr/goRXnMg8LQqq80teQxFat9RW4Wxjqpnhb2xNwFtOxYFK818q+L7/7CC9MqaQcnYwn+mJUlMQTam8rPI1v1VRLfdzXcmHgQCXrlaTAMFOBF5zPO2vr3+OES/XS8r4K9V4f
+  template:
+    metadata:
+      creationTimestamp: null
+      name: outline-app-secrets
+      namespace: outline

apps/outline/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: outline
+  labels:
+    name: outline

apps/outline/oidc-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: outline-oidc
+  namespace: outline
+spec:
+  encryptedData:
+    client-id: AgAvHl9m4WXOYPIFG7rrYzPeA0Ksm2DUs2DS9gJismHLL5fX63e2KaD66W18NIFZQvtkfe8QelvBi3J2YAH2I1qrdZjn0SVJSq/7cdR+0X6lb71/hAkbHUr2xS+uOV86/O+6/htLhdNl0IcAjvyNMXWh5ptVBW/pbTQHJqGYN1dHHDQKri/3XIpxipKIXL22oPnowv0jcW1tgTZPPFkFOpahVlVP9n0pxM/2X3bb/bbsNVuTk0tWeDTqj0XTUN/0Kowq8P4rJFVdbJPyyV/37PnPlu3+F1i/flApo3HKQmoKR09ogCS6uZEP10SO7lmz0+JCMCIxYAHr/ELaynEXWhsCNJsLTE+TtMdMNciy7n8TZ75cS+Azgjhc5cimreimmxuFI1NRuMHnLMCLam1Hve2190sPhcIT+O5Revrc+BKsSXuscDZtrUugOChqYM9EW0T3AwmbIAyBQsY3MoLc2wFuq1fpYMx+Yu/0ajPBZvUV2JRbVt5YtMHoo3VbtKXkbfZW9IaBGcBFDxg7Dd8cZju4ljieRWJm6hISO6OPrMjvGG1IvBTv9c4SEGhnCHbjpYeoB9UPlL/FKVQgyrK1v5qhnZ0ZoiKYecxfj6Jytrf/qLdrBZDlCHghKpn3OM2OeubOak+xH1zgENK1UhjhkuqELoBXy9WDTVkOvrm5Tiw4WAAUm9jmjQiKusdwUGMPBt49sXOfyzsu8iPWIb8FDhH3DLF13d9wH7vtwlGQBuxhC1+FiSI=
+    client-secret: AgAHp4jsX5igXGHwKiQJuOKgyIIOssntobUhHnIKtSrNDK5mBZ0EvvwijvF9bqOp2i542WxVcjabk9a06/h6QCXymmo9Lcq4kK3cZyrQ6V7efjNFhogHOnE/f+y+lZy3AawTd1uokLiURf9z2X9rxi+2ppLhqExNFa7W3WIuEoVnk55H6nMMOxBK3SFfXi7PQUt+gJWGdZRMltVnySjqnqVbHwsO73uNoHLrOI4pEUeQiPaSQngkQRGa5LAYm/A6jYGYkTFQtvT2gee+aLTITzynzwdUmCdrJ0GcL/bi122dI3f+Kpkdj61Z2G2VZY2r2blXtQZcm2vt2j+4ZRWwMejWtpnpfBAnFm2wThvFiXqIYit8AtXu6isuj2bmwqtgZT5ieQeop7ieu1LBuqCutNkn+coxdmQWH+/Ct5oYl7GTHsPWqGlXF9Irel9J05SxZGmCtjw7sdXeGXwRItP7/eh6attCJZJJaPCD5/1/cNSVoLrifBQnNV/NoxWYzXsrYpz6LdNq38kK8eEB/YtNWMoCLN/NUtY2SbrJBlQChdWNeovxz9D3ddujFb+792iQ7qHcm2DGAEX2fKqeYTFY1J1Rp/eDF89UMocjJ5pgvfdjqYNuGXdARYxjBJybMskS+MhNd87JkjNAyRDDhU7hIe09Q0KtR9eeoe8wQ+8R2oOXoivAsm/KmyNGDTKPM5xUCvDABHjaaMmHczSAAdZcc/dYnrdgMGPcONeyVkhPgr90Fw==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: outline-oidc
+      namespace: outline

apps/outline/outline.yaml

@@ -0,0 +1,255 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: outline-data
+  namespace: outline
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: outline
+  namespace: outline
+  labels:
+    app: outline
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: outline
+  template:
+    metadata:
+      labels:
+        app: outline
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until pg_isready -h outline-postgres -p 5432 -U "$POSTGRES_USER"; do
+            echo "Waiting for outline-postgres..."; sleep 2;
+          done
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: outline-postgres-credentials
+              key: POSTGRES_USER
+      - name: wait-for-redis
+        image: redis:7-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until redis-cli -h outline-redis -p 6379 ping | grep -q PONG; do
+            echo "Waiting for outline-redis..."; sleep 2;
+          done
+      containers:
+      - name: outline
+        image: docker.io/outlinewiki/outline:1.10.0
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 3000
+        env:
+        - name: NODE_ENV
+          value: "production"
+        - name: PORT
+          value: "3000"
+        - name: URL
+          value: "https://outline.basicstack.de"
+        - name: FORCE_HTTPS
+          value: "false"
+        - name: ENABLE_UPDATES
+          value: "false"
+        - name: WEB_CONCURRENCY
+          value: "2"
+        - name: DEFAULT_LANGUAGE
+          value: "en_US"
+        - name: RATE_LIMITER_ENABLED
+          value: "true"
+        - name: LOG_LEVEL
+          value: "info"
+        - name: FILE_STORAGE
+          value: "s3"
+        - name: FILE_STORAGE_UPLOAD_MAX_SIZE
+          value: "262144000"
+        - name: PGSSLMODE
+          value: "disable"
+        - name: DATABASE_CONNECTION_POOL_MIN
+          value: "5"
+        - name: DATABASE_CONNECTION_POOL_MAX
+          value: "20"
+        # Secrets
+        - name: SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: outline-app-secrets
+              key: SECRET_KEY
+        - name: UTILS_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: outline-app-secrets
+              key: UTILS_SECRET
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: outline-app-secrets
+              key: DATABASE_URL
+        - name: REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              name: outline-app-secrets
+              key: REDIS_URL
+        # S3 (Hetzner Object Storage)
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: outline-s3
+              key: access-key
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: outline-s3
+              key: secret-key
+        - name: AWS_REGION
+          valueFrom:
+            secretKeyRef:
+              name: outline-s3
+              key: region
+        - name: AWS_S3_UPLOAD_BUCKET_URL
+          valueFrom:
+            secretKeyRef:
+              name: outline-s3
+              key: endpoint
+        - name: AWS_S3_UPLOAD_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              name: outline-s3
+              key: bucket
+        - name: AWS_S3_FORCE_PATH_STYLE
+          value: "true"
+        - name: AWS_S3_ACL
+          value: "private"
+        # OIDC (Pocket ID)
+        - name: OIDC_CLIENT_ID
+          valueFrom:
+            secretKeyRef:
+              name: outline-oidc
+              key: client-id
+        - name: OIDC_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: outline-oidc
+              key: client-secret
+        - name: OIDC_AUTH_URI
+          value: "https://auth.basicstack.de/authorize"
+        - name: OIDC_TOKEN_URI
+          value: "https://auth.basicstack.de/api/oidc/token"
+        - name: OIDC_USERINFO_URI
+          value: "https://auth.basicstack.de/api/oidc/userinfo"
+        - name: OIDC_LOGOUT_URI
+          value: "https://auth.basicstack.de/api/oidc/end-session"
+        - name: OIDC_USERNAME_CLAIM
+          value: "email"
+        - name: OIDC_DISPLAY_NAME
+          value: "Pocket ID"
+        - name: OIDC_SCOPES
+          value: "openid profile email"
+        - name: OIDC_DISABLE_REDIRECT
+          value: "false"
+        # Persist local cache directory
+        volumeMounts:
+        - name: outline-data
+          mountPath: /var/lib/outline/data
+        resources:
+          requests:
+            cpu: 200m
+            memory: 512Mi
+          limits:
+            cpu: 1000m
+            memory: 2Gi
+        livenessProbe:
+          httpGet:
+            path: /_health
+            port: 3000
+          initialDelaySeconds: 60
+          periodSeconds: 30
+          timeoutSeconds: 10
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /_health
+            port: 3000
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+      volumes:
+      - name: outline-data
+        persistentVolumeClaim:
+          claimName: outline-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: outline
+  namespace: outline
+  labels:
+    app: outline
+spec:
+  type: ClusterIP
+  selector:
+    app: outline
+  ports:
+  - name: http
+    port: 80
+    targetPort: 3000
+    protocol: TCP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: outline
+  namespace: outline
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  tls:
+  - hosts:
+    - outline.basicstack.de
+    secretName: outline-tls
+  rules:
+  - host: outline.basicstack.de
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: outline
+            port:
+              number: 80

apps/outline/postgres-credentials-sealed.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: outline-postgres-credentials
+  namespace: outline
+spec:
+  encryptedData:
+    POSTGRES_DB: AgCbJr6ng8tQwTMyNfrvubnlA5CvamfSZtzR4AwuIn/UUcK6mb68B5ucCUyA0GeeFVOGx63rlibDCKb42Da0AHrSYDTzZ1YbYAiwEzdlrmgh5tCfMiqbcUHUqSR4L7QAjkS50ASQXPXxSHuxuYa9cO7EQCasN7Fm5cmTkbcGHTKOSJ7fE7VQDBXyf6Z4Rga8HBys20xjglDFy7meBLEXDOKRUkQDC/PGgqlF98ZVtMH148ziaRxiu2tkbpsFchQ/zFc6x5rtllYGnlic8ssbgAXwUL6WZO6CcSbjhSkq4JcIOJh60GudsthTaW5HTldrD0VwaLXw6BNSD9+7+Zd1UIKSN9+imBOCmpOglLG2LFCxUD02lr3sPBdlztQwZnvxvFhfKRXstn/Qo8ujzKlRFyHdLsX0IQf7vj9kb+Giq54MAgelqivl7HMg8O73DIHe/GUmwDr3cXt1+aVoPKnQHe5wkQ4bIsTDzkNZzig1Lisl/bBxF2tf48ZXP5QZXpcL80TX6ZLxh0u1MVTpmIT6fZziw8lOJfLdE917UokNPnFVWx9X7Z+zWh07b/AMIx1Wa5JNKl1rpmJazYY42pvEGU2crR0NETNYavnM95Pl9sbRh1f/grp3v8Y4J31Oyvy2+MMChe1FdUbOnMN3Aozc3OLFuu94aiJFd1r2YU/1VCSa/0VuaDcCPwtFvKf92B/ADh0HT0Lxe7VW
+    POSTGRES_PASSWORD: AgAGj5wpTMEcDHT20f1SKTd+J0ZSC6bhfBP2WOSa3zBbxqI7Rb0OEuR+tsuQJyEbL8uKPrKKvgcjf4/GGTNChpkoe/n9Z5Vsc5cvdel59egtIca3R/2g+EhUqGhrb/R4cVMylqANcpK3O25GJNL6K9TfVU03qlORazVVp2qNr+cQ/fWUHqAJUz77ui6KPjGTjsrKH4Vnh5eP40ZoYRDXOmM7AACgMfw4dtaUID/QhslaGE8vi0o/sv/MyhC0Pic98V/cwmwPgQCYgH44v56EtOR7X0Rqz08so3GGi4hvrQCs1i40Sd2k0SayBNmvF0LGO9stlkDIpmMmnsEeCgxk0u5L9tRxsDtvGX4AAiU9FByWhVp4JRYd4QAF509p3aF+1/3kNQen+4aHefy4Xzpx1dDQOZoWArz/kXhyh8u+wkkDwsf/6VXSORdd2w4M2lDyhP61pGY8FC0QOCKWNkfOTTgFOJf8guqwSfmLNdS+lhJmtpgKpNwPnG90CgV8b6xd3uPXYa6FQPd7I9sFpN4I8ZC9SJZE4karU7Fys/SArdBB2proemw6SYepBvo41bcdftxYpL5RvfCkFjLj6CDTYbQtjPzV0YeG03l1qKqbfiKroC242mGprJweGdC4kM4L28aWaMq0oGg6og7u0A84YIukQlncIMAQ3t4L78Zp/OCOWbkPkWykjUVCkOtxMSnIjfWvFVN8vr+zN0AD/PwPu976D1RdCWXzXyZu1gWZmnBEUQweYw1mFMsaqAxQl000UWc=
+    POSTGRES_USER: AgAqZajhuIMyXzZtwm3C6KCN6njShH2tdlj4U8IsZxc3v0rHdtlCDfjk4db/fkrmRZXGzJmjizVfMwlBjVAXlO7WiGpZ0eDG/UQMIR7f56UD40Ccj2rJOExw0tB/mnfCmXVVMsUyeA3IPJLY+lUB40IE6nw7K3n8A5XpyaGTCPa8QFlCHvwpQ/seDNykA4sc+NX4So5sPUS3h/NFo9T1XhI6prvDWOd5d0YEFYBoHSgQU/QcIizT6sgudVl4cKtWtRrJdRSaq6msREB4hFfc4DnaxGsB3fLmsfUC881nkSc25ayGvnJXvWhFCnIy5v15cHFT/Yo8yHqVQdWVgJWKxb8fZfLGtXuRGXYoLHbH+VGhAcPUDSERmBoyCKG4ABGWkXG6+xLG7meN9q2/u0zbmCoYZRSxflJIUOWaZIGuvJq6KmpKsEJ5A58M90Y308oSS7yHyX3pgGJVV3zWZC6ddOoPPGbpipxAESUq7XCJnTth9SkIttlGBIgiFZ0wErL31LileoEGvhMcv5H5MSzebZbfW+uJIMRJF5QfaL8BS7UCmdI9j73VxT7pxupxKeuQCJ/zsF3B13IR/+w1hEwNW0TugqRlaqF8agU0jCLF0tlD5aCnrqHJp7MB/zD3Lb9Pk30mUEeye++kx4kqTE2Vf5sPAHbIjBlOvm/y1lKxDjASoab+I6mp4/fTBe4+S+Unqw+R7uWxBRUb
+  template:
+    metadata:
+      creationTimestamp: null
+      name: outline-postgres-credentials
+      namespace: outline

apps/outline/postgres.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: outline-postgres
+  namespace: outline
+  labels:
+    app: outline-postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: outline-postgres
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: outline-postgres
+  namespace: outline
+  labels:
+    app: outline-postgres
+spec:
+  serviceName: outline-postgres
+  replicas: 0
+  selector:
+    matchLabels:
+      app: outline-postgres
+  template:
+    metadata:
+      labels:
+        app: outline-postgres
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: postgres
+          containerPort: 5432
+          protocol: TCP
+        envFrom:
+        - secretRef:
+            name: outline-postgres-credentials
+        env:
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 1Gi
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 6
+        livenessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 5
+          failureThreshold: 6
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+          subPath: pgdata
+  volumeClaimTemplates:
+  - metadata:
+      name: postgres-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: hcloud-volumes-encrypted
+      resources:
+        requests:
+          storage: 10Gi

apps/outline/redis.yaml

@@ -0,0 +1,89 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: outline-redis
+  namespace: outline
+  labels:
+    app: outline-redis
+spec:
+  type: ClusterIP
+  selector:
+    app: outline-redis
+  ports:
+  - name: redis
+    port: 6379
+    targetPort: 6379
+    protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: outline-redis
+  namespace: outline
+  labels:
+    app: outline-redis
+spec:
+  serviceName: outline-redis
+  replicas: 0
+  selector:
+    matchLabels:
+      app: outline-redis
+  template:
+    metadata:
+      labels:
+        app: outline-redis
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: redis
+        image: redis:7-alpine
+        imagePullPolicy: IfNotPresent
+        args:
+        - --appendonly
+        - "yes"
+        - --save
+        - "60"
+        - "1000"
+        ports:
+        - name: redis
+          containerPort: 6379
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            cpu: 200m
+            memory: 256Mi
+        readinessProbe:
+          exec:
+            command: ["redis-cli", "ping"]
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        livenessProbe:
+          exec:
+            command: ["redis-cli", "ping"]
+          initialDelaySeconds: 15
+          periodSeconds: 30
+        volumeMounts:
+        - name: redis-data
+          mountPath: /data
+  volumeClaimTemplates:
+  - metadata:
+      name: redis-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: hcloud-volumes-encrypted
+      resources:
+        requests:
+          storage: 2Gi

apps/outline/s3-sealed.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: outline-s3
+  namespace: outline
+spec:
+  encryptedData:
+    access-key: AgCMbJDgvGRG0ritkltoJF/nFpb7p4JRbl57Tje3EUx/+jCZDqhvM2J6DAxLL4ELy6xINuk7bByMU4Ry5z6lmYTyZ6UvgYpJIkrg9xS7dA4WU2474oGbDWx/u2xRvI0M3t/JqlfkShaX/jxDVSibYGpGvN/+uZ/F308EvUOai/PLgEz2VpA/6y/5gtFdrDjDFoBLs5eZsrBPDAIweKQ0s+5wiPlHv+vDhIvw99efS4wiZZQXuD7j7zKJIxwGKG0k6VGIoRm207xxvHoSXmxmhZp0I6whKJ4XJqXo/S1nzQllQWOaaUyA2Y+sGXfjbcm1YdNN48/MZK6uWJNjXaIphmW6DehW8+i3aJLbv/XSmbxcY9fowRyEfFz3aqqtGW7gP4CzUc+0DwQhfE+9z/lBoWOTKsDBf8kUYnF5GPCMB1TjAAP12MWCsBmecP9kppMawH2SmRZbl+ImZgiQTXlJ79gNRoF2ri4TF2xz/tcjP3Rnt82p8yV3gj/NDsIMLUG1GPiNDMdz8dNR723V9pb+lIoNUD+pb7+Jr1N2wCq21YKNlUGPC6ZXK5flQVwlKXbxNSyu1KEyhzgEnRjKBXTQOBlRcUkkaXNbA4FreuyvfCJcRkScf3YsfzJQcxxiCxK+A5UDd9sgiVwi3rYYfQhwzPUDcvjVMTX93qZaKPfmfcxg0STV5M+A5pAkSVvs/8ORS0HoPqvXZRlH/fOT040n4z1b4ALnBg==
+    bucket: AgAiu0w47e6zbhFOX7dOGgDJVRAy5ZDfWmjgYGMibCCFm+FglEmKaj5Vi+70nZMsUigtH98WKKfEdSsJ4+mGuITNSxU2+p0uth2D2JapkEmOtzbzMzpagoXLnRV2xZW/HKyiUhRct7ygawUQ6TA3A4JBb3G/6WK+T1AxF2Z0NtDTULmM+mx8a4XNE1Anmuo6zOUQ2GIUfbBOI6RCBP693fVI8m+eB4bzkbmwrTXdT1bHPQ95wWbX28wDI3i9PQwRlM2oM5OYOTn46l3V+5mwPTUntuGuMis5P1jH4O7EkUObCbeVsC4S4orZ/sYPBs580MXFzBGeKM/Ce1fGVICC6kMWqt1SFnlYvEGRej1qj8N6uS5lmLNG6HM6Z3nHSiM/6Pi3LzfvnEvum8x6qmqRGsergIHTAcaHBOOiVDrjRKRqlCTRDmOjoTbB0vD00cWXAFGTjTXKsFtrcAmzq3EF3vhrA69Vf2FeJsF4m+ixlFdZd7WqrL5H2W7tDEHtbMhMHiii5cfHkKD9orLFpfMzirSR+sLdeRzR5t5pvGIkPRVECEHU+8w9vWSxm2aSFXOJUH4nOMM0TNzlJNlD/vsDL+Z/JQjQDzBKyBIHp1hKo45JxjwgmZKjsDy4Ts8CEwPp/zPwSFY96hqFVVs14PqjWb17y9oCpca181rQ+7bmrW0F4OCXTGg4wRBxk0ikDB7JbwCzZ1d7VqNB5lOfzeDPgZq8d98=
+    endpoint: AgB6zhkvtiqfkoSMExAy5Imd9LrBkC04F7Ifx/YSGfGtcjFSNsACNNL2vnVKUyKWvuDPa+ptdJh6HLUvmk5D9myMxlH8Bc7p69x4/wUqvwRiBgwm/4Gge3XlYa16KzMJIhkaVu6UxNSEI3c8S94Rvv7uiCk+IDkROUnutVC3dzRE6OiFz5GL56AO1f7bOj7t+0Rdpq7rSDoBq1e1Wftm6xnHa56BGXGlpaH/7UU0h0MDlnAyAbE4Xi04FLk3s7gjPPA4tetbtEBEBbKTA7GokWpt4rXrqivLr4hKd/OSklI9SByd9SxpdbO2m6QrunIqQDjjHkDrPIOAelM1MOVSTJKG1JWfXqywA2l/WeqIGEQt8FW4wWCU0E4XKixsJdFEDjtFAOyVJPYoyObaVUvkXrZdxPuc+KFtocSXR9KM0CaPQ7jteqOfYI+4VDwvE/YRVHJrolUzqBpu/pKkHnDDottfJ/O7OJCGl+jfQ1ZzP4ovOVH+GIBmo3rjVWZM+f8KlA8HA6NAdAWuSE8z3phP7n8N2ZzICr5KsW0vNSI5Ue7/mvjCbm9nUushAuzvFX7s4YH1KtwQrToMJTylfUdxyBjLzJ2NOOsg7ZiaWw9j9tjrAv1SRhPps/i7cy2Fg6Z1qeX+mvWgeIjKaQvTdYH0HREUXpzAMe1UWUkCJQtCrk8nk7gigmxF3T/fO0AIxMQGknRKYdqp3Hr06M09P1WF5QjrLU7uydWbFhA6Js5DWd3d2MurpQ==
+    region: AgBFM4EAaxptRlqN8EnsrmeX950mS10T7ZlfZmdpOsQJSR1Z0CzN5gmsJmVlznrRqw1FGGXU3feBrg/eA01IP5ykqq06iLWyjla3oLVWaYzwK/wASiY7Ycl2tkuyZwpOnQNWuxmXFX2WHl0NNKbd+mklmBxaIPRQVdYqtDAUeblyTII8oBFmv1zc4kpf4MCCHhsCG7m4VG+b+EI/DZZRXK5LFaVoOFy1pHGhN8um/OerDICHuOleOQ+hWj3w5mkigdKzFZkeIG9RQ0WnvS1muwECKvanDBaS0RGDxt+Ipm09WPk4VyGAzHyDd5wOd/uckD/yYFzAodjlZHDQipgNE1MAs43gqt6X3lKNAwqOEc6TLmQv6FUTU9+R7J+2FOgSKQ/ACTVyGHiebOqjb2L9bk1oj/j0cKARRQyRyL8B/TFHPVfyPH+kiTmRa+qlsg7FN7oTeFSOWaNZbA0jz+ONM9IzRpE33VHS/r9cCmfe2rFySNXCjHIkiKJC9qMYBqorM1kM4Ab00hlfhdO0aspTW9KC0pEauPs0n84sl/HHbOQ+sLcYW7QBvZvuRi1OyVM99XoKjI7Jnu7Aq55NX+rt9zZ+iJpnXeUm8hxCKC/CyUS5mAEmqQpJoKqLVMfhvpXMn8I5lXs3YpshHto649EPTVS6A23YBG4Q3toZ8UmbXp2JtxlL+JnfU0/mZvAPOBkfrmx5MToa
+    secret-key: AgDCSueSKDzmfI/5zhEusMX6AaIpjF526UdnWdTnmUfsX+LW+w/uQvgM0aPIoR/sMyEQoo/UxouU3V1PQIHj9Q/3acDuFBkXCFFb02dFv+gXF3SAEGnCaHviH8Bce12SfMePyOlKT2eDI8vTyehw3istEMTJHc7rMr/PNZprKG1T35wxdaklS88uLP5oAnHVKeqEvAvgNFXWrMqJ1GM1l8/rTd5GCJ8xsE1mGU/26Ac518nzo2PMARZYwZ0Rfiv2+QZswT7DDQNH6ZoY0DPzL+SW9ecMGc933L6EejNzco1esttEtRa+ZJRZ2tqUbm8LTZOFHq500VRtbEAcAtySMtjxX0VF4sSGrxGGooEkY9mqOcy2yQLZdb9UGObF8wMBXcGx67SAcXbppWeNSOvp4p7QVrzVbNXIbnV+7e8XrKBHf9inBW2j9oCgPjPcTKW7VfHOQKIYKwnHS81hgOsTQcX+BcYT9Bkea4E+/R0ovYJh68bd3Wwcm/dECnj3y3Kzgj2j4/xv7aO4cNacAt1INILYY9P0edYHbCnO4gHXt9itvKTbiOslYRuxfgff490ReXunkTl/FFawmwCZoVY7nlp59IKwTB/PahPctGmrbHstwI6UK5kLaB93Cf6bNengLCt+GYAmuuiTAOqfTLvGRQBu074mqWAO9ThYkdU0ebtOO9sY8bALk2ZlSdEN0T5Aqmp+LaSz6W91nLDF8/XaGD6zIpHHjwF6oCPg6BPn//43Rx/unbjJf82j
+  template:
+    metadata:
+      creationTimestamp: null
+      name: outline-s3
+      namespace: outline

apps/passbolt/passbolt-deployment.yaml

@@ -33,6 +33,7 @@ spec:
       containers:
         - name: passbolt
           image: passbolt/passbolt:latest-ce
+          imagePullPolicy: Always
           ports:
             - containerPort: 80
               name: http

apps/pocket-id/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: pocket-id
-        image: ghcr.io/pocket-id/pocket-id:v2.14.0
+        image: ghcr.io/pocket-id/pocket-id:v2.19.1
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 1411

apps/velero/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: velero
+description: Velero cluster-reconstruction backups for BasicStack (Kopia + Hetzner S3)
+type: application
+version: 1.0.0
+appVersion: "1.18.1"
+
+dependencies:
+  - name: velero
+    version: 12.1.0
+    repository: https://vmware-tanzu.github.io/helm-charts

apps/velero/values.yaml

@@ -0,0 +1,105 @@
+velero:
+  # Pinned chart version 12.1.0 (Velero appVersion 1.18.1).
+  # AWS plugin follows Velero minor version: v1.14.x for Velero 1.18.x.
+  image:
+    repository: velero/velero
+    tag: v1.18.1
+    pullPolicy: IfNotPresent
+
+  # Bring the AWS object-storage plugin in as an initContainer that copies
+  # the binary into /plugins so the Velero server can load it.
+  initContainers:
+    - name: velero-plugin-for-aws
+      image: velero/velero-plugin-for-aws:v1.14.2
+      imagePullPolicy: IfNotPresent
+      volumeMounts:
+        - mountPath: /target
+          name: plugins
+
+  # Kopia file-system backup requires the node-agent daemonset on every
+  # node whose PVC data we back up.
+  deployNodeAgent: true
+
+  # BackupStorageLocation credentials come from the SealedSecret
+  # apps/velero/velero-s3-credentials-sealed.yaml (key: cloud, INI format
+  # with aws_access_key_id / aws_secret_access_key for the dedicated
+  # `velero-prod` Hetzner OS credential from DEV-570).
+  credentials:
+    useSecret: true
+    existingSecret: velero-s3-credentials
+
+  # Kopia backup-repository password (velero-repo-credentials Secret,
+  # key `repository-password`) is auto-discovered by Velero — this is a
+  # hard-coded constant in the Velero server (see pkg/repository/keys),
+  # so no extra wiring is needed. SealedSecret lives at
+  # apps/velero/velero-repo-credentials-sealed.yaml.
+
+  # Turn OFF volume-snapshot CRDs — Hetzner Cloud Volumes have no CSI
+  # snapshot support, so filesystem backup via Kopia is the only path.
+  snapshotsEnabled: false
+  backupsEnabled: true
+
+  configuration:
+    # Kopia uploader (unified repo). This is the Velero server default in
+    # 1.18 but we set it explicitly so any future default change does not
+    # silently rewrite our repos to restic.
+    uploaderType: kopia
+
+    # No CSI feature flag — see snapshotsEnabled comment above.
+    features: ""
+
+    backupStorageLocation:
+      - name: default
+        provider: aws
+        bucket: basicstack-velero
+        default: true
+        credential:
+          name: velero-s3-credentials
+          key: cloud
+        config:
+          # Hetzner Object Storage in Helsinki (hel1). Region is used by
+          # the AWS SDK signer; endpoint overrides the AWS endpoint URL.
+          region: hel1
+          s3ForcePathStyle: "false"
+          s3Url: https://hel1.your-objectstorage.com
+          # publicUrl left unset — this backend is not user-facing.
+
+    # No volumeSnapshotLocation entries — snapshotsEnabled: false above
+    # already suppresses the default snapshot location CRD.
+
+  # Small resource footprint until we know real backup sizing (DEV-573).
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+  # Small memory request so node-agent fits on the tight 4 GiB CX22 workers
+  # (worker-1/2/3 sit around 95 % memory-request already). Actual peak usage
+  # during Kopia backup lands under 300 MiB per node in practice; limit is
+  # generous so a spike does not OOM-kill mid-backup.
+  nodeAgent:
+    resources:
+      requests:
+        cpu: 50m
+        memory: 64Mi
+      limits:
+        cpu: 1000m
+        memory: 1Gi
+
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
+  # Do not install the bundled `velero` CLI kubectl image — we run the
+  # CLI from cp-1.
+  kubectl:
+    image:
+      repository: bitnami/kubectl
+    resources: {}
+
+  # No schedules here — those live in DEV-573.
+  schedules: {}

apps/velero/velero-repo-credentials-sealed.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: velero-repo-credentials
+  namespace: velero
+spec:
+  encryptedData:
+    repository-password: AgCn0k/wQK76oVgVu9Eay7yMdxTKOE3RmJIojQcmPMerivAk72vw9F1jjmMDWW3f9X4HMc5MyQS9EXw9aes49047r1fZ4fA1yFgoBhuicS8ULpI4yI1ei9AVov6IQWx3FvarOlUIlQM40KC/es7mwlL/fSE2W5//u1zNp3Z+ujGdqAgBlvO0UQ/pGDNVKhWh6TBMNbzak4IDh6uNSNVAcqrzsh+2+RKPV4RRXnD1oJlTvtSri8EHQZdAAUaWxcIuyhnqeulStvViJKDC3Xw3PamrDXTYEPcAs8P90BVoDoJQwtxJ12Q+yqR1mjLvIZj81CKWXqNSUZDkAatRFjrfUYeiBAgmvnY5HL6f0eoJDVY+F7LfHslQHc4W/2z3YS19++oSKhffM4vwJxAWJqTbsKghvrA6hgFQM6btqGrhuHk3Q5NslKtlFnSgbRBst0FhBoyme0uWeBGk6szKKD8ClRuREJTWfof6pFGRM2O1IcKAhQqgJbamNHtU2yUCkLw732VBW3MJGQ+yLmlegjAmPKVLFDt4Yk5NrtTKrx9HZ3vohoPKwCz1CUizByVOXCsqEO1aogEszaQX9qCVsTCo6M8Zemw6OYCMMcbUcfPHRZmzd8o0j0Y+/6KTjP0fL5l9vR/ocJH+kXLnVsWZiFa3/TXQ0KEG1pPeCSVjQP7WHB1eLtrPH13ikvpyT8u6Ng+FPvOoTD36JrwSj8Een5HVc8oaoV50pXbbGqkepFIXZQ1te5nQL9XSP0AbRMcrqg==
+  template:
+    metadata:
+      annotations:
+        argocd.argoproj.io/compare-options: IgnoreExtraneous
+      creationTimestamp: null
+      name: velero-repo-credentials
+      namespace: velero
+    type: Opaque

apps/velero/velero-s3-credentials-sealed.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: velero-s3-credentials
+  namespace: velero
+spec:
+  encryptedData:
+    cloud: AgAo2duq77obiTyFYdVyNANrMtx2iiOT6/cRHyYezhtr6YKlV2whmQBHXuRNhnjQyrmt3Gyl9+t/UZpJbJljqzesstfwRDZ9q1usXILVZE0ZsJB1GyQ5kwN/EQUeqhAYYUSZEUMXLAEkYTD2HE+sbWkNQrGXl1UJfPeEuDkRtCtQ+WNdZ3L53DLCjXQh3JV/4jIG+5bc56Sb33l/xtSSs5YBlUI/cgCM/Di0Xj5pyujxaqiU+vSCrpxiltkK7QDgcti1MbbL5YqeDox5/DwmJZvL1Of2PzODRpgTrDXbylWH8AaNRhzYcSJgv0ZBUwiqsLjFRdU9Oroevg2Uf5dO8FLUbAvx8ER97Afkt6fX7wTgcn3uj9/Ye3+BVOPRIj2l6VaJ3ul15Z+i+UCAAHe3jeujQhVHhZMRXGp5CnjkjJbiFIIL83cq6SfKq21pTEUkk5Ty5V844zDMymHHKvB52/LXS1i0LrELT/QzEKr0y+B3wp/JRFtvsliN+Bsweg4RroBsnVL4d0Aqp0zr3dHaUmTVDTycFGb/GFRTdsC+0JAJlplonJXcU/9h8mSRaBekusFTwMs25C6JHXPcXbI9qf8+qw6n/4vC86FkpyOWKNS2fs5r/Aut+8PC8Z2RcNUjnNUtt/i9FSkoQtIOwHRR0J8Zy+07TyysR0Zz2tB6C895SmHPI5isxQGZoAJHp+ypJwP/nR+RTdB648IfLLZyRINLEt58Nk4NM7B9tXOvk0JBXfB/knXy0wO54PvrnmLfxcE3mIy2/KKM5HK/b+QbSy97VqpXZekbcKJl18foUMsP7YitmP5pWfxBkg9m+3ZDemhCL48mLjr05KVrJ04r8UTMAlWwUw==
+  template:
+    metadata:
+      annotations:
+        argocd.argoproj.io/compare-options: IgnoreExtraneous
+      creationTimestamp: null
+      name: velero-s3-credentials
+      namespace: velero
+    type: Opaque

apps/vikunja/README.md

@@ -0,0 +1,39 @@
+# Vikunja (vikunja.basicstack.de)
+
+Self-hosted task manager. Unified `vikunja/vikunja` container (API + frontend on port 3456). Backed by a namespace-local PostgreSQL. OIDC via Pocket ID (`auth.basicstack.de`); local login disabled.
+
+## Layout
+
+| File | Purpose |
+|---|---|
+| `vikunja-db-pvc.yaml` | 10 Gi encrypted PVC for Postgres data |
+| `vikunja-files-pvc.yaml` | 10 Gi encrypted PVC for user-uploaded attachments (`/app/vikunja/files`) |
+| `vikunja-postgres-deployment.yaml` | Postgres 16, pinned to `csi.hetzner.cloud/location=fsn1` |
+| `vikunja-postgres-service.yaml` | ClusterIP `vikunja-postgres:5432` |
+| `vikunja-deployment.yaml` | Vikunja app, pinned to `fsn1`, waits on Postgres |
+| `vikunja-service.yaml` | ClusterIP `vikunja:80` → pod `3456` |
+| `vikunja-ingress.yaml` | Traefik + `letsencrypt-prod` cert |
+| `vikunja-db-credentials-sealed.yaml` | Postgres user password |
+| `vikunja-oidc-secret-sealed.yaml` | Pocket ID client id + secret |
+| `vikunja-app-secrets-sealed.yaml` | Vikunja JWT secret |
+
+## Node placement
+
+Both the app and Postgres pin to Hetzner location `fsn1` because their PVCs live on `hcloud-volumes-encrypted` (datacenter-local).
+
+## OIDC
+
+- Pocket ID client `Vikunja` (id `cdfff4ee-28ba-4aab-9f47-7e41b4b6b847`), callback `https://vikunja.basicstack.de/auth/openid/basicstack`.
+- Pocket ID groups `vikunja_admins` and `vikunja_members`.
+- Vikunja does not map OIDC groups to the admin flag. Promote the first user post-login: `kubectl -n vikunja exec deploy/vikunja -- /app/vikunja/vikunja user set-admin --admin <username>`.
+- **PKCE must be OFF on the Pocket ID client.** Vikunja 2.x's OAuth redirect (`frontend/src/helpers/redirectToProvider.ts`) does not send `code_challenge`, and the Go backend does not implement PKCE either. With `pkceEnabled: true` on Pocket ID the flow fails at `/authorize` with `invalid_request … 'code_challenge' parameter is missing`. Client is confidential (`isPublic: false`), so `client_secret` still authenticates the token exchange.
+
+## SMTP
+
+Reminder emails go through Stalwart on `stalwart-smtp.stalwart.svc.cluster.local:25` using unauthenticated in-cluster relay. Stalwart's port-25 listener trusts the pod-network CIDR (`10.244.0.0/16`), so no SMTP AUTH is required and no sealed secret is needed. Envelope-from is `vikunja@basicstack.de` (the mailbox already exists in Stalwart as principal `g`).
+
+If a future policy tightens the port-25 CIDR trust, switch to submission on `stalwart-smtp:587` with STARTTLS and a `vikunja-mailer` sealed secret (`VIKUNJA_MAILER_USERNAME`/`VIKUNJA_MAILER_PASSWORD` from a secretKeyRef, `VIKUNJA_MAILER_AUTHTYPE=plain`); creating that secret requires a working Stalwart admin session to (re)set the vikunja principal's password.
+
+## Change control
+
+Manifest changes flow through Argo CD (Application: `apps/app-vikunja.yaml`, auto-sync + prune + selfHeal). Sealed secrets are encrypted against the cluster-active sealed-secrets key.

apps/vikunja/vikunja-app-secrets-sealed.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: vikunja-app-secrets
+  namespace: vikunja
+spec:
+  encryptedData:
+    jwt-secret: AgA2gSMP5tLy+2BeEe/vO3DHi7v4WO2HRZc2kTD/+BRII/CxCd3SIJYOh5O7OPNOq5Vp+F6oCi6GUg7ljyXuHU1IlHSSYWfjdGKS/mIvlKf2XRASOS1CefNSDKjU+x2FB3LPJG10v/0OG7bLKCj7QGq5y+QNc8dQrV7kqwAonG18iZzl5rcL4vN38B2V6kaO+s/tcz0uwmVOlUqblmv6WwrkMm5bVt9RQxcO5UjmDuf2KlnHBgvhRUhcBCbmrnI7J+FkVtq+2hiWwyz3hXAQAyy6ULMltYZvSy4ADseTV3yBdArUvY9e13kwfVZBjLe5EBzn0SKoQyY0XZQUYDmfQU+U5zFxsZLbYeekf7FUN8zHJ+mgvrT35PGrVazoz2rKI+9BrA7sstYJZ0ZUlj+nKnrbHYyaUhrlu3DYFvTHCOz8ylRxIsM5Ux819+RISpBhKkg6+mIhH2L26larI7UEY576c83TMhmCEOeX0es+MoomsAqeQi8vZ+OT5qz9dFZpzshXmwRYy1Ezm9d992iBF9EskB9IuyaozpA+GU9WwKAOj5GF3izj89nel9QPPQVBGHYvxXXfaXi3yQIpqD54nVPKC1oEhPNeCfVp3YcGSd/HOsl3tT/8G7ipD40jGVk5tgWxpymbvNZmcpm1CIb4SPYHOavJtqBcPY1F9k8a5yYl9sMqd64qzDmTlOQhRVWjD0mqCf2W/W3OvHo4dTu6Fy95jHrzoHsmmfNQQDVytCJK/3SmYUvh51nD8/CyiJxRAq3zPViFHKmRS0E5oCR0+zm3GlZTGwUmu1ROSIlsDgik/khmrZvYFLtF
+  template:
+    metadata:
+      creationTimestamp: null
+      name: vikunja-app-secrets
+      namespace: vikunja

apps/vikunja/vikunja-db-credentials-sealed.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: vikunja-db-credentials
+  namespace: vikunja
+spec:
+  encryptedData:
+    postgres-password: AgB+P7iBg+tPIDKMGmiPh9HREVb7mKCf1GvQfBLcF61QuWHt0/S1YBYCd4fF3PSA9ok916m6VeOoIQKB0p57XDSEeNewYNsXQgudS0k0JHNdRZER8JhbXpK53E7EY0xw4TD8Tc6WFqyzCFwsL1WpYvvDH3p25W8f3JJfB39D6Up9j5xT5Hufi0G5BJ+tbAujNJooa0SWZbtQPxMWG96fvjmTV5FKxzvwg1JOL4gq0JHBfzo9sKIbQdGpGJjX4JkPLCxA0jU6mcJKEFn80zkDbQBL3g5dXZaYmIAEogZkvytltR4ebI0zyDrSTPqXhJF5bFcRpM43VXz6ZcjgdtG5VPelvAuMqfJ1U7HdYhhJqTt0lvwUi03BjulIJJ52gUjlg2yQioOwfON/y3gRM1TgMHMFCyw32S7vCfR00MKVh7xrb4YceRUTbeHkTd+4TMcdpI11YHeah2scu82tzyD/cGPtpFv2LYyH7JrBcIXXhfIDccBRX4bb2se9VyhpmvEP5Q9SQjazo/BxdnJUYF6kzpJA9oOIjFMXcSDxZh/nwDkA90w4a9iyT9l0+7p3z/RjbXOMKybuQlXpfPrhtCRS6PM3Am5sgnIi/APRaujDa/2a3kCe7K0zZl4Wk3wlAc4mMjnoWs9w//HfMpQByM6GWeHWeoQ/Bj3xCVtACb6SNqTxIs4OP9hAVBnHqouH0O/zjqZNuBa5v7GDwmW6JU1cA+Y/UI4Z/ycu2X6AXBHp5CROheV9PdINTrgS
+  template:
+    metadata:
+      creationTimestamp: null
+      name: vikunja-db-credentials
+      namespace: vikunja

apps/vikunja/vikunja-db-pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: vikunja-db
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: database
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 10Gi

apps/vikunja/vikunja-deployment.yaml

@@ -0,0 +1,144 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vikunja
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: app
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: vikunja
+      component: app
+  template:
+    metadata:
+      labels:
+        app: vikunja
+        component: app
+    spec:
+      hostname: vikunja
+      subdomain: basicstack
+      setHostnameAsFQDN: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+        - sh
+        - -c
+        - |
+          until pg_isready -h vikunja-postgres -U vikunja; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+      securityContext:
+        fsGroup: 1000
+      containers:
+      - name: vikunja
+        image: vikunja/vikunja:2.6.0
+        ports:
+        - name: http
+          containerPort: 3456
+        env:
+        - name: VIKUNJA_SERVICE_PUBLICURL
+          value: https://vikunja.basicstack.de/
+        - name: VIKUNJA_SERVICE_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: vikunja-app-secrets
+              key: jwt-secret
+        - name: VIKUNJA_SERVICE_ENABLEREGISTRATION
+          value: "false"
+        - name: VIKUNJA_SERVICE_TIMEZONE
+          value: Europe/Berlin
+        - name: VIKUNJA_DATABASE_TYPE
+          value: postgres
+        - name: VIKUNJA_DATABASE_HOST
+          value: vikunja-postgres
+        - name: VIKUNJA_DATABASE_PORT
+          value: "5432"
+        - name: VIKUNJA_DATABASE_DATABASE
+          value: vikunja
+        - name: VIKUNJA_DATABASE_USER
+          value: vikunja
+        - name: VIKUNJA_DATABASE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: vikunja-db-credentials
+              key: postgres-password
+        - name: VIKUNJA_DATABASE_SSLMODE
+          value: disable
+        - name: VIKUNJA_FILES_BASEPATH
+          value: /app/vikunja/files
+        - name: VIKUNJA_AUTH_LOCAL_ENABLED
+          value: "false"
+        - name: VIKUNJA_AUTH_OPENID_ENABLED
+          value: "true"
+        - name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_NAME
+          value: BasicStack SSO
+        - name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_AUTHURL
+          value: https://auth.basicstack.de
+        - name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_SCOPE
+          value: openid profile email vikunja_scope
+        - name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_CLIENTID
+          valueFrom:
+            secretKeyRef:
+              name: vikunja-oidc-secret
+              key: client-id
+        - name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_CLIENTSECRET
+          valueFrom:
+            secretKeyRef:
+              name: vikunja-oidc-secret
+              key: client-secret
+        - name: VIKUNJA_MAILER_ENABLED
+          value: "true"
+        - name: VIKUNJA_MAILER_HOST
+          value: stalwart-smtp.stalwart.svc.cluster.local
+        - name: VIKUNJA_MAILER_PORT
+          value: "25"
+        - name: VIKUNJA_MAILER_FORCESSL
+          value: "false"
+        - name: VIKUNJA_MAILER_SKIPTLSVERIFY
+          value: "true"
+        - name: VIKUNJA_MAILER_FROMEMAIL
+          value: vikunja@basicstack.de
+        resources:
+          requests:
+            cpu: 100m
+            memory: 128Mi
+          limits:
+            cpu: "1"
+            memory: 512Mi
+        livenessProbe:
+          httpGet:
+            path: /api/v1/info
+            port: 3456
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /api/v1/info
+            port: 3456
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 3
+        volumeMounts:
+        - name: files
+          mountPath: /app/vikunja/files
+      volumes:
+      - name: files
+        persistentVolumeClaim:
+          claimName: vikunja-files

apps/vikunja/vikunja-files-pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: vikunja-files
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: files
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: hcloud-volumes-encrypted
+  resources:
+    requests:
+      storage: 10Gi

apps/vikunja/vikunja-ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vikunja
+  namespace: vikunja
+  labels:
+    app: vikunja
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  tls:
+  - hosts:
+    - vikunja.basicstack.de
+    secretName: vikunja-tls
+  rules:
+  - host: vikunja.basicstack.de
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: vikunja
+            port:
+              number: 80

apps/vikunja/vikunja-oidc-secret-sealed.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: vikunja-oidc-secret
+  namespace: vikunja
+spec:
+  encryptedData:
+    client-id: AgBdzdW0SxhmOyhOpC/jkBJ4UtD0qrRrytVp7fInK3bxiiVBHsIU8r3Hoe2iYZU75IyITogmcGImKH9pivxPMhj5jBqFeCExQadt/pqXHL9TmxmNpIx7FmnlrxaZ+zeqeKRMDhYJnxSeNiuRDVZEEnsBhmUc0gvz8Pq/x82ExY++MQjQQu2xB6XQ7DTyzVDx32yZAvBU6aRFhTtMTqPAvjsMHp6pKmciauS7LmffCVIJvnJUaUKIELHPDyWsMcQvWpPQrPKGkvmGruhXbct6LxPFKdkbsQMUWvj7emSemmbJk0VmGPtVxI7cTmHIJ6l4kr8aJfJ9fv2gMzbgprYVZHJ17qhPvvUv5iTUcYu5FiMiy3QtNHYMP+VyhWvj9aUH8ypFYwf0S3/fRobEtyBxnBLCmnIt4lvS0ABQ49YwdtWnBUXT1jP4Xzk3gWM8fe1LsOuUZXMgFoGMXtxuodhZFhTELjdKPaYTY/DDU6iRMpk/nUsQy1IGls1CySEB+taiQDRNMZmQAFHdoFOxX8sZyPKdYL3wgKDweqcLoI0UKjapkHdpcoWccqg0ky/f5VWWlD7kHWLXYeosH6ucLAKR/mRHoyOfIy1dEpYYevfv5UM4Lewnr8epLse/x3ymsBNaSIhxt8PxnFYe/Gi1rO8MAgTPNcw2pr2z5jiW8CXUPLdXezleoctl4VGg97pkYVibn0BJ+VrZOTgO/p/df5D0v7h7DSsvdbpA7XxkaTnL62JRhC3RDXU=
+    client-secret: AgA3ZQ5Da1D2tcNBA9RVf7YYNLBvBjKf+OArLLQSKKUDGAULoC0BQUn+T7rctuOANFeKLOYE2OjJQhURoi69xFyQ3d868s4a2Ipo3QwUV2vtoVJnz7l2fGXCzAOTcJhKonHdJooklfeK+gwi80sRXV1IxxDu778yKSWnFUd61vUPbhKgVgtXDM4IItNtctGc4Drz6dvrOHkyM1m4kxGbXUclpN+HztufQrO7W+ba23bZRCejaYz3sQ6IHmM50HXzRPD5Rdb2ALQFm9V7zdSuUQ8njNrXHfgqMSiDWoCzdpTKG50mBPqmait/yyNVnOAd8J906RZk27LnAY9zV1WiMUSImNS6IzuLV0DZTn6I4cWgnemIwNAdNR4/x7TwWNdVS3A3jH8q6S1IP5g65P40SmP27PfFx+L7jx72Om8HXBY8TpMtWwGFcjyEGy61tKuimXd65+o0UkXEmQfJKfnugnSM3yCufEat2PIE2+Arb7+pRh6VtzaBowLdo32OcqS5GMmFLhisv5cR016IIIbSnaLkIGpiZEHSvpcLoGOJm8H+1P6MPsHWfjWtKZy6y8Ri/B9zNHlUx506Kij5YOR+tH3B3ip4PIa2LRlRCAbzBb/WCwzP72WzK8PU/sSujTa+JgxBoNGt3fg2iqBmoaU3UrhVoTTcXhEn7pnvIlk60jZKTscC+thS++VHMnf7LjLAA2ksu9G6qKpWy5DAERYi/MC6YdYz0eoIBSa/NEHLi94zFA==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: vikunja-oidc-secret
+      namespace: vikunja

apps/vikunja/vikunja-postgres-deployment.yaml

@@ -0,0 +1,79 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vikunja-postgres
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: database
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: vikunja
+      component: database
+  template:
+    metadata:
+      labels:
+        app: vikunja
+        component: database
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: csi.hetzner.cloud/location
+                operator: In
+                values:
+                - fsn1
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        ports:
+        - name: postgres
+          containerPort: 5432
+        env:
+        - name: POSTGRES_DB
+          value: vikunja
+        - name: POSTGRES_USER
+          value: vikunja
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: vikunja-db-credentials
+              key: postgres-password
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: "1"
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - vikunja
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - pg_isready
+            - -U
+            - vikunja
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: vikunja-db

apps/vikunja/vikunja-postgres-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vikunja-postgres
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: database
+spec:
+  type: ClusterIP
+  selector:
+    app: vikunja
+    component: database
+  ports:
+  - name: postgres
+    port: 5432
+    targetPort: 5432

apps/vikunja/vikunja-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vikunja
+  namespace: vikunja
+  labels:
+    app: vikunja
+    component: app
+spec:
+  type: ClusterIP
+  selector:
+    app: vikunja
+    component: app
+  ports:
+  - name: http
+    port: 80
+    targetPort: 3456

infrastructure/K3S_OPERATIONS.md

@@ -3,9 +3,9 @@
 ## Current Cluster Status
 
 - **Version:** v1.36.2+k3s1
-- **Last Updated:** 2026-07-06
-- **Control Plane:** k3s-cp-1 (178.105.17.239 / 10.42.1.1)
-- **Workers:** k3s-worker-1, k3s-worker-2, k3s-worker-3, k3s-update-runner
+- **Last Updated:** 2026-08-30
+- **Control Plane:** k3s-cp-1 (178.105.17.239 / 10.42.1.1), k3s-cp-2 (188.245.85.199 / 10.42.1.8), k3s-cp-3 (49.13.92.162 / 10.42.1.9) — etcd HA
+- **Workers:** k3s-worker-1, k3s-worker-2, k3s-worker-3, k3s-worker-4 (nbg1), k3s-worker-5, k3s-worker-6, k3s-update-runner
 - **Update Method:** k3s system-upgrade-controller (automated)
 
 ## Quick Reference
@@ -79,12 +79,15 @@ The script will:
    ```bash
    ssh root@<new-node-ip> "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v${CLUSTER_VERSION}' sh -s - agent \
      --server https://10.42.1.1:6443 \
-     --token '${K3S_TOKEN}'"
+     --token '${K3S_TOKEN}' \
+     --node-ip <new-node-private-ip>"
    ```
 
+   **Critical:** The `--node-ip=<private-ip>` flag is required. Without it, k3s defaults to the public IP as the node's InternalIP, which breaks LoadBalancer service routing (see DEV-224). Verify with `kubectl get nodes -o wide` — the INTERNAL-IP column MUST show the private (10.42.1.x) IP.
+
 5. **Verify node joined:**
    ```bash
-   kubectl get nodes
+   kubectl get nodes -o wide  # INTERNAL-IP must be private (10.42.1.x)
    kubectl get node <node-name> -o yaml | grep flannel
    ```
 
@@ -410,13 +413,20 @@ failed to register flannel network: failed to acquire lease: subnet "X.X.X.X/16"
 
 ### Node CIDRs
 
-| Node | Pod CIDR | Private IP | Public IP |
-|------|----------|-----------|-----------|
-| k3s-cp-1 | 10.244.0.0/24 | 10.42.1.1 | 178.105.17.239 |
-| k3s-worker-1 | 10.244.1.0/24 | 10.42.1.2 | - |
-| k3s-worker-2 | 10.244.2.0/24 | 10.42.1.3 | - |
-| k3s-update-runner | 10.244.3.0/24 | 167.233.79.65 | 167.233.79.65 |
-| k3s-worker-3 | 10.244.5.0/24 | 10.42.1.5 | 167.233.121.121 |
+| Node | Private IP | Public IP | Datacenter | Role |
+|------|-----------|-----------|------------|------|
+| k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | control-plane, etcd |
+| k3s-cp-2 | 10.42.1.8 | 188.245.85.199 | fsn1 | control-plane, etcd |
+| k3s-cp-3 | 10.42.1.9 | 49.13.92.162 | fsn1 | control-plane, etcd |
+| k3s-worker-1 | 10.42.1.2 | 178.105.216.48 | fsn1 | worker |
+| k3s-worker-2 | 10.42.1.3 | 49.13.134.255 | fsn1 | worker |
+| k3s-worker-3 | 10.42.1.5 | 167.233.121.121 | fsn1 | worker |
+| k3s-worker-4 | 10.42.1.6 | 128.140.3.80 | nbg1 | worker (stateless workloads only) |
+| k3s-worker-5 | 10.42.1.7 | 167.233.192.86 | fsn1 | worker (DEV-410) |
+| k3s-worker-6 | 10.42.1.10 | 49.12.223.133 | fsn1 | worker (cx33 4vCPU/8GB) |
+| k3s-update-runner | 167.233.79.65 | 167.233.79.65 | fsn1 | update-runner |
+
+**Datacenter note:** Hetzner Cloud Volumes are datacenter-scoped. Pods with PersistentVolumes only schedule on nodes in the same datacenter as the PV. Provision stateful-workload workers in **fsn1** to match the existing PV pool. k3s-worker-4 remains in nbg1 and is only usable for stateless workloads.
 
 ### Critical Firewall Rule
 
@@ -428,6 +438,7 @@ failed to register flannel network: failed to acquire lease: subnet "X.X.X.X/16"
 
 | Date | Version | Changes | By |
 |------|---------|---------|-----|
+| 2026-08-30 | v1.36.2+k3s1 | DEV-410: Added k3s-worker-5 (fsn1) to unblock Stalwart scheduling; k3s-worker-4 (nbg1) retained for stateless workloads; provision script now sets --node-ip; refreshed node table with cp-2/cp-3 and worker-4/5/6 | CTO Agent |
 | 2026-07-06 | v1.36.2+k3s1 | Fixed DNS service CIDR mismatch (kubelet cluster-dns configuration), corrected Service CIDR documentation to 10.96.0.0/16, added DNS configuration guide | CTO Agent |
 | 2026-07-06 | v1.36.2+k3s1 | Fixed API server instability by adding explicit --cluster-cidr=10.244.0.0/16 to k3s service, updated all nodes to v1.36.2+k3s1 | CTO Agent |
 | 2026-07-06 | v1.36.2+k3s1 | Initial documentation, upgraded from v1.35.5, installed system-upgrade-controller | CTO Agent |

infrastructure/OS_UPDATE_PROCEDURE.md

@@ -1,6 +1,6 @@
 # Weekly Rolling Ubuntu OS Update Procedure
 
-**Purpose:** Keep every k3s node's Ubuntu OS patched (kernel, security, package updates) with a **rolling, zero-downtime** update — one node at a time, drain → update → reboot → verify → uncordon → cluster health check → next node.
+**Purpose:** Keep every k3s node's Ubuntu OS patched (kernel, security, package updates) with a **rolling, zero-downtime** update — one node at a time, drain → update → reboot → verify → uncordon → cluster health check → next node. Since [DEV-553](/DEV/issues/DEV-553) (2026-08-30) this covers **workers AND control-plane nodes** in a single cycle; workers run first, then CPs one at a time with etcd leader last.
 
 **Audience:** Agents (currently CTO, optionally a dedicated ClusterOps agent). Also usable manually by an operator.
 
@@ -22,9 +22,9 @@
 
 | Role | Node | Private IP | Public IP | Datacenter | Notes |
 |------|------|-----------|-----------|------------|-------|
-| control-plane | k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | EXCLUDED from `os-update.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
-| control-plane | k3s-cp-2 | — | 188.245.85.199 | fsn1 | EXCLUDED — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
-| control-plane | k3s-cp-3 | — | 49.13.92.162 | fsn1 | EXCLUDED — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
+| control-plane | k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
+| control-plane | k3s-cp-2 | — | 188.245.85.199 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
+| control-plane | k3s-cp-3 | — | 49.13.92.162 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
 | worker | k3s-worker-1 | 10.42.1.2 | (via CP) | fsn1 | |
 | worker | k3s-worker-2 | 10.42.1.3 | (via CP) | fsn1 | |
 | worker | k3s-worker-3 | 10.42.1.5 | 167.233.121.121 | fsn1 | |
@@ -38,11 +38,19 @@ Always re-derive the live list before running — nodes may have been added/remo
 ssh root@178.105.17.239 'kubectl get nodes -o wide'
 ```
 
-**Order rule:** update workers only. Within workers, update in this order to protect stateful workloads:
+**Order rule:** update workers first, then control-plane nodes. Within each group, update in this order to protect stateful workloads:
+
+Workers:
 1. runner + workers that host no PVs (safest — lowest disruption)
 2. remaining workers
 3. **Stalwart-hosting fsn1 workers last among workers** — Stalwart has hard fsn1 affinity, so draining a fsn1 worker while another fsn1 worker is also unavailable can leave Stalwart Pending. Never have two fsn1 workers cordoned/down at the same time.
-4. **ALL control-plane nodes excluded** — `os-update.sh` selects nodes without the `node-role.kubernetes.io/control-plane` label, so `k3s-cp-1`, `k3s-cp-2`, and `k3s-cp-3` are all skipped automatically. CPs have their own procedure ([`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) — HA-aware, one CP at a time, leader last).
+
+Control planes (integrated with the weekly cycle since [DEV-553](/DEV/issues/DEV-553); previously excluded and run manually via `CP_UPDATE_PROCEDURE.md`):
+1. `os-update.sh` queries the current etcd leader before entering the CP phase.
+2. Followers are updated first (in any order), leader last. Rationale: draining a follower is a no-op for raft leadership; draining the leader forces a re-election. Doing followers first minimises leader flapping.
+3. Exactly one CP is drained or rebooting at a time — two CPs down means etcd quorum loss and a write outage across the cluster.
+4. Between CPs `os-update.sh` re-runs `cluster-health.sh` and re-checks `etcdctl endpoint status --cluster`; any yellow signal halts the cycle at that CP. Settle wait between CPs is `POST_CP_WAIT_SECONDS` (default 180 s).
+5. Each CP goes through `update-cp-node.sh <cp> --run` with `ASSUME_YES=1`: Phase A (add-swap, idempotent) → B preflight → C drain → D apt → E reboot with `/livez` polling and etcd rejoin check → F uncordon + cluster health.
 
 **Concurrency:** exactly one node at a time. Never in parallel.
 
@@ -76,6 +84,12 @@ Environment variables the scripts expect:
 - Drain timeout: `DRAIN_TIMEOUT_SECONDS` — default `600`
 - Reboot wait: `REBOOT_MAX_WAIT_SECONDS` — default `600`
 - Post-uncordon settle: `POST_UNCORDON_WAIT_SECONDS` — default `180`
+- Post-uncordon bounded health retry (DEV-595):
+  - `HEALTH_MAX_RETRIES` — default `10` (extra attempts after the first)
+  - `HEALTH_RETRY_INTERVAL_SECONDS` — default `30`
+  - Short-circuits immediately on persistent pod errors
+    (`CrashLoopBackOff`, `ImagePullBackOff`, `ErrImagePull`,
+    `CreateContainerConfigError`, `InvalidImageName`).
 
 ---
 
@@ -249,15 +263,19 @@ Only proceed to the next node when ALL of the above are green. If not:
 - Give it another 3 minutes and re-check (workloads with large images may still be pulling).
 - If still not green: STOP the cycle. Uncordon everything, leave the cluster in a stable state, and open a follow-up issue with the failing workloads. Do NOT proceed to update more nodes.
 
-### 8. Control plane special handling
+### 8. Control plane handling
 
-All three CPs (`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`) are **NOT** updated by the weekly `os-update.sh` cycle — they have their own dedicated procedure and script: [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) and `scripts/os-update/update-cp-node.sh`. Reasons:
+Since [DEV-553](/DEV/issues/DEV-553) (2026-08-30) the weekly `os-update.sh` cycle **includes** the three CPs (`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`) after all workers finish. The CP steps are delegated to `scripts/os-update/update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) for the full design. Key properties:
 
-- CPs are `NoSchedule`-tainted and host no StatefulSet workloads today, but they are the etcd quorum. Two of three CPs cordoned/rebooting at once = write outage. One CP at a time, leader last.
-- All three CPs currently have zero swap. The CP procedure adds a durable 4 GiB swapfile per CP before draining.
-- The CP procedure polls the target's `/livez` from the operator machine and cross-checks `etcdctl endpoint status --cluster` from a peer CP to confirm the just-rebooted member rejoined.
+- CPs are `NoSchedule`-tainted and host no StatefulSet workloads today, but they are the etcd quorum. Two of three CPs cordoned/rebooting at once = write outage. `os-update.sh` walks CPs sequentially with a health gate between each.
+- `os-update.sh` queries `etcdctl endpoint status --cluster` before the CP phase to identify the current leader, then updates followers first and leader last.
+- `update-cp-node.sh` polls the target's `/livez` from the operator machine and cross-checks `etcdctl endpoint status --cluster` from a peer CP to confirm the just-rebooted member rejoined etcd as `started`.
+- Phase A (`--add-swap`) is idempotent: creates `/swapfile` (4 GiB default) with `vm.swappiness=10` on any CP without swap, and installs `etcd-client` if `etcdctl` is missing. Re-runs are no-ops when swap is already active.
 
-`os-update.sh` excludes any node carrying the `node-role.kubernetes.io/control-plane` label (see `--dry-run` output) and prints a pointer to the CP procedure.
+Skip flags on `os-update.sh`:
+- `--skip-cps` — workers only (legacy behavior for narrow patch cycles).
+- `--only-cps` — skip workers and only walk the CP phase.
+- `--only <node>` — single node, routed automatically to worker or CP.
 
 ---
 
@@ -318,8 +336,8 @@ If a node was skipped or errored → mark `blocked` with the unblock action, or
 
 ## Automation entry points
 
-- `infrastructure/scripts/os-update/os-update.sh` — full cycle runner (preflight → per-node loop → finalization). Idempotent, resumable via `--start-from <node>`. Use `--dry-run` to print the plan without touching anything. Explicitly skips `k3s-cp-1`.
-- `infrastructure/scripts/os-update/update-node.sh <node>` — single-node update (all 7 per-node steps). Callable standalone for retry. Not for `k3s-cp-1`.
+- `infrastructure/scripts/os-update/os-update.sh` — full cycle runner (preflight → worker loop → CP loop → finalization). Idempotent, resumable via `--start-from <node>` (works for workers or CPs). Use `--dry-run` to print the plan without touching anything. Use `--skip-cps` / `--only-cps` to run only one half of the cycle.
+- `infrastructure/scripts/os-update/update-node.sh <node>` — single-node update for **workers** (all 7 per-node steps). Callable standalone for retry. Never call this for a CP.
 - `infrastructure/scripts/os-update/update-cp-node.sh <node>` — HA-aware CP update flow (add swap, preflight, drain, apt, reboot, finalize). Accepts `k3s-cp-1`, `k3s-cp-2`, or `k3s-cp-3`. See [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md).
 - `infrastructure/scripts/os-update/update-cp-1.sh` — thin wrapper (calls `update-cp-node.sh k3s-cp-1 "$@"`). Kept for backwards compat.
 - `infrastructure/scripts/os-update/cluster-health.sh` — the preflight/post-node health check as a standalone command; exits non-zero on any failure.
@@ -341,3 +359,4 @@ A Paperclip routine (see `infrastructure/OS_UPDATE_ROUTINE.md`) fires this proce
 | 2026-08-09 | Initial procedure + automation scripts | CTO agent (DEV-462) |
 | 2026-08-16 | Added kine thundering-herd guardrails after DEV-495 incident (worker-3 drain caused CP kine cascade + near-OOM on cp-1). Four new rules: pre-plan StatefulSet drain order, swap on cp-1, halt on kubectl >5 s, cp-1 update as separate design task. | CTO agent (DEV-478/DEV-495) |
 | 2026-08-23 | Downgraded the kine cascade section for the HA-etcd era (DEV-510 migrated cp-1 from kine SQLite to embedded etcd, cp-2/cp-3 joined as full members). Pointed §8 and the "Automation entry points" list at the new HA-aware `CP_UPDATE_PROCEDURE.md` / `update-cp-node.sh`. Old `CP1_UPDATE_PROCEDURE.md` and `update-cp-1.sh` are now redirect stubs. | CTO agent (DEV-515) |
+| 2026-08-30 | Integrated the three CPs into the weekly `os-update.sh` cycle (leader-last ordering, health gate between CPs, delegates each CP to `update-cp-node.sh --run` with `ASSUME_YES=1`). Added `--skip-cps` / `--only-cps` flags. `--only <node>` and `--start-from <node>` now route between worker/CP lists automatically. | CTO agent (DEV-553) |

infrastructure/scripts/os-update/README.md

@@ -23,10 +23,11 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure.
 ## Guardrails the scripts enforce
 
 - Preflight cluster health failure → refuse to start.
+- SSH-drift preflight (DEV-587): before draining anything, verify the operator can SSH to every planned node and that cp-1's pubkey is present in each target's `/root/.ssh/authorized_keys`. Missing pubkeys are auto-appended via the operator's own SSH; nodes that reject the operator entirely fail the whole cycle with the exact repair command. Preflight report is written to `/tmp/os-update-<UTC-ts>/ssh-preflight.txt`. Escape hatch: `--skip-ssh-preflight` for bootstrap.
 - Drain with a PDB conflict → uncordon and mark the node `SKIPPED_DRAIN_FAILED`, never `--force`.
 - Node reboot fails to come back within timeout → hard stop, escalate. Do NOT rebuild the node or touch k3s config.
 - kubelet doesn't return `Ready` → hard stop, escalate. Do NOT touch k3s config.
-- Post-node cluster health check red → hard stop, do not proceed to the next node.
+- Post-node cluster health check red → first retry up to `HEALTH_MAX_RETRIES` (default 10) more times at `HEALTH_RETRY_INTERVAL_SECONDS` (default 30s) to let transient reinit windows settle (DEV-595). Short-circuit halt immediately on any `CrashLoopBackOff`/`ImagePullBackOff`/`ErrImagePull`/`CreateContainerConfigError`/`InvalidImageName`. Still red after all attempts → hard stop, do not proceed to the next node.
 - Absolute rule: **no k3s config, no manifests, no PVs, no service files touched** — apt/dpkg only.
 
 ## Typical invocations

infrastructure/scripts/os-update/os-update.sh

@@ -4,32 +4,54 @@
 # Behavior:
 #   1. preflight cluster health (fail-closed)
 #   2. take an etcd snapshot
-#   3. compute node order (workers first, control plane last;
-#      Stalwart-hosting fsn1 workers moved to end of workers group)
-#   4. call update-node.sh for each node, halting on any failure
+#   3. compute node order:
+#        workers (non-Stalwart first, Stalwart-hosting fsn1 workers last)
+#        then control-plane nodes (etcd followers first, leader last).
+#   3b. SSH-key drift preflight: verify operator can reach every planned node
+#        and that cp-1's pubkey is present in root's authorized_keys (auto-heal
+#        by appending if missing; hard-fail if unreachable). See DEV-587.
+#   4. call update-node.sh for each worker and update-cp-node.sh for each CP,
+#      halting on any failure
 #   5. finalization: health snapshot + apt history digest
 #
 # Usage:
 #   os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
+#                [--skip-cps] [--only-cps] [--skip-ssh-preflight]
+#
+# Flags:
+#   --skip-cps            run workers only (legacy behavior, useful for narrow patches).
+#   --only-cps            skip workers, only walk the CP phase.
+#   --only <n>            update a single node (worker or CP, routed automatically).
+#   --skip-ssh-preflight  skip the SSH-drift preflight (escape hatch for bootstrap
+#                         scenarios where cp-1's pubkey is intentionally absent).
 #
 # Environment:
 #   CONTROL_PLANE_HOST (default 178.105.17.239)
-#   All env vars honored by update-node.sh are honored here as well.
+#   POST_CP_WAIT_SECONDS (default 180) — settle between CPs after uncordon.
+#   All env vars honored by update-node.sh / update-cp-node.sh are honored here.
 #
 # Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it
-# step-for-step and the doc is the authoritative reference.
+# step-for-step and the doc is the authoritative reference. CP handling is
+# delegated to update-cp-node.sh; see CP_UPDATE_PROCEDURE.md.
 set -euo pipefail
 
 CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
+POST_CP_WAIT_SECONDS="${POST_CP_WAIT_SECONDS:-180}"
 DRY_RUN=0
 START_FROM=""
 ONLY=""
+SKIP_CPS=0
+ONLY_CPS=0
+SKIP_SSH_PREFLIGHT="${SKIP_SSH_PREFLIGHT:-0}"
 
 while [ $# -gt 0 ]; do
   case "$1" in
     --dry-run)             DRY_RUN=1; shift ;;
     --start-from)          START_FROM="$2"; shift 2 ;;
     --only)                ONLY="$2"; shift 2 ;;
+    --skip-cps)            SKIP_CPS=1; shift ;;
+    --only-cps)            ONLY_CPS=1; shift ;;
+    --skip-ssh-preflight)  SKIP_SSH_PREFLIGHT=1; shift ;;
     -h|--help)             grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
     *)                     echo "unknown arg: $1" >&2; exit 2 ;;
   esac
@@ -43,6 +65,27 @@ mkdir -p "$LOG_DIR"
 log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
 die() { log "FATAL: $*"; exit 1; }
 
+# node_ssh_endpoint <node-name>
+# Prints the ssh argv fragment (may include `-J bastion`) used to reach a node.
+# Private-IP workers go through cp-1 via ProxyJump; every other node has a
+# public IP. Returns non-zero for an unknown node. Callers pass the output
+# unquoted so `-J` is treated as its own arg.
+node_ssh_endpoint() {
+  case "$1" in
+    k3s-cp-1)     echo "root@178.105.17.239" ;;
+    k3s-cp-2)     echo "root@188.245.85.199" ;;
+    k3s-cp-3)     echo "root@49.13.92.162" ;;
+    k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
+    k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
+    k3s-worker-3) echo "root@167.233.121.121" ;;
+    k3s-worker-4) echo "root@128.140.3.80" ;;
+    k3s-worker-5) echo "root@167.233.192.86" ;;
+    k3s-worker-6) echo "root@49.12.223.133" ;;
+    k3s-update-runner) echo "root@167.233.79.65" ;;
+    *)            return 1 ;;
+  esac
+}
+
 log "=== os-update.sh cycle $STAMP ==="
 log "log dir: $LOG_DIR"
 
@@ -68,17 +111,17 @@ fi
 
 # --- 3. node ordering ---------------------------------------------------------
 # Ordering rule:
-#   - workers only
-#   - within workers: nodes NOT hosting Stalwart first, Stalwart-hosting fsn1 nodes last
-#   - ALL control-plane nodes are EXCLUDED and never updated by this script. Rationale:
-#     * kine/etcd write-path is sensitive to concurrent drains (see kine thundering-herd
-#       guardrails in OS_UPDATE_PROCEDURE.md, added after DEV-495).
-#     * rebooting a CP removes one apiserver — needs external liveness monitoring.
-#     * CP nodes host their own StatefulSet workloads that need batched eviction.
-#   Since DEV-510 (2026-08-22) the cluster runs HA control plane (cp-1/cp-2/cp-3). The
-#   exclusion here is role-based so ALL current and future CPs are covered automatically.
-#   To update a CP, use `scripts/os-update/update-cp-1.sh` (currently cp-1-only; will be
-#   generalized to any CP as part of the HA-aware CP OS-update procedure follow-up).
+#   - workers first (non-Stalwart, then Stalwart-hosting fsn1 last)
+#   - control-plane nodes AFTER workers, one at a time, etcd followers first,
+#     leader last. Rationale (see CP_UPDATE_PROCEDURE.md):
+#     * two of three CPs down = etcd quorum loss = write outage. Only one CP
+#       cordoned/rebooting at a time.
+#     * draining a follower is a no-op for raft leadership; draining the leader
+#       forces a re-election, so doing followers first minimises leader flapping.
+#   Since DEV-510 (2026-08-22) the cluster runs HA embedded etcd across
+#   cp-1/cp-2/cp-3 with 3-node quorum. This script delegates all CP steps to
+#   update-cp-node.sh, which handles swap (idempotent), preflight, drain, apt,
+#   reboot with /livez polling, etcd rejoin check, and finalize.
 STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true)
 # If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers.
 
@@ -108,38 +151,222 @@ for n in $ALL_NODES; do
   fi
 done
 
-ORDER=("${workers[@]}" "${stalwart_workers[@]}")
-# NOTE: CP nodes intentionally excluded. To update a CP, see CP1_UPDATE_PROCEDURE.md.
+WORKER_ORDER=("${workers[@]}" "${stalwart_workers[@]}")
+
+# --- 3b. control-plane ordering (leader last) --------------------------------
+# CP-node → public IP map. Kept out-of-cluster so a full-cluster incident
+# doesn't hide etcd from the operator machine.
+declare -A CP_HOST=(
+  [k3s-cp-1]="178.105.17.239"
+  [k3s-cp-2]="188.245.85.199"
+  [k3s-cp-3]="49.13.92.162"
+)
+
+etcd_leader_ip() {
+  # Query etcd leader endpoint from the operator's control-plane host. Uses
+  # the same TLS material as update-cp-node.sh. Prints "IP" on success,
+  # nothing on failure.
+  ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -o BatchMode=yes \
+    root@"$CONTROL_PLANE_HOST" \
+    'ETCDCTL_API=3 etcdctl \
+      --endpoints=https://127.0.0.1:2379 \
+      --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+      --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
+      --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
+      endpoint status --cluster -w simple 2>/dev/null' \
+    | awk -F, '$5 ~ /true/ {print $1}' \
+    | sed -E 's|https?://([^:]+):.*|\1|'
+}
+
+CP_ORDER=()
 if [ -n "$CP_NODES" ]; then
-  log "[plan] EXCLUDING control-plane nodes: $(echo $CP_NODES | tr '\n' ' ')— use scripts/os-update/update-cp-1.sh (see CP1_UPDATE_PROCEDURE.md)"
+  LEADER_IP="$(etcd_leader_ip || true)"
+  leader_node=""
+  followers=()
+  for n in $CP_NODES; do
+    ip="${CP_HOST[$n]:-}"
+    if [ -n "$ip" ] && [ "$ip" = "$LEADER_IP" ]; then
+      leader_node="$n"
+    else
+      followers+=("$n")
+    fi
+  done
+  if [ -n "$leader_node" ]; then
+    CP_ORDER=("${followers[@]}" "$leader_node")
+    log "[plan] etcd leader is $leader_node ($LEADER_IP) — updating followers first, leader last"
+  else
+    # Couldn't identify leader — refuse to guess. Fall back to no CPs.
+    CP_ORDER=()
+    log "[plan] WARN could not identify etcd leader — skipping CP phase this cycle"
+  fi
 fi
 
+# Route --only <node> automatically to worker or CP list.
 if [ -n "$ONLY" ]; then
-  ORDER=("$ONLY")
+  if is_cp "$ONLY"; then
+    CP_ORDER=("$ONLY")
+    WORKER_ORDER=()
+  else
+    WORKER_ORDER=("$ONLY")
+    CP_ORDER=()
+  fi
 elif [ -n "$START_FROM" ]; then
   new=()
   skip=1
-  for n in "${ORDER[@]}"; do
+  for n in "${WORKER_ORDER[@]}"; do
     [ "$n" = "$START_FROM" ] && skip=0
     [ $skip -eq 0 ] && new+=("$n")
   done
-  ORDER=("${new[@]}")
+  WORKER_ORDER=("${new[@]}")
+  # If START_FROM is a CP, trim CP_ORDER instead.
+  if is_cp "$START_FROM"; then
+    new_cp=()
+    skip=1
+    for n in "${CP_ORDER[@]:-}"; do
+      [ -z "$n" ] && continue
+      [ "$n" = "$START_FROM" ] && skip=0
+      [ $skip -eq 0 ] && new_cp+=("$n")
+    done
+    CP_ORDER=("${new_cp[@]}")
+    WORKER_ORDER=()
+  fi
 fi
 
-log "[plan] ordered nodes (${#ORDER[@]}): ${ORDER[*]}"
-printf '%s\n' "${ORDER[@]}" > "$LOG_DIR/plan.txt"
+[ "$SKIP_CPS" -eq 1 ] && CP_ORDER=()
+[ "$ONLY_CPS" -eq 1 ] && WORKER_ORDER=()
+
+log "[plan] workers (${#WORKER_ORDER[@]}): ${WORKER_ORDER[*]:-none}"
+log "[plan] control-planes (${#CP_ORDER[@]}): ${CP_ORDER[*]:-none}"
+{
+  echo "# workers"
+  printf '%s\n' "${WORKER_ORDER[@]:-}"
+  echo "# control-planes"
+  printf '%s\n' "${CP_ORDER[@]:-}"
+} > "$LOG_DIR/plan.txt"
 
 if [ "$DRY_RUN" -eq 1 ]; then
   log "DRY-RUN — plan written, no node touched. Exiting."
   exit 0
 fi
 
-# --- 4. per-node loop ---------------------------------------------------------
+# --- 3c. SSH-key drift preflight ---------------------------------------------
+# DEV-587 / DEV-586 (2026-09-27) — some k3s nodes drift to lose cp-1's SSH
+# pubkey from /root/.ssh/authorized_keys (probable Hetzner cloud-init resync
+# from the HC SSH-keys API; the operator key is NOT in that keyset). When this
+# happens mid-cycle the whole run aborts partway through drain / peer-probe.
+# This preflight proves the operator can SSH to every planned node and that
+# cp-1's pubkey is present in root's authorized_keys — auto-heals drift via
+# the operator's own SSH access, and hard-fails BEFORE anything is drained if
+# a node is genuinely unreachable.
+ssh_drift_preflight() {
+  local ok=() repaired=() problem=()
+
+  log "[preflight] SSH-key drift check for planned nodes"
+
+  local cp1_pubkey cp1_key_body
+  if ! cp1_pubkey=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
+                         root@"$CONTROL_PLANE_HOST" 'cat /root/.ssh/id_rsa.pub' 2>/dev/null); then
+    die "SSH-drift preflight: cannot read cp-1 pubkey from $CONTROL_PLANE_HOST — fix operator->cp-1 SSH first"
+  fi
+  cp1_pubkey=$(printf '%s' "$cp1_pubkey" | tr -d '\r\n')
+  cp1_key_body=$(printf '%s' "$cp1_pubkey" | awk '{print $2}')
+  [ -n "$cp1_key_body" ] || die "SSH-drift preflight: could not parse cp-1 pubkey (got: $cp1_pubkey)"
+  log "  cp-1 pubkey OK (${cp1_key_body:0:32}...)"
+
+  local plan_nodes=()
+  for n in "${WORKER_ORDER[@]:-}" "${CP_ORDER[@]:-}"; do
+    [ -z "$n" ] && continue
+    plan_nodes+=("$n")
+  done
+  if [ ${#plan_nodes[@]} -eq 0 ]; then
+    log "  (no planned nodes — nothing to preflight)"
+    return 0
+  fi
+
+  for n in "${plan_nodes[@]}"; do
+    local endpoint
+    if ! endpoint=$(node_ssh_endpoint "$n"); then
+      log "  $n: unknown ssh endpoint — refusing"
+      problem+=("$n (no ssh mapping in node_ssh_endpoint)")
+      continue
+    fi
+
+    # 1. reachability from the operator machine.
+    # shellcheck disable=SC2086  # $endpoint may include `-J bastion`; deliberate word-split.
+    if ! ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
+             $endpoint true 2>/dev/null; then
+      problem+=("$n ($endpoint - operator SSH refused)")
+      continue
+    fi
+
+    # 2. cp-1 is the source of the key; drift check would be pointless on it.
+    if [ "$n" = "k3s-cp-1" ]; then
+      ok+=("$n")
+      continue
+    fi
+
+    # 3. cp-1 pubkey present?
+    # shellcheck disable=SC2086
+    if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
+           $endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
+      ok+=("$n")
+      continue
+    fi
+
+    # 4. auto-heal — append cp-1's pubkey via the (already verified) operator SSH.
+    log "  $n: cp-1 pubkey MISSING from /root/.ssh/authorized_keys — appending"
+    # shellcheck disable=SC2086
+    if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
+           $endpoint "install -d -m 700 /root/.ssh && printf '%s\n' '$cp1_pubkey' >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys" \
+           2>/dev/null; then
+      # shellcheck disable=SC2086
+      if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
+             $endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
+        repaired+=("$n")
+      else
+        problem+=("$n (auto-heal wrote but grep-verify failed)")
+      fi
+    else
+      problem+=("$n ($endpoint - auto-heal write refused)")
+    fi
+  done
+
+  log "[preflight] SSH-drift summary:"
+  log "  ok        (${#ok[@]}): ${ok[*]:-none}"
+  log "  repaired  (${#repaired[@]}): ${repaired[*]:-none}"
+  log "  problem   (${#problem[@]}): ${problem[*]:-none}"
+
+  {
+    echo "# SSH-drift preflight — $(date -u +%FT%TZ)"
+    echo "# cp-1 pubkey: $cp1_pubkey"
+    printf 'ok\t%s\n'        "${ok[*]:-}"
+    printf 'repaired\t%s\n'  "${repaired[*]:-}"
+    printf 'problem\t%s\n'   "${problem[*]:-}"
+  } > "$LOG_DIR/ssh-preflight.txt"
+
+  if [ ${#problem[@]} -gt 0 ]; then
+    log "SSH-drift preflight FAILED for: ${problem[*]}"
+    log "  cp-1 pubkey to install:"
+    log "    $cp1_pubkey"
+    log "  From any machine that CAN reach the failing node, run:"
+    log "    ssh root@<node> \"grep -qF '$cp1_key_body' /root/.ssh/authorized_keys || echo '$cp1_pubkey' >> /root/.ssh/authorized_keys\""
+    die "SSH-drift preflight failed — refuse to start rolling update"
+  fi
+}
+
+if [ "$SKIP_SSH_PREFLIGHT" -eq 1 ]; then
+  log "[preflight] SSH-drift check SKIPPED (--skip-ssh-preflight)"
+else
+  ssh_drift_preflight
+fi
+
+# --- 4. worker loop ----------------------------------------------------------
 updated=()
 skipped=()
-for n in "${ORDER[@]}"; do
+for n in "${WORKER_ORDER[@]}"; do
+  [ -z "$n" ] && continue
   log "===================================================================="
-  log "==> updating $n"
+  log "==> updating worker $n"
   log "===================================================================="
   NODE_LOG="$LOG_DIR/${n}.log"
   set +e
@@ -153,6 +380,50 @@ for n in "${ORDER[@]}"; do
   esac
 done
 
+# --- 4b. control-plane loop --------------------------------------------------
+# One CP at a time, followers first, leader last. Delegates to update-cp-node.sh
+# --run (Phase A add-swap is idempotent, Phase B..F run with ASSUME_YES=1).
+# Between CPs we run cluster-health.sh and re-verify etcd cluster status; halt
+# on any yellow signal.
+if [ ${#CP_ORDER[@]} -gt 0 ]; then
+  log "===================================================================="
+  log "==> control-plane phase (${#CP_ORDER[@]} nodes): ${CP_ORDER[*]}"
+  log "===================================================================="
+  first_cp=1
+  for cp in "${CP_ORDER[@]}"; do
+    [ -z "$cp" ] && continue
+    if [ $first_cp -eq 0 ]; then
+      log "-- health gate between CPs (settle ${POST_CP_WAIT_SECONDS}s, then re-check)"
+      sleep "$POST_CP_WAIT_SECONDS"
+      if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" 2>&1 | tee "$LOG_DIR/inter-cp-health-before-${cp}.log"; then
+        die "cluster health degraded between CPs — halting before $cp. See $LOG_DIR/"
+      fi
+      # Re-check etcd cluster status; every member must still be started.
+      ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
+        'ETCDCTL_API=3 etcdctl \
+          --endpoints=https://127.0.0.1:2379 \
+          --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+          --cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
+          --key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
+          endpoint status --cluster -w table' 2>&1 | tee -a "$LOG_DIR/inter-cp-health-before-${cp}.log" || \
+        die "etcd endpoint status failed between CPs — halting before $cp"
+    fi
+    log "===================================================================="
+    log "==> updating control-plane $cp"
+    log "===================================================================="
+    CP_LOG="$LOG_DIR/${cp}.log"
+    set +e
+    ASSUME_YES=1 "$SCRIPT_DIR/update-cp-node.sh" "$cp" --run 2>&1 | tee "$CP_LOG"
+    rc=${PIPESTATUS[0]}
+    set -e
+    case $rc in
+      0) updated+=("$cp") ;;
+      *) die "update-cp-node.sh failed for $cp (rc=$rc). Cycle halted. See $CP_LOG" ;;
+    esac
+    first_cp=0
+  done
+fi
+
 # --- 5. finalization ----------------------------------------------------------
 log "===================================================================="
 log "==> finalization"
@@ -166,17 +437,11 @@ log "[final] apt history digest"
 {
   for n in "${updated[@]}"; do
     echo "=== $n ==="
-    # resolve ssh target via a mini-eval of node_ssh_target-equivalent
-    case "$n" in
-      k3s-cp-1)           t="root@178.105.17.239" ;;
-      k3s-worker-1)       t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
-      k3s-worker-2)       t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
-      k3s-worker-3)       t="root@167.233.121.121" ;;
-      k3s-worker-4)       t="root@128.140.3.80" ;;
-      k3s-worker-5)       t="root@167.233.192.86" ;;
-      k3s-update-runner)  t="root@167.233.79.65" ;;
-      *)                  echo "  (unknown ssh target)"; continue ;;
-    esac
+    if ! t=$(node_ssh_endpoint "$n"); then
+      echo "  (unknown ssh target)"
+      continue
+    fi
+    # shellcheck disable=SC2086  # $t may include `-J bastion`; deliberate word-split.
     ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
       'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
       || echo "  (could not read apt history)"

infrastructure/scripts/os-update/update-cp-node.sh

@@ -473,15 +473,24 @@ phase_reboot() {
   if [ "$DRY_RUN" != "1" ]; then
     local deadline=$(( $(date +%s) + REBOOT_MAX_WAIT_SECONDS ))
     local code=000
+    # /livez is auth-gated on this cluster (returns 401 without a bearer token).
+    # Any HTTP response from the api-server (200/401/403) means it is up and
+    # serving; only network-level failures (000, connection refused) count as
+    # "not yet back". curl --max-time keeps the poll cycle snappy after reboot.
     while [ $(date +%s) -lt $deadline ]; do
-      code=$(curl -sk -o /dev/null -w '%{http_code}' "https://$TARGET_HOST:6443/livez" 2>/dev/null || echo 000)
-      if [ "$code" = "200" ]; then
-        log "  $NODE api-server /livez=200"
+      code=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "https://$TARGET_HOST:6443/livez" 2>/dev/null || echo 000)
+      case "$code" in
+        200|401|403)
+          log "  $NODE api-server responding (/livez HTTP $code)"
           break
-      fi
+          ;;
+      esac
       sleep 5
     done
-    [ "$code" = "200" ] || die "$NODE api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s -- escalate; check 'hcloud server describe $NODE' and Hetzner console"
+    case "$code" in
+      200|401|403) : ;;
+      *) die "$NODE api-server did not return within ${REBOOT_MAX_WAIT_SECONDS}s (last code=$code) -- escalate; check 'hcloud server describe $NODE' and Hetzner console" ;;
+    esac
   fi
 
   log "-- verify etcd cluster status from peer ($PEER_HOST) -- $NODE should be 'started'"

infrastructure/scripts/os-update/update-node.sh

@@ -15,6 +15,8 @@
 #   DRAIN_TIMEOUT_SECONDS (default 600)
 #   REBOOT_MAX_WAIT_SECONDS (default 600)
 #   POST_UNCORDON_WAIT_SECONDS (default 180)
+#   HEALTH_MAX_RETRIES (default 10)             — extra post-uncordon health retries
+#   HEALTH_RETRY_INTERVAL_SECONDS (default 30)  — delay between those retries
 #   CONTROL_PLANE_HOST (default 178.105.17.239)
 #   SSH_OPTS (default "-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new")
 #   ASSUME_YES=1 to skip interactive confirmations
@@ -29,9 +31,15 @@ fi
 DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-600}"
 REBOOT_MAX_WAIT_SECONDS="${REBOOT_MAX_WAIT_SECONDS:-600}"
 POST_UNCORDON_WAIT_SECONDS="${POST_UNCORDON_WAIT_SECONDS:-180}"
+HEALTH_MAX_RETRIES="${HEALTH_MAX_RETRIES:-10}"
+HEALTH_RETRY_INTERVAL_SECONDS="${HEALTH_RETRY_INTERVAL_SECONDS:-30}"
 CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
 SSH_OPTS="${SSH_OPTS:--o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new}"
 
+# Pod-level signatures that do not recover on their own — short-circuit the
+# post-uncordon retry loop and halt immediately (DEV-595).
+PERSISTENT_FAILURE_PATTERN='CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|InvalidImageName'
+
 log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; }
 die() { log "FATAL: $*"; exit 1; }
 
@@ -44,6 +52,7 @@ node_ssh_target() {
     k3s-worker-3)        echo "root@167.233.121.121" ;;
     k3s-worker-4)        echo "root@128.140.3.80" ;;
     k3s-worker-5)        echo "root@167.233.192.86" ;;
+    k3s-worker-6)        echo "root@49.12.223.133" ;;
     k3s-update-runner)   echo "root@167.233.79.65" ;;
     *)                   die "unknown node $1 — update node_ssh_target() in $0" ;;
   esac
@@ -187,11 +196,39 @@ log "[6/7] uncordon $NODE"
 kubectl uncordon "$NODE"
 
 # --- 7. post-node settle ------------------------------------------------------
+# First check at POST_UNCORDON_WAIT_SECONDS, then up to HEALTH_MAX_RETRIES more
+# attempts at HEALTH_RETRY_INTERVAL_SECONDS. Transient yellow signals (Init:*,
+# Running-but-not-Ready, deployment-below-replicas during reschedule) often
+# clear within a minute or two; previously they tripped a FATAL on the first
+# check. A pod in CrashLoopBackOff (or image-pull failure) is persistent and
+# short-circuits the loop immediately. (DEV-595)
 log "[7/7] post-node settle (${POST_UNCORDON_WAIT_SECONDS}s) + health check"
 sleep "$POST_UNCORDON_WAIT_SECONDS"
 
-if RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh"; then
-  log "=== $NODE update: OK ==="
-else
-  die "cluster health failed after updating $NODE — STOP the cycle, do NOT continue"
-fi
+attempt=1
+max_attempts=$((HEALTH_MAX_RETRIES + 1))
+while :; do
+  set +e
+  HEALTH_OUT=$(RETRY_ON_TRANSIENT=0 "$SCRIPT_DIR/cluster-health.sh" 2>&1)
+  HEALTH_RC=$?
+  set -e
+
+  echo "$HEALTH_OUT"
+
+  if [ "$HEALTH_RC" -eq 0 ]; then
+    log "=== $NODE update: OK (health attempt $attempt/$max_attempts) ==="
+    break
+  fi
+
+  if echo "$HEALTH_OUT" | grep -qE "$PERSISTENT_FAILURE_PATTERN"; then
+    die "cluster health failed after updating $NODE with persistent pod error (CrashLoopBackOff/ImagePullBackOff/etc.) on attempt $attempt — STOP the cycle, do NOT continue"
+  fi
+
+  if [ "$attempt" -ge "$max_attempts" ]; then
+    die "cluster health still failing after $attempt attempts (interval ${HEALTH_RETRY_INTERVAL_SECONDS}s) following update of $NODE — STOP the cycle, do NOT continue"
+  fi
+
+  log "  cluster health yellow on attempt $attempt/$max_attempts (no persistent signatures) — retrying in ${HEALTH_RETRY_INTERVAL_SECONDS}s"
+  sleep "$HEALTH_RETRY_INTERVAL_SECONDS"
+  attempt=$((attempt + 1))
+done

infrastructure/scripts/provision-k3s-worker.sh

@@ -82,7 +82,8 @@ echo "This may take a few minutes..."
 
 ssh root@${NEW_NODE_IP} "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v${CLUSTER_VERSION}' sh -s - agent \
   --server https://${CONTROL_PLANE_IP}:6443 \
-  --token '${K3S_TOKEN}'"
+  --token '${K3S_TOKEN}' \
+  --node-ip ${NEW_NODE_IP}"
 
 if [ $? -ne 0 ]; then
   echo -e "${RED}Error: k3s installation failed${NC}"
@@ -111,3 +112,4 @@ echo "Next steps:"
 echo "  1. Label the node if needed: kubectl label nodes <node-name> node-role.kubernetes.io/worker=worker"
 echo "  2. Monitor pod scheduling: kubectl get pods --all-namespaces -o wide | grep <node-name>"
 echo "  3. Verify flannel annotation: kubectl get node <node-name> -o yaml | grep flannel"
+echo "  4. Verify INTERNAL-IP is the private IP (not public): kubectl get node <node-name> -o wide"

velero/prometheusrule.yaml

@@ -0,0 +1,108 @@
+---
+# Velero alerting for cluster-reconstruction backups (DEV-569 / DEV-574).
+#
+# Metrics come from the Velero server on :8085/metrics, scraped via
+# velero/servicemonitor.yaml.
+#
+# All our Velero Schedules are daily (see velero/schedules/*), so the
+# freshness alert uses a 26h window (24h cadence + 2h grace).
+#
+# Notes:
+#   - `velero_backup_last_successful_timestamp` is emitted only after a
+#     schedule has had at least one successful run. Freshly created
+#     schedules produce no series until their first success, so the
+#     "missed" alert doesn't fire on day 0 — it fires once a schedule
+#     has succeeded at least once and then goes >26h without another.
+#   - `schedule=""` is Velero's rollup for one-off (non-Schedule)
+#     backups; we filter it out so ad-hoc backups don't page.
+#   - Velero v1.18.1 does not expose `velero_backup_repository_available`
+#     (mentioned in the DEV-574 spec), so the repo-health alert here
+#     watches `velero_backup_location_status_gauge` instead — that
+#     covers the same operational failure mode ("backups can't land
+#     because the target storage is unreachable").
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: velero-alerts
+  namespace: velero
+  labels:
+    app.kubernetes.io/name: velero
+    app.kubernetes.io/part-of: velero
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: velero.backups
+      interval: 60s
+      rules:
+        - alert: VeleroBackupFailed
+          expr: sum by (schedule) (increase(velero_backup_failure_total{schedule!=""}[30m])) > 0
+          for: 0m
+          labels:
+            team: infra
+            severity: warning
+            service: velero
+            component: backup
+          annotations:
+            summary: "Velero backup failed ({{ $labels.schedule }})"
+            description: |
+              Velero recorded {{ $value | printf "%.0f" }} new backup
+              failure(s) for schedule `{{ $labels.schedule }}` in the
+              last 30 minutes.
+              Investigate: `velero backup get -n velero` and
+              `velero backup describe <name> --details`.
+        - alert: VeleroBackupPartial
+          expr: sum by (schedule) (increase(velero_backup_partial_failure_total{schedule!=""}[30m])) > 0
+          for: 0m
+          labels:
+            team: infra
+            severity: warning
+            service: velero
+            component: backup
+          annotations:
+            summary: "Velero backup completed with partial failures ({{ $labels.schedule }})"
+            description: |
+              Velero recorded {{ $value | printf "%.0f" }} partial
+              failure(s) for schedule `{{ $labels.schedule }}` in the
+              last 30 minutes. Some items were skipped or errored even
+              though the backup finished.
+              Investigate: `velero backup describe <name> --details`.
+        - alert: VeleroBackupMissed
+          # All schedules are daily → alert if the last success is
+          # older than 26h (24h cadence + 2h grace).
+          expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (26 * 3600)
+          for: 15m
+          labels:
+            team: infra
+            severity: warning
+            service: velero
+            component: backup
+          annotations:
+            summary: "Velero schedule missed daily window ({{ $labels.schedule }})"
+            description: |
+              Schedule `{{ $labels.schedule }}` has not produced a
+              successful backup in {{ $value | humanizeDuration }}
+              (>26h). Expected cadence: daily.
+              Check: `kubectl -n velero get schedule {{ $labels.schedule }}`
+              and Velero server logs.
+    - name: velero.storage
+      interval: 60s
+      rules:
+        - alert: VeleroStorageLocationUnavailable
+          # Substitutes for the spec's VeleroRepoUnhealthy — see file
+          # header. `velero_backup_location_status_gauge` is 1 when the
+          # BackupStorageLocation is Available, 0 when Unavailable.
+          expr: velero_backup_location_status_gauge == 0
+          for: 15m
+          labels:
+            team: infra
+            severity: warning
+            service: velero
+            component: backup
+          annotations:
+            summary: "Velero BackupStorageLocation unavailable ({{ $labels.backup_location_name }})"
+            description: |
+              BackupStorageLocation `{{ $labels.backup_location_name }}`
+              has reported Unavailable for >15m. New backups will fail
+              until it is restored.
+              Check: `kubectl -n velero get backupstoragelocation` and
+              `velero backup-location get`.

velero/restore-drill/00-serviceaccount-rbac.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: restore-drill
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+---
+# Ops permissions: create/list/get/delete velero Backups+Restores, create+delete
+# restore-test-* namespaces. Namespace verbs cannot be name-scoped in RBAC, so we
+# accept cluster-wide namespace create/delete. Drill script enforces the
+# 'restore-test-' name prefix before deleting.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: velero-restore-drill-ops
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+rules:
+  - apiGroups: ["velero.io"]
+    resources: ["backups", "restores"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: velero-restore-drill-ops
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+subjects:
+  - kind: ServiceAccount
+    name: restore-drill
+    namespace: velero
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: velero-restore-drill-ops
+---
+# Read-only diff permissions: cluster-scoped read of the workload resources we
+# compare between the source and restored namespaces.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: velero-restore-drill-view
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+subjects:
+  - kind: ServiceAccount
+    name: restore-drill
+    namespace: velero
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view

velero/restore-drill/10-drill-script-configmap.yaml

@@ -0,0 +1,224 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: restore-drill-script
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+data:
+  drill.sh: |
+    #!/bin/bash
+    # Monthly Velero restore drill for openwebui (DEV-575).
+    # Picks the latest openwebui-manifests backup (or ${DRILL_BACKUP_OVERRIDE}),
+    # restores it into restore-test-<ts>, diffs counts vs source, cleans up.
+    # Writes result to ConfigMap velero-restore-drill-last-result and job stdout.
+    # Exits non-zero on any failure (surfaces via kube_job_status_failed alert).
+    set -uo pipefail
+
+    SRC_NS="${DRILL_SOURCE_NS:-openwebui}"
+    SCHEDULE_NAME="${DRILL_SCHEDULE_NAME:-openwebui-manifests}"
+    RESULT_CM="${DRILL_RESULT_CM:-velero-restore-drill-last-result}"
+    RESULT_NS="${DRILL_RESULT_NS:-velero}"
+    RESTORE_TIMEOUT="${DRILL_RESTORE_TIMEOUT:-1800}"
+    NS_DELETE_TIMEOUT="${DRILL_NS_DELETE_TIMEOUT:-600}"
+
+    TS="$(date -u +%Y%m%d-%H%M%S)"
+    TGT_NS="restore-test-${TS}"
+    RESTORE_NAME="drill-${TS}"
+
+    LOG=""
+    log() { local line; line="[$(date -u +%FT%TZ)] $*"; echo "$line"; LOG+="${line}"$'\n'; }
+
+    DRILL_STATUS="fail"
+    BACKUP_NAME=""
+    BACKUP_START=""
+    BACKUP_COMPLETION=""
+    RESTORE_PHASE=""
+    RESTORE_DURATION_S=""
+    DIFF_SUMMARY=""
+
+    write_result_cm() {
+      local report="$1"
+      kubectl -n "${RESULT_NS}" create configmap "${RESULT_CM}" \
+        --from-literal=result="${report}" \
+        --from-literal=log="${LOG}" \
+        --dry-run=client -o yaml \
+        | kubectl -n "${RESULT_NS}" apply -f - >/dev/null 2>&1 \
+        || log "WARN: failed to write result ConfigMap"
+    }
+
+    post_webhook() {
+      local status="$1" backup="$2" restore="$3" restore_phase="$4" \
+            restore_duration_s="$5" target_ns="$6" source_ns="$7" diff_summary="$8"
+      if [[ -z "${PAPERCLIP_WEBHOOK_URL:-}" || -z "${PAPERCLIP_WEBHOOK_SECRET:-}" ]]; then
+        log "webhook: no PAPERCLIP_WEBHOOK_URL/SECRET, skipping"
+        return
+      fi
+      local body_file=/tmp/webhook.json
+      printf '{"source":"velero-restore-drill","status":"%s","timestamp":"%s","backup":"%s","restore":"%s","restore_phase":"%s","restore_duration_s":"%s","target_namespace":"%s","source_namespace":"%s","diff_summary":"%s"}\n' \
+        "${status}" "${TS}" "${backup}" "${restore}" "${restore_phase}" \
+        "${restore_duration_s}" "${target_ns}" "${source_ns}" "${diff_summary}" > "${body_file}"
+      # DEV-590: trigger is in github_hmac signingMode. Compute HMAC-SHA256 over
+      # the raw body and send X-Hub-Signature-256; sending Authorization: Bearer
+      # here is a trap because the home Paperclip's global actor middleware
+      # rejects any Bearer header that is not a board API key or agent JWT
+      # before the per-trigger handler runs.
+      local sig
+      sig=$(openssl dgst -sha256 -hmac "${PAPERCLIP_WEBHOOK_SECRET}" -r "${body_file}" | awk '{print $1}')
+      local hc
+      hc=$(curl -sS -o /tmp/webhook.out -w '%{http_code}' -m 30 \
+             -X POST -H "X-Hub-Signature-256: sha256=${sig}" \
+             -H "Content-Type: application/json" \
+             --data-binary "@${body_file}" \
+             "${PAPERCLIP_WEBHOOK_URL}" 2>&1) || hc="000"
+      log "webhook: HTTP ${hc}"
+    }
+
+    finalize() {
+      local end
+      end="$(date -u +%FT%TZ)"
+
+      if [[ "${TGT_NS}" == restore-test-* ]] && kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
+        log "cleanup: deleting namespace ${TGT_NS}"
+        if kubectl delete ns "${TGT_NS}" --timeout="${NS_DELETE_TIMEOUT}s" >/tmp/nsdelete.log 2>&1; then
+          log "cleanup: namespace ${TGT_NS} deleted"
+        else
+          log "cleanup: WARN failed to fully delete ${TGT_NS}, may be stuck terminating"
+        fi
+      fi
+
+      if [[ -n "${RESTORE_NAME}" ]] && kubectl -n velero get restore.velero.io "${RESTORE_NAME}" >/dev/null 2>&1; then
+        kubectl -n velero delete restore.velero.io "${RESTORE_NAME}" --wait=false >/dev/null 2>&1 || true
+      fi
+
+      local report
+      report=$(printf 'drill=%s\ntimestamp=%s\nbackup=%s\nbackup_started=%s\nbackup_completed=%s\nrestore=%s\nrestore_phase=%s\nrestore_duration_s=%s\ntarget_namespace=%s\nsource_namespace=%s\ndiff_summary=%s\nfinished_at=%s\n' \
+        "${DRILL_STATUS}" "${TS}" "${BACKUP_NAME}" "${BACKUP_START}" \
+        "${BACKUP_COMPLETION}" "${RESTORE_NAME}" "${RESTORE_PHASE}" \
+        "${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" "${DIFF_SUMMARY}" "${end}")
+
+      log "----- FINAL REPORT -----"
+      printf '%s\n' "${report}"
+
+      write_result_cm "${report}"
+      post_webhook "${DRILL_STATUS}" "${BACKUP_NAME}" "${RESTORE_NAME}" \
+        "${RESTORE_PHASE}" "${RESTORE_DURATION_S}" "${TGT_NS}" "${SRC_NS}" \
+        "${DIFF_SUMMARY}"
+
+      if [[ "${DRILL_STATUS}" == "pass" ]]; then
+        exit 0
+      else
+        exit 1
+      fi
+    }
+    trap finalize EXIT
+
+    log "drill start ts=${TS} src_ns=${SRC_NS} schedule=${SCHEDULE_NAME} target=${TGT_NS}"
+
+    if [[ -n "${DRILL_BACKUP_OVERRIDE:-}" ]]; then
+      BACKUP_NAME="${DRILL_BACKUP_OVERRIDE}"
+      log "using override backup name: ${BACKUP_NAME}"
+    else
+      BACKUP_NAME=$(kubectl -n velero get backups.velero.io \
+        -l "velero.io/schedule-name=${SCHEDULE_NAME}" \
+        -o go-template='{{range .items}}{{if eq .status.phase "Completed"}}{{.status.completionTimestamp}}|{{.metadata.name}}{{"\n"}}{{end}}{{end}}' 2>/dev/null \
+        | sort -r | head -n1 | cut -d'|' -f2)
+      if [[ -z "${BACKUP_NAME}" ]]; then
+        log "ERROR: no Completed backup found for schedule=${SCHEDULE_NAME}"
+        exit 1
+      fi
+      log "latest Completed backup: ${BACKUP_NAME}"
+    fi
+
+    BACKUP_START=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.startTimestamp}' 2>/dev/null || true)
+    BACKUP_COMPLETION=$(kubectl -n velero get backup.velero.io "${BACKUP_NAME}" -o jsonpath='{.status.completionTimestamp}' 2>/dev/null || true)
+
+    log "creating restore ${RESTORE_NAME} from backup ${BACKUP_NAME} -> ns ${TGT_NS}"
+    RESTORE_YAML=/tmp/restore.yaml
+    {
+      printf 'apiVersion: velero.io/v1\n'
+      printf 'kind: Restore\n'
+      printf 'metadata:\n'
+      printf '  name: %s\n' "${RESTORE_NAME}"
+      printf '  namespace: velero\n'
+      printf '  labels:\n'
+      printf '    app.kubernetes.io/part-of: velero-restore-drill\n'
+      printf 'spec:\n'
+      printf '  backupName: %s\n' "${BACKUP_NAME}"
+      printf '  includedNamespaces:\n'
+      printf '    - %s\n' "${SRC_NS}"
+      printf '  namespaceMapping:\n'
+      printf '    %s: %s\n' "${SRC_NS}" "${TGT_NS}"
+      printf '  includeClusterResources: false\n'
+      printf '  restorePVs: false\n'
+      printf '  existingResourcePolicy: none\n'
+      printf '  hooks: {}\n'
+    } > "${RESTORE_YAML}"
+    if ! kubectl apply -f "${RESTORE_YAML}" >/tmp/apply.log 2>&1; then
+      log "ERROR: failed to create Restore CR"
+      cat /tmp/apply.log | while IFS= read -r l; do log "  $l"; done
+      exit 1
+    fi
+
+    log "waiting up to ${RESTORE_TIMEOUT}s for restore to finish"
+    start_epoch=$(date +%s)
+    while :; do
+      RESTORE_PHASE=$(kubectl -n velero get restore.velero.io "${RESTORE_NAME}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
+      case "${RESTORE_PHASE}" in
+        Completed|PartiallyFailed|Failed|FailedValidation)
+          break
+          ;;
+      esac
+      now=$(date +%s)
+      if (( now - start_epoch > RESTORE_TIMEOUT )); then
+        log "ERROR: restore did not reach terminal phase within ${RESTORE_TIMEOUT}s (last=${RESTORE_PHASE})"
+        exit 1
+      fi
+      sleep 10
+    done
+    RESTORE_DURATION_S=$(( $(date +%s) - start_epoch ))
+    log "restore phase=${RESTORE_PHASE} duration=${RESTORE_DURATION_S}s"
+
+    if [[ "${RESTORE_PHASE}" != "Completed" && "${RESTORE_PHASE}" != "PartiallyFailed" ]]; then
+      log "ERROR: restore did not complete (phase=${RESTORE_PHASE})"
+      kubectl -n velero describe restore.velero.io "${RESTORE_NAME}" 2>&1 | tail -40 | while IFS= read -r l; do log "  $l"; done
+      exit 1
+    fi
+
+    if ! kubectl get ns "${TGT_NS}" >/dev/null 2>&1; then
+      log "ERROR: restore reported ${RESTORE_PHASE} but namespace ${TGT_NS} does not exist"
+      exit 1
+    fi
+
+    count_kind() {
+      local ns="$1" kind="$2"
+      kubectl -n "${ns}" get "${kind}" --no-headers 2>/dev/null | wc -l | tr -d ' '
+    }
+
+    KINDS=(deployments statefulsets daemonsets replicasets services configmaps secrets serviceaccounts persistentvolumeclaims cronjobs jobs)
+    DIFF_SUMMARY=""
+    for k in "${KINDS[@]}"; do
+      sc=$(count_kind "${SRC_NS}" "${k}")
+      tc=$(count_kind "${TGT_NS}" "${k}")
+      DIFF_SUMMARY+="${k}=${sc}/${tc};"
+    done
+    log "diff (src/tgt): ${DIFF_SUMMARY}"
+
+    STRICT_KINDS=(deployments statefulsets services configmaps persistentvolumeclaims)
+    strict_ok=1
+    for k in "${STRICT_KINDS[@]}"; do
+      sc=$(count_kind "${SRC_NS}" "${k}")
+      tc=$(count_kind "${TGT_NS}" "${k}")
+      if [[ "${sc}" != "${tc}" ]]; then
+        log "STRICT MISMATCH kind=${k} src=${sc} tgt=${tc}"
+        strict_ok=0
+      fi
+    done
+
+    if (( strict_ok == 1 )); then
+      DRILL_STATUS="pass"
+      log "drill result: PASS"
+    else
+      log "drill result: FAIL (strict kind mismatch)"
+      exit 1
+    fi

velero/restore-drill/20-restore-drill-cronjob.yaml

@@ -0,0 +1,108 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: velero-restore-drill
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-restore-drill
+spec:
+  # Monthly at 04:00 UTC on the 1st (per DEV-575 acceptance).
+  schedule: "0 4 1 * *"
+  timeZone: "UTC"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: velero-restore-drill
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 3600
+      ttlSecondsAfterFinished: 604800   # keep completed jobs for 7d for debug
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/part-of: velero-restore-drill
+        spec:
+          restartPolicy: Never
+          serviceAccountName: restore-drill
+          # DEV-579: public DNS for paperclip.home.imicros.de returns the LAN IP
+          # 10.0.0.2 (split-brain), which is unroutable from the cluster. The
+          # home reverse proxy at home.imicros.de (46.225.155.185) already
+          # serves the paperclip.home.imicros.de vhost publicly, so overriding
+          # the hostname to that public IP lets the webhook reach the routine.
+          # If the home ISP rotates the public IP, refresh this entry from
+          # `dig +short home.imicros.de` (DDNS record).
+          hostAliases:
+            - ip: "46.225.155.185"
+              hostnames:
+                - "paperclip.home.imicros.de"
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            fsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: drill
+              image: alpine/k8s:1.30.5
+              imagePullPolicy: IfNotPresent
+              command: ["/bin/bash", "/scripts/drill.sh"]
+              env:
+                - name: DRILL_SOURCE_NS
+                  value: "openwebui"
+                - name: DRILL_SCHEDULE_NAME
+                  value: "openwebui-manifests"
+                - name: DRILL_RESULT_NS
+                  value: "velero"
+                - name: DRILL_RESULT_CM
+                  value: "velero-restore-drill-last-result"
+                - name: DRILL_RESTORE_TIMEOUT
+                  value: "1800"
+                - name: DRILL_NS_DELETE_TIMEOUT
+                  value: "600"
+                # Optional Paperclip webhook (mounted from SealedSecret if wired).
+                - name: PAPERCLIP_WEBHOOK_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: velero-restore-drill-webhook
+                      key: webhookUrl
+                      optional: true
+                - name: PAPERCLIP_WEBHOOK_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: velero-restore-drill-webhook
+                      key: webhookSecret
+                      optional: true
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 128Mi
+                limits:
+                  cpu: 500m
+                  memory: 512Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                  readOnly: true
+                - name: tmp
+                  mountPath: /tmp
+                - name: kube
+                  mountPath: /.kube
+          volumes:
+            - name: script
+              configMap:
+                name: restore-drill-script
+                defaultMode: 0555
+            - name: tmp
+              emptyDir: {}
+            - name: kube
+              emptyDir: {}

velero/restore-drill/30-webhook-sealed.yaml

@@ -0,0 +1,17 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: velero-restore-drill-webhook
+  namespace: velero
+spec:
+  encryptedData:
+    webhookSecret: AgB4J8oWfPjZDar6zOp9ezv7M5DrtQFCvN6JgJmT3f6LJBngssanDW0N2nFg76a8L/Pht2RZK8aQq0GyGVbYjxtiQ0lagZGR/7j2yoKgkUyRie99YL7SMBPZ2SM9yBDRzsctHKeLZtmhsqyrmKLFnVs49Au3v2GtpknwOePr/BDZgdTGhaR/NftFuWMak6yPeoXNkGpYK7spJwoWUDN1LyQY/ZoEcgz2iQJ7odKqVE7hmt21an1xv7a/nmIDS8uNWWfYlPspTTxfQvTX4Ti28e5Z7S9kj+fqAVX6zH7tZ/jKODE72+uOya2FnjZ/SjEJ7Lz0+4eZTMWo0Kk0rK+WFY5fQvwOFlOZiCQsL3S2oxIiPjy11ZJh8gBFCXUR1KJNsPNveifsXvOC+kwWA/YaYQRHUArKZ50QOXhkwbEoS7XB1LklJil0IRzScvDSNSwtrpxg80rWVTVZEC5XJVpnRaNOSvfQI7NBYPA/rYRU3OcT0cT7vkBUIK4kgFV/4uIIoalaKcAGtPoHTEciA0avkCOgDSzlE18YZ///1hdtgJWx1OQ2ewpdawmsK1XJPjSreuf8zqxiVQosxI3Q0pTjv47aznjnxV2EDuvhj/gXYfrvO9B47chi2PdesAZt2IeOEOphl94EkVANHAuNgwnsymbK8mdFyybnCcIrWpgaIRMZYy3cQv+tqRwHJrk1cqPnul7fWKu4V7t6SfVY/ML3vsGQOevE4dce/rDquMuAbnWYqmV8YoD3jxPwe8bkhKUcrZM=
+    webhookUrl: AgBVLLkdlaxiqWyQQXcLj5uewljVQhBQpJw1OG9IMNw+MjQoBss6gqZhHgK2xAddsbnzOEAr/BJ6kJgR8DXldWsdQ9MnQZDc7VLOXs+h399KrIYeDTbuRHwWzp2BaCBSwOxmnd7S6v6h10Qq9rC02F8uRQ5gQ0B/JOVb1kwh+h6whCL3cSC511Wp6dnON2u0QJb3NoIVqGJd24ZcFgOs8w53fSgQgakvbMPYe34/RoHtT/SZmoaIUUzzAC+C1FzRKaJPKVDoPoYy5IhnIMNkecPmyg1hjJzmRQR64grIhO02WBm+lbjXnRqcauAyNeE5NNglYKaQmSr3Ya8zaB1pxm2L5DiUgCyR4O4nlIPH/37zvqa35fVnELmuwYprBCVmCxjulkPlqrh3tmRcR6EEPdd2ECND4E6XUenof57W/VsPg+iiRn/vcqAM3e0NidHhf5GtSxW5VDBmorrzqyWsEZbnNxIA01rNX1pupVTC1YssTdGF1k5/ccWUlwCpZJMeMyR0fHfn6kCsFHbF8lZ6w3VILObiKJnDU7Kkn3RXZ1k9plN21ZipffoSSXDLStQitJhX3a6ADhRazRw/xZjdqgO5iYNfSXih6GI1/6LgbE5UTZA7nqvAX9OqnH1B/7qMGCcwt18l+lF26uStN0OiDOaPdsgdzSpdv0BPibSvc4RdtvVEHiarV11geaMdxS8wZnS+C7GO2Nvx+qcGS6v9G9czIEvvrogcuBd8S3F7z0oiLmNNEeVW/CJwknO+5OkJm/wmFhQTZbRozgte8v9pjsaPZ6k/CzUdAilb+aEdkZ8DesXh5zYlGTMedclc
+  template:
+    metadata:
+      creationTimestamp: null
+      name: velero-restore-drill-webhook
+      namespace: velero
+    type: Opaque
+---

velero/restore-drill/README.md

@@ -0,0 +1,81 @@
+# Velero restore drill
+
+Monthly Kubernetes CronJob that exercises the Velero backup chain by restoring
+the `openwebui` namespace into a transient `restore-test-<ts>` namespace,
+diffing resource counts against the live namespace, and cleaning up.
+
+Wired by [DEV-575](https://forgejo.basicstack.de/basicstack/stack.basicstack.de/wiki/DEV-575)
+under the [DEV-569](https://forgejo.basicstack.de/basicstack/stack.basicstack.de/wiki/DEV-569)
+Velero cluster-reconstruction backup layer.
+
+## What it does
+
+1. Picks the latest `Completed` backup for schedule `openwebui-manifests`
+   (env `DRILL_BACKUP_OVERRIDE` short-circuits this — used by the failure
+   drill).
+2. Creates a `Restore` CR with namespace mapping
+   `openwebui: restore-test-<ts>`.
+3. Waits for the `Restore` to reach a terminal phase (`Completed`,
+   `PartiallyFailed`, `Failed`, `FailedValidation`) or times out at 30 min.
+4. Counts core kinds (deployments, statefulsets, services, configmaps, PVCs, …)
+   in source vs target and requires strict equality on the manifest-carrying
+   kinds to pass.
+5. Deletes the `restore-test-<ts>` namespace and the `Restore` CR.
+6. Writes the final report to ConfigMap `velero/velero-restore-drill-last-result`
+   and logs it to stdout.
+7. Optionally POSTs a JSON payload to the Paperclip routine webhook if
+   the `velero-restore-drill-webhook` Secret is present. The CronJob uses a
+   `hostAliases` entry to route `paperclip.home.imicros.de` at the home
+   public IP so the webhook reaches the routine from Hetzner (DEV-579).
+   Authentication is `github_hmac` — the script computes an HMAC-SHA256
+   of the raw body with the webhook secret as key and sends it as
+   `X-Hub-Signature-256: sha256=<hex>`. The matching trigger `signingMode`
+   is `github_hmac` on routine `c9d941fa-fb97-4e6a-a305-8e46f296e0cd`.
+   Previously it was `bearer`, but the home Paperclip's global actor
+   middleware rejects any `Authorization: Bearer …` whose value is not a
+   board API key or an agent JWT before the per-trigger handler runs,
+   making the bearer path unreachable (DEV-590).
+
+## Failure signalling
+
+The Job exits non-zero on any failure. The alerting stack (DEV-574) will fire
+`KubeJobFailed` on the `velero-restore-drill-*` job name and route the alert
+through existing Alertmanager receivers. Ops can inspect the last result via:
+
+```
+kubectl -n velero get cm velero-restore-drill-last-result -o jsonpath='{.data.result}'
+kubectl -n velero get cm velero-restore-drill-last-result -o jsonpath='{.data.log}'
+```
+
+## Manual runs
+
+Kick a drill on demand:
+
+```
+kubectl -n velero create job --from=cronjob/velero-restore-drill drill-adhoc-$(date -u +%Y%m%d-%H%M%S)
+```
+
+Force a failure to test the alert wiring (uses a bogus backup name):
+
+```
+kubectl -n velero create job drill-adhoc-fail-$(date -u +%s) \
+  --image=alpine/k8s:1.30.5 \
+  --dry-run=client -o yaml \
+  | kubectl -n velero apply -f -   # only if you need a scratch, prefer:
+kubectl -n velero create job --from=cronjob/velero-restore-drill drill-fail-$(date -u +%Y%m%d-%H%M%S) --dry-run=client -o yaml \
+  | sed 's/value: "openwebui-manifests"/value: "openwebui-manifests"\n            - name: DRILL_BACKUP_OVERRIDE\n              value: "no-such-backup"/' \
+  | kubectl apply -f -
+```
+
+For a simpler failure drill, apply the CronJob-derived Job then patch env before scheduling — see the parent DEV-575 comment for the exact runbook.
+
+## RBAC scope
+
+- `velero.io/backups,restores`: get/list/create/delete in-cluster (namespace
+  scope is enforced by resource type — Backups/Restores live in `velero`).
+- `namespaces`: cluster-wide get/list/create/delete. Cannot be name-scoped in
+  RBAC; the drill script enforces the `restore-test-` prefix before ever
+  deleting a namespace.
+- `configmaps`: get/list/create/update/patch (only used to write the result
+  ConfigMap in `velero` — pod-level namespace scoping via ServiceAccount).
+- Cluster-wide `view` ClusterRole for the counting diff.

velero/schedules/00-cluster-all.yaml

@@ -0,0 +1,24 @@
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: cluster-all
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: cluster
+spec:
+  schedule: "0 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - '*'
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: true
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default

velero/schedules/10-namespace-manifests.yaml

@@ -0,0 +1,780 @@
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: argocd-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: argocd
+spec:
+  schedule: "15 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - argocd
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: basicstack-web-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: basicstack-web
+spec:
+  schedule: "16 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - basicstack-web
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: bookstack-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: bookstack
+spec:
+  schedule: "17 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - bookstack
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: cert-manager-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: cert-manager
+spec:
+  schedule: "18 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - cert-manager
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: container-registry-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: container-registry
+spec:
+  schedule: "19 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - container-registry
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: directus-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: directus
+spec:
+  schedule: "20 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - directus
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: dozzle-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: dozzle
+spec:
+  schedule: "21 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - dozzle
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: forgejo-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: forgejo
+spec:
+  schedule: "22 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - forgejo
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: grist-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: grist
+spec:
+  schedule: "23 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - grist
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: harbor-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: harbor
+spec:
+  schedule: "24 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - harbor
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: headlamp-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: headlamp
+spec:
+  schedule: "25 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - headlamp
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: kaneo-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: kaneo
+spec:
+  schedule: "26 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - kaneo
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: keycloak-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: keycloak
+spec:
+  schedule: "27 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - keycloak
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: monitoring-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: monitoring
+spec:
+  schedule: "28 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - monitoring
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: nextcloud-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: nextcloud
+spec:
+  schedule: "29 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - nextcloud
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: observability-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: observability
+spec:
+  schedule: "30 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - observability
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: opencloud-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: opencloud
+spec:
+  schedule: "31 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - opencloud
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: openwebui-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: openwebui
+spec:
+  schedule: "32 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - openwebui
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: outline-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: outline
+spec:
+  schedule: "33 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - outline
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: pangolin-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: pangolin
+spec:
+  schedule: "34 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - pangolin
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: paperclip-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: paperclip
+spec:
+  schedule: "35 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - paperclip
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: paperclip-system-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: paperclip-system
+spec:
+  schedule: "36 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - paperclip-system
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: passbolt-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: passbolt
+spec:
+  schedule: "37 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - passbolt
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: planka-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: planka
+spec:
+  schedule: "38 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - planka
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: platform-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: platform
+spec:
+  schedule: "39 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - platform
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: pocket-id-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: pocket-id
+spec:
+  schedule: "40 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - pocket-id
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: reloader-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: reloader
+spec:
+  schedule: "41 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - reloader
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: stalwart-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: stalwart
+spec:
+  schedule: "42 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - stalwart
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: vaultwarden-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: vaultwarden
+spec:
+  schedule: "43 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - vaultwarden
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: vikunja-manifests
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-manifests
+    velero.basicstack.de/namespace: vikunja
+spec:
+  schedule: "44 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 720h0m0s
+    includedNamespaces:
+      - vikunja
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: false
+    storageLocation: default

velero/schedules/20-namespace-data.yaml

@@ -0,0 +1,104 @@
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: pocket-id-data
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-data
+    velero.basicstack.de/namespace: pocket-id
+spec:
+  schedule: "45 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 336h0m0s
+    includedNamespaces:
+      - pocket-id
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: true
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: vaultwarden-data
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-data
+    velero.basicstack.de/namespace: vaultwarden
+spec:
+  schedule: "47 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 336h0m0s
+    includedNamespaces:
+      - vaultwarden
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: true
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: vikunja-data
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-data
+    velero.basicstack.de/namespace: vikunja
+spec:
+  schedule: "49 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 336h0m0s
+    includedNamespaces:
+      - vikunja
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: true
+    storageLocation: default
+---
+apiVersion: velero.io/v1
+kind: Schedule
+metadata:
+  name: passbolt-data
+  namespace: velero
+  labels:
+    app.kubernetes.io/part-of: velero-schedules
+    tier: namespace-data
+    velero.basicstack.de/namespace: passbolt
+spec:
+  schedule: "51 3 * * *"
+  useOwnerReferencesInBackup: false
+  template:
+    ttl: 336h0m0s
+    includedNamespaces:
+      - passbolt
+    excludedNamespaces:
+      - velero
+      - kube-system
+      - kube-node-lease
+      - kube-public
+    includeClusterResources: false
+    snapshotVolumes: false
+    defaultVolumesToFsBackup: true
+    storageLocation: default

velero/servicemonitor.yaml

@@ -0,0 +1,28 @@
+---
+# Scrape Velero's built-in Prometheus endpoint (http-monitoring / :8085)
+# so kube-prometheus-stack can drive the accompanying PrometheusRule
+# (velero/prometheusrule.yaml). The `release: kube-prometheus-stack`
+# label is what the operator's serviceMonitorSelector matches.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: velero
+  namespace: velero
+  labels:
+    app.kubernetes.io/name: velero
+    app.kubernetes.io/part-of: velero
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - velero
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: velero
+      app.kubernetes.io/name: velero
+  endpoints:
+    - port: http-monitoring
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      scheme: http