basicstack/stack.basicstack.de
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

Compare commits

base: basicstack:main
Branches Tags
basicstack:main
basicstack:feat/DEV-474-rocksdb-teardown
basicstack:fix/DEV-527-dns-config-hostnetwork
basicstack:fix/DEV-523-argocd-oidc-secret-label
basicstack:fix/DEV-523-argocd-oidc-secret-ref
basicstack:feat/DEV-521-argocd-helm
basicstack:feat/DEV-469-stalwart-postgres
basicstack:chore/DEV-460-delete-platform-prod
..
compare: basicstack:fix/DEV-527-dns-config-hostnetwork
Branches Tags
basicstack:main
basicstack:feat/DEV-474-rocksdb-teardown
basicstack:fix/DEV-527-dns-config-hostnetwork
basicstack:fix/DEV-523-argocd-oidc-secret-label
basicstack:fix/DEV-523-argocd-oidc-secret-ref
basicstack:feat/DEV-521-argocd-helm
basicstack:feat/DEV-469-stalwart-postgres
basicstack:chore/DEV-460-delete-platform-prod

1 commit
main ... fix/DEV-52

Author SHA1 Message Date
CTO Agent
1c3cd31381 fix(dns): cap hostNetwork/coredns pod DNS at 3 upstreams (DEV-527) 
Hetzner OS publishes 4 systemd-resolved upstreams and Kubernetes limits
pod resolv.conf to 3 nameservers, so kubelet drops the 4th and fires a
DNSConfigForming Warning event on every hostNetwork or dnsPolicy=Default
pod restart. Silence the noise by pinning the pods to 3 explicit servers
(same 3 kubelet was already picking).

- apps/observability/patches/node-exporter-dns-config.yaml — strategic-
  merge patch adding dnsPolicy=None + dnsConfig to the
  kube-prometheus-stack node-exporter DaemonSet (Helm-managed, applied
  by hand)
- apps/observability/patches/coredns-dns-config.yaml — companion patch
  for the k3s built-in CoreDNS Deployment. kubectl patch alone is not
  durable because the k3s addon controller reverts dnsPolicy; kept as a
  quick manual re-apply hook
- infrastructure/k3s-manifests/coredns.yaml — the authoritative modified
  k3s addon manifest that must live at
  /var/lib/rancher/k3s/server/manifests/coredns.yaml on all 3 CP nodes
- infrastructure/k3s-manifests/README-DEV-527.md — apply procedure,
  verification steps, and upgrade caveat

Applied and verified on the live cluster:
- node-exporter DaemonSet rolled with dnsPolicy=None; no
  DNSConfigForming events on current pods
- coredns Deployment reconciled after pushing the modified manifest to
  all 3 CPs; new pod runs with dnsPolicy=None and 3-nameserver dnsConfig
- internal + external DNS resolution still works

Co-Authored-By: Paperclip <noreply@paperclip.ing>
 2026-08-23 12:05:12 +00:00
113 changed files with 1019 additions and 4887 deletions
Show stats Expand all files Collapse all files

20
apps/app-grist.yaml
View file

@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: grist
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/grist
destination:
server: https://kubernetes.default.svc
namespace: grist
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

20
apps/app-kaneo.yaml
View file

@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: kaneo
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/kaneo
destination:
server: https://kubernetes.default.svc
namespace: kaneo
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

20
apps/app-openwebui.yaml
View file

@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: openwebui
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/openwebui
destination:
server: https://kubernetes.default.svc
namespace: openwebui
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

20
apps/app-outline.yaml
View file

@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: outline
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/outline
destination:
server: https://kubernetes.default.svc
namespace: outline
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

23
apps/app-velero-monitoring.yaml
View file

@ -1,23 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: velero-monitoring
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: velero
directory:
recurse: false
include: '{servicemonitor.yaml,prometheusrule.yaml}'
destination:
server: https://kubernetes.default.svc
namespace: velero
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true

23
apps/app-velero-restore-drill.yaml
View file

@ -1,23 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: velero-restore-drill
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: velero/restore-drill
directory:
recurse: false
include: '*.yaml'
destination:
server: https://kubernetes.default.svc
namespace: velero
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true

23
apps/app-velero-schedules.yaml
View file

@ -1,23 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: velero-schedules
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: velero/schedules
directory:
recurse: false
include: '*.yaml'
destination:
server: https://kubernetes.default.svc
namespace: velero
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- ServerSideApply=true

21
apps/app-velero.yaml
View file

@ -1,21 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: velero
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/velero
destination:
server: https://kubernetes.default.svc
namespace: velero
syncPolicy:
automated:
prune: false
selfHeal: true
syncOptions:
- CreateNamespace=true
- ServerSideApply=true

20
apps/app-vikunja.yaml
View file

@ -1,20 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: vikunja
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/vikunja
destination:
server: https://kubernetes.default.svc
namespace: vikunja
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

2
apps/argocd/README.md
View file

@ -27,7 +27,7 @@ Memory limits were added after DEV-281 (resource exhaustion incident on 2026-07-
| Component | Memory Limit | Memory Request | | Component | Memory Limit | Memory Request |
|-----------|--------------|----------------| |-----------|--------------|----------------|
| application-controller | 1Gi | 512Mi | | application-controller | 512Mi | 256Mi |
| repo-server | 512Mi | 256Mi | | repo-server | 512Mi | 256Mi |
| redis | 256Mi | 128Mi | | redis | 256Mi | 128Mi |
| server | 256Mi | 128Mi | | server | 256Mi | 128Mi |

4
apps/argocd/values.yaml
View file

@ -45,9 +45,9 @@ configs:
controller: controller:
resources: resources:
requests: requests:
memory: 512Mi memory: 256Mi
limits: limits:
memory: 1Gi memory: 512Mi
repoServer: repoServer:
resources: resources:

3
apps/bookstack/bookstack-deployment.yaml
View file

@ -100,8 +100,6 @@ metadata:
namespace: bookstack namespace: bookstack
spec: spec:
replicas: 1 replicas: 1
strategy:
type: Recreate
selector: selector:
matchLabels: matchLabels:
app: bookstack app: bookstack
@ -125,7 +123,6 @@ spec:
containers: containers:
- name: bookstack - name: bookstack
image: lscr.io/linuxserver/bookstack:latest image: lscr.io/linuxserver/bookstack:latest
imagePullPolicy: Always
ports: ports:
- containerPort: 80 - containerPort: 80
name: http name: http

2
apps/directus/directus-deployment.yaml
View file

@ -35,7 +35,7 @@ spec:
fsGroup: 1000 fsGroup: 1000
containers: containers:
- name: directus - name: directus
image: directus/directus:12.3.0 image: directus/directus:12.1.1
ports: ports:
- name: http - name: http
containerPort: 8055 containerPort: 8055

2
apps/dozzle/deployment.yaml
View file

@ -18,7 +18,7 @@ spec:
serviceAccountName: dozzle serviceAccountName: dozzle
containers: containers:
- name: dozzle - name: dozzle
image: amir20/dozzle:v10.7.3 image: amir20/dozzle:v10.6.10
ports: ports:
- containerPort: 8080 - containerPort: 8080
name: dozzle-http name: dozzle-http

2
apps/forgejo/forgejo-backup-cronjob.yaml
View file

@ -96,7 +96,7 @@ spec:
memory: 512Mi memory: 512Mi
containers: containers:
- name: restic - name: restic
image: harbor.basicstack.de/library/restic:0.19.1 image: harbor.basicstack.de/library/restic:0.17.3
env: env:
- name: AWS_ACCESS_KEY_ID - name: AWS_ACCESS_KEY_ID
valueFrom: valueFrom:

94
apps/grist/README.md
View file

@ -1,94 +0,0 @@
# Grist Deployment
This directory contains the Kubernetes manifests for the Grist deployment at https://grist.basicstack.de
Grist is an open-source, self-hostable spreadsheet / database hybrid ([grist-core](https://github.com/gristlabs/grist-core)).
## Components
- **grist-deployment.yaml**: Main Grist application deployment
- Image: `gristlabs/grist:1.7.7`
- Port: 8484
- OIDC login via Pocket ID (`https://auth.basicstack.de`)
- Data persisted at `/persist`
- **grist-service.yaml**: ClusterIP service exposing port 80 -> 8484
- **grist-data-pvc.yaml**: PVC for Grist app data (`/persist`), 20 GiB on `hcloud-volumes-encrypted`
- **grist-postgres-deployment.yaml** / **grist-postgres-service.yaml**: In-namespace PostgreSQL 16 for the Grist home database (TypeORM connection)
- **grist-db-pvc.yaml**: PVC for PostgreSQL data, 20 GiB on `hcloud-volumes-encrypted`
- **grist-ingress.yaml**: Traefik ingress with TLS
- Host: `grist.basicstack.de`
- TLS via cert-manager (`letsencrypt-prod`, DNS-01 via Hetzner webhook)
- **grist-db-credentials-sealed.yaml**: SealedSecret for PostgreSQL user password
- **grist-app-secrets-sealed.yaml**: SealedSecret with `GRIST_SESSION_SECRET`
- **grist-oidc-secret-sealed.yaml**: SealedSecret with Pocket ID OIDC client credentials
## OIDC Configuration (Pocket ID)
- Pocket ID group: `grist`
- Client name in Pocket ID: `Grist`
- Callback URL: `https://grist.basicstack.de/oauth2/callback`
- Client is restricted to members of the `grist` group
Grist environment variables driving OIDC (from https://pocket-id.org/docs/client-examples/grist):
```
GRIST_OIDC_IDP_ISSUER=https://auth.basicstack.de
GRIST_OIDC_IDP_CLIENT_ID=<from secret>
GRIST_OIDC_IDP_CLIENT_SECRET=<from secret>
GRIST_OIDC_SP_HOST=https://grist.basicstack.de
GRIST_OIDC_IDP_SCOPES="openid email profile"
GRIST_OIDC_IDP_END_SESSION_ENDPOINT=https://auth.basicstack.de/api/oidc/end-session
GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED=true
GRIST_DEFAULT_EMAIL=andreas.leinen@basicstack.de
GRIST_FORCE_LOGIN=true
```
`GRIST_DEFAULT_EMAIL` is used by Grist as the primary admin identity - the value must be a valid Pocket ID user.
## Storage
Both PVCs (`grist-data`, `grist-db`) use `hcloud-volumes-encrypted` (Retain reclaim policy) as required for
sensitive workload data on Hetzner.
## GitOps
Managed by Argo CD via `apps/app-grist.yaml`:
```yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: grist
namespace: argocd
spec:
project: default
source:
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
targetRevision: main
path: apps/grist
destination:
server: https://kubernetes.default.svc
namespace: grist
syncPolicy:
syncOptions:
- CreateNamespace=true
```
## Verification
```
kubectl -n grist get pods
kubectl -n grist get pvc
kubectl -n grist get ingress
curl -I https://grist.basicstack.de
```
## Backup
The PostgreSQL PVC uses reclaim policy `Retain`. For scheduled logical backups add a `CronJob` like the one in `apps/directus/directus-backup-cronjob.yaml`.

15
apps/grist/grist-app-secrets-sealed.yaml
View file

@ -1,15 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: grist-app-secrets
namespace: grist
spec:
encryptedData:
session-secret: AgBN2KDg6QQgvhD2WvTG/r4+y6FrY7EXc8kU8N4hUDIh/VJDoyK+xiEHFvuOXMH3ZpNw4qTEpL+a+6LVWM1/c6mdcTIW5J52+hOOoXEtSXJhoBCjljgddlTTxHy9sIdE0694dMgeBJwmnvT9vm1TNQWMbiBaD05Xh4VCX3D908L4l/CiM11M9CLkoMGK7cTz9OCuGtd36k4OUxmw3/mQnafCDyzKf07NRW70Fr/bMGCdKE4Jt1rqWBfIaTb2mq96FCHosFq37keTCh1gu2nKf3iWesfv40rsVNN3wEhATaG2ghmjP2ul21zMOIO5GuCnMcawZv8VR1gyb5f98WOinL1zf5NMPCPRdTl/3G2gpqPc9VRNxOGguup3NLCaV2J+vbH5dgEj/h6zeTXw5bRARRQHAaGhUgFS3PALzubTzaUyNyFOyaMoXbAnE79HU+7QEd3Yjpdujup4Pe6OioK2k7jGWJj/jxZokffCjUsG9+hvNlg2E4ETtXxmHXIkwgTx7nHON6UApnprCtAUatRuMT3H+4VzfgkjV6y2aIHcp21EbH+qDBLo83+sBeUeWYyRtA/W2BGoAhx1CXeBysmEjD/Cls2h2PVhIvUdfNBY9fDgLemuK7MxdKbR8hRb3zHeedLXM4wevg2zCTyyGtizeC3JmpnqO1UHsKBZMGyUh43r+7sh3SyBKVUJxZt0q2JOWibaZNbIsrqcAyb8Te/QZlayZ2s2BxvMvvNrNpcOTYs/KzwwaHva8OUl
template:
metadata:
creationTimestamp: null
name: grist-app-secrets
namespace: grist

15
apps/grist/grist-data-pvc.yaml
View file

@ -1,15 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: grist-data
namespace: grist
labels:
app: grist
component: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi

15
apps/grist/grist-db-credentials-sealed.yaml
View file

@ -1,15 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: grist-db-credentials
namespace: grist
spec:
encryptedData:
postgres-password: AgA7MFNJ8//EOg4CdfHlSe2yGW6zcJyGsoiWVYlbVkb2GgMZmGQDH/ImYOTfeZXkEQhAWhA0braRB7Prx8fwpON2+6PsOczg+S1N1ykuDMJ+KDuXHhJtm5LNqbgibddX9y6YYInorgxM5mh/ORt2D1IdTwf1AXACkEkp8Ws4IGIfuUCym7kNM77mVD9jW4xyf/c7RE/J3va7L/6SRdx2rdoil8loMzMTHH+lvc738UdFA67hoLkKuseiYgA6NZap5Ppb48GTQul47ji3wH24F/u8DTKiHWbMBxwWO48gQJvAISgXSCW4eiKhkAcm28JACCOd1jT7Asen/yd5qEISB8Fr8q2qwxQwM3kXFIJRf9VVN7BWv6hEMnDO3AhBmcRr+gCPhKial17Varakh0ruBXB4eQDlxGQZ09trwtRPZkOsQnNf38iMQBpDH5U12tASmi1t5ddGKzQQh3EWx1vJyap8xNFf5zK6MuoDEMekHrjcclqgoMWppMH26Daghp2XZl740bV0UhLQBO1TOFEORHBs2DA2A3msmN25selVxjFBHvwLZHQJOHoVHbcqlbF7e2bFSLnij1TBMlPre/llBzoiVYrjHQoHw3fg7xVMrPBT+U1wn4ItDFZTP+nCxnWejtQ/dxzBfMrvBcHVhACsw7VhVkM5nhNFJBda65czyud0BWCSMYJYZEBZjDs220/mhQWIh2/RLANN6pcwJ2ONGyNJs3uo7Q245DlmDTjH5Qppmg==
template:
metadata:
creationTimestamp: null
name: grist-db-credentials
namespace: grist

15
apps/grist/grist-db-pvc.yaml
View file

@ -1,15 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: grist-db
namespace: grist
labels:
app: grist
component: database
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi

127
apps/grist/grist-deployment.yaml
View file

@ -1,127 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: grist
namespace: grist
labels:
app: grist
component: app
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: grist
component: app
template:
metadata:
labels:
app: grist
component: app
spec:
initContainers:
- name: wait-for-postgres
image: postgres:16-alpine
command:
- sh
- -c
- |
until pg_isready -h grist-postgres -U grist; do
echo "Waiting for PostgreSQL..."
sleep 2
done
securityContext:
fsGroup: 1000
containers:
- name: grist
image: gristlabs/grist:1.7.19
ports:
- name: http
containerPort: 8484
env:
- name: APP_HOME_URL
value: https://grist.basicstack.de
- name: GRIST_DOMAIN
value: grist.basicstack.de
- name: GRIST_SINGLE_ORG
value: docs
- name: GRIST_ORG_IN_PATH
value: "false"
- name: GRIST_DEFAULT_PRODUCT
value: Free
- name: GRIST_HIDE_UI_ELEMENTS
value: billing
- name: PORT
value: "8484"
- name: TYPEORM_TYPE
value: postgres
- name: TYPEORM_HOST
value: grist-postgres
- name: TYPEORM_PORT
value: "5432"
- name: TYPEORM_DATABASE
value: grist
- name: TYPEORM_USERNAME
value: grist
- name: TYPEORM_PASSWORD
valueFrom:
secretKeyRef:
name: grist-db-credentials
key: postgres-password
- name: GRIST_SESSION_SECRET
valueFrom:
secretKeyRef:
name: grist-app-secrets
key: session-secret
- name: GRIST_OIDC_IDP_ISSUER
value: https://auth.basicstack.de
- name: GRIST_OIDC_IDP_CLIENT_ID
valueFrom:
secretKeyRef:
name: grist-oidc-secret
key: client-id
- name: GRIST_OIDC_IDP_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: grist-oidc-secret
key: client-secret
- name: GRIST_OIDC_SP_HOST
value: https://grist.basicstack.de
- name: GRIST_OIDC_IDP_SCOPES
value: openid email profile
- name: GRIST_OIDC_IDP_END_SESSION_ENDPOINT
value: https://auth.basicstack.de/api/oidc/end-session
- name: GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED
value: "true"
- name: GRIST_DEFAULT_EMAIL
value: andreas.leinen@basicstack.de
- name: GRIST_FORCE_LOGIN
value: "true"
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 2Gi
livenessProbe:
tcpSocket:
port: 8484
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /status
port: 8484
initialDelaySeconds: 30
periodSeconds: 5
timeoutSeconds: 3
volumeMounts:
- name: data
mountPath: /persist
volumes:
- name: data
persistentVolumeClaim:
claimName: grist-data

26
apps/grist/grist-ingress.yaml
View file

@ -1,26 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: grist
namespace: grist
labels:
app: grist
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
ingressClassName: traefik
tls:
- hosts:
- grist.basicstack.de
secretName: grist-tls
rules:
- host: grist.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grist
port:
number: 80

16
apps/grist/grist-oidc-secret-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: grist-oidc-secret
namespace: grist
spec:
encryptedData:
client-id: AgAAQEIKFFtfc3pm1yp3HKu7uIiTsYtcer6CYJbrUAY6bwNBE+Cj2NF3wvDVeW1M2padhXqyuuhuOIaYbbQ8NOgR3kVg6UIKgn5oNimpXgkwef4xCuGNbNN7kfRfVTJfwrJm8jON0nzmmvxnmsNsz7hJgRszQ2Tp1qJlCI/KIPsIZ0vcXehy+VWV5xBOQOnP+b/wxR+tbt9BnDX/acRi95JAJLXYabqkD7FDc0rLGiWmEsyl4prhhRcT2Ux7O6CJuyegAOOodPDsylkV2iyaI6L3EEY9IrrV5w4ta2tDDPEcnuLkUo/1LipfJUSGzO/tqYJMAbd6klTtfI3cIjimmQXUP/0uiy4YXT7LJud/yEwjSajRIMAdIMQNhAcXPM3EnW85nck4NYSO8qUgu04+lpPiQKiE1eSjCa/LWBIqOMG69nE/hZCCceNBa2krvJIhjrPewBEMxBvg4aCUUS9GO3YanoDN3Y5T9HcpjvbhtLvLMGu20uAKxw63m8DTyNnzruYkL4Qr/5KxyhquCGv9YuwJMtaiUlWc36mh6YKEyyiYSe56cMegUnIZKBWWjsJwJm+ApPKZSI3u9/AGiHodF9tKmqt8vCzuqyzhsAskQXpewdn3iHe2IY2KwuWPPoObUaaSYWcKdoEZYVgaY9IofXf/x4GbJ5GVqbrwCGFpXJGk3GFFtvlyFICINU+CCp5PXgoMiqvpNvUVEHVclCmcTMi8lWufWQi/6Xu4gH+08QHVMkzf55o=
client-secret: AgBrF2e/M48zFC0PdXm08IavK+qa3Sl3a0qmO0+eH/z5WYnRpgos5dFdCLxcrdDcJUgKy//pU+I23qoTXqgybxRnfVzsOdYb4UksGgWh9640BQ/NRuxqhdnfXjQN7PrJj+D+fumwRdTQU1PqyzEePM/6CVf6Ro9vdP6iTMDzbAnPnHoA8L8Hd+f3YKo43OKMR2GMwjNKAhpD20wMCYt1vTiEQX4drR9JW4mjcKfSCdaIrEFwIYn4EBkr1SwuC4Ek/KTn4CNQO82BQH+jjWO5CHRvLP6b4v2bVkgdQEqLR4+HpNjMm0JBKOjHvhfKv952T5mwROamWcpNlN4KW02YzIZMBU+dh7WhouZkDkQCalP4G7AGoLrjhWmLIgS56JqTuoiobFf90Uvz6ewcDmHAzPuYZMfn2mKndPYvoi/KSGzM3ceeNrFxpHCPhsG+UHbOfSCBmg7jqIeE9ial6fMsRRLeEqc3ay4lAN7PBXwJ/o3XMrezUy8RQsXLex9oS0IS9rktimwQNWEpY1adwWRERdBKqGAIEbVofVwai/SuC/dcgQsoX2r9kJOiA8ahM+eDxUg1534hb21v3tFWykp3y/EbFBxbPPtCsUY4ghZZ8O/lLtXsZdHxaq68RKbVqDbRGa/eLqCqStWS8Bk1XzwkJFBBHFcN9hDvNwoT+33QSSsuuiw8gLiztlSrewo33rVKEc9Enqb+f/uUX6HY9j3Iz7hvKZ2aYX/U8ayNN7hQaJryYQ==
template:
metadata:
creationTimestamp: null
name: grist-oidc-secret
namespace: grist

70
apps/grist/grist-postgres-deployment.yaml
View file

@ -1,70 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: grist-postgres
namespace: grist
labels:
app: grist
component: database
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: grist
component: database
template:
metadata:
labels:
app: grist
component: database
spec:
containers:
- name: postgres
image: postgres:16-alpine
ports:
- name: postgres
containerPort: 5432
env:
- name: POSTGRES_DB
value: grist
- name: POSTGRES_USER
value: grist
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: grist-db-credentials
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
exec:
command:
- pg_isready
- -U
- grist
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- pg_isready
- -U
- grist
initialDelaySeconds: 5
periodSeconds: 5
volumeMounts:
- name: data
mountPath: /var/lib/postgresql/data
volumes:
- name: data
persistentVolumeClaim:
claimName: grist-db

17
apps/grist/grist-postgres-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: grist-postgres
namespace: grist
labels:
app: grist
component: database
spec:
type: ClusterIP
selector:
app: grist
component: database
ports:
- name: postgres
port: 5432
targetPort: 5432

17
apps/grist/grist-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: grist
namespace: grist
labels:
app: grist
component: app
spec:
type: ClusterIP
selector:
app: grist
component: app
ports:
- name: http
port: 80
targetPort: 8484

6
apps/harbor/values.yaml
View file

@ -95,9 +95,6 @@ harbor:
cpu: 500m cpu: 500m
registry: registry:
# Skip Kopia fs-backup of the registry blob PVC (DEV-573 safety net).
podAnnotations:
backup.velero.io/backup-volumes-excludes: registry-data
registry: registry:
resources: resources:
requests: requests:
@ -109,9 +106,6 @@ harbor:
trivy: trivy:
enabled: true enabled: true
# Skip Kopia fs-backup of the trivy scan cache PVC (DEV-573 safety net).
podAnnotations:
backup.velero.io/backup-volumes-excludes: data
resources: resources:
requests: requests:
memory: 512Mi memory: 512Mi

2
apps/headlamp/deployment.yaml
View file

@ -16,7 +16,7 @@ spec:
serviceAccountName: headlamp-admin serviceAccountName: headlamp-admin
containers: containers:
- name: headlamp - name: headlamp
image: ghcr.io/headlamp-k8s/headlamp:v0.45.0 image: ghcr.io/headlamp-k8s/headlamp:v0.43.0
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
args: args:
- "-in-cluster" - "-in-cluster"

61
apps/kaneo/README.md
View file

@ -1,61 +0,0 @@
# Kaneo
Self-hosted [Kaneo](https://github.com/usekaneo/kaneo) — open-source project
management — deployed on the BasicStack Kubernetes cluster.
- **URL:** https://kaneo.basicstack.de
- **Namespace:** `kaneo`
- **Distribution:** hand-rolled manifests, GitOps-managed by Argo CD
(see `apps/app-kaneo.yaml`).
- **Auth:** OIDC via Pocket ID (`https://auth.basicstack.de`) with
`CUSTOM_OAUTH_*` env vars. Client id + secret live in
`kaneo-oidc-secret` (SealedSecret). Access is restricted to
Pocket ID groups `kaneo-admins` / `kaneo-members`.
- **Database:** dedicated PostgreSQL in-namespace on
`hcloud-volumes-encrypted` (20Gi). Both the Postgres and Kaneo
pods pin to the `fsn1` datacenter via
`csi.hetzner.cloud/location=fsn1` node affinity — the Hetzner
volume is DC-local.
- **TLS:** Traefik ingress, cert issued by `letsencrypt-prod`
(DNS-01 via the Hetzner cert-manager webhook).
- **Image:** `ghcr.io/usekaneo/kaneo:2.22.0` (single container
bundling API + web on port 5173, health at `/api/health`).
## Files
| File | Purpose |
| --- | --- |
| `kaneo-postgres-pvc.yaml` | 20 Gi encrypted Hetzner volume for Postgres data. |
| `kaneo-postgres-deployment.yaml` | Postgres 16-alpine, pinned to `fsn1`. |
| `kaneo-postgres-service.yaml` | ClusterIP service `kaneo-postgres:5432`. |
| `kaneo-db-credentials-sealed.yaml` | Postgres user password (SealedSecret). |
| `kaneo-app-secrets-sealed.yaml` | `AUTH_SECRET` (Better Auth JWT secret, ≥32 chars). |
| `kaneo-oidc-secret-sealed.yaml` | OIDC client id + secret from Pocket ID. |
| `kaneo-smtp-sealed.yaml` | SMTP submission credentials for `kaneo@basicstack.de` (SealedSecret). |
| `kaneo-deployment.yaml` | Kaneo app deployment (pinned to `fsn1`). |
| `kaneo-service.yaml` | ClusterIP service on port 80 → container 5173. |
| `kaneo-ingress.yaml` | Traefik ingress + `letsencrypt-prod` cert. |
## Post-deploy setup
1. Point browser at https://kaneo.basicstack.de and sign in via the
"Sign in with Pocket ID" (custom OAuth) button.
2. The first admin (`andreas.leinen@basicstack.de`, member of
`kaneo-admins`) must be promoted inside Kaneo — the current
upstream release does not map OIDC groups to roles.
## SMTP
Outbound SMTP is wired to the Stalwart submission relay at
`mail.basicstack.de:587` (STARTTLS) using the `kaneo@basicstack.de`
mailbox. Credentials live in the `kaneo-smtp` SealedSecret
(`kaneo-smtp-sealed.yaml`) with keys `smtp-user` and `smtp-password`.
SMTP is used only for workspace invites and notifications;
`DISABLE_EMAIL_OTP_SIGN_IN=true` remains set so users still sign in via
Pocket ID SSO, not email codes.
To rotate the mailbox password: update it in Stalwart, re-seal
`kaneo-smtp` with the new value, commit, and let Argo apply. The
existing pod picks up the new secret on next restart
(`kubectl -n kaneo rollout restart deploy/kaneo`).

15
apps/kaneo/kaneo-app-secrets-sealed.yaml
View file

@ -1,15 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: kaneo-app-secrets
namespace: kaneo
spec:
encryptedData:
auth-secret: AgArCARyFt7NinaeQOfDDEpTKKWVba8IlKh7NN/vflgdOkE5bjz/P/ZzGUVhUOTyqCs+3dU+hX6sqc7IYbYckXdprwKJSXy47HLEsZ/cuJ0FRIBShRyXm+I62Yu8WWLHNJLTnlWyfirBC6Pg65kXckoKiYPwcVLSD5I+YHzo3Dt3Hl6MeW/iVocM9xgjBD/5yRt1ag+S4JYv0t1y39Cz80XxbFvo55RzYWr3OlYNI325b9W/d5uylqlDemln8NymL5iCdfnLDZoc+1r+eHAf1HDP6Uch1dG+c/xsrlvoWpzyMeSEbEY1tGeYFZ1oKm4B1pyUH9dfdxWEaNThheL5Xw2QX/klpZdTq5Jl6Lfxme5aLyqNtP9J6ovd5b1oVYW1DWc+RAqBPhFlHnkfc+8ZeP6vrmnUULq0ssSVqpXOz4zLgDHU61UzSllNdpZWXokigranj3dq5CM7WkYLEj9kfdDIGS3gDUD+EmIkIj/N96URNpOINSwFVAcnE73k3ij9mZWECLl++lzh1fqHrdYQiTI6A7UpxPudc7BoujBQFqGhUDXGdzBWsrqr11tldD+kb/kcKQYxs6sFHyw0pDgCLVZukPv7TWRNfVRYUv68h/1ZKAwmj8KkuGlNtAQ+NyvUWyct3SA+pQYWabr33pIPekWPIXMIty6/zfKu92hLFglwkauewxqLtEBZ6FFukKMyyfowV+iwDIMyY3//MvxiRzxD7sc5sLwcfvPCBzd9XGaJO1ZLX6r43gfOu6HFHiq4FqdjYJ9fqG3zUKeo5bnav7E6
template:
metadata:
creationTimestamp: null
name: kaneo-app-secrets
namespace: kaneo

16
apps/kaneo/kaneo-db-credentials-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: kaneo-db-credentials
namespace: kaneo
spec:
encryptedData:
database-url: AgDEiqPdexD58tc4Sv63mk1erIHAeweJzGpKgoOcro6ioW4eX9pWnfVnVfmtsqQK8WXkRuEP5pAnj1LZAhuo6hfwk/lOpoYItqZKl6KGK0EQZ3u/hcmUt0yeEkTJuDPGBbgk1RuT1G2u2uDDYYJYHmAQtIIFbAoeJS2iDKnKkl6UHZdEJhO10w6IHGiNCT6Z9NovLJau5cw9tTv7xnbGO8F1QqwLh/dL0kI2oYNTYg1RxpegXRFeKGz+QSJ3rbxlH2P/E01Fevma2/ZhOBjALS1bpW9R6pgQm4n/FDu/fu72T62yw2oNvj6nIeRtIsm9ze02jQqYwF7mgct6IJOdqY7RoYnTZJQ+/Xl/iiBIGpzXan9bU/JZw6sEeyxS+tyeaolz4oiTeNfRXp8QNdX3qbXM9OE5Q+bjqx1q+cZcnFcBIvGK8vGqlWwDjwNicucuNBZuv4JWEOp2N8axoFVRUsUn/Frv7McqKT1Q2mRXf0aiGYh9m7EIdnFAx3JeiBJv392jBWcwvJpQ+J0LIsh/bJEBtdC4c16Y+38nbqfjYEUJWVIoj9SvJEo3QZPitUz99cwdtDn+cOVVYp+lv5BiunM7OgvAgBbX7F4Ri/QlL8rt3/8lO0Augwp933lSTuGGnMCrXPz2QCagI6B0l8F5q5OTnljgR7JXslyGFaemtj9jZEnCiSsw9KAiDdOUyk36CJSfHmRedJJ6o2nalb3zdFDF7iMXxz6fpTQK0QQGAccO518zUHOpb3Jb6XbOBKDgTBOWPEGlroiavrOtoRDJ+B5nR/N4c3Indpn7d0TTDU/4rz4V1uNkRrHnA26ya9Q=
postgres-password: AgBo5f/NzqZ/9jsOzXmoivPVVeLA3xk9GWocQQlLv34Rc1K5fzhSoCYdy8o7k06R83VeNsJylxfAsOI7HNIo0xWUPqC8HnbW482ajtREh93wOFCaZzjKu8ClbFIkvTb19GRCPLSmzBMFsTCJzjSa5rWGqK2Ezj7LFeTTiehsqS/PbPUYi6w4X6ZnChS3VuMXJVJAvbCF8fFS3x/YuuJVjjd+TmVsPmGOa6J+r/6RG7l4DsR+SmV3HQ7M6bLqTsUPjzg5Foz11yc8ER8xQQTgbbfrBPwgJQoTX622yHf0Q313ns7vyiKGMqge9DSQvDmGmMDmAxqtYInycVBH1CwmfKVbw3CyFGt0D5LJXiBaTE89uLErsNCQCK6NKAODqffKlOWpZ1KVF+ZIQ6nqJETtj49uqdKGowauCs3tDl/Hi82+hxycEVMWUwgu81CJ6GT7+vSZfz4+9FRbLoXlbdfR/duG14f+AkPirikeDcy3bFy5XiBgPVQXL0EbAY8TaPkYu+TS1Roo2F6Gqj3D1+i4GapVodrBntl1gMPvMnd3dcGGl/VzREGZvRsZgdRprFi+nGPmotRhIO3eGcPautlZ5m0WMqKCpA4jW0sG1sgkVwRpn2Zdq4Uw2+IKNHTkJKclIXCaw2/b6WNCmiwbe1ssFoR/QgikzshshoB5XpIYFe7Jr5Sj9mmNH6L4PpCIZ/sCe8MF5eMD8IuyWSVmflID07sfBD6GL3XB9TFLzSyviHE/VJdGY7l+ZeR64UZZ1ti4bwI=
template:
metadata:
creationTimestamp: null
name: kaneo-db-credentials
namespace: kaneo

157
apps/kaneo/kaneo-deployment.yaml
View file

@ -1,157 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: kaneo
namespace: kaneo
labels:
app: kaneo
component: app
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: kaneo
component: app
template:
metadata:
labels:
app: kaneo
component: app
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
initContainers:
- name: wait-for-postgres
image: postgres:16-alpine
command:
- sh
- -c
- |
until pg_isready -h kaneo-postgres -U kaneo -d kaneo; do
echo "Waiting for PostgreSQL..."
sleep 2
done
containers:
- name: kaneo
image: ghcr.io/usekaneo/kaneo:2.22.0
ports:
- name: http
containerPort: 5173
env:
# Public URL of this Kaneo instance.
- name: KANEO_CLIENT_URL
value: https://kaneo.basicstack.de
- name: CORS_ORIGINS
value: https://kaneo.basicstack.de
# Better Auth JWT secret (>=32 chars).
- name: AUTH_SECRET
valueFrom:
secretKeyRef:
name: kaneo-app-secrets
key: auth-secret
# Database credentials. DATABASE_URL is preferred; POSTGRES_* are
# accepted as a fallback and useful for the built-in migrations.
- name: POSTGRES_DB
value: kaneo
- name: POSTGRES_USER
value: kaneo
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: kaneo-db-credentials
key: postgres-password
- name: POSTGRES_HOST
value: kaneo-postgres
- name: POSTGRES_PORT
value: "5432"
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: kaneo-db-credentials
key: database-url
# Self-hosted: keep public registration off, log in via OIDC only.
- name: DISABLE_REGISTRATION
value: "true"
- name: DISABLE_PASSWORD_REGISTRATION
value: "true"
- name: DISABLE_EMAIL_OTP_SIGN_IN
value: "true"
- name: KANEO_CLOUD
value: "false"
# Custom OAuth / OIDC via Pocket ID.
- name: CUSTOM_OAUTH_CLIENT_ID
valueFrom:
secretKeyRef:
name: kaneo-oidc-secret
key: client-id
- name: CUSTOM_OAUTH_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: kaneo-oidc-secret
key: client-secret
- name: CUSTOM_OAUTH_DISCOVERY_URL
value: https://auth.basicstack.de/.well-known/openid-configuration
- name: CUSTOM_OAUTH_AUTHORIZATION_URL
value: https://auth.basicstack.de/authorize
- name: CUSTOM_OAUTH_TOKEN_URL
value: https://auth.basicstack.de/api/oidc/token
- name: CUSTOM_OAUTH_USER_INFO_URL
value: https://auth.basicstack.de/api/oidc/userinfo
- name: CUSTOM_OAUTH_LOGOUT_URL
value: https://auth.basicstack.de/api/oidc/end-session
- name: CUSTOM_OAUTH_SCOPES
value: openid,profile,email,groups
- name: CUSTOM_AUTH_PKCE
value: "true"
# SMTP: outbound only, used for workspace invites / notifications.
# Sign-in still requires OIDC (DISABLE_EMAIL_OTP_SIGN_IN=true above).
- name: SMTP_HOST
value: mail.basicstack.de
- name: SMTP_PORT
value: "587"
- name: SMTP_SECURE
value: "false"
- name: SMTP_REQUIRE_TLS
value: "true"
- name: SMTP_USER
valueFrom:
secretKeyRef:
name: kaneo-smtp
key: smtp-user
- name: SMTP_PASSWORD
valueFrom:
secretKeyRef:
name: kaneo-smtp
key: smtp-password
- name: SMTP_FROM
value: "Kaneo <kaneo@basicstack.de>"
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
httpGet:
path: /api/health
port: http
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /api/health
port: http
initialDelaySeconds: 15
periodSeconds: 5
timeoutSeconds: 3

26
apps/kaneo/kaneo-ingress.yaml
View file

@ -1,26 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kaneo
namespace: kaneo
labels:
app: kaneo
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
ingressClassName: traefik
tls:
- hosts:
- kaneo.basicstack.de
secretName: kaneo-tls
rules:
- host: kaneo.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: kaneo
port:
number: 80

16
apps/kaneo/kaneo-oidc-secret-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: kaneo-oidc-secret
namespace: kaneo
spec:
encryptedData:
client-id: AgBGmlu+mGYlHcWPCRHZ3IbevGB5kGntqPasFBnny6ZydTeCmN8NWpy0bmoorecJSbvdeP38UDgfEjtmusfkipnrlfsZd7BugZLiijq5+T+9al5JTl/UChMl6u/gReaVPgLi9iketDN6oB2XC/56mV+MdMb/bO4d1u3CRFUJ/Pq1znWeQpVsXO1yOLoOge4+PYDYjgd4O6mJsbAQcmGxQ7pwcByw//+50Mvuw0gJBqI5ZSGKd0jSNnaVcECtvjY6v3xN1NCeVF1FoWKl4s7o/Nrq5mDltW/y6g0h3N1Fzan/D47UVQjGLYu6cTmM7ReqToYk65cznttGkyXfqrlVL/oEiYENtaqp/BZvwPBMKEAjJsufwGvbNgsxa3eGs+p3L+HwzdgTQbVGu2wVp+TXW8zwZvkDyprUU91Fu6QFsspV/GRYgqVmp4RxrJ0PzZInp4ZhLt/zOyWzbEkILMPV1A5fzPjrPmwpmhln/K0VTdpRN62XQRgH0P7EP6Ao0sP2/ae39eTkGlhb5fwzhSqK3LSmBYmoXNcffA4nBLFohSndmwgoxzZXM1FJ1kXl4wuevU8Eh8BD3dVIZkuYacKiIUFoNn74NbPtJpl3/MWfXef9Eb7IBR8KyurbdwSmKSAZDqjmeGTa271+KUiGfST+FYQgSDfFfQQEmh+K49WqqbOiod4Mh5RTGWJx3WNwEONsTRKVjiGgLw4A+ENNDB39PoRChoRoRRogg+ZJZE27EjxbmrKIjrw=
client-secret: AgBJQi6fIlTkOIZa1zr9jjyNOFAyZUjHWy0ZBjfcNh+iY2V3qIpG8kNPBntmvtVtemYI+UxEvvlalq0iEue+AtVmUoJeeO/lYuEDv+BrJGUWeZITbItuyoG+xGZM4x41X6FPgfRcnLCfii3dmwOeDZh1asR2Ahy2szdXrd2QaDeZMCCXNR+IeuP9U+IqJncfm+KjCqPABVbsZuGfDrT7r4XRv4mHQGnILD4IyA9N0/XyfY/cZRJryWeF5xbrOAP/ZWJduqYGIm2ahJfRZWIzk1Ujknq35iJ4bOZiw5eWb+aYhD9cxJj2UMBwWdqnXnpH+p55lrBqWlCzI0yDOfk4UQCI2RzQBJc/RvVS0z16k7El4dl9lOFTcgwBg13C/JC8Y0wo4M1jEap2ty/2ifwVZBZdflT8Uy5sk8dKBOv9wCupA/N60ACwlsavCUlgtQetVZUHIlI1U1rKCzArpGZ6PuREZokjdHw01u/j9ANEt1AaMe3G5jLcGohyvpcgL+zqI2S0rCqbSdRqWzZvr2kM8uFoeZSJLG3y4wdwZMgvRGMfoAKD1YlUGuhWeeZTda77ScM6ii8iEZEjvIgJXrfSCUf1vJ4QIAdMnNi05d3pTTDVHNTRjWBrklVb6oTpe/rFF1BBP6zhwGylWtb71xszbQyX8lzD+7v1O4iXr7ZWo86L9mGKz6FMnj0bO4LLVoqeVMy5DHo2fCuplix03P3WQCyjGKranAWdLalEV59QdnEhaQ==
template:
metadata:
creationTimestamp: null
name: kaneo-oidc-secret
namespace: kaneo

83
apps/kaneo/kaneo-postgres-deployment.yaml
View file

@ -1,83 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: kaneo-postgres
namespace: kaneo
labels:
app: kaneo
component: database
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: kaneo
component: database
template:
metadata:
labels:
app: kaneo
component: database
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: postgres
image: postgres:16-alpine
ports:
- name: postgres
containerPort: 5432
env:
- name: POSTGRES_DB
value: kaneo
- name: POSTGRES_USER
value: kaneo
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: kaneo-db-credentials
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
exec:
command:
- pg_isready
- -U
- kaneo
- -d
- kaneo
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- pg_isready
- -U
- kaneo
- -d
- kaneo
initialDelaySeconds: 5
periodSeconds: 5
volumeMounts:
- name: data
mountPath: /var/lib/postgresql/data
volumes:
- name: data
persistentVolumeClaim:
claimName: kaneo-postgres-data

15
apps/kaneo/kaneo-postgres-pvc.yaml
View file

@ -1,15 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: kaneo-postgres-data
namespace: kaneo
labels:
app: kaneo
component: database
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi

17
apps/kaneo/kaneo-postgres-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: kaneo-postgres
namespace: kaneo
labels:
app: kaneo
component: database
spec:
type: ClusterIP
selector:
app: kaneo
component: database
ports:
- name: postgres
port: 5432
targetPort: 5432

17
apps/kaneo/kaneo-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: kaneo
namespace: kaneo
labels:
app: kaneo
component: app
spec:
type: ClusterIP
selector:
app: kaneo
component: app
ports:
- name: http
port: 80
targetPort: 5173

16
apps/kaneo/kaneo-smtp-sealed.yaml
View file

@ -1,16 +0,0 @@
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: kaneo-smtp
namespace: kaneo
spec:
encryptedData:
smtp-password: AgByMIhrSeQbPJezoqPf4jtTCx/b/R+TGZPHjjvUcdZJimgZY8XO72BA2nI/419UWqY3rBmWyquYMnEFJT0/48Rn0Nk4aFxBwBIxVuiG9y1+f5yXIhT2JuOqGoBnImYezCCtYOcgn4Byo6Twldce1ElRCqwH6grsVOLKtogZvLJC0sz6EWbqPfLrDuGyOFz2wg5P3Cjmy8W9ZgcTDTpVI4JxZh2JKbIJpq4kGDQD0miW8Av0BOmSi75uvT2PvaV/uinJ72VJFLP/GTHWjf/9kpOiIC2TL4B+Kk0HtMeWSzNF79itsD53ow7bya/bq//zUiwdaIeSaAcSJL8XqL9q38LIEL0Vr9ccXN9xsPZttLw836NDeLsryE+RhF3pIK2vEs76YIeO03AEnXNjEYOcwy7ePGYh6Y/Uf/B75QNzbzrq8H4B5lKFVmQgTBLJnvxTzOc6Sd/Yssgl1wJw90jgw2HYcLHXt/eQrB4cGKWSwrh2x6fnU79UpSGatdTqIyP2Q8vbgKmxqqmg6NDzjgeatT6ADKOqJKAjnXfrr8Bm3NjvAVWOGWuIfWdS0wHiEOhAoUNNAQ+bD7f/SbJE24q8W6juhzTmRLuU1BM2K51YPHA3BqyEpzygaboSi/bgHVOuoIAEAt7jAM6mbW5RQBwiNglF/C2Oem8J3LOdDlndJoSMaWKm5p1lKc3M59jr47jaDXm9HKM4ypC58cqty3varqxBwwdZHirZ6No0Ar4Sd78I4jo4kXk9zu33UjCcR/X2NX1abIYZ8eNoUga+I9AOkpM=
smtp-user: AgAq1mT9fzt9/VHmcbn0Yzr/8HWadEBe1XjJOTonao7Tdc3EO1PvCgEE9nBkrqOK5x2uDP1jZuP98XxJXr6mHuCnEOn8SHMN93TqK5rLhsKir1ImdNWKC+ruVI3TPQfTkeEz5m3u0DjHWSymva7BjmTIIW5gE4SDt/7WOFrsWbX1fLqQKfuB/k1qIBd+rFL7rVSe3XnRzxxwwrIbYfczSCknMbOot65FRcR+pC6ox7hOYC7pMIU76WMrVYx0cmDfAZAidcOQP9+K/H4LY0vkwPnaNk7+Is5gJifWR37lTnCkjRknMWB+yyDZF3BkmmdHNvXE6OlMcBF8l/jrO7S0rTtkCDaXllpRt1tWoSY2Kyr2DTaLt9QQvd1v8Rz2mw1qeDZg0mO/BswPF10SEKwjD3RlGFkA6mK9hZE5G+eDpgMH9u5HEKftI6jD/wdzElY+GdEFhlmj/ATZuNk+oC+eVnHGHUnY39lQYX+7PEtOujsKAjsuCPBaMDMkTP+f+LOUguuaRaanpLctmRAnT5BK19dmNuJtZPZAPgzCLu44884BH1qpKpVXkaUZSUbYA1RpKflcf1+uWJaqp7oBOZLSVoVSLiVG8ypGv7IPeTiILQcWQK27z6H0VxiAZ0S8yzNcCmeTWg8/wUGVzk0VJ9imPIShpiq515X6GuU1v1LsMiuJtFfL3EpNdDln04dLFWVb0ghIdt4Hn4U3IpJRIcP+XEEpv4+6
template:
metadata:
creationTimestamp: null
name: kaneo-smtp
namespace: kaneo
---

17
apps/monitoring/README.md
View file

@ -2,10 +2,11 @@
Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context. Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context.
- `backup-k8s-resources-cronjob.yaml` — daily dump of cluster-scoped and per-namespace Kubernetes resources, streamed through `restic backup --stdin` to `hetzner-s3:${BUCKET}/restic/k8s-resources`. Uses `serviceAccountName: backup-sa` and no PVC mount (init container `alpine/k8s:1.29.4` writes an emptyDir, main container `restic/restic:0.17.3` reads it on stdin). Rewritten from the local-path tarball per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-487](/DEV/issues/DEV-487)).
- `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)). - `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)).
- `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)). - `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)).
- `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block. - `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block.
- `backup-restic-alerts.yaml` + `backup-restic-alerts.test.yaml` — `PrometheusRule` with freshness, integrity, and repo-size alerts covering the three restic repos (loki/grafana/prometheus), plus a `promtool test rules` unit test proving each alert fires against synthetic samples ([DEV-490](/DEV/issues/DEV-490), extended in [DEV-492](/DEV/issues/DEV-492)). The `k8s-resources` restic repo was superseded by Velero cluster-reconstruction backups and retired in [DEV-576](/DEV/issues/DEV-576). - `backup-restic-alerts.yaml` + `backup-restic-alerts.test.yaml` — `PrometheusRule` with freshness, integrity, and repo-size alerts covering the four restic repos (loki/grafana/k8s-resources/prometheus), plus a `promtool test rules` unit test proving each alert fires against synthetic samples ([DEV-490](/DEV/issues/DEV-490), extended in [DEV-492](/DEV/issues/DEV-492)).
## Shared SealedSecret ## Shared SealedSecret
@ -17,7 +18,7 @@ All four restic CronJobs read Hetzner S3 credentials + the restic repository pas
| `secret-key` | Hetzner Object Storage secret access key | | `secret-key` | Hetzner Object Storage secret access key |
| `endpoint` | S3 endpoint hostname (e.g. `fsn1.your-objectstorage.com`) | | `endpoint` | S3 endpoint hostname (e.g. `fsn1.your-objectstorage.com`) |
| `bucket` | Bucket name (single bucket, per-prefix repos) | | `bucket` | Bucket name (single bucket, per-prefix repos) |
| `restic-password`| 32-byte random string sealed at [DEV-484](/DEV/issues/DEV-484); plaintext copy in Passbolt entry `restic / monitoring backups`. Rotation: `restic key add` → seal new value → `restic key remove` old id. Same key protects all three remaining repos (loki/grafana/prometheus) — rotating rewrites the key file on every repo. | | `restic-password`| 32-byte random string sealed at [DEV-484](/DEV/issues/DEV-484); plaintext copy in Passbolt entry `restic / monitoring backups`. Rotation: `restic key add` → seal new value → `restic key remove` old id. Same key protects all four repos (loki/grafana/k8s-resources/prometheus) — rotating rewrites the key file on every repo. |
## Restore ## Restore
@ -29,20 +30,20 @@ Each restic CronJob writes `/metrics/backup_<kind>.prom` (atomic — `.prom.tmp`
| Metric | Emitted by | | Metric | Emitted by |
|--------------------------------------------|----------------------------------------------------------------------------------| |--------------------------------------------|----------------------------------------------------------------------------------|
| `backup_<kind>_success` (0/1) | `backup-{loki,grafana}-*-cronjob.yaml`, `prometheus-backup-cronjob.yaml` | | `backup_<kind>_success` (0/1) | `backup-{loki,grafana,k8s-resources}-*-cronjob.yaml`, `prometheus-backup-cronjob.yaml` |
| `backup_<kind>_timestamp_seconds` | same | | `backup_<kind>_timestamp_seconds` | same |
| `backup_<kind>_check_status` (exit code) | same — from `restic check --read-data-subset=5%` | | `backup_<kind>_check_status` (exit code) | same — from `restic check --read-data-subset=5%` |
| `backup_prometheus_backup_status` | `prometheus-backup-cronjob.yaml` — `restic backup` exit code (3 = accepted compaction race) | | `backup_prometheus_backup_status` | `prometheus-backup-cronjob.yaml` — `restic backup` exit code (3 = accepted compaction race) |
| `restic_repo_size_bytes{repo="<kind>"}` | same — from `restic stats --json --mode raw-data` (added in DEV-490) | | `restic_repo_size_bytes{repo="<kind>"}` | same — from `restic stats --json --mode raw-data` (added in DEV-490) |
**Cross-node staleness note.** Because a backup CronJob may run on a different worker across days (loki/prometheus follow their app pods), a `.prom` file can linger on a node the job has since left and node-exporter keeps exposing it. The freshness alerts collapse the per-node samples with `max()` so the freshest sample wins; check/size alerts fire when *any* node reports a bad value, which is intentional — a recent failure is still a signal until the file is manually cleaned or the job returns to that node. **Cross-node staleness note.** Because a backup CronJob may run on a different worker across days (loki/prometheus follow their app pods, `k8s-resources` is unpinned), a `.prom` file can linger on a node the job has since left and node-exporter keeps exposing it. The freshness alerts collapse the per-node samples with `max()` so the freshest sample wins; check/size alerts fire when *any* node reports a bad value, which is intentional — a recent failure is still a signal until the file is manually cleaned or the job returns to that node.
## Alerts ## Alerts
`backup-restic-alerts.yaml` defines seven alerts (all `severity: warning`): `backup-restic-alerts.yaml` defines nine alerts (all `severity: warning`):
- `BackupLokiStale` / `BackupGrafanaStale` / `BackupPrometheusStale` — `time() - max(backup_<kind>_timestamp_seconds) > 28h`. Daily schedule + 4 h grace. `max()` collapses per-node samples so a stale `.prom` on a node the job has left does not fire. - `BackupLokiStale` / `BackupGrafanaStale` / `BackupK8sResourcesStale` / `BackupPrometheusStale` — `time() - max(backup_<kind>_timestamp_seconds) > 28h`. Daily schedule + 4 h grace. `max()` collapses per-node samples so a stale `.prom` on a node the job has left does not fire.
- `BackupLokiCheckFailed` / `BackupGrafanaCheckFailed` / `BackupPrometheusCheckFailed` — `backup_<kind>_check_status != 0`. - `BackupLokiCheckFailed` / `BackupGrafanaCheckFailed` / `BackupK8sResourcesCheckFailed` / `BackupPrometheusCheckFailed` — `backup_<kind>_check_status != 0`.
- `ResticRepoOversize` — `restic_repo_size_bytes > 20 GiB`. Baseline expected < 5 GiB; catches retention/prune regressions. - `ResticRepoOversize` — `restic_repo_size_bytes > 20 GiB`. Baseline expected < 5 GiB; catches retention/prune regressions.
All alerts carry a `Runbook: docs/monitoring/restic-restore.md` annotation. To iterate on the rule file locally: All alerts carry a `Runbook: docs/monitoring/restic-restore.md` annotation. To iterate on the rule file locally:
@ -54,4 +55,4 @@ promtool check rules /tmp/backup-restic-rules.yaml
promtool test rules apps/monitoring/backup-restic-alerts.test.yaml promtool test rules apps/monitoring/backup-restic-alerts.test.yaml
``` ```
The legacy `backup-volumes` CronJob and its 100 Gi local-path `backup-storage` PVC were retired in [DEV-489](/DEV/issues/DEV-489) once the restic pipeline was proven end-to-end. With the shared destination PVC gone, the DEV-483 bridge `nodeSelector` pinning Loki to `k3s-worker-2` was also removed — `backup-loki-restic` follows the Loki pod via `podAffinity` regardless of which node the RWO CSI volume lands on. `backup-grafana-restic` still nodeSelects `k3s-worker-2` because its source PV (`grafana-storage`, local-path) is anchored there. `prometheus-backup` uses `podAffinity` on `app=prometheus` (RWO PVC on Hetzner CSI, single-node attach) and follows the Prometheus pod between workers. The `backup-k8s-resources` CronJob and its restic repo were retired in [DEV-576](/DEV/issues/DEV-576) after Velero `cluster-all` took over cluster-manifest coverage. The legacy `backup-volumes` CronJob and its 100 Gi local-path `backup-storage` PVC were retired in [DEV-489](/DEV/issues/DEV-489) once the restic pipeline was proven end-to-end. With the shared destination PVC gone, the DEV-483 bridge `nodeSelector` pinning Loki to `k3s-worker-2` was also removed — `backup-loki-restic` follows the Loki pod via `podAffinity` regardless of which node the RWO CSI volume lands on. `backup-grafana-restic` still nodeSelects `k3s-worker-2` because its source PV (`grafana-storage`, local-path) is anchored there. `backup-k8s-resources` has no PVC dep and stays unpinned. `prometheus-backup` uses `podAffinity` on `app=prometheus` (RWO PVC on Hetzner CSI, single-node attach) and follows the Prometheus pod between workers.

4
apps/monitoring/backup-grafana-restic-cronjob.yaml
View file

@ -44,8 +44,8 @@ spec:
kubernetes.io/hostname: k3s-worker-2 kubernetes.io/hostname: k3s-worker-2
containers: containers:
- name: restic - name: restic
image: harbor.basicstack.de/library/restic:0.19.1 image: harbor.basicstack.de/library/restic:0.17.3
# Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) — # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
# deterministic ingress via Harbor. Retag procedure in # deterministic ingress via Harbor. Retag procedure in
# docs/monitoring/restic-restore.md § "Tag-bump procedure". # docs/monitoring/restic-restore.md § "Tag-bump procedure".
env: env:

205
apps/monitoring/backup-k8s-resources-cronjob.yaml Normal file
View file

@ -0,0 +1,205 @@
---
# Kubernetes-resource backup via restic to Hetzner Object Storage
# (DEV-487, DEV-482 Option 4). Step 4 of the Option 4 rollout.
#
# Streams a concatenated YAML dump of cluster-scoped and per-namespace
# resources through `restic backup --stdin` into
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/k8s-resources`. No PVC mount
# (drops the local-path `backup-storage` dependency), no node pin.
#
# Two-container pattern:
# 1. `kubectl-dump` init container (`alpine/k8s:1.29.4`) writes
# /dump/cluster.yaml into an emptyDir. Uses `serviceAccountName:
# backup-sa` (unchanged from the legacy job).
# 2. `restic` main container (`restic/restic:0.17.3`, matches the
# loki/grafana siblings) reads that file on stdin and streams it
# into the restic repo with `--stdin-filename cluster.yaml`.
apiVersion: batch/v1
kind: CronJob
metadata:
name: backup-k8s-resources
namespace: monitoring
labels:
app: backup
type: k8s-resources
backend: restic
spec:
schedule: "0 2 * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
metadata:
labels:
app: backup
type: k8s-resources
backend: restic
spec:
backoffLimit: 2
activeDeadlineSeconds: 3600
template:
metadata:
labels:
app: backup
type: k8s-resources
backend: restic
spec:
restartPolicy: OnFailure
serviceAccountName: backup-sa
initContainers:
- name: kubectl-dump
image: alpine/k8s:1.29.4
command:
- /bin/sh
- -c
- |
set -eu
echo "=== kubectl-dump started at $(date -u +%FT%TZ) ==="
DUMP=/dump/cluster.yaml
: > "${DUMP}"
echo "--- namespaces ---"
kubectl get namespaces -o yaml >> "${DUMP}"
echo "---" >> "${DUMP}"
echo "--- cluster-scoped resources ---"
kubectl get persistentvolumes,storageclasses,clusterroles,clusterrolebindings \
-o yaml >> "${DUMP}"
echo "---" >> "${DUMP}"
echo "--- namespaced resources ---"
for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do
echo " ns=${ns}"
kubectl get \
configmaps,secrets,services,deployments,statefulsets,daemonsets,jobs,cronjobs,ingresses,persistentvolumeclaims \
-n "${ns}" -o yaml >> "${DUMP}" 2>/dev/null || true
echo "---" >> "${DUMP}"
done
echo "dump size: $(wc -c < ${DUMP}) bytes"
echo "=== kubectl-dump finished at $(date -u +%FT%TZ) ==="
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: dump
mountPath: /dump
containers:
- name: restic
image: harbor.basicstack.de/library/restic:0.17.3
# Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
# deterministic ingress via Harbor. Retag procedure in
# docs/monitoring/restic-restore.md § "Tag-bump procedure".
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: monitoring-s3-backup
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: monitoring-s3-backup
key: secret-key
- name: RESTIC_PASSWORD
valueFrom:
secretKeyRef:
name: monitoring-s3-backup
key: restic-password
- name: S3_ENDPOINT
valueFrom:
secretKeyRef:
name: monitoring-s3-backup
key: endpoint
- name: S3_BUCKET
valueFrom:
secretKeyRef:
name: monitoring-s3-backup
key: bucket
- name: RESTIC_REPOSITORY
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/k8s-resources"
command:
- /bin/sh
- -c
- |
set -eu
echo "=== backup-k8s-resources-restic started at $(date -u +%FT%TZ) ==="
echo "Repository: ${RESTIC_REPOSITORY}"
# First-run tolerance: init if the repo isn't there yet.
if restic snapshots >/dev/null 2>&1; then
echo "Repo exists, skipping init."
else
echo "Repo missing, initialising..."
restic init
fi
echo "--- restic backup --stdin cluster.yaml ---"
restic backup --stdin \
--stdin-filename cluster.yaml \
--tag k8s-resources \
--host k3s < /dump/cluster.yaml
echo "--- restic forget/prune ---"
restic forget --tag k8s-resources \
--keep-daily 7 \
--keep-weekly 4 \
--keep-monthly 6 \
--prune
echo "--- restic check --read-data-subset=5% ---"
CHECK_STATUS=0
restic check --read-data-subset=5% || CHECK_STATUS=$?
echo "restic check exit: ${CHECK_STATUS}"
echo "--- restic stats (repo size) ---"
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
| grep -oE '"total_size":[0-9]+' \
| head -1 \
| cut -d: -f2)
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
# Textfile-collector metrics; identical wiring to the
# loki/grafana siblings. See loki cronjob for the
# atomic-write rationale (DEV-494).
{
echo "backup_k8s_resources_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
echo "backup_k8s_resources_timestamp_seconds $(date +%s)"
echo "backup_k8s_resources_check_status ${CHECK_STATUS}"
echo "restic_repo_size_bytes{repo=\"k8s-resources\"} ${REPO_SIZE_BYTES}"
} > /metrics/backup_k8s_resources.prom.tmp
mv /metrics/backup_k8s_resources.prom.tmp /metrics/backup_k8s_resources.prom
echo "=== backup-k8s-resources-restic finished at $(date -u +%FT%TZ) ==="
exit ${CHECK_STATUS}
volumeMounts:
- name: dump
mountPath: /dump
readOnly: true
- name: metrics
mountPath: /metrics
- name: cache
mountPath: /root/.cache/restic
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 1500m
memory: 1Gi
volumes:
- name: dump
emptyDir: {}
- name: metrics
hostPath:
# node-exporter's textfile-collector directory
# (DEV-494). See sibling loki cronjob for detail.
path: /var/lib/node_exporter/textfile_collector
type: DirectoryOrCreate
- name: cache
emptyDir: {}

4
apps/monitoring/backup-loki-restic-cronjob.yaml
View file

@ -53,8 +53,8 @@ spec:
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
containers: containers:
- name: restic - name: restic
image: harbor.basicstack.de/library/restic:0.19.1 image: harbor.basicstack.de/library/restic:0.17.3
# Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) — # Mirrored from docker.io/restic/restic:0.17.3 (DEV-493) —
# deterministic ingress via Harbor. Retag procedure in # deterministic ingress via Harbor. Retag procedure in
# docs/monitoring/restic-restore.md § "Tag-bump procedure". # docs/monitoring/restic-restore.md § "Tag-bump procedure".
env: env:

44
apps/monitoring/backup-restic-alerts.test.yaml
View file

@ -35,6 +35,8 @@ tests:
values: '-104400x60' values: '-104400x60'
- series: 'backup_grafana_timestamp_seconds' - series: 'backup_grafana_timestamp_seconds'
values: '-104400x60' values: '-104400x60'
- series: 'backup_k8s_resources_timestamp_seconds'
values: '-104400x60'
- series: 'backup_prometheus_timestamp_seconds' - series: 'backup_prometheus_timestamp_seconds'
values: '-104400x60' values: '-104400x60'
alert_rule_test: alert_rule_test:
@ -68,6 +70,21 @@ tests:
`backup_grafana_timestamp_seconds` sample in more `backup_grafana_timestamp_seconds` sample in more
than 28 hours. Expected daily at 03:15 UTC. than 28 hours. Expected daily at 03:15 UTC.
Runbook: docs/monitoring/restic-restore.md Runbook: docs/monitoring/restic-restore.md
- eval_time: 30m
alertname: BackupK8sResourcesStale
exp_alerts:
- exp_labels:
severity: warning
service: monitoring
component: backup
repo: k8s-resources
exp_annotations:
summary: "K8s-resources restic backup is stale (>28h)"
description: |
`backup-k8s-resources` has not written a fresh
`backup_k8s_resources_timestamp_seconds` sample in
more than 28 hours. Expected daily at 02:00 UTC.
Runbook: docs/monitoring/restic-restore.md
- eval_time: 30m - eval_time: 30m
alertname: BackupPrometheusStale alertname: BackupPrometheusStale
exp_alerts: exp_alerts:
@ -95,6 +112,8 @@ tests:
values: '-3600x60' values: '-3600x60'
- series: 'backup_grafana_timestamp_seconds' - series: 'backup_grafana_timestamp_seconds'
values: '-3600x60' values: '-3600x60'
- series: 'backup_k8s_resources_timestamp_seconds'
values: '-3600x60'
- series: 'backup_prometheus_timestamp_seconds' - series: 'backup_prometheus_timestamp_seconds'
values: '-3600x60' values: '-3600x60'
alert_rule_test: alert_rule_test:
@ -104,6 +123,9 @@ tests:
- eval_time: 30m - eval_time: 30m
alertname: BackupGrafanaStale alertname: BackupGrafanaStale
exp_alerts: [] exp_alerts: []
- eval_time: 30m
alertname: BackupK8sResourcesStale
exp_alerts: []
- eval_time: 30m - eval_time: 30m
alertname: BackupPrometheusStale alertname: BackupPrometheusStale
exp_alerts: [] exp_alerts: []
@ -117,6 +139,8 @@ tests:
values: '1x30' values: '1x30'
- series: 'backup_grafana_check_status' - series: 'backup_grafana_check_status'
values: '2x30' values: '2x30'
- series: 'backup_k8s_resources_check_status'
values: '1x30'
- series: 'backup_prometheus_check_status' - series: 'backup_prometheus_check_status'
values: '1x30' values: '1x30'
alert_rule_test: alert_rule_test:
@ -148,6 +172,21 @@ tests:
`restic check --read-data-subset=5%` returned `restic check --read-data-subset=5%` returned
exit code 2 on the grafana repository. exit code 2 on the grafana repository.
Runbook: docs/monitoring/restic-restore.md Runbook: docs/monitoring/restic-restore.md
- eval_time: 15m
alertname: BackupK8sResourcesCheckFailed
exp_alerts:
- exp_labels:
severity: warning
service: monitoring
component: backup
repo: k8s-resources
exp_annotations:
summary: "restic check failed on k8s-resources repo"
description: |
`restic check --read-data-subset=5%` returned
exit code 1 on the k8s-resources
repository.
Runbook: docs/monitoring/restic-restore.md
- eval_time: 15m - eval_time: 15m
alertname: BackupPrometheusCheckFailed alertname: BackupPrometheusCheckFailed
exp_alerts: exp_alerts:
@ -173,6 +212,8 @@ tests:
values: '0x30' values: '0x30'
- series: 'backup_grafana_check_status' - series: 'backup_grafana_check_status'
values: '0x30' values: '0x30'
- series: 'backup_k8s_resources_check_status'
values: '0x30'
- series: 'backup_prometheus_check_status' - series: 'backup_prometheus_check_status'
values: '0x30' values: '0x30'
alert_rule_test: alert_rule_test:
@ -182,6 +223,9 @@ tests:
- eval_time: 15m - eval_time: 15m
alertname: BackupGrafanaCheckFailed alertname: BackupGrafanaCheckFailed
exp_alerts: [] exp_alerts: []
- eval_time: 15m
alertname: BackupK8sResourcesCheckFailed
exp_alerts: []
- eval_time: 15m - eval_time: 15m
alertname: BackupPrometheusCheckFailed alertname: BackupPrometheusCheckFailed
exp_alerts: [] exp_alerts: []

32
apps/monitoring/backup-restic-alerts.yaml
View file

@ -19,7 +19,7 @@
# kube-prometheus-stack node-exporter DaemonSet (DEV-494). # kube-prometheus-stack node-exporter DaemonSet (DEV-494).
# #
# Because a CronJob may run on a different worker across days (loki # Because a CronJob may run on a different worker across days (loki
# backup follows the loki pod), stale # backup follows the loki pod; k8s-resources is unpinned), stale
# `.prom` files can linger on nodes the job has since left. That # `.prom` files can linger on nodes the job has since left. That
# would leave a per-node series with an old timestamp/check-status # would leave a per-node series with an old timestamp/check-status
# indefinitely. The alerts below aggregate across instances so a # indefinitely. The alerts below aggregate across instances so a
@ -74,6 +74,21 @@ spec:
`backup_grafana_timestamp_seconds` sample in more `backup_grafana_timestamp_seconds` sample in more
than 28 hours. Expected daily at 03:15 UTC. than 28 hours. Expected daily at 03:15 UTC.
Runbook: docs/monitoring/restic-restore.md Runbook: docs/monitoring/restic-restore.md
- alert: BackupK8sResourcesStale
expr: time() - max(backup_k8s_resources_timestamp_seconds) > 28 * 3600
for: 15m
labels:
severity: warning
service: monitoring
component: backup
repo: k8s-resources
annotations:
summary: "K8s-resources restic backup is stale (>28h)"
description: |
`backup-k8s-resources` has not written a fresh
`backup_k8s_resources_timestamp_seconds` sample in
more than 28 hours. Expected daily at 02:00 UTC.
Runbook: docs/monitoring/restic-restore.md
- alert: BackupPrometheusStale - alert: BackupPrometheusStale
expr: time() - max(backup_prometheus_timestamp_seconds) > 28 * 3600 expr: time() - max(backup_prometheus_timestamp_seconds) > 28 * 3600
for: 15m for: 15m
@ -135,6 +150,21 @@ spec:
`restic check --read-data-subset=5%` returned `restic check --read-data-subset=5%` returned
exit code {{ $value }} on the grafana repository. exit code {{ $value }} on the grafana repository.
Runbook: docs/monitoring/restic-restore.md Runbook: docs/monitoring/restic-restore.md
- alert: BackupK8sResourcesCheckFailed
expr: backup_k8s_resources_check_status != 0
for: 5m
labels:
severity: warning
service: monitoring
component: backup
repo: k8s-resources
annotations:
summary: "restic check failed on k8s-resources repo"
description: |
`restic check --read-data-subset=5%` returned
exit code {{ $value }} on the k8s-resources
repository.
Runbook: docs/monitoring/restic-restore.md
- alert: BackupPrometheusCheckFailed - alert: BackupPrometheusCheckFailed
expr: backup_prometheus_check_status != 0 expr: backup_prometheus_check_status != 0
for: 5m for: 5m

6
apps/monitoring/prometheus-backup-cronjob.yaml
View file

@ -48,7 +48,7 @@ metadata:
type: prometheus type: prometheus
backend: restic backend: restic
spec: spec:
schedule: "30 3 * * *" # daily 03:30, offset from loki/grafana schedule: "30 3 * * *" # daily 03:30, offset from loki/grafana/k8s-resources
concurrencyPolicy: Forbid concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3 successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3 failedJobsHistoryLimit: 3
@ -81,7 +81,7 @@ spec:
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
containers: containers:
- name: restic - name: restic
image: harbor.basicstack.de/library/restic:0.19.1 image: harbor.basicstack.de/library/restic:0.17.3
env: env:
- name: AWS_ACCESS_KEY_ID - name: AWS_ACCESS_KEY_ID
valueFrom: valueFrom:
@ -173,7 +173,7 @@ spec:
echo "restic repo size: ${REPO_SIZE_BYTES} bytes" echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
# Textfile-collector metrics. Same hostPath pattern as # Textfile-collector metrics. Same hostPath pattern as
# the loki/grafana siblings (DEV-494) — # the loki/grafana/k8s-resources siblings (DEV-494) —
# written atomically via `.tmp` + rename so a mid-write # written atomically via `.tmp` + rename so a mid-write
# read never surfaces a truncated sample. # read never surfaces a truncated sample.
# backup_prometheus_success rolls in both stages: the # backup_prometheus_success rolls in both stages: the

110
apps/monitoring/prometheus-deployment.yaml
View file

@ -1,110 +0,0 @@
---
# Prometheus data volume — RWO on hcloud-volumes-encrypted (fsn1). Because the
# volume is RWO, the Deployment below must use Recreate; a rolling update would
# spin up a surge pod that cannot mount the volume while the old pod holds it
# and the rollout would stall at Init:0/1. See DEV-583 / mirrors DEV-538.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-data-encrypted
namespace: monitoring
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 50Gi
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: monitoring
labels:
app: prometheus
spec:
type: ClusterIP
ports:
- port: 9090
targetPort: 9090
name: web
selector:
app: prometheus
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
namespace: monitoring
labels:
app: prometheus
spec:
replicas: 1
# prometheus-data-encrypted is RWO on hcloud-volumes-encrypted, so a rolling
# update deadlocks (new pod cannot attach the PVC while the old pod holds
# it). Recreate drops the old pod first so the CSI detaches the volume
# before the new pod tries to attach it. Same pattern as loki/opencloud.
strategy:
type: Recreate
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
annotations:
# Exclude the prometheus TSDB volume from Velero fs-backup — TSDB is
# backed up separately by prometheus-backup-cronjob and streaming it
# through Kopia would double-copy tens of GiB on every backup.
backup.velero.io/backup-volumes-excludes: prometheus-storage
spec:
serviceAccountName: prometheus
initContainers:
- name: fix-permissions
image: busybox:latest
command:
- sh
- -c
- chown -R 65534:65534 /prometheus && chmod -R 755 /prometheus
volumeMounts:
- name: prometheus-storage
mountPath: /prometheus
containers:
- name: prometheus
image: prom/prometheus:v2.47.0
args:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=30d'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
- '--web.enable-lifecycle'
ports:
- containerPort: 9090
name: web
volumeMounts:
- name: prometheus-config
mountPath: /etc/prometheus
- name: prometheus-rules
mountPath: /etc/prometheus/rules
- name: prometheus-storage
mountPath: /prometheus
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: 1000m
memory: 2Gi
volumes:
- name: prometheus-config
configMap:
name: prometheus-config
- name: prometheus-rules
configMap:
name: prometheus-rules
- name: prometheus-storage
persistentVolumeClaim:
claimName: prometheus-data-encrypted

62
apps/observability/patches/README.md
View file

@ -36,3 +36,65 @@ The kube-prometheus-stack chart is not currently tracked in ArgoCD;
if it moves under GitOps, fold these values into the chart values as if it moves under GitOps, fold these values into the chart values as
`prometheus-node-exporter.extraArgs` + `.extraHostVolumeMounts` `prometheus-node-exporter.extraArgs` + `.extraHostVolumeMounts`
instead of maintaining this patch. instead of maintaining this patch.
## `node-exporter-dns-config.yaml` (DEV-527)
Caps the `hostNetwork` node-exporter pod's DNS at three upstream
servers to silence Kubernetes' `DNSConfigForming` Warning event.
Hetzner's OS publishes four systemd-resolved upstreams
(`2a01:4ff:ff00::add:2`, `2a01:4ff:ff00::add:1`, `185.12.64.1`,
`185.12.64.2`), and kubelet drops the fourth because Kubernetes
pods are limited to three nameservers.
Sets:
1. `dnsPolicy: None` so `dnsConfig` is authoritative (with `Default`
or the auto-coerced `ClusterFirst→Default` on a hostNetwork pod,
kubelet still merges the node's resolv.conf on top and the fourth
nameserver keeps re-triggering the warning), and
2. an explicit `dnsConfig` with the two Hetzner IPv6 anycast entries
plus the first IPv4 entry, matching the three servers kubelet was
already picking, plus `edns0`/`trust-ad` resolv.conf options.
Apply / re-apply:
```bash
kubectl -n observability patch daemonset \
kube-prometheus-stack-prometheus-node-exporter \
--type=strategic \
--patch-file=apps/observability/patches/node-exporter-dns-config.yaml
kubectl -n observability rollout status daemonset \
kube-prometheus-stack-prometheus-node-exporter
```
If the chart moves under GitOps, fold these values into the chart
values as `prometheus-node-exporter.dnsPolicy` +
`.dnsConfig` instead of maintaining this patch.
## `coredns-dns-config.yaml` (DEV-527)
Companion to the node-exporter patch for the k3s built-in CoreDNS
Deployment (`kube-system/coredns`, `dnsPolicy: Default`), which also
triggers the same `DNSConfigForming` warning on every pod restart.
Because CoreDNS is a **k3s Addon** whose source manifest lives at
`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control
plane node, `kubectl patch` alone is NOT durable — the k3s addon
controller re-applies the source manifest and reverts `dnsPolicy`
back to `Default`. This patch file is kept in the repo as the
canonical description of the fix and can be used for a quick manual
re-apply (until the addon controller next reconciles), but the
authoritative fix is applied by editing the same `dnsPolicy` /
`dnsConfig` block into the CoreDNS Deployment stanza of the k3s
`coredns.yaml` on each CP node. See
`infrastructure/k3s-manifests/README-DEV-527.md` for the procedure.
Quick manual re-apply after a wrangler reconciliation reverted the
change (rare):
```bash
kubectl -n kube-system patch deployment coredns \
--type=strategic \
--patch-file=apps/observability/patches/coredns-dns-config.yaml
kubectl -n kube-system rollout status deployment coredns
```

47
apps/observability/patches/coredns-dns-config.yaml Normal file
View file

@ -0,0 +1,47 @@
---
# Strategic-merge patch capping the CoreDNS pod's DNS at three
# upstream servers to silence Kubernetes' DNSConfigForming warning
# (DEV-527).
#
# The Hetzner OS publishes four systemd-resolved upstreams
# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2),
# and kubelet drops the fourth because Kubernetes limits pod resolv.conf
# to three nameservers. CoreDNS uses `dnsPolicy: Default` (inherit the
# node's resolv.conf) so it triggers the warning too.
#
# We keep the same three servers kubelet would have picked (the two
# Hetzner IPv6 anycast entries plus the first IPv4 entry) so CoreDNS's
# upstream forwarder (`forward . /etc/resolv.conf`) sees the same
# upstreams it already uses; only the noisy warning goes away.
# `dnsPolicy: None` is required so `dnsConfig` is authoritative —
# otherwise kubelet still merges the node's resolv.conf and we exceed
# the three-nameserver limit again.
#
# CoreDNS in k3s is an *Addon* whose source manifest lives on each
# control-plane node at `/var/lib/rancher/k3s/server/manifests/coredns.yaml`
# and is reconciled by the k3s addon controller. Applying this file
# via `kubectl patch` is durable in steady state (the addon controller
# only re-applies when the source file's checksum changes), but if the
# source file is ever regenerated (k3s upgrade, disaster recovery) the
# patch will be reverted. To keep the fix on regeneration, mirror the
# same `dnsPolicy` / `dnsConfig` block into the Deployment stanza of
# each CP node's `coredns.yaml` — see `docs/DEV-527-coredns-dns-config.md`.
#
# Apply / re-apply with:
# kubectl -n kube-system patch deployment coredns \
# --type=strategic \
# --patch-file=apps/observability/patches/coredns-dns-config.yaml
spec:
template:
spec:
dnsPolicy: None
dnsConfig:
nameservers:
- 2a01:4ff:ff00::add:2
- 2a01:4ff:ff00::add:1
- 185.12.64.1
searches:
- .
options:
- name: edns0
- name: trust-ad

48
apps/observability/patches/node-exporter-dns-config.yaml Normal file
View file

@ -0,0 +1,48 @@
---
# Strategic-merge patch capping the node-exporter pod's DNS at three
# upstream servers to silence Kubernetes' DNSConfigForming warning
# (DEV-527).
#
# The Hetzner OS publishes four systemd-resolved upstreams
# (2a01:4ff:ff00::add:2, 2a01:4ff:ff00::add:1, 185.12.64.1, 185.12.64.2),
# and kubelet drops the fourth because Kubernetes limits pod resolv.conf
# to three nameservers. On hostNetwork pods that inherit the node's
# resolv.conf, this fires a per-pod `DNSConfigForming` Warning event on
# every kubelet DNS refresh.
#
# We keep the same three servers kubelet would have picked (the two
# Hetzner IPv6 anycast entries plus the first IPv4 entry) so runtime
# behaviour is unchanged; only the noisy warning goes away. `dnsPolicy:
# None` is required for `dnsConfig` to be authoritative — otherwise
# kubelet still merges the node's resolv.conf on top and we would still
# exceed the three-nameserver limit.
#
# node-exporter is `hostNetwork: true` and does not talk to cluster DNS,
# so upstream-only resolution is correct.
#
# The kube-prometheus-stack chart is Helm-managed (release
# `kube-prometheus-stack` in `observability`) and is NOT currently
# tracked in ArgoCD, so a direct DaemonSet patch is the pragmatic
# wiring path. If the chart moves under GitOps, fold these values
# into the chart values as `prometheus-node-exporter.dnsPolicy` +
# `.dnsConfig` instead of maintaining this patch.
#
# Apply / re-apply with:
# kubectl -n observability patch daemonset \
# kube-prometheus-stack-prometheus-node-exporter \
# --type=strategic \
# --patch-file=apps/observability/patches/node-exporter-dns-config.yaml
spec:
template:
spec:
dnsPolicy: None
dnsConfig:
nameservers:
- 2a01:4ff:ff00::add:2
- 2a01:4ff:ff00::add:1
- 185.12.64.1
searches:
- .
options:
- name: edns0
- name: trust-ad

5
apps/opencloud/backup-cronjob.yaml
View file

@ -6,7 +6,6 @@ metadata:
name: opencloud-backup name: opencloud-backup
namespace: opencloud namespace: opencloud
spec: spec:
suspend: true # CronJob deaktiviert
schedule: "0 2 * * *" # Daily at 2 AM UTC schedule: "0 2 * * *" # Daily at 2 AM UTC
concurrencyPolicy: Forbid concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3 successfulJobsHistoryLimit: 3
@ -91,8 +90,8 @@ spec:
readOnly: true readOnly: true
resources: resources:
requests: requests:
memory: "16Mi" memory: "256Mi"
cpu: "20m" cpu: "100m"
limits: limits:
memory: "512Mi" memory: "512Mi"
cpu: "500m" cpu: "500m"

16
apps/opencloud/opencloud-deployment.yaml
View file

@ -41,9 +41,7 @@ metadata:
name: opencloud name: opencloud
namespace: opencloud namespace: opencloud
spec: spec:
replicas: 0 replicas: 1
strategy:
type: Recreate
selector: selector:
matchLabels: matchLabels:
app: opencloud app: opencloud
@ -51,12 +49,6 @@ spec:
metadata: metadata:
labels: labels:
app: opencloud app: opencloud
annotations:
# Skip Kopia fs-backup of the 100 GiB opencloud object-store PVC — this
# blob store is out of scope for the DEV-573 data schedules; the
# exclusion is a safety net so an accidental defaultVolumesToFsBackup
# run never streams it into the Kopia repo.
backup.velero.io/backup-volumes-excludes: opencloud-data
spec: spec:
securityContext: securityContext:
fsGroup: 1000 fsGroup: 1000
@ -64,7 +56,7 @@ spec:
runAsNonRoot: true runAsNonRoot: true
initContainers: initContainers:
- name: init-dirs - name: init-dirs
image: opencloudeu/opencloud-rolling:7.4.0 image: opencloudeu/opencloud-rolling:7.2.0
command: command:
- sh - sh
- -c - -c
@ -79,7 +71,7 @@ spec:
runAsNonRoot: true runAsNonRoot: true
containers: containers:
- name: opencloud - name: opencloud
image: opencloudeu/opencloud-rolling:7.4.0 image: opencloudeu/opencloud-rolling:7.2.0
command: command:
- /bin/sh - /bin/sh
- -c - -c
@ -454,7 +446,7 @@ spec:
resources: resources:
requests: requests:
memory: "512Mi" memory: "512Mi"
cpu: "50m" cpu: "250m"
limits: limits:
memory: "2Gi" memory: "2Gi"
cpu: "1000m" cpu: "1000m"

2
apps/opencloud/tika-deployment.yaml
View file

@ -22,7 +22,7 @@ metadata:
name: tika name: tika
namespace: opencloud namespace: opencloud
spec: spec:
replicas: 0 replicas: 1
selector: selector:
matchLabels: matchLabels:
app: tika app: tika

46
apps/openwebui/README.md
View file

@ -1,46 +0,0 @@
# Open WebUI (openwebui.basicstack.de)
Self-hosted LLM front-end — [Open WebUI](https://openwebui.com/).
## Stack
- **App:** `ghcr.io/open-webui/open-webui:v0.11.3` (Deployment, 1 replica, `Recreate` strategy)
- **Database:** Postgres 16 + pgvector (`pgvector/pgvector:pg16`, StatefulSet, PVC 20Gi on `hcloud-volumes-encrypted`)
- **Vector DB:** `pgvector` in the same Postgres instance (`VECTOR_DB=pgvector`, extension auto-created on first start)
- **Cache / websocket manager:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`, `WEBSOCKET_MANAGER=redis` on `db=1`)
- **Object storage:** Hetzner Object Storage bucket `openwebui-basicstack` (region `hel1`, path-style)
- **Auth:** Pocket ID OIDC — `Open WebUI` client, restricted to `openwebui_admins` / `openwebui_users`. Sign-up via OAuth only (`ENABLE_LOGIN_FORM=false`, `ENABLE_SIGNUP=false`)
- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
- **Node placement:** all stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
## Files
| File | Purpose |
| ---- | ------- |
| `namespace.yaml` | Namespace declaration |
| `postgres.yaml` | Postgres+pgvector StatefulSet + Service (fsn1-pinned) |
| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
| `openwebui.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
| `app-secrets-sealed.yaml` | Sealed `WEBUI_SECRET_KEY`, `DATABASE_URL`, `PGVECTOR_DB_URL`, `REDIS_URL`, `WEBSOCKET_REDIS_URL` |
| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
## Deployment references
- Argo Application: `apps/app-openwebui.yaml`
- Pocket ID OIDC client id: `3ffbd06b-12cc-4562-bbe5-da4433962870` (PKCE **enabled**, group restriction on, allowed groups: `openwebui_admins`, `openwebui_users`)
- Pocket ID groups: `openwebui_admins` (`737f4b7f-63f2-42b5-98bb-4e96c4a80546`), `openwebui_users` (`74b0af52-0942-4cb6-b37a-83d87d203e32`)
- OIDC redirect URI: `https://openwebui.basicstack.de/oauth/oidc/callback`
- Role mapping: OIDC `groups` claim → Open WebUI roles. `openwebui_admins` → admin, `openwebui_users` → user.
- Initial admin: `andreas.leinen@basicstack.de` (member of both groups; `OAUTH_ADMIN_ROLES=openwebui_admins`)
- DNS: `openwebui.basicstack.de` A → `138.199.128.63` (cluster ingress)
- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `openwebui-basicstack` (CORS whitelisted to `https://openwebui.basicstack.de`). S3 credentials are shared with the Hetzner project (same key used for `outline-basicstack`, `basicstack-backup`).
## Deferred / follow-ups
- **Model providers (Ollama / OpenAI):** none wired yet. Add `OLLAMA_BASE_URL` or `OPENAI_API_BASE_URL`/`OPENAI_API_KEY` via a sealed secret when a backend is chosen.
- **RAG embeddings:** `RAG_EMBEDDING_ENGINE=default` (local). Switch to `openai` + `RAG_OPENAI_*` for hosted embeddings when a provider is picked.
- **Content extraction (Tika):** not deployed. Add `CONTENT_EXTRACTION_ENGINE=tika` + `TIKA_SERVER_URL` once a Tika service is available if PDF/Docx ingestion is required.
- **SMTP:** Open WebUI has no built-in mailer; not wired.
- **HA:** single replica with `Recreate` strategy; Redis websocket manager already in place, so scaling to N replicas only requires bumping `spec.replicas` (Postgres/Redis remain single-instance).

19
apps/openwebui/app-secrets-sealed.yaml
View file

@ -1,19 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: openwebui-app-secrets
namespace: openwebui
spec:
encryptedData:
DATABASE_URL: AgBZ3lRcF8KC9OApWa7g3oUoJG8DT8KG0/mgN6O1VwfnC5JZ9V5Lsp5sFtvYg5OXkFW4iBhzFu67n4zAR34EXwCSs977nt2uH7ShgZdrQSOlBTyaKO33ixOcY6WtGQVyJCzl0vjVHBKLwbsVkqEhBDLtqey1CZ9nAW/RZh8MahMSHhRNgJ4XtTyNK/5z2nmJyjW1oVwZab/cWeChlQUtJgt8qWxiQenlgu319P9plXrBDAPmDN/UKDbwu0ivH1s3sBQikuBYpCc9EJS+7NXBNnphsyJ90cF1oyqA5zt+0Q00fZnqE0qLdSOYbVCW058lK7NkeyecxeQwpFgPZ2XaLmU1rIl1wayYpYWCjZ9sLsImnIpFRz+0ouoWoKCXnJvgQ0inGjO5tCmMfHt0H6xTm/Etb3455raEujMcYbRD+mnC2UFmLHshuEb+X5JVUx+wfabVIFps8Vmf34OOf9yODvM127e8DdBtXQ5LAw9JP43tHT38+5XK9koCJIUtv0OQcsp9AAN5Y/BIPEloyt9iFWSQC/vNufYprzdeRGo5DkRvaJOij+s/nw3irOFw1oMF+V9+xTOfsgbxgn+gKJcNSyjHL2Z0gtm8oMLWHIxbjZHxpcY427cvqxITkb1am0QgNDLsTpO+Ie76qGtDiDdEZfgLNeGXaWbJEvIO6L71STj5RmdqSptMxquNIZkhGRz5XRaTupqcmyXAuIUCLIdejXMLBLXNjkOIPut6p8S3keGj3PdIO3AUFnLlpZt+idWMpn7sk07HKSj+qs3DdZcOpkHL0EH5A/qiNVXi9M1NoQgDqvEzF2KMYkvRyJJGwkyg+4snhb+rkV+ioQA=
PGVECTOR_DB_URL: AgAF2jka+S+kBgNuEGKupNdThqJesdGRTtcgo760wCMOza4yXJ/ivHMr8onLmykeZLV87Yv3rBKEYvssUciT/RD7QxCVEW31aKKPdi+MHg24/Fi3zXwQKoop05ksXizGVxWia5vrtG93j/XxWKi7JEb9glHe0Taj6TX0Hl+JsEBJELSTPrdngeOKzkYRFDvHP43AadUhvOFwxFoidbLGLulU9t3AdPpXb/T0wLNOcukZFpQxaB36MIIipaJl9leZ1sQ6C0e/MeimUYw3W5+JNf1P2lrL75dH/lqkkJZ0rLetSB7qRmpPyH8dMJumkv+16fkMJMt+LV64KdE2KdqtcxuXFO+RjYkMfzf8lFEtWPNA3qNuPXtwXZAfmLWEcjIJaTXv33Ed+3kAwiHil5Veh1q1eBhGQ0HKvkRGSnxxSyvVfoznLlLvNQVlNLgNfhd407K8ZOutushxaf/NJFW0RFdQcsq69SaQUFpIbv6IGaAM0TRN2O+qNatP3709U6dYQPYdGTJN4LOoKmfIIXaWhiqeYzOiP01QMhML0GvUVbwi3b3fK8twDx5O85Q8QSUgEOYkKnarSciUE/XYhQKi+E7Czv6vGbKmB3G5Oo9Bw6ONDvxd29kpkO2Ko9zwSFaqUooONy3PzkIwPiQ7Vp/ygsoFaVJNU1vsUWtDT8wJMVNcLPLCnd4ERhsMk/r+gY0SXZe3Slo4KIrqoAInf0Y1RBvKZ+XLQks1HxHmarA7uRe3Px/lMXm1kFRtlLwsNxwMUU5u65CbgehGphqq8UZRpByT8dFGhfnJK+rN4ORRX0Fd3fxgxtxyb5VgCe40rgQrH9LtdYculZDpRVc=
REDIS_URL: AgBuATUR1mMRwov35kA8dxOUqgl8O480/qxvsbBj3vt51Zo1s5cAzWBG0LlDLtAVh2NEAowSn+egj0mmLsAXJrndreQfAw+UpHDVHi4DbNbPeg7IUCivw/rMjLHyTMO4zsxqreKspwSNE38JU9vzgDBqv62HnQB7hzeGgpwUpsrWXiHNBji8xK/WJtIAGikbPEEzccDC0aCX9Ut/+Yb9ynv1AOtq/xMFIhsWTQ5r2vmbuMS/f59MNWMkWCZBApJL510+QUGzKXC/m553Mk9wHF4G+wX8dd2T4dCQxZ+A68gonvVeFiEYkSHzd5c7s+RYoSBGABrhJ7SziQOG5KWQSbXjjlu2o7JPmsxbgj9tpS0ErmCF/wzgEzd89DjtVNqJKV08nT8X4x3yb2L7oDtFPvEUZ9Ro+IpXYIR/ZnpXUcUgzIoaMJR9RsWBCrAKMFRlAtRsxWnHF0Mx+P6Di92mn2zyJ729FiB8QH6V2lmmko3xRhEZ9l7houxeoTG+37APzpD7U34VaO0W92ybvpcjfrL5BHZEXhpFqE3DXOXD21rf6fmxdOkNqJVJ45CXmweCbKzS3L/0l+GsTNrb8EZALj3RThcNMDgBTzpeTt4iHsodDXb3ziaVufbtw9BVCCrVNb01e2hemAgor5Z/TIkOxK7bLQhNTrpwvs2r7skhOUhzRNuusFi4tz1oFX0JO1Rj96X7Q37fQn5/fKkG6FeOtTIlLJ0mM8FoNijGiRkVgoU=
WEBSOCKET_REDIS_URL: AgBjDV7t/UTomB4Ml+7iX5pL1menP1c5kPZOSMlTBPaee6tXnPa7cdEs9/vu7PxfFtP03BczSoobXy4Q/4wvLe4sRAUHRQrcEk+hkJ7PbDT5n/K5zayaqlQJm5MQukytYe5EhlinsjR3EAphq5oCJB8NUAtdC0S+KQHckbxhUa96V7YEeHGu2U1oUJrZCCfQgdfHbFKY8FKm3lEd//uNxmmNovWcC+N4fwWmwj2Z9I68oNK3/H4E0GZwnL/1xxEDgaty68xB6wa7Cg3ZfeX9NLqZk0Xz5cyTziAPQ9rPwo/sShBnplDd7JZRR1podSASUEMhr904LRpbwHv+bfoEt6hUlq3g7gilF4gBVuMM0AwjWkdepFjLGSW++N7C16rei8l1uFthvVvMwdExd9dI8y2h+8ghkWfqwnFuomIxY9NgyAW4ijW7GmVfhKsnD0K9khUe8eiJtR4aqyuQWYsB7QkTMjuFPGZLe3qEg2q3Cl3h7H9tlQxUPqfvaN1Sb56rzQJV3dEMl55SAhoHrPzCjaTibZsuSkRW9djqEyc8lJ/PfTtln7aI0CG9h17WCc8cIAHHIhRIcGpNednIfmkDZjtrKVh415rdMMHZsX5W0/VRnKIkZK/nl0G2HDHw3MJVKaIk05mp0nPb7IFjE3i3t8p/jRgEhgdtIh/s0wma8OijzaXIbnKb8cpQ+AVYAh2TfEzU1bZuWRYzUck/lAzJ4xR4e/yWpsl9L/0tFN+TWco=
WEBUI_SECRET_KEY: AgBqvo0l+vmqUBvOtln8zf7UVDaVM739x2/xY3lCf/i1/GlmCK/3aGDLj6tqODFSXuKTKxMS3liFVDleR/7EdYvu9/7H0ThSfO2xhheLzXdooxpIGHnfriKL58UpyWcieo3Y6VIOAQzrSyjBSd6S3GHxjtBdqyqvuGSUXkojf+jOjOKC2yOf+0qySklm/LAww8MigPrM55rlUcqRwb0D/KdpAIxw/wplkAMdFz0RGGfF2Xr03K0AKqJ2egUb9exI0JDUyGQBDJv9isVQPr+2xZHvoSb1QcPCExL5FA4YLN/EhwKhlp56itlrGT44aT3Cx6e8k8cEeqUyP0/4Fjz6xZnQLnTt1dvFAmEAusT9riqTkcAqhHR87MmkPIYmlJLkud1aT8G3LpXUmghrQgYaam5G4DKbQbQ1SBJ5eNC19VNW7hcL5zOvdB+Vm41/OzM4n2AW+Oa0UjFMgWO27IVIGVNjwn2VKapUWsRHLkidGfeVQIYyWznXRFPiBhSwvHhO767Bt1TASst3R3f3bZDcjASLr+0VfRm0MbJ0iXZkK8IrQk0un8zHEZRwCWk5Jl/LM0AnS5NeRgO9XAPlXm8xv5Xy8D2DKsIZGMkXZWeoEJ57/l2i/mn9JMWhq+va5+jyHKQwrqAH9ZqZpWvmip+LeIrl/AWiKGSzarhSIw3DLin+HTJs0rNswVdU4FgA7CDi1Dg1rTW1bbDMrebSB8KNLYqrWlokW4MQ2d9FuzrN+UhMYKbiq/TOS26/oBlWvMtdyO2qVTHjWJPWUDpYny1yAVHJ
template:
metadata:
creationTimestamp: null
name: openwebui-app-secrets
namespace: openwebui

6
apps/openwebui/namespace.yaml
View file

@ -1,6 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: openwebui
labels:
name: openwebui

16
apps/openwebui/oidc-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: openwebui-oidc
namespace: openwebui
spec:
encryptedData:
client-id: AgBPXc6HeT9gaqtXK9W/Bo0Jgt7UA/2Aygb7bHmwXSJHjv00Ja/ciISsmHQD7fovmG6eoM/fi3fRdJZqJkekRX+3Od+09I3BYQGzXjNwBkplydPLPEuzqDcCLdRTqZCpDFIz3If08++23hyeH233Vp6BwDxcZXunqzzLMNY+i907QhGm0ULb+T54zmZ49NBElXv8/moD9cag807ySCDw+I5pcWQMIZRv1FKtso8aeA/KA8Mzgfmbhxe+3HpPZ5svuOhFlzHED8bZ39c1l8cakjPDuRvrrIXzwogNhuIUArmXIyBhgq9G5W0yTsvqrsep1gMWEtYnZSNgVkK3vzodH2oOwstsOMgKThd1c5bnIPbN3bkLxwd/TbyHmQdLSCMNwOCVo8XGbeRhSABMxVacLfDI7hR+WIyRdkiMwKKQnQMqdBynzcWWXeSlnzIBpS3N+oodO7gS+l+6G6KX2ZXyUyRlxdDo7m9BJhc+HhxJ66L8vVj1P8QKczrixWJUqrUIFxM3jKY1A7XOI/x20/gQeodCJWIHetKvIHwwQh+w63mwLP8gRFglBnjNFok2zeYNBo3eSv7zhXEYve0y83al86/TrLggSfI7juJmLcy3bCy98ZExXdf818JslqmDutavUkUeYn8wIvdkeZbRtWgYLCu7izJZ1UDN8zkP6VQuhzLHO2OrhUuKnwpiI7Jp4/gIrwuGI9Ahal1OX+8T6R8JrAd4lfz5MfyoNzFZ7qrp7xe5zlTZZxE=
client-secret: AgABqIXEErUfaN5XW7SxGr/yJgs4GoPufPmNZF4d34STcCi7s+kE181h985ubaL12mwt6zNTDxUlGdz5p7ABLePRf8gEMUtzFNom6BOqIiJjhQuWLJRR1//DIfu+9G1KjmSFKS4iLkoHV7bh2+b+q+u3NXFVfClV8bVMICN1Nzp+doSdwTAh2CyQTK5JXmu6jXDJxuOkJClQdEH+rwdE5/It8ZHhC3y568m5H1JO4dxgvBvZv7jLxX8b+hGGLD8N9P6zMRwiKUlup+UPCv1V/SjppAP5kLmPKumqhtWqEY0j4XIu6xsIaZz5WeIAzlpQoRLshd1VGk9ebfXN5SfXcJdHyZlH5zy2W9HL/iBmr5kv2RkzpH2Sludewk2pWPatXtEcE4N1sat1cR2TnfW4wxJySZ2ni2d7H5jPFNu6Md7MFVVneYhanSdt++CtWol5bikEkzuSk2kjPT7Bf1wRhkz68VLo3Faf6ltktvpHV6eGfAegW8HgTSYjzwA55MdqWNrLlN/iZfcfhR6+8A2xblpUFXBgvCkTzLQzM2d7gK9mr1j+6osWpUDzD+Os8NSsnPBM4ywm2xfW5ZSEHdX2TmF9s7hDLCffQfd+ILSre/uyxNXuSNesE4pC0bFF9OndE2IvIQkuMPy306NnCEBzZsp+P8J0NHIr7EKWg+IMFRsYMQrIl5Gv+opyTssr9fTMAUN9ns0LdQPxBnjce+0egHV6g27zg3QVIc6K6vsZNciYyQ==
template:
metadata:
creationTimestamp: null
name: openwebui-oidc
namespace: openwebui

279
apps/openwebui/openwebui.yaml
View file

@ -1,279 +0,0 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: openwebui-data
namespace: openwebui
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: openwebui
namespace: openwebui
labels:
app: openwebui
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: openwebui
template:
metadata:
labels:
app: openwebui
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
initContainers:
- name: wait-for-postgres
image: postgres:16-alpine
command:
- sh
- -c
- |
until pg_isready -h openwebui-postgres -p 5432 -U "$POSTGRES_USER"; do
echo "Waiting for openwebui-postgres..."; sleep 2;
done
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: openwebui-postgres-credentials
key: POSTGRES_USER
- name: wait-for-redis
image: redis:7-alpine
command:
- sh
- -c
- |
until redis-cli -h openwebui-redis -p 6379 ping | grep -q PONG; do
echo "Waiting for openwebui-redis..."; sleep 2;
done
containers:
- name: openwebui
image: ghcr.io/open-webui/open-webui:v0.11.3
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8080
env:
# Core
- name: PORT
value: "8080"
- name: WEBUI_URL
value: "https://openwebui.basicstack.de"
- name: WEBUI_SECRET_KEY
valueFrom:
secretKeyRef:
name: openwebui-app-secrets
key: WEBUI_SECRET_KEY
- name: ENV
value: "prod"
- name: WEBUI_AUTH
value: "true"
- name: ENABLE_SIGNUP
value: "false"
- name: ENABLE_LOGIN_FORM
value: "false"
# Database (Postgres)
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: openwebui-app-secrets
key: DATABASE_URL
# Vector DB (pgvector)
- name: VECTOR_DB
value: "pgvector"
- name: PGVECTOR_DB_URL
valueFrom:
secretKeyRef:
name: openwebui-app-secrets
key: PGVECTOR_DB_URL
- name: PGVECTOR_CREATE_EXTENSION
value: "true"
# Redis (main + websocket manager)
- name: REDIS_URL
valueFrom:
secretKeyRef:
name: openwebui-app-secrets
key: REDIS_URL
- name: ENABLE_WEBSOCKET_SUPPORT
value: "true"
- name: WEBSOCKET_MANAGER
value: "redis"
- name: WEBSOCKET_REDIS_URL
valueFrom:
secretKeyRef:
name: openwebui-app-secrets
key: WEBSOCKET_REDIS_URL
# Object Storage (Hetzner S3-compatible)
- name: STORAGE_PROVIDER
value: "s3"
- name: S3_BUCKET_NAME
valueFrom:
secretKeyRef:
name: openwebui-s3
key: bucket
- name: S3_REGION_NAME
valueFrom:
secretKeyRef:
name: openwebui-s3
key: region
- name: S3_ENDPOINT_URL
valueFrom:
secretKeyRef:
name: openwebui-s3
key: endpoint
- name: S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: openwebui-s3
key: access-key
- name: S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: openwebui-s3
key: secret-key
- name: S3_ADDRESSING_STYLE
value: "path"
- name: S3_USE_ACCELERATE_ENDPOINT
value: "false"
# OIDC (Pocket ID)
- name: ENABLE_OAUTH_SIGNUP
value: "true"
- name: OAUTH_MERGE_ACCOUNTS_BY_EMAIL
value: "true"
- name: OAUTH_UPDATE_PICTURE_ON_LOGIN
value: "true"
- name: DEFAULT_USER_ROLE
value: "user"
- name: OAUTH_PROVIDER_NAME
value: "Pocket ID"
- name: OPENID_PROVIDER_URL
value: "https://auth.basicstack.de/.well-known/openid-configuration"
- name: OAUTH_SCOPES
value: "openid email profile groups"
- name: OAUTH_CLIENT_ID
valueFrom:
secretKeyRef:
name: openwebui-oidc
key: client-id
- name: OAUTH_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: openwebui-oidc
key: client-secret
- name: ENABLE_OAUTH_ROLE_MANAGEMENT
value: "true"
- name: ENABLE_OAUTH_GROUP_MANAGEMENT
value: "true"
- name: ENABLE_OAUTH_GROUP_CREATION
value: "true"
- name: OAUTH_ROLES_CLAIM
value: "groups"
- name: OAUTH_ALLOWED_ROLES
value: "openwebui_users,openwebui_admins"
- name: OAUTH_ADMIN_ROLES
value: "openwebui_admins"
- name: OAUTH_GROUPS_CLAIM
value: "groups"
# Runtime
- name: UVICORN_WORKERS
value: "1"
volumeMounts:
- name: openwebui-data
mountPath: /app/backend/data
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: 2000m
memory: 4Gi
startupProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 60
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
readinessProbe:
httpGet:
path: /health/db
port: 8080
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
volumes:
- name: openwebui-data
persistentVolumeClaim:
claimName: openwebui-data
---
apiVersion: v1
kind: Service
metadata:
name: openwebui
namespace: openwebui
labels:
app: openwebui
spec:
type: ClusterIP
selector:
app: openwebui
ports:
- name: http
port: 80
targetPort: 8080
protocol: TCP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: openwebui
namespace: openwebui
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
ingressClassName: traefik
tls:
- hosts:
- openwebui.basicstack.de
secretName: openwebui-tls
rules:
- host: openwebui.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: openwebui
port:
number: 80

17
apps/openwebui/postgres-credentials-sealed.yaml
View file

@ -1,17 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: openwebui-postgres-credentials
namespace: openwebui
spec:
encryptedData:
POSTGRES_DB: AgC27bglkH6N5Ule5F23XbirvuqWgiQlwd8yyqEqpcZYsvT8605y70g2mS4zZBtvVqdYFcORJF5abdqWdVhmPhZAV71tesJBTOynMQb9Zd39EbBcEb/6Shresa7FdaLQgaQGLxIOOt1xWLmh/eRv9bLiIGNtz1ezKCa9fXYuCrD1Eg0EGmUjdioXbkuVoVNON5rBwUTWWpfVq7/d/UNQGUvAuhOX2RORD4hy/5IRZG5LvAF7C9oK1LaCbi5B65kbC9DOtlCPfB499tt8fSQDYic8YBLkmPKQgHd3+8NlNrH+TtTZMTO6UOR53IqJW/xxoJZapAl/40XkFj7Q59X+xDzG+p81DfwwOu9U+Fu5a2zpdduCAVQCIRSVZBL/ziYkRjFR08/5ToqHF9+p0C6khIgt1EkyvyUTxZfKPlcq68soY9cl4UzV3UPe6DyyEXwKsPmaUXPG4ckGEAWIIIoEd4M2cFN6jyWTRyYiWfe9uaPSIkjBGi8TIXk+Tk2ZTcZG5UQd48MOtc9gnXQwoFo2JzAqqBYa4HcATzPPiDzQCKhMXjGelINzExqvjOm26L8Zi5yCsaAoP3Ev/Kedl3ohdWW7CpwTxcffnd+CTYo5oz9K8HD0CvS2gc46HL2zCzZSF8Zpap/0bsM4FWFcEEI8ru9mdkhXZ9EygI97fcxtAi5eDtYm2UjKRDzCLLb3a+eorUEKfpXPUTG5E8A=
POSTGRES_PASSWORD: AgAislLVvgaIxtm+U9LzD1lK0Rz05MaiphMWHuHVpGayAqTorWy7jz+iSI4aQH7QQFzClv6WbSgzcwSjYiTZhH9+GxpRHpIAcyYbjHVG8qU3n99A8WXXCd5V8K4alQK4z5kA6OxZGOuYRQ2mfAbHGdAUBNIVTqNv7DtpNuATKzZbkLd/tMX3ezIcpJqOEtb0AqgMvcH0gsw2j7I04WBlCdH7qQ6e8EUqHjxWBc1YMF7INzAxlkVgIcaNkpfcErEL4y7X1E1yo0KRzAg0O18rzUqs+hOi1XVSNaEuA0yno+a1ZXzGnmiwJFK+BboL3BZUtev6Xlbe3DqcCwUjmw3v18suO84gXGfLEJa/OYsys1vgHQ4qt0tP9guSWaqnTfdkc2ZVilw7M+d2TS5PtQC0U2/MW+SNmS8EdC+/q6FvTgaMr2uptqgC6hrrLFemQ1KFzJibut1s5o7HuLKqD915gSbHTPJ0vtPOstIJo11ggmvLw37JGqhbHkME7SwkJzZXWAFPvAWfsJTRqL8fzdk1RLlVYQeak7VnX/iL5q9bUomsGvYXMV90/DH3vdkhFuxOUx1kdjnROtx8PYg3rnzXPvfR49+x4EhUWRqaF1XDXWdWhCKrZFrSskvcm7h01gHvLQBhbKK/X0nOz6CM/Ht8zzIsnsmjssoZaz7iFrvYYyhSWyp/U/N9GdFkcDA6qzRsZSM8WGnRnEeQIHklzIL1fZlvsPl57rFjHSYZzdXD1Iolzn9v8KU4vt7kGxOiPh6Abfg=
POSTGRES_USER: AgC5pjM5LU5zyzqvfsrf1u5koa/IwO0wTnDUv4GcEzRLKD/d0Z5iDlT6bdyCS1bFr159NWUyz0FyqfMDqsGFzJiGwEd6Bq2PHaaa+h5wfhY/KguVPndb2SCRJWDcxddH85GGRo6iTkVLeNBO5mOI3imQnDNyKaBpowRINoLNnWDoTEZ7COrTrLTWBlJz8ar4JyIWAToS08cBIhQqIBCWJkUpXSS52Frju3yYKBCeR0lDBrL97WghqnM7cD38luPXajLZeysck8iAwZm2Wt57ZmLUCLSo9cxkhL7Wx/mV2A/s43KzL0m7w+YkQsk6x8anug7+hr2lKAzb58ytKkyPsJ/OirGB9fRfIujEyU6B75AFDePtvv5W6Nf3ngWn4hH25K9dDA+LyMKo0KVXQi5uMnVFitwMUMRFLFlnyreydt/aQyG/G+XjE1TZtBgjF2OHdPKSaxqOh9F2vZ8GLRNV70EVVOO54/4lqeLChNRBbuxgn5v1MtD8lOUMtoS4Ho+82995MARCzZeukfP61tqMbKvTn9HtbbQq7jBfoWCUgCY4KeiQ9sf5+Ks9jfmgDvdgBckiO0IJCq0oaiVvUGW5fkLwa273sypVo6sglLoa1I+YCkUJ+Ym7nOw51r+zATrNmzzAG8W04ynz5Jx6iF6mS0K29giqAx0POxFpw72CPI67COYLQdXr4b/UoSAlWWwCH0eGCJUfcA/LYIE=
template:
metadata:
creationTimestamp: null
name: openwebui-postgres-credentials
namespace: openwebui

100
apps/openwebui/postgres.yaml
View file

@ -1,100 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: openwebui-postgres
namespace: openwebui
labels:
app: openwebui-postgres
spec:
type: ClusterIP
selector:
app: openwebui-postgres
ports:
- name: postgres
port: 5432
targetPort: 5432
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: openwebui-postgres
namespace: openwebui
labels:
app: openwebui-postgres
spec:
serviceName: openwebui-postgres
replicas: 1
selector:
matchLabels:
app: openwebui-postgres
template:
metadata:
labels:
app: openwebui-postgres
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: postgres
image: pgvector/pgvector:pg16
imagePullPolicy: IfNotPresent
ports:
- name: postgres
containerPort: 5432
protocol: TCP
envFrom:
- secretRef:
name: openwebui-postgres-credentials
env:
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 1000m
memory: 1Gi
readinessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 6
volumeMounts:
- name: postgres-data
mountPath: /var/lib/postgresql/data
subPath: pgdata
volumeClaimTemplates:
- metadata:
name: postgres-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 20Gi

89
apps/openwebui/redis.yaml
View file

@ -1,89 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: openwebui-redis
namespace: openwebui
labels:
app: openwebui-redis
spec:
type: ClusterIP
selector:
app: openwebui-redis
ports:
- name: redis
port: 6379
targetPort: 6379
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: openwebui-redis
namespace: openwebui
labels:
app: openwebui-redis
spec:
serviceName: openwebui-redis
replicas: 1
selector:
matchLabels:
app: openwebui-redis
template:
metadata:
labels:
app: openwebui-redis
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: redis
image: redis:7-alpine
imagePullPolicy: IfNotPresent
args:
- --appendonly
- "yes"
- --save
- "60"
- "1000"
ports:
- name: redis
containerPort: 6379
protocol: TCP
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
readinessProbe:
exec:
command: ["redis-cli", "ping"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["redis-cli", "ping"]
initialDelaySeconds: 15
periodSeconds: 30
volumeMounts:
- name: redis-data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: redis-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 2Gi

19
apps/openwebui/s3-sealed.yaml
View file

@ -1,19 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: openwebui-s3
namespace: openwebui
spec:
encryptedData:
access-key: AgDBruwsmBhUuqOg34A8jWTT+9Ko9K4erxigpVtcQRxfvoBu0Jn9O/W80XDPO4+HD6krMlxZyPb0uP/qwsLU99Ll2gcxsUDVOvhJwkqzI1ohZbl+iueJ7qG7Ty5HO8CKCe8AiDgXnEW5/hWUxYT+0lX23aIv3SaJdhOHaScMCw5DSGlR4/JA43uEfK+Rbx+FDYokw17pi60LAhuQhZpbzoy0sgd4A5nKaSyNRis/YPnIZzN5gb8yyuGYa5fHSLGnowsuAlfj4x1MzNsw8pzuygnti6KsoMJyWnqW1TsaWb8WkhWDRKEN2AO4tTD+2BuvixV6LBynomkLfwZ8DSmN0Di5Db3bKGEkECiRAloeIc4efr5L5W4ucAxPE1YFOM7VINt5k/7pdymQapGRyQymzEsC6dWZNl54SjN65zzZIerkM2N2II0oqyzhtYW9WZ1ihPYvXSSp2JoaiKny1ljzqyXeizX/Xkl4ZjUZETYV8KIIFMx3SwTGBvShGwmv3T/v7XnMDxG7lIvjtHvCmcaiW3fegz3reckuczuG/c4rtKQo+bzPnkWznKlXVrHNg7Hks9DisIQPngn/u1bJPK8ncEYaX0MRGIIqb5yn8eoT9Ggxk9eY7QHym9Y/bjFMbR5MMR7I5fSQSP2cUI1K6OQ8eqkteCUB2i0wCfDXHqnkE44NJM38hWUl8b3rNH2yo2gnLyYhkaeqdetqNbW992quRUp7UXshrA==
bucket: AgAm7zODlaJ10GoHumXsgXJC4An3GNegi/Gm3aNhKlyNV3ItVU8q43/UPMck3ed14XZ1gbMUIkkIU8L9IIIofwEam4YrfQYfoQomtaYF2EHnRjH56+xrnRP+WKzTbqDpiZIcRfr6E1+ajBM70H7A0TfztLiPmFNJaGv8YPLSdWh8E5m9vgiJJ4ErEDwMxMrhLpfjZ6iNp/H9vchPLOtSP3CtAJvrrdWO8My1xZX3bnMpoU9YLlSSfPAGpglZ5dsxeRBA+HSSRo4PouhLKMm7PULk4eFFwtetNgPh0DwHXhD7hboRcZJECmZMPz2xdWwu8+KqKmMywTaN+gy6IDOw7pZV0hHwVd2AWhlE5dXnV5pVEm0vUKbC5ceS4Swq27RphO5g5/7lfnkV1I9JqTPETFXCydxQ+vhW8UxbEunTWp7PxE/XCNMvYBvecksWEmLix3iVFGj2chH2tnDLx/m/K2HxQkqNo25c1PnDO3aL5WTSjcK9VWSwrv4ucoROtDA4MiD33FCSF4eBwzUm6AUPuWIIsoobjGOKZcwNiRDDtom6O52h6+0/xdplrNdj5X/5GHe7jnEM/IwaYdNsYBu0/Hi9eohA453brJfGzShe9mcKw0TWktq9xrmfk57PuWAKny6dg6VWixS0YHNPd7v1EkANmnm1nagNMcurScolwgY/MXYEbtQbbHPk5Zn/yQGrDCFoHSkE83gC7go9dTvnmEi4anDfYQ==
endpoint: AgAlRlwvCwNLdPXgWgpSYTz7MhtX+wsK9TNCep0C6oGJi6HMlEzT8s1YW+58HTGmc5wFkoB8P48JDwcfdb7BjAgFMVgkiBz3WrbtYYgUda8EeiD+5Kt04lTcetPibi/R/sbe1Hvxp2SIsJ4Qgt6x6qElR0GM3hfiavP95mazIL6zqX0MImXtKFEqGBwhSiFIQ2w0ldbbBxpl2B0w9JpOO0f9TxEZW4do4JyKkTSfUnBpJdpN1tqDP+j18PV/H7s7O4zdpVWmwlp8l96xSVZtg0VZvVvdC201mPuAKF7EV4bIy6/a6jIgKeUv0k0aeZqqTgeNkGmCzxI43ujShMxVEuNaDDVt1PAzcTl2r8IImyJlf/B3AipgyGpwXUzn2+A0DiBIqjOXC1dezkezS0+iq5yekO20d3N73KswIsWJu25Dvc5j+c97nPSQmOYGcoYtZmoIac2GNf+UTApbLiJdzSaP0hQZ4c0p6g+xtCMjesLl1xI1ihMOJtetW/mDsKokGWfACWcNdkN4mKbgBHtOS8GvfK13NXpDRDfgl95avhVgj0hdGANre3fSk7cmEX4fcFCyyj3wUqHOBK98oNL8zaNVmHjEDoajw5RI6EwHOM1lnnUfMyEuJT+Ye6vRCmJTLoMrVqCOUkcX0rfIA7+VRJjSoeeoz7+iFJcsNGH6HIdE6rjOH9HdC4v/L5wNH5CUbiPLG8tFcFiFF2k9EUg6V2YymkrgCM8WZb2PN0QQloohe1VkRQ==
region: AgBWLZOWPXCo8H5D9fKTqK/UWSOGsB32Swr0H3mhoOvZntLkqtxR0WsToH1hYzLnsVz41O5ntBRiw4nx00SvKs8MdKT3rW2gcuJcJMznUMXN8WoWLfUIrRHO5xRR4bh3BDeqaJo80iGL0Ql1/460BYwf3F5mBnKDEBdbaJCNKraRGLUVSOJlFFkebzwTsgOszQgOOZFFA3oRUNgZugHFzeOq9HYYqf+XQ1WSA5AyKjgR2xYK55vOiw/qag1keX31kWGgB039M5IcNhdI1kVUrQWgBeDWU4zp30983FFBaxJENKqTXEhtkfJGgi33+kPuMGdz7KQfNShd6bQNVpfGeX9ZsTPY+ZIwlsiljQzuJ2FTu84Xnmf3jNsoDuOnq4BmpYHV7+t9iGPFmiiTHCYN5hTlNWptLvBa0ZYE1dOwEz41ZEEisj8RAMWBNBiHZDBPyVEvWjkmld0Lro71F6XGb50jlaQlOV31b3zVksatzgnkQmtNfeZucHY6AyxIrwlbw+UxTIJ1dfzFBRyrPAdGZamUgyqZLqXWkJmwy6ktD/tRJrfpso3QjkPjAlCXYWNIp34Qx7GHAf4t8jaCJx+qjlpSuQ4iv28xFL5xug0I02cfvZwV441ANVP5i4ATnVpJ/TIb1zbUtl5ffuGrD5fY58DQ74ntsHSQ8h8JIpQDAu7WL+ZSWjGpV5tBUz3vIP4wsDJnHN7p
secret-key: AgAHkBSjTxORjuS5i75zd5Sk3FytIYwfQTgRs64LUHl2SAD86znxUh+4T+AGCO6dZHoKp+madkx84ghUn31LdA/2FRfXrrujxHLlom4LKHkv/jDc4hSRtJPMBqpi0KUXUtXk83nJ1s1BN5er93S60IiaYZOBcS0XojTfh+TbwlKJJsKstJcOB1PNVb0V4/OC+fSWpOJNkSI5PAEkzVVcSFsEaIuIRpAazwm7hBbnvWCGwd/WlSzZEHNVnf/fpI2QstVYGe42fg0NzEqnFk/I5QGRcKhUvM3wQ5smNidb2mors8FE0UzdvFcP4cytYb7tDf0GKclX4p2kmc7Js7EWTRmin1N31EVdGPGsJ2sV5u5JQoAg0b381H4ZqQckInB8NIoAx5L2p1gqnPKPt2yTV192jes7COzNhmODTVaSKY0z3v//u6tVpsanfiDvkSl4y47BKeJfxp4Ihd4wJNgN3vkaxi4mDYBiNiY2QXSczZ+2qyacZeGowl1ZpGH13KSg6bOV/hQ/T0tr0lbwiZ04V7v5pORAw8SnvHT6rqzHLrE6PxDiLQsY36bIpMvqPLNke0ErEfqXMwGeYClij0fpU/7qZ5t9Ag2PvClroxY3wuLdpKE9ffeHtH50BgyabOS3DsMXcB1pODyXf2QaEzB5NaGqMp/PEbmoy3JLYwM9We5NCQObxfjEI6gkgBitBY1PDBbU6DGfTM3Smf8y6qJ6Y4kHx31VUU2eVHP7+O912OH9yzguyq1DpkiI
template:
metadata:
creationTimestamp: null
name: openwebui-s3
namespace: openwebui

39
apps/outline/README.md
View file

@ -1,39 +0,0 @@
# Outline (outline.basicstack.de)
Self-hosted knowledge base — [Outline](https://www.getoutline.com/).
## Stack
- **App:** `outlinewiki/outline:1.10.0` (Deployment, 1 replica, `Recreate` strategy)
- **Database:** Postgres 16 (StatefulSet, PVC 10Gi on `hcloud-volumes-encrypted`)
- **Cache/queue:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`)
- **Object storage:** Hetzner Object Storage bucket `outline-basicstack` (region `hel1`)
- **Auth:** Pocket ID OIDC (`Outline` client, restricted to `outline_admins` / `outline_members`)
- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
- **Node placement:** stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
## Files
| File | Purpose |
| ---- | ------- |
| `namespace.yaml` | Namespace declaration |
| `postgres.yaml` | Postgres StatefulSet + Service (fsn1-pinned) |
| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
| `outline.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
| `app-secrets-sealed.yaml` | Sealed `SECRET_KEY`, `UTILS_SECRET`, `DATABASE_URL`, `REDIS_URL` |
| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
## Deployment references
- Argo Application: `apps/app-outline.yaml`
- Pocket ID OIDC client id: `b4a84bbd-fae4-445e-868e-5c1197e5d155` (PKCE **disabled** — Outline's OIDC plugin does not send `code_challenge`; group restriction on, allowed groups: `outline_admins`, `outline_members`)
- Pocket ID groups: `outline_admins` (`2b41fc67-eb7b-43d8-8954-85426214f1a8`), `outline_members` (`aa6df56f-b868-4de1-adf7-8696e990aa53`)
- DNS: `outline.basicstack.de` A → `138.199.128.63` (cluster ingress)
- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `outline-basicstack` (CORS whitelisted to `https://outline.basicstack.de`)
## Deferred items
- **SMTP:** Outline runs without email notifications. To enable, create/reuse a Stalwart mailbox (e.g. `outline@basicstack.de`) and add a sealed `outline-smtp` secret with `smtp-host`, `smtp-port`, `smtp-username`, `smtp-password`, then extend `outline.yaml` with the `SMTP_*` env vars.
- **Admin promotion:** first Pocket ID user in `outline_admins` becomes the initial team owner via first-login. If the admin role does not stick automatically, promote in Settings → Members after login.

18
apps/outline/app-secrets-sealed.yaml
View file

@ -1,18 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: outline-app-secrets
namespace: outline
spec:
encryptedData:
DATABASE_URL: AgA0XHjlu1iWM7AJaStmEtMdVtYMwKzXxk7/nMCtIaNxaXTa7XH3K40277Wc5a/mfM4KLiF6iLExquNIa7YI6R3EROpUC23cTJbstpQNP6tffzZwr3FeTeod4qWYJ3HfIHaBlocNc0p1ntM9H/9k+Vy3rQCn9T5kLJ1c6LqUGhPinWwQxwiDgNU6JQyEfILIAX35th5fe24GH4Ka10k56wq5Fnx9CkYYxsVQ21NNKCdtqZecR+C2RqfYhXharvtio1ojWP4KNQZuXEXGDAH/nb7QPtaNPCV8hhsS66FAjm3hHTOmhJ4XXbJQpWYCiPOWBa9MVHxMg3+IDfdR2OWLDOpgFzN/vMKx+iqa3aVHXfdrXcFUqEC48d0h3MXjdCge4c8wY469PUTar+74d4BcljjG6R1FL5CJ1Wmxygt6rG2Qk+pWihZ00Frc0gttsxowS7c9s8iPL6VF1ShyaTXdYq0W7pHlX/Yq927C7h8E6TNQ+Rxw3UI4qPXN7xkU85FTQvWZjqoyyej5GaPbGy7Qp/SO52wsq4Fh5qPrZKoGl5xQVcmuF4LZKbb8WQDQcFDatHdWZiHZqsX1wKxRP0jR8RkT3Q7Z/M2itMN9lODtddCv/hjrwd1ih950kaZoVW3cdZCVb+S/M4pbkuB0BAasRbntJx/hOy/9zp21MAWfTGTyscBetIjHeC/oy4S1WQyo5TwkMSeugqXJ/mn/zTEpDYrxuqso7Aeu9H658WeenwgUVh2VB9SVWFWxZnHe0rB+keMPDwJ7Dx7NsdydybsRmORtTIc4zj+OsqDhmg0wwCq6pSslp+/h9MwfUB8YkYv9ioPP
REDIS_URL: AgCKT7MxyQJV4Yk+I8TO52x8x5nfXd0ECB9H2w8p2gRFMyiTbrTMvdzaF9fqrgTiGRuB3VgDeeQpsnFGvemoYWqbLZ+WI1UfQKlyar5y3tHh88GMoI53dmkaBc6V7Fr353rkcy9NmRJC+g37V5PWasiX5aWFSdYoslmaUOC/o4pQwyDZEwo3xFHysbm9tD5jGVSsCgnUGeiG0Urva8pNrM3aTfvSoVOhts312eKk3+x+8mda7ZchH1DtQ9EZ1V8emEEjOv1Jh6gh5wLjl710cHAS0ch/9RTj8ooCH//pX4V5gj0ue5EWL1vk0xgV+kv5tGG2pwR4aUZWa1d1eQzbFDshHwkbAWKm0pjCq6eIMg6QLQRdagGXe4ZKsRAahUJ+7Qo1MqDXthoJHDEDDhB+/SbbPKTOrgm+eh3oA3EuWDcp+iKAqboBc1WkzUWh7kRVRt4bkVBHkeK3UjFvjGtuwC8cI2yOQ3DyRh7Ija+Fp1o8ust86FKYS1/VUQRqBX2HTBamognfS+EN4PyrHJ1XZzoL85njGm7v6nOG+EQN5QTlRkmLTLqLVWzUk9tuQbpm57r0ULqbkMKYzfnaj6opWo6EsCH2uQg2KGMVgxAhY6jFuFc8cLXdz44Fb7tjTlSgq1Z5S3hqHRrP2fd5upjFICP/ECa+JRb6ZGpTSvIJ2MpH/0oCTFkgl5+A2KUUP7PDNkqCoVhujRLS05XuSize+owTbjeEz7fJCRy66A==
SECRET_KEY: AgBYcN97oeke6wlfUel6cki6sYIUyM2zJUuQfBSSFoZRawSurDhL7MboCbVXXtgold8QMp1aF541442TO0DdR1eMvOvNqGHIQog3sNDlbjr3LOk3vV7FcZFoME866tsQ0uzfIR5Z7SBvkD6EqdnrIe+sNfSQHt1Jn3/QCPhgiYbsN4tPD1t7HFAuuPmI/gs91wnv40r7PGxK2cYYcPgGrT1P+prjSsca5sbVvy2e/jkW8FWLoRxrNw4Q7FImgDPP180Adt2V0n+7HfY8u5gSS94oY+5CBmWuR7W6GhStuI/LfKDDVVgWdKtnP90Eq9NiGRg1Xk+EwjNLWSm9aTLZ/2h1jzDGqmrJ75W+Z5OHV2bdgxCFb7lZJRU1mDTCjqMNf1jKEzSQ2pFYHHmyRhz0dQ7FVD7wUS3SSTkC8epxxRf+s8Jb2O67x5sM87V/QPyMGlCPs/634OOwj4+7V8owcc+NaC9iiGTjwd+6AurFCF3O2t2YcYgsJXoDp7Rd0hf/kk3cEKAhKMqO2DDAew9Og3fpJrF2mgYnMJb5fDAaEq7FrA8y7P9a1vyL6nZMmfXZcZ/a9g+r7uoFzoE2wxsaCpmlYvgep/X+a1iQgE2+LwjB+lquYFH+Ztam5HOneY7wNMD8EpLbyUq6RO54+f/1COipeLZxyUc8tI1zMlsWlBKNSl0qljZqA3HAE3a49xQBPTiEHlfsLk9me55zkR2t/H9+HBlJ5LQvGb43sYhVssSfv7mYTAzEOJPiAjUiLfpg47cu+dt+c5x4qenFmqUDh78J
UTILS_SECRET: AgCAxEUaZS1TqU9HA2a0Ze1lxsP97DYgyUnWFaVAWc2qSM3sqp4BffN9ZJ8mVUXkZdZXmY5Yoirui/p4z/Ie1BpZBH5dbcuVGWu1DopCcs0UROpftJTWLKLZjoYGxp6RbazUNSVc500KvLuwMX+E0wG7OUSAt+nRN65/ebHRWl4kg1DceFLVQVmies/uHHz1++1jkVTGrr2ClLdmj0z/VYc5dYifxouYUEgjKWzB9ERkso/J5CCjG7hRRTbAeEaRH/SeyV6qw9kvhgCK8Kd0wicJatu8EcbHBDZs+0IiQ6vkqkdQE47MlFqfwem/lLWM9ce9rGBMkJ/rcMTGgW+pP6GKaQVKrhP3+0G6lz7IHneFvEF+W5tEaRjf7VbQdbgoX4lfykjPQ8pk3cIGPWLOrFQTxdl5EOhwS3dQ/WoAQs/GLgDsNUO8TlQziyURGCJT/LIfXvEVZsbOTcrbayd3ywwkL5aGN0TAcnWb0TPXgZVhw91N7Z7e2gRoGx2CoSpZzjv0k3liBrffCRYvvc85HrQnKtd8l0uutpCrtVOStc1wqRA5g0s9j0bgqspvQp7CGfyxs7vnGP6PMBoOtgH8mR14oI41SXTUn30uRWzGR3yebGLRXZ6ORgtzB8lQxDZAo6VUcc4afzyL8RXdhtJNmr/goRXnMg8LQqq80teQxFat9RW4Wxjqpnhb2xNwFtOxYFK818q+L7/7CC9MqaQcnYwn+mJUlMQTam8rPI1v1VRLfdzXcmHgQCXrlaTAMFOBF5zPO2vr3+OES/XS8r4K9V4f
template:
metadata:
creationTimestamp: null
name: outline-app-secrets
namespace: outline

7
apps/outline/namespace.yaml
View file

@ -1,7 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: outline
labels:
name: outline

16
apps/outline/oidc-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: outline-oidc
namespace: outline
spec:
encryptedData:
client-id: AgAvHl9m4WXOYPIFG7rrYzPeA0Ksm2DUs2DS9gJismHLL5fX63e2KaD66W18NIFZQvtkfe8QelvBi3J2YAH2I1qrdZjn0SVJSq/7cdR+0X6lb71/hAkbHUr2xS+uOV86/O+6/htLhdNl0IcAjvyNMXWh5ptVBW/pbTQHJqGYN1dHHDQKri/3XIpxipKIXL22oPnowv0jcW1tgTZPPFkFOpahVlVP9n0pxM/2X3bb/bbsNVuTk0tWeDTqj0XTUN/0Kowq8P4rJFVdbJPyyV/37PnPlu3+F1i/flApo3HKQmoKR09ogCS6uZEP10SO7lmz0+JCMCIxYAHr/ELaynEXWhsCNJsLTE+TtMdMNciy7n8TZ75cS+Azgjhc5cimreimmxuFI1NRuMHnLMCLam1Hve2190sPhcIT+O5Revrc+BKsSXuscDZtrUugOChqYM9EW0T3AwmbIAyBQsY3MoLc2wFuq1fpYMx+Yu/0ajPBZvUV2JRbVt5YtMHoo3VbtKXkbfZW9IaBGcBFDxg7Dd8cZju4ljieRWJm6hISO6OPrMjvGG1IvBTv9c4SEGhnCHbjpYeoB9UPlL/FKVQgyrK1v5qhnZ0ZoiKYecxfj6Jytrf/qLdrBZDlCHghKpn3OM2OeubOak+xH1zgENK1UhjhkuqELoBXy9WDTVkOvrm5Tiw4WAAUm9jmjQiKusdwUGMPBt49sXOfyzsu8iPWIb8FDhH3DLF13d9wH7vtwlGQBuxhC1+FiSI=
client-secret: AgAHp4jsX5igXGHwKiQJuOKgyIIOssntobUhHnIKtSrNDK5mBZ0EvvwijvF9bqOp2i542WxVcjabk9a06/h6QCXymmo9Lcq4kK3cZyrQ6V7efjNFhogHOnE/f+y+lZy3AawTd1uokLiURf9z2X9rxi+2ppLhqExNFa7W3WIuEoVnk55H6nMMOxBK3SFfXi7PQUt+gJWGdZRMltVnySjqnqVbHwsO73uNoHLrOI4pEUeQiPaSQngkQRGa5LAYm/A6jYGYkTFQtvT2gee+aLTITzynzwdUmCdrJ0GcL/bi122dI3f+Kpkdj61Z2G2VZY2r2blXtQZcm2vt2j+4ZRWwMejWtpnpfBAnFm2wThvFiXqIYit8AtXu6isuj2bmwqtgZT5ieQeop7ieu1LBuqCutNkn+coxdmQWH+/Ct5oYl7GTHsPWqGlXF9Irel9J05SxZGmCtjw7sdXeGXwRItP7/eh6attCJZJJaPCD5/1/cNSVoLrifBQnNV/NoxWYzXsrYpz6LdNq38kK8eEB/YtNWMoCLN/NUtY2SbrJBlQChdWNeovxz9D3ddujFb+792iQ7qHcm2DGAEX2fKqeYTFY1J1Rp/eDF89UMocjJ5pgvfdjqYNuGXdARYxjBJybMskS+MhNd87JkjNAyRDDhU7hIe09Q0KtR9eeoe8wQ+8R2oOXoivAsm/KmyNGDTKPM5xUCvDABHjaaMmHczSAAdZcc/dYnrdgMGPcONeyVkhPgr90Fw==
template:
metadata:
creationTimestamp: null
name: outline-oidc
namespace: outline

255
apps/outline/outline.yaml
View file

@ -1,255 +0,0 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: outline-data
namespace: outline
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: outline
namespace: outline
labels:
app: outline
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: outline
template:
metadata:
labels:
app: outline
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
initContainers:
- name: wait-for-postgres
image: postgres:16-alpine
command:
- sh
- -c
- |
until pg_isready -h outline-postgres -p 5432 -U "$POSTGRES_USER"; do
echo "Waiting for outline-postgres..."; sleep 2;
done
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: outline-postgres-credentials
key: POSTGRES_USER
- name: wait-for-redis
image: redis:7-alpine
command:
- sh
- -c
- |
until redis-cli -h outline-redis -p 6379 ping | grep -q PONG; do
echo "Waiting for outline-redis..."; sleep 2;
done
containers:
- name: outline
image: docker.io/outlinewiki/outline:1.10.0
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3000
env:
- name: NODE_ENV
value: "production"
- name: PORT
value: "3000"
- name: URL
value: "https://outline.basicstack.de"
- name: FORCE_HTTPS
value: "false"
- name: ENABLE_UPDATES
value: "false"
- name: WEB_CONCURRENCY
value: "2"
- name: DEFAULT_LANGUAGE
value: "en_US"
- name: RATE_LIMITER_ENABLED
value: "true"
- name: LOG_LEVEL
value: "info"
- name: FILE_STORAGE
value: "s3"
- name: FILE_STORAGE_UPLOAD_MAX_SIZE
value: "262144000"
- name: PGSSLMODE
value: "disable"
- name: DATABASE_CONNECTION_POOL_MIN
value: "5"
- name: DATABASE_CONNECTION_POOL_MAX
value: "20"
# Secrets
- name: SECRET_KEY
valueFrom:
secretKeyRef:
name: outline-app-secrets
key: SECRET_KEY
- name: UTILS_SECRET
valueFrom:
secretKeyRef:
name: outline-app-secrets
key: UTILS_SECRET
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: outline-app-secrets
key: DATABASE_URL
- name: REDIS_URL
valueFrom:
secretKeyRef:
name: outline-app-secrets
key: REDIS_URL
# S3 (Hetzner Object Storage)
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: outline-s3
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: outline-s3
key: secret-key
- name: AWS_REGION
valueFrom:
secretKeyRef:
name: outline-s3
key: region
- name: AWS_S3_UPLOAD_BUCKET_URL
valueFrom:
secretKeyRef:
name: outline-s3
key: endpoint
- name: AWS_S3_UPLOAD_BUCKET_NAME
valueFrom:
secretKeyRef:
name: outline-s3
key: bucket
- name: AWS_S3_FORCE_PATH_STYLE
value: "true"
- name: AWS_S3_ACL
value: "private"
# OIDC (Pocket ID)
- name: OIDC_CLIENT_ID
valueFrom:
secretKeyRef:
name: outline-oidc
key: client-id
- name: OIDC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: outline-oidc
key: client-secret
- name: OIDC_AUTH_URI
value: "https://auth.basicstack.de/authorize"
- name: OIDC_TOKEN_URI
value: "https://auth.basicstack.de/api/oidc/token"
- name: OIDC_USERINFO_URI
value: "https://auth.basicstack.de/api/oidc/userinfo"
- name: OIDC_LOGOUT_URI
value: "https://auth.basicstack.de/api/oidc/end-session"
- name: OIDC_USERNAME_CLAIM
value: "email"
- name: OIDC_DISPLAY_NAME
value: "Pocket ID"
- name: OIDC_SCOPES
value: "openid profile email"
- name: OIDC_DISABLE_REDIRECT
value: "false"
# Persist local cache directory
volumeMounts:
- name: outline-data
mountPath: /var/lib/outline/data
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 1000m
memory: 2Gi
livenessProbe:
httpGet:
path: /_health
port: 3000
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 5
readinessProbe:
httpGet:
path: /_health
port: 3000
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
volumes:
- name: outline-data
persistentVolumeClaim:
claimName: outline-data
---
apiVersion: v1
kind: Service
metadata:
name: outline
namespace: outline
labels:
app: outline
spec:
type: ClusterIP
selector:
app: outline
ports:
- name: http
port: 80
targetPort: 3000
protocol: TCP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: outline
namespace: outline
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
ingressClassName: traefik
tls:
- hosts:
- outline.basicstack.de
secretName: outline-tls
rules:
- host: outline.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: outline
port:
number: 80

17
apps/outline/postgres-credentials-sealed.yaml
View file

@ -1,17 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: outline-postgres-credentials
namespace: outline
spec:
encryptedData:
POSTGRES_DB: AgCbJr6ng8tQwTMyNfrvubnlA5CvamfSZtzR4AwuIn/UUcK6mb68B5ucCUyA0GeeFVOGx63rlibDCKb42Da0AHrSYDTzZ1YbYAiwEzdlrmgh5tCfMiqbcUHUqSR4L7QAjkS50ASQXPXxSHuxuYa9cO7EQCasN7Fm5cmTkbcGHTKOSJ7fE7VQDBXyf6Z4Rga8HBys20xjglDFy7meBLEXDOKRUkQDC/PGgqlF98ZVtMH148ziaRxiu2tkbpsFchQ/zFc6x5rtllYGnlic8ssbgAXwUL6WZO6CcSbjhSkq4JcIOJh60GudsthTaW5HTldrD0VwaLXw6BNSD9+7+Zd1UIKSN9+imBOCmpOglLG2LFCxUD02lr3sPBdlztQwZnvxvFhfKRXstn/Qo8ujzKlRFyHdLsX0IQf7vj9kb+Giq54MAgelqivl7HMg8O73DIHe/GUmwDr3cXt1+aVoPKnQHe5wkQ4bIsTDzkNZzig1Lisl/bBxF2tf48ZXP5QZXpcL80TX6ZLxh0u1MVTpmIT6fZziw8lOJfLdE917UokNPnFVWx9X7Z+zWh07b/AMIx1Wa5JNKl1rpmJazYY42pvEGU2crR0NETNYavnM95Pl9sbRh1f/grp3v8Y4J31Oyvy2+MMChe1FdUbOnMN3Aozc3OLFuu94aiJFd1r2YU/1VCSa/0VuaDcCPwtFvKf92B/ADh0HT0Lxe7VW
POSTGRES_PASSWORD: AgAGj5wpTMEcDHT20f1SKTd+J0ZSC6bhfBP2WOSa3zBbxqI7Rb0OEuR+tsuQJyEbL8uKPrKKvgcjf4/GGTNChpkoe/n9Z5Vsc5cvdel59egtIca3R/2g+EhUqGhrb/R4cVMylqANcpK3O25GJNL6K9TfVU03qlORazVVp2qNr+cQ/fWUHqAJUz77ui6KPjGTjsrKH4Vnh5eP40ZoYRDXOmM7AACgMfw4dtaUID/QhslaGE8vi0o/sv/MyhC0Pic98V/cwmwPgQCYgH44v56EtOR7X0Rqz08so3GGi4hvrQCs1i40Sd2k0SayBNmvF0LGO9stlkDIpmMmnsEeCgxk0u5L9tRxsDtvGX4AAiU9FByWhVp4JRYd4QAF509p3aF+1/3kNQen+4aHefy4Xzpx1dDQOZoWArz/kXhyh8u+wkkDwsf/6VXSORdd2w4M2lDyhP61pGY8FC0QOCKWNkfOTTgFOJf8guqwSfmLNdS+lhJmtpgKpNwPnG90CgV8b6xd3uPXYa6FQPd7I9sFpN4I8ZC9SJZE4karU7Fys/SArdBB2proemw6SYepBvo41bcdftxYpL5RvfCkFjLj6CDTYbQtjPzV0YeG03l1qKqbfiKroC242mGprJweGdC4kM4L28aWaMq0oGg6og7u0A84YIukQlncIMAQ3t4L78Zp/OCOWbkPkWykjUVCkOtxMSnIjfWvFVN8vr+zN0AD/PwPu976D1RdCWXzXyZu1gWZmnBEUQweYw1mFMsaqAxQl000UWc=
POSTGRES_USER: AgAqZajhuIMyXzZtwm3C6KCN6njShH2tdlj4U8IsZxc3v0rHdtlCDfjk4db/fkrmRZXGzJmjizVfMwlBjVAXlO7WiGpZ0eDG/UQMIR7f56UD40Ccj2rJOExw0tB/mnfCmXVVMsUyeA3IPJLY+lUB40IE6nw7K3n8A5XpyaGTCPa8QFlCHvwpQ/seDNykA4sc+NX4So5sPUS3h/NFo9T1XhI6prvDWOd5d0YEFYBoHSgQU/QcIizT6sgudVl4cKtWtRrJdRSaq6msREB4hFfc4DnaxGsB3fLmsfUC881nkSc25ayGvnJXvWhFCnIy5v15cHFT/Yo8yHqVQdWVgJWKxb8fZfLGtXuRGXYoLHbH+VGhAcPUDSERmBoyCKG4ABGWkXG6+xLG7meN9q2/u0zbmCoYZRSxflJIUOWaZIGuvJq6KmpKsEJ5A58M90Y308oSS7yHyX3pgGJVV3zWZC6ddOoPPGbpipxAESUq7XCJnTth9SkIttlGBIgiFZ0wErL31LileoEGvhMcv5H5MSzebZbfW+uJIMRJF5QfaL8BS7UCmdI9j73VxT7pxupxKeuQCJ/zsF3B13IR/+w1hEwNW0TugqRlaqF8agU0jCLF0tlD5aCnrqHJp7MB/zD3Lb9Pk30mUEeye++kx4kqTE2Vf5sPAHbIjBlOvm/y1lKxDjASoab+I6mp4/fTBe4+S+Unqw+R7uWxBRUb
template:
metadata:
creationTimestamp: null
name: outline-postgres-credentials
namespace: outline

100
apps/outline/postgres.yaml
View file

@ -1,100 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: outline-postgres
namespace: outline
labels:
app: outline-postgres
spec:
type: ClusterIP
selector:
app: outline-postgres
ports:
- name: postgres
port: 5432
targetPort: 5432
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: outline-postgres
namespace: outline
labels:
app: outline-postgres
spec:
serviceName: outline-postgres
replicas: 0
selector:
matchLabels:
app: outline-postgres
template:
metadata:
labels:
app: outline-postgres
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: postgres
image: postgres:16-alpine
imagePullPolicy: IfNotPresent
ports:
- name: postgres
containerPort: 5432
protocol: TCP
envFrom:
- secretRef:
name: outline-postgres-credentials
env:
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 1Gi
readinessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
exec:
command:
- /bin/sh
- -c
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
initialDelaySeconds: 30
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 6
volumeMounts:
- name: postgres-data
mountPath: /var/lib/postgresql/data
subPath: pgdata
volumeClaimTemplates:
- metadata:
name: postgres-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 10Gi

89
apps/outline/redis.yaml
View file

@ -1,89 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: outline-redis
namespace: outline
labels:
app: outline-redis
spec:
type: ClusterIP
selector:
app: outline-redis
ports:
- name: redis
port: 6379
targetPort: 6379
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: outline-redis
namespace: outline
labels:
app: outline-redis
spec:
serviceName: outline-redis
replicas: 0
selector:
matchLabels:
app: outline-redis
template:
metadata:
labels:
app: outline-redis
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: redis
image: redis:7-alpine
imagePullPolicy: IfNotPresent
args:
- --appendonly
- "yes"
- --save
- "60"
- "1000"
ports:
- name: redis
containerPort: 6379
protocol: TCP
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
readinessProbe:
exec:
command: ["redis-cli", "ping"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["redis-cli", "ping"]
initialDelaySeconds: 15
periodSeconds: 30
volumeMounts:
- name: redis-data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: redis-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 2Gi

19
apps/outline/s3-sealed.yaml
View file

@ -1,19 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: outline-s3
namespace: outline
spec:
encryptedData:
access-key: AgCMbJDgvGRG0ritkltoJF/nFpb7p4JRbl57Tje3EUx/+jCZDqhvM2J6DAxLL4ELy6xINuk7bByMU4Ry5z6lmYTyZ6UvgYpJIkrg9xS7dA4WU2474oGbDWx/u2xRvI0M3t/JqlfkShaX/jxDVSibYGpGvN/+uZ/F308EvUOai/PLgEz2VpA/6y/5gtFdrDjDFoBLs5eZsrBPDAIweKQ0s+5wiPlHv+vDhIvw99efS4wiZZQXuD7j7zKJIxwGKG0k6VGIoRm207xxvHoSXmxmhZp0I6whKJ4XJqXo/S1nzQllQWOaaUyA2Y+sGXfjbcm1YdNN48/MZK6uWJNjXaIphmW6DehW8+i3aJLbv/XSmbxcY9fowRyEfFz3aqqtGW7gP4CzUc+0DwQhfE+9z/lBoWOTKsDBf8kUYnF5GPCMB1TjAAP12MWCsBmecP9kppMawH2SmRZbl+ImZgiQTXlJ79gNRoF2ri4TF2xz/tcjP3Rnt82p8yV3gj/NDsIMLUG1GPiNDMdz8dNR723V9pb+lIoNUD+pb7+Jr1N2wCq21YKNlUGPC6ZXK5flQVwlKXbxNSyu1KEyhzgEnRjKBXTQOBlRcUkkaXNbA4FreuyvfCJcRkScf3YsfzJQcxxiCxK+A5UDd9sgiVwi3rYYfQhwzPUDcvjVMTX93qZaKPfmfcxg0STV5M+A5pAkSVvs/8ORS0HoPqvXZRlH/fOT040n4z1b4ALnBg==
bucket: AgAiu0w47e6zbhFOX7dOGgDJVRAy5ZDfWmjgYGMibCCFm+FglEmKaj5Vi+70nZMsUigtH98WKKfEdSsJ4+mGuITNSxU2+p0uth2D2JapkEmOtzbzMzpagoXLnRV2xZW/HKyiUhRct7ygawUQ6TA3A4JBb3G/6WK+T1AxF2Z0NtDTULmM+mx8a4XNE1Anmuo6zOUQ2GIUfbBOI6RCBP693fVI8m+eB4bzkbmwrTXdT1bHPQ95wWbX28wDI3i9PQwRlM2oM5OYOTn46l3V+5mwPTUntuGuMis5P1jH4O7EkUObCbeVsC4S4orZ/sYPBs580MXFzBGeKM/Ce1fGVICC6kMWqt1SFnlYvEGRej1qj8N6uS5lmLNG6HM6Z3nHSiM/6Pi3LzfvnEvum8x6qmqRGsergIHTAcaHBOOiVDrjRKRqlCTRDmOjoTbB0vD00cWXAFGTjTXKsFtrcAmzq3EF3vhrA69Vf2FeJsF4m+ixlFdZd7WqrL5H2W7tDEHtbMhMHiii5cfHkKD9orLFpfMzirSR+sLdeRzR5t5pvGIkPRVECEHU+8w9vWSxm2aSFXOJUH4nOMM0TNzlJNlD/vsDL+Z/JQjQDzBKyBIHp1hKo45JxjwgmZKjsDy4Ts8CEwPp/zPwSFY96hqFVVs14PqjWb17y9oCpca181rQ+7bmrW0F4OCXTGg4wRBxk0ikDB7JbwCzZ1d7VqNB5lOfzeDPgZq8d98=
endpoint: AgB6zhkvtiqfkoSMExAy5Imd9LrBkC04F7Ifx/YSGfGtcjFSNsACNNL2vnVKUyKWvuDPa+ptdJh6HLUvmk5D9myMxlH8Bc7p69x4/wUqvwRiBgwm/4Gge3XlYa16KzMJIhkaVu6UxNSEI3c8S94Rvv7uiCk+IDkROUnutVC3dzRE6OiFz5GL56AO1f7bOj7t+0Rdpq7rSDoBq1e1Wftm6xnHa56BGXGlpaH/7UU0h0MDlnAyAbE4Xi04FLk3s7gjPPA4tetbtEBEBbKTA7GokWpt4rXrqivLr4hKd/OSklI9SByd9SxpdbO2m6QrunIqQDjjHkDrPIOAelM1MOVSTJKG1JWfXqywA2l/WeqIGEQt8FW4wWCU0E4XKixsJdFEDjtFAOyVJPYoyObaVUvkXrZdxPuc+KFtocSXR9KM0CaPQ7jteqOfYI+4VDwvE/YRVHJrolUzqBpu/pKkHnDDottfJ/O7OJCGl+jfQ1ZzP4ovOVH+GIBmo3rjVWZM+f8KlA8HA6NAdAWuSE8z3phP7n8N2ZzICr5KsW0vNSI5Ue7/mvjCbm9nUushAuzvFX7s4YH1KtwQrToMJTylfUdxyBjLzJ2NOOsg7ZiaWw9j9tjrAv1SRhPps/i7cy2Fg6Z1qeX+mvWgeIjKaQvTdYH0HREUXpzAMe1UWUkCJQtCrk8nk7gigmxF3T/fO0AIxMQGknRKYdqp3Hr06M09P1WF5QjrLU7uydWbFhA6Js5DWd3d2MurpQ==
region: AgBFM4EAaxptRlqN8EnsrmeX950mS10T7ZlfZmdpOsQJSR1Z0CzN5gmsJmVlznrRqw1FGGXU3feBrg/eA01IP5ykqq06iLWyjla3oLVWaYzwK/wASiY7Ycl2tkuyZwpOnQNWuxmXFX2WHl0NNKbd+mklmBxaIPRQVdYqtDAUeblyTII8oBFmv1zc4kpf4MCCHhsCG7m4VG+b+EI/DZZRXK5LFaVoOFy1pHGhN8um/OerDICHuOleOQ+hWj3w5mkigdKzFZkeIG9RQ0WnvS1muwECKvanDBaS0RGDxt+Ipm09WPk4VyGAzHyDd5wOd/uckD/yYFzAodjlZHDQipgNE1MAs43gqt6X3lKNAwqOEc6TLmQv6FUTU9+R7J+2FOgSKQ/ACTVyGHiebOqjb2L9bk1oj/j0cKARRQyRyL8B/TFHPVfyPH+kiTmRa+qlsg7FN7oTeFSOWaNZbA0jz+ONM9IzRpE33VHS/r9cCmfe2rFySNXCjHIkiKJC9qMYBqorM1kM4Ab00hlfhdO0aspTW9KC0pEauPs0n84sl/HHbOQ+sLcYW7QBvZvuRi1OyVM99XoKjI7Jnu7Aq55NX+rt9zZ+iJpnXeUm8hxCKC/CyUS5mAEmqQpJoKqLVMfhvpXMn8I5lXs3YpshHto649EPTVS6A23YBG4Q3toZ8UmbXp2JtxlL+JnfU0/mZvAPOBkfrmx5MToa
secret-key: AgDCSueSKDzmfI/5zhEusMX6AaIpjF526UdnWdTnmUfsX+LW+w/uQvgM0aPIoR/sMyEQoo/UxouU3V1PQIHj9Q/3acDuFBkXCFFb02dFv+gXF3SAEGnCaHviH8Bce12SfMePyOlKT2eDI8vTyehw3istEMTJHc7rMr/PNZprKG1T35wxdaklS88uLP5oAnHVKeqEvAvgNFXWrMqJ1GM1l8/rTd5GCJ8xsE1mGU/26Ac518nzo2PMARZYwZ0Rfiv2+QZswT7DDQNH6ZoY0DPzL+SW9ecMGc933L6EejNzco1esttEtRa+ZJRZ2tqUbm8LTZOFHq500VRtbEAcAtySMtjxX0VF4sSGrxGGooEkY9mqOcy2yQLZdb9UGObF8wMBXcGx67SAcXbppWeNSOvp4p7QVrzVbNXIbnV+7e8XrKBHf9inBW2j9oCgPjPcTKW7VfHOQKIYKwnHS81hgOsTQcX+BcYT9Bkea4E+/R0ovYJh68bd3Wwcm/dECnj3y3Kzgj2j4/xv7aO4cNacAt1INILYY9P0edYHbCnO4gHXt9itvKTbiOslYRuxfgff490ReXunkTl/FFawmwCZoVY7nlp59IKwTB/PahPctGmrbHstwI6UK5kLaB93Cf6bNengLCt+GYAmuuiTAOqfTLvGRQBu074mqWAO9ThYkdU0ebtOO9sY8bALk2ZlSdEN0T5Aqmp+LaSz6W91nLDF8/XaGD6zIpHHjwF6oCPg6BPn//43Rx/unbjJf82j
template:
metadata:
creationTimestamp: null
name: outline-s3
namespace: outline

4
apps/pangolin/pangolin-deployment.yaml
View file

@ -35,7 +35,7 @@ spec:
spec: spec:
initContainers: initContainers:
- name: wait-for-postgres - name: wait-for-postgres
image: postgres:17.11 image: postgres:17.5
command: command:
- /bin/sh - /bin/sh
- -c - -c
@ -58,7 +58,7 @@ spec:
name: pangolin-postgres-secrets name: pangolin-postgres-secrets
key: postgres-db key: postgres-db
- name: render-config - name: render-config
image: busybox:1.38.0 image: busybox:1.37
command: command:
- /bin/sh - /bin/sh
- -c - -c

2
apps/pangolin/postgres.yaml
View file

@ -57,7 +57,7 @@ spec:
fsGroup: 999 fsGroup: 999
containers: containers:
- name: postgres - name: postgres
image: postgres:17.11 image: postgres:17.5
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
ports: ports:
- containerPort: 5432 - containerPort: 5432

2
apps/paperclip/deployment.yaml
View file

@ -20,7 +20,7 @@ spec:
fsGroupChangePolicy: Always fsGroupChangePolicy: Always
initContainers: initContainers:
- name: fix-permissions - name: fix-permissions
image: busybox:1.38.0 image: busybox:1.36
command: command:
- sh - sh
- -c - -c

1
apps/passbolt/passbolt-deployment.yaml
View file

@ -33,7 +33,6 @@ spec:
containers: containers:
- name: passbolt - name: passbolt
image: passbolt/passbolt:latest-ce image: passbolt/passbolt:latest-ce
imagePullPolicy: Always
ports: ports:
- containerPort: 80 - containerPort: 80
name: http name: http

2
apps/pocket-id/deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec: spec:
containers: containers:
- name: pocket-id - name: pocket-id
image: ghcr.io/pocket-id/pocket-id:v2.19.1 image: ghcr.io/pocket-id/pocket-id:v2.11.0
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
ports: ports:
- containerPort: 1411 - containerPort: 1411

123
apps/stalwart/stalwart-allow-cluster-ips-job.yaml Normal file
View file

@ -0,0 +1,123 @@
---
# One-time Job to configure Stalwart HTTP listener to allow internal cluster IPs
#
# Problem: Stalwart blocks the HTTP port from Traefik's pod IP (10.244.2.227),
# causing 502/503 errors when accessing mail.basicstack.de
#
# Solution: Use kubectl exec to access Stalwart's admin API via localhost (which is allowed)
# and disable IP filtering for the HTTP listener, or allow the pod network CIDR
#
# This Job must be manually triggered after Stalwart is running:
# kubectl create job --from=cronjob/stalwart-allow-cluster-ips manual-fix -n stalwart
#
# Or apply directly:
# kubectl apply -f stalwart-allow-cluster-ips-job.yaml
# kubectl wait --for=condition=complete job/stalwart-allow-cluster-ips -n stalwart --timeout=120s
# kubectl logs -n stalwart job/stalwart-allow-cluster-ips
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: stalwart-config-access
namespace: stalwart
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: stalwart-config-access
namespace: stalwart
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
resourceNames: ["stalwart-admin-credentials"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: stalwart-config-access
namespace: stalwart
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: stalwart-config-access
subjects:
- kind: ServiceAccount
name: stalwart-config-access
namespace: stalwart
---
apiVersion: batch/v1
kind: Job
metadata:
name: stalwart-allow-cluster-ips
namespace: stalwart
spec:
ttlSecondsAfterFinished: 3600 # Keep logs for 1 hour
backoffLimit: 3
template:
metadata:
labels:
app: stalwart-security-fix
spec:
serviceAccountName: stalwart-config-access
restartPolicy: OnFailure
containers:
- name: fix-security
image: bitnami/kubectl:latest
command:
- /bin/bash
- -c
- |
set -e
echo "=== Stalwart HTTP Listener Security Fix ==="
echo "Configuring Stalwart to allow internal cluster IPs for the HTTP listener"
echo ""
# Wait for Stalwart pod to be ready
echo "Waiting for stalwart-0 pod to be ready..."
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=180s
# Get admin credentials
echo "Retrieving admin credentials..."
ADMIN_EMAIL=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-email}' | base64 -d)
ADMIN_PASSWORD=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-password}' | base64 -d)
echo "Admin email: $ADMIN_EMAIL"
# Use kubectl exec to access Stalwart's admin API from localhost
# The HTTP listener allows localhost connections even when blocking other IPs
echo ""
echo "Accessing Stalwart admin API via kubectl exec..."
# Test API access first
echo "Testing API connectivity..."
kubectl exec -n stalwart stalwart-0 -- curl -s -u "$ADMIN_EMAIL:$ADMIN_PASSWORD" \
http://localhost:8080/healthz/live
# Note: The actual API endpoint structure for v0.16.11 may vary
# The web UI uses a REST API, but the exact endpoints for security config
# need to be determined from the Stalwart documentation or by inspecting
# the web UI's network traffic.
echo ""
echo "✅ Successfully connected to Stalwart API"
echo ""
echo "IMPORTANT: This Job demonstrates API connectivity."
echo "The actual security configuration change requires:"
echo "1. Identifying the correct API endpoint for security settings"
echo "2. Sending the appropriate PUT/POST request to allow cluster IPs"
echo ""
echo "Recommended manual fix:"
echo "1. Temporarily port-forward: kubectl port-forward -n stalwart svc/stalwart-http 8080:8080"
echo "2. Access https://mail.basicstack.de from your browser"
echo "3. Login with admin credentials"
echo "4. Navigate to Settings > Security"
echo "5. Disable IP filtering for the HTTP listener or add 10.244.0.0/16 to allowed IPs"
exit 0

4
apps/stalwart/stalwart-fresh-deployment.yaml
View file

@ -122,7 +122,7 @@ spec:
mountPath: /var/lib/stalwart mountPath: /var/lib/stalwart
containers: containers:
- name: stalwart - name: stalwart
image: stalwartlabs/stalwart:v0.16.18 image: stalwartlabs/stalwart:v0.16.11
ports: ports:
- containerPort: 25 - containerPort: 25
name: smtp name: smtp
@ -316,7 +316,7 @@ spec:
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
containers: containers:
- name: backup - name: backup
image: alpine:3.24 image: alpine:3.19
command: command:
- /bin/sh - /bin/sh
- -c - -c

11
apps/velero/Chart.yaml
View file

@ -1,11 +0,0 @@
apiVersion: v2
name: velero
description: Velero cluster-reconstruction backups for BasicStack (Kopia + Hetzner S3)
type: application
version: 1.0.0
appVersion: "1.18.1"
dependencies:
- name: velero
version: 12.1.0
repository: https://vmware-tanzu.github.io/helm-charts

105
apps/velero/values.yaml
View file

@ -1,105 +0,0 @@
velero:
# Pinned chart version 12.1.0 (Velero appVersion 1.18.1).
# AWS plugin follows Velero minor version: v1.14.x for Velero 1.18.x.
image:
repository: velero/velero
tag: v1.18.1
pullPolicy: IfNotPresent
# Bring the AWS object-storage plugin in as an initContainer that copies
# the binary into /plugins so the Velero server can load it.
initContainers:
- name: velero-plugin-for-aws
image: velero/velero-plugin-for-aws:v1.14.2
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /target
name: plugins
# Kopia file-system backup requires the node-agent daemonset on every
# node whose PVC data we back up.
deployNodeAgent: true
# BackupStorageLocation credentials come from the SealedSecret
# apps/velero/velero-s3-credentials-sealed.yaml (key: cloud, INI format
# with aws_access_key_id / aws_secret_access_key for the dedicated
# `velero-prod` Hetzner OS credential from DEV-570).
credentials:
useSecret: true
existingSecret: velero-s3-credentials
# Kopia backup-repository password (velero-repo-credentials Secret,
# key `repository-password`) is auto-discovered by Velero — this is a
# hard-coded constant in the Velero server (see pkg/repository/keys),
# so no extra wiring is needed. SealedSecret lives at
# apps/velero/velero-repo-credentials-sealed.yaml.
# Turn OFF volume-snapshot CRDs — Hetzner Cloud Volumes have no CSI
# snapshot support, so filesystem backup via Kopia is the only path.
snapshotsEnabled: false
backupsEnabled: true
configuration:
# Kopia uploader (unified repo). This is the Velero server default in
# 1.18 but we set it explicitly so any future default change does not
# silently rewrite our repos to restic.
uploaderType: kopia
# No CSI feature flag — see snapshotsEnabled comment above.
features: ""
backupStorageLocation:
- name: default
provider: aws
bucket: basicstack-velero
default: true
credential:
name: velero-s3-credentials
key: cloud
config:
# Hetzner Object Storage in Helsinki (hel1). Region is used by
# the AWS SDK signer; endpoint overrides the AWS endpoint URL.
region: hel1
s3ForcePathStyle: "false"
s3Url: https://hel1.your-objectstorage.com
# publicUrl left unset — this backend is not user-facing.
# No volumeSnapshotLocation entries — snapshotsEnabled: false above
# already suppresses the default snapshot location CRD.
# Small resource footprint until we know real backup sizing (DEV-573).
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
# Small memory request so node-agent fits on the tight 4 GiB CX22 workers
# (worker-1/2/3 sit around 95 % memory-request already). Actual peak usage
# during Kopia backup lands under 300 MiB per node in practice; limit is
# generous so a spike does not OOM-kill mid-backup.
nodeAgent:
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 1000m
memory: 1Gi
metrics:
enabled: true
serviceMonitor:
enabled: false
# Do not install the bundled `velero` CLI kubectl image — we run the
# CLI from cp-1.
kubectl:
image:
repository: bitnami/kubectl
resources: {}
# No schedules here — those live in DEV-573.
schedules: {}

18
apps/velero/velero-repo-credentials-sealed.yaml
View file

@ -1,18 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: velero-repo-credentials
namespace: velero
spec:
encryptedData:
repository-password: AgCn0k/wQK76oVgVu9Eay7yMdxTKOE3RmJIojQcmPMerivAk72vw9F1jjmMDWW3f9X4HMc5MyQS9EXw9aes49047r1fZ4fA1yFgoBhuicS8ULpI4yI1ei9AVov6IQWx3FvarOlUIlQM40KC/es7mwlL/fSE2W5//u1zNp3Z+ujGdqAgBlvO0UQ/pGDNVKhWh6TBMNbzak4IDh6uNSNVAcqrzsh+2+RKPV4RRXnD1oJlTvtSri8EHQZdAAUaWxcIuyhnqeulStvViJKDC3Xw3PamrDXTYEPcAs8P90BVoDoJQwtxJ12Q+yqR1mjLvIZj81CKWXqNSUZDkAatRFjrfUYeiBAgmvnY5HL6f0eoJDVY+F7LfHslQHc4W/2z3YS19++oSKhffM4vwJxAWJqTbsKghvrA6hgFQM6btqGrhuHk3Q5NslKtlFnSgbRBst0FhBoyme0uWeBGk6szKKD8ClRuREJTWfof6pFGRM2O1IcKAhQqgJbamNHtU2yUCkLw732VBW3MJGQ+yLmlegjAmPKVLFDt4Yk5NrtTKrx9HZ3vohoPKwCz1CUizByVOXCsqEO1aogEszaQX9qCVsTCo6M8Zemw6OYCMMcbUcfPHRZmzd8o0j0Y+/6KTjP0fL5l9vR/ocJH+kXLnVsWZiFa3/TXQ0KEG1pPeCSVjQP7WHB1eLtrPH13ikvpyT8u6Ng+FPvOoTD36JrwSj8Een5HVc8oaoV50pXbbGqkepFIXZQ1te5nQL9XSP0AbRMcrqg==
template:
metadata:
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
creationTimestamp: null
name: velero-repo-credentials
namespace: velero
type: Opaque

18
apps/velero/velero-s3-credentials-sealed.yaml
View file

@ -1,18 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: velero-s3-credentials
namespace: velero
spec:
encryptedData:
cloud: AgAo2duq77obiTyFYdVyNANrMtx2iiOT6/cRHyYezhtr6YKlV2whmQBHXuRNhnjQyrmt3Gyl9+t/UZpJbJljqzesstfwRDZ9q1usXILVZE0ZsJB1GyQ5kwN/EQUeqhAYYUSZEUMXLAEkYTD2HE+sbWkNQrGXl1UJfPeEuDkRtCtQ+WNdZ3L53DLCjXQh3JV/4jIG+5bc56Sb33l/xtSSs5YBlUI/cgCM/Di0Xj5pyujxaqiU+vSCrpxiltkK7QDgcti1MbbL5YqeDox5/DwmJZvL1Of2PzODRpgTrDXbylWH8AaNRhzYcSJgv0ZBUwiqsLjFRdU9Oroevg2Uf5dO8FLUbAvx8ER97Afkt6fX7wTgcn3uj9/Ye3+BVOPRIj2l6VaJ3ul15Z+i+UCAAHe3jeujQhVHhZMRXGp5CnjkjJbiFIIL83cq6SfKq21pTEUkk5Ty5V844zDMymHHKvB52/LXS1i0LrELT/QzEKr0y+B3wp/JRFtvsliN+Bsweg4RroBsnVL4d0Aqp0zr3dHaUmTVDTycFGb/GFRTdsC+0JAJlplonJXcU/9h8mSRaBekusFTwMs25C6JHXPcXbI9qf8+qw6n/4vC86FkpyOWKNS2fs5r/Aut+8PC8Z2RcNUjnNUtt/i9FSkoQtIOwHRR0J8Zy+07TyysR0Zz2tB6C895SmHPI5isxQGZoAJHp+ypJwP/nR+RTdB648IfLLZyRINLEt58Nk4NM7B9tXOvk0JBXfB/knXy0wO54PvrnmLfxcE3mIy2/KKM5HK/b+QbSy97VqpXZekbcKJl18foUMsP7YitmP5pWfxBkg9m+3ZDemhCL48mLjr05KVrJ04r8UTMAlWwUw==
template:
metadata:
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
creationTimestamp: null
name: velero-s3-credentials
namespace: velero
type: Opaque

39
apps/vikunja/README.md
View file

@ -1,39 +0,0 @@
# Vikunja (vikunja.basicstack.de)
Self-hosted task manager. Unified `vikunja/vikunja` container (API + frontend on port 3456). Backed by a namespace-local PostgreSQL. OIDC via Pocket ID (`auth.basicstack.de`); local login disabled.
## Layout
| File | Purpose |
|---|---|
| `vikunja-db-pvc.yaml` | 10 Gi encrypted PVC for Postgres data |
| `vikunja-files-pvc.yaml` | 10 Gi encrypted PVC for user-uploaded attachments (`/app/vikunja/files`) |
| `vikunja-postgres-deployment.yaml` | Postgres 16, pinned to `csi.hetzner.cloud/location=fsn1` |
| `vikunja-postgres-service.yaml` | ClusterIP `vikunja-postgres:5432` |
| `vikunja-deployment.yaml` | Vikunja app, pinned to `fsn1`, waits on Postgres |
| `vikunja-service.yaml` | ClusterIP `vikunja:80` → pod `3456` |
| `vikunja-ingress.yaml` | Traefik + `letsencrypt-prod` cert |
| `vikunja-db-credentials-sealed.yaml` | Postgres user password |
| `vikunja-oidc-secret-sealed.yaml` | Pocket ID client id + secret |
| `vikunja-app-secrets-sealed.yaml` | Vikunja JWT secret |
## Node placement
Both the app and Postgres pin to Hetzner location `fsn1` because their PVCs live on `hcloud-volumes-encrypted` (datacenter-local).
## OIDC
- Pocket ID client `Vikunja` (id `cdfff4ee-28ba-4aab-9f47-7e41b4b6b847`), callback `https://vikunja.basicstack.de/auth/openid/basicstack`.
- Pocket ID groups `vikunja_admins` and `vikunja_members`.
- Vikunja does not map OIDC groups to the admin flag. Promote the first user post-login: `kubectl -n vikunja exec deploy/vikunja -- /app/vikunja/vikunja user set-admin --admin <username>`.
- **PKCE must be OFF on the Pocket ID client.** Vikunja 2.x's OAuth redirect (`frontend/src/helpers/redirectToProvider.ts`) does not send `code_challenge`, and the Go backend does not implement PKCE either. With `pkceEnabled: true` on Pocket ID the flow fails at `/authorize` with `invalid_request … 'code_challenge' parameter is missing`. Client is confidential (`isPublic: false`), so `client_secret` still authenticates the token exchange.
## SMTP
Reminder emails go through Stalwart on `stalwart-smtp.stalwart.svc.cluster.local:25` using unauthenticated in-cluster relay. Stalwart's port-25 listener trusts the pod-network CIDR (`10.244.0.0/16`), so no SMTP AUTH is required and no sealed secret is needed. Envelope-from is `vikunja@basicstack.de` (the mailbox already exists in Stalwart as principal `g`).
If a future policy tightens the port-25 CIDR trust, switch to submission on `stalwart-smtp:587` with STARTTLS and a `vikunja-mailer` sealed secret (`VIKUNJA_MAILER_USERNAME`/`VIKUNJA_MAILER_PASSWORD` from a secretKeyRef, `VIKUNJA_MAILER_AUTHTYPE=plain`); creating that secret requires a working Stalwart admin session to (re)set the vikunja principal's password.
## Change control
Manifest changes flow through Argo CD (Application: `apps/app-vikunja.yaml`, auto-sync + prune + selfHeal). Sealed secrets are encrypted against the cluster-active sealed-secrets key.

15
apps/vikunja/vikunja-app-secrets-sealed.yaml
View file

@ -1,15 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: vikunja-app-secrets
namespace: vikunja
spec:
encryptedData:
jwt-secret: AgA2gSMP5tLy+2BeEe/vO3DHi7v4WO2HRZc2kTD/+BRII/CxCd3SIJYOh5O7OPNOq5Vp+F6oCi6GUg7ljyXuHU1IlHSSYWfjdGKS/mIvlKf2XRASOS1CefNSDKjU+x2FB3LPJG10v/0OG7bLKCj7QGq5y+QNc8dQrV7kqwAonG18iZzl5rcL4vN38B2V6kaO+s/tcz0uwmVOlUqblmv6WwrkMm5bVt9RQxcO5UjmDuf2KlnHBgvhRUhcBCbmrnI7J+FkVtq+2hiWwyz3hXAQAyy6ULMltYZvSy4ADseTV3yBdArUvY9e13kwfVZBjLe5EBzn0SKoQyY0XZQUYDmfQU+U5zFxsZLbYeekf7FUN8zHJ+mgvrT35PGrVazoz2rKI+9BrA7sstYJZ0ZUlj+nKnrbHYyaUhrlu3DYFvTHCOz8ylRxIsM5Ux819+RISpBhKkg6+mIhH2L26larI7UEY576c83TMhmCEOeX0es+MoomsAqeQi8vZ+OT5qz9dFZpzshXmwRYy1Ezm9d992iBF9EskB9IuyaozpA+GU9WwKAOj5GF3izj89nel9QPPQVBGHYvxXXfaXi3yQIpqD54nVPKC1oEhPNeCfVp3YcGSd/HOsl3tT/8G7ipD40jGVk5tgWxpymbvNZmcpm1CIb4SPYHOavJtqBcPY1F9k8a5yYl9sMqd64qzDmTlOQhRVWjD0mqCf2W/W3OvHo4dTu6Fy95jHrzoHsmmfNQQDVytCJK/3SmYUvh51nD8/CyiJxRAq3zPViFHKmRS0E5oCR0+zm3GlZTGwUmu1ROSIlsDgik/khmrZvYFLtF
template:
metadata:
creationTimestamp: null
name: vikunja-app-secrets
namespace: vikunja

15
apps/vikunja/vikunja-db-credentials-sealed.yaml
View file

@ -1,15 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: vikunja-db-credentials
namespace: vikunja
spec:
encryptedData:
postgres-password: AgB+P7iBg+tPIDKMGmiPh9HREVb7mKCf1GvQfBLcF61QuWHt0/S1YBYCd4fF3PSA9ok916m6VeOoIQKB0p57XDSEeNewYNsXQgudS0k0JHNdRZER8JhbXpK53E7EY0xw4TD8Tc6WFqyzCFwsL1WpYvvDH3p25W8f3JJfB39D6Up9j5xT5Hufi0G5BJ+tbAujNJooa0SWZbtQPxMWG96fvjmTV5FKxzvwg1JOL4gq0JHBfzo9sKIbQdGpGJjX4JkPLCxA0jU6mcJKEFn80zkDbQBL3g5dXZaYmIAEogZkvytltR4ebI0zyDrSTPqXhJF5bFcRpM43VXz6ZcjgdtG5VPelvAuMqfJ1U7HdYhhJqTt0lvwUi03BjulIJJ52gUjlg2yQioOwfON/y3gRM1TgMHMFCyw32S7vCfR00MKVh7xrb4YceRUTbeHkTd+4TMcdpI11YHeah2scu82tzyD/cGPtpFv2LYyH7JrBcIXXhfIDccBRX4bb2se9VyhpmvEP5Q9SQjazo/BxdnJUYF6kzpJA9oOIjFMXcSDxZh/nwDkA90w4a9iyT9l0+7p3z/RjbXOMKybuQlXpfPrhtCRS6PM3Am5sgnIi/APRaujDa/2a3kCe7K0zZl4Wk3wlAc4mMjnoWs9w//HfMpQByM6GWeHWeoQ/Bj3xCVtACb6SNqTxIs4OP9hAVBnHqouH0O/zjqZNuBa5v7GDwmW6JU1cA+Y/UI4Z/ycu2X6AXBHp5CROheV9PdINTrgS
template:
metadata:
creationTimestamp: null
name: vikunja-db-credentials
namespace: vikunja

15
apps/vikunja/vikunja-db-pvc.yaml
View file

@ -1,15 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: vikunja-db
namespace: vikunja
labels:
app: vikunja
component: database
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 10Gi

144
apps/vikunja/vikunja-deployment.yaml
View file

@ -1,144 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: vikunja
namespace: vikunja
labels:
app: vikunja
component: app
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: vikunja
component: app
template:
metadata:
labels:
app: vikunja
component: app
spec:
hostname: vikunja
subdomain: basicstack
setHostnameAsFQDN: true
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
initContainers:
- name: wait-for-postgres
image: postgres:16-alpine
command:
- sh
- -c
- |
until pg_isready -h vikunja-postgres -U vikunja; do
echo "Waiting for PostgreSQL..."
sleep 2
done
securityContext:
fsGroup: 1000
containers:
- name: vikunja
image: vikunja/vikunja:2.6.0
ports:
- name: http
containerPort: 3456
env:
- name: VIKUNJA_SERVICE_PUBLICURL
value: https://vikunja.basicstack.de/
- name: VIKUNJA_SERVICE_SECRET
valueFrom:
secretKeyRef:
name: vikunja-app-secrets
key: jwt-secret
- name: VIKUNJA_SERVICE_ENABLEREGISTRATION
value: "false"
- name: VIKUNJA_SERVICE_TIMEZONE
value: Europe/Berlin
- name: VIKUNJA_DATABASE_TYPE
value: postgres
- name: VIKUNJA_DATABASE_HOST
value: vikunja-postgres
- name: VIKUNJA_DATABASE_PORT
value: "5432"
- name: VIKUNJA_DATABASE_DATABASE
value: vikunja
- name: VIKUNJA_DATABASE_USER
value: vikunja
- name: VIKUNJA_DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: vikunja-db-credentials
key: postgres-password
- name: VIKUNJA_DATABASE_SSLMODE
value: disable
- name: VIKUNJA_FILES_BASEPATH
value: /app/vikunja/files
- name: VIKUNJA_AUTH_LOCAL_ENABLED
value: "false"
- name: VIKUNJA_AUTH_OPENID_ENABLED
value: "true"
- name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_NAME
value: BasicStack SSO
- name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_AUTHURL
value: https://auth.basicstack.de
- name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_SCOPE
value: openid profile email vikunja_scope
- name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_CLIENTID
valueFrom:
secretKeyRef:
name: vikunja-oidc-secret
key: client-id
- name: VIKUNJA_AUTH_OPENID_PROVIDERS_BASICSTACK_CLIENTSECRET
valueFrom:
secretKeyRef:
name: vikunja-oidc-secret
key: client-secret
- name: VIKUNJA_MAILER_ENABLED
value: "true"
- name: VIKUNJA_MAILER_HOST
value: stalwart-smtp.stalwart.svc.cluster.local
- name: VIKUNJA_MAILER_PORT
value: "25"
- name: VIKUNJA_MAILER_FORCESSL
value: "false"
- name: VIKUNJA_MAILER_SKIPTLSVERIFY
value: "true"
- name: VIKUNJA_MAILER_FROMEMAIL
value: vikunja@basicstack.de
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
livenessProbe:
httpGet:
path: /api/v1/info
port: 3456
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /api/v1/info
port: 3456
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 3
volumeMounts:
- name: files
mountPath: /app/vikunja/files
volumes:
- name: files
persistentVolumeClaim:
claimName: vikunja-files

15
apps/vikunja/vikunja-files-pvc.yaml
View file

@ -1,15 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: vikunja-files
namespace: vikunja
labels:
app: vikunja
component: files
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes-encrypted
resources:
requests:
storage: 10Gi

26
apps/vikunja/vikunja-ingress.yaml
View file

@ -1,26 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: vikunja
namespace: vikunja
labels:
app: vikunja
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
ingressClassName: traefik
tls:
- hosts:
- vikunja.basicstack.de
secretName: vikunja-tls
rules:
- host: vikunja.basicstack.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: vikunja
port:
number: 80

16
apps/vikunja/vikunja-oidc-secret-sealed.yaml
View file

@ -1,16 +0,0 @@
---
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
creationTimestamp: null
name: vikunja-oidc-secret
namespace: vikunja
spec:
encryptedData:
client-id: AgBdzdW0SxhmOyhOpC/jkBJ4UtD0qrRrytVp7fInK3bxiiVBHsIU8r3Hoe2iYZU75IyITogmcGImKH9pivxPMhj5jBqFeCExQadt/pqXHL9TmxmNpIx7FmnlrxaZ+zeqeKRMDhYJnxSeNiuRDVZEEnsBhmUc0gvz8Pq/x82ExY++MQjQQu2xB6XQ7DTyzVDx32yZAvBU6aRFhTtMTqPAvjsMHp6pKmciauS7LmffCVIJvnJUaUKIELHPDyWsMcQvWpPQrPKGkvmGruhXbct6LxPFKdkbsQMUWvj7emSemmbJk0VmGPtVxI7cTmHIJ6l4kr8aJfJ9fv2gMzbgprYVZHJ17qhPvvUv5iTUcYu5FiMiy3QtNHYMP+VyhWvj9aUH8ypFYwf0S3/fRobEtyBxnBLCmnIt4lvS0ABQ49YwdtWnBUXT1jP4Xzk3gWM8fe1LsOuUZXMgFoGMXtxuodhZFhTELjdKPaYTY/DDU6iRMpk/nUsQy1IGls1CySEB+taiQDRNMZmQAFHdoFOxX8sZyPKdYL3wgKDweqcLoI0UKjapkHdpcoWccqg0ky/f5VWWlD7kHWLXYeosH6ucLAKR/mRHoyOfIy1dEpYYevfv5UM4Lewnr8epLse/x3ymsBNaSIhxt8PxnFYe/Gi1rO8MAgTPNcw2pr2z5jiW8CXUPLdXezleoctl4VGg97pkYVibn0BJ+VrZOTgO/p/df5D0v7h7DSsvdbpA7XxkaTnL62JRhC3RDXU=
client-secret: AgA3ZQ5Da1D2tcNBA9RVf7YYNLBvBjKf+OArLLQSKKUDGAULoC0BQUn+T7rctuOANFeKLOYE2OjJQhURoi69xFyQ3d868s4a2Ipo3QwUV2vtoVJnz7l2fGXCzAOTcJhKonHdJooklfeK+gwi80sRXV1IxxDu778yKSWnFUd61vUPbhKgVgtXDM4IItNtctGc4Drz6dvrOHkyM1m4kxGbXUclpN+HztufQrO7W+ba23bZRCejaYz3sQ6IHmM50HXzRPD5Rdb2ALQFm9V7zdSuUQ8njNrXHfgqMSiDWoCzdpTKG50mBPqmait/yyNVnOAd8J906RZk27LnAY9zV1WiMUSImNS6IzuLV0DZTn6I4cWgnemIwNAdNR4/x7TwWNdVS3A3jH8q6S1IP5g65P40SmP27PfFx+L7jx72Om8HXBY8TpMtWwGFcjyEGy61tKuimXd65+o0UkXEmQfJKfnugnSM3yCufEat2PIE2+Arb7+pRh6VtzaBowLdo32OcqS5GMmFLhisv5cR016IIIbSnaLkIGpiZEHSvpcLoGOJm8H+1P6MPsHWfjWtKZy6y8Ri/B9zNHlUx506Kij5YOR+tH3B3ip4PIa2LRlRCAbzBb/WCwzP72WzK8PU/sSujTa+JgxBoNGt3fg2iqBmoaU3UrhVoTTcXhEn7pnvIlk60jZKTscC+thS++VHMnf7LjLAA2ksu9G6qKpWy5DAERYi/MC6YdYz0eoIBSa/NEHLi94zFA==
template:
metadata:
creationTimestamp: null
name: vikunja-oidc-secret
namespace: vikunja

79
apps/vikunja/vikunja-postgres-deployment.yaml
View file

@ -1,79 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: vikunja-postgres
namespace: vikunja
labels:
app: vikunja
component: database
spec:
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: vikunja
component: database
template:
metadata:
labels:
app: vikunja
component: database
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: csi.hetzner.cloud/location
operator: In
values:
- fsn1
containers:
- name: postgres
image: postgres:16-alpine
ports:
- name: postgres
containerPort: 5432
env:
- name: POSTGRES_DB
value: vikunja
- name: POSTGRES_USER
value: vikunja
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: vikunja-db-credentials
key: postgres-password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
exec:
command:
- pg_isready
- -U
- vikunja
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- pg_isready
- -U
- vikunja
initialDelaySeconds: 5
periodSeconds: 5
volumeMounts:
- name: data
mountPath: /var/lib/postgresql/data
volumes:
- name: data
persistentVolumeClaim:
claimName: vikunja-db

17
apps/vikunja/vikunja-postgres-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: vikunja-postgres
namespace: vikunja
labels:
app: vikunja
component: database
spec:
type: ClusterIP
selector:
app: vikunja
component: database
ports:
- name: postgres
port: 5432
targetPort: 5432

17
apps/vikunja/vikunja-service.yaml
View file

@ -1,17 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: vikunja
namespace: vikunja
labels:
app: vikunja
component: app
spec:
type: ClusterIP
selector:
app: vikunja
component: app
ports:
- name: http
port: 80
targetPort: 3456

8
docs/monitoring/restic-restore.md
View file

@ -70,7 +70,7 @@ spec:
restartPolicy: Never restartPolicy: Never
containers: containers:
- name: restic - name: restic
image: harbor.basicstack.de/library/restic:0.19.1 # matches CronJob image (DEV-493, bumped in DEV-541) image: harbor.basicstack.de/library/restic:0.17.3 # matches CronJob image (DEV-493)
command: ["sleep", "3600"] command: ["sleep", "3600"]
env: env:
- { name: AWS_ACCESS_KEY_ID, valueFrom: { secretKeyRef: { name: monitoring-s3-backup, key: access-key } } } - { name: AWS_ACCESS_KEY_ID, valueFrom: { secretKeyRef: { name: monitoring-s3-backup, key: access-key } } }
@ -240,9 +240,9 @@ All three restic CronJobs (`backup-loki-restic`, `backup-grafana-restic`,
keep the backup pipeline off the Docker Hub pull path and immune to keep the backup pipeline off the Docker Hub pull path and immune to
upstream retagging: upstream retagging:
- Manifest reference: `harbor.basicstack.de/library/restic:0.19.1` - Manifest reference: `harbor.basicstack.de/library/restic:0.17.3`
- Upstream: `docker.io/restic/restic:0.19.1` - Upstream: `docker.io/restic/restic:0.17.3`
- Ticket: [DEV-493](/DEV/issues/DEV-493) (bumped to 0.19.1 in [DEV-541](/DEV/issues/DEV-541)) - Ticket: [DEV-493](/DEV/issues/DEV-493)
The `library` project is public, so no `imagePullSecret` is required The `library` project is public, so no `imagePullSecret` is required
on the CronJob pods. on the CronJob pods.

35
infrastructure/K3S_OPERATIONS.md
View file

@ -3,9 +3,9 @@
## Current Cluster Status ## Current Cluster Status
- **Version:** v1.36.2+k3s1 - **Version:** v1.36.2+k3s1
- **Last Updated:** 2026-08-30 - **Last Updated:** 2026-07-06
- **Control Plane:** k3s-cp-1 (178.105.17.239 / 10.42.1.1), k3s-cp-2 (188.245.85.199 / 10.42.1.8), k3s-cp-3 (49.13.92.162 / 10.42.1.9) — etcd HA - **Control Plane:** k3s-cp-1 (178.105.17.239 / 10.42.1.1)
- **Workers:** k3s-worker-1, k3s-worker-2, k3s-worker-3, k3s-worker-4 (nbg1), k3s-worker-5, k3s-worker-6, k3s-update-runner - **Workers:** k3s-worker-1, k3s-worker-2, k3s-worker-3, k3s-update-runner
- **Update Method:** k3s system-upgrade-controller (automated) - **Update Method:** k3s system-upgrade-controller (automated)
## Quick Reference ## Quick Reference
@ -79,15 +79,12 @@ The script will:
```bash ```bash
ssh root@<new-node-ip> "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v${CLUSTER_VERSION}' sh -s - agent \ ssh root@<new-node-ip> "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION='v${CLUSTER_VERSION}' sh -s - agent \
--server https://10.42.1.1:6443 \ --server https://10.42.1.1:6443 \
--token '${K3S_TOKEN}' \ --token '${K3S_TOKEN}'"
--node-ip <new-node-private-ip>"
``` ```
**Critical:** The `--node-ip=<private-ip>` flag is required. Without it, k3s defaults to the public IP as the node's InternalIP, which breaks LoadBalancer service routing (see DEV-224). Verify with `kubectl get nodes -o wide` — the INTERNAL-IP column MUST show the private (10.42.1.x) IP.
5. **Verify node joined:** 5. **Verify node joined:**
```bash ```bash
kubectl get nodes -o wide # INTERNAL-IP must be private (10.42.1.x) kubectl get nodes
kubectl get node <node-name> -o yaml | grep flannel kubectl get node <node-name> -o yaml | grep flannel
``` ```
@ -413,20 +410,13 @@ failed to register flannel network: failed to acquire lease: subnet "X.X.X.X/16"
### Node CIDRs ### Node CIDRs
| Node | Private IP | Public IP | Datacenter | Role | | Node | Pod CIDR | Private IP | Public IP |
|------|-----------|-----------|------------|------| |------|----------|-----------|-----------|
| k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | control-plane, etcd | | k3s-cp-1 | 10.244.0.0/24 | 10.42.1.1 | 178.105.17.239 |
| k3s-cp-2 | 10.42.1.8 | 188.245.85.199 | fsn1 | control-plane, etcd | | k3s-worker-1 | 10.244.1.0/24 | 10.42.1.2 | - |
| k3s-cp-3 | 10.42.1.9 | 49.13.92.162 | fsn1 | control-plane, etcd | | k3s-worker-2 | 10.244.2.0/24 | 10.42.1.3 | - |
| k3s-worker-1 | 10.42.1.2 | 178.105.216.48 | fsn1 | worker | | k3s-update-runner | 10.244.3.0/24 | 167.233.79.65 | 167.233.79.65 |
| k3s-worker-2 | 10.42.1.3 | 49.13.134.255 | fsn1 | worker | | k3s-worker-3 | 10.244.5.0/24 | 10.42.1.5 | 167.233.121.121 |
| k3s-worker-3 | 10.42.1.5 | 167.233.121.121 | fsn1 | worker |
| k3s-worker-4 | 10.42.1.6 | 128.140.3.80 | nbg1 | worker (stateless workloads only) |
| k3s-worker-5 | 10.42.1.7 | 167.233.192.86 | fsn1 | worker (DEV-410) |
| k3s-worker-6 | 10.42.1.10 | 49.12.223.133 | fsn1 | worker (cx33 4vCPU/8GB) |
| k3s-update-runner | 167.233.79.65 | 167.233.79.65 | fsn1 | update-runner |
**Datacenter note:** Hetzner Cloud Volumes are datacenter-scoped. Pods with PersistentVolumes only schedule on nodes in the same datacenter as the PV. Provision stateful-workload workers in **fsn1** to match the existing PV pool. k3s-worker-4 remains in nbg1 and is only usable for stateless workloads.
### Critical Firewall Rule ### Critical Firewall Rule
@ -438,7 +428,6 @@ failed to register flannel network: failed to acquire lease: subnet "X.X.X.X/16"
| Date | Version | Changes | By | | Date | Version | Changes | By |
|------|---------|---------|-----| |------|---------|---------|-----|
| 2026-08-30 | v1.36.2+k3s1 | DEV-410: Added k3s-worker-5 (fsn1) to unblock Stalwart scheduling; k3s-worker-4 (nbg1) retained for stateless workloads; provision script now sets --node-ip; refreshed node table with cp-2/cp-3 and worker-4/5/6 | CTO Agent |
| 2026-07-06 | v1.36.2+k3s1 | Fixed DNS service CIDR mismatch (kubelet cluster-dns configuration), corrected Service CIDR documentation to 10.96.0.0/16, added DNS configuration guide | CTO Agent | | 2026-07-06 | v1.36.2+k3s1 | Fixed DNS service CIDR mismatch (kubelet cluster-dns configuration), corrected Service CIDR documentation to 10.96.0.0/16, added DNS configuration guide | CTO Agent |
| 2026-07-06 | v1.36.2+k3s1 | Fixed API server instability by adding explicit --cluster-cidr=10.244.0.0/16 to k3s service, updated all nodes to v1.36.2+k3s1 | CTO Agent | | 2026-07-06 | v1.36.2+k3s1 | Fixed API server instability by adding explicit --cluster-cidr=10.244.0.0/16 to k3s service, updated all nodes to v1.36.2+k3s1 | CTO Agent |
| 2026-07-06 | v1.36.2+k3s1 | Initial documentation, upgraded from v1.35.5, installed system-upgrade-controller | CTO Agent | | 2026-07-06 | v1.36.2+k3s1 | Initial documentation, upgraded from v1.35.5, installed system-upgrade-controller | CTO Agent |

47
infrastructure/OS_UPDATE_PROCEDURE.md
View file

@ -1,6 +1,6 @@
# Weekly Rolling Ubuntu OS Update Procedure # Weekly Rolling Ubuntu OS Update Procedure
**Purpose:** Keep every k3s node's Ubuntu OS patched (kernel, security, package updates) with a **rolling, zero-downtime** update — one node at a time, drain → update → reboot → verify → uncordon → cluster health check → next node. Since [DEV-553](/DEV/issues/DEV-553) (2026-08-30) this covers **workers AND control-plane nodes** in a single cycle; workers run first, then CPs one at a time with etcd leader last. **Purpose:** Keep every k3s node's Ubuntu OS patched (kernel, security, package updates) with a **rolling, zero-downtime** update — one node at a time, drain → update → reboot → verify → uncordon → cluster health check → next node.
**Audience:** Agents (currently CTO, optionally a dedicated ClusterOps agent). Also usable manually by an operator. **Audience:** Agents (currently CTO, optionally a dedicated ClusterOps agent). Also usable manually by an operator.
@ -22,9 +22,9 @@
| Role | Node | Private IP | Public IP | Datacenter | Notes | | Role | Node | Private IP | Public IP | Datacenter | Notes |
|------|------|-----------|-----------|------------|-------| |------|------|-----------|-----------|------------|-------|
| control-plane | k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) | | control-plane | k3s-cp-1 | 10.42.1.1 | 178.105.17.239 | fsn1 | EXCLUDED from `os-update.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
| control-plane | k3s-cp-2 | — | 188.245.85.199 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) | | control-plane | k3s-cp-2 | — | 188.245.85.199 | fsn1 | EXCLUDED — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
| control-plane | k3s-cp-3 | — | 49.13.92.162 | fsn1 | included via `update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) | | control-plane | k3s-cp-3 | — | 49.13.92.162 | fsn1 | EXCLUDED — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) |
| worker | k3s-worker-1 | 10.42.1.2 | (via CP) | fsn1 | | | worker | k3s-worker-1 | 10.42.1.2 | (via CP) | fsn1 | |
| worker | k3s-worker-2 | 10.42.1.3 | (via CP) | fsn1 | | | worker | k3s-worker-2 | 10.42.1.3 | (via CP) | fsn1 | |
| worker | k3s-worker-3 | 10.42.1.5 | 167.233.121.121 | fsn1 | | | worker | k3s-worker-3 | 10.42.1.5 | 167.233.121.121 | fsn1 | |
@ -38,19 +38,11 @@ Always re-derive the live list before running — nodes may have been added/remo
ssh root@178.105.17.239 'kubectl get nodes -o wide' ssh root@178.105.17.239 'kubectl get nodes -o wide'
``` ```
**Order rule:** update workers first, then control-plane nodes. Within each group, update in this order to protect stateful workloads: **Order rule:** update workers only. Within workers, update in this order to protect stateful workloads:
Workers:
1. runner + workers that host no PVs (safest — lowest disruption) 1. runner + workers that host no PVs (safest — lowest disruption)
2. remaining workers 2. remaining workers
3. **Stalwart-hosting fsn1 workers last among workers** — Stalwart has hard fsn1 affinity, so draining a fsn1 worker while another fsn1 worker is also unavailable can leave Stalwart Pending. Never have two fsn1 workers cordoned/down at the same time. 3. **Stalwart-hosting fsn1 workers last among workers** — Stalwart has hard fsn1 affinity, so draining a fsn1 worker while another fsn1 worker is also unavailable can leave Stalwart Pending. Never have two fsn1 workers cordoned/down at the same time.
4. **ALL control-plane nodes excluded** — `os-update.sh` selects nodes without the `node-role.kubernetes.io/control-plane` label, so `k3s-cp-1`, `k3s-cp-2`, and `k3s-cp-3` are all skipped automatically. CPs have their own procedure ([`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) — HA-aware, one CP at a time, leader last).
Control planes (integrated with the weekly cycle since [DEV-553](/DEV/issues/DEV-553); previously excluded and run manually via `CP_UPDATE_PROCEDURE.md`):
1. `os-update.sh` queries the current etcd leader before entering the CP phase.
2. Followers are updated first (in any order), leader last. Rationale: draining a follower is a no-op for raft leadership; draining the leader forces a re-election. Doing followers first minimises leader flapping.
3. Exactly one CP is drained or rebooting at a time — two CPs down means etcd quorum loss and a write outage across the cluster.
4. Between CPs `os-update.sh` re-runs `cluster-health.sh` and re-checks `etcdctl endpoint status --cluster`; any yellow signal halts the cycle at that CP. Settle wait between CPs is `POST_CP_WAIT_SECONDS` (default 180 s).
5. Each CP goes through `update-cp-node.sh <cp> --run` with `ASSUME_YES=1`: Phase A (add-swap, idempotent) → B preflight → C drain → D apt → E reboot with `/livez` polling and etcd rejoin check → F uncordon + cluster health.
**Concurrency:** exactly one node at a time. Never in parallel. **Concurrency:** exactly one node at a time. Never in parallel.
@ -84,12 +76,6 @@ Environment variables the scripts expect:
- Drain timeout: `DRAIN_TIMEOUT_SECONDS` — default `600` - Drain timeout: `DRAIN_TIMEOUT_SECONDS` — default `600`
- Reboot wait: `REBOOT_MAX_WAIT_SECONDS` — default `600` - Reboot wait: `REBOOT_MAX_WAIT_SECONDS` — default `600`
- Post-uncordon settle: `POST_UNCORDON_WAIT_SECONDS` — default `180` - Post-uncordon settle: `POST_UNCORDON_WAIT_SECONDS` — default `180`
- Post-uncordon bounded health retry (DEV-595):
- `HEALTH_MAX_RETRIES` — default `10` (extra attempts after the first)
- `HEALTH_RETRY_INTERVAL_SECONDS` — default `30`
- Short-circuits immediately on persistent pod errors
(`CrashLoopBackOff`, `ImagePullBackOff`, `ErrImagePull`,
`CreateContainerConfigError`, `InvalidImageName`).
--- ---
@ -263,19 +249,15 @@ Only proceed to the next node when ALL of the above are green. If not:
- Give it another 3 minutes and re-check (workloads with large images may still be pulling). - Give it another 3 minutes and re-check (workloads with large images may still be pulling).
- If still not green: STOP the cycle. Uncordon everything, leave the cluster in a stable state, and open a follow-up issue with the failing workloads. Do NOT proceed to update more nodes. - If still not green: STOP the cycle. Uncordon everything, leave the cluster in a stable state, and open a follow-up issue with the failing workloads. Do NOT proceed to update more nodes.
### 8. Control plane handling ### 8. Control plane special handling
Since [DEV-553](/DEV/issues/DEV-553) (2026-08-30) the weekly `os-update.sh` cycle **includes** the three CPs (`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`) after all workers finish. The CP steps are delegated to `scripts/os-update/update-cp-node.sh` — see [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) for the full design. Key properties: All three CPs (`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`) are **NOT** updated by the weekly `os-update.sh` cycle — they have their own dedicated procedure and script: [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md) and `scripts/os-update/update-cp-node.sh`. Reasons:
- CPs are `NoSchedule`-tainted and host no StatefulSet workloads today, but they are the etcd quorum. Two of three CPs cordoned/rebooting at once = write outage. `os-update.sh` walks CPs sequentially with a health gate between each. - CPs are `NoSchedule`-tainted and host no StatefulSet workloads today, but they are the etcd quorum. Two of three CPs cordoned/rebooting at once = write outage. One CP at a time, leader last.
- `os-update.sh` queries `etcdctl endpoint status --cluster` before the CP phase to identify the current leader, then updates followers first and leader last. - All three CPs currently have zero swap. The CP procedure adds a durable 4 GiB swapfile per CP before draining.
- `update-cp-node.sh` polls the target's `/livez` from the operator machine and cross-checks `etcdctl endpoint status --cluster` from a peer CP to confirm the just-rebooted member rejoined etcd as `started`. - The CP procedure polls the target's `/livez` from the operator machine and cross-checks `etcdctl endpoint status --cluster` from a peer CP to confirm the just-rebooted member rejoined.
- Phase A (`--add-swap`) is idempotent: creates `/swapfile` (4 GiB default) with `vm.swappiness=10` on any CP without swap, and installs `etcd-client` if `etcdctl` is missing. Re-runs are no-ops when swap is already active.
Skip flags on `os-update.sh`: `os-update.sh` excludes any node carrying the `node-role.kubernetes.io/control-plane` label (see `--dry-run` output) and prints a pointer to the CP procedure.
- `--skip-cps` — workers only (legacy behavior for narrow patch cycles).
- `--only-cps` — skip workers and only walk the CP phase.
- `--only <node>` — single node, routed automatically to worker or CP.
--- ---
@ -336,8 +318,8 @@ If a node was skipped or errored → mark `blocked` with the unblock action, or
## Automation entry points ## Automation entry points
- `infrastructure/scripts/os-update/os-update.sh` — full cycle runner (preflight → worker loop → CP loop → finalization). Idempotent, resumable via `--start-from <node>` (works for workers or CPs). Use `--dry-run` to print the plan without touching anything. Use `--skip-cps` / `--only-cps` to run only one half of the cycle. - `infrastructure/scripts/os-update/os-update.sh` — full cycle runner (preflight → per-node loop → finalization). Idempotent, resumable via `--start-from <node>`. Use `--dry-run` to print the plan without touching anything. Explicitly skips `k3s-cp-1`.
- `infrastructure/scripts/os-update/update-node.sh <node>` — single-node update for **workers** (all 7 per-node steps). Callable standalone for retry. Never call this for a CP. - `infrastructure/scripts/os-update/update-node.sh <node>` — single-node update (all 7 per-node steps). Callable standalone for retry. Not for `k3s-cp-1`.
- `infrastructure/scripts/os-update/update-cp-node.sh <node>` — HA-aware CP update flow (add swap, preflight, drain, apt, reboot, finalize). Accepts `k3s-cp-1`, `k3s-cp-2`, or `k3s-cp-3`. See [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md). - `infrastructure/scripts/os-update/update-cp-node.sh <node>` — HA-aware CP update flow (add swap, preflight, drain, apt, reboot, finalize). Accepts `k3s-cp-1`, `k3s-cp-2`, or `k3s-cp-3`. See [`CP_UPDATE_PROCEDURE.md`](CP_UPDATE_PROCEDURE.md).
- `infrastructure/scripts/os-update/update-cp-1.sh` — thin wrapper (calls `update-cp-node.sh k3s-cp-1 "$@"`). Kept for backwards compat. - `infrastructure/scripts/os-update/update-cp-1.sh` — thin wrapper (calls `update-cp-node.sh k3s-cp-1 "$@"`). Kept for backwards compat.
- `infrastructure/scripts/os-update/cluster-health.sh` — the preflight/post-node health check as a standalone command; exits non-zero on any failure. - `infrastructure/scripts/os-update/cluster-health.sh` — the preflight/post-node health check as a standalone command; exits non-zero on any failure.
@ -359,4 +341,3 @@ A Paperclip routine (see `infrastructure/OS_UPDATE_ROUTINE.md`) fires this proce
| 2026-08-09 | Initial procedure + automation scripts | CTO agent (DEV-462) | | 2026-08-09 | Initial procedure + automation scripts | CTO agent (DEV-462) |
| 2026-08-16 | Added kine thundering-herd guardrails after DEV-495 incident (worker-3 drain caused CP kine cascade + near-OOM on cp-1). Four new rules: pre-plan StatefulSet drain order, swap on cp-1, halt on kubectl >5 s, cp-1 update as separate design task. | CTO agent (DEV-478/DEV-495) | | 2026-08-16 | Added kine thundering-herd guardrails after DEV-495 incident (worker-3 drain caused CP kine cascade + near-OOM on cp-1). Four new rules: pre-plan StatefulSet drain order, swap on cp-1, halt on kubectl >5 s, cp-1 update as separate design task. | CTO agent (DEV-478/DEV-495) |
| 2026-08-23 | Downgraded the kine cascade section for the HA-etcd era (DEV-510 migrated cp-1 from kine SQLite to embedded etcd, cp-2/cp-3 joined as full members). Pointed §8 and the "Automation entry points" list at the new HA-aware `CP_UPDATE_PROCEDURE.md` / `update-cp-node.sh`. Old `CP1_UPDATE_PROCEDURE.md` and `update-cp-1.sh` are now redirect stubs. | CTO agent (DEV-515) | | 2026-08-23 | Downgraded the kine cascade section for the HA-etcd era (DEV-510 migrated cp-1 from kine SQLite to embedded etcd, cp-2/cp-3 joined as full members). Pointed §8 and the "Automation entry points" list at the new HA-aware `CP_UPDATE_PROCEDURE.md` / `update-cp-node.sh`. Old `CP1_UPDATE_PROCEDURE.md` and `update-cp-1.sh` are now redirect stubs. | CTO agent (DEV-515) |
| 2026-08-30 | Integrated the three CPs into the weekly `os-update.sh` cycle (leader-last ordering, health gate between CPs, delegates each CP to `update-cp-node.sh --run` with `ASSUME_YES=1`). Added `--skip-cps` / `--only-cps` flags. `--only <node>` and `--start-from <node>` now route between worker/CP lists automatically. | CTO agent (DEV-553) |

104
infrastructure/k3s-manifests/README-DEV-527.md Normal file
View file

@ -0,0 +1,104 @@
# k3s CoreDNS manifest override (DEV-527)
## Context
The k3s server bundles a stock CoreDNS Deployment and reconciles it
via the k3s addon controller. The manifest source is
`/var/lib/rancher/k3s/server/manifests/coredns.yaml` on each control
plane node. Any `kubectl patch` on the `kube-system/coredns`
Deployment that touches a field the source manifest declares is
reverted by the addon controller.
Hetzner Cloud nodes publish four systemd-resolved DNS upstreams:
```
2a01:4ff:ff00::add:2
2a01:4ff:ff00::add:1
185.12.64.1
185.12.64.2
```
Kubernetes limits pod `resolv.conf` to three nameservers, so kubelet
drops one entry and fires a `DNSConfigForming` Warning event every
time a pod using `dnsPolicy: Default` (or a `hostNetwork` pod that
coerces `ClusterFirst` to `Default`) is created. The stock CoreDNS
Deployment uses `dnsPolicy: Default`, so the warning fires on every
CoreDNS pod restart.
## Fix
The authoritative fix is to modify the source manifest so
CoreDNS runs with an explicit three-nameserver `dnsConfig` and
`dnsPolicy: None`. The three servers match what kubelet was already
picking (both Hetzner IPv6 anycast entries plus the first IPv4
entry), so CoreDNS's `forward . /etc/resolv.conf` upstreams are
unchanged; only the noisy Warning event is silenced.
The `coredns.yaml` file in this directory is the modified manifest
that must live at `/var/lib/rancher/k3s/server/manifests/coredns.yaml`
on **all three** k3s control plane nodes
(`k3s-cp-1`, `k3s-cp-2`, `k3s-cp-3`).
## Apply / re-apply procedure
Run from any host with SSH access to the CPs:
```bash
TS="$(date -u +%Y%m%dT%H%M%SZ)"
for IP in 178.105.17.239 188.245.85.199 49.13.92.162; do
echo "== $IP =="
scp infrastructure/k3s-manifests/coredns.yaml \
root@$IP:/tmp/coredns.yaml.new
ssh root@$IP "\
cp -a /var/lib/rancher/k3s/server/manifests/coredns.yaml \
/var/lib/rancher/k3s/server/manifests/coredns.yaml.bak-\$(date -u +%Y%m%dT%H%M%SZ) && \
install -m 0600 -o root -g root /tmp/coredns.yaml.new \
/var/lib/rancher/k3s/server/manifests/coredns.yaml && \
rm /tmp/coredns.yaml.new && \
sha256sum /var/lib/rancher/k3s/server/manifests/coredns.yaml"
done
kubectl -n kube-system rollout status deployment coredns --timeout=120s
```
The k3s addon controller detects the file's checksum change and
re-applies the Deployment; the rolling update replaces the CoreDNS
pod with the new spec.
## Verification
```bash
kubectl get deploy -n kube-system coredns \
-o jsonpath='{.spec.template.spec.dnsPolicy}{"\n"}{.spec.template.spec.dnsConfig}{"\n"}'
# Expect: dnsPolicy=None + dnsConfig with 3 nameservers.
# No DNSConfigForming event on the current CoreDNS pod:
POD=$(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].metadata.name}')
kubectl get events -n kube-system \
--field-selector involvedObject.name=$POD,reason=DNSConfigForming
# Expect: No resources found.
# Cluster DNS still resolves internal + external:
kubectl run dns-test --restart=Never --image=busybox:1.36 --rm=false --command -- \
sh -c "nslookup kubernetes.default.svc.cluster.local && nslookup forgejo.basicstack.de && echo OK"
kubectl logs dns-test | tail -20
kubectl delete pod dns-test
```
## Upgrade caveat
When the k3s server binary is upgraded to a version that ships a new
CoreDNS manifest (new CoreDNS image, new resource limits, new probes,
etc.), the upgrade will **not** overwrite this file — k3s only creates
the file if it does not exist. Instead, k3s emits an event/log noting
that the on-disk manifest differs from the bundled default. On such an
upgrade, diff the bundled manifest against this one, fold the upstream
changes in, and re-apply.
## Related
- Companion strategic-merge patch:
`apps/observability/patches/coredns-dns-config.yaml`
- Companion node-exporter patch:
`apps/observability/patches/node-exporter-dns-config.yaml`
- Parent issue: `DEV-525` (Kubernetes events)
- Sibling k3s coredns.yaml edit: `DEV-526` (clusterIP change)

232
infrastructure/k3s-manifests/coredns.yaml Normal file
View file

@ -0,0 +1,232 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
}
hosts /etc/coredns/NodeHosts {
ttl 60
reload 15s
fallthrough
}
prometheus :9153
cache 30
loop
reload
loadbalance
import /etc/coredns/custom/*.override
forward . /etc/resolv.conf
}
import /etc/coredns/custom/*.server
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: "CoreDNS"
spec:
revisionHistoryLimit: 0
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
priorityClassName: "system-cluster-critical"
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
nodeSelector:
kubernetes.io/os: linux
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
k8s-app: kube-dns
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
k8s-app: kube-dns
containers:
- name: coredns
image: "rancher/mirrored-coredns-coredns:1.14.4"
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
- name: custom-config-volume
mountPath: /etc/coredns/custom
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
initialDelaySeconds: 0
periodSeconds: 2
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
dnsPolicy: None
dnsConfig:
nameservers:
- 2a01:4ff:ff00::add:2
- 2a01:4ff:ff00::add:1
- 185.12.64.1
searches:
- .
options:
- name: edns0
- name: trust-ad
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
- key: NodeHosts
path: NodeHosts
- name: custom-config-volume
configMap:
name: coredns-custom
optional: true
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.96.0.10
clusterIPs: [10.96.0.10]
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
ipFamilyPolicy: SingleStack

3
infrastructure/scripts/os-update/README.md
View file

@ -23,11 +23,10 @@ Scripts that implement the weekly rolling Ubuntu OS-update procedure.
## Guardrails the scripts enforce ## Guardrails the scripts enforce
- Preflight cluster health failure → refuse to start. - Preflight cluster health failure → refuse to start.
- SSH-drift preflight (DEV-587): before draining anything, verify the operator can SSH to every planned node and that cp-1's pubkey is present in each target's `/root/.ssh/authorized_keys`. Missing pubkeys are auto-appended via the operator's own SSH; nodes that reject the operator entirely fail the whole cycle with the exact repair command. Preflight report is written to `/tmp/os-update-<UTC-ts>/ssh-preflight.txt`. Escape hatch: `--skip-ssh-preflight` for bootstrap.
- Drain with a PDB conflict → uncordon and mark the node `SKIPPED_DRAIN_FAILED`, never `--force`. - Drain with a PDB conflict → uncordon and mark the node `SKIPPED_DRAIN_FAILED`, never `--force`.
- Node reboot fails to come back within timeout → hard stop, escalate. Do NOT rebuild the node or touch k3s config. - Node reboot fails to come back within timeout → hard stop, escalate. Do NOT rebuild the node or touch k3s config.
- kubelet doesn't return `Ready` → hard stop, escalate. Do NOT touch k3s config. - kubelet doesn't return `Ready` → hard stop, escalate. Do NOT touch k3s config.
- Post-node cluster health check red → first retry up to `HEALTH_MAX_RETRIES` (default 10) more times at `HEALTH_RETRY_INTERVAL_SECONDS` (default 30s) to let transient reinit windows settle (DEV-595). Short-circuit halt immediately on any `CrashLoopBackOff`/`ImagePullBackOff`/`ErrImagePull`/`CreateContainerConfigError`/`InvalidImageName`. Still red after all attempts → hard stop, do not proceed to the next node. - Post-node cluster health check red → hard stop, do not proceed to the next node.
- Absolute rule: **no k3s config, no manifests, no PVs, no service files touched** — apt/dpkg only. - Absolute rule: **no k3s config, no manifests, no PVs, no service files touched** — apt/dpkg only.
## Typical invocations ## Typical invocations

351
infrastructure/scripts/os-update/os-update.sh
View file

@ -4,56 +4,34 @@
# Behavior: # Behavior:
# 1. preflight cluster health (fail-closed) # 1. preflight cluster health (fail-closed)
# 2. take an etcd snapshot # 2. take an etcd snapshot
# 3. compute node order: # 3. compute node order (workers first, control plane last;
# workers (non-Stalwart first, Stalwart-hosting fsn1 workers last) # Stalwart-hosting fsn1 workers moved to end of workers group)
# then control-plane nodes (etcd followers first, leader last). # 4. call update-node.sh for each node, halting on any failure
# 3b. SSH-key drift preflight: verify operator can reach every planned node
# and that cp-1's pubkey is present in root's authorized_keys (auto-heal
# by appending if missing; hard-fail if unreachable). See DEV-587.
# 4. call update-node.sh for each worker and update-cp-node.sh for each CP,
# halting on any failure
# 5. finalization: health snapshot + apt history digest # 5. finalization: health snapshot + apt history digest
# #
# Usage: # Usage:
# os-update.sh [--dry-run] [--start-from <node>] [--only <node>] # os-update.sh [--dry-run] [--start-from <node>] [--only <node>]
# [--skip-cps] [--only-cps] [--skip-ssh-preflight]
#
# Flags:
# --skip-cps run workers only (legacy behavior, useful for narrow patches).
# --only-cps skip workers, only walk the CP phase.
# --only <n> update a single node (worker or CP, routed automatically).
# --skip-ssh-preflight skip the SSH-drift preflight (escape hatch for bootstrap
# scenarios where cp-1's pubkey is intentionally absent).
# #
# Environment: # Environment:
# CONTROL_PLANE_HOST (default 178.105.17.239) # CONTROL_PLANE_HOST (default 178.105.17.239)
# POST_CP_WAIT_SECONDS (default 180) — settle between CPs after uncordon. # All env vars honored by update-node.sh are honored here as well.
# All env vars honored by update-node.sh / update-cp-node.sh are honored here.
# #
# Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it # Read OS_UPDATE_PROCEDURE.md alongside this script; the script mirrors it
# step-for-step and the doc is the authoritative reference. CP handling is # step-for-step and the doc is the authoritative reference.
# delegated to update-cp-node.sh; see CP_UPDATE_PROCEDURE.md.
set -euo pipefail set -euo pipefail
CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}" CONTROL_PLANE_HOST="${CONTROL_PLANE_HOST:-178.105.17.239}"
POST_CP_WAIT_SECONDS="${POST_CP_WAIT_SECONDS:-180}"
DRY_RUN=0 DRY_RUN=0
START_FROM="" START_FROM=""
ONLY="" ONLY=""
SKIP_CPS=0
ONLY_CPS=0
SKIP_SSH_PREFLIGHT="${SKIP_SSH_PREFLIGHT:-0}"
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
--dry-run) DRY_RUN=1; shift ;; --dry-run) DRY_RUN=1; shift ;;
--start-from) START_FROM="$2"; shift 2 ;; --start-from) START_FROM="$2"; shift 2 ;;
--only) ONLY="$2"; shift 2 ;; --only) ONLY="$2"; shift 2 ;;
--skip-cps) SKIP_CPS=1; shift ;; -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
--only-cps) ONLY_CPS=1; shift ;; *) echo "unknown arg: $1" >&2; exit 2 ;;
--skip-ssh-preflight) SKIP_SSH_PREFLIGHT=1; shift ;;
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac esac
done done
@ -65,27 +43,6 @@ mkdir -p "$LOG_DIR"
log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; } log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_DIR/main.log"; }
die() { log "FATAL: $*"; exit 1; } die() { log "FATAL: $*"; exit 1; }
# node_ssh_endpoint <node-name>
# Prints the ssh argv fragment (may include `-J bastion`) used to reach a node.
# Private-IP workers go through cp-1 via ProxyJump; every other node has a
# public IP. Returns non-zero for an unknown node. Callers pass the output
# unquoted so `-J` is treated as its own arg.
node_ssh_endpoint() {
case "$1" in
k3s-cp-1) echo "root@178.105.17.239" ;;
k3s-cp-2) echo "root@188.245.85.199" ;;
k3s-cp-3) echo "root@49.13.92.162" ;;
k3s-worker-1) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
k3s-worker-2) echo "-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
k3s-worker-3) echo "root@167.233.121.121" ;;
k3s-worker-4) echo "root@128.140.3.80" ;;
k3s-worker-5) echo "root@167.233.192.86" ;;
k3s-worker-6) echo "root@49.12.223.133" ;;
k3s-update-runner) echo "root@167.233.79.65" ;;
*) return 1 ;;
esac
}
log "=== os-update.sh cycle $STAMP ===" log "=== os-update.sh cycle $STAMP ==="
log "log dir: $LOG_DIR" log "log dir: $LOG_DIR"
@ -111,17 +68,17 @@ fi
# --- 3. node ordering --------------------------------------------------------- # --- 3. node ordering ---------------------------------------------------------
# Ordering rule: # Ordering rule:
# - workers first (non-Stalwart, then Stalwart-hosting fsn1 last) # - workers only
# - control-plane nodes AFTER workers, one at a time, etcd followers first, # - within workers: nodes NOT hosting Stalwart first, Stalwart-hosting fsn1 nodes last
# leader last. Rationale (see CP_UPDATE_PROCEDURE.md): # - ALL control-plane nodes are EXCLUDED and never updated by this script. Rationale:
# * two of three CPs down = etcd quorum loss = write outage. Only one CP # * kine/etcd write-path is sensitive to concurrent drains (see kine thundering-herd
# cordoned/rebooting at a time. # guardrails in OS_UPDATE_PROCEDURE.md, added after DEV-495).
# * draining a follower is a no-op for raft leadership; draining the leader # * rebooting a CP removes one apiserver — needs external liveness monitoring.
# forces a re-election, so doing followers first minimises leader flapping. # * CP nodes host their own StatefulSet workloads that need batched eviction.
# Since DEV-510 (2026-08-22) the cluster runs HA embedded etcd across # Since DEV-510 (2026-08-22) the cluster runs HA control plane (cp-1/cp-2/cp-3). The
# cp-1/cp-2/cp-3 with 3-node quorum. This script delegates all CP steps to # exclusion here is role-based so ALL current and future CPs are covered automatically.
# update-cp-node.sh, which handles swap (idempotent), preflight, drain, apt, # To update a CP, use `scripts/os-update/update-cp-1.sh` (currently cp-1-only; will be
# reboot with /livez polling, etcd rejoin check, and finalize. # generalized to any CP as part of the HA-aware CP OS-update procedure follow-up).
STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true) STALWART_NODE=$(kubectl -n stalwart get pod -l app=stalwart -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null | tr ' ' '\n' | sort -u || true)
# If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers. # If the pod's not currently up (e.g. Pending) we still want to protect fsn1 workers.
@ -151,222 +108,38 @@ for n in $ALL_NODES; do
fi fi
done done
WORKER_ORDER=("${workers[@]}" "${stalwart_workers[@]}") ORDER=("${workers[@]}" "${stalwart_workers[@]}")
# NOTE: CP nodes intentionally excluded. To update a CP, see CP1_UPDATE_PROCEDURE.md.
# --- 3b. control-plane ordering (leader last) --------------------------------
# CP-node → public IP map. Kept out-of-cluster so a full-cluster incident
# doesn't hide etcd from the operator machine.
declare -A CP_HOST=(
[k3s-cp-1]="178.105.17.239"
[k3s-cp-2]="188.245.85.199"
[k3s-cp-3]="49.13.92.162"
)
etcd_leader_ip() {
# Query etcd leader endpoint from the operator's control-plane host. Uses
# the same TLS material as update-cp-node.sh. Prints "IP" on success,
# nothing on failure.
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -o BatchMode=yes \
root@"$CONTROL_PLANE_HOST" \
'ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
--cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
--key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
endpoint status --cluster -w simple 2>/dev/null' \
| awk -F, '$5 ~ /true/ {print $1}' \
| sed -E 's|https?://([^:]+):.*|\1|'
}
CP_ORDER=()
if [ -n "$CP_NODES" ]; then if [ -n "$CP_NODES" ]; then
LEADER_IP="$(etcd_leader_ip || true)" log "[plan] EXCLUDING control-plane nodes: $(echo $CP_NODES | tr '\n' ' ')— use scripts/os-update/update-cp-1.sh (see CP1_UPDATE_PROCEDURE.md)"
leader_node=""
followers=()
for n in $CP_NODES; do
ip="${CP_HOST[$n]:-}"
if [ -n "$ip" ] && [ "$ip" = "$LEADER_IP" ]; then
leader_node="$n"
else
followers+=("$n")
fi
done
if [ -n "$leader_node" ]; then
CP_ORDER=("${followers[@]}" "$leader_node")
log "[plan] etcd leader is $leader_node ($LEADER_IP) — updating followers first, leader last"
else
# Couldn't identify leader — refuse to guess. Fall back to no CPs.
CP_ORDER=()
log "[plan] WARN could not identify etcd leader — skipping CP phase this cycle"
fi
fi fi
# Route --only <node> automatically to worker or CP list.
if [ -n "$ONLY" ]; then if [ -n "$ONLY" ]; then
if is_cp "$ONLY"; then ORDER=("$ONLY")
CP_ORDER=("$ONLY")
WORKER_ORDER=()
else
WORKER_ORDER=("$ONLY")
CP_ORDER=()
fi
elif [ -n "$START_FROM" ]; then elif [ -n "$START_FROM" ]; then
new=() new=()
skip=1 skip=1
for n in "${WORKER_ORDER[@]}"; do for n in "${ORDER[@]}"; do
[ "$n" = "$START_FROM" ] && skip=0 [ "$n" = "$START_FROM" ] && skip=0
[ $skip -eq 0 ] && new+=("$n") [ $skip -eq 0 ] && new+=("$n")
done done
WORKER_ORDER=("${new[@]}") ORDER=("${new[@]}")
# If START_FROM is a CP, trim CP_ORDER instead.
if is_cp "$START_FROM"; then
new_cp=()
skip=1
for n in "${CP_ORDER[@]:-}"; do
[ -z "$n" ] && continue
[ "$n" = "$START_FROM" ] && skip=0
[ $skip -eq 0 ] && new_cp+=("$n")
done
CP_ORDER=("${new_cp[@]}")
WORKER_ORDER=()
fi
fi fi
[ "$SKIP_CPS" -eq 1 ] && CP_ORDER=() log "[plan] ordered nodes (${#ORDER[@]}): ${ORDER[*]}"
[ "$ONLY_CPS" -eq 1 ] && WORKER_ORDER=() printf '%s\n' "${ORDER[@]}" > "$LOG_DIR/plan.txt"
log "[plan] workers (${#WORKER_ORDER[@]}): ${WORKER_ORDER[*]:-none}"
log "[plan] control-planes (${#CP_ORDER[@]}): ${CP_ORDER[*]:-none}"
{
echo "# workers"
printf '%s\n' "${WORKER_ORDER[@]:-}"
echo "# control-planes"
printf '%s\n' "${CP_ORDER[@]:-}"
} > "$LOG_DIR/plan.txt"
if [ "$DRY_RUN" -eq 1 ]; then if [ "$DRY_RUN" -eq 1 ]; then
log "DRY-RUN — plan written, no node touched. Exiting." log "DRY-RUN — plan written, no node touched. Exiting."
exit 0 exit 0
fi fi
# --- 3c. SSH-key drift preflight --------------------------------------------- # --- 4. per-node loop ---------------------------------------------------------
# DEV-587 / DEV-586 (2026-09-27) — some k3s nodes drift to lose cp-1's SSH
# pubkey from /root/.ssh/authorized_keys (probable Hetzner cloud-init resync
# from the HC SSH-keys API; the operator key is NOT in that keyset). When this
# happens mid-cycle the whole run aborts partway through drain / peer-probe.
# This preflight proves the operator can SSH to every planned node and that
# cp-1's pubkey is present in root's authorized_keys — auto-heals drift via
# the operator's own SSH access, and hard-fails BEFORE anything is drained if
# a node is genuinely unreachable.
ssh_drift_preflight() {
local ok=() repaired=() problem=()
log "[preflight] SSH-key drift check for planned nodes"
local cp1_pubkey cp1_key_body
if ! cp1_pubkey=$(ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
root@"$CONTROL_PLANE_HOST" 'cat /root/.ssh/id_rsa.pub' 2>/dev/null); then
die "SSH-drift preflight: cannot read cp-1 pubkey from $CONTROL_PLANE_HOST — fix operator->cp-1 SSH first"
fi
cp1_pubkey=$(printf '%s' "$cp1_pubkey" | tr -d '\r\n')
cp1_key_body=$(printf '%s' "$cp1_pubkey" | awk '{print $2}')
[ -n "$cp1_key_body" ] || die "SSH-drift preflight: could not parse cp-1 pubkey (got: $cp1_pubkey)"
log " cp-1 pubkey OK (${cp1_key_body:0:32}...)"
local plan_nodes=()
for n in "${WORKER_ORDER[@]:-}" "${CP_ORDER[@]:-}"; do
[ -z "$n" ] && continue
plan_nodes+=("$n")
done
if [ ${#plan_nodes[@]} -eq 0 ]; then
log " (no planned nodes — nothing to preflight)"
return 0
fi
for n in "${plan_nodes[@]}"; do
local endpoint
if ! endpoint=$(node_ssh_endpoint "$n"); then
log " $n: unknown ssh endpoint — refusing"
problem+=("$n (no ssh mapping in node_ssh_endpoint)")
continue
fi
# 1. reachability from the operator machine.
# shellcheck disable=SC2086 # $endpoint may include `-J bastion`; deliberate word-split.
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
$endpoint true 2>/dev/null; then
problem+=("$n ($endpoint - operator SSH refused)")
continue
fi
# 2. cp-1 is the source of the key; drift check would be pointless on it.
if [ "$n" = "k3s-cp-1" ]; then
ok+=("$n")
continue
fi
# 3. cp-1 pubkey present?
# shellcheck disable=SC2086
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
$endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
ok+=("$n")
continue
fi
# 4. auto-heal — append cp-1's pubkey via the (already verified) operator SSH.
log " $n: cp-1 pubkey MISSING from /root/.ssh/authorized_keys — appending"
# shellcheck disable=SC2086
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
$endpoint "install -d -m 700 /root/.ssh && printf '%s\n' '$cp1_pubkey' >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys" \
2>/dev/null; then
# shellcheck disable=SC2086
if ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \
$endpoint "grep -qF '$cp1_key_body' /root/.ssh/authorized_keys" 2>/dev/null; then
repaired+=("$n")
else
problem+=("$n (auto-heal wrote but grep-verify failed)")
fi
else
problem+=("$n ($endpoint - auto-heal write refused)")
fi
done
log "[preflight] SSH-drift summary:"
log " ok (${#ok[@]}): ${ok[*]:-none}"
log " repaired (${#repaired[@]}): ${repaired[*]:-none}"
log " problem (${#problem[@]}): ${problem[*]:-none}"
{
echo "# SSH-drift preflight — $(date -u +%FT%TZ)"
echo "# cp-1 pubkey: $cp1_pubkey"
printf 'ok\t%s\n' "${ok[*]:-}"
printf 'repaired\t%s\n' "${repaired[*]:-}"
printf 'problem\t%s\n' "${problem[*]:-}"
} > "$LOG_DIR/ssh-preflight.txt"
if [ ${#problem[@]} -gt 0 ]; then
log "SSH-drift preflight FAILED for: ${problem[*]}"
log " cp-1 pubkey to install:"
log " $cp1_pubkey"
log " From any machine that CAN reach the failing node, run:"
log " ssh root@<node> \"grep -qF '$cp1_key_body' /root/.ssh/authorized_keys || echo '$cp1_pubkey' >> /root/.ssh/authorized_keys\""
die "SSH-drift preflight failed — refuse to start rolling update"
fi
}
if [ "$SKIP_SSH_PREFLIGHT" -eq 1 ]; then
log "[preflight] SSH-drift check SKIPPED (--skip-ssh-preflight)"
else
ssh_drift_preflight
fi
# --- 4. worker loop ----------------------------------------------------------
updated=() updated=()
skipped=() skipped=()
for n in "${WORKER_ORDER[@]}"; do for n in "${ORDER[@]}"; do
[ -z "$n" ] && continue
log "====================================================================" log "===================================================================="
log "==> updating worker $n" log "==> updating $n"
log "====================================================================" log "===================================================================="
NODE_LOG="$LOG_DIR/${n}.log" NODE_LOG="$LOG_DIR/${n}.log"
set +e set +e
@ -380,50 +153,6 @@ for n in "${WORKER_ORDER[@]}"; do
esac esac
done done
# --- 4b. control-plane loop --------------------------------------------------
# One CP at a time, followers first, leader last. Delegates to update-cp-node.sh
# --run (Phase A add-swap is idempotent, Phase B..F run with ASSUME_YES=1).
# Between CPs we run cluster-health.sh and re-verify etcd cluster status; halt
# on any yellow signal.
if [ ${#CP_ORDER[@]} -gt 0 ]; then
log "===================================================================="
log "==> control-plane phase (${#CP_ORDER[@]} nodes): ${CP_ORDER[*]}"
log "===================================================================="
first_cp=1
for cp in "${CP_ORDER[@]}"; do
[ -z "$cp" ] && continue
if [ $first_cp -eq 0 ]; then
log "-- health gate between CPs (settle ${POST_CP_WAIT_SECONDS}s, then re-check)"
sleep "$POST_CP_WAIT_SECONDS"
if ! RETRY_ON_TRANSIENT=1 "$SCRIPT_DIR/cluster-health.sh" 2>&1 | tee "$LOG_DIR/inter-cp-health-before-${cp}.log"; then
die "cluster health degraded between CPs — halting before $cp. See $LOG_DIR/"
fi
# Re-check etcd cluster status; every member must still be started.
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$CONTROL_PLANE_HOST" \
'ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
--cert=/var/lib/rancher/k3s/server/tls/etcd/server-client.crt \
--key=/var/lib/rancher/k3s/server/tls/etcd/server-client.key \
endpoint status --cluster -w table' 2>&1 | tee -a "$LOG_DIR/inter-cp-health-before-${cp}.log" || \
die "etcd endpoint status failed between CPs — halting before $cp"
fi
log "===================================================================="
log "==> updating control-plane $cp"
log "===================================================================="
CP_LOG="$LOG_DIR/${cp}.log"
set +e
ASSUME_YES=1 "$SCRIPT_DIR/update-cp-node.sh" "$cp" --run 2>&1 | tee "$CP_LOG"
rc=${PIPESTATUS[0]}
set -e
case $rc in
0) updated+=("$cp") ;;
*) die "update-cp-node.sh failed for $cp (rc=$rc). Cycle halted. See $CP_LOG" ;;
esac
first_cp=0
done
fi
# --- 5. finalization ---------------------------------------------------------- # --- 5. finalization ----------------------------------------------------------
log "====================================================================" log "===================================================================="
log "==> finalization" log "==> finalization"
@ -437,11 +166,17 @@ log "[final] apt history digest"
{ {
for n in "${updated[@]}"; do for n in "${updated[@]}"; do
echo "=== $n ===" echo "=== $n ==="
if ! t=$(node_ssh_endpoint "$n"); then # resolve ssh target via a mini-eval of node_ssh_target-equivalent
echo " (unknown ssh target)" case "$n" in
continue k3s-cp-1) t="root@178.105.17.239" ;;
fi k3s-worker-1) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.2" ;;
# shellcheck disable=SC2086 # $t may include `-J bastion`; deliberate word-split. k3s-worker-2) t="-J root@$CONTROL_PLANE_HOST root@10.42.1.3" ;;
k3s-worker-3) t="root@167.233.121.121" ;;
k3s-worker-4) t="root@128.140.3.80" ;;
k3s-worker-5) t="root@167.233.192.86" ;;
k3s-update-runner) t="root@167.233.79.65" ;;
*) echo " (unknown ssh target)"; continue ;;
esac
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \ ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new $t \
'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \ 'zgrep -h "Commandline\|Install\|Upgrade\|Remove" /var/log/apt/history.log* 2>/dev/null | tail -40' 2>/dev/null \
|| echo " (could not read apt history)" || echo " (could not read apt history)"

Some files were not shown because too many files have changed in this diff Show more