Compare commits
1 commit
main
...
chore/DEV-
| Author | SHA1 | Date | |
|---|---|---|---|
|
a2f21a4d96 |
132 changed files with 33713 additions and 9393 deletions
|
|
@ -3,27 +3,16 @@ kind: Application
|
|||
metadata:
|
||||
name: argocd
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/argocd
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: argocd
|
||||
sources:
|
||||
- repoURL: https://argoproj.github.io/argo-helm
|
||||
chart: argo-cd
|
||||
targetRevision: 10.4.0
|
||||
helm:
|
||||
releaseName: argocd
|
||||
valueFiles:
|
||||
- $values/apps/argocd/values.yaml
|
||||
- repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/argocd
|
||||
ref: values
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: grist
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/grist
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: grist
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: kaneo
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/kaneo
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kaneo
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: openwebui
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/openwebui
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: openwebui
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: outline
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/outline
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: outline
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: velero-monitoring
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: velero
|
||||
directory:
|
||||
recurse: false
|
||||
include: '{servicemonitor.yaml,prometheusrule.yaml}'
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: velero
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: velero-restore-drill
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: velero/restore-drill
|
||||
directory:
|
||||
recurse: false
|
||||
include: '*.yaml'
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: velero
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: velero-schedules
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: velero/schedules
|
||||
directory:
|
||||
recurse: false
|
||||
include: '*.yaml'
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: velero
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: velero
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/velero
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: velero
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: false
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: vikunja
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/vikunja
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: vikunja
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
|
@ -2,83 +2,69 @@
|
|||
|
||||
This directory contains the Argo CD deployment configuration for the basicstack.de k3s cluster.
|
||||
|
||||
Argo CD itself is installed from the community Helm chart (`argoproj/argo-helm`, chart `argo-cd`). This directory holds the chart values file plus a small kustomize wrapper for the ingress and sealed secrets that stay in git.
|
||||
|
||||
## Files
|
||||
|
||||
- `values.yaml` - Helm values for the `argo-cd` chart (image tag, OIDC/RBAC config, resource limits, ingress disabled).
|
||||
- `kustomization.yaml` - Kustomize wrapper for the ingress + sealed secrets (does NOT install Argo CD itself).
|
||||
- `argocd-ingress.yaml` - Ingress configuration for the Argo CD UI (Traefik + cert-manager `letsencrypt-prod`).
|
||||
- `argocd-oidc-secret-sealed.yaml` - Sealed secret for Pocket ID OIDC integration.
|
||||
- `repo-*.yaml` - Sealed secrets for Git repository access.
|
||||
- `argocd-install.yaml` - Auto-generated Argo CD installation manifest (DO NOT EDIT DIRECTLY)
|
||||
- `kustomization.yaml` - Kustomize overlay that adds resource limits and other customizations
|
||||
- `argocd-ingress.yaml` - Ingress configuration for Argo CD UI
|
||||
- `argocd-oidc-secret-sealed.yaml` - Sealed secret for OIDC integration
|
||||
- `repo-*.yaml` - Sealed secrets for Git repository access
|
||||
|
||||
## How the install is wired
|
||||
## Resource Limits
|
||||
|
||||
The root [`../app-argocd.yaml`](../app-argocd.yaml) is an Argo CD `Application` with two sources:
|
||||
**IMPORTANT**: Resource limits were added after DEV-281 (resource exhaustion incident on 2026-07-12).
|
||||
|
||||
1. The public Helm chart at `https://argoproj.github.io/argo-helm`, chart `argo-cd`, `targetRevision` pinned in git.
|
||||
2. This repo (`ref: values`) providing the `values.yaml` used by source (1) AND the ingress/sealed secrets applied via `kustomize`.
|
||||
|
||||
Once bootstrapped, Argo CD manages its own install by syncing this Application.
|
||||
|
||||
## Resource limits
|
||||
|
||||
Memory limits were added after DEV-281 (resource exhaustion incident on 2026-07-12) and are now driven by `values.yaml`:
|
||||
All Argo CD components now have memory limits to prevent OOM incidents:
|
||||
|
||||
| Component | Memory Limit | Memory Request |
|
||||
|-----------|--------------|----------------|
|
||||
| application-controller | 1Gi | 512Mi |
|
||||
| application-controller | 512Mi | 256Mi |
|
||||
| repo-server | 512Mi | 256Mi |
|
||||
| redis | 256Mi | 128Mi |
|
||||
| server | 256Mi | 128Mi |
|
||||
| notifications-controller | 128Mi | 64Mi |
|
||||
| applicationset-controller | 256Mi | 128Mi |
|
||||
|
||||
These limits are based on observed usage patterns and provide headroom while preventing unlimited memory consumption.
|
||||
|
||||
## Deployment
|
||||
|
||||
### Steady state (managed by Argo CD)
|
||||
|
||||
Once the cluster is bootstrapped, changes to this directory are picked up by the root `argocd` Application on the next sync. No manual `kubectl apply` is required.
|
||||
|
||||
### First-time / disaster-recovery bootstrap
|
||||
|
||||
Argo CD cannot install itself while it is gone. Bootstrap with helm, then hand ownership back:
|
||||
### Option 1: Apply with kustomize (RECOMMENDED)
|
||||
|
||||
```bash
|
||||
helm repo add argo https://argoproj.github.io/argo-helm
|
||||
helm repo update
|
||||
helm install argocd argo/argo-cd \
|
||||
--version 10.4.0 \
|
||||
--namespace argocd --create-namespace \
|
||||
--values apps/argocd/values.yaml \
|
||||
--wait --timeout 10m
|
||||
|
||||
kubectl apply -k apps/argocd/ # ingress + sealed secrets
|
||||
kubectl apply -f apps/app-argocd.yaml # hand ownership back to GitOps
|
||||
kubectl apply -k apps/argocd/
|
||||
```
|
||||
|
||||
This will apply the base manifests plus all patches defined in `kustomization.yaml`.
|
||||
|
||||
### Option 2: Direct apply (not recommended)
|
||||
|
||||
```bash
|
||||
kubectl apply -f apps/argocd/argocd-install.yaml
|
||||
kubectl apply -f apps/argocd/argocd-ingress.yaml
|
||||
# etc.
|
||||
```
|
||||
|
||||
**Note**: This skips the resource limit patches and is NOT recommended.
|
||||
|
||||
## Updating Argo CD
|
||||
|
||||
Bump the chart and the image tag in a single PR:
|
||||
When updating to a new Argo CD version:
|
||||
|
||||
1. Refresh the local helm repo cache and check what's available:
|
||||
1. Download the new install manifest:
|
||||
```bash
|
||||
helm repo update
|
||||
helm search repo argo/argo-cd --versions | head
|
||||
curl -sSL https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml > argocd-install.yaml
|
||||
```
|
||||
|
||||
2. Bump both fields together, keeping them in sync with the chart's `appVersion`:
|
||||
- `apps/app-argocd.yaml` -> `spec.sources[0].targetRevision` (chart version, e.g. `10.4.0`)
|
||||
- `apps/argocd/values.yaml` -> `global.image.tag` (app version, e.g. `v3.5.1`)
|
||||
|
||||
3. Optionally render locally to sanity-check the output before opening the PR:
|
||||
2. Apply with kustomize (resource limits will be automatically applied):
|
||||
```bash
|
||||
helm template argocd argo/argo-cd \
|
||||
--version <new-chart-version> \
|
||||
-f apps/argocd/values.yaml -n argocd | less
|
||||
kubectl apply -k apps/argocd/
|
||||
```
|
||||
|
||||
4. Open the PR. After merge, Argo CD syncs itself onto the new version.
|
||||
3. Verify resource limits are in place:
|
||||
```bash
|
||||
kubectl get statefulset,deployment -n argocd -o custom-columns='NAME:.metadata.name,MEMORY_LIMIT:.spec.template.spec.containers[0].resources.limits.memory'
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
@ -88,21 +74,22 @@ Bump the chart and the image tag in a single PR:
|
|||
kubectl top pods -n argocd
|
||||
```
|
||||
|
||||
### Check applied resource limits
|
||||
### Check if resource limits are applied
|
||||
|
||||
```bash
|
||||
kubectl get deployment,statefulset -n argocd -o custom-columns='NAME:.metadata.name,MEMORY_LIMIT:.spec.template.spec.containers[0].resources.limits.memory'
|
||||
kubectl get deployment,statefulset -n argocd -o json | jq '.items[] | {name: .metadata.name, limits: .spec.template.spec.containers[0].resources.limits}'
|
||||
```
|
||||
|
||||
### Rollback if a chart upgrade misbehaves
|
||||
### Rollback if needed
|
||||
|
||||
If there are issues after applying resource limits:
|
||||
|
||||
```bash
|
||||
helm -n argocd history argocd
|
||||
helm -n argocd rollback argocd <previous-revision>
|
||||
# Remove limits from a specific component
|
||||
kubectl patch deployment -n argocd argocd-server --type='json' -p='[{"op": "remove", "path": "/spec/template/spec/containers/0/resources"}]'
|
||||
```
|
||||
|
||||
## History
|
||||
|
||||
- **2026-08-23**: Switched to the community Helm chart, bumped to `v3.5.1` / chart `10.4.0` (DEV-519).
|
||||
- **2026-07-12**: Added resource limits via kustomization to prevent OOM incidents (DEV-281).
|
||||
- **2026-07-11**: Initial deployment (vendored `install.yaml`).
|
||||
- **2026-07-12**: Added resource limits via kustomization to prevent OOM incidents (DEV-281)
|
||||
- **2026-07-11**: Initial deployment
|
||||
|
|
|
|||
33364
apps/argocd/argocd-install.yaml
Normal file
33364
apps/argocd/argocd-install.yaml
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -16,6 +16,4 @@ spec:
|
|||
creationTimestamp: null
|
||||
name: argocd-oidc-secret
|
||||
namespace: argocd
|
||||
labels:
|
||||
app.kubernetes.io/part-of: argocd
|
||||
type: Opaque
|
||||
|
|
|
|||
|
|
@ -4,7 +4,82 @@ kind: Kustomization
|
|||
namespace: argocd
|
||||
|
||||
resources:
|
||||
- argocd-install.yaml
|
||||
- argocd-ingress.yaml
|
||||
- argocd-oidc-secret-sealed.yaml
|
||||
- repo-basicstack-org-secret-sealed.yaml
|
||||
- repo-stack-basicstack-de-secret-sealed.yaml
|
||||
|
||||
patches:
|
||||
# Add memory limits to prevent OOM incidents (DEV-281)
|
||||
- target:
|
||||
kind: StatefulSet
|
||||
name: argocd-application-controller
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 512Mi
|
||||
requests:
|
||||
memory: 256Mi
|
||||
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: argocd-repo-server
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 512Mi
|
||||
requests:
|
||||
memory: 256Mi
|
||||
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: argocd-redis
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
requests:
|
||||
memory: 128Mi
|
||||
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: argocd-server
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
requests:
|
||||
memory: 128Mi
|
||||
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: argocd-notifications-controller
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 128Mi
|
||||
requests:
|
||||
memory: 64Mi
|
||||
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: argocd-applicationset-controller
|
||||
patch: |-
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/resources
|
||||
value:
|
||||
limits:
|
||||
memory: 256Mi
|
||||
requests:
|
||||
memory: 128Mi
|
||||
|
|
|
|||
|
|
@ -1,87 +0,0 @@
|
|||
global:
|
||||
image:
|
||||
tag: v3.5.1
|
||||
|
||||
configs:
|
||||
cm:
|
||||
url: https://argo.basicstack.de
|
||||
application.instanceLabelKey: argocd.argoproj.io/instance
|
||||
resource.exclusions: |
|
||||
- apiGroups: [cilium.io]
|
||||
kinds: [CiliumIdentity, CiliumEndpoint, CiliumEndpointSlice]
|
||||
- apiGroups: [kyverno.io, reports.kyverno.io, wgpolicyk8s.io]
|
||||
kinds: [PolicyReport, ClusterPolicyReport, EphemeralReport,
|
||||
ClusterEphemeralReport, AdmissionReport, ClusterAdmissionReport,
|
||||
BackgroundScanReport, ClusterBackgroundScanReport, UpdateRequest]
|
||||
oidc.config: |
|
||||
name: Pocket ID
|
||||
issuer: https://auth.basicstack.de
|
||||
clientID: $argocd-oidc-secret:oidc.pocketid.clientId
|
||||
clientSecret: $argocd-oidc-secret:oidc.pocketid.clientSecret
|
||||
requestedScopes: [openid, profile, email, groups]
|
||||
requestedIDTokenClaims:
|
||||
groups: {essential: true}
|
||||
|
||||
rbac:
|
||||
policy.default: role:readonly
|
||||
policy.csv: |
|
||||
g, argo_admins, role:admin
|
||||
p, role:admin, applications, *, */*, allow
|
||||
p, role:admin, clusters, *, *, allow
|
||||
p, role:admin, repositories, *, *, allow
|
||||
p, role:admin, projects, *, *, allow
|
||||
p, role:admin, accounts, *, *, allow
|
||||
p, role:admin, gpgkeys, *, *, allow
|
||||
p, role:admin, certificates, *, *, allow
|
||||
p, role:admin, exec, *, *, allow
|
||||
|
||||
params:
|
||||
server.insecure: "true"
|
||||
|
||||
ssh:
|
||||
extraHosts: |
|
||||
forgejo.forgejo.svc.cluster.local ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCnFlDTGVmri7geuOfZ1UvHzCRi1U5BqCq+9MwkrbGkZCmBmUt3ek95JBZzurLvR/0rE624B9tB1AvSJIEIQ1W1YXD4ydiDaHX6J0sip6lEfqhREx0y91C15zHgi9jN0UKidse1g0xlLHpwePUmz8/2BJcLHJuwbZSdUu7+uhDIYtIJ5+3trX3IABNeluRA5TkspPAC0ViEaz4saWgWQWjKL8dsb3jIR94DiWAVpQnhaCBEILkIStJrmWl0O5B10Jr3KWy18szr9UVd8edCkoEXriCR8gx69jHdmuem5WlZPGvsK5Adf0mXE8S3rdyHqWOkDRs6Wlwd9p/1nDY8c8wtE3vqhefCpt1BwpTys9PMmgUfG0at/W+NdvalRqgQz26Bso8Tf7hcfyA/B69U2pvjA0tbgdIGJ5kLeCzpz9kpCYn8wSIuNIJ86BpyfnjRYFCYJVc6Ls86i8j3fEZAjEX7bmbeDBHQnyyH+rjq+Llo8aUd2Uf7HoSA93EjeuWq/Ta+YbWEWrp9Mrd48jnypxHXsiwDzqkm+YGQbXHfasiarxji/eQ5UMMG8hCxpLp1lJLhGN2th4eCLkpwchFr5jZGWgyZCth1WzQGj7NHvDTxKRU6n4MfsEX1B6GF0D60qtM5vZynpmO902mkn6wtxo+pCkDMroj668a62zw3rSS5Bw==
|
||||
|
||||
controller:
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
|
||||
repoServer:
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
|
||||
server:
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
redis:
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
|
||||
notifications:
|
||||
resources:
|
||||
requests:
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
|
||||
applicationSet:
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
|
|
@ -100,8 +100,6 @@ metadata:
|
|||
namespace: bookstack
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: bookstack
|
||||
|
|
@ -125,7 +123,6 @@ spec:
|
|||
containers:
|
||||
- name: bookstack
|
||||
image: lscr.io/linuxserver/bookstack:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
|
|
@ -203,7 +200,7 @@ spec:
|
|||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
path: /
|
||||
port: 80
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
|
|
@ -211,7 +208,7 @@ spec:
|
|||
failureThreshold: 6
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
path: /
|
||||
port: 80
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ spec:
|
|||
fsGroup: 1000
|
||||
containers:
|
||||
- name: directus
|
||||
image: directus/directus:12.3.0
|
||||
image: directus/directus:12.1.1
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8055
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ spec:
|
|||
serviceAccountName: dozzle
|
||||
containers:
|
||||
- name: dozzle
|
||||
image: amir20/dozzle:v10.7.3
|
||||
image: amir20/dozzle:v10.6.10
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: dozzle-http
|
||||
|
|
|
|||
|
|
@ -50,40 +50,17 @@ kubectl apply -f apps/app-forgejo-runner.yaml
|
|||
|
||||
## Runner Configuration
|
||||
|
||||
The runner is deployed as a Deployment (single replica) that bind-mounts the
|
||||
host's `/var/run/docker.sock` to execute job containers. This requires the
|
||||
Docker Engine (package `docker.io`) to be installed and running on the target
|
||||
node — see **Node prerequisites** below.
|
||||
The runner is deployed as a StatefulSet with Docker-in-Docker (dind) sidecar for proper isolation and state management.
|
||||
|
||||
Configuration:
|
||||
- **Deployment type**: Deployment (replicas=1)
|
||||
- **Docker execution**: Host Docker socket (`/var/run/docker.sock`)
|
||||
- **Deployment type**: StatefulSet (stable pod identity, persistent storage)
|
||||
- **Docker execution**: Docker-in-Docker sidecar (privileged init container)
|
||||
- **Concurrent jobs**: 2 (configurable via config.yaml)
|
||||
- **Labels**: ubuntu-latest:docker://node:24-bookworm, ubuntu-22.04:docker://node:24-bookworm
|
||||
- **Forgejo URL**: https://forgejo.basicstack.de (external URL for proper webhook/API access)
|
||||
- **Node selector**: `basicstack.de/docker=true` — schedules only on nodes with the
|
||||
Docker Engine installed. See **Node prerequisites** below.
|
||||
|
||||
## Node prerequisites (required)
|
||||
|
||||
The runner uses the host's Docker daemon. Every worker node that should be
|
||||
eligible to run the runner MUST have `docker.io` installed, the `docker`
|
||||
systemd unit enabled, and be labeled `basicstack.de/docker=true`. Bootstrap a
|
||||
worker with:
|
||||
|
||||
```bash
|
||||
ssh root@<node>
|
||||
DEBIAN_FRONTEND=noninteractive apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io
|
||||
systemctl enable --now docker
|
||||
# from the control plane:
|
||||
kubectl label node <node-name> basicstack.de/docker=true --overwrite
|
||||
```
|
||||
|
||||
Rolling OS updates that reimage a node will remove Docker. Re-run the bootstrap
|
||||
after any node reimage. Without Docker, the runner pod fails to mount
|
||||
`/var/run/docker.sock` (hostPath type check for `Socket` fails); the
|
||||
nodeSelector prevents that scheduling mistake by pinning to labeled nodes.
|
||||
- **Persistent volumes**:
|
||||
- runner-data (1Gi): Runner registration and config
|
||||
- docker-data (20Gi): Docker image cache
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ spec:
|
|||
app: forgejo-runner
|
||||
spec:
|
||||
serviceAccountName: forgejo-runner
|
||||
nodeSelector:
|
||||
basicstack.de/docker: "true"
|
||||
containers:
|
||||
- name: runner
|
||||
image: code.forgejo.org/forgejo/runner:4.0.1
|
||||
|
|
|
|||
|
|
@ -1,18 +1,3 @@
|
|||
---
|
||||
# Forgejo pg_dump → restic → Hetzner Object Storage (DEV-514).
|
||||
#
|
||||
# Replaces the legacy volume-based backup that wrote pg_dump files to
|
||||
# an RWO PVC (`platform-backup-data`). The hcloud volume backing that
|
||||
# PVC was deleted externally on 2026-08-17; rather than re-provision
|
||||
# the same legacy pattern, this migrates to the restic→S3 pipeline
|
||||
# already used for loki/grafana/prometheus (DEV-485…DEV-492).
|
||||
#
|
||||
# initContainer runs pg_dump -F c into an emptyDir; main container
|
||||
# runs `restic backup --stdin-from-command` isn't used here because
|
||||
# the dump has to complete before restic starts (need a proper exit
|
||||
# code, and the custom-format dump is not resumable). Instead we
|
||||
# stage the dump on emptyDir and let restic dedup it into
|
||||
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/forgejo`.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
|
|
@ -20,185 +5,76 @@ metadata:
|
|||
namespace: forgejo
|
||||
labels:
|
||||
app: forgejo-backup
|
||||
backend: restic
|
||||
spec:
|
||||
schedule: "0 3 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
successfulJobsHistoryLimit: 7
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app: forgejo-backup
|
||||
backend: restic
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: forgejo-backup
|
||||
backend: restic
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
initContainers:
|
||||
- name: pgdump
|
||||
image: postgres:16-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
echo "=== forgejo pg_dump started at $(date -u +%FT%TZ) ==="
|
||||
mkdir -p /dump
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
|
||||
-h forgejo-postgres.forgejo.svc.cluster.local \
|
||||
-U "${POSTGRES_USER}" \
|
||||
-d "${POSTGRES_DB}" \
|
||||
-F c \
|
||||
-f /dump/forgejo-db.dump
|
||||
echo "pg_dump size: $(wc -c < /dump/forgejo-db.dump) bytes"
|
||||
# Manifest lets a restic-restore consumer identify the
|
||||
# dump without touching the binary.
|
||||
{
|
||||
echo "backup_timestamp=$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
echo "type=postgresql_custom_dump"
|
||||
echo "database=${POSTGRES_DB}"
|
||||
echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U ${POSTGRES_USER} -d ${POSTGRES_DB} -F c forgejo-db.dump"
|
||||
} > /dump/manifest.txt
|
||||
echo "=== forgejo pg_dump finished at $(date -u +%FT%TZ) ==="
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_USER
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: POSTGRES_DB
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_DB
|
||||
volumeMounts:
|
||||
- name: dump
|
||||
mountPath: /dump
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 512Mi
|
||||
containers:
|
||||
- name: restic
|
||||
image: harbor.basicstack.de/library/restic:0.19.1
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-s3-backup
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-s3-backup
|
||||
key: secret-key
|
||||
- name: RESTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-s3-backup
|
||||
key: restic-password
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-s3-backup
|
||||
key: endpoint
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-s3-backup
|
||||
key: bucket
|
||||
- name: RESTIC_REPOSITORY
|
||||
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/forgejo"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
echo "=== backup-forgejo-restic started at $(date -u +%FT%TZ) ==="
|
||||
echo "Repository: ${RESTIC_REPOSITORY}"
|
||||
- name: forgejo-backup
|
||||
image: postgres:16-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -lc
|
||||
- |
|
||||
set -euo pipefail
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
BACKUP_DIR="/data/forgejo-${ts}"
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
if restic snapshots >/dev/null 2>&1; then
|
||||
echo "Repo exists, skipping init."
|
||||
else
|
||||
echo "Repo missing, initialising..."
|
||||
restic init
|
||||
fi
|
||||
echo "[${ts}] Starting Forgejo backup..."
|
||||
|
||||
echo "--- restic backup /source ---"
|
||||
restic backup /source \
|
||||
--tag forgejo \
|
||||
--host k3s
|
||||
# Backup PostgreSQL database
|
||||
echo "[$(date -u +%H:%M:%S)] Dumping PostgreSQL database..."
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
|
||||
-h forgejo-postgres.forgejo.svc.cluster.local \
|
||||
-U "${POSTGRES_USER}" \
|
||||
-d "${POSTGRES_DB}" \
|
||||
-F c \
|
||||
-f "${BACKUP_DIR}/forgejo-db.dump"
|
||||
|
||||
echo "--- restic forget/prune ---"
|
||||
restic forget --tag forgejo \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 6 \
|
||||
--prune
|
||||
echo "[$(date -u +%H:%M:%S)] Database backup complete: $(ls -lh ${BACKUP_DIR}/forgejo-db.dump | awk '{print $5}')"
|
||||
|
||||
echo "--- restic check --read-data-subset=5% ---"
|
||||
CHECK_STATUS=0
|
||||
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
||||
echo "restic check exit: ${CHECK_STATUS}"
|
||||
# Create backup manifest
|
||||
echo "backup_timestamp=${ts}" > "${BACKUP_DIR}/manifest.txt"
|
||||
echo "type=postgresql_custom_dump" >> "${BACKUP_DIR}/manifest.txt"
|
||||
echo "database=forgejo" >> "${BACKUP_DIR}/manifest.txt"
|
||||
echo "restore_cmd=PGPASSWORD=<pass> pg_restore -h <host> -U forgejo -d forgejo -F c forgejo-db.dump" >> "${BACKUP_DIR}/manifest.txt"
|
||||
|
||||
echo "--- restic stats (repo size) ---"
|
||||
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
|
||||
| grep -oE '"total_size":[0-9]+' \
|
||||
| head -1 \
|
||||
| cut -d: -f2)
|
||||
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
|
||||
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
|
||||
# Cleanup old backups (keep 14 days)
|
||||
find /data -maxdepth 1 -type d -name "forgejo-*" -mtime +14 -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
# Textfile-collector metrics, same wiring as
|
||||
# loki/grafana siblings (DEV-494). Atomic write.
|
||||
{
|
||||
echo "backup_forgejo_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
|
||||
echo "backup_forgejo_timestamp_seconds $(date +%s)"
|
||||
echo "backup_forgejo_check_status ${CHECK_STATUS}"
|
||||
echo "restic_repo_size_bytes{repo=\"forgejo\"} ${REPO_SIZE_BYTES}"
|
||||
} > /metrics/backup_forgejo.prom.tmp
|
||||
mv /metrics/backup_forgejo.prom.tmp /metrics/backup_forgejo.prom
|
||||
echo "[$(date -u +%H:%M:%S)] Backup complete. Files:"
|
||||
ls -lh "${BACKUP_DIR}/"
|
||||
|
||||
echo "=== backup-forgejo-restic finished at $(date -u +%FT%TZ) ==="
|
||||
exit ${CHECK_STATUS}
|
||||
volumeMounts:
|
||||
- name: dump
|
||||
mountPath: /source
|
||||
readOnly: true
|
||||
- name: metrics
|
||||
mountPath: /metrics
|
||||
- name: cache
|
||||
mountPath: /root/.cache/restic
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 1Gi
|
||||
echo "[$(date -u +%H:%M:%S)] All backups in storage:"
|
||||
ls -1 /data | grep "forgejo-"
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_USER
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: POSTGRES_DB
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-postgres-secret
|
||||
key: POSTGRES_DB
|
||||
volumeMounts:
|
||||
- name: backup-data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: dump
|
||||
emptyDir:
|
||||
sizeLimit: 5Gi
|
||||
- name: metrics
|
||||
hostPath:
|
||||
path: /var/lib/node_exporter/textfile_collector
|
||||
type: DirectoryOrCreate
|
||||
- name: cache
|
||||
emptyDir: {}
|
||||
- name: backup-data
|
||||
persistentVolumeClaim:
|
||||
claimName: platform-backup-data
|
||||
|
|
|
|||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: forgejo-s3-backup
|
||||
namespace: forgejo
|
||||
spec:
|
||||
encryptedData:
|
||||
access-key: AgAu/XawJy0uhi3l8vwtd+loShP2ITnbOEZ/9eu6dkypz7l1MmFJhnEGq8Sy1lsg7BKj6epYRI40kDJIAG6KMBD5i7FvLYCvbFuPJaS3yehA8b1ECreHXvx/VYRUZL4K3WtiPO0fXmUNkxAqIcfHOcHtOZ7YcgSSyTw4DaXzngtek4xcTQlisyIVeBgc1dLPY+vHGn10BGkU5R/LnsUQqgGdp6u24Y+m7OrQGqMH8xwtZR7siEEkasu4IWG0QcY+okcMeiCi8mgAF64rdeZnIeEWqRnKHXxwEurtm20NmmCeRR8OgZfWDesgasThsnNSlRySd4EKlP3DMz8Yyluz/LlSkPt24brnP4qKuEoRpCHHOWgOizX8hkkS+Al7Rnpmxv2pt5CGlRw9s/5uPY81PAN9WLIgjs+XIylvaJBSW06nw3v+MjCUr5jfimW9riTLN32cBSurdpNMzlgb5+DJzEvaGNEG+jSjjFj2eSLci23UWySh7oFdmXeBp3kDWCrrNlJOYqJEwAAunTwvtM/1XknW2qJyyJrD7hbWN7SX4Ymb5Q5q18fMGH+IzB/Si1jdBli8pi8B2Gd/jo6RSmhXUheO3xsB2sXYc09GuSBBBBNzr9MsRL3mvUVr22pay+YPcCbwbwtdR0qmrdhEHAychYSJpNGuwGvs0AthRsNGN6hr1/IJqnnSRkGdJPr6k+T3v7XJzouYy+8Dh8r6oPW3hOhHo4HpJw==
|
||||
bucket: AgAyQVCK+wE0wWxwVjx5DVnI2falTGNzsUrolv1YyoG2Bx6msYe2G00ovxCMJ6/dWws1Q/Sf6yoiFT0IByYflQFhY/ABzXRi7/gFxCWyYU/7yTo7w1Ue0+eEnBKFtkP/ZhIvtfubo6GDDtbmshpp4UtzPq7kB6qvE6oCFyf6awQD2pe/GKX8RZaDx5TCbxPxhedLkYdc7srR/8rFZBnQiXm/2ohtDOwvLA+RVOpnCW/bsVLvLYQmXyhszMv2M3zqXwM9ODYzQ4Y7lIw6T3Hsmel/1hHWJtN8/mlq+Q3Z/NbzhVeEWs8/xbK/ShC9RwZUgBWxAiW04HvwcxVYs60KaJ4oOMLtO4Iko4+iPTNonjra2wmziEpUtsotvtReRjwHdvk1bBn3q6uqzNaJFv9wLc/RVCvKSpqE5KOxmdbke9EDUs4wQHVQCAfFfwnzzpcF/Rf6ML3TUWc+U0W6UjCfVTWwROznpi0E18tA07apx+xm1y+MyuVYnSpZv++vXetnCjA+KFvqZSon/WnKQrmSO2RFR07FcMs6gQ+EOSaRXBPCnEcnF2IJUbJLdFsgpQbzYG6zo3m2KmxN7gaGrTUdhNg/N7LklnNbJML9KRa7e2Rv8t4NGoigOpZG6+soIijZOSfEJF6owN6MP3oecgUgNsnQ5ftKK/RznNaDtIeZ5KByHlQ7CsZ0MIkHBmoEkDt64xWNEoW4NrWPSxas6pOQGZMPMA==
|
||||
endpoint: AgBhXqp3TdUU6GxsexRIoeHNwjvxnd2v4Any9WGs7gtyZvfdGIuWHIQ+JpsuRDrEq+BvTaX49D0QYoqnoQ6ZgmmGUI7kJxc+Ombcqo/8PAsOmzfdUaz1NcLAeR/UTGu8BDxAQcySSFax2dyK/zkY3jcsmuUCJNJLB57EZwtwx4fyz+ota+zNBE7pY+sDpWkXKHHtWrvNkD7tSLYHQuS3uzATidRixngL+iaH8rQX3ZK6KaK4H4ZdFG3/1Fw0wjLt8+yVBntBnh/bCL9/JLWQWIiHtooQnKXDdEwYQHWhrktti1KDzy/S++o4XVcRG1alzsQ0vV7uzSZAh4xCmnzximJxeJKawzFXRgFUcoEPrOI/nVedYrAU0DzkuwPdqtQDi/fGoqzZPx3PYhB9wv2PJsLg20ZlFcfHYUCad3O6OmO+VlrpYYcUe4oStkOygrmK5bv89fQ4Tov7BB14+RQIo9ogKmLzZu3xStqIVK+/Y8OUC64zyo7FpPMoYKtRW5oawu8pxo7Gtb+27BMMXQBiMEPNf38C4aj3GGJOvm4CFxVHV510/7o2hSW2ITt5X2RzNOEzULJOHD3DvO8wdWzvpBykRFnZ8JTYv77Kbx7bd9Ylx62VXm2a+Bd0K1Nc/sNJE63pglVhdl05oLYsUTrcSsNoJ6Dz3pMKeue3pYrn9KNurG0s+BFfO6I2Aa4/L8iAggXBzf9G1ALSkn7A30C9rMeyV+/1n+q8GpdPqq4ltqLOHVwIZw==
|
||||
restic-password: AgB1JGFnSDe3N5ZaiT9xShfzUQiyDtGwH2wotYO5Ekk9nqAbS4xTNBXu/1Z1+cB8zVUP80hrPHHbVXjNu47AcLI+Bn8Ozn9fKIZFsnVHXqicG0Y+WHRMpQxuMijVEMl48oUNE+vw+xRhYTzibVWoxur+VEzyPCn8rLUX07tcNhY+jYptNVBYlVVXDrHcOs0s9wpvIlSWmc9RsINMu+6FaedXAs3nKY4gUpWxmNLEpLG8V7D2mNJZHk1adghTN9p8RMGcLtNylDY9W1Sdnziga/NG0WulZf+4IWV6e7wc2oBCSekTofCZuCva8HazJ1EIeqBvt2GuXoG1CjjbPNb2UwZeH21Iol1W0C5So1VFguxHZq5WL44mqD79NLlnSAMmQMSvxf7gXQVXEdtT5UlF9nYCkuQ+K+mT+8iRMy1YSCQSQ9kkD36unkrlYcLXKoPLYxHcoo7fGZ0MPIW8J+P77Hp9Z8jNPCyCaeSJYApCxybO8ujbUGdIr2FCP1z/JDP4/zZat9MkTA6G3H9kDNso7AJBHL/NfkWImvQnveLGj0CuhPpf2zwOylCqTeXrgoN8zztO2/qVDa3VRQbznap0njr0qzVrvOfTxYoiRmkGh3PXZ2mb/NqbeIvCFhj2VyVgavdMchlIbM3fCzXGEdCa2RN+ZYVtcFe/b6wR/qf7lrMaPHsTZiMfYuMJoC9F9wdbkOttT9c36Sob4x+aQYLs7/+XGCJdbw672qot6x6P7GHjY8zVKlMYLSZjKcoD48c=
|
||||
secret-key: AgAHz7lfqTLG0Kldt/IZGW/o8KKazICHGbdVklXs3Gzs6Mc2KaG3nggkZDRWwY6noochg4Wc9mDDMGY3XIQnEy3PXs/rjQgwym0Dbvc54CvjWUcfmxCJyIqY1uCUxoXp/WAEV18lpWFSkVx+5U9wSccYVmkOJXNJ/MgH0vzJPaNW6xpYDg1XRC4nj72vwqyvZ/gM/LEX41NOpPjBROLGql3uBk5UfzNZG4dBun0aaPiYx5KKAbz8JAbK0umvOOOctTPSv1hIhyv9xJRRB67MW/J8T6mB2YfezsisXSSOAVNuE2JWrafHFxKq/hSNQHSrQRKjt18k6vfxmfkICbgQUZl/7AnoKi/f+76Feu+3wuMT+XPsaJN+iWb3jNzcb1T+Aiv4rGyIA7uCB6FNOc9Ngf5ngDQJw0nDNHDSnUoowgcE96kuPefU1rmn2V+JJcwoYdzYE3Ds48C0rNT+WSIit4551SiyWuj7iCRLhyEf5vhojxkuoyCN+EC1a50XwIkt223EBmy6FFPeBxql6HyJbOE313BPsQU6enPrTKNFd7iZYqJXq1k+SRuGwJhMeo1BBNhmVqxFGOpyxkSVs3VPybeXd82Zk/xF0SWKyBp3XQnLB39ZmT83Z5jwrYfkZmWkkdRvCME6NmlQtftRDDk3efGbgw7bz4Eq0d08KPCezsaDSAGNewonYy3UOiv/CNQg5hQ5G6R55y7+kEMV4nNkjGj0qF1hrZURS0mV4I/7H0OwXpDYiVetfaYI
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: forgejo-s3-backup
|
||||
namespace: forgejo
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
# Grist Deployment
|
||||
|
||||
This directory contains the Kubernetes manifests for the Grist deployment at https://grist.basicstack.de
|
||||
|
||||
Grist is an open-source, self-hostable spreadsheet / database hybrid ([grist-core](https://github.com/gristlabs/grist-core)).
|
||||
|
||||
## Components
|
||||
|
||||
- **grist-deployment.yaml**: Main Grist application deployment
|
||||
- Image: `gristlabs/grist:1.7.7`
|
||||
- Port: 8484
|
||||
- OIDC login via Pocket ID (`https://auth.basicstack.de`)
|
||||
- Data persisted at `/persist`
|
||||
|
||||
- **grist-service.yaml**: ClusterIP service exposing port 80 -> 8484
|
||||
|
||||
- **grist-data-pvc.yaml**: PVC for Grist app data (`/persist`), 20 GiB on `hcloud-volumes-encrypted`
|
||||
|
||||
- **grist-postgres-deployment.yaml** / **grist-postgres-service.yaml**: In-namespace PostgreSQL 16 for the Grist home database (TypeORM connection)
|
||||
|
||||
- **grist-db-pvc.yaml**: PVC for PostgreSQL data, 20 GiB on `hcloud-volumes-encrypted`
|
||||
|
||||
- **grist-ingress.yaml**: Traefik ingress with TLS
|
||||
- Host: `grist.basicstack.de`
|
||||
- TLS via cert-manager (`letsencrypt-prod`, DNS-01 via Hetzner webhook)
|
||||
|
||||
- **grist-db-credentials-sealed.yaml**: SealedSecret for PostgreSQL user password
|
||||
- **grist-app-secrets-sealed.yaml**: SealedSecret with `GRIST_SESSION_SECRET`
|
||||
- **grist-oidc-secret-sealed.yaml**: SealedSecret with Pocket ID OIDC client credentials
|
||||
|
||||
## OIDC Configuration (Pocket ID)
|
||||
|
||||
- Pocket ID group: `grist`
|
||||
- Client name in Pocket ID: `Grist`
|
||||
- Callback URL: `https://grist.basicstack.de/oauth2/callback`
|
||||
- Client is restricted to members of the `grist` group
|
||||
|
||||
Grist environment variables driving OIDC (from https://pocket-id.org/docs/client-examples/grist):
|
||||
|
||||
```
|
||||
GRIST_OIDC_IDP_ISSUER=https://auth.basicstack.de
|
||||
GRIST_OIDC_IDP_CLIENT_ID=<from secret>
|
||||
GRIST_OIDC_IDP_CLIENT_SECRET=<from secret>
|
||||
GRIST_OIDC_SP_HOST=https://grist.basicstack.de
|
||||
GRIST_OIDC_IDP_SCOPES="openid email profile"
|
||||
GRIST_OIDC_IDP_END_SESSION_ENDPOINT=https://auth.basicstack.de/api/oidc/end-session
|
||||
GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED=true
|
||||
GRIST_DEFAULT_EMAIL=andreas.leinen@basicstack.de
|
||||
GRIST_FORCE_LOGIN=true
|
||||
```
|
||||
|
||||
`GRIST_DEFAULT_EMAIL` is used by Grist as the primary admin identity - the value must be a valid Pocket ID user.
|
||||
|
||||
## Storage
|
||||
|
||||
Both PVCs (`grist-data`, `grist-db`) use `hcloud-volumes-encrypted` (Retain reclaim policy) as required for
|
||||
sensitive workload data on Hetzner.
|
||||
|
||||
## GitOps
|
||||
|
||||
Managed by Argo CD via `apps/app-grist.yaml`:
|
||||
|
||||
```yaml
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: grist
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@forgejo.forgejo.svc.cluster.local:basicstack/stack.basicstack.de.git
|
||||
targetRevision: main
|
||||
path: apps/grist
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: grist
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
```
|
||||
kubectl -n grist get pods
|
||||
kubectl -n grist get pvc
|
||||
kubectl -n grist get ingress
|
||||
curl -I https://grist.basicstack.de
|
||||
```
|
||||
|
||||
## Backup
|
||||
|
||||
The PostgreSQL PVC uses reclaim policy `Retain`. For scheduled logical backups add a `CronJob` like the one in `apps/directus/directus-backup-cronjob.yaml`.
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-app-secrets
|
||||
namespace: grist
|
||||
spec:
|
||||
encryptedData:
|
||||
session-secret: AgBN2KDg6QQgvhD2WvTG/r4+y6FrY7EXc8kU8N4hUDIh/VJDoyK+xiEHFvuOXMH3ZpNw4qTEpL+a+6LVWM1/c6mdcTIW5J52+hOOoXEtSXJhoBCjljgddlTTxHy9sIdE0694dMgeBJwmnvT9vm1TNQWMbiBaD05Xh4VCX3D908L4l/CiM11M9CLkoMGK7cTz9OCuGtd36k4OUxmw3/mQnafCDyzKf07NRW70Fr/bMGCdKE4Jt1rqWBfIaTb2mq96FCHosFq37keTCh1gu2nKf3iWesfv40rsVNN3wEhATaG2ghmjP2ul21zMOIO5GuCnMcawZv8VR1gyb5f98WOinL1zf5NMPCPRdTl/3G2gpqPc9VRNxOGguup3NLCaV2J+vbH5dgEj/h6zeTXw5bRARRQHAaGhUgFS3PALzubTzaUyNyFOyaMoXbAnE79HU+7QEd3Yjpdujup4Pe6OioK2k7jGWJj/jxZokffCjUsG9+hvNlg2E4ETtXxmHXIkwgTx7nHON6UApnprCtAUatRuMT3H+4VzfgkjV6y2aIHcp21EbH+qDBLo83+sBeUeWYyRtA/W2BGoAhx1CXeBysmEjD/Cls2h2PVhIvUdfNBY9fDgLemuK7MxdKbR8hRb3zHeedLXM4wevg2zCTyyGtizeC3JmpnqO1UHsKBZMGyUh43r+7sh3SyBKVUJxZt0q2JOWibaZNbIsrqcAyb8Te/QZlayZ2s2BxvMvvNrNpcOTYs/KzwwaHva8OUl
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-app-secrets
|
||||
namespace: grist
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: grist-data
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-db-credentials
|
||||
namespace: grist
|
||||
spec:
|
||||
encryptedData:
|
||||
postgres-password: AgA7MFNJ8//EOg4CdfHlSe2yGW6zcJyGsoiWVYlbVkb2GgMZmGQDH/ImYOTfeZXkEQhAWhA0braRB7Prx8fwpON2+6PsOczg+S1N1ykuDMJ+KDuXHhJtm5LNqbgibddX9y6YYInorgxM5mh/ORt2D1IdTwf1AXACkEkp8Ws4IGIfuUCym7kNM77mVD9jW4xyf/c7RE/J3va7L/6SRdx2rdoil8loMzMTHH+lvc738UdFA67hoLkKuseiYgA6NZap5Ppb48GTQul47ji3wH24F/u8DTKiHWbMBxwWO48gQJvAISgXSCW4eiKhkAcm28JACCOd1jT7Asen/yd5qEISB8Fr8q2qwxQwM3kXFIJRf9VVN7BWv6hEMnDO3AhBmcRr+gCPhKial17Varakh0ruBXB4eQDlxGQZ09trwtRPZkOsQnNf38iMQBpDH5U12tASmi1t5ddGKzQQh3EWx1vJyap8xNFf5zK6MuoDEMekHrjcclqgoMWppMH26Daghp2XZl740bV0UhLQBO1TOFEORHBs2DA2A3msmN25selVxjFBHvwLZHQJOHoVHbcqlbF7e2bFSLnij1TBMlPre/llBzoiVYrjHQoHw3fg7xVMrPBT+U1wn4ItDFZTP+nCxnWejtQ/dxzBfMrvBcHVhACsw7VhVkM5nhNFJBda65czyud0BWCSMYJYZEBZjDs220/mhQWIh2/RLANN6pcwJ2ONGyNJs3uo7Q245DlmDTjH5Qppmg==
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-db-credentials
|
||||
namespace: grist
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: grist-db
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: database
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: grist
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: app
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: grist
|
||||
component: app
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: grist
|
||||
component: app
|
||||
spec:
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:16-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h grist-postgres -U grist; do
|
||||
echo "Waiting for PostgreSQL..."
|
||||
sleep 2
|
||||
done
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: grist
|
||||
image: gristlabs/grist:1.7.19
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8484
|
||||
env:
|
||||
- name: APP_HOME_URL
|
||||
value: https://grist.basicstack.de
|
||||
- name: GRIST_DOMAIN
|
||||
value: grist.basicstack.de
|
||||
- name: GRIST_SINGLE_ORG
|
||||
value: docs
|
||||
- name: GRIST_ORG_IN_PATH
|
||||
value: "false"
|
||||
- name: GRIST_DEFAULT_PRODUCT
|
||||
value: Free
|
||||
- name: GRIST_HIDE_UI_ELEMENTS
|
||||
value: billing
|
||||
- name: PORT
|
||||
value: "8484"
|
||||
- name: TYPEORM_TYPE
|
||||
value: postgres
|
||||
- name: TYPEORM_HOST
|
||||
value: grist-postgres
|
||||
- name: TYPEORM_PORT
|
||||
value: "5432"
|
||||
- name: TYPEORM_DATABASE
|
||||
value: grist
|
||||
- name: TYPEORM_USERNAME
|
||||
value: grist
|
||||
- name: TYPEORM_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grist-db-credentials
|
||||
key: postgres-password
|
||||
- name: GRIST_SESSION_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grist-app-secrets
|
||||
key: session-secret
|
||||
- name: GRIST_OIDC_IDP_ISSUER
|
||||
value: https://auth.basicstack.de
|
||||
- name: GRIST_OIDC_IDP_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grist-oidc-secret
|
||||
key: client-id
|
||||
- name: GRIST_OIDC_IDP_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grist-oidc-secret
|
||||
key: client-secret
|
||||
- name: GRIST_OIDC_SP_HOST
|
||||
value: https://grist.basicstack.de
|
||||
- name: GRIST_OIDC_IDP_SCOPES
|
||||
value: openid email profile
|
||||
- name: GRIST_OIDC_IDP_END_SESSION_ENDPOINT
|
||||
value: https://auth.basicstack.de/api/oidc/end-session
|
||||
- name: GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED
|
||||
value: "true"
|
||||
- name: GRIST_DEFAULT_EMAIL
|
||||
value: andreas.leinen@basicstack.de
|
||||
- name: GRIST_FORCE_LOGIN
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 8484
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 8484
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /persist
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: grist-data
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: grist
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- grist.basicstack.de
|
||||
secretName: grist-tls
|
||||
rules:
|
||||
- host: grist.basicstack.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: grist
|
||||
port:
|
||||
number: 80
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-oidc-secret
|
||||
namespace: grist
|
||||
spec:
|
||||
encryptedData:
|
||||
client-id: AgAAQEIKFFtfc3pm1yp3HKu7uIiTsYtcer6CYJbrUAY6bwNBE+Cj2NF3wvDVeW1M2padhXqyuuhuOIaYbbQ8NOgR3kVg6UIKgn5oNimpXgkwef4xCuGNbNN7kfRfVTJfwrJm8jON0nzmmvxnmsNsz7hJgRszQ2Tp1qJlCI/KIPsIZ0vcXehy+VWV5xBOQOnP+b/wxR+tbt9BnDX/acRi95JAJLXYabqkD7FDc0rLGiWmEsyl4prhhRcT2Ux7O6CJuyegAOOodPDsylkV2iyaI6L3EEY9IrrV5w4ta2tDDPEcnuLkUo/1LipfJUSGzO/tqYJMAbd6klTtfI3cIjimmQXUP/0uiy4YXT7LJud/yEwjSajRIMAdIMQNhAcXPM3EnW85nck4NYSO8qUgu04+lpPiQKiE1eSjCa/LWBIqOMG69nE/hZCCceNBa2krvJIhjrPewBEMxBvg4aCUUS9GO3YanoDN3Y5T9HcpjvbhtLvLMGu20uAKxw63m8DTyNnzruYkL4Qr/5KxyhquCGv9YuwJMtaiUlWc36mh6YKEyyiYSe56cMegUnIZKBWWjsJwJm+ApPKZSI3u9/AGiHodF9tKmqt8vCzuqyzhsAskQXpewdn3iHe2IY2KwuWPPoObUaaSYWcKdoEZYVgaY9IofXf/x4GbJ5GVqbrwCGFpXJGk3GFFtvlyFICINU+CCp5PXgoMiqvpNvUVEHVclCmcTMi8lWufWQi/6Xu4gH+08QHVMkzf55o=
|
||||
client-secret: AgBrF2e/M48zFC0PdXm08IavK+qa3Sl3a0qmO0+eH/z5WYnRpgos5dFdCLxcrdDcJUgKy//pU+I23qoTXqgybxRnfVzsOdYb4UksGgWh9640BQ/NRuxqhdnfXjQN7PrJj+D+fumwRdTQU1PqyzEePM/6CVf6Ro9vdP6iTMDzbAnPnHoA8L8Hd+f3YKo43OKMR2GMwjNKAhpD20wMCYt1vTiEQX4drR9JW4mjcKfSCdaIrEFwIYn4EBkr1SwuC4Ek/KTn4CNQO82BQH+jjWO5CHRvLP6b4v2bVkgdQEqLR4+HpNjMm0JBKOjHvhfKv952T5mwROamWcpNlN4KW02YzIZMBU+dh7WhouZkDkQCalP4G7AGoLrjhWmLIgS56JqTuoiobFf90Uvz6ewcDmHAzPuYZMfn2mKndPYvoi/KSGzM3ceeNrFxpHCPhsG+UHbOfSCBmg7jqIeE9ial6fMsRRLeEqc3ay4lAN7PBXwJ/o3XMrezUy8RQsXLex9oS0IS9rktimwQNWEpY1adwWRERdBKqGAIEbVofVwai/SuC/dcgQsoX2r9kJOiA8ahM+eDxUg1534hb21v3tFWykp3y/EbFBxbPPtCsUY4ghZZ8O/lLtXsZdHxaq68RKbVqDbRGa/eLqCqStWS8Bk1XzwkJFBBHFcN9hDvNwoT+33QSSsuuiw8gLiztlSrewo33rVKEc9Enqb+f/uUX6HY9j3Iz7hvKZ2aYX/U8ayNN7hQaJryYQ==
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: grist-oidc-secret
|
||||
namespace: grist
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: grist-postgres
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: database
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: grist
|
||||
component: database
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: grist
|
||||
component: database
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16-alpine
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
env:
|
||||
- name: POSTGRES_DB
|
||||
value: grist
|
||||
- name: POSTGRES_USER
|
||||
value: grist
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: grist-db-credentials
|
||||
key: postgres-password
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pg_isready
|
||||
- -U
|
||||
- grist
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pg_isready
|
||||
- -U
|
||||
- grist
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: grist-db
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: grist-postgres
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: database
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: grist
|
||||
component: database
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: grist
|
||||
namespace: grist
|
||||
labels:
|
||||
app: grist
|
||||
component: app
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: grist
|
||||
component: app
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8484
|
||||
|
|
@ -95,9 +95,6 @@ harbor:
|
|||
cpu: 500m
|
||||
|
||||
registry:
|
||||
# Skip Kopia fs-backup of the registry blob PVC (DEV-573 safety net).
|
||||
podAnnotations:
|
||||
backup.velero.io/backup-volumes-excludes: registry-data
|
||||
registry:
|
||||
resources:
|
||||
requests:
|
||||
|
|
@ -109,9 +106,6 @@ harbor:
|
|||
|
||||
trivy:
|
||||
enabled: true
|
||||
# Skip Kopia fs-backup of the trivy scan cache PVC (DEV-573 safety net).
|
||||
podAnnotations:
|
||||
backup.velero.io/backup-volumes-excludes: data
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ spec:
|
|||
serviceAccountName: headlamp-admin
|
||||
containers:
|
||||
- name: headlamp
|
||||
image: ghcr.io/headlamp-k8s/headlamp:v0.45.0
|
||||
image: ghcr.io/headlamp-k8s/headlamp:v0.43.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- "-in-cluster"
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
# Kaneo
|
||||
|
||||
Self-hosted [Kaneo](https://github.com/usekaneo/kaneo) — open-source project
|
||||
management — deployed on the BasicStack Kubernetes cluster.
|
||||
|
||||
- **URL:** https://kaneo.basicstack.de
|
||||
- **Namespace:** `kaneo`
|
||||
- **Distribution:** hand-rolled manifests, GitOps-managed by Argo CD
|
||||
(see `apps/app-kaneo.yaml`).
|
||||
- **Auth:** OIDC via Pocket ID (`https://auth.basicstack.de`) with
|
||||
`CUSTOM_OAUTH_*` env vars. Client id + secret live in
|
||||
`kaneo-oidc-secret` (SealedSecret). Access is restricted to
|
||||
Pocket ID groups `kaneo-admins` / `kaneo-members`.
|
||||
- **Database:** dedicated PostgreSQL in-namespace on
|
||||
`hcloud-volumes-encrypted` (20Gi). Both the Postgres and Kaneo
|
||||
pods pin to the `fsn1` datacenter via
|
||||
`csi.hetzner.cloud/location=fsn1` node affinity — the Hetzner
|
||||
volume is DC-local.
|
||||
- **TLS:** Traefik ingress, cert issued by `letsencrypt-prod`
|
||||
(DNS-01 via the Hetzner cert-manager webhook).
|
||||
- **Image:** `ghcr.io/usekaneo/kaneo:2.22.0` (single container
|
||||
bundling API + web on port 5173, health at `/api/health`).
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `kaneo-postgres-pvc.yaml` | 20 Gi encrypted Hetzner volume for Postgres data. |
|
||||
| `kaneo-postgres-deployment.yaml` | Postgres 16-alpine, pinned to `fsn1`. |
|
||||
| `kaneo-postgres-service.yaml` | ClusterIP service `kaneo-postgres:5432`. |
|
||||
| `kaneo-db-credentials-sealed.yaml` | Postgres user password (SealedSecret). |
|
||||
| `kaneo-app-secrets-sealed.yaml` | `AUTH_SECRET` (Better Auth JWT secret, ≥32 chars). |
|
||||
| `kaneo-oidc-secret-sealed.yaml` | OIDC client id + secret from Pocket ID. |
|
||||
| `kaneo-smtp-sealed.yaml` | SMTP submission credentials for `kaneo@basicstack.de` (SealedSecret). |
|
||||
| `kaneo-deployment.yaml` | Kaneo app deployment (pinned to `fsn1`). |
|
||||
| `kaneo-service.yaml` | ClusterIP service on port 80 → container 5173. |
|
||||
| `kaneo-ingress.yaml` | Traefik ingress + `letsencrypt-prod` cert. |
|
||||
|
||||
## Post-deploy setup
|
||||
|
||||
1. Point browser at https://kaneo.basicstack.de and sign in via the
|
||||
"Sign in with Pocket ID" (custom OAuth) button.
|
||||
2. The first admin (`andreas.leinen@basicstack.de`, member of
|
||||
`kaneo-admins`) must be promoted inside Kaneo — the current
|
||||
upstream release does not map OIDC groups to roles.
|
||||
|
||||
## SMTP
|
||||
|
||||
Outbound SMTP is wired to the Stalwart submission relay at
|
||||
`mail.basicstack.de:587` (STARTTLS) using the `kaneo@basicstack.de`
|
||||
mailbox. Credentials live in the `kaneo-smtp` SealedSecret
|
||||
(`kaneo-smtp-sealed.yaml`) with keys `smtp-user` and `smtp-password`.
|
||||
|
||||
SMTP is used only for workspace invites and notifications;
|
||||
`DISABLE_EMAIL_OTP_SIGN_IN=true` remains set so users still sign in via
|
||||
Pocket ID SSO, not email codes.
|
||||
|
||||
To rotate the mailbox password: update it in Stalwart, re-seal
|
||||
`kaneo-smtp` with the new value, commit, and let Argo apply. The
|
||||
existing pod picks up the new secret on next restart
|
||||
(`kubectl -n kaneo rollout restart deploy/kaneo`).
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-app-secrets
|
||||
namespace: kaneo
|
||||
spec:
|
||||
encryptedData:
|
||||
auth-secret: AgArCARyFt7NinaeQOfDDEpTKKWVba8IlKh7NN/vflgdOkE5bjz/P/ZzGUVhUOTyqCs+3dU+hX6sqc7IYbYckXdprwKJSXy47HLEsZ/cuJ0FRIBShRyXm+I62Yu8WWLHNJLTnlWyfirBC6Pg65kXckoKiYPwcVLSD5I+YHzo3Dt3Hl6MeW/iVocM9xgjBD/5yRt1ag+S4JYv0t1y39Cz80XxbFvo55RzYWr3OlYNI325b9W/d5uylqlDemln8NymL5iCdfnLDZoc+1r+eHAf1HDP6Uch1dG+c/xsrlvoWpzyMeSEbEY1tGeYFZ1oKm4B1pyUH9dfdxWEaNThheL5Xw2QX/klpZdTq5Jl6Lfxme5aLyqNtP9J6ovd5b1oVYW1DWc+RAqBPhFlHnkfc+8ZeP6vrmnUULq0ssSVqpXOz4zLgDHU61UzSllNdpZWXokigranj3dq5CM7WkYLEj9kfdDIGS3gDUD+EmIkIj/N96URNpOINSwFVAcnE73k3ij9mZWECLl++lzh1fqHrdYQiTI6A7UpxPudc7BoujBQFqGhUDXGdzBWsrqr11tldD+kb/kcKQYxs6sFHyw0pDgCLVZukPv7TWRNfVRYUv68h/1ZKAwmj8KkuGlNtAQ+NyvUWyct3SA+pQYWabr33pIPekWPIXMIty6/zfKu92hLFglwkauewxqLtEBZ6FFukKMyyfowV+iwDIMyY3//MvxiRzxD7sc5sLwcfvPCBzd9XGaJO1ZLX6r43gfOu6HFHiq4FqdjYJ9fqG3zUKeo5bnav7E6
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-app-secrets
|
||||
namespace: kaneo
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-db-credentials
|
||||
namespace: kaneo
|
||||
spec:
|
||||
encryptedData:
|
||||
database-url: AgDEiqPdexD58tc4Sv63mk1erIHAeweJzGpKgoOcro6ioW4eX9pWnfVnVfmtsqQK8WXkRuEP5pAnj1LZAhuo6hfwk/lOpoYItqZKl6KGK0EQZ3u/hcmUt0yeEkTJuDPGBbgk1RuT1G2u2uDDYYJYHmAQtIIFbAoeJS2iDKnKkl6UHZdEJhO10w6IHGiNCT6Z9NovLJau5cw9tTv7xnbGO8F1QqwLh/dL0kI2oYNTYg1RxpegXRFeKGz+QSJ3rbxlH2P/E01Fevma2/ZhOBjALS1bpW9R6pgQm4n/FDu/fu72T62yw2oNvj6nIeRtIsm9ze02jQqYwF7mgct6IJOdqY7RoYnTZJQ+/Xl/iiBIGpzXan9bU/JZw6sEeyxS+tyeaolz4oiTeNfRXp8QNdX3qbXM9OE5Q+bjqx1q+cZcnFcBIvGK8vGqlWwDjwNicucuNBZuv4JWEOp2N8axoFVRUsUn/Frv7McqKT1Q2mRXf0aiGYh9m7EIdnFAx3JeiBJv392jBWcwvJpQ+J0LIsh/bJEBtdC4c16Y+38nbqfjYEUJWVIoj9SvJEo3QZPitUz99cwdtDn+cOVVYp+lv5BiunM7OgvAgBbX7F4Ri/QlL8rt3/8lO0Augwp933lSTuGGnMCrXPz2QCagI6B0l8F5q5OTnljgR7JXslyGFaemtj9jZEnCiSsw9KAiDdOUyk36CJSfHmRedJJ6o2nalb3zdFDF7iMXxz6fpTQK0QQGAccO518zUHOpb3Jb6XbOBKDgTBOWPEGlroiavrOtoRDJ+B5nR/N4c3Indpn7d0TTDU/4rz4V1uNkRrHnA26ya9Q=
|
||||
postgres-password: AgBo5f/NzqZ/9jsOzXmoivPVVeLA3xk9GWocQQlLv34Rc1K5fzhSoCYdy8o7k06R83VeNsJylxfAsOI7HNIo0xWUPqC8HnbW482ajtREh93wOFCaZzjKu8ClbFIkvTb19GRCPLSmzBMFsTCJzjSa5rWGqK2Ezj7LFeTTiehsqS/PbPUYi6w4X6ZnChS3VuMXJVJAvbCF8fFS3x/YuuJVjjd+TmVsPmGOa6J+r/6RG7l4DsR+SmV3HQ7M6bLqTsUPjzg5Foz11yc8ER8xQQTgbbfrBPwgJQoTX622yHf0Q313ns7vyiKGMqge9DSQvDmGmMDmAxqtYInycVBH1CwmfKVbw3CyFGt0D5LJXiBaTE89uLErsNCQCK6NKAODqffKlOWpZ1KVF+ZIQ6nqJETtj49uqdKGowauCs3tDl/Hi82+hxycEVMWUwgu81CJ6GT7+vSZfz4+9FRbLoXlbdfR/duG14f+AkPirikeDcy3bFy5XiBgPVQXL0EbAY8TaPkYu+TS1Roo2F6Gqj3D1+i4GapVodrBntl1gMPvMnd3dcGGl/VzREGZvRsZgdRprFi+nGPmotRhIO3eGcPautlZ5m0WMqKCpA4jW0sG1sgkVwRpn2Zdq4Uw2+IKNHTkJKclIXCaw2/b6WNCmiwbe1ssFoR/QgikzshshoB5XpIYFe7Jr5Sj9mmNH6L4PpCIZ/sCe8MF5eMD8IuyWSVmflID07sfBD6GL3XB9TFLzSyviHE/VJdGY7l+ZeR64UZZ1ti4bwI=
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-db-credentials
|
||||
namespace: kaneo
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kaneo
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
component: app
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kaneo
|
||||
component: app
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: kaneo
|
||||
component: app
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:16-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h kaneo-postgres -U kaneo -d kaneo; do
|
||||
echo "Waiting for PostgreSQL..."
|
||||
sleep 2
|
||||
done
|
||||
containers:
|
||||
- name: kaneo
|
||||
image: ghcr.io/usekaneo/kaneo:2.22.0
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 5173
|
||||
env:
|
||||
# Public URL of this Kaneo instance.
|
||||
- name: KANEO_CLIENT_URL
|
||||
value: https://kaneo.basicstack.de
|
||||
- name: CORS_ORIGINS
|
||||
value: https://kaneo.basicstack.de
|
||||
# Better Auth JWT secret (>=32 chars).
|
||||
- name: AUTH_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-app-secrets
|
||||
key: auth-secret
|
||||
# Database credentials. DATABASE_URL is preferred; POSTGRES_* are
|
||||
# accepted as a fallback and useful for the built-in migrations.
|
||||
- name: POSTGRES_DB
|
||||
value: kaneo
|
||||
- name: POSTGRES_USER
|
||||
value: kaneo
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-db-credentials
|
||||
key: postgres-password
|
||||
- name: POSTGRES_HOST
|
||||
value: kaneo-postgres
|
||||
- name: POSTGRES_PORT
|
||||
value: "5432"
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-db-credentials
|
||||
key: database-url
|
||||
# Self-hosted: keep public registration off, log in via OIDC only.
|
||||
- name: DISABLE_REGISTRATION
|
||||
value: "true"
|
||||
- name: DISABLE_PASSWORD_REGISTRATION
|
||||
value: "true"
|
||||
- name: DISABLE_EMAIL_OTP_SIGN_IN
|
||||
value: "true"
|
||||
- name: KANEO_CLOUD
|
||||
value: "false"
|
||||
# Custom OAuth / OIDC via Pocket ID.
|
||||
- name: CUSTOM_OAUTH_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-oidc-secret
|
||||
key: client-id
|
||||
- name: CUSTOM_OAUTH_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-oidc-secret
|
||||
key: client-secret
|
||||
- name: CUSTOM_OAUTH_DISCOVERY_URL
|
||||
value: https://auth.basicstack.de/.well-known/openid-configuration
|
||||
- name: CUSTOM_OAUTH_AUTHORIZATION_URL
|
||||
value: https://auth.basicstack.de/authorize
|
||||
- name: CUSTOM_OAUTH_TOKEN_URL
|
||||
value: https://auth.basicstack.de/api/oidc/token
|
||||
- name: CUSTOM_OAUTH_USER_INFO_URL
|
||||
value: https://auth.basicstack.de/api/oidc/userinfo
|
||||
- name: CUSTOM_OAUTH_LOGOUT_URL
|
||||
value: https://auth.basicstack.de/api/oidc/end-session
|
||||
- name: CUSTOM_OAUTH_SCOPES
|
||||
value: openid,profile,email,groups
|
||||
- name: CUSTOM_AUTH_PKCE
|
||||
value: "true"
|
||||
# SMTP: outbound only, used for workspace invites / notifications.
|
||||
# Sign-in still requires OIDC (DISABLE_EMAIL_OTP_SIGN_IN=true above).
|
||||
- name: SMTP_HOST
|
||||
value: mail.basicstack.de
|
||||
- name: SMTP_PORT
|
||||
value: "587"
|
||||
- name: SMTP_SECURE
|
||||
value: "false"
|
||||
- name: SMTP_REQUIRE_TLS
|
||||
value: "true"
|
||||
- name: SMTP_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-smtp
|
||||
key: smtp-user
|
||||
- name: SMTP_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-smtp
|
||||
key: smtp-password
|
||||
- name: SMTP_FROM
|
||||
value: "Kaneo <kaneo@basicstack.de>"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: kaneo
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- kaneo.basicstack.de
|
||||
secretName: kaneo-tls
|
||||
rules:
|
||||
- host: kaneo.basicstack.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: kaneo
|
||||
port:
|
||||
number: 80
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-oidc-secret
|
||||
namespace: kaneo
|
||||
spec:
|
||||
encryptedData:
|
||||
client-id: AgBGmlu+mGYlHcWPCRHZ3IbevGB5kGntqPasFBnny6ZydTeCmN8NWpy0bmoorecJSbvdeP38UDgfEjtmusfkipnrlfsZd7BugZLiijq5+T+9al5JTl/UChMl6u/gReaVPgLi9iketDN6oB2XC/56mV+MdMb/bO4d1u3CRFUJ/Pq1znWeQpVsXO1yOLoOge4+PYDYjgd4O6mJsbAQcmGxQ7pwcByw//+50Mvuw0gJBqI5ZSGKd0jSNnaVcECtvjY6v3xN1NCeVF1FoWKl4s7o/Nrq5mDltW/y6g0h3N1Fzan/D47UVQjGLYu6cTmM7ReqToYk65cznttGkyXfqrlVL/oEiYENtaqp/BZvwPBMKEAjJsufwGvbNgsxa3eGs+p3L+HwzdgTQbVGu2wVp+TXW8zwZvkDyprUU91Fu6QFsspV/GRYgqVmp4RxrJ0PzZInp4ZhLt/zOyWzbEkILMPV1A5fzPjrPmwpmhln/K0VTdpRN62XQRgH0P7EP6Ao0sP2/ae39eTkGlhb5fwzhSqK3LSmBYmoXNcffA4nBLFohSndmwgoxzZXM1FJ1kXl4wuevU8Eh8BD3dVIZkuYacKiIUFoNn74NbPtJpl3/MWfXef9Eb7IBR8KyurbdwSmKSAZDqjmeGTa271+KUiGfST+FYQgSDfFfQQEmh+K49WqqbOiod4Mh5RTGWJx3WNwEONsTRKVjiGgLw4A+ENNDB39PoRChoRoRRogg+ZJZE27EjxbmrKIjrw=
|
||||
client-secret: AgBJQi6fIlTkOIZa1zr9jjyNOFAyZUjHWy0ZBjfcNh+iY2V3qIpG8kNPBntmvtVtemYI+UxEvvlalq0iEue+AtVmUoJeeO/lYuEDv+BrJGUWeZITbItuyoG+xGZM4x41X6FPgfRcnLCfii3dmwOeDZh1asR2Ahy2szdXrd2QaDeZMCCXNR+IeuP9U+IqJncfm+KjCqPABVbsZuGfDrT7r4XRv4mHQGnILD4IyA9N0/XyfY/cZRJryWeF5xbrOAP/ZWJduqYGIm2ahJfRZWIzk1Ujknq35iJ4bOZiw5eWb+aYhD9cxJj2UMBwWdqnXnpH+p55lrBqWlCzI0yDOfk4UQCI2RzQBJc/RvVS0z16k7El4dl9lOFTcgwBg13C/JC8Y0wo4M1jEap2ty/2ifwVZBZdflT8Uy5sk8dKBOv9wCupA/N60ACwlsavCUlgtQetVZUHIlI1U1rKCzArpGZ6PuREZokjdHw01u/j9ANEt1AaMe3G5jLcGohyvpcgL+zqI2S0rCqbSdRqWzZvr2kM8uFoeZSJLG3y4wdwZMgvRGMfoAKD1YlUGuhWeeZTda77ScM6ii8iEZEjvIgJXrfSCUf1vJ4QIAdMnNi05d3pTTDVHNTRjWBrklVb6oTpe/rFF1BBP6zhwGylWtb71xszbQyX8lzD+7v1O4iXr7ZWo86L9mGKz6FMnj0bO4LLVoqeVMy5DHo2fCuplix03P3WQCyjGKranAWdLalEV59QdnEhaQ==
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-oidc-secret
|
||||
namespace: kaneo
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kaneo-postgres
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
component: database
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kaneo
|
||||
component: database
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: kaneo
|
||||
component: database
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16-alpine
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
env:
|
||||
- name: POSTGRES_DB
|
||||
value: kaneo
|
||||
- name: POSTGRES_USER
|
||||
value: kaneo
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kaneo-db-credentials
|
||||
key: postgres-password
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pg_isready
|
||||
- -U
|
||||
- kaneo
|
||||
- -d
|
||||
- kaneo
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pg_isready
|
||||
- -U
|
||||
- kaneo
|
||||
- -d
|
||||
- kaneo
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: kaneo-postgres-data
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: kaneo-postgres-data
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
component: database
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kaneo-postgres
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
component: database
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: kaneo
|
||||
component: database
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kaneo
|
||||
namespace: kaneo
|
||||
labels:
|
||||
app: kaneo
|
||||
component: app
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: kaneo
|
||||
component: app
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 5173
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-smtp
|
||||
namespace: kaneo
|
||||
spec:
|
||||
encryptedData:
|
||||
smtp-password: AgByMIhrSeQbPJezoqPf4jtTCx/b/R+TGZPHjjvUcdZJimgZY8XO72BA2nI/419UWqY3rBmWyquYMnEFJT0/48Rn0Nk4aFxBwBIxVuiG9y1+f5yXIhT2JuOqGoBnImYezCCtYOcgn4Byo6Twldce1ElRCqwH6grsVOLKtogZvLJC0sz6EWbqPfLrDuGyOFz2wg5P3Cjmy8W9ZgcTDTpVI4JxZh2JKbIJpq4kGDQD0miW8Av0BOmSi75uvT2PvaV/uinJ72VJFLP/GTHWjf/9kpOiIC2TL4B+Kk0HtMeWSzNF79itsD53ow7bya/bq//zUiwdaIeSaAcSJL8XqL9q38LIEL0Vr9ccXN9xsPZttLw836NDeLsryE+RhF3pIK2vEs76YIeO03AEnXNjEYOcwy7ePGYh6Y/Uf/B75QNzbzrq8H4B5lKFVmQgTBLJnvxTzOc6Sd/Yssgl1wJw90jgw2HYcLHXt/eQrB4cGKWSwrh2x6fnU79UpSGatdTqIyP2Q8vbgKmxqqmg6NDzjgeatT6ADKOqJKAjnXfrr8Bm3NjvAVWOGWuIfWdS0wHiEOhAoUNNAQ+bD7f/SbJE24q8W6juhzTmRLuU1BM2K51YPHA3BqyEpzygaboSi/bgHVOuoIAEAt7jAM6mbW5RQBwiNglF/C2Oem8J3LOdDlndJoSMaWKm5p1lKc3M59jr47jaDXm9HKM4ypC58cqty3varqxBwwdZHirZ6No0Ar4Sd78I4jo4kXk9zu33UjCcR/X2NX1abIYZ8eNoUga+I9AOkpM=
|
||||
smtp-user: AgAq1mT9fzt9/VHmcbn0Yzr/8HWadEBe1XjJOTonao7Tdc3EO1PvCgEE9nBkrqOK5x2uDP1jZuP98XxJXr6mHuCnEOn8SHMN93TqK5rLhsKir1ImdNWKC+ruVI3TPQfTkeEz5m3u0DjHWSymva7BjmTIIW5gE4SDt/7WOFrsWbX1fLqQKfuB/k1qIBd+rFL7rVSe3XnRzxxwwrIbYfczSCknMbOot65FRcR+pC6ox7hOYC7pMIU76WMrVYx0cmDfAZAidcOQP9+K/H4LY0vkwPnaNk7+Is5gJifWR37lTnCkjRknMWB+yyDZF3BkmmdHNvXE6OlMcBF8l/jrO7S0rTtkCDaXllpRt1tWoSY2Kyr2DTaLt9QQvd1v8Rz2mw1qeDZg0mO/BswPF10SEKwjD3RlGFkA6mK9hZE5G+eDpgMH9u5HEKftI6jD/wdzElY+GdEFhlmj/ATZuNk+oC+eVnHGHUnY39lQYX+7PEtOujsKAjsuCPBaMDMkTP+f+LOUguuaRaanpLctmRAnT5BK19dmNuJtZPZAPgzCLu44884BH1qpKpVXkaUZSUbYA1RpKflcf1+uWJaqp7oBOZLSVoVSLiVG8ypGv7IPeTiILQcWQK27z6H0VxiAZ0S8yzNcCmeTWg8/wUGVzk0VJ9imPIShpiq515X6GuU1v1LsMiuJtFfL3EpNdDln04dLFWVb0ghIdt4Hn4U3IpJRIcP+XEEpv4+6
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: kaneo-smtp
|
||||
namespace: kaneo
|
||||
---
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
# monitoring — backup CronJobs
|
||||
|
||||
Manifests recording the cluster-side monitoring backup CronJobs that were previously applied out-of-band. These files are the authoritative source (`kubectl apply -f apps/monitoring/`). See [DEV-464](/DEV/issues/DEV-464) for the repair context.
|
||||
|
||||
- `backup-loki-restic-cronjob.yaml` — daily restic backup of `loki-storage-encrypted` to `hetzner-s3:${BUCKET}/restic/loki`. Co-schedules with the Loki pod via `podAffinity` (RWO permits additional read-only mounts on the same node). Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-485](/DEV/issues/DEV-485)).
|
||||
- `backup-grafana-restic-cronjob.yaml` — daily restic backup of `grafana-storage` to `hetzner-s3:${BUCKET}/restic/grafana`. Pinned to `k3s-worker-2` via `nodeSelector` (the local-path PV anchors the grafana pod there already, no `podAffinity` needed). Schedule `15 3 * * *` — offset from the loki run at `03:00`. Deployed per the [DEV-482](/DEV/issues/DEV-482) Option 4 rollout ([DEV-486](/DEV/issues/DEV-486)).
|
||||
- `prometheus-backup-cronjob.yaml` + `prometheus-backup-sealed.yaml` — daily restic backup of `prometheus-data-encrypted` to `hetzner-s3:${BUCKET}/restic/prometheus`. Co-schedules with the Prometheus pod via `podAffinity` so the RWO PVC attaches on the same node. Schedule `30 3 * * *` — offset from the loki (03:00) and grafana (03:15) runs. Migrated from the DEV-465 `rclone sync` job to restic client-side encryption per [DEV-492](/DEV/issues/DEV-492) / [DEV-482](/DEV/issues/DEV-482) Option 4. **Compaction-race mitigation:** `--exclude wal/*` + `--exclude chunks_head/*` + accept `restic backup` exit code 3 (source file vanished mid-walk) as a warning, not a failure; restore drill re-runs `promtool tsdb analyze` per block.
|
||||
- `backup-restic-alerts.yaml` + `backup-restic-alerts.test.yaml` — `PrometheusRule` with freshness, integrity, and repo-size alerts covering the three restic repos (loki/grafana/prometheus), plus a `promtool test rules` unit test proving each alert fires against synthetic samples ([DEV-490](/DEV/issues/DEV-490), extended in [DEV-492](/DEV/issues/DEV-492)). The `k8s-resources` restic repo was superseded by Velero cluster-reconstruction backups and retired in [DEV-576](/DEV/issues/DEV-576).
|
||||
|
||||
## Shared SealedSecret
|
||||
|
||||
All four restic CronJobs read Hetzner S3 credentials + the restic repository password from **SealedSecret `monitoring-s3-backup`** (namespace `monitoring`). Keys:
|
||||
|
||||
| Key | Purpose |
|
||||
|------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| `access-key` | Hetzner Object Storage access key ID |
|
||||
| `secret-key` | Hetzner Object Storage secret access key |
|
||||
| `endpoint` | S3 endpoint hostname (e.g. `fsn1.your-objectstorage.com`) |
|
||||
| `bucket` | Bucket name (single bucket, per-prefix repos) |
|
||||
| `restic-password`| 32-byte random string sealed at [DEV-484](/DEV/issues/DEV-484); plaintext copy in Passbolt entry `restic / monitoring backups`. Rotation: `restic key add` → seal new value → `restic key remove` old id. Same key protects all three remaining repos (loki/grafana/prometheus) — rotating rewrites the key file on every repo. |
|
||||
|
||||
## Restore
|
||||
|
||||
Restore procedure for all four restic repos: [`docs/monitoring/restic-restore.md`](../../docs/monitoring/restic-restore.md). First drill (2026-08-16, loki/k8s-resources) passed — see [DEV-488](/DEV/issues/DEV-488). Prometheus repo added and drilled same day — see [DEV-492](/DEV/issues/DEV-492).
|
||||
|
||||
## Emitted metrics (textfile-collector format)
|
||||
|
||||
Each restic CronJob writes `/metrics/backup_<kind>.prom` (atomic — `.prom.tmp` + `mv`) into a `hostPath` volume mounted at the node-exporter textfile-collector directory (`/var/lib/node_exporter/textfile_collector`). The kube-prometheus-stack node-exporter DaemonSet has `--collector.textfile.directory=/host/textfile_collector` enabled ([DEV-494](/DEV/issues/DEV-494), applied via [`apps/observability/patches/node-exporter-textfile-collector.yaml`](../observability/patches/node-exporter-textfile-collector.yaml)) and surfaces those samples in Prometheus.
|
||||
|
||||
| Metric | Emitted by |
|
||||
|--------------------------------------------|----------------------------------------------------------------------------------|
|
||||
| `backup_<kind>_success` (0/1) | `backup-{loki,grafana}-*-cronjob.yaml`, `prometheus-backup-cronjob.yaml` |
|
||||
| `backup_<kind>_timestamp_seconds` | same |
|
||||
| `backup_<kind>_check_status` (exit code) | same — from `restic check --read-data-subset=5%` |
|
||||
| `backup_prometheus_backup_status` | `prometheus-backup-cronjob.yaml` — `restic backup` exit code (3 = accepted compaction race) |
|
||||
| `restic_repo_size_bytes{repo="<kind>"}` | same — from `restic stats --json --mode raw-data` (added in DEV-490) |
|
||||
|
||||
**Cross-node staleness note.** Because a backup CronJob may run on a different worker across days (loki/prometheus follow their app pods), a `.prom` file can linger on a node the job has since left and node-exporter keeps exposing it. The freshness alerts collapse the per-node samples with `max()` so the freshest sample wins; check/size alerts fire when *any* node reports a bad value, which is intentional — a recent failure is still a signal until the file is manually cleaned or the job returns to that node.
|
||||
|
||||
## Alerts
|
||||
|
||||
`backup-restic-alerts.yaml` defines seven alerts (all `severity: warning`):
|
||||
|
||||
- `BackupLokiStale` / `BackupGrafanaStale` / `BackupPrometheusStale` — `time() - max(backup_<kind>_timestamp_seconds) > 28h`. Daily schedule + 4 h grace. `max()` collapses per-node samples so a stale `.prom` on a node the job has left does not fire.
|
||||
- `BackupLokiCheckFailed` / `BackupGrafanaCheckFailed` / `BackupPrometheusCheckFailed` — `backup_<kind>_check_status != 0`.
|
||||
- `ResticRepoOversize` — `restic_repo_size_bytes > 20 GiB`. Baseline expected < 5 GiB; catches retention/prune regressions.
|
||||
|
||||
All alerts carry a `Runbook: docs/monitoring/restic-restore.md` annotation. To iterate on the rule file locally:
|
||||
|
||||
```bash
|
||||
awk '/^spec:/{f=1;next} f{sub(/^ /,"");print}' \
|
||||
apps/monitoring/backup-restic-alerts.yaml > /tmp/backup-restic-rules.yaml
|
||||
promtool check rules /tmp/backup-restic-rules.yaml
|
||||
promtool test rules apps/monitoring/backup-restic-alerts.test.yaml
|
||||
```
|
||||
|
||||
The legacy `backup-volumes` CronJob and its 100 Gi local-path `backup-storage` PVC were retired in [DEV-489](/DEV/issues/DEV-489) once the restic pipeline was proven end-to-end. With the shared destination PVC gone, the DEV-483 bridge `nodeSelector` pinning Loki to `k3s-worker-2` was also removed — `backup-loki-restic` follows the Loki pod via `podAffinity` regardless of which node the RWO CSI volume lands on. `backup-grafana-restic` still nodeSelects `k3s-worker-2` because its source PV (`grafana-storage`, local-path) is anchored there. `prometheus-backup` uses `podAffinity` on `app=prometheus` (RWO PVC on Hetzner CSI, single-node attach) and follows the Prometheus pod between workers. The `backup-k8s-resources` CronJob and its restic repo were retired in [DEV-576](/DEV/issues/DEV-576) after Velero `cluster-all` took over cluster-manifest coverage.
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
---
|
||||
# Grafana data backup via restic to Hetzner Object Storage (DEV-486,
|
||||
# DEV-482 Option 4). Step 3 of the Option 4 rollout.
|
||||
#
|
||||
# Streams the RWO PVC `grafana-storage` (mounted read-only) into
|
||||
# `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/grafana`, a client-side
|
||||
# encrypted restic repository.
|
||||
#
|
||||
# `grafana-storage` is a local-path PV anchored on k3s-worker-2, so
|
||||
# the grafana pod is already pinned there; a plain nodeSelector on
|
||||
# the same host is enough (no podAffinity like the loki job needed).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: backup-grafana-restic
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: backup
|
||||
type: grafana
|
||||
backend: restic
|
||||
spec:
|
||||
schedule: "15 3 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: grafana
|
||||
backend: restic
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: grafana
|
||||
backend: restic
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: k3s-worker-2
|
||||
containers:
|
||||
- name: restic
|
||||
image: harbor.basicstack.de/library/restic:0.19.1
|
||||
# Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
|
||||
# deterministic ingress via Harbor. Retag procedure in
|
||||
# docs/monitoring/restic-restore.md § "Tag-bump procedure".
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: secret-key
|
||||
- name: RESTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: restic-password
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: endpoint
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: bucket
|
||||
- name: RESTIC_REPOSITORY
|
||||
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/grafana"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
echo "=== backup-grafana-restic started at $(date -u +%FT%TZ) ==="
|
||||
echo "Repository: ${RESTIC_REPOSITORY}"
|
||||
|
||||
# First-run tolerance: init if the repo isn't there yet.
|
||||
if restic snapshots >/dev/null 2>&1; then
|
||||
echo "Repo exists, skipping init."
|
||||
else
|
||||
echo "Repo missing, initialising..."
|
||||
restic init
|
||||
fi
|
||||
|
||||
echo "--- restic backup /source ---"
|
||||
restic backup /source \
|
||||
--tag grafana \
|
||||
--host k3s \
|
||||
--exclude '*.tmp'
|
||||
|
||||
echo "--- restic forget/prune ---"
|
||||
restic forget --tag grafana \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 6 \
|
||||
--prune
|
||||
|
||||
echo "--- restic check --read-data-subset=5% ---"
|
||||
CHECK_STATUS=0
|
||||
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
||||
echo "restic check exit: ${CHECK_STATUS}"
|
||||
|
||||
echo "--- restic stats (repo size) ---"
|
||||
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
|
||||
| grep -oE '"total_size":[0-9]+' \
|
||||
| head -1 \
|
||||
| cut -d: -f2)
|
||||
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
|
||||
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
|
||||
|
||||
# Textfile-collector metrics; identical wiring to the
|
||||
# loki sibling. See that file for the atomic-write
|
||||
# rationale (DEV-494).
|
||||
{
|
||||
echo "backup_grafana_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
|
||||
echo "backup_grafana_timestamp_seconds $(date +%s)"
|
||||
echo "backup_grafana_check_status ${CHECK_STATUS}"
|
||||
echo "restic_repo_size_bytes{repo=\"grafana\"} ${REPO_SIZE_BYTES}"
|
||||
} > /metrics/backup_grafana.prom.tmp
|
||||
mv /metrics/backup_grafana.prom.tmp /metrics/backup_grafana.prom
|
||||
|
||||
echo "=== backup-grafana-restic finished at $(date -u +%FT%TZ) ==="
|
||||
exit ${CHECK_STATUS}
|
||||
volumeMounts:
|
||||
- name: grafana-data
|
||||
mountPath: /source
|
||||
readOnly: true
|
||||
- name: metrics
|
||||
mountPath: /metrics
|
||||
- name: cache
|
||||
mountPath: /root/.cache/restic
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: grafana-data
|
||||
persistentVolumeClaim:
|
||||
claimName: grafana-storage
|
||||
- name: metrics
|
||||
hostPath:
|
||||
# node-exporter's textfile-collector directory
|
||||
# (DEV-494). See sibling loki cronjob for detail.
|
||||
path: /var/lib/node_exporter/textfile_collector
|
||||
type: DirectoryOrCreate
|
||||
- name: cache
|
||||
emptyDir: {}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
---
|
||||
# Loki data backup via restic to Hetzner Object Storage (DEV-485,
|
||||
# DEV-482 Option 4). Step 2 of the Option 4 rollout.
|
||||
#
|
||||
# Streams the RWO PVC `loki-storage-encrypted` (mounted read-only)
|
||||
# into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/loki`, a client-side
|
||||
# encrypted restic repository.
|
||||
#
|
||||
# podAffinity co-schedules with the Loki pod (app=loki, topology
|
||||
# kubernetes.io/hostname). RWO permits additional read-only mounts
|
||||
# on the node that holds the PVC's VolumeAttachment, so this
|
||||
# survives Loki being rescheduled to a different worker.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: backup-loki-restic
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: backup
|
||||
type: loki
|
||||
backend: restic
|
||||
spec:
|
||||
schedule: "0 3 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: loki
|
||||
backend: restic
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: loki
|
||||
backend: restic
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
affinity:
|
||||
podAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- loki
|
||||
topologyKey: kubernetes.io/hostname
|
||||
containers:
|
||||
- name: restic
|
||||
image: harbor.basicstack.de/library/restic:0.19.1
|
||||
# Mirrored from docker.io/restic/restic:0.19.1 (DEV-493) —
|
||||
# deterministic ingress via Harbor. Retag procedure in
|
||||
# docs/monitoring/restic-restore.md § "Tag-bump procedure".
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: secret-key
|
||||
- name: RESTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: restic-password
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: endpoint
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: bucket
|
||||
- name: RESTIC_REPOSITORY
|
||||
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/loki"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
echo "=== backup-loki-restic started at $(date -u +%FT%TZ) ==="
|
||||
echo "Repository: ${RESTIC_REPOSITORY}"
|
||||
|
||||
# First-run tolerance: init if the repo isn't there yet.
|
||||
# `restic cat config` is the tightest existence probe; use
|
||||
# `snapshots` per plan spec — either exits 0 iff the repo
|
||||
# is initialised.
|
||||
if restic snapshots >/dev/null 2>&1; then
|
||||
echo "Repo exists, skipping init."
|
||||
else
|
||||
echo "Repo missing, initialising..."
|
||||
restic init
|
||||
fi
|
||||
|
||||
echo "--- restic backup /source ---"
|
||||
restic backup /source \
|
||||
--tag loki \
|
||||
--host k3s \
|
||||
--exclude '*.tmp'
|
||||
|
||||
echo "--- restic forget/prune ---"
|
||||
restic forget --tag loki \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 6 \
|
||||
--prune
|
||||
|
||||
echo "--- restic check --read-data-subset=5% ---"
|
||||
CHECK_STATUS=0
|
||||
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
||||
echo "restic check exit: ${CHECK_STATUS}"
|
||||
|
||||
echo "--- restic stats (repo size) ---"
|
||||
# `restic stats --json --mode raw-data` prints e.g.
|
||||
# {"total_size":123,"total_file_count":45,...}. Extract
|
||||
# total_size without jq (not present in the restic image)
|
||||
# via grep/cut; fall back to 0 on empty output.
|
||||
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
|
||||
| grep -oE '"total_size":[0-9]+' \
|
||||
| head -1 \
|
||||
| cut -d: -f2)
|
||||
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
|
||||
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
|
||||
|
||||
# Textfile-collector metrics. Written to the shared
|
||||
# host directory that node-exporter's textfile collector
|
||||
# scrapes (DEV-494). Atomic write: build the file with a
|
||||
# `.tmp` extension (ignored by node-exporter) and rename
|
||||
# into place, so a mid-write read never surfaces a
|
||||
# truncated sample.
|
||||
{
|
||||
echo "backup_loki_success $([ ${CHECK_STATUS} -eq 0 ] && echo 1 || echo 0)"
|
||||
echo "backup_loki_timestamp_seconds $(date +%s)"
|
||||
echo "backup_loki_check_status ${CHECK_STATUS}"
|
||||
echo "restic_repo_size_bytes{repo=\"loki\"} ${REPO_SIZE_BYTES}"
|
||||
} > /metrics/backup_loki.prom.tmp
|
||||
mv /metrics/backup_loki.prom.tmp /metrics/backup_loki.prom
|
||||
|
||||
echo "=== backup-loki-restic finished at $(date -u +%FT%TZ) ==="
|
||||
exit ${CHECK_STATUS}
|
||||
volumeMounts:
|
||||
- name: loki-data
|
||||
mountPath: /source
|
||||
readOnly: true
|
||||
- name: metrics
|
||||
mountPath: /metrics
|
||||
- name: cache
|
||||
mountPath: /root/.cache/restic
|
||||
resources:
|
||||
# Requests deliberately lowered from the plan doc's
|
||||
# 200m/256Mi — worker-2 (Loki node) has ~150m free CPU
|
||||
# and podAffinity forces us onto it. 100m/128Mi mirrors
|
||||
# the sibling backup CronJobs; limits stay generous so
|
||||
# restic can burst during pack/check.
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: loki-data
|
||||
persistentVolumeClaim:
|
||||
claimName: loki-storage-encrypted
|
||||
- name: metrics
|
||||
hostPath:
|
||||
# node-exporter's textfile-collector directory
|
||||
# (DEV-494). `DirectoryOrCreate` lets kubelet create
|
||||
# the dir on the current node if it does not yet
|
||||
# exist — nodes were pre-created out-of-band, this
|
||||
# is a safety net.
|
||||
path: /var/lib/node_exporter/textfile_collector
|
||||
type: DirectoryOrCreate
|
||||
- name: cache
|
||||
emptyDir: {}
|
||||
|
|
@ -1,216 +0,0 @@
|
|||
---
|
||||
# promtool test file for backup-restic-alerts.yaml (DEV-490 DoD).
|
||||
#
|
||||
# Extract the rules from the PrometheusRule wrapper first:
|
||||
# awk '/^spec:/{f=1;next} f{sub(/^ /,"");print}' \
|
||||
# apps/monitoring/backup-restic-alerts.yaml > /tmp/rules.yaml
|
||||
# Then run:
|
||||
# promtool test rules apps/monitoring/backup-restic-alerts.test.yaml
|
||||
# (this file uses `rule_files: [/tmp/rules.yaml]` — pass the extracted
|
||||
# path explicitly).
|
||||
rule_files:
|
||||
- /tmp/backup-restic-rules.yaml
|
||||
|
||||
# Evaluation cadence for the test scenarios. The alerts' `for:` uses 5m/15m/30m,
|
||||
# so we run 60m of series and evaluate at 60m so all alerts have had time to
|
||||
# settle into the firing state.
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Freshness alerts fire when timestamp is stale (>28h old).
|
||||
# ---------------------------------------------------------------------------
|
||||
- interval: 1m
|
||||
input_series:
|
||||
# Timestamp value = "current time - 29h" in Unix seconds. During the
|
||||
# test, promtool's clock starts at the Unix epoch (0). We fix the
|
||||
# value to a constant far in the past so `time() - value > 28*3600`
|
||||
# for the entire test run.
|
||||
#
|
||||
# Concretely: time() returns the sample timestamp in seconds. Over a
|
||||
# 60-minute test starting at t=0, time() runs 0 .. 3600. A constant
|
||||
# value of -104400 (=-29h) keeps `time() - value` >= 29h across the
|
||||
# whole window, safely above the 28h threshold.
|
||||
- series: 'backup_loki_timestamp_seconds'
|
||||
values: '-104400x60'
|
||||
- series: 'backup_grafana_timestamp_seconds'
|
||||
values: '-104400x60'
|
||||
- series: 'backup_prometheus_timestamp_seconds'
|
||||
values: '-104400x60'
|
||||
alert_rule_test:
|
||||
- eval_time: 30m
|
||||
alertname: BackupLokiStale
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: loki
|
||||
exp_annotations:
|
||||
summary: "Loki restic backup is stale (>28h)"
|
||||
description: |
|
||||
`backup-loki-restic` has not written a fresh
|
||||
`backup_loki_timestamp_seconds` sample in more than
|
||||
28 hours. Expected daily at 03:00 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- eval_time: 30m
|
||||
alertname: BackupGrafanaStale
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: grafana
|
||||
exp_annotations:
|
||||
summary: "Grafana restic backup is stale (>28h)"
|
||||
description: |
|
||||
`backup-grafana-restic` has not written a fresh
|
||||
`backup_grafana_timestamp_seconds` sample in more
|
||||
than 28 hours. Expected daily at 03:15 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- eval_time: 30m
|
||||
alertname: BackupPrometheusStale
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: prometheus
|
||||
exp_annotations:
|
||||
summary: "Prometheus restic backup is stale (>28h)"
|
||||
description: |
|
||||
`prometheus-backup` has not written a fresh
|
||||
`backup_prometheus_timestamp_seconds` sample in more
|
||||
than 28 hours. Expected daily at 03:30 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Freshness alerts stay silent when timestamp is fresh (<28h).
|
||||
# ---------------------------------------------------------------------------
|
||||
- interval: 1m
|
||||
input_series:
|
||||
# Value = -3600 (=-1h) → time() - value stays around 1-2h, well below
|
||||
# the 28h threshold across the run.
|
||||
- series: 'backup_loki_timestamp_seconds'
|
||||
values: '-3600x60'
|
||||
- series: 'backup_grafana_timestamp_seconds'
|
||||
values: '-3600x60'
|
||||
- series: 'backup_prometheus_timestamp_seconds'
|
||||
values: '-3600x60'
|
||||
alert_rule_test:
|
||||
- eval_time: 30m
|
||||
alertname: BackupLokiStale
|
||||
exp_alerts: []
|
||||
- eval_time: 30m
|
||||
alertname: BackupGrafanaStale
|
||||
exp_alerts: []
|
||||
- eval_time: 30m
|
||||
alertname: BackupPrometheusStale
|
||||
exp_alerts: []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Check-status alerts fire when the metric is non-zero.
|
||||
# ---------------------------------------------------------------------------
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'backup_loki_check_status'
|
||||
values: '1x30'
|
||||
- series: 'backup_grafana_check_status'
|
||||
values: '2x30'
|
||||
- series: 'backup_prometheus_check_status'
|
||||
values: '1x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: BackupLokiCheckFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: loki
|
||||
exp_annotations:
|
||||
summary: "restic check failed on loki repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code 1 on the loki repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- eval_time: 15m
|
||||
alertname: BackupGrafanaCheckFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: grafana
|
||||
exp_annotations:
|
||||
summary: "restic check failed on grafana repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code 2 on the grafana repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- eval_time: 15m
|
||||
alertname: BackupPrometheusCheckFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: prometheus
|
||||
exp_annotations:
|
||||
summary: "restic check failed on prometheus repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code 1 on the prometheus
|
||||
repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Check-status alerts stay silent on 0.
|
||||
# ---------------------------------------------------------------------------
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'backup_loki_check_status'
|
||||
values: '0x30'
|
||||
- series: 'backup_grafana_check_status'
|
||||
values: '0x30'
|
||||
- series: 'backup_prometheus_check_status'
|
||||
values: '0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: BackupLokiCheckFailed
|
||||
exp_alerts: []
|
||||
- eval_time: 15m
|
||||
alertname: BackupGrafanaCheckFailed
|
||||
exp_alerts: []
|
||||
- eval_time: 15m
|
||||
alertname: BackupPrometheusCheckFailed
|
||||
exp_alerts: []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Repo-size alert fires when >20 GiB (21474836480 bytes).
|
||||
# ---------------------------------------------------------------------------
|
||||
- interval: 1m
|
||||
input_series:
|
||||
# 25 GiB = 26843545600 bytes.
|
||||
- series: 'restic_repo_size_bytes{repo="loki"}'
|
||||
values: '26843545600x45'
|
||||
# 15 GiB = 16106127360 bytes (below threshold).
|
||||
- series: 'restic_repo_size_bytes{repo="grafana"}'
|
||||
values: '16106127360x45'
|
||||
alert_rule_test:
|
||||
- eval_time: 40m
|
||||
alertname: ResticRepoOversize
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: loki
|
||||
exp_annotations:
|
||||
summary: "restic repository loki exceeds 20 GiB"
|
||||
description: |
|
||||
Repository loki is
|
||||
25GiB, above the 20 GiB
|
||||
guard. Baseline is <5 GiB per repo. Investigate
|
||||
retention/prune (see forget flags in the CronJob)
|
||||
and dedup effectiveness.
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
---
|
||||
# Prometheus alerting rules for the restic-based monitoring backups
|
||||
# (DEV-490 / DEV-482 Option 4). Freshness alerts fire when a
|
||||
# CronJob has not emitted its `backup_<kind>_timestamp_seconds`
|
||||
# metric within 28h (schedules are daily; 28h gives one missed
|
||||
# run + 4h grace before we page).
|
||||
#
|
||||
# Check-status alerts fire on the first non-zero exit from
|
||||
# `restic check --read-data-subset=5%`.
|
||||
#
|
||||
# Repo-size alerts fire when a repository exceeds 20 GiB — the
|
||||
# baseline is expected < 5 GiB per repo. The alert catches
|
||||
# retention/prune bugs and runaway growth. The metric is emitted
|
||||
# by the same CronJob step (see backup-*-restic-cronjob.yaml,
|
||||
# `restic stats --json --mode raw-data`).
|
||||
#
|
||||
# The metrics are written to the node's textfile-collector directory
|
||||
# (`/var/lib/node_exporter/textfile_collector`) and scraped by the
|
||||
# kube-prometheus-stack node-exporter DaemonSet (DEV-494).
|
||||
#
|
||||
# Because a CronJob may run on a different worker across days (loki
|
||||
# backup follows the loki pod), stale
|
||||
# `.prom` files can linger on nodes the job has since left. That
|
||||
# would leave a per-node series with an old timestamp/check-status
|
||||
# indefinitely. The alerts below aggregate across instances so a
|
||||
# single fresh sample from the node where the job currently runs is
|
||||
# enough to keep the freshness alert quiet, and check/size alerts
|
||||
# fire when *any* node reports a bad value (which is the correct
|
||||
# behaviour — a recent failure is still a signal).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: backup-restic-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: backup
|
||||
backend: restic
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: backup-restic.freshness
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: BackupLokiStale
|
||||
# `max()` collapses per-node samples so a stale `.prom` file
|
||||
# left behind on a node the loki backup no longer runs on
|
||||
# does not fire this alert; the freshest sample wins.
|
||||
expr: time() - max(backup_loki_timestamp_seconds) > 28 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: loki
|
||||
annotations:
|
||||
summary: "Loki restic backup is stale (>28h)"
|
||||
description: |
|
||||
`backup-loki-restic` has not written a fresh
|
||||
`backup_loki_timestamp_seconds` sample in more than
|
||||
28 hours. Expected daily at 03:00 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupGrafanaStale
|
||||
expr: time() - max(backup_grafana_timestamp_seconds) > 28 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: grafana
|
||||
annotations:
|
||||
summary: "Grafana restic backup is stale (>28h)"
|
||||
description: |
|
||||
`backup-grafana-restic` has not written a fresh
|
||||
`backup_grafana_timestamp_seconds` sample in more
|
||||
than 28 hours. Expected daily at 03:15 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupPrometheusStale
|
||||
expr: time() - max(backup_prometheus_timestamp_seconds) > 28 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: prometheus
|
||||
annotations:
|
||||
summary: "Prometheus restic backup is stale (>28h)"
|
||||
description: |
|
||||
`prometheus-backup` has not written a fresh
|
||||
`backup_prometheus_timestamp_seconds` sample in more
|
||||
than 28 hours. Expected daily at 03:30 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupForgejoStale
|
||||
expr: time() - max(backup_forgejo_timestamp_seconds) > 28 * 3600
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: forgejo
|
||||
annotations:
|
||||
summary: "Forgejo restic backup is stale (>28h)"
|
||||
description: |
|
||||
`forgejo-backup` has not written a fresh
|
||||
`backup_forgejo_timestamp_seconds` sample in more
|
||||
than 28 hours. Expected daily at 03:00 UTC.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- name: backup-restic.integrity
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: BackupLokiCheckFailed
|
||||
expr: backup_loki_check_status != 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: loki
|
||||
annotations:
|
||||
summary: "restic check failed on loki repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code {{ $value }} on the loki repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupGrafanaCheckFailed
|
||||
expr: backup_grafana_check_status != 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: grafana
|
||||
annotations:
|
||||
summary: "restic check failed on grafana repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code {{ $value }} on the grafana repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupPrometheusCheckFailed
|
||||
expr: backup_prometheus_check_status != 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: prometheus
|
||||
annotations:
|
||||
summary: "restic check failed on prometheus repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code {{ $value }} on the prometheus
|
||||
repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- alert: BackupForgejoCheckFailed
|
||||
expr: backup_forgejo_check_status != 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
repo: forgejo
|
||||
annotations:
|
||||
summary: "restic check failed on forgejo repo"
|
||||
description: |
|
||||
`restic check --read-data-subset=5%` returned
|
||||
exit code {{ $value }} on the forgejo
|
||||
repository.
|
||||
Runbook: docs/monitoring/restic-restore.md
|
||||
- name: backup-restic.size
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: ResticRepoOversize
|
||||
# 20 GiB = 20 * 1024^3 = 21474836480 bytes.
|
||||
expr: restic_repo_size_bytes > 21474836480
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
service: monitoring
|
||||
component: backup
|
||||
annotations:
|
||||
summary: "restic repository {{ $labels.repo }} exceeds 20 GiB"
|
||||
description: |
|
||||
Repository {{ $labels.repo }} is
|
||||
{{ $value | humanize1024 }}B, above the 20 GiB
|
||||
guard. Baseline is <5 GiB per repo. Investigate
|
||||
retention/prune (see forget flags in the CronJob)
|
||||
and dedup effectiveness.
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: loki
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: loki
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: loki
|
||||
# loki-storage-encrypted is RWO on hcloud-volumes-encrypted, so a rolling
|
||||
# update deadlocks (new pod cannot attach the PVC while the old pod holds
|
||||
# it). Recreate drops the old pod first so the CSI detaches the volume
|
||||
# before the new pod tries to attach it. Same pattern as harbor's RWO fix.
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: loki
|
||||
spec:
|
||||
containers:
|
||||
- name: loki
|
||||
image: grafana/loki:2.9.2
|
||||
args:
|
||||
- -config.file=/etc/loki/loki.yaml
|
||||
ports:
|
||||
- containerPort: 3100
|
||||
name: http
|
||||
- containerPort: 9096
|
||||
name: grpc
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: loki-config
|
||||
mountPath: /etc/loki
|
||||
- name: loki-storage
|
||||
mountPath: /loki
|
||||
volumes:
|
||||
- name: loki-config
|
||||
configMap:
|
||||
name: loki-config
|
||||
- name: loki-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: loki-storage-encrypted
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
---
|
||||
# Prometheus data backup via restic to Hetzner Object Storage
|
||||
# (DEV-492, DEV-482 Option 4). Replaces the DEV-465 rclone-sync job so
|
||||
# every monitoring backup ships client-side-encrypted; Hetzner Object
|
||||
# Storage has no SSE-S3/SSE-KMS, so the previous plaintext-at-rest
|
||||
# object layout was the only remaining gap.
|
||||
#
|
||||
# Streams the RWO PVC `prometheus-data-encrypted` (mounted read-only)
|
||||
# into `s3:${S3_ENDPOINT}/${S3_BUCKET}/restic/prometheus`, a client-side
|
||||
# encrypted restic repository (tag=`prometheus`, host=`k3s`).
|
||||
#
|
||||
# Node scheduling matches the previous job: podAffinity co-schedules
|
||||
# with the Prometheus pod (app=prometheus, topology
|
||||
# kubernetes.io/hostname). Hetzner CSI RWO permits additional read-only
|
||||
# mounts on the node that holds the PVC's VolumeAttachment, so this
|
||||
# survives Prometheus being rescheduled to a different worker.
|
||||
#
|
||||
# Prometheus TSDB compaction race
|
||||
# --------------------------------
|
||||
# Prometheus rewrites the on-disk store roughly every 2 h: it creates a
|
||||
# new block dir, then deletes the source dirs. restic walks the source
|
||||
# tree once and may catch a file that disappeared mid-walk; restic
|
||||
# 0.17.3 exits 3 ("at least one source file could not be read") in
|
||||
# that case, and the snapshot excludes only the missing file. The
|
||||
# next daily run picks up the successor block, so the race is not a
|
||||
# data-loss risk — but we must not treat exit 3 as a hard failure, or
|
||||
# the daily job will alert-flap.
|
||||
#
|
||||
# Mitigation:
|
||||
# - exclude `wal/*` (WAL is replayed from a fresh instance on
|
||||
# restart; we accept losing the last ~15 s of ingested samples
|
||||
# rather than snapshotting a moving segment)
|
||||
# - exclude `chunks_head/*` (in-memory head block; ephemeral, would
|
||||
# be rebuilt from WAL which we do not keep)
|
||||
# - exclude Prometheus lock/scratch files (`lock`, `queries.active`,
|
||||
# `*.tmp`, `lost+found/*`)
|
||||
# - treat restic exit code 3 as a soft warning (log, continue);
|
||||
# any other non-zero exit is still fatal
|
||||
# - restore drill re-runs `promtool tsdb analyze` against every
|
||||
# block so a corrupted snapshot is caught end-to-end
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: prometheus-backup
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: backup
|
||||
type: prometheus
|
||||
backend: restic
|
||||
spec:
|
||||
schedule: "30 3 * * *" # daily 03:30, offset from loki/grafana
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: prometheus
|
||||
backend: restic
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup
|
||||
type: prometheus
|
||||
backend: restic
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
affinity:
|
||||
podAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- prometheus
|
||||
topologyKey: kubernetes.io/hostname
|
||||
containers:
|
||||
- name: restic
|
||||
image: harbor.basicstack.de/library/restic:0.19.1
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: secret-key
|
||||
- name: RESTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: restic-password
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: endpoint
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: monitoring-s3-backup
|
||||
key: bucket
|
||||
- name: RESTIC_REPOSITORY
|
||||
value: "s3:$(S3_ENDPOINT)/$(S3_BUCKET)/restic/prometheus"
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
echo "=== backup-prometheus-restic started at $(date -u +%FT%TZ) ==="
|
||||
echo "Repository: ${RESTIC_REPOSITORY}"
|
||||
|
||||
# First-run tolerance: init if the repo isn't there yet.
|
||||
if restic snapshots >/dev/null 2>&1; then
|
||||
echo "Repo exists, skipping init."
|
||||
else
|
||||
echo "Repo missing, initialising..."
|
||||
restic init
|
||||
fi
|
||||
|
||||
# See "Prometheus TSDB compaction race" in the file
|
||||
# header. WAL + chunks_head are excluded on purpose;
|
||||
# exit code 3 (source file vanished mid-walk during a
|
||||
# compaction) is accepted and reported, any other
|
||||
# non-zero exit is fatal.
|
||||
echo "--- restic backup /source (exclude wal/chunks_head + lock files) ---"
|
||||
BACKUP_STATUS=0
|
||||
restic backup /source \
|
||||
--tag prometheus \
|
||||
--host k3s \
|
||||
--exclude 'wal/*' \
|
||||
--exclude 'chunks_head/*' \
|
||||
--exclude 'lock' \
|
||||
--exclude 'queries.active' \
|
||||
--exclude 'lost+found/*' \
|
||||
--exclude '*.tmp' || BACKUP_STATUS=$?
|
||||
echo "restic backup exit: ${BACKUP_STATUS}"
|
||||
if [ "${BACKUP_STATUS}" -eq 0 ]; then
|
||||
echo "backup: all files captured cleanly"
|
||||
elif [ "${BACKUP_STATUS}" -eq 3 ]; then
|
||||
echo "backup: exit 3 (source files vanished mid-walk) — expected under Prometheus compaction, continuing"
|
||||
else
|
||||
echo "backup: FATAL exit ${BACKUP_STATUS} (not compaction-race)"
|
||||
exit ${BACKUP_STATUS}
|
||||
fi
|
||||
|
||||
echo "--- restic forget/prune ---"
|
||||
restic forget --tag prometheus \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 6 \
|
||||
--prune
|
||||
|
||||
echo "--- restic check --read-data-subset=5% ---"
|
||||
CHECK_STATUS=0
|
||||
restic check --read-data-subset=5% || CHECK_STATUS=$?
|
||||
echo "restic check exit: ${CHECK_STATUS}"
|
||||
|
||||
echo "--- restic stats (repo size) ---"
|
||||
REPO_SIZE_BYTES=$(restic stats --json --mode raw-data 2>/dev/null \
|
||||
| grep -oE '"total_size":[0-9]+' \
|
||||
| head -1 \
|
||||
| cut -d: -f2)
|
||||
REPO_SIZE_BYTES=${REPO_SIZE_BYTES:-0}
|
||||
echo "restic repo size: ${REPO_SIZE_BYTES} bytes"
|
||||
|
||||
# Textfile-collector metrics. Same hostPath pattern as
|
||||
# the loki/grafana siblings (DEV-494) —
|
||||
# written atomically via `.tmp` + rename so a mid-write
|
||||
# read never surfaces a truncated sample.
|
||||
# backup_prometheus_success rolls in both stages: the
|
||||
# backup step (accepting exit 3) and restic check.
|
||||
BACKUP_OK=0
|
||||
if [ "${BACKUP_STATUS}" -eq 0 ] || [ "${BACKUP_STATUS}" -eq 3 ]; then
|
||||
BACKUP_OK=1
|
||||
fi
|
||||
SUCCESS=0
|
||||
if [ "${BACKUP_OK}" -eq 1 ] && [ "${CHECK_STATUS}" -eq 0 ]; then
|
||||
SUCCESS=1
|
||||
fi
|
||||
{
|
||||
echo "backup_prometheus_success ${SUCCESS}"
|
||||
echo "backup_prometheus_timestamp_seconds $(date +%s)"
|
||||
echo "backup_prometheus_check_status ${CHECK_STATUS}"
|
||||
echo "backup_prometheus_backup_status ${BACKUP_STATUS}"
|
||||
echo "restic_repo_size_bytes{repo=\"prometheus\"} ${REPO_SIZE_BYTES}"
|
||||
} > /metrics/backup_prometheus.prom.tmp
|
||||
mv /metrics/backup_prometheus.prom.tmp /metrics/backup_prometheus.prom
|
||||
|
||||
echo "=== backup-prometheus-restic finished at $(date -u +%FT%TZ) ==="
|
||||
exit ${CHECK_STATUS}
|
||||
volumeMounts:
|
||||
- name: prometheus-data
|
||||
mountPath: /source
|
||||
readOnly: true
|
||||
- name: metrics
|
||||
mountPath: /metrics
|
||||
- name: cache
|
||||
mountPath: /root/.cache/restic
|
||||
resources:
|
||||
# Prometheus TSDB is ~8-10 GiB; give restic room to
|
||||
# burst during pack/check but keep steady-state small.
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: prometheus-data
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data-encrypted
|
||||
- name: metrics
|
||||
hostPath:
|
||||
# node-exporter's textfile-collector directory
|
||||
# (DEV-494). See sibling loki cronjob for detail.
|
||||
path: /var/lib/node_exporter/textfile_collector
|
||||
type: DirectoryOrCreate
|
||||
- name: cache
|
||||
emptyDir: {}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: monitoring-s3-backup
|
||||
namespace: monitoring
|
||||
spec:
|
||||
encryptedData:
|
||||
access-key: AgBf1XxCF3/Uv1vVlJHyTSEMONqWBqPfSOax8jJJHy3NOpgGga3SsxDfAbPXe2rBOYSgGzRZcaT8TcNg8Ng1ipv5Dhu7LzmRb6v6fW0NimrHSU/dtzzJSIR7jQ1sf+Qn4CPS73ReDRvUJ7g4o57ClZIVFBk9oPUDJYdjtg9kAM5Me0ZyqVp4Cnyj2TRXYcQPM+RN1r/PXqK4MhlUsmhsbQCPYwocM0liqv0T8L5xR1g2845oMaMx2nmLs1c1R8nCRugK8IeBKf/CCKLYg8yTqWKS2arrfxm+dkl1mrqLdOgnXrqeeo/63lT1sbd06ub86PJr83MXWW0yh1Uc4r7DX7xK/JIGJ4w+jtQa0vRutYANDcdeVlzjVl+ghoxxw/ZizYVQ/mUFCc+ApJD9u709mI39xof77U5yYBRhnZEExRseL756APubmCc0tPfUSNA1GmMhiYaMnyWu6F5Okreto/03AgI3t7lQm9SPRRM7hcHETG/QC8PrZ0NZEN/A2Nm/ZzwDG4HPDaNxH+IV90f5X4wuh6Mh2lffidOix88rkaizQAeVY5gt8tvfSfyfRc5RoOOov+fvUVCqJHnWaD1A2lKScVeCgLv8/0CJTyR0zM6cg2XN8z0HvU/sahfZrG23fVs3E/IFM5D7//CDp+GfcwQOCBZKPLvJtTRh5nA5EbXQusGEaMOTBiiGG0IJKPejKcPRWS1g+owu+G5R6QQjZgNDoP0cPg==
|
||||
bucket: AgCYDCrQalif3+XZYhe4Xxyypg2OMGvzdNyBmHzMaP+ou1g2Q0wIjH6S1+YpPu+JgmaATZVISoSZ+OIl0mNOibiFMr0kFjuMhvKRt9Pq05+6axnqnJ17U2GNsZQtY3Ic93qxkkhJKCOQ87mvAJ5AIVBc3pAqfkD5YYTllgnSKwUwyaWK3iX3hxuKLYDvsQ2WTtcc/gsnlCkm1dwPE4E7DFt4wh+jQJRD314Mt7KSpvPBpz7yVVFFX+xRKCfRxjcwshUSUI5OgagfOQvsivkxlL0oSqA9veX2MJpd0SJH0T1Y0na0evkJitxxe+ECwM4FluK/iwW4cdUu+EpQIdd5otY9C5b8e8PJp8ah1k1hYzXdX+j975MeyXWMUn/NgXfyT2CUef6hmmmImialrSYsHT72bz3xpAnZ/oQ/0qcZpNje2oJeokNgkoiC0E5XRnfwEF9jL3XL+aB6gQ3BD9YRptaxLLxOQNj51Oe82vVsOnUz+utzJO1DtqpF+8I0lvHfgcUhFQytUuLscilfBQO/ru8wh47ZI+LRiS3qCZp2Pqi9+r7Y7jJKcbb2awWp6lgrrqgLMvnbVMuvxBi2hUy0C+EkP8tayRJDzz72utx8FY/v0FWoRDl9BAi8kn11+kbbo8a2OXe+20UkmNzunz5H0ijIxhJWzVh/oLuuLFCS0YJtJNjuqUB0VctAFcR4p5Agua8LA9+73HIBsHBYexzmvEZ6Ng==
|
||||
endpoint: AgCV6YtzikWYMRwHZM1tT8epq6XiSVqhbxMhS7htVjVUfdR4V7/KIcNExxSBul+TU+RCq3S7fppL7CxKbI/IsEdirRMj2PvESXEmzLY/xId7+wJFEdxeXxn59cbV2MfCyysKIwzsVXbmzO66yGaeMKU2oyqJVYu+00L9xS0RbtQRbi0UksMxtw0PhZVMee0aHfu9jynWI3lmPzC/EPRuwxQqBIjYM2v2JCDmYjP0fsBSZSJc+SnAaNmmIf7ypcFwy14qIF7C9tIg02lBx+Sg30oMPjn600AbyF6RmZ+mdclp/Qp+zXOUsbMym0Lu9BjbM7G/z0ExpR5sgE/6+05gJhPF7MuHkrYIMo9LHWWdgzNTnjikSTGEeuFzzRlThPMoUfVi/S7KhG9NtQRnNqEoVCGGW6tt5sygXVE/lWSnmmd//4jXvF9ZJHU+5LrjL7UfVIYNH4y7aYtXpoTgmUe85wWqeMdqr1EiSVueoSwCggGITYoXoKYfERfij8ln/xnX1Kt2xxuyLtpDOczBYZK6LLwTpaXVVe+lO/xBgevWaoEz6si2hyj+glluMQWxPsOnPwUfRi5qqlQxgMZK/9aALYhPLquc16iMG+4FdtoOUfQ0XlLvurHsbCt9lUy3YrsmpDhv42FfamVlcK+gMz7mqYhNhd3fVjxCiUuwvzd01QHcx8ikv48YOnDuRiweRszbPNvlaK2KH6fo7b4l9mmaa4W+MaxZCxG3KSMOafwRmyusEUVNWw==
|
||||
secret-key: AgBItdfzRgcgJvK5HALDnkbGH1DveAgcmS+Q4f16VwX1KY8TzGWpjoaOdsLCwzTWOFME5xjrRyOwwAcdtP65ccWlEZvvoS4kpqv49G1ebblD1wmApnpa8vD63Rd028vUYFF3CnwntX2USxF6t4rD68sZqqzMDyuG9Av93HAKVMNdx9V4AJ45nvfF2WR7ohk37NhQifOdKuelgRV6ahl3/fm/XPRGzf+u7OER8jll9n/ywE0mSkoKBXocbkQ2Xrn+utzplh9BoR9KSPTdnmI8ZXUkDTR9p+7PN4r/HheEdTUG0aaefdEDCbdDlv1EW84l5KzhY4v9g0DW8GQQdyHWi3sMm0hSGFg9NV/Vs1HM9FZZCyjS+mzR4CbuHgkY9mvj+Svn7oS+Fy5d24s1ekmPheakBffXKy2Q2CLMRGNymjUIBQsDwoGC8o4IF+yPyDcw/OvXlr0lruxfRM6IN0oxgRmLAQiqwRzsZb3RPgILEZI9VXKxzCBEuCO7fE2pC05K/mujhUutWmJ5tQ0NFzP1mhNK59cAIVMl7y7Llq5/GZ5ax+voP0c3VandXJeaSRr6O40vyzfCMgpgEgRs9t2WfxCmVYEN2NWpFv07JNICWzJX40PLT9n/euRXzSmJld6UfjzRteaTfmo+OxFczJ6Zl8XdC+6x6wO0Nk0hVYo27s2LH4o0MsM84x9Z3StCCGl77/NnyavpxfDkj3qJQ4BgPtCw26tHTt9DIWYfgeAVqvoRZVlg8YxM3N1k
|
||||
restic-password: AgBuoBFK5Y2Vlf1Vj6Q7y5OwWV2iOLMATe+Kjca8uwHgiYnpka+619ab52P1P+tkxbGyGdnwF+3q0KzvJJaNfuoduwlPuRjiYmaLLoCuQahZmTLNbzwEFst1cbm4oG6vCTBGsK9IMngT6VMfmvz029X+jsGaJHaECHlJjWnMteKN2pwxbzt4/3Wx6wP3//qAk1ah4QLRZUEWvn6O7djTFWJtzM4MRfZPI6EmM9emlQRN9iHwWSUmgBZ5zvaAylRHHUHK1lkNgWJJr7+tC4D8DbZ+c4+FO6XefXDheG6jO2Ial09KTISC0+v86xKFq/aSko5TKczpdXEdYNfyf/aPJTRd4LTcTySUdIpflmjIUnJBdn2Yn4Jox8LKc6ecUJ+IxMXhWlToFNtSFJwx744uv9WsrQ7p/lH7VTZOT+R+0wKTe9vn07h6B78xX6JJWKHMepb9aJB0gB1fn9cx1KHzm4/Y30SFwuOpYpVx19lyl1BRfSry4kmPuDKouSO6D415NJ/lMNYqTvafRho5SzlFfle/n7wAK7OIRdNbgsabO2mfHjHpw6w8Bvo9K7+3QJiwpeesY8OKAubcpxy1RBDw3fp2JQCNNqIer41vUtprsRU+tBnzDboImhOTXpP71AgTa+L68k/iGVJB0YqjFDxjcJB1f1hKGvH+yCV0GPFSeUFOZ/gmxV+Q0Fmoq/ezLrqpjoTbjp+ZbAi6qbVrMN/3t3lheKf47dU5HEOQMyZkXqDrkyuc7kXhmzN/V/ISmW0=
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: monitoring-s3-backup
|
||||
namespace: monitoring
|
||||
type: Opaque
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
---
|
||||
# Prometheus data volume — RWO on hcloud-volumes-encrypted (fsn1). Because the
|
||||
# volume is RWO, the Deployment below must use Recreate; a rolling update would
|
||||
# spin up a surge pod that cannot mount the volume while the old pod holds it
|
||||
# and the rollout would stall at Init:0/1. See DEV-583 / mirrors DEV-538.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: prometheus-data-encrypted
|
||||
namespace: monitoring
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: prometheus
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 9090
|
||||
targetPort: 9090
|
||||
name: web
|
||||
selector:
|
||||
app: prometheus
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: prometheus
|
||||
spec:
|
||||
replicas: 1
|
||||
# prometheus-data-encrypted is RWO on hcloud-volumes-encrypted, so a rolling
|
||||
# update deadlocks (new pod cannot attach the PVC while the old pod holds
|
||||
# it). Recreate drops the old pod first so the CSI detaches the volume
|
||||
# before the new pod tries to attach it. Same pattern as loki/opencloud.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: prometheus
|
||||
annotations:
|
||||
# Exclude the prometheus TSDB volume from Velero fs-backup — TSDB is
|
||||
# backed up separately by prometheus-backup-cronjob and streaming it
|
||||
# through Kopia would double-copy tens of GiB on every backup.
|
||||
backup.velero.io/backup-volumes-excludes: prometheus-storage
|
||||
spec:
|
||||
serviceAccountName: prometheus
|
||||
initContainers:
|
||||
- name: fix-permissions
|
||||
image: busybox:latest
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chown -R 65534:65534 /prometheus && chmod -R 755 /prometheus
|
||||
volumeMounts:
|
||||
- name: prometheus-storage
|
||||
mountPath: /prometheus
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: prom/prometheus:v2.47.0
|
||||
args:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=30d'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
- '--web.enable-lifecycle'
|
||||
ports:
|
||||
- containerPort: 9090
|
||||
name: web
|
||||
volumeMounts:
|
||||
- name: prometheus-config
|
||||
mountPath: /etc/prometheus
|
||||
- name: prometheus-rules
|
||||
mountPath: /etc/prometheus/rules
|
||||
- name: prometheus-storage
|
||||
mountPath: /prometheus
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
volumes:
|
||||
- name: prometheus-config
|
||||
configMap:
|
||||
name: prometheus-config
|
||||
- name: prometheus-rules
|
||||
configMap:
|
||||
name: prometheus-rules
|
||||
- name: prometheus-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data-encrypted
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# observability patches
|
||||
|
||||
Strategic-merge patches applied on top of Helm-managed observability
|
||||
resources. Each file is idempotent (re-applying is a no-op) and is
|
||||
reasserted by hand rather than by a controller, so re-run after any
|
||||
`helm upgrade` of the affected release.
|
||||
|
||||
## `node-exporter-textfile-collector.yaml` (DEV-494)
|
||||
|
||||
Enables the node-exporter textfile collector on the
|
||||
`kube-prometheus-stack-prometheus-node-exporter` DaemonSet by:
|
||||
|
||||
1. adding `--collector.textfile.directory=/host/textfile_collector`
|
||||
to the container args, and
|
||||
2. mounting the host directory `/var/lib/node_exporter/textfile_collector`
|
||||
read-only at `/host/textfile_collector` (`hostPath` type
|
||||
`DirectoryOrCreate`, kubelet creates it on nodes where the
|
||||
directory does not exist yet).
|
||||
|
||||
The four monitoring backup CronJobs in `apps/monitoring/` write their
|
||||
textfile-collector `.prom` files into that same host directory, so the
|
||||
metrics surface in Prometheus via node-exporter's normal scrape.
|
||||
|
||||
Apply / re-apply:
|
||||
|
||||
```bash
|
||||
kubectl -n observability patch daemonset \
|
||||
kube-prometheus-stack-prometheus-node-exporter \
|
||||
--type=strategic \
|
||||
--patch-file=apps/observability/patches/node-exporter-textfile-collector.yaml
|
||||
kubectl -n observability rollout status daemonset \
|
||||
kube-prometheus-stack-prometheus-node-exporter
|
||||
```
|
||||
|
||||
The kube-prometheus-stack chart is not currently tracked in ArgoCD;
|
||||
if it moves under GitOps, fold these values into the chart values as
|
||||
`prometheus-node-exporter.extraArgs` + `.extraHostVolumeMounts`
|
||||
instead of maintaining this patch.
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
---
|
||||
# Strategic-merge patch enabling node-exporter's textfile collector on
|
||||
# the kube-prometheus-stack node-exporter DaemonSet (DEV-494).
|
||||
#
|
||||
# The chart is Helm-managed (release `kube-prometheus-stack` in
|
||||
# namespace `observability`, chart kube-prometheus-stack-86.2.2 /
|
||||
# prometheus-node-exporter-4.55.0) and is NOT currently tracked in
|
||||
# ArgoCD, so a direct DaemonSet patch is the pragmatic wiring path.
|
||||
# The change:
|
||||
#
|
||||
# 1. adds the `--collector.textfile.directory=/host/textfile_collector`
|
||||
# arg to the node-exporter container, and
|
||||
# 2. mounts the host directory `/var/lib/node_exporter/textfile_collector`
|
||||
# read-only at `/host/textfile_collector` (type DirectoryOrCreate so
|
||||
# kubelet creates the dir on nodes where it does not yet exist).
|
||||
#
|
||||
# Apply with:
|
||||
# kubectl -n observability patch daemonset \
|
||||
# kube-prometheus-stack-prometheus-node-exporter \
|
||||
# --type=strategic \
|
||||
# --patch-file=apps/observability/patches/node-exporter-textfile-collector.yaml
|
||||
#
|
||||
# If the Helm release is ever `helm upgrade`d without folding these
|
||||
# values into the chart values, this patch will be reverted — re-apply
|
||||
# it after the upgrade (or move it into a repo-owned values file).
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: node-exporter
|
||||
args:
|
||||
- --path.procfs=/host/proc
|
||||
- --path.sysfs=/host/sys
|
||||
- --path.rootfs=/host/root
|
||||
- --path.udev.data=/host/root/run/udev/data
|
||||
- --web.listen-address=[$(HOST_IP)]:9100
|
||||
- --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
|
||||
- --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$
|
||||
- --collector.textfile.directory=/host/textfile_collector
|
||||
volumeMounts:
|
||||
- mountPath: /host/proc
|
||||
name: proc
|
||||
readOnly: true
|
||||
- mountPath: /host/sys
|
||||
name: sys
|
||||
readOnly: true
|
||||
- mountPath: /host/root
|
||||
mountPropagation: HostToContainer
|
||||
name: root
|
||||
readOnly: true
|
||||
- mountPath: /host/textfile_collector
|
||||
name: textfile-collector
|
||||
readOnly: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /proc
|
||||
type: ""
|
||||
name: proc
|
||||
- hostPath:
|
||||
path: /sys
|
||||
type: ""
|
||||
name: sys
|
||||
- hostPath:
|
||||
path: /
|
||||
type: ""
|
||||
name: root
|
||||
- hostPath:
|
||||
path: /var/lib/node_exporter/textfile_collector
|
||||
type: DirectoryOrCreate
|
||||
name: textfile-collector
|
||||
|
|
@ -6,7 +6,6 @@ metadata:
|
|||
name: opencloud-backup
|
||||
namespace: opencloud
|
||||
spec:
|
||||
suspend: true # CronJob deaktiviert
|
||||
schedule: "0 2 * * *" # Daily at 2 AM UTC
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
|
|
@ -91,8 +90,8 @@ spec:
|
|||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
memory: "16Mi"
|
||||
cpu: "20m"
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
|
|
|
|||
|
|
@ -41,9 +41,7 @@ metadata:
|
|||
name: opencloud
|
||||
namespace: opencloud
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: opencloud
|
||||
|
|
@ -51,12 +49,6 @@ spec:
|
|||
metadata:
|
||||
labels:
|
||||
app: opencloud
|
||||
annotations:
|
||||
# Skip Kopia fs-backup of the 100 GiB opencloud object-store PVC — this
|
||||
# blob store is out of scope for the DEV-573 data schedules; the
|
||||
# exclusion is a safety net so an accidental defaultVolumesToFsBackup
|
||||
# run never streams it into the Kopia repo.
|
||||
backup.velero.io/backup-volumes-excludes: opencloud-data
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
|
|
@ -64,7 +56,7 @@ spec:
|
|||
runAsNonRoot: true
|
||||
initContainers:
|
||||
- name: init-dirs
|
||||
image: opencloudeu/opencloud-rolling:7.4.0
|
||||
image: opencloudeu/opencloud-rolling:7.2.0
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
|
|
@ -79,7 +71,7 @@ spec:
|
|||
runAsNonRoot: true
|
||||
containers:
|
||||
- name: opencloud
|
||||
image: opencloudeu/opencloud-rolling:7.4.0
|
||||
image: opencloudeu/opencloud-rolling:7.2.0
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
|
|
@ -454,7 +446,7 @@ spec:
|
|||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "50m"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "1000m"
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ metadata:
|
|||
name: tika
|
||||
namespace: opencloud
|
||||
spec:
|
||||
replicas: 0
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tika
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
# Open WebUI (openwebui.basicstack.de)
|
||||
|
||||
Self-hosted LLM front-end — [Open WebUI](https://openwebui.com/).
|
||||
|
||||
## Stack
|
||||
|
||||
- **App:** `ghcr.io/open-webui/open-webui:v0.11.3` (Deployment, 1 replica, `Recreate` strategy)
|
||||
- **Database:** Postgres 16 + pgvector (`pgvector/pgvector:pg16`, StatefulSet, PVC 20Gi on `hcloud-volumes-encrypted`)
|
||||
- **Vector DB:** `pgvector` in the same Postgres instance (`VECTOR_DB=pgvector`, extension auto-created on first start)
|
||||
- **Cache / websocket manager:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`, `WEBSOCKET_MANAGER=redis` on `db=1`)
|
||||
- **Object storage:** Hetzner Object Storage bucket `openwebui-basicstack` (region `hel1`, path-style)
|
||||
- **Auth:** Pocket ID OIDC — `Open WebUI` client, restricted to `openwebui_admins` / `openwebui_users`. Sign-up via OAuth only (`ENABLE_LOGIN_FORM=false`, `ENABLE_SIGNUP=false`)
|
||||
- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
|
||||
- **Node placement:** all stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
| ---- | ------- |
|
||||
| `namespace.yaml` | Namespace declaration |
|
||||
| `postgres.yaml` | Postgres+pgvector StatefulSet + Service (fsn1-pinned) |
|
||||
| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
|
||||
| `openwebui.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
|
||||
| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
|
||||
| `app-secrets-sealed.yaml` | Sealed `WEBUI_SECRET_KEY`, `DATABASE_URL`, `PGVECTOR_DB_URL`, `REDIS_URL`, `WEBSOCKET_REDIS_URL` |
|
||||
| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
|
||||
| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
|
||||
|
||||
## Deployment references
|
||||
|
||||
- Argo Application: `apps/app-openwebui.yaml`
|
||||
- Pocket ID OIDC client id: `3ffbd06b-12cc-4562-bbe5-da4433962870` (PKCE **enabled**, group restriction on, allowed groups: `openwebui_admins`, `openwebui_users`)
|
||||
- Pocket ID groups: `openwebui_admins` (`737f4b7f-63f2-42b5-98bb-4e96c4a80546`), `openwebui_users` (`74b0af52-0942-4cb6-b37a-83d87d203e32`)
|
||||
- OIDC redirect URI: `https://openwebui.basicstack.de/oauth/oidc/callback`
|
||||
- Role mapping: OIDC `groups` claim → Open WebUI roles. `openwebui_admins` → admin, `openwebui_users` → user.
|
||||
- Initial admin: `andreas.leinen@basicstack.de` (member of both groups; `OAUTH_ADMIN_ROLES=openwebui_admins`)
|
||||
- DNS: `openwebui.basicstack.de` A → `138.199.128.63` (cluster ingress)
|
||||
- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `openwebui-basicstack` (CORS whitelisted to `https://openwebui.basicstack.de`). S3 credentials are shared with the Hetzner project (same key used for `outline-basicstack`, `basicstack-backup`).
|
||||
|
||||
## Deferred / follow-ups
|
||||
|
||||
- **Model providers (Ollama / OpenAI):** none wired yet. Add `OLLAMA_BASE_URL` or `OPENAI_API_BASE_URL`/`OPENAI_API_KEY` via a sealed secret when a backend is chosen.
|
||||
- **RAG embeddings:** `RAG_EMBEDDING_ENGINE=default` (local). Switch to `openai` + `RAG_OPENAI_*` for hosted embeddings when a provider is picked.
|
||||
- **Content extraction (Tika):** not deployed. Add `CONTENT_EXTRACTION_ENGINE=tika` + `TIKA_SERVER_URL` once a Tika service is available if PDF/Docx ingestion is required.
|
||||
- **SMTP:** Open WebUI has no built-in mailer; not wired.
|
||||
- **HA:** single replica with `Recreate` strategy; Redis websocket manager already in place, so scaling to N replicas only requires bumping `spec.replicas` (Postgres/Redis remain single-instance).
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-app-secrets
|
||||
namespace: openwebui
|
||||
spec:
|
||||
encryptedData:
|
||||
DATABASE_URL: AgBZ3lRcF8KC9OApWa7g3oUoJG8DT8KG0/mgN6O1VwfnC5JZ9V5Lsp5sFtvYg5OXkFW4iBhzFu67n4zAR34EXwCSs977nt2uH7ShgZdrQSOlBTyaKO33ixOcY6WtGQVyJCzl0vjVHBKLwbsVkqEhBDLtqey1CZ9nAW/RZh8MahMSHhRNgJ4XtTyNK/5z2nmJyjW1oVwZab/cWeChlQUtJgt8qWxiQenlgu319P9plXrBDAPmDN/UKDbwu0ivH1s3sBQikuBYpCc9EJS+7NXBNnphsyJ90cF1oyqA5zt+0Q00fZnqE0qLdSOYbVCW058lK7NkeyecxeQwpFgPZ2XaLmU1rIl1wayYpYWCjZ9sLsImnIpFRz+0ouoWoKCXnJvgQ0inGjO5tCmMfHt0H6xTm/Etb3455raEujMcYbRD+mnC2UFmLHshuEb+X5JVUx+wfabVIFps8Vmf34OOf9yODvM127e8DdBtXQ5LAw9JP43tHT38+5XK9koCJIUtv0OQcsp9AAN5Y/BIPEloyt9iFWSQC/vNufYprzdeRGo5DkRvaJOij+s/nw3irOFw1oMF+V9+xTOfsgbxgn+gKJcNSyjHL2Z0gtm8oMLWHIxbjZHxpcY427cvqxITkb1am0QgNDLsTpO+Ie76qGtDiDdEZfgLNeGXaWbJEvIO6L71STj5RmdqSptMxquNIZkhGRz5XRaTupqcmyXAuIUCLIdejXMLBLXNjkOIPut6p8S3keGj3PdIO3AUFnLlpZt+idWMpn7sk07HKSj+qs3DdZcOpkHL0EH5A/qiNVXi9M1NoQgDqvEzF2KMYkvRyJJGwkyg+4snhb+rkV+ioQA=
|
||||
PGVECTOR_DB_URL: AgAF2jka+S+kBgNuEGKupNdThqJesdGRTtcgo760wCMOza4yXJ/ivHMr8onLmykeZLV87Yv3rBKEYvssUciT/RD7QxCVEW31aKKPdi+MHg24/Fi3zXwQKoop05ksXizGVxWia5vrtG93j/XxWKi7JEb9glHe0Taj6TX0Hl+JsEBJELSTPrdngeOKzkYRFDvHP43AadUhvOFwxFoidbLGLulU9t3AdPpXb/T0wLNOcukZFpQxaB36MIIipaJl9leZ1sQ6C0e/MeimUYw3W5+JNf1P2lrL75dH/lqkkJZ0rLetSB7qRmpPyH8dMJumkv+16fkMJMt+LV64KdE2KdqtcxuXFO+RjYkMfzf8lFEtWPNA3qNuPXtwXZAfmLWEcjIJaTXv33Ed+3kAwiHil5Veh1q1eBhGQ0HKvkRGSnxxSyvVfoznLlLvNQVlNLgNfhd407K8ZOutushxaf/NJFW0RFdQcsq69SaQUFpIbv6IGaAM0TRN2O+qNatP3709U6dYQPYdGTJN4LOoKmfIIXaWhiqeYzOiP01QMhML0GvUVbwi3b3fK8twDx5O85Q8QSUgEOYkKnarSciUE/XYhQKi+E7Czv6vGbKmB3G5Oo9Bw6ONDvxd29kpkO2Ko9zwSFaqUooONy3PzkIwPiQ7Vp/ygsoFaVJNU1vsUWtDT8wJMVNcLPLCnd4ERhsMk/r+gY0SXZe3Slo4KIrqoAInf0Y1RBvKZ+XLQks1HxHmarA7uRe3Px/lMXm1kFRtlLwsNxwMUU5u65CbgehGphqq8UZRpByT8dFGhfnJK+rN4ORRX0Fd3fxgxtxyb5VgCe40rgQrH9LtdYculZDpRVc=
|
||||
REDIS_URL: AgBuATUR1mMRwov35kA8dxOUqgl8O480/qxvsbBj3vt51Zo1s5cAzWBG0LlDLtAVh2NEAowSn+egj0mmLsAXJrndreQfAw+UpHDVHi4DbNbPeg7IUCivw/rMjLHyTMO4zsxqreKspwSNE38JU9vzgDBqv62HnQB7hzeGgpwUpsrWXiHNBji8xK/WJtIAGikbPEEzccDC0aCX9Ut/+Yb9ynv1AOtq/xMFIhsWTQ5r2vmbuMS/f59MNWMkWCZBApJL510+QUGzKXC/m553Mk9wHF4G+wX8dd2T4dCQxZ+A68gonvVeFiEYkSHzd5c7s+RYoSBGABrhJ7SziQOG5KWQSbXjjlu2o7JPmsxbgj9tpS0ErmCF/wzgEzd89DjtVNqJKV08nT8X4x3yb2L7oDtFPvEUZ9Ro+IpXYIR/ZnpXUcUgzIoaMJR9RsWBCrAKMFRlAtRsxWnHF0Mx+P6Di92mn2zyJ729FiB8QH6V2lmmko3xRhEZ9l7houxeoTG+37APzpD7U34VaO0W92ybvpcjfrL5BHZEXhpFqE3DXOXD21rf6fmxdOkNqJVJ45CXmweCbKzS3L/0l+GsTNrb8EZALj3RThcNMDgBTzpeTt4iHsodDXb3ziaVufbtw9BVCCrVNb01e2hemAgor5Z/TIkOxK7bLQhNTrpwvs2r7skhOUhzRNuusFi4tz1oFX0JO1Rj96X7Q37fQn5/fKkG6FeOtTIlLJ0mM8FoNijGiRkVgoU=
|
||||
WEBSOCKET_REDIS_URL: AgBjDV7t/UTomB4Ml+7iX5pL1menP1c5kPZOSMlTBPaee6tXnPa7cdEs9/vu7PxfFtP03BczSoobXy4Q/4wvLe4sRAUHRQrcEk+hkJ7PbDT5n/K5zayaqlQJm5MQukytYe5EhlinsjR3EAphq5oCJB8NUAtdC0S+KQHckbxhUa96V7YEeHGu2U1oUJrZCCfQgdfHbFKY8FKm3lEd//uNxmmNovWcC+N4fwWmwj2Z9I68oNK3/H4E0GZwnL/1xxEDgaty68xB6wa7Cg3ZfeX9NLqZk0Xz5cyTziAPQ9rPwo/sShBnplDd7JZRR1podSASUEMhr904LRpbwHv+bfoEt6hUlq3g7gilF4gBVuMM0AwjWkdepFjLGSW++N7C16rei8l1uFthvVvMwdExd9dI8y2h+8ghkWfqwnFuomIxY9NgyAW4ijW7GmVfhKsnD0K9khUe8eiJtR4aqyuQWYsB7QkTMjuFPGZLe3qEg2q3Cl3h7H9tlQxUPqfvaN1Sb56rzQJV3dEMl55SAhoHrPzCjaTibZsuSkRW9djqEyc8lJ/PfTtln7aI0CG9h17WCc8cIAHHIhRIcGpNednIfmkDZjtrKVh415rdMMHZsX5W0/VRnKIkZK/nl0G2HDHw3MJVKaIk05mp0nPb7IFjE3i3t8p/jRgEhgdtIh/s0wma8OijzaXIbnKb8cpQ+AVYAh2TfEzU1bZuWRYzUck/lAzJ4xR4e/yWpsl9L/0tFN+TWco=
|
||||
WEBUI_SECRET_KEY: AgBqvo0l+vmqUBvOtln8zf7UVDaVM739x2/xY3lCf/i1/GlmCK/3aGDLj6tqODFSXuKTKxMS3liFVDleR/7EdYvu9/7H0ThSfO2xhheLzXdooxpIGHnfriKL58UpyWcieo3Y6VIOAQzrSyjBSd6S3GHxjtBdqyqvuGSUXkojf+jOjOKC2yOf+0qySklm/LAww8MigPrM55rlUcqRwb0D/KdpAIxw/wplkAMdFz0RGGfF2Xr03K0AKqJ2egUb9exI0JDUyGQBDJv9isVQPr+2xZHvoSb1QcPCExL5FA4YLN/EhwKhlp56itlrGT44aT3Cx6e8k8cEeqUyP0/4Fjz6xZnQLnTt1dvFAmEAusT9riqTkcAqhHR87MmkPIYmlJLkud1aT8G3LpXUmghrQgYaam5G4DKbQbQ1SBJ5eNC19VNW7hcL5zOvdB+Vm41/OzM4n2AW+Oa0UjFMgWO27IVIGVNjwn2VKapUWsRHLkidGfeVQIYyWznXRFPiBhSwvHhO767Bt1TASst3R3f3bZDcjASLr+0VfRm0MbJ0iXZkK8IrQk0un8zHEZRwCWk5Jl/LM0AnS5NeRgO9XAPlXm8xv5Xy8D2DKsIZGMkXZWeoEJ57/l2i/mn9JMWhq+va5+jyHKQwrqAH9ZqZpWvmip+LeIrl/AWiKGSzarhSIw3DLin+HTJs0rNswVdU4FgA7CDi1Dg1rTW1bbDMrebSB8KNLYqrWlokW4MQ2d9FuzrN+UhMYKbiq/TOS26/oBlWvMtdyO2qVTHjWJPWUDpYny1yAVHJ
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-app-secrets
|
||||
namespace: openwebui
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: openwebui
|
||||
labels:
|
||||
name: openwebui
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-oidc
|
||||
namespace: openwebui
|
||||
spec:
|
||||
encryptedData:
|
||||
client-id: AgBPXc6HeT9gaqtXK9W/Bo0Jgt7UA/2Aygb7bHmwXSJHjv00Ja/ciISsmHQD7fovmG6eoM/fi3fRdJZqJkekRX+3Od+09I3BYQGzXjNwBkplydPLPEuzqDcCLdRTqZCpDFIz3If08++23hyeH233Vp6BwDxcZXunqzzLMNY+i907QhGm0ULb+T54zmZ49NBElXv8/moD9cag807ySCDw+I5pcWQMIZRv1FKtso8aeA/KA8Mzgfmbhxe+3HpPZ5svuOhFlzHED8bZ39c1l8cakjPDuRvrrIXzwogNhuIUArmXIyBhgq9G5W0yTsvqrsep1gMWEtYnZSNgVkK3vzodH2oOwstsOMgKThd1c5bnIPbN3bkLxwd/TbyHmQdLSCMNwOCVo8XGbeRhSABMxVacLfDI7hR+WIyRdkiMwKKQnQMqdBynzcWWXeSlnzIBpS3N+oodO7gS+l+6G6KX2ZXyUyRlxdDo7m9BJhc+HhxJ66L8vVj1P8QKczrixWJUqrUIFxM3jKY1A7XOI/x20/gQeodCJWIHetKvIHwwQh+w63mwLP8gRFglBnjNFok2zeYNBo3eSv7zhXEYve0y83al86/TrLggSfI7juJmLcy3bCy98ZExXdf818JslqmDutavUkUeYn8wIvdkeZbRtWgYLCu7izJZ1UDN8zkP6VQuhzLHO2OrhUuKnwpiI7Jp4/gIrwuGI9Ahal1OX+8T6R8JrAd4lfz5MfyoNzFZ7qrp7xe5zlTZZxE=
|
||||
client-secret: AgABqIXEErUfaN5XW7SxGr/yJgs4GoPufPmNZF4d34STcCi7s+kE181h985ubaL12mwt6zNTDxUlGdz5p7ABLePRf8gEMUtzFNom6BOqIiJjhQuWLJRR1//DIfu+9G1KjmSFKS4iLkoHV7bh2+b+q+u3NXFVfClV8bVMICN1Nzp+doSdwTAh2CyQTK5JXmu6jXDJxuOkJClQdEH+rwdE5/It8ZHhC3y568m5H1JO4dxgvBvZv7jLxX8b+hGGLD8N9P6zMRwiKUlup+UPCv1V/SjppAP5kLmPKumqhtWqEY0j4XIu6xsIaZz5WeIAzlpQoRLshd1VGk9ebfXN5SfXcJdHyZlH5zy2W9HL/iBmr5kv2RkzpH2Sludewk2pWPatXtEcE4N1sat1cR2TnfW4wxJySZ2ni2d7H5jPFNu6Md7MFVVneYhanSdt++CtWol5bikEkzuSk2kjPT7Bf1wRhkz68VLo3Faf6ltktvpHV6eGfAegW8HgTSYjzwA55MdqWNrLlN/iZfcfhR6+8A2xblpUFXBgvCkTzLQzM2d7gK9mr1j+6osWpUDzD+Os8NSsnPBM4ywm2xfW5ZSEHdX2TmF9s7hDLCffQfd+ILSre/uyxNXuSNesE4pC0bFF9OndE2IvIQkuMPy306NnCEBzZsp+P8J0NHIr7EKWg+IMFRsYMQrIl5Gv+opyTssr9fTMAUN9ns0LdQPxBnjce+0egHV6g27zg3QVIc6K6vsZNciYyQ==
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-oidc
|
||||
namespace: openwebui
|
||||
|
|
@ -1,279 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: openwebui-data
|
||||
namespace: openwebui
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: openwebui
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: openwebui
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: openwebui
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:16-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h openwebui-postgres -p 5432 -U "$POSTGRES_USER"; do
|
||||
echo "Waiting for openwebui-postgres..."; sleep 2;
|
||||
done
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-postgres-credentials
|
||||
key: POSTGRES_USER
|
||||
- name: wait-for-redis
|
||||
image: redis:7-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until redis-cli -h openwebui-redis -p 6379 ping | grep -q PONG; do
|
||||
echo "Waiting for openwebui-redis..."; sleep 2;
|
||||
done
|
||||
containers:
|
||||
- name: openwebui
|
||||
image: ghcr.io/open-webui/open-webui:v0.11.3
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
# Core
|
||||
- name: PORT
|
||||
value: "8080"
|
||||
- name: WEBUI_URL
|
||||
value: "https://openwebui.basicstack.de"
|
||||
- name: WEBUI_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-app-secrets
|
||||
key: WEBUI_SECRET_KEY
|
||||
- name: ENV
|
||||
value: "prod"
|
||||
- name: WEBUI_AUTH
|
||||
value: "true"
|
||||
- name: ENABLE_SIGNUP
|
||||
value: "false"
|
||||
- name: ENABLE_LOGIN_FORM
|
||||
value: "false"
|
||||
# Database (Postgres)
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-app-secrets
|
||||
key: DATABASE_URL
|
||||
# Vector DB (pgvector)
|
||||
- name: VECTOR_DB
|
||||
value: "pgvector"
|
||||
- name: PGVECTOR_DB_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-app-secrets
|
||||
key: PGVECTOR_DB_URL
|
||||
- name: PGVECTOR_CREATE_EXTENSION
|
||||
value: "true"
|
||||
# Redis (main + websocket manager)
|
||||
- name: REDIS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-app-secrets
|
||||
key: REDIS_URL
|
||||
- name: ENABLE_WEBSOCKET_SUPPORT
|
||||
value: "true"
|
||||
- name: WEBSOCKET_MANAGER
|
||||
value: "redis"
|
||||
- name: WEBSOCKET_REDIS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-app-secrets
|
||||
key: WEBSOCKET_REDIS_URL
|
||||
# Object Storage (Hetzner S3-compatible)
|
||||
- name: STORAGE_PROVIDER
|
||||
value: "s3"
|
||||
- name: S3_BUCKET_NAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-s3
|
||||
key: bucket
|
||||
- name: S3_REGION_NAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-s3
|
||||
key: region
|
||||
- name: S3_ENDPOINT_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-s3
|
||||
key: endpoint
|
||||
- name: S3_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-s3
|
||||
key: access-key
|
||||
- name: S3_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-s3
|
||||
key: secret-key
|
||||
- name: S3_ADDRESSING_STYLE
|
||||
value: "path"
|
||||
- name: S3_USE_ACCELERATE_ENDPOINT
|
||||
value: "false"
|
||||
# OIDC (Pocket ID)
|
||||
- name: ENABLE_OAUTH_SIGNUP
|
||||
value: "true"
|
||||
- name: OAUTH_MERGE_ACCOUNTS_BY_EMAIL
|
||||
value: "true"
|
||||
- name: OAUTH_UPDATE_PICTURE_ON_LOGIN
|
||||
value: "true"
|
||||
- name: DEFAULT_USER_ROLE
|
||||
value: "user"
|
||||
- name: OAUTH_PROVIDER_NAME
|
||||
value: "Pocket ID"
|
||||
- name: OPENID_PROVIDER_URL
|
||||
value: "https://auth.basicstack.de/.well-known/openid-configuration"
|
||||
- name: OAUTH_SCOPES
|
||||
value: "openid email profile groups"
|
||||
- name: OAUTH_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-oidc
|
||||
key: client-id
|
||||
- name: OAUTH_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: openwebui-oidc
|
||||
key: client-secret
|
||||
- name: ENABLE_OAUTH_ROLE_MANAGEMENT
|
||||
value: "true"
|
||||
- name: ENABLE_OAUTH_GROUP_MANAGEMENT
|
||||
value: "true"
|
||||
- name: ENABLE_OAUTH_GROUP_CREATION
|
||||
value: "true"
|
||||
- name: OAUTH_ROLES_CLAIM
|
||||
value: "groups"
|
||||
- name: OAUTH_ALLOWED_ROLES
|
||||
value: "openwebui_users,openwebui_admins"
|
||||
- name: OAUTH_ADMIN_ROLES
|
||||
value: "openwebui_admins"
|
||||
- name: OAUTH_GROUPS_CLAIM
|
||||
value: "groups"
|
||||
# Runtime
|
||||
- name: UVICORN_WORKERS
|
||||
value: "1"
|
||||
volumeMounts:
|
||||
- name: openwebui-data
|
||||
mountPath: /app/backend/data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 4Gi
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 60
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health/db
|
||||
port: 8080
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
volumes:
|
||||
- name: openwebui-data
|
||||
persistentVolumeClaim:
|
||||
claimName: openwebui-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: openwebui
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: openwebui
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: openwebui
|
||||
namespace: openwebui
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- openwebui.basicstack.de
|
||||
secretName: openwebui-tls
|
||||
rules:
|
||||
- host: openwebui.basicstack.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: openwebui
|
||||
port:
|
||||
number: 80
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-postgres-credentials
|
||||
namespace: openwebui
|
||||
spec:
|
||||
encryptedData:
|
||||
POSTGRES_DB: AgC27bglkH6N5Ule5F23XbirvuqWgiQlwd8yyqEqpcZYsvT8605y70g2mS4zZBtvVqdYFcORJF5abdqWdVhmPhZAV71tesJBTOynMQb9Zd39EbBcEb/6Shresa7FdaLQgaQGLxIOOt1xWLmh/eRv9bLiIGNtz1ezKCa9fXYuCrD1Eg0EGmUjdioXbkuVoVNON5rBwUTWWpfVq7/d/UNQGUvAuhOX2RORD4hy/5IRZG5LvAF7C9oK1LaCbi5B65kbC9DOtlCPfB499tt8fSQDYic8YBLkmPKQgHd3+8NlNrH+TtTZMTO6UOR53IqJW/xxoJZapAl/40XkFj7Q59X+xDzG+p81DfwwOu9U+Fu5a2zpdduCAVQCIRSVZBL/ziYkRjFR08/5ToqHF9+p0C6khIgt1EkyvyUTxZfKPlcq68soY9cl4UzV3UPe6DyyEXwKsPmaUXPG4ckGEAWIIIoEd4M2cFN6jyWTRyYiWfe9uaPSIkjBGi8TIXk+Tk2ZTcZG5UQd48MOtc9gnXQwoFo2JzAqqBYa4HcATzPPiDzQCKhMXjGelINzExqvjOm26L8Zi5yCsaAoP3Ev/Kedl3ohdWW7CpwTxcffnd+CTYo5oz9K8HD0CvS2gc46HL2zCzZSF8Zpap/0bsM4FWFcEEI8ru9mdkhXZ9EygI97fcxtAi5eDtYm2UjKRDzCLLb3a+eorUEKfpXPUTG5E8A=
|
||||
POSTGRES_PASSWORD: AgAislLVvgaIxtm+U9LzD1lK0Rz05MaiphMWHuHVpGayAqTorWy7jz+iSI4aQH7QQFzClv6WbSgzcwSjYiTZhH9+GxpRHpIAcyYbjHVG8qU3n99A8WXXCd5V8K4alQK4z5kA6OxZGOuYRQ2mfAbHGdAUBNIVTqNv7DtpNuATKzZbkLd/tMX3ezIcpJqOEtb0AqgMvcH0gsw2j7I04WBlCdH7qQ6e8EUqHjxWBc1YMF7INzAxlkVgIcaNkpfcErEL4y7X1E1yo0KRzAg0O18rzUqs+hOi1XVSNaEuA0yno+a1ZXzGnmiwJFK+BboL3BZUtev6Xlbe3DqcCwUjmw3v18suO84gXGfLEJa/OYsys1vgHQ4qt0tP9guSWaqnTfdkc2ZVilw7M+d2TS5PtQC0U2/MW+SNmS8EdC+/q6FvTgaMr2uptqgC6hrrLFemQ1KFzJibut1s5o7HuLKqD915gSbHTPJ0vtPOstIJo11ggmvLw37JGqhbHkME7SwkJzZXWAFPvAWfsJTRqL8fzdk1RLlVYQeak7VnX/iL5q9bUomsGvYXMV90/DH3vdkhFuxOUx1kdjnROtx8PYg3rnzXPvfR49+x4EhUWRqaF1XDXWdWhCKrZFrSskvcm7h01gHvLQBhbKK/X0nOz6CM/Ht8zzIsnsmjssoZaz7iFrvYYyhSWyp/U/N9GdFkcDA6qzRsZSM8WGnRnEeQIHklzIL1fZlvsPl57rFjHSYZzdXD1Iolzn9v8KU4vt7kGxOiPh6Abfg=
|
||||
POSTGRES_USER: AgC5pjM5LU5zyzqvfsrf1u5koa/IwO0wTnDUv4GcEzRLKD/d0Z5iDlT6bdyCS1bFr159NWUyz0FyqfMDqsGFzJiGwEd6Bq2PHaaa+h5wfhY/KguVPndb2SCRJWDcxddH85GGRo6iTkVLeNBO5mOI3imQnDNyKaBpowRINoLNnWDoTEZ7COrTrLTWBlJz8ar4JyIWAToS08cBIhQqIBCWJkUpXSS52Frju3yYKBCeR0lDBrL97WghqnM7cD38luPXajLZeysck8iAwZm2Wt57ZmLUCLSo9cxkhL7Wx/mV2A/s43KzL0m7w+YkQsk6x8anug7+hr2lKAzb58ytKkyPsJ/OirGB9fRfIujEyU6B75AFDePtvv5W6Nf3ngWn4hH25K9dDA+LyMKo0KVXQi5uMnVFitwMUMRFLFlnyreydt/aQyG/G+XjE1TZtBgjF2OHdPKSaxqOh9F2vZ8GLRNV70EVVOO54/4lqeLChNRBbuxgn5v1MtD8lOUMtoS4Ho+82995MARCzZeukfP61tqMbKvTn9HtbbQq7jBfoWCUgCY4KeiQ9sf5+Ks9jfmgDvdgBckiO0IJCq0oaiVvUGW5fkLwa273sypVo6sglLoa1I+YCkUJ+Ym7nOw51r+zATrNmzzAG8W04ynz5Jx6iF6mS0K29giqAx0POxFpw72CPI67COYLQdXr4b/UoSAlWWwCH0eGCJUfcA/LYIE=
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-postgres-credentials
|
||||
namespace: openwebui
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: openwebui-postgres
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui-postgres
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: openwebui-postgres
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: openwebui-postgres
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui-postgres
|
||||
spec:
|
||||
serviceName: openwebui-postgres
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: openwebui-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: openwebui-postgres
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: postgres
|
||||
image: pgvector/pgvector:pg16
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: openwebui-postgres-credentials
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: postgres-data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
subPath: pgdata
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: postgres-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: openwebui-redis
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui-redis
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: openwebui-redis
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: openwebui-redis
|
||||
namespace: openwebui
|
||||
labels:
|
||||
app: openwebui-redis
|
||||
spec:
|
||||
serviceName: openwebui-redis
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: openwebui-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: openwebui-redis
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --save
|
||||
- "60"
|
||||
- "1000"
|
||||
ports:
|
||||
- name: redis
|
||||
containerPort: 6379
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: redis-data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: redis-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-s3
|
||||
namespace: openwebui
|
||||
spec:
|
||||
encryptedData:
|
||||
access-key: AgDBruwsmBhUuqOg34A8jWTT+9Ko9K4erxigpVtcQRxfvoBu0Jn9O/W80XDPO4+HD6krMlxZyPb0uP/qwsLU99Ll2gcxsUDVOvhJwkqzI1ohZbl+iueJ7qG7Ty5HO8CKCe8AiDgXnEW5/hWUxYT+0lX23aIv3SaJdhOHaScMCw5DSGlR4/JA43uEfK+Rbx+FDYokw17pi60LAhuQhZpbzoy0sgd4A5nKaSyNRis/YPnIZzN5gb8yyuGYa5fHSLGnowsuAlfj4x1MzNsw8pzuygnti6KsoMJyWnqW1TsaWb8WkhWDRKEN2AO4tTD+2BuvixV6LBynomkLfwZ8DSmN0Di5Db3bKGEkECiRAloeIc4efr5L5W4ucAxPE1YFOM7VINt5k/7pdymQapGRyQymzEsC6dWZNl54SjN65zzZIerkM2N2II0oqyzhtYW9WZ1ihPYvXSSp2JoaiKny1ljzqyXeizX/Xkl4ZjUZETYV8KIIFMx3SwTGBvShGwmv3T/v7XnMDxG7lIvjtHvCmcaiW3fegz3reckuczuG/c4rtKQo+bzPnkWznKlXVrHNg7Hks9DisIQPngn/u1bJPK8ncEYaX0MRGIIqb5yn8eoT9Ggxk9eY7QHym9Y/bjFMbR5MMR7I5fSQSP2cUI1K6OQ8eqkteCUB2i0wCfDXHqnkE44NJM38hWUl8b3rNH2yo2gnLyYhkaeqdetqNbW992quRUp7UXshrA==
|
||||
bucket: AgAm7zODlaJ10GoHumXsgXJC4An3GNegi/Gm3aNhKlyNV3ItVU8q43/UPMck3ed14XZ1gbMUIkkIU8L9IIIofwEam4YrfQYfoQomtaYF2EHnRjH56+xrnRP+WKzTbqDpiZIcRfr6E1+ajBM70H7A0TfztLiPmFNJaGv8YPLSdWh8E5m9vgiJJ4ErEDwMxMrhLpfjZ6iNp/H9vchPLOtSP3CtAJvrrdWO8My1xZX3bnMpoU9YLlSSfPAGpglZ5dsxeRBA+HSSRo4PouhLKMm7PULk4eFFwtetNgPh0DwHXhD7hboRcZJECmZMPz2xdWwu8+KqKmMywTaN+gy6IDOw7pZV0hHwVd2AWhlE5dXnV5pVEm0vUKbC5ceS4Swq27RphO5g5/7lfnkV1I9JqTPETFXCydxQ+vhW8UxbEunTWp7PxE/XCNMvYBvecksWEmLix3iVFGj2chH2tnDLx/m/K2HxQkqNo25c1PnDO3aL5WTSjcK9VWSwrv4ucoROtDA4MiD33FCSF4eBwzUm6AUPuWIIsoobjGOKZcwNiRDDtom6O52h6+0/xdplrNdj5X/5GHe7jnEM/IwaYdNsYBu0/Hi9eohA453brJfGzShe9mcKw0TWktq9xrmfk57PuWAKny6dg6VWixS0YHNPd7v1EkANmnm1nagNMcurScolwgY/MXYEbtQbbHPk5Zn/yQGrDCFoHSkE83gC7go9dTvnmEi4anDfYQ==
|
||||
endpoint: AgAlRlwvCwNLdPXgWgpSYTz7MhtX+wsK9TNCep0C6oGJi6HMlEzT8s1YW+58HTGmc5wFkoB8P48JDwcfdb7BjAgFMVgkiBz3WrbtYYgUda8EeiD+5Kt04lTcetPibi/R/sbe1Hvxp2SIsJ4Qgt6x6qElR0GM3hfiavP95mazIL6zqX0MImXtKFEqGBwhSiFIQ2w0ldbbBxpl2B0w9JpOO0f9TxEZW4do4JyKkTSfUnBpJdpN1tqDP+j18PV/H7s7O4zdpVWmwlp8l96xSVZtg0VZvVvdC201mPuAKF7EV4bIy6/a6jIgKeUv0k0aeZqqTgeNkGmCzxI43ujShMxVEuNaDDVt1PAzcTl2r8IImyJlf/B3AipgyGpwXUzn2+A0DiBIqjOXC1dezkezS0+iq5yekO20d3N73KswIsWJu25Dvc5j+c97nPSQmOYGcoYtZmoIac2GNf+UTApbLiJdzSaP0hQZ4c0p6g+xtCMjesLl1xI1ihMOJtetW/mDsKokGWfACWcNdkN4mKbgBHtOS8GvfK13NXpDRDfgl95avhVgj0hdGANre3fSk7cmEX4fcFCyyj3wUqHOBK98oNL8zaNVmHjEDoajw5RI6EwHOM1lnnUfMyEuJT+Ye6vRCmJTLoMrVqCOUkcX0rfIA7+VRJjSoeeoz7+iFJcsNGH6HIdE6rjOH9HdC4v/L5wNH5CUbiPLG8tFcFiFF2k9EUg6V2YymkrgCM8WZb2PN0QQloohe1VkRQ==
|
||||
region: AgBWLZOWPXCo8H5D9fKTqK/UWSOGsB32Swr0H3mhoOvZntLkqtxR0WsToH1hYzLnsVz41O5ntBRiw4nx00SvKs8MdKT3rW2gcuJcJMznUMXN8WoWLfUIrRHO5xRR4bh3BDeqaJo80iGL0Ql1/460BYwf3F5mBnKDEBdbaJCNKraRGLUVSOJlFFkebzwTsgOszQgOOZFFA3oRUNgZugHFzeOq9HYYqf+XQ1WSA5AyKjgR2xYK55vOiw/qag1keX31kWGgB039M5IcNhdI1kVUrQWgBeDWU4zp30983FFBaxJENKqTXEhtkfJGgi33+kPuMGdz7KQfNShd6bQNVpfGeX9ZsTPY+ZIwlsiljQzuJ2FTu84Xnmf3jNsoDuOnq4BmpYHV7+t9iGPFmiiTHCYN5hTlNWptLvBa0ZYE1dOwEz41ZEEisj8RAMWBNBiHZDBPyVEvWjkmld0Lro71F6XGb50jlaQlOV31b3zVksatzgnkQmtNfeZucHY6AyxIrwlbw+UxTIJ1dfzFBRyrPAdGZamUgyqZLqXWkJmwy6ktD/tRJrfpso3QjkPjAlCXYWNIp34Qx7GHAf4t8jaCJx+qjlpSuQ4iv28xFL5xug0I02cfvZwV441ANVP5i4ATnVpJ/TIb1zbUtl5ffuGrD5fY58DQ74ntsHSQ8h8JIpQDAu7WL+ZSWjGpV5tBUz3vIP4wsDJnHN7p
|
||||
secret-key: AgAHkBSjTxORjuS5i75zd5Sk3FytIYwfQTgRs64LUHl2SAD86znxUh+4T+AGCO6dZHoKp+madkx84ghUn31LdA/2FRfXrrujxHLlom4LKHkv/jDc4hSRtJPMBqpi0KUXUtXk83nJ1s1BN5er93S60IiaYZOBcS0XojTfh+TbwlKJJsKstJcOB1PNVb0V4/OC+fSWpOJNkSI5PAEkzVVcSFsEaIuIRpAazwm7hBbnvWCGwd/WlSzZEHNVnf/fpI2QstVYGe42fg0NzEqnFk/I5QGRcKhUvM3wQ5smNidb2mors8FE0UzdvFcP4cytYb7tDf0GKclX4p2kmc7Js7EWTRmin1N31EVdGPGsJ2sV5u5JQoAg0b381H4ZqQckInB8NIoAx5L2p1gqnPKPt2yTV192jes7COzNhmODTVaSKY0z3v//u6tVpsanfiDvkSl4y47BKeJfxp4Ihd4wJNgN3vkaxi4mDYBiNiY2QXSczZ+2qyacZeGowl1ZpGH13KSg6bOV/hQ/T0tr0lbwiZ04V7v5pORAw8SnvHT6rqzHLrE6PxDiLQsY36bIpMvqPLNke0ErEfqXMwGeYClij0fpU/7qZ5t9Ag2PvClroxY3wuLdpKE9ffeHtH50BgyabOS3DsMXcB1pODyXf2QaEzB5NaGqMp/PEbmoy3JLYwM9We5NCQObxfjEI6gkgBitBY1PDBbU6DGfTM3Smf8y6qJ6Y4kHx31VUU2eVHP7+O912OH9yzguyq1DpkiI
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: openwebui-s3
|
||||
namespace: openwebui
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# Outline (outline.basicstack.de)
|
||||
|
||||
Self-hosted knowledge base — [Outline](https://www.getoutline.com/).
|
||||
|
||||
## Stack
|
||||
|
||||
- **App:** `outlinewiki/outline:1.10.0` (Deployment, 1 replica, `Recreate` strategy)
|
||||
- **Database:** Postgres 16 (StatefulSet, PVC 10Gi on `hcloud-volumes-encrypted`)
|
||||
- **Cache/queue:** Redis 7 (StatefulSet, PVC 2Gi on `hcloud-volumes-encrypted`)
|
||||
- **Object storage:** Hetzner Object Storage bucket `outline-basicstack` (region `hel1`)
|
||||
- **Auth:** Pocket ID OIDC (`Outline` client, restricted to `outline_admins` / `outline_members`)
|
||||
- **Ingress:** Traefik, TLS via `letsencrypt-prod` (DNS-01)
|
||||
- **Node placement:** stateful pods pinned to Hetzner location `fsn1` (matches CSI-encrypted PVCs)
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
| ---- | ------- |
|
||||
| `namespace.yaml` | Namespace declaration |
|
||||
| `postgres.yaml` | Postgres StatefulSet + Service (fsn1-pinned) |
|
||||
| `redis.yaml` | Redis StatefulSet + Service (fsn1-pinned) |
|
||||
| `outline.yaml` | App Deployment + Service + Ingress (+ PVC for local cache) |
|
||||
| `postgres-credentials-sealed.yaml` | Sealed Postgres user/db/password |
|
||||
| `app-secrets-sealed.yaml` | Sealed `SECRET_KEY`, `UTILS_SECRET`, `DATABASE_URL`, `REDIS_URL` |
|
||||
| `oidc-sealed.yaml` | Sealed OIDC client id + secret |
|
||||
| `s3-sealed.yaml` | Sealed S3 access key, secret, endpoint, region, bucket |
|
||||
|
||||
## Deployment references
|
||||
|
||||
- Argo Application: `apps/app-outline.yaml`
|
||||
- Pocket ID OIDC client id: `b4a84bbd-fae4-445e-868e-5c1197e5d155` (PKCE **disabled** — Outline's OIDC plugin does not send `code_challenge`; group restriction on, allowed groups: `outline_admins`, `outline_members`)
|
||||
- Pocket ID groups: `outline_admins` (`2b41fc67-eb7b-43d8-8954-85426214f1a8`), `outline_members` (`aa6df56f-b868-4de1-adf7-8696e990aa53`)
|
||||
- DNS: `outline.basicstack.de` A → `138.199.128.63` (cluster ingress)
|
||||
- S3 endpoint: `https://hel1.your-objectstorage.com` bucket `outline-basicstack` (CORS whitelisted to `https://outline.basicstack.de`)
|
||||
|
||||
## Deferred items
|
||||
|
||||
- **SMTP:** Outline runs without email notifications. To enable, create/reuse a Stalwart mailbox (e.g. `outline@basicstack.de`) and add a sealed `outline-smtp` secret with `smtp-host`, `smtp-port`, `smtp-username`, `smtp-password`, then extend `outline.yaml` with the `SMTP_*` env vars.
|
||||
- **Admin promotion:** first Pocket ID user in `outline_admins` becomes the initial team owner via first-login. If the admin role does not stick automatically, promote in Settings → Members after login.
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-app-secrets
|
||||
namespace: outline
|
||||
spec:
|
||||
encryptedData:
|
||||
DATABASE_URL: AgA0XHjlu1iWM7AJaStmEtMdVtYMwKzXxk7/nMCtIaNxaXTa7XH3K40277Wc5a/mfM4KLiF6iLExquNIa7YI6R3EROpUC23cTJbstpQNP6tffzZwr3FeTeod4qWYJ3HfIHaBlocNc0p1ntM9H/9k+Vy3rQCn9T5kLJ1c6LqUGhPinWwQxwiDgNU6JQyEfILIAX35th5fe24GH4Ka10k56wq5Fnx9CkYYxsVQ21NNKCdtqZecR+C2RqfYhXharvtio1ojWP4KNQZuXEXGDAH/nb7QPtaNPCV8hhsS66FAjm3hHTOmhJ4XXbJQpWYCiPOWBa9MVHxMg3+IDfdR2OWLDOpgFzN/vMKx+iqa3aVHXfdrXcFUqEC48d0h3MXjdCge4c8wY469PUTar+74d4BcljjG6R1FL5CJ1Wmxygt6rG2Qk+pWihZ00Frc0gttsxowS7c9s8iPL6VF1ShyaTXdYq0W7pHlX/Yq927C7h8E6TNQ+Rxw3UI4qPXN7xkU85FTQvWZjqoyyej5GaPbGy7Qp/SO52wsq4Fh5qPrZKoGl5xQVcmuF4LZKbb8WQDQcFDatHdWZiHZqsX1wKxRP0jR8RkT3Q7Z/M2itMN9lODtddCv/hjrwd1ih950kaZoVW3cdZCVb+S/M4pbkuB0BAasRbntJx/hOy/9zp21MAWfTGTyscBetIjHeC/oy4S1WQyo5TwkMSeugqXJ/mn/zTEpDYrxuqso7Aeu9H658WeenwgUVh2VB9SVWFWxZnHe0rB+keMPDwJ7Dx7NsdydybsRmORtTIc4zj+OsqDhmg0wwCq6pSslp+/h9MwfUB8YkYv9ioPP
|
||||
REDIS_URL: AgCKT7MxyQJV4Yk+I8TO52x8x5nfXd0ECB9H2w8p2gRFMyiTbrTMvdzaF9fqrgTiGRuB3VgDeeQpsnFGvemoYWqbLZ+WI1UfQKlyar5y3tHh88GMoI53dmkaBc6V7Fr353rkcy9NmRJC+g37V5PWasiX5aWFSdYoslmaUOC/o4pQwyDZEwo3xFHysbm9tD5jGVSsCgnUGeiG0Urva8pNrM3aTfvSoVOhts312eKk3+x+8mda7ZchH1DtQ9EZ1V8emEEjOv1Jh6gh5wLjl710cHAS0ch/9RTj8ooCH//pX4V5gj0ue5EWL1vk0xgV+kv5tGG2pwR4aUZWa1d1eQzbFDshHwkbAWKm0pjCq6eIMg6QLQRdagGXe4ZKsRAahUJ+7Qo1MqDXthoJHDEDDhB+/SbbPKTOrgm+eh3oA3EuWDcp+iKAqboBc1WkzUWh7kRVRt4bkVBHkeK3UjFvjGtuwC8cI2yOQ3DyRh7Ija+Fp1o8ust86FKYS1/VUQRqBX2HTBamognfS+EN4PyrHJ1XZzoL85njGm7v6nOG+EQN5QTlRkmLTLqLVWzUk9tuQbpm57r0ULqbkMKYzfnaj6opWo6EsCH2uQg2KGMVgxAhY6jFuFc8cLXdz44Fb7tjTlSgq1Z5S3hqHRrP2fd5upjFICP/ECa+JRb6ZGpTSvIJ2MpH/0oCTFkgl5+A2KUUP7PDNkqCoVhujRLS05XuSize+owTbjeEz7fJCRy66A==
|
||||
SECRET_KEY: AgBYcN97oeke6wlfUel6cki6sYIUyM2zJUuQfBSSFoZRawSurDhL7MboCbVXXtgold8QMp1aF541442TO0DdR1eMvOvNqGHIQog3sNDlbjr3LOk3vV7FcZFoME866tsQ0uzfIR5Z7SBvkD6EqdnrIe+sNfSQHt1Jn3/QCPhgiYbsN4tPD1t7HFAuuPmI/gs91wnv40r7PGxK2cYYcPgGrT1P+prjSsca5sbVvy2e/jkW8FWLoRxrNw4Q7FImgDPP180Adt2V0n+7HfY8u5gSS94oY+5CBmWuR7W6GhStuI/LfKDDVVgWdKtnP90Eq9NiGRg1Xk+EwjNLWSm9aTLZ/2h1jzDGqmrJ75W+Z5OHV2bdgxCFb7lZJRU1mDTCjqMNf1jKEzSQ2pFYHHmyRhz0dQ7FVD7wUS3SSTkC8epxxRf+s8Jb2O67x5sM87V/QPyMGlCPs/634OOwj4+7V8owcc+NaC9iiGTjwd+6AurFCF3O2t2YcYgsJXoDp7Rd0hf/kk3cEKAhKMqO2DDAew9Og3fpJrF2mgYnMJb5fDAaEq7FrA8y7P9a1vyL6nZMmfXZcZ/a9g+r7uoFzoE2wxsaCpmlYvgep/X+a1iQgE2+LwjB+lquYFH+Ztam5HOneY7wNMD8EpLbyUq6RO54+f/1COipeLZxyUc8tI1zMlsWlBKNSl0qljZqA3HAE3a49xQBPTiEHlfsLk9me55zkR2t/H9+HBlJ5LQvGb43sYhVssSfv7mYTAzEOJPiAjUiLfpg47cu+dt+c5x4qenFmqUDh78J
|
||||
UTILS_SECRET: AgCAxEUaZS1TqU9HA2a0Ze1lxsP97DYgyUnWFaVAWc2qSM3sqp4BffN9ZJ8mVUXkZdZXmY5Yoirui/p4z/Ie1BpZBH5dbcuVGWu1DopCcs0UROpftJTWLKLZjoYGxp6RbazUNSVc500KvLuwMX+E0wG7OUSAt+nRN65/ebHRWl4kg1DceFLVQVmies/uHHz1++1jkVTGrr2ClLdmj0z/VYc5dYifxouYUEgjKWzB9ERkso/J5CCjG7hRRTbAeEaRH/SeyV6qw9kvhgCK8Kd0wicJatu8EcbHBDZs+0IiQ6vkqkdQE47MlFqfwem/lLWM9ce9rGBMkJ/rcMTGgW+pP6GKaQVKrhP3+0G6lz7IHneFvEF+W5tEaRjf7VbQdbgoX4lfykjPQ8pk3cIGPWLOrFQTxdl5EOhwS3dQ/WoAQs/GLgDsNUO8TlQziyURGCJT/LIfXvEVZsbOTcrbayd3ywwkL5aGN0TAcnWb0TPXgZVhw91N7Z7e2gRoGx2CoSpZzjv0k3liBrffCRYvvc85HrQnKtd8l0uutpCrtVOStc1wqRA5g0s9j0bgqspvQp7CGfyxs7vnGP6PMBoOtgH8mR14oI41SXTUn30uRWzGR3yebGLRXZ6ORgtzB8lQxDZAo6VUcc4afzyL8RXdhtJNmr/goRXnMg8LQqq80teQxFat9RW4Wxjqpnhb2xNwFtOxYFK818q+L7/7CC9MqaQcnYwn+mJUlMQTam8rPI1v1VRLfdzXcmHgQCXrlaTAMFOBF5zPO2vr3+OES/XS8r4K9V4f
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-app-secrets
|
||||
namespace: outline
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: outline
|
||||
labels:
|
||||
name: outline
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-oidc
|
||||
namespace: outline
|
||||
spec:
|
||||
encryptedData:
|
||||
client-id: AgAvHl9m4WXOYPIFG7rrYzPeA0Ksm2DUs2DS9gJismHLL5fX63e2KaD66W18NIFZQvtkfe8QelvBi3J2YAH2I1qrdZjn0SVJSq/7cdR+0X6lb71/hAkbHUr2xS+uOV86/O+6/htLhdNl0IcAjvyNMXWh5ptVBW/pbTQHJqGYN1dHHDQKri/3XIpxipKIXL22oPnowv0jcW1tgTZPPFkFOpahVlVP9n0pxM/2X3bb/bbsNVuTk0tWeDTqj0XTUN/0Kowq8P4rJFVdbJPyyV/37PnPlu3+F1i/flApo3HKQmoKR09ogCS6uZEP10SO7lmz0+JCMCIxYAHr/ELaynEXWhsCNJsLTE+TtMdMNciy7n8TZ75cS+Azgjhc5cimreimmxuFI1NRuMHnLMCLam1Hve2190sPhcIT+O5Revrc+BKsSXuscDZtrUugOChqYM9EW0T3AwmbIAyBQsY3MoLc2wFuq1fpYMx+Yu/0ajPBZvUV2JRbVt5YtMHoo3VbtKXkbfZW9IaBGcBFDxg7Dd8cZju4ljieRWJm6hISO6OPrMjvGG1IvBTv9c4SEGhnCHbjpYeoB9UPlL/FKVQgyrK1v5qhnZ0ZoiKYecxfj6Jytrf/qLdrBZDlCHghKpn3OM2OeubOak+xH1zgENK1UhjhkuqELoBXy9WDTVkOvrm5Tiw4WAAUm9jmjQiKusdwUGMPBt49sXOfyzsu8iPWIb8FDhH3DLF13d9wH7vtwlGQBuxhC1+FiSI=
|
||||
client-secret: AgAHp4jsX5igXGHwKiQJuOKgyIIOssntobUhHnIKtSrNDK5mBZ0EvvwijvF9bqOp2i542WxVcjabk9a06/h6QCXymmo9Lcq4kK3cZyrQ6V7efjNFhogHOnE/f+y+lZy3AawTd1uokLiURf9z2X9rxi+2ppLhqExNFa7W3WIuEoVnk55H6nMMOxBK3SFfXi7PQUt+gJWGdZRMltVnySjqnqVbHwsO73uNoHLrOI4pEUeQiPaSQngkQRGa5LAYm/A6jYGYkTFQtvT2gee+aLTITzynzwdUmCdrJ0GcL/bi122dI3f+Kpkdj61Z2G2VZY2r2blXtQZcm2vt2j+4ZRWwMejWtpnpfBAnFm2wThvFiXqIYit8AtXu6isuj2bmwqtgZT5ieQeop7ieu1LBuqCutNkn+coxdmQWH+/Ct5oYl7GTHsPWqGlXF9Irel9J05SxZGmCtjw7sdXeGXwRItP7/eh6attCJZJJaPCD5/1/cNSVoLrifBQnNV/NoxWYzXsrYpz6LdNq38kK8eEB/YtNWMoCLN/NUtY2SbrJBlQChdWNeovxz9D3ddujFb+792iQ7qHcm2DGAEX2fKqeYTFY1J1Rp/eDF89UMocjJ5pgvfdjqYNuGXdARYxjBJybMskS+MhNd87JkjNAyRDDhU7hIe09Q0KtR9eeoe8wQ+8R2oOXoivAsm/KmyNGDTKPM5xUCvDABHjaaMmHczSAAdZcc/dYnrdgMGPcONeyVkhPgr90Fw==
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-oidc
|
||||
namespace: outline
|
||||
|
|
@ -1,255 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: outline-data
|
||||
namespace: outline
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: outline
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline
|
||||
spec:
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: outline
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: outline
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:16-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h outline-postgres -p 5432 -U "$POSTGRES_USER"; do
|
||||
echo "Waiting for outline-postgres..."; sleep 2;
|
||||
done
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-postgres-credentials
|
||||
key: POSTGRES_USER
|
||||
- name: wait-for-redis
|
||||
image: redis:7-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until redis-cli -h outline-redis -p 6379 ping | grep -q PONG; do
|
||||
echo "Waiting for outline-redis..."; sleep 2;
|
||||
done
|
||||
containers:
|
||||
- name: outline
|
||||
image: docker.io/outlinewiki/outline:1.10.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: NODE_ENV
|
||||
value: "production"
|
||||
- name: PORT
|
||||
value: "3000"
|
||||
- name: URL
|
||||
value: "https://outline.basicstack.de"
|
||||
- name: FORCE_HTTPS
|
||||
value: "false"
|
||||
- name: ENABLE_UPDATES
|
||||
value: "false"
|
||||
- name: WEB_CONCURRENCY
|
||||
value: "2"
|
||||
- name: DEFAULT_LANGUAGE
|
||||
value: "en_US"
|
||||
- name: RATE_LIMITER_ENABLED
|
||||
value: "true"
|
||||
- name: LOG_LEVEL
|
||||
value: "info"
|
||||
- name: FILE_STORAGE
|
||||
value: "s3"
|
||||
- name: FILE_STORAGE_UPLOAD_MAX_SIZE
|
||||
value: "262144000"
|
||||
- name: PGSSLMODE
|
||||
value: "disable"
|
||||
- name: DATABASE_CONNECTION_POOL_MIN
|
||||
value: "5"
|
||||
- name: DATABASE_CONNECTION_POOL_MAX
|
||||
value: "20"
|
||||
# Secrets
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-app-secrets
|
||||
key: SECRET_KEY
|
||||
- name: UTILS_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-app-secrets
|
||||
key: UTILS_SECRET
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-app-secrets
|
||||
key: DATABASE_URL
|
||||
- name: REDIS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-app-secrets
|
||||
key: REDIS_URL
|
||||
# S3 (Hetzner Object Storage)
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-s3
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-s3
|
||||
key: secret-key
|
||||
- name: AWS_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-s3
|
||||
key: region
|
||||
- name: AWS_S3_UPLOAD_BUCKET_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-s3
|
||||
key: endpoint
|
||||
- name: AWS_S3_UPLOAD_BUCKET_NAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-s3
|
||||
key: bucket
|
||||
- name: AWS_S3_FORCE_PATH_STYLE
|
||||
value: "true"
|
||||
- name: AWS_S3_ACL
|
||||
value: "private"
|
||||
# OIDC (Pocket ID)
|
||||
- name: OIDC_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-oidc
|
||||
key: client-id
|
||||
- name: OIDC_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: outline-oidc
|
||||
key: client-secret
|
||||
- name: OIDC_AUTH_URI
|
||||
value: "https://auth.basicstack.de/authorize"
|
||||
- name: OIDC_TOKEN_URI
|
||||
value: "https://auth.basicstack.de/api/oidc/token"
|
||||
- name: OIDC_USERINFO_URI
|
||||
value: "https://auth.basicstack.de/api/oidc/userinfo"
|
||||
- name: OIDC_LOGOUT_URI
|
||||
value: "https://auth.basicstack.de/api/oidc/end-session"
|
||||
- name: OIDC_USERNAME_CLAIM
|
||||
value: "email"
|
||||
- name: OIDC_DISPLAY_NAME
|
||||
value: "Pocket ID"
|
||||
- name: OIDC_SCOPES
|
||||
value: "openid profile email"
|
||||
- name: OIDC_DISABLE_REDIRECT
|
||||
value: "false"
|
||||
# Persist local cache directory
|
||||
volumeMounts:
|
||||
- name: outline-data
|
||||
mountPath: /var/lib/outline/data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /_health
|
||||
port: 3000
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /_health
|
||||
port: 3000
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
- name: outline-data
|
||||
persistentVolumeClaim:
|
||||
claimName: outline-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: outline
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: outline
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 3000
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: outline
|
||||
namespace: outline
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- outline.basicstack.de
|
||||
secretName: outline-tls
|
||||
rules:
|
||||
- host: outline.basicstack.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: outline
|
||||
port:
|
||||
number: 80
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-postgres-credentials
|
||||
namespace: outline
|
||||
spec:
|
||||
encryptedData:
|
||||
POSTGRES_DB: AgCbJr6ng8tQwTMyNfrvubnlA5CvamfSZtzR4AwuIn/UUcK6mb68B5ucCUyA0GeeFVOGx63rlibDCKb42Da0AHrSYDTzZ1YbYAiwEzdlrmgh5tCfMiqbcUHUqSR4L7QAjkS50ASQXPXxSHuxuYa9cO7EQCasN7Fm5cmTkbcGHTKOSJ7fE7VQDBXyf6Z4Rga8HBys20xjglDFy7meBLEXDOKRUkQDC/PGgqlF98ZVtMH148ziaRxiu2tkbpsFchQ/zFc6x5rtllYGnlic8ssbgAXwUL6WZO6CcSbjhSkq4JcIOJh60GudsthTaW5HTldrD0VwaLXw6BNSD9+7+Zd1UIKSN9+imBOCmpOglLG2LFCxUD02lr3sPBdlztQwZnvxvFhfKRXstn/Qo8ujzKlRFyHdLsX0IQf7vj9kb+Giq54MAgelqivl7HMg8O73DIHe/GUmwDr3cXt1+aVoPKnQHe5wkQ4bIsTDzkNZzig1Lisl/bBxF2tf48ZXP5QZXpcL80TX6ZLxh0u1MVTpmIT6fZziw8lOJfLdE917UokNPnFVWx9X7Z+zWh07b/AMIx1Wa5JNKl1rpmJazYY42pvEGU2crR0NETNYavnM95Pl9sbRh1f/grp3v8Y4J31Oyvy2+MMChe1FdUbOnMN3Aozc3OLFuu94aiJFd1r2YU/1VCSa/0VuaDcCPwtFvKf92B/ADh0HT0Lxe7VW
|
||||
POSTGRES_PASSWORD: AgAGj5wpTMEcDHT20f1SKTd+J0ZSC6bhfBP2WOSa3zBbxqI7Rb0OEuR+tsuQJyEbL8uKPrKKvgcjf4/GGTNChpkoe/n9Z5Vsc5cvdel59egtIca3R/2g+EhUqGhrb/R4cVMylqANcpK3O25GJNL6K9TfVU03qlORazVVp2qNr+cQ/fWUHqAJUz77ui6KPjGTjsrKH4Vnh5eP40ZoYRDXOmM7AACgMfw4dtaUID/QhslaGE8vi0o/sv/MyhC0Pic98V/cwmwPgQCYgH44v56EtOR7X0Rqz08so3GGi4hvrQCs1i40Sd2k0SayBNmvF0LGO9stlkDIpmMmnsEeCgxk0u5L9tRxsDtvGX4AAiU9FByWhVp4JRYd4QAF509p3aF+1/3kNQen+4aHefy4Xzpx1dDQOZoWArz/kXhyh8u+wkkDwsf/6VXSORdd2w4M2lDyhP61pGY8FC0QOCKWNkfOTTgFOJf8guqwSfmLNdS+lhJmtpgKpNwPnG90CgV8b6xd3uPXYa6FQPd7I9sFpN4I8ZC9SJZE4karU7Fys/SArdBB2proemw6SYepBvo41bcdftxYpL5RvfCkFjLj6CDTYbQtjPzV0YeG03l1qKqbfiKroC242mGprJweGdC4kM4L28aWaMq0oGg6og7u0A84YIukQlncIMAQ3t4L78Zp/OCOWbkPkWykjUVCkOtxMSnIjfWvFVN8vr+zN0AD/PwPu976D1RdCWXzXyZu1gWZmnBEUQweYw1mFMsaqAxQl000UWc=
|
||||
POSTGRES_USER: AgAqZajhuIMyXzZtwm3C6KCN6njShH2tdlj4U8IsZxc3v0rHdtlCDfjk4db/fkrmRZXGzJmjizVfMwlBjVAXlO7WiGpZ0eDG/UQMIR7f56UD40Ccj2rJOExw0tB/mnfCmXVVMsUyeA3IPJLY+lUB40IE6nw7K3n8A5XpyaGTCPa8QFlCHvwpQ/seDNykA4sc+NX4So5sPUS3h/NFo9T1XhI6prvDWOd5d0YEFYBoHSgQU/QcIizT6sgudVl4cKtWtRrJdRSaq6msREB4hFfc4DnaxGsB3fLmsfUC881nkSc25ayGvnJXvWhFCnIy5v15cHFT/Yo8yHqVQdWVgJWKxb8fZfLGtXuRGXYoLHbH+VGhAcPUDSERmBoyCKG4ABGWkXG6+xLG7meN9q2/u0zbmCoYZRSxflJIUOWaZIGuvJq6KmpKsEJ5A58M90Y308oSS7yHyX3pgGJVV3zWZC6ddOoPPGbpipxAESUq7XCJnTth9SkIttlGBIgiFZ0wErL31LileoEGvhMcv5H5MSzebZbfW+uJIMRJF5QfaL8BS7UCmdI9j73VxT7pxupxKeuQCJ/zsF3B13IR/+w1hEwNW0TugqRlaqF8agU0jCLF0tlD5aCnrqHJp7MB/zD3Lb9Pk30mUEeye++kx4kqTE2Vf5sPAHbIjBlOvm/y1lKxDjASoab+I6mp4/fTBe4+S+Unqw+R7uWxBRUb
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-postgres-credentials
|
||||
namespace: outline
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: outline-postgres
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline-postgres
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: outline-postgres
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: outline-postgres
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline-postgres
|
||||
spec:
|
||||
serviceName: outline-postgres
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: outline-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: outline-postgres
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: outline-postgres-credentials
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: postgres-data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
subPath: pgdata
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: postgres-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: outline-redis
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline-redis
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: outline-redis
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: outline-redis
|
||||
namespace: outline
|
||||
labels:
|
||||
app: outline-redis
|
||||
spec:
|
||||
serviceName: outline-redis
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: outline-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: outline-redis
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --save
|
||||
- "60"
|
||||
- "1000"
|
||||
ports:
|
||||
- name: redis
|
||||
containerPort: 6379
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: redis-data
|
||||
mountPath: /data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: redis-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-s3
|
||||
namespace: outline
|
||||
spec:
|
||||
encryptedData:
|
||||
access-key: AgCMbJDgvGRG0ritkltoJF/nFpb7p4JRbl57Tje3EUx/+jCZDqhvM2J6DAxLL4ELy6xINuk7bByMU4Ry5z6lmYTyZ6UvgYpJIkrg9xS7dA4WU2474oGbDWx/u2xRvI0M3t/JqlfkShaX/jxDVSibYGpGvN/+uZ/F308EvUOai/PLgEz2VpA/6y/5gtFdrDjDFoBLs5eZsrBPDAIweKQ0s+5wiPlHv+vDhIvw99efS4wiZZQXuD7j7zKJIxwGKG0k6VGIoRm207xxvHoSXmxmhZp0I6whKJ4XJqXo/S1nzQllQWOaaUyA2Y+sGXfjbcm1YdNN48/MZK6uWJNjXaIphmW6DehW8+i3aJLbv/XSmbxcY9fowRyEfFz3aqqtGW7gP4CzUc+0DwQhfE+9z/lBoWOTKsDBf8kUYnF5GPCMB1TjAAP12MWCsBmecP9kppMawH2SmRZbl+ImZgiQTXlJ79gNRoF2ri4TF2xz/tcjP3Rnt82p8yV3gj/NDsIMLUG1GPiNDMdz8dNR723V9pb+lIoNUD+pb7+Jr1N2wCq21YKNlUGPC6ZXK5flQVwlKXbxNSyu1KEyhzgEnRjKBXTQOBlRcUkkaXNbA4FreuyvfCJcRkScf3YsfzJQcxxiCxK+A5UDd9sgiVwi3rYYfQhwzPUDcvjVMTX93qZaKPfmfcxg0STV5M+A5pAkSVvs/8ORS0HoPqvXZRlH/fOT040n4z1b4ALnBg==
|
||||
bucket: AgAiu0w47e6zbhFOX7dOGgDJVRAy5ZDfWmjgYGMibCCFm+FglEmKaj5Vi+70nZMsUigtH98WKKfEdSsJ4+mGuITNSxU2+p0uth2D2JapkEmOtzbzMzpagoXLnRV2xZW/HKyiUhRct7ygawUQ6TA3A4JBb3G/6WK+T1AxF2Z0NtDTULmM+mx8a4XNE1Anmuo6zOUQ2GIUfbBOI6RCBP693fVI8m+eB4bzkbmwrTXdT1bHPQ95wWbX28wDI3i9PQwRlM2oM5OYOTn46l3V+5mwPTUntuGuMis5P1jH4O7EkUObCbeVsC4S4orZ/sYPBs580MXFzBGeKM/Ce1fGVICC6kMWqt1SFnlYvEGRej1qj8N6uS5lmLNG6HM6Z3nHSiM/6Pi3LzfvnEvum8x6qmqRGsergIHTAcaHBOOiVDrjRKRqlCTRDmOjoTbB0vD00cWXAFGTjTXKsFtrcAmzq3EF3vhrA69Vf2FeJsF4m+ixlFdZd7WqrL5H2W7tDEHtbMhMHiii5cfHkKD9orLFpfMzirSR+sLdeRzR5t5pvGIkPRVECEHU+8w9vWSxm2aSFXOJUH4nOMM0TNzlJNlD/vsDL+Z/JQjQDzBKyBIHp1hKo45JxjwgmZKjsDy4Ts8CEwPp/zPwSFY96hqFVVs14PqjWb17y9oCpca181rQ+7bmrW0F4OCXTGg4wRBxk0ikDB7JbwCzZ1d7VqNB5lOfzeDPgZq8d98=
|
||||
endpoint: AgB6zhkvtiqfkoSMExAy5Imd9LrBkC04F7Ifx/YSGfGtcjFSNsACNNL2vnVKUyKWvuDPa+ptdJh6HLUvmk5D9myMxlH8Bc7p69x4/wUqvwRiBgwm/4Gge3XlYa16KzMJIhkaVu6UxNSEI3c8S94Rvv7uiCk+IDkROUnutVC3dzRE6OiFz5GL56AO1f7bOj7t+0Rdpq7rSDoBq1e1Wftm6xnHa56BGXGlpaH/7UU0h0MDlnAyAbE4Xi04FLk3s7gjPPA4tetbtEBEBbKTA7GokWpt4rXrqivLr4hKd/OSklI9SByd9SxpdbO2m6QrunIqQDjjHkDrPIOAelM1MOVSTJKG1JWfXqywA2l/WeqIGEQt8FW4wWCU0E4XKixsJdFEDjtFAOyVJPYoyObaVUvkXrZdxPuc+KFtocSXR9KM0CaPQ7jteqOfYI+4VDwvE/YRVHJrolUzqBpu/pKkHnDDottfJ/O7OJCGl+jfQ1ZzP4ovOVH+GIBmo3rjVWZM+f8KlA8HA6NAdAWuSE8z3phP7n8N2ZzICr5KsW0vNSI5Ue7/mvjCbm9nUushAuzvFX7s4YH1KtwQrToMJTylfUdxyBjLzJ2NOOsg7ZiaWw9j9tjrAv1SRhPps/i7cy2Fg6Z1qeX+mvWgeIjKaQvTdYH0HREUXpzAMe1UWUkCJQtCrk8nk7gigmxF3T/fO0AIxMQGknRKYdqp3Hr06M09P1WF5QjrLU7uydWbFhA6Js5DWd3d2MurpQ==
|
||||
region: AgBFM4EAaxptRlqN8EnsrmeX950mS10T7ZlfZmdpOsQJSR1Z0CzN5gmsJmVlznrRqw1FGGXU3feBrg/eA01IP5ykqq06iLWyjla3oLVWaYzwK/wASiY7Ycl2tkuyZwpOnQNWuxmXFX2WHl0NNKbd+mklmBxaIPRQVdYqtDAUeblyTII8oBFmv1zc4kpf4MCCHhsCG7m4VG+b+EI/DZZRXK5LFaVoOFy1pHGhN8um/OerDICHuOleOQ+hWj3w5mkigdKzFZkeIG9RQ0WnvS1muwECKvanDBaS0RGDxt+Ipm09WPk4VyGAzHyDd5wOd/uckD/yYFzAodjlZHDQipgNE1MAs43gqt6X3lKNAwqOEc6TLmQv6FUTU9+R7J+2FOgSKQ/ACTVyGHiebOqjb2L9bk1oj/j0cKARRQyRyL8B/TFHPVfyPH+kiTmRa+qlsg7FN7oTeFSOWaNZbA0jz+ONM9IzRpE33VHS/r9cCmfe2rFySNXCjHIkiKJC9qMYBqorM1kM4Ab00hlfhdO0aspTW9KC0pEauPs0n84sl/HHbOQ+sLcYW7QBvZvuRi1OyVM99XoKjI7Jnu7Aq55NX+rt9zZ+iJpnXeUm8hxCKC/CyUS5mAEmqQpJoKqLVMfhvpXMn8I5lXs3YpshHto649EPTVS6A23YBG4Q3toZ8UmbXp2JtxlL+JnfU0/mZvAPOBkfrmx5MToa
|
||||
secret-key: AgDCSueSKDzmfI/5zhEusMX6AaIpjF526UdnWdTnmUfsX+LW+w/uQvgM0aPIoR/sMyEQoo/UxouU3V1PQIHj9Q/3acDuFBkXCFFb02dFv+gXF3SAEGnCaHviH8Bce12SfMePyOlKT2eDI8vTyehw3istEMTJHc7rMr/PNZprKG1T35wxdaklS88uLP5oAnHVKeqEvAvgNFXWrMqJ1GM1l8/rTd5GCJ8xsE1mGU/26Ac518nzo2PMARZYwZ0Rfiv2+QZswT7DDQNH6ZoY0DPzL+SW9ecMGc933L6EejNzco1esttEtRa+ZJRZ2tqUbm8LTZOFHq500VRtbEAcAtySMtjxX0VF4sSGrxGGooEkY9mqOcy2yQLZdb9UGObF8wMBXcGx67SAcXbppWeNSOvp4p7QVrzVbNXIbnV+7e8XrKBHf9inBW2j9oCgPjPcTKW7VfHOQKIYKwnHS81hgOsTQcX+BcYT9Bkea4E+/R0ovYJh68bd3Wwcm/dECnj3y3Kzgj2j4/xv7aO4cNacAt1INILYY9P0edYHbCnO4gHXt9itvKTbiOslYRuxfgff490ReXunkTl/FFawmwCZoVY7nlp59IKwTB/PahPctGmrbHstwI6UK5kLaB93Cf6bNengLCt+GYAmuuiTAOqfTLvGRQBu074mqWAO9ThYkdU0ebtOO9sY8bALk2ZlSdEN0T5Aqmp+LaSz6W91nLDF8/XaGD6zIpHHjwF6oCPg6BPn//43Rx/unbjJf82j
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: outline-s3
|
||||
namespace: outline
|
||||
|
|
@ -1,240 +0,0 @@
|
|||
# Pangolin
|
||||
|
||||
Pangolin is BasicStack's self-hosted reverse proxy and SSO gateway. It sits in front of
|
||||
selected cluster services (e.g. `paperclip.basicstack.de`) and forces every request through
|
||||
Pocket-ID before it reaches the backend. Pangolin does not terminate TLS or handle the
|
||||
raw HTTP request path itself — it publishes a Traefik dynamic config, and the
|
||||
`pangolin-kube-controller` translates that config into Traefik CRDs that our existing
|
||||
Traefik ingress controller enforces.
|
||||
|
||||
Public URL: <https://pangolin.basicstack.de> — App source: <https://github.com/fosrl/pangolin>
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Browser ──HTTPS──► Traefik (ns=kube-system)
|
||||
│
|
||||
│ IngressRoute + Middleware (badger) + TraefikService
|
||||
│ ▲ managed by pangolin-kube-controller (ns=pangolin)
|
||||
│ │
|
||||
│ │ polls /api/v1/traefik-config every ~15s
|
||||
│ │
|
||||
▼ │
|
||||
Pangolin API (ns=pangolin)
|
||||
├─ port 3000 external dashboard API (session auth, behind Ingress)
|
||||
├─ port 3001 internal traefik-config API (Bearer auth, cluster-internal only)
|
||||
└─ port 3002 UI HTTP (behind Ingress)
|
||||
│
|
||||
├── PostgreSQL (ns=pangolin, pangolin-postgres)
|
||||
└── OIDC provider: Pocket-ID at https://auth.basicstack.de
|
||||
|
||||
Auth flow for a protected resource (e.g. paperclip.basicstack.de):
|
||||
Browser ──► Traefik ──► badger middleware ──callback──► Pangolin API (3001)
|
||||
│
|
||||
▼
|
||||
Pocket-ID / session
|
||||
```
|
||||
|
||||
Two long-running workloads live in the `pangolin` namespace:
|
||||
|
||||
- **Pangolin** — the app itself: dashboard, resource/target CRUD, OIDC handshake,
|
||||
session cookies, and the internal `/api/v1/traefik-config` endpoint that
|
||||
publishes the current desired Traefik state.
|
||||
- **pangolin-kube-controller** — a Kubernetes controller (leader-elected) that
|
||||
polls the Pangolin API, then upserts Traefik `IngressRoute`, `Middleware`,
|
||||
`TraefikService`, etc. via server-side apply into the `pangolin` namespace.
|
||||
|
||||
The controller only writes CRDs into `pangolin`; the backing Service typically
|
||||
lives in the target app's own namespace. Traefik is started with
|
||||
`--providers.kubernetescrd.allowCrossNamespace=true` so those cross-namespace
|
||||
refs resolve.
|
||||
|
||||
## Components (files in this directory)
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `namespace.yaml` | Creates the `pangolin` namespace. |
|
||||
| `pangolin-config.yaml` | Pangolin `config.yml` template (ConfigMap). `__DATABASE_URL__` is rendered at startup by the `render-config` init container from the sealed secret. |
|
||||
| `pangolin-deployment.yaml` | Pangolin app Deployment, Service (ports 3000/3001/3002), Ingress, and the `pangolin-data` PVC. |
|
||||
| `pangolin-secrets-sealed.yaml` | Sealed OIDC client id/secret and local admin password. |
|
||||
| `pangolin-postgres-secrets-sealed.yaml` | Sealed Postgres credentials plus the rendered `DATABASE_URL`. |
|
||||
| `postgres.yaml` | Single-replica Postgres 17.5 + `pangolin-postgres-data` PVC. |
|
||||
| `pangolin-controller-config.yaml` | `pangolin-kube-controller` env: `CONFIG_ENDPOINT`, target namespace, log level. |
|
||||
| `pangolin-controller-deployment.yaml` | Controller Deployment (image `fosrl/pangolin-kube-controller:0.1.0-alpha.1`). |
|
||||
| `pangolin-controller-rbac.yaml` | ServiceAccount + ClusterRole/Binding: CRUD on Traefik CRDs and leases. |
|
||||
| `pangolin-controller-service.yaml` | ClusterIP for controller metrics. |
|
||||
| `pangolin-controller-api-key-sealed.yaml` | Sealed `Authorization` header the controller sends to the internal API. |
|
||||
|
||||
ArgoCD Application: [`apps/app-pangolin.yaml`](../app-pangolin.yaml) — sync policy is
|
||||
`automated: { prune: true, selfHeal: true }`. Any local edit must land in git.
|
||||
|
||||
## SSO integration (Pocket-ID)
|
||||
|
||||
Pangolin authenticates users against Pocket-ID via OIDC. Two independent SSO
|
||||
flows exist and should not be confused:
|
||||
|
||||
1. **Logging into the Pangolin dashboard.** Configured in
|
||||
`pangolin-config.yaml` (`oidc.*`) and via env in `pangolin-deployment.yaml`:
|
||||
- Issuer: `https://auth.basicstack.de`
|
||||
- Callback URL: `https://pangolin.basicstack.de/auth/callback`
|
||||
- Scopes: `openid profile email groups`
|
||||
- Client id/secret: `pangolin-secrets` sealed secret keys
|
||||
`oidc-client-id` / `oidc-client-secret`.
|
||||
2. **Protecting a downstream resource (e.g. paperclip.basicstack.de).**
|
||||
Handled by the badger Traefik middleware that the controller emits.
|
||||
The middleware calls back into Pangolin's internal API (port 3001) to
|
||||
check the session cookie, and if unauthenticated redirects the browser
|
||||
through Pangolin's own OIDC flow. There is nothing to configure per-resource
|
||||
in Kubernetes — the resource's auth policy is set inside the Pangolin UI.
|
||||
|
||||
The Pocket-ID client is provisioned separately (see [pocket-id docs](https://pocket-id.org/docs/client-examples/pangolin)).
|
||||
Do not reconfigure the OIDC client from the Pangolin side once it is working.
|
||||
|
||||
## Ingress protection workflow
|
||||
|
||||
The end-to-end flow for a protected hostname like `paperclip.basicstack.de`:
|
||||
|
||||
1. Operator creates a **Resource** in the Pangolin UI (hostname, backend
|
||||
Service, port, auth policy).
|
||||
2. Pangolin persists it in Postgres and exposes the new desired state at
|
||||
`http://pangolin.pangolin.svc.cluster.local:3001/api/v1/traefik-config`.
|
||||
3. `pangolin-kube-controller` polls that endpoint on a ~15s loop.
|
||||
4. On change, the controller server-side-applies into `ns=pangolin`:
|
||||
- `IngressRoute` for the hostname (with `tls.certResolver: letsencrypt`).
|
||||
- `Middleware/badger` (session check + OIDC redirect).
|
||||
- `TraefikService/1-<resource>-service` pointing at the backend
|
||||
Service in the app's namespace, port from the DB.
|
||||
5. Traefik picks the new CRDs up and starts serving the route.
|
||||
6. First request → redirected to Pocket-ID → session cookie → subsequent
|
||||
requests are proxied to the backend Service.
|
||||
|
||||
## Adding a new protected ingress
|
||||
|
||||
Prereqs the target app must already satisfy:
|
||||
|
||||
- A Kubernetes `Service` in its own namespace, listening on a known port.
|
||||
- A DNS record for the hostname pointing at the cluster ingress IP.
|
||||
- No competing plain `Ingress` in another namespace for the same hostname.
|
||||
If one exists (e.g. a legacy direct ingress), delete it first — Traefik
|
||||
will otherwise route the pre-existing ingress and Pangolin's IngressRoute
|
||||
never wins.
|
||||
|
||||
Steps:
|
||||
|
||||
1. **Log in** to <https://pangolin.basicstack.de> (SSO via Pocket-ID, or
|
||||
local admin from the `pangolin-secrets` sealed secret).
|
||||
2. **Create the Resource.**
|
||||
- Type: HTTP/HTTPS.
|
||||
- Domain: the exact FQDN (e.g. `newapp.basicstack.de`).
|
||||
- Base domain: `basicstack.de`.
|
||||
3. **Target.**
|
||||
- Target type: HTTP.
|
||||
- Backend URL: `http://<service>.<namespace>.svc.cluster.local:<port>`
|
||||
— the port here is written to the `targets.port` column in Pangolin's
|
||||
Postgres and used by the controller when it generates the `TraefikService`.
|
||||
If the Service port later changes, update the target port in the UI (or
|
||||
via SQL: `UPDATE targets SET port=<N> WHERE "targetId"=<id>;`).
|
||||
- Path match: `/`, strip prefix off.
|
||||
4. **TLS.** Enable TLS, force HTTPS redirect. The Pangolin-generated IngressRoute
|
||||
hardcodes `tls.certResolver: letsencrypt`, which is provisioned on our Traefik
|
||||
install (HTTP-01, persistent `/data` PVC). Do not disable that resolver in the
|
||||
Traefik `HelmChartConfig` without also removing every Pangolin-generated route
|
||||
— Traefik will otherwise fail with `nonexistent certificate resolver`.
|
||||
5. **Auth.** Enable authentication, method SSO/OIDC. In Community Edition (what we
|
||||
run) there are no shared policies or group rules — add authorised users
|
||||
individually by email. If you need group-based rules, that's an Enterprise
|
||||
feature.
|
||||
6. **Save** — controller applies CRDs within a poll interval (~15s).
|
||||
7. **Verify.**
|
||||
|
||||
```bash
|
||||
kubectl -n pangolin get ingressroute,middleware,traefikservice \
|
||||
| grep -i <resource-name>
|
||||
kubectl -n pangolin logs -l app=pangolin-controller --tail=50
|
||||
# HTTP smoke test
|
||||
curl -sSI https://<host>/ # expect 302 to Pocket-ID
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **badger middleware returns HTTP 500.** The middleware is trying to call
|
||||
`apiBaseUrl` and either the URL is not an FQDN or Pangolin isn't reachable
|
||||
from `kube-system`. Confirm `pangolin-config.yaml` sets
|
||||
`server.internal_hostname` and `server.badger_override` to
|
||||
`pangolin.pangolin.svc.cluster.local` — the short name `pangolin` does not
|
||||
resolve from Traefik's namespace.
|
||||
- **Controller logs `conflict with "controller" using traefik.io/v1alpha1: .spec…`.**
|
||||
A pre-existing CRD was created with the `Update` verb and the current
|
||||
controller uses server-side `Apply`. Delete the offending
|
||||
`Middleware/TraefikService/IngressRoute` once — the controller recreates it
|
||||
via SSA and the conflict is gone.
|
||||
- **Traefik logs `service port not found: <N>`.** The port stored in Pangolin's
|
||||
`targets` table doesn't match the Kubernetes Service port. Fix in the UI, or:
|
||||
`UPDATE targets SET port=<N> WHERE "targetId"=<id>;` (identifier case matters,
|
||||
double-quote it).
|
||||
- **Traefik logs `nonexistent certificate resolver: letsencrypt`.** The
|
||||
`letsencrypt` resolver isn't loaded in Traefik. Every Pangolin IngressRoute
|
||||
references it — either define the resolver in Traefik (see
|
||||
`infrastructure/networking/traefik-helmchartconfig.yaml`) or override
|
||||
`traefik.cert_resolver` in `pangolin-config.yaml`. Grep the Traefik startup
|
||||
log for `unable to get ACME account`; that appears when the resolver was
|
||||
skipped entirely.
|
||||
- **Route doesn't resolve cross-namespace.** Traefik must run with
|
||||
`--providers.kubernetescrd.allowCrossNamespace=true`. Pangolin's CRDs live in
|
||||
`pangolin`; backend Services usually don't.
|
||||
- **CRDs never appear after saving a Resource.**
|
||||
```bash
|
||||
kubectl -n pangolin get pods -l app=pangolin-controller
|
||||
kubectl -n pangolin logs -l app=pangolin-controller --tail=200
|
||||
kubectl -n pangolin get configmap pangolin-controller-config -o yaml
|
||||
```
|
||||
Common causes: controller CrashLoop, wrong `CONFIG_ENDPOINT`, missing auth
|
||||
header sealed secret, RBAC missing after a controller image upgrade.
|
||||
- **Login loop / redirects to Pocket-ID forever.** Clear cookies for both
|
||||
`pangolin.basicstack.de` and `auth.basicstack.de`, confirm the OIDC callback
|
||||
URL matches the Pocket-ID client, and check Pangolin logs for OIDC errors:
|
||||
`kubectl -n pangolin logs -l app=pangolin`.
|
||||
- **Pangolin pod stuck in `Init:0/2`.** The `wait-for-postgres` init container
|
||||
is looping on `pg_isready`. Check `pangolin-postgres` health and the
|
||||
`pangolin-postgres-secrets` connection details.
|
||||
|
||||
## Critical notes — DO NOT DELETE
|
||||
|
||||
Pangolin holds live user/account/config state. Losing a PVC here means
|
||||
re-provisioning every protected resource, every target port, every user grant,
|
||||
and re-doing the OIDC client trust from scratch — plus temporary outages on
|
||||
every downstream site that Pangolin fronts.
|
||||
|
||||
- **`pangolin-postgres-data` PVC (ns=pangolin, 10 Gi, `hcloud-volumes-encrypted`).**
|
||||
Backs Postgres. Contains resources, targets, users, sessions, and everything
|
||||
the controller reads. Annotated with
|
||||
`argocd.argoproj.io/sync-options: Delete=false` — do not remove that
|
||||
annotation, and do not delete the PVC as a "reset" step. Snapshot the volume
|
||||
before any migration or destructive DB operation.
|
||||
- **`pangolin-data` PVC (ns=pangolin, 10 Gi, `hcloud-volumes-encrypted`).**
|
||||
Runtime files under `/app/config`. Also marked `Delete=false`. Do not delete.
|
||||
- **Sealed secrets** (`pangolin-secrets`, `pangolin-postgres-secrets`,
|
||||
`pangolin-controller-api-key`) — rotating them out of order will lock the
|
||||
controller out of the internal API or take Pangolin off Postgres. Rotate one
|
||||
secret at a time and verify the corresponding pod comes back before
|
||||
proceeding.
|
||||
- **`pangolin-config` ConfigMap.** Editing `server.internal_hostname` /
|
||||
`server.badger_override` back to the default short hostname will break the
|
||||
badger middleware cluster-wide (see Troubleshooting).
|
||||
- **The Traefik `HelmChartConfig`** in `kube-system`
|
||||
(`infrastructure/networking/traefik-helmchartconfig.yaml`) — Pangolin depends
|
||||
on `allowCrossNamespace=true` and on the `letsencrypt` resolver being loaded
|
||||
with a persistent `/data` PVC. Removing either silently breaks every
|
||||
Pangolin-managed route.
|
||||
|
||||
If access to Pangolin is broken, **restore from a snapshot first; do not
|
||||
"reset" or re-provision** the admin/OIDC layer. That is the standing rule for
|
||||
every stateful service in this cluster.
|
||||
|
||||
## Related
|
||||
|
||||
- History and gotchas: internal memory `pangolin-kube-integration` (from DEV-457).
|
||||
- Configuration walkthrough for the `paperclip.basicstack.de` resource:
|
||||
[`pangolin-paperclip-configuration-guide-community.md`](pangolin-paperclip-configuration-guide-community.md).
|
||||
- Pangolin docs: <https://docs.pangolin.net/>
|
||||
- pangolin-kube-controller: <https://github.com/fosrl/pangolin-kube-controller>
|
||||
|
|
@ -35,7 +35,7 @@ spec:
|
|||
spec:
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:17.11
|
||||
image: postgres:17.5
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
|
|
@ -58,7 +58,7 @@ spec:
|
|||
name: pangolin-postgres-secrets
|
||||
key: postgres-db
|
||||
- name: render-config
|
||||
image: busybox:1.38.0
|
||||
image: busybox:1.37
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ spec:
|
|||
fsGroup: 999
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:17.11
|
||||
image: postgres:17.5
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ spec:
|
|||
fsGroupChangePolicy: Always
|
||||
initContainers:
|
||||
- name: fix-permissions
|
||||
image: busybox:1.38.0
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ spec:
|
|||
containers:
|
||||
- name: passbolt
|
||||
image: passbolt/passbolt:latest-ce
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ spec:
|
|||
spec:
|
||||
containers:
|
||||
- name: pocket-id
|
||||
image: ghcr.io/pocket-id/pocket-id:v2.19.1
|
||||
image: ghcr.io/pocket-id/pocket-id:v2.11.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 1411
|
||||
|
|
|
|||
|
|
@ -71,20 +71,14 @@ This repository (`https://forgejo.basicstack.de/basicstack/stack.basicstack.de`)
|
|||
|
||||
### Investigation Results (2026-07-13)
|
||||
|
||||
**Infrastructure Status**: ✅ ALL VERIFIED WORKING (as of 21:00 UTC)
|
||||
**Infrastructure Status**: ✅ ALL VERIFIED WORKING
|
||||
- SMTP (ports 25, 587, 465): Responding with correct ESMTP greeting
|
||||
- IMAP (ports 143, 993): Responding with correct IMAP greeting
|
||||
- LoadBalancer external IP: 178.105.17.239 (correct)
|
||||
- Pod status: Running, started at 19:26 UTC
|
||||
- Service configuration: Correct port mappings and selectors verified
|
||||
- Pod status: Running, 0 restarts
|
||||
- Monitoring: PrometheusRules and ServiceMonitors in place
|
||||
|
||||
**SMTP Issue Timeline (2026-07-13)**:
|
||||
- 20:16 UTC: LoadBalancer IP updates visible in events (10.42.1.1 → 178.105.17.239)
|
||||
- 20:31 UTC: User reported SMTP port returning IMAP greeting (Thunderbird error)
|
||||
- 21:00 UTC: External port tests confirm all ports working correctly with proper greetings
|
||||
- **Assessment**: Transient issue during LoadBalancer IP transition, now resolved
|
||||
- **Action Required**: User to retest SMTP sending with fresh client connection
|
||||
**User-Reported SMTP Issue**: Client configuration problem (Thunderbird connecting to port 143 instead of 587), not infrastructure issue
|
||||
|
||||
### Architecture Overview
|
||||
|
||||
|
|
|
|||
123
apps/stalwart/stalwart-allow-cluster-ips-job.yaml
Normal file
123
apps/stalwart/stalwart-allow-cluster-ips-job.yaml
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
---
|
||||
# One-time Job to configure Stalwart HTTP listener to allow internal cluster IPs
|
||||
#
|
||||
# Problem: Stalwart blocks the HTTP port from Traefik's pod IP (10.244.2.227),
|
||||
# causing 502/503 errors when accessing mail.basicstack.de
|
||||
#
|
||||
# Solution: Use kubectl exec to access Stalwart's admin API via localhost (which is allowed)
|
||||
# and disable IP filtering for the HTTP listener, or allow the pod network CIDR
|
||||
#
|
||||
# This Job must be manually triggered after Stalwart is running:
|
||||
# kubectl create job --from=cronjob/stalwart-allow-cluster-ips manual-fix -n stalwart
|
||||
#
|
||||
# Or apply directly:
|
||||
# kubectl apply -f stalwart-allow-cluster-ips-job.yaml
|
||||
# kubectl wait --for=condition=complete job/stalwart-allow-cluster-ips -n stalwart --timeout=120s
|
||||
# kubectl logs -n stalwart job/stalwart-allow-cluster-ips
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: stalwart-config-access
|
||||
namespace: stalwart
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: stalwart-config-access
|
||||
namespace: stalwart
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get"]
|
||||
resourceNames: ["stalwart-admin-credentials"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: stalwart-config-access
|
||||
namespace: stalwart
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: stalwart-config-access
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: stalwart-config-access
|
||||
namespace: stalwart
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: stalwart-allow-cluster-ips
|
||||
namespace: stalwart
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 3600 # Keep logs for 1 hour
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: stalwart-security-fix
|
||||
spec:
|
||||
serviceAccountName: stalwart-config-access
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: fix-security
|
||||
image: bitnami/kubectl:latest
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
echo "=== Stalwart HTTP Listener Security Fix ==="
|
||||
echo "Configuring Stalwart to allow internal cluster IPs for the HTTP listener"
|
||||
echo ""
|
||||
|
||||
# Wait for Stalwart pod to be ready
|
||||
echo "Waiting for stalwart-0 pod to be ready..."
|
||||
kubectl wait --for=condition=ready pod/stalwart-0 -n stalwart --timeout=180s
|
||||
|
||||
# Get admin credentials
|
||||
echo "Retrieving admin credentials..."
|
||||
ADMIN_EMAIL=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-email}' | base64 -d)
|
||||
ADMIN_PASSWORD=$(kubectl get secret stalwart-admin-credentials -n stalwart -o jsonpath='{.data.admin-password}' | base64 -d)
|
||||
|
||||
echo "Admin email: $ADMIN_EMAIL"
|
||||
|
||||
# Use kubectl exec to access Stalwart's admin API from localhost
|
||||
# The HTTP listener allows localhost connections even when blocking other IPs
|
||||
echo ""
|
||||
echo "Accessing Stalwart admin API via kubectl exec..."
|
||||
|
||||
# Test API access first
|
||||
echo "Testing API connectivity..."
|
||||
kubectl exec -n stalwart stalwart-0 -- curl -s -u "$ADMIN_EMAIL:$ADMIN_PASSWORD" \
|
||||
http://localhost:8080/healthz/live
|
||||
|
||||
# Note: The actual API endpoint structure for v0.16.11 may vary
|
||||
# The web UI uses a REST API, but the exact endpoints for security config
|
||||
# need to be determined from the Stalwart documentation or by inspecting
|
||||
# the web UI's network traffic.
|
||||
|
||||
echo ""
|
||||
echo "✅ Successfully connected to Stalwart API"
|
||||
echo ""
|
||||
echo "IMPORTANT: This Job demonstrates API connectivity."
|
||||
echo "The actual security configuration change requires:"
|
||||
echo "1. Identifying the correct API endpoint for security settings"
|
||||
echo "2. Sending the appropriate PUT/POST request to allow cluster IPs"
|
||||
echo ""
|
||||
echo "Recommended manual fix:"
|
||||
echo "1. Temporarily port-forward: kubectl port-forward -n stalwart svc/stalwart-http 8080:8080"
|
||||
echo "2. Access https://mail.basicstack.de from your browser"
|
||||
echo "3. Login with admin credentials"
|
||||
echo "4. Navigate to Settings > Security"
|
||||
echo "5. Disable IP filtering for the HTTP listener or add 10.244.0.0/16 to allowed IPs"
|
||||
|
||||
exit 0
|
||||
|
|
@ -5,7 +5,4 @@ metadata:
|
|||
name: stalwart-bootstrap-config
|
||||
namespace: stalwart
|
||||
data:
|
||||
# DEV-476: Bootstrap now points at the PostgreSQL config/data store.
|
||||
# PGPASSWORD is injected into the stalwart container from
|
||||
# secret/stalwart-postgres-credentials (see stalwart-fresh-deployment.yaml).
|
||||
config.json: '{"@type":"PostgreSql","host":"stalwart-postgres","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"PGPASSWORD"},"useTls":false}'
|
||||
config.json: '{"@type":"RocksDb","path":"/var/lib/stalwart"}'
|
||||
|
|
|
|||
|
|
@ -10,22 +10,12 @@ data:
|
|||
# Stalwart Mail Server Configuration
|
||||
#
|
||||
|
||||
# DEV-476: primary store is PostgreSQL (see stalwart-bootstrap-config.yaml).
|
||||
# Note: the container starts with `--config /etc/stalwart/config.json`, so
|
||||
# this stalwart.toml is not read at runtime for a live pod — the bootstrap
|
||||
# JSON is authoritative. Kept in sync here as documentation and for any
|
||||
# one-shot tooling that references the toml.
|
||||
[store]
|
||||
data = "postgres"
|
||||
data = "rocksdb"
|
||||
|
||||
[store.postgres]
|
||||
type = "postgresql"
|
||||
host = "stalwart-postgres"
|
||||
port = 5432
|
||||
database = "stalwart"
|
||||
user = "stalwart"
|
||||
password = "%{env:PGPASSWORD}%"
|
||||
tls.enable = false
|
||||
[store.rocksdb]
|
||||
type = "rocksdb"
|
||||
path = "/var/lib/stalwart"
|
||||
|
||||
#
|
||||
# Server Configuration
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ spec:
|
|||
mountPath: /var/lib/stalwart
|
||||
containers:
|
||||
- name: stalwart
|
||||
image: stalwartlabs/stalwart:v0.16.18
|
||||
image: stalwartlabs/stalwart:v0.16.11
|
||||
ports:
|
||||
- containerPort: 25
|
||||
name: smtp
|
||||
|
|
@ -145,12 +145,6 @@ spec:
|
|||
value: "/etc/stalwart/certs/tls.crt"
|
||||
- name: TLS_PRIVATE_KEY
|
||||
value: "/etc/stalwart/certs/tls.key"
|
||||
# DEV-476: bootstrap config.json resolves authSecret via this env var.
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-postgres-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/stalwart
|
||||
|
|
@ -295,7 +289,7 @@ spec:
|
|||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 1800 # 30 minute timeout (was 600s; retries + affinity settling can push past 10 min)
|
||||
activeDeadlineSeconds: 600 # 10 minute timeout
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
|
|
@ -303,20 +297,9 @@ spec:
|
|||
spec:
|
||||
serviceAccountName: stalwart-backup
|
||||
restartPolicy: OnFailure
|
||||
# Co-locate with stalwart-0 so both pods share the same hcloud block volume
|
||||
# attachment (RWO). Without this the backup pod can be scheduled on a
|
||||
# different node and hits FailedAttachVolume/Multi-Attach (DEV-468).
|
||||
affinity:
|
||||
podAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app: stalwart
|
||||
statefulset.kubernetes.io/pod-name: stalwart-0
|
||||
topologyKey: kubernetes.io/hostname
|
||||
containers:
|
||||
- name: backup
|
||||
image: alpine:3.24
|
||||
image: alpine:3.19
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: stalwart-postgres-backup
|
||||
namespace: stalwart
|
||||
labels:
|
||||
app: stalwart-postgres-backup
|
||||
spec:
|
||||
schedule: "30 2 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 1800
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: stalwart-postgres-backup
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: backup
|
||||
image: postgres:16-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
BACKUP_START=$(date +%s)
|
||||
echo "=== Starting Stalwart Postgres backup at $(date -u) ==="
|
||||
|
||||
# pg_dump already in image; restic is small and only needed here.
|
||||
apk add --no-cache restic
|
||||
|
||||
export RESTIC_REPOSITORY="s3:${S3_ENDPOINT}/${S3_BUCKET}/stalwart"
|
||||
echo "Repository: ${RESTIC_REPOSITORY}"
|
||||
|
||||
# Repo is shared with the RocksDB backup, so it already exists;
|
||||
# init only if the RocksDB side has not created it yet.
|
||||
restic snapshots > /dev/null 2>&1 || restic init
|
||||
|
||||
echo "Streaming pg_dump | gzip -> restic (tag stalwart-postgres)..."
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
|
||||
-h stalwart-postgres.stalwart.svc.cluster.local \
|
||||
-U "${POSTGRES_USER}" \
|
||||
-d "${POSTGRES_DB}" \
|
||||
--no-owner --no-acl \
|
||||
| gzip -c \
|
||||
| restic backup \
|
||||
--stdin \
|
||||
--stdin-filename stalwart-postgres.sql.gz \
|
||||
--tag stalwart-postgres \
|
||||
--tag daily \
|
||||
--host stalwart-postgres-k8s
|
||||
|
||||
echo "Forget/prune (tag=stalwart-postgres, independent retention)..."
|
||||
restic forget \
|
||||
--tag stalwart-postgres \
|
||||
--keep-daily 14 \
|
||||
--keep-weekly 8 \
|
||||
--keep-monthly 6 \
|
||||
--prune
|
||||
|
||||
echo "Latest stalwart-postgres snapshots:"
|
||||
restic snapshots --tag stalwart-postgres --compact || true
|
||||
|
||||
BACKUP_END=$(date +%s)
|
||||
echo "=== Postgres backup complete in $((BACKUP_END - BACKUP_START))s at $(date -u) ==="
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-postgres-credentials
|
||||
key: POSTGRES_USER
|
||||
- name: POSTGRES_DB
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-postgres-credentials
|
||||
key: POSTGRES_DB
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-postgres-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: RESTIC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-s3-backup
|
||||
key: restic-password
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-s3-backup
|
||||
key: access-key
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-s3-backup
|
||||
key: secret-key
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-s3-backup
|
||||
key: endpoint
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-s3-backup
|
||||
key: bucket
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: stalwart-postgres-credentials
|
||||
namespace: stalwart
|
||||
spec:
|
||||
encryptedData:
|
||||
POSTGRES_DB: AgBl0xkIxrdRjAlv0JP/rPXwV4sbfFcLc1TwPKZ0VC7+MXd9K1s+FMEottjxe3jUMeVk1ddI0OelZ7PlZcVeocvC4fq02djtagLcjSBn5AFqnn0vJO+sAKsnqxuyQv7IIuNhqL5a88obOeuKJPB325IsXLEhFUrCjXDz4KTjn1RW4p9sWcbn1jBbGYHSRpp7GMn6LgkJOjhbpiQeptr270dTiOjHAxI3Ky507v5qAA8m5VwjHK9pXN8QkkYqIq3mXp4WWRt9LVAfnaQ+tnxPpeZBgzf3pV2Hj4yDpFuans8xGft9e5Er2ygEwpUD+AOEJdjSntzvkx4K5Lprv3ud1PXY+f5LZb7w41Ex2fwQtGKWepQyAD1LSqtapkOVirgWOfmGxR0NcyWTpqu6VuVYPgUq+I37CCmTdVKbBmqhSIMRll0E7Cnzse1wiLIugyHtidg5whWpYiHiByEKCxwmd0LKAGZ2gTrIOctRmpLb5UYvDxr+T/mrG20p4SymFucaT7MrvMXRnewB5HSvhooXgFyTq7zNL6mlVgK5oQI1RRJ71dNtrjs2Z/RLCzffbV8K3oSM8lFuN0BBea5M8QorJoraqwa7cHPyCCCbuJuXBMo8ZDC96etvoICiBFwt4hFcxuOS43XX6CLZiVq516/TN4VLm558ai9KfIZvEkaIivJeFqdDuR2oVUV9nAQc6nqMHHnRiFmPJpUpXg==
|
||||
POSTGRES_PASSWORD: AgASUNVrGocpv2dDca4E/cRK8IISrCS5z40ObiAzxsZql/+NdRfMzk1fUl5QF94KAgm3xe1V901SWNJvNVG/qKfHt83K8ARlWyHxFH8etc9diMQgXD4EuW21wVyWbI1lRaDSP56yqMMlOpgrgfLhN45BToiD0Dqe6s1lAdO7ozyBcNO9kEkHzM/bFmGgbDX/u8/c4bXXV3VSQueBPVDPMToEzIazCJPDQKn7PlmOzmANQ8xWa5avRpQfpsPcoatevDMnYDGyVEAlxBIAbaf8muyVD7yce6rFCYAQCUhqwyiu3YPVNBJT1PdmtNwZ2WHu42nuLcOkUoEWLtX/Ex/fnBhlvKvFBKCLBR2XPLjY/GkfoYXFN/+RbwCdvJnj77v11AxBLPrNpLx7sQmS0Gxz2ivzxexjuSVibyVO2pYqsi4idOOc8OtUEwbxN1WjncKMRz0WB5k2bOE4/DIjYIZ5KYz/+RVi6fR0CMKPuCgq76j3DHazMEKCicCdqw0iZaqyXKTIf3jzaAkdC0v6c/1by4d5z2NppDiGKwvLcwHYHlq2IKPPCjSdpu/efGI2CxijcHLtYpa5N2gg40T9oJRsf6NWu9lrvtdsoKiaWCo4hnkqmhGsNXiAqVerd7BslEkD8h5bnZYZPZ6IVZCZw86rwDOmICOj79lqtIcHDJyJra+n7wC3FlokBwR1bR3QcVJcqL2V2OBqlHQ2SCHv+uxL8d8IkHibnj23FJHlwYRBFNvw
|
||||
POSTGRES_USER: AgA6zhC4W9S/+X+o2Gcj+0WfapqtUjZEmxpnMxZwQpYk5qaR1TrfFezKgKuGbnFWZ1HeUJM1gTlsOvg2wIlbKLHd8on8KeK4JhtR5GUd75icaW4/SSwWdODlHP6UlY1rxbc2uYxwLkbauRcjY1DKl633n/BXNYzAKFd1MbN0XsD3cMHgwDHy2MD6qW8VS80BEqj6ljNUPC/wJTR1CISH+rAiWmuuX+mf8whjqZxulqFFWaPuscC9ceHZT2bdhkC5uSBRXc5t6ZTiroRW50cT08J+u0W2Xm56ygJ6uuIxBWgTWKcAE1UNRIIXLOlK19j5yR6e216TaDTj06Spu6l7thfdPiRAx1kXMJhCK3Rhh5umWVrXxyomfA6IweUeI/kfLC3q3I5UGezZUMTou4QUsRVYwxnuyVsiDQJ0MWd0p5RVN690PbLxFebsg3YFW1R33UAvzhukO/Ct+QvWG+FfVUnaVrMh3nM+UnJau7w+9hdToG9wwvwBwCJRNRpCDo3ev1d+0AuBURXWnOSCUcifDsF6rKfCVxRMEVB5dLj5IYN3j0XRRMkvsE5UAHCFC5QFaUQD/5V6LSRvX1vV/aD88Ver57P0P4bYnNHLj02GP9OIV7HTv2jDJz3ZycfchXjRw9cIM7WmmF2HWlyCT608HD8NKf+dZlF7FJr+REYGCmNCvA4PIB6g6kwIOZzQ0KjVqFEJ0z+G7xXXjQ==
|
||||
stalwart-db-password: AgBhrofehsfXWbdixnkOqdsNu8y9eDNIBW1g5ZgHcNAX0uq5Qa/84KguJZcNm4nzIuTs5KZTjTLvuync4cypKVKoqzthRwGWL01uAc3I8+gzjAsSLIzfWa1Dq9BZTEA0Q7oO3dBmdooAy+e5WR0MRhsF9VQWWwxM/UzQad4ihM/PBlkfvx+6Ql0pTp8m/K+E5YqXzNnla5eCra+zlzG3JWF7SOF82AnM6sLKzsHQzTv31z4I6pJTyoEtBNxbjsmmuCIh+fGPBusqTEY3vjEwVJJEaviq2B6rR9uz/sNF5bc6XtX0nAeyeQR4WURrTmGBDgwI85Z0/vDBsD9D+NTuoTitw4kTO8JcE2H4FUY8afurxpODvx6mmLaYnV8c7hLz4l33k9zCpjb8gHfNqQR/sKRu+iSeej12E0MgxUXnSL9nhLBltwA/6KpGHz4TxTG58/AopF9yDQIKOsZF7Lj4mJC3AQ7qlpwF8j/5wiMEr8OqyhAiOm6wmzIArZzb91SVlKitLmQEPBdOzx2kboZmeSJi/P52R6qFYm/bamGPTI5m6x+MnEXgur37eubnSlY/v65sfhdJb+Gm4B1QyS06CSG4VCgom1T5Fpc1zTWxgYN85OaHQVFR1T5jhOCgZPS01/rN3R1D/wryve8n2aGLZhf+ad8vKyXbcg25lIchzwWOXYk5nM0NOshw3Dk2/IcFfwja3QGak0ps7syZUHPY+wCPL9mrd7F4OqaLrZmbZHrv
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: stalwart-postgres-credentials
|
||||
namespace: stalwart
|
||||
type: Opaque
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: stalwart-postgres
|
||||
namespace: stalwart
|
||||
labels:
|
||||
app: stalwart-postgres
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: stalwart-postgres
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: stalwart-postgres
|
||||
namespace: stalwart
|
||||
labels:
|
||||
app: stalwart-postgres
|
||||
spec:
|
||||
serviceName: stalwart-postgres
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: stalwart-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: stalwart-postgres
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: csi.hetzner.cloud/location
|
||||
operator: In
|
||||
values:
|
||||
- fsn1
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: stalwart-postgres-credentials
|
||||
env:
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: postgres-data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
subPath: pgdata
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: postgres-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
apiVersion: v2
|
||||
name: velero
|
||||
description: Velero cluster-reconstruction backups for BasicStack (Kopia + Hetzner S3)
|
||||
type: application
|
||||
version: 1.0.0
|
||||
appVersion: "1.18.1"
|
||||
|
||||
dependencies:
|
||||
- name: velero
|
||||
version: 12.1.0
|
||||
repository: https://vmware-tanzu.github.io/helm-charts
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
velero:
|
||||
# Pinned chart version 12.1.0 (Velero appVersion 1.18.1).
|
||||
# AWS plugin follows Velero minor version: v1.14.x for Velero 1.18.x.
|
||||
image:
|
||||
repository: velero/velero
|
||||
tag: v1.18.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Bring the AWS object-storage plugin in as an initContainer that copies
|
||||
# the binary into /plugins so the Velero server can load it.
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: velero/velero-plugin-for-aws:v1.14.2
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- mountPath: /target
|
||||
name: plugins
|
||||
|
||||
# Kopia file-system backup requires the node-agent daemonset on every
|
||||
# node whose PVC data we back up.
|
||||
deployNodeAgent: true
|
||||
|
||||
# BackupStorageLocation credentials come from the SealedSecret
|
||||
# apps/velero/velero-s3-credentials-sealed.yaml (key: cloud, INI format
|
||||
# with aws_access_key_id / aws_secret_access_key for the dedicated
|
||||
# `velero-prod` Hetzner OS credential from DEV-570).
|
||||
credentials:
|
||||
useSecret: true
|
||||
existingSecret: velero-s3-credentials
|
||||
|
||||
# Kopia backup-repository password (velero-repo-credentials Secret,
|
||||
# key `repository-password`) is auto-discovered by Velero — this is a
|
||||
# hard-coded constant in the Velero server (see pkg/repository/keys),
|
||||
# so no extra wiring is needed. SealedSecret lives at
|
||||
# apps/velero/velero-repo-credentials-sealed.yaml.
|
||||
|
||||
# Turn OFF volume-snapshot CRDs — Hetzner Cloud Volumes have no CSI
|
||||
# snapshot support, so filesystem backup via Kopia is the only path.
|
||||
snapshotsEnabled: false
|
||||
backupsEnabled: true
|
||||
|
||||
configuration:
|
||||
# Kopia uploader (unified repo). This is the Velero server default in
|
||||
# 1.18 but we set it explicitly so any future default change does not
|
||||
# silently rewrite our repos to restic.
|
||||
uploaderType: kopia
|
||||
|
||||
# No CSI feature flag — see snapshotsEnabled comment above.
|
||||
features: ""
|
||||
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
bucket: basicstack-velero
|
||||
default: true
|
||||
credential:
|
||||
name: velero-s3-credentials
|
||||
key: cloud
|
||||
config:
|
||||
# Hetzner Object Storage in Helsinki (hel1). Region is used by
|
||||
# the AWS SDK signer; endpoint overrides the AWS endpoint URL.
|
||||
region: hel1
|
||||
s3ForcePathStyle: "false"
|
||||
s3Url: https://hel1.your-objectstorage.com
|
||||
# publicUrl left unset — this backend is not user-facing.
|
||||
|
||||
# No volumeSnapshotLocation entries — snapshotsEnabled: false above
|
||||
# already suppresses the default snapshot location CRD.
|
||||
|
||||
# Small resource footprint until we know real backup sizing (DEV-573).
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# Small memory request so node-agent fits on the tight 4 GiB CX22 workers
|
||||
# (worker-1/2/3 sit around 95 % memory-request already). Actual peak usage
|
||||
# during Kopia backup lands under 300 MiB per node in practice; limit is
|
||||
# generous so a spike does not OOM-kill mid-backup.
|
||||
nodeAgent:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
# Do not install the bundled `velero` CLI kubectl image — we run the
|
||||
# CLI from cp-1.
|
||||
kubectl:
|
||||
image:
|
||||
repository: bitnami/kubectl
|
||||
resources: {}
|
||||
|
||||
# No schedules here — those live in DEV-573.
|
||||
schedules: {}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: velero-repo-credentials
|
||||
namespace: velero
|
||||
spec:
|
||||
encryptedData:
|
||||
repository-password: AgCn0k/wQK76oVgVu9Eay7yMdxTKOE3RmJIojQcmPMerivAk72vw9F1jjmMDWW3f9X4HMc5MyQS9EXw9aes49047r1fZ4fA1yFgoBhuicS8ULpI4yI1ei9AVov6IQWx3FvarOlUIlQM40KC/es7mwlL/fSE2W5//u1zNp3Z+ujGdqAgBlvO0UQ/pGDNVKhWh6TBMNbzak4IDh6uNSNVAcqrzsh+2+RKPV4RRXnD1oJlTvtSri8EHQZdAAUaWxcIuyhnqeulStvViJKDC3Xw3PamrDXTYEPcAs8P90BVoDoJQwtxJ12Q+yqR1mjLvIZj81CKWXqNSUZDkAatRFjrfUYeiBAgmvnY5HL6f0eoJDVY+F7LfHslQHc4W/2z3YS19++oSKhffM4vwJxAWJqTbsKghvrA6hgFQM6btqGrhuHk3Q5NslKtlFnSgbRBst0FhBoyme0uWeBGk6szKKD8ClRuREJTWfof6pFGRM2O1IcKAhQqgJbamNHtU2yUCkLw732VBW3MJGQ+yLmlegjAmPKVLFDt4Yk5NrtTKrx9HZ3vohoPKwCz1CUizByVOXCsqEO1aogEszaQX9qCVsTCo6M8Zemw6OYCMMcbUcfPHRZmzd8o0j0Y+/6KTjP0fL5l9vR/ocJH+kXLnVsWZiFa3/TXQ0KEG1pPeCSVjQP7WHB1eLtrPH13ikvpyT8u6Ng+FPvOoTD36JrwSj8Een5HVc8oaoV50pXbbGqkepFIXZQ1te5nQL9XSP0AbRMcrqg==
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/compare-options: IgnoreExtraneous
|
||||
creationTimestamp: null
|
||||
name: velero-repo-credentials
|
||||
namespace: velero
|
||||
type: Opaque
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: velero-s3-credentials
|
||||
namespace: velero
|
||||
spec:
|
||||
encryptedData:
|
||||
cloud: AgAo2duq77obiTyFYdVyNANrMtx2iiOT6/cRHyYezhtr6YKlV2whmQBHXuRNhnjQyrmt3Gyl9+t/UZpJbJljqzesstfwRDZ9q1usXILVZE0ZsJB1GyQ5kwN/EQUeqhAYYUSZEUMXLAEkYTD2HE+sbWkNQrGXl1UJfPeEuDkRtCtQ+WNdZ3L53DLCjXQh3JV/4jIG+5bc56Sb33l/xtSSs5YBlUI/cgCM/Di0Xj5pyujxaqiU+vSCrpxiltkK7QDgcti1MbbL5YqeDox5/DwmJZvL1Of2PzODRpgTrDXbylWH8AaNRhzYcSJgv0ZBUwiqsLjFRdU9Oroevg2Uf5dO8FLUbAvx8ER97Afkt6fX7wTgcn3uj9/Ye3+BVOPRIj2l6VaJ3ul15Z+i+UCAAHe3jeujQhVHhZMRXGp5CnjkjJbiFIIL83cq6SfKq21pTEUkk5Ty5V844zDMymHHKvB52/LXS1i0LrELT/QzEKr0y+B3wp/JRFtvsliN+Bsweg4RroBsnVL4d0Aqp0zr3dHaUmTVDTycFGb/GFRTdsC+0JAJlplonJXcU/9h8mSRaBekusFTwMs25C6JHXPcXbI9qf8+qw6n/4vC86FkpyOWKNS2fs5r/Aut+8PC8Z2RcNUjnNUtt/i9FSkoQtIOwHRR0J8Zy+07TyysR0Zz2tB6C895SmHPI5isxQGZoAJHp+ypJwP/nR+RTdB648IfLLZyRINLEt58Nk4NM7B9tXOvk0JBXfB/knXy0wO54PvrnmLfxcE3mIy2/KKM5HK/b+QbSy97VqpXZekbcKJl18foUMsP7YitmP5pWfxBkg9m+3ZDemhCL48mLjr05KVrJ04r8UTMAlWwUw==
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/compare-options: IgnoreExtraneous
|
||||
creationTimestamp: null
|
||||
name: velero-s3-credentials
|
||||
namespace: velero
|
||||
type: Opaque
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# Vikunja (vikunja.basicstack.de)
|
||||
|
||||
Self-hosted task manager. Unified `vikunja/vikunja` container (API + frontend on port 3456). Backed by a namespace-local PostgreSQL. OIDC via Pocket ID (`auth.basicstack.de`); local login disabled.
|
||||
|
||||
## Layout
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `vikunja-db-pvc.yaml` | 10 Gi encrypted PVC for Postgres data |
|
||||
| `vikunja-files-pvc.yaml` | 10 Gi encrypted PVC for user-uploaded attachments (`/app/vikunja/files`) |
|
||||
| `vikunja-postgres-deployment.yaml` | Postgres 16, pinned to `csi.hetzner.cloud/location=fsn1` |
|
||||
| `vikunja-postgres-service.yaml` | ClusterIP `vikunja-postgres:5432` |
|
||||
| `vikunja-deployment.yaml` | Vikunja app, pinned to `fsn1`, waits on Postgres |
|
||||
| `vikunja-service.yaml` | ClusterIP `vikunja:80` → pod `3456` |
|
||||
| `vikunja-ingress.yaml` | Traefik + `letsencrypt-prod` cert |
|
||||
| `vikunja-db-credentials-sealed.yaml` | Postgres user password |
|
||||
| `vikunja-oidc-secret-sealed.yaml` | Pocket ID client id + secret |
|
||||
| `vikunja-app-secrets-sealed.yaml` | Vikunja JWT secret |
|
||||
|
||||
## Node placement
|
||||
|
||||
Both the app and Postgres pin to Hetzner location `fsn1` because their PVCs live on `hcloud-volumes-encrypted` (datacenter-local).
|
||||
|
||||
## OIDC
|
||||
|
||||
- Pocket ID client `Vikunja` (id `cdfff4ee-28ba-4aab-9f47-7e41b4b6b847`), callback `https://vikunja.basicstack.de/auth/openid/basicstack`.
|
||||
- Pocket ID groups `vikunja_admins` and `vikunja_members`.
|
||||
- Vikunja does not map OIDC groups to the admin flag. Promote the first user post-login: `kubectl -n vikunja exec deploy/vikunja -- /app/vikunja/vikunja user set-admin --admin <username>`.
|
||||
- **PKCE must be OFF on the Pocket ID client.** Vikunja 2.x's OAuth redirect (`frontend/src/helpers/redirectToProvider.ts`) does not send `code_challenge`, and the Go backend does not implement PKCE either. With `pkceEnabled: true` on Pocket ID the flow fails at `/authorize` with `invalid_request … 'code_challenge' parameter is missing`. Client is confidential (`isPublic: false`), so `client_secret` still authenticates the token exchange.
|
||||
|
||||
## SMTP
|
||||
|
||||
Reminder emails go through Stalwart on `stalwart-smtp.stalwart.svc.cluster.local:25` using unauthenticated in-cluster relay. Stalwart's port-25 listener trusts the pod-network CIDR (`10.244.0.0/16`), so no SMTP AUTH is required and no sealed secret is needed. Envelope-from is `vikunja@basicstack.de` (the mailbox already exists in Stalwart as principal `g`).
|
||||
|
||||
If a future policy tightens the port-25 CIDR trust, switch to submission on `stalwart-smtp:587` with STARTTLS and a `vikunja-mailer` sealed secret (`VIKUNJA_MAILER_USERNAME`/`VIKUNJA_MAILER_PASSWORD` from a secretKeyRef, `VIKUNJA_MAILER_AUTHTYPE=plain`); creating that secret requires a working Stalwart admin session to (re)set the vikunja principal's password.
|
||||
|
||||
## Change control
|
||||
|
||||
Manifest changes flow through Argo CD (Application: `apps/app-vikunja.yaml`, auto-sync + prune + selfHeal). Sealed secrets are encrypted against the cluster-active sealed-secrets key.
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: vikunja-app-secrets
|
||||
namespace: vikunja
|
||||
spec:
|
||||
encryptedData:
|
||||
jwt-secret: AgA2gSMP5tLy+2BeEe/vO3DHi7v4WO2HRZc2kTD/+BRII/CxCd3SIJYOh5O7OPNOq5Vp+F6oCi6GUg7ljyXuHU1IlHSSYWfjdGKS/mIvlKf2XRASOS1CefNSDKjU+x2FB3LPJG10v/0OG7bLKCj7QGq5y+QNc8dQrV7kqwAonG18iZzl5rcL4vN38B2V6kaO+s/tcz0uwmVOlUqblmv6WwrkMm5bVt9RQxcO5UjmDuf2KlnHBgvhRUhcBCbmrnI7J+FkVtq+2hiWwyz3hXAQAyy6ULMltYZvSy4ADseTV3yBdArUvY9e13kwfVZBjLe5EBzn0SKoQyY0XZQUYDmfQU+U5zFxsZLbYeekf7FUN8zHJ+mgvrT35PGrVazoz2rKI+9BrA7sstYJZ0ZUlj+nKnrbHYyaUhrlu3DYFvTHCOz8ylRxIsM5Ux819+RISpBhKkg6+mIhH2L26larI7UEY576c83TMhmCEOeX0es+MoomsAqeQi8vZ+OT5qz9dFZpzshXmwRYy1Ezm9d992iBF9EskB9IuyaozpA+GU9WwKAOj5GF3izj89nel9QPPQVBGHYvxXXfaXi3yQIpqD54nVPKC1oEhPNeCfVp3YcGSd/HOsl3tT/8G7ipD40jGVk5tgWxpymbvNZmcpm1CIb4SPYHOavJtqBcPY1F9k8a5yYl9sMqd64qzDmTlOQhRVWjD0mqCf2W/W3OvHo4dTu6Fy95jHrzoHsmmfNQQDVytCJK/3SmYUvh51nD8/CyiJxRAq3zPViFHKmRS0E5oCR0+zm3GlZTGwUmu1ROSIlsDgik/khmrZvYFLtF
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: vikunja-app-secrets
|
||||
namespace: vikunja
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
apiVersion: bitnami.com/v1alpha1
|
||||
kind: SealedSecret
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: vikunja-db-credentials
|
||||
namespace: vikunja
|
||||
spec:
|
||||
encryptedData:
|
||||
postgres-password: AgB+P7iBg+tPIDKMGmiPh9HREVb7mKCf1GvQfBLcF61QuWHt0/S1YBYCd4fF3PSA9ok916m6VeOoIQKB0p57XDSEeNewYNsXQgudS0k0JHNdRZER8JhbXpK53E7EY0xw4TD8Tc6WFqyzCFwsL1WpYvvDH3p25W8f3JJfB39D6Up9j5xT5Hufi0G5BJ+tbAujNJooa0SWZbtQPxMWG96fvjmTV5FKxzvwg1JOL4gq0JHBfzo9sKIbQdGpGJjX4JkPLCxA0jU6mcJKEFn80zkDbQBL3g5dXZaYmIAEogZkvytltR4ebI0zyDrSTPqXhJF5bFcRpM43VXz6ZcjgdtG5VPelvAuMqfJ1U7HdYhhJqTt0lvwUi03BjulIJJ52gUjlg2yQioOwfON/y3gRM1TgMHMFCyw32S7vCfR00MKVh7xrb4YceRUTbeHkTd+4TMcdpI11YHeah2scu82tzyD/cGPtpFv2LYyH7JrBcIXXhfIDccBRX4bb2se9VyhpmvEP5Q9SQjazo/BxdnJUYF6kzpJA9oOIjFMXcSDxZh/nwDkA90w4a9iyT9l0+7p3z/RjbXOMKybuQlXpfPrhtCRS6PM3Am5sgnIi/APRaujDa/2a3kCe7K0zZl4Wk3wlAc4mMjnoWs9w//HfMpQByM6GWeHWeoQ/Bj3xCVtACb6SNqTxIs4OP9hAVBnHqouH0O/zjqZNuBa5v7GDwmW6JU1cA+Y/UI4Z/ycu2X6AXBHp5CROheV9PdINTrgS
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
name: vikunja-db-credentials
|
||||
namespace: vikunja
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vikunja-db
|
||||
namespace: vikunja
|
||||
labels:
|
||||
app: vikunja
|
||||
component: database
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hcloud-volumes-encrypted
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue