diff --git a/k8s-ng/apps/llamacpp/deployment.yaml b/k8s-ng/apps/llamacpp/deployment.yaml index dee5a00..a24208b 100644 --- a/k8s-ng/apps/llamacpp/deployment.yaml +++ b/k8s-ng/apps/llamacpp/deployment.yaml @@ -20,6 +20,8 @@ spec: spec: dnsPolicy: Default nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: internal workload/ai: "true" tolerations: - key: workload diff --git a/k8s-ng/apps/web-petting/deployment.yaml b/k8s-ng/apps/web-petting/deployment.yaml index a3ed845..3d37057 100644 --- a/k8s-ng/apps/web-petting/deployment.yaml +++ b/k8s-ng/apps/web-petting/deployment.yaml @@ -16,6 +16,9 @@ spec: annotations: reloader.stakater.com/auto: "true" spec: + nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external containers: - name: web-petting image: ultradesu/web-petting:latest diff --git a/k8s-ng/core/argocd/values.yaml b/k8s-ng/core/argocd/values.yaml index d456377..18ff37b 100644 --- a/k8s-ng/core/argocd/values.yaml +++ b/k8s-ng/core/argocd/values.yaml @@ -1,6 +1,9 @@ # Argo CD Helm Chart Values (k8s-ng cluster) global: + nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external domain: ag.btwiusearch.net logging: format: text diff --git a/k8s-ng/core/cert-manager/values.yaml b/k8s-ng/core/cert-manager/values.yaml index c526c4b..e923bda 100644 --- a/k8s-ng/core/cert-manager/values.yaml +++ b/k8s-ng/core/cert-manager/values.yaml @@ -1,2 +1,12 @@ crds: enabled: true + +nodeSelector: &externalCore + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external +webhook: + nodeSelector: *externalCore +cainjector: + nodeSelector: *externalCore +startupapicheck: + nodeSelector: *externalCore diff --git a/k8s-ng/core/cnpg/values.yaml b/k8s-ng/core/cnpg/values.yaml index 0915930..91646f3 100644 --- a/k8s-ng/core/cnpg/values.yaml +++ b/k8s-ng/core/cnpg/values.yaml @@ -1,4 +1,8 @@ # CloudNativePG operator (chart 0.29.x = operator 1.30.x) +nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external + crds: create: true diff --git a/k8s-ng/core/external-secrets/values.yaml b/k8s-ng/core/external-secrets/values.yaml index 1b4551c..06c7d12 100644 --- a/k8s-ng/core/external-secrets/values.yaml +++ b/k8s-ng/core/external-secrets/values.yaml @@ -1 +1,9 @@ installCRDs: true + +nodeSelector: &externalCore + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external +webhook: + nodeSelector: *externalCore +certController: + nodeSelector: *externalCore diff --git a/k8s-ng/core/gitea/deployment.yaml b/k8s-ng/core/gitea/deployment.yaml index 8c04ae0..96bf586 100644 --- a/k8s-ng/core/gitea/deployment.yaml +++ b/k8s-ng/core/gitea/deployment.yaml @@ -20,6 +20,8 @@ spec: app: gitea spec: nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music volumes: - name: storage diff --git a/k8s-ng/core/gitea/runner.yaml b/k8s-ng/core/gitea/runner.yaml index d2c5d02..8a46d7d 100644 --- a/k8s-ng/core/gitea/runner.yaml +++ b/k8s-ng/core/gitea/runner.yaml @@ -35,6 +35,8 @@ spec: - name: ndots value: "2" nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: internal workload/ai: "true" tolerations: - key: workload diff --git a/k8s-ng/core/gitea/user-unban-cronjob.yaml b/k8s-ng/core/gitea/user-unban-cronjob.yaml index 672320f..20c0386 100644 --- a/k8s-ng/core/gitea/user-unban-cronjob.yaml +++ b/k8s-ng/core/gitea/user-unban-cronjob.yaml @@ -18,6 +18,8 @@ spec: spec: restartPolicy: OnFailure nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music volumes: - name: storage diff --git a/k8s-ng/core/gpu/kustomization.yaml b/k8s-ng/core/gpu/kustomization.yaml index 8a3c212..47d3d13 100644 --- a/k8s-ng/core/gpu/kustomization.yaml +++ b/k8s-ng/core/gpu/kustomization.yaml @@ -22,5 +22,7 @@ patches: - op: replace path: /spec/template/spec/nodeSelector value: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: internal kubernetes.io/arch: amd64 workload/ai: "true" diff --git a/k8s-ng/core/gpu/values.yaml b/k8s-ng/core/gpu/values.yaml index d9090ed..717f62c 100644 --- a/k8s-ng/core/gpu/values.yaml +++ b/k8s-ng/core/gpu/values.yaml @@ -31,5 +31,7 @@ tolerations: node_selector_enabled: true node_selector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: internal kubernetes.io/arch: amd64 workload/ai: "true" diff --git a/k8s-ng/core/immich-db/cluster.yaml b/k8s-ng/core/immich-db/cluster.yaml index 4723c0b..da28e14 100644 --- a/k8s-ng/core/immich-db/cluster.yaml +++ b/k8s-ng/core/immich-db/cluster.yaml @@ -31,6 +31,8 @@ spec: immich.hexor.cy/volume: postgres affinity: nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music postgresql: shared_preload_libraries: [vectors.so] diff --git a/k8s-ng/core/immich-db/storage.yaml b/k8s-ng/core/immich-db/storage.yaml index 4142318..e3baf19 100644 --- a/k8s-ng/core/immich-db/storage.yaml +++ b/k8s-ng/core/immich-db/storage.yaml @@ -11,6 +11,8 @@ spec: spec: restartPolicy: OnFailure nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music containers: - name: prepare diff --git a/k8s-ng/core/keycloak/auth-proxy.yaml b/k8s-ng/core/keycloak/auth-proxy.yaml index caa9f5d..8bcf71d 100644 --- a/k8s-ng/core/keycloak/auth-proxy.yaml +++ b/k8s-ng/core/keycloak/auth-proxy.yaml @@ -20,6 +20,9 @@ spec: labels: app: auth-proxy spec: + nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external containers: - name: auth-proxy image: ultradesu/rsauth2-proxy:latest diff --git a/k8s-ng/core/keycloak/values.yaml b/k8s-ng/core/keycloak/values.yaml index 795760d..f50e756 100644 --- a/k8s-ng/core/keycloak/values.yaml +++ b/k8s-ng/core/keycloak/values.yaml @@ -1,5 +1,9 @@ replicas: 1 +nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external + image: repository: quay.io/keycloak/keycloak tag: "26.5.6" diff --git a/k8s-ng/core/postgresql/cluster.yaml b/k8s-ng/core/postgresql/cluster.yaml index 5cf00b8..897b9e5 100644 --- a/k8s-ng/core/postgresql/cluster.yaml +++ b/k8s-ng/core/postgresql/cluster.yaml @@ -6,7 +6,7 @@ # master. A failover swaps the roles; switch back with # kubectl cnpg promote main # -# Nodes eligible for instances carry the label workload/postgres=true. To move a +# Eligible nodes are core/external and carry workload/postgres=true. To move a # replica: label the new node, unlabel the old one, then recreate the instance # (kubectl cnpg destroy main ); the operator re-clones it from the primary. # @@ -36,6 +36,8 @@ spec: podAntiAffinityType: required topologyKey: kubernetes.io/hostname nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external workload/postgres: "true" resources: diff --git a/k8s-ng/core/prom-stack/README.md b/k8s-ng/core/prom-stack/README.md index 84cac7f..3b9325f 100644 --- a/k8s-ng/core/prom-stack/README.md +++ b/k8s-ng/core/prom-stack/README.md @@ -14,6 +14,12 @@ chunks, or Grafana database are copied from the old cluster. - Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on the large local disk of node `music`. +Stack services select `homelab.hexor.cy/pool=core` and +`homelab.hexor.cy/exposure=external`. Prometheus, Alertmanager and Loki also +select `music` because their volumes are local. Grafana, the operators, Alloy +and the Loki gateway can run on either `master` or `music`. Node Exporter and +Loki Canary are not restricted to these groups. + ## Storage and retention The PreSync hook creates these directories on `music`: diff --git a/k8s-ng/core/prom-stack/alloy-values.yaml b/k8s-ng/core/prom-stack/alloy-values.yaml index b952628..ea11a37 100644 --- a/k8s-ng/core/prom-stack/alloy-values.yaml +++ b/k8s-ng/core/prom-stack/alloy-values.yaml @@ -123,7 +123,8 @@ controller: type: deployment replicas: 1 nodeSelector: - kubernetes.io/hostname: music + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical minReadySeconds: 10 terminationGracePeriodSeconds: 60 diff --git a/k8s-ng/core/prom-stack/loki-values.yaml b/k8s-ng/core/prom-stack/loki-values.yaml index fcbba38..c5f9c74 100644 --- a/k8s-ng/core/prom-stack/loki-values.yaml +++ b/k8s-ng/core/prom-stack/loki-values.yaml @@ -37,6 +37,8 @@ singleBinary: replicas: 1 sidecar: false nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music priorityClassName: monitoring-critical terminationGracePeriodSeconds: 120 @@ -78,14 +80,12 @@ sidecar: gateway: enabled: true - # A second replica cannot improve availability while both replicas are pinned - # to the only storage node, and the chart's anti-affinity prevents scheduling. + # Gateway replicas can run on either external core node; Loki storage stays on music. replicas: 1 nodeSelector: - kubernetes.io/hostname: music - # The chart defaults to required pod anti-affinity. With a single eligible - # node that deadlocks every rolling update because the surge pod cannot be - # scheduled until the old pod is gone. Both pods may briefly share music. + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external + # Allow a surge pod to share a node during rolling updates. affinity: null priorityClassName: monitoring-critical verboseLogging: false diff --git a/k8s-ng/core/prom-stack/prom-values.yaml b/k8s-ng/core/prom-stack/prom-values.yaml index df9ab6d..f9ba1bb 100644 --- a/k8s-ng/core/prom-stack/prom-values.yaml +++ b/k8s-ng/core/prom-stack/prom-values.yaml @@ -41,6 +41,8 @@ alertmanager: replicas: 1 retention: 120h nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music priorityClassName: monitoring-critical alertmanagerConfigSelector: @@ -73,6 +75,8 @@ prometheus: prometheusSpec: replicas: 1 nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music priorityClassName: monitoring-critical scrapeInterval: 30s @@ -147,7 +151,8 @@ grafana: annotations: reloader.stakater.com/auto: "true" nodeSelector: - kubernetes.io/hostname: music + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: @@ -229,8 +234,14 @@ grafana: - gf.hexor.cy prometheusOperator: + admissionWebhooks: + patch: + nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external nodeSelector: - kubernetes.io/hostname: music + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: @@ -250,7 +261,8 @@ prometheusOperator: kube-state-metrics: nodeSelector: - kubernetes.io/hostname: music + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: diff --git a/k8s-ng/core/prom-stack/storage.yaml b/k8s-ng/core/prom-stack/storage.yaml index 9f7e441..d3266cf 100644 --- a/k8s-ng/core/prom-stack/storage.yaml +++ b/k8s-ng/core/prom-stack/storage.yaml @@ -15,6 +15,8 @@ spec: spec: restartPolicy: OnFailure nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music containers: - name: prepare diff --git a/k8s-ng/core/reloader/values.yaml b/k8s-ng/core/reloader/values.yaml index dd43454..2b86d03 100644 --- a/k8s-ng/core/reloader/values.yaml +++ b/k8s-ng/core/reloader/values.yaml @@ -16,6 +16,9 @@ reloader: rbac: enabled: true deployment: + nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external replicas: 2 resources: requests: diff --git a/k8s-ng/core/secret-stores/bitwarden-cli.yaml b/k8s-ng/core/secret-stores/bitwarden-cli.yaml index 6b679de..41aae8a 100644 --- a/k8s-ng/core/secret-stores/bitwarden-cli.yaml +++ b/k8s-ng/core/secret-stores/bitwarden-cli.yaml @@ -29,6 +29,8 @@ spec: app.kubernetes.io/instance: bitwarden-cli spec: nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/arch: amd64 containers: - name: bitwarden-cli diff --git a/k8s-ng/core/vaultwarden/deployment.yaml b/k8s-ng/core/vaultwarden/deployment.yaml index b380b47..76bd63d 100644 --- a/k8s-ng/core/vaultwarden/deployment.yaml +++ b/k8s-ng/core/vaultwarden/deployment.yaml @@ -17,6 +17,8 @@ spec: app: vaultwarden spec: nodeSelector: + homelab.hexor.cy/pool: core + homelab.hexor.cy/exposure: external kubernetes.io/hostname: music containers: - name: vaultwarden