Place core services and web-petting on external core nodes

k8s-ng/core/prom-stack/README.md

@@ -14,6 +14,12 @@ chunks, or Grafana database are copied from the old cluster.
 - Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on
   the large local disk of node `music`.
 
+Stack services select `homelab.hexor.cy/pool=core` and
+`homelab.hexor.cy/exposure=external`. Prometheus, Alertmanager and Loki also
+select `music` because their volumes are local. Grafana, the operators, Alloy
+and the Loki gateway can run on either `master` or `music`. Node Exporter and
+Loki Canary are not restricted to these groups.
+
 ## Storage and retention
 
 The PreSync hook creates these directories on `music`:

k8s-ng/core/prom-stack/alloy-values.yaml

@@ -123,7 +123,8 @@ controller:
   type: deployment
   replicas: 1
   nodeSelector:
-    kubernetes.io/hostname: music
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   priorityClassName: monitoring-critical
   minReadySeconds: 10
   terminationGracePeriodSeconds: 60

k8s-ng/core/prom-stack/loki-values.yaml

@@ -37,6 +37,8 @@ singleBinary:
   replicas: 1
   sidecar: false
   nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
     kubernetes.io/hostname: music
   priorityClassName: monitoring-critical
   terminationGracePeriodSeconds: 120
@@ -78,14 +80,12 @@ sidecar:
 
 gateway:
   enabled: true
-  # A second replica cannot improve availability while both replicas are pinned
-  # to the only storage node, and the chart's anti-affinity prevents scheduling.
+  # Gateway replicas can run on either external core node; Loki storage stays on music.
   replicas: 1
   nodeSelector:
-    kubernetes.io/hostname: music
-  # The chart defaults to required pod anti-affinity. With a single eligible
-  # node that deadlocks every rolling update because the surge pod cannot be
-  # scheduled until the old pod is gone. Both pods may briefly share music.
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  # Allow a surge pod to share a node during rolling updates.
   affinity: null
   priorityClassName: monitoring-critical
   verboseLogging: false

k8s-ng/core/prom-stack/prom-values.yaml

@@ -41,6 +41,8 @@ alertmanager:
     replicas: 1
     retention: 120h
     nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
       kubernetes.io/hostname: music
     priorityClassName: monitoring-critical
     alertmanagerConfigSelector:
@@ -73,6 +75,8 @@ prometheus:
   prometheusSpec:
     replicas: 1
     nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
       kubernetes.io/hostname: music
     priorityClassName: monitoring-critical
     scrapeInterval: 30s
@@ -147,7 +151,8 @@ grafana:
   annotations:
     reloader.stakater.com/auto: "true"
   nodeSelector:
-    kubernetes.io/hostname: music
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   priorityClassName: monitoring-critical
   resources:
     requests:
@@ -229,8 +234,14 @@ grafana:
           - gf.hexor.cy
 
 prometheusOperator:
+  admissionWebhooks:
+    patch:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
   nodeSelector:
-    kubernetes.io/hostname: music
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   priorityClassName: monitoring-critical
   resources:
     requests:
@@ -250,7 +261,8 @@ prometheusOperator:
 
 kube-state-metrics:
   nodeSelector:
-    kubernetes.io/hostname: music
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   priorityClassName: monitoring-critical
   resources:
     requests:

k8s-ng/core/prom-stack/storage.yaml

@@ -15,6 +15,8 @@ spec:
     spec:
       restartPolicy: OnFailure
       nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
         kubernetes.io/hostname: music
       containers:
         - name: prepare