Fixed alert reporting, instance label, gitea limits

k8s-ng/core/gitea/deployment.yaml

@@ -31,11 +31,11 @@ spec:
           image: gitea/gitea:28.0.0
           resources:
             requests:
-              memory: 512Mi
-              cpu: 200m
+              memory: 1Gi
+              cpu: "1"
             limits:
-              memory: 2Gi
-              cpu: 1500m
+              memory: 4Gi
+              cpu: "4"
           env:
             - name: GITEA__service__REGISTER_MANUAL_CONFIRM
               value: "true"

k8s-ng/core/prom-stack/README.md

@@ -36,6 +36,17 @@ them. This protects data from a manifest rename, but it is not a backup. Back up
   custom fields `IDM Client ID`, `IDM Client Secret`, `TG Token`, and `TG ID`;
 - DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster.
 
+## Alerting and target identities
+
+Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and
+`InfoInhibitor` remain visible in Prometheus but route to the null receiver.
+Warning and critical notifications are grouped for two minutes and rendered as
+Telegram HTML with escaped annotations and labelled graph/runbook links.
+
+Node Exporter keeps scraping the discovered node address, but rewrites the
+public `instance` label from Kubernetes discovery to `<node-name>:9100`. Grafana
+variables and alerts therefore remain stable when node IP addresses change.
+
 After Argo CD reports the application healthy, verify targets, rules and log
 delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana
 uses its own Keycloak OIDC client.

k8s-ng/core/prom-stack/alertmanager-config.yaml

@@ -8,9 +8,11 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   route:
-    groupBy: [namespace, alertname]
-    groupWait: 30s
-    groupInterval: 5m
+    groupBy: [cluster, namespace, alertname]
+    # A two-minute grace period absorbs alerts which resolve immediately and
+    # never need to interrupt a human.
+    groupWait: 2m
+    groupInterval: 10m
     repeatInterval: 12h
     receiver: telegram
     routes:
@@ -19,6 +21,13 @@ spec:
           - name: alertname
             value: Watchdog|InfoInhibitor
             matchType: =~
+      # Informational alerts remain queryable in Prometheus but do not page the
+      # Telegram chat. Warning and critical alerts still use the parent route.
+      - receiver: "null"
+        matchers:
+          - name: severity
+            value: info
+            matchType: =
   receivers:
     - name: telegram
       telegramConfigs:
@@ -29,15 +38,23 @@ spec:
           # AlertmanagerConfig only supports an integer here, not a Secret ref.
           chatID: 124317807
           sendResolved: true
+          parseMode: HTML
           message: |-
-            {{ if eq .Status "firing" }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
+            {{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
+            {{ with .CommonLabels.alertname }}<b>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
             {{ range .Alerts }}
-            {{ .Labels.alertname }} ({{ .Labels.severity }})
-            {{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}
-            {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
-            {{ if .Labels.namespace }}Namespace: {{ .Labels.namespace }}{{ end }}
-            {{ if .Labels.pod }}Pod: {{ .Labels.pod }}{{ end }}
-            {{ if .Labels.node }}Node: {{ .Labels.node }}{{ end }}
-            {{ .GeneratorURL }}
+
+            {{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ with .Annotations.summary }}<b>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</b>{{ else }}<b>{{ .Labels.alertname | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</b>{{ end }}
+            {{ with .Annotations.description }}{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}{{ end }}
+            {{ if or .Labels.namespace .Labels.pod .Labels.container .Labels.node .Labels.instance }}
+            <b>Context</b>
+            {{ with .Labels.namespace }}Namespace: <code>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
+            {{ with .Labels.pod }}Pod: <code>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
+            {{ with .Labels.container }}Container: <code>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
+            {{ with .Labels.node }}Node: <code>{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
+            {{ if and (not .Labels.node) .Labels.instance }}Instance: <code>{{ .Labels.instance | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}</code>{{ end }}
+            {{ end }}
+            Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
+            <a href="{{ .GeneratorURL | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}">📈 Open graph</a>{{ with .Annotations.runbook_url }} · <a href="{{ . | reReplaceAll "&" "&amp;" | reReplaceAll "<" "&lt;" | reReplaceAll ">" "&gt;" }}">📖 Runbook</a>{{ end }}
             {{ end }}
     - name: "null"

k8s-ng/core/prom-stack/prom-values.yaml

@@ -188,11 +188,11 @@ grafana:
   sidecar:
     resources:
       requests:
-        cpu: 20m
-        memory: 64Mi
+        cpu: 100m
+        memory: 128Mi
       limits:
-        cpu: 200m
-        memory: 256Mi
+        cpu: "1"
+        memory: 512Mi
     dashboards:
       enabled: true
       searchNamespace: ALL
@@ -259,6 +259,16 @@ prometheus-node-exporter:
   priorityClassName: monitoring-critical
   tolerations:
     - operator: Exists
+  prometheus:
+    monitor:
+      # Keep dashboard variables and alerts stable when a node IP changes.
+      # The scrape address remains the discovered node IP; only the public
+      # instance label is rewritten to the Kubernetes node hostname.
+      relabelings:
+        - action: replace
+          sourceLabels: [__meta_kubernetes_endpoint_node_name]
+          targetLabel: instance
+          replacement: "${1}:9100"
   resources:
     requests:
       cpu: 50m