diff --git a/k8s-ng/core/gitea/deployment.yaml b/k8s-ng/core/gitea/deployment.yaml index ee33f4e..8c04ae0 100644 --- a/k8s-ng/core/gitea/deployment.yaml +++ b/k8s-ng/core/gitea/deployment.yaml @@ -31,11 +31,11 @@ spec: image: gitea/gitea:28.0.0 resources: requests: - memory: 512Mi - cpu: 200m + memory: 1Gi + cpu: "1" limits: - memory: 2Gi - cpu: 1500m + memory: 4Gi + cpu: "4" env: - name: GITEA__service__REGISTER_MANUAL_CONFIRM value: "true" diff --git a/k8s-ng/core/prom-stack/README.md b/k8s-ng/core/prom-stack/README.md index b63e1e8..84cac7f 100644 --- a/k8s-ng/core/prom-stack/README.md +++ b/k8s-ng/core/prom-stack/README.md @@ -36,6 +36,17 @@ them. This protects data from a manifest rename, but it is not a backup. Back up custom fields `IDM Client ID`, `IDM Client Secret`, `TG Token`, and `TG ID`; - DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster. +## Alerting and target identities + +Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and +`InfoInhibitor` remain visible in Prometheus but route to the null receiver. +Warning and critical notifications are grouped for two minutes and rendered as +Telegram HTML with escaped annotations and labelled graph/runbook links. + +Node Exporter keeps scraping the discovered node address, but rewrites the +public `instance` label from Kubernetes discovery to `:9100`. Grafana +variables and alerts therefore remain stable when node IP addresses change. + After Argo CD reports the application healthy, verify targets, rules and log delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana uses its own Keycloak OIDC client. diff --git a/k8s-ng/core/prom-stack/alertmanager-config.yaml b/k8s-ng/core/prom-stack/alertmanager-config.yaml index e51d52f..9753674 100644 --- a/k8s-ng/core/prom-stack/alertmanager-config.yaml +++ b/k8s-ng/core/prom-stack/alertmanager-config.yaml @@ -8,9 +8,11 @@ metadata: argocd.argoproj.io/sync-wave: "1" spec: route: - groupBy: [namespace, alertname] - groupWait: 30s - groupInterval: 5m + groupBy: [cluster, namespace, alertname] + # A two-minute grace period absorbs alerts which resolve immediately and + # never need to interrupt a human. + groupWait: 2m + groupInterval: 10m repeatInterval: 12h receiver: telegram routes: @@ -19,6 +21,13 @@ spec: - name: alertname value: Watchdog|InfoInhibitor matchType: =~ + # Informational alerts remain queryable in Prometheus but do not page the + # Telegram chat. Warning and critical alerts still use the parent route. + - receiver: "null" + matchers: + - name: severity + value: info + matchType: = receivers: - name: telegram telegramConfigs: @@ -29,15 +38,23 @@ spec: # AlertmanagerConfig only supports an integer here, not a Secret ref. chatID: 124317807 sendResolved: true + parseMode: HTML message: |- - {{ if eq .Status "firing" }}๐Ÿ”ฅ FIRING{{ else }}โœ… RESOLVED{{ end }} + {{ if eq .Status "firing" }}๐Ÿ”ฅ FIRING ยท {{ len .Alerts.Firing }}{{ else }}โœ… RESOLVED ยท {{ len .Alerts.Resolved }}{{ end }} + {{ with .CommonLabels.alertname }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }}{{ with .CommonLabels.severity }} ยท {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} {{ range .Alerts }} - {{ .Labels.alertname }} ({{ .Labels.severity }}) - {{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }} - {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }} - {{ if .Labels.namespace }}Namespace: {{ .Labels.namespace }}{{ end }} - {{ if .Labels.pod }}Pod: {{ .Labels.pod }}{{ end }} - {{ if .Labels.node }}Node: {{ .Labels.node }}{{ end }} - {{ .GeneratorURL }} + + {{ if eq .Status "firing" }}๐Ÿ”ด{{ else }}๐ŸŸข{{ end }} {{ with .Annotations.summary }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ else }}{{ .Labels.alertname | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ with .Annotations.description }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ if or .Labels.namespace .Labels.pod .Labels.container .Labels.node .Labels.instance }} + Context + {{ with .Labels.namespace }}Namespace: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ with .Labels.pod }}Pod: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ with .Labels.container }}Container: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ with .Labels.node }}Node: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ if and (not .Labels.node) .Labels.instance }}Instance: {{ .Labels.instance | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} + {{ end }} + Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }} + " ">" }}">๐Ÿ“ˆ Open graph{{ with .Annotations.runbook_url }} ยท " ">" }}">๐Ÿ“– Runbook{{ end }} {{ end }} - name: "null" diff --git a/k8s-ng/core/prom-stack/prom-values.yaml b/k8s-ng/core/prom-stack/prom-values.yaml index ef8980a..ce2ee16 100644 --- a/k8s-ng/core/prom-stack/prom-values.yaml +++ b/k8s-ng/core/prom-stack/prom-values.yaml @@ -188,11 +188,11 @@ grafana: sidecar: resources: requests: - cpu: 20m - memory: 64Mi + cpu: 100m + memory: 128Mi limits: - cpu: 200m - memory: 256Mi + cpu: "1" + memory: 512Mi dashboards: enabled: true searchNamespace: ALL @@ -259,6 +259,16 @@ prometheus-node-exporter: priorityClassName: monitoring-critical tolerations: - operator: Exists + prometheus: + monitor: + # Keep dashboard variables and alerts stable when a node IP changes. + # The scrape address remains the discovered node IP; only the public + # instance label is rewritten to the Kubernetes node hostname. + relabelings: + - action: replace + sourceLabels: [__meta_kubernetes_endpoint_node_name] + targetLabel: instance + replacement: "${1}:9100" resources: requests: cpu: 50m