diff --git a/k8s-ng/core/prom-stack/README.md b/k8s-ng/core/prom-stack/README.md index 3b9325f..ef643b9 100644 --- a/k8s-ng/core/prom-stack/README.md +++ b/k8s-ng/core/prom-stack/README.md @@ -47,7 +47,12 @@ them. This protects data from a manifest rename, but it is not a backup. Back up Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and `InfoInhibitor` remain visible in Prometheus but route to the null receiver. Warning and critical notifications are grouped for two minutes and rendered as -Telegram HTML with escaped annotations and labelled graph/runbook links. +Telegram HTML with escaped annotations and labelled alerts/runbook links. +Notifications show at most three alerts, with each alert's escaped text limited +to 500 characters. A single short Prometheus alerts link replaces repeated graph +URLs; the full group remains visible there. Workload alerts show the affected +DaemonSet, Deployment or StatefulSet rather than the kube-state-metrics pod +which supplied the metric. Node Exporter keeps scraping the discovered node address, but rewrites the public `instance` label from Kubernetes discovery to `:9100`. Grafana diff --git a/k8s-ng/core/prom-stack/alertmanager-config.yaml b/k8s-ng/core/prom-stack/alertmanager-config.yaml index 6b1b177..6684bfc 100644 --- a/k8s-ng/core/prom-stack/alertmanager-config.yaml +++ b/k8s-ng/core/prom-stack/alertmanager-config.yaml @@ -39,24 +39,33 @@ spec: chatID: 124317807 sendResolved: true parseMode: HTML + # Bound the rendered HTML, including escaped text, before Telegram's + # 4096-character limit. Keep the full group accessible in Prometheus. message: |- + {{- define "telegram.alert.text" -}} + {{- $escaped := html . -}} + {{- $short := printf "%.500s" $escaped -}} + {{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}โ€ฆ{{ end -}} + {{- end -}} + {{- $alerts := .Alerts.Resolved -}} + {{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}} {{ if eq .Status "firing" }}๐Ÿ”ฅ FIRING ยท {{ len .Alerts.Firing }}{{ else }}โœ… RESOLVED ยท {{ len .Alerts.Resolved }}{{ end }} - {{ with .CommonLabels.alertname }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }}{{ with .CommonLabels.severity }} ยท {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ range .Alerts }} + {{ with .CommonLabels.alertname }}{{ printf "%.64s" . }}{{ end }}{{ with .CommonLabels.severity }} ยท {{ printf "%.16s" . }}{{ end }} + {{ range $index, $alert := $alerts }}{{ if lt $index 3 }} + {{- $context := "" -}} + {{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}} + {{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}} + {{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}} + {{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}} + {{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}} + {{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}} + {{- end -}} + {{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}} + {{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}} - {{ if eq .Status "firing" }}๐Ÿ”ด{{ else }}๐ŸŸข{{ end }} {{ with .Annotations.summary }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ else }}{{ .Labels.alertname | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ with .Annotations.description }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ if or .Labels.namespace .Labels.job_name .Labels.pod .Labels.container .Labels.node .Labels.instance }} - Context - {{ with .Labels.namespace }}Namespace: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ with .Labels.job_name }}Job: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ else }} - {{ with .Labels.pod }}Pod: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ with .Labels.container }}Container: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ with .Labels.node }}Node: {{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ if and (not .Labels.node) .Labels.instance }}Instance: {{ .Labels.instance | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }} - {{ end }} - {{ end }} + {{ if eq .Status "firing" }}๐Ÿ”ด{{ else }}๐ŸŸข{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }} Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }} - " ">" }}">๐Ÿ“ˆ Open graph{{ with .Annotations.runbook_url }} ยท " ">" }}">๐Ÿ“– Runbook{{ end }} - {{ end }} + {{ end }}{{ end }} + {{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }} + ๐Ÿ“ˆ Open alerts{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} ยท ๐Ÿ“– Runbook{{ end }}{{ end }} - name: "null"