Bound Telegram alert messages and identify affected workloads
Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 9s
Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 9s
This commit is contained in:
Aleksandr Bogomiakov committed
1 parent
ba1fcae807
commit
9b0530fb70
2 files changed
+31
-17
No files matched your search
@@ -47,7 +47,12 @@ them. This protects data from a manifest rename, but it is not a backup. Back up
|
||||
Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and
|
||||
`InfoInhibitor` remain visible in Prometheus but route to the null receiver.
|
||||
Warning and critical notifications are grouped for two minutes and rendered as
|
||||
Telegram HTML with escaped annotations and labelled graph/runbook links.
|
||||
Telegram HTML with escaped annotations and labelled alerts/runbook links.
|
||||
Notifications show at most three alerts, with each alert's escaped text limited
|
||||
to 500 characters. A single short Prometheus alerts link replaces repeated graph
|
||||
URLs; the full group remains visible there. Workload alerts show the affected
|
||||
DaemonSet, Deployment or StatefulSet rather than the kube-state-metrics pod
|
||||
which supplied the metric.
|
||||
|
||||
Node Exporter keeps scraping the discovered node address, but rewrites the
|
||||
public `instance` label from Kubernetes discovery to `<node-name>:9100`. Grafana
|
||||
|
||||
@@ -39,24 +39,33 @@ spec:
|
||||
chatID: 124317807
|
||||
sendResolved: true
|
||||
parseMode: HTML
|
||||
# Bound the rendered HTML, including escaped text, before Telegram's
|
||||
# 4096-character limit. Keep the full group accessible in Prometheus.
|
||||
message: |-
|
||||
{{- define "telegram.alert.text" -}}
|
||||
{{- $escaped := html . -}}
|
||||
{{- $short := printf "%.500s" $escaped -}}
|
||||
{{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}…{{ end -}}
|
||||
{{- end -}}
|
||||
{{- $alerts := .Alerts.Resolved -}}
|
||||
{{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}}
|
||||
{{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
|
||||
{{ with .CommonLabels.alertname }}<b>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ range .Alerts }}
|
||||
{{ with .CommonLabels.alertname }}<b>{{ printf "%.64s" . }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ printf "%.16s" . }}</code>{{ end }}
|
||||
{{ range $index, $alert := $alerts }}{{ if lt $index 3 }}
|
||||
{{- $context := "" -}}
|
||||
{{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}}
|
||||
{{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}}
|
||||
{{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}}
|
||||
{{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}}
|
||||
{{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}}
|
||||
{{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}}
|
||||
{{- end -}}
|
||||
{{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}}
|
||||
{{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}}
|
||||
|
||||
{{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ with .Annotations.summary }}<b>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</b>{{ else }}<b>{{ .Labels.alertname | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</b>{{ end }}
|
||||
{{ with .Annotations.description }}{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}{{ end }}
|
||||
{{ if or .Labels.namespace .Labels.job_name .Labels.pod .Labels.container .Labels.node .Labels.instance }}
|
||||
<b>Context</b>
|
||||
{{ with .Labels.namespace }}Namespace: <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ with .Labels.job_name }}Job: <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ else }}
|
||||
{{ with .Labels.pod }}Pod: <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ with .Labels.container }}Container: <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ with .Labels.node }}Node: <code>{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ if and (not .Labels.node) .Labels.instance }}Instance: <code>{{ .Labels.instance | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}</code>{{ end }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }}
|
||||
Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
|
||||
<a href="{{ .GeneratorURL | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}">📈 Open graph</a>{{ with .Annotations.runbook_url }} · <a href="{{ . | reReplaceAll "&" "&" | reReplaceAll "<" "<" | reReplaceAll ">" ">" }}">📖 Runbook</a>{{ end }}
|
||||
{{ end }}
|
||||
{{ end }}{{ end }}
|
||||
{{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }}
|
||||
<a href="https://prom.hexor.cy/alerts">📈 Open alerts</a>{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} · <a href="{{ . }}">📖 Runbook</a>{{ end }}{{ end }}
|
||||
- name: "null"
|
||||
Reference in new issue
Block a user