homelab/k8s-ng/core/prom-stack/alertmanager-config.yaml

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: telegram
  labels:
    alertmanager: main
  annotations:
    argocd.argoproj.io/sync-wave: "1"
spec:
  route:
    groupBy: [cluster, namespace, alertname]
    # A two-minute grace period absorbs alerts which resolve immediately and
    # never need to interrupt a human.
    groupWait: 2m
    groupInterval: 10m
    repeatInterval: 12h
    receiver: telegram
    routes:
      - receiver: "null"
        matchers:
          - name: alertname
            value: Watchdog|InfoInhibitor
            matchType: =~
      # Informational alerts remain queryable in Prometheus but do not page the
      # Telegram chat. Warning and critical alerts still use the parent route.
      - receiver: "null"
        matchers:
          - name: severity
            value: info
            matchType: =
  receivers:
    - name: telegram
      telegramConfigs:
        - botToken:
            name: alertmanager-telegram
            key: bot-token
          # Telegram chat IDs are routing identifiers rather than credentials;
          # AlertmanagerConfig only supports an integer here, not a Secret ref.
          chatID: 124317807
          sendResolved: true
          parseMode: HTML
          # Bound the rendered HTML, including escaped text, before Telegram's
          # 4096-character limit. Keep the full group accessible in Prometheus.
          message: |-
            {{- define "telegram.alert.text" -}}
            {{- $escaped := html . -}}
            {{- $short := printf "%.500s" $escaped -}}
            {{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}…{{ end -}}
            {{- end -}}
            {{- $alerts := .Alerts.Resolved -}}
            {{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}}
            {{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
            {{ with .CommonLabels.alertname }}<b>{{ printf "%.64s" . }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ printf "%.16s" . }}</code>{{ end }}
            {{ range $index, $alert := $alerts }}{{ if lt $index 3 }}
            {{- $context := "" -}}
            {{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}}
            {{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}}
            {{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}}
            {{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}}
            {{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}}
            {{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}}
            {{- end -}}
            {{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}}
            {{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}}

            {{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }}
            Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
            {{ end }}{{ end }}
            {{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }}
            <a href="https://prom.hexor.cy/alerts">📈 Open alerts</a>{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} · <a href="{{ . }}">📖 Runbook</a>{{ end }}{{ end }}
    - name: "null"