diff --git a/k8s/core/prom-stack/grafana-alerting-configmap.yaml b/k8s/core/prom-stack/grafana-alerting-configmap.yaml index 2d0ee84c..27a412ca 100644 --- a/k8s/core/prom-stack/grafana-alerting-configmap.yaml +++ b/k8s/core/prom-stack/grafana-alerting-configmap.yaml @@ -59,6 +59,57 @@ data: summary: 'VPN node throttling CPU' labels: severity: warning + + - orgId: 1 + name: kubernetes_alerts + folder: Kubernetes + interval: 30s + rules: + - uid: node_not_ready + title: Kubernetes Node Not Ready + condition: B + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: P76F38748CEC837F0 + model: + expr: 'kube_node_status_condition{condition="Ready",status="true"} == 0' + refId: A + intervalMs: 1000 + maxDataPoints: 43200 + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: [] + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + noDataState: Alerting + execErrState: Alerting + for: 0s + annotations: + node: '{{ $labels.node }}' + condition: '{{ $labels.condition }}' + summary: 'Kubernetes node is not ready' + labels: + severity: critical contactpoints.yaml: | apiVersion: 1 @@ -79,11 +130,10 @@ data: 📊 {{ .Labels.alertname }} {{ .Annotations.summary }} - 🎯 Details: - • Pod: {{ .Annotations.pod }} - • Node: {{ .Annotations.node }} - • Namespace: {{ .Annotations.namespace }} - • Throttling rate: {{ .Annotations.throttle_rate }} + {{ if .Annotations.node }}🖥 Node: {{ .Annotations.node }}{{ end }} + {{ if .Annotations.pod }}📦 Pod: {{ .Annotations.pod }}{{ end }} + {{ if .Annotations.namespace }}📁 Namespace: {{ .Annotations.namespace }}{{ end }} + {{ if .Annotations.throttle_rate }}⚠️ Throttling rate: {{ .Annotations.throttle_rate }}{{ end }} 🔗 View in Grafana {{ end }}