Configured alerts in grafana and TG endpoint

k8s/core/prom-stack/external-secrets.yaml

@@ -119,4 +119,43 @@ spec:
         key: eca0fb0b-3939-40a8-890a-6294863e5a65
         property: fields[1].value
 
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-telegram
+spec:
+  target:
+    name: grafana-telegram
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        bot-token: |-
+          {{ .bot_token }}
+        chat-id: |-
+          {{ .chat_id }}
+  data:
+    - secretKey: bot_token
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        metadataPolicy: None
+        key: eca0fb0b-3939-40a8-890a-6294863e5a65
+        property: fields[0].value
+    - secretKey: chat_id
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        conversionStrategy: Default
+        decodingStrategy: None
+        metadataPolicy: None
+        key: eca0fb0b-3939-40a8-890a-6294863e5a65
+        property: fields[1].value
 

k8s/core/prom-stack/grafana-alerting.yaml

@@ -0,0 +1,69 @@
+rules.yaml: |
+  apiVersion: 1
+  groups:
+    - orgId: 1
+      name: pasarguard_alerts
+      folder: Kubernetes
+      interval: 1m
+      rules:
+        - uid: pasarguard_cpu_throttling
+          title: VPN CPU Throttle
+          condition: A
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: prometheus
+              model:
+                expr: 'rate(container_cpu_cfs_throttled_periods_total{container="pasarguard-node"}[5m]) > 0.1'
+                refId: A
+          noDataState: NoData
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            description: 'Throttling rate: {{ printf "%.2f" $values.A.Value }}'
+            summary: 'VPN node throttling CPU on {{ $labels.node }}'
+          labels:
+            severity: warning
+
+contactpoints.yaml: |
+  apiVersion: 1
+  contactPoints:
+    - orgId: 1
+      name: telegram
+      receivers:
+        - uid: telegram_default
+          type: telegram
+          settings:
+            bottoken: $TELEGRAM_BOT_TOKEN
+            chatid: $TELEGRAM_CHAT_ID
+            message: |
+              {{ if eq .Status "firing" }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
+              
+              {{ range .Alerts }}
+              📊 <b>{{ .Labels.alertname }}</b>
+              {{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}
+              
+              🎯 <b>Details:</b>
+              • Pod: <code>{{ .Labels.pod }}</code>
+              • Node: <code>{{ .Labels.node }}</code>
+              • Namespace: <code>{{ .Labels.namespace }}</code>
+              {{ if .Annotations.description }}• {{ .Annotations.description }}{{ end }}
+              
+              🔗 <a href="{{ .GeneratorURL }}">View in Grafana</a>
+              {{ end }}
+            parse_mode: HTML
+          disableResolveMessage: false
+
+policies.yaml: |
+  apiVersion: 1
+  policies:
+    - orgId: 1
+      receiver: telegram
+      group_by:
+        - grafana_folder
+        - alertname
+      group_wait: 10s
+      group_interval: 5m
+      repeat_interval: 4h

k8s/core/prom-stack/grafana-values.yaml

@@ -56,3 +56,19 @@ ingress:
       hosts:
         - '*.hexor.cy'
 
+extraConfigmapMounts:
+  - name: grafana-alerting
+    mountPath: /etc/grafana/provisioning/alerting
+    configMap: grafana-alerting
+    readOnly: true
+
+envValueFrom:
+  TELEGRAM_BOT_TOKEN:
+    secretKeyRef:
+      name: grafana-telegram
+      key: bot-token
+  TELEGRAM_CHAT_ID:
+    secretKeyRef:
+      name: grafana-telegram
+      key: chat-id
+

k8s/core/prom-stack/kustomization.yaml

@@ -6,6 +6,12 @@ resources:
   - persistentVolume.yaml
   - external-secrets.yaml
 
+configMapGenerator:
+  - name: grafana-alerting
+    namespace: prometheus
+    files:
+      - grafana-alerting.yaml
+
 helmCharts:
   - name: kube-prometheus-stack
     repo: https://prometheus-community.github.io/helm-charts