diff --git a/k8s/core/prom-stack/external-secrets.yaml b/k8s/core/prom-stack/external-secrets.yaml
index b923cdb..49c2332 100644
--- a/k8s/core/prom-stack/external-secrets.yaml
+++ b/k8s/core/prom-stack/external-secrets.yaml
@@ -119,4 +119,43 @@ spec:
key: eca0fb0b-3939-40a8-890a-6294863e5a65
property: fields[1].value
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+ name: grafana-telegram
+spec:
+ target:
+ name: grafana-telegram
+ deletionPolicy: Delete
+ template:
+ type: Opaque
+ data:
+ bot-token: |-
+ {{ .bot_token }}
+ chat-id: |-
+ {{ .chat_id }}
+ data:
+ - secretKey: bot_token
+ sourceRef:
+ storeRef:
+ name: vaultwarden-login
+ kind: ClusterSecretStore
+ remoteRef:
+ conversionStrategy: Default
+ decodingStrategy: None
+ metadataPolicy: None
+ key: eca0fb0b-3939-40a8-890a-6294863e5a65
+ property: fields[0].value
+ - secretKey: chat_id
+ sourceRef:
+ storeRef:
+ name: vaultwarden-login
+ kind: ClusterSecretStore
+ remoteRef:
+ conversionStrategy: Default
+ decodingStrategy: None
+ metadataPolicy: None
+ key: eca0fb0b-3939-40a8-890a-6294863e5a65
+ property: fields[1].value
diff --git a/k8s/core/prom-stack/grafana-alerting.yaml b/k8s/core/prom-stack/grafana-alerting.yaml
new file mode 100644
index 0000000..ca00996
--- /dev/null
+++ b/k8s/core/prom-stack/grafana-alerting.yaml
@@ -0,0 +1,69 @@
+rules.yaml: |
+ apiVersion: 1
+ groups:
+ - orgId: 1
+ name: pasarguard_alerts
+ folder: Kubernetes
+ interval: 1m
+ rules:
+ - uid: pasarguard_cpu_throttling
+ title: VPN CPU Throttle
+ condition: A
+ data:
+ - refId: A
+ relativeTimeRange:
+ from: 600
+ to: 0
+ datasourceUid: prometheus
+ model:
+ expr: 'rate(container_cpu_cfs_throttled_periods_total{container="pasarguard-node"}[5m]) > 0.1'
+ refId: A
+ noDataState: NoData
+ execErrState: Alerting
+ for: 5m
+ annotations:
+ description: 'Throttling rate: {{ printf "%.2f" $values.A.Value }}'
+ summary: 'VPN node throttling CPU on {{ $labels.node }}'
+ labels:
+ severity: warning
+
+contactpoints.yaml: |
+ apiVersion: 1
+ contactPoints:
+ - orgId: 1
+ name: telegram
+ receivers:
+ - uid: telegram_default
+ type: telegram
+ settings:
+ bottoken: $TELEGRAM_BOT_TOKEN
+ chatid: $TELEGRAM_CHAT_ID
+ message: |
+ {{ if eq .Status "firing" }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
+
+ {{ range .Alerts }}
+ 📊 {{ .Labels.alertname }}
+ {{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}
+
+ 🎯 Details:
+ • Pod: {{ .Labels.pod }}
+ • Node: {{ .Labels.node }}
+ • Namespace: {{ .Labels.namespace }}
+ {{ if .Annotations.description }}• {{ .Annotations.description }}{{ end }}
+
+ 🔗 View in Grafana
+ {{ end }}
+ parse_mode: HTML
+ disableResolveMessage: false
+
+policies.yaml: |
+ apiVersion: 1
+ policies:
+ - orgId: 1
+ receiver: telegram
+ group_by:
+ - grafana_folder
+ - alertname
+ group_wait: 10s
+ group_interval: 5m
+ repeat_interval: 4h
diff --git a/k8s/core/prom-stack/grafana-values.yaml b/k8s/core/prom-stack/grafana-values.yaml
index 9e23a52..2fae72b 100644
--- a/k8s/core/prom-stack/grafana-values.yaml
+++ b/k8s/core/prom-stack/grafana-values.yaml
@@ -56,3 +56,19 @@ ingress:
hosts:
- '*.hexor.cy'
+extraConfigmapMounts:
+ - name: grafana-alerting
+ mountPath: /etc/grafana/provisioning/alerting
+ configMap: grafana-alerting
+ readOnly: true
+
+envValueFrom:
+ TELEGRAM_BOT_TOKEN:
+ secretKeyRef:
+ name: grafana-telegram
+ key: bot-token
+ TELEGRAM_CHAT_ID:
+ secretKeyRef:
+ name: grafana-telegram
+ key: chat-id
+
diff --git a/k8s/core/prom-stack/kustomization.yaml b/k8s/core/prom-stack/kustomization.yaml
index 701d6d6..af2eb85 100644
--- a/k8s/core/prom-stack/kustomization.yaml
+++ b/k8s/core/prom-stack/kustomization.yaml
@@ -6,6 +6,12 @@ resources:
- persistentVolume.yaml
- external-secrets.yaml
+configMapGenerator:
+ - name: grafana-alerting
+ namespace: prometheus
+ files:
+ - grafana-alerting.yaml
+
helmCharts:
- name: kube-prometheus-stack
repo: https://prometheus-community.github.io/helm-charts