diff --git a/k8s-ng/apps/furumi/kustomization.yaml b/k8s-ng/apps/furumi/kustomization.yaml index 267f514..1793d68 100644 --- a/k8s-ng/apps/furumi/kustomization.yaml +++ b/k8s-ng/apps/furumi/kustomization.yaml @@ -5,4 +5,5 @@ resources: - secrets.yaml - deployment.yaml - service.yaml + - servicemonitor.yaml - ingress.yaml diff --git a/k8s-ng/apps/furumi/servicemonitor.yaml b/k8s-ng/apps/furumi/servicemonitor.yaml new file mode 100644 index 0000000..89fe7fc --- /dev/null +++ b/k8s-ng/apps/furumi/servicemonitor.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: furumi-player +spec: + selector: + matchLabels: + app: furumi-player + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/k8s-ng/core-state/values.yaml b/k8s-ng/core-state/values.yaml index eb3f451..3748585 100644 --- a/k8s-ng/core-state/values.yaml +++ b/k8s-ng/core-state/values.yaml @@ -62,7 +62,7 @@ apps: project: apps path: k8s-ng/apps/furumi namespace: furumi - wave: 5 # pinned to the music node; must not hold back the other waves + wave: 6 # after prom-stack installs the ServiceMonitor CRD skipDryRunOnMissingResource: true - name: web-petting project: apps @@ -87,3 +87,8 @@ apps: wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node prune: false # never delete the instance because of a manifest change skipDryRunOnMissingResource: true + - name: prom-stack + path: k8s-ng/core/prom-stack + namespace: prometheus + wave: 5 # after secret-stores; installs monitoring before application workloads + skipDryRunOnMissingResource: true # monitoring, cert-manager and ExternalSecret CRDs diff --git a/k8s-ng/core/postgresql/databases/grafana.yaml b/k8s-ng/core/postgresql/databases/grafana.yaml new file mode 100644 index 0000000..f9228e4 --- /dev/null +++ b/k8s-ng/core/postgresql/databases/grafana.yaml @@ -0,0 +1,78 @@ +# Database "grafana": generated credentials, role, database and read access for +# the monitoring namespace. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: grafana-db + namespace: psql +spec: + refreshPolicy: CreatedOnce + target: + name: grafana-db + template: + type: kubernetes.io/basic-auth + metadata: + labels: + cnpg.io/reload: "true" + data: + username: grafana + password: "{{ .password }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: db-password +--- +apiVersion: postgresql.cnpg.io/v1 +kind: DatabaseRole +metadata: + name: main-grafana + namespace: psql +spec: + cluster: + name: main + name: grafana + login: true + passwordSecret: + name: grafana-db +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: main-grafana + namespace: psql +spec: + cluster: + name: main + name: grafana + owner: grafana + databaseReclaimPolicy: retain +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana-db-reader + namespace: psql +rules: + - apiGroups: [""] + resources: [secrets] + resourceNames: [grafana-db] + verbs: [get, list, watch] + - apiGroups: [authorization.k8s.io] + resources: [selfsubjectrulesreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana-db-reader + namespace: psql +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana-db-reader +subjects: + - kind: ServiceAccount + name: db-credentials-reader + namespace: prometheus diff --git a/k8s-ng/core/postgresql/kustomization.yaml b/k8s-ng/core/postgresql/kustomization.yaml index ba81bee..7d38c2e 100644 --- a/k8s-ng/core/postgresql/kustomization.yaml +++ b/k8s-ng/core/postgresql/kustomization.yaml @@ -9,3 +9,4 @@ resources: - databases/furumi.yaml - databases/web-petting.yaml - databases/meguri.yaml + - databases/grafana.yaml diff --git a/k8s-ng/core/prom-stack/README.md b/k8s-ng/core/prom-stack/README.md new file mode 100644 index 0000000..b12767d --- /dev/null +++ b/k8s-ng/core/prom-stack/README.md @@ -0,0 +1,41 @@ +# Monitoring stack (new cluster) + +The stack is intentionally a fresh installation: no Prometheus TSDB, Loki log +chunks, or Grafana database are copied from the old cluster. + +## Layout + +- `kube-prometheus-stack`: Prometheus, Alertmanager, Grafana, exporters and + Kubernetes alerting rules. +- `loki`: one filesystem-backed monolithic instance. A second replica would + require shared object storage and is deliberately not faked with a local PV. +- `alloy`: unprivileged Kubernetes API log and event collection. This replaces + EOL Promtail and does not require host log mounts or privileged sysctls. +- Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on + the large local disk of node `music`. + +## Storage and retention + +The PreSync hook creates these directories on `music`: + +| Path | PV | Retention | +| --- | ---: | --- | +| `/data/k8s/monitoring/prometheus` | 500 Gi | up to 365 days / 425 GB | +| `/data/k8s/monitoring/loki` | 200 Gi | 90 days | +| `/data/k8s/monitoring/alertmanager` | 5 Gi | 120 hours of notification state | + +All PVs and the StorageClass use `Retain`; Argo CD is also told not to prune +them. This protects data from a manifest rename, but it is not a backup. Back up +`/data/k8s/monitoring` with the rest of the `music` node if history matters. + +## Dependencies + +- node `music` must exist and `/data` must be the intended large filesystem; +- `main-rw.psql.svc` and the generated `grafana-db` Secret must be ready; +- `vaultwarden-login` must resolve the existing Grafana OIDC and Telegram item + UUIDs used in `external-secrets.yaml`; +- DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster. + +After Argo CD reports the application healthy, verify targets, rules and log +delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana +uses its own Keycloak OIDC client. diff --git a/k8s-ng/core/prom-stack/alertmanager-config.yaml b/k8s-ng/core/prom-stack/alertmanager-config.yaml new file mode 100644 index 0000000..f63ca12 --- /dev/null +++ b/k8s-ng/core/prom-stack/alertmanager-config.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: telegram + labels: + alertmanager: main +spec: + route: + groupBy: [namespace, alertname] + groupWait: 30s + groupInterval: 5m + repeatInterval: 12h + receiver: telegram + routes: + - receiver: "null" + matchers: + - name: alertname + value: Watchdog|InfoInhibitor + matchType: =~ + receivers: + - name: telegram + telegramConfigs: + - botToken: + name: alertmanager-telegram + key: bot-token + # Telegram chat IDs are routing identifiers rather than credentials; + # AlertmanagerConfig only supports an integer here, not a Secret ref. + chatID: 124317807 + sendResolved: true + message: |- + {{ if eq .Status "firing" }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }} + {{ range .Alerts }} + {{ .Labels.alertname }} ({{ .Labels.severity }}) + {{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }} + {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }} + {{ if .Labels.namespace }}Namespace: {{ .Labels.namespace }}{{ end }} + {{ if .Labels.pod }}Pod: {{ .Labels.pod }}{{ end }} + {{ if .Labels.node }}Node: {{ .Labels.node }}{{ end }} + {{ .GeneratorURL }} + {{ end }} + - name: "null" diff --git a/k8s-ng/core/prom-stack/alloy-values.yaml b/k8s-ng/core/prom-stack/alloy-values.yaml new file mode 100644 index 0000000..b952628 --- /dev/null +++ b/k8s-ng/core/prom-stack/alloy-values.yaml @@ -0,0 +1,160 @@ +alloy: + enableReporting: false + storagePath: /var/lib/alloy + configMap: + create: true + content: |- + discovery.kubernetes "pods" { + role = "pod" + } + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + regex = `(.+)` + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = `(.+)` + target_label = "app" + } + rule { + source_labels = ["namespace", "app"] + separator = "/" + regex = `(.+)/(.+)` + replacement = "$1/$2" + target_label = "job" + } + } + + loki.source.kubernetes "pods" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pods.receiver] + } + + loki.process "pods" { + stage.static_labels { + values = { + cluster = "hexor-ng", + } + } + forward_to = [loki.write.local.receiver] + } + + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = [loki.process.cluster_events.receiver] + } + + loki.process "cluster_events" { + stage.static_labels { + values = { + cluster = "hexor-ng", + } + } + forward_to = [loki.write.local.receiver] + } + + loki.write "local" { + endpoint { + url = "http://loki-gateway.prometheus.svc/loki/api/v1/push" + } + } + mounts: + extra: + - name: data + mountPath: /var/lib/alloy + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 473 + runAsGroup: 473 + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + livenessProbe: + httpGet: + path: /metrics + port: 12345 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 2 + failureThreshold: 3 + +global: + podSecurityContext: + fsGroup: 473 + fsGroupChangePolicy: OnRootMismatch + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +controller: + type: deployment + replicas: 1 + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + minReadySeconds: 10 + terminationGracePeriodSeconds: 60 + volumes: + extra: + - name: data + emptyDir: {} + +configReloader: + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 128Mi + +# Only the API permissions used by pod-log discovery and cluster events. +rbac: + create: true + rules: + - apiGroups: [""] + resources: [pods, pods/log, namespaces, events] + verbs: [get, list, watch] + # Kept separate because the Alloy chart concatenates this list into the + # ClusterRole. Node metadata is useful for diagnosing a log source. + clusterRules: + - apiGroups: [""] + resources: [nodes] + verbs: [get, list, watch] + +serviceMonitor: + enabled: true + interval: 30s diff --git a/k8s-ng/core/prom-stack/dashboards/auth-proxy-dashboard-cm.yaml b/k8s-ng/core/prom-stack/dashboards/auth-proxy-dashboard-cm.yaml new file mode 100644 index 0000000..16ce5cb --- /dev/null +++ b/k8s-ng/core/prom-stack/dashboards/auth-proxy-dashboard-cm.yaml @@ -0,0 +1,367 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: auth-proxy-dashboard + labels: + grafana_dashboard: "1" +data: + auth-proxy.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "routes_loaded_total", + "refId": "A" + } + ], + "title": "Loaded Routes", + "type": "stat" + }, + { + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(increase(auth_requests_total[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Auth Requests (total)", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + } + }, + "overrides": [] + } + }, + { + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(increase(auth_requests_total{status=\"403\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Forbidden (403)", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + } + }, + { + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(increase(callback_requests_total{result=\"success\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Successful Logins", + "type": "stat", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + } + }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 5, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (status) (rate(auth_requests_total[$__rate_interval]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Auth Request Rate by Status", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { "color": "green", "value": null } ] + }, + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 6, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "auth_request_duration_seconds{quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "auth_request_duration_seconds{quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "auth_request_duration_seconds{quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Auth Request Duration (p50, p95, p99)", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { "color": "green", "value": null } ] + }, + "unit": "s" + }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 7, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (result) (rate(callback_requests_total[$__rate_interval]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "title": "OIDC Callback Rate by Result", + "type": "timeseries", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { "color": "green", "value": null } ] + }, + "unit": "reqps" + }, + "overrides": [] + } + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "auth-proxy", + "rsauth2-proxy" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Auth Proxy (rsauth2-proxy)", + "uid": "auth-proxy-metrics", + "version": 1 + } diff --git a/k8s-ng/core/prom-stack/dashboards/furumi-dashboard-cm.yaml b/k8s-ng/core/prom-stack/dashboards/furumi-dashboard-cm.yaml new file mode 100644 index 0000000..c132c05 --- /dev/null +++ b/k8s-ng/core/prom-stack/dashboards/furumi-dashboard-cm.yaml @@ -0,0 +1,504 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: furumi-dashboard + labels: + grafana_dashboard: "1" +data: + furumi.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "title": "Build Version", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "max by (version) (furumusic_build_info{namespace=~\"$namespace\"})", + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "name" } + }, + { + "id": 2, + "title": "HTTP RPS", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" } + ], + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" } + }, + { + "id": 3, + "title": "5xx Error Ratio", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "refId": "A" } + ], + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" } + }, + { + "id": 4, + "title": "HTTP p95 Latency", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "refId": "A" } + ], + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" } + }, + { + "id": 5, + "title": "Active Users 15m", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_active_users{namespace=~\"$namespace\",window=\"15m\"})", "refId": "A" } + ], + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" } + }, + { + "id": 6, + "title": "Stream Throughput", + "type": "stat", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "Bps", "decimals": 1 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" } + ], + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" } + }, + { "id": 10, "title": "HTTP", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "collapsed": false, "panels": [] }, + { + "id": 11, + "title": "Request Rate by Route and Status", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route, status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "{{route}} {{status}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 12, + "title": "Request Duration Quantiles", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 13, + "title": "In-flight Requests", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route) (furumusic_http_in_flight_requests{namespace=~\"$namespace\",route=~\"$route\"})", "legendFormat": "{{route}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 14, + "title": "HTTP Body Throughput", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 }, + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_request_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "request", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_response_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "response", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 15, + "title": "Status Code Mix", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { "id": 20, "title": "Auth and Users", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "collapsed": false, "panels": [] }, + { + "id": 21, + "title": "Users by Role", + "type": "bargauge", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (role) (furumusic_users_total{namespace=~\"$namespace\"})", "legendFormat": "{{role}}", "refId": "A" } + ], + "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "showUnfilled": true } + }, + { + "id": 22, + "title": "Active Users", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (furumusic_active_users{namespace=~\"$namespace\"})", "legendFormat": "{{window}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 23, + "title": "Auth Events", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method, outcome, reason) (rate(furumusic_auth_login_attempts_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "login {{method}} {{outcome}} {{reason}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method) (rate(furumusic_auth_sessions_created_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "session {{method}}", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(furumusic_auth_denied_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "denied {{kind}}", "refId": "C" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { "id": 30, "title": "Playback and Streaming", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "collapsed": false, "panels": [] }, + { + "id": 31, + "title": "Listens Rate", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (completed) (rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "completed={{completed}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 32, + "title": "Listen Completion Ratio", + "description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 }, + "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 33, + "title": "Streaming", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 }, + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (range) (rate(furumusic_stream_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "requests range={{range}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 34, + "title": "Listened Hours and Play History Activity", + "description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 }, + "fieldConfig": { + "defaults": { "unit": "h", "decimals": 2 }, + "overrides": [ + { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] } + ] + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { "id": 40, "title": "Library and Storage", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] }, + { + "id": 41, + "title": "Library Inventory", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_artists_total{namespace=~\"$namespace\"})", "legendFormat": "artists", "refId": "C" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_playlists_total{namespace=~\"$namespace\"})", "legendFormat": "playlists", "refId": "D" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 42, + "title": "Media Bytes by Type", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 }, + "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 43, + "title": "Storage Used Ratio", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 }, + "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "1 - (sum by (path_kind) (furumusic_storage_free_bytes{namespace=~\"$namespace\"}) / sum by (path_kind) (furumusic_storage_total_bytes{namespace=~\"$namespace\"}))", "legendFormat": "{{path_kind}}", "refId": "A" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { "id": 50, "title": "AI Agent", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "collapsed": false, "panels": [] }, + { + "id": 51, + "title": "Agent Queue and Reviews", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 52, + "title": "Agent Processing and Failures", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 52 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome, decision) (rate(furumusic_agent_files_processed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "processed {{outcome}} {{decision}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (stage) (rate(furumusic_agent_failed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "failed {{stage}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 53, + "title": "Agent Confidence", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 52 }, + "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 54, + "title": "Discover Runs and Duration", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_agent_discover_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "runs {{outcome}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(furumusic_agent_discover_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{outcome}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 55, + "title": "Discover File Flow", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_seen_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "seen", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_queued_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "queued", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_discover_files_skipped_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}", "refId": "C" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 56, + "title": "RAG and LLM Requests", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind, outcome) (rate(furumusic_agent_rag_queries_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, outcome) (rate(furumusic_agent_llm_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 57, + "title": "RAG and LLM Latency p95", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, kind, outcome) (rate(furumusic_agent_rag_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, model, outcome) (rate(furumusic_agent_llm_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 58, + "title": "LLM Token Rate, Batch Splits and Parse Failures", + "description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 }, + "fieldConfig": { + "defaults": { "unit": "suffix: tok/s" }, + "overrides": [ + { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }, + { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] } + ] + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model) (rate(furumusic_agent_llm_parse_failures_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "parse failures {{model}}", "refId": "C" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 59, + "title": "Cover Pipeline", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, outcome) (rate(furumusic_agent_cover_lookup_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "lookup {{source}} {{outcome}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (variant, outcome) (rate(furumusic_agent_cover_variant_generation_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "variant {{variant}} {{outcome}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { "id": 70, "title": "Scheduler and Torrents", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "collapsed": false, "panels": [] }, + { + "id": 71, + "title": "Scheduler Jobs", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_running{namespace=~\"$namespace\"})", "legendFormat": "running {{job}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_enabled{namespace=~\"$namespace\"})", "legendFormat": "enabled {{job}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 72, + "title": "Scheduler Runs and Duration p95", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job, trigger, outcome) (rate(furumusic_scheduler_job_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{job}} {{trigger}} {{outcome}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, job, trigger, outcome) (rate(furumusic_scheduler_job_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{job}} {{outcome}}", "refId": "B" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 73, + "title": "Torrents", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_torrent_sessions_total{namespace=~\"$namespace\"})", "legendFormat": "sessions {{status}}", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_downloads_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "downloads {{outcome}}", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_selected_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes {{outcome}}", "refId": "C" } + ], + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } } + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ "furumi", "furumusic", "music" ], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(furumusic_build_info, namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": { "query": "label_values(furumusic_build_info, namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "route", + "options": [], + "query": { "query": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Furumi Music", + "uid": "furumi-music", + "version": 1 + } diff --git a/k8s-ng/core/prom-stack/dashboards/telemt-dashboard-cm.yaml b/k8s-ng/core/prom-stack/dashboards/telemt-dashboard-cm.yaml new file mode 100644 index 0000000..1bc0f97 --- /dev/null +++ b/k8s-ng/core/prom-stack/dashboards/telemt-dashboard-cm.yaml @@ -0,0 +1,409 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: telemt-dashboard + labels: + grafana_dashboard: "1" +data: + telemt.json: |- + { + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "title": "Nodes Overview", + "type": "table", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "inspect": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Uptime" }, + "properties": [ + { "id": "unit", "value": "dtdurations" }, + { "id": "custom.width", "value": 140 } + ] + }, + { + "matcher": { "id": "byName", "options": "Bad Conn" }, + "properties": [ + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 100 }] } }, + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Writers" }, + "properties": [ + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Node", "desc": false }], + "frameIndex": 0, + "footer": { "show": false } + }, + "transformations": [ + { + "id": "joinByField", + "options": { "byField": "node", "mode": "outer" } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { "pattern": "^(node|Value.*)$" } + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "node": "Node", + "Value #uptime": "Uptime", + "Value #writers": "Writers", + "Value #buffers": "Buffers In Use", + "Value #connections": "Connections", + "Value #bad": "Bad Conn", + "Value #hs_timeout": "HS Timeouts" + } + } + } + ], + "targets": [ + { + "expr": "telemt_uptime_seconds{node=~\"$node\"}", + "legendFormat": "", + "refId": "uptime", + "format": "table", + "instant": true + }, + { + "expr": "telemt_me_writers_active_current{node=~\"$node\"}", + "legendFormat": "", + "refId": "writers", + "format": "table", + "instant": true + }, + { + "expr": "telemt_buffer_pool_buffers_total{node=~\"$node\", kind=\"in_use\"}", + "legendFormat": "", + "refId": "buffers", + "format": "table", + "instant": true + }, + { + "expr": "telemt_connections_total{node=~\"$node\"}", + "legendFormat": "", + "refId": "connections", + "format": "table", + "instant": true + }, + { + "expr": "telemt_connections_bad_total{node=~\"$node\"}", + "legendFormat": "", + "refId": "bad", + "format": "table", + "instant": true + }, + { + "expr": "telemt_handshake_timeouts_total{node=~\"$node\"}", + "legendFormat": "", + "refId": "hs_timeout", + "format": "table", + "instant": true + } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Connections Rate", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 10, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "cps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_connections_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} accepted", "refId": "A" }, + { "expr": "rate(telemt_connections_bad_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} bad", "refId": "B" }, + { "expr": "rate(telemt_handshake_timeouts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs timeout", "refId": "C" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Upstream Connect", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 11, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "cps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_upstream_connect_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} success", "refId": "A" }, + { "expr": "rate(telemt_upstream_connect_fail_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} fail", "refId": "B" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Upstream Connect Duration (success)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "id": 12, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "increase(telemt_upstream_connect_duration_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{bucket}}", "refId": "A" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "ME Writers & Pool", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "id": 13, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "telemt_me_writers_active_current{node=~\"$node\"}", "legendFormat": "{{node}} active", "refId": "A" }, + { "expr": "telemt_me_writers_warm_current{node=~\"$node\"}", "legendFormat": "{{node}} warm", "refId": "B" }, + { "expr": "telemt_pool_drain_active{node=~\"$node\"}", "legendFormat": "{{node}} draining", "refId": "C" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Per-User Active Connections", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 20, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "telemt_user_connections_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}}", "refId": "A" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Per-User Traffic", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 21, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_user_octets_from_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} rx", "refId": "A" }, + { "expr": "rate(telemt_user_octets_to_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} tx", "refId": "B" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "DC->Client Payload", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "id": 30, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_me_d2c_payload_bytes_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} payload", "refId": "A" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "ME Errors & Anomalies", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, + "id": 31, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "cps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_me_reconnect_attempts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} reconnect", "refId": "A" }, + { "expr": "rate(telemt_me_handshake_reject_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs reject", "refId": "B" }, + { "expr": "rate(telemt_me_crc_mismatch_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} crc mismatch", "refId": "C" }, + { "expr": "rate(telemt_desync_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} desync", "refId": "D" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Per-User Unique IPs", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "id": 40, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "telemt_user_unique_ips_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} active", "refId": "A" }, + { "expr": "telemt_user_unique_ips_recent_window{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} recent", "refId": "B" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }, + { + "title": "Conntrack", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "id": 41, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" }, + "unit": "cps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "targets": [ + { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"attempt\"}[5m])", "legendFormat": "{{node}} delete attempt", "refId": "A" }, + { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"error\"}[5m])", "legendFormat": "{{node}} delete error", "refId": "B" }, + { "expr": "telemt_conntrack_event_queue_depth{node=~\"$node\"}", "legendFormat": "{{node}} queue depth", "refId": "C" } + ], + "datasource": { "type": "prometheus", "uid": "${datasource}" } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["telemt", "mtproxy", "telegram"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(telemt_uptime_seconds, node)", + "hide": 0, + "includeAll": true, + "label": "Node", + "multi": true, + "name": "node", + "query": "label_values(telemt_uptime_seconds, node)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "title": "Telemt MTProxy", + "uid": "telemt-mtproxy" + } diff --git a/k8s-ng/core/prom-stack/external-secrets.yaml b/k8s-ng/core/prom-stack/external-secrets.yaml new file mode 100644 index 0000000..cb7a763 --- /dev/null +++ b/k8s-ng/core/prom-stack/external-secrets.yaml @@ -0,0 +1,125 @@ +# Read-only access to the Grafana database credentials in the psql namespace. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: db-credentials-reader +--- +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: psql-credentials +spec: + provider: + kubernetes: + remoteNamespace: psql + server: + caProvider: + type: ConfigMap + name: kube-root-ca.crt + key: ca.crt + auth: + serviceAccount: + name: db-credentials-reader +--- +# A local break-glass account. OIDC remains the normal login path. +apiVersion: generators.external-secrets.io/v1alpha1 +kind: Password +metadata: + name: grafana-admin +spec: + length: 40 + digits: 10 + symbols: 0 + noUpper: false + allowRepeat: true +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: grafana-admin +spec: + refreshPolicy: CreatedOnce + target: + name: grafana-admin + template: + type: Opaque + data: + admin-user: admin + admin-password: "{{ .password }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: grafana-admin +--- +# Grafana's database credentials come from CNPG. The existing Grafana Keycloak +# client remains in Vaultwarden; item UUIDs are retained so no secret migration +# or rename is required. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: grafana-env +spec: + refreshInterval: 1h + target: + name: grafana-env + deletionPolicy: Delete + template: + type: Opaque + data: + GF_DATABASE_USER: "{{ .db_username }}" + GF_DATABASE_PASSWORD: "{{ .db_password }}" + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .client_id }}" + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .client_secret }}" + data: + - secretKey: db_username + remoteRef: + key: grafana-db + property: username + - secretKey: db_password + remoteRef: + key: grafana-db + property: password + - secretKey: client_id + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: 6a03357b-616e-4973-897a-93ccd675ddcb + property: fields[0].value + - secretKey: client_secret + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: 6a03357b-616e-4973-897a-93ccd675ddcb + property: fields[1].value + secretStoreRef: + kind: SecretStore + name: psql-credentials +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: alertmanager-telegram +spec: + refreshInterval: 1h + target: + name: alertmanager-telegram + deletionPolicy: Delete + template: + type: Opaque + data: + bot-token: "{{ .bot_token }}" + data: + - secretKey: bot_token + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: eca0fb0b-3939-40a8-890a-6294863e5a65 + property: fields[0].value diff --git a/k8s-ng/core/prom-stack/ingress.yaml b/k8s-ng/core/prom-stack/ingress.yaml new file mode 100644 index 0000000..023e85e --- /dev/null +++ b/k8s-ng/core/prom-stack/ingress.yaml @@ -0,0 +1,41 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: prometheus-auth +spec: + forwardAuth: + address: http://auth-proxy.keycloak.svc:80/auth + trustForwardHeader: true + authResponseHeaders: + - X-Auth-Request-User + - X-Auth-Request-Email + - X-Auth-Request-Groups +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: prometheus +spec: + entryPoints: [websecure] + routes: + - match: Host(`prom.hexor.cy`) + kind: Rule + middlewares: + - name: prometheus-auth + services: + - name: prometheus-kube-prometheus-prometheus + port: 9090 + tls: + secretName: prometheus-tls +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: prometheus-tls +spec: + secretName: prometheus-tls + issuerRef: + name: letsencrypt + kind: ClusterIssuer + dnsNames: + - prom.hexor.cy diff --git a/k8s-ng/core/prom-stack/kustomization.yaml b/k8s-ng/core/prom-stack/kustomization.yaml new file mode 100644 index 0000000..26f56ad --- /dev/null +++ b/k8s-ng/core/prom-stack/kustomization.yaml @@ -0,0 +1,37 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - storage.yaml + - external-secrets.yaml + - alertmanager-config.yaml + - ingress.yaml + - service-monitors.yaml + - dashboards/auth-proxy-dashboard-cm.yaml + - dashboards/furumi-dashboard-cm.yaml + - dashboards/telemt-dashboard-cm.yaml + +helmCharts: + - name: kube-prometheus-stack + repo: https://prometheus-community.github.io/helm-charts + version: 92.1.0 + releaseName: prometheus + namespace: prometheus + valuesFile: prom-values.yaml + includeCRDs: true + + - name: loki + repo: https://grafana-community.github.io/helm-charts + version: 18.13.8 + releaseName: loki + namespace: prometheus + valuesFile: loki-values.yaml + includeCRDs: true + + - name: alloy + repo: https://grafana.github.io/helm-charts + version: 1.13.0 + releaseName: alloy + namespace: prometheus + valuesFile: alloy-values.yaml + includeCRDs: true diff --git a/k8s-ng/core/prom-stack/loki-values.yaml b/k8s-ng/core/prom-stack/loki-values.yaml new file mode 100644 index 0000000..d98b3b1 --- /dev/null +++ b/k8s-ng/core/prom-stack/loki-values.yaml @@ -0,0 +1,118 @@ +deploymentMode: Monolithic + +loki: + auth_enabled: false + commonConfig: + path_prefix: /var/loki + replication_factor: 1 + schemaConfig: + configs: + - from: "2026-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + storage: + type: filesystem + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + compactor: + working_directory: /var/loki/compactor + retention_enabled: true + delete_request_store: filesystem + limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + retention_period: 2160h # 90 days + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + max_query_parallelism: 16 + query_timeout: 5m + volume_enabled: true + +singleBinary: + replicas: 1 + sidecar: false + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + terminationGracePeriodSeconds: 120 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "4" + memory: 8Gi + podDisruptionBudget: + enabled: false + persistence: + enabled: true + size: 200Gi + storageClass: monitoring-local + selector: + matchLabels: + monitoring.hexor.cy/volume: loki + whenDeleted: Retain + whenScaled: Retain + +# These must be zero in filesystem-backed monolithic mode. +read: + replicas: 0 +write: + replicas: 0 +backend: + replicas: 0 + +chunksCache: + enabled: false +resultsCache: + enabled: false + +sidecar: + rules: + enabled: false + +gateway: + enabled: true + replicas: 2 + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + verboseLogging: false + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + +test: + enabled: false +lokiCanary: + enabled: true + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + +monitoring: + dashboards: + enabled: true + rules: + enabled: true + alerts: + enabled: true + serviceMonitor: + enabled: true + interval: 30s diff --git a/k8s-ng/core/prom-stack/prom-values.yaml b/k8s-ng/core/prom-stack/prom-values.yaml new file mode 100644 index 0000000..ef8980a --- /dev/null +++ b/k8s-ng/core/prom-stack/prom-values.yaml @@ -0,0 +1,268 @@ +defaultRules: + create: true + rules: + # k3s embeds these control-plane components and does not expose their + # conventional metrics Services. Disabling the groups prevents false pages. + etcd: false + kubeControllerManager: false + kubeProxy: false + kubeSchedulerAlerting: false + kubeSchedulerRecording: false + runbookUrl: https://runbooks.prometheus-operator.dev/runbooks + +kubeControllerManager: + enabled: false +kubeScheduler: + enabled: false +kubeProxy: + enabled: false +kubeEtcd: + enabled: false + +alertmanager: + enabled: true + config: + global: + resolve_timeout: 5m + route: + group_by: [namespace, alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: "null" + receivers: + - name: "null" + alertmanagerSpec: + replicas: 1 + retention: 120h + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + alertmanagerConfigSelector: + matchLabels: + alertmanager: main + # The single global routing object must be allowed to receive alerts from + # every namespace, rather than only alerts labelled namespace=prometheus. + alertmanagerConfigMatcherStrategy: + type: None + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + storage: + volumeClaimTemplate: + spec: + storageClassName: monitoring-local + accessModes: [ReadWriteOnce] + selector: + matchLabels: + monitoring.hexor.cy/volume: alertmanager + resources: + requests: + storage: 5Gi + +prometheus: + prometheusSpec: + replicas: 1 + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + scrapeInterval: 30s + scrapeTimeout: 10s + evaluationInterval: 30s + externalUrl: https://prom.hexor.cy + externalLabels: + cluster: hexor-ng + enableRemoteWriteReceiver: true + walCompression: true + retention: 365d + # Time and size retention are both applied; whichever is reached first + # wins, leaving headroom for compaction and WAL spikes on the 500 Gi PV. + retentionSize: 425GB + resources: + requests: + cpu: "1" + memory: 4Gi + limits: + cpu: "4" + memory: 12Gi + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: monitoring-local + accessModes: [ReadWriteOnce] + selector: + matchLabels: + monitoring.hexor.cy/volume: prometheus + resources: + requests: + storage: 500Gi + + # Discover monitoring resources in every namespace without requiring the + # kube-prometheus-stack release label on application-owned monitors. + ruleSelectorNilUsesHelmValues: false + ruleSelector: {} + ruleNamespaceSelector: {} + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorSelector: {} + serviceMonitorNamespaceSelector: {} + podMonitorSelectorNilUsesHelmValues: false + podMonitorSelector: {} + podMonitorNamespaceSelector: {} + probeSelectorNilUsesHelmValues: false + probeSelector: {} + probeNamespaceSelector: {} + scrapeConfigSelectorNilUsesHelmValues: false + scrapeConfigSelector: {} + scrapeConfigNamespaceSelector: {} + + additionalScrapeConfigs: + - job_name: xray-vpn + metrics_path: /scrape + static_configs: + - targets: [cy.tail2fe2d.ts.net:9550] + labels: {site: cy} + - targets: [x86.tail2fe2d.ts.net:9550] + labels: {site: am} + - targets: [jp.tail2fe2d.ts.net:9550] + labels: {site: jp} + - job_name: macos-node-exporter + static_configs: + - targets: [macbook-pro.tail2fe2d.ts.net:9100] + labels: {instance: ultrabook-pro} + +grafana: + enabled: true + replicas: 1 + testFramework: + enabled: false + annotations: + reloader.stakater.com/auto: "true" + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + admin: + existingSecret: grafana-admin + userKey: admin-user + passwordKey: admin-password + envFromSecret: grafana-env + grafana.ini: + server: + root_url: https://gf.hexor.cy + auth: + signout_redirect_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2Fgf.hexor.cy%2Flogin&client_id=Grafana + oauth_allow_insecure_email_lookup: true + auth.generic_oauth: + name: Keycloak + enabled: true + scopes: openid profile email + allow_sign_up: true + auth_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/auth + token_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/token + api_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/userinfo + email_attribute_path: email + login_attribute_path: preferred_username + name_attribute_path: name + role_attribute_path: >- + contains(groups[*], 'hexor-admin') && 'Admin' || + contains(groups[*], 'hexor-guest') && 'Viewer' || + 'Viewer' + role_attribute_strict: false + database: + type: postgres + host: main-rw.psql.svc:5432 + name: grafana + ssl_mode: disable + log: + level: info + sidecar: + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + dashboards: + enabled: true + searchNamespace: ALL + datasources: + uid: prometheus + alertmanager: + uid: alertmanager + handleGrafanaManagedAlerts: false + additionalDataSources: + - name: Loki + uid: loki + type: loki + url: http://loki-gateway.prometheus.svc:80 + access: proxy + orgId: 1 + jsonData: + maxLines: 2000 + ingress: + enabled: true + ingressClassName: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + hosts: + - gf.hexor.cy + tls: + - secretName: grafana-tls + hosts: + - gf.hexor.cy + +prometheusOperator: + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + prometheusConfigReloader: + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + +kube-state-metrics: + nodeSelector: + kubernetes.io/hostname: music + priorityClassName: monitoring-critical + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +prometheus-node-exporter: + priorityClassName: monitoring-critical + tolerations: + - operator: Exists + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/k8s-ng/core/prom-stack/service-monitors.yaml b/k8s-ng/core/prom-stack/service-monitors.yaml new file mode 100644 index 0000000..6511319 --- /dev/null +++ b/k8s-ng/core/prom-stack/service-monitors.yaml @@ -0,0 +1,31 @@ +# Monitors for core services that already exist before this stack is installed. +# Later application waves keep their ServiceMonitors next to the applications. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: auth-proxy + namespace: keycloak +spec: + selector: + matchLabels: + app: auth-proxy + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: postgresql + namespace: psql +spec: + selector: + matchLabels: + cnpg.io/cluster: main + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/k8s-ng/core/prom-stack/storage.yaml b/k8s-ng/core/prom-stack/storage.yaml new file mode 100644 index 0000000..9f7e441 --- /dev/null +++ b/k8s-ng/core/prom-stack/storage.yaml @@ -0,0 +1,144 @@ +# Prepare the node-local directories before the local PersistentVolumes are +# made available. The hook is idempotent and runs before every Argo CD sync. +apiVersion: batch/v1 +kind: Job +metadata: + name: monitoring-storage-prepare + namespace: prometheus + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded + argocd.argoproj.io/sync-wave: "-2" +spec: + backoffLimit: 3 + template: + spec: + restartPolicy: OnFailure + nodeSelector: + kubernetes.io/hostname: music + containers: + - name: prepare + image: docker.io/busybox:1.37.0 + command: + - sh + - -ec + - | + mkdir -p /storage/prometheus /storage/alertmanager /storage/loki + chown 1000:2000 /storage/prometheus /storage/alertmanager + chown 10001:10001 /storage/loki + chmod 0770 /storage/prometheus /storage/alertmanager /storage/loki + securityContext: + runAsNonRoot: false + runAsUser: 0 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: storage + mountPath: /storage + volumes: + - name: storage + hostPath: + path: /data/k8s/monitoring + type: DirectoryOrCreate +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: monitoring-local + annotations: + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Retain +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: monitoring-prometheus-music + labels: + monitoring.hexor.cy/volume: prometheus + annotations: + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false +spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: monitoring-local + local: + path: /data/k8s/monitoring/prometheus + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [music] +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: monitoring-alertmanager-music + labels: + monitoring.hexor.cy/volume: alertmanager + annotations: + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false +spec: + capacity: + storage: 5Gi + volumeMode: Filesystem + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: monitoring-local + local: + path: /data/k8s/monitoring/alertmanager + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [music] +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: monitoring-loki-music + labels: + monitoring.hexor.cy/volume: loki + annotations: + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false +spec: + capacity: + storage: 200Gi + volumeMode: Filesystem + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: monitoring-local + local: + path: /data/k8s/monitoring/loki + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [music] +--- +# Keep monitoring schedulable under application-node pressure without competing +# with Kubernetes' system-cluster-critical classes. +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: monitoring-critical + annotations: + argocd.argoproj.io/sync-wave: "-1" +value: 1000000 +globalDefault: false +description: Monitoring control plane and storage workloads