Added prom-stack
Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 31s
Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 31s
This commit is contained in:
Aleksandr Bogomiakov committed
1 parent
089bc9bfb5
commit
52bfa88a33
18 files changed
+2385
-1
No files matched your search
@@ -5,4 +5,5 @@ resources:
|
|||||||
- secrets.yaml
|
- secrets.yaml
|
||||||
- deployment.yaml
|
- deployment.yaml
|
||||||
- service.yaml
|
- service.yaml
|
||||||
|
- servicemonitor.yaml
|
||||||
- ingress.yaml
|
- ingress.yaml
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: furumi-player
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: furumi-player
|
||||||
|
endpoints:
|
||||||
|
- port: http
|
||||||
|
path: /metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
@@ -62,7 +62,7 @@ apps:
|
|||||||
project: apps
|
project: apps
|
||||||
path: k8s-ng/apps/furumi
|
path: k8s-ng/apps/furumi
|
||||||
namespace: furumi
|
namespace: furumi
|
||||||
wave: 5 # pinned to the music node; must not hold back the other waves
|
wave: 6 # after prom-stack installs the ServiceMonitor CRD
|
||||||
skipDryRunOnMissingResource: true
|
skipDryRunOnMissingResource: true
|
||||||
- name: web-petting
|
- name: web-petting
|
||||||
project: apps
|
project: apps
|
||||||
@@ -87,3 +87,8 @@ apps:
|
|||||||
wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node
|
wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node
|
||||||
prune: false # never delete the instance because of a manifest change
|
prune: false # never delete the instance because of a manifest change
|
||||||
skipDryRunOnMissingResource: true
|
skipDryRunOnMissingResource: true
|
||||||
|
- name: prom-stack
|
||||||
|
path: k8s-ng/core/prom-stack
|
||||||
|
namespace: prometheus
|
||||||
|
wave: 5 # after secret-stores; installs monitoring before application workloads
|
||||||
|
skipDryRunOnMissingResource: true # monitoring, cert-manager and ExternalSecret CRDs
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# Database "grafana": generated credentials, role, database and read access for
|
||||||
|
# the monitoring namespace.
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: grafana-db
|
||||||
|
namespace: psql
|
||||||
|
spec:
|
||||||
|
refreshPolicy: CreatedOnce
|
||||||
|
target:
|
||||||
|
name: grafana-db
|
||||||
|
template:
|
||||||
|
type: kubernetes.io/basic-auth
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
cnpg.io/reload: "true"
|
||||||
|
data:
|
||||||
|
username: grafana
|
||||||
|
password: "{{ .password }}"
|
||||||
|
dataFrom:
|
||||||
|
- sourceRef:
|
||||||
|
generatorRef:
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
name: db-password
|
||||||
|
---
|
||||||
|
apiVersion: postgresql.cnpg.io/v1
|
||||||
|
kind: DatabaseRole
|
||||||
|
metadata:
|
||||||
|
name: main-grafana
|
||||||
|
namespace: psql
|
||||||
|
spec:
|
||||||
|
cluster:
|
||||||
|
name: main
|
||||||
|
name: grafana
|
||||||
|
login: true
|
||||||
|
passwordSecret:
|
||||||
|
name: grafana-db
|
||||||
|
---
|
||||||
|
apiVersion: postgresql.cnpg.io/v1
|
||||||
|
kind: Database
|
||||||
|
metadata:
|
||||||
|
name: main-grafana
|
||||||
|
namespace: psql
|
||||||
|
spec:
|
||||||
|
cluster:
|
||||||
|
name: main
|
||||||
|
name: grafana
|
||||||
|
owner: grafana
|
||||||
|
databaseReclaimPolicy: retain
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: grafana-db-reader
|
||||||
|
namespace: psql
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [secrets]
|
||||||
|
resourceNames: [grafana-db]
|
||||||
|
verbs: [get, list, watch]
|
||||||
|
- apiGroups: [authorization.k8s.io]
|
||||||
|
resources: [selfsubjectrulesreviews]
|
||||||
|
verbs: [create]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: grafana-db-reader
|
||||||
|
namespace: psql
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: grafana-db-reader
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: db-credentials-reader
|
||||||
|
namespace: prometheus
|
||||||
@@ -9,3 +9,4 @@ resources:
|
|||||||
- databases/furumi.yaml
|
- databases/furumi.yaml
|
||||||
- databases/web-petting.yaml
|
- databases/web-petting.yaml
|
||||||
- databases/meguri.yaml
|
- databases/meguri.yaml
|
||||||
|
- databases/grafana.yaml
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Monitoring stack (new cluster)
|
||||||
|
|
||||||
|
The stack is intentionally a fresh installation: no Prometheus TSDB, Loki log
|
||||||
|
chunks, or Grafana database are copied from the old cluster.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
- `kube-prometheus-stack`: Prometheus, Alertmanager, Grafana, exporters and
|
||||||
|
Kubernetes alerting rules.
|
||||||
|
- `loki`: one filesystem-backed monolithic instance. A second replica would
|
||||||
|
require shared object storage and is deliberately not faked with a local PV.
|
||||||
|
- `alloy`: unprivileged Kubernetes API log and event collection. This replaces
|
||||||
|
EOL Promtail and does not require host log mounts or privileged sysctls.
|
||||||
|
- Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on
|
||||||
|
the large local disk of node `music`.
|
||||||
|
|
||||||
|
## Storage and retention
|
||||||
|
|
||||||
|
The PreSync hook creates these directories on `music`:
|
||||||
|
|
||||||
|
| Path | PV | Retention |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| `/data/k8s/monitoring/prometheus` | 500 Gi | up to 365 days / 425 GB |
|
||||||
|
| `/data/k8s/monitoring/loki` | 200 Gi | 90 days |
|
||||||
|
| `/data/k8s/monitoring/alertmanager` | 5 Gi | 120 hours of notification state |
|
||||||
|
|
||||||
|
All PVs and the StorageClass use `Retain`; Argo CD is also told not to prune
|
||||||
|
them. This protects data from a manifest rename, but it is not a backup. Back up
|
||||||
|
`/data/k8s/monitoring` with the rest of the `music` node if history matters.
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- node `music` must exist and `/data` must be the intended large filesystem;
|
||||||
|
- `main-rw.psql.svc` and the generated `grafana-db` Secret must be ready;
|
||||||
|
- `vaultwarden-login` must resolve the existing Grafana OIDC and Telegram item
|
||||||
|
UUIDs used in `external-secrets.yaml`;
|
||||||
|
- DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster.
|
||||||
|
|
||||||
|
After Argo CD reports the application healthy, verify targets, rules and log
|
||||||
|
delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana
|
||||||
|
uses its own Keycloak OIDC client.
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1alpha1
|
||||||
|
kind: AlertmanagerConfig
|
||||||
|
metadata:
|
||||||
|
name: telegram
|
||||||
|
labels:
|
||||||
|
alertmanager: main
|
||||||
|
spec:
|
||||||
|
route:
|
||||||
|
groupBy: [namespace, alertname]
|
||||||
|
groupWait: 30s
|
||||||
|
groupInterval: 5m
|
||||||
|
repeatInterval: 12h
|
||||||
|
receiver: telegram
|
||||||
|
routes:
|
||||||
|
- receiver: "null"
|
||||||
|
matchers:
|
||||||
|
- name: alertname
|
||||||
|
value: Watchdog|InfoInhibitor
|
||||||
|
matchType: =~
|
||||||
|
receivers:
|
||||||
|
- name: telegram
|
||||||
|
telegramConfigs:
|
||||||
|
- botToken:
|
||||||
|
name: alertmanager-telegram
|
||||||
|
key: bot-token
|
||||||
|
# Telegram chat IDs are routing identifiers rather than credentials;
|
||||||
|
# AlertmanagerConfig only supports an integer here, not a Secret ref.
|
||||||
|
chatID: 124317807
|
||||||
|
sendResolved: true
|
||||||
|
message: |-
|
||||||
|
{{ if eq .Status "firing" }}🔥 FIRING{{ else }}✅ RESOLVED{{ end }}
|
||||||
|
{{ range .Alerts }}
|
||||||
|
{{ .Labels.alertname }} ({{ .Labels.severity }})
|
||||||
|
{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ end }}
|
||||||
|
{{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
|
||||||
|
{{ if .Labels.namespace }}Namespace: {{ .Labels.namespace }}{{ end }}
|
||||||
|
{{ if .Labels.pod }}Pod: {{ .Labels.pod }}{{ end }}
|
||||||
|
{{ if .Labels.node }}Node: {{ .Labels.node }}{{ end }}
|
||||||
|
{{ .GeneratorURL }}
|
||||||
|
{{ end }}
|
||||||
|
- name: "null"
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
alloy:
|
||||||
|
enableReporting: false
|
||||||
|
storagePath: /var/lib/alloy
|
||||||
|
configMap:
|
||||||
|
create: true
|
||||||
|
content: |-
|
||||||
|
discovery.kubernetes "pods" {
|
||||||
|
role = "pod"
|
||||||
|
}
|
||||||
|
|
||||||
|
discovery.relabel "pod_logs" {
|
||||||
|
targets = discovery.kubernetes.pods.targets
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_namespace"]
|
||||||
|
target_label = "namespace"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_name"]
|
||||||
|
target_label = "pod"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||||
|
target_label = "container"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||||
|
target_label = "node"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_label_app"]
|
||||||
|
regex = `(.+)`
|
||||||
|
target_label = "app"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||||
|
regex = `(.+)`
|
||||||
|
target_label = "app"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["namespace", "app"]
|
||||||
|
separator = "/"
|
||||||
|
regex = `(.+)/(.+)`
|
||||||
|
replacement = "$1/$2"
|
||||||
|
target_label = "job"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.kubernetes "pods" {
|
||||||
|
targets = discovery.relabel.pod_logs.output
|
||||||
|
forward_to = [loki.process.pods.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.process "pods" {
|
||||||
|
stage.static_labels {
|
||||||
|
values = {
|
||||||
|
cluster = "hexor-ng",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
forward_to = [loki.write.local.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.kubernetes_events "cluster_events" {
|
||||||
|
job_name = "integrations/kubernetes/eventhandler"
|
||||||
|
log_format = "logfmt"
|
||||||
|
forward_to = [loki.process.cluster_events.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.process "cluster_events" {
|
||||||
|
stage.static_labels {
|
||||||
|
values = {
|
||||||
|
cluster = "hexor-ng",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
forward_to = [loki.write.local.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.write "local" {
|
||||||
|
endpoint {
|
||||||
|
url = "http://loki-gateway.prometheus.svc/loki/api/v1/push"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mounts:
|
||||||
|
extra:
|
||||||
|
- name: data
|
||||||
|
mountPath: /var/lib/alloy
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 473
|
||||||
|
runAsGroup: 473
|
||||||
|
capabilities:
|
||||||
|
drop: [ALL]
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /metrics
|
||||||
|
port: 12345
|
||||||
|
scheme: HTTP
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 30
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 3
|
||||||
|
|
||||||
|
global:
|
||||||
|
podSecurityContext:
|
||||||
|
fsGroup: 473
|
||||||
|
fsGroupChangePolicy: OnRootMismatch
|
||||||
|
runAsNonRoot: true
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
|
||||||
|
controller:
|
||||||
|
type: deployment
|
||||||
|
replicas: 1
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
minReadySeconds: 10
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
volumes:
|
||||||
|
extra:
|
||||||
|
- name: data
|
||||||
|
emptyDir: {}
|
||||||
|
|
||||||
|
configReloader:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 50Mi
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
|
||||||
|
# Only the API permissions used by pod-log discovery and cluster events.
|
||||||
|
rbac:
|
||||||
|
create: true
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [pods, pods/log, namespaces, events]
|
||||||
|
verbs: [get, list, watch]
|
||||||
|
# Kept separate because the Alloy chart concatenates this list into the
|
||||||
|
# ClusterRole. Node metadata is useful for diagnosing a log source.
|
||||||
|
clusterRules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [nodes]
|
||||||
|
verbs: [get, list, watch]
|
||||||
|
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: true
|
||||||
|
interval: 30s
|
||||||
@@ -0,0 +1,367 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: auth-proxy-dashboard
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
auth-proxy.json: |-
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "-- Grafana --"
|
||||||
|
},
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "routes_loaded_total",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Loaded Routes",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(increase(auth_requests_total[$__rate_interval]))",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Auth Requests (total)",
|
||||||
|
"type": "stat",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(increase(auth_requests_total{status=\"403\"}[$__rate_interval]))",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Forbidden (403)",
|
||||||
|
"type": "stat",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "red", "value": 1 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(increase(callback_requests_total{result=\"success\"}[$__rate_interval]))",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Successful Logins",
|
||||||
|
"type": "stat",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "single", "sort": "none" }
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum by (status) (rate(auth_requests_total[$__rate_interval]))",
|
||||||
|
"legendFormat": "{{status}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Auth Request Rate by Status",
|
||||||
|
"type": "timeseries",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 2,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [ { "color": "green", "value": null } ]
|
||||||
|
},
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||||
|
"id": 6,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "single", "sort": "none" }
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "auth_request_duration_seconds{quantile=\"0.5\"}",
|
||||||
|
"legendFormat": "p50",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "auth_request_duration_seconds{quantile=\"0.95\"}",
|
||||||
|
"legendFormat": "p95",
|
||||||
|
"refId": "B"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "auth_request_duration_seconds{quantile=\"0.99\"}",
|
||||||
|
"legendFormat": "p99",
|
||||||
|
"refId": "C"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Auth Request Duration (p50, p95, p99)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 0,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 2,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [ { "color": "green", "value": null } ]
|
||||||
|
},
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||||
|
"id": 7,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "single", "sort": "none" }
|
||||||
|
},
|
||||||
|
"pluginVersion": "10.0.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum by (result) (rate(callback_requests_total[$__rate_interval]))",
|
||||||
|
"legendFormat": "{{result}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "OIDC Callback Rate by Result",
|
||||||
|
"type": "timeseries",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"fillOpacity": 50,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "normal" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [ { "color": "green", "value": null } ]
|
||||||
|
},
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": [
|
||||||
|
"auth-proxy",
|
||||||
|
"rsauth2-proxy"
|
||||||
|
],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"current": {
|
||||||
|
"selected": false,
|
||||||
|
"text": "Prometheus",
|
||||||
|
"value": "Prometheus"
|
||||||
|
},
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"multi": false,
|
||||||
|
"name": "datasource",
|
||||||
|
"options": [],
|
||||||
|
"query": "prometheus",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"type": "datasource"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-1h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Auth Proxy (rsauth2-proxy)",
|
||||||
|
"uid": "auth-proxy-metrics",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,504 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: furumi-dashboard
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
furumi.json: |-
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "Build Version",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "max by (version) (furumusic_build_info{namespace=~\"$namespace\"})",
|
||||||
|
"legendFormat": "{{version}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "name" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "HTTP RPS",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "5xx Error Ratio",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "HTTP p95 Latency",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Active Users 15m",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_active_users{namespace=~\"$namespace\",window=\"15m\"})", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Stream Throughput",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "Bps", "decimals": 1 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
|
||||||
|
},
|
||||||
|
{ "id": 10, "title": "HTTP", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"title": "Request Rate by Route and Status",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route, status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "{{route}} {{status}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"title": "Request Duration Quantiles",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"title": "In-flight Requests",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route) (furumusic_http_in_flight_requests{namespace=~\"$namespace\",route=~\"$route\"})", "legendFormat": "{{route}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"title": "HTTP Body Throughput",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_request_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "request", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_response_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "response", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 15,
|
||||||
|
"title": "Status Code Mix",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{ "id": 20, "title": "Auth and Users", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 21,
|
||||||
|
"title": "Users by Role",
|
||||||
|
"type": "bargauge",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (role) (furumusic_users_total{namespace=~\"$namespace\"})", "legendFormat": "{{role}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "showUnfilled": true }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 22,
|
||||||
|
"title": "Active Users",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (furumusic_active_users{namespace=~\"$namespace\"})", "legendFormat": "{{window}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 23,
|
||||||
|
"title": "Auth Events",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method, outcome, reason) (rate(furumusic_auth_login_attempts_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "login {{method}} {{outcome}} {{reason}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method) (rate(furumusic_auth_sessions_created_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "session {{method}}", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(furumusic_auth_denied_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "denied {{kind}}", "refId": "C" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{ "id": 30, "title": "Playback and Streaming", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 31,
|
||||||
|
"title": "Listens Rate",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (completed) (rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "completed={{completed}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 32,
|
||||||
|
"title": "Listen Completion Ratio",
|
||||||
|
"description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 33,
|
||||||
|
"title": "Streaming",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (range) (rate(furumusic_stream_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "requests range={{range}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 34,
|
||||||
|
"title": "Listened Hours and Play History Activity",
|
||||||
|
"description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "h", "decimals": 2 },
|
||||||
|
"overrides": [
|
||||||
|
{ "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{ "id": 40, "title": "Library and Storage", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 41,
|
||||||
|
"title": "Library Inventory",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_artists_total{namespace=~\"$namespace\"})", "legendFormat": "artists", "refId": "C" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_playlists_total{namespace=~\"$namespace\"})", "legendFormat": "playlists", "refId": "D" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 42,
|
||||||
|
"title": "Media Bytes by Type",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 43,
|
||||||
|
"title": "Storage Used Ratio",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "1 - (sum by (path_kind) (furumusic_storage_free_bytes{namespace=~\"$namespace\"}) / sum by (path_kind) (furumusic_storage_total_bytes{namespace=~\"$namespace\"}))", "legendFormat": "{{path_kind}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{ "id": 50, "title": "AI Agent", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 51,
|
||||||
|
"title": "Agent Queue and Reviews",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 52,
|
||||||
|
"title": "Agent Processing and Failures",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 52 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome, decision) (rate(furumusic_agent_files_processed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "processed {{outcome}} {{decision}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (stage) (rate(furumusic_agent_failed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "failed {{stage}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 53,
|
||||||
|
"title": "Agent Confidence",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 52 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 54,
|
||||||
|
"title": "Discover Runs and Duration",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_agent_discover_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "runs {{outcome}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(furumusic_agent_discover_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{outcome}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 55,
|
||||||
|
"title": "Discover File Flow",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_seen_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "seen", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_queued_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "queued", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_discover_files_skipped_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}", "refId": "C" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 56,
|
||||||
|
"title": "RAG and LLM Requests",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind, outcome) (rate(furumusic_agent_rag_queries_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, outcome) (rate(furumusic_agent_llm_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 57,
|
||||||
|
"title": "RAG and LLM Latency p95",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, kind, outcome) (rate(furumusic_agent_rag_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, model, outcome) (rate(furumusic_agent_llm_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 58,
|
||||||
|
"title": "LLM Token Rate, Batch Splits and Parse Failures",
|
||||||
|
"description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "suffix: tok/s" },
|
||||||
|
"overrides": [
|
||||||
|
{ "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] },
|
||||||
|
{ "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model) (rate(furumusic_agent_llm_parse_failures_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "parse failures {{model}}", "refId": "C" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 59,
|
||||||
|
"title": "Cover Pipeline",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, outcome) (rate(furumusic_agent_cover_lookup_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "lookup {{source}} {{outcome}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (variant, outcome) (rate(furumusic_agent_cover_variant_generation_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "variant {{variant}} {{outcome}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{ "id": 70, "title": "Scheduler and Torrents", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "collapsed": false, "panels": [] },
|
||||||
|
{
|
||||||
|
"id": 71,
|
||||||
|
"title": "Scheduler Jobs",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_running{namespace=~\"$namespace\"})", "legendFormat": "running {{job}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_enabled{namespace=~\"$namespace\"})", "legendFormat": "enabled {{job}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 72,
|
||||||
|
"title": "Scheduler Runs and Duration p95",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job, trigger, outcome) (rate(furumusic_scheduler_job_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{job}} {{trigger}} {{outcome}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, job, trigger, outcome) (rate(furumusic_scheduler_job_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{job}} {{outcome}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 73,
|
||||||
|
"title": "Torrents",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_torrent_sessions_total{namespace=~\"$namespace\"})", "legendFormat": "sessions {{status}}", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_downloads_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "downloads {{outcome}}", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_selected_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes {{outcome}}", "refId": "C" }
|
||||||
|
],
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": [ "furumi", "furumusic", "music" ],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"multi": false,
|
||||||
|
"name": "datasource",
|
||||||
|
"options": [],
|
||||||
|
"query": "prometheus",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"type": "datasource"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"allValue": ".*",
|
||||||
|
"current": { "selected": true, "text": "All", "value": "$__all" },
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"definition": "label_values(furumusic_build_info, namespace)",
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": true,
|
||||||
|
"multi": true,
|
||||||
|
"name": "namespace",
|
||||||
|
"options": [],
|
||||||
|
"query": { "query": "label_values(furumusic_build_info, namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"sort": 1,
|
||||||
|
"type": "query"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"allValue": ".*",
|
||||||
|
"current": { "selected": true, "text": "All", "value": "$__all" },
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"definition": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)",
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": true,
|
||||||
|
"multi": true,
|
||||||
|
"name": "route",
|
||||||
|
"options": [],
|
||||||
|
"query": { "query": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"sort": 1,
|
||||||
|
"type": "query"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Furumi Music",
|
||||||
|
"uid": "furumi-music",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,409 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: telemt-dashboard
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
telemt.json: |-
|
||||||
|
{
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"title": "Nodes Overview",
|
||||||
|
"type": "table",
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
|
||||||
|
"id": 1,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {
|
||||||
|
"align": "auto",
|
||||||
|
"cellOptions": { "type": "auto" },
|
||||||
|
"inspect": false
|
||||||
|
},
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"matcher": { "id": "byName", "options": "Uptime" },
|
||||||
|
"properties": [
|
||||||
|
{ "id": "unit", "value": "dtdurations" },
|
||||||
|
{ "id": "custom.width", "value": 140 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"matcher": { "id": "byName", "options": "Bad Conn" },
|
||||||
|
"properties": [
|
||||||
|
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 100 }] } },
|
||||||
|
{ "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"matcher": { "id": "byName", "options": "Writers" },
|
||||||
|
"properties": [
|
||||||
|
{ "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
|
||||||
|
{ "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"showHeader": true,
|
||||||
|
"sortBy": [{ "displayName": "Node", "desc": false }],
|
||||||
|
"frameIndex": 0,
|
||||||
|
"footer": { "show": false }
|
||||||
|
},
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"id": "joinByField",
|
||||||
|
"options": { "byField": "node", "mode": "outer" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "filterFieldsByName",
|
||||||
|
"options": {
|
||||||
|
"include": { "pattern": "^(node|Value.*)$" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "organize",
|
||||||
|
"options": {
|
||||||
|
"renameByName": {
|
||||||
|
"node": "Node",
|
||||||
|
"Value #uptime": "Uptime",
|
||||||
|
"Value #writers": "Writers",
|
||||||
|
"Value #buffers": "Buffers In Use",
|
||||||
|
"Value #connections": "Connections",
|
||||||
|
"Value #bad": "Bad Conn",
|
||||||
|
"Value #hs_timeout": "HS Timeouts"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "telemt_uptime_seconds{node=~\"$node\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "uptime",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "telemt_me_writers_active_current{node=~\"$node\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "writers",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "telemt_buffer_pool_buffers_total{node=~\"$node\", kind=\"in_use\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "buffers",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "telemt_connections_total{node=~\"$node\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "connections",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "telemt_connections_bad_total{node=~\"$node\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "bad",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "telemt_handshake_timeouts_total{node=~\"$node\"}",
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "hs_timeout",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Connections Rate",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||||
|
"id": 10,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "cps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_connections_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} accepted", "refId": "A" },
|
||||||
|
{ "expr": "rate(telemt_connections_bad_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} bad", "refId": "B" },
|
||||||
|
{ "expr": "rate(telemt_handshake_timeouts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs timeout", "refId": "C" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Upstream Connect",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||||
|
"id": 11,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "cps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_upstream_connect_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} success", "refId": "A" },
|
||||||
|
{ "expr": "rate(telemt_upstream_connect_fail_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} fail", "refId": "B" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Upstream Connect Duration (success)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||||
|
"id": 12,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } },
|
||||||
|
"unit": "short",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "increase(telemt_upstream_connect_duration_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{bucket}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "ME Writers & Pool",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||||
|
"id": 13,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "telemt_me_writers_active_current{node=~\"$node\"}", "legendFormat": "{{node}} active", "refId": "A" },
|
||||||
|
{ "expr": "telemt_me_writers_warm_current{node=~\"$node\"}", "legendFormat": "{{node}} warm", "refId": "B" },
|
||||||
|
{ "expr": "telemt_pool_drain_active{node=~\"$node\"}", "legendFormat": "{{node}} draining", "refId": "C" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Per-User Active Connections",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
||||||
|
"id": 20,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "telemt_user_connections_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Per-User Traffic",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
||||||
|
"id": 21,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "Bps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_user_octets_from_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} rx", "refId": "A" },
|
||||||
|
{ "expr": "rate(telemt_user_octets_to_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} tx", "refId": "B" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "DC->Client Payload",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
|
||||||
|
"id": 30,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "Bps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_me_d2c_payload_bytes_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} payload", "refId": "A" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "ME Errors & Anomalies",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
|
||||||
|
"id": 31,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "cps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_me_reconnect_attempts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} reconnect", "refId": "A" },
|
||||||
|
{ "expr": "rate(telemt_me_handshake_reject_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs reject", "refId": "B" },
|
||||||
|
{ "expr": "rate(telemt_me_crc_mismatch_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} crc mismatch", "refId": "C" },
|
||||||
|
{ "expr": "rate(telemt_desync_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} desync", "refId": "D" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Per-User Unique IPs",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
|
||||||
|
"id": 40,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "telemt_user_unique_ips_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} active", "refId": "A" },
|
||||||
|
{ "expr": "telemt_user_unique_ips_recent_window{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} recent", "refId": "B" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Conntrack",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
|
||||||
|
"id": 41,
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
|
||||||
|
"unit": "cps",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"attempt\"}[5m])", "legendFormat": "{{node}} delete attempt", "refId": "A" },
|
||||||
|
{ "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"error\"}[5m])", "legendFormat": "{{node}} delete error", "refId": "B" },
|
||||||
|
{ "expr": "telemt_conntrack_event_queue_depth{node=~\"$node\"}", "legendFormat": "{{node}} queue depth", "refId": "C" }
|
||||||
|
],
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" }
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["telemt", "mtproxy", "telegram"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"current": {},
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"label": "Datasource",
|
||||||
|
"multi": false,
|
||||||
|
"name": "datasource",
|
||||||
|
"options": [],
|
||||||
|
"query": "prometheus",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"type": "datasource"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"current": {},
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"definition": "label_values(telemt_uptime_seconds, node)",
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": true,
|
||||||
|
"label": "Node",
|
||||||
|
"multi": true,
|
||||||
|
"name": "node",
|
||||||
|
"query": "label_values(telemt_uptime_seconds, node)",
|
||||||
|
"refresh": 2,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"sort": 1,
|
||||||
|
"type": "query"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"title": "Telemt MTProxy",
|
||||||
|
"uid": "telemt-mtproxy"
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
# Read-only access to the Grafana database credentials in the psql namespace.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: db-credentials-reader
|
||||||
|
---
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: SecretStore
|
||||||
|
metadata:
|
||||||
|
name: psql-credentials
|
||||||
|
spec:
|
||||||
|
provider:
|
||||||
|
kubernetes:
|
||||||
|
remoteNamespace: psql
|
||||||
|
server:
|
||||||
|
caProvider:
|
||||||
|
type: ConfigMap
|
||||||
|
name: kube-root-ca.crt
|
||||||
|
key: ca.crt
|
||||||
|
auth:
|
||||||
|
serviceAccount:
|
||||||
|
name: db-credentials-reader
|
||||||
|
---
|
||||||
|
# A local break-glass account. OIDC remains the normal login path.
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
metadata:
|
||||||
|
name: grafana-admin
|
||||||
|
spec:
|
||||||
|
length: 40
|
||||||
|
digits: 10
|
||||||
|
symbols: 0
|
||||||
|
noUpper: false
|
||||||
|
allowRepeat: true
|
||||||
|
---
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: grafana-admin
|
||||||
|
spec:
|
||||||
|
refreshPolicy: CreatedOnce
|
||||||
|
target:
|
||||||
|
name: grafana-admin
|
||||||
|
template:
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
admin-user: admin
|
||||||
|
admin-password: "{{ .password }}"
|
||||||
|
dataFrom:
|
||||||
|
- sourceRef:
|
||||||
|
generatorRef:
|
||||||
|
apiVersion: generators.external-secrets.io/v1alpha1
|
||||||
|
kind: Password
|
||||||
|
name: grafana-admin
|
||||||
|
---
|
||||||
|
# Grafana's database credentials come from CNPG. The existing Grafana Keycloak
|
||||||
|
# client remains in Vaultwarden; item UUIDs are retained so no secret migration
|
||||||
|
# or rename is required.
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: grafana-env
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
target:
|
||||||
|
name: grafana-env
|
||||||
|
deletionPolicy: Delete
|
||||||
|
template:
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
GF_DATABASE_USER: "{{ .db_username }}"
|
||||||
|
GF_DATABASE_PASSWORD: "{{ .db_password }}"
|
||||||
|
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .client_id }}"
|
||||||
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .client_secret }}"
|
||||||
|
data:
|
||||||
|
- secretKey: db_username
|
||||||
|
remoteRef:
|
||||||
|
key: grafana-db
|
||||||
|
property: username
|
||||||
|
- secretKey: db_password
|
||||||
|
remoteRef:
|
||||||
|
key: grafana-db
|
||||||
|
property: password
|
||||||
|
- secretKey: client_id
|
||||||
|
sourceRef:
|
||||||
|
storeRef:
|
||||||
|
name: vaultwarden-login
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
remoteRef:
|
||||||
|
key: 6a03357b-616e-4973-897a-93ccd675ddcb
|
||||||
|
property: fields[0].value
|
||||||
|
- secretKey: client_secret
|
||||||
|
sourceRef:
|
||||||
|
storeRef:
|
||||||
|
name: vaultwarden-login
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
remoteRef:
|
||||||
|
key: 6a03357b-616e-4973-897a-93ccd675ddcb
|
||||||
|
property: fields[1].value
|
||||||
|
secretStoreRef:
|
||||||
|
kind: SecretStore
|
||||||
|
name: psql-credentials
|
||||||
|
---
|
||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: alertmanager-telegram
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
target:
|
||||||
|
name: alertmanager-telegram
|
||||||
|
deletionPolicy: Delete
|
||||||
|
template:
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
bot-token: "{{ .bot_token }}"
|
||||||
|
data:
|
||||||
|
- secretKey: bot_token
|
||||||
|
sourceRef:
|
||||||
|
storeRef:
|
||||||
|
name: vaultwarden-login
|
||||||
|
kind: ClusterSecretStore
|
||||||
|
remoteRef:
|
||||||
|
key: eca0fb0b-3939-40a8-890a-6294863e5a65
|
||||||
|
property: fields[0].value
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: prometheus-auth
|
||||||
|
spec:
|
||||||
|
forwardAuth:
|
||||||
|
address: http://auth-proxy.keycloak.svc:80/auth
|
||||||
|
trustForwardHeader: true
|
||||||
|
authResponseHeaders:
|
||||||
|
- X-Auth-Request-User
|
||||||
|
- X-Auth-Request-Email
|
||||||
|
- X-Auth-Request-Groups
|
||||||
|
---
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
spec:
|
||||||
|
entryPoints: [websecure]
|
||||||
|
routes:
|
||||||
|
- match: Host(`prom.hexor.cy`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: prometheus-auth
|
||||||
|
services:
|
||||||
|
- name: prometheus-kube-prometheus-prometheus
|
||||||
|
port: 9090
|
||||||
|
tls:
|
||||||
|
secretName: prometheus-tls
|
||||||
|
---
|
||||||
|
apiVersion: cert-manager.io/v1
|
||||||
|
kind: Certificate
|
||||||
|
metadata:
|
||||||
|
name: prometheus-tls
|
||||||
|
spec:
|
||||||
|
secretName: prometheus-tls
|
||||||
|
issuerRef:
|
||||||
|
name: letsencrypt
|
||||||
|
kind: ClusterIssuer
|
||||||
|
dnsNames:
|
||||||
|
- prom.hexor.cy
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- storage.yaml
|
||||||
|
- external-secrets.yaml
|
||||||
|
- alertmanager-config.yaml
|
||||||
|
- ingress.yaml
|
||||||
|
- service-monitors.yaml
|
||||||
|
- dashboards/auth-proxy-dashboard-cm.yaml
|
||||||
|
- dashboards/furumi-dashboard-cm.yaml
|
||||||
|
- dashboards/telemt-dashboard-cm.yaml
|
||||||
|
|
||||||
|
helmCharts:
|
||||||
|
- name: kube-prometheus-stack
|
||||||
|
repo: https://prometheus-community.github.io/helm-charts
|
||||||
|
version: 92.1.0
|
||||||
|
releaseName: prometheus
|
||||||
|
namespace: prometheus
|
||||||
|
valuesFile: prom-values.yaml
|
||||||
|
includeCRDs: true
|
||||||
|
|
||||||
|
- name: loki
|
||||||
|
repo: https://grafana-community.github.io/helm-charts
|
||||||
|
version: 18.13.8
|
||||||
|
releaseName: loki
|
||||||
|
namespace: prometheus
|
||||||
|
valuesFile: loki-values.yaml
|
||||||
|
includeCRDs: true
|
||||||
|
|
||||||
|
- name: alloy
|
||||||
|
repo: https://grafana.github.io/helm-charts
|
||||||
|
version: 1.13.0
|
||||||
|
releaseName: alloy
|
||||||
|
namespace: prometheus
|
||||||
|
valuesFile: alloy-values.yaml
|
||||||
|
includeCRDs: true
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
deploymentMode: Monolithic
|
||||||
|
|
||||||
|
loki:
|
||||||
|
auth_enabled: false
|
||||||
|
commonConfig:
|
||||||
|
path_prefix: /var/loki
|
||||||
|
replication_factor: 1
|
||||||
|
schemaConfig:
|
||||||
|
configs:
|
||||||
|
- from: "2026-01-01"
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
storage:
|
||||||
|
type: filesystem
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /var/loki/chunks
|
||||||
|
rules_directory: /var/loki/rules
|
||||||
|
compactor:
|
||||||
|
working_directory: /var/loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
limits_config:
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
retention_period: 2160h # 90 days
|
||||||
|
ingestion_rate_mb: 16
|
||||||
|
ingestion_burst_size_mb: 32
|
||||||
|
max_query_parallelism: 16
|
||||||
|
query_timeout: 5m
|
||||||
|
volume_enabled: true
|
||||||
|
|
||||||
|
singleBinary:
|
||||||
|
replicas: 1
|
||||||
|
sidecar: false
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
terminationGracePeriodSeconds: 120
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 2Gi
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: 8Gi
|
||||||
|
podDisruptionBudget:
|
||||||
|
enabled: false
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
size: 200Gi
|
||||||
|
storageClass: monitoring-local
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
monitoring.hexor.cy/volume: loki
|
||||||
|
whenDeleted: Retain
|
||||||
|
whenScaled: Retain
|
||||||
|
|
||||||
|
# These must be zero in filesystem-backed monolithic mode.
|
||||||
|
read:
|
||||||
|
replicas: 0
|
||||||
|
write:
|
||||||
|
replicas: 0
|
||||||
|
backend:
|
||||||
|
replicas: 0
|
||||||
|
|
||||||
|
chunksCache:
|
||||||
|
enabled: false
|
||||||
|
resultsCache:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
sidecar:
|
||||||
|
rules:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
gateway:
|
||||||
|
enabled: true
|
||||||
|
replicas: 2
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
verboseLogging: false
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 256Mi
|
||||||
|
podDisruptionBudget:
|
||||||
|
enabled: true
|
||||||
|
maxUnavailable: 1
|
||||||
|
|
||||||
|
test:
|
||||||
|
enabled: false
|
||||||
|
lokiCanary:
|
||||||
|
enabled: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
|
||||||
|
monitoring:
|
||||||
|
dashboards:
|
||||||
|
enabled: true
|
||||||
|
rules:
|
||||||
|
enabled: true
|
||||||
|
alerts:
|
||||||
|
enabled: true
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: true
|
||||||
|
interval: 30s
|
||||||
@@ -0,0 +1,268 @@
|
|||||||
|
defaultRules:
|
||||||
|
create: true
|
||||||
|
rules:
|
||||||
|
# k3s embeds these control-plane components and does not expose their
|
||||||
|
# conventional metrics Services. Disabling the groups prevents false pages.
|
||||||
|
etcd: false
|
||||||
|
kubeControllerManager: false
|
||||||
|
kubeProxy: false
|
||||||
|
kubeSchedulerAlerting: false
|
||||||
|
kubeSchedulerRecording: false
|
||||||
|
runbookUrl: https://runbooks.prometheus-operator.dev/runbooks
|
||||||
|
|
||||||
|
kubeControllerManager:
|
||||||
|
enabled: false
|
||||||
|
kubeScheduler:
|
||||||
|
enabled: false
|
||||||
|
kubeProxy:
|
||||||
|
enabled: false
|
||||||
|
kubeEtcd:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
enabled: true
|
||||||
|
config:
|
||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
route:
|
||||||
|
group_by: [namespace, alertname]
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 12h
|
||||||
|
receiver: "null"
|
||||||
|
receivers:
|
||||||
|
- name: "null"
|
||||||
|
alertmanagerSpec:
|
||||||
|
replicas: 1
|
||||||
|
retention: 120h
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
alertmanagerConfigSelector:
|
||||||
|
matchLabels:
|
||||||
|
alertmanager: main
|
||||||
|
# The single global routing object must be allowed to receive alerts from
|
||||||
|
# every namespace, rather than only alerts labelled namespace=prometheus.
|
||||||
|
alertmanagerConfigMatcherStrategy:
|
||||||
|
type: None
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 1Gi
|
||||||
|
storage:
|
||||||
|
volumeClaimTemplate:
|
||||||
|
spec:
|
||||||
|
storageClassName: monitoring-local
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
monitoring.hexor.cy/volume: alertmanager
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 5Gi
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
prometheusSpec:
|
||||||
|
replicas: 1
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
scrapeInterval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
evaluationInterval: 30s
|
||||||
|
externalUrl: https://prom.hexor.cy
|
||||||
|
externalLabels:
|
||||||
|
cluster: hexor-ng
|
||||||
|
enableRemoteWriteReceiver: true
|
||||||
|
walCompression: true
|
||||||
|
retention: 365d
|
||||||
|
# Time and size retention are both applied; whichever is reached first
|
||||||
|
# wins, leaving headroom for compaction and WAL spikes on the 500 Gi PV.
|
||||||
|
retentionSize: 425GB
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 4Gi
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: 12Gi
|
||||||
|
storageSpec:
|
||||||
|
volumeClaimTemplate:
|
||||||
|
spec:
|
||||||
|
storageClassName: monitoring-local
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
monitoring.hexor.cy/volume: prometheus
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 500Gi
|
||||||
|
|
||||||
|
# Discover monitoring resources in every namespace without requiring the
|
||||||
|
# kube-prometheus-stack release label on application-owned monitors.
|
||||||
|
ruleSelectorNilUsesHelmValues: false
|
||||||
|
ruleSelector: {}
|
||||||
|
ruleNamespaceSelector: {}
|
||||||
|
serviceMonitorSelectorNilUsesHelmValues: false
|
||||||
|
serviceMonitorSelector: {}
|
||||||
|
serviceMonitorNamespaceSelector: {}
|
||||||
|
podMonitorSelectorNilUsesHelmValues: false
|
||||||
|
podMonitorSelector: {}
|
||||||
|
podMonitorNamespaceSelector: {}
|
||||||
|
probeSelectorNilUsesHelmValues: false
|
||||||
|
probeSelector: {}
|
||||||
|
probeNamespaceSelector: {}
|
||||||
|
scrapeConfigSelectorNilUsesHelmValues: false
|
||||||
|
scrapeConfigSelector: {}
|
||||||
|
scrapeConfigNamespaceSelector: {}
|
||||||
|
|
||||||
|
additionalScrapeConfigs:
|
||||||
|
- job_name: xray-vpn
|
||||||
|
metrics_path: /scrape
|
||||||
|
static_configs:
|
||||||
|
- targets: [cy.tail2fe2d.ts.net:9550]
|
||||||
|
labels: {site: cy}
|
||||||
|
- targets: [x86.tail2fe2d.ts.net:9550]
|
||||||
|
labels: {site: am}
|
||||||
|
- targets: [jp.tail2fe2d.ts.net:9550]
|
||||||
|
labels: {site: jp}
|
||||||
|
- job_name: macos-node-exporter
|
||||||
|
static_configs:
|
||||||
|
- targets: [macbook-pro.tail2fe2d.ts.net:9100]
|
||||||
|
labels: {instance: ultrabook-pro}
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
enabled: true
|
||||||
|
replicas: 1
|
||||||
|
testFramework:
|
||||||
|
enabled: false
|
||||||
|
annotations:
|
||||||
|
reloader.stakater.com/auto: "true"
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 2Gi
|
||||||
|
admin:
|
||||||
|
existingSecret: grafana-admin
|
||||||
|
userKey: admin-user
|
||||||
|
passwordKey: admin-password
|
||||||
|
envFromSecret: grafana-env
|
||||||
|
grafana.ini:
|
||||||
|
server:
|
||||||
|
root_url: https://gf.hexor.cy
|
||||||
|
auth:
|
||||||
|
signout_redirect_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2Fgf.hexor.cy%2Flogin&client_id=Grafana
|
||||||
|
oauth_allow_insecure_email_lookup: true
|
||||||
|
auth.generic_oauth:
|
||||||
|
name: Keycloak
|
||||||
|
enabled: true
|
||||||
|
scopes: openid profile email
|
||||||
|
allow_sign_up: true
|
||||||
|
auth_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/auth
|
||||||
|
token_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/token
|
||||||
|
api_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/userinfo
|
||||||
|
email_attribute_path: email
|
||||||
|
login_attribute_path: preferred_username
|
||||||
|
name_attribute_path: name
|
||||||
|
role_attribute_path: >-
|
||||||
|
contains(groups[*], 'hexor-admin') && 'Admin' ||
|
||||||
|
contains(groups[*], 'hexor-guest') && 'Viewer' ||
|
||||||
|
'Viewer'
|
||||||
|
role_attribute_strict: false
|
||||||
|
database:
|
||||||
|
type: postgres
|
||||||
|
host: main-rw.psql.svc:5432
|
||||||
|
name: grafana
|
||||||
|
ssl_mode: disable
|
||||||
|
log:
|
||||||
|
level: info
|
||||||
|
sidecar:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 20m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
dashboards:
|
||||||
|
enabled: true
|
||||||
|
searchNamespace: ALL
|
||||||
|
datasources:
|
||||||
|
uid: prometheus
|
||||||
|
alertmanager:
|
||||||
|
uid: alertmanager
|
||||||
|
handleGrafanaManagedAlerts: false
|
||||||
|
additionalDataSources:
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
url: http://loki-gateway.prometheus.svc:80
|
||||||
|
access: proxy
|
||||||
|
orgId: 1
|
||||||
|
jsonData:
|
||||||
|
maxLines: 2000
|
||||||
|
ingress:
|
||||||
|
enabled: true
|
||||||
|
ingressClassName: traefik
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt
|
||||||
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||||
|
hosts:
|
||||||
|
- gf.hexor.cy
|
||||||
|
tls:
|
||||||
|
- secretName: grafana-tls
|
||||||
|
hosts:
|
||||||
|
- gf.hexor.cy
|
||||||
|
|
||||||
|
prometheusOperator:
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
prometheusConfigReloader:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 20m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 256Mi
|
||||||
|
|
||||||
|
kube-state-metrics:
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
|
||||||
|
prometheus-node-exporter:
|
||||||
|
priorityClassName: monitoring-critical
|
||||||
|
tolerations:
|
||||||
|
- operator: Exists
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 256Mi
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# Monitors for core services that already exist before this stack is installed.
|
||||||
|
# Later application waves keep their ServiceMonitors next to the applications.
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: auth-proxy
|
||||||
|
namespace: keycloak
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: auth-proxy
|
||||||
|
endpoints:
|
||||||
|
- port: http
|
||||||
|
path: /metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PodMonitor
|
||||||
|
metadata:
|
||||||
|
name: postgresql
|
||||||
|
namespace: psql
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
cnpg.io/cluster: main
|
||||||
|
podMetricsEndpoints:
|
||||||
|
- port: metrics
|
||||||
|
path: /metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
# Prepare the node-local directories before the local PersistentVolumes are
|
||||||
|
# made available. The hook is idempotent and runs before every Argo CD sync.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: monitoring-storage-prepare
|
||||||
|
namespace: prometheus
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/hook: PreSync
|
||||||
|
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
|
||||||
|
argocd.argoproj.io/sync-wave: "-2"
|
||||||
|
spec:
|
||||||
|
backoffLimit: 3
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: music
|
||||||
|
containers:
|
||||||
|
- name: prepare
|
||||||
|
image: docker.io/busybox:1.37.0
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -ec
|
||||||
|
- |
|
||||||
|
mkdir -p /storage/prometheus /storage/alertmanager /storage/loki
|
||||||
|
chown 1000:2000 /storage/prometheus /storage/alertmanager
|
||||||
|
chown 10001:10001 /storage/loki
|
||||||
|
chmod 0770 /storage/prometheus /storage/alertmanager /storage/loki
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: false
|
||||||
|
runAsUser: 0
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
volumeMounts:
|
||||||
|
- name: storage
|
||||||
|
mountPath: /storage
|
||||||
|
volumes:
|
||||||
|
- name: storage
|
||||||
|
hostPath:
|
||||||
|
path: /data/k8s/monitoring
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
---
|
||||||
|
apiVersion: storage.k8s.io/v1
|
||||||
|
kind: StorageClass
|
||||||
|
metadata:
|
||||||
|
name: monitoring-local
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-1"
|
||||||
|
argocd.argoproj.io/sync-options: Prune=false
|
||||||
|
provisioner: kubernetes.io/no-provisioner
|
||||||
|
volumeBindingMode: WaitForFirstConsumer
|
||||||
|
reclaimPolicy: Retain
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolume
|
||||||
|
metadata:
|
||||||
|
name: monitoring-prometheus-music
|
||||||
|
labels:
|
||||||
|
monitoring.hexor.cy/volume: prometheus
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-1"
|
||||||
|
argocd.argoproj.io/sync-options: Prune=false
|
||||||
|
spec:
|
||||||
|
capacity:
|
||||||
|
storage: 500Gi
|
||||||
|
volumeMode: Filesystem
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
persistentVolumeReclaimPolicy: Retain
|
||||||
|
storageClassName: monitoring-local
|
||||||
|
local:
|
||||||
|
path: /data/k8s/monitoring/prometheus
|
||||||
|
nodeAffinity:
|
||||||
|
required:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: In
|
||||||
|
values: [music]
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolume
|
||||||
|
metadata:
|
||||||
|
name: monitoring-alertmanager-music
|
||||||
|
labels:
|
||||||
|
monitoring.hexor.cy/volume: alertmanager
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-1"
|
||||||
|
argocd.argoproj.io/sync-options: Prune=false
|
||||||
|
spec:
|
||||||
|
capacity:
|
||||||
|
storage: 5Gi
|
||||||
|
volumeMode: Filesystem
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
persistentVolumeReclaimPolicy: Retain
|
||||||
|
storageClassName: monitoring-local
|
||||||
|
local:
|
||||||
|
path: /data/k8s/monitoring/alertmanager
|
||||||
|
nodeAffinity:
|
||||||
|
required:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: In
|
||||||
|
values: [music]
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolume
|
||||||
|
metadata:
|
||||||
|
name: monitoring-loki-music
|
||||||
|
labels:
|
||||||
|
monitoring.hexor.cy/volume: loki
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-1"
|
||||||
|
argocd.argoproj.io/sync-options: Prune=false
|
||||||
|
spec:
|
||||||
|
capacity:
|
||||||
|
storage: 200Gi
|
||||||
|
volumeMode: Filesystem
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
persistentVolumeReclaimPolicy: Retain
|
||||||
|
storageClassName: monitoring-local
|
||||||
|
local:
|
||||||
|
path: /data/k8s/monitoring/loki
|
||||||
|
nodeAffinity:
|
||||||
|
required:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: In
|
||||||
|
values: [music]
|
||||||
|
---
|
||||||
|
# Keep monitoring schedulable under application-node pressure without competing
|
||||||
|
# with Kubernetes' system-cluster-critical classes.
|
||||||
|
apiVersion: scheduling.k8s.io/v1
|
||||||
|
kind: PriorityClass
|
||||||
|
metadata:
|
||||||
|
name: monitoring-critical
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-1"
|
||||||
|
value: 1000000
|
||||||
|
globalDefault: false
|
||||||
|
description: Monitoring control plane and storage workloads
|
||||||
Reference in new issue
Block a user