defaultRules: create: true # Default rule descriptions reference $labels.cluster. Prometheus external # labels are added only when alerts leave Prometheus, so attach the label to # the rules as well to avoid messages ending with "on cluster .". additionalRuleLabels: cluster: hexor-ng rules: # k3s embeds these control-plane components and does not expose their # conventional metrics Services. Disabling the groups prevents false pages. etcd: false kubeControllerManager: false kubeProxy: false kubeSchedulerAlerting: false kubeSchedulerRecording: false runbookUrl: https://runbooks.prometheus-operator.dev/runbooks kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeProxy: enabled: false kubeEtcd: enabled: false alertmanager: enabled: true config: global: resolve_timeout: 5m route: group_by: [namespace, alertname] group_wait: 30s group_interval: 5m repeat_interval: 12h receiver: "null" receivers: - name: "null" alertmanagerSpec: replicas: 1 retention: 120h nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external kubernetes.io/hostname: music priorityClassName: monitoring-critical alertmanagerConfigSelector: matchLabels: alertmanager: main # The single global routing object must be allowed to receive alerts from # every namespace, rather than only alerts labelled namespace=prometheus. alertmanagerConfigMatcherStrategy: type: None resources: requests: cpu: 100m memory: 256Mi limits: cpu: 500m memory: 1Gi storage: volumeClaimTemplate: spec: storageClassName: monitoring-local accessModes: [ReadWriteOnce] selector: matchLabels: monitoring.hexor.cy/volume: alertmanager resources: requests: storage: 5Gi prometheus: prometheusSpec: replicas: 1 nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external kubernetes.io/hostname: music priorityClassName: monitoring-critical scrapeInterval: 30s scrapeTimeout: 10s evaluationInterval: 30s externalUrl: https://prom.hexor.cy externalLabels: cluster: hexor-ng enableRemoteWriteReceiver: true walCompression: true retention: 365d # Time and size retention are both applied; whichever is reached first # wins, leaving headroom for compaction and WAL spikes on the 500 Gi PV. retentionSize: 425GB resources: requests: cpu: "1" memory: 4Gi limits: cpu: "4" memory: 12Gi storageSpec: volumeClaimTemplate: spec: storageClassName: monitoring-local accessModes: [ReadWriteOnce] selector: matchLabels: monitoring.hexor.cy/volume: prometheus resources: requests: storage: 500Gi # Discover monitoring resources in every namespace without requiring the # kube-prometheus-stack release label on application-owned monitors. ruleSelectorNilUsesHelmValues: false ruleSelector: {} ruleNamespaceSelector: {} serviceMonitorSelectorNilUsesHelmValues: false serviceMonitorSelector: {} serviceMonitorNamespaceSelector: {} podMonitorSelectorNilUsesHelmValues: false podMonitorSelector: {} podMonitorNamespaceSelector: {} probeSelectorNilUsesHelmValues: false probeSelector: {} probeNamespaceSelector: {} scrapeConfigSelectorNilUsesHelmValues: false scrapeConfigSelector: {} scrapeConfigNamespaceSelector: {} #additionalScrapeConfigs: # - job_name: xray-vpn # metrics_path: /scrape # static_configs: # - targets: [cy.tail2fe2d.ts.net:9550] # labels: {site: cy} # - targets: [x86.tail2fe2d.ts.net:9550] # labels: {site: am} # - targets: [jp.tail2fe2d.ts.net:9550] # labels: {site: jp} # - job_name: macos-node-exporter # static_configs: # - targets: [macbook-pro.tail2fe2d.ts.net:9100] # labels: {instance: ultrabook-pro} grafana: enabled: true replicas: 1 testFramework: enabled: false annotations: reloader.stakater.com/auto: "true" nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: cpu: 250m memory: 512Mi limits: cpu: "2" memory: 2Gi admin: existingSecret: grafana-admin userKey: admin-user passwordKey: admin-password envFromSecret: grafana-env grafana.ini: server: root_url: https://gf.hexor.cy auth: signout_redirect_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2Fgf.hexor.cy%2Flogin&client_id=Grafana oauth_allow_insecure_email_lookup: true auth.generic_oauth: name: Keycloak enabled: true scopes: openid profile email allow_sign_up: true auth_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/auth token_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/token api_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/userinfo email_attribute_path: email login_attribute_path: preferred_username name_attribute_path: name role_attribute_path: >- contains(groups[*], 'hexor-admin') && 'Admin' || contains(groups[*], 'hexor-guest') && 'Viewer' || 'Viewer' role_attribute_strict: false database: type: postgres host: main-rw.psql.svc:5432 name: grafana ssl_mode: disable log: level: info sidecar: resources: requests: cpu: 100m memory: 128Mi limits: cpu: "1" memory: 512Mi dashboards: enabled: true searchNamespace: ALL datasources: uid: prometheus alertmanager: uid: alertmanager handleGrafanaManagedAlerts: false additionalDataSources: - name: Loki uid: loki type: loki url: http://loki-gateway.prometheus.svc:80 access: proxy orgId: 1 jsonData: maxLines: 2000 ingress: enabled: true ingressClassName: traefik annotations: cert-manager.io/cluster-issuer: letsencrypt traefik.ingress.kubernetes.io/router.entrypoints: websecure hosts: - gf.hexor.cy tls: - secretName: grafana-tls hosts: - gf.hexor.cy prometheusOperator: admissionWebhooks: patch: nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: cpu: 100m memory: 128Mi limits: cpu: 500m memory: 512Mi prometheusConfigReloader: resources: requests: cpu: 20m memory: 64Mi limits: cpu: 200m memory: 256Mi kube-state-metrics: nodeSelector: homelab.hexor.cy/pool: core homelab.hexor.cy/exposure: external priorityClassName: monitoring-critical resources: requests: cpu: 50m memory: 128Mi limits: cpu: 500m memory: 512Mi prometheus-node-exporter: priorityClassName: monitoring-critical # `home` has stopped reporting to the cluster. A DaemonSet pod there keeps # prom-stack Progressing and blocks later core-state sync waves. # Remove this exclusion when the node rejoins. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: NotIn values: [home] tolerations: - operator: Exists prometheus: monitor: # Keep dashboard variables and alerts stable when a node IP changes. # The scrape address remains the discovered node IP; only the public # instance label is rewritten to the Kubernetes node hostname. relabelings: - action: replace sourceLabels: [__meta_kubernetes_endpoint_node_name] targetLabel: instance replacement: "${1}:9100" resources: requests: cpu: 50m memory: 64Mi limits: cpu: 500m memory: 256Mi