Auto-update README with current k8s applications

Generated by CI/CD workflow on 2026-10-18 10:24:11

This PR updates the README.md file with the current list of applications found in the k8s/ directory structure.

.gitea/workflows/authentik-apps.yaml

@@ -30,9 +30,11 @@ jobs:
         cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
 
     - name: Setup kubeconfig
+      env:
+        CI_KUBECONFIG: ${{ secrets.KUBECONFIG }}
       run: |
         mkdir -p ~/.kube
-        echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
+        printf '%s' "$CI_KUBECONFIG" > ~/.kube/config
         chmod 600 ~/.kube/config
 
     - name: Terraform Init

README.md

@@ -42,7 +42,11 @@ ArgoCD homelab project
 
 | Application | Status |
 | :--- | :---: |
+| **airtrail** | [![airtrail](https://ag.hexor.cy/api/badge?name=airtrail&revision=true)](https://ag.hexor.cy/applications/argocd/airtrail) |
+| **amnezia** | [![amnezia](https://ag.hexor.cy/api/badge?name=amnezia&revision=true)](https://ag.hexor.cy/applications/argocd/amnezia) |
 | **comfyui** | [![comfyui](https://ag.hexor.cy/api/badge?name=comfyui&revision=true)](https://ag.hexor.cy/applications/argocd/comfyui) |
+| **doka2-lobby-list** | [![doka2-lobby-list](https://ag.hexor.cy/api/badge?name=doka2-lobby-list&revision=true)](https://ag.hexor.cy/applications/argocd/doka2-lobby-list) |
+| **firefly-iii** | [![firefly-iii](https://ag.hexor.cy/api/badge?name=firefly-iii&revision=true)](https://ag.hexor.cy/applications/argocd/firefly-iii) |
 | **furumi** | [![furumi](https://ag.hexor.cy/api/badge?name=furumi&revision=true)](https://ag.hexor.cy/applications/argocd/furumi) |
 | **gitea** | [![gitea](https://ag.hexor.cy/api/badge?name=gitea&revision=true)](https://ag.hexor.cy/applications/argocd/gitea) |
 | **greece-notifier** | [![greece-notifier](https://ag.hexor.cy/api/badge?name=greece-notifier&revision=true)](https://ag.hexor.cy/applications/argocd/greece-notifier) |
@@ -52,7 +56,9 @@ ArgoCD homelab project
 | **jellyfin** | [![jellyfin](https://ag.hexor.cy/api/badge?name=jellyfin&revision=true)](https://ag.hexor.cy/applications/argocd/jellyfin) |
 | **k8s-secrets** | [![k8s-secrets](https://ag.hexor.cy/api/badge?name=k8s-secrets&revision=true)](https://ag.hexor.cy/applications/argocd/k8s-secrets) |
 | **khm** | [![khm](https://ag.hexor.cy/api/badge?name=khm&revision=true)](https://ag.hexor.cy/applications/argocd/khm) |
+| **lan-play** | [![lan-play](https://ag.hexor.cy/api/badge?name=lan-play&revision=true)](https://ag.hexor.cy/applications/argocd/lan-play) |
 | **lidarr** | [![lidarr](https://ag.hexor.cy/api/badge?name=lidarr&revision=true)](https://ag.hexor.cy/applications/argocd/lidarr) |
+| **llamacpp** | [![llamacpp](https://ag.hexor.cy/api/badge?name=llamacpp&revision=true)](https://ag.hexor.cy/applications/argocd/llamacpp) |
 | **matrix** | [![matrix](https://ag.hexor.cy/api/badge?name=matrix&revision=true)](https://ag.hexor.cy/applications/argocd/matrix) |
 | **mtproxy** | [![mtproxy](https://ag.hexor.cy/api/badge?name=mtproxy&revision=true)](https://ag.hexor.cy/applications/argocd/mtproxy) |
 | **n8n** | [![n8n](https://ag.hexor.cy/api/badge?name=n8n&revision=true)](https://ag.hexor.cy/applications/argocd/n8n) |

k8s-ng/apps/furumi/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - servicemonitor.yaml
+  - ingress.yaml

k8s-ng/apps/furumi/servicemonitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: furumi-player
+spec:
+  selector:
+    matchLabels:
+      app: furumi-player
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s-ng/apps/llamacpp/configmap.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llamacpp-config
+data:
+  LLAMA_CACHE: /models
+  LLAMA_ARG_HOST: 0.0.0.0
+  LLAMA_ARG_PORT: "8080"
+  # The model is chosen here: <hf-user>/<repo>:<quant>. It is downloaded into
+  # /models on first start; changing it restarts the pod (reloader).
+  #LLAMA_ARG_HF_REPO: "unsloth/Qwen3.8-27B-GGUF:UD-Q6_K"
+  LLAMA_ARG_HF_REPO: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K"
+  LLAMA_ARG_CTX_SIZE: "131072"
+  LLAMA_ARG_N_GPU_LAYERS: all
+  LLAMA_ARG_FLASH_ATTN: "on"
+  LLAMA_ARG_FIT: "on"
+  # Strix Halo: mmap-ed weights hang or corrupt on ROCm; read them directly.
+  LLAMA_ARG_LOAD_MODE: dio

k8s-ng/apps/llamacpp/deployment.yaml

@@ -0,0 +1,76 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llamacpp
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: llamacpp
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate # one GPU, one pod
+  selector:
+    matchLabels:
+      app: llamacpp
+  template:
+    metadata:
+      labels:
+        app: llamacpp
+    spec:
+      dnsPolicy: Default
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: internal
+        workload/ai: "true"
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      containers:
+        - name: llamacpp
+          image: ghcr.io/ggml-org/llama.cpp:server-rocm-b11434
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: llamacpp-config
+          env:
+            - name: HF_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: llamacpp-hf-token
+                  key: token
+                  optional: true
+            # hipBLASLt gives the biggest prompt-processing gain on ROCm 7.x.
+            - name: ROCBLAS_USE_HIPBLASLT
+              value: "1"
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          resources:
+            limits:
+              amd.com/gpu: 1
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 180 # first start downloads the model
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 3
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      volumes:
+        - name: models
+          hostPath:
+            path: /var/lib/llamacpp/models
+            type: DirectoryOrCreate

k8s-ng/apps/llamacpp/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml

k8s-ng/apps/llamacpp/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: llamacpp
+  labels:
+    app: llamacpp
+spec:
+  type: ClusterIP
+  selector:
+    app: llamacpp
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+      protocol: TCP

k8s-ng/apps/minecraft/configmaps.yaml

@@ -0,0 +1,561 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-config
+  namespace: minecraft
+data:
+  nginx.conf: |
+    user nginx;
+    worker_processes 1;
+    error_log /var/log/nginx/error.log warn;
+    pid /var/run/nginx.pid;
+
+    events {
+        worker_connections 1024;
+    }
+
+    http {
+        include /etc/nginx/mime.types;
+        default_type application/octet-stream;
+
+        server {
+            listen 80;
+
+            # Custom 502 error page with auto-refresh
+            error_page 502 /502.html;
+            location = /502.html {
+                internal;
+                return 200 '<!DOCTYPE html><html><head><meta charset="utf-8"><title>Server Loading</title><style>body{font-family:Arial,sans-serif;text-align:center;margin-top:100px;background:#f0f0f0}h1{color:#333}p{color:#666;font-size:18px}</style></head><body><h1>Server is loading probably...</h1><p>Please wait a moment and try refreshing the page.</p><script>setTimeout(function(){window.location.reload();}, 10000);</script></body></html>';
+                add_header Content-Type text/html;
+            }
+
+            # Main location - proxy to Minecraft Dynmap
+            location / {
+                # Proxy configuration for Dynmap server
+                proxy_pass http://localhost:8123;
+                proxy_set_header Host $host;
+                proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-Proto $scheme;
+
+                # Inject user authentication meta tag into HTML head
+                sub_filter '<head>' '<head><meta name="remote-user" content="$http_x_authentik_username">';
+
+                # Replace default Dynmap title with custom server name
+                sub_filter 'Minecraft Dynamic Map' "Hexor's MC server";
+
+                # Inject custom script before closing body tag
+                sub_filter "</body>" '<script src="/clients/inject.js"></script></body>';
+
+                # Apply sub_filter replacements globally (not just once)
+                sub_filter_once off;
+            }
+
+            # Serve inject.js and .ps1 scripts inline (no forced download)
+            location = /clients/inject.js {
+                alias /mc/clients/inject.js;
+                default_type application/javascript;
+            }
+
+            location ~ ^/clients/(.+\.ps1)$ {
+                alias /mc/clients/$1;
+                default_type text/plain;
+            }
+
+            # Static file serving for client downloads
+            location /clients/ {
+                alias /mc/clients/;
+                sendfile on;                          # Enable efficient file serving
+                add_header Content-Disposition "attachment";  # Force download
+                autoindex on;                         # Enable directory listing
+                gzip off;                            # Disable compression for downloads
+                chunked_transfer_encoding off;       # Disable chunked encoding
+            }
+        }
+    }
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: client-scripts
+  namespace: minecraft
+data:
+  inject.js: |
+    (function() {
+      function getUsername() {
+        var metas = document.querySelectorAll("meta");
+        for (var i = 0; i < metas.length; i++) {
+          if (metas[i].getAttribute("name") === "remote-user") {
+            return metas[i].getAttribute("content");
+          }
+        }
+        var jwt = document.cookie.split("; ").find(function(row) {
+          return row.startsWith("authentik_session=");
+        });
+        if (jwt) {
+          try {
+            var token = jwt.split("=")[1];
+            var payload = JSON.parse(atob(token.split(".")[1]));
+            return payload.sub || payload.username || "web-user";
+          } catch(e) {}
+        }
+        return "web-user";
+      }
+
+      var username = getUsername();
+      console.log("Username found:", username);
+
+      if (username && username !== "web-user" &&
+          window.location.search.indexOf("playername=") === -1) {
+        var currentUrl = new URL(window.location.href);
+        currentUrl.searchParams.set("playername", username);
+        console.log("Redirecting to:", currentUrl.href);
+        window.location.href = currentUrl.href;
+      }
+
+      document.addEventListener("DOMContentLoaded", function() {
+        // User block
+        var userBlock = document.createElement("div");
+        userBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:8px;text-align:center;font-size:medium;border-radius:4px;position:absolute;top:10px;right:150px;max-width:200px;";
+        userBlock.innerHTML = "Logged in as: <b>" + username + "</b>";
+        document.body.appendChild(userBlock);
+
+        // Info block
+        var infoBlock = document.createElement("p");
+        infoBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:10px;text-align:center;font-size:large;display:inline-block;border-radius:4px;position:absolute;top:10px;left:150px;";
+        infoBlock.innerHTML = 'GEYMERSKIY SOYUZ Server<br>'
+          + 'Get <a href="https://github.com/PrismLauncher/PrismLauncher/releases/tag/8.4">Prism Launcher</a> '
+          + 'and <a href="/clients/1.12.2.zip">client.zip</a> for this server. '
+          + 'Server address <b>minecraft.hexor.cy:30565</b><br>'
+          + 'Requires <a href="https://www.java.com/en/download/manual.jsp">Java 8</a><br><br>'
+          + '<a href="#" id="showInstallBtn" style="color:black;text-decoration:underline;">Windows Install Script</a>';
+        document.body.appendChild(infoBlock);
+
+        // Modal
+        var modal = document.createElement("div");
+        modal.id = "installModal";
+        modal.style.cssText = "display:none;position:fixed;z-index:1000;left:0;top:0;width:100%;height:100%;background-color:rgba(0,0,0,0.5);";
+        modal.innerHTML = '<div style="background-color:#CEC6CB;margin:15% auto;padding:10px;border-radius:4px;width:70%;max-width:500px;text-align:center;color:black;font-size:large;">'
+          + '<h3 style="margin-top:0;color:black;">Windows Installation</h3>'
+          + '<p style="color:black;">Copy and paste this command into PowerShell:</p>'
+          + '<textarea id="scriptCommand" readonly style="width:90%;height:60px;font-family:monospace;padding:8px;border:1px solid #888;border-radius:4px;resize:none;background-color:white;color:black;"></textarea>'
+          + '<br><br>'
+          + '<button id="copyButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;margin-right:10px;font-size:large;">Copy</button>'
+          + '<button id="closeButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;font-size:large;">Close</button>'
+          + '</div>';
+        document.body.appendChild(modal);
+
+        // Generate PowerShell command with username
+        function buildPsCommand() {
+          var d = "$";
+          var q = "'";
+          return d + 'f="' + d + 'env:TEMP\\mc-install.ps1"; iwr -useb https://minecraft.hexor.cy/clients/win-install.ps1 -OutFile '
+            + d + 'f; powershell -ExecutionPolicy Bypass -File ' + d + 'f -Username ' + q + username + q + '; Remove-Item ' + d + 'f';
+        }
+
+        document.getElementById("showInstallBtn").addEventListener("click", function(e) {
+          e.preventDefault();
+          modal.style.display = "block";
+          document.getElementById("scriptCommand").value = buildPsCommand();
+        });
+
+        document.getElementById("closeButton").addEventListener("click", function() {
+          modal.style.display = "none";
+        });
+
+        document.getElementById("copyButton").addEventListener("click", function() {
+          var textarea = document.getElementById("scriptCommand");
+          textarea.select();
+          textarea.setSelectionRange(0, 99999);
+          document.execCommand("copy");
+          var btn = document.getElementById("copyButton");
+          btn.style.borderColor = "#4CAF50";
+          setTimeout(function() { btn.style.borderColor = "#888"; }, 2000);
+        });
+
+        modal.addEventListener("click", function(event) {
+          if (event.target === modal) {
+            modal.style.display = "none";
+          }
+        });
+      });
+    })();
+
+  win-install.ps1: |
+    # Game Setup Script for PrismLauncher and Minecraft Client
+    # This script downloads and configures PrismLauncher with Hexor client
+
+    param(
+        [string]$Username = "",
+        [string]$InstallPath = "$env:USERPROFILE\Games\PrismLauncher"
+    )
+
+    # Enable TLS 1.2 for downloads
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+    # Function to write colored log messages
+    function Write-Log {
+        param(
+            [string]$Message,
+            [string]$Level = "INFO"
+        )
+
+        switch ($Level) {
+            "INFO" {
+                Write-Host "[" -NoNewline
+                Write-Host "INFO" -ForegroundColor Blue -NoNewline
+                Write-Host "] $Message"
+            }
+            "WARN" {
+                Write-Host "[" -NoNewline
+                Write-Host "WARN" -ForegroundColor Yellow -NoNewline
+                Write-Host "] $Message"
+            }
+            "ERROR" {
+                Write-Host "[" -NoNewline
+                Write-Host "ERROR" -ForegroundColor Red -NoNewline
+                Write-Host "] $Message"
+            }
+            "SUCCESS" {
+                Write-Host "[" -NoNewline
+                Write-Host "SUCCESS" -ForegroundColor Green -NoNewline
+                Write-Host "] $Message"
+            }
+        }
+    }
+
+    # Function to download file with progress
+    function Download-File {
+        param(
+            [string]$Url,
+            [string]$OutputPath
+        )
+
+        try {
+            Write-Log "Downloading from: $Url"
+            Write-Log "Saving to: $OutputPath"
+
+            if (Test-Path $OutputPath) {
+                Remove-Item $OutputPath -Force
+                Write-Log "Removed existing file to avoid corruption" "WARN"
+            }
+
+            $webClient = New-Object System.Net.WebClient
+            $webClient.DownloadFile($Url, $OutputPath)
+
+            if (Test-Path $OutputPath) {
+                $fileSize = (Get-Item $OutputPath).Length
+                Write-Log "Download completed successfully ($fileSize bytes)" "SUCCESS"
+                return $true
+            } else {
+                Write-Log "Download failed - file not found" "ERROR"
+                return $false
+            }
+        }
+        catch {
+            Write-Log "Download failed: $($_.Exception.Message)" "ERROR"
+            return $false
+        }
+    }
+
+    # Function to extract ZIP archive
+    function Extract-Archive {
+        param(
+            [string]$ArchivePath,
+            [string]$DestinationPath
+        )
+
+        try {
+            Write-Log "Extracting archive: $ArchivePath"
+            Write-Log "Destination: $DestinationPath"
+
+            if (!(Test-Path $ArchivePath)) {
+                Write-Log "Archive file not found: $ArchivePath" "ERROR"
+                return $false
+            }
+
+            $fileSize = (Get-Item $ArchivePath).Length
+            if ($fileSize -lt 1000) {
+                Write-Log "Archive file too small ($fileSize bytes), probably corrupted" "ERROR"
+                return $false
+            }
+
+            if (!(Test-Path $DestinationPath)) {
+                New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null
+            }
+
+            $existingFiles = Get-ChildItem -Path $DestinationPath -File | Where-Object { $_.Extension -ne ".zip" }
+            $existingDirs = Get-ChildItem -Path $DestinationPath -Directory
+
+            if ($existingFiles.Count -gt 0 -or $existingDirs.Count -gt 0) {
+                Write-Log "Clearing existing files in destination directory" "WARN"
+                $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
+                $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
+            }
+
+            Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+            try {
+                $zip = [System.IO.Compression.ZipFile]::OpenRead($ArchivePath)
+                $entryCount = $zip.Entries.Count
+                $zip.Dispose()
+                Write-Log "ZIP file validated ($entryCount entries)" "SUCCESS"
+            }
+            catch {
+                Write-Log "ZIP file validation failed: $($_.Exception.Message)" "ERROR"
+                return $false
+            }
+
+            [System.IO.Compression.ZipFile]::ExtractToDirectory($ArchivePath, $DestinationPath)
+
+            Write-Log "Archive extracted successfully" "SUCCESS"
+            return $true
+        }
+        catch {
+            Write-Log "Archive extraction failed: $($_.Exception.Message)" "ERROR"
+
+            if (Test-Path $ArchivePath) {
+                Write-Log "Removing potentially corrupted archive file" "WARN"
+                Remove-Item $ArchivePath -Force -ErrorAction SilentlyContinue
+            }
+
+            return $false
+        }
+    }
+
+    # Function to check Java installation
+    function Check-JavaInstallation {
+        try {
+            $javaVersion = java -version 2>&1 | Select-String "version"
+            if ($javaVersion) {
+                Write-Log "Java is installed: $($javaVersion.ToString().Trim())" "SUCCESS"
+                return $true
+            }
+        }
+        catch {
+            Write-Log "Java is not installed or not in PATH" "WARN"
+            Write-Log "Please download Java from: https://www.java.com/en/download/manual.jsp" "WARN"
+            Write-Log "Look for 'Windows Offline (64-bit)' version" "WARN"
+
+            $response = Read-Host "Do you want to continue without Java? (y/n)"
+            if ($response -eq 'y' -or $response -eq 'Y') {
+                Write-Log "Continuing without Java verification" "WARN"
+                return $true
+            } else {
+                Write-Log "Installation cancelled. Please install Java first." "ERROR"
+                return $false
+            }
+        }
+    }
+
+    # Function to generate a random hex string (UUID-like without dashes)
+    function New-RandomHexId {
+        param([int]$Length = 32)
+        $bytes = New-Object byte[] ($Length / 2)
+        $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
+        $rng.GetBytes($bytes)
+        $rng.Dispose()
+        return ($bytes | ForEach-Object { $_.ToString("x2") }) -join ''
+    }
+
+    # Function to create accounts.json file with offline account
+    function Create-AccountsFile {
+        param(
+            [string]$TargetPath,
+            [string]$PlayerName
+        )
+
+        $accountsPath = Join-Path $TargetPath "accounts.json"
+        $profileId = New-RandomHexId -Length 32
+        $clientToken = New-RandomHexId -Length 32
+        $iat = [int][double]::Parse((Get-Date -UFormat %s))
+
+        $accountsObj = @{
+            accounts = @(
+                @{
+                    entitlement = @{
+                        canPlayMinecraft = $true
+                        ownsMinecraft = $true
+                    }
+                    "msa-client-id" = ""
+                    type = "MSA"
+                },
+                @{
+                    active = $true
+                    entitlement = @{
+                        canPlayMinecraft = $true
+                        ownsMinecraft = $true
+                    }
+                    profile = @{
+                        capes = @()
+                        id = $profileId
+                        name = $PlayerName
+                        skin = @{
+                            id = ""
+                            url = ""
+                            variant = ""
+                        }
+                    }
+                    type = "Offline"
+                    ygg = @{
+                        extra = @{
+                            clientToken = $clientToken
+                            userName = $PlayerName
+                        }
+                        iat = $iat
+                        token = "0"
+                    }
+                }
+            )
+            formatVersion = 3
+        }
+
+        try {
+            $accountsObj | ConvertTo-Json -Depth 10 | Out-File -FilePath $accountsPath -Encoding UTF8
+            Write-Log "Created accounts.json at: $accountsPath" "SUCCESS"
+            Write-Log "Player name: $PlayerName" "SUCCESS"
+            return $true
+        }
+        catch {
+            Write-Log "Failed to create accounts.json: $($_.Exception.Message)" "ERROR"
+            return $false
+        }
+    }
+
+    # Main installation process
+    Write-Log "Starting PrismLauncher and Minecraft client setup"
+    Write-Log "Player name: $Username"
+    Write-Log "Installation path: $InstallPath"
+
+    # Create installation directory and clear if exists (but keep ZIP files)
+    if (Test-Path $InstallPath) {
+        Write-Log "Installation directory exists, clearing contents" "WARN"
+        $existingFiles = Get-ChildItem -Path $InstallPath -File | Where-Object { $_.Extension -ne ".zip" }
+        $existingDirs = Get-ChildItem -Path $InstallPath -Directory
+
+        $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
+        $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
+    } else {
+        New-Item -ItemType Directory -Path $InstallPath -Force | Out-Null
+        Write-Log "Created installation directory: $InstallPath"
+    }
+
+    # Step 1: Download PrismLauncher
+    Write-Log "Step 1: Downloading PrismLauncher..."
+    $launcherUrl = "https://github.com/PrismLauncher/PrismLauncher/releases/download/8.4/PrismLauncher-Windows-MSVC-Portable-8.4.zip"
+    $launcherZip = Join-Path $InstallPath "PrismLauncher-8.4.zip"
+    $launcherExtractPath = Join-Path $InstallPath "PrismLauncher-Windows-MSVC-Portable-8.4"
+
+    if (!(Download-File -Url $launcherUrl -OutputPath $launcherZip)) {
+        Write-Log "Failed to download PrismLauncher. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 2: Extract PrismLauncher
+    Write-Log "Step 2: Extracting PrismLauncher..."
+    if (!(Extract-Archive -ArchivePath $launcherZip -DestinationPath $InstallPath)) {
+        Write-Log "Failed to extract PrismLauncher. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Find the actual extracted directory
+    $extractedDirs = Get-ChildItem -Path $InstallPath -Directory | Where-Object { $_.Name -like "*PrismLauncher*" }
+    if ($extractedDirs.Count -gt 0) {
+        $launcherExtractPath = $extractedDirs[0].FullName
+        Write-Log "Found PrismLauncher directory: $launcherExtractPath" "SUCCESS"
+    } else {
+        Write-Log "Could not find extracted PrismLauncher directory. Checking for direct extraction..." "WARN"
+        $prismExe = Join-Path $InstallPath "prismlauncher.exe"
+        if (Test-Path $prismExe) {
+            $launcherExtractPath = $InstallPath
+            Write-Log "PrismLauncher extracted directly to: $launcherExtractPath" "SUCCESS"
+        } else {
+            Write-Log "Failed to locate PrismLauncher files. Exiting." "ERROR"
+            exit 1
+        }
+    }
+
+    # Step 3: Create accounts.json in InstallPath (only if username provided)
+    if ($Username -and $Username -ne "web-user") {
+        Write-Log "Step 3: Creating accounts configuration for $Username..."
+        if (!(Create-AccountsFile -TargetPath $InstallPath -PlayerName $Username)) {
+            Write-Log "Failed to create accounts.json. Exiting." "ERROR"
+            exit 1
+        }
+    } else {
+        Write-Log "Step 3: No username provided, skipping accounts.json creation" "WARN"
+    }
+
+    # Step 4: Check Java installation
+    Write-Log "Step 4: Checking Java installation..."
+    if (!(Check-JavaInstallation)) {
+        Write-Log "Java check failed. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 5: Download Minecraft client
+    Write-Log "Step 5: Downloading Minecraft client..."
+    $minecraftUrl = "https://minecraft.hexor.cy/clients/1.12.2.zip"
+    $minecraftZip = Join-Path $InstallPath "minecraft-1.12.2.zip"
+    $instancesPath = Join-Path $launcherExtractPath "instances"
+    $hexorPath = Join-Path $instancesPath "Hexor"
+
+    if (!(Download-File -Url $minecraftUrl -OutputPath $minecraftZip)) {
+        Write-Log "Failed to download Minecraft client. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 6: Create instances directory and extract Minecraft client
+    Write-Log "Step 6: Setting up Minecraft client..."
+    if (!(Test-Path $instancesPath)) {
+        New-Item -ItemType Directory -Path $instancesPath -Force | Out-Null
+        Write-Log "Created instances directory"
+    }
+
+    if (!(Test-Path $hexorPath)) {
+        New-Item -ItemType Directory -Path $hexorPath -Force | Out-Null
+        Write-Log "Created Hexor instance directory"
+    }
+
+    if (!(Extract-Archive -ArchivePath $minecraftZip -DestinationPath $hexorPath)) {
+        Write-Log "Failed to extract Minecraft client. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 7: Cleanup temporary files
+    Write-Log "Step 7: Cleaning up temporary files..."
+    try {
+        Remove-Item $launcherZip -Force
+        Remove-Item $minecraftZip -Force
+        Write-Log "Temporary files cleaned up" "SUCCESS"
+    }
+    catch {
+        Write-Log "Could not remove temporary files: $($_.Exception.Message)" "WARN"
+    }
+
+    # Final success message
+    Write-Log "=== INSTALLATION COMPLETED SUCCESSFULLY ===" "SUCCESS"
+    Write-Log "PrismLauncher location: $launcherExtractPath" "SUCCESS"
+    Write-Log "Executable: $(Join-Path $launcherExtractPath 'prismlauncher.exe')" "SUCCESS"
+    Write-Log "Minecraft client installed in: $hexorPath" "SUCCESS"
+    if ($Username -and $Username -ne "web-user") {
+        Write-Log "Player name configured: $Username" "SUCCESS"
+        Write-Log "Accounts file: $(Join-Path $InstallPath 'accounts.json')" "SUCCESS"
+    }
+    Write-Log ""
+    Write-Log "You can now run PrismLauncher and the Hexor instance should be available!"
+    Write-Log "If Java was not installed, please download it from: https://www.java.com/en/download/manual.jsp"
+
+    # Ask if user wants to launch the game
+    $launchResponse = Read-Host "Do you want to launch PrismLauncher now? (y/n)"
+    if ($launchResponse -eq 'y' -or $launchResponse -eq 'Y') {
+        $launcherExe = Join-Path $launcherExtractPath "prismlauncher.exe"
+        if (Test-Path $launcherExe) {
+            Write-Log "Launching PrismLauncher..." "INFO"
+            Start-Process -FilePath $launcherExe -WorkingDirectory $launcherExtractPath
+        } else {
+            Write-Log "Launcher executable not found!" "ERROR"
+        }
+    }

k8s-ng/apps/minecraft/deployments.yaml

@@ -0,0 +1,137 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minecraft
+  namespace: minecraft
+  labels:
+    app: minecraft
+spec:
+  selector:
+    matchLabels:
+      app: minecraft
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: minecraft
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      volumes:
+      - name: storage
+        hostPath:
+          path: /data/k8s/mc-server/
+          type: Directory
+      - name: nginx-config
+        configMap:
+          name: nginx-config
+      - name: client-scripts
+        configMap:
+          name: client-scripts
+
+      terminationGracePeriodSeconds: 180
+      containers:
+        - name: minecraft
+          image: eclipse-temurin:8-jdk-ubi10-minimal@sha256:85c943a7093a276b0f6ae13dee2ffecb98e390ab6a32fc7e4b8b1fc6bb103dd7
+          command: ["java"]
+          args:
+            - -Xms4G
+            - -Xmx4G
+            - -XX:+UseG1GC
+            - -XX:+ParallelRefProcEnabled
+            - -XX:MaxGCPauseMillis=200
+            - -XX:+UnlockExperimentalVMOptions
+            - -XX:+DisableExplicitGC
+            - -XX:+AlwaysPreTouch
+            - -XX:G1NewSizePercent=30
+            - -XX:G1MaxNewSizePercent=40
+            - -XX:G1HeapRegionSize=8M
+            - -XX:G1ReservePercent=20
+            - -XX:G1HeapWastePercent=5
+            - -XX:G1MixedGCCountTarget=4
+            - -XX:InitiatingHeapOccupancyPercent=15
+            - -XX:G1MixedGCLiveThresholdPercent=90
+            - -XX:G1RSetUpdatingPauseTimePercent=5
+            - -XX:SurvivorRatio=32
+            - -XX:+PerfDisableSharedMem
+            - -XX:MaxTenuringThreshold=1
+            - -jar
+            - forge-1.12.2-14.23.5.2854.jar
+            - nogui
+          workingDir: /mc/
+          resources:
+            requests:
+              memory: "8Gi"
+              cpu: "2000m"
+            limits:
+              memory: "12Gi"
+              cpu: "4000m"
+          ports:
+            - name: game
+              containerPort: 25565
+              protocol: TCP
+            - name: dynmap
+              containerPort: 8123
+              protocol: TCP
+            - name: webstatus-mod
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 10
+            failureThreshold: 90
+          livenessProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: storage
+              mountPath: /mc
+        - name: nginx
+          image: nginx@sha256:f9ea18bfa4fad859e1ed38259d711da7ccad2c3516e875cec3351a57c859f571
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "256Mi"
+              cpu: "200m"
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: nginx-config
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
+            - name: storage
+              mountPath: /mc
+            - name: client-scripts
+              mountPath: /mc/clients/win-install.ps1
+              subPath: win-install.ps1
+            - name: client-scripts
+              mountPath: /mc/clients/inject.js
+              subPath: inject.js
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-exporter
+  namespace: minecraft
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 19565
+      targetPort: 19565

k8s-ng/apps/minecraft/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: minecraft
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: minecraft.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: minecraft-dynmap
+                port:
+                  number: 80
+  tls:
+    - secretName: minecraft-tls
+      hosts:
+        - minecraft.hexor.cy

k8s-ng/apps/minecraft/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: minecraft
+resources:
+  - deployments.yaml
+  - configmaps.yaml
+  - services.yaml
+  - ingress.yaml

k8s-ng/apps/minecraft/services.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-dynmap
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-webstatus-mod
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-game
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    - port: 30565
+      targetPort: 25565
+  selector:
+    app: minecraft

k8s-ng/apps/rustdesk/deployment.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rustdesk-hbbs
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: rustdesk-hbbs
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rustdesk-hbbs
+  template:
+    metadata:
+      labels:
+        app: rustdesk-hbbs
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      containers:
+      - name: hbbs
+        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "750m"
+        command: ["hbbs"]
+        args:
+          - "--relay-servers"
+          - "rd.hexor.cy:21117"
+          - "--port"
+          - "21116"
+        ports:
+        - name: registry
+          containerPort: 21116
+          protocol: TCP
+        - name: registry-udp
+          containerPort: 21116
+          protocol: UDP
+        - name: nat
+          containerPort: 21115
+          protocol: TCP
+        readinessProbe:
+          tcpSocket:
+            port: registry
+        volumeMounts:
+        - name: keys
+          mountPath: /data
+          readOnly: true
+        - name: data
+          mountPath: /data-persistent
+        env:
+        - name: RUST_LOG
+          value: "info"
+        - name: DB_URL
+          value: "/data-persistent/db_v2.sqlite3"
+        workingDir: /data
+      volumes:
+      - name: keys
+        secret:
+          secretName: rustdesk-keys
+          items:
+          - key: id_ed25519
+            path: id_ed25519
+            mode: 0600
+          - key: id_ed25519.pub
+            path: id_ed25519.pub
+            mode: 0644
+      - name: data
+        hostPath:
+          path: /data/k8s/rustdesk/hbbs
+          type: Directory
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rustdesk-hbbr
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: rustdesk-hbbr
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rustdesk-hbbr
+  template:
+    metadata:
+      labels:
+        app: rustdesk-hbbr
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      containers:
+      - name: hbbr
+        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "750m"
+        command: ["hbbr"]
+        args:
+          - "--port"
+          - "21117"
+        ports:
+        - name: relay
+          containerPort: 21117
+          protocol: TCP
+        readinessProbe:
+          tcpSocket:
+            port: relay
+        volumeMounts:
+        - name: keys
+          mountPath: /data
+          readOnly: true
+        - name: data
+          mountPath: /data-persistent
+        env:
+        - name: RUST_LOG
+          value: "info"
+        workingDir: /data
+      volumes:
+      - name: keys
+        secret:
+          secretName: rustdesk-keys
+          items:
+          - key: id_ed25519
+            path: id_ed25519
+            mode: 0600
+          - key: id_ed25519.pub
+            path: id_ed25519.pub
+            mode: 0644
+      - name: data
+        hostPath:
+          path: /data/k8s/rustdesk/hbbr
+          type: Directory

k8s-ng/apps/rustdesk/external-secrets.yaml

@@ -0,0 +1,27 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: rustdesk-keys
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: rustdesk-keys
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        id_ed25519: "{{ .private_key }}"
+        id_ed25519.pub: "{{ .public_key }}"
+  data:
+    - secretKey: private_key
+      remoteRef:
+        key: k8s/rustdesk/secrets
+        property: Cert
+    - secretKey: public_key
+      remoteRef:
+        key: k8s/rustdesk/secrets
+        property: Key

k8s-ng/apps/rustdesk/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: rustdesk
+resources:
+  - external-secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - network-policy.yaml

k8s-ng/apps/rustdesk/network-policy.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: rustdesk-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: rustdesk-hbbs
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  # Allow all incoming connections to RustDesk ports
+  - from: []
+    ports:
+    - protocol: TCP
+      port: 21115
+    - protocol: TCP
+      port: 21116
+    - protocol: UDP
+      port: 21116
+  egress:
+  # Allow DNS
+  - to: []
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53
+  # Allow communication between HBBS and HBBR
+  - to:
+    - podSelector:
+        matchLabels:
+          app: rustdesk-hbbr
+    ports:
+    - protocol: TCP
+      port: 21117
+
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: rustdesk-hbbr-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: rustdesk-hbbr
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  # Allow all incoming connections to relay port
+  - from: []
+    ports:
+    - protocol: TCP
+      port: 21117
+  # Allow connections from HBBS
+  - from:
+    - podSelector:
+        matchLabels:
+          app: rustdesk-hbbs
+    ports:
+    - protocol: TCP
+      port: 21117
+  egress:
+  # Allow DNS
+  - to: []
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53

k8s-ng/apps/rustdesk/service.yaml

@@ -0,0 +1,59 @@
+# Preserve client source IPs for RustDesk NAT detection. Public DNS points to music,
+# where both servers and their persistent data live.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbs
+  labels:
+    app: rustdesk-hbbs
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbs
+  ports:
+  - name: registry-tcp
+    port: 21116
+    targetPort: 21116
+    protocol: TCP
+  - name: nat
+    port: 21115
+    targetPort: 21115
+    protocol: TCP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbs-udp
+  labels:
+    app: rustdesk-hbbs
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbs
+  ports:
+  - name: registry-udp
+    port: 21116
+    targetPort: 21116
+    protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbr
+  labels:
+    app: rustdesk-hbbr
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbr
+  ports:
+  - name: relay
+    port: 21117
+    targetPort: 21117
+    protocol: TCP

k8s-ng/apps/shadowsocks/README.md

@@ -0,0 +1,18 @@
+# Shadowsocks on spb
+
+One shadowsocks-rust pod runs on `spb`, exposing `8388/TCP` and `8388/UDP`
+through host ports. Client settings:
+
+- Server: `spb.hexor.cy`
+- Port: `8388`
+- Method: `chacha20-ietf-poly1305`
+- Password: the `Password` custom field of the Vaultwarden item named
+  `k8s/shadowsocks/secrets` (exact name, not an item UUID).
+
+Create that item in the server secrets vault accessible to the External Secrets
+bot. Use a long random password, for example generated with `openssl rand -base64 32`.
+External Secrets creates `shadowsocks-config`; Reloader restarts the deployment
+when the Secret changes. No persistent storage is required.
+
+The spb NixOS configuration opens port 8388 for both protocols. Apply that
+configuration when deploying this application for the first time.

k8s-ng/apps/shadowsocks/deployment.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shadowsocks
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: shadowsocks
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: shadowsocks
+  template:
+    metadata:
+      labels:
+        app: shadowsocks
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: spb
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        runAsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: shadowsocks
+          # shadowsocks-rust v1.25.0; multi-arch image includes linux/arm64.
+          image: ghcr.io/shadowsocks/ssserver-rust:v1.25.0@sha256:22240c07d8ce3a36421ede770bd775dcc5fc56d5d829943196957b05452dc36f
+          imagePullPolicy: IfNotPresent
+          command: ["/usr/bin/ssserver"]
+          args:
+            - --log-without-time
+            - -c
+            - /etc/shadowsocks-rust/config.json
+          ports:
+            - name: proxy-tcp
+              containerPort: 8388
+              hostPort: 8388
+              protocol: TCP
+            - name: proxy-udp
+              containerPort: 8388
+              hostPort: 8388
+              protocol: UDP
+          readinessProbe:
+            tcpSocket:
+              port: proxy-tcp
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /etc/shadowsocks-rust
+              readOnly: true
+      volumes:
+        - name: config
+          secret:
+            secretName: shadowsocks-config
+            defaultMode: 0444

k8s-ng/apps/shadowsocks/external-secrets.yaml

@@ -0,0 +1,29 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: shadowsocks-config
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: shadowsocks-config
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        config.json: |
+          {
+            "server": "0.0.0.0",
+            "server_port": 8388,
+            "method": "chacha20-ietf-poly1305",
+            "password": {{ .password | toJson }},
+            "mode": "tcp_and_udp"
+          }
+  data:
+    - secretKey: password
+      remoteRef:
+        key: k8s/shadowsocks/secrets
+        property: Password

k8s-ng/apps/shadowsocks/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: shadowsocks
+resources:
+  - external-secrets.yaml
+  - deployment.yaml

k8s-ng/apps/web-petting/deployment.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web-petting
+  labels:
+    app: web-petting
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: web-petting
+  template:
+    metadata:
+      labels:
+        app: web-petting
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+      containers:
+        - name: web-petting
+          image: ultradesu/web-petting:latest
+          imagePullPolicy: Always
+          args:
+            - web-petting
+            - -l
+            - 0.0.0.0:3000
+          ports:
+            - name: http
+              containerPort: 3000
+          env:
+            - name: RUST_LOG
+              value: info
+            - name: WEB_PETTING_DEBUG
+              value: "false"
+            - name: WEB_PETTING_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: web-petting-secrets
+                  key: WEB_PETTING_DATABASE_URL
+          # Everything persistent is in PostgreSQL and R2; /data is scratch space only.
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+      volumes:
+        - name: data
+          emptyDir: {}

k8s-ng/apps/web-petting/ingress.yaml

@@ -0,0 +1,37 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: web-petting
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: pet.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: web-petting
+                port:
+                  number: 80
+    - host: xn--l1acako8eb.xn--p1ai
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: web-petting
+                port:
+                  number: 80
+  tls:
+    - secretName: web-petting-tls
+      hosts:
+        - pet.hexor.cy
+    - secretName: web-petting-murnyanya-tls
+      hosts:
+        - xn--l1acako8eb.xn--p1ai

k8s-ng/apps/web-petting/secrets.yaml

@@ -0,0 +1,47 @@
+# Read-only access to the web-petting-db Secret in the psql namespace
+# (see postgresql/databases/web-petting.yaml for the other half).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: db-credentials-reader
+---
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: psql-credentials
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: psql
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          key: ca.crt
+      auth:
+        serviceAccount:
+          name: db-credentials-reader
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: web-petting-secrets
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: psql-credentials
+  target:
+    name: web-petting-secrets
+    template:
+      data:
+        WEB_PETTING_DATABASE_URL: "postgresql://{{ .username }}:{{ .password }}@main-rw.psql.svc:5432/web_petting"
+  data:
+    - secretKey: username
+      remoteRef:
+        key: web-petting-db
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: web-petting-db
+        property: password

k8s-ng/apps/web-petting/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-petting
+  labels:
+    app: web-petting
+spec:
+  type: ClusterIP
+  selector:
+    app: web-petting
+  ports:
+    - name: http
+      port: 80
+      targetPort: 3000
+      protocol: TCP

k8s-ng/core-state/templates/applications.yaml

@@ -23,6 +23,9 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ServerSideApply=true
+      {{- if and (hasKey . "validate") (not .validate) }}
+      - Validate=false
+      {{- end }}
       {{- if .skipDryRunOnMissingResource }}
       - SkipDryRunOnMissingResource=true
       {{- end }}

k8s-ng/core-state/values.yaml

@@ -24,10 +24,19 @@ apps:
     path: k8s-ng/core/traefik-config
     namespace: kube-system
     wave: 1
+  - name: coredns-config
+    path: k8s-ng/core/coredns-config
+    namespace: kube-system
+    wave: 1
+    validate: false # partial Deployment: only the fields we manage
   - name: reloader
     path: k8s-ng/core/reloader
     namespace: reloader
     wave: 1
+  - name: gpu-system
+    path: k8s-ng/core/gpu
+    namespace: gpu-system
+    wave: 1
   - name: cert-manager
     path: k8s-ng/core/cert-manager
     namespace: cert-manager
@@ -53,13 +62,67 @@ apps:
     namespace: keycloak
     wave: 3
     skipDryRunOnMissingResource: true # ExternalSecret/Certificate CRDs come from earlier waves
+  # Not a core service: lives in k8s-ng/apps and belongs to the "apps" project.
   - name: furumi
-    path: k8s-ng/core/furumi
+    project: apps
+    path: k8s-ng/apps/furumi
     namespace: furumi
-    wave: 5 # pinned to the music node; must not hold back the other waves
+    wave: 6 # after prom-stack installs the ServiceMonitor CRD
+    skipDryRunOnMissingResource: true
+  - name: web-petting
+    project: apps
+    path: k8s-ng/apps/web-petting
+    namespace: web-petting
+    wave: 5
+    skipDryRunOnMissingResource: true
+  - name: k8s-secrets
+    project: apps
+    path: k8s/apps/k8s-secrets
+    namespace: k8s-secret
+    wave: 6 # after secret-stores and auth-proxy
+    skipDryRunOnMissingResource: true
+  - name: llamacpp
+    project: apps
+    path: k8s-ng/apps/llamacpp
+    namespace: llamacpp
+    wave: 5 # pinned to the ai node (GPU); must not hold back the other waves
     skipDryRunOnMissingResource: true
   - name: secret-stores
     path: k8s-ng/core/secret-stores
     namespace: external-secrets
     wave: 4 # reads the Vaultwarden vault, so it comes after vaultwarden
     skipDryRunOnMissingResource: true # ClusterSecretStore CRD comes from the external-secrets app
+  - name: gitea
+    path: k8s-ng/core/gitea
+    namespace: gitea
+    wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node
+    prune: false # never delete the instance because of a manifest change
+    skipDryRunOnMissingResource: true
+  - name: prom-stack
+    path: k8s-ng/core/prom-stack
+    namespace: prometheus
+    wave: 5 # after secret-stores; installs monitoring before application workloads
+    skipDryRunOnMissingResource: true # monitoring, cert-manager and ExternalSecret CRDs
+  - name: immich-db
+    path: k8s-ng/core/immich-db
+    namespace: immich
+    wave: 2
+    prune: false
+    skipDryRunOnMissingResource: true
+  - name: minecraft
+    project: games
+    path: k8s-ng/apps/minecraft
+    namespace: minecraft
+    wave: 6
+  - name: rustdesk
+    project: apps
+    path: k8s-ng/apps/rustdesk
+    namespace: rustdesk
+    wave: 6
+    skipDryRunOnMissingResource: true
+  - name: shadowsocks
+    project: apps
+    path: k8s-ng/apps/shadowsocks
+    namespace: shadowsocks
+    wave: 6
+    skipDryRunOnMissingResource: true

k8s-ng/core/argocd/values.yaml

@@ -1,6 +1,9 @@
 # Argo CD Helm Chart Values (k8s-ng cluster)
 
 global:
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   domain: ag.btwiusearch.net
   logging:
     format: text

k8s-ng/core/cert-manager/issuer.yaml

@@ -1,4 +1,5 @@
-# Public certificates through DNS-01. The credential Secrets are created by hand
+# Public certificates through DNS-01 for the zones below and HTTP-01 (Traefik) for
+# every other domain, e.g. the .рф one. The credential Secrets are created by hand
 # (they will move to Vaultwarden + External Secrets):
 #   kubectl -n cert-manager create secret generic cloudflare-api-token --from-literal=apiToken=<token>
 #     (Zone:DNS:Edit on the btwiusearch.net zone)
@@ -36,3 +37,8 @@ spec:
             secretAccessKeySecretRef:
               name: aws-api-key-secret
               key: secretKey
+      # Fallback for domains outside the zones above: their A records must point
+      # at the cluster nodes (Traefik answers on port 80 everywhere).
+      - http01:
+          ingress:
+            ingressClassName: traefik

k8s-ng/core/cert-manager/values.yaml

@@ -1,2 +1,12 @@
 crds:
   enabled: true
+
+nodeSelector: &externalCore
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+webhook:
+  nodeSelector: *externalCore
+cainjector:
+  nodeSelector: *externalCore
+startupapicheck:
+  nodeSelector: *externalCore

k8s-ng/core/cnpg/values.yaml

@@ -1,4 +1,8 @@
 # CloudNativePG operator (chart 0.29.x = operator 1.30.x)
 
+nodeSelector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+
 crds:
   create: true

k8s-ng/core/coredns-config/deployment.yaml

@@ -0,0 +1,33 @@
+# K3s owns the CoreDNS Deployment. Apply only the fields customized here so
+# upstream image, probes, Corefile mounts and topology spread stay intact.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  replicas: 3
+  template:
+    spec:
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        # DNS also belongs on the internal core node used by Actions jobs.
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: homelab.hexor.cy/pool
+                    operator: In
+                    values:
+                      - core

k8s-ng/core/coredns-config/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml

k8s-ng/core/coredns-config/service.yaml

@@ -0,0 +1,11 @@
+# Keep DNS on the caller's node when it has a healthy CoreDNS replica. K3s owns
+# the rest of this Service; server-side apply preserves its ports and selector.
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  trafficDistribution: PreferSameNode

k8s-ng/core/external-secrets/values.yaml

@@ -1 +1,9 @@
 installCRDs: true
+
+nodeSelector: &externalCore
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+webhook:
+  nodeSelector: *externalCore
+certController:
+  nodeSelector: *externalCore

k8s-ng/core/gitea/deployment.yaml

@@ -0,0 +1,72 @@
+# Gitea with SQLite; everything (repositories, database, SSH host keys) lives in
+# /var/lib/gitea on the music node.
+# `replicas` is deliberately not set: Argo CD does not manage it, so the instance
+# can be scaled to 0 with kubectl during a data migration without being healed back.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea
+  labels:
+    app: gitea
+spec:
+  selector:
+    matchLabels:
+      app: gitea
+  strategy:
+    type: Recreate # two instances must never open the same SQLite database
+  template:
+    metadata:
+      labels:
+        app: gitea
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
+      volumes:
+        - name: storage
+          hostPath:
+            path: /var/lib/gitea
+            type: DirectoryOrCreate
+      containers:
+        - name: gitea
+          image: gitea/gitea:28.0.0
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: "1"
+            limits:
+              memory: 4Gi
+              cpu: "4"
+          env:
+            - name: GITEA__service__REGISTER_MANUAL_CONFIRM
+              value: "true"
+            - name: GITEA__service__ENABLE_CAPTCHA
+              value: "true"
+            - name: GITEA__service__REQUIRE_CAPTCHA_FOR_LOGIN
+              value: "false"
+            - name: GITEA__service__REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA
+              value: "true"
+            - name: GITEA__service__CAPTCHA_TYPE
+              value: cfturnstile
+            - name: GITEA__security__ALLOWED_HOST_LIST
+              value: "*"
+          envFrom:
+            - secretRef:
+                name: gitea-runner-act-runner-secrets
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+            - name: ssh
+              containerPort: 22
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /api/healthz
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: storage
+              mountPath: /data

k8s-ng/core/gitea/external-secrets.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-runner-token
+spec:
+  refreshInterval: 1m
+  target:
+    name: gitea-runner-act-runner-secrets
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        token: |-
+          {{ .password }}
+        GITEA__service__CF_TURNSTILE_SITEKEY: |-
+          {{ .CF_TURNSTILE_SITEKEY }}
+        GITEA__service__CF_TURNSTILE_SECRET: |-
+          {{ .CF_TURNSTILE_SECRET }}
+  data:
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: login.password
+    - secretKey: CF_TURNSTILE_SITEKEY
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: fields[0].value
+    - secretKey: CF_TURNSTILE_SECRET
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: fields[1].value

k8s-ng/core/gitea/ingress.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitea
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  rules:
+    - host: gt.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gitea
+                port:
+                  number: 80
+  tls:
+    - secretName: gitea-tls
+      hosts:
+        - gt.hexor.cy

k8s-ng/core/gitea/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - external-secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+  - runner.yaml
+  - runner-rbac.yaml
+  - user-unban-cronjob.yaml

k8s-ng/core/gitea/runner-rbac.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gitea-runner-kubernetes-token
+  annotations:
+    kubernetes.io/service-account.name: gitea-runner
+type: kubernetes.io/service-account-token
+---
+# CI may edit the routes of the auth proxy (the ConfigMap lives in the keycloak namespace here).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auth-proxy-routes-manager
+  namespace: keycloak
+rules:
+  - apiGroups: [""]
+    resources: [configmaps]
+    resourceNames: [auth-proxy-routes]
+    verbs: [get, update, patch]
+  - apiGroups: [""]
+    resources: [configmaps]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-auth-proxy-routes
+  namespace: keycloak
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: auth-proxy-routes-manager
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner
+    namespace: gitea

k8s-ng/core/gitea/runner.yaml

@@ -0,0 +1,86 @@
+# Gitea Actions runner on the ai node. Jobs run as containers in the host's Docker
+# (virtualisation.docker in hosts/ai of the nix repo) through the mounted socket.
+# `replicas` is not set (see deployment.yaml): scale it with kubectl when needed.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-config
+data:
+  config.yaml: |
+    runner:
+      capacity: 8 # jobs in parallel; the ai node has 32 threads and 124 GiB
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea-runner
+  labels:
+    app: gitea-runner
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  selector:
+    matchLabels:
+      app: gitea-runner
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: gitea-runner
+    spec:
+      serviceAccountName: gitea-runner
+      dnsConfig:
+        options:
+          - name: ndots
+            value: "2"
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: internal
+        workload/ai: "true"
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      volumes:
+        - name: docker-sock
+          hostPath:
+            path: /var/run/docker.sock
+            type: Socket
+        - name: runner-data
+          emptyDir:
+            sizeLimit: 30Gi
+        - name: config
+          configMap:
+            name: gitea-runner-config
+      containers:
+        - name: gitea-runner
+          image: gitea/act_runner:nightly
+          volumeMounts:
+            - name: docker-sock
+              mountPath: /var/run/docker.sock
+            - name: runner-data
+              mountPath: /data
+            - name: config
+              mountPath: /config.yaml
+              subPath: config.yaml
+          env:
+            - name: CONFIG_FILE
+              value: /config.yaml
+            - name: GITEA_INSTANCE_URL
+              # Job containers run outside the cluster network, so they need the public URL.
+              value: https://gt.hexor.cy
+            - name: GITEA_RUNNER_REGISTRATION_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-runner-act-runner-secrets
+                  key: token
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GITEA_RUNNER_NAME
+              value: $(NODE_NAME)
+            - name: GITEA_RUNNER_LABELS
+              value: ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04,ubuntu-20.04:docker://ghcr.io/catthehacker/ubuntu:act-20.04

k8s-ng/core/gitea/service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea
+spec:
+  selector:
+    app: gitea
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 3000
+---
+# Git over SSH on every node (servicelb); the node firewall opens 30022 (nix repo, k3s-base.nix).
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-ssh
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 30022
+      targetPort: 22
+  selector:
+    app: gitea

k8s-ng/core/gitea/user-unban-cronjob.yaml

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-user-unban
+  labels:
+    app: gitea-user-unban
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: gitea-user-unban
+        spec:
+          restartPolicy: OnFailure
+          nodeSelector:
+            homelab.hexor.cy/pool: core
+            homelab.hexor.cy/exposure: external
+            kubernetes.io/hostname: music
+          volumes:
+            - name: storage
+              hostPath:
+                path: /var/lib/gitea
+                type: Directory
+          containers:
+            - name: sqlite-unban
+              image: gitea/gitea:28.0.0
+              imagePullPolicy: IfNotPresent
+              resources:
+                requests:
+                  memory: 32Mi
+                  cpu: 10m
+                limits:
+                  memory: 128Mi
+                  cpu: 100m
+              command:
+                - /bin/sh
+                - -ec
+                - |
+                  sqlite3 -cmd ".timeout 30000" /data/gitea/gitea.db "
+                    UPDATE \"user\"
+                       SET is_active = 1,
+                           prohibit_login = 0,
+                           updated_unix = unixepoch()
+                     WHERE lower(email) = lower('ab@hexor.cy')
+                       AND (is_active <> 1 OR prohibit_login <> 0);
+
+                    SELECT printf(
+                      'gitea user watchdog: id=%d login=%s email=%s is_active=%d prohibit_login=%d updated_unix=%d',
+                      id, lower_name, email, is_active, prohibit_login, updated_unix
+                    )
+                      FROM \"user\"
+                     WHERE lower(email) = lower('ab@hexor.cy');
+                  "
+              volumeMounts:
+                - name: storage
+                  mountPath: /data

k8s-ng/core/gpu/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+helmCharts:
+  - name: amd-gpu
+    repo: https://rocm.github.io/k8s-device-plugin/
+    version: 0.22.0
+    releaseName: amd-gpu-device-plugin
+    namespace: gpu-system
+    valuesFile: values.yaml
+    includeCRDs: true
+
+# The chart always adds the NFD label feature.node.kubernetes.io/pci-0300_1002.present
+# to the selector; NFD is not installed (and would not match this iGPU), so replace it.
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: DaemonSet
+      name: amd-gpu-device-plugin-daemonset
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/nodeSelector
+        value:
+          homelab.hexor.cy/pool: core
+          homelab.hexor.cy/exposure: internal
+          kubernetes.io/arch: amd64
+          workload/ai: "true"

k8s-ng/core/gpu/values.yaml

@@ -0,0 +1,37 @@
+# AMD GPU device plugin: exposes the Strix Halo iGPU of the "ai" node as the
+# amd.com/gpu resource (and injects /dev/kfd and /dev/dri into the pod).
+nfd:
+  enabled: false
+
+labeller:
+  enabled: false
+
+dp:
+  image:
+    repository: docker.io/rocm/k8s-device-plugin
+    tag: "1.31.0.9"
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+
+# The ai node carries the label workload/ai=true and the taint workload=ai:NoSchedule
+# (hosts/ai in the nix repo).
+tolerations:
+  - key: workload
+    operator: Equal
+    value: ai
+    effect: NoSchedule
+
+node_selector_enabled: true
+node_selector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: internal
+  kubernetes.io/arch: amd64
+  workload/ai: "true"

k8s-ng/core/immich-db/cluster.yaml

@@ -0,0 +1,47 @@
+# Keep PostgreSQL and both vector extensions identical to the source during
+# migration. The application/media migration and any upgrades happen later.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: immich
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  description: Immich database migrated from the old cluster; media remain on NAS
+  imageName: ghcr.io/tensorchord/cloudnative-pgvecto.rs:14-v0.3.0@sha256:df7848916fea2c486faee8155a481dc5009f30ec513b5561d9ab76d1e101a594
+  instances: 1
+  enableSuperuserAccess: true
+  bootstrap:
+    initdb:
+      database: immich
+      owner: immich
+      dataChecksums: true
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+  postgresUID: 26
+  postgresGID: 26
+  storage:
+    size: 55Gi
+    storageClass: immich-local
+    pvcTemplate:
+      accessModes: [ReadWriteOnce]
+      selector:
+        matchLabels:
+          immich.hexor.cy/volume: postgres
+  affinity:
+    nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
+      kubernetes.io/hostname: music
+  postgresql:
+    shared_preload_libraries: [vectors.so]
+    parameters:
+      shared_buffers: 512MB
+      max_connections: "200"
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      memory: 2Gi

k8s-ng/core/immich-db/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: immich
+resources:
+  - storage.yaml
+  - cluster.yaml

k8s-ng/core/immich-db/storage.yaml

@@ -0,0 +1,66 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: immich-db-storage-prepare
+  annotations:
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
+      containers:
+        - name: prepare
+          image: docker.io/busybox:1.37.0
+          command: [sh, -ec, "mkdir -p /storage/postgresql; chown 26:26 /storage/postgresql; chmod 0700 /storage/postgresql"]
+          securityContext:
+            runAsUser: 0
+            allowPrivilegeEscalation: false
+          volumeMounts:
+            - name: storage
+              mountPath: /storage
+      volumes:
+        - name: storage
+          hostPath:
+            path: /data/k8s/immich
+            type: DirectoryOrCreate
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: immich-local
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Retain
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: immich-postgres-music
+  labels:
+    immich.hexor.cy/volume: postgres
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 55Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: immich-local
+  local:
+    path: /data/k8s/immich/postgresql
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]

k8s-ng/core/keycloak/auth-proxy.yaml

@@ -20,9 +20,12 @@ spec:
       labels:
         app: auth-proxy
     spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
       containers:
         - name: auth-proxy
-          image: ultradesu/rsauth2-proxy:latest
+          image: ultradesu/rsauth2-proxy:v0.1.3
           ports:
             - containerPort: 8080
               name: http

k8s-ng/core/keycloak/values.yaml

@@ -1,5 +1,9 @@
 replicas: 1
 
+nodeSelector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+
 image:
   repository: quay.io/keycloak/keycloak
   tag: "26.5.6"

k8s-ng/core/postgresql/cluster.yaml

@@ -2,7 +2,11 @@
 # own node with a node-local volume. Replication is asynchronous (nodes sit in
 # different regions), so a failover can lose the last few transactions.
 #
-# Nodes eligible for instances carry the label workload/postgres=true. To move a
+# Layout: the primary runs on the music node (the powerful one), the replica on
+# master. A failover swaps the roles; switch back with
+#   kubectl cnpg promote main <instance on music>
+#
+# Eligible nodes are core/external and carry workload/postgres=true. To move a
 # replica: label the new node, unlabel the old one, then recreate the instance
 # (kubectl cnpg destroy main <N>); the operator re-clones it from the primary.
 #
@@ -32,6 +36,8 @@ spec:
     podAntiAffinityType: required
     topologyKey: kubernetes.io/hostname
     nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
       workload/postgres: "true"
 
   resources:

k8s-ng/core/postgresql/databases/furumi.yaml

@@ -50,7 +50,10 @@ spec:
   name: furumi
   owner: furumi
   extensions:
+    # `ensure` is defaulted by the CRD inside list items; spelled out so Argo CD
+    # does not see it as drift.
     - name: pg_trgm
+      ensure: present
   databaseReclaimPolicy: retain
 ---
 # Lets the furumi namespace read only its own credentials Secret.

k8s-ng/core/postgresql/databases/grafana.yaml

@@ -0,0 +1,78 @@
+# Database "grafana": generated credentials, role, database and read access for
+# the monitoring namespace.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: grafana-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: grafana
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-grafana
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: grafana
+  login: true
+  passwordSecret:
+    name: grafana-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-grafana
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: grafana
+  owner: grafana
+  databaseReclaimPolicy: retain
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: grafana-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [grafana-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: grafana-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: grafana-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: prometheus

k8s-ng/core/postgresql/databases/meguri.yaml

@@ -0,0 +1,81 @@
+# Database "meguri": password, role, database and read access for the app.
+# To add another database copy this file and replace the name.
+
+# Generated once; the role and the application both read this Secret.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: meguri-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: meguri-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: meguri
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-meguri
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: meguri
+  login: true
+  passwordSecret:
+    name: meguri-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-meguri
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: meguri
+  owner: meguri
+  databaseReclaimPolicy: retain
+---
+# Lets the meguri namespace read only its own credentials Secret.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: meguri-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [meguri-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: meguri-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: meguri-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: meguri

k8s-ng/core/postgresql/databases/web-petting.yaml

@@ -0,0 +1,81 @@
+# Database "web_petting": password, role, database and read access for the app.
+# To add another database copy this file and replace the name.
+
+# Generated once; the role and the application both read this Secret.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: web-petting-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: web-petting-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: web_petting
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-web-petting
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: web_petting
+  login: true
+  passwordSecret:
+    name: web-petting-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-web-petting
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: web_petting
+  owner: web_petting
+  databaseReclaimPolicy: retain
+---
+# Lets the web-petting namespace read only its own credentials Secret.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: web-petting-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [web-petting-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: web-petting-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: web-petting-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: web-petting

k8s-ng/core/postgresql/kustomization.yaml

@@ -7,3 +7,6 @@ resources:
   - databases/vaultwarden.yaml
   - databases/keycloak.yaml
   - databases/furumi.yaml
+  - databases/web-petting.yaml
+  - databases/meguri.yaml
+  - databases/grafana.yaml

k8s-ng/core/prom-stack/README.md

@@ -0,0 +1,63 @@
+# Monitoring stack (new cluster)
+
+The stack is intentionally a fresh installation: no Prometheus TSDB, Loki log
+chunks, or Grafana database are copied from the old cluster.
+
+## Layout
+
+- `kube-prometheus-stack`: Prometheus, Alertmanager, Grafana, exporters and
+  Kubernetes alerting rules.
+- `loki`: one filesystem-backed monolithic instance. A second replica would
+  require shared object storage and is deliberately not faked with a local PV.
+- `alloy`: unprivileged Kubernetes API log and event collection. This replaces
+  EOL Promtail and does not require host log mounts or privileged sysctls.
+- Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on
+  the large local disk of node `music`.
+
+Stack services select `homelab.hexor.cy/pool=core` and
+`homelab.hexor.cy/exposure=external`. Prometheus, Alertmanager and Loki also
+select `music` because their volumes are local. Grafana, the operators, Alloy
+and the Loki gateway can run on either `master` or `music`. Node Exporter and
+Loki Canary are not restricted to these groups.
+
+## Storage and retention
+
+The PreSync hook creates these directories on `music`:
+
+| Path | PV | Retention |
+| --- | ---: | --- |
+| `/data/k8s/monitoring/prometheus` | 500 Gi | up to 365 days / 425 GB |
+| `/data/k8s/monitoring/loki` | 200 Gi | 90 days |
+| `/data/k8s/monitoring/alertmanager` | 5 Gi | 120 hours of notification state |
+
+All PVs and the StorageClass use `Retain`; Argo CD is also told not to prune
+them. This protects data from a manifest rename, but it is not a backup. Back up
+`/data/k8s/monitoring` with the rest of the `music` node if history matters.
+
+## Dependencies
+
+- node `music` must exist and `/data` must be the intended large filesystem;
+- `main-rw.psql.svc` and the generated `grafana-db` Secret must be ready;
+- `vaultwarden-fields` must resolve `k8s/prometheus/grafana-secrets` with the
+  custom fields `IDM Client ID`, `IDM Client Secret`, `TG Token`, and `TG ID`;
+- DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster.
+
+## Alerting and target identities
+
+Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and
+`InfoInhibitor` remain visible in Prometheus but route to the null receiver.
+Warning and critical notifications are grouped for two minutes and rendered as
+Telegram HTML with escaped annotations and labelled alerts/runbook links.
+Notifications show at most three alerts, with each alert's escaped text limited
+to 500 characters. A single short Prometheus alerts link replaces repeated graph
+URLs; the full group remains visible there. Workload alerts show the affected
+DaemonSet, Deployment or StatefulSet rather than the kube-state-metrics pod
+which supplied the metric.
+
+Node Exporter keeps scraping the discovered node address, but rewrites the
+public `instance` label from Kubernetes discovery to `<node-name>:9100`. Grafana
+variables and alerts therefore remain stable when node IP addresses change.
+
+After Argo CD reports the application healthy, verify targets, rules and log
+delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana
+uses its own Keycloak OIDC client.

k8s-ng/core/prom-stack/alertmanager-config.yaml

@@ -0,0 +1,71 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: telegram
+  labels:
+    alertmanager: main
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  route:
+    groupBy: [cluster, namespace, alertname]
+    # A two-minute grace period absorbs alerts which resolve immediately and
+    # never need to interrupt a human.
+    groupWait: 2m
+    groupInterval: 10m
+    repeatInterval: 12h
+    receiver: telegram
+    routes:
+      - receiver: "null"
+        matchers:
+          - name: alertname
+            value: Watchdog|InfoInhibitor
+            matchType: =~
+      # Informational alerts remain queryable in Prometheus but do not page the
+      # Telegram chat. Warning and critical alerts still use the parent route.
+      - receiver: "null"
+        matchers:
+          - name: severity
+            value: info
+            matchType: =
+  receivers:
+    - name: telegram
+      telegramConfigs:
+        - botToken:
+            name: alertmanager-telegram
+            key: bot-token
+          # Telegram chat IDs are routing identifiers rather than credentials;
+          # AlertmanagerConfig only supports an integer here, not a Secret ref.
+          chatID: 124317807
+          sendResolved: true
+          parseMode: HTML
+          # Bound the rendered HTML, including escaped text, before Telegram's
+          # 4096-character limit. Keep the full group accessible in Prometheus.
+          message: |-
+            {{- define "telegram.alert.text" -}}
+            {{- $escaped := html . -}}
+            {{- $short := printf "%.500s" $escaped -}}
+            {{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}…{{ end -}}
+            {{- end -}}
+            {{- $alerts := .Alerts.Resolved -}}
+            {{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}}
+            {{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
+            {{ with .CommonLabels.alertname }}<b>{{ printf "%.64s" . }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ printf "%.16s" . }}</code>{{ end }}
+            {{ range $index, $alert := $alerts }}{{ if lt $index 3 }}
+            {{- $context := "" -}}
+            {{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}}
+            {{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}}
+            {{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}}
+            {{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}}
+            {{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}}
+            {{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}}
+            {{- end -}}
+            {{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}}
+            {{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}}
+
+            {{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }}
+            Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
+            {{ end }}{{ end }}
+            {{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }}
+            <a href="https://prom.hexor.cy/alerts">📈 Open alerts</a>{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} · <a href="{{ . }}">📖 Runbook</a>{{ end }}{{ end }}
+    - name: "null"

k8s-ng/core/prom-stack/alloy-values.yaml

@@ -0,0 +1,161 @@
+alloy:
+  enableReporting: false
+  storagePath: /var/lib/alloy
+  configMap:
+    create: true
+    content: |-
+      discovery.kubernetes "pods" {
+        role = "pod"
+      }
+
+      discovery.relabel "pod_logs" {
+        targets = discovery.kubernetes.pods.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace"]
+          target_label  = "namespace"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          target_label  = "pod"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_name"]
+          target_label  = "container"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_node_name"]
+          target_label  = "node"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_label_app"]
+          regex         = `(.+)`
+          target_label  = "app"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+          regex         = `(.+)`
+          target_label  = "app"
+        }
+        rule {
+          source_labels = ["namespace", "app"]
+          separator     = "/"
+          regex         = `(.+)/(.+)`
+          replacement   = "$1/$2"
+          target_label  = "job"
+        }
+      }
+
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pod_logs.output
+        forward_to = [loki.process.pods.receiver]
+      }
+
+      loki.process "pods" {
+        stage.static_labels {
+          values = {
+            cluster = "hexor-ng",
+          }
+        }
+        forward_to = [loki.write.local.receiver]
+      }
+
+      loki.source.kubernetes_events "cluster_events" {
+        job_name   = "integrations/kubernetes/eventhandler"
+        log_format = "logfmt"
+        forward_to = [loki.process.cluster_events.receiver]
+      }
+
+      loki.process "cluster_events" {
+        stage.static_labels {
+          values = {
+            cluster = "hexor-ng",
+          }
+        }
+        forward_to = [loki.write.local.receiver]
+      }
+
+      loki.write "local" {
+        endpoint {
+          url = "http://loki-gateway.prometheus.svc/loki/api/v1/push"
+        }
+      }
+  mounts:
+    extra:
+      - name: data
+        mountPath: /var/lib/alloy
+  securityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 473
+    runAsGroup: 473
+    capabilities:
+      drop: [ALL]
+    seccompProfile:
+      type: RuntimeDefault
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 1Gi
+  livenessProbe:
+    httpGet:
+      path: /metrics
+      port: 12345
+      scheme: HTTP
+    initialDelaySeconds: 30
+    periodSeconds: 30
+    timeoutSeconds: 2
+    failureThreshold: 3
+
+global:
+  podSecurityContext:
+    fsGroup: 473
+    fsGroupChangePolicy: OnRootMismatch
+    runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
+
+controller:
+  type: deployment
+  replicas: 1
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  priorityClassName: monitoring-critical
+  minReadySeconds: 10
+  terminationGracePeriodSeconds: 60
+  volumes:
+    extra:
+      - name: data
+        emptyDir: {}
+
+configReloader:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 50Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# Only the API permissions used by pod-log discovery and cluster events.
+rbac:
+  create: true
+  rules:
+    - apiGroups: [""]
+      resources: [pods, pods/log, namespaces, events]
+      verbs: [get, list, watch]
+  # Kept separate because the Alloy chart concatenates this list into the
+  # ClusterRole. Node metadata is useful for diagnosing a log source.
+  clusterRules:
+    - apiGroups: [""]
+      resources: [nodes]
+      verbs: [get, list, watch]
+
+serviceMonitor:
+  enabled: true
+  interval: 30s

k8s-ng/core/prom-stack/dashboards/auth-proxy-dashboard-cm.yaml

@@ -0,0 +1,367 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: auth-proxy-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  auth-proxy.json: |-
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": {
+              "type": "grafana",
+              "uid": "-- Grafana --"
+            },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "type": "dashboard"
+          }
+        ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+          "id": 1,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "routes_loaded_total",
+              "refId": "A"
+            }
+          ],
+          "title": "Loaded Routes",
+          "type": "stat"
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+          "id": 2,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(auth_requests_total[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Auth Requests (total)",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+          "id": 3,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(auth_requests_total{status=\"403\"}[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Forbidden (403)",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "red", "value": 1 }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+          "id": 4,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(callback_requests_total{result=\"success\"}[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Successful Logins",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+          "id": 5,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum by (status) (rate(auth_requests_total[$__rate_interval]))",
+              "legendFormat": "{{status}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Auth Request Rate by Status",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "reqps"
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+          "id": 6,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.5\"}",
+              "legendFormat": "p50",
+              "refId": "A"
+            },
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.95\"}",
+              "legendFormat": "p95",
+              "refId": "B"
+            },
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.99\"}",
+              "legendFormat": "p99",
+              "refId": "C"
+            }
+          ],
+          "title": "Auth Request Duration (p50, p95, p99)",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+          "id": 7,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum by (result) (rate(callback_requests_total[$__rate_interval]))",
+              "legendFormat": "{{result}}",
+              "refId": "A"
+            }
+          ],
+          "title": "OIDC Callback Rate by Result",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 50,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "normal" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "reqps"
+            },
+            "overrides": []
+          }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "style": "dark",
+      "tags": [
+        "auth-proxy",
+        "rsauth2-proxy"
+      ],
+      "templating": {
+        "list": [
+          {
+            "current": {
+              "selected": false,
+              "text": "Prometheus",
+              "value": "Prometheus"
+            },
+            "hide": 0,
+            "includeAll": false,
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          }
+        ]
+      },
+      "time": {
+        "from": "now-1h",
+        "to": "now"
+      },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Auth Proxy (rsauth2-proxy)",
+      "uid": "auth-proxy-metrics",
+      "version": 1
+    }

k8s-ng/core/prom-stack/dashboards/furumi-dashboard-cm.yaml

@@ -0,0 +1,504 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: furumi-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  furumi.json: |-
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "type": "dashboard"
+          }
+        ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "id": 1,
+          "title": "Build Version",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "max by (version) (furumusic_build_info{namespace=~\"$namespace\"})",
+              "legendFormat": "{{version}}",
+              "refId": "A"
+            }
+          ],
+          "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "name" }
+        },
+        {
+          "id": 2,
+          "title": "HTTP RPS",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 3,
+          "title": "5xx Error Ratio",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 4,
+          "title": "HTTP p95 Latency",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 5,
+          "title": "Active Users 15m",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_active_users{namespace=~\"$namespace\",window=\"15m\"})", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 6,
+          "title": "Stream Throughput",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "Bps", "decimals": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        { "id": 10, "title": "HTTP", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "collapsed": false, "panels": [] },
+        {
+          "id": 11,
+          "title": "Request Rate by Route and Status",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
+          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route, status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "{{route}} {{status}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 12,
+          "title": "Request Duration Quantiles",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 13,
+          "title": "In-flight Requests",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route) (furumusic_http_in_flight_requests{namespace=~\"$namespace\",route=~\"$route\"})", "legendFormat": "{{route}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 14,
+          "title": "HTTP Body Throughput",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_request_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "request", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_response_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "response", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 15,
+          "title": "Status Code Mix",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 20, "title": "Auth and Users", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "collapsed": false, "panels": [] },
+        {
+          "id": 21,
+          "title": "Users by Role",
+          "type": "bargauge",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (role) (furumusic_users_total{namespace=~\"$namespace\"})", "legendFormat": "{{role}}", "refId": "A" }
+          ],
+          "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "showUnfilled": true }
+        },
+        {
+          "id": 22,
+          "title": "Active Users",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (furumusic_active_users{namespace=~\"$namespace\"})", "legendFormat": "{{window}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 23,
+          "title": "Auth Events",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method, outcome, reason) (rate(furumusic_auth_login_attempts_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "login {{method}} {{outcome}} {{reason}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method) (rate(furumusic_auth_sessions_created_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "session {{method}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(furumusic_auth_denied_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "denied {{kind}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 30, "title": "Playback and Streaming", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "collapsed": false, "panels": [] },
+        {
+          "id": 31,
+          "title": "Listens Rate",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (completed) (rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "completed={{completed}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 32,
+          "title": "Listen Completion Ratio",
+          "description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 33,
+          "title": "Streaming",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (range) (rate(furumusic_stream_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "requests range={{range}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 34,
+          "title": "Listened Hours and Play History Activity",
+          "description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 },
+          "fieldConfig": {
+            "defaults": { "unit": "h", "decimals": 2 },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 40, "title": "Library and Storage", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] },
+        {
+          "id": 41,
+          "title": "Library Inventory",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_artists_total{namespace=~\"$namespace\"})", "legendFormat": "artists", "refId": "C" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_playlists_total{namespace=~\"$namespace\"})", "legendFormat": "playlists", "refId": "D" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 42,
+          "title": "Media Bytes by Type",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 43,
+          "title": "Storage Used Ratio",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "1 - (sum by (path_kind) (furumusic_storage_free_bytes{namespace=~\"$namespace\"}) / sum by (path_kind) (furumusic_storage_total_bytes{namespace=~\"$namespace\"}))", "legendFormat": "{{path_kind}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 50, "title": "AI Agent", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "collapsed": false, "panels": [] },
+        {
+          "id": 51,
+          "title": "Agent Queue and Reviews",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 52,
+          "title": "Agent Processing and Failures",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome, decision) (rate(furumusic_agent_files_processed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "processed {{outcome}} {{decision}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (stage) (rate(furumusic_agent_failed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "failed {{stage}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 53,
+          "title": "Agent Confidence",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 54,
+          "title": "Discover Runs and Duration",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_agent_discover_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "runs {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(furumusic_agent_discover_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 55,
+          "title": "Discover File Flow",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_seen_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "seen", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_queued_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "queued", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_discover_files_skipped_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 56,
+          "title": "RAG and LLM Requests",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind, outcome) (rate(furumusic_agent_rag_queries_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, outcome) (rate(furumusic_agent_llm_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 57,
+          "title": "RAG and LLM Latency p95",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, kind, outcome) (rate(furumusic_agent_rag_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, model, outcome) (rate(furumusic_agent_llm_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 58,
+          "title": "LLM Token Rate, Batch Splits and Parse Failures",
+          "description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 },
+          "fieldConfig": {
+            "defaults": { "unit": "suffix: tok/s" },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] },
+              { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model) (rate(furumusic_agent_llm_parse_failures_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "parse failures {{model}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 59,
+          "title": "Cover Pipeline",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, outcome) (rate(furumusic_agent_cover_lookup_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "lookup {{source}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (variant, outcome) (rate(furumusic_agent_cover_variant_generation_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "variant {{variant}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 70, "title": "Scheduler and Torrents", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "collapsed": false, "panels": [] },
+        {
+          "id": 71,
+          "title": "Scheduler Jobs",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_running{namespace=~\"$namespace\"})", "legendFormat": "running {{job}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_enabled{namespace=~\"$namespace\"})", "legendFormat": "enabled {{job}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 72,
+          "title": "Scheduler Runs and Duration p95",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job, trigger, outcome) (rate(furumusic_scheduler_job_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{job}} {{trigger}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, job, trigger, outcome) (rate(furumusic_scheduler_job_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{job}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 73,
+          "title": "Torrents",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_torrent_sessions_total{namespace=~\"$namespace\"})", "legendFormat": "sessions {{status}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_downloads_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "downloads {{outcome}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_selected_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes {{outcome}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "style": "dark",
+      "tags": [ "furumi", "furumusic", "music" ],
+      "templating": {
+        "list": [
+          {
+            "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+            "hide": 0,
+            "includeAll": false,
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          },
+          {
+            "allValue": ".*",
+            "current": { "selected": true, "text": "All", "value": "$__all" },
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(furumusic_build_info, namespace)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "namespace",
+            "options": [],
+            "query": { "query": "label_values(furumusic_build_info, namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          },
+          {
+            "allValue": ".*",
+            "current": { "selected": true, "text": "All", "value": "$__all" },
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "route",
+            "options": [],
+            "query": { "query": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          }
+        ]
+      },
+      "time": { "from": "now-6h", "to": "now" },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Furumi Music",
+      "uid": "furumi-music",
+      "version": 1
+    }

k8s-ng/core/prom-stack/dashboards/telemt-dashboard-cm.yaml

@@ -0,0 +1,409 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: telemt-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  telemt.json: |-
+    {
+      "annotations": { "list": [] },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "title": "Nodes Overview",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
+          "id": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {
+                "align": "auto",
+                "cellOptions": { "type": "auto" },
+                "inspect": false
+              },
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": { "id": "byName", "options": "Uptime" },
+                "properties": [
+                  { "id": "unit", "value": "dtdurations" },
+                  { "id": "custom.width", "value": 140 }
+                ]
+              },
+              {
+                "matcher": { "id": "byName", "options": "Bad Conn" },
+                "properties": [
+                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 100 }] } },
+                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
+                ]
+              },
+              {
+                "matcher": { "id": "byName", "options": "Writers" },
+                "properties": [
+                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
+                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
+                ]
+              }
+            ]
+          },
+          "options": {
+            "showHeader": true,
+            "sortBy": [{ "displayName": "Node", "desc": false }],
+            "frameIndex": 0,
+            "footer": { "show": false }
+          },
+          "transformations": [
+            {
+              "id": "joinByField",
+              "options": { "byField": "node", "mode": "outer" }
+            },
+            {
+              "id": "filterFieldsByName",
+              "options": {
+                "include": { "pattern": "^(node|Value.*)$" }
+              }
+            },
+            {
+              "id": "organize",
+              "options": {
+                "renameByName": {
+                  "node": "Node",
+                  "Value #uptime": "Uptime",
+                  "Value #writers": "Writers",
+                  "Value #buffers": "Buffers In Use",
+                  "Value #connections": "Connections",
+                  "Value #bad": "Bad Conn",
+                  "Value #hs_timeout": "HS Timeouts"
+                }
+              }
+            }
+          ],
+          "targets": [
+            {
+              "expr": "telemt_uptime_seconds{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "uptime",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_me_writers_active_current{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "writers",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_buffer_pool_buffers_total{node=~\"$node\", kind=\"in_use\"}",
+              "legendFormat": "",
+              "refId": "buffers",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_connections_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "connections",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_connections_bad_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "bad",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_handshake_timeouts_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "hs_timeout",
+              "format": "table",
+              "instant": true
+            }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Connections Rate",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+          "id": 10,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_connections_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} accepted", "refId": "A" },
+            { "expr": "rate(telemt_connections_bad_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} bad", "refId": "B" },
+            { "expr": "rate(telemt_handshake_timeouts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs timeout", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Upstream Connect",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+          "id": 11,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_upstream_connect_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} success", "refId": "A" },
+            { "expr": "rate(telemt_upstream_connect_fail_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} fail", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Upstream Connect Duration (success)",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+          "id": 12,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } },
+              "unit": "short",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "increase(telemt_upstream_connect_duration_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{bucket}}", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "ME Writers & Pool",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+          "id": 13,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_me_writers_active_current{node=~\"$node\"}", "legendFormat": "{{node}} active", "refId": "A" },
+            { "expr": "telemt_me_writers_warm_current{node=~\"$node\"}", "legendFormat": "{{node}} warm", "refId": "B" },
+            { "expr": "telemt_pool_drain_active{node=~\"$node\"}", "legendFormat": "{{node}} draining", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Active Connections",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+          "id": 20,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_user_connections_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}}", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Traffic",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+          "id": 21,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "Bps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_user_octets_from_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} rx", "refId": "A" },
+            { "expr": "rate(telemt_user_octets_to_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} tx", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "DC->Client Payload",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+          "id": 30,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "Bps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_me_d2c_payload_bytes_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} payload", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "ME Errors & Anomalies",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+          "id": 31,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_me_reconnect_attempts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} reconnect", "refId": "A" },
+            { "expr": "rate(telemt_me_handshake_reject_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs reject", "refId": "B" },
+            { "expr": "rate(telemt_me_crc_mismatch_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} crc mismatch", "refId": "C" },
+            { "expr": "rate(telemt_desync_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} desync", "refId": "D" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Unique IPs",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+          "id": 40,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_user_unique_ips_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} active", "refId": "A" },
+            { "expr": "telemt_user_unique_ips_recent_window{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} recent", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Conntrack",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+          "id": 41,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"attempt\"}[5m])", "legendFormat": "{{node}} delete attempt", "refId": "A" },
+            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"error\"}[5m])", "legendFormat": "{{node}} delete error", "refId": "B" },
+            { "expr": "telemt_conntrack_event_queue_depth{node=~\"$node\"}", "legendFormat": "{{node}} queue depth", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 39,
+      "tags": ["telemt", "mtproxy", "telegram"],
+      "templating": {
+        "list": [
+          {
+            "current": {},
+            "hide": 0,
+            "includeAll": false,
+            "label": "Datasource",
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          },
+          {
+            "current": {},
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(telemt_uptime_seconds, node)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Node",
+            "multi": true,
+            "name": "node",
+            "query": "label_values(telemt_uptime_seconds, node)",
+            "refresh": 2,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          }
+        ]
+      },
+      "time": { "from": "now-6h", "to": "now" },
+      "title": "Telemt MTProxy",
+      "uid": "telemt-mtproxy"
+    }

k8s-ng/core/prom-stack/external-secrets.yaml

@@ -0,0 +1,146 @@
+# Read-only access to the Grafana database credentials in the psql namespace.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: db-credentials-reader
+  annotations:
+    argocd.argoproj.io/sync-wave: "-3"
+---
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: psql-credentials
+  annotations:
+    argocd.argoproj.io/sync-wave: "-2"
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: psql
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          key: ca.crt
+      auth:
+        serviceAccount:
+          name: db-credentials-reader
+---
+# A local break-glass account. OIDC remains the normal login path.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: grafana-admin
+  annotations:
+    argocd.argoproj.io/sync-wave: "-2"
+spec:
+  length: 40
+  digits: 10
+  symbols: 0
+  noUpper: false
+  allowRepeat: true
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-admin
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: grafana-admin
+    template:
+      type: Opaque
+      data:
+        admin-user: admin
+        admin-password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: grafana-admin
+---
+# Grafana's database credentials come from CNPG. OIDC credentials are read by
+# stable Vaultwarden item and custom-field names, so imports and field ordering
+# do not invalidate the references.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-env
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: 1h
+  target:
+    name: grafana-env
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        GF_DATABASE_USER: "{{ .db_username }}"
+        GF_DATABASE_PASSWORD: "{{ .db_password }}"
+        GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .client_id }}"
+        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .client_secret }}"
+  data:
+    - secretKey: db_username
+      remoteRef:
+        key: grafana-db
+        property: username
+    - secretKey: db_password
+      remoteRef:
+        key: grafana-db
+        property: password
+    - secretKey: client_id
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: IDM Client ID
+    - secretKey: client_secret
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: IDM Client Secret
+  secretStoreRef:
+    kind: SecretStore
+    name: psql-credentials
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-telegram
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: 1h
+  target:
+    name: alertmanager-telegram
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        bot-token: "{{ .bot_token }}"
+        chat-id: "{{ .chat_id }}"
+  data:
+    - secretKey: bot_token
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: TG Token
+    - secretKey: chat_id
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: TG ID

k8s-ng/core/prom-stack/ingress.yaml

@@ -0,0 +1,41 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: prometheus-auth
+spec:
+  forwardAuth:
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: true
+    authResponseHeaders:
+      - X-Auth-Request-User
+      - X-Auth-Request-Email
+      - X-Auth-Request-Groups
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: prometheus
+spec:
+  entryPoints: [websecure]
+  routes:
+    - match: Host(`prom.hexor.cy`)
+      kind: Rule
+      middlewares:
+        - name: prometheus-auth
+      services:
+        - name: prometheus-kube-prometheus-prometheus
+          port: 9090
+  tls:
+    secretName: prometheus-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: prometheus-tls
+spec:
+  secretName: prometheus-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - prom.hexor.cy

k8s-ng/core/prom-stack/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - storage.yaml
+  - external-secrets.yaml
+  - alertmanager-config.yaml
+  - ingress.yaml
+  - loki-ingress.yaml
+  - service-monitors.yaml
+  - dashboards/auth-proxy-dashboard-cm.yaml
+  - dashboards/furumi-dashboard-cm.yaml
+  - dashboards/telemt-dashboard-cm.yaml
+
+helmCharts:
+  - name: kube-prometheus-stack
+    repo: https://prometheus-community.github.io/helm-charts
+    version: 92.1.0
+    releaseName: prometheus
+    namespace: prometheus
+    valuesFile: prom-values.yaml
+    includeCRDs: true
+
+  - name: loki
+    repo: https://grafana-community.github.io/helm-charts
+    version: 18.13.8
+    releaseName: loki
+    namespace: prometheus
+    valuesFile: loki-values.yaml
+    includeCRDs: true
+
+  - name: alloy
+    repo: https://grafana.github.io/helm-charts
+    version: 1.13.0
+    releaseName: alloy
+    namespace: prometheus
+    valuesFile: alloy-values.yaml
+    includeCRDs: true

k8s-ng/core/prom-stack/loki-ingress.yaml

@@ -0,0 +1,48 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: loki-machine-auth
+spec:
+  forwardAuth:
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: false
+---
+# Do not pass the Keycloak client secret through to Loki after ForwardAuth.
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: loki-strip-authorization
+spec:
+  headers:
+    customRequestHeaders:
+      Authorization: ""
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: loki-push
+spec:
+  entryPoints: [websecure]
+  routes:
+    - match: Host(`loki.hexor.cy`) && Path(`/loki/api/v1/push`) && Method(`POST`)
+      kind: Rule
+      middlewares:
+        - name: loki-machine-auth
+        - name: loki-strip-authorization
+      services:
+        - name: loki-gateway
+          port: 80
+  tls:
+    secretName: loki-push-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: loki-push-tls
+spec:
+  secretName: loki-push-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - loki.hexor.cy

k8s-ng/core/prom-stack/loki-values.yaml

@@ -0,0 +1,123 @@
+deploymentMode: Monolithic
+
+loki:
+  auth_enabled: false
+  commonConfig:
+    path_prefix: /var/loki
+    replication_factor: 1
+  schemaConfig:
+    configs:
+      - from: "2026-01-01"
+        store: tsdb
+        object_store: filesystem
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+  storage:
+    type: filesystem
+    filesystem:
+      chunks_directory: /var/loki/chunks
+      rules_directory: /var/loki/rules
+  compactor:
+    working_directory: /var/loki/compactor
+    retention_enabled: true
+    delete_request_store: filesystem
+  limits_config:
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    retention_period: 2160h # 90 days
+    ingestion_rate_mb: 16
+    ingestion_burst_size_mb: 32
+    max_query_parallelism: 16
+    query_timeout: 5m
+    volume_enabled: true
+
+singleBinary:
+  replicas: 1
+  sidecar: false
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+    kubernetes.io/hostname: music
+  priorityClassName: monitoring-critical
+  terminationGracePeriodSeconds: 120
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: "4"
+      memory: 8Gi
+  podDisruptionBudget:
+    enabled: false
+  persistence:
+    enabled: true
+    size: 200Gi
+    storageClass: monitoring-local
+    selector:
+      matchLabels:
+        monitoring.hexor.cy/volume: loki
+    whenDeleted: Retain
+    whenScaled: Retain
+
+# These must be zero in filesystem-backed monolithic mode.
+read:
+  replicas: 0
+write:
+  replicas: 0
+backend:
+  replicas: 0
+
+chunksCache:
+  enabled: false
+resultsCache:
+  enabled: false
+
+sidecar:
+  rules:
+    enabled: false
+
+gateway:
+  enabled: true
+  # Gateway replicas can run on either external core node; Loki storage stays on music.
+  replicas: 2
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  # Allow a surge pod to share a node during rolling updates.
+  affinity: null
+  priorityClassName: monitoring-critical
+  verboseLogging: false
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 500m
+      memory: 256Mi
+  podDisruptionBudget:
+    enabled: true
+
+test:
+  enabled: false
+lokiCanary:
+  enabled: true
+  resources:
+    requests:
+      cpu: 10m
+      memory: 32Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+monitoring:
+  dashboards:
+    enabled: true
+  rules:
+    enabled: true
+  alerts:
+    enabled: true
+  serviceMonitor:
+    enabled: true
+    interval: 30s

k8s-ng/core/prom-stack/prom-values.yaml

@@ -0,0 +1,295 @@
+defaultRules:
+  create: true
+  # Default rule descriptions reference $labels.cluster. Prometheus external
+  # labels are added only when alerts leave Prometheus, so attach the label to
+  # the rules as well to avoid messages ending with "on cluster .".
+  additionalRuleLabels:
+    cluster: hexor-ng
+  rules:
+    # k3s embeds these control-plane components and does not expose their
+    # conventional metrics Services. Disabling the groups prevents false pages.
+    etcd: false
+    kubeControllerManager: false
+    kubeProxy: false
+    kubeSchedulerAlerting: false
+    kubeSchedulerRecording: false
+  runbookUrl: https://runbooks.prometheus-operator.dev/runbooks
+
+kubeControllerManager:
+  enabled: false
+kubeScheduler:
+  enabled: false
+kubeProxy:
+  enabled: false
+kubeEtcd:
+  enabled: false
+
+alertmanager:
+  enabled: true
+  config:
+    global:
+      resolve_timeout: 5m
+    route:
+      group_by: [namespace, alertname]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 12h
+      receiver: "null"
+    receivers:
+      - name: "null"
+  alertmanagerSpec:
+    replicas: 1
+    retention: 120h
+    nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
+      kubernetes.io/hostname: music
+    priorityClassName: monitoring-critical
+    alertmanagerConfigSelector:
+      matchLabels:
+        alertmanager: main
+    # The single global routing object must be allowed to receive alerts from
+    # every namespace, rather than only alerts labelled namespace=prometheus.
+    alertmanagerConfigMatcherStrategy:
+      type: None
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 500m
+        memory: 1Gi
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: monitoring-local
+          accessModes: [ReadWriteOnce]
+          selector:
+            matchLabels:
+              monitoring.hexor.cy/volume: alertmanager
+          resources:
+            requests:
+              storage: 5Gi
+
+prometheus:
+  prometheusSpec:
+    replicas: 1
+    nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
+      kubernetes.io/hostname: music
+    priorityClassName: monitoring-critical
+    scrapeInterval: 30s
+    scrapeTimeout: 10s
+    evaluationInterval: 30s
+    externalUrl: https://prom.hexor.cy
+    externalLabels:
+      cluster: hexor-ng
+    enableRemoteWriteReceiver: true
+    walCompression: true
+    retention: 365d
+    # Time and size retention are both applied; whichever is reached first
+    # wins, leaving headroom for compaction and WAL spikes on the 500 Gi PV.
+    retentionSize: 425GB
+    resources:
+      requests:
+        cpu: "1"
+        memory: 4Gi
+      limits:
+        cpu: "4"
+        memory: 12Gi
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: monitoring-local
+          accessModes: [ReadWriteOnce]
+          selector:
+            matchLabels:
+              monitoring.hexor.cy/volume: prometheus
+          resources:
+            requests:
+              storage: 500Gi
+
+    # Discover monitoring resources in every namespace without requiring the
+    # kube-prometheus-stack release label on application-owned monitors.
+    ruleSelectorNilUsesHelmValues: false
+    ruleSelector: {}
+    ruleNamespaceSelector: {}
+    serviceMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorSelector: {}
+    serviceMonitorNamespaceSelector: {}
+    podMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelector: {}
+    podMonitorNamespaceSelector: {}
+    probeSelectorNilUsesHelmValues: false
+    probeSelector: {}
+    probeNamespaceSelector: {}
+    scrapeConfigSelectorNilUsesHelmValues: false
+    scrapeConfigSelector: {}
+    scrapeConfigNamespaceSelector: {}
+
+   #additionalScrapeConfigs:
+   #  - job_name: xray-vpn
+   #    metrics_path: /scrape
+   #    static_configs:
+   #      - targets: [cy.tail2fe2d.ts.net:9550]
+   #        labels: {site: cy}
+   #      - targets: [x86.tail2fe2d.ts.net:9550]
+   #        labels: {site: am}
+   #      - targets: [jp.tail2fe2d.ts.net:9550]
+   #        labels: {site: jp}
+   #  - job_name: macos-node-exporter
+   #    static_configs:
+   #      - targets: [macbook-pro.tail2fe2d.ts.net:9100]
+   #        labels: {instance: ultrabook-pro}
+
+grafana:
+  enabled: true
+  replicas: 1
+  testFramework:
+    enabled: false
+  annotations:
+    reloader.stakater.com/auto: "true"
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  priorityClassName: monitoring-critical
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: "2"
+      memory: 2Gi
+  admin:
+    existingSecret: grafana-admin
+    userKey: admin-user
+    passwordKey: admin-password
+  envFromSecret: grafana-env
+  grafana.ini:
+    server:
+      root_url: https://gf.hexor.cy
+    auth:
+      signout_redirect_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/logout?post_logout_redirect_uri=https%3A%2F%2Fgf.hexor.cy%2Flogin&client_id=Grafana
+      oauth_allow_insecure_email_lookup: true
+    auth.generic_oauth:
+      name: Keycloak
+      enabled: true
+      scopes: openid profile email
+      allow_sign_up: true
+      auth_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/auth
+      token_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/token
+      api_url: https://auth.hexor.cy/auth/realms/hexor/protocol/openid-connect/userinfo
+      email_attribute_path: email
+      login_attribute_path: preferred_username
+      name_attribute_path: name
+      role_attribute_path: >-
+        contains(groups[*], 'hexor-admin') && 'Admin' ||
+        contains(groups[*], 'hexor-guest') && 'Viewer' ||
+        'Viewer'
+      role_attribute_strict: false
+    database:
+      type: postgres
+      host: main-rw.psql.svc:5432
+      name: grafana
+      ssl_mode: disable
+    log:
+      level: info
+  sidecar:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: "1"
+        memory: 512Mi
+    dashboards:
+      enabled: true
+      searchNamespace: ALL
+    datasources:
+      uid: prometheus
+      alertmanager:
+        uid: alertmanager
+        handleGrafanaManagedAlerts: false
+  additionalDataSources:
+    - name: Loki
+      uid: loki
+      type: loki
+      url: http://loki-gateway.prometheus.svc:80
+      access: proxy
+      orgId: 1
+      jsonData:
+        maxLines: 2000
+  ingress:
+    enabled: true
+    ingressClassName: traefik
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt
+      traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    hosts:
+      - gf.hexor.cy
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - gf.hexor.cy
+
+prometheusOperator:
+  admissionWebhooks:
+    patch:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  priorityClassName: monitoring-critical
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+  prometheusConfigReloader:
+    resources:
+      requests:
+        cpu: 20m
+        memory: 64Mi
+      limits:
+        cpu: 200m
+        memory: 256Mi
+
+kube-state-metrics:
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  priorityClassName: monitoring-critical
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+prometheus-node-exporter:
+  priorityClassName: monitoring-critical
+  tolerations:
+    - operator: Exists
+  prometheus:
+    monitor:
+      # Keep dashboard variables and alerts stable when a node IP changes.
+      # The scrape address remains the discovered node IP; only the public
+      # instance label is rewritten to the Kubernetes node hostname.
+      relabelings:
+        - action: replace
+          sourceLabels: [__meta_kubernetes_endpoint_node_name]
+          targetLabel: instance
+          replacement: "${1}:9100"
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 500m
+      memory: 256Mi

k8s-ng/core/prom-stack/service-monitors.yaml

@@ -0,0 +1,31 @@
+# Monitors for core services that already exist before this stack is installed.
+# Later application waves keep their ServiceMonitors next to the applications.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: auth-proxy
+  namespace: keycloak
+spec:
+  selector:
+    matchLabels:
+      app: auth-proxy
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: postgresql
+  namespace: psql
+spec:
+  selector:
+    matchLabels:
+      cnpg.io/cluster: main
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s-ng/core/prom-stack/storage.yaml

@@ -0,0 +1,146 @@
+# Prepare the node-local directories before the local PersistentVolumes are
+# made available. The hook is idempotent and runs before every Argo CD sync.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: monitoring-storage-prepare
+  namespace: prometheus
+  annotations:
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
+    argocd.argoproj.io/sync-wave: "-2"
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
+      containers:
+        - name: prepare
+          image: docker.io/busybox:1.37.0
+          command:
+            - sh
+            - -ec
+            - |
+              mkdir -p /storage/prometheus /storage/alertmanager /storage/loki
+              chown 1000:2000 /storage/prometheus /storage/alertmanager
+              chown 10001:10001 /storage/loki
+              chmod 0770 /storage/prometheus /storage/alertmanager /storage/loki
+          securityContext:
+            runAsNonRoot: false
+            runAsUser: 0
+            allowPrivilegeEscalation: false
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: storage
+              mountPath: /storage
+      volumes:
+        - name: storage
+          hostPath:
+            path: /data/k8s/monitoring
+            type: DirectoryOrCreate
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: monitoring-local
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: Prune=false
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Retain
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: monitoring-prometheus-music
+  labels:
+    monitoring.hexor.cy/volume: prometheus
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 500Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: monitoring-local
+  local:
+    path: /data/k8s/monitoring/prometheus
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: monitoring-alertmanager-music
+  labels:
+    monitoring.hexor.cy/volume: alertmanager
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 5Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: monitoring-local
+  local:
+    path: /data/k8s/monitoring/alertmanager
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: monitoring-loki-music
+  labels:
+    monitoring.hexor.cy/volume: loki
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 200Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: monitoring-local
+  local:
+    path: /data/k8s/monitoring/loki
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]
+---
+# Keep monitoring schedulable under application-node pressure without competing
+# with Kubernetes' system-cluster-critical classes.
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: monitoring-critical
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+value: 1000000
+globalDefault: false
+description: Monitoring control plane and storage workloads

k8s-ng/core/reloader/values.yaml

@@ -6,7 +6,9 @@ reloader:
   reloadStrategy: annotations
   ignoreConfigMaps: false
   ignoreSecrets: false
-  ignoreJobs: false
+  # Jobs are one-shot executions. Restarting completed Helm hooks after a
+  # referenced Secret changes can rerun migrations outside the release flow.
+  ignoreJobs: true
   ignoreCronJobs: false
   enableHA: true
   syncAfterRestart: true
@@ -14,6 +16,9 @@ reloader:
   rbac:
     enabled: true
   deployment:
+    nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
     replicas: 2
     resources:
       requests:

k8s-ng/core/secret-stores/bitwarden-cli.yaml

@@ -3,7 +3,7 @@
 # with email + password, so keep password login enabled for it, i.e. no
 # SSO_ONLY, or switch the image to API-key login):
 #   kubectl -n external-secrets create secret generic bitwarden-cli \
-#     --from-literal=BW_HOST=https://vw.btwiusearch.net \
+#     --from-literal=BW_HOST=https://vw.hexor.cy \
 #     --from-literal=BW_USERNAME=<bot email> --from-literal=BW_PASSWORD=<password>
 # Image source: Dockerfile-bitwarden-cli in this directory.
 apiVersion: apps/v1
@@ -29,6 +29,8 @@ spec:
         app.kubernetes.io/instance: bitwarden-cli
     spec:
       nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
         kubernetes.io/arch: amd64
       containers:
         - name: bitwarden-cli
@@ -40,7 +42,9 @@ spec:
               cpu: "300m"
             limits:
               memory: "512Mi"
-              cpu: "1000m"
+              # `bw sync` is bursty and was reaching the 1-core CFS quota.
+              # Keep the scheduler reservation modest, but allow short sync bursts.
+              cpu: "2"
           env:
             - name: BW_HOST
               valueFrom:
@@ -62,15 +66,19 @@ spec:
               containerPort: 8087
               protocol: TCP
           livenessProbe:
+            # Syncs the vault and requires the bridge to be logged in: `bw serve` can
+            # lose its login (status "unauthenticated") while /sync still answers, and
+            # every lookup then fails with 500. Failing the probe restarts the pod.
             exec:
               command:
-                - wget
-                - -q
-                - http://127.0.0.1:8087/sync
-                - --post-data=''
+                - sh
+                - -c
+                - >-
+                  wget -q -O /dev/null --post-data='' http://127.0.0.1:8087/sync &&
+                  wget -q -O - http://127.0.0.1:8087/status | grep -q '"status":"unlocked"'
             initialDelaySeconds: 20
             failureThreshold: 3
-            timeoutSeconds: 1
+            timeoutSeconds: 10
             periodSeconds: 120
           readinessProbe:
             tcpSocket:

k8s-ng/core/traefik-config/helmchartconfig.yaml

@@ -11,6 +11,20 @@ metadata:
   namespace: kube-system
 spec:
   valuesContent: |-
+    deployment:
+      kind: DaemonSet
+    service:
+      spec:
+        externalTrafficPolicy: Local
+        # ServiceLB masquerades clients as pod traffic before the NodePort hop,
+        # which bypasses externalTrafficPolicy. Prefer the ingress on this node
+        # for that cluster-policy path as well.
+        trafficDistribution: PreferSameNode
+    tolerations:
+      - key: workload
+        operator: Equal
+        value: ai
+        effect: NoSchedule
     ports:
       websecure:
         asDefault: true

k8s-ng/core/vaultwarden/deployment.yaml

@@ -16,6 +16,10 @@ spec:
       labels:
         app: vaultwarden
     spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
       containers:
         - name: vaultwarden
           image: vaultwarden/server:1.37.3
@@ -34,7 +38,7 @@ spec:
                 optional: true
           env:
             - name: DOMAIN
-              value: https://vw.btwiusearch.net
+              value: https://vw.hexor.cy
             - name: ORG_GROUPS_ENABLED
               value: "true"
             - name: SSO_AUTHORITY
@@ -74,4 +78,4 @@ spec:
       volumes:
         - name: data
           persistentVolumeClaim:
-            claimName: vaultwarden-data
+            claimName: vaultwarden-data-music

k8s-ng/core/vaultwarden/ingress.yaml

@@ -7,7 +7,7 @@ metadata:
     traefik.ingress.kubernetes.io/router.entrypoints: websecure
 spec:
   rules:
-    - host: vw.btwiusearch.net
+    - host: vw.hexor.cy
       http:
         paths:
           - path: /
@@ -20,4 +20,4 @@ spec:
   tls:
     - secretName: vaultwarden-tls
       hosts:
-        - vw.btwiusearch.net
+        - vw.hexor.cy

k8s-ng/core/vaultwarden/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 
 resources:
   - secrets.yaml
+  - storage.yaml
   - pvc.yaml
   - deployment.yaml
   - service.yaml

k8s-ng/core/vaultwarden/pvc.yaml

@@ -1,12 +1,13 @@
 # Attachments, sends, icon cache and RSA keys live here; the database is in PostgreSQL.
-# A local-path volume is bound to one node: the pod stays on the node it first ran on.
+# Bind explicitly to the retained local volume on music (see storage.yaml).
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: vaultwarden-data
+  name: vaultwarden-data-music
 spec:
   accessModes: [ReadWriteOnce]
-  storageClassName: local-path
+  storageClassName: ""
+  volumeName: vaultwarden-data-music
   resources:
     requests:
       storage: 2Gi

k8s-ng/core/vaultwarden/storage.yaml

@@ -0,0 +1,28 @@
+# The directory must contain Vaultwarden's migrated /data before the pod starts.
+# Retain the files when the claim or Argo CD application is removed.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: vaultwarden-data-music
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 2Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  claimRef:
+    namespace: vaultwarden
+    name: vaultwarden-data-music
+  local:
+    path: /data/k8s/vaultwarden
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]

k8s/apps/k8s-secrets/app.yaml

@@ -1,21 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: k8s-secrets
-  namespace: argocd
-spec:
-  project: apps
-  destination:
-    namespace: k8s-secret
-    server: https://kubernetes.default.svc
-  source:
-    repoURL: ssh://git@gt.hexor.cy:30022/ab/homelab.git
-    targetRevision: HEAD
-    path: k8s/apps/k8s-secrets
-  syncPolicy:
-    automated:
-      selfHeal: true
-      prune: true
-    syncOptions:
-      - CreateNamespace=true
-

k8s/apps/k8s-secrets/deployment.yaml

@@ -16,11 +16,12 @@ spec:
     spec:
       serviceAccountName: secret-reader
       nodeSelector:
-        kubernetes.io/os: linux
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
       containers:
       - name: secret-reader
         image: ultradesu/k8s-secrets:0.2.1
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         args:
           - "--secrets"
           - "openai-creds"
@@ -28,7 +29,6 @@ spec:
           - "k8s-secret"
           - "--port"
           - "3000"
-          - "--webhook"
         ports:
         - containerPort: 3000
           name: http

k8s/apps/k8s-secrets/external-secret.yaml

@@ -1,44 +1,32 @@
----
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
   name: openai-creds
 spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vaultwarden-login
   target:
     name: openai-creds
     deletionPolicy: Delete
     template:
       type: Opaque
       data:
-        USER: |-
-          {{ .user }}
-        PASS: |-
-          {{ .pass }}
-        TOTP: |-
-          {{ .totp }}
+        USER: "{{ .user }}"
+        PASS: "{{ .pass }}"
+        TOTP: "{{ .totp }}"
   data:
     - secretKey: user
-      sourceRef:
-        storeRef:
-          name: vaultwarden-login
-          kind: ClusterSecretStore
       remoteRef:
-        key: a485f323-fd47-40ee-a5cf-40891b1f963c
+        key: k8s/openai/creds
         property: login.username
     - secretKey: pass
-      sourceRef:
-        storeRef:
-          name: vaultwarden-login
-          kind: ClusterSecretStore
       remoteRef:
-        key: a485f323-fd47-40ee-a5cf-40891b1f963c
+        key: k8s/openai/creds
         property: login.password
     - secretKey: totp
-      sourceRef:
-        storeRef:
-          name: vaultwarden-login
-          kind: ClusterSecretStore
       remoteRef:
-        key: a485f323-fd47-40ee-a5cf-40891b1f963c
+        key: k8s/openai/creds
         property: login.totp
 

k8s/apps/k8s-secrets/ingress.yaml

@@ -5,12 +5,8 @@ metadata:
   name: auth-proxy
 spec:
   forwardAuth:
-    address: http://auth-proxy.auth-proxy.svc:80/auth
-    trustForwardHeader: true
-    authResponseHeaders:
-      - X-Auth-Request-User
-      - X-Auth-Request-Email
-      - X-Auth-Request-Groups
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: false
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute

k8s/apps/k8s-secrets/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - service-account.yaml
+  - rbac.yaml
+  - external-secret.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

k8s/apps/k8s-secrets/rbac.yaml

@@ -5,7 +5,8 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["get", "list"]
+  resourceNames: ["openai-creds"]
+  verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

terraform/keycloak/main.tf

@@ -204,6 +204,34 @@ resource "keycloak_openid_client_default_scopes" "rsauth2_proxy" {
   ]
 }
 
+# A single machine identity shared by the host-level Vector agents. The secret
+# is generated by Keycloak and delivered to Nix hosts through sops-nix.
+resource "keycloak_openid_client" "tsunagi_vector" {
+  realm_id                     = keycloak_realm.hexor.id
+  client_id                    = "tsunagi-vector"
+  name                         = "tsunagi Vector log writer"
+  enabled                      = true
+  access_type                  = "CONFIDENTIAL"
+  standard_flow_enabled        = false
+  direct_access_grants_enabled = false
+  service_accounts_enabled     = true
+  access_token_lifespan        = "600"
+}
+
+# Handoff to the Nix sops secret. Terraform CI has the generated Keycloak
+# credential; the operator can read this Secret and encrypt it for the hosts.
+resource "kubernetes_secret_v1" "tsunagi_vector_client" {
+  metadata {
+    name      = "tsunagi-vector-client"
+    namespace = "keycloak"
+  }
+
+  data = {
+    client_id     = keycloak_openid_client.tsunagi_vector.client_id
+    client_secret = keycloak_openid_client.tsunagi_vector.client_secret
+  }
+}
+
 # =============================================================================
 # Proxy applications — auto-created groups + routes ConfigMap
 # =============================================================================
@@ -298,11 +326,21 @@ resource "kubernetes_config_map_v1" "auth_proxy_routes" {
 
   data = {
     "routes.yaml" = yamlencode({
-      routes = {
+      routes = merge({
         for k, v in var.proxy_applications : v.domain => {
           allowed_groups = local.app_allowed_groups[k]
         }
-      }
+        }, {
+        "loki.hexor.cy" = {
+          browser_access = false
+          allowed_groups = []
+          machine_access = {
+            client_ids = [keycloak_openid_client.tsunagi_vector.client_id]
+            method     = "POST"
+            path       = "/loki/api/v1/push"
+          }
+        }
+      })
     })
   }
 }

terraform/keycloak/outputs.tf

@@ -15,6 +15,11 @@ output "rsauth2_proxy_client_secret" {
   sensitive = true
 }
 
+output "tsunagi_vector_client_secret" {
+  value     = keycloak_openid_client.tsunagi_vector.client_secret
+  sensitive = true
+}
+
 output "standalone_groups" {
   value = [for g in keycloak_group.standalone : g.name]
 }

terraform/keycloak/providers.tf

@@ -19,6 +19,6 @@ provider "keycloak" {
 }
 
 provider "kubernetes" {
-  config_path    = "~/.kube/config"
-  config_context = "hexor-lab"
+  # CI supplies the current cluster context through its KUBECONFIG secret.
+  config_path = "~/.kube/config"
 }

terraform/keycloak/terraform.tfvars

@@ -4,7 +4,10 @@ groups = [
   "game-servers-managers",
   "argocd-admins",
   "web-petting",
-  "Furumusic-Admins"
+  "Furumusic-Admins",
+  "meguri-admins",
+  "meguri-leads",
+  "meguri-team"
 ]
 
 proxy_applications = {
@@ -43,9 +46,9 @@ oauth2_applications = {
     extra_optional_scopes     = ["offline_access"]
   }
   Vaultwarden = {
-    redirect_uris             = ["https://vw.btwiusearch.net/identity/connect/oidc-signin"]
-    web_origins               = ["https://vw.btwiusearch.net"]
-    post_logout_redirect_uris = ["https://vw.btwiusearch.net/*"]
+    redirect_uris             = ["https://vw.hexor.cy/identity/connect/oidc-signin"]
+    web_origins               = ["https://vw.hexor.cy"]
+    post_logout_redirect_uris = ["https://vw.hexor.cy/*"]
     extra_optional_scopes     = ["offline_access"]
     # Bitwarden clients treat tokens shorter than ~10 minutes as expired.
     access_token_lifespan = "900"
@@ -111,6 +114,13 @@ oauth2_applications = {
     web_origins               = ["https://auth.matrix.hexor.cy"]
     post_logout_redirect_uris = ["https://auth.matrix.hexor.cy/*"]
   }
+  # Meguri matches the group names below exactly (the groups mapper sends plain names, not paths):
+  # hexor-admin and meguri-admins = instance administrators, meguri-leads / meguri-team = the "Core" workspace.
+  Meguri = {
+    redirect_uris             = ["https://meguri.hexor.cy/api/oidc/callback"]
+    web_origins               = ["https://meguri.hexor.cy"]
+    post_logout_redirect_uris = ["https://meguri.hexor.cy/*"]
+  }
   Amnezia-Fellow = {
     redirect_uris             = ["https://awg.hexor.cy/auth/oidc/callback"]
     web_origins               = ["https://awg.hexor.cy"]