Provision Jacob database with pgvector through GitOps

`bw serve` can lose its login while /sync still answers; every lookup then
fails with 500. The probe now also checks /status, so the pod restarts itself.

Co-Authored-By: Claude <noreply@anthropic.com>

.gitea/workflows/authentik-apps.yaml

@@ -30,9 +30,11 @@ jobs:
         cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
 
     - name: Setup kubeconfig
+      env:
+        CI_KUBECONFIG: ${{ secrets.KUBECONFIG }}
       run: |
         mkdir -p ~/.kube
-        echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
+        printf '%s' "$CI_KUBECONFIG" > ~/.kube/config
         chmod 600 ~/.kube/config
 
     - name: Terraform Init

k8s-ng/apps/furumi/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - servicemonitor.yaml
+  - ingress.yaml

k8s-ng/apps/furumi/servicemonitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: furumi-player
+spec:
+  selector:
+    matchLabels:
+      app: furumi-player
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s-ng/apps/k8s-secrets/README.md

@@ -0,0 +1,12 @@
+# k8s-secrets
+
+The `k8s-secrets` Argo CD Application reads this directory through
+`core-state/values.yaml` and deploys into namespace `k8s-secret`.
+
+External Secrets loads the `k8s/openai/creds` Vaultwarden item into
+`openai-creds`. The reader at `pass.hexor.cy` can only read that Secret and is
+protected by the Keycloak forward-auth proxy. Its access policy is managed in
+`terraform/keycloak/terraform.tfvars`.
+
+The Kubernetes resource names and namespace match the former deployment so
+Argo CD can switch the source path without replacing the service or certificate.

k8s-ng/apps/k8s-secrets/external-secret.yaml

@@ -0,0 +1,32 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: openai-creds
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    name: vaultwarden-login
+    kind: ClusterSecretStore
+  target:
+    name: openai-creds
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        USER: "{{ .user }}"
+        PASS: "{{ .pass }}"
+        TOTP: "{{ .totp }}"
+  data:
+    - secretKey: user
+      remoteRef:
+        key: k8s/openai/creds
+        property: login.username
+    - secretKey: pass
+      remoteRef:
+        key: k8s/openai/creds
+        property: login.password
+    - secretKey: totp
+      remoteRef:
+        key: k8s/openai/creds
+        property: login.totp

k8s-ng/apps/k8s-secrets/ingress.yaml

@@ -0,0 +1,38 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: auth-proxy
+spec:
+  forwardAuth:
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: false
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: secret-reader
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`pass.hexor.cy`)
+      kind: Rule
+      middlewares:
+        - name: auth-proxy
+      services:
+        - name: secret-reader
+          port: 80
+  tls:
+    secretName: secret-reader-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: secret-reader-tls
+spec:
+  secretName: secret-reader-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - pass.hexor.cy

k8s-ng/apps/k8s-secrets/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: k8s-secret
+resources:
+  - external-secret.yaml
+  - reader.yaml
+  - ingress.yaml

k8s-ng/apps/k8s-secrets/reader.yaml

@@ -0,0 +1,109 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-reader
+  labels:
+    app: secret-reader
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-reader
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["openai-creds"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: secret-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: secret-reader
+subjects:
+  - kind: ServiceAccount
+    name: secret-reader
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: secret-reader
+  labels:
+    app: secret-reader
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: secret-reader
+  template:
+    metadata:
+      labels:
+        app: secret-reader
+    spec:
+      serviceAccountName: secret-reader
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: secret-reader
+          image: ultradesu/k8s-secrets:0.2.2@sha256:f206942d21ca9dcd131d6e19ba26cc17671815919217aef657b30a9f85854778
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secrets
+            - openai-creds
+            - --namespace
+            - k8s-secret
+            - --port
+            - "3000"
+          ports:
+            - name: http
+              containerPort: 3000
+          env:
+            - name: RUST_LOG
+              value: info
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 150m
+              memory: 128Mi
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: secret-reader
+  labels:
+    app: secret-reader
+spec:
+  selector:
+    app: secret-reader
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

k8s-ng/apps/llamacpp/configmap.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llamacpp-config
+data:
+  LLAMA_CACHE: /models
+  LLAMA_ARG_HOST: 0.0.0.0
+  LLAMA_ARG_PORT: "8080"
+  # The model is chosen here: <hf-user>/<repo>:<quant>. It is downloaded into
+  # /models on first start; changing it restarts the pod (reloader).
+  #LLAMA_ARG_HF_REPO: "unsloth/Qwen3.8-27B-GGUF:UD-Q6_K"
+  LLAMA_ARG_HF_REPO: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K"
+  LLAMA_ARG_CTX_SIZE: "131072"
+  LLAMA_ARG_N_GPU_LAYERS: all
+  LLAMA_ARG_FLASH_ATTN: "on"
+  LLAMA_ARG_FIT: "on"
+  # Strix Halo: mmap-ed weights hang or corrupt on ROCm; read them directly.
+  LLAMA_ARG_LOAD_MODE: dio

k8s-ng/apps/llamacpp/deployment.yaml

@@ -0,0 +1,76 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llamacpp
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: llamacpp
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate # one GPU, one pod
+  selector:
+    matchLabels:
+      app: llamacpp
+  template:
+    metadata:
+      labels:
+        app: llamacpp
+    spec:
+      dnsPolicy: Default
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: internal
+        workload/ai: "true"
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      containers:
+        - name: llamacpp
+          image: ghcr.io/ggml-org/llama.cpp:server-rocm-b11434
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: llamacpp-config
+          env:
+            - name: HF_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: llamacpp-hf-token
+                  key: token
+                  optional: true
+            # hipBLASLt gives the biggest prompt-processing gain on ROCm 7.x.
+            - name: ROCBLAS_USE_HIPBLASLT
+              value: "1"
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          resources:
+            limits:
+              amd.com/gpu: 1
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 180 # first start downloads the model
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 3
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      volumes:
+        - name: models
+          hostPath:
+            path: /var/lib/llamacpp/models
+            type: DirectoryOrCreate

k8s-ng/apps/llamacpp/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml

k8s-ng/apps/llamacpp/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: llamacpp
+  labels:
+    app: llamacpp
+spec:
+  type: ClusterIP
+  selector:
+    app: llamacpp
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+      protocol: TCP

k8s-ng/apps/minecraft/configmaps.yaml

@@ -0,0 +1,561 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-config
+  namespace: minecraft
+data:
+  nginx.conf: |
+    user nginx;
+    worker_processes 1;
+    error_log /var/log/nginx/error.log warn;
+    pid /var/run/nginx.pid;
+
+    events {
+        worker_connections 1024;
+    }
+
+    http {
+        include /etc/nginx/mime.types;
+        default_type application/octet-stream;
+
+        server {
+            listen 80;
+
+            # Custom 502 error page with auto-refresh
+            error_page 502 /502.html;
+            location = /502.html {
+                internal;
+                return 200 '<!DOCTYPE html><html><head><meta charset="utf-8"><title>Server Loading</title><style>body{font-family:Arial,sans-serif;text-align:center;margin-top:100px;background:#f0f0f0}h1{color:#333}p{color:#666;font-size:18px}</style></head><body><h1>Server is loading probably...</h1><p>Please wait a moment and try refreshing the page.</p><script>setTimeout(function(){window.location.reload();}, 10000);</script></body></html>';
+                add_header Content-Type text/html;
+            }
+
+            # Main location - proxy to Minecraft Dynmap
+            location / {
+                # Proxy configuration for Dynmap server
+                proxy_pass http://localhost:8123;
+                proxy_set_header Host $host;
+                proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-Proto $scheme;
+
+                # Inject user authentication meta tag into HTML head
+                sub_filter '<head>' '<head><meta name="remote-user" content="$http_x_authentik_username">';
+
+                # Replace default Dynmap title with custom server name
+                sub_filter 'Minecraft Dynamic Map' "Hexor's MC server";
+
+                # Inject custom script before closing body tag
+                sub_filter "</body>" '<script src="/clients/inject.js"></script></body>';
+
+                # Apply sub_filter replacements globally (not just once)
+                sub_filter_once off;
+            }
+
+            # Serve inject.js and .ps1 scripts inline (no forced download)
+            location = /clients/inject.js {
+                alias /mc/clients/inject.js;
+                default_type application/javascript;
+            }
+
+            location ~ ^/clients/(.+\.ps1)$ {
+                alias /mc/clients/$1;
+                default_type text/plain;
+            }
+
+            # Static file serving for client downloads
+            location /clients/ {
+                alias /mc/clients/;
+                sendfile on;                          # Enable efficient file serving
+                add_header Content-Disposition "attachment";  # Force download
+                autoindex on;                         # Enable directory listing
+                gzip off;                            # Disable compression for downloads
+                chunked_transfer_encoding off;       # Disable chunked encoding
+            }
+        }
+    }
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: client-scripts
+  namespace: minecraft
+data:
+  inject.js: |
+    (function() {
+      function getUsername() {
+        var metas = document.querySelectorAll("meta");
+        for (var i = 0; i < metas.length; i++) {
+          if (metas[i].getAttribute("name") === "remote-user") {
+            return metas[i].getAttribute("content");
+          }
+        }
+        var jwt = document.cookie.split("; ").find(function(row) {
+          return row.startsWith("authentik_session=");
+        });
+        if (jwt) {
+          try {
+            var token = jwt.split("=")[1];
+            var payload = JSON.parse(atob(token.split(".")[1]));
+            return payload.sub || payload.username || "web-user";
+          } catch(e) {}
+        }
+        return "web-user";
+      }
+
+      var username = getUsername();
+      console.log("Username found:", username);
+
+      if (username && username !== "web-user" &&
+          window.location.search.indexOf("playername=") === -1) {
+        var currentUrl = new URL(window.location.href);
+        currentUrl.searchParams.set("playername", username);
+        console.log("Redirecting to:", currentUrl.href);
+        window.location.href = currentUrl.href;
+      }
+
+      document.addEventListener("DOMContentLoaded", function() {
+        // User block
+        var userBlock = document.createElement("div");
+        userBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:8px;text-align:center;font-size:medium;border-radius:4px;position:absolute;top:10px;right:150px;max-width:200px;";
+        userBlock.innerHTML = "Logged in as: <b>" + username + "</b>";
+        document.body.appendChild(userBlock);
+
+        // Info block
+        var infoBlock = document.createElement("p");
+        infoBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:10px;text-align:center;font-size:large;display:inline-block;border-radius:4px;position:absolute;top:10px;left:150px;";
+        infoBlock.innerHTML = 'GEYMERSKIY SOYUZ Server<br>'
+          + 'Get <a href="https://github.com/PrismLauncher/PrismLauncher/releases/tag/8.4">Prism Launcher</a> '
+          + 'and <a href="/clients/1.12.2.zip">client.zip</a> for this server. '
+          + 'Server address <b>minecraft.hexor.cy:30565</b><br>'
+          + 'Requires <a href="https://www.java.com/en/download/manual.jsp">Java 8</a><br><br>'
+          + '<a href="#" id="showInstallBtn" style="color:black;text-decoration:underline;">Windows Install Script</a>';
+        document.body.appendChild(infoBlock);
+
+        // Modal
+        var modal = document.createElement("div");
+        modal.id = "installModal";
+        modal.style.cssText = "display:none;position:fixed;z-index:1000;left:0;top:0;width:100%;height:100%;background-color:rgba(0,0,0,0.5);";
+        modal.innerHTML = '<div style="background-color:#CEC6CB;margin:15% auto;padding:10px;border-radius:4px;width:70%;max-width:500px;text-align:center;color:black;font-size:large;">'
+          + '<h3 style="margin-top:0;color:black;">Windows Installation</h3>'
+          + '<p style="color:black;">Copy and paste this command into PowerShell:</p>'
+          + '<textarea id="scriptCommand" readonly style="width:90%;height:60px;font-family:monospace;padding:8px;border:1px solid #888;border-radius:4px;resize:none;background-color:white;color:black;"></textarea>'
+          + '<br><br>'
+          + '<button id="copyButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;margin-right:10px;font-size:large;">Copy</button>'
+          + '<button id="closeButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;font-size:large;">Close</button>'
+          + '</div>';
+        document.body.appendChild(modal);
+
+        // Generate PowerShell command with username
+        function buildPsCommand() {
+          var d = "$";
+          var q = "'";
+          return d + 'f="' + d + 'env:TEMP\\mc-install.ps1"; iwr -useb https://minecraft.hexor.cy/clients/win-install.ps1 -OutFile '
+            + d + 'f; powershell -ExecutionPolicy Bypass -File ' + d + 'f -Username ' + q + username + q + '; Remove-Item ' + d + 'f';
+        }
+
+        document.getElementById("showInstallBtn").addEventListener("click", function(e) {
+          e.preventDefault();
+          modal.style.display = "block";
+          document.getElementById("scriptCommand").value = buildPsCommand();
+        });
+
+        document.getElementById("closeButton").addEventListener("click", function() {
+          modal.style.display = "none";
+        });
+
+        document.getElementById("copyButton").addEventListener("click", function() {
+          var textarea = document.getElementById("scriptCommand");
+          textarea.select();
+          textarea.setSelectionRange(0, 99999);
+          document.execCommand("copy");
+          var btn = document.getElementById("copyButton");
+          btn.style.borderColor = "#4CAF50";
+          setTimeout(function() { btn.style.borderColor = "#888"; }, 2000);
+        });
+
+        modal.addEventListener("click", function(event) {
+          if (event.target === modal) {
+            modal.style.display = "none";
+          }
+        });
+      });
+    })();
+
+  win-install.ps1: |
+    # Game Setup Script for PrismLauncher and Minecraft Client
+    # This script downloads and configures PrismLauncher with Hexor client
+
+    param(
+        [string]$Username = "",
+        [string]$InstallPath = "$env:USERPROFILE\Games\PrismLauncher"
+    )
+
+    # Enable TLS 1.2 for downloads
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+    # Function to write colored log messages
+    function Write-Log {
+        param(
+            [string]$Message,
+            [string]$Level = "INFO"
+        )
+
+        switch ($Level) {
+            "INFO" {
+                Write-Host "[" -NoNewline
+                Write-Host "INFO" -ForegroundColor Blue -NoNewline
+                Write-Host "] $Message"
+            }
+            "WARN" {
+                Write-Host "[" -NoNewline
+                Write-Host "WARN" -ForegroundColor Yellow -NoNewline
+                Write-Host "] $Message"
+            }
+            "ERROR" {
+                Write-Host "[" -NoNewline
+                Write-Host "ERROR" -ForegroundColor Red -NoNewline
+                Write-Host "] $Message"
+            }
+            "SUCCESS" {
+                Write-Host "[" -NoNewline
+                Write-Host "SUCCESS" -ForegroundColor Green -NoNewline
+                Write-Host "] $Message"
+            }
+        }
+    }
+
+    # Function to download file with progress
+    function Download-File {
+        param(
+            [string]$Url,
+            [string]$OutputPath
+        )
+
+        try {
+            Write-Log "Downloading from: $Url"
+            Write-Log "Saving to: $OutputPath"
+
+            if (Test-Path $OutputPath) {
+                Remove-Item $OutputPath -Force
+                Write-Log "Removed existing file to avoid corruption" "WARN"
+            }
+
+            $webClient = New-Object System.Net.WebClient
+            $webClient.DownloadFile($Url, $OutputPath)
+
+            if (Test-Path $OutputPath) {
+                $fileSize = (Get-Item $OutputPath).Length
+                Write-Log "Download completed successfully ($fileSize bytes)" "SUCCESS"
+                return $true
+            } else {
+                Write-Log "Download failed - file not found" "ERROR"
+                return $false
+            }
+        }
+        catch {
+            Write-Log "Download failed: $($_.Exception.Message)" "ERROR"
+            return $false
+        }
+    }
+
+    # Function to extract ZIP archive
+    function Extract-Archive {
+        param(
+            [string]$ArchivePath,
+            [string]$DestinationPath
+        )
+
+        try {
+            Write-Log "Extracting archive: $ArchivePath"
+            Write-Log "Destination: $DestinationPath"
+
+            if (!(Test-Path $ArchivePath)) {
+                Write-Log "Archive file not found: $ArchivePath" "ERROR"
+                return $false
+            }
+
+            $fileSize = (Get-Item $ArchivePath).Length
+            if ($fileSize -lt 1000) {
+                Write-Log "Archive file too small ($fileSize bytes), probably corrupted" "ERROR"
+                return $false
+            }
+
+            if (!(Test-Path $DestinationPath)) {
+                New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null
+            }
+
+            $existingFiles = Get-ChildItem -Path $DestinationPath -File | Where-Object { $_.Extension -ne ".zip" }
+            $existingDirs = Get-ChildItem -Path $DestinationPath -Directory
+
+            if ($existingFiles.Count -gt 0 -or $existingDirs.Count -gt 0) {
+                Write-Log "Clearing existing files in destination directory" "WARN"
+                $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
+                $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
+            }
+
+            Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+            try {
+                $zip = [System.IO.Compression.ZipFile]::OpenRead($ArchivePath)
+                $entryCount = $zip.Entries.Count
+                $zip.Dispose()
+                Write-Log "ZIP file validated ($entryCount entries)" "SUCCESS"
+            }
+            catch {
+                Write-Log "ZIP file validation failed: $($_.Exception.Message)" "ERROR"
+                return $false
+            }
+
+            [System.IO.Compression.ZipFile]::ExtractToDirectory($ArchivePath, $DestinationPath)
+
+            Write-Log "Archive extracted successfully" "SUCCESS"
+            return $true
+        }
+        catch {
+            Write-Log "Archive extraction failed: $($_.Exception.Message)" "ERROR"
+
+            if (Test-Path $ArchivePath) {
+                Write-Log "Removing potentially corrupted archive file" "WARN"
+                Remove-Item $ArchivePath -Force -ErrorAction SilentlyContinue
+            }
+
+            return $false
+        }
+    }
+
+    # Function to check Java installation
+    function Check-JavaInstallation {
+        try {
+            $javaVersion = java -version 2>&1 | Select-String "version"
+            if ($javaVersion) {
+                Write-Log "Java is installed: $($javaVersion.ToString().Trim())" "SUCCESS"
+                return $true
+            }
+        }
+        catch {
+            Write-Log "Java is not installed or not in PATH" "WARN"
+            Write-Log "Please download Java from: https://www.java.com/en/download/manual.jsp" "WARN"
+            Write-Log "Look for 'Windows Offline (64-bit)' version" "WARN"
+
+            $response = Read-Host "Do you want to continue without Java? (y/n)"
+            if ($response -eq 'y' -or $response -eq 'Y') {
+                Write-Log "Continuing without Java verification" "WARN"
+                return $true
+            } else {
+                Write-Log "Installation cancelled. Please install Java first." "ERROR"
+                return $false
+            }
+        }
+    }
+
+    # Function to generate a random hex string (UUID-like without dashes)
+    function New-RandomHexId {
+        param([int]$Length = 32)
+        $bytes = New-Object byte[] ($Length / 2)
+        $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
+        $rng.GetBytes($bytes)
+        $rng.Dispose()
+        return ($bytes | ForEach-Object { $_.ToString("x2") }) -join ''
+    }
+
+    # Function to create accounts.json file with offline account
+    function Create-AccountsFile {
+        param(
+            [string]$TargetPath,
+            [string]$PlayerName
+        )
+
+        $accountsPath = Join-Path $TargetPath "accounts.json"
+        $profileId = New-RandomHexId -Length 32
+        $clientToken = New-RandomHexId -Length 32
+        $iat = [int][double]::Parse((Get-Date -UFormat %s))
+
+        $accountsObj = @{
+            accounts = @(
+                @{
+                    entitlement = @{
+                        canPlayMinecraft = $true
+                        ownsMinecraft = $true
+                    }
+                    "msa-client-id" = ""
+                    type = "MSA"
+                },
+                @{
+                    active = $true
+                    entitlement = @{
+                        canPlayMinecraft = $true
+                        ownsMinecraft = $true
+                    }
+                    profile = @{
+                        capes = @()
+                        id = $profileId
+                        name = $PlayerName
+                        skin = @{
+                            id = ""
+                            url = ""
+                            variant = ""
+                        }
+                    }
+                    type = "Offline"
+                    ygg = @{
+                        extra = @{
+                            clientToken = $clientToken
+                            userName = $PlayerName
+                        }
+                        iat = $iat
+                        token = "0"
+                    }
+                }
+            )
+            formatVersion = 3
+        }
+
+        try {
+            $accountsObj | ConvertTo-Json -Depth 10 | Out-File -FilePath $accountsPath -Encoding UTF8
+            Write-Log "Created accounts.json at: $accountsPath" "SUCCESS"
+            Write-Log "Player name: $PlayerName" "SUCCESS"
+            return $true
+        }
+        catch {
+            Write-Log "Failed to create accounts.json: $($_.Exception.Message)" "ERROR"
+            return $false
+        }
+    }
+
+    # Main installation process
+    Write-Log "Starting PrismLauncher and Minecraft client setup"
+    Write-Log "Player name: $Username"
+    Write-Log "Installation path: $InstallPath"
+
+    # Create installation directory and clear if exists (but keep ZIP files)
+    if (Test-Path $InstallPath) {
+        Write-Log "Installation directory exists, clearing contents" "WARN"
+        $existingFiles = Get-ChildItem -Path $InstallPath -File | Where-Object { $_.Extension -ne ".zip" }
+        $existingDirs = Get-ChildItem -Path $InstallPath -Directory
+
+        $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
+        $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
+    } else {
+        New-Item -ItemType Directory -Path $InstallPath -Force | Out-Null
+        Write-Log "Created installation directory: $InstallPath"
+    }
+
+    # Step 1: Download PrismLauncher
+    Write-Log "Step 1: Downloading PrismLauncher..."
+    $launcherUrl = "https://github.com/PrismLauncher/PrismLauncher/releases/download/8.4/PrismLauncher-Windows-MSVC-Portable-8.4.zip"
+    $launcherZip = Join-Path $InstallPath "PrismLauncher-8.4.zip"
+    $launcherExtractPath = Join-Path $InstallPath "PrismLauncher-Windows-MSVC-Portable-8.4"
+
+    if (!(Download-File -Url $launcherUrl -OutputPath $launcherZip)) {
+        Write-Log "Failed to download PrismLauncher. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 2: Extract PrismLauncher
+    Write-Log "Step 2: Extracting PrismLauncher..."
+    if (!(Extract-Archive -ArchivePath $launcherZip -DestinationPath $InstallPath)) {
+        Write-Log "Failed to extract PrismLauncher. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Find the actual extracted directory
+    $extractedDirs = Get-ChildItem -Path $InstallPath -Directory | Where-Object { $_.Name -like "*PrismLauncher*" }
+    if ($extractedDirs.Count -gt 0) {
+        $launcherExtractPath = $extractedDirs[0].FullName
+        Write-Log "Found PrismLauncher directory: $launcherExtractPath" "SUCCESS"
+    } else {
+        Write-Log "Could not find extracted PrismLauncher directory. Checking for direct extraction..." "WARN"
+        $prismExe = Join-Path $InstallPath "prismlauncher.exe"
+        if (Test-Path $prismExe) {
+            $launcherExtractPath = $InstallPath
+            Write-Log "PrismLauncher extracted directly to: $launcherExtractPath" "SUCCESS"
+        } else {
+            Write-Log "Failed to locate PrismLauncher files. Exiting." "ERROR"
+            exit 1
+        }
+    }
+
+    # Step 3: Create accounts.json in InstallPath (only if username provided)
+    if ($Username -and $Username -ne "web-user") {
+        Write-Log "Step 3: Creating accounts configuration for $Username..."
+        if (!(Create-AccountsFile -TargetPath $InstallPath -PlayerName $Username)) {
+            Write-Log "Failed to create accounts.json. Exiting." "ERROR"
+            exit 1
+        }
+    } else {
+        Write-Log "Step 3: No username provided, skipping accounts.json creation" "WARN"
+    }
+
+    # Step 4: Check Java installation
+    Write-Log "Step 4: Checking Java installation..."
+    if (!(Check-JavaInstallation)) {
+        Write-Log "Java check failed. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 5: Download Minecraft client
+    Write-Log "Step 5: Downloading Minecraft client..."
+    $minecraftUrl = "https://minecraft.hexor.cy/clients/1.12.2.zip"
+    $minecraftZip = Join-Path $InstallPath "minecraft-1.12.2.zip"
+    $instancesPath = Join-Path $launcherExtractPath "instances"
+    $hexorPath = Join-Path $instancesPath "Hexor"
+
+    if (!(Download-File -Url $minecraftUrl -OutputPath $minecraftZip)) {
+        Write-Log "Failed to download Minecraft client. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 6: Create instances directory and extract Minecraft client
+    Write-Log "Step 6: Setting up Minecraft client..."
+    if (!(Test-Path $instancesPath)) {
+        New-Item -ItemType Directory -Path $instancesPath -Force | Out-Null
+        Write-Log "Created instances directory"
+    }
+
+    if (!(Test-Path $hexorPath)) {
+        New-Item -ItemType Directory -Path $hexorPath -Force | Out-Null
+        Write-Log "Created Hexor instance directory"
+    }
+
+    if (!(Extract-Archive -ArchivePath $minecraftZip -DestinationPath $hexorPath)) {
+        Write-Log "Failed to extract Minecraft client. Exiting." "ERROR"
+        exit 1
+    }
+
+    # Step 7: Cleanup temporary files
+    Write-Log "Step 7: Cleaning up temporary files..."
+    try {
+        Remove-Item $launcherZip -Force
+        Remove-Item $minecraftZip -Force
+        Write-Log "Temporary files cleaned up" "SUCCESS"
+    }
+    catch {
+        Write-Log "Could not remove temporary files: $($_.Exception.Message)" "WARN"
+    }
+
+    # Final success message
+    Write-Log "=== INSTALLATION COMPLETED SUCCESSFULLY ===" "SUCCESS"
+    Write-Log "PrismLauncher location: $launcherExtractPath" "SUCCESS"
+    Write-Log "Executable: $(Join-Path $launcherExtractPath 'prismlauncher.exe')" "SUCCESS"
+    Write-Log "Minecraft client installed in: $hexorPath" "SUCCESS"
+    if ($Username -and $Username -ne "web-user") {
+        Write-Log "Player name configured: $Username" "SUCCESS"
+        Write-Log "Accounts file: $(Join-Path $InstallPath 'accounts.json')" "SUCCESS"
+    }
+    Write-Log ""
+    Write-Log "You can now run PrismLauncher and the Hexor instance should be available!"
+    Write-Log "If Java was not installed, please download it from: https://www.java.com/en/download/manual.jsp"
+
+    # Ask if user wants to launch the game
+    $launchResponse = Read-Host "Do you want to launch PrismLauncher now? (y/n)"
+    if ($launchResponse -eq 'y' -or $launchResponse -eq 'Y') {
+        $launcherExe = Join-Path $launcherExtractPath "prismlauncher.exe"
+        if (Test-Path $launcherExe) {
+            Write-Log "Launching PrismLauncher..." "INFO"
+            Start-Process -FilePath $launcherExe -WorkingDirectory $launcherExtractPath
+        } else {
+            Write-Log "Launcher executable not found!" "ERROR"
+        }
+    }

k8s-ng/apps/minecraft/deployments.yaml

@@ -0,0 +1,137 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minecraft
+  namespace: minecraft
+  labels:
+    app: minecraft
+spec:
+  selector:
+    matchLabels:
+      app: minecraft
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: minecraft
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      volumes:
+      - name: storage
+        hostPath:
+          path: /data/k8s/mc-server/
+          type: Directory
+      - name: nginx-config
+        configMap:
+          name: nginx-config
+      - name: client-scripts
+        configMap:
+          name: client-scripts
+
+      terminationGracePeriodSeconds: 180
+      containers:
+        - name: minecraft
+          image: eclipse-temurin:8-jdk-ubi10-minimal@sha256:85c943a7093a276b0f6ae13dee2ffecb98e390ab6a32fc7e4b8b1fc6bb103dd7
+          command: ["java"]
+          args:
+            - -Xms4G
+            - -Xmx4G
+            - -XX:+UseG1GC
+            - -XX:+ParallelRefProcEnabled
+            - -XX:MaxGCPauseMillis=200
+            - -XX:+UnlockExperimentalVMOptions
+            - -XX:+DisableExplicitGC
+            - -XX:+AlwaysPreTouch
+            - -XX:G1NewSizePercent=30
+            - -XX:G1MaxNewSizePercent=40
+            - -XX:G1HeapRegionSize=8M
+            - -XX:G1ReservePercent=20
+            - -XX:G1HeapWastePercent=5
+            - -XX:G1MixedGCCountTarget=4
+            - -XX:InitiatingHeapOccupancyPercent=15
+            - -XX:G1MixedGCLiveThresholdPercent=90
+            - -XX:G1RSetUpdatingPauseTimePercent=5
+            - -XX:SurvivorRatio=32
+            - -XX:+PerfDisableSharedMem
+            - -XX:MaxTenuringThreshold=1
+            - -jar
+            - forge-1.12.2-14.23.5.2854.jar
+            - nogui
+          workingDir: /mc/
+          resources:
+            requests:
+              memory: "8Gi"
+              cpu: "2000m"
+            limits:
+              memory: "12Gi"
+              cpu: "4000m"
+          ports:
+            - name: game
+              containerPort: 25565
+              protocol: TCP
+            - name: dynmap
+              containerPort: 8123
+              protocol: TCP
+            - name: webstatus-mod
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 10
+            failureThreshold: 90
+          livenessProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            tcpSocket:
+              port: game
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: storage
+              mountPath: /mc
+        - name: nginx
+          image: nginx@sha256:f9ea18bfa4fad859e1ed38259d711da7ccad2c3516e875cec3351a57c859f571
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "256Mi"
+              cpu: "200m"
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: nginx-config
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
+            - name: storage
+              mountPath: /mc
+            - name: client-scripts
+              mountPath: /mc/clients/win-install.ps1
+              subPath: win-install.ps1
+            - name: client-scripts
+              mountPath: /mc/clients/inject.js
+              subPath: inject.js
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-exporter
+  namespace: minecraft
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 19565
+      targetPort: 19565

k8s-ng/apps/minecraft/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: minecraft
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: minecraft.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: minecraft-dynmap
+                port:
+                  number: 80
+  tls:
+    - secretName: minecraft-tls
+      hosts:
+        - minecraft.hexor.cy

k8s-ng/apps/minecraft/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: minecraft
+resources:
+  - deployments.yaml
+  - configmaps.yaml
+  - services.yaml
+  - ingress.yaml

k8s-ng/apps/minecraft/services.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-dynmap
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-webstatus-mod
+spec:
+  selector:
+    app: minecraft
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minecraft-game
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    - port: 30565
+      targetPort: 25565
+  selector:
+    app: minecraft

k8s-ng/apps/mtproxy/README.md

@@ -0,0 +1,23 @@
+# MTProxy
+
+Argo CD deploys this directory through the `mtproxy` entry in
+`core-state/values.yaml`. Two telemt DaemonSets run on every node labeled
+`homelab.hexor.cy/exposure=external`: classic MTProxy on 30443/TCP and
+fake-TLS on 30444/TCP. The classic listener replaces the old amd64-only
+`telegrammessenger/proxy` image so the arm64 `spb` node is covered.
+
+Vaultwarden item `k8s/mtproxy/secret` must contain a custom field named
+`Secret` with the existing 32-character hexadecimal MTProxy secret. External
+Secrets renders both configurations from that field. Reloader restarts the
+DaemonSets when it changes. Changing this value invalidates existing links.
+
+`mtproxy-link-reconciler` lists external nodes every five minutes and creates
+`mtproxy-links` and `telemt-links` Secrets. A node's public DNS name must be
+`<node>.hexor.cy` and must point to that node. The reader at `proxy.hexor.cy`
+has read-only access to those two Secrets and is protected by Keycloak's
+forward auth; its access policy is in `terraform/keycloak/terraform.tfvars`.
+The proxy Pods have no Kubernetes API access.
+
+Before traffic is switched over, allow inbound 30443/TCP and 30444/TCP on
+every external node and verify both DNS and connectivity on each node.
+Neither the cluster nor host firewall is changed by these manifests.

k8s-ng/apps/mtproxy/daemonsets.yaml

@@ -0,0 +1,149 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: mtproxy-classic
+  labels:
+    app: mtproxy-classic
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-classic
+  template:
+    metadata:
+      labels:
+        app: mtproxy-classic
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: telemt
+          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
+          imagePullPolicy: IfNotPresent
+          workingDir: /run/telemt
+          args: ["/app/config.toml"]
+          ports:
+            - name: proxy
+              containerPort: 30443
+              hostPort: 30443
+              protocol: TCP
+            - name: metrics
+              containerPort: 9090
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: proxy
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /app/config.toml
+              subPath: classic.toml
+              readOnly: true
+            - name: workdir
+              mountPath: /run/telemt
+      volumes:
+        - name: config
+          secret:
+            secretName: mtproxy-config
+            defaultMode: 0444
+            items:
+              - key: classic.toml
+                path: classic.toml
+        - name: workdir
+          emptyDir:
+            medium: Memory
+            sizeLimit: 4Mi
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: mtproxy-tls
+  labels:
+    app: mtproxy-tls
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-tls
+  template:
+    metadata:
+      labels:
+        app: mtproxy-tls
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: telemt
+          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
+          imagePullPolicy: IfNotPresent
+          workingDir: /run/telemt
+          args: ["/app/config.toml"]
+          ports:
+            - name: proxy
+              containerPort: 30444
+              hostPort: 30444
+              protocol: TCP
+            - name: metrics
+              containerPort: 9090
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: proxy
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /app/config.toml
+              subPath: tls.toml
+              readOnly: true
+            - name: workdir
+              mountPath: /run/telemt
+      volumes:
+        - name: config
+          secret:
+            secretName: mtproxy-config
+            defaultMode: 0444
+            items:
+              - key: tls.toml
+                path: tls.toml
+        - name: workdir
+          emptyDir:
+            medium: Memory
+            sizeLimit: 4Mi

k8s-ng/apps/mtproxy/external-secrets.yaml

@@ -0,0 +1,74 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: mtproxy-config
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: mtproxy-config
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        SECRET: "{{ .secret }}"
+        classic.toml: |
+          [general]
+          use_middle_proxy = true
+          log_level = "normal"
+
+          [general.modes]
+          classic = true
+          secure = false
+          tls = false
+
+          [server]
+          port = 30443
+          metrics_port = 9090
+          metrics_whitelist = ["0.0.0.0/0"]
+
+          [server.api]
+          enabled = false
+
+          [[server.listeners]]
+          ip = "0.0.0.0"
+
+          [access.users]
+          user = "{{ .secret }}"
+        tls.toml: |
+          [general]
+          use_middle_proxy = true
+          log_level = "normal"
+
+          [general.modes]
+          classic = false
+          secure = false
+          tls = true
+
+          [server]
+          port = 30444
+          metrics_port = 9090
+          metrics_whitelist = ["0.0.0.0/0"]
+
+          [server.api]
+          enabled = false
+
+          [[server.listeners]]
+          ip = "0.0.0.0"
+
+          [censorship]
+          tls_domain = "ya.ru"
+          mask = true
+          tls_emulation = true
+          tls_front_dir = "tlsfront"
+
+          [access.users]
+          user = "{{ .secret }}"
+  data:
+    - secretKey: secret
+      remoteRef:
+        key: k8s/mtproxy/secret
+        property: Secret

k8s-ng/apps/mtproxy/ingress.yaml

@@ -0,0 +1,38 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: mtproxy-auth
+spec:
+  forwardAuth:
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: false
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: mtproxy-links
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`proxy.hexor.cy`)
+      kind: Rule
+      middlewares:
+        - name: mtproxy-auth
+      services:
+        - name: mtproxy-link-reader
+          port: 80
+  tls:
+    secretName: mtproxy-links-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mtproxy-links-tls
+spec:
+  secretName: mtproxy-links-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - proxy.hexor.cy

k8s-ng/apps/mtproxy/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: mtproxy
+resources:
+  - external-secrets.yaml
+  - daemonsets.yaml
+  - metrics.yaml
+  - links.yaml
+  - reader.yaml
+  - ingress.yaml
+configMapGenerator:
+  - name: mtproxy-link-reconciler
+    files:
+      - reconcile.py=link-reconciler.py
+generatorOptions:
+  disableNameSuffixHash: true

k8s-ng/apps/mtproxy/link-reconciler.py

@@ -0,0 +1,75 @@
+"""Publish client links for every external node, using its public DNS name."""
+
+import base64
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+API = "https://kubernetes.default.svc"
+NAMESPACE = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", encoding="utf-8").read().strip()
+TOKEN = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
+CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+
+def request(method, path, body=None):
+    import ssl
+
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(
+        API + path,
+        data=data,
+        method=method,
+        headers={
+            "Authorization": "Bearer " + TOKEN,
+            "Content-Type": "application/json",
+        },
+    )
+    with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=15) as response:
+        return json.load(response)
+
+
+def publish(name, links):
+    path = f"/api/v1/namespaces/{NAMESPACE}/secrets"
+    data = {node: base64.b64encode(link.encode()).decode() for node, link in links.items()}
+    try:
+        current = request("GET", f"{path}/{name}")
+    except urllib.error.HTTPError as error:
+        if error.code != 404:
+            raise
+        request("POST", path, {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, "type": "Opaque", "data": data})
+        return
+    if current.get("data", {}) == data:
+        return
+    # PUT replaces the data map, removing links for nodes that are no longer external.
+    request("PUT", f"{path}/{name}", {
+        "apiVersion": "v1",
+        "kind": "Secret",
+        "metadata": {"name": name, "resourceVersion": current["metadata"]["resourceVersion"]},
+        "type": "Opaque",
+        "data": data,
+    })
+
+
+secret = open("/config/SECRET", encoding="utf-8").read().strip()
+if len(secret) != 32 or any(char not in "0123456789abcdefABCDEF" for char in secret):
+    raise ValueError("Vaultwarden Secret must be 32 hexadecimal characters")
+
+selector = urllib.parse.quote("homelab.hexor.cy/exposure=external", safe="")
+nodes = request("GET", f"/api/v1/nodes?labelSelector={selector}")["items"]
+if not nodes:
+    raise ValueError("No external nodes found")
+
+classic = {}
+tls = {}
+for node in nodes:
+    name = node["metadata"]["name"]
+    # DNS is maintained outside the cluster; no public IP needs to be stored on a Node.
+    host = f"{name}.hexor.cy"
+    classic[name] = f"tg://proxy?server={host}&port=30443&secret={secret}"
+    tls[name] = f"tg://proxy?server={host}&port=30444&secret=ee{secret}79612e7275"
+
+publish("mtproxy-links", classic)
+publish("telemt-links", tls)
+print(f"Published MTProxy links for {len(nodes)} external nodes")

k8s-ng/apps/mtproxy/links.yaml

@@ -0,0 +1,108 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mtproxy-link-reconciler
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mtproxy-external-node-reader
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: mtproxy-external-node-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mtproxy-external-node-reader
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reconciler
+    namespace: mtproxy
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mtproxy-link-writer
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mtproxy-links", "telemt-links"]
+    verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mtproxy-link-writer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mtproxy-link-writer
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reconciler
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: mtproxy-link-reconciler
+spec:
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          serviceAccountName: mtproxy-link-reconciler
+          restartPolicy: OnFailure
+          nodeSelector:
+            homelab.hexor.cy/exposure: external
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: reconcile
+              image: python:3.13-alpine3.22@sha256:e81548ac35b07a3bd4805f275107592ef458b1e893c0e04d45aedaa19416cca5
+              command: ["python", "/scripts/reconcile.py"]
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 32Mi
+                limits:
+                  memory: 128Mi
+              securityContext:
+                readOnlyRootFilesystem: true
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: scripts
+                  mountPath: /scripts
+                  readOnly: true
+                - name: config
+                  mountPath: /config
+                  readOnly: true
+          volumes:
+            - name: scripts
+              configMap:
+                name: mtproxy-link-reconciler
+            - name: config
+              secret:
+                secretName: mtproxy-config
+                items:
+                  - key: SECRET
+                    path: SECRET

k8s-ng/apps/mtproxy/metrics.yaml

@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-classic-metrics
+  labels:
+    app: mtproxy-classic
+spec:
+  clusterIP: None
+  selector:
+    app: mtproxy-classic
+  ports:
+    - name: metrics
+      port: 9090
+      targetPort: metrics
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-tls-metrics
+  labels:
+    app: mtproxy-tls
+spec:
+  clusterIP: None
+  selector:
+    app: mtproxy-tls
+  ports:
+    - name: metrics
+      port: 9090
+      targetPort: metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: mtproxy-classic
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-classic
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      relabelings:
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: mtproxy-tls
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-tls
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      relabelings:
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node

k8s-ng/apps/mtproxy/reader.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mtproxy-link-reader
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mtproxy-link-reader
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mtproxy-links", "telemt-links"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mtproxy-link-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mtproxy-link-reader
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reader
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mtproxy-link-reader
+  labels:
+    app: mtproxy-link-reader
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mtproxy-link-reader
+  template:
+    metadata:
+      labels:
+        app: mtproxy-link-reader
+    spec:
+      serviceAccountName: mtproxy-link-reader
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: reader
+          image: ultradesu/k8s-secrets:0.2.2@sha256:f206942d21ca9dcd131d6e19ba26cc17671815919217aef657b30a9f85854778
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secrets
+            - mtproxy-links,telemt-links
+            - --namespace
+            - mtproxy
+            - --port
+            - "3000"
+          ports:
+            - name: http
+              containerPort: 3000
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 128Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-link-reader
+spec:
+  selector:
+    app: mtproxy-link-reader
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

k8s-ng/apps/rustdesk/deployment.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rustdesk-hbbs
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: rustdesk-hbbs
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rustdesk-hbbs
+  template:
+    metadata:
+      labels:
+        app: rustdesk-hbbs
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      containers:
+      - name: hbbs
+        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "750m"
+        command: ["hbbs"]
+        args:
+          - "--relay-servers"
+          - "rd.hexor.cy:21117"
+          - "--port"
+          - "21116"
+        ports:
+        - name: registry
+          containerPort: 21116
+          protocol: TCP
+        - name: registry-udp
+          containerPort: 21116
+          protocol: UDP
+        - name: nat
+          containerPort: 21115
+          protocol: TCP
+        readinessProbe:
+          tcpSocket:
+            port: registry
+        volumeMounts:
+        - name: keys
+          mountPath: /data
+          readOnly: true
+        - name: data
+          mountPath: /data-persistent
+        env:
+        - name: RUST_LOG
+          value: "info"
+        - name: DB_URL
+          value: "/data-persistent/db_v2.sqlite3"
+        workingDir: /data
+      volumes:
+      - name: keys
+        secret:
+          secretName: rustdesk-keys
+          items:
+          - key: id_ed25519
+            path: id_ed25519
+            mode: 0600
+          - key: id_ed25519.pub
+            path: id_ed25519.pub
+            mode: 0644
+      - name: data
+        hostPath:
+          path: /data/k8s/rustdesk/hbbs
+          type: Directory
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rustdesk-hbbr
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: rustdesk-hbbr
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rustdesk-hbbr
+  template:
+    metadata:
+      labels:
+        app: rustdesk-hbbr
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: music
+      containers:
+      - name: hbbr
+        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "750m"
+        command: ["hbbr"]
+        args:
+          - "--port"
+          - "21117"
+        ports:
+        - name: relay
+          containerPort: 21117
+          protocol: TCP
+        readinessProbe:
+          tcpSocket:
+            port: relay
+        volumeMounts:
+        - name: keys
+          mountPath: /data
+          readOnly: true
+        - name: data
+          mountPath: /data-persistent
+        env:
+        - name: RUST_LOG
+          value: "info"
+        workingDir: /data
+      volumes:
+      - name: keys
+        secret:
+          secretName: rustdesk-keys
+          items:
+          - key: id_ed25519
+            path: id_ed25519
+            mode: 0600
+          - key: id_ed25519.pub
+            path: id_ed25519.pub
+            mode: 0644
+      - name: data
+        hostPath:
+          path: /data/k8s/rustdesk/hbbr
+          type: Directory

k8s-ng/apps/rustdesk/external-secrets.yaml

@@ -0,0 +1,27 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: rustdesk-keys
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: rustdesk-keys
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        id_ed25519: "{{ .private_key }}"
+        id_ed25519.pub: "{{ .public_key }}"
+  data:
+    - secretKey: private_key
+      remoteRef:
+        key: k8s/rustdesk/secrets
+        property: Cert
+    - secretKey: public_key
+      remoteRef:
+        key: k8s/rustdesk/secrets
+        property: Key

k8s-ng/apps/rustdesk/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: rustdesk
+resources:
+  - external-secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - network-policy.yaml

k8s-ng/apps/rustdesk/network-policy.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: rustdesk-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: rustdesk-hbbs
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  # Allow all incoming connections to RustDesk ports
+  - from: []
+    ports:
+    - protocol: TCP
+      port: 21115
+    - protocol: TCP
+      port: 21116
+    - protocol: UDP
+      port: 21116
+  egress:
+  # Allow DNS
+  - to: []
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53
+  # Allow communication between HBBS and HBBR
+  - to:
+    - podSelector:
+        matchLabels:
+          app: rustdesk-hbbr
+    ports:
+    - protocol: TCP
+      port: 21117
+
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: rustdesk-hbbr-network-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: rustdesk-hbbr
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  # Allow all incoming connections to relay port
+  - from: []
+    ports:
+    - protocol: TCP
+      port: 21117
+  # Allow connections from HBBS
+  - from:
+    - podSelector:
+        matchLabels:
+          app: rustdesk-hbbs
+    ports:
+    - protocol: TCP
+      port: 21117
+  egress:
+  # Allow DNS
+  - to: []
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53

k8s-ng/apps/rustdesk/service.yaml

@@ -0,0 +1,59 @@
+# Preserve client source IPs for RustDesk NAT detection. Public DNS points to music,
+# where both servers and their persistent data live.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbs
+  labels:
+    app: rustdesk-hbbs
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbs
+  ports:
+  - name: registry-tcp
+    port: 21116
+    targetPort: 21116
+    protocol: TCP
+  - name: nat
+    port: 21115
+    targetPort: 21115
+    protocol: TCP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbs-udp
+  labels:
+    app: rustdesk-hbbs
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbs
+  ports:
+  - name: registry-udp
+    port: 21116
+    targetPort: 21116
+    protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustdesk-hbbr
+  labels:
+    app: rustdesk-hbbr
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app: rustdesk-hbbr
+  ports:
+  - name: relay
+    port: 21117
+    targetPort: 21117
+    protocol: TCP

k8s-ng/apps/shadowsocks/README.md

@@ -0,0 +1,18 @@
+# Shadowsocks on spb
+
+One shadowsocks-rust pod runs on `spb`, exposing `8388/TCP` and `8388/UDP`
+through host ports. Client settings:
+
+- Server: `spb.hexor.cy`
+- Port: `8388`
+- Method: `chacha20-ietf-poly1305`
+- Password: the `Password` custom field of the Vaultwarden item named
+  `k8s/shadowsocks/secrets` (exact name, not an item UUID).
+
+Create that item in the server secrets vault accessible to the External Secrets
+bot. Use a long random password, for example generated with `openssl rand -base64 32`.
+External Secrets creates `shadowsocks-config`; Reloader restarts the deployment
+when the Secret changes. No persistent storage is required.
+
+The spb NixOS configuration opens port 8388 for both protocols. Apply that
+configuration when deploying this application for the first time.

k8s-ng/apps/shadowsocks/deployment.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shadowsocks
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: shadowsocks
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: shadowsocks
+  template:
+    metadata:
+      labels:
+        app: shadowsocks
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: spb
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        runAsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: shadowsocks
+          # shadowsocks-rust v1.25.0; multi-arch image includes linux/arm64.
+          image: ghcr.io/shadowsocks/ssserver-rust:v1.25.0@sha256:22240c07d8ce3a36421ede770bd775dcc5fc56d5d829943196957b05452dc36f
+          imagePullPolicy: IfNotPresent
+          command: ["/usr/bin/ssserver"]
+          args:
+            - --log-without-time
+            - -c
+            - /etc/shadowsocks-rust/config.json
+          ports:
+            - name: proxy-tcp
+              containerPort: 8388
+              hostPort: 8388
+              protocol: TCP
+            - name: proxy-udp
+              containerPort: 8388
+              hostPort: 8388
+              protocol: UDP
+          readinessProbe:
+            tcpSocket:
+              port: proxy-tcp
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /etc/shadowsocks-rust
+              readOnly: true
+      volumes:
+        - name: config
+          secret:
+            secretName: shadowsocks-config
+            defaultMode: 0444

k8s-ng/apps/shadowsocks/external-secrets.yaml

@@ -0,0 +1,29 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: shadowsocks-config
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: shadowsocks-config
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        config.json: |
+          {
+            "server": "0.0.0.0",
+            "server_port": 8388,
+            "method": "chacha20-ietf-poly1305",
+            "password": {{ .password | toJson }},
+            "mode": "tcp_and_udp"
+          }
+  data:
+    - secretKey: password
+      remoteRef:
+        key: k8s/shadowsocks/secrets
+        property: Password

k8s-ng/apps/shadowsocks/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: shadowsocks
+resources:
+  - external-secrets.yaml
+  - deployment.yaml

k8s-ng/apps/web-petting/deployment.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web-petting
+  labels:
+    app: web-petting
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: web-petting
+  template:
+    metadata:
+      labels:
+        app: web-petting
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+      containers:
+        - name: web-petting
+          image: ultradesu/web-petting:latest
+          imagePullPolicy: Always
+          args:
+            - web-petting
+            - -l
+            - 0.0.0.0:3000
+          ports:
+            - name: http
+              containerPort: 3000
+          env:
+            - name: RUST_LOG
+              value: info
+            - name: WEB_PETTING_DEBUG
+              value: "false"
+            - name: WEB_PETTING_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: web-petting-secrets
+                  key: WEB_PETTING_DATABASE_URL
+          # Everything persistent is in PostgreSQL and R2; /data is scratch space only.
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+      volumes:
+        - name: data
+          emptyDir: {}

k8s-ng/apps/web-petting/ingress.yaml

@@ -0,0 +1,37 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: web-petting
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: pet.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: web-petting
+                port:
+                  number: 80
+    - host: xn--l1acako8eb.xn--p1ai
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: web-petting
+                port:
+                  number: 80
+  tls:
+    - secretName: web-petting-tls
+      hosts:
+        - pet.hexor.cy
+    - secretName: web-petting-murnyanya-tls
+      hosts:
+        - xn--l1acako8eb.xn--p1ai

k8s-ng/apps/web-petting/secrets.yaml

@@ -0,0 +1,47 @@
+# Read-only access to the web-petting-db Secret in the psql namespace
+# (see postgresql/databases/web-petting.yaml for the other half).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: db-credentials-reader
+---
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: psql-credentials
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: psql
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          key: ca.crt
+      auth:
+        serviceAccount:
+          name: db-credentials-reader
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: web-petting-secrets
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: psql-credentials
+  target:
+    name: web-petting-secrets
+    template:
+      data:
+        WEB_PETTING_DATABASE_URL: "postgresql://{{ .username }}:{{ .password }}@main-rw.psql.svc:5432/web_petting"
+  data:
+    - secretKey: username
+      remoteRef:
+        key: web-petting-db
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: web-petting-db
+        property: password

k8s-ng/apps/web-petting/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-petting
+  labels:
+    app: web-petting
+spec:
+  type: ClusterIP
+  selector:
+    app: web-petting
+  ports:
+    - name: http
+      port: 80
+      targetPort: 3000
+      protocol: TCP

k8s-ng/core-state/templates/applications.yaml

@@ -23,6 +23,9 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ServerSideApply=true
+      {{- if and (hasKey . "validate") (not .validate) }}
+      - Validate=false
+      {{- end }}
       {{- if .skipDryRunOnMissingResource }}
       - SkipDryRunOnMissingResource=true
       {{- end }}

k8s-ng/core-state/values.yaml

@@ -24,10 +24,19 @@ apps:
     path: k8s-ng/core/traefik-config
     namespace: kube-system
     wave: 1
+  - name: coredns-config
+    path: k8s-ng/core/coredns-config
+    namespace: kube-system
+    wave: 1
+    validate: false # partial Deployment: only the fields we manage
   - name: reloader
     path: k8s-ng/core/reloader
     namespace: reloader
     wave: 1
+  - name: gpu-system
+    path: k8s-ng/core/gpu
+    namespace: gpu-system
+    wave: 1
   - name: cert-manager
     path: k8s-ng/core/cert-manager
     namespace: cert-manager
@@ -53,13 +62,73 @@ apps:
     namespace: keycloak
     wave: 3
     skipDryRunOnMissingResource: true # ExternalSecret/Certificate CRDs come from earlier waves
+  # Not a core service: lives in k8s-ng/apps and belongs to the "apps" project.
   - name: furumi
-    path: k8s-ng/core/furumi
+    project: apps
+    path: k8s-ng/apps/furumi
     namespace: furumi
-    wave: 5 # pinned to the music node; must not hold back the other waves
+    wave: 6 # after prom-stack installs the ServiceMonitor CRD
+    skipDryRunOnMissingResource: true
+  - name: web-petting
+    project: apps
+    path: k8s-ng/apps/web-petting
+    namespace: web-petting
+    wave: 5
+    skipDryRunOnMissingResource: true
+  - name: k8s-secrets
+    project: apps
+    path: k8s-ng/apps/k8s-secrets
+    namespace: k8s-secret
+    wave: 5 # after secret-stores and auth-proxy; independent of prom-stack health
+    skipDryRunOnMissingResource: true
+  - name: llamacpp
+    project: apps
+    path: k8s-ng/apps/llamacpp
+    namespace: llamacpp
+    wave: 5 # pinned to the ai node (GPU); must not hold back the other waves
     skipDryRunOnMissingResource: true
   - name: secret-stores
     path: k8s-ng/core/secret-stores
     namespace: external-secrets
     wave: 4 # reads the Vaultwarden vault, so it comes after vaultwarden
     skipDryRunOnMissingResource: true # ClusterSecretStore CRD comes from the external-secrets app
+  - name: gitea
+    path: k8s-ng/core/gitea
+    namespace: gitea
+    wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node
+    prune: false # never delete the instance because of a manifest change
+    skipDryRunOnMissingResource: true
+  - name: prom-stack
+    path: k8s-ng/core/prom-stack
+    namespace: prometheus
+    wave: 5 # after secret-stores; installs monitoring before application workloads
+    skipDryRunOnMissingResource: true # monitoring, cert-manager and ExternalSecret CRDs
+  - name: immich-db
+    path: k8s-ng/core/immich-db
+    namespace: immich
+    wave: 2
+    prune: false
+    skipDryRunOnMissingResource: true
+  - name: minecraft
+    project: games
+    path: k8s-ng/apps/minecraft
+    namespace: minecraft
+    wave: 6
+  - name: rustdesk
+    project: apps
+    path: k8s-ng/apps/rustdesk
+    namespace: rustdesk
+    wave: 6
+    skipDryRunOnMissingResource: true
+  - name: shadowsocks
+    project: apps
+    path: k8s-ng/apps/shadowsocks
+    namespace: shadowsocks
+    wave: 6
+    skipDryRunOnMissingResource: true
+  - name: mtproxy
+    project: apps
+    path: k8s-ng/apps/mtproxy
+    namespace: mtproxy
+    wave: 6 # after secret-stores and prom-stack
+    skipDryRunOnMissingResource: true

k8s-ng/core/argocd/values.yaml

@@ -1,6 +1,9 @@
 # Argo CD Helm Chart Values (k8s-ng cluster)
 
 global:
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
   domain: ag.btwiusearch.net
   logging:
     format: text

k8s-ng/core/cert-manager/issuer.yaml

@@ -1,4 +1,5 @@
-# Public certificates through DNS-01. The credential Secrets are created by hand
+# Public certificates through DNS-01 for the zones below and HTTP-01 (Traefik) for
+# every other domain, e.g. the .рф one. The credential Secrets are created by hand
 # (they will move to Vaultwarden + External Secrets):
 #   kubectl -n cert-manager create secret generic cloudflare-api-token --from-literal=apiToken=<token>
 #     (Zone:DNS:Edit on the btwiusearch.net zone)
@@ -36,3 +37,8 @@ spec:
             secretAccessKeySecretRef:
               name: aws-api-key-secret
               key: secretKey
+      # Fallback for domains outside the zones above: their A records must point
+      # at the cluster nodes (Traefik answers on port 80 everywhere).
+      - http01:
+          ingress:
+            ingressClassName: traefik

k8s-ng/core/cert-manager/values.yaml

@@ -1,2 +1,12 @@
 crds:
   enabled: true
+
+nodeSelector: &externalCore
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+webhook:
+  nodeSelector: *externalCore
+cainjector:
+  nodeSelector: *externalCore
+startupapicheck:
+  nodeSelector: *externalCore

k8s-ng/core/cnpg/values.yaml

@@ -1,4 +1,8 @@
 # CloudNativePG operator (chart 0.29.x = operator 1.30.x)
 
+nodeSelector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+
 crds:
   create: true

k8s-ng/core/coredns-config/deployment.yaml

@@ -0,0 +1,33 @@
+# K3s owns the CoreDNS Deployment. Apply only the fields customized here so
+# upstream image, probes, Corefile mounts and topology spread stay intact.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  replicas: 3
+  template:
+    spec:
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        # DNS also belongs on the internal core node used by Actions jobs.
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: homelab.hexor.cy/pool
+                    operator: In
+                    values:
+                      - core

k8s-ng/core/coredns-config/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml

k8s-ng/core/coredns-config/service.yaml

@@ -0,0 +1,11 @@
+# Keep DNS on the caller's node when it has a healthy CoreDNS replica. K3s owns
+# the rest of this Service; server-side apply preserves its ports and selector.
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false,Delete=false
+spec:
+  trafficDistribution: PreferSameNode

k8s-ng/core/external-secrets/values.yaml

@@ -1 +1,9 @@
 installCRDs: true
+
+nodeSelector: &externalCore
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+webhook:
+  nodeSelector: *externalCore
+certController:
+  nodeSelector: *externalCore

k8s-ng/core/gitea/deployment.yaml

@@ -0,0 +1,72 @@
+# Gitea with SQLite; everything (repositories, database, SSH host keys) lives in
+# /var/lib/gitea on the music node.
+# `replicas` is deliberately not set: Argo CD does not manage it, so the instance
+# can be scaled to 0 with kubectl during a data migration without being healed back.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea
+  labels:
+    app: gitea
+spec:
+  selector:
+    matchLabels:
+      app: gitea
+  strategy:
+    type: Recreate # two instances must never open the same SQLite database
+  template:
+    metadata:
+      labels:
+        app: gitea
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
+      volumes:
+        - name: storage
+          hostPath:
+            path: /var/lib/gitea
+            type: DirectoryOrCreate
+      containers:
+        - name: gitea
+          image: gitea/gitea:28.0.0
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: "1"
+            limits:
+              memory: 4Gi
+              cpu: "4"
+          env:
+            - name: GITEA__service__REGISTER_MANUAL_CONFIRM
+              value: "true"
+            - name: GITEA__service__ENABLE_CAPTCHA
+              value: "true"
+            - name: GITEA__service__REQUIRE_CAPTCHA_FOR_LOGIN
+              value: "false"
+            - name: GITEA__service__REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA
+              value: "true"
+            - name: GITEA__service__CAPTCHA_TYPE
+              value: cfturnstile
+            - name: GITEA__security__ALLOWED_HOST_LIST
+              value: "*"
+          envFrom:
+            - secretRef:
+                name: gitea-runner-act-runner-secrets
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+            - name: ssh
+              containerPort: 22
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /api/healthz
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: storage
+              mountPath: /data

k8s-ng/core/gitea/external-secrets.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-runner-token
+spec:
+  refreshInterval: 1m
+  target:
+    name: gitea-runner-act-runner-secrets
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        token: |-
+          {{ .password }}
+        GITEA__service__CF_TURNSTILE_SITEKEY: |-
+          {{ .CF_TURNSTILE_SITEKEY }}
+        GITEA__service__CF_TURNSTILE_SECRET: |-
+          {{ .CF_TURNSTILE_SECRET }}
+  data:
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: login.password
+    - secretKey: CF_TURNSTILE_SITEKEY
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: fields[0].value
+    - secretKey: CF_TURNSTILE_SECRET
+      sourceRef:
+        storeRef:
+          name: vaultwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/gitea/secrets
+        property: fields[1].value

k8s-ng/core/gitea/ingress.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitea
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+spec:
+  rules:
+    - host: gt.hexor.cy
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gitea
+                port:
+                  number: 80
+  tls:
+    - secretName: gitea-tls
+      hosts:
+        - gt.hexor.cy

k8s-ng/core/gitea/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - external-secrets.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+  - runner.yaml
+  - runner-rbac.yaml
+  - user-unban-cronjob.yaml

k8s-ng/core/gitea/runner-rbac.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gitea-runner-kubernetes-token
+  annotations:
+    kubernetes.io/service-account.name: gitea-runner
+type: kubernetes.io/service-account-token
+---
+# CI may edit the routes of the auth proxy (the ConfigMap lives in the keycloak namespace here).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auth-proxy-routes-manager
+  namespace: keycloak
+rules:
+  - apiGroups: [""]
+    resources: [configmaps]
+    resourceNames: [auth-proxy-routes]
+    verbs: [get, update, patch]
+  - apiGroups: [""]
+    resources: [configmaps]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-auth-proxy-routes
+  namespace: keycloak
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: auth-proxy-routes-manager
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner
+    namespace: gitea

k8s-ng/core/gitea/runner.yaml

@@ -0,0 +1,86 @@
+# Gitea Actions runner on the ai node. Jobs run as containers in the host's Docker
+# (virtualisation.docker in hosts/ai of the nix repo) through the mounted socket.
+# `replicas` is not set (see deployment.yaml): scale it with kubectl when needed.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gitea-runner-config
+data:
+  config.yaml: |
+    runner:
+      capacity: 8 # jobs in parallel; the ai node has 32 threads and 124 GiB
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea-runner
+  labels:
+    app: gitea-runner
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  selector:
+    matchLabels:
+      app: gitea-runner
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: gitea-runner
+    spec:
+      serviceAccountName: gitea-runner
+      dnsConfig:
+        options:
+          - name: ndots
+            value: "2"
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: internal
+        workload/ai: "true"
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      volumes:
+        - name: docker-sock
+          hostPath:
+            path: /var/run/docker.sock
+            type: Socket
+        - name: runner-data
+          emptyDir:
+            sizeLimit: 30Gi
+        - name: config
+          configMap:
+            name: gitea-runner-config
+      containers:
+        - name: gitea-runner
+          image: gitea/act_runner:nightly
+          volumeMounts:
+            - name: docker-sock
+              mountPath: /var/run/docker.sock
+            - name: runner-data
+              mountPath: /data
+            - name: config
+              mountPath: /config.yaml
+              subPath: config.yaml
+          env:
+            - name: CONFIG_FILE
+              value: /config.yaml
+            - name: GITEA_INSTANCE_URL
+              # Job containers run outside the cluster network, so they need the public URL.
+              value: https://gt.hexor.cy
+            - name: GITEA_RUNNER_REGISTRATION_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-runner-act-runner-secrets
+                  key: token
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GITEA_RUNNER_NAME
+              value: $(NODE_NAME)
+            - name: GITEA_RUNNER_LABELS
+              value: ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04,ubuntu-20.04:docker://ghcr.io/catthehacker/ubuntu:act-20.04

k8s-ng/core/gitea/service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea
+spec:
+  selector:
+    app: gitea
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 3000
+---
+# Git over SSH on every node (servicelb); the node firewall opens 30022 (nix repo, k3s-base.nix).
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-ssh
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 30022
+      targetPort: 22
+  selector:
+    app: gitea

k8s-ng/core/gitea/user-unban-cronjob.yaml

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-user-unban
+  labels:
+    app: gitea-user-unban
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: gitea-user-unban
+        spec:
+          restartPolicy: OnFailure
+          nodeSelector:
+            homelab.hexor.cy/pool: core
+            homelab.hexor.cy/exposure: external
+            kubernetes.io/hostname: music
+          volumes:
+            - name: storage
+              hostPath:
+                path: /var/lib/gitea
+                type: Directory
+          containers:
+            - name: sqlite-unban
+              image: gitea/gitea:28.0.0
+              imagePullPolicy: IfNotPresent
+              resources:
+                requests:
+                  memory: 32Mi
+                  cpu: 10m
+                limits:
+                  memory: 128Mi
+                  cpu: 100m
+              command:
+                - /bin/sh
+                - -ec
+                - |
+                  sqlite3 -cmd ".timeout 30000" /data/gitea/gitea.db "
+                    UPDATE \"user\"
+                       SET is_active = 1,
+                           prohibit_login = 0,
+                           updated_unix = unixepoch()
+                     WHERE lower(email) = lower('ab@hexor.cy')
+                       AND (is_active <> 1 OR prohibit_login <> 0);
+
+                    SELECT printf(
+                      'gitea user watchdog: id=%d login=%s email=%s is_active=%d prohibit_login=%d updated_unix=%d',
+                      id, lower_name, email, is_active, prohibit_login, updated_unix
+                    )
+                      FROM \"user\"
+                     WHERE lower(email) = lower('ab@hexor.cy');
+                  "
+              volumeMounts:
+                - name: storage
+                  mountPath: /data

k8s-ng/core/gpu/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+helmCharts:
+  - name: amd-gpu
+    repo: https://rocm.github.io/k8s-device-plugin/
+    version: 0.22.0
+    releaseName: amd-gpu-device-plugin
+    namespace: gpu-system
+    valuesFile: values.yaml
+    includeCRDs: true
+
+# The chart always adds the NFD label feature.node.kubernetes.io/pci-0300_1002.present
+# to the selector; NFD is not installed (and would not match this iGPU), so replace it.
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: DaemonSet
+      name: amd-gpu-device-plugin-daemonset
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/nodeSelector
+        value:
+          homelab.hexor.cy/pool: core
+          homelab.hexor.cy/exposure: internal
+          kubernetes.io/arch: amd64
+          workload/ai: "true"

k8s-ng/core/gpu/values.yaml

@@ -0,0 +1,37 @@
+# AMD GPU device plugin: exposes the Strix Halo iGPU of the "ai" node as the
+# amd.com/gpu resource (and injects /dev/kfd and /dev/dri into the pod).
+nfd:
+  enabled: false
+
+labeller:
+  enabled: false
+
+dp:
+  image:
+    repository: docker.io/rocm/k8s-device-plugin
+    tag: "1.31.0.9"
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+
+# The ai node carries the label workload/ai=true and the taint workload=ai:NoSchedule
+# (hosts/ai in the nix repo).
+tolerations:
+  - key: workload
+    operator: Equal
+    value: ai
+    effect: NoSchedule
+
+node_selector_enabled: true
+node_selector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: internal
+  kubernetes.io/arch: amd64
+  workload/ai: "true"

k8s-ng/core/immich-db/cluster.yaml

@@ -0,0 +1,47 @@
+# Keep PostgreSQL and both vector extensions identical to the source during
+# migration. The application/media migration and any upgrades happen later.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: immich
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  description: Immich database migrated from the old cluster; media remain on NAS
+  imageName: ghcr.io/tensorchord/cloudnative-pgvecto.rs:14-v0.3.0@sha256:df7848916fea2c486faee8155a481dc5009f30ec513b5561d9ab76d1e101a594
+  instances: 1
+  enableSuperuserAccess: true
+  bootstrap:
+    initdb:
+      database: immich
+      owner: immich
+      dataChecksums: true
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+  postgresUID: 26
+  postgresGID: 26
+  storage:
+    size: 55Gi
+    storageClass: immich-local
+    pvcTemplate:
+      accessModes: [ReadWriteOnce]
+      selector:
+        matchLabels:
+          immich.hexor.cy/volume: postgres
+  affinity:
+    nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
+      kubernetes.io/hostname: music
+  postgresql:
+    shared_preload_libraries: [vectors.so]
+    parameters:
+      shared_buffers: 512MB
+      max_connections: "200"
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      memory: 2Gi

k8s-ng/core/immich-db/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: immich
+resources:
+  - storage.yaml
+  - cluster.yaml

k8s-ng/core/immich-db/storage.yaml

@@ -0,0 +1,66 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: immich-db-storage-prepare
+  annotations:
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      restartPolicy: OnFailure
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
+        kubernetes.io/hostname: music
+      containers:
+        - name: prepare
+          image: docker.io/busybox:1.37.0
+          command: [sh, -ec, "mkdir -p /storage/postgresql; chown 26:26 /storage/postgresql; chmod 0700 /storage/postgresql"]
+          securityContext:
+            runAsUser: 0
+            allowPrivilegeEscalation: false
+          volumeMounts:
+            - name: storage
+              mountPath: /storage
+      volumes:
+        - name: storage
+          hostPath:
+            path: /data/k8s/immich
+            type: DirectoryOrCreate
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: immich-local
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Retain
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: immich-postgres-music
+  labels:
+    immich.hexor.cy/volume: postgres
+  annotations:
+    argocd.argoproj.io/sync-options: Prune=false
+spec:
+  capacity:
+    storage: 55Gi
+  volumeMode: Filesystem
+  accessModes: [ReadWriteOnce]
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: immich-local
+  local:
+    path: /data/k8s/immich/postgresql
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values: [music]

k8s-ng/core/keycloak/auth-proxy.yaml

@@ -20,9 +20,12 @@ spec:
       labels:
         app: auth-proxy
     spec:
+      nodeSelector:
+        homelab.hexor.cy/pool: core
+        homelab.hexor.cy/exposure: external
       containers:
         - name: auth-proxy
-          image: ultradesu/rsauth2-proxy:latest
+          image: ultradesu/rsauth2-proxy:v0.1.3
           ports:
             - containerPort: 8080
               name: http

k8s-ng/core/keycloak/hacker-theme-configmap.yaml

@@ -1,366 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: keycloak-hacker-theme
-  namespace: keycloak
-data:
-  theme.properties: |
-    parent=keycloak.v2
-    import=common/keycloak
-
-    styles=css/hacker.css
-
-  hacker.css: |
-    /* ============================================================
-       HEXOR — Hacker Terminal Theme for Keycloak
-       ============================================================ */
-
-    @import url('https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;600&display=swap');
-
-    :root {
-      --hk-bg: #0a0a0a;
-      --hk-panel: #0d0d0d;
-      --hk-border: #00ff4180;
-      --hk-green: #00ff41;
-      --hk-green-dim: #00cc33;
-      --hk-green-glow: rgba(0, 255, 65, 0.15);
-      --hk-green-glow-strong: rgba(0, 255, 65, 0.35);
-      --hk-cyan: #00e5ff;
-      --hk-red: #ff3333;
-      --hk-text: #b0ffb0;
-      --hk-text-dim: #5a8a5a;
-      --hk-input-bg: #050505;
-      --hk-font: 'Fira Code', 'Cascadia Code', 'JetBrains Mono', monospace;
-    }
-
-    /* -- Force dark always --------------------------------------------------- */
-    html.login-pf,
-    html.login-pf.pf-v5-theme-dark {
-      color-scheme: dark;
-    }
-
-    /* -- Background: scanlines + CRT ---------------------------------------- */
-    body#keycloak-bg {
-      font-family: var(--hk-font) !important;
-      background: var(--hk-bg) !important;
-      color: var(--hk-text) !important;
-      min-height: 100vh;
-      position: relative;
-    }
-
-    body#keycloak-bg::before {
-      content: '';
-      position: fixed;
-      inset: 0;
-      background: repeating-linear-gradient(
-        0deg,
-        transparent,
-        transparent 2px,
-        rgba(0, 255, 65, 0.03) 2px,
-        rgba(0, 255, 65, 0.03) 4px
-      );
-      pointer-events: none;
-      z-index: 9999;
-    }
-
-    /* -- Login container ----------------------------------------------------- */
-    .pf-v5-c-login {
-      background: transparent !important;
-    }
-
-    .pf-v5-c-login__container {
-      max-width: 480px !important;
-    }
-
-    /* -- Header / Brand ------------------------------------------------------ */
-    #kc-header-wrapper {
-      font-family: var(--hk-font) !important;
-      color: var(--hk-green) !important;
-      text-shadow: 0 0 10px var(--hk-green), 0 0 30px var(--hk-green-glow);
-      font-size: 1.5rem !important;
-      letter-spacing: 0.15em;
-      text-transform: uppercase;
-    }
-
-    #kc-header-wrapper .kc-logo-text span::before {
-      content: '> ';
-      opacity: 0.6;
-    }
-
-    /* -- Main card ----------------------------------------------------------- */
-    .pf-v5-c-login__main {
-      background: var(--hk-panel) !important;
-      border: 1px solid var(--hk-border) !important;
-      border-radius: 0 !important;
-      box-shadow:
-        0 0 15px var(--hk-green-glow),
-        inset 0 0 30px rgba(0, 0, 0, 0.5) !important;
-    }
-
-    /* -- Title --------------------------------------------------------------- */
-    h1#kc-page-title,
-    .pf-v5-c-title {
-      font-family: var(--hk-font) !important;
-      color: var(--hk-green) !important;
-      text-shadow: 0 0 8px var(--hk-green-glow);
-      font-weight: 600 !important;
-      font-size: 1.25rem !important;
-      letter-spacing: 0.05em;
-    }
-
-    /* -- Labels -------------------------------------------------------------- */
-    .pf-v5-c-form__label-text {
-      font-family: var(--hk-font) !important;
-      color: var(--hk-green-dim) !important;
-      font-size: 0.85rem !important;
-      text-transform: uppercase;
-      letter-spacing: 0.08em;
-    }
-
-    /* -- Input fields -------------------------------------------------------- */
-    .pf-v5-c-form-control {
-      background-color: var(--hk-input-bg) !important;
-      border: 1px solid var(--hk-border) !important;
-      border-radius: 0 !important;
-      color: var(--hk-green) !important;
-      font-family: var(--hk-font) !important;
-      font-size: 0.9rem !important;
-      caret-color: var(--hk-green);
-      transition: border-color 0.2s, box-shadow 0.2s;
-    }
-
-    .pf-v5-c-form-control:focus-within,
-    .pf-v5-c-form-control:focus {
-      border-color: var(--hk-green) !important;
-      box-shadow: 0 0 8px var(--hk-green-glow), inset 0 0 4px var(--hk-green-glow) !important;
-      outline: none !important;
-    }
-
-    .pf-v5-c-form-control input {
-      color: var(--hk-green) !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    .pf-v5-c-form-control input::placeholder {
-      color: var(--hk-text-dim) !important;
-    }
-
-    /* -- Primary button (Sign In) -------------------------------------------- */
-    .pf-v5-c-button.pf-m-primary {
-      background: transparent !important;
-      border: 1px solid var(--hk-green) !important;
-      border-radius: 0 !important;
-      color: var(--hk-green) !important;
-      font-family: var(--hk-font) !important;
-      font-weight: 600 !important;
-      text-transform: uppercase;
-      letter-spacing: 0.12em;
-      font-size: 0.9rem !important;
-      transition: all 0.2s;
-      position: relative;
-      overflow: hidden;
-    }
-
-    .pf-v5-c-button.pf-m-primary:hover {
-      background: var(--hk-green-glow) !important;
-      box-shadow: 0 0 15px var(--hk-green-glow-strong), inset 0 0 15px var(--hk-green-glow) !important;
-      color: #fff !important;
-      text-shadow: 0 0 5px var(--hk-green);
-    }
-
-    /* -- Secondary button (Try Another Way, Passkey) ------------------------- */
-    .pf-v5-c-button.pf-m-secondary {
-      background: transparent !important;
-      border: 1px solid var(--hk-cyan) !important;
-      border-radius: 0 !important;
-      color: var(--hk-cyan) !important;
-      font-family: var(--hk-font) !important;
-      text-transform: uppercase;
-      letter-spacing: 0.1em;
-      font-size: 0.85rem !important;
-      transition: all 0.2s;
-    }
-
-    .pf-v5-c-button.pf-m-secondary:hover {
-      background: rgba(0, 229, 255, 0.1) !important;
-      box-shadow: 0 0 12px rgba(0, 229, 255, 0.3) !important;
-      color: #fff !important;
-    }
-
-    /* -- Passkey authenticate button ----------------------------------------- */
-    #authenticateWebAuthnButton {
-      background: transparent !important;
-      border: 1px solid var(--hk-cyan) !important;
-      border-radius: 0 !important;
-      color: var(--hk-cyan) !important;
-      font-family: var(--hk-font) !important;
-      text-transform: uppercase;
-      letter-spacing: 0.1em;
-      font-size: 0.85rem !important;
-      padding: 0.75rem 1rem;
-      display: block;
-      width: 100%;
-      text-align: center;
-      text-decoration: none;
-      transition: all 0.2s;
-      margin-top: 1rem;
-    }
-
-    #authenticateWebAuthnButton::before {
-      content: '\f084  ';
-      font-family: 'Font Awesome 6 Free', 'FontAwesome';
-      font-weight: 900;
-    }
-
-    #authenticateWebAuthnButton:hover {
-      background: rgba(0, 229, 255, 0.1) !important;
-      box-shadow: 0 0 15px rgba(0, 229, 255, 0.3), inset 0 0 10px rgba(0, 229, 255, 0.1) !important;
-      color: #fff !important;
-      text-shadow: 0 0 5px var(--hk-cyan);
-    }
-
-    /* -- Show password button ------------------------------------------------ */
-    .pf-v5-c-button.pf-m-control {
-      background: var(--hk-input-bg) !important;
-      border: 1px solid var(--hk-border) !important;
-      border-left: none !important;
-      border-radius: 0 !important;
-      color: var(--hk-text-dim) !important;
-    }
-
-    .pf-v5-c-button.pf-m-control:hover {
-      color: var(--hk-green) !important;
-    }
-
-    /* -- Social providers (Google, etc) -------------------------------------- */
-    .pf-v5-c-login__main-footer {
-      border-top: 1px solid var(--hk-border) !important;
-    }
-
-    .kc-social-section .pf-v5-c-button.pf-m-secondary {
-      border-color: var(--hk-text-dim) !important;
-      color: var(--hk-text) !important;
-    }
-
-    .kc-social-section .pf-v5-c-button.pf-m-secondary:hover {
-      border-color: var(--hk-green) !important;
-      color: var(--hk-green) !important;
-      box-shadow: 0 0 10px var(--hk-green-glow) !important;
-    }
-
-    /* -- Try another way link ------------------------------------------------ */
-    #try-another-way {
-      color: var(--hk-cyan) !important;
-      text-decoration: none !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    #try-another-way:hover {
-      text-shadow: 0 0 8px rgba(0, 229, 255, 0.5);
-    }
-
-    /* -- Links --------------------------------------------------------------- */
-    a {
-      color: var(--hk-cyan) !important;
-      text-decoration: none;
-      transition: text-shadow 0.2s;
-    }
-
-    a:hover {
-      text-shadow: 0 0 6px rgba(0, 229, 255, 0.4);
-    }
-
-    /* -- Alerts / messages --------------------------------------------------- */
-    .pf-v5-c-alert {
-      background: rgba(0, 255, 65, 0.05) !important;
-      border: 1px solid var(--hk-border) !important;
-      border-radius: 0 !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    .pf-v5-c-alert.pf-m-danger {
-      border-color: var(--hk-red) !important;
-      background: rgba(255, 51, 51, 0.05) !important;
-    }
-
-    .pf-v5-c-alert__title {
-      color: var(--hk-text) !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    /* -- Checkbox (Remember me) ---------------------------------------------- */
-    .pf-v5-c-check__label {
-      color: var(--hk-text-dim) !important;
-      font-family: var(--hk-font) !important;
-      font-size: 0.8rem !important;
-    }
-
-    /* -- Helper text (Forgot password, etc.) --------------------------------- */
-    .pf-v5-c-helper-text .pf-v5-c-button.pf-m-link {
-      color: var(--hk-cyan) !important;
-      font-family: var(--hk-font) !important;
-      font-size: 0.8rem !important;
-    }
-
-    /* -- Select auth list (Try another way options) -------------------------- */
-    .select-auth-container {
-      background: var(--hk-panel) !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    .pf-v5-c-data-list__item {
-      border-color: var(--hk-border) !important;
-      background: transparent !important;
-    }
-
-    .pf-v5-c-data-list__item:hover {
-      background: var(--hk-green-glow) !important;
-    }
-
-    .select-auth-box-headline {
-      color: var(--hk-green) !important;
-    }
-
-    .select-auth-box-desc {
-      color: var(--hk-text-dim) !important;
-    }
-
-    /* -- Footer -------------------------------------------------------------- */
-    .pf-v5-c-login__main-footer-band {
-      background: transparent !important;
-      border-top: 1px solid var(--hk-border) !important;
-    }
-
-    /* -- Language selector --------------------------------------------------- */
-    select#login-select-toggle {
-      background: var(--hk-input-bg) !important;
-      color: var(--hk-green) !important;
-      border-color: var(--hk-border) !important;
-      font-family: var(--hk-font) !important;
-    }
-
-    /* -- Blinking cursor animation for header -------------------------------- */
-    @keyframes blink {
-      0%, 100% { opacity: 1; }
-      50% { opacity: 0; }
-    }
-
-    #kc-header-wrapper::after {
-      content: '_';
-      animation: blink 1s step-end infinite;
-      color: var(--hk-green);
-    }
-
-    /* -- Subtle CRT flicker ------------------------------------------------- */
-    @keyframes flicker {
-      0% { opacity: 0.97; }
-      5% { opacity: 0.95; }
-      10% { opacity: 0.98; }
-      15% { opacity: 0.96; }
-      20% { opacity: 0.99; }
-      100% { opacity: 0.98; }
-    }
-
-    .pf-v5-c-login__main {
-      animation: flicker 4s infinite;
-    }

k8s-ng/core/keycloak/kustomization.yaml

@@ -3,10 +3,24 @@ kind: Kustomization
 
 resources:
   - secrets.yaml
-  - hacker-theme-configmap.yaml
   - auth-proxy.yaml
 # ConfigMap auth-proxy-routes is managed by Terraform (terraform/keycloak, kubernetes_config_map_v1).
 
+configMapGenerator:
+  - name: keycloak-hexor-lab-theme
+    namespace: keycloak
+    files:
+      - theme.properties=themes/hexor-lab/login/theme.properties
+      - template.ftl=themes/hexor-lab/login/template.ftl
+      - hexor.css=themes/hexor-lab/login/resources/css/hexor.css
+      - keycloak.css=themes/hexor-lab/login/resources/css/keycloak.css
+      - background.js=themes/hexor-lab/login/resources/js/background.js
+      - favicon.svg=themes/hexor-lab/login/resources/img/favicon.svg
+
+generatorOptions:
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+
 helmCharts:
   - name: keycloakx
     repo: https://codecentric.github.io/helm-charts

k8s-ng/core/keycloak/preview/app.js

@@ -0,0 +1,110 @@
+/* Preview behavior only. Keycloak will supply the real authentication later. */
+"use strict";
+const $ = (selector) => document.querySelector(selector);
+const screens = {
+  login: ["Sign in", ""],
+  otp: ["Verify identity", ""],
+  reset: ["Reset password", "We’ll send a reset link to your email."],
+  success: ["Access granted", ""],
+};
+
+function notice(message = "", error = false) {
+  $("#notice").textContent = message;
+  $("#notice").hidden = !message;
+  $("#notice").classList.toggle("error", error);
+}
+
+function showScreen(requested, focus = false) {
+  const screen = requested === "error" ? "login" : (Object.hasOwn(screens, requested) ? requested : "login");
+  const [title, description] = screens[screen];
+  document.querySelectorAll("[data-view]").forEach((element) => { element.hidden = element.dataset.view !== screen; });
+  $("#page-title").replaceChildren(document.createTextNode(title));
+  const cursor = document.createElement("span");
+  cursor.className = "cursor";
+  cursor.setAttribute("aria-hidden", "true");
+  cursor.textContent = "_";
+  $("#page-title").append(cursor);
+  $("#page-description").textContent = description;
+  $("#page-description").hidden = !description;
+  document.querySelectorAll("[data-screen]").forEach((button) => button.setAttribute("aria-pressed", String(button.dataset.screen === requested)));
+  document.querySelectorAll(".field-error").forEach((element) => { element.hidden = true; });
+  document.querySelectorAll("[aria-invalid]").forEach((element) => element.removeAttribute("aria-invalid"));
+  notice(requested === "error" ? "[ DENIED ] Invalid username or password. Please try again." : "", requested === "error");
+  $("#password").value = "";
+  $("#password").type = "password";
+  $("#reveal-password").textContent = "SHOW";
+  $("#reveal-password").setAttribute("aria-label", "Show password");
+  $("#reveal-password").setAttribute("aria-pressed", "false");
+  $("#otp").value = "";
+  if (focus) document.querySelector(`[data-view="${screen}"] input`)?.focus();
+}
+
+function navigate(screen) {
+  if (location.hash === `#${screen}`) showScreen(screen, true);
+  else location.hash = screen;
+}
+
+function fieldError(field, output, message) {
+  field.setAttribute("aria-invalid", "true");
+  output.textContent = message;
+  output.hidden = false;
+  field.focus();
+}
+
+document.querySelectorAll("[data-screen]").forEach((button) => button.addEventListener("click", () => navigate(button.dataset.screen)));
+window.addEventListener("hashchange", () => showScreen(location.hash.slice(1), true));
+
+$("#login-form").addEventListener("submit", (event) => {
+  event.preventDefault();
+  notice();
+  if (!$("#username").value.trim()) return fieldError($("#username"), $("#login-error"), "Enter your username or email address.");
+  if (!$("#password").value) return fieldError($("#password"), $("#login-error"), "Enter your password.");
+  $("#password").value = "";
+  navigate("otp");
+});
+$("#otp-form").addEventListener("submit", (event) => {
+  event.preventDefault();
+  if (!/^[0-9]{6}$/.test($("#otp").value)) return fieldError($("#otp"), $("#otp-error"), "Enter a 6-digit verification code.");
+  $("#otp").value = "";
+  navigate("success");
+});
+$("#reset-form").addEventListener("submit", (event) => {
+  event.preventDefault();
+  if (!$("#reset-email").validity.valid) return fieldError($("#reset-email"), $("#reset-error"), "Enter a valid email address.");
+  $("#reset-error").hidden = true;
+  $("#reset-email").removeAttribute("aria-invalid");
+  notice("[ OK ] Reset-link confirmation preview. No email was sent.");
+});
+$("#reveal-password").addEventListener("click", () => {
+  const reveal = $("#password").type === "password";
+  $("#password").type = reveal ? "text" : "password";
+  $("#reveal-password").textContent = reveal ? "HIDE" : "SHOW";
+  $("#reveal-password").setAttribute("aria-label", reveal ? "Hide password" : "Show password");
+  $("#reveal-password").setAttribute("aria-pressed", String(reveal));
+});
+$("#passkey").addEventListener("click", () => notice("[ DEMO ] Passkey authentication will be handled by Keycloak. No device prompt is opened here."));
+$("#google").addEventListener("click", () => notice("[ DEMO ] Google sign-in will be handled by Keycloak. This preview stays local."));
+document.querySelectorAll("input").forEach((input) => input.addEventListener("input", () => {
+  input.removeAttribute("aria-invalid");
+  input.closest("form")?.querySelectorAll(".field-error").forEach((element) => { element.hidden = true; });
+}));
+
+showScreen(location.hash.slice(1));
+
+// The optional stdlib server reloads the page when HTML/CSS/JS files change.
+if (location.protocol === "http:" && ["127.0.0.1", "localhost"].includes(location.hostname)) {
+  let version;
+  let checking = false;
+  setInterval(async () => {
+    if (checking || document.hidden) return;
+    checking = true;
+    try {
+      const response = await fetch("/__version", { cache: "no-store" });
+      if (!response.ok || !response.headers.get("content-type")?.includes("application/json")) return;
+      const current = (await response.json()).version;
+      if (version !== undefined && current !== version) location.reload();
+      version = current;
+    } catch { /* The preview also works without the local server. */ }
+    finally { checking = false; }
+  }, 1000);
+}

k8s-ng/core/keycloak/preview/background.js

@@ -0,0 +1 @@
+../themes/hexor-lab/login/resources/js/background.js

k8s-ng/core/keycloak/preview/index.html

@@ -0,0 +1,62 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="color-scheme" content="dark">
+  <meta name="description" content="Local prototype of the Hexor terminal login theme.">
+  <title>Hexor · Access terminal</title>
+  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' fill='%230c100c'/%3E%3Ctext x='8' y='45' fill='%23b9ef70' font-family='monospace' font-size='40'%3E%26gt;_%3C/text%3E%3C/svg%3E">
+  <link rel="stylesheet" href="styles.css">
+  <link rel="stylesheet" href="preview.css">
+  <script src="app.js" defer></script>
+  <script src="background.js" defer></script>
+</head>
+<body>
+  <canvas id="code-background" aria-hidden="true"></canvas>
+  <header class="site-header">
+    <a class="wordmark" href="#login">Hexor Lab</a>
+  </header>
+
+  <main class="page-main">
+    <div class="terminal-wrap">
+      <section class="terminal" aria-labelledby="page-title">
+        <div class="titlebar"><span><span aria-hidden="true">&gt;_</span> auth@hexor</span></div>
+        <div class="terminal-body">
+          <div class="heading"><h1 id="page-title">Sign in<span class="cursor" aria-hidden="true">_</span></h1><p id="page-description" class="description" hidden></p></div>
+
+          <div id="notice" class="notice" role="status" hidden></div>
+
+          <form id="login-form" data-view="login" novalidate>
+            <div class="field"><label for="username"><span class="prompt" aria-hidden="true">&gt;</span> username <span class="label-hint">/ email</span></label><input id="username" name="username" autocomplete="username" autocapitalize="none" spellcheck="false" placeholder="you@hexor.cy" required aria-describedby="login-error"></div>
+            <div class="field"><label for="password"><span class="prompt" aria-hidden="true">&gt;</span> password</label><div class="password-wrap"><input id="password" name="password" type="password" autocomplete="current-password" placeholder="Enter your password" required aria-describedby="login-error"><button class="reveal" id="reveal-password" type="button" aria-label="Show password" aria-pressed="false">SHOW</button></div></div>
+            <p class="field-error" id="login-error" role="alert" hidden></p>
+            <div class="form-options"><label class="checkbox"><input type="checkbox" name="rememberMe"> Keep me signed in</label><a href="#reset">Forgot password?</a></div>
+            <button class="primary" type="submit"><span>Sign in</span><span aria-hidden="true">↵</span></button>
+            <div class="divider"><span>OR CONTINUE WITH</span></div>
+            <div class="alternatives"><button class="secondary" id="passkey" type="button"><span aria-hidden="true">⌘</span> Passkey</button><button class="secondary" id="google" type="button"><span class="google-icon" aria-hidden="true">G</span> Google</button></div>
+          </form>
+
+          <form id="otp-form" data-view="otp" novalidate hidden>
+            <div class="field"><label for="otp"><span class="prompt" aria-hidden="true">&gt;</span> verification code</label><input id="otp" name="otp" class="otp-input" type="text" inputmode="numeric" pattern="[0-9]{6}" maxlength="6" autocomplete="one-time-code" placeholder="000000" required aria-describedby="otp-help otp-error"><p id="otp-help" class="field-help">Enter the 6-digit code from your authenticator.</p></div>
+            <p id="otp-error" class="field-error" role="alert" hidden></p>
+            <button class="primary" type="submit"><span>Verify identity</span><span aria-hidden="true">↵</span></button><a class="back-link" href="#login">← Back to sign in</a>
+          </form>
+
+          <form id="reset-form" data-view="reset" novalidate hidden>
+            <div class="field"><label for="reset-email"><span class="prompt" aria-hidden="true">&gt;</span> email address</label><input id="reset-email" name="email" type="email" autocomplete="email" autocapitalize="none" spellcheck="false" placeholder="you@hexor.cy" required aria-describedby="reset-error"></div><p id="reset-error" class="field-error" role="alert" hidden></p>
+            <button class="primary" type="submit"><span>Send reset link</span><span aria-hidden="true">↵</span></button><a class="back-link" href="#login">← Back to sign in</a>
+          </form>
+
+          <div data-view="success" class="success-panel" hidden><div class="success-mark" aria-hidden="true">[ OK ]</div><p>Identity verified.</p><a class="secondary back-link" href="#login">Back to sign in →</a></div>
+        </div>
+      </section>
+    </div>
+  </main>
+
+  <aside class="preview" aria-label="Настройки локального предпросмотра" lang="ru">
+    <div class="preview-heading"><span class="preview-label">LOCAL PREVIEW</span><span>Демо · данные никуда не отправляются</span></div>
+    <div class="preview-controls"><div class="preview-group" role="group" aria-label="Экран"><button type="button" data-screen="login" aria-pressed="true">Логин</button><button type="button" data-screen="otp" aria-pressed="false">OTP</button><button type="button" data-screen="reset" aria-pressed="false">Сброс</button><button type="button" data-screen="error" aria-pressed="false">Ошибка</button></div></div>
+  </aside>
+</body>
+</html>

k8s-ng/core/keycloak/preview/preview.css

@@ -0,0 +1,11 @@
+.preview { border-top: 1px solid #ffffff0c; padding: 18px 24px 21px; color: var(--muted); font-size: 12px; background: var(--bg); }
+.preview-heading, .preview-controls { max-width: 760px; margin: 0 auto; display: flex; gap: 15px; flex-wrap: wrap; justify-content: space-between; align-items: center; }
+.preview-heading { margin-bottom: 12px; }
+.preview-label { color: var(--text); letter-spacing: 1px; font-size: 11px; }
+.preview-group { display: flex; align-items: center; gap: 7px; flex-wrap: wrap; }
+.preview button, .preview select { border: 1px solid var(--border); border-radius: 0; background: transparent; padding: 5px 9px; color: var(--muted); font-size: 13px; }
+.preview select { background: var(--bg); }
+.preview button[aria-pressed="true"] { color: var(--accent); border-color: var(--accent); }
+.preview button:hover { color: var(--text); }
+.preview .checkbox { margin-left: 8px; }
+@media (max-width: 580px) { .preview { padding: 16px; } .preview-controls { gap: 12px; } }

k8s-ng/core/keycloak/preview/realm.json

@@ -0,0 +1,49 @@
+{
+  "realm": "hexor-preview",
+  "enabled": true,
+  "displayName": "Hexor Lab",
+  "loginTheme": "hexor-lab",
+  "sslRequired": "none",
+  "registrationAllowed": false,
+  "resetPasswordAllowed": true,
+  "rememberMe": true,
+  "loginWithEmailAllowed": true,
+  "clients": [
+    {
+      "clientId": "theme-preview",
+      "enabled": true,
+      "publicClient": true,
+      "standardFlowEnabled": true,
+      "redirectUris": ["http://127.0.0.1:8765/*"],
+      "webOrigins": ["http://127.0.0.1:8765"]
+    }
+  ],
+  "users": [
+    {
+      "username": "demo",
+      "enabled": true,
+      "emailVerified": true,
+      "email": "demo@example.test",
+      "firstName": "Demo",
+      "lastName": "User",
+      "credentials": [{"type": "password", "value": "demo-password", "temporary": false}]
+    },
+    {
+      "username": "demo-otp",
+      "enabled": true,
+      "emailVerified": true,
+      "email": "demo-otp@example.test",
+      "firstName": "Demo",
+      "lastName": "OTP",
+      "credentials": [
+        {"type": "password", "value": "demo-password", "temporary": false},
+        {
+          "type": "otp",
+          "userLabel": "Local preview",
+          "secretData": "{\"value\":\"local-preview-secret\"}",
+          "credentialData": "{\"subType\":\"totp\",\"digits\":6,\"counter\":0,\"period\":30,\"algorithm\":\"HmacSHA1\"}"
+        }
+      ]
+    }
+  ]
+}

k8s-ng/core/keycloak/preview/run-keycloak.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run the real login theme in an isolated, disposable Keycloak 26.5.6 instance.
+# Test users: demo / demo-password and demo-otp / demo-password.
+# The existing deployment, Terraform and production realm are not involved.
+preview_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+theme_dir="$(cd -- "$preview_dir/../themes/hexor-lab" && pwd)"
+
+printf '%s\n' \
+  'After Keycloak starts, open:' \
+  'http://127.0.0.1:8780/realms/hexor-preview/protocol/openid-connect/auth?client_id=theme-preview&redirect_uri=http%3A%2F%2F127.0.0.1%3A8765%2F&response_type=code&scope=openid' \
+  'Test account: demo / demo-password'
+
+docker run --rm --name hexor-lab-preview \
+  --publish 127.0.0.1:8780:8080 \
+  --mount "type=bind,source=$theme_dir,target=/opt/keycloak/themes/hexor-lab,readonly" \
+  --mount "type=bind,source=$preview_dir/realm.json,target=/opt/keycloak/data/import/hexor-preview.json,readonly" \
+  quay.io/keycloak/keycloak:26.5.6 \
+  start-dev --import-realm \
+  --spi-theme--static-max-age=-1 \
+  --spi-theme--cache-themes=false \
+  --spi-theme--cache-templates=false

k8s-ng/core/keycloak/preview/serve.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Local theme preview: python3 serve.py [--port 8765]. No dependencies."""
+
+import argparse
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+import json
+from pathlib import Path
+from urllib.parse import urlsplit
+
+ROOT = Path(__file__).resolve().parent
+ASSETS = ("index.html", "styles.css", "preview.css", "app.js", "background.js")
+
+
+class PreviewHandler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header("Cache-Control", "no-store")
+        super().end_headers()
+
+    def do_GET(self):
+        if urlsplit(self.path).path == "/__version":
+            version = ":".join(str((ROOT / name).stat().st_mtime_ns) for name in ASSETS)
+            body = json.dumps({"version": version}).encode()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        else:
+            super().do_GET()
+
+    def log_message(self, format, *args):
+        if not self.path.startswith("/__version"):
+            super().log_message(format, *args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--port", type=int, default=8765)
+    args = parser.parse_args()
+    server = ThreadingHTTPServer(("127.0.0.1", args.port), partial(PreviewHandler, directory=str(ROOT)))
+    print(f"Local: http://127.0.0.1:{server.server_port}", flush=True)
+    print("Edit index.html, styles.css or app.js; the browser reloads on save. Ctrl+C to stop.", flush=True)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.server_close()

k8s-ng/core/keycloak/preview/styles.css

@@ -0,0 +1 @@
+../themes/hexor-lab/login/resources/css/hexor.css

k8s-ng/core/keycloak/themes/hexor-lab/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

k8s-ng/core/keycloak/themes/hexor-lab/login/resources/css/hexor.css

@@ -0,0 +1,83 @@
+/* Start here: all palette and layout settings are ordinary CSS variables. */
+:root {
+  --bg: #0c0f0d;
+  --surface: #111612;
+  --input-bg: #0c110d;
+  --accent: #b9ef70;
+  --accent-ink: #121c09;
+  --text: #e1e7dd;
+  --muted: #8b9989;
+  --border: #354031;
+  --error: #ffa595;
+  --font: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+  --terminal-width: 560px;
+  --code-opacity: .16;
+}
+* { box-sizing: border-box; }
+[hidden] { display: none !important; }
+body { margin: 0; min-height: 100svh; color: var(--text); background: var(--bg); font: 16px/1.6 var(--font); display: flex; flex-direction: column; isolation: isolate; }
+#code-background { position: fixed; inset: 0; width: 100%; height: 100%; pointer-events: none; z-index: -1; opacity: var(--code-opacity); mask-image: linear-gradient(to bottom, transparent, #000 15%, #000 85%, transparent); }
+button, input, select { font: inherit; }
+button, select { cursor: pointer; }
+button, a, input, select { -webkit-tap-highlight-color: transparent; }
+button, a, select { touch-action: manipulation; }
+a { color: var(--muted); text-underline-offset: 4px; text-decoration-thickness: 1px; }
+a:hover { color: var(--accent); }
+button:focus-visible, a:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 4px; }
+::selection { background: var(--accent); color: var(--accent-ink); }
+.site-header { padding: 27px 38px; display: flex; justify-content: space-between; gap: 20px; align-items: center; }
+.wordmark { font: 500 clamp(26px, 4vw, 34px)/1.2 var(--font); letter-spacing: -1px; text-decoration: none; color: var(--text); }
+.page-main { flex: 1; display: grid; place-items: center; padding: 35px 24px 64px; }
+.terminal-wrap { width: min(100%, var(--terminal-width)); }
+.terminal { background: var(--surface); border: 1px solid var(--border); box-shadow: 0 24px 90px #0004; }
+.titlebar { display: flex; padding: 12px 20px; border-bottom: 1px solid var(--border); font-size: 14px; color: var(--muted); }
+.titlebar span span { color: var(--accent); margin-right: 9px; }
+.terminal-body { padding: 30px 36px 34px; }
+.heading { margin: 0 0 28px; }
+h1 { margin: 0 0 8px; font-size: clamp(28px, 6vw, 34px); line-height: 1.3; font-weight: 500; letter-spacing: -1px; }
+.cursor { color: var(--accent); animation: blink 1.2s step-end infinite; }
+@keyframes blink { 50% { opacity: 0; } }
+.description { color: var(--muted); margin: 0; font-size: 15px; }
+.field { margin-bottom: 21px; }
+.field > label { display: block; margin-bottom: 8px; font-size: 16px; }
+.prompt { color: var(--accent); margin-right: 7px; }
+.label-hint { color: var(--muted); font-size: 14px; }
+input:not([type="checkbox"]) { width: 100%; min-height: 50px; padding: 12px 14px; background: var(--input-bg); border: 1px solid var(--border); border-radius: 0; color: var(--text); caret-color: var(--accent); outline: none; box-shadow: none; font-size: 16px; }
+input:not([type="checkbox"]):hover { border-color: var(--muted); }
+input:not([type="checkbox"]):focus { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
+input::placeholder { color: var(--muted); opacity: .7; }
+input[aria-invalid="true"]:not([type="checkbox"]) { border-color: var(--error); }
+input:-webkit-autofill { -webkit-text-fill-color: var(--text); box-shadow: 0 0 0 1000px var(--input-bg) inset; }
+.password-wrap { position: relative; }
+.password-wrap input { padding-right: 72px; }
+.reveal { position: absolute; right: 5px; top: 5px; bottom: 5px; padding: 0 10px; background: transparent; color: var(--muted); border: 0; font-size: 12px; letter-spacing: .5px; }
+.reveal:hover { color: var(--accent); }
+.form-options { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 10px; margin: 1px 0 26px; font-size: 14px; }
+.checkbox { display: inline-flex; align-items: center; gap: 8px; cursor: pointer; color: var(--muted); }
+input[type="checkbox"] { width: 16px; height: 16px; margin: 0; accent-color: var(--accent); cursor: pointer; }
+.primary { width: 100%; min-height: 50px; display: flex; align-items: center; justify-content: space-between; padding: 11px 16px; background: var(--accent); color: var(--accent-ink); border: 1px solid var(--accent); border-radius: 0; font-weight: bold; transition: filter .15s; }
+.primary:hover { filter: brightness(1.1); }
+.primary:active { transform: translateY(1px); }
+.primary span:last-child { font-size: 20px; font-weight: normal; line-height: 1; }
+.divider { display: flex; align-items: center; gap: 15px; margin: 24px 0 18px; color: var(--muted); font-size: 11px; letter-spacing: 1px; }
+.divider::before, .divider::after { content: ""; height: 1px; background: var(--border); flex: 1; }
+.alternatives { display: flex; gap: 12px; }
+.secondary { display: inline-flex; justify-content: center; align-items: center; gap: 10px; min-height: 48px; padding: 10px 15px; background: transparent; color: var(--text); border: 1px solid var(--border); border-radius: 0; font-size: 16px; text-decoration: none; }
+.alternatives .secondary { flex: 1; }
+.secondary:hover { border-color: var(--accent); color: var(--accent); background: #ffffff03; }
+.secondary > span { color: var(--muted); font-size: 16px; }
+.google-icon { font-family: sans-serif; font-weight: bold; }
+.notice { border-left: 2px solid var(--accent); padding: 10px 13px; background: #ffffff04; color: var(--text); font-size: 15px; margin: 0 0 22px; overflow-wrap: anywhere; }
+.notice.error { border-color: var(--error); color: var(--error); }
+.field-error { color: var(--error); font-size: 14px; margin: -8px 0 18px; }
+.field-help { margin: 10px 0 0; color: var(--muted); font-size: 14px; }
+input.otp-input { letter-spacing: 10px; font-size: 24px; padding: 15px; }
+.back-link { display: block; text-align: center; margin-top: 24px; font-size: 15px; }
+.success-panel { text-align: center; padding: 15px 0; }
+.success-mark { color: var(--accent); font-size: 32px; }
+@media (max-width: 580px) {
+  .site-header { padding: 20px; }
+  .page-main { padding: 25px 16px 40px; }
+  .terminal-body { padding: 24px; }
+}
+@media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation: none !important; transition: none !important; } }

k8s-ng/core/keycloak/themes/hexor-lab/login/resources/css/keycloak.css

@@ -0,0 +1,130 @@
+/* Adapt the native Keycloak 26.5.6 form components to the shared design. */
+html.login-pf {
+  color-scheme: dark;
+  --pf-v5-global--FontFamily--text: var(--font);
+  --pf-v5-global--FontFamily--heading: var(--font);
+  --pf-v5-global--FontFamily--monospace: var(--font);
+  --pf-v5-global--FontSize--md: 16px;
+  --pf-v5-global--Color--100: var(--text);
+  --pf-v5-global--Color--200: var(--muted);
+  --pf-v5-global--primary-color--100: var(--accent);
+  --pf-v5-global--primary-color--200: var(--accent);
+  --pf-v5-global--link--Color: var(--accent);
+  --pf-v5-global--link--Color--hover: var(--text);
+  --pf-v5-global--BackgroundColor--100: var(--surface);
+  --pf-v5-global--BackgroundColor--200: var(--input-bg);
+  --pf-v5-global--BorderColor--100: var(--border);
+  --pf-v5-global--danger-color--100: var(--error);
+  --pf-v5-global--success-color--100: var(--accent);
+}
+html.login-pf body#keycloak-bg {
+  background: var(--bg);
+  color: var(--text);
+  font: 16px/1.6 var(--font);
+  height: auto;
+}
+.hexor-heading { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; padding: 30px 36px 0; }
+.hexor-title { margin: 0; font: 500 clamp(28px, 6vw, 34px)/1.3 var(--font); letter-spacing: -1px; }
+.hexor-title::after { content: "_"; color: var(--accent); animation: blink 1.2s step-end infinite; }
+.hexor-body { padding: 28px 36px 34px; }
+.hexor-body > :last-child { margin-bottom: 0; }
+.hexor-locale { max-width: 100%; }
+.hexor-locale select { max-width: 180px; }
+.hexor-body .pf-v5-c-form { gap: 21px; }
+.hexor-body .pf-v5-c-form__group { gap: 8px; }
+.hexor-body .pf-v5-c-form__group-label { padding-bottom: 0 !important; }
+.hexor-body .pf-v5-c-form__label { color: var(--text); font-size: 16px; line-height: 1.6; }
+.hexor-body .pf-v5-c-form__label-text::before { content: ">"; color: var(--accent); margin-inline-end: 12px; }
+.hexor-body .pf-v5-c-form__helper-text { margin-top: 10px; }
+.hexor-body .pf-v5-c-helper-text { gap: 10px; flex-wrap: wrap; }
+.hexor-body .pf-v5-c-helper-text__item-text { color: var(--muted); font-size: 14px; }
+.hexor-body .pf-v5-c-check { color: var(--muted); font-size: 14px; align-items: center; }
+.hexor-body .pf-v5-c-check__label { color: var(--muted); font-size: 14px; }
+.hexor-body .pf-v5-c-form-control, .hexor-locale .pf-v5-c-form-control {
+  background: var(--input-bg);
+  color: var(--text);
+  border: 1px solid var(--border);
+  border-radius: 0;
+  --pf-v5-c-form-control--after--BorderBottomWidth: 0;
+  --pf-v5-c-form-control--before--BorderWidth: 0;
+  --pf-v5-c-form-control--Height: auto;
+}
+.hexor-body .pf-v5-c-form-control::before,
+.hexor-body .pf-v5-c-form-control::after,
+.hexor-locale .pf-v5-c-form-control::before,
+.hexor-locale .pf-v5-c-form-control::after { display: none; }
+.hexor-body .pf-v5-c-form-control:focus-within { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
+.hexor-body .pf-v5-c-form-control.pf-m-error { border-color: var(--error); }
+.hexor-body .pf-v5-c-form-control > :is(input, textarea, select),
+.hexor-locale .pf-v5-c-form-control > select {
+  border: 0;
+  border-radius: 0;
+  padding: 12px 14px;
+  min-height: 50px;
+  background: transparent;
+  color: var(--text);
+  font: 16px/1.6 var(--font);
+  box-shadow: none;
+}
+.hexor-body .pf-v5-c-form-control > input:focus { box-shadow: none; }
+.hexor-body .pf-v5-c-form-control input[readonly] { color: var(--muted); }
+.hexor-body .pf-v5-c-input-group { gap: 0; }
+.hexor-body .pf-v5-c-input-group__item { min-width: 0; }
+.hexor-body .pf-v5-c-form-control__utilities { background: transparent; color: var(--muted); }
+.hexor-body .pf-v5-c-button {
+  min-height: 48px;
+  border: 1px solid var(--border);
+  border-radius: 0;
+  padding: 10px 15px;
+  background: transparent;
+  color: var(--text);
+  font: 16px/1.6 var(--font);
+  box-shadow: none;
+  text-decoration: none;
+  white-space: normal;
+}
+.hexor-body .pf-v5-c-button::after { display: none; }
+.hexor-body .pf-v5-c-button:hover { border-color: var(--accent); color: var(--accent); }
+.hexor-body .pf-v5-c-button:focus-visible { outline: 2px solid var(--accent); outline-offset: 4px; }
+.hexor-body .pf-v5-c-button.pf-m-primary {
+  width: 100%;
+  min-height: 50px;
+  background: var(--accent);
+  border-color: var(--accent);
+  color: var(--accent-ink);
+  font-weight: bold;
+}
+.hexor-body .pf-v5-c-button.pf-m-primary:hover { filter: brightness(1.1); }
+.hexor-body .pf-v5-c-button:disabled, .hexor-body .pf-m-aria-disabled { opacity: .5; pointer-events: none; }
+.hexor-body .pf-v5-c-button.pf-m-control { min-width: 46px; height: 100%; color: var(--muted); background: var(--input-bg); }
+.hexor-body .pf-v5-c-button.pf-m-link { min-height: auto; border-color: transparent; padding-inline: 0; color: var(--accent); }
+.hexor-body .pf-v5-c-form__actions { padding-top: 0 !important; }
+.hexor-body a:not(.pf-v5-c-button) { color: var(--muted); }
+.hexor-body a:not(.pf-v5-c-button):hover { color: var(--accent); }
+.hexor-body .pf-v5-c-alert { border: 0; border-inline-start: 2px solid var(--accent); padding: 10px 13px; background: #ffffff04; color: var(--text); box-shadow: none; }
+.hexor-body .pf-v5-c-alert.pf-m-danger { border-color: var(--error); }
+.hexor-body .pf-v5-c-alert__icon { display: none; }
+.hexor-body .pf-v5-c-alert__title { color: var(--text); font: 15px/1.6 var(--font); }
+.hexor-body .pf-v5-c-alert.pf-m-danger .pf-v5-c-alert__title,
+.hexor-body .pf-v5-c-helper-text__item.pf-m-error .pf-v5-c-helper-text__item-text { color: var(--error); }
+.hexor-footer:empty { display: none; }
+.hexor-footer-band { padding: 0; margin-top: 22px; color: var(--muted); font-size: 14px; }
+.hexor-divider-label { display: flex; align-items: center; gap: 15px; margin: 24px 0 18px; font-size: 11px; letter-spacing: 1px; }
+.hexor-divider-label::before, .hexor-divider-label::after { content: ""; flex: 1; height: 1px; background: var(--border); }
+#kc-social-providers ul { display: flex; flex-wrap: wrap; gap: 12px; padding: 0; margin: 0; list-style: none; }
+#kc-social-providers li { flex: 1 1 160px; padding: 0; }
+#kc-social-providers a { display: flex; justify-content: center !important; align-items: center; gap: 10px; }
+#kc-social-providers svg { width: 20px; height: 20px; flex-shrink: 0; fill: currentColor; }
+#kc-social-providers svg:not(.google) { filter: none; }
+#kc-social-providers .pf-v5-u-m-auto { margin: 0 !important; }
+#kc-select-try-another-way-form { margin-top: 20px; }
+#kc-info-wrapper, #kc-registration { color: var(--muted); font-size: 14px; }
+#kc-otp-login-form #otp { font-size: 24px; letter-spacing: 8px; }
+.hexor-body :is(.pf-v5-c-tile, .pf-v5-c-data-list__item, .pf-v5-c-panel) { background: var(--input-bg); color: var(--text); border: 1px solid var(--border); box-shadow: none; }
+.hexor-body .pf-v5-c-tile.pf-m-selected { border-color: var(--accent); }
+.hexor-body img, .hexor-body canvas, .hexor-body svg { max-width: 100%; }
+.hexor-body pre { overflow-wrap: anywhere; white-space: pre-wrap; }
+@media (max-width: 580px) {
+  .hexor-heading { padding: 24px 24px 0; }
+  .hexor-body { padding: 24px; }
+}

k8s-ng/core/keycloak/themes/hexor-lab/login/resources/img/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><rect width="64" height="64" fill="#0c0f0d"/><text x="7" y="43" fill="#b9ef70" font-family="monospace" font-size="34">&gt;_</text></svg>

k8s-ng/core/keycloak/themes/hexor-lab/login/resources/js/background.js

@@ -0,0 +1,126 @@
+/* Decorative code stream. Brightness: --code-opacity in css/hexor.css. */
+(() => {
+  "use strict";
+  const canvas = document.querySelector("#code-background");
+  const context = canvas?.getContext("2d");
+  if (!context) return;
+
+  const source = [
+    "use std::sync::Arc;",
+    "use tokio::net::TcpListener;",
+    "",
+    "async fn main() -> Result<()> {",
+    "    let config = Config::load()?;",
+    "    let state = Arc::new(State::new());",
+    "    let router = routes(state.clone());",
+    "",
+    "    server::bind(config.address)",
+    "        .serve(router)",
+    "        .await?;",
+    "    Ok(())",
+    "}",
+    "",
+    "impl Session {",
+    "    pub fn verify(&self) -> bool {",
+    "        self.expires_at > clock::now()",
+    "            && self.signature.is_valid()",
+    "    }",
+    "}",
+    "",
+    "fn routes(state: Arc<State>) -> Router {",
+    "    Router::new()",
+    '        .route("/", get(index))',
+    '        .route("/session", post(create))',
+    "        .with_state(state)",
+    "}",
+    "",
+    "match identity.authenticate().await {",
+    "    Ok(session) => session.commit(),",
+    "    Err(error) => trace!(?error),",
+    "}",
+    "",
+    "let transport = Transport::builder()",
+    "    .with_timeout(Duration::from_secs(30))",
+    "    .with_keepalive(true)",
+    "    .connect(endpoint).await?;",
+    "",
+  ];
+  const motion = window.matchMedia("(prefers-reduced-motion: reduce)");
+  const lineHeight = 24;
+  let streams = [];
+  let width = 0;
+  let height = 0;
+  let color;
+
+  function updateColor() {
+    color = getComputedStyle(document.documentElement).getPropertyValue("--accent").trim();
+  }
+
+  function resize() {
+    width = window.innerWidth;
+    height = window.innerHeight;
+    const ratio = Math.min(window.devicePixelRatio || 1, 1.5);
+    canvas.width = Math.round(width * ratio);
+    canvas.height = Math.round(height * ratio);
+    context.setTransform(ratio, 0, 0, ratio, 0, 0);
+    context.font = '13px "SFMono-Regular", Consolas, "Liberation Mono", monospace';
+    const columns = Math.max(1, Math.ceil(width / 470));
+    const rows = Math.ceil(height / lineHeight);
+    streams = Array.from({ length: columns }, (_, column) => {
+      const start = Math.floor(Math.random() * source.length);
+      const count = Math.max(4, rows - 7 + Math.floor(Math.random() * 5));
+      return {
+        x: column * (width / columns) + 24,
+        y: 44 - Math.random() * 80,
+        lines: Array.from({ length: count }, (_, i) => source[(start + i) % source.length]),
+        next: (start + count) % source.length,
+        typed: Math.random() * 12,
+        pause: 0,
+        number: 20 + start,
+        speed: .55 + Math.random() * .6,
+      };
+    });
+    draw(false);
+  }
+
+  function draw(advance) {
+    context.clearRect(0, 0, width, height);
+    context.fillStyle = color;
+    for (const stream of streams) {
+      const current = source[stream.next];
+      if (advance) {
+        if (stream.typed < current.length) stream.typed += stream.speed;
+        else if (++stream.pause > 8) {
+          stream.lines.push(current);
+          stream.next = (stream.next + 1) % source.length;
+          stream.typed = 0;
+          stream.pause = 0;
+        }
+        // Scroll gently as new lines reach the bottom of the viewport.
+        const overflow = stream.y + (stream.lines.length + 2) * lineHeight - height;
+        if (overflow > 0) stream.y -= Math.min(1.5, overflow);
+        if (stream.y < -lineHeight) {
+          stream.lines.shift();
+          stream.y += lineHeight;
+          stream.number++;
+        }
+      }
+      stream.lines.concat(current.slice(0, Math.floor(stream.typed))).forEach((line, i) => {
+        const y = stream.y + i * lineHeight;
+        context.globalAlpha = .3;
+        context.fillText(String(stream.number + i).padStart(3, "0"), stream.x, y);
+        context.globalAlpha = i === stream.lines.length ? .95 : .6;
+        context.fillText(line, stream.x + 42, y);
+      });
+    }
+    context.globalAlpha = 1;
+  }
+
+  updateColor();
+  resize();
+  window.addEventListener("resize", resize);
+  // Ten frames per second is enough for slow typing; no work in hidden tabs.
+  setInterval(() => {
+    if (!document.hidden && !motion.matches) draw(true);
+  }, 100);
+})();

k8s-ng/core/keycloak/themes/hexor-lab/login/template.ftl

@@ -0,0 +1,247 @@
+<#-- Based on Keycloak 26.5.6 keycloak.v2/login/template.ftl (Apache-2.0). -->
+<#import "field.ftl" as field>
+<#import "footer.ftl" as loginFooter>
+<#macro username>
+  <#assign label>
+    <#if !realm.loginWithEmailAllowed>${msg("username")}<#elseif !realm.registrationEmailAsUsername>${msg("usernameOrEmail")}<#else>${msg("email")}</#if>
+  </#assign>
+  <@field.group name="username" label=label>
+    <div class="${properties.kcInputGroup}">
+      <div class="${properties.kcInputGroupItemClass} ${properties.kcFill}">
+        <span class="${properties.kcInputClass} ${properties.kcFormReadOnlyClass}">
+          <input id="kc-attempted-username" value="${auth.attemptedUsername}" readonly>
+        </span>
+      </div>
+      <div class="${properties.kcInputGroupItemClass}">
+        <button id="reset-login" class="${properties.kcFormPasswordVisibilityButtonClass} kc-login-tooltip" type="button"
+              aria-label="${msg('restartLoginTooltip')}" onclick="location.href='${url.loginRestartFlowUrl}'">
+            <i class="fa-sync-alt fas" aria-hidden="true"></i>
+            <span class="kc-tooltip-text">${msg("restartLoginTooltip")}</span>
+        </button>
+      </div>
+    </div>
+  </@field.group>
+</#macro>
+
+<#macro registrationLayout bodyClass="" displayInfo=false displayMessage=true displayRequiredFields=false>
+<!DOCTYPE html>
+<html class="${properties.kcHtmlClass!}" lang="${lang}"<#if realm.internationalizationEnabled> dir="${(locale.rtl)?then('rtl','ltr')}"</#if>>
+
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta name="color-scheme" content="dark">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <#if properties.meta?has_content>
+        <#list properties.meta?split(' ') as meta>
+            <meta name="${meta?split('==')[0]}" content="${meta?split('==')[1]}"/>
+        </#list>
+    </#if>
+    <title>${msg("loginTitle",(realm.displayName!''))}</title>
+    <link rel="icon" href="${url.resourcesPath}/img/favicon.svg" />
+    <#if properties.stylesCommon?has_content>
+        <#list properties.stylesCommon?split(' ') as style>
+            <link href="${url.resourcesCommonPath}/${style}" rel="stylesheet" />
+        </#list>
+    </#if>
+    <#if properties.styles?has_content>
+        <#list properties.styles?split(' ') as style>
+            <link href="${url.resourcesPath}/${style}" rel="stylesheet" />
+        </#list>
+    </#if>
+    <script type="importmap">
+        {
+            "imports": {
+                "rfc4648": "${url.resourcesCommonPath}/vendor/rfc4648/rfc4648.js"
+            }
+        }
+    </script>
+    <#if properties.scripts?has_content>
+        <#list properties.scripts?split(' ') as script>
+            <script src="${url.resourcesPath}/${script}" defer></script>
+        </#list>
+    </#if>
+    <#if scripts??>
+        <#list scripts as script>
+            <script src="${script}" type="text/javascript"></script>
+        </#list>
+    </#if>
+    <script type="module" src="${url.resourcesPath}/js/passwordVisibility.js"></script>
+    <script type="module">
+        <#outputformat "JavaScript">
+        import { startSessionPolling } from "${url.resourcesPath}/js/authChecker.js";
+
+        startSessionPolling(
+            ${url.ssoLoginInOtherTabsUrl?c}
+        );
+        </#outputformat>
+    </script>
+    <script type="module">
+        document.addEventListener("click", (event) => {
+            const link = event.target.closest("a[data-once-link]");
+
+            if (!link) {
+                return;
+            }
+
+            if (link.getAttribute("aria-disabled") === "true") {
+                event.preventDefault();
+                return;
+            }
+
+            const { disabledClass } = link.dataset;
+
+            if (disabledClass) {
+                link.classList.add(...disabledClass.trim().split(/\s+/));
+            }
+
+            link.setAttribute("role", "link");
+            link.setAttribute("aria-disabled", "true");
+        });
+    </script>
+    <#if authenticationSession??>
+        <script type="module">
+             <#outputformat "JavaScript">
+            import { checkAuthSession } from "${url.resourcesPath}/js/authChecker.js";
+
+            checkAuthSession(
+                ${authenticationSession.authSessionIdHash?c}
+            );
+            </#outputformat>
+        </script>
+    </#if>
+    <script>
+      // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=1404468
+      const isFirefox = true;
+    </script>
+</head>
+
+<body id="keycloak-bg" class="${properties.kcBodyClass!}" data-page-id="login-${pageId}">
+<canvas id="code-background" aria-hidden="true"></canvas>
+<header class="site-header"><span class="wordmark">Hexor Lab</span></header>
+<div class="${properties.kcLogin!} page-main">
+  <div class="${properties.kcLoginContainer!}">
+    <main class="${properties.kcLoginMain!} terminal">
+      <div class="titlebar"><span><span aria-hidden="true">&gt;_</span> auth@hexor</span></div>
+      <div class="${properties.kcLoginMainHeader!}">
+        <h1 class="${properties.kcLoginMainTitle!}" id="kc-page-title"><#nested "header"></h1>
+        <#if realm.internationalizationEnabled  && locale.supported?size gt 1>
+        <div class="${properties.kcLoginMainHeaderUtilities!}">
+          <div class="${properties.kcInputClass!}">
+            <select
+              aria-label="${msg("languages")}"
+              id="login-select-toggle"
+              onchange="if (this.value) window.location.href=this.value"
+            >
+              <#list locale.supported?sort_by("label") as l>
+                <option
+                  value="${l.url}"
+                  ${(l.languageTag == locale.currentLanguageTag)?then('selected','')}
+                >
+                  ${l.label}
+                </option>
+              </#list>
+            </select>
+            <span class="${properties.kcFormControlUtilClass}">
+              <span class="${properties.kcFormControlToggleIcon!}">
+                <svg
+                  class="pf-v5-svg"
+                  viewBox="0 0 320 512"
+                  fill="currentColor"
+                  aria-hidden="true"
+                  role="img"
+                  width="1em"
+                  height="1em"
+                >
+                  <path
+                    d="M31.3 192h257.3c17.8 0 26.7 21.5 14.1 34.1L174.1 354.8c-7.8 7.8-20.5 7.8-28.3 0L17.2 226.1C4.6 213.5 13.5 192 31.3 192z"
+                  >
+                  </path>
+                </svg>
+              </span>
+            </span>
+          </div>
+        </div>
+        </#if>
+      </div>
+      <div class="${properties.kcLoginMainBody!}">
+        <#if !(auth?has_content && auth.showUsername() && !auth.showResetCredentials())>
+            <#if displayRequiredFields>
+                <div class="${properties.kcContentWrapperClass!}">
+                    <div class="${properties.kcLabelWrapperClass!} subtitle">
+                        <span class="${properties.kcInputHelperTextItemTextClass!}">
+                          <span class="${properties.kcInputRequiredClass!}">*</span> ${msg("requiredFields")}
+                        </span>
+                    </div>
+                </div>
+            </#if>
+        <#else>
+            <#if displayRequiredFields>
+                <div class="${properties.kcContentWrapperClass!}">
+                    <div class="${properties.kcLabelWrapperClass!} subtitle">
+                        <span class="${properties.kcInputHelperTextItemTextClass!}">
+                          <span class="${properties.kcInputRequiredClass!}">*</span> ${msg("requiredFields")}
+                        </span>
+                    </div>
+                    <div class="${properties.kcFormClass} ${properties.kcContentWrapperClass}">
+                        <#nested "show-username">
+                        <@username />
+                    </div>
+                </div>
+            <#else>
+                <div class="${properties.kcFormClass} ${properties.kcContentWrapperClass}">
+                  <#nested "show-username">
+                  <@username />
+                </div>
+            </#if>
+        </#if>
+
+        <#-- App-initiated actions should not see warning messages about the need to complete the action -->
+        <#-- during login.                                                                               -->
+        <#if displayMessage && message?has_content && (message.type != 'warning' || !isAppInitiatedAction??)>
+            <div class="${properties.kcAlertClass!} pf-m-${(message.type = 'error')?then('danger', message.type)}">
+                <div class="${properties.kcAlertIconClass!}">
+                    <#if message.type = 'success'><span class="${properties.kcFeedbackSuccessIcon!}"></span></#if>
+                    <#if message.type = 'warning'><span class="${properties.kcFeedbackWarningIcon!}"></span></#if>
+                    <#if message.type = 'error'><span class="${properties.kcFeedbackErrorIcon!}"></span></#if>
+                    <#if message.type = 'info'><span class="${properties.kcFeedbackInfoIcon!}"></span></#if>
+                </div>
+                <span class="${properties.kcAlertTitleClass!} kc-feedback-text">${message.summary}</span>
+            </div>
+        </#if>
+
+        <#nested "form">
+
+        <#if auth?has_content && auth.showTryAnotherWayLink()>
+          <form id="kc-select-try-another-way-form" action="${url.loginAction}" method="post" novalidate="novalidate">
+              <input type="hidden" name="tryAnotherWay" value="on"/>
+              <a id="try-another-way" href="javascript:document.forms['kc-select-try-another-way-form'].requestSubmit()"
+                  class="${properties.kcButtonSecondaryClass} ${properties.kcButtonBlockClass} ${properties.kcMarginTopClass}">
+                    ${msg("doTryAnotherWay")}
+              </a>
+          </form>
+        </#if>
+
+          <div class="${properties.kcLoginMainFooter!}">
+              <#nested "socialProviders">
+
+              <#if displayInfo>
+                  <div id="kc-info" class="${properties.kcLoginMainFooterBand!} ${properties.kcFormClass}">
+                      <div id="kc-info-wrapper" class="${properties.kcLoginMainFooterBandItem!}">
+                          <#nested "info">
+                      </div>
+                  </div>
+              </#if>
+          </div>
+      </div>
+
+        <div class="${properties.kcLoginMainFooter!}">
+            <@loginFooter.content/>
+        </div>
+    </main>
+  </div>
+</div>
+</body>
+</html>
+</#macro>

k8s-ng/core/keycloak/themes/hexor-lab/login/theme.properties

@@ -0,0 +1,18 @@
+parent=keycloak.v2
+import=common/keycloak
+styles=css/styles.css css/hexor.css css/keycloak.css
+scripts=js/background.js
+darkMode=false
+
+# Own layout; inherit Keycloak's form fields, authentication and translations.
+kcLogin=hexor-page
+kcLoginContainer=terminal-wrap
+kcLoginMain=hexor-main
+kcLoginMainHeader=hexor-heading
+kcLoginMainTitle=hexor-title
+kcLoginMainBody=hexor-body
+kcLoginMainHeaderUtilities=hexor-locale
+kcLoginMainFooter=hexor-footer
+kcLoginMainFooterBand=hexor-footer-band
+kcLoginMainFooterBandItem=hexor-footer-item
+kcLoginMainFooterHelperText=hexor-divider-label

k8s-ng/core/keycloak/values.yaml

@@ -1,5 +1,9 @@
 replicas: 1
 
+nodeSelector:
+  homelab.hexor.cy/pool: core
+  homelab.hexor.cy/exposure: external
+
 image:
   repository: quay.io/keycloak/keycloak
   tag: "26.5.6"
@@ -24,17 +28,27 @@ extraEnv: |
     value: "-Djgroups.dns.query=keycloak-keycloakx-headless.keycloak.svc"
 
 extraVolumes: |
-  - name: hacker-theme
+  - name: hexor-lab-theme
     configMap:
-      name: keycloak-hacker-theme
+      name: keycloak-hexor-lab-theme
+      items:
+        - key: theme.properties
+          path: theme.properties
+        - key: template.ftl
+          path: template.ftl
+        - key: hexor.css
+          path: resources/css/hexor.css
+        - key: keycloak.css
+          path: resources/css/keycloak.css
+        - key: background.js
+          path: resources/js/background.js
+        - key: favicon.svg
+          path: resources/img/favicon.svg
 
 extraVolumeMounts: |
-  - name: hacker-theme
-    mountPath: /opt/keycloak/themes/hacker/login/theme.properties
-    subPath: theme.properties
-  - name: hacker-theme
-    mountPath: /opt/keycloak/themes/hacker/login/css/hacker.css
-    subPath: hacker.css
+  - name: hexor-lab-theme
+    mountPath: /opt/keycloak/themes/hexor-lab/login
+    readOnly: true
 
 dbchecker:
   enabled: true

k8s-ng/core/postgresql/cluster.yaml

@@ -2,7 +2,11 @@
 # own node with a node-local volume. Replication is asynchronous (nodes sit in
 # different regions), so a failover can lose the last few transactions.
 #
-# Nodes eligible for instances carry the label workload/postgres=true. To move a
+# Layout: the primary runs on the music node (the powerful one), the replica on
+# master. A failover swaps the roles; switch back with
+#   kubectl cnpg promote main <instance on music>
+#
+# Eligible nodes are core/external and carry workload/postgres=true. To move a
 # replica: label the new node, unlabel the old one, then recreate the instance
 # (kubectl cnpg destroy main <N>); the operator re-clones it from the primary.
 #
@@ -32,6 +36,8 @@ spec:
     podAntiAffinityType: required
     topologyKey: kubernetes.io/hostname
     nodeSelector:
+      homelab.hexor.cy/pool: core
+      homelab.hexor.cy/exposure: external
       workload/postgres: "true"
 
   resources:

k8s-ng/core/postgresql/databases/furumi.yaml

@@ -50,7 +50,10 @@ spec:
   name: furumi
   owner: furumi
   extensions:
+    # `ensure` is defaulted by the CRD inside list items; spelled out so Argo CD
+    # does not see it as drift.
     - name: pg_trgm
+      ensure: present
   databaseReclaimPolicy: retain
 ---
 # Lets the furumi namespace read only its own credentials Secret.

k8s-ng/core/postgresql/databases/grafana.yaml

@@ -0,0 +1,78 @@
+# Database "grafana": generated credentials, role, database and read access for
+# the monitoring namespace.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: grafana-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: grafana
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-grafana
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: grafana
+  login: true
+  passwordSecret:
+    name: grafana-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-grafana
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: grafana
+  owner: grafana
+  databaseReclaimPolicy: retain
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: grafana-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [grafana-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: grafana-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: grafana-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: prometheus

k8s-ng/core/postgresql/databases/jacob.yaml

@@ -0,0 +1,55 @@
+# Database "jacob": password, role, database and read access for the app.
+# To add another database copy this file and replace the name.
+
+# Generated once; the role and the application both read this Secret.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: jacob-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: jacob-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: jacob
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-jacob
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: jacob
+  login: true
+  passwordSecret:
+    name: jacob-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-jacob
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: jacob
+  owner: jacob
+  databaseReclaimPolicy: retain
+  extensions:
+    - name: vector
+      ensure: present

k8s-ng/core/postgresql/databases/meguri.yaml

@@ -0,0 +1,81 @@
+# Database "meguri": password, role, database and read access for the app.
+# To add another database copy this file and replace the name.
+
+# Generated once; the role and the application both read this Secret.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: meguri-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: meguri-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: meguri
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-meguri
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: meguri
+  login: true
+  passwordSecret:
+    name: meguri-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-meguri
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: meguri
+  owner: meguri
+  databaseReclaimPolicy: retain
+---
+# Lets the meguri namespace read only its own credentials Secret.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: meguri-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [meguri-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: meguri-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: meguri-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: meguri

k8s-ng/core/postgresql/databases/web-petting.yaml

@@ -0,0 +1,81 @@
+# Database "web_petting": password, role, database and read access for the app.
+# To add another database copy this file and replace the name.
+
+# Generated once; the role and the application both read this Secret.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: web-petting-db
+  namespace: psql
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: web-petting-db
+    template:
+      type: kubernetes.io/basic-auth
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        username: web_petting
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: db-password
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
+metadata:
+  name: main-web-petting
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: web_petting
+  login: true
+  passwordSecret:
+    name: web-petting-db
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: main-web-petting
+  namespace: psql
+spec:
+  cluster:
+    name: main
+  name: web_petting
+  owner: web_petting
+  databaseReclaimPolicy: retain
+---
+# Lets the web-petting namespace read only its own credentials Secret.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: web-petting-db-reader
+  namespace: psql
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [web-petting-db]
+    verbs: [get, list, watch]
+  - apiGroups: [authorization.k8s.io]
+    resources: [selfsubjectrulesreviews]
+    verbs: [create]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: web-petting-db-reader
+  namespace: psql
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: web-petting-db-reader
+subjects:
+  - kind: ServiceAccount
+    name: db-credentials-reader
+    namespace: web-petting

k8s-ng/core/postgresql/kustomization.yaml

@@ -7,3 +7,7 @@ resources:
   - databases/vaultwarden.yaml
   - databases/keycloak.yaml
   - databases/furumi.yaml
+  - databases/web-petting.yaml
+  - databases/meguri.yaml
+  - databases/grafana.yaml
+  - databases/jacob.yaml

k8s-ng/core/prom-stack/README.md

@@ -0,0 +1,63 @@
+# Monitoring stack (new cluster)
+
+The stack is intentionally a fresh installation: no Prometheus TSDB, Loki log
+chunks, or Grafana database are copied from the old cluster.
+
+## Layout
+
+- `kube-prometheus-stack`: Prometheus, Alertmanager, Grafana, exporters and
+  Kubernetes alerting rules.
+- `loki`: one filesystem-backed monolithic instance. A second replica would
+  require shared object storage and is deliberately not faked with a local PV.
+- `alloy`: unprivileged Kubernetes API log and event collection. This replaces
+  EOL Promtail and does not require host log mounts or privileged sysctls.
+- Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on
+  the large local disk of node `music`.
+
+Stack services select `homelab.hexor.cy/pool=core` and
+`homelab.hexor.cy/exposure=external`. Prometheus, Alertmanager and Loki also
+select `music` because their volumes are local. Grafana, the operators, Alloy
+and the Loki gateway can run on either `master` or `music`. Node Exporter and
+Loki Canary are not restricted to these groups.
+
+## Storage and retention
+
+The PreSync hook creates these directories on `music`:
+
+| Path | PV | Retention |
+| --- | ---: | --- |
+| `/data/k8s/monitoring/prometheus` | 500 Gi | up to 365 days / 425 GB |
+| `/data/k8s/monitoring/loki` | 200 Gi | 90 days |
+| `/data/k8s/monitoring/alertmanager` | 5 Gi | 120 hours of notification state |
+
+All PVs and the StorageClass use `Retain`; Argo CD is also told not to prune
+them. This protects data from a manifest rename, but it is not a backup. Back up
+`/data/k8s/monitoring` with the rest of the `music` node if history matters.
+
+## Dependencies
+
+- node `music` must exist and `/data` must be the intended large filesystem;
+- `main-rw.psql.svc` and the generated `grafana-db` Secret must be ready;
+- `vaultwarden-fields` must resolve `k8s/prometheus/grafana-secrets` with the
+  custom fields `IDM Client ID`, `IDM Client Secret`, `TG Token`, and `TG ID`;
+- DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster.
+
+## Alerting and target identities
+
+Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and
+`InfoInhibitor` remain visible in Prometheus but route to the null receiver.
+Warning and critical notifications are grouped for two minutes and rendered as
+Telegram HTML with escaped annotations and labelled alerts/runbook links.
+Notifications show at most three alerts, with each alert's escaped text limited
+to 500 characters. A single short Prometheus alerts link replaces repeated graph
+URLs; the full group remains visible there. Workload alerts show the affected
+DaemonSet, Deployment or StatefulSet rather than the kube-state-metrics pod
+which supplied the metric.
+
+Node Exporter keeps scraping the discovered node address, but rewrites the
+public `instance` label from Kubernetes discovery to `<node-name>:9100`. Grafana
+variables and alerts therefore remain stable when node IP addresses change.
+
+After Argo CD reports the application healthy, verify targets, rules and log
+delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana
+uses its own Keycloak OIDC client.

k8s-ng/core/prom-stack/alertmanager-config.yaml

@@ -0,0 +1,71 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: telegram
+  labels:
+    alertmanager: main
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  route:
+    groupBy: [cluster, namespace, alertname]
+    # A two-minute grace period absorbs alerts which resolve immediately and
+    # never need to interrupt a human.
+    groupWait: 2m
+    groupInterval: 10m
+    repeatInterval: 12h
+    receiver: telegram
+    routes:
+      - receiver: "null"
+        matchers:
+          - name: alertname
+            value: Watchdog|InfoInhibitor
+            matchType: =~
+      # Informational alerts remain queryable in Prometheus but do not page the
+      # Telegram chat. Warning and critical alerts still use the parent route.
+      - receiver: "null"
+        matchers:
+          - name: severity
+            value: info
+            matchType: =
+  receivers:
+    - name: telegram
+      telegramConfigs:
+        - botToken:
+            name: alertmanager-telegram
+            key: bot-token
+          # Telegram chat IDs are routing identifiers rather than credentials;
+          # AlertmanagerConfig only supports an integer here, not a Secret ref.
+          chatID: 124317807
+          sendResolved: true
+          parseMode: HTML
+          # Bound the rendered HTML, including escaped text, before Telegram's
+          # 4096-character limit. Keep the full group accessible in Prometheus.
+          message: |-
+            {{- define "telegram.alert.text" -}}
+            {{- $escaped := html . -}}
+            {{- $short := printf "%.500s" $escaped -}}
+            {{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}…{{ end -}}
+            {{- end -}}
+            {{- $alerts := .Alerts.Resolved -}}
+            {{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}}
+            {{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
+            {{ with .CommonLabels.alertname }}<b>{{ printf "%.64s" . }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ printf "%.16s" . }}</code>{{ end }}
+            {{ range $index, $alert := $alerts }}{{ if lt $index 3 }}
+            {{- $context := "" -}}
+            {{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}}
+            {{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}}
+            {{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}}
+            {{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}}
+            {{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}}
+            {{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}}
+            {{- end -}}
+            {{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}}
+            {{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}}
+
+            {{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }}
+            Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
+            {{ end }}{{ end }}
+            {{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }}
+            <a href="https://prom.hexor.cy/alerts">📈 Open alerts</a>{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} · <a href="{{ . }}">📖 Runbook</a>{{ end }}{{ end }}
+    - name: "null"

k8s-ng/core/prom-stack/alloy-values.yaml

@@ -0,0 +1,161 @@
+alloy:
+  enableReporting: false
+  storagePath: /var/lib/alloy
+  configMap:
+    create: true
+    content: |-
+      discovery.kubernetes "pods" {
+        role = "pod"
+      }
+
+      discovery.relabel "pod_logs" {
+        targets = discovery.kubernetes.pods.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace"]
+          target_label  = "namespace"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          target_label  = "pod"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_name"]
+          target_label  = "container"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_node_name"]
+          target_label  = "node"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_label_app"]
+          regex         = `(.+)`
+          target_label  = "app"
+        }
+        rule {
+          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+          regex         = `(.+)`
+          target_label  = "app"
+        }
+        rule {
+          source_labels = ["namespace", "app"]
+          separator     = "/"
+          regex         = `(.+)/(.+)`
+          replacement   = "$1/$2"
+          target_label  = "job"
+        }
+      }
+
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pod_logs.output
+        forward_to = [loki.process.pods.receiver]
+      }
+
+      loki.process "pods" {
+        stage.static_labels {
+          values = {
+            cluster = "hexor-ng",
+          }
+        }
+        forward_to = [loki.write.local.receiver]
+      }
+
+      loki.source.kubernetes_events "cluster_events" {
+        job_name   = "integrations/kubernetes/eventhandler"
+        log_format = "logfmt"
+        forward_to = [loki.process.cluster_events.receiver]
+      }
+
+      loki.process "cluster_events" {
+        stage.static_labels {
+          values = {
+            cluster = "hexor-ng",
+          }
+        }
+        forward_to = [loki.write.local.receiver]
+      }
+
+      loki.write "local" {
+        endpoint {
+          url = "http://loki-gateway.prometheus.svc/loki/api/v1/push"
+        }
+      }
+  mounts:
+    extra:
+      - name: data
+        mountPath: /var/lib/alloy
+  securityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 473
+    runAsGroup: 473
+    capabilities:
+      drop: [ALL]
+    seccompProfile:
+      type: RuntimeDefault
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 1Gi
+  livenessProbe:
+    httpGet:
+      path: /metrics
+      port: 12345
+      scheme: HTTP
+    initialDelaySeconds: 30
+    periodSeconds: 30
+    timeoutSeconds: 2
+    failureThreshold: 3
+
+global:
+  podSecurityContext:
+    fsGroup: 473
+    fsGroupChangePolicy: OnRootMismatch
+    runAsNonRoot: true
+    seccompProfile:
+      type: RuntimeDefault
+
+controller:
+  type: deployment
+  replicas: 1
+  nodeSelector:
+    homelab.hexor.cy/pool: core
+    homelab.hexor.cy/exposure: external
+  priorityClassName: monitoring-critical
+  minReadySeconds: 10
+  terminationGracePeriodSeconds: 60
+  volumes:
+    extra:
+      - name: data
+        emptyDir: {}
+
+configReloader:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 50Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# Only the API permissions used by pod-log discovery and cluster events.
+rbac:
+  create: true
+  rules:
+    - apiGroups: [""]
+      resources: [pods, pods/log, namespaces, events]
+      verbs: [get, list, watch]
+  # Kept separate because the Alloy chart concatenates this list into the
+  # ClusterRole. Node metadata is useful for diagnosing a log source.
+  clusterRules:
+    - apiGroups: [""]
+      resources: [nodes]
+      verbs: [get, list, watch]
+
+serviceMonitor:
+  enabled: true
+  interval: 30s

k8s-ng/core/prom-stack/dashboards/auth-proxy-dashboard-cm.yaml

@@ -0,0 +1,367 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: auth-proxy-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  auth-proxy.json: |-
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": {
+              "type": "grafana",
+              "uid": "-- Grafana --"
+            },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "type": "dashboard"
+          }
+        ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+          "id": 1,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "routes_loaded_total",
+              "refId": "A"
+            }
+          ],
+          "title": "Loaded Routes",
+          "type": "stat"
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+          "id": 2,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(auth_requests_total[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Auth Requests (total)",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+          "id": 3,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(auth_requests_total{status=\"403\"}[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Forbidden (403)",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "red", "value": 1 }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+          "id": 4,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum(increase(callback_requests_total{result=\"success\"}[$__rate_interval]))",
+              "refId": "A"
+            }
+          ],
+          "title": "Successful Logins",
+          "type": "stat",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+          "id": 5,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum by (status) (rate(auth_requests_total[$__rate_interval]))",
+              "legendFormat": "{{status}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Auth Request Rate by Status",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "reqps"
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+          "id": 6,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.5\"}",
+              "legendFormat": "p50",
+              "refId": "A"
+            },
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.95\"}",
+              "legendFormat": "p95",
+              "refId": "B"
+            },
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "auth_request_duration_seconds{quantile=\"0.99\"}",
+              "legendFormat": "p99",
+              "refId": "C"
+            }
+          ],
+          "title": "Auth Request Duration (p50, p95, p99)",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          }
+        },
+        {
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+          "id": 7,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "pluginVersion": "10.0.0",
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "sum by (result) (rate(callback_requests_total[$__rate_interval]))",
+              "legendFormat": "{{result}}",
+              "refId": "A"
+            }
+          ],
+          "title": "OIDC Callback Rate by Result",
+          "type": "timeseries",
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 50,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "normal" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [ { "color": "green", "value": null } ]
+              },
+              "unit": "reqps"
+            },
+            "overrides": []
+          }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "style": "dark",
+      "tags": [
+        "auth-proxy",
+        "rsauth2-proxy"
+      ],
+      "templating": {
+        "list": [
+          {
+            "current": {
+              "selected": false,
+              "text": "Prometheus",
+              "value": "Prometheus"
+            },
+            "hide": 0,
+            "includeAll": false,
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          }
+        ]
+      },
+      "time": {
+        "from": "now-1h",
+        "to": "now"
+      },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Auth Proxy (rsauth2-proxy)",
+      "uid": "auth-proxy-metrics",
+      "version": 1
+    }

k8s-ng/core/prom-stack/dashboards/furumi-dashboard-cm.yaml

@@ -0,0 +1,504 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: furumi-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  furumi.json: |-
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "type": "dashboard"
+          }
+        ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "id": 1,
+          "title": "Build Version",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
+          "targets": [
+            {
+              "datasource": { "type": "prometheus", "uid": "${datasource}" },
+              "expr": "max by (version) (furumusic_build_info{namespace=~\"$namespace\"})",
+              "legendFormat": "{{version}}",
+              "refId": "A"
+            }
+          ],
+          "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "name" }
+        },
+        {
+          "id": 2,
+          "title": "HTTP RPS",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 3,
+          "title": "5xx Error Ratio",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 4,
+          "title": "HTTP p95 Latency",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 5,
+          "title": "Active Users 15m",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_active_users{namespace=~\"$namespace\",window=\"15m\"})", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        {
+          "id": 6,
+          "title": "Stream Throughput",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
+          "fieldConfig": { "defaults": { "unit": "Bps", "decimals": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
+          ],
+          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
+        },
+        { "id": 10, "title": "HTTP", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "collapsed": false, "panels": [] },
+        {
+          "id": 11,
+          "title": "Request Rate by Route and Status",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
+          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route, status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "{{route}} {{status}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 12,
+          "title": "Request Duration Quantiles",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 13,
+          "title": "In-flight Requests",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route) (furumusic_http_in_flight_requests{namespace=~\"$namespace\",route=~\"$route\"})", "legendFormat": "{{route}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 14,
+          "title": "HTTP Body Throughput",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_request_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "request", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_response_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "response", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 15,
+          "title": "Status Code Mix",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 },
+          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 20, "title": "Auth and Users", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "collapsed": false, "panels": [] },
+        {
+          "id": 21,
+          "title": "Users by Role",
+          "type": "bargauge",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (role) (furumusic_users_total{namespace=~\"$namespace\"})", "legendFormat": "{{role}}", "refId": "A" }
+          ],
+          "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "showUnfilled": true }
+        },
+        {
+          "id": 22,
+          "title": "Active Users",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (furumusic_active_users{namespace=~\"$namespace\"})", "legendFormat": "{{window}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 23,
+          "title": "Auth Events",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method, outcome, reason) (rate(furumusic_auth_login_attempts_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "login {{method}} {{outcome}} {{reason}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method) (rate(furumusic_auth_sessions_created_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "session {{method}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(furumusic_auth_denied_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "denied {{kind}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 30, "title": "Playback and Streaming", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "collapsed": false, "panels": [] },
+        {
+          "id": 31,
+          "title": "Listens Rate",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (completed) (rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "completed={{completed}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 32,
+          "title": "Listen Completion Ratio",
+          "description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 33,
+          "title": "Streaming",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 },
+          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (range) (rate(furumusic_stream_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "requests range={{range}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 34,
+          "title": "Listened Hours and Play History Activity",
+          "description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 },
+          "fieldConfig": {
+            "defaults": { "unit": "h", "decimals": 2 },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 40, "title": "Library and Storage", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] },
+        {
+          "id": 41,
+          "title": "Library Inventory",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_artists_total{namespace=~\"$namespace\"})", "legendFormat": "artists", "refId": "C" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_playlists_total{namespace=~\"$namespace\"})", "legendFormat": "playlists", "refId": "D" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 42,
+          "title": "Media Bytes by Type",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 43,
+          "title": "Storage Used Ratio",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "1 - (sum by (path_kind) (furumusic_storage_free_bytes{namespace=~\"$namespace\"}) / sum by (path_kind) (furumusic_storage_total_bytes{namespace=~\"$namespace\"}))", "legendFormat": "{{path_kind}}", "refId": "A" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 50, "title": "AI Agent", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "collapsed": false, "panels": [] },
+        {
+          "id": 51,
+          "title": "Agent Queue and Reviews",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 52,
+          "title": "Agent Processing and Failures",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome, decision) (rate(furumusic_agent_files_processed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "processed {{outcome}} {{decision}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (stage) (rate(furumusic_agent_failed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "failed {{stage}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 53,
+          "title": "Agent Confidence",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 52 },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 54,
+          "title": "Discover Runs and Duration",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_agent_discover_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "runs {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(furumusic_agent_discover_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 55,
+          "title": "Discover File Flow",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_seen_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "seen", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_queued_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "queued", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_discover_files_skipped_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 56,
+          "title": "RAG and LLM Requests",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind, outcome) (rate(furumusic_agent_rag_queries_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, outcome) (rate(furumusic_agent_llm_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 57,
+          "title": "RAG and LLM Latency p95",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 },
+          "fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, kind, outcome) (rate(furumusic_agent_rag_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, model, outcome) (rate(furumusic_agent_llm_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 58,
+          "title": "LLM Token Rate, Batch Splits and Parse Failures",
+          "description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 },
+          "fieldConfig": {
+            "defaults": { "unit": "suffix: tok/s" },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] },
+              { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model) (rate(furumusic_agent_llm_parse_failures_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "parse failures {{model}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 59,
+          "title": "Cover Pipeline",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, outcome) (rate(furumusic_agent_cover_lookup_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "lookup {{source}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (variant, outcome) (rate(furumusic_agent_cover_variant_generation_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "variant {{variant}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        { "id": 70, "title": "Scheduler and Torrents", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "collapsed": false, "panels": [] },
+        {
+          "id": 71,
+          "title": "Scheduler Jobs",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_running{namespace=~\"$namespace\"})", "legendFormat": "running {{job}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_enabled{namespace=~\"$namespace\"})", "legendFormat": "enabled {{job}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 72,
+          "title": "Scheduler Runs and Duration p95",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job, trigger, outcome) (rate(furumusic_scheduler_job_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{job}} {{trigger}} {{outcome}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, job, trigger, outcome) (rate(furumusic_scheduler_job_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{job}} {{outcome}}", "refId": "B" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        },
+        {
+          "id": 73,
+          "title": "Torrents",
+          "type": "timeseries",
+          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 },
+          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_torrent_sessions_total{namespace=~\"$namespace\"})", "legendFormat": "sessions {{status}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_downloads_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "downloads {{outcome}}", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_selected_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes {{outcome}}", "refId": "C" }
+          ],
+          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "style": "dark",
+      "tags": [ "furumi", "furumusic", "music" ],
+      "templating": {
+        "list": [
+          {
+            "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+            "hide": 0,
+            "includeAll": false,
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          },
+          {
+            "allValue": ".*",
+            "current": { "selected": true, "text": "All", "value": "$__all" },
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(furumusic_build_info, namespace)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "namespace",
+            "options": [],
+            "query": { "query": "label_values(furumusic_build_info, namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          },
+          {
+            "allValue": ".*",
+            "current": { "selected": true, "text": "All", "value": "$__all" },
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)",
+            "hide": 0,
+            "includeAll": true,
+            "multi": true,
+            "name": "route",
+            "options": [],
+            "query": { "query": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          }
+        ]
+      },
+      "time": { "from": "now-6h", "to": "now" },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Furumi Music",
+      "uid": "furumi-music",
+      "version": 1
+    }

k8s-ng/core/prom-stack/dashboards/telemt-dashboard-cm.yaml

@@ -0,0 +1,409 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: telemt-dashboard
+  labels:
+    grafana_dashboard: "1"
+data:
+  telemt.json: |-
+    {
+      "annotations": { "list": [] },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "liveNow": false,
+      "panels": [
+        {
+          "title": "Nodes Overview",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
+          "id": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {
+                "align": "auto",
+                "cellOptions": { "type": "auto" },
+                "inspect": false
+              },
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": { "id": "byName", "options": "Uptime" },
+                "properties": [
+                  { "id": "unit", "value": "dtdurations" },
+                  { "id": "custom.width", "value": 140 }
+                ]
+              },
+              {
+                "matcher": { "id": "byName", "options": "Bad Conn" },
+                "properties": [
+                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 100 }] } },
+                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
+                ]
+              },
+              {
+                "matcher": { "id": "byName", "options": "Writers" },
+                "properties": [
+                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
+                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
+                ]
+              }
+            ]
+          },
+          "options": {
+            "showHeader": true,
+            "sortBy": [{ "displayName": "Node", "desc": false }],
+            "frameIndex": 0,
+            "footer": { "show": false }
+          },
+          "transformations": [
+            {
+              "id": "joinByField",
+              "options": { "byField": "node", "mode": "outer" }
+            },
+            {
+              "id": "filterFieldsByName",
+              "options": {
+                "include": { "pattern": "^(node|Value.*)$" }
+              }
+            },
+            {
+              "id": "organize",
+              "options": {
+                "renameByName": {
+                  "node": "Node",
+                  "Value #uptime": "Uptime",
+                  "Value #writers": "Writers",
+                  "Value #buffers": "Buffers In Use",
+                  "Value #connections": "Connections",
+                  "Value #bad": "Bad Conn",
+                  "Value #hs_timeout": "HS Timeouts"
+                }
+              }
+            }
+          ],
+          "targets": [
+            {
+              "expr": "telemt_uptime_seconds{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "uptime",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_me_writers_active_current{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "writers",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_buffer_pool_buffers_total{node=~\"$node\", kind=\"in_use\"}",
+              "legendFormat": "",
+              "refId": "buffers",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_connections_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "connections",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_connections_bad_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "bad",
+              "format": "table",
+              "instant": true
+            },
+            {
+              "expr": "telemt_handshake_timeouts_total{node=~\"$node\"}",
+              "legendFormat": "",
+              "refId": "hs_timeout",
+              "format": "table",
+              "instant": true
+            }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Connections Rate",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+          "id": 10,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_connections_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} accepted", "refId": "A" },
+            { "expr": "rate(telemt_connections_bad_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} bad", "refId": "B" },
+            { "expr": "rate(telemt_handshake_timeouts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs timeout", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Upstream Connect",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+          "id": 11,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_upstream_connect_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} success", "refId": "A" },
+            { "expr": "rate(telemt_upstream_connect_fail_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} fail", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Upstream Connect Duration (success)",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+          "id": 12,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } },
+              "unit": "short",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "increase(telemt_upstream_connect_duration_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{bucket}}", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "ME Writers & Pool",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+          "id": 13,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_me_writers_active_current{node=~\"$node\"}", "legendFormat": "{{node}} active", "refId": "A" },
+            { "expr": "telemt_me_writers_warm_current{node=~\"$node\"}", "legendFormat": "{{node}} warm", "refId": "B" },
+            { "expr": "telemt_pool_drain_active{node=~\"$node\"}", "legendFormat": "{{node}} draining", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Active Connections",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+          "id": 20,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_user_connections_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}}", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Traffic",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+          "id": 21,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "Bps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_user_octets_from_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} rx", "refId": "A" },
+            { "expr": "rate(telemt_user_octets_to_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} tx", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "DC->Client Payload",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+          "id": 30,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "Bps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_me_d2c_payload_bytes_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} payload", "refId": "A" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "ME Errors & Anomalies",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+          "id": 31,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_me_reconnect_attempts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} reconnect", "refId": "A" },
+            { "expr": "rate(telemt_me_handshake_reject_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs reject", "refId": "B" },
+            { "expr": "rate(telemt_me_crc_mismatch_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} crc mismatch", "refId": "C" },
+            { "expr": "rate(telemt_desync_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} desync", "refId": "D" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Per-User Unique IPs",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+          "id": 40,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "telemt_user_unique_ips_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} active", "refId": "A" },
+            { "expr": "telemt_user_unique_ips_recent_window{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} recent", "refId": "B" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        },
+        {
+          "title": "Conntrack",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+          "id": 41,
+          "fieldConfig": {
+            "defaults": {
+              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
+              "unit": "cps",
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
+            },
+            "overrides": []
+          },
+          "options": {
+            "tooltip": { "mode": "multi", "sort": "desc" },
+            "legend": { "displayMode": "list", "placement": "bottom" }
+          },
+          "targets": [
+            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"attempt\"}[5m])", "legendFormat": "{{node}} delete attempt", "refId": "A" },
+            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"error\"}[5m])", "legendFormat": "{{node}} delete error", "refId": "B" },
+            { "expr": "telemt_conntrack_event_queue_depth{node=~\"$node\"}", "legendFormat": "{{node}} queue depth", "refId": "C" }
+          ],
+          "datasource": { "type": "prometheus", "uid": "${datasource}" }
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 39,
+      "tags": ["telemt", "mtproxy", "telegram"],
+      "templating": {
+        "list": [
+          {
+            "current": {},
+            "hide": 0,
+            "includeAll": false,
+            "label": "Datasource",
+            "multi": false,
+            "name": "datasource",
+            "options": [],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "type": "datasource"
+          },
+          {
+            "current": {},
+            "datasource": { "type": "prometheus", "uid": "${datasource}" },
+            "definition": "label_values(telemt_uptime_seconds, node)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Node",
+            "multi": true,
+            "name": "node",
+            "query": "label_values(telemt_uptime_seconds, node)",
+            "refresh": 2,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "type": "query"
+          }
+        ]
+      },
+      "time": { "from": "now-6h", "to": "now" },
+      "title": "Telemt MTProxy",
+      "uid": "telemt-mtproxy"
+    }

k8s-ng/core/prom-stack/external-secrets.yaml

@@ -0,0 +1,146 @@
+# Read-only access to the Grafana database credentials in the psql namespace.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: db-credentials-reader
+  annotations:
+    argocd.argoproj.io/sync-wave: "-3"
+---
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: psql-credentials
+  annotations:
+    argocd.argoproj.io/sync-wave: "-2"
+spec:
+  provider:
+    kubernetes:
+      remoteNamespace: psql
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          key: ca.crt
+      auth:
+        serviceAccount:
+          name: db-credentials-reader
+---
+# A local break-glass account. OIDC remains the normal login path.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: grafana-admin
+  annotations:
+    argocd.argoproj.io/sync-wave: "-2"
+spec:
+  length: 40
+  digits: 10
+  symbols: 0
+  noUpper: false
+  allowRepeat: true
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-admin
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshPolicy: CreatedOnce
+  target:
+    name: grafana-admin
+    template:
+      type: Opaque
+      data:
+        admin-user: admin
+        admin-password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: grafana-admin
+---
+# Grafana's database credentials come from CNPG. OIDC credentials are read by
+# stable Vaultwarden item and custom-field names, so imports and field ordering
+# do not invalidate the references.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-env
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: 1h
+  target:
+    name: grafana-env
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        GF_DATABASE_USER: "{{ .db_username }}"
+        GF_DATABASE_PASSWORD: "{{ .db_password }}"
+        GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .client_id }}"
+        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .client_secret }}"
+  data:
+    - secretKey: db_username
+      remoteRef:
+        key: grafana-db
+        property: username
+    - secretKey: db_password
+      remoteRef:
+        key: grafana-db
+        property: password
+    - secretKey: client_id
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: IDM Client ID
+    - secretKey: client_secret
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: IDM Client Secret
+  secretStoreRef:
+    kind: SecretStore
+    name: psql-credentials
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-telegram
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: 1h
+  target:
+    name: alertmanager-telegram
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        bot-token: "{{ .bot_token }}"
+        chat-id: "{{ .chat_id }}"
+  data:
+    - secretKey: bot_token
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: TG Token
+    - secretKey: chat_id
+      sourceRef:
+        storeRef:
+          name: vaultwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: k8s/prometheus/grafana-secrets
+        property: TG ID