Provision Jacob database with pgvector through GitOps

Enable Hexor Lab login theme in Keycloak realm
Deploy Hexor Lab Phosphor login theme
2026-10-27 01:35:38 +02:00 · 2026-10-21 21:36:21 +01:00 · 2026-10-21 21:31:25 +01:00 · 2026-10-21 12:33:00 +01:00 · 2026-10-21 12:10:56 +01:00 · 2026-10-21 11:54:18 +01:00
125 changed files with 7365 additions and 456 deletions
@@ -30,9 +30,11 @@ jobs:
        cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
    - name: Setup kubeconfig
      env:
        CI_KUBECONFIG: ${{ secrets.KUBECONFIG }}
      run: |
        mkdir -p ~/.kube
-        echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
+        printf '%s' "$CI_KUBECONFIG" > ~/.kube/config
        chmod 600 ~/.kube/config
    - name: Terraform Init
@@ -0,0 +1,50 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: furumi-player
  labels:
    app: furumi-player
 spec:
  replicas: 1
  strategy:
    type: Recreate # the library is a hostPath on a single node
  selector:
    matchLabels:
      app: furumi-player
  template:
    metadata:
      labels:
        app: furumi-player
      annotations:
        reloader.stakater.com/auto: "true"
    spec:
      # The music library lives on this node's /data (LVM raid, not replicated).
      nodeSelector:
        kubernetes.io/hostname: music
      containers:
        - name: furumi-player
          image: ultradesu/furumusic:latest
          imagePullPolicy: Always
          env:
            - name: FURU_DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: furumi-creds
                  key: PG_STRING
          ports:
            - name: http
              containerPort: 8000
          volumeMounts:
            - name: library
              mountPath: /media
            - name: inbox
              mountPath: /inbox
      volumes:
        - name: library
          hostPath:
            path: /data/furumi/library
            type: Directory
        - name: inbox
          hostPath:
            path: /data/furumi/inbox
            type: Directory
@@ -0,0 +1,24 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: furumi
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
 spec:
  ingressClassName: traefik
  rules:
    - host: music.hexor.cy
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: furumi-player
                port:
                  number: 8000
  tls:
    - secretName: furumi-tls
      hosts:
        - music.hexor.cy
@@ -0,0 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - secrets.yaml
  - deployment.yaml
  - service.yaml
  - servicemonitor.yaml
  - ingress.yaml
@@ -0,0 +1,47 @@
 # Read-only access to the furumi-db Secret in the psql namespace
 # (see postgresql/databases/furumi.yaml for the other half).
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: db-credentials-reader
 ---
 apiVersion: external-secrets.io/v1
 kind: SecretStore
 metadata:
  name: psql-credentials
 spec:
  provider:
    kubernetes:
      remoteNamespace: psql
      server:
        caProvider:
          type: ConfigMap
          name: kube-root-ca.crt
          key: ca.crt
      auth:
        serviceAccount:
          name: db-credentials-reader
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: furumi-creds
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: SecretStore
    name: psql-credentials
  target:
    name: furumi-creds
    template:
      data:
        PG_STRING: "postgresql://{{ .username }}:{{ .password }}@main-rw.psql.svc:5432/furumi"
  data:
    - secretKey: username
      remoteRef:
        key: furumi-db
        property: username
    - secretKey: password
      remoteRef:
        key: furumi-db
        property: password
@@ -0,0 +1,16 @@
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: furumi-player
  labels:
    app: furumi-player
 spec:
  type: ClusterIP
  selector:
    app: furumi-player
  ports:
    - name: http
      protocol: TCP
      port: 8000
      targetPort: 8000
@@ -0,0 +1,13 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: furumi-player
 spec:
  selector:
    matchLabels:
      app: furumi-player
  endpoints:
    - port: http
      path: /metrics
      interval: 30s
      scrapeTimeout: 10s
@@ -0,0 +1,12 @@
 # k8s-secrets
 The `k8s-secrets` Argo CD Application reads this directory through
 `core-state/values.yaml` and deploys into namespace `k8s-secret`.
 External Secrets loads the `k8s/openai/creds` Vaultwarden item into
 `openai-creds`. The reader at `pass.hexor.cy` can only read that Secret and is
 protected by the Keycloak forward-auth proxy. Its access policy is managed in
 `terraform/keycloak/terraform.tfvars`.
 The Kubernetes resource names and namespace match the former deployment so
 Argo CD can switch the source path without replacing the service or certificate.
@@ -0,0 +1,32 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: openai-creds
 spec:
  refreshInterval: 5m
  secretStoreRef:
    name: vaultwarden-login
    kind: ClusterSecretStore
  target:
    name: openai-creds
    creationPolicy: Owner
    deletionPolicy: Retain
    template:
      type: Opaque
      data:
        USER: "{{ .user }}"
        PASS: "{{ .pass }}"
        TOTP: "{{ .totp }}"
  data:
    - secretKey: user
      remoteRef:
        key: k8s/openai/creds
        property: login.username
    - secretKey: pass
      remoteRef:
        key: k8s/openai/creds
        property: login.password
    - secretKey: totp
      remoteRef:
        key: k8s/openai/creds
        property: login.totp
@@ -0,0 +1,38 @@
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: auth-proxy
 spec:
  forwardAuth:
    address: http://auth-proxy.keycloak.svc:80/auth
    trustForwardHeader: false
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
 metadata:
  name: secret-reader
 spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`pass.hexor.cy`)
      kind: Rule
      middlewares:
        - name: auth-proxy
      services:
        - name: secret-reader
          port: 80
  tls:
    secretName: secret-reader-tls
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: secret-reader-tls
 spec:
  secretName: secret-reader-tls
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer
  dnsNames:
    - pass.hexor.cy
@@ -0,0 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: k8s-secret
 resources:
  - external-secret.yaml
  - reader.yaml
  - ingress.yaml
@@ -0,0 +1,109 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: secret-reader
  labels:
    app: secret-reader
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: secret-reader
 rules:
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["openai-creds"]
    verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: secret-reader
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: secret-reader
 subjects:
  - kind: ServiceAccount
    name: secret-reader
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: secret-reader
  labels:
    app: secret-reader
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: secret-reader
  template:
    metadata:
      labels:
        app: secret-reader
    spec:
      serviceAccountName: secret-reader
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: external
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: secret-reader
          image: ultradesu/k8s-secrets:0.2.2@sha256:f206942d21ca9dcd131d6e19ba26cc17671815919217aef657b30a9f85854778
          imagePullPolicy: IfNotPresent
          args:
            - --secrets
            - openai-creds
            - --namespace
            - k8s-secret
            - --port
            - "3000"
          ports:
            - name: http
              containerPort: 3000
          env:
            - name: RUST_LOG
              value: info
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 150m
              memory: 128Mi
          readinessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 5
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 10
            periodSeconds: 10
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: secret-reader
  labels:
    app: secret-reader
 spec:
  selector:
    app: secret-reader
  ports:
    - name: http
      port: 80
      targetPort: http
@@ -0,0 +1,18 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: llamacpp-config
 data:
  LLAMA_CACHE: /models
  LLAMA_ARG_HOST: 0.0.0.0
  LLAMA_ARG_PORT: "8080"
  # The model is chosen here: <hf-user>/<repo>:<quant>. It is downloaded into
  # /models on first start; changing it restarts the pod (reloader).
  #LLAMA_ARG_HF_REPO: "unsloth/Qwen3.8-27B-GGUF:UD-Q6_K"
  LLAMA_ARG_HF_REPO: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K"
  LLAMA_ARG_CTX_SIZE: "131072"
  LLAMA_ARG_N_GPU_LAYERS: all
  LLAMA_ARG_FLASH_ATTN: "on"
  LLAMA_ARG_FIT: "on"
  # Strix Halo: mmap-ed weights hang or corrupt on ROCm; read them directly.
  LLAMA_ARG_LOAD_MODE: dio
@@ -0,0 +1,76 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: llamacpp
  annotations:
    reloader.stakater.com/auto: "true"
  labels:
    app: llamacpp
 spec:
  replicas: 1
  strategy:
    type: Recreate # one GPU, one pod
  selector:
    matchLabels:
      app: llamacpp
  template:
    metadata:
      labels:
        app: llamacpp
    spec:
      dnsPolicy: Default
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: internal
        workload/ai: "true"
      tolerations:
        - key: workload
          operator: Equal
          value: ai
          effect: NoSchedule
      containers:
        - name: llamacpp
          image: ghcr.io/ggml-org/llama.cpp:server-rocm-b11434
          imagePullPolicy: IfNotPresent
          envFrom:
            - configMapRef:
                name: llamacpp-config
          env:
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef:
                  name: llamacpp-hf-token
                  key: token
                  optional: true
            # hipBLASLt gives the biggest prompt-processing gain on ROCm 7.x.
            - name: ROCBLAS_USE_HIPBLASLT
              value: "1"
          ports:
            - name: http
              containerPort: 8080
              protocol: TCP
          resources:
            limits:
              amd.com/gpu: 1
          startupProbe:
            httpGet:
              path: /health
              port: http
            failureThreshold: 180 # first start downloads the model
            periodSeconds: 10
            timeoutSeconds: 5
          readinessProbe:
            httpGet:
              path: /health
              port: http
            failureThreshold: 3
            periodSeconds: 10
            timeoutSeconds: 5
          volumeMounts:
            - name: models
              mountPath: /models
      volumes:
        - name: models
          hostPath:
            path: /var/lib/llamacpp/models
            type: DirectoryOrCreate
@@ -0,0 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - configmap.yaml
  - deployment.yaml
  - service.yaml
@@ -0,0 +1,15 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: llamacpp
  labels:
    app: llamacpp
 spec:
  type: ClusterIP
  selector:
    app: llamacpp
  ports:
    - name: http
      port: 8080
      targetPort: http
      protocol: TCP
@@ -0,0 +1,561 @@
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: nginx-config
  namespace: minecraft
 data:
  nginx.conf: |
    user nginx;
    worker_processes 1;
    error_log /var/log/nginx/error.log warn;
    pid /var/run/nginx.pid;
    events {
        worker_connections 1024;
    }
    http {
        include /etc/nginx/mime.types;
        default_type application/octet-stream;
        server {
            listen 80;
            # Custom 502 error page with auto-refresh
            error_page 502 /502.html;
            location = /502.html {
                internal;
                return 200 '<!DOCTYPE html><html><head><meta charset="utf-8"><title>Server Loading</title><style>body{font-family:Arial,sans-serif;text-align:center;margin-top:100px;background:#f0f0f0}h1{color:#333}p{color:#666;font-size:18px}</style></head><body><h1>Server is loading probably...</h1><p>Please wait a moment and try refreshing the page.</p><script>setTimeout(function(){window.location.reload();}, 10000);</script></body></html>';
                add_header Content-Type text/html;
            }
            # Main location - proxy to Minecraft Dynmap
            location / {
                # Proxy configuration for Dynmap server
                proxy_pass http://localhost:8123;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                # Inject user authentication meta tag into HTML head
                sub_filter '<head>' '<head><meta name="remote-user" content="$http_x_authentik_username">';
                # Replace default Dynmap title with custom server name
                sub_filter 'Minecraft Dynamic Map' "Hexor's MC server";
                # Inject custom script before closing body tag
                sub_filter "</body>" '<script src="/clients/inject.js"></script></body>';
                # Apply sub_filter replacements globally (not just once)
                sub_filter_once off;
            }
            # Serve inject.js and .ps1 scripts inline (no forced download)
            location = /clients/inject.js {
                alias /mc/clients/inject.js;
                default_type application/javascript;
            }
            location ~ ^/clients/(.+\.ps1)$ {
                alias /mc/clients/$1;
                default_type text/plain;
            }
            # Static file serving for client downloads
            location /clients/ {
                alias /mc/clients/;
                sendfile on;                          # Enable efficient file serving
                add_header Content-Disposition "attachment";  # Force download
                autoindex on;                         # Enable directory listing
                gzip off;                            # Disable compression for downloads
                chunked_transfer_encoding off;       # Disable chunked encoding
            }
        }
    }
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: client-scripts
  namespace: minecraft
 data:
  inject.js: |
    (function() {
      function getUsername() {
        var metas = document.querySelectorAll("meta");
        for (var i = 0; i < metas.length; i++) {
          if (metas[i].getAttribute("name") === "remote-user") {
            return metas[i].getAttribute("content");
          }
        }
        var jwt = document.cookie.split("; ").find(function(row) {
          return row.startsWith("authentik_session=");
        });
        if (jwt) {
          try {
            var token = jwt.split("=")[1];
            var payload = JSON.parse(atob(token.split(".")[1]));
            return payload.sub || payload.username || "web-user";
          } catch(e) {}
        }
        return "web-user";
      }
      var username = getUsername();
      console.log("Username found:", username);
      if (username && username !== "web-user" &&
          window.location.search.indexOf("playername=") === -1) {
        var currentUrl = new URL(window.location.href);
        currentUrl.searchParams.set("playername", username);
        console.log("Redirecting to:", currentUrl.href);
        window.location.href = currentUrl.href;
      }
      document.addEventListener("DOMContentLoaded", function() {
        // User block
        var userBlock = document.createElement("div");
        userBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:8px;text-align:center;font-size:medium;border-radius:4px;position:absolute;top:10px;right:150px;max-width:200px;";
        userBlock.innerHTML = "Logged in as: <b>" + username + "</b>";
        document.body.appendChild(userBlock);
        // Info block
        var infoBlock = document.createElement("p");
        infoBlock.style.cssText = "background-color:#CEC6CB;color:black;padding:10px;text-align:center;font-size:large;display:inline-block;border-radius:4px;position:absolute;top:10px;left:150px;";
        infoBlock.innerHTML = 'GEYMERSKIY SOYUZ Server<br>'
          + 'Get <a href="https://github.com/PrismLauncher/PrismLauncher/releases/tag/8.4">Prism Launcher</a> '
          + 'and <a href="/clients/1.12.2.zip">client.zip</a> for this server. '
          + 'Server address <b>minecraft.hexor.cy:30565</b><br>'
          + 'Requires <a href="https://www.java.com/en/download/manual.jsp">Java 8</a><br><br>'
          + '<a href="#" id="showInstallBtn" style="color:black;text-decoration:underline;">Windows Install Script</a>';
        document.body.appendChild(infoBlock);
        // Modal
        var modal = document.createElement("div");
        modal.id = "installModal";
        modal.style.cssText = "display:none;position:fixed;z-index:1000;left:0;top:0;width:100%;height:100%;background-color:rgba(0,0,0,0.5);";
        modal.innerHTML = '<div style="background-color:#CEC6CB;margin:15% auto;padding:10px;border-radius:4px;width:70%;max-width:500px;text-align:center;color:black;font-size:large;">'
          + '<h3 style="margin-top:0;color:black;">Windows Installation</h3>'
          + '<p style="color:black;">Copy and paste this command into PowerShell:</p>'
          + '<textarea id="scriptCommand" readonly style="width:90%;height:60px;font-family:monospace;padding:8px;border:1px solid #888;border-radius:4px;resize:none;background-color:white;color:black;"></textarea>'
          + '<br><br>'
          + '<button id="copyButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;margin-right:10px;font-size:large;">Copy</button>'
          + '<button id="closeButton" style="background-color:#CEC6CB;color:black;padding:10px 15px;border:1px solid #888;border-radius:4px;cursor:pointer;font-size:large;">Close</button>'
          + '</div>';
        document.body.appendChild(modal);
        // Generate PowerShell command with username
        function buildPsCommand() {
          var d = "$";
          var q = "'";
          return d + 'f="' + d + 'env:TEMP\\mc-install.ps1"; iwr -useb https://minecraft.hexor.cy/clients/win-install.ps1 -OutFile '
            + d + 'f; powershell -ExecutionPolicy Bypass -File ' + d + 'f -Username ' + q + username + q + '; Remove-Item ' + d + 'f';
        }
        document.getElementById("showInstallBtn").addEventListener("click", function(e) {
          e.preventDefault();
          modal.style.display = "block";
          document.getElementById("scriptCommand").value = buildPsCommand();
        });
        document.getElementById("closeButton").addEventListener("click", function() {
          modal.style.display = "none";
        });
        document.getElementById("copyButton").addEventListener("click", function() {
          var textarea = document.getElementById("scriptCommand");
          textarea.select();
          textarea.setSelectionRange(0, 99999);
          document.execCommand("copy");
          var btn = document.getElementById("copyButton");
          btn.style.borderColor = "#4CAF50";
          setTimeout(function() { btn.style.borderColor = "#888"; }, 2000);
        });
        modal.addEventListener("click", function(event) {
          if (event.target === modal) {
            modal.style.display = "none";
          }
        });
      });
    })();
  win-install.ps1: |
    # Game Setup Script for PrismLauncher and Minecraft Client
    # This script downloads and configures PrismLauncher with Hexor client
    param(
        [string]$Username = "",
        [string]$InstallPath = "$env:USERPROFILE\Games\PrismLauncher"
    )
    # Enable TLS 1.2 for downloads
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    # Function to write colored log messages
    function Write-Log {
        param(
            [string]$Message,
            [string]$Level = "INFO"
        )
        switch ($Level) {
            "INFO" {
                Write-Host "[" -NoNewline
                Write-Host "INFO" -ForegroundColor Blue -NoNewline
                Write-Host "] $Message"
            }
            "WARN" {
                Write-Host "[" -NoNewline
                Write-Host "WARN" -ForegroundColor Yellow -NoNewline
                Write-Host "] $Message"
            }
            "ERROR" {
                Write-Host "[" -NoNewline
                Write-Host "ERROR" -ForegroundColor Red -NoNewline
                Write-Host "] $Message"
            }
            "SUCCESS" {
                Write-Host "[" -NoNewline
                Write-Host "SUCCESS" -ForegroundColor Green -NoNewline
                Write-Host "] $Message"
            }
        }
    }
    # Function to download file with progress
    function Download-File {
        param(
            [string]$Url,
            [string]$OutputPath
        )
        try {
            Write-Log "Downloading from: $Url"
            Write-Log "Saving to: $OutputPath"
            if (Test-Path $OutputPath) {
                Remove-Item $OutputPath -Force
                Write-Log "Removed existing file to avoid corruption" "WARN"
            }
            $webClient = New-Object System.Net.WebClient
            $webClient.DownloadFile($Url, $OutputPath)
            if (Test-Path $OutputPath) {
                $fileSize = (Get-Item $OutputPath).Length
                Write-Log "Download completed successfully ($fileSize bytes)" "SUCCESS"
                return $true
            } else {
                Write-Log "Download failed - file not found" "ERROR"
                return $false
            }
        }
        catch {
            Write-Log "Download failed: $($_.Exception.Message)" "ERROR"
            return $false
        }
    }
    # Function to extract ZIP archive
    function Extract-Archive {
        param(
            [string]$ArchivePath,
            [string]$DestinationPath
        )
        try {
            Write-Log "Extracting archive: $ArchivePath"
            Write-Log "Destination: $DestinationPath"
            if (!(Test-Path $ArchivePath)) {
                Write-Log "Archive file not found: $ArchivePath" "ERROR"
                return $false
            }
            $fileSize = (Get-Item $ArchivePath).Length
            if ($fileSize -lt 1000) {
                Write-Log "Archive file too small ($fileSize bytes), probably corrupted" "ERROR"
                return $false
            }
            if (!(Test-Path $DestinationPath)) {
                New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null
            }
            $existingFiles = Get-ChildItem -Path $DestinationPath -File | Where-Object { $_.Extension -ne ".zip" }
            $existingDirs = Get-ChildItem -Path $DestinationPath -Directory
            if ($existingFiles.Count -gt 0 -or $existingDirs.Count -gt 0) {
                Write-Log "Clearing existing files in destination directory" "WARN"
                $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
                $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
            }
            Add-Type -AssemblyName System.IO.Compression.FileSystem
            try {
                $zip = [System.IO.Compression.ZipFile]::OpenRead($ArchivePath)
                $entryCount = $zip.Entries.Count
                $zip.Dispose()
                Write-Log "ZIP file validated ($entryCount entries)" "SUCCESS"
            }
            catch {
                Write-Log "ZIP file validation failed: $($_.Exception.Message)" "ERROR"
                return $false
            }
            [System.IO.Compression.ZipFile]::ExtractToDirectory($ArchivePath, $DestinationPath)
            Write-Log "Archive extracted successfully" "SUCCESS"
            return $true
        }
        catch {
            Write-Log "Archive extraction failed: $($_.Exception.Message)" "ERROR"
            if (Test-Path $ArchivePath) {
                Write-Log "Removing potentially corrupted archive file" "WARN"
                Remove-Item $ArchivePath -Force -ErrorAction SilentlyContinue
            }
            return $false
        }
    }
    # Function to check Java installation
    function Check-JavaInstallation {
        try {
            $javaVersion = java -version 2>&1 | Select-String "version"
            if ($javaVersion) {
                Write-Log "Java is installed: $($javaVersion.ToString().Trim())" "SUCCESS"
                return $true
            }
        }
        catch {
            Write-Log "Java is not installed or not in PATH" "WARN"
            Write-Log "Please download Java from: https://www.java.com/en/download/manual.jsp" "WARN"
            Write-Log "Look for 'Windows Offline (64-bit)' version" "WARN"
            $response = Read-Host "Do you want to continue without Java? (y/n)"
            if ($response -eq 'y' -or $response -eq 'Y') {
                Write-Log "Continuing without Java verification" "WARN"
                return $true
            } else {
                Write-Log "Installation cancelled. Please install Java first." "ERROR"
                return $false
            }
        }
    }
    # Function to generate a random hex string (UUID-like without dashes)
    function New-RandomHexId {
        param([int]$Length = 32)
        $bytes = New-Object byte[] ($Length / 2)
        $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
        $rng.GetBytes($bytes)
        $rng.Dispose()
        return ($bytes | ForEach-Object { $_.ToString("x2") }) -join ''
    }
    # Function to create accounts.json file with offline account
    function Create-AccountsFile {
        param(
            [string]$TargetPath,
            [string]$PlayerName
        )
        $accountsPath = Join-Path $TargetPath "accounts.json"
        $profileId = New-RandomHexId -Length 32
        $clientToken = New-RandomHexId -Length 32
        $iat = [int][double]::Parse((Get-Date -UFormat %s))
        $accountsObj = @{
            accounts = @(
                @{
                    entitlement = @{
                        canPlayMinecraft = $true
                        ownsMinecraft = $true
                    }
                    "msa-client-id" = ""
                    type = "MSA"
                },
                @{
                    active = $true
                    entitlement = @{
                        canPlayMinecraft = $true
                        ownsMinecraft = $true
                    }
                    profile = @{
                        capes = @()
                        id = $profileId
                        name = $PlayerName
                        skin = @{
                            id = ""
                            url = ""
                            variant = ""
                        }
                    }
                    type = "Offline"
                    ygg = @{
                        extra = @{
                            clientToken = $clientToken
                            userName = $PlayerName
                        }
                        iat = $iat
                        token = "0"
                    }
                }
            )
            formatVersion = 3
        }
        try {
            $accountsObj | ConvertTo-Json -Depth 10 | Out-File -FilePath $accountsPath -Encoding UTF8
            Write-Log "Created accounts.json at: $accountsPath" "SUCCESS"
            Write-Log "Player name: $PlayerName" "SUCCESS"
            return $true
        }
        catch {
            Write-Log "Failed to create accounts.json: $($_.Exception.Message)" "ERROR"
            return $false
        }
    }
    # Main installation process
    Write-Log "Starting PrismLauncher and Minecraft client setup"
    Write-Log "Player name: $Username"
    Write-Log "Installation path: $InstallPath"
    # Create installation directory and clear if exists (but keep ZIP files)
    if (Test-Path $InstallPath) {
        Write-Log "Installation directory exists, clearing contents" "WARN"
        $existingFiles = Get-ChildItem -Path $InstallPath -File | Where-Object { $_.Extension -ne ".zip" }
        $existingDirs = Get-ChildItem -Path $InstallPath -Directory
        $existingFiles | Remove-Item -Force -ErrorAction SilentlyContinue
        $existingDirs | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
    } else {
        New-Item -ItemType Directory -Path $InstallPath -Force | Out-Null
        Write-Log "Created installation directory: $InstallPath"
    }
    # Step 1: Download PrismLauncher
    Write-Log "Step 1: Downloading PrismLauncher..."
    $launcherUrl = "https://github.com/PrismLauncher/PrismLauncher/releases/download/8.4/PrismLauncher-Windows-MSVC-Portable-8.4.zip"
    $launcherZip = Join-Path $InstallPath "PrismLauncher-8.4.zip"
    $launcherExtractPath = Join-Path $InstallPath "PrismLauncher-Windows-MSVC-Portable-8.4"
    if (!(Download-File -Url $launcherUrl -OutputPath $launcherZip)) {
        Write-Log "Failed to download PrismLauncher. Exiting." "ERROR"
        exit 1
    }
    # Step 2: Extract PrismLauncher
    Write-Log "Step 2: Extracting PrismLauncher..."
    if (!(Extract-Archive -ArchivePath $launcherZip -DestinationPath $InstallPath)) {
        Write-Log "Failed to extract PrismLauncher. Exiting." "ERROR"
        exit 1
    }
    # Find the actual extracted directory
    $extractedDirs = Get-ChildItem -Path $InstallPath -Directory | Where-Object { $_.Name -like "*PrismLauncher*" }
    if ($extractedDirs.Count -gt 0) {
        $launcherExtractPath = $extractedDirs[0].FullName
        Write-Log "Found PrismLauncher directory: $launcherExtractPath" "SUCCESS"
    } else {
        Write-Log "Could not find extracted PrismLauncher directory. Checking for direct extraction..." "WARN"
        $prismExe = Join-Path $InstallPath "prismlauncher.exe"
        if (Test-Path $prismExe) {
            $launcherExtractPath = $InstallPath
            Write-Log "PrismLauncher extracted directly to: $launcherExtractPath" "SUCCESS"
        } else {
            Write-Log "Failed to locate PrismLauncher files. Exiting." "ERROR"
            exit 1
        }
    }
    # Step 3: Create accounts.json in InstallPath (only if username provided)
    if ($Username -and $Username -ne "web-user") {
        Write-Log "Step 3: Creating accounts configuration for $Username..."
        if (!(Create-AccountsFile -TargetPath $InstallPath -PlayerName $Username)) {
            Write-Log "Failed to create accounts.json. Exiting." "ERROR"
            exit 1
        }
    } else {
        Write-Log "Step 3: No username provided, skipping accounts.json creation" "WARN"
    }
    # Step 4: Check Java installation
    Write-Log "Step 4: Checking Java installation..."
    if (!(Check-JavaInstallation)) {
        Write-Log "Java check failed. Exiting." "ERROR"
        exit 1
    }
    # Step 5: Download Minecraft client
    Write-Log "Step 5: Downloading Minecraft client..."
    $minecraftUrl = "https://minecraft.hexor.cy/clients/1.12.2.zip"
    $minecraftZip = Join-Path $InstallPath "minecraft-1.12.2.zip"
    $instancesPath = Join-Path $launcherExtractPath "instances"
    $hexorPath = Join-Path $instancesPath "Hexor"
    if (!(Download-File -Url $minecraftUrl -OutputPath $minecraftZip)) {
        Write-Log "Failed to download Minecraft client. Exiting." "ERROR"
        exit 1
    }
    # Step 6: Create instances directory and extract Minecraft client
    Write-Log "Step 6: Setting up Minecraft client..."
    if (!(Test-Path $instancesPath)) {
        New-Item -ItemType Directory -Path $instancesPath -Force | Out-Null
        Write-Log "Created instances directory"
    }
    if (!(Test-Path $hexorPath)) {
        New-Item -ItemType Directory -Path $hexorPath -Force | Out-Null
        Write-Log "Created Hexor instance directory"
    }
    if (!(Extract-Archive -ArchivePath $minecraftZip -DestinationPath $hexorPath)) {
        Write-Log "Failed to extract Minecraft client. Exiting." "ERROR"
        exit 1
    }
    # Step 7: Cleanup temporary files
    Write-Log "Step 7: Cleaning up temporary files..."
    try {
        Remove-Item $launcherZip -Force
        Remove-Item $minecraftZip -Force
        Write-Log "Temporary files cleaned up" "SUCCESS"
    }
    catch {
        Write-Log "Could not remove temporary files: $($_.Exception.Message)" "WARN"
    }
    # Final success message
    Write-Log "=== INSTALLATION COMPLETED SUCCESSFULLY ===" "SUCCESS"
    Write-Log "PrismLauncher location: $launcherExtractPath" "SUCCESS"
    Write-Log "Executable: $(Join-Path $launcherExtractPath 'prismlauncher.exe')" "SUCCESS"
    Write-Log "Minecraft client installed in: $hexorPath" "SUCCESS"
    if ($Username -and $Username -ne "web-user") {
        Write-Log "Player name configured: $Username" "SUCCESS"
        Write-Log "Accounts file: $(Join-Path $InstallPath 'accounts.json')" "SUCCESS"
    }
    Write-Log ""
    Write-Log "You can now run PrismLauncher and the Hexor instance should be available!"
    Write-Log "If Java was not installed, please download it from: https://www.java.com/en/download/manual.jsp"
    # Ask if user wants to launch the game
    $launchResponse = Read-Host "Do you want to launch PrismLauncher now? (y/n)"
    if ($launchResponse -eq 'y' -or $launchResponse -eq 'Y') {
        $launcherExe = Join-Path $launcherExtractPath "prismlauncher.exe"
        if (Test-Path $launcherExe) {
            Write-Log "Launching PrismLauncher..." "INFO"
            Start-Process -FilePath $launcherExe -WorkingDirectory $launcherExtractPath
        } else {
            Write-Log "Launcher executable not found!" "ERROR"
        }
    }
@@ -0,0 +1,137 @@
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: minecraft
  namespace: minecraft
  labels:
    app: minecraft
 spec:
  selector:
    matchLabels:
      app: minecraft
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: minecraft
    spec:
      nodeSelector:
        kubernetes.io/hostname: music
      volumes:
      - name: storage
        hostPath:
          path: /data/k8s/mc-server/
          type: Directory
      - name: nginx-config
        configMap:
          name: nginx-config
      - name: client-scripts
        configMap:
          name: client-scripts
      terminationGracePeriodSeconds: 180
      containers:
        - name: minecraft
          image: eclipse-temurin:8-jdk-ubi10-minimal@sha256:85c943a7093a276b0f6ae13dee2ffecb98e390ab6a32fc7e4b8b1fc6bb103dd7
          command: ["java"]
          args:
            - -Xms4G
            - -Xmx4G
            - -XX:+UseG1GC
            - -XX:+ParallelRefProcEnabled
            - -XX:MaxGCPauseMillis=200
            - -XX:+UnlockExperimentalVMOptions
            - -XX:+DisableExplicitGC
            - -XX:+AlwaysPreTouch
            - -XX:G1NewSizePercent=30
            - -XX:G1MaxNewSizePercent=40
            - -XX:G1HeapRegionSize=8M
            - -XX:G1ReservePercent=20
            - -XX:G1HeapWastePercent=5
            - -XX:G1MixedGCCountTarget=4
            - -XX:InitiatingHeapOccupancyPercent=15
            - -XX:G1MixedGCLiveThresholdPercent=90
            - -XX:G1RSetUpdatingPauseTimePercent=5
            - -XX:SurvivorRatio=32
            - -XX:+PerfDisableSharedMem
            - -XX:MaxTenuringThreshold=1
            - -jar
            - forge-1.12.2-14.23.5.2854.jar
            - nogui
          workingDir: /mc/
          resources:
            requests:
              memory: "8Gi"
              cpu: "2000m"
            limits:
              memory: "12Gi"
              cpu: "4000m"
          ports:
            - name: game
              containerPort: 25565
              protocol: TCP
            - name: dynmap
              containerPort: 8123
              protocol: TCP
            - name: webstatus-mod
              containerPort: 8080
              protocol: TCP
          startupProbe:
            tcpSocket:
              port: game
            periodSeconds: 10
            failureThreshold: 90
          livenessProbe:
            tcpSocket:
              port: game
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 3
          readinessProbe:
            tcpSocket:
              port: game
            periodSeconds: 10
            timeoutSeconds: 5
          volumeMounts:
            - name: storage
              mountPath: /mc
        - name: nginx
          image: nginx@sha256:f9ea18bfa4fad859e1ed38259d711da7ccad2c3516e875cec3351a57c859f571
          resources:
            requests:
              memory: "64Mi"
              cpu: "50m"
            limits:
              memory: "256Mi"
              cpu: "200m"
          ports:
            - containerPort: 80
          volumeMounts:
            - name: nginx-config
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
            - name: storage
              mountPath: /mc
            - name: client-scripts
              mountPath: /mc/clients/win-install.ps1
              subPath: win-install.ps1
            - name: client-scripts
              mountPath: /mc/clients/inject.js
              subPath: inject.js
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: minecraft-exporter
  namespace: minecraft
 spec:
  selector:
    app: minecraft
  ports:
    - protocol: TCP
      port: 19565
      targetPort: 19565
@@ -0,0 +1,24 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: minecraft
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
 spec:
  ingressClassName: traefik
  rules:
    - host: minecraft.hexor.cy
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: minecraft-dynmap
                port:
                  number: 80
  tls:
    - secretName: minecraft-tls
      hosts:
        - minecraft.hexor.cy
@@ -0,0 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: minecraft
 resources:
  - deployments.yaml
  - configmaps.yaml
  - services.yaml
  - ingress.yaml
@@ -0,0 +1,39 @@
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: minecraft-dynmap
 spec:
  selector:
    app: minecraft
  ports:
    - protocol: TCP
      port: 80
      targetPort: 80
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: minecraft-webstatus-mod
 spec:
  selector:
    app: minecraft
  ports:
    - protocol: TCP
      port: 80
      targetPort: 8080
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: minecraft-game
 spec:
  type: LoadBalancer
  externalTrafficPolicy: Cluster
  ports:
    - port: 30565
      targetPort: 25565
  selector:
    app: minecraft
@@ -0,0 +1,23 @@
 # MTProxy
 Argo CD deploys this directory through the `mtproxy` entry in
 `core-state/values.yaml`. Two telemt DaemonSets run on every node labeled
 `homelab.hexor.cy/exposure=external`: classic MTProxy on 30443/TCP and
 fake-TLS on 30444/TCP. The classic listener replaces the old amd64-only
 `telegrammessenger/proxy` image so the arm64 `spb` node is covered.
 Vaultwarden item `k8s/mtproxy/secret` must contain a custom field named
 `Secret` with the existing 32-character hexadecimal MTProxy secret. External
 Secrets renders both configurations from that field. Reloader restarts the
 DaemonSets when it changes. Changing this value invalidates existing links.
 `mtproxy-link-reconciler` lists external nodes every five minutes and creates
 `mtproxy-links` and `telemt-links` Secrets. A node's public DNS name must be
 `<node>.hexor.cy` and must point to that node. The reader at `proxy.hexor.cy`
 has read-only access to those two Secrets and is protected by Keycloak's
 forward auth; its access policy is in `terraform/keycloak/terraform.tfvars`.
 The proxy Pods have no Kubernetes API access.
 Before traffic is switched over, allow inbound 30443/TCP and 30444/TCP on
 every external node and verify both DNS and connectivity on each node.
 Neither the cluster nor host firewall is changed by these manifests.
@@ -0,0 +1,149 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
  name: mtproxy-classic
  labels:
    app: mtproxy-classic
 spec:
  selector:
    matchLabels:
      app: mtproxy-classic
  template:
    metadata:
      labels:
        app: mtproxy-classic
      annotations:
        reloader.stakater.com/auto: "true"
    spec:
      nodeSelector:
        homelab.hexor.cy/exposure: external
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 65532
        runAsGroup: 65532
        fsGroup: 65532
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: telemt
          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
          imagePullPolicy: IfNotPresent
          workingDir: /run/telemt
          args: ["/app/config.toml"]
          ports:
            - name: proxy
              containerPort: 30443
              hostPort: 30443
              protocol: TCP
            - name: metrics
              containerPort: 9090
              protocol: TCP
          readinessProbe:
            tcpSocket:
              port: proxy
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              memory: 256Mi
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          volumeMounts:
            - name: config
              mountPath: /app/config.toml
              subPath: classic.toml
              readOnly: true
            - name: workdir
              mountPath: /run/telemt
      volumes:
        - name: config
          secret:
            secretName: mtproxy-config
            defaultMode: 0444
            items:
              - key: classic.toml
                path: classic.toml
        - name: workdir
          emptyDir:
            medium: Memory
            sizeLimit: 4Mi
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
  name: mtproxy-tls
  labels:
    app: mtproxy-tls
 spec:
  selector:
    matchLabels:
      app: mtproxy-tls
  template:
    metadata:
      labels:
        app: mtproxy-tls
      annotations:
        reloader.stakater.com/auto: "true"
    spec:
      nodeSelector:
        homelab.hexor.cy/exposure: external
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 65532
        runAsGroup: 65532
        fsGroup: 65532
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: telemt
          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
          imagePullPolicy: IfNotPresent
          workingDir: /run/telemt
          args: ["/app/config.toml"]
          ports:
            - name: proxy
              containerPort: 30444
              hostPort: 30444
              protocol: TCP
            - name: metrics
              containerPort: 9090
              protocol: TCP
          readinessProbe:
            tcpSocket:
              port: proxy
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              memory: 256Mi
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          volumeMounts:
            - name: config
              mountPath: /app/config.toml
              subPath: tls.toml
              readOnly: true
            - name: workdir
              mountPath: /run/telemt
      volumes:
        - name: config
          secret:
            secretName: mtproxy-config
            defaultMode: 0444
            items:
              - key: tls.toml
                path: tls.toml
        - name: workdir
          emptyDir:
            medium: Memory
            sizeLimit: 4Mi
@@ -0,0 +1,74 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: mtproxy-config
 spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vaultwarden-fields
    kind: ClusterSecretStore
  target:
    name: mtproxy-config
    creationPolicy: Owner
    deletionPolicy: Retain
    template:
      type: Opaque
      data:
        SECRET: "{{ .secret }}"
        classic.toml: |
          [general]
          use_middle_proxy = true
          log_level = "normal"
          [general.modes]
          classic = true
          secure = false
          tls = false
          [server]
          port = 30443
          metrics_port = 9090
          metrics_whitelist = ["0.0.0.0/0"]
          [server.api]
          enabled = false
          [[server.listeners]]
          ip = "0.0.0.0"
          [access.users]
          user = "{{ .secret }}"
        tls.toml: |
          [general]
          use_middle_proxy = true
          log_level = "normal"
          [general.modes]
          classic = false
          secure = false
          tls = true
          [server]
          port = 30444
          metrics_port = 9090
          metrics_whitelist = ["0.0.0.0/0"]
          [server.api]
          enabled = false
          [[server.listeners]]
          ip = "0.0.0.0"
          [censorship]
          tls_domain = "ya.ru"
          mask = true
          tls_emulation = true
          tls_front_dir = "tlsfront"
          [access.users]
          user = "{{ .secret }}"
  data:
    - secretKey: secret
      remoteRef:
        key: k8s/mtproxy/secret
        property: Secret
@@ -0,0 +1,38 @@
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: mtproxy-auth
 spec:
  forwardAuth:
    address: http://auth-proxy.keycloak.svc:80/auth
    trustForwardHeader: false
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
 metadata:
  name: mtproxy-links
 spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`proxy.hexor.cy`)
      kind: Rule
      middlewares:
        - name: mtproxy-auth
      services:
        - name: mtproxy-link-reader
          port: 80
  tls:
    secretName: mtproxy-links-tls
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: mtproxy-links-tls
 spec:
  secretName: mtproxy-links-tls
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer
  dnsNames:
    - proxy.hexor.cy
@@ -0,0 +1,16 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: mtproxy
 resources:
  - external-secrets.yaml
  - daemonsets.yaml
  - metrics.yaml
  - links.yaml
  - reader.yaml
  - ingress.yaml
 configMapGenerator:
  - name: mtproxy-link-reconciler
    files:
      - reconcile.py=link-reconciler.py
 generatorOptions:
  disableNameSuffixHash: true
@@ -0,0 +1,75 @@
 """Publish client links for every external node, using its public DNS name."""
 import base64
 import json
 import urllib.error
 import urllib.parse
 import urllib.request
 API = "https://kubernetes.default.svc"
 NAMESPACE = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", encoding="utf-8").read().strip()
 TOKEN = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
 CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
 def request(method, path, body=None):
    import ssl
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(
        API + path,
        data=data,
        method=method,
        headers={
            "Authorization": "Bearer " + TOKEN,
            "Content-Type": "application/json",
        },
    )
    with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=15) as response:
        return json.load(response)
 def publish(name, links):
    path = f"/api/v1/namespaces/{NAMESPACE}/secrets"
    data = {node: base64.b64encode(link.encode()).decode() for node, link in links.items()}
    try:
        current = request("GET", f"{path}/{name}")
    except urllib.error.HTTPError as error:
        if error.code != 404:
            raise
        request("POST", path, {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, "type": "Opaque", "data": data})
        return
    if current.get("data", {}) == data:
        return
    # PUT replaces the data map, removing links for nodes that are no longer external.
    request("PUT", f"{path}/{name}", {
        "apiVersion": "v1",
        "kind": "Secret",
        "metadata": {"name": name, "resourceVersion": current["metadata"]["resourceVersion"]},
        "type": "Opaque",
        "data": data,
    })
 secret = open("/config/SECRET", encoding="utf-8").read().strip()
 if len(secret) != 32 or any(char not in "0123456789abcdefABCDEF" for char in secret):
    raise ValueError("Vaultwarden Secret must be 32 hexadecimal characters")
 selector = urllib.parse.quote("homelab.hexor.cy/exposure=external", safe="")
 nodes = request("GET", f"/api/v1/nodes?labelSelector={selector}")["items"]
 if not nodes:
    raise ValueError("No external nodes found")
 classic = {}
 tls = {}
 for node in nodes:
    name = node["metadata"]["name"]
    # DNS is maintained outside the cluster; no public IP needs to be stored on a Node.
    host = f"{name}.hexor.cy"
    classic[name] = f"tg://proxy?server={host}&port=30443&secret={secret}"
    tls[name] = f"tg://proxy?server={host}&port=30444&secret=ee{secret}79612e7275"
 publish("mtproxy-links", classic)
 publish("telemt-links", tls)
 print(f"Published MTProxy links for {len(nodes)} external nodes")
@@ -0,0 +1,108 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: mtproxy-link-reconciler
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: mtproxy-external-node-reader
 rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: mtproxy-external-node-reader
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: mtproxy-external-node-reader
 subjects:
  - kind: ServiceAccount
    name: mtproxy-link-reconciler
    namespace: mtproxy
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: mtproxy-link-writer
 rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["create"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["mtproxy-links", "telemt-links"]
    verbs: ["get", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: mtproxy-link-writer
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: mtproxy-link-writer
 subjects:
  - kind: ServiceAccount
    name: mtproxy-link-reconciler
 ---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
  name: mtproxy-link-reconciler
 spec:
  schedule: "*/5 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 2
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          serviceAccountName: mtproxy-link-reconciler
          restartPolicy: OnFailure
          nodeSelector:
            homelab.hexor.cy/exposure: external
          securityContext:
            runAsNonRoot: true
            runAsUser: 65534
            runAsGroup: 65534
            seccompProfile:
              type: RuntimeDefault
          containers:
            - name: reconcile
              image: python:3.13-alpine3.22@sha256:e81548ac35b07a3bd4805f275107592ef458b1e893c0e04d45aedaa19416cca5
              command: ["python", "/scripts/reconcile.py"]
              resources:
                requests:
                  cpu: 10m
                  memory: 32Mi
                limits:
                  memory: 128Mi
              securityContext:
                readOnlyRootFilesystem: true
                allowPrivilegeEscalation: false
                capabilities:
                  drop: ["ALL"]
              volumeMounts:
                - name: scripts
                  mountPath: /scripts
                  readOnly: true
                - name: config
                  mountPath: /config
                  readOnly: true
          volumes:
            - name: scripts
              configMap:
                name: mtproxy-link-reconciler
            - name: config
              secret:
                secretName: mtproxy-config
                items:
                  - key: SECRET
                    path: SECRET
@@ -0,0 +1,65 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: mtproxy-classic-metrics
  labels:
    app: mtproxy-classic
 spec:
  clusterIP: None
  selector:
    app: mtproxy-classic
  ports:
    - name: metrics
      port: 9090
      targetPort: metrics
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: mtproxy-tls-metrics
  labels:
    app: mtproxy-tls
 spec:
  clusterIP: None
  selector:
    app: mtproxy-tls
  ports:
    - name: metrics
      port: 9090
      targetPort: metrics
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: mtproxy-classic
 spec:
  selector:
    matchLabels:
      app: mtproxy-classic
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s
      scrapeTimeout: 10s
      relabelings:
        - action: replace
          sourceLabels: [__meta_kubernetes_pod_node_name]
          targetLabel: node
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: mtproxy-tls
 spec:
  selector:
    matchLabels:
      app: mtproxy-tls
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s
      scrapeTimeout: 10s
      relabelings:
        - action: replace
          sourceLabels: [__meta_kubernetes_pod_node_name]
          targetLabel: node
@@ -0,0 +1,96 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: mtproxy-link-reader
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: mtproxy-link-reader
 rules:
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["mtproxy-links", "telemt-links"]
    verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: mtproxy-link-reader
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: mtproxy-link-reader
 subjects:
  - kind: ServiceAccount
    name: mtproxy-link-reader
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: mtproxy-link-reader
  labels:
    app: mtproxy-link-reader
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: mtproxy-link-reader
  template:
    metadata:
      labels:
        app: mtproxy-link-reader
    spec:
      serviceAccountName: mtproxy-link-reader
      nodeSelector:
        homelab.hexor.cy/exposure: external
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: reader
          image: ultradesu/k8s-secrets:0.2.2@sha256:f206942d21ca9dcd131d6e19ba26cc17671815919217aef657b30a9f85854778
          imagePullPolicy: IfNotPresent
          args:
            - --secrets
            - mtproxy-links,telemt-links
            - --namespace
            - mtproxy
            - --port
            - "3000"
          ports:
            - name: http
              containerPort: 3000
          readinessProbe:
            httpGet:
              path: /health
              port: http
          livenessProbe:
            httpGet:
              path: /health
              port: http
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              memory: 128Mi
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: mtproxy-link-reader
 spec:
  selector:
    app: mtproxy-link-reader
  ports:
    - name: http
      port: 80
      targetPort: http
@@ -0,0 +1,151 @@
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: rustdesk-hbbs
  annotations:
    reloader.stakater.com/auto: "true"
  labels:
    app: rustdesk-hbbs
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: rustdesk-hbbs
  template:
    metadata:
      labels:
        app: rustdesk-hbbs
    spec:
      nodeSelector:
        kubernetes.io/hostname: music
      containers:
      - name: hbbs
        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
        imagePullPolicy: Always
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "512Mi"
            cpu: "750m"
        command: ["hbbs"]
        args:
          - "--relay-servers"
          - "rd.hexor.cy:21117"
          - "--port"
          - "21116"
        ports:
        - name: registry
          containerPort: 21116
          protocol: TCP
        - name: registry-udp
          containerPort: 21116
          protocol: UDP
        - name: nat
          containerPort: 21115
          protocol: TCP
        readinessProbe:
          tcpSocket:
            port: registry
        volumeMounts:
        - name: keys
          mountPath: /data
          readOnly: true
        - name: data
          mountPath: /data-persistent
        env:
        - name: RUST_LOG
          value: "info"
        - name: DB_URL
          value: "/data-persistent/db_v2.sqlite3"
        workingDir: /data
      volumes:
      - name: keys
        secret:
          secretName: rustdesk-keys
          items:
          - key: id_ed25519
            path: id_ed25519
            mode: 0600
          - key: id_ed25519.pub
            path: id_ed25519.pub
            mode: 0644
      - name: data
        hostPath:
          path: /data/k8s/rustdesk/hbbs
          type: Directory
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: rustdesk-hbbr
  annotations:
    reloader.stakater.com/auto: "true"
  labels:
    app: rustdesk-hbbr
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: rustdesk-hbbr
  template:
    metadata:
      labels:
        app: rustdesk-hbbr
    spec:
      nodeSelector:
        kubernetes.io/hostname: music
      containers:
      - name: hbbr
        image: rustdesk/rustdesk-server@sha256:8ecdab65deb7c84652a626380e31d11a8f1fbafd97916d57f95c20628f943c00
        imagePullPolicy: Always
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "512Mi"
            cpu: "750m"
        command: ["hbbr"]
        args:
          - "--port"
          - "21117"
        ports:
        - name: relay
          containerPort: 21117
          protocol: TCP
        readinessProbe:
          tcpSocket:
            port: relay
        volumeMounts:
        - name: keys
          mountPath: /data
          readOnly: true
        - name: data
          mountPath: /data-persistent
        env:
        - name: RUST_LOG
          value: "info"
        workingDir: /data
      volumes:
      - name: keys
        secret:
          secretName: rustdesk-keys
          items:
          - key: id_ed25519
            path: id_ed25519
            mode: 0600
          - key: id_ed25519.pub
            path: id_ed25519.pub
            mode: 0644
      - name: data
        hostPath:
          path: /data/k8s/rustdesk/hbbr
          type: Directory
@@ -0,0 +1,27 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: rustdesk-keys
 spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vaultwarden-fields
    kind: ClusterSecretStore
  target:
    name: rustdesk-keys
    creationPolicy: Owner
    deletionPolicy: Retain
    template:
      type: Opaque
      data:
        id_ed25519: "{{ .private_key }}"
        id_ed25519.pub: "{{ .public_key }}"
  data:
    - secretKey: private_key
      remoteRef:
        key: k8s/rustdesk/secrets
        property: Cert
    - secretKey: public_key
      remoteRef:
        key: k8s/rustdesk/secrets
        property: Key
@@ -0,0 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: rustdesk
 resources:
  - external-secrets.yaml
  - deployment.yaml
  - service.yaml
  - network-policy.yaml
@@ -0,0 +1,73 @@
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
  name: rustdesk-network-policy
 spec:
  podSelector:
    matchLabels:
      app: rustdesk-hbbs
  policyTypes:
  - Ingress
  - Egress
  ingress:
  # Allow all incoming connections to RustDesk ports
  - from: []
    ports:
    - protocol: TCP
      port: 21115
    - protocol: TCP
      port: 21116
    - protocol: UDP
      port: 21116
  egress:
  # Allow DNS
  - to: []
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
  # Allow communication between HBBS and HBBR
  - to:
    - podSelector:
        matchLabels:
          app: rustdesk-hbbr
    ports:
    - protocol: TCP
      port: 21117
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
  name: rustdesk-hbbr-network-policy
 spec:
  podSelector:
    matchLabels:
      app: rustdesk-hbbr
  policyTypes:
  - Ingress
  - Egress
  ingress:
  # Allow all incoming connections to relay port
  - from: []
    ports:
    - protocol: TCP
      port: 21117
  # Allow connections from HBBS
  - from:
    - podSelector:
        matchLabels:
          app: rustdesk-hbbs
    ports:
    - protocol: TCP
      port: 21117
  egress:
  # Allow DNS
  - to: []
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
@@ -0,0 +1,59 @@
 # Preserve client source IPs for RustDesk NAT detection. Public DNS points to music,
 # where both servers and their persistent data live.
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: rustdesk-hbbs
  labels:
    app: rustdesk-hbbs
 spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  selector:
    app: rustdesk-hbbs
  ports:
  - name: registry-tcp
    port: 21116
    targetPort: 21116
    protocol: TCP
  - name: nat
    port: 21115
    targetPort: 21115
    protocol: TCP
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: rustdesk-hbbs-udp
  labels:
    app: rustdesk-hbbs
 spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  selector:
    app: rustdesk-hbbs
  ports:
  - name: registry-udp
    port: 21116
    targetPort: 21116
    protocol: UDP
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: rustdesk-hbbr
  labels:
    app: rustdesk-hbbr
 spec:
  type: LoadBalancer
  externalTrafficPolicy: Local
  selector:
    app: rustdesk-hbbr
  ports:
  - name: relay
    port: 21117
    targetPort: 21117
    protocol: TCP
@@ -0,0 +1,18 @@
 # Shadowsocks on spb
 One shadowsocks-rust pod runs on `spb`, exposing `8388/TCP` and `8388/UDP`
 through host ports. Client settings:
 - Server: `spb.hexor.cy`
 - Port: `8388`
 - Method: `chacha20-ietf-poly1305`
 - Password: the `Password` custom field of the Vaultwarden item named
  `k8s/shadowsocks/secrets` (exact name, not an item UUID).
 Create that item in the server secrets vault accessible to the External Secrets
 bot. Use a long random password, for example generated with `openssl rand -base64 32`.
 External Secrets creates `shadowsocks-config`; Reloader restarts the deployment
 when the Secret changes. No persistent storage is required.
 The spb NixOS configuration opens port 8388 for both protocols. Apply that
 configuration when deploying this application for the first time.
@@ -0,0 +1,71 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: shadowsocks
  annotations:
    reloader.stakater.com/auto: "true"
  labels:
    app: shadowsocks
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: shadowsocks
  template:
    metadata:
      labels:
        app: shadowsocks
    spec:
      nodeSelector:
        kubernetes.io/hostname: spb
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
        runAsGroup: 65534
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: shadowsocks
          # shadowsocks-rust v1.25.0; multi-arch image includes linux/arm64.
          image: ghcr.io/shadowsocks/ssserver-rust:v1.25.0@sha256:22240c07d8ce3a36421ede770bd775dcc5fc56d5d829943196957b05452dc36f
          imagePullPolicy: IfNotPresent
          command: ["/usr/bin/ssserver"]
          args:
            - --log-without-time
            - -c
            - /etc/shadowsocks-rust/config.json
          ports:
            - name: proxy-tcp
              containerPort: 8388
              hostPort: 8388
              protocol: TCP
            - name: proxy-udp
              containerPort: 8388
              hostPort: 8388
              protocol: UDP
          readinessProbe:
            tcpSocket:
              port: proxy-tcp
          resources:
            requests:
              cpu: 50m
              memory: 32Mi
            limits:
              memory: 256Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
          volumeMounts:
            - name: config
              mountPath: /etc/shadowsocks-rust
              readOnly: true
      volumes:
        - name: config
          secret:
            secretName: shadowsocks-config
            defaultMode: 0444
@@ -0,0 +1,29 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: shadowsocks-config
 spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vaultwarden-fields
    kind: ClusterSecretStore
  target:
    name: shadowsocks-config
    creationPolicy: Owner
    deletionPolicy: Retain
    template:
      type: Opaque
      data:
        config.json: |
          {
            "server": "0.0.0.0",
            "server_port": 8388,
            "method": "chacha20-ietf-poly1305",
            "password": {{ .password | toJson }},
            "mode": "tcp_and_udp"
          }
  data:
    - secretKey: password
      remoteRef:
        key: k8s/shadowsocks/secrets
        property: Password
@@ -0,0 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: shadowsocks
 resources:
  - external-secrets.yaml
  - deployment.yaml
@@ -0,0 +1,54 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: web-petting
  labels:
    app: web-petting
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: web-petting
  template:
    metadata:
      labels:
        app: web-petting
      annotations:
        reloader.stakater.com/auto: "true"
    spec:
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: external
      containers:
        - name: web-petting
          image: ultradesu/web-petting:latest
          imagePullPolicy: Always
          args:
            - web-petting
            - -l
            - 0.0.0.0:3000
          ports:
            - name: http
              containerPort: 3000
          env:
            - name: RUST_LOG
              value: info
            - name: WEB_PETTING_DEBUG
              value: "false"
            - name: WEB_PETTING_DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: web-petting-secrets
                  key: WEB_PETTING_DATABASE_URL
          # Everything persistent is in PostgreSQL and R2; /data is scratch space only.
          volumeMounts:
            - name: data
              mountPath: /data
          resources:
            requests:
              memory: 256Mi
            limits:
              memory: 1Gi
      volumes:
        - name: data
          emptyDir: {}
@@ -0,0 +1,37 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: web-petting
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
 spec:
  ingressClassName: traefik
  rules:
    - host: pet.hexor.cy
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: web-petting
                port:
                  number: 80
    - host: xn--l1acako8eb.xn--p1ai
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: web-petting
                port:
                  number: 80
  tls:
    - secretName: web-petting-tls
      hosts:
        - pet.hexor.cy
    - secretName: web-petting-murnyanya-tls
      hosts:
        - xn--l1acako8eb.xn--p1ai
@@ -0,0 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - secrets.yaml
  - deployment.yaml
  - service.yaml
  - ingress.yaml
@@ -0,0 +1,47 @@
 # Read-only access to the web-petting-db Secret in the psql namespace
 # (see postgresql/databases/web-petting.yaml for the other half).
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: db-credentials-reader
 ---
 apiVersion: external-secrets.io/v1
 kind: SecretStore
 metadata:
  name: psql-credentials
 spec:
  provider:
    kubernetes:
      remoteNamespace: psql
      server:
        caProvider:
          type: ConfigMap
          name: kube-root-ca.crt
          key: ca.crt
      auth:
        serviceAccount:
          name: db-credentials-reader
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: web-petting-secrets
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: SecretStore
    name: psql-credentials
  target:
    name: web-petting-secrets
    template:
      data:
        WEB_PETTING_DATABASE_URL: "postgresql://{{ .username }}:{{ .password }}@main-rw.psql.svc:5432/web_petting"
  data:
    - secretKey: username
      remoteRef:
        key: web-petting-db
        property: username
    - secretKey: password
      remoteRef:
        key: web-petting-db
        property: password
@@ -0,0 +1,15 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: web-petting
  labels:
    app: web-petting
 spec:
  type: ClusterIP
  selector:
    app: web-petting
  ports:
    - name: http
      port: 80
      targetPort: 3000
      protocol: TCP
@@ -23,6 +23,9 @@ spec:
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true
      {{- if and (hasKey . "validate") (not .validate) }}
      - Validate=false
      {{- end }}
      {{- if .skipDryRunOnMissingResource }}
      - SkipDryRunOnMissingResource=true
      {{- end }}
@@ -24,10 +24,19 @@ apps:
    path: k8s-ng/core/traefik-config
    namespace: kube-system
    wave: 1
  - name: coredns-config
    path: k8s-ng/core/coredns-config
    namespace: kube-system
    wave: 1
    validate: false # partial Deployment: only the fields we manage
  - name: reloader
    path: k8s-ng/core/reloader
    namespace: reloader
    wave: 1
  - name: gpu-system
    path: k8s-ng/core/gpu
    namespace: gpu-system
    wave: 1
  - name: cert-manager
    path: k8s-ng/core/cert-manager
    namespace: cert-manager
@@ -53,8 +62,73 @@ apps:
    namespace: keycloak
    wave: 3
    skipDryRunOnMissingResource: true # ExternalSecret/Certificate CRDs come from earlier waves
  # Not a core service: lives in k8s-ng/apps and belongs to the "apps" project.
  - name: furumi
    project: apps
    path: k8s-ng/apps/furumi
    namespace: furumi
    wave: 6 # after prom-stack installs the ServiceMonitor CRD
    skipDryRunOnMissingResource: true
  - name: web-petting
    project: apps
    path: k8s-ng/apps/web-petting
    namespace: web-petting
    wave: 5
    skipDryRunOnMissingResource: true
  - name: k8s-secrets
    project: apps
    path: k8s-ng/apps/k8s-secrets
    namespace: k8s-secret
    wave: 5 # after secret-stores and auth-proxy; independent of prom-stack health
    skipDryRunOnMissingResource: true
  - name: llamacpp
    project: apps
    path: k8s-ng/apps/llamacpp
    namespace: llamacpp
    wave: 5 # pinned to the ai node (GPU); must not hold back the other waves
    skipDryRunOnMissingResource: true
  - name: secret-stores
    path: k8s-ng/core/secret-stores
    namespace: external-secrets
    wave: 4 # reads the Vaultwarden vault, so it comes after vaultwarden
    skipDryRunOnMissingResource: true # ClusterSecretStore CRD comes from the external-secrets app
  - name: gitea
    path: k8s-ng/core/gitea
    namespace: gitea
    wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node
    prune: false # never delete the instance because of a manifest change
    skipDryRunOnMissingResource: true
  - name: prom-stack
    path: k8s-ng/core/prom-stack
    namespace: prometheus
    wave: 5 # after secret-stores; installs monitoring before application workloads
    skipDryRunOnMissingResource: true # monitoring, cert-manager and ExternalSecret CRDs
  - name: immich-db
    path: k8s-ng/core/immich-db
    namespace: immich
    wave: 2
    prune: false
    skipDryRunOnMissingResource: true
  - name: minecraft
    project: games
    path: k8s-ng/apps/minecraft
    namespace: minecraft
    wave: 6
  - name: rustdesk
    project: apps
    path: k8s-ng/apps/rustdesk
    namespace: rustdesk
    wave: 6
    skipDryRunOnMissingResource: true
  - name: shadowsocks
    project: apps
    path: k8s-ng/apps/shadowsocks
    namespace: shadowsocks
    wave: 6
    skipDryRunOnMissingResource: true
  - name: mtproxy
    project: apps
    path: k8s-ng/apps/mtproxy
    namespace: mtproxy
    wave: 6 # after secret-stores and prom-stack
    skipDryRunOnMissingResource: true
@@ -1,6 +1,9 @@
 # Argo CD Helm Chart Values (k8s-ng cluster)
 global:
  nodeSelector:
    homelab.hexor.cy/pool: core
    homelab.hexor.cy/exposure: external
  domain: ag.btwiusearch.net
  logging:
    format: text
@@ -1,4 +1,5 @@
-# Public certificates through DNS-01. The credential Secrets are created by hand
+# Public certificates through DNS-01 for the zones below and HTTP-01 (Traefik) for
 # every other domain, e.g. the .рф one. The credential Secrets are created by hand
 # (they will move to Vaultwarden + External Secrets):
 #   kubectl -n cert-manager create secret generic cloudflare-api-token --from-literal=apiToken=<token>
 #     (Zone:DNS:Edit on the btwiusearch.net zone)
@@ -36,3 +37,8 @@ spec:
            secretAccessKeySecretRef:
              name: aws-api-key-secret
              key: secretKey
      # Fallback for domains outside the zones above: their A records must point
      # at the cluster nodes (Traefik answers on port 80 everywhere).
      - http01:
          ingress:
            ingressClassName: traefik
@@ -1,2 +1,12 @@
 crds:
  enabled: true
 nodeSelector: &externalCore
  homelab.hexor.cy/pool: core
  homelab.hexor.cy/exposure: external
 webhook:
  nodeSelector: *externalCore
 cainjector:
  nodeSelector: *externalCore
 startupapicheck:
  nodeSelector: *externalCore
@@ -1,4 +1,8 @@
 # CloudNativePG operator (chart 0.29.x = operator 1.30.x)
 nodeSelector:
  homelab.hexor.cy/pool: core
  homelab.hexor.cy/exposure: external
 crds:
  create: true
@@ -0,0 +1,33 @@
 # K3s owns the CoreDNS Deployment. Apply only the fields customized here so
 # upstream image, probes, Corefile mounts and topology spread stay intact.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: coredns
  namespace: kube-system
  annotations:
    argocd.argoproj.io/sync-options: Prune=false,Delete=false
 spec:
  replicas: 3
  template:
    spec:
      tolerations:
        - key: CriticalAddonsOnly
          operator: Exists
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
        # DNS also belongs on the internal core node used by Actions jobs.
        - key: workload
          operator: Equal
          value: ai
          effect: NoSchedule
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: homelab.hexor.cy/pool
                    operator: In
                    values:
                      - core
@@ -0,0 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - deployment.yaml
  - service.yaml
@@ -0,0 +1,11 @@
 # Keep DNS on the caller's node when it has a healthy CoreDNS replica. K3s owns
 # the rest of this Service; server-side apply preserves its ports and selector.
 apiVersion: v1
 kind: Service
 metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    argocd.argoproj.io/sync-options: Prune=false,Delete=false
 spec:
  trafficDistribution: PreferSameNode
@@ -1 +1,9 @@
 installCRDs: true
 nodeSelector: &externalCore
  homelab.hexor.cy/pool: core
  homelab.hexor.cy/exposure: external
 webhook:
  nodeSelector: *externalCore
 certController:
  nodeSelector: *externalCore
@@ -0,0 +1,72 @@
 # Gitea with SQLite; everything (repositories, database, SSH host keys) lives in
 # /var/lib/gitea on the music node.
 # `replicas` is deliberately not set: Argo CD does not manage it, so the instance
 # can be scaled to 0 with kubectl during a data migration without being healed back.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: gitea
  labels:
    app: gitea
 spec:
  selector:
    matchLabels:
      app: gitea
  strategy:
    type: Recreate # two instances must never open the same SQLite database
  template:
    metadata:
      labels:
        app: gitea
    spec:
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: external
        kubernetes.io/hostname: music
      volumes:
        - name: storage
          hostPath:
            path: /var/lib/gitea
            type: DirectoryOrCreate
      containers:
        - name: gitea
          image: gitea/gitea:28.0.0
          resources:
            requests:
              memory: 1Gi
              cpu: "1"
            limits:
              memory: 4Gi
              cpu: "4"
          env:
            - name: GITEA__service__REGISTER_MANUAL_CONFIRM
              value: "true"
            - name: GITEA__service__ENABLE_CAPTCHA
              value: "true"
            - name: GITEA__service__REQUIRE_CAPTCHA_FOR_LOGIN
              value: "false"
            - name: GITEA__service__REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA
              value: "true"
            - name: GITEA__service__CAPTCHA_TYPE
              value: cfturnstile
            - name: GITEA__security__ALLOWED_HOST_LIST
              value: "*"
          envFrom:
            - secretRef:
                name: gitea-runner-act-runner-secrets
          ports:
            - name: http
              containerPort: 3000
              protocol: TCP
            - name: ssh
              containerPort: 22
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /api/healthz
              port: http
            periodSeconds: 10
            timeoutSeconds: 5
          volumeMounts:
            - name: storage
              mountPath: /data
@@ -0,0 +1,44 @@
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: gitea-runner-token
 spec:
  refreshInterval: 1m
  target:
    name: gitea-runner-act-runner-secrets
    deletionPolicy: Delete
    template:
      type: Opaque
      data:
        token: |-
          {{ .password }}
        GITEA__service__CF_TURNSTILE_SITEKEY: |-
          {{ .CF_TURNSTILE_SITEKEY }}
        GITEA__service__CF_TURNSTILE_SECRET: |-
          {{ .CF_TURNSTILE_SECRET }}
  data:
    - secretKey: password
      sourceRef:
        storeRef:
          name: vaultwarden-login
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/gitea/secrets
        property: login.password
    - secretKey: CF_TURNSTILE_SITEKEY
      sourceRef:
        storeRef:
          name: vaultwarden-login
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/gitea/secrets
        property: fields[0].value
    - secretKey: CF_TURNSTILE_SECRET
      sourceRef:
        storeRef:
          name: vaultwarden-login
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/gitea/secrets
        property: fields[1].value
@@ -0,0 +1,23 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: gitea
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt
    traefik.ingress.kubernetes.io/router.entrypoints: websecure
 spec:
  rules:
    - host: gt.hexor.cy
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: gitea
                port:
                  number: 80
  tls:
    - secretName: gitea-tls
      hosts:
        - gt.hexor.cy
@@ -0,0 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - external-secrets.yaml
  - deployment.yaml
  - service.yaml
  - ingress.yaml
  - runner.yaml
  - runner-rbac.yaml
  - user-unban-cronjob.yaml
@@ -0,0 +1,41 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: gitea-runner
 ---
 apiVersion: v1
 kind: Secret
 metadata:
  name: gitea-runner-kubernetes-token
  annotations:
    kubernetes.io/service-account.name: gitea-runner
 type: kubernetes.io/service-account-token
 ---
 # CI may edit the routes of the auth proxy (the ConfigMap lives in the keycloak namespace here).
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: auth-proxy-routes-manager
  namespace: keycloak
 rules:
  - apiGroups: [""]
    resources: [configmaps]
    resourceNames: [auth-proxy-routes]
    verbs: [get, update, patch]
  - apiGroups: [""]
    resources: [configmaps]
    verbs: [create]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: gitea-runner-auth-proxy-routes
  namespace: keycloak
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: auth-proxy-routes-manager
 subjects:
  - kind: ServiceAccount
    name: gitea-runner
    namespace: gitea
@@ -0,0 +1,86 @@
 # Gitea Actions runner on the ai node. Jobs run as containers in the host's Docker
 # (virtualisation.docker in hosts/ai of the nix repo) through the mounted socket.
 # `replicas` is not set (see deployment.yaml): scale it with kubectl when needed.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: gitea-runner-config
 data:
  config.yaml: |
    runner:
      capacity: 8 # jobs in parallel; the ai node has 32 threads and 124 GiB
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: gitea-runner
  labels:
    app: gitea-runner
  annotations:
    reloader.stakater.com/auto: "true"
 spec:
  selector:
    matchLabels:
      app: gitea-runner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: gitea-runner
    spec:
      serviceAccountName: gitea-runner
      dnsConfig:
        options:
          - name: ndots
            value: "2"
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: internal
        workload/ai: "true"
      tolerations:
        - key: workload
          operator: Equal
          value: ai
          effect: NoSchedule
      volumes:
        - name: docker-sock
          hostPath:
            path: /var/run/docker.sock
            type: Socket
        - name: runner-data
          emptyDir:
            sizeLimit: 30Gi
        - name: config
          configMap:
            name: gitea-runner-config
      containers:
        - name: gitea-runner
          image: gitea/act_runner:nightly
          volumeMounts:
            - name: docker-sock
              mountPath: /var/run/docker.sock
            - name: runner-data
              mountPath: /data
            - name: config
              mountPath: /config.yaml
              subPath: config.yaml
          env:
            - name: CONFIG_FILE
              value: /config.yaml
            - name: GITEA_INSTANCE_URL
              # Job containers run outside the cluster network, so they need the public URL.
              value: https://gt.hexor.cy
            - name: GITEA_RUNNER_REGISTRATION_TOKEN
              valueFrom:
                secretKeyRef:
                  name: gitea-runner-act-runner-secrets
                  key: token
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: GITEA_RUNNER_NAME
              value: $(NODE_NAME)
            - name: GITEA_RUNNER_LABELS
              value: ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04,ubuntu-20.04:docker://ghcr.io/catthehacker/ubuntu:act-20.04
@@ -0,0 +1,24 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: gitea
 spec:
  selector:
    app: gitea
  ports:
    - protocol: TCP
      port: 80
      targetPort: 3000
 ---
 # Git over SSH on every node (servicelb); the node firewall opens 30022 (nix repo, k3s-base.nix).
 apiVersion: v1
 kind: Service
 metadata:
  name: gitea-ssh
 spec:
  type: LoadBalancer
  ports:
    - port: 30022
      targetPort: 22
  selector:
    app: gitea
@@ -0,0 +1,61 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
  name: gitea-user-unban
  labels:
    app: gitea-user-unban
 spec:
  schedule: "*/10 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            app: gitea-user-unban
        spec:
          restartPolicy: OnFailure
          nodeSelector:
            homelab.hexor.cy/pool: core
            homelab.hexor.cy/exposure: external
            kubernetes.io/hostname: music
          volumes:
            - name: storage
              hostPath:
                path: /var/lib/gitea
                type: Directory
          containers:
            - name: sqlite-unban
              image: gitea/gitea:28.0.0
              imagePullPolicy: IfNotPresent
              resources:
                requests:
                  memory: 32Mi
                  cpu: 10m
                limits:
                  memory: 128Mi
                  cpu: 100m
              command:
                - /bin/sh
                - -ec
                - |
                  sqlite3 -cmd ".timeout 30000" /data/gitea/gitea.db "
                    UPDATE \"user\"
                       SET is_active = 1,
                           prohibit_login = 0,
                           updated_unix = unixepoch()
                     WHERE lower(email) = lower('ab@hexor.cy')
                       AND (is_active <> 1 OR prohibit_login <> 0);
                    SELECT printf(
                      'gitea user watchdog: id=%d login=%s email=%s is_active=%d prohibit_login=%d updated_unix=%d',
                      id, lower_name, email, is_active, prohibit_login, updated_unix
                    )
                      FROM \"user\"
                     WHERE lower(email) = lower('ab@hexor.cy');
                  "
              volumeMounts:
                - name: storage
                  mountPath: /data
@@ -0,0 +1,28 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 helmCharts:
  - name: amd-gpu
    repo: https://rocm.github.io/k8s-device-plugin/
    version: 0.22.0
    releaseName: amd-gpu-device-plugin
    namespace: gpu-system
    valuesFile: values.yaml
    includeCRDs: true
 # The chart always adds the NFD label feature.node.kubernetes.io/pci-0300_1002.present
 # to the selector; NFD is not installed (and would not match this iGPU), so replace it.
 patches:
  - target:
      group: apps
      version: v1
      kind: DaemonSet
      name: amd-gpu-device-plugin-daemonset
    patch: |-
      - op: replace
        path: /spec/template/spec/nodeSelector
        value:
          homelab.hexor.cy/pool: core
          homelab.hexor.cy/exposure: internal
          kubernetes.io/arch: amd64
          workload/ai: "true"
@@ -0,0 +1,37 @@
 # AMD GPU device plugin: exposes the Strix Halo iGPU of the "ai" node as the
 # amd.com/gpu resource (and injects /dev/kfd and /dev/dri into the pod).
 nfd:
  enabled: false
 labeller:
  enabled: false
 dp:
  image:
    repository: docker.io/rocm/k8s-device-plugin
    tag: "1.31.0.9"
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
 securityContext:
  allowPrivilegeEscalation: false
  capabilities:
    drop:
      - ALL
 # The ai node carries the label workload/ai=true and the taint workload=ai:NoSchedule
 # (hosts/ai in the nix repo).
 tolerations:
  - key: workload
    operator: Equal
    value: ai
    effect: NoSchedule
 node_selector_enabled: true
 node_selector:
  homelab.hexor.cy/pool: core
  homelab.hexor.cy/exposure: internal
  kubernetes.io/arch: amd64
  workload/ai: "true"
@@ -0,0 +1,47 @@
 # Keep PostgreSQL and both vector extensions identical to the source during
 # migration. The application/media migration and any upgrades happen later.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: immich
  annotations:
    argocd.argoproj.io/sync-options: Prune=false
 spec:
  description: Immich database migrated from the old cluster; media remain on NAS
  imageName: ghcr.io/tensorchord/cloudnative-pgvecto.rs:14-v0.3.0@sha256:df7848916fea2c486faee8155a481dc5009f30ec513b5561d9ab76d1e101a594
  instances: 1
  enableSuperuserAccess: true
  bootstrap:
    initdb:
      database: immich
      owner: immich
      dataChecksums: true
      encoding: UTF8
      localeCType: C
      localeCollate: C
  postgresUID: 26
  postgresGID: 26
  storage:
    size: 55Gi
    storageClass: immich-local
    pvcTemplate:
      accessModes: [ReadWriteOnce]
      selector:
        matchLabels:
          immich.hexor.cy/volume: postgres
  affinity:
    nodeSelector:
      homelab.hexor.cy/pool: core
      homelab.hexor.cy/exposure: external
      kubernetes.io/hostname: music
  postgresql:
    shared_preload_libraries: [vectors.so]
    parameters:
      shared_buffers: 512MB
      max_connections: "200"
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      memory: 2Gi
@@ -0,0 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: immich
 resources:
  - storage.yaml
  - cluster.yaml
@@ -0,0 +1,66 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
  name: immich-db-storage-prepare
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
 spec:
  backoffLimit: 3
  template:
    spec:
      restartPolicy: OnFailure
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: external
        kubernetes.io/hostname: music
      containers:
        - name: prepare
          image: docker.io/busybox:1.37.0
          command: [sh, -ec, "mkdir -p /storage/postgresql; chown 26:26 /storage/postgresql; chmod 0700 /storage/postgresql"]
          securityContext:
            runAsUser: 0
            allowPrivilegeEscalation: false
          volumeMounts:
            - name: storage
              mountPath: /storage
      volumes:
        - name: storage
          hostPath:
            path: /data/k8s/immich
            type: DirectoryOrCreate
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
  name: immich-local
  annotations:
    argocd.argoproj.io/sync-options: Prune=false
 provisioner: kubernetes.io/no-provisioner
 volumeBindingMode: WaitForFirstConsumer
 reclaimPolicy: Retain
 ---
 apiVersion: v1
 kind: PersistentVolume
 metadata:
  name: immich-postgres-music
  labels:
    immich.hexor.cy/volume: postgres
  annotations:
    argocd.argoproj.io/sync-options: Prune=false
 spec:
  capacity:
    storage: 55Gi
  volumeMode: Filesystem
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: immich-local
  local:
    path: /data/k8s/immich/postgresql
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values: [music]
@@ -20,9 +20,12 @@ spec:
      labels:
        app: auth-proxy
    spec:
      nodeSelector:
        homelab.hexor.cy/pool: core
        homelab.hexor.cy/exposure: external
      containers:
        - name: auth-proxy
-          image: ultradesu/rsauth2-proxy:latest
+          image: ultradesu/rsauth2-proxy:v0.1.3
          ports:
            - containerPort: 8080
              name: http
@@ -1,366 +0,0 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: keycloak-hacker-theme
  namespace: keycloak
 data:
  theme.properties: |
    parent=keycloak.v2
    import=common/keycloak
    styles=css/hacker.css
  hacker.css: |
    /* ============================================================
       HEXOR — Hacker Terminal Theme for Keycloak
       ============================================================ */
    @import url('https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;600&display=swap');
    :root {
      --hk-bg: #0a0a0a;
      --hk-panel: #0d0d0d;
      --hk-border: #00ff4180;
      --hk-green: #00ff41;
      --hk-green-dim: #00cc33;
      --hk-green-glow: rgba(0, 255, 65, 0.15);
      --hk-green-glow-strong: rgba(0, 255, 65, 0.35);
      --hk-cyan: #00e5ff;
      --hk-red: #ff3333;
      --hk-text: #b0ffb0;
      --hk-text-dim: #5a8a5a;
      --hk-input-bg: #050505;
      --hk-font: 'Fira Code', 'Cascadia Code', 'JetBrains Mono', monospace;
    }
    /* -- Force dark always --------------------------------------------------- */
    html.login-pf,
    html.login-pf.pf-v5-theme-dark {
      color-scheme: dark;
    }
    /* -- Background: scanlines + CRT ---------------------------------------- */
    body#keycloak-bg {
      font-family: var(--hk-font) !important;
      background: var(--hk-bg) !important;
      color: var(--hk-text) !important;
      min-height: 100vh;
      position: relative;
    }
    body#keycloak-bg::before {
      content: '';
      position: fixed;
      inset: 0;
      background: repeating-linear-gradient(
        0deg,
        transparent,
        transparent 2px,
        rgba(0, 255, 65, 0.03) 2px,
        rgba(0, 255, 65, 0.03) 4px
      );
      pointer-events: none;
      z-index: 9999;
    }
    /* -- Login container ----------------------------------------------------- */
    .pf-v5-c-login {
      background: transparent !important;
    }
    .pf-v5-c-login__container {
      max-width: 480px !important;
    }
    /* -- Header / Brand ------------------------------------------------------ */
    #kc-header-wrapper {
      font-family: var(--hk-font) !important;
      color: var(--hk-green) !important;
      text-shadow: 0 0 10px var(--hk-green), 0 0 30px var(--hk-green-glow);
      font-size: 1.5rem !important;
      letter-spacing: 0.15em;
      text-transform: uppercase;
    }
    #kc-header-wrapper .kc-logo-text span::before {
      content: '> ';
      opacity: 0.6;
    }
    /* -- Main card ----------------------------------------------------------- */
    .pf-v5-c-login__main {
      background: var(--hk-panel) !important;
      border: 1px solid var(--hk-border) !important;
      border-radius: 0 !important;
      box-shadow:
        0 0 15px var(--hk-green-glow),
        inset 0 0 30px rgba(0, 0, 0, 0.5) !important;
    }
    /* -- Title --------------------------------------------------------------- */
    h1#kc-page-title,
    .pf-v5-c-title {
      font-family: var(--hk-font) !important;
      color: var(--hk-green) !important;
      text-shadow: 0 0 8px var(--hk-green-glow);
      font-weight: 600 !important;
      font-size: 1.25rem !important;
      letter-spacing: 0.05em;
    }
    /* -- Labels -------------------------------------------------------------- */
    .pf-v5-c-form__label-text {
      font-family: var(--hk-font) !important;
      color: var(--hk-green-dim) !important;
      font-size: 0.85rem !important;
      text-transform: uppercase;
      letter-spacing: 0.08em;
    }
    /* -- Input fields -------------------------------------------------------- */
    .pf-v5-c-form-control {
      background-color: var(--hk-input-bg) !important;
      border: 1px solid var(--hk-border) !important;
      border-radius: 0 !important;
      color: var(--hk-green) !important;
      font-family: var(--hk-font) !important;
      font-size: 0.9rem !important;
      caret-color: var(--hk-green);
      transition: border-color 0.2s, box-shadow 0.2s;
    }
    .pf-v5-c-form-control:focus-within,
    .pf-v5-c-form-control:focus {
      border-color: var(--hk-green) !important;
      box-shadow: 0 0 8px var(--hk-green-glow), inset 0 0 4px var(--hk-green-glow) !important;
      outline: none !important;
    }
    .pf-v5-c-form-control input {
      color: var(--hk-green) !important;
      font-family: var(--hk-font) !important;
    }
    .pf-v5-c-form-control input::placeholder {
      color: var(--hk-text-dim) !important;
    }
    /* -- Primary button (Sign In) -------------------------------------------- */
    .pf-v5-c-button.pf-m-primary {
      background: transparent !important;
      border: 1px solid var(--hk-green) !important;
      border-radius: 0 !important;
      color: var(--hk-green) !important;
      font-family: var(--hk-font) !important;
      font-weight: 600 !important;
      text-transform: uppercase;
      letter-spacing: 0.12em;
      font-size: 0.9rem !important;
      transition: all 0.2s;
      position: relative;
      overflow: hidden;
    }
    .pf-v5-c-button.pf-m-primary:hover {
      background: var(--hk-green-glow) !important;
      box-shadow: 0 0 15px var(--hk-green-glow-strong), inset 0 0 15px var(--hk-green-glow) !important;
      color: #fff !important;
      text-shadow: 0 0 5px var(--hk-green);
    }
    /* -- Secondary button (Try Another Way, Passkey) ------------------------- */
    .pf-v5-c-button.pf-m-secondary {
      background: transparent !important;
      border: 1px solid var(--hk-cyan) !important;
      border-radius: 0 !important;
      color: var(--hk-cyan) !important;
      font-family: var(--hk-font) !important;
      text-transform: uppercase;
      letter-spacing: 0.1em;
      font-size: 0.85rem !important;
      transition: all 0.2s;
    }
    .pf-v5-c-button.pf-m-secondary:hover {
      background: rgba(0, 229, 255, 0.1) !important;
      box-shadow: 0 0 12px rgba(0, 229, 255, 0.3) !important;
      color: #fff !important;
    }
    /* -- Passkey authenticate button ----------------------------------------- */
    #authenticateWebAuthnButton {
      background: transparent !important;
      border: 1px solid var(--hk-cyan) !important;
      border-radius: 0 !important;
      color: var(--hk-cyan) !important;
      font-family: var(--hk-font) !important;
      text-transform: uppercase;
      letter-spacing: 0.1em;
      font-size: 0.85rem !important;
      padding: 0.75rem 1rem;
      display: block;
      width: 100%;
      text-align: center;
      text-decoration: none;
      transition: all 0.2s;
      margin-top: 1rem;
    }
    #authenticateWebAuthnButton::before {
      content: '\f084  ';
      font-family: 'Font Awesome 6 Free', 'FontAwesome';
      font-weight: 900;
    }
    #authenticateWebAuthnButton:hover {
      background: rgba(0, 229, 255, 0.1) !important;
      box-shadow: 0 0 15px rgba(0, 229, 255, 0.3), inset 0 0 10px rgba(0, 229, 255, 0.1) !important;
      color: #fff !important;
      text-shadow: 0 0 5px var(--hk-cyan);
    }
    /* -- Show password button ------------------------------------------------ */
    .pf-v5-c-button.pf-m-control {
      background: var(--hk-input-bg) !important;
      border: 1px solid var(--hk-border) !important;
      border-left: none !important;
      border-radius: 0 !important;
      color: var(--hk-text-dim) !important;
    }
    .pf-v5-c-button.pf-m-control:hover {
      color: var(--hk-green) !important;
    }
    /* -- Social providers (Google, etc) -------------------------------------- */
    .pf-v5-c-login__main-footer {
      border-top: 1px solid var(--hk-border) !important;
    }
    .kc-social-section .pf-v5-c-button.pf-m-secondary {
      border-color: var(--hk-text-dim) !important;
      color: var(--hk-text) !important;
    }
    .kc-social-section .pf-v5-c-button.pf-m-secondary:hover {
      border-color: var(--hk-green) !important;
      color: var(--hk-green) !important;
      box-shadow: 0 0 10px var(--hk-green-glow) !important;
    }
    /* -- Try another way link ------------------------------------------------ */
    #try-another-way {
      color: var(--hk-cyan) !important;
      text-decoration: none !important;
      font-family: var(--hk-font) !important;
    }
    #try-another-way:hover {
      text-shadow: 0 0 8px rgba(0, 229, 255, 0.5);
    }
    /* -- Links --------------------------------------------------------------- */
    a {
      color: var(--hk-cyan) !important;
      text-decoration: none;
      transition: text-shadow 0.2s;
    }
    a:hover {
      text-shadow: 0 0 6px rgba(0, 229, 255, 0.4);
    }
    /* -- Alerts / messages --------------------------------------------------- */
    .pf-v5-c-alert {
      background: rgba(0, 255, 65, 0.05) !important;
      border: 1px solid var(--hk-border) !important;
      border-radius: 0 !important;
      font-family: var(--hk-font) !important;
    }
    .pf-v5-c-alert.pf-m-danger {
      border-color: var(--hk-red) !important;
      background: rgba(255, 51, 51, 0.05) !important;
    }
    .pf-v5-c-alert__title {
      color: var(--hk-text) !important;
      font-family: var(--hk-font) !important;
    }
    /* -- Checkbox (Remember me) ---------------------------------------------- */
    .pf-v5-c-check__label {
      color: var(--hk-text-dim) !important;
      font-family: var(--hk-font) !important;
      font-size: 0.8rem !important;
    }
    /* -- Helper text (Forgot password, etc.) --------------------------------- */
    .pf-v5-c-helper-text .pf-v5-c-button.pf-m-link {
      color: var(--hk-cyan) !important;
      font-family: var(--hk-font) !important;
      font-size: 0.8rem !important;
    }
    /* -- Select auth list (Try another way options) -------------------------- */
    .select-auth-container {
      background: var(--hk-panel) !important;
      font-family: var(--hk-font) !important;
    }
    .pf-v5-c-data-list__item {
      border-color: var(--hk-border) !important;
      background: transparent !important;
    }
    .pf-v5-c-data-list__item:hover {
      background: var(--hk-green-glow) !important;
    }
    .select-auth-box-headline {
      color: var(--hk-green) !important;
    }
    .select-auth-box-desc {
      color: var(--hk-text-dim) !important;
    }
    /* -- Footer -------------------------------------------------------------- */
    .pf-v5-c-login__main-footer-band {
      background: transparent !important;
      border-top: 1px solid var(--hk-border) !important;
    }
    /* -- Language selector --------------------------------------------------- */
    select#login-select-toggle {
      background: var(--hk-input-bg) !important;
      color: var(--hk-green) !important;
      border-color: var(--hk-border) !important;
      font-family: var(--hk-font) !important;
    }
    /* -- Blinking cursor animation for header -------------------------------- */
    @keyframes blink {
      0%, 100% { opacity: 1; }
      50% { opacity: 0; }
    }
    #kc-header-wrapper::after {
      content: '_';
      animation: blink 1s step-end infinite;
      color: var(--hk-green);
    }
    /* -- Subtle CRT flicker ------------------------------------------------- */
    @keyframes flicker {
      0% { opacity: 0.97; }
      5% { opacity: 0.95; }
      10% { opacity: 0.98; }
      15% { opacity: 0.96; }
      20% { opacity: 0.99; }
      100% { opacity: 0.98; }
    }
    .pf-v5-c-login__main {
      animation: flicker 4s infinite;
    }
@@ -3,10 +3,24 @@ kind: Kustomization
 resources:
  - secrets.yaml
  - hacker-theme-configmap.yaml
  - auth-proxy.yaml
 # ConfigMap auth-proxy-routes is managed by Terraform (terraform/keycloak, kubernetes_config_map_v1).
 configMapGenerator:
  - name: keycloak-hexor-lab-theme
    namespace: keycloak
    files:
      - theme.properties=themes/hexor-lab/login/theme.properties
      - template.ftl=themes/hexor-lab/login/template.ftl
      - hexor.css=themes/hexor-lab/login/resources/css/hexor.css
      - keycloak.css=themes/hexor-lab/login/resources/css/keycloak.css
      - background.js=themes/hexor-lab/login/resources/js/background.js
      - favicon.svg=themes/hexor-lab/login/resources/img/favicon.svg
 generatorOptions:
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
 helmCharts:
  - name: keycloakx
    repo: https://codecentric.github.io/helm-charts
@@ -0,0 +1,110 @@
 /* Preview behavior only. Keycloak will supply the real authentication later. */
 "use strict";
 const $ = (selector) => document.querySelector(selector);
 const screens = {
  login: ["Sign in", ""],
  otp: ["Verify identity", ""],
  reset: ["Reset password", "We’ll send a reset link to your email."],
  success: ["Access granted", ""],
 };
 function notice(message = "", error = false) {
  $("#notice").textContent = message;
  $("#notice").hidden = !message;
  $("#notice").classList.toggle("error", error);
 }
 function showScreen(requested, focus = false) {
  const screen = requested === "error" ? "login" : (Object.hasOwn(screens, requested) ? requested : "login");
  const [title, description] = screens[screen];
  document.querySelectorAll("[data-view]").forEach((element) => { element.hidden = element.dataset.view !== screen; });
  $("#page-title").replaceChildren(document.createTextNode(title));
  const cursor = document.createElement("span");
  cursor.className = "cursor";
  cursor.setAttribute("aria-hidden", "true");
  cursor.textContent = "_";
  $("#page-title").append(cursor);
  $("#page-description").textContent = description;
  $("#page-description").hidden = !description;
  document.querySelectorAll("[data-screen]").forEach((button) => button.setAttribute("aria-pressed", String(button.dataset.screen === requested)));
  document.querySelectorAll(".field-error").forEach((element) => { element.hidden = true; });
  document.querySelectorAll("[aria-invalid]").forEach((element) => element.removeAttribute("aria-invalid"));
  notice(requested === "error" ? "[ DENIED ] Invalid username or password. Please try again." : "", requested === "error");
  $("#password").value = "";
  $("#password").type = "password";
  $("#reveal-password").textContent = "SHOW";
  $("#reveal-password").setAttribute("aria-label", "Show password");
  $("#reveal-password").setAttribute("aria-pressed", "false");
  $("#otp").value = "";
  if (focus) document.querySelector(`[data-view="${screen}"] input`)?.focus();
 }
 function navigate(screen) {
  if (location.hash === `#${screen}`) showScreen(screen, true);
  else location.hash = screen;
 }
 function fieldError(field, output, message) {
  field.setAttribute("aria-invalid", "true");
  output.textContent = message;
  output.hidden = false;
  field.focus();
 }
 document.querySelectorAll("[data-screen]").forEach((button) => button.addEventListener("click", () => navigate(button.dataset.screen)));
 window.addEventListener("hashchange", () => showScreen(location.hash.slice(1), true));
 $("#login-form").addEventListener("submit", (event) => {
  event.preventDefault();
  notice();
  if (!$("#username").value.trim()) return fieldError($("#username"), $("#login-error"), "Enter your username or email address.");
  if (!$("#password").value) return fieldError($("#password"), $("#login-error"), "Enter your password.");
  $("#password").value = "";
  navigate("otp");
 });
 $("#otp-form").addEventListener("submit", (event) => {
  event.preventDefault();
  if (!/^[0-9]{6}$/.test($("#otp").value)) return fieldError($("#otp"), $("#otp-error"), "Enter a 6-digit verification code.");
  $("#otp").value = "";
  navigate("success");
 });
 $("#reset-form").addEventListener("submit", (event) => {
  event.preventDefault();
  if (!$("#reset-email").validity.valid) return fieldError($("#reset-email"), $("#reset-error"), "Enter a valid email address.");
  $("#reset-error").hidden = true;
  $("#reset-email").removeAttribute("aria-invalid");
  notice("[ OK ] Reset-link confirmation preview. No email was sent.");
 });
 $("#reveal-password").addEventListener("click", () => {
  const reveal = $("#password").type === "password";
  $("#password").type = reveal ? "text" : "password";
  $("#reveal-password").textContent = reveal ? "HIDE" : "SHOW";
  $("#reveal-password").setAttribute("aria-label", reveal ? "Hide password" : "Show password");
  $("#reveal-password").setAttribute("aria-pressed", String(reveal));
 });
 $("#passkey").addEventListener("click", () => notice("[ DEMO ] Passkey authentication will be handled by Keycloak. No device prompt is opened here."));
 $("#google").addEventListener("click", () => notice("[ DEMO ] Google sign-in will be handled by Keycloak. This preview stays local."));
 document.querySelectorAll("input").forEach((input) => input.addEventListener("input", () => {
  input.removeAttribute("aria-invalid");
  input.closest("form")?.querySelectorAll(".field-error").forEach((element) => { element.hidden = true; });
 }));
 showScreen(location.hash.slice(1));
 // The optional stdlib server reloads the page when HTML/CSS/JS files change.
 if (location.protocol === "http:" && ["127.0.0.1", "localhost"].includes(location.hostname)) {
  let version;
  let checking = false;
  setInterval(async () => {
    if (checking || document.hidden) return;
    checking = true;
    try {
      const response = await fetch("/__version", { cache: "no-store" });
      if (!response.ok || !response.headers.get("content-type")?.includes("application/json")) return;
      const current = (await response.json()).version;
      if (version !== undefined && current !== version) location.reload();
      version = current;
    } catch { /* The preview also works without the local server. */ }
    finally { checking = false; }
  }, 1000);
 }
@@ -0,0 +1 @@
 ../themes/hexor-lab/login/resources/js/background.js
@@ -0,0 +1,62 @@
 <!doctype html>
 <html lang="en">
 <head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <meta name="color-scheme" content="dark">
  <meta name="description" content="Local prototype of the Hexor terminal login theme.">
  <title>Hexor · Access terminal</title>
  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' fill='%230c100c'/%3E%3Ctext x='8' y='45' fill='%23b9ef70' font-family='monospace' font-size='40'%3E%26gt;_%3C/text%3E%3C/svg%3E">
  <link rel="stylesheet" href="styles.css">
  <link rel="stylesheet" href="preview.css">
  <script src="app.js" defer></script>
  <script src="background.js" defer></script>
 </head>
 <body>
  <canvas id="code-background" aria-hidden="true"></canvas>
  <header class="site-header">
    <a class="wordmark" href="#login">Hexor Lab</a>
  </header>
  <main class="page-main">
    <div class="terminal-wrap">
      <section class="terminal" aria-labelledby="page-title">
        <div class="titlebar"><span><span aria-hidden="true">&gt;_</span> auth@hexor</span></div>
        <div class="terminal-body">
          <div class="heading"><h1 id="page-title">Sign in<span class="cursor" aria-hidden="true">_</span></h1><p id="page-description" class="description" hidden></p></div>
          <div id="notice" class="notice" role="status" hidden></div>
          <form id="login-form" data-view="login" novalidate>
            <div class="field"><label for="username"><span class="prompt" aria-hidden="true">&gt;</span> username <span class="label-hint">/ email</span></label><input id="username" name="username" autocomplete="username" autocapitalize="none" spellcheck="false" placeholder="you@hexor.cy" required aria-describedby="login-error"></div>
            <div class="field"><label for="password"><span class="prompt" aria-hidden="true">&gt;</span> password</label><div class="password-wrap"><input id="password" name="password" type="password" autocomplete="current-password" placeholder="Enter your password" required aria-describedby="login-error"><button class="reveal" id="reveal-password" type="button" aria-label="Show password" aria-pressed="false">SHOW</button></div></div>
            <p class="field-error" id="login-error" role="alert" hidden></p>
            <div class="form-options"><label class="checkbox"><input type="checkbox" name="rememberMe"> Keep me signed in</label><a href="#reset">Forgot password?</a></div>
            <button class="primary" type="submit"><span>Sign in</span><span aria-hidden="true">↵</span></button>
            <div class="divider"><span>OR CONTINUE WITH</span></div>
            <div class="alternatives"><button class="secondary" id="passkey" type="button"><span aria-hidden="true">⌘</span> Passkey</button><button class="secondary" id="google" type="button"><span class="google-icon" aria-hidden="true">G</span> Google</button></div>
          </form>
          <form id="otp-form" data-view="otp" novalidate hidden>
            <div class="field"><label for="otp"><span class="prompt" aria-hidden="true">&gt;</span> verification code</label><input id="otp" name="otp" class="otp-input" type="text" inputmode="numeric" pattern="[0-9]{6}" maxlength="6" autocomplete="one-time-code" placeholder="000000" required aria-describedby="otp-help otp-error"><p id="otp-help" class="field-help">Enter the 6-digit code from your authenticator.</p></div>
            <p id="otp-error" class="field-error" role="alert" hidden></p>
            <button class="primary" type="submit"><span>Verify identity</span><span aria-hidden="true">↵</span></button><a class="back-link" href="#login">← Back to sign in</a>
          </form>
          <form id="reset-form" data-view="reset" novalidate hidden>
            <div class="field"><label for="reset-email"><span class="prompt" aria-hidden="true">&gt;</span> email address</label><input id="reset-email" name="email" type="email" autocomplete="email" autocapitalize="none" spellcheck="false" placeholder="you@hexor.cy" required aria-describedby="reset-error"></div><p id="reset-error" class="field-error" role="alert" hidden></p>
            <button class="primary" type="submit"><span>Send reset link</span><span aria-hidden="true">↵</span></button><a class="back-link" href="#login">← Back to sign in</a>
          </form>
          <div data-view="success" class="success-panel" hidden><div class="success-mark" aria-hidden="true">[ OK ]</div><p>Identity verified.</p><a class="secondary back-link" href="#login">Back to sign in →</a></div>
        </div>
      </section>
    </div>
  </main>
  <aside class="preview" aria-label="Настройки локального предпросмотра" lang="ru">
    <div class="preview-heading"><span class="preview-label">LOCAL PREVIEW</span><span>Демо · данные никуда не отправляются</span></div>
    <div class="preview-controls"><div class="preview-group" role="group" aria-label="Экран"><button type="button" data-screen="login" aria-pressed="true">Логин</button><button type="button" data-screen="otp" aria-pressed="false">OTP</button><button type="button" data-screen="reset" aria-pressed="false">Сброс</button><button type="button" data-screen="error" aria-pressed="false">Ошибка</button></div></div>
  </aside>
 </body>
 </html>
@@ -0,0 +1,11 @@
 .preview { border-top: 1px solid #ffffff0c; padding: 18px 24px 21px; color: var(--muted); font-size: 12px; background: var(--bg); }
 .preview-heading, .preview-controls { max-width: 760px; margin: 0 auto; display: flex; gap: 15px; flex-wrap: wrap; justify-content: space-between; align-items: center; }
 .preview-heading { margin-bottom: 12px; }
 .preview-label { color: var(--text); letter-spacing: 1px; font-size: 11px; }
 .preview-group { display: flex; align-items: center; gap: 7px; flex-wrap: wrap; }
 .preview button, .preview select { border: 1px solid var(--border); border-radius: 0; background: transparent; padding: 5px 9px; color: var(--muted); font-size: 13px; }
 .preview select { background: var(--bg); }
 .preview button[aria-pressed="true"] { color: var(--accent); border-color: var(--accent); }
 .preview button:hover { color: var(--text); }
 .preview .checkbox { margin-left: 8px; }
@media (max-width: 580px) { .preview { padding: 16px; } .preview-controls { gap: 12px; } }
@@ -0,0 +1,49 @@
 {
  "realm": "hexor-preview",
  "enabled": true,
  "displayName": "Hexor Lab",
  "loginTheme": "hexor-lab",
  "sslRequired": "none",
  "registrationAllowed": false,
  "resetPasswordAllowed": true,
  "rememberMe": true,
  "loginWithEmailAllowed": true,
  "clients": [
    {
      "clientId": "theme-preview",
      "enabled": true,
      "publicClient": true,
      "standardFlowEnabled": true,
      "redirectUris": ["http://127.0.0.1:8765/*"],
      "webOrigins": ["http://127.0.0.1:8765"]
    }
  ],
  "users": [
    {
      "username": "demo",
      "enabled": true,
      "emailVerified": true,
      "email": "demo@example.test",
      "firstName": "Demo",
      "lastName": "User",
      "credentials": [{"type": "password", "value": "demo-password", "temporary": false}]
    },
    {
      "username": "demo-otp",
      "enabled": true,
      "emailVerified": true,
      "email": "demo-otp@example.test",
      "firstName": "Demo",
      "lastName": "OTP",
      "credentials": [
        {"type": "password", "value": "demo-password", "temporary": false},
        {
          "type": "otp",
          "userLabel": "Local preview",
          "secretData": "{\"value\":\"local-preview-secret\"}",
          "credentialData": "{\"subType\":\"totp\",\"digits\":6,\"counter\":0,\"period\":30,\"algorithm\":\"HmacSHA1\"}"
        }
      ]
    }
  ]
 }
@@ -0,0 +1,23 @@
 #!/usr/bin/env bash
 set -euo pipefail
 # Run the real login theme in an isolated, disposable Keycloak 26.5.6 instance.
 # Test users: demo / demo-password and demo-otp / demo-password.
 # The existing deployment, Terraform and production realm are not involved.
 preview_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 theme_dir="$(cd -- "$preview_dir/../themes/hexor-lab" && pwd)"
 printf '%s\n' \
  'After Keycloak starts, open:' \
  'http://127.0.0.1:8780/realms/hexor-preview/protocol/openid-connect/auth?client_id=theme-preview&redirect_uri=http%3A%2F%2F127.0.0.1%3A8765%2F&response_type=code&scope=openid' \
  'Test account: demo / demo-password'
 docker run --rm --name hexor-lab-preview \
  --publish 127.0.0.1:8780:8080 \
  --mount "type=bind,source=$theme_dir,target=/opt/keycloak/themes/hexor-lab,readonly" \
  --mount "type=bind,source=$preview_dir/realm.json,target=/opt/keycloak/data/import/hexor-preview.json,readonly" \
  quay.io/keycloak/keycloak:26.5.6 \
  start-dev --import-realm \
  --spi-theme--static-max-age=-1 \
  --spi-theme--cache-themes=false \
  --spi-theme--cache-templates=false
@@ -0,0 +1,49 @@
 #!/usr/bin/env python3
 """Local theme preview: python3 serve.py [--port 8765]. No dependencies."""
 import argparse
 from functools import partial
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 import json
 from pathlib import Path
 from urllib.parse import urlsplit
 ROOT = Path(__file__).resolve().parent
 ASSETS = ("index.html", "styles.css", "preview.css", "app.js", "background.js")
 class PreviewHandler(SimpleHTTPRequestHandler):
    def end_headers(self):
        self.send_header("Cache-Control", "no-store")
        super().end_headers()
    def do_GET(self):
        if urlsplit(self.path).path == "/__version":
            version = ":".join(str((ROOT / name).stat().st_mtime_ns) for name in ASSETS)
            body = json.dumps({"version": version}).encode()
            self.send_response(200)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)
        else:
            super().do_GET()
    def log_message(self, format, *args):
        if not self.path.startswith("/__version"):
            super().log_message(format, *args)
 if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--port", type=int, default=8765)
    args = parser.parse_args()
    server = ThreadingHTTPServer(("127.0.0.1", args.port), partial(PreviewHandler, directory=str(ROOT)))
    print(f"Local: http://127.0.0.1:{server.server_port}", flush=True)
    print("Edit index.html, styles.css or app.js; the browser reloads on save. Ctrl+C to stop.", flush=True)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass
    finally:
        server.server_close()
@@ -0,0 +1 @@
 ../themes/hexor-lab/login/resources/css/hexor.css
@@ -0,0 +1,202 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright [yyyy] [name of copyright owner]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
@@ -0,0 +1,83 @@
 /* Start here: all palette and layout settings are ordinary CSS variables. */
 :root {
  --bg: #0c0f0d;
  --surface: #111612;
  --input-bg: #0c110d;
  --accent: #b9ef70;
  --accent-ink: #121c09;
  --text: #e1e7dd;
  --muted: #8b9989;
  --border: #354031;
  --error: #ffa595;
  --font: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
  --terminal-width: 560px;
  --code-opacity: .16;
 }
 * { box-sizing: border-box; }
 [hidden] { display: none !important; }
 body { margin: 0; min-height: 100svh; color: var(--text); background: var(--bg); font: 16px/1.6 var(--font); display: flex; flex-direction: column; isolation: isolate; }
 #code-background { position: fixed; inset: 0; width: 100%; height: 100%; pointer-events: none; z-index: -1; opacity: var(--code-opacity); mask-image: linear-gradient(to bottom, transparent, #000 15%, #000 85%, transparent); }
 button, input, select { font: inherit; }
 button, select { cursor: pointer; }
 button, a, input, select { -webkit-tap-highlight-color: transparent; }
 button, a, select { touch-action: manipulation; }
 a { color: var(--muted); text-underline-offset: 4px; text-decoration-thickness: 1px; }
 a:hover { color: var(--accent); }
 button:focus-visible, a:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 4px; }
 ::selection { background: var(--accent); color: var(--accent-ink); }
 .site-header { padding: 27px 38px; display: flex; justify-content: space-between; gap: 20px; align-items: center; }
 .wordmark { font: 500 clamp(26px, 4vw, 34px)/1.2 var(--font); letter-spacing: -1px; text-decoration: none; color: var(--text); }
 .page-main { flex: 1; display: grid; place-items: center; padding: 35px 24px 64px; }
 .terminal-wrap { width: min(100%, var(--terminal-width)); }
 .terminal { background: var(--surface); border: 1px solid var(--border); box-shadow: 0 24px 90px #0004; }
 .titlebar { display: flex; padding: 12px 20px; border-bottom: 1px solid var(--border); font-size: 14px; color: var(--muted); }
 .titlebar span span { color: var(--accent); margin-right: 9px; }
 .terminal-body { padding: 30px 36px 34px; }
 .heading { margin: 0 0 28px; }
 h1 { margin: 0 0 8px; font-size: clamp(28px, 6vw, 34px); line-height: 1.3; font-weight: 500; letter-spacing: -1px; }
 .cursor { color: var(--accent); animation: blink 1.2s step-end infinite; }
@keyframes blink { 50% { opacity: 0; } }
 .description { color: var(--muted); margin: 0; font-size: 15px; }
 .field { margin-bottom: 21px; }
 .field > label { display: block; margin-bottom: 8px; font-size: 16px; }
 .prompt { color: var(--accent); margin-right: 7px; }
 .label-hint { color: var(--muted); font-size: 14px; }
 input:not([type="checkbox"]) { width: 100%; min-height: 50px; padding: 12px 14px; background: var(--input-bg); border: 1px solid var(--border); border-radius: 0; color: var(--text); caret-color: var(--accent); outline: none; box-shadow: none; font-size: 16px; }
 input:not([type="checkbox"]):hover { border-color: var(--muted); }
 input:not([type="checkbox"]):focus { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
 input::placeholder { color: var(--muted); opacity: .7; }
 input[aria-invalid="true"]:not([type="checkbox"]) { border-color: var(--error); }
 input:-webkit-autofill { -webkit-text-fill-color: var(--text); box-shadow: 0 0 0 1000px var(--input-bg) inset; }
 .password-wrap { position: relative; }
 .password-wrap input { padding-right: 72px; }
 .reveal { position: absolute; right: 5px; top: 5px; bottom: 5px; padding: 0 10px; background: transparent; color: var(--muted); border: 0; font-size: 12px; letter-spacing: .5px; }
 .reveal:hover { color: var(--accent); }
 .form-options { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 10px; margin: 1px 0 26px; font-size: 14px; }
 .checkbox { display: inline-flex; align-items: center; gap: 8px; cursor: pointer; color: var(--muted); }
 input[type="checkbox"] { width: 16px; height: 16px; margin: 0; accent-color: var(--accent); cursor: pointer; }
 .primary { width: 100%; min-height: 50px; display: flex; align-items: center; justify-content: space-between; padding: 11px 16px; background: var(--accent); color: var(--accent-ink); border: 1px solid var(--accent); border-radius: 0; font-weight: bold; transition: filter .15s; }
 .primary:hover { filter: brightness(1.1); }
 .primary:active { transform: translateY(1px); }
 .primary span:last-child { font-size: 20px; font-weight: normal; line-height: 1; }
 .divider { display: flex; align-items: center; gap: 15px; margin: 24px 0 18px; color: var(--muted); font-size: 11px; letter-spacing: 1px; }
 .divider::before, .divider::after { content: ""; height: 1px; background: var(--border); flex: 1; }
 .alternatives { display: flex; gap: 12px; }
 .secondary { display: inline-flex; justify-content: center; align-items: center; gap: 10px; min-height: 48px; padding: 10px 15px; background: transparent; color: var(--text); border: 1px solid var(--border); border-radius: 0; font-size: 16px; text-decoration: none; }
 .alternatives .secondary { flex: 1; }
 .secondary:hover { border-color: var(--accent); color: var(--accent); background: #ffffff03; }
 .secondary > span { color: var(--muted); font-size: 16px; }
 .google-icon { font-family: sans-serif; font-weight: bold; }
 .notice { border-left: 2px solid var(--accent); padding: 10px 13px; background: #ffffff04; color: var(--text); font-size: 15px; margin: 0 0 22px; overflow-wrap: anywhere; }
 .notice.error { border-color: var(--error); color: var(--error); }
 .field-error { color: var(--error); font-size: 14px; margin: -8px 0 18px; }
 .field-help { margin: 10px 0 0; color: var(--muted); font-size: 14px; }
 input.otp-input { letter-spacing: 10px; font-size: 24px; padding: 15px; }
 .back-link { display: block; text-align: center; margin-top: 24px; font-size: 15px; }
 .success-panel { text-align: center; padding: 15px 0; }
 .success-mark { color: var(--accent); font-size: 32px; }
@media (max-width: 580px) {
  .site-header { padding: 20px; }
  .page-main { padding: 25px 16px 40px; }
  .terminal-body { padding: 24px; }
 }
@media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation: none !important; transition: none !important; } }
@@ -0,0 +1,130 @@
 /* Adapt the native Keycloak 26.5.6 form components to the shared design. */
 html.login-pf {
  color-scheme: dark;
  --pf-v5-global--FontFamily--text: var(--font);
  --pf-v5-global--FontFamily--heading: var(--font);
  --pf-v5-global--FontFamily--monospace: var(--font);
  --pf-v5-global--FontSize--md: 16px;
  --pf-v5-global--Color--100: var(--text);
  --pf-v5-global--Color--200: var(--muted);
  --pf-v5-global--primary-color--100: var(--accent);
  --pf-v5-global--primary-color--200: var(--accent);
  --pf-v5-global--link--Color: var(--accent);
  --pf-v5-global--link--Color--hover: var(--text);
  --pf-v5-global--BackgroundColor--100: var(--surface);
  --pf-v5-global--BackgroundColor--200: var(--input-bg);
  --pf-v5-global--BorderColor--100: var(--border);
  --pf-v5-global--danger-color--100: var(--error);
  --pf-v5-global--success-color--100: var(--accent);
 }
 html.login-pf body#keycloak-bg {
  background: var(--bg);
  color: var(--text);
  font: 16px/1.6 var(--font);
  height: auto;
 }
 .hexor-heading { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; padding: 30px 36px 0; }
 .hexor-title { margin: 0; font: 500 clamp(28px, 6vw, 34px)/1.3 var(--font); letter-spacing: -1px; }
 .hexor-title::after { content: "_"; color: var(--accent); animation: blink 1.2s step-end infinite; }
 .hexor-body { padding: 28px 36px 34px; }
 .hexor-body > :last-child { margin-bottom: 0; }
 .hexor-locale { max-width: 100%; }
 .hexor-locale select { max-width: 180px; }
 .hexor-body .pf-v5-c-form { gap: 21px; }
 .hexor-body .pf-v5-c-form__group { gap: 8px; }
 .hexor-body .pf-v5-c-form__group-label { padding-bottom: 0 !important; }
 .hexor-body .pf-v5-c-form__label { color: var(--text); font-size: 16px; line-height: 1.6; }
 .hexor-body .pf-v5-c-form__label-text::before { content: ">"; color: var(--accent); margin-inline-end: 12px; }
 .hexor-body .pf-v5-c-form__helper-text { margin-top: 10px; }
 .hexor-body .pf-v5-c-helper-text { gap: 10px; flex-wrap: wrap; }
 .hexor-body .pf-v5-c-helper-text__item-text { color: var(--muted); font-size: 14px; }
 .hexor-body .pf-v5-c-check { color: var(--muted); font-size: 14px; align-items: center; }
 .hexor-body .pf-v5-c-check__label { color: var(--muted); font-size: 14px; }
 .hexor-body .pf-v5-c-form-control, .hexor-locale .pf-v5-c-form-control {
  background: var(--input-bg);
  color: var(--text);
  border: 1px solid var(--border);
  border-radius: 0;
  --pf-v5-c-form-control--after--BorderBottomWidth: 0;
  --pf-v5-c-form-control--before--BorderWidth: 0;
  --pf-v5-c-form-control--Height: auto;
 }
 .hexor-body .pf-v5-c-form-control::before,
 .hexor-body .pf-v5-c-form-control::after,
 .hexor-locale .pf-v5-c-form-control::before,
 .hexor-locale .pf-v5-c-form-control::after { display: none; }
 .hexor-body .pf-v5-c-form-control:focus-within { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
 .hexor-body .pf-v5-c-form-control.pf-m-error { border-color: var(--error); }
 .hexor-body .pf-v5-c-form-control > :is(input, textarea, select),
 .hexor-locale .pf-v5-c-form-control > select {
  border: 0;
  border-radius: 0;
  padding: 12px 14px;
  min-height: 50px;
  background: transparent;
  color: var(--text);
  font: 16px/1.6 var(--font);
  box-shadow: none;
 }
 .hexor-body .pf-v5-c-form-control > input:focus { box-shadow: none; }
 .hexor-body .pf-v5-c-form-control input[readonly] { color: var(--muted); }
 .hexor-body .pf-v5-c-input-group { gap: 0; }
 .hexor-body .pf-v5-c-input-group__item { min-width: 0; }
 .hexor-body .pf-v5-c-form-control__utilities { background: transparent; color: var(--muted); }
 .hexor-body .pf-v5-c-button {
  min-height: 48px;
  border: 1px solid var(--border);
  border-radius: 0;
  padding: 10px 15px;
  background: transparent;
  color: var(--text);
  font: 16px/1.6 var(--font);
  box-shadow: none;
  text-decoration: none;
  white-space: normal;
 }
 .hexor-body .pf-v5-c-button::after { display: none; }
 .hexor-body .pf-v5-c-button:hover { border-color: var(--accent); color: var(--accent); }
 .hexor-body .pf-v5-c-button:focus-visible { outline: 2px solid var(--accent); outline-offset: 4px; }
 .hexor-body .pf-v5-c-button.pf-m-primary {
  width: 100%;
  min-height: 50px;
  background: var(--accent);
  border-color: var(--accent);
  color: var(--accent-ink);
  font-weight: bold;
 }
 .hexor-body .pf-v5-c-button.pf-m-primary:hover { filter: brightness(1.1); }
 .hexor-body .pf-v5-c-button:disabled, .hexor-body .pf-m-aria-disabled { opacity: .5; pointer-events: none; }
 .hexor-body .pf-v5-c-button.pf-m-control { min-width: 46px; height: 100%; color: var(--muted); background: var(--input-bg); }
 .hexor-body .pf-v5-c-button.pf-m-link { min-height: auto; border-color: transparent; padding-inline: 0; color: var(--accent); }
 .hexor-body .pf-v5-c-form__actions { padding-top: 0 !important; }
 .hexor-body a:not(.pf-v5-c-button) { color: var(--muted); }
 .hexor-body a:not(.pf-v5-c-button):hover { color: var(--accent); }
 .hexor-body .pf-v5-c-alert { border: 0; border-inline-start: 2px solid var(--accent); padding: 10px 13px; background: #ffffff04; color: var(--text); box-shadow: none; }
 .hexor-body .pf-v5-c-alert.pf-m-danger { border-color: var(--error); }
 .hexor-body .pf-v5-c-alert__icon { display: none; }
 .hexor-body .pf-v5-c-alert__title { color: var(--text); font: 15px/1.6 var(--font); }
 .hexor-body .pf-v5-c-alert.pf-m-danger .pf-v5-c-alert__title,
 .hexor-body .pf-v5-c-helper-text__item.pf-m-error .pf-v5-c-helper-text__item-text { color: var(--error); }
 .hexor-footer:empty { display: none; }
 .hexor-footer-band { padding: 0; margin-top: 22px; color: var(--muted); font-size: 14px; }
 .hexor-divider-label { display: flex; align-items: center; gap: 15px; margin: 24px 0 18px; font-size: 11px; letter-spacing: 1px; }
 .hexor-divider-label::before, .hexor-divider-label::after { content: ""; flex: 1; height: 1px; background: var(--border); }
 #kc-social-providers ul { display: flex; flex-wrap: wrap; gap: 12px; padding: 0; margin: 0; list-style: none; }
 #kc-social-providers li { flex: 1 1 160px; padding: 0; }
 #kc-social-providers a { display: flex; justify-content: center !important; align-items: center; gap: 10px; }
 #kc-social-providers svg { width: 20px; height: 20px; flex-shrink: 0; fill: currentColor; }
 #kc-social-providers svg:not(.google) { filter: none; }
 #kc-social-providers .pf-v5-u-m-auto { margin: 0 !important; }
 #kc-select-try-another-way-form { margin-top: 20px; }
 #kc-info-wrapper, #kc-registration { color: var(--muted); font-size: 14px; }
 #kc-otp-login-form #otp { font-size: 24px; letter-spacing: 8px; }
 .hexor-body :is(.pf-v5-c-tile, .pf-v5-c-data-list__item, .pf-v5-c-panel) { background: var(--input-bg); color: var(--text); border: 1px solid var(--border); box-shadow: none; }
 .hexor-body .pf-v5-c-tile.pf-m-selected { border-color: var(--accent); }
 .hexor-body img, .hexor-body canvas, .hexor-body svg { max-width: 100%; }
 .hexor-body pre { overflow-wrap: anywhere; white-space: pre-wrap; }
@media (max-width: 580px) {
  .hexor-heading { padding: 24px 24px 0; }
  .hexor-body { padding: 24px; }
 }
@@ -0,0 +1 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><rect width="64" height="64" fill="#0c0f0d"/><text x="7" y="43" fill="#b9ef70" font-family="monospace" font-size="34">&gt;_</text></svg>
@@ -0,0 +1,126 @@
 /* Decorative code stream. Brightness: --code-opacity in css/hexor.css. */
 (() => {
  "use strict";
  const canvas = document.querySelector("#code-background");
  const context = canvas?.getContext("2d");
  if (!context) return;
  const source = [
    "use std::sync::Arc;",
    "use tokio::net::TcpListener;",
    "",
    "async fn main() -> Result<()> {",
    "    let config = Config::load()?;",
    "    let state = Arc::new(State::new());",
    "    let router = routes(state.clone());",
    "",
    "    server::bind(config.address)",
    "        .serve(router)",
    "        .await?;",
    "    Ok(())",
    "}",
    "",
    "impl Session {",
    "    pub fn verify(&self) -> bool {",
    "        self.expires_at > clock::now()",
    "            && self.signature.is_valid()",
    "    }",
    "}",
    "",
    "fn routes(state: Arc<State>) -> Router {",
    "    Router::new()",
    '        .route("/", get(index))',
    '        .route("/session", post(create))',
    "        .with_state(state)",
    "}",
    "",
    "match identity.authenticate().await {",
    "    Ok(session) => session.commit(),",
    "    Err(error) => trace!(?error),",
    "}",
    "",
    "let transport = Transport::builder()",
    "    .with_timeout(Duration::from_secs(30))",
    "    .with_keepalive(true)",
    "    .connect(endpoint).await?;",
    "",
  ];
  const motion = window.matchMedia("(prefers-reduced-motion: reduce)");
  const lineHeight = 24;
  let streams = [];
  let width = 0;
  let height = 0;
  let color;
  function updateColor() {
    color = getComputedStyle(document.documentElement).getPropertyValue("--accent").trim();
  }
  function resize() {
    width = window.innerWidth;
    height = window.innerHeight;
    const ratio = Math.min(window.devicePixelRatio || 1, 1.5);
    canvas.width = Math.round(width * ratio);
    canvas.height = Math.round(height * ratio);
    context.setTransform(ratio, 0, 0, ratio, 0, 0);
    context.font = '13px "SFMono-Regular", Consolas, "Liberation Mono", monospace';
    const columns = Math.max(1, Math.ceil(width / 470));
    const rows = Math.ceil(height / lineHeight);
    streams = Array.from({ length: columns }, (_, column) => {
      const start = Math.floor(Math.random() * source.length);
      const count = Math.max(4, rows - 7 + Math.floor(Math.random() * 5));
      return {
        x: column * (width / columns) + 24,
        y: 44 - Math.random() * 80,
        lines: Array.from({ length: count }, (_, i) => source[(start + i) % source.length]),
        next: (start + count) % source.length,
        typed: Math.random() * 12,
        pause: 0,
        number: 20 + start,
        speed: .55 + Math.random() * .6,
      };
    });
    draw(false);
  }
  function draw(advance) {
    context.clearRect(0, 0, width, height);
    context.fillStyle = color;
    for (const stream of streams) {
      const current = source[stream.next];
      if (advance) {
        if (stream.typed < current.length) stream.typed += stream.speed;
        else if (++stream.pause > 8) {
          stream.lines.push(current);
          stream.next = (stream.next + 1) % source.length;
          stream.typed = 0;
          stream.pause = 0;
        }
        // Scroll gently as new lines reach the bottom of the viewport.
        const overflow = stream.y + (stream.lines.length + 2) * lineHeight - height;
        if (overflow > 0) stream.y -= Math.min(1.5, overflow);
        if (stream.y < -lineHeight) {
          stream.lines.shift();
          stream.y += lineHeight;
          stream.number++;
        }
      }
      stream.lines.concat(current.slice(0, Math.floor(stream.typed))).forEach((line, i) => {
        const y = stream.y + i * lineHeight;
        context.globalAlpha = .3;
        context.fillText(String(stream.number + i).padStart(3, "0"), stream.x, y);
        context.globalAlpha = i === stream.lines.length ? .95 : .6;
        context.fillText(line, stream.x + 42, y);
      });
    }
    context.globalAlpha = 1;
  }
  updateColor();
  resize();
  window.addEventListener("resize", resize);
  // Ten frames per second is enough for slow typing; no work in hidden tabs.
  setInterval(() => {
    if (!document.hidden && !motion.matches) draw(true);
  }, 100);
 })();
@@ -0,0 +1,247 @@
 <#-- Based on Keycloak 26.5.6 keycloak.v2/login/template.ftl (Apache-2.0). -->
 <#import "field.ftl" as field>
 <#import "footer.ftl" as loginFooter>
 <#macro username>
  <#assign label>
    <#if !realm.loginWithEmailAllowed>${msg("username")}<#elseif !realm.registrationEmailAsUsername>${msg("usernameOrEmail")}<#else>${msg("email")}</#if>
  </#assign>
  <@field.group name="username" label=label>
    <div class="${properties.kcInputGroup}">
      <div class="${properties.kcInputGroupItemClass} ${properties.kcFill}">
        <span class="${properties.kcInputClass} ${properties.kcFormReadOnlyClass}">
          <input id="kc-attempted-username" value="${auth.attemptedUsername}" readonly>
        </span>
      </div>
      <div class="${properties.kcInputGroupItemClass}">
        <button id="reset-login" class="${properties.kcFormPasswordVisibilityButtonClass} kc-login-tooltip" type="button"
              aria-label="${msg('restartLoginTooltip')}" onclick="location.href='${url.loginRestartFlowUrl}'">
            <i class="fa-sync-alt fas" aria-hidden="true"></i>
            <span class="kc-tooltip-text">${msg("restartLoginTooltip")}</span>
        </button>
      </div>
    </div>
  </@field.group>
 </#macro>
 <#macro registrationLayout bodyClass="" displayInfo=false displayMessage=true displayRequiredFields=false>
 <!DOCTYPE html>
 <html class="${properties.kcHtmlClass!}" lang="${lang}"<#if realm.internationalizationEnabled> dir="${(locale.rtl)?then('rtl','ltr')}"</#if>>
 <head>
    <meta charset="utf-8">
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
    <meta name="color-scheme" content="dark">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <#if properties.meta?has_content>
        <#list properties.meta?split(' ') as meta>
            <meta name="${meta?split('==')[0]}" content="${meta?split('==')[1]}"/>
        </#list>
    </#if>
    <title>${msg("loginTitle",(realm.displayName!''))}</title>
    <link rel="icon" href="${url.resourcesPath}/img/favicon.svg" />
    <#if properties.stylesCommon?has_content>
        <#list properties.stylesCommon?split(' ') as style>
            <link href="${url.resourcesCommonPath}/${style}" rel="stylesheet" />
        </#list>
    </#if>
    <#if properties.styles?has_content>
        <#list properties.styles?split(' ') as style>
            <link href="${url.resourcesPath}/${style}" rel="stylesheet" />
        </#list>
    </#if>
    <script type="importmap">
        {
            "imports": {
                "rfc4648": "${url.resourcesCommonPath}/vendor/rfc4648/rfc4648.js"
            }
        }
    </script>
    <#if properties.scripts?has_content>
        <#list properties.scripts?split(' ') as script>
            <script src="${url.resourcesPath}/${script}" defer></script>
        </#list>
    </#if>
    <#if scripts??>
        <#list scripts as script>
            <script src="${script}" type="text/javascript"></script>
        </#list>
    </#if>
    <script type="module" src="${url.resourcesPath}/js/passwordVisibility.js"></script>
    <script type="module">
        <#outputformat "JavaScript">
        import { startSessionPolling } from "${url.resourcesPath}/js/authChecker.js";
        startSessionPolling(
            ${url.ssoLoginInOtherTabsUrl?c}
        );
        </#outputformat>
    </script>
    <script type="module">
        document.addEventListener("click", (event) => {
            const link = event.target.closest("a[data-once-link]");
            if (!link) {
                return;
            }
            if (link.getAttribute("aria-disabled") === "true") {
                event.preventDefault();
                return;
            }
            const { disabledClass } = link.dataset;
            if (disabledClass) {
                link.classList.add(...disabledClass.trim().split(/\s+/));
            }
            link.setAttribute("role", "link");
            link.setAttribute("aria-disabled", "true");
        });
    </script>
    <#if authenticationSession??>
        <script type="module">
             <#outputformat "JavaScript">
            import { checkAuthSession } from "${url.resourcesPath}/js/authChecker.js";
            checkAuthSession(
                ${authenticationSession.authSessionIdHash?c}
            );
            </#outputformat>
        </script>
    </#if>
    <script>
      // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=1404468
      const isFirefox = true;
    </script>
 </head>
 <body id="keycloak-bg" class="${properties.kcBodyClass!}" data-page-id="login-${pageId}">
 <canvas id="code-background" aria-hidden="true"></canvas>
 <header class="site-header"><span class="wordmark">Hexor Lab</span></header>
 <div class="${properties.kcLogin!} page-main">
  <div class="${properties.kcLoginContainer!}">
    <main class="${properties.kcLoginMain!} terminal">
      <div class="titlebar"><span><span aria-hidden="true">&gt;_</span> auth@hexor</span></div>
      <div class="${properties.kcLoginMainHeader!}">
        <h1 class="${properties.kcLoginMainTitle!}" id="kc-page-title"><#nested "header"></h1>
        <#if realm.internationalizationEnabled  && locale.supported?size gt 1>
        <div class="${properties.kcLoginMainHeaderUtilities!}">
          <div class="${properties.kcInputClass!}">
            <select
              aria-label="${msg("languages")}"
              id="login-select-toggle"
              onchange="if (this.value) window.location.href=this.value"
            >
              <#list locale.supported?sort_by("label") as l>
                <option
                  value="${l.url}"
                  ${(l.languageTag == locale.currentLanguageTag)?then('selected','')}
                >
                  ${l.label}
                </option>
              </#list>
            </select>
            <span class="${properties.kcFormControlUtilClass}">
              <span class="${properties.kcFormControlToggleIcon!}">
                <svg
                  class="pf-v5-svg"
                  viewBox="0 0 320 512"
                  fill="currentColor"
                  aria-hidden="true"
                  role="img"
                  width="1em"
                  height="1em"
                >
                  <path
                    d="M31.3 192h257.3c17.8 0 26.7 21.5 14.1 34.1L174.1 354.8c-7.8 7.8-20.5 7.8-28.3 0L17.2 226.1C4.6 213.5 13.5 192 31.3 192z"
                  >
                  </path>
                </svg>
              </span>
            </span>
          </div>
        </div>
        </#if>
      </div>
      <div class="${properties.kcLoginMainBody!}">
        <#if !(auth?has_content && auth.showUsername() && !auth.showResetCredentials())>
            <#if displayRequiredFields>
                <div class="${properties.kcContentWrapperClass!}">
                    <div class="${properties.kcLabelWrapperClass!} subtitle">
                        <span class="${properties.kcInputHelperTextItemTextClass!}">
                          <span class="${properties.kcInputRequiredClass!}">*</span> ${msg("requiredFields")}
                        </span>
                    </div>
                </div>
            </#if>
        <#else>
            <#if displayRequiredFields>
                <div class="${properties.kcContentWrapperClass!}">
                    <div class="${properties.kcLabelWrapperClass!} subtitle">
                        <span class="${properties.kcInputHelperTextItemTextClass!}">
                          <span class="${properties.kcInputRequiredClass!}">*</span> ${msg("requiredFields")}
                        </span>
                    </div>
                    <div class="${properties.kcFormClass} ${properties.kcContentWrapperClass}">
                        <#nested "show-username">
                        <@username />
                    </div>
                </div>
            <#else>
                <div class="${properties.kcFormClass} ${properties.kcContentWrapperClass}">
                  <#nested "show-username">
                  <@username />
                </div>
            </#if>
        </#if>
        <#-- App-initiated actions should not see warning messages about the need to complete the action -->
        <#-- during login.                                                                               -->
        <#if displayMessage && message?has_content && (message.type != 'warning' || !isAppInitiatedAction??)>
            <div class="${properties.kcAlertClass!} pf-m-${(message.type = 'error')?then('danger', message.type)}">
                <div class="${properties.kcAlertIconClass!}">
                    <#if message.type = 'success'><span class="${properties.kcFeedbackSuccessIcon!}"></span></#if>
                    <#if message.type = 'warning'><span class="${properties.kcFeedbackWarningIcon!}"></span></#if>
                    <#if message.type = 'error'><span class="${properties.kcFeedbackErrorIcon!}"></span></#if>
                    <#if message.type = 'info'><span class="${properties.kcFeedbackInfoIcon!}"></span></#if>
                </div>
                <span class="${properties.kcAlertTitleClass!} kc-feedback-text">${message.summary}</span>
            </div>
        </#if>
        <#nested "form">
        <#if auth?has_content && auth.showTryAnotherWayLink()>
          <form id="kc-select-try-another-way-form" action="${url.loginAction}" method="post" novalidate="novalidate">
              <input type="hidden" name="tryAnotherWay" value="on"/>
              <a id="try-another-way" href="javascript:document.forms['kc-select-try-another-way-form'].requestSubmit()"
                  class="${properties.kcButtonSecondaryClass} ${properties.kcButtonBlockClass} ${properties.kcMarginTopClass}">
                    ${msg("doTryAnotherWay")}
              </a>
          </form>
        </#if>
          <div class="${properties.kcLoginMainFooter!}">
              <#nested "socialProviders">
              <#if displayInfo>
                  <div id="kc-info" class="${properties.kcLoginMainFooterBand!} ${properties.kcFormClass}">
                      <div id="kc-info-wrapper" class="${properties.kcLoginMainFooterBandItem!}">
                          <#nested "info">
                      </div>
                  </div>
              </#if>
          </div>
      </div>
        <div class="${properties.kcLoginMainFooter!}">
            <@loginFooter.content/>
        </div>
    </main>
  </div>
 </div>
 </body>
 </html>
 </#macro>
@@ -0,0 +1,18 @@
 parent=keycloak.v2
 import=common/keycloak
 styles=css/styles.css css/hexor.css css/keycloak.css
 scripts=js/background.js
 darkMode=false
 # Own layout; inherit Keycloak's form fields, authentication and translations.
 kcLogin=hexor-page
 kcLoginContainer=terminal-wrap
 kcLoginMain=hexor-main
 kcLoginMainHeader=hexor-heading
 kcLoginMainTitle=hexor-title
 kcLoginMainBody=hexor-body
 kcLoginMainHeaderUtilities=hexor-locale
 kcLoginMainFooter=hexor-footer
 kcLoginMainFooterBand=hexor-footer-band
 kcLoginMainFooterBandItem=hexor-footer-item
 kcLoginMainFooterHelperText=hexor-divider-label
@@ -1,5 +1,9 @@
 replicas: 1
 nodeSelector:
  homelab.hexor.cy/pool: core
  homelab.hexor.cy/exposure: external
 image:
  repository: quay.io/keycloak/keycloak
  tag: "26.5.6"
@@ -24,17 +28,27 @@ extraEnv: |
    value: "-Djgroups.dns.query=keycloak-keycloakx-headless.keycloak.svc"
 extraVolumes: |
-  - name: hacker-theme
+  - name: hexor-lab-theme
    configMap:
-      name: keycloak-hacker-theme
+      name: keycloak-hexor-lab-theme
      items:
        - key: theme.properties
          path: theme.properties
        - key: template.ftl
          path: template.ftl
        - key: hexor.css
          path: resources/css/hexor.css
        - key: keycloak.css
          path: resources/css/keycloak.css
        - key: background.js
          path: resources/js/background.js
        - key: favicon.svg
          path: resources/img/favicon.svg
 extraVolumeMounts: |
-  - name: hacker-theme
+  - name: hexor-lab-theme
-    mountPath: /opt/keycloak/themes/hacker/login/theme.properties
+    mountPath: /opt/keycloak/themes/hexor-lab/login
-    subPath: theme.properties
+    readOnly: true
  - name: hacker-theme
    mountPath: /opt/keycloak/themes/hacker/login/css/hacker.css
    subPath: hacker.css
 dbchecker:
  enabled: true
@@ -2,7 +2,11 @@
 # own node with a node-local volume. Replication is asynchronous (nodes sit in
 # different regions), so a failover can lose the last few transactions.
 #
-# Nodes eligible for instances carry the label workload/postgres=true. To move a
+# Layout: the primary runs on the music node (the powerful one), the replica on
 # master. A failover swaps the roles; switch back with
 #   kubectl cnpg promote main <instance on music>
 #
 # Eligible nodes are core/external and carry workload/postgres=true. To move a
 # replica: label the new node, unlabel the old one, then recreate the instance
 # (kubectl cnpg destroy main <N>); the operator re-clones it from the primary.
 #
@@ -32,6 +36,8 @@ spec:
    podAntiAffinityType: required
    topologyKey: kubernetes.io/hostname
    nodeSelector:
      homelab.hexor.cy/pool: core
      homelab.hexor.cy/exposure: external
      workload/postgres: "true"
  resources:
@@ -0,0 +1,86 @@
 # Database "furumi": password, role, database and read access for the app.
 # To add another database copy this file and replace the name.
 # Generated once; the role and the application both read this Secret.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: furumi-db
  namespace: psql
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: furumi-db
    template:
      type: kubernetes.io/basic-auth
      metadata:
        labels:
          cnpg.io/reload: "true"
      data:
        username: furumi
        password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: db-password
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
 metadata:
  name: main-furumi
  namespace: psql
 spec:
  cluster:
    name: main
  name: furumi
  login: true
  passwordSecret:
    name: furumi-db
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:
  name: main-furumi
  namespace: psql
 spec:
  cluster:
    name: main
  name: furumi
  owner: furumi
  extensions:
    # `ensure` is defaulted by the CRD inside list items; spelled out so Argo CD
    # does not see it as drift.
    - name: pg_trgm
      ensure: present
  databaseReclaimPolicy: retain
 ---
 # Lets the furumi namespace read only its own credentials Secret.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: furumi-db-reader
  namespace: psql
 rules:
  - apiGroups: [""]
    resources: [secrets]
    resourceNames: [furumi-db]
    verbs: [get, list, watch]
  - apiGroups: [authorization.k8s.io]
    resources: [selfsubjectrulesreviews]
    verbs: [create]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: furumi-db-reader
  namespace: psql
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: furumi-db-reader
 subjects:
  - kind: ServiceAccount
    name: db-credentials-reader
    namespace: furumi
@@ -0,0 +1,78 @@
 # Database "grafana": generated credentials, role, database and read access for
 # the monitoring namespace.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: grafana-db
  namespace: psql
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: grafana-db
    template:
      type: kubernetes.io/basic-auth
      metadata:
        labels:
          cnpg.io/reload: "true"
      data:
        username: grafana
        password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: db-password
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
 metadata:
  name: main-grafana
  namespace: psql
 spec:
  cluster:
    name: main
  name: grafana
  login: true
  passwordSecret:
    name: grafana-db
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:
  name: main-grafana
  namespace: psql
 spec:
  cluster:
    name: main
  name: grafana
  owner: grafana
  databaseReclaimPolicy: retain
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: grafana-db-reader
  namespace: psql
 rules:
  - apiGroups: [""]
    resources: [secrets]
    resourceNames: [grafana-db]
    verbs: [get, list, watch]
  - apiGroups: [authorization.k8s.io]
    resources: [selfsubjectrulesreviews]
    verbs: [create]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: grafana-db-reader
  namespace: psql
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: grafana-db-reader
 subjects:
  - kind: ServiceAccount
    name: db-credentials-reader
    namespace: prometheus
@@ -0,0 +1,55 @@
 # Database "jacob": password, role, database and read access for the app.
 # To add another database copy this file and replace the name.
 # Generated once; the role and the application both read this Secret.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: jacob-db
  namespace: psql
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: jacob-db
    template:
      type: kubernetes.io/basic-auth
      metadata:
        labels:
          cnpg.io/reload: "true"
      data:
        username: jacob
        password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: db-password
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
 metadata:
  name: main-jacob
  namespace: psql
 spec:
  cluster:
    name: main
  name: jacob
  login: true
  passwordSecret:
    name: jacob-db
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:
  name: main-jacob
  namespace: psql
 spec:
  cluster:
    name: main
  name: jacob
  owner: jacob
  databaseReclaimPolicy: retain
  extensions:
    - name: vector
      ensure: present
@@ -0,0 +1,81 @@
 # Database "meguri": password, role, database and read access for the app.
 # To add another database copy this file and replace the name.
 # Generated once; the role and the application both read this Secret.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: meguri-db
  namespace: psql
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: meguri-db
    template:
      type: kubernetes.io/basic-auth
      metadata:
        labels:
          cnpg.io/reload: "true"
      data:
        username: meguri
        password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: db-password
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
 metadata:
  name: main-meguri
  namespace: psql
 spec:
  cluster:
    name: main
  name: meguri
  login: true
  passwordSecret:
    name: meguri-db
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:
  name: main-meguri
  namespace: psql
 spec:
  cluster:
    name: main
  name: meguri
  owner: meguri
  databaseReclaimPolicy: retain
 ---
 # Lets the meguri namespace read only its own credentials Secret.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: meguri-db-reader
  namespace: psql
 rules:
  - apiGroups: [""]
    resources: [secrets]
    resourceNames: [meguri-db]
    verbs: [get, list, watch]
  - apiGroups: [authorization.k8s.io]
    resources: [selfsubjectrulesreviews]
    verbs: [create]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: meguri-db-reader
  namespace: psql
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: meguri-db-reader
 subjects:
  - kind: ServiceAccount
    name: db-credentials-reader
    namespace: meguri
@@ -0,0 +1,81 @@
 # Database "web_petting": password, role, database and read access for the app.
 # To add another database copy this file and replace the name.
 # Generated once; the role and the application both read this Secret.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: web-petting-db
  namespace: psql
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: web-petting-db
    template:
      type: kubernetes.io/basic-auth
      metadata:
        labels:
          cnpg.io/reload: "true"
      data:
        username: web_petting
        password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: db-password
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
 metadata:
  name: main-web-petting
  namespace: psql
 spec:
  cluster:
    name: main
  name: web_petting
  login: true
  passwordSecret:
    name: web-petting-db
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:
  name: main-web-petting
  namespace: psql
 spec:
  cluster:
    name: main
  name: web_petting
  owner: web_petting
  databaseReclaimPolicy: retain
 ---
 # Lets the web-petting namespace read only its own credentials Secret.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: web-petting-db-reader
  namespace: psql
 rules:
  - apiGroups: [""]
    resources: [secrets]
    resourceNames: [web-petting-db]
    verbs: [get, list, watch]
  - apiGroups: [authorization.k8s.io]
    resources: [selfsubjectrulesreviews]
    verbs: [create]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: web-petting-db-reader
  namespace: psql
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: web-petting-db-reader
 subjects:
  - kind: ServiceAccount
    name: db-credentials-reader
    namespace: web-petting
@@ -6,3 +6,8 @@ resources:
  - password-generator.yaml
  - databases/vaultwarden.yaml
  - databases/keycloak.yaml
  - databases/furumi.yaml
  - databases/web-petting.yaml
  - databases/meguri.yaml
  - databases/grafana.yaml
  - databases/jacob.yaml
@@ -0,0 +1,63 @@
 # Monitoring stack (new cluster)
 The stack is intentionally a fresh installation: no Prometheus TSDB, Loki log
 chunks, or Grafana database are copied from the old cluster.
 ## Layout
 - `kube-prometheus-stack`: Prometheus, Alertmanager, Grafana, exporters and
  Kubernetes alerting rules.
 - `loki`: one filesystem-backed monolithic instance. A second replica would
  require shared object storage and is deliberately not faked with a local PV.
 - `alloy`: unprivileged Kubernetes API log and event collection. This replaces
  EOL Promtail and does not require host log mounts or privileged sysctls.
 - Grafana state is stored in the `main` CNPG cluster; metrics and logs stay on
  the large local disk of node `music`.
 Stack services select `homelab.hexor.cy/pool=core` and
 `homelab.hexor.cy/exposure=external`. Prometheus, Alertmanager and Loki also
 select `music` because their volumes are local. Grafana, the operators, Alloy
 and the Loki gateway can run on either `master` or `music`. Node Exporter and
 Loki Canary are not restricted to these groups.
 ## Storage and retention
 The PreSync hook creates these directories on `music`:
 | Path | PV | Retention |
 | --- | ---: | --- |
 | `/data/k8s/monitoring/prometheus` | 500 Gi | up to 365 days / 425 GB |
 | `/data/k8s/monitoring/loki` | 200 Gi | 90 days |
 | `/data/k8s/monitoring/alertmanager` | 5 Gi | 120 hours of notification state |
 All PVs and the StorageClass use `Retain`; Argo CD is also told not to prune
 them. This protects data from a manifest rename, but it is not a backup. Back up
 `/data/k8s/monitoring` with the rest of the `music` node if history matters.
 ## Dependencies
 - node `music` must exist and `/data` must be the intended large filesystem;
 - `main-rw.psql.svc` and the generated `grafana-db` Secret must be ready;
 - `vaultwarden-fields` must resolve `k8s/prometheus/grafana-secrets` with the
  custom fields `IDM Client ID`, `IDM Client Secret`, `TG Token`, and `TG ID`;
 - DNS for `gf.hexor.cy` and `prom.hexor.cy` must point to the new cluster.
 ## Alerting and target identities
 Telegram is reserved for actionable alerts: `severity=info`, `Watchdog`, and
 `InfoInhibitor` remain visible in Prometheus but route to the null receiver.
 Warning and critical notifications are grouped for two minutes and rendered as
 Telegram HTML with escaped annotations and labelled alerts/runbook links.
 Notifications show at most three alerts, with each alert's escaped text limited
 to 500 characters. A single short Prometheus alerts link replaces repeated graph
 URLs; the full group remains visible there. Workload alerts show the affected
 DaemonSet, Deployment or StatefulSet rather than the kube-state-metrics pod
 which supplied the metric.
 Node Exporter keeps scraping the discovered node address, but rewrites the
 public `instance` label from Kubernetes discovery to `<node-name>:9100`. Grafana
 variables and alerts therefore remain stable when node IP addresses change.
 After Argo CD reports the application healthy, verify targets, rules and log
 delivery in Grafana. Prometheus is protected by the Keycloak auth proxy; Grafana
 uses its own Keycloak OIDC client.
@@ -0,0 +1,71 @@
 apiVersion: monitoring.coreos.com/v1alpha1
 kind: AlertmanagerConfig
 metadata:
  name: telegram
  labels:
    alertmanager: main
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  route:
    groupBy: [cluster, namespace, alertname]
    # A two-minute grace period absorbs alerts which resolve immediately and
    # never need to interrupt a human.
    groupWait: 2m
    groupInterval: 10m
    repeatInterval: 12h
    receiver: telegram
    routes:
      - receiver: "null"
        matchers:
          - name: alertname
            value: Watchdog|InfoInhibitor
            matchType: =~
      # Informational alerts remain queryable in Prometheus but do not page the
      # Telegram chat. Warning and critical alerts still use the parent route.
      - receiver: "null"
        matchers:
          - name: severity
            value: info
            matchType: =
  receivers:
    - name: telegram
      telegramConfigs:
        - botToken:
            name: alertmanager-telegram
            key: bot-token
          # Telegram chat IDs are routing identifiers rather than credentials;
          # AlertmanagerConfig only supports an integer here, not a Secret ref.
          chatID: 124317807
          sendResolved: true
          parseMode: HTML
          # Bound the rendered HTML, including escaped text, before Telegram's
          # 4096-character limit. Keep the full group accessible in Prometheus.
          message: |-
            {{- define "telegram.alert.text" -}}
            {{- $escaped := html . -}}
            {{- $short := printf "%.500s" $escaped -}}
            {{- $short | reReplaceAll "&[^;]*$" "" | safeHtml -}}{{- if ne $short $escaped }}…{{ end -}}
            {{- end -}}
            {{- $alerts := .Alerts.Resolved -}}
            {{- if eq .Status "firing" }}{{ $alerts = .Alerts.Firing }}{{ end -}}
            {{ if eq .Status "firing" }}🔥 <b>FIRING · {{ len .Alerts.Firing }}</b>{{ else }}✅ <b>RESOLVED · {{ len .Alerts.Resolved }}</b>{{ end }}
            {{ with .CommonLabels.alertname }}<b>{{ printf "%.64s" . }}</b>{{ end }}{{ with .CommonLabels.severity }} · <code>{{ printf "%.16s" . }}</code>{{ end }}
            {{ range $index, $alert := $alerts }}{{ if lt $index 3 }}
            {{- $context := "" -}}
            {{- with .Labels.namespace }}{{ $context = printf "Namespace: %s\n" . }}{{ end -}}
            {{- if .Labels.daemonset }}{{ $context = printf "%sDaemonSet: %s\n" $context .Labels.daemonset -}}
            {{- else if .Labels.deployment }}{{ $context = printf "%sDeployment: %s\n" $context .Labels.deployment -}}
            {{- else if .Labels.statefulset }}{{ $context = printf "%sStatefulSet: %s\n" $context .Labels.statefulset -}}
            {{- else if .Labels.job_name }}{{ $context = printf "%sJob: %s\n" $context .Labels.job_name -}}
            {{- else if .Labels.pod }}{{ $context = printf "%sPod: %s\n" $context .Labels.pod -}}
            {{- end -}}
            {{- with .Labels.node }}{{ $context = printf "%sNode: %s\n" $context . -}}
            {{- else }}{{ if and (not .Labels.daemonset) (not .Labels.deployment) (not .Labels.statefulset) .Labels.instance }}{{ $context = printf "%sInstance: %s\n" $context .Labels.instance }}{{ end }}{{ end -}}
            {{ if eq .Status "firing" }}🔴{{ else }}🟢{{ end }} {{ template "telegram.alert.text" (printf "%s%s\n%s" $context .Annotations.summary .Annotations.description) }}
            Started: {{ .StartsAt | tz "Europe/London" | date "2006-01-02 15:04 MST" }}
            {{ end }}{{ end }}
            {{ if gt (len $alerts) 3 }}Showing the first 3 of {{ len $alerts }} alerts; open Prometheus for the full group.{{ end }}
            <a href="https://prom.hexor.cy/alerts">📈 Open alerts</a>{{ with .CommonAnnotations.runbook_url }}{{ if le (len .) 256 }} · <a href="{{ . }}">📖 Runbook</a>{{ end }}{{ end }}
    - name: "null"
@@ -0,0 +1,161 @@
 alloy:
  enableReporting: false
  storagePath: /var/lib/alloy
  configMap:
    create: true
    content: |-
      discovery.kubernetes "pods" {
        role = "pod"
      }
      discovery.relabel "pod_logs" {
        targets = discovery.kubernetes.pods.targets
        rule {
          source_labels = ["__meta_kubernetes_namespace"]
          target_label  = "namespace"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_name"]
          target_label  = "pod"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_container_name"]
          target_label  = "container"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_node_name"]
          target_label  = "node"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_label_app"]
          regex         = `(.+)`
          target_label  = "app"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
          regex         = `(.+)`
          target_label  = "app"
        }
        rule {
          source_labels = ["namespace", "app"]
          separator     = "/"
          regex         = `(.+)/(.+)`
          replacement   = "$1/$2"
          target_label  = "job"
        }
      }
      loki.source.kubernetes "pods" {
        targets    = discovery.relabel.pod_logs.output
        forward_to = [loki.process.pods.receiver]
      }
      loki.process "pods" {
        stage.static_labels {
          values = {
            cluster = "hexor-ng",
          }
        }
        forward_to = [loki.write.local.receiver]
      }
      loki.source.kubernetes_events "cluster_events" {
        job_name   = "integrations/kubernetes/eventhandler"
        log_format = "logfmt"
        forward_to = [loki.process.cluster_events.receiver]
      }
      loki.process "cluster_events" {
        stage.static_labels {
          values = {
            cluster = "hexor-ng",
          }
        }
        forward_to = [loki.write.local.receiver]
      }
      loki.write "local" {
        endpoint {
          url = "http://loki-gateway.prometheus.svc/loki/api/v1/push"
        }
      }
  mounts:
    extra:
      - name: data
        mountPath: /var/lib/alloy
  securityContext:
    allowPrivilegeEscalation: false
    readOnlyRootFilesystem: true
    runAsNonRoot: true
    runAsUser: 473
    runAsGroup: 473
    capabilities:
      drop: [ALL]
    seccompProfile:
      type: RuntimeDefault
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: "1"
      memory: 1Gi
  livenessProbe:
    httpGet:
      path: /metrics
      port: 12345
      scheme: HTTP
    initialDelaySeconds: 30
    periodSeconds: 30
    timeoutSeconds: 2
    failureThreshold: 3
 global:
  podSecurityContext:
    fsGroup: 473
    fsGroupChangePolicy: OnRootMismatch
    runAsNonRoot: true
    seccompProfile:
      type: RuntimeDefault
 controller:
  type: deployment
  replicas: 1
  nodeSelector:
    homelab.hexor.cy/pool: core
    homelab.hexor.cy/exposure: external
  priorityClassName: monitoring-critical
  minReadySeconds: 10
  terminationGracePeriodSeconds: 60
  volumes:
    extra:
      - name: data
        emptyDir: {}
 configReloader:
  resources:
    requests:
      cpu: 10m
      memory: 50Mi
    limits:
      cpu: 100m
      memory: 128Mi
 # Only the API permissions used by pod-log discovery and cluster events.
 rbac:
  create: true
  rules:
    - apiGroups: [""]
      resources: [pods, pods/log, namespaces, events]
      verbs: [get, list, watch]
  # Kept separate because the Alloy chart concatenates this list into the
  # ClusterRole. Node metadata is useful for diagnosing a log source.
  clusterRules:
    - apiGroups: [""]
      resources: [nodes]
      verbs: [get, list, watch]
 serviceMonitor:
  enabled: true
  interval: 30s
@@ -0,0 +1,367 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: auth-proxy-dashboard
  labels:
    grafana_dashboard: "1"
 data:
  auth-proxy.json: |-
    {
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": {
              "type": "grafana",
              "uid": "-- Grafana --"
            },
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "editable": true,
      "fiscalYearStartMonth": 0,
      "graphTooltip": 1,
      "id": null,
      "links": [],
      "liveNow": false,
      "panels": [
        {
          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
          "id": 1,
          "options": {
            "colorMode": "value",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
            "textMode": "auto"
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "routes_loaded_total",
              "refId": "A"
            }
          ],
          "title": "Loaded Routes",
          "type": "stat"
        },
        {
          "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
          "id": 2,
          "options": {
            "colorMode": "value",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
            "textMode": "auto"
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "sum(increase(auth_requests_total[$__rate_interval]))",
              "refId": "A"
            }
          ],
          "title": "Auth Requests (total)",
          "type": "stat",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "thresholds" },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "color": "green", "value": null }
                ]
              }
            },
            "overrides": []
          }
        },
        {
          "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
          "id": 3,
          "options": {
            "colorMode": "value",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
            "textMode": "auto"
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "sum(increase(auth_requests_total{status=\"403\"}[$__rate_interval]))",
              "refId": "A"
            }
          ],
          "title": "Forbidden (403)",
          "type": "stat",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "thresholds" },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "color": "green", "value": null },
                  { "color": "red", "value": 1 }
                ]
              }
            },
            "overrides": []
          }
        },
        {
          "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
          "id": 4,
          "options": {
            "colorMode": "value",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false },
            "textMode": "auto"
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "sum(increase(callback_requests_total{result=\"success\"}[$__rate_interval]))",
              "refId": "A"
            }
          ],
          "title": "Successful Logins",
          "type": "stat",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "thresholds" },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "color": "green", "value": null }
                ]
              }
            },
            "overrides": []
          }
        },
        {
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
          "id": 5,
          "options": {
            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
            "tooltip": { "mode": "single", "sort": "none" }
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "sum by (status) (rate(auth_requests_total[$__rate_interval]))",
              "legendFormat": "{{status}}",
              "refId": "A"
            }
          ],
          "title": "Auth Request Rate by Status",
          "type": "timeseries",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "palette-classic" },
              "custom": {
                "axisBorderShow": false,
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
                "insertNulls": false,
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 5,
                "scaleDistribution": { "type": "linear" },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": { "group": "A", "mode": "none" },
                "thresholdsStyle": { "mode": "off" }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [ { "color": "green", "value": null } ]
              },
              "unit": "reqps"
            },
            "overrides": []
          }
        },
        {
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
          "id": 6,
          "options": {
            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
            "tooltip": { "mode": "single", "sort": "none" }
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "auth_request_duration_seconds{quantile=\"0.5\"}",
              "legendFormat": "p50",
              "refId": "A"
            },
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "auth_request_duration_seconds{quantile=\"0.95\"}",
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "auth_request_duration_seconds{quantile=\"0.99\"}",
              "legendFormat": "p99",
              "refId": "C"
            }
          ],
          "title": "Auth Request Duration (p50, p95, p99)",
          "type": "timeseries",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "palette-classic" },
              "custom": {
                "axisBorderShow": false,
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
                "insertNulls": false,
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 5,
                "scaleDistribution": { "type": "linear" },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": { "group": "A", "mode": "none" },
                "thresholdsStyle": { "mode": "off" }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [ { "color": "green", "value": null } ]
              },
              "unit": "s"
            },
            "overrides": []
          }
        },
        {
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
          "id": 7,
          "options": {
            "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
            "tooltip": { "mode": "single", "sort": "none" }
          },
          "pluginVersion": "10.0.0",
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "sum by (result) (rate(callback_requests_total[$__rate_interval]))",
              "legendFormat": "{{result}}",
              "refId": "A"
            }
          ],
          "title": "OIDC Callback Rate by Result",
          "type": "timeseries",
          "fieldConfig": {
            "defaults": {
              "color": { "mode": "palette-classic" },
              "custom": {
                "axisBorderShow": false,
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "bars",
                "fillOpacity": 50,
                "gradientMode": "none",
                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
                "insertNulls": false,
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": { "type": "linear" },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": { "group": "A", "mode": "normal" },
                "thresholdsStyle": { "mode": "off" }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [ { "color": "green", "value": null } ]
              },
              "unit": "reqps"
            },
            "overrides": []
          }
        }
      ],
      "refresh": "30s",
      "schemaVersion": 38,
      "style": "dark",
      "tags": [
        "auth-proxy",
        "rsauth2-proxy"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "selected": false,
              "text": "Prometheus",
              "value": "Prometheus"
            },
            "hide": 0,
            "includeAll": false,
            "multi": false,
            "name": "datasource",
            "options": [],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "type": "datasource"
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {},
      "timezone": "",
      "title": "Auth Proxy (rsauth2-proxy)",
      "uid": "auth-proxy-metrics",
      "version": 1
    }
@@ -0,0 +1,504 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: furumi-dashboard
  labels:
    grafana_dashboard: "1"
 data:
  furumi.json: |-
    {
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": { "type": "grafana", "uid": "-- Grafana --" },
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "editable": true,
      "fiscalYearStartMonth": 0,
      "graphTooltip": 1,
      "id": null,
      "links": [],
      "liveNow": false,
      "panels": [
        {
          "id": 1,
          "title": "Build Version",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${datasource}" },
              "expr": "max by (version) (furumusic_build_info{namespace=~\"$namespace\"})",
              "legendFormat": "{{version}}",
              "refId": "A"
            }
          ],
          "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "name" }
        },
        {
          "id": 2,
          "title": "HTTP RPS",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
          "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
          ],
          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
        },
        {
          "id": 3,
          "title": "5xx Error Ratio",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "refId": "A" }
          ],
          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
        },
        {
          "id": 4,
          "title": "HTTP p95 Latency",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "refId": "A" }
          ],
          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
        },
        {
          "id": 5,
          "title": "Active Users 15m",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_active_users{namespace=~\"$namespace\",window=\"15m\"})", "refId": "A" }
          ],
          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
        },
        {
          "id": 6,
          "title": "Stream Throughput",
          "type": "stat",
          "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
          "fieldConfig": { "defaults": { "unit": "Bps", "decimals": 1 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "refId": "A" }
          ],
          "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "textMode": "auto" }
        },
        { "id": 10, "title": "HTTP", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "collapsed": false, "panels": [] },
        {
          "id": 11,
          "title": "Request Rate by Route and Status",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route, status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "{{route}} {{status}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 12,
          "title": "Request Duration Quantiles",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
          "fieldConfig": { "defaults": { "unit": "s", "decimals": 3 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(furumusic_http_request_duration_seconds_bucket{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 13,
          "title": "In-flight Requests",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 },
          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (route) (furumusic_http_in_flight_requests{namespace=~\"$namespace\",route=~\"$route\"})", "legendFormat": "{{route}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 14,
          "title": "HTTP Body Throughput",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 },
          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_request_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "request", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_http_response_body_bytes_total{namespace=~\"$namespace\",route=~\"$route\"}[$__rate_interval]))", "legendFormat": "response", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 15,
          "title": "Status Code Mix",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 },
          "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (rate(furumusic_http_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{status}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        { "id": 20, "title": "Auth and Users", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "collapsed": false, "panels": [] },
        {
          "id": 21,
          "title": "Users by Role",
          "type": "bargauge",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 },
          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (role) (furumusic_users_total{namespace=~\"$namespace\"})", "legendFormat": "{{role}}", "refId": "A" }
          ],
          "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calc": "lastNotNull", "fields": "", "values": false }, "showUnfilled": true }
        },
        {
          "id": 22,
          "title": "Active Users",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 },
          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (furumusic_active_users{namespace=~\"$namespace\"})", "legendFormat": "{{window}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 23,
          "title": "Auth Events",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method, outcome, reason) (rate(furumusic_auth_login_attempts_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "login {{method}} {{outcome}} {{reason}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (method) (rate(furumusic_auth_sessions_created_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "session {{method}}", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(furumusic_auth_denied_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "denied {{kind}}", "refId": "C" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        { "id": 30, "title": "Playback and Streaming", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "collapsed": false, "panels": [] },
        {
          "id": 31,
          "title": "Listens Rate",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (completed) (rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "completed={{completed}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 32,
          "title": "Listen Completion Ratio",
          "description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 },
          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 33,
          "title": "Streaming",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 },
          "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_stream_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (range) (rate(furumusic_stream_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "requests range={{range}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 34,
          "title": "Listened Hours and Play History Activity",
          "description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 },
          "fieldConfig": {
            "defaults": { "unit": "h", "decimals": 2 },
            "overrides": [
              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] }
            ]
          },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        { "id": 40, "title": "Library and Storage", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] },
        {
          "id": 41,
          "title": "Library Inventory",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 },
          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_artists_total{namespace=~\"$namespace\"})", "legendFormat": "artists", "refId": "C" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_playlists_total{namespace=~\"$namespace\"})", "legendFormat": "playlists", "refId": "D" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 42,
          "title": "Media Bytes by Type",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 },
          "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 43,
          "title": "Storage Used Ratio",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 },
          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "1 - (sum by (path_kind) (furumusic_storage_free_bytes{namespace=~\"$namespace\"}) / sum by (path_kind) (furumusic_storage_total_bytes{namespace=~\"$namespace\"}))", "legendFormat": "{{path_kind}}", "refId": "A" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        { "id": 50, "title": "AI Agent", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "collapsed": false, "panels": [] },
        {
          "id": 51,
          "title": "Agent Queue and Reviews",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 },
          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 52,
          "title": "Agent Processing and Failures",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 52 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome, decision) (rate(furumusic_agent_files_processed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "processed {{outcome}} {{decision}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (stage) (rate(furumusic_agent_failed_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "failed {{stage}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 53,
          "title": "Agent Confidence",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 52 },
          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(furumusic_agent_confidence_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 54,
          "title": "Discover Runs and Duration",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 59 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_agent_discover_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "runs {{outcome}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(furumusic_agent_discover_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{outcome}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 55,
          "title": "Discover File Flow",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 59 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_seen_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "seen", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_agent_discover_files_queued_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "queued", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_discover_files_skipped_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}", "refId": "C" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 56,
          "title": "RAG and LLM Requests",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 66 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind, outcome) (rate(furumusic_agent_rag_queries_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, outcome) (rate(furumusic_agent_llm_requests_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 57,
          "title": "RAG and LLM Latency p95",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 66 },
          "fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, kind, outcome) (rate(furumusic_agent_rag_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "rag {{kind}} {{outcome}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, model, outcome) (rate(furumusic_agent_llm_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "llm {{model}} {{outcome}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 58,
          "title": "LLM Token Rate, Batch Splits and Parse Failures",
          "description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 },
          "fieldConfig": {
            "defaults": { "unit": "suffix: tok/s" },
            "overrides": [
              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] },
              { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }
            ]
          },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model) (rate(furumusic_agent_llm_parse_failures_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "parse failures {{model}}", "refId": "C" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 59,
          "title": "Cover Pipeline",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 73 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, outcome) (rate(furumusic_agent_cover_lookup_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "lookup {{source}} {{outcome}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (variant, outcome) (rate(furumusic_agent_cover_variant_generation_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "variant {{variant}} {{outcome}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        { "id": 70, "title": "Scheduler and Torrents", "type": "row", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "collapsed": false, "panels": [] },
        {
          "id": 71,
          "title": "Scheduler Jobs",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 0, "y": 81 },
          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_running{namespace=~\"$namespace\"})", "legendFormat": "running {{job}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job) (furumusic_scheduler_job_enabled{namespace=~\"$namespace\"})", "legendFormat": "enabled {{job}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 72,
          "title": "Scheduler Runs and Duration p95",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 8, "y": 81 },
          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (job, trigger, outcome) (rate(furumusic_scheduler_job_runs_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "{{job}} {{trigger}} {{outcome}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le, job, trigger, outcome) (rate(furumusic_scheduler_job_duration_seconds_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", "legendFormat": "p95 {{job}} {{outcome}}", "refId": "B" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        },
        {
          "id": 73,
          "title": "Torrents",
          "type": "timeseries",
          "gridPos": { "h": 7, "w": 8, "x": 16, "y": 81 },
          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
          "targets": [
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_torrent_sessions_total{namespace=~\"$namespace\"})", "legendFormat": "sessions {{status}}", "refId": "A" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_downloads_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "downloads {{outcome}}", "refId": "B" },
            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (outcome) (rate(furumusic_torrent_selected_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "bytes {{outcome}}", "refId": "C" }
          ],
          "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
        }
      ],
      "refresh": "30s",
      "schemaVersion": 38,
      "style": "dark",
      "tags": [ "furumi", "furumusic", "music" ],
      "templating": {
        "list": [
          {
            "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
            "hide": 0,
            "includeAll": false,
            "multi": false,
            "name": "datasource",
            "options": [],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "type": "datasource"
          },
          {
            "allValue": ".*",
            "current": { "selected": true, "text": "All", "value": "$__all" },
            "datasource": { "type": "prometheus", "uid": "${datasource}" },
            "definition": "label_values(furumusic_build_info, namespace)",
            "hide": 0,
            "includeAll": true,
            "multi": true,
            "name": "namespace",
            "options": [],
            "query": { "query": "label_values(furumusic_build_info, namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "sort": 1,
            "type": "query"
          },
          {
            "allValue": ".*",
            "current": { "selected": true, "text": "All", "value": "$__all" },
            "datasource": { "type": "prometheus", "uid": "${datasource}" },
            "definition": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)",
            "hide": 0,
            "includeAll": true,
            "multi": true,
            "name": "route",
            "options": [],
            "query": { "query": "label_values(furumusic_http_requests_total{namespace=~\"$namespace\"}, route)", "refId": "PrometheusVariableQueryEditor-VariableQuery" },
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "sort": 1,
            "type": "query"
          }
        ]
      },
      "time": { "from": "now-6h", "to": "now" },
      "timepicker": {},
      "timezone": "",
      "title": "Furumi Music",
      "uid": "furumi-music",
      "version": 1
    }
@@ -0,0 +1,409 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: telemt-dashboard
  labels:
    grafana_dashboard: "1"
 data:
  telemt.json: |-
    {
      "annotations": { "list": [] },
      "editable": true,
      "fiscalYearStartMonth": 0,
      "graphTooltip": 1,
      "id": null,
      "links": [],
      "liveNow": false,
      "panels": [
        {
          "title": "Nodes Overview",
          "type": "table",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
          "id": 1,
          "fieldConfig": {
            "defaults": {
              "custom": {
                "align": "auto",
                "cellOptions": { "type": "auto" },
                "inspect": false
              },
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "color": "green", "value": null }
                ]
              }
            },
            "overrides": [
              {
                "matcher": { "id": "byName", "options": "Uptime" },
                "properties": [
                  { "id": "unit", "value": "dtdurations" },
                  { "id": "custom.width", "value": 140 }
                ]
              },
              {
                "matcher": { "id": "byName", "options": "Bad Conn" },
                "properties": [
                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 100 }] } },
                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
                ]
              },
              {
                "matcher": { "id": "byName", "options": "Writers" },
                "properties": [
                  { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
                  { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }
                ]
              }
            ]
          },
          "options": {
            "showHeader": true,
            "sortBy": [{ "displayName": "Node", "desc": false }],
            "frameIndex": 0,
            "footer": { "show": false }
          },
          "transformations": [
            {
              "id": "joinByField",
              "options": { "byField": "node", "mode": "outer" }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": { "pattern": "^(node|Value.*)$" }
              }
            },
            {
              "id": "organize",
              "options": {
                "renameByName": {
                  "node": "Node",
                  "Value #uptime": "Uptime",
                  "Value #writers": "Writers",
                  "Value #buffers": "Buffers In Use",
                  "Value #connections": "Connections",
                  "Value #bad": "Bad Conn",
                  "Value #hs_timeout": "HS Timeouts"
                }
              }
            }
          ],
          "targets": [
            {
              "expr": "telemt_uptime_seconds{node=~\"$node\"}",
              "legendFormat": "",
              "refId": "uptime",
              "format": "table",
              "instant": true
            },
            {
              "expr": "telemt_me_writers_active_current{node=~\"$node\"}",
              "legendFormat": "",
              "refId": "writers",
              "format": "table",
              "instant": true
            },
            {
              "expr": "telemt_buffer_pool_buffers_total{node=~\"$node\", kind=\"in_use\"}",
              "legendFormat": "",
              "refId": "buffers",
              "format": "table",
              "instant": true
            },
            {
              "expr": "telemt_connections_total{node=~\"$node\"}",
              "legendFormat": "",
              "refId": "connections",
              "format": "table",
              "instant": true
            },
            {
              "expr": "telemt_connections_bad_total{node=~\"$node\"}",
              "legendFormat": "",
              "refId": "bad",
              "format": "table",
              "instant": true
            },
            {
              "expr": "telemt_handshake_timeouts_total{node=~\"$node\"}",
              "legendFormat": "",
              "refId": "hs_timeout",
              "format": "table",
              "instant": true
            }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Connections Rate",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
          "id": 10,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "cps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_connections_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} accepted", "refId": "A" },
            { "expr": "rate(telemt_connections_bad_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} bad", "refId": "B" },
            { "expr": "rate(telemt_handshake_timeouts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs timeout", "refId": "C" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Upstream Connect",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
          "id": 11,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "cps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_upstream_connect_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} success", "refId": "A" },
            { "expr": "rate(telemt_upstream_connect_fail_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} fail", "refId": "B" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Upstream Connect Duration (success)",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
          "id": 12,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } },
              "unit": "short",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "increase(telemt_upstream_connect_duration_success_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{bucket}}", "refId": "A" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "ME Writers & Pool",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
          "id": 13,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "telemt_me_writers_active_current{node=~\"$node\"}", "legendFormat": "{{node}} active", "refId": "A" },
            { "expr": "telemt_me_writers_warm_current{node=~\"$node\"}", "legendFormat": "{{node}} warm", "refId": "B" },
            { "expr": "telemt_pool_drain_active{node=~\"$node\"}", "legendFormat": "{{node}} draining", "refId": "C" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Per-User Active Connections",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
          "id": 20,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "telemt_user_connections_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}}", "refId": "A" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Per-User Traffic",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
          "id": 21,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "Bps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_user_octets_from_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} rx", "refId": "A" },
            { "expr": "rate(telemt_user_octets_to_client{node=~\"$node\"}[5m])", "legendFormat": "{{node}} {{user}} tx", "refId": "B" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "DC->Client Payload",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
          "id": 30,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "Bps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_me_d2c_payload_bytes_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} payload", "refId": "A" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "ME Errors & Anomalies",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
          "id": 31,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "cps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_me_reconnect_attempts_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} reconnect", "refId": "A" },
            { "expr": "rate(telemt_me_handshake_reject_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} hs reject", "refId": "B" },
            { "expr": "rate(telemt_me_crc_mismatch_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} crc mismatch", "refId": "C" },
            { "expr": "rate(telemt_desync_total{node=~\"$node\"}[5m])", "legendFormat": "{{node}} desync", "refId": "D" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Per-User Unique IPs",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
          "id": 40,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "telemt_user_unique_ips_current{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} active", "refId": "A" },
            { "expr": "telemt_user_unique_ips_recent_window{node=~\"$node\"}", "legendFormat": "{{node}} {{user}} recent", "refId": "B" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        },
        {
          "title": "Conntrack",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
          "id": 41,
          "fieldConfig": {
            "defaults": {
              "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "showPoints": "auto" },
              "unit": "cps",
              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
            },
            "overrides": []
          },
          "options": {
            "tooltip": { "mode": "multi", "sort": "desc" },
            "legend": { "displayMode": "list", "placement": "bottom" }
          },
          "targets": [
            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"attempt\"}[5m])", "legendFormat": "{{node}} delete attempt", "refId": "A" },
            { "expr": "rate(telemt_conntrack_delete_total{node=~\"$node\", result=\"error\"}[5m])", "legendFormat": "{{node}} delete error", "refId": "B" },
            { "expr": "telemt_conntrack_event_queue_depth{node=~\"$node\"}", "legendFormat": "{{node}} queue depth", "refId": "C" }
          ],
          "datasource": { "type": "prometheus", "uid": "${datasource}" }
        }
      ],
      "refresh": "30s",
      "schemaVersion": 39,
      "tags": ["telemt", "mtproxy", "telegram"],
      "templating": {
        "list": [
          {
            "current": {},
            "hide": 0,
            "includeAll": false,
            "label": "Datasource",
            "multi": false,
            "name": "datasource",
            "options": [],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "type": "datasource"
          },
          {
            "current": {},
            "datasource": { "type": "prometheus", "uid": "${datasource}" },
            "definition": "label_values(telemt_uptime_seconds, node)",
            "hide": 0,
            "includeAll": true,
            "label": "Node",
            "multi": true,
            "name": "node",
            "query": "label_values(telemt_uptime_seconds, node)",
            "refresh": 2,
            "regex": "",
            "skipUrlSync": false,
            "sort": 1,
            "type": "query"
          }
        ]
      },
      "time": { "from": "now-6h", "to": "now" },
      "title": "Telemt MTProxy",
      "uid": "telemt-mtproxy"
    }
@@ -0,0 +1,146 @@
 # Read-only access to the Grafana database credentials in the psql namespace.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: db-credentials-reader
  annotations:
    argocd.argoproj.io/sync-wave: "-3"
 ---
 apiVersion: external-secrets.io/v1
 kind: SecretStore
 metadata:
  name: psql-credentials
  annotations:
    argocd.argoproj.io/sync-wave: "-2"
 spec:
  provider:
    kubernetes:
      remoteNamespace: psql
      server:
        caProvider:
          type: ConfigMap
          name: kube-root-ca.crt
          key: ca.crt
      auth:
        serviceAccount:
          name: db-credentials-reader
 ---
 # A local break-glass account. OIDC remains the normal login path.
 apiVersion: generators.external-secrets.io/v1alpha1
 kind: Password
 metadata:
  name: grafana-admin
  annotations:
    argocd.argoproj.io/sync-wave: "-2"
 spec:
  length: 40
  digits: 10
  symbols: 0
  noUpper: false
  allowRepeat: true
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: grafana-admin
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
 spec:
  refreshPolicy: CreatedOnce
  target:
    name: grafana-admin
    template:
      type: Opaque
      data:
        admin-user: admin
        admin-password: "{{ .password }}"
  dataFrom:
    - sourceRef:
        generatorRef:
          apiVersion: generators.external-secrets.io/v1alpha1
          kind: Password
          name: grafana-admin
 ---
 # Grafana's database credentials come from CNPG. OIDC credentials are read by
 # stable Vaultwarden item and custom-field names, so imports and field ordering
 # do not invalidate the references.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: grafana-env
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
 spec:
  refreshInterval: 1h
  target:
    name: grafana-env
    deletionPolicy: Delete
    template:
      type: Opaque
      data:
        GF_DATABASE_USER: "{{ .db_username }}"
        GF_DATABASE_PASSWORD: "{{ .db_password }}"
        GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .client_id }}"
        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .client_secret }}"
  data:
    - secretKey: db_username
      remoteRef:
        key: grafana-db
        property: username
    - secretKey: db_password
      remoteRef:
        key: grafana-db
        property: password
    - secretKey: client_id
      sourceRef:
        storeRef:
          name: vaultwarden-fields
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/prometheus/grafana-secrets
        property: IDM Client ID
    - secretKey: client_secret
      sourceRef:
        storeRef:
          name: vaultwarden-fields
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/prometheus/grafana-secrets
        property: IDM Client Secret
  secretStoreRef:
    kind: SecretStore
    name: psql-credentials
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: alertmanager-telegram
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
 spec:
  refreshInterval: 1h
  target:
    name: alertmanager-telegram
    deletionPolicy: Delete
    template:
      type: Opaque
      data:
        bot-token: "{{ .bot_token }}"
        chat-id: "{{ .chat_id }}"
  data:
    - secretKey: bot_token
      sourceRef:
        storeRef:
          name: vaultwarden-fields
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/prometheus/grafana-secrets
        property: TG Token
    - secretKey: chat_id
      sourceRef:
        storeRef:
          name: vaultwarden-fields
          kind: ClusterSecretStore
      remoteRef:
        key: k8s/prometheus/grafana-secrets
        property: TG ID
Author	SHA1	Message	Date
jacob	33a91223a3	Provision Jacob database with pgvector through GitOps Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 8s Details	2026-10-27 01:35:38 +02:00
Aleksandr Bogomiakov	87e7bcebcf	Enable Hexor Lab login theme in Keycloak realm Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 19s Details Keycloak Terraform / Terraform (push) Successful in 45s Details	2026-10-21 21:36:21 +01:00
Aleksandr Bogomiakov	cd8cb1f586	Deploy Hexor Lab Phosphor login theme Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 12s Details	2026-10-21 21:31:25 +01:00
Ultradesu	729e808b13	Avoid Traefik NodePort collision with telemt TLS Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details	2026-10-21 12:33:00 +01:00
Ultradesu	af717cc530	Update k8s-secrets image digest for 0.2.2 Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 7s Details	2026-10-21 12:10:56 +01:00
Ultradesu	56c4c3871a	Sync k8s-secrets before monitoring wave completes Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details	2026-10-21 11:54:18 +01:00
Ultradesu	8dd1868c25	Move k8s-secrets into k8s-ng and update reader image Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 6s Details	2026-10-21 11:52:13 +01:00
Ultradesu	5eaee8c7b7	Match ServiceMonitor relabel defaults Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details	2026-10-21 11:38:14 +01:00
Ultradesu	7e7813bfe0	Mount telemt config as a regular file Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 6s Details	2026-10-21 11:36:08 +01:00
Ultradesu	a79c695a40	Exclude unreachable home node from exporter rollout Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details	2026-10-21 11:26:37 +01:00
Ultradesu	29d95cd853	Deploy MTProxy on external nodes via Argo CD Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 9s Details	2026-10-21 11:23:21 +01:00
Ultradesu	8a90756af0	Restore authenticated secret reader in current cluster Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 7s Details Auto-update README / Generate README and Create MR (push) Successful in 7s Details Check with kubeconform / lint (push) Successful in 10s Details	2026-10-18 11:23:14 +01:00
Ultradesu	52780edbb5	Expose authenticated Loki push endpoint for tsunagi logs Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details	2026-10-18 00:35:03 +01:00
Ultradesu	8a4849733c	Update Keycloak CI kubeconfig handling for current cluster Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details Keycloak Terraform / Terraform (push) Successful in 11s Details	2026-10-18 00:28:56 +01:00
Ultradesu	8e99f42815	Use active Kubernetes context in Keycloak Terraform CI Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 5s Details Keycloak Terraform / Terraform (push) Failing after 11s Details	2026-10-18 00:26:58 +01:00
Ultradesu	c4b608b621	Create tsunagi Vector machine client and Loki auth route Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 8s Details Keycloak Terraform / Terraform (push) Failing after 21s Details	2026-10-18 00:23:48 +01:00
Aleksandr Bogomiakov	dd0f664324	Add Shadowsocks deployment on spb with Vaultwarden credentials Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 6s Details	2026-10-15 10:58:52 +01:00
Aleksandr Bogomiakov	04610cc6e2	Changed llama model unsloth/Qwen3.8-27B-GGUF:UD-Q6_K -> unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q6_K Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 19s Details	2026-10-15 01:19:37 +01:00
Aleksandr Bogomiakov	aaa1e9ae7e	Run CoreDNS on the AI core node and prefer local DNS replicas Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 21s Details	2026-10-15 00:41:04 +01:00
Aleksandr Bogomiakov	47589988bf	Prefer local ingress endpoints behind k3s ServiceLB Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 31s Details	2026-10-15 00:37:13 +01:00
Ultradesu	2a08898a23	Configure three CoreDNS replicas on core nodes Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 17s Details	2026-10-14 13:57:36 +01:00
Aleksandr Bogomiakov	9b0530fb70	Bound Telegram alert messages and identify affected workloads Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 9s Details	2026-10-12 18:23:21 +01:00
Aleksandr Bogomiakov	ba1fcae807	Disabled some prom scrape configs Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 12s Details	2026-10-12 14:25:09 +01:00
Aleksandr Bogomiakov	da97f66448	Keep ingress traffic on the node selected by DNS Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 10s Details	2026-10-12 14:10:30 +01:00
Aleksandr Bogomiakov	2a89c89754	Run Traefik on every node, including the internal AI node Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 37s Details	2026-10-12 14:02:40 +01:00
Aleksandr Bogomiakov	8e4201621d	Place core services and web-petting on external core nodes Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 1m6s Details	2026-10-12 13:44:16 +01:00
Aleksandr Bogomiakov	ee7a82b0b8	Document PostgreSQL primary on music and replica on master Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 40s Details	2026-10-12 13:15:25 +01:00
Aleksandr Bogomiakov	6da0a39428	Preserve RustDesk client IP addresses during NAT detection Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 34s Details	2026-10-12 11:49:24 +01:00
Aleksandr Bogomiakov	4c744e539c	Normalize migrated manifest whitespace Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 2m32s Details	2026-10-12 11:46:25 +01:00
Aleksandr Bogomiakov	e94caf8054	Migrate Minecraft, RustDesk and Immich database to hexor-lab Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 3m3s Details	2026-10-12 11:45:52 +01:00
Aleksandr Bogomiakov	46bc178dab	Move Vaultwarden local storage and workload to music Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 1m53s Details	2026-10-11 13:00:46 +01:00
Aleksandr Bogomiakov	d1d28776cd	Fix migration job reloads and alert context Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Failing after 17m22s Details	2026-10-08 18:00:44 +01:00
Aleksandr Bogomiakov	1991ffc1e3	Allow Loki gateway rolling updates on music Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 15s Details	2026-10-08 17:15:34 +01:00
Aleksandr Bogomiakov	d8973e2daa	Tune Bitwarden CLI CPU burst limit Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 19s Details	2026-10-08 17:11:31 +01:00
Aleksandr Bogomiakov	f2d5b97465	Fixed alert reporting, instance label, gitea limits Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 29s Details	2026-10-08 17:03:46 +01:00
Aleksandr Bogomiakov	ac8e5e127c	NG: Fixed prom-stak Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 17s Details	2026-10-07 18:52:06 +01:00
Aleksandr Bogomiakov	52bfa88a33	Added prom-stack Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 31s Details	2026-10-07 18:35:17 +01:00
Aleksandr Bogomiakov	089bc9bfb5	Added meguri Keycloak Terraform / Terraform (push) Failing after 41s Details Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 2m59s Details	2026-10-07 18:09:48 +01:00
Aleksandr Bogomiakov	070176030c	Adde meguri demo Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 11s Details	2026-10-07 14:24:11 +01:00
Aleksandr BogomiakovandClaude	b6c671bc23	NG: bitwarden-cli bridge: liveness probe requires the unlocked status Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 31s Details `bw serve` can lose its login while /sync still answers; every lookup then fails with 500. The probe now also checks /status, so the pod restarts itself. Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 21:04:29 +01:00
Aleksandr Bogomiakov	04c35083ef	NG: moved gitea Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 27s Details	2026-10-06 20:26:25 +01:00
Aleksandr Bogomiakov	be9ace99bc	NG: moved gitea Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 29s Details	2026-10-06 20:20:49 +01:00
Aleksandr BogomiakovandClaude	1b41eee364	NG: gpu-system: drop the NFD nodeSelector the chart adds to the device plugin Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 23s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 19:27:35 +01:00
Aleksandr BogomiakovandClaude	373ce77ac8	NG: llamacpp on the ai node (Qwen3.8-27B) with the AMD GPU device plugin Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 44s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 19:20:06 +01:00
Aleksandr Bogomiakov	bd88daa109	NG: fixed furumi DB Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 30s Details	2026-10-06 19:07:58 +01:00
Aleksandr BogomiakovandClaude	1c7185aed6	NG: Vaultwarden moves to vw.hexor.cy (vw.btwiusearch.net dropped) Keycloak Terraform / Terraform (push) Failing after 32s Details Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 16s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 18:05:41 +01:00
Aleksandr BogomiakovandClaude	6d39028e2f	NG: HTTP-01 fallback solver for domains outside the DNS-01 zones Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 17s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 17:57:36 +01:00
Aleksandr BogomiakovandClaude	4159a6c045	NG: web-petting app Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 31s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 17:42:49 +01:00
Aleksandr BogomiakovandClaude	5f0a3ca3b6	NG: web_petting database in main PostgreSQL cluster Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 17:42:49 +01:00
Aleksandr BogomiakovandClaude	07cdc88568	NG: document PostgreSQL layout (primary on music, replica on ams-1) Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 17:31:18 +01:00
Aleksandr BogomiakovandClaude	b11b7c902d	NG: move furumi to apps project Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 17s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 16:30:44 +01:00
Aleksandr BogomiakovandClaude	ff4e87aca0	NG: furumi player pinned to the music node Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 1m19s Details Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 16:18:36 +01:00
Aleksandr BogomiakovandClaude	01561ea2cd	NG: furumi database in main PostgreSQL cluster Co-Authored-By: Claude <noreply@anthropic.com>	2026-10-06 16:18:36 +01:00
		`@@ -0,0 +1 @@`
							`../themes/hexor-lab/login/resources/js/background.js`
		`@@ -0,0 +1 @@`
							`../themes/hexor-lab/login/resources/css/hexor.css`
		`@@ -0,0 +1 @@`
							`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><rect width="64" height="64" fill="#0c0f0d"/><text x="7" y="43" fill="#b9ef70" font-family="monospace" font-size="34">>_</text></svg>`