Deploy MTProxy on external nodes via Argo CD

k8s-ng/apps/mtproxy/README.md

@@ -0,0 +1,23 @@
+# MTProxy
+
+Argo CD deploys this directory through the `mtproxy` entry in
+`core-state/values.yaml`. Two telemt DaemonSets run on every node labeled
+`homelab.hexor.cy/exposure=external`: classic MTProxy on 30443/TCP and
+fake-TLS on 30444/TCP. The classic listener replaces the old amd64-only
+`telegrammessenger/proxy` image so the arm64 `spb` node is covered.
+
+Vaultwarden item `k8s/mtproxy/secret` must contain a custom field named
+`Secret` with the existing 32-character hexadecimal MTProxy secret. External
+Secrets renders both configurations from that field. Reloader restarts the
+DaemonSets when it changes. Changing this value invalidates existing links.
+
+`mtproxy-link-reconciler` lists external nodes every five minutes and creates
+`mtproxy-links` and `telemt-links` Secrets. A node's public DNS name must be
+`<node>.hexor.cy` and must point to that node. The reader at `proxy.hexor.cy`
+has read-only access to those two Secrets and is protected by Keycloak's
+forward auth; its access policy is in `terraform/keycloak/terraform.tfvars`.
+The proxy Pods have no Kubernetes API access.
+
+Before traffic is switched over, allow inbound 30443/TCP and 30444/TCP on
+every external node and verify both DNS and connectivity on each node.
+Neither the cluster nor host firewall is changed by these manifests.

k8s-ng/apps/mtproxy/daemonsets.yaml

@@ -0,0 +1,147 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: mtproxy-classic
+  labels:
+    app: mtproxy-classic
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-classic
+  template:
+    metadata:
+      labels:
+        app: mtproxy-classic
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: telemt
+          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
+          imagePullPolicy: IfNotPresent
+          workingDir: /run/telemt
+          args: ["/etc/telemt/config.toml"]
+          ports:
+            - name: proxy
+              containerPort: 30443
+              hostPort: 30443
+              protocol: TCP
+            - name: metrics
+              containerPort: 9090
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: proxy
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /etc/telemt
+              readOnly: true
+            - name: workdir
+              mountPath: /run/telemt
+      volumes:
+        - name: config
+          secret:
+            secretName: mtproxy-config
+            defaultMode: 0444
+            items:
+              - key: classic.toml
+                path: config.toml
+        - name: workdir
+          emptyDir:
+            medium: Memory
+            sizeLimit: 4Mi
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: mtproxy-tls
+  labels:
+    app: mtproxy-tls
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-tls
+  template:
+    metadata:
+      labels:
+        app: mtproxy-tls
+      annotations:
+        reloader.stakater.com/auto: "true"
+    spec:
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: telemt
+          image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a
+          imagePullPolicy: IfNotPresent
+          workingDir: /run/telemt
+          args: ["/etc/telemt/config.toml"]
+          ports:
+            - name: proxy
+              containerPort: 30444
+              hostPort: 30444
+              protocol: TCP
+            - name: metrics
+              containerPort: 9090
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: proxy
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: config
+              mountPath: /etc/telemt
+              readOnly: true
+            - name: workdir
+              mountPath: /run/telemt
+      volumes:
+        - name: config
+          secret:
+            secretName: mtproxy-config
+            defaultMode: 0444
+            items:
+              - key: tls.toml
+                path: config.toml
+        - name: workdir
+          emptyDir:
+            medium: Memory
+            sizeLimit: 4Mi

k8s-ng/apps/mtproxy/external-secrets.yaml

@@ -0,0 +1,74 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: mtproxy-config
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vaultwarden-fields
+    kind: ClusterSecretStore
+  target:
+    name: mtproxy-config
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      type: Opaque
+      data:
+        SECRET: "{{ .secret }}"
+        classic.toml: |
+          [general]
+          use_middle_proxy = true
+          log_level = "normal"
+
+          [general.modes]
+          classic = true
+          secure = false
+          tls = false
+
+          [server]
+          port = 30443
+          metrics_port = 9090
+          metrics_whitelist = ["0.0.0.0/0"]
+
+          [server.api]
+          enabled = false
+
+          [[server.listeners]]
+          ip = "0.0.0.0"
+
+          [access.users]
+          user = "{{ .secret }}"
+        tls.toml: |
+          [general]
+          use_middle_proxy = true
+          log_level = "normal"
+
+          [general.modes]
+          classic = false
+          secure = false
+          tls = true
+
+          [server]
+          port = 30444
+          metrics_port = 9090
+          metrics_whitelist = ["0.0.0.0/0"]
+
+          [server.api]
+          enabled = false
+
+          [[server.listeners]]
+          ip = "0.0.0.0"
+
+          [censorship]
+          tls_domain = "ya.ru"
+          mask = true
+          tls_emulation = true
+          tls_front_dir = "tlsfront"
+
+          [access.users]
+          user = "{{ .secret }}"
+  data:
+    - secretKey: secret
+      remoteRef:
+        key: k8s/mtproxy/secret
+        property: Secret

k8s-ng/apps/mtproxy/ingress.yaml

@@ -0,0 +1,38 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: mtproxy-auth
+spec:
+  forwardAuth:
+    address: http://auth-proxy.keycloak.svc:80/auth
+    trustForwardHeader: false
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: mtproxy-links
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`proxy.hexor.cy`)
+      kind: Rule
+      middlewares:
+        - name: mtproxy-auth
+      services:
+        - name: mtproxy-link-reader
+          port: 80
+  tls:
+    secretName: mtproxy-links-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: mtproxy-links-tls
+spec:
+  secretName: mtproxy-links-tls
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+    - proxy.hexor.cy

k8s-ng/apps/mtproxy/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: mtproxy
+resources:
+  - external-secrets.yaml
+  - daemonsets.yaml
+  - metrics.yaml
+  - links.yaml
+  - reader.yaml
+  - ingress.yaml
+configMapGenerator:
+  - name: mtproxy-link-reconciler
+    files:
+      - reconcile.py=link-reconciler.py
+generatorOptions:
+  disableNameSuffixHash: true

k8s-ng/apps/mtproxy/link-reconciler.py

@@ -0,0 +1,75 @@
+"""Publish client links for every external node, using its public DNS name."""
+
+import base64
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+API = "https://kubernetes.default.svc"
+NAMESPACE = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", encoding="utf-8").read().strip()
+TOKEN = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
+CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+
+def request(method, path, body=None):
+    import ssl
+
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(
+        API + path,
+        data=data,
+        method=method,
+        headers={
+            "Authorization": "Bearer " + TOKEN,
+            "Content-Type": "application/json",
+        },
+    )
+    with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=15) as response:
+        return json.load(response)
+
+
+def publish(name, links):
+    path = f"/api/v1/namespaces/{NAMESPACE}/secrets"
+    data = {node: base64.b64encode(link.encode()).decode() for node, link in links.items()}
+    try:
+        current = request("GET", f"{path}/{name}")
+    except urllib.error.HTTPError as error:
+        if error.code != 404:
+            raise
+        request("POST", path, {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, "type": "Opaque", "data": data})
+        return
+    if current.get("data", {}) == data:
+        return
+    # PUT replaces the data map, removing links for nodes that are no longer external.
+    request("PUT", f"{path}/{name}", {
+        "apiVersion": "v1",
+        "kind": "Secret",
+        "metadata": {"name": name, "resourceVersion": current["metadata"]["resourceVersion"]},
+        "type": "Opaque",
+        "data": data,
+    })
+
+
+secret = open("/config/SECRET", encoding="utf-8").read().strip()
+if len(secret) != 32 or any(char not in "0123456789abcdefABCDEF" for char in secret):
+    raise ValueError("Vaultwarden Secret must be 32 hexadecimal characters")
+
+selector = urllib.parse.quote("homelab.hexor.cy/exposure=external", safe="")
+nodes = request("GET", f"/api/v1/nodes?labelSelector={selector}")["items"]
+if not nodes:
+    raise ValueError("No external nodes found")
+
+classic = {}
+tls = {}
+for node in nodes:
+    name = node["metadata"]["name"]
+    # DNS is maintained outside the cluster; no public IP needs to be stored on a Node.
+    host = f"{name}.hexor.cy"
+    classic[name] = f"tg://proxy?server={host}&port=30443&secret={secret}"
+    tls[name] = f"tg://proxy?server={host}&port=30444&secret=ee{secret}79612e7275"
+
+publish("mtproxy-links", classic)
+publish("telemt-links", tls)
+print(f"Published MTProxy links for {len(nodes)} external nodes")

k8s-ng/apps/mtproxy/links.yaml

@@ -0,0 +1,108 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mtproxy-link-reconciler
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mtproxy-external-node-reader
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: mtproxy-external-node-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mtproxy-external-node-reader
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reconciler
+    namespace: mtproxy
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mtproxy-link-writer
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mtproxy-links", "telemt-links"]
+    verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mtproxy-link-writer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mtproxy-link-writer
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reconciler
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: mtproxy-link-reconciler
+spec:
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          serviceAccountName: mtproxy-link-reconciler
+          restartPolicy: OnFailure
+          nodeSelector:
+            homelab.hexor.cy/exposure: external
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: reconcile
+              image: python:3.13-alpine3.22@sha256:e81548ac35b07a3bd4805f275107592ef458b1e893c0e04d45aedaa19416cca5
+              command: ["python", "/scripts/reconcile.py"]
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 32Mi
+                limits:
+                  memory: 128Mi
+              securityContext:
+                readOnlyRootFilesystem: true
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: scripts
+                  mountPath: /scripts
+                  readOnly: true
+                - name: config
+                  mountPath: /config
+                  readOnly: true
+          volumes:
+            - name: scripts
+              configMap:
+                name: mtproxy-link-reconciler
+            - name: config
+              secret:
+                secretName: mtproxy-config
+                items:
+                  - key: SECRET
+                    path: SECRET

k8s-ng/apps/mtproxy/metrics.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-classic-metrics
+  labels:
+    app: mtproxy-classic
+spec:
+  clusterIP: None
+  selector:
+    app: mtproxy-classic
+  ports:
+    - name: metrics
+      port: 9090
+      targetPort: metrics
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-tls-metrics
+  labels:
+    app: mtproxy-tls
+spec:
+  clusterIP: None
+  selector:
+    app: mtproxy-tls
+  ports:
+    - name: metrics
+      port: 9090
+      targetPort: metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: mtproxy-classic
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-classic
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: mtproxy-tls
+spec:
+  selector:
+    matchLabels:
+      app: mtproxy-tls
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_pod_node_name]
+          targetLabel: node

k8s-ng/apps/mtproxy/reader.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mtproxy-link-reader
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mtproxy-link-reader
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["mtproxy-links", "telemt-links"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mtproxy-link-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mtproxy-link-reader
+subjects:
+  - kind: ServiceAccount
+    name: mtproxy-link-reader
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mtproxy-link-reader
+  labels:
+    app: mtproxy-link-reader
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mtproxy-link-reader
+  template:
+    metadata:
+      labels:
+        app: mtproxy-link-reader
+    spec:
+      serviceAccountName: mtproxy-link-reader
+      nodeSelector:
+        homelab.hexor.cy/exposure: external
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: reader
+          image: ultradesu/k8s-secrets:0.2.1@sha256:1cb1d2458879e91f98b6ca613c556f63dbb1e6b92a46f49d562ac23bc2b5fd2b
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secrets
+            - mtproxy-links,telemt-links
+            - --namespace
+            - mtproxy
+            - --port
+            - "3000"
+          ports:
+            - name: http
+              containerPort: 3000
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 128Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mtproxy-link-reader
+spec:
+  selector:
+    app: mtproxy-link-reader
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

k8s-ng/core-state/values.yaml

@@ -126,3 +126,9 @@ apps:
     namespace: shadowsocks
     wave: 6
     skipDryRunOnMissingResource: true
+  - name: mtproxy
+    project: apps
+    path: k8s-ng/apps/mtproxy
+    namespace: mtproxy
+    wave: 6 # after secret-stores and prom-stack
+    skipDryRunOnMissingResource: true