diff --git a/k8s-ng/apps/mtproxy/README.md b/k8s-ng/apps/mtproxy/README.md new file mode 100644 index 0000000..0f49845 --- /dev/null +++ b/k8s-ng/apps/mtproxy/README.md @@ -0,0 +1,23 @@ +# MTProxy + +Argo CD deploys this directory through the `mtproxy` entry in +`core-state/values.yaml`. Two telemt DaemonSets run on every node labeled +`homelab.hexor.cy/exposure=external`: classic MTProxy on 30443/TCP and +fake-TLS on 30444/TCP. The classic listener replaces the old amd64-only +`telegrammessenger/proxy` image so the arm64 `spb` node is covered. + +Vaultwarden item `k8s/mtproxy/secret` must contain a custom field named +`Secret` with the existing 32-character hexadecimal MTProxy secret. External +Secrets renders both configurations from that field. Reloader restarts the +DaemonSets when it changes. Changing this value invalidates existing links. + +`mtproxy-link-reconciler` lists external nodes every five minutes and creates +`mtproxy-links` and `telemt-links` Secrets. A node's public DNS name must be +`.hexor.cy` and must point to that node. The reader at `proxy.hexor.cy` +has read-only access to those two Secrets and is protected by Keycloak's +forward auth; its access policy is in `terraform/keycloak/terraform.tfvars`. +The proxy Pods have no Kubernetes API access. + +Before traffic is switched over, allow inbound 30443/TCP and 30444/TCP on +every external node and verify both DNS and connectivity on each node. +Neither the cluster nor host firewall is changed by these manifests. diff --git a/k8s-ng/apps/mtproxy/daemonsets.yaml b/k8s-ng/apps/mtproxy/daemonsets.yaml new file mode 100644 index 0000000..4237cda --- /dev/null +++ b/k8s-ng/apps/mtproxy/daemonsets.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: mtproxy-classic + labels: + app: mtproxy-classic +spec: + selector: + matchLabels: + app: mtproxy-classic + template: + metadata: + labels: + app: mtproxy-classic + annotations: + reloader.stakater.com/auto: "true" + spec: + nodeSelector: + homelab.hexor.cy/exposure: external + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: telemt + image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a + imagePullPolicy: IfNotPresent + workingDir: /run/telemt + args: ["/etc/telemt/config.toml"] + ports: + - name: proxy + containerPort: 30443 + hostPort: 30443 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + readinessProbe: + tcpSocket: + port: proxy + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/telemt + readOnly: true + - name: workdir + mountPath: /run/telemt + volumes: + - name: config + secret: + secretName: mtproxy-config + defaultMode: 0444 + items: + - key: classic.toml + path: config.toml + - name: workdir + emptyDir: + medium: Memory + sizeLimit: 4Mi +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: mtproxy-tls + labels: + app: mtproxy-tls +spec: + selector: + matchLabels: + app: mtproxy-tls + template: + metadata: + labels: + app: mtproxy-tls + annotations: + reloader.stakater.com/auto: "true" + spec: + nodeSelector: + homelab.hexor.cy/exposure: external + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: telemt + image: ghcr.io/telemt/telemt@sha256:b311c20596213a3941dcd11f499be982de74f554eecfffe57fab51c1bf80d50a + imagePullPolicy: IfNotPresent + workingDir: /run/telemt + args: ["/etc/telemt/config.toml"] + ports: + - name: proxy + containerPort: 30444 + hostPort: 30444 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + readinessProbe: + tcpSocket: + port: proxy + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/telemt + readOnly: true + - name: workdir + mountPath: /run/telemt + volumes: + - name: config + secret: + secretName: mtproxy-config + defaultMode: 0444 + items: + - key: tls.toml + path: config.toml + - name: workdir + emptyDir: + medium: Memory + sizeLimit: 4Mi diff --git a/k8s-ng/apps/mtproxy/external-secrets.yaml b/k8s-ng/apps/mtproxy/external-secrets.yaml new file mode 100644 index 0000000..23e3574 --- /dev/null +++ b/k8s-ng/apps/mtproxy/external-secrets.yaml @@ -0,0 +1,74 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: mtproxy-config +spec: + refreshInterval: 1h + secretStoreRef: + name: vaultwarden-fields + kind: ClusterSecretStore + target: + name: mtproxy-config + creationPolicy: Owner + deletionPolicy: Retain + template: + type: Opaque + data: + SECRET: "{{ .secret }}" + classic.toml: | + [general] + use_middle_proxy = true + log_level = "normal" + + [general.modes] + classic = true + secure = false + tls = false + + [server] + port = 30443 + metrics_port = 9090 + metrics_whitelist = ["0.0.0.0/0"] + + [server.api] + enabled = false + + [[server.listeners]] + ip = "0.0.0.0" + + [access.users] + user = "{{ .secret }}" + tls.toml: | + [general] + use_middle_proxy = true + log_level = "normal" + + [general.modes] + classic = false + secure = false + tls = true + + [server] + port = 30444 + metrics_port = 9090 + metrics_whitelist = ["0.0.0.0/0"] + + [server.api] + enabled = false + + [[server.listeners]] + ip = "0.0.0.0" + + [censorship] + tls_domain = "ya.ru" + mask = true + tls_emulation = true + tls_front_dir = "tlsfront" + + [access.users] + user = "{{ .secret }}" + data: + - secretKey: secret + remoteRef: + key: k8s/mtproxy/secret + property: Secret diff --git a/k8s-ng/apps/mtproxy/ingress.yaml b/k8s-ng/apps/mtproxy/ingress.yaml new file mode 100644 index 0000000..5611f4d --- /dev/null +++ b/k8s-ng/apps/mtproxy/ingress.yaml @@ -0,0 +1,38 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: mtproxy-auth +spec: + forwardAuth: + address: http://auth-proxy.keycloak.svc:80/auth + trustForwardHeader: false +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: mtproxy-links +spec: + entryPoints: + - websecure + routes: + - match: Host(`proxy.hexor.cy`) + kind: Rule + middlewares: + - name: mtproxy-auth + services: + - name: mtproxy-link-reader + port: 80 + tls: + secretName: mtproxy-links-tls +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: mtproxy-links-tls +spec: + secretName: mtproxy-links-tls + issuerRef: + name: letsencrypt + kind: ClusterIssuer + dnsNames: + - proxy.hexor.cy diff --git a/k8s-ng/apps/mtproxy/kustomization.yaml b/k8s-ng/apps/mtproxy/kustomization.yaml new file mode 100644 index 0000000..89cb4ea --- /dev/null +++ b/k8s-ng/apps/mtproxy/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: mtproxy +resources: + - external-secrets.yaml + - daemonsets.yaml + - metrics.yaml + - links.yaml + - reader.yaml + - ingress.yaml +configMapGenerator: + - name: mtproxy-link-reconciler + files: + - reconcile.py=link-reconciler.py +generatorOptions: + disableNameSuffixHash: true diff --git a/k8s-ng/apps/mtproxy/link-reconciler.py b/k8s-ng/apps/mtproxy/link-reconciler.py new file mode 100644 index 0000000..0eddb43 --- /dev/null +++ b/k8s-ng/apps/mtproxy/link-reconciler.py @@ -0,0 +1,75 @@ +"""Publish client links for every external node, using its public DNS name.""" + +import base64 +import json +import urllib.error +import urllib.parse +import urllib.request + + +API = "https://kubernetes.default.svc" +NAMESPACE = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", encoding="utf-8").read().strip() +TOKEN = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip() +CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + +def request(method, path, body=None): + import ssl + + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + API + path, + data=data, + method=method, + headers={ + "Authorization": "Bearer " + TOKEN, + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=15) as response: + return json.load(response) + + +def publish(name, links): + path = f"/api/v1/namespaces/{NAMESPACE}/secrets" + data = {node: base64.b64encode(link.encode()).decode() for node, link in links.items()} + try: + current = request("GET", f"{path}/{name}") + except urllib.error.HTTPError as error: + if error.code != 404: + raise + request("POST", path, {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, "type": "Opaque", "data": data}) + return + if current.get("data", {}) == data: + return + # PUT replaces the data map, removing links for nodes that are no longer external. + request("PUT", f"{path}/{name}", { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": name, "resourceVersion": current["metadata"]["resourceVersion"]}, + "type": "Opaque", + "data": data, + }) + + +secret = open("/config/SECRET", encoding="utf-8").read().strip() +if len(secret) != 32 or any(char not in "0123456789abcdefABCDEF" for char in secret): + raise ValueError("Vaultwarden Secret must be 32 hexadecimal characters") + +selector = urllib.parse.quote("homelab.hexor.cy/exposure=external", safe="") +nodes = request("GET", f"/api/v1/nodes?labelSelector={selector}")["items"] +if not nodes: + raise ValueError("No external nodes found") + +classic = {} +tls = {} +for node in nodes: + name = node["metadata"]["name"] + # DNS is maintained outside the cluster; no public IP needs to be stored on a Node. + host = f"{name}.hexor.cy" + classic[name] = f"tg://proxy?server={host}&port=30443&secret={secret}" + tls[name] = f"tg://proxy?server={host}&port=30444&secret=ee{secret}79612e7275" + +publish("mtproxy-links", classic) +publish("telemt-links", tls) +print(f"Published MTProxy links for {len(nodes)} external nodes") diff --git a/k8s-ng/apps/mtproxy/links.yaml b/k8s-ng/apps/mtproxy/links.yaml new file mode 100644 index 0000000..4bc4822 --- /dev/null +++ b/k8s-ng/apps/mtproxy/links.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mtproxy-link-reconciler +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mtproxy-external-node-reader +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mtproxy-external-node-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mtproxy-external-node-reader +subjects: + - kind: ServiceAccount + name: mtproxy-link-reconciler + namespace: mtproxy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: mtproxy-link-writer +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mtproxy-links", "telemt-links"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: mtproxy-link-writer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mtproxy-link-writer +subjects: + - kind: ServiceAccount + name: mtproxy-link-reconciler +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mtproxy-link-reconciler +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: mtproxy-link-reconciler + restartPolicy: OnFailure + nodeSelector: + homelab.hexor.cy/exposure: external + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: reconcile + image: python:3.13-alpine3.22@sha256:e81548ac35b07a3bd4805f275107592ef458b1e893c0e04d45aedaa19416cca5 + command: ["python", "/scripts/reconcile.py"] + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: config + mountPath: /config + readOnly: true + volumes: + - name: scripts + configMap: + name: mtproxy-link-reconciler + - name: config + secret: + secretName: mtproxy-config + items: + - key: SECRET + path: SECRET diff --git a/k8s-ng/apps/mtproxy/metrics.yaml b/k8s-ng/apps/mtproxy/metrics.yaml new file mode 100644 index 0000000..b4c69bc --- /dev/null +++ b/k8s-ng/apps/mtproxy/metrics.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + name: mtproxy-classic-metrics + labels: + app: mtproxy-classic +spec: + clusterIP: None + selector: + app: mtproxy-classic + ports: + - name: metrics + port: 9090 + targetPort: metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: mtproxy-tls-metrics + labels: + app: mtproxy-tls +spec: + clusterIP: None + selector: + app: mtproxy-tls + ports: + - name: metrics + port: 9090 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: mtproxy-classic +spec: + selector: + matchLabels: + app: mtproxy-classic + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: mtproxy-tls +spec: + selector: + matchLabels: + app: mtproxy-tls + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node diff --git a/k8s-ng/apps/mtproxy/reader.yaml b/k8s-ng/apps/mtproxy/reader.yaml new file mode 100644 index 0000000..7a2b399 --- /dev/null +++ b/k8s-ng/apps/mtproxy/reader.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mtproxy-link-reader +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: mtproxy-link-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mtproxy-links", "telemt-links"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: mtproxy-link-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mtproxy-link-reader +subjects: + - kind: ServiceAccount + name: mtproxy-link-reader +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mtproxy-link-reader + labels: + app: mtproxy-link-reader +spec: + replicas: 1 + selector: + matchLabels: + app: mtproxy-link-reader + template: + metadata: + labels: + app: mtproxy-link-reader + spec: + serviceAccountName: mtproxy-link-reader + nodeSelector: + homelab.hexor.cy/exposure: external + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: reader + image: ultradesu/k8s-secrets:0.2.1@sha256:1cb1d2458879e91f98b6ca613c556f63dbb1e6b92a46f49d562ac23bc2b5fd2b + imagePullPolicy: IfNotPresent + args: + - --secrets + - mtproxy-links,telemt-links + - --namespace + - mtproxy + - --port + - "3000" + ports: + - name: http + containerPort: 3000 + readinessProbe: + httpGet: + path: /health + port: http + livenessProbe: + httpGet: + path: /health + port: http + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: mtproxy-link-reader +spec: + selector: + app: mtproxy-link-reader + ports: + - name: http + port: 80 + targetPort: http diff --git a/k8s-ng/core-state/values.yaml b/k8s-ng/core-state/values.yaml index b42e696..70b11fa 100644 --- a/k8s-ng/core-state/values.yaml +++ b/k8s-ng/core-state/values.yaml @@ -126,3 +126,9 @@ apps: namespace: shadowsocks wave: 6 skipDryRunOnMissingResource: true + - name: mtproxy + project: apps + path: k8s-ng/apps/mtproxy + namespace: mtproxy + wave: 6 # after secret-stores and prom-stack + skipDryRunOnMissingResource: true