Reworked pasarguard nodes daemonset.

2025-12-08 17:31:51 +02:00
parent fc1bc1a65e
commit b2a77a6572
3 changed files with 502 additions and 0 deletions
--- a/k8s/apps/pasarguard/configmap-scripts-ingress.yaml
+++ b/k8s/apps/pasarguard/configmap-scripts-ingress.yaml
@@ -0,0 +1,275 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pasarguard-scripts-ingress
+  labels:
+    app: pasarguard-node-ingress
+data:
+  init-uuid-ingress.sh: |
+    #!/bin/bash
+    set -e
+    echo "Started"
+    # NODE_NAME is already set via environment variable
+    NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
+
+    # Get DNS name from node label xray-node-address
+    DNS_NAME=$(kubectl get node "${NODE_NAME}" -o jsonpath='{.metadata.labels.xray-node-address}')
+
+    if [ -z "${DNS_NAME}" ]; then
+      echo "ERROR: Node ${NODE_NAME} does not have label 'xray-node-address'"
+      exit 1
+    fi
+
+    echo "Node: ${NODE_NAME}"
+    echo "DNS Name from label: ${DNS_NAME}"
+
+    # Use DNS name for ConfigMap name to ensure uniqueness
+    CONFIGMAP_NAME="node-uuid-ingress-${DNS_NAME//./-}"
+
+    echo "Checking ConfigMap: ${CONFIGMAP_NAME}"
+
+    # Check if ConfigMap exists and get UUID
+    if kubectl get configmap "${CONFIGMAP_NAME}" -n "${NAMESPACE}" &>/dev/null; then
+      echo "ConfigMap exists, reading UUID..."
+      API_KEY=$(kubectl get configmap "${CONFIGMAP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.data.API_KEY}')
+
+      if [ -z "${API_KEY}" ]; then
+        echo "UUID not found in ConfigMap, generating new one..."
+        API_KEY=$(cat /proc/sys/kernel/random/uuid)
+        kubectl patch configmap "${CONFIGMAP_NAME}" -n "${NAMESPACE}" --type merge -p "{\"data\":{\"API_KEY\":\"${API_KEY}\"}}"
+      else
+        echo "Using existing UUID from ConfigMap"
+      fi
+    else
+      echo "ConfigMap does not exist, creating new one..."
+      API_KEY=$(cat /proc/sys/kernel/random/uuid)
+      kubectl create configmap "${CONFIGMAP_NAME}" -n "${NAMESPACE}" \
+        --from-literal=API_KEY="${API_KEY}" \
+        --from-literal=NODE_NAME="${NODE_NAME}"
+    fi
+
+    # Save UUID and node info to shared volume for the main container
+    echo -n "${API_KEY}" > /shared/api-key
+    echo -n "${NODE_NAME}" > /shared/node-name
+    echo -n "${CONFIGMAP_NAME}" > /shared/configmap-name
+    echo "UUID initialized: ${API_KEY}"
+    echo "Node name: ${NODE_NAME}"
+    echo "ConfigMap: ${CONFIGMAP_NAME}"
+
+    # Create Certificate for this node using DNS name from label
+    CERT_NAME="pasarguard-node-ingress-${DNS_NAME//./-}"
+
+    echo "Creating Certificate: ${CERT_NAME} for ${DNS_NAME}"
+
+    # Check if Certificate already exists
+    if ! kubectl get certificate "${CERT_NAME}" -n "${NAMESPACE}" &>/dev/null; then
+      echo "Certificate does not exist, creating..."
+      cat <<EOF | kubectl apply -f -
+    apiVersion: cert-manager.io/v1
+    kind: Certificate
+    metadata:
+      name: ${CERT_NAME}
+      namespace: ${NAMESPACE}
+    spec:
+      secretName: ${CERT_NAME}-tls
+      issuerRef:
+        name: letsencrypt
+        kind: ClusterIssuer
+      dnsNames:
+        - ${DNS_NAME}
+    EOF
+    else
+      echo "Certificate already exists"
+    fi
+
+    # Wait for certificate to be ready
+    
+    echo "Waiting for certificate to be ready..."
+    for i in {1..600}; do
+      if kubectl get secret "${CERT_NAME}-tls" -n "${NAMESPACE}" &>/dev/null; then
+        echo "Certificate secret is ready!"
+        break
+      fi
+      echo "Waiting for certificate... ($i/600)"
+      sleep 1
+    done
+
+    if ! kubectl get secret "${CERT_NAME}-tls" -n "${NAMESPACE}" &>/dev/null; then
+      echo "WARNING: Certificate secret not ready after 600 seconds"
+    else
+      # Extract certificate and key from secret to shared volume
+      echo "Extracting certificate and key..."
+      kubectl get secret "${CERT_NAME}-tls" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}' | base64 -d > /shared/tls.crt
+      kubectl get secret "${CERT_NAME}-tls" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.key}' | base64 -d > /shared/tls.key
+      echo "Certificate and key extracted successfully."
+      cat /shared/tls.crt
+    fi
+
+    # Create ClusterIP Service for this node (pod selector based)
+    NODE_SHORT_NAME="${NODE_NAME%%.*}"
+    SERVICE_NAME="${NODE_SHORT_NAME}-ingress"
+
+    echo "Creating Service: ${SERVICE_NAME} for node ${NODE_NAME} (short: ${NODE_SHORT_NAME})"
+
+    # Create Service with pod selector
+    cat <<EOF | kubectl apply -f -
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ${SERVICE_NAME}
+      namespace: ${NAMESPACE}
+      labels:
+        app: pasarguard-node-ingress
+        node: ${NODE_NAME}
+    spec:
+      type: ClusterIP
+      selector:
+        app: pasarguard-node-ingress
+      ports:
+        - name: proxy
+          port: 443
+          protocol: TCP
+          targetPort: 443
+        - name: api
+          port: 62050
+          protocol: TCP
+          targetPort: 62050
+        - name: metrics
+          port: 9550
+          protocol: TCP
+          targetPort: 9550
+    EOF
+
+    echo "Service created: ${SERVICE_NAME}.${NAMESPACE}.svc.cluster.local"
+
+    # Create IngressRouteTCP for this DNS name with TLS passthrough
+    INGRESS_NAME="pasarguard-tcp-${DNS_NAME//./-}"
+
+    echo "Creating IngressRouteTCP: ${INGRESS_NAME} for ${DNS_NAME}"
+
+    cat <<EOF | kubectl apply -f -
+    apiVersion: traefik.io/v1alpha1
+    kind: IngressRouteTCP
+    metadata:
+      name: ${INGRESS_NAME}
+      namespace: ${NAMESPACE}
+      labels:
+        app: pasarguard-node-ingress
+        node: ${NODE_NAME}
+    spec:
+      entryPoints:
+        - websecure
+      routes:
+        - match: HostSNI(\`${DNS_NAME}\`)
+          services:
+            - name: ${SERVICE_NAME}
+              port: 443
+      tls:
+        passthrough: true
+    EOF
+
+    echo "IngressRouteTCP created: ${INGRESS_NAME}"
+    echo "Traffic to ${DNS_NAME}:443 will be routed to ${SERVICE_NAME}:443"
+
+  exporter-start.sh: |
+    #!/bin/sh
+    # Install required tools
+    apk add --no-cache wget curl iproute2-ss bash
+
+    # Download v2ray-exporter
+    echo "Downloading v2ray-exporter..."
+    ARCH=$(uname -m)
+    case $ARCH in
+      x86_64)
+        BINARY_ARCH="amd64"
+        ;;
+      aarch64|arm64)
+        BINARY_ARCH="arm64"
+        ;;
+      *)
+        echo "Unsupported architecture: $ARCH"
+        exit 1
+        ;;
+    esac
+    
+    echo "Detected architecture: $ARCH, using binary: v2ray-exporter_linux_$BINARY_ARCH"
+    wget -L -O /tmp/v2ray-exporter "https://github.com/wi1dcard/v2ray-exporter/releases/download/v0.6.0/v2ray-exporter_linux_$BINARY_ARCH"
+    mv /tmp/v2ray-exporter /usr/local/bin/v2ray-exporter
+    chmod +x /usr/local/bin/v2ray-exporter
+
+    # Wait for initial API port file
+    echo "Waiting for initial xray API port file..."
+    while [ ! -f /shared/xray-api-port ]; do
+      echo "Waiting for API port file..."
+      sleep 2
+    done
+
+    # Main loop - restart exporter if it crashes or port changes
+    while true; do
+      if [ -f /shared/xray-api-port ]; then
+        API_PORT=$(cat /shared/xray-api-port)
+        if [ -n "$API_PORT" ]; then
+          echo "Starting v2ray-exporter with endpoint 127.0.0.1:$API_PORT"
+          /usr/local/bin/v2ray-exporter --v2ray-endpoint "127.0.0.1:$API_PORT" --listen ":9550" &
+          EXPORTER_PID=$!
+          
+          # Wait for exporter to exit or port file to change
+          while kill -0 $EXPORTER_PID 2>/dev/null; do
+            if [ -f /shared/xray-api-port ]; then
+              NEW_PORT=$(cat /shared/xray-api-port)
+              if [ "$NEW_PORT" != "$API_PORT" ]; then
+                echo "API port changed from $API_PORT to $NEW_PORT, restarting exporter"
+                kill $EXPORTER_PID 2>/dev/null
+                wait $EXPORTER_PID 2>/dev/null
+                break
+              fi
+            fi
+            sleep 5
+          done
+          
+          echo "Exporter stopped, restarting..."
+          wait $EXPORTER_PID 2>/dev/null
+        fi
+      fi
+      sleep 2
+    done
+
+  pasarguard-start.sh: |
+    #!/bin/sh
+    # Read API_KEY from shared volume created by init container
+    if [ -f /shared/api-key ]; then
+      export API_KEY=$(cat /shared/api-key)
+      echo "Loaded API_KEY from shared volume"
+    else
+      echo "WARNING: API_KEY file not found, using default"
+    fi
+
+    cd /app
+    
+    # Start main process in background
+    ./main &
+    MAIN_PID=$!
+    
+    # Start continuous port monitoring in background
+    {
+      sleep 10  # Wait for xray to start initially
+      LAST_PORT=""
+      
+      while true; do
+        API_PORT=$(netstat -tlpn | grep xray | grep 127.0.0.1 | awk '{print $4}' | cut -d: -f2 | head -1)
+        if [ -n "$API_PORT" ] && [ "$API_PORT" != "$LAST_PORT" ]; then
+          echo "Found xray API port: $API_PORT"
+          echo -n "$API_PORT" > /shared/xray-api-port
+          LAST_PORT="$API_PORT"
+        fi
+        sleep 5  # Check every 5 seconds
+      done
+    } &
+    PORT_MONITOR_PID=$!
+    
+    # Wait for main process to finish
+    wait $MAIN_PID
+    
+    # Clean up port monitor
+    kill $PORT_MONITOR_PID 2>/dev/null
--- a/k8s/apps/pasarguard/daemonset-ingress.yaml
+++ b/k8s/apps/pasarguard/daemonset-ingress.yaml
@@ -0,0 +1,225 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pasarguard-node-ingress
+  labels:
+    app: pasarguard-node-ingress
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pasarguard-node-ingress-configmap
+  labels:
+    app: pasarguard-node-ingress
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "update", "patch"]
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["get", "list", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["get", "list", "create", "update", "patch", "delete"]
+  - apiGroups: ["traefik.io", "traefik.containo.us"]
+    resources: ["ingressroutetcps"]
+    verbs: ["get", "list", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pasarguard-node-ingress-configmap
+  labels:
+    app: pasarguard-node-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pasarguard-node-ingress-configmap
+subjects:
+  - kind: ServiceAccount
+    name: pasarguard-node-ingress
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pasarguard-node-ingress-reader
+  labels:
+    app: pasarguard-node-ingress
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pasarguard-node-ingress-reader
+  labels:
+    app: pasarguard-node-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pasarguard-node-ingress-reader
+subjects:
+  - kind: ServiceAccount
+    name: pasarguard-node-ingress
+    namespace: pasarguard
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: pasarguard-node-ingress
+  labels:
+    app: pasarguard-node-ingress
+spec:
+  selector:
+    matchLabels:
+      app: pasarguard-node-ingress
+  revisionHistoryLimit: 3
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: pasarguard-node-ingress
+    spec:
+      serviceAccountName: pasarguard-node-ingress
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: xray-node-address
+                operator: Exists
+      initContainers:
+        - name: init-uuid
+          image: bitnami/kubectl:latest
+          env:
+            - name: GODEBUG
+              value: "x509sha1=1"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          command:
+            - /bin/bash
+            - /scripts/init-uuid-ingress.sh
+          volumeMounts:
+            - name: shared-data
+              mountPath: /shared
+            - name: scripts
+              mountPath: /scripts
+      containers:
+        - name: xray-exporter
+          image: alpine:3.18
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+            - /scripts/exporter-start.sh
+          ports:
+            - name: metrics
+              containerPort: 9550
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /scrape
+              port: metrics
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /scrape
+              port: metrics
+            initialDelaySeconds: 45
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "128Mi"
+              cpu: "150m"
+          volumeMounts:
+            - name: shared-data
+              mountPath: /shared
+              readOnly: true
+            - name: scripts
+              mountPath: /scripts
+        - name: pasarguard-node
+          image: 'pasarguard/node:v0.1.3'
+          imagePullPolicy: Always
+          command:
+            - /bin/sh
+            - /scripts/pasarguard-start.sh
+          ports:
+            - name: api
+              containerPort: 62050
+              protocol: TCP
+            - name: proxy
+              containerPort: 443
+              protocol: TCP
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: NODE_HOST
+              value: "0.0.0.0"
+            - name: SERVICE_PORT
+              value: "62050"
+            - name: SERVICE_PROTOCOL
+              value: "grpc"
+            - name: DEBUG
+              value: "true"
+            - name: SSL_CERT_FILE
+              value: "/shared/tls.crt"
+            - name: SSL_KEY_FILE
+              value: "/shared/tls.key"
+            - name: XRAY_EXECUTABLE_PATH
+              value: "/usr/local/bin/xray"
+            - name: XRAY_ASSETS_PATH
+              value: "/usr/local/share/xray"
+            - name: API_KEY
+              value: "change-this-to-a-secure-uuid"
+          livenessProbe:
+            tcpSocket:
+              port: 62050
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            tcpSocket:
+              port: 62050
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "512Mi"
+              cpu: "750m"
+          volumeMounts:
+            - name: shared-data
+              mountPath: /shared
+              readOnly: false
+            - name: scripts
+              mountPath: /scripts
+      volumes:
+        - name: shared-data
+          emptyDir: {}
+        - name: scripts
+          configMap:
+            name: pasarguard-scripts-ingress
+            defaultMode: 0755
--- a/k8s/apps/pasarguard/kustomization.yaml
+++ b/k8s/apps/pasarguard/kustomization.yaml
@@ -9,3 +9,5 @@ resources:
  - ./certificate.yaml
  - ./configmap-scripts.yaml
  - ./servicemonitor.yaml
+  - ./configmap-scripts-ingress.yaml
+  - ./daemonset-ingress.yaml