Fixed gitea runner scheduling. Bump amnezia

k8s/apps/amnezia/configmap-scripts.yaml

@@ -10,7 +10,10 @@ data:
 
     PORT="${1:-5847}"
     VPN_CIDR="${2:-10.8.0.0/16}"
+    POLICY_FILE="${3:-/run/amnezia/policy/policy.conf}"
     FORWARD_CHAIN="AMNEZIAWG-FORWARD"
+    POLICY_CHAIN_A="AMNEZIAWG-POLICY-A"
+    POLICY_CHAIN_B="AMNEZIAWG-POLICY-B"
     IPTABLES=(iptables -w 30)
 
     external_interface() {
@@ -49,10 +52,84 @@ data:
       done
     }
 
-    reset_chain() {
-      local chain="$1"
-      "${IPTABLES[@]}" -N "${chain}" >/dev/null 2>&1 || true
-      "${IPTABLES[@]}" -F "${chain}"
+    ensure_chain() {
+      "${IPTABLES[@]}" -N "$1" >/dev/null 2>&1 || true
+    }
+
+    valid_ipv4_32() {
+      local value="$1"
+      [[ "${value}" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/32$ ]] || return 1
+      local address="${value%/32}"
+      local octet
+      IFS=. read -r -a octets <<< "${address}"
+      [ "${#octets[@]}" -eq 4 ] || return 1
+      for octet in "${octets[@]}"; do
+        [[ "${octet}" =~ ^[0-9]+$ ]] || return 1
+        [ "${octet}" -le 255 ] || return 1
+      done
+    }
+
+    routed_through_awg0() {
+      ip -4 route get "${1%/32}" 2>/dev/null | grep -Eq '(^|[[:space:]])dev awg0([[:space:]]|$)'
+    }
+
+    active_policy_chain() {
+      "${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null \
+        | awk '$1 == "-A" && $3 == "-j" && $4 ~ /^AMNEZIAWG-POLICY-[AB]$/ { print $4; exit }'
+    }
+
+    apply_policy() {
+      local active inactive source destination extra
+      active="$(active_policy_chain || true)"
+      if [ "${active}" = "${POLICY_CHAIN_A}" ]; then
+        inactive="${POLICY_CHAIN_B}"
+      else
+        inactive="${POLICY_CHAIN_A}"
+      fi
+
+      ensure_chain "${inactive}"
+      "${IPTABLES[@]}" -F "${inactive}"
+
+      if [ -f "${POLICY_FILE}" ]; then
+        while read -r source destination extra; do
+          [ -n "${source:-}" ] || continue
+          [[ "${source}" == \#* ]] && continue
+          if [ -n "${extra:-}" ] \
+            || ! valid_ipv4_32 "${source}" \
+            || ! valid_ipv4_32 "${destination:-}" \
+            || ! routed_through_awg0 "${source}" \
+            || ! routed_through_awg0 "${destination}"; then
+            echo "Invalid or out-of-tunnel policy line: ${source:-} ${destination:-} ${extra:-}" >&2
+            "${IPTABLES[@]}" -F "${inactive}"
+            return 1
+          fi
+          "${IPTABLES[@]}" -A "${inactive}" \
+            -i awg0 -o awg0 -s "${source}" -d "${destination}" \
+            -m comment --comment amneziawg-group-pair -j ACCEPT
+        done < "${POLICY_FILE}"
+      fi
+
+      # This DROP is deliberately restricted to packets entering and leaving
+      # awg0. It cannot match Kubernetes, CNI, kubelet, Tailscale, or host traffic.
+      "${IPTABLES[@]}" -A "${inactive}" \
+        -i awg0 -o awg0 -s "${VPN_CIDR}" -d "${VPN_CIDR}" \
+        -m comment --comment amneziawg-client-isolation -j DROP
+      "${IPTABLES[@]}" -A "${inactive}" \
+        -i awg0 -o "${EXT_IF}" -s "${VPN_CIDR}" \
+        -m conntrack --ctstate NEW,ESTABLISHED,RELATED \
+        -m comment --comment amneziawg-internet-out -j ACCEPT
+      "${IPTABLES[@]}" -A "${inactive}" \
+        -i "${EXT_IF}" -o awg0 -d "${VPN_CIDR}" \
+        -m conntrack --ctstate ESTABLISHED,RELATED \
+        -m comment --comment amneziawg-internet-return -j ACCEPT
+      "${IPTABLES[@]}" -A "${inactive}" -j RETURN
+
+      # Insert the complete new policy before removing the old jump. There is
+      # never a window with a partially built or absent active policy.
+      "${IPTABLES[@]}" -I "${FORWARD_CHAIN}" 1 -j "${inactive}"
+      if [ -n "${active}" ]; then
+        "${IPTABLES[@]}" -D "${FORWARD_CHAIN}" -j "${active}"
+      fi
     }
 
     EXT_IF="$(external_interface || true)"
@@ -65,6 +142,7 @@ data:
     fi
 
     sysctl -w net.ipv4.ip_forward=1
+    sysctl -w net.ipv4.conf.awg0.send_redirects=0
 
     # Remove rules written by older revisions. In particular, the unqualified
     # tailscale UDP DROP also blocks Flannel VXLAN (UDP/8472).
@@ -79,11 +157,21 @@ data:
     "${IPTABLES[@]}" -F AMNEZIAWG-INPUT >/dev/null 2>&1 || true
     "${IPTABLES[@]}" -X AMNEZIAWG-INPUT >/dev/null 2>&1 || true
 
-    reset_chain "${FORWARD_CHAIN}"
-    "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT
-    "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT
-    "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -j RETURN
-    ensure_rule -A FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}"
+    ensure_chain "${FORWARD_CHAIN}"
+    ensure_chain "${POLICY_CHAIN_A}"
+    ensure_chain "${POLICY_CHAIN_B}"
+    if ! "${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null | grep -q -- '-j RETURN'; then
+      "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -j RETURN
+    fi
+    apply_policy
+    delete_rule "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT
+    delete_rule "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT
+
+    # The jump is first, but non-awg traffic immediately RETURNs to the original
+    # host FORWARD chain without an ACCEPT or DROP decision.
+    delete_rule FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}"
+    "${IPTABLES[@]}" -I FORWARD 1 \
+      -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}"
 
     ensure_rule -A -t nat POSTROUTING -s "${VPN_CIDR}" -o "${EXT_IF}" -m comment --comment amneziawg-masquerade -j MASQUERADE
 
@@ -96,6 +184,8 @@ data:
     PORT="${1:-5847}"
     VPN_CIDR="${2:-10.8.0.0/16}"
     FORWARD_CHAIN="AMNEZIAWG-FORWARD"
+    POLICY_CHAIN_A="AMNEZIAWG-POLICY-A"
+    POLICY_CHAIN_B="AMNEZIAWG-POLICY-B"
     IPTABLES=(iptables -w 30)
 
     external_interface() {
@@ -139,6 +229,10 @@ data:
     "${IPTABLES[@]}" -F AMNEZIAWG-INPUT >/dev/null 2>&1 || true
     "${IPTABLES[@]}" -X AMNEZIAWG-INPUT >/dev/null 2>&1 || true
     "${IPTABLES[@]}" -F "${FORWARD_CHAIN}" >/dev/null 2>&1 || true
+    "${IPTABLES[@]}" -F "${POLICY_CHAIN_A}" >/dev/null 2>&1 || true
+    "${IPTABLES[@]}" -F "${POLICY_CHAIN_B}" >/dev/null 2>&1 || true
+    "${IPTABLES[@]}" -X "${POLICY_CHAIN_A}" >/dev/null 2>&1 || true
+    "${IPTABLES[@]}" -X "${POLICY_CHAIN_B}" >/dev/null 2>&1 || true
     "${IPTABLES[@]}" -X "${FORWARD_CHAIN}" >/dev/null 2>&1 || true
 
   firewall-check.sh: |
@@ -176,11 +270,26 @@ data:
       exit 1
     fi
 
-    "${IPTABLES[@]}" -C FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}"
-    "${IPTABLES[@]}" -C "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT
-    "${IPTABLES[@]}" -C "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT
+    FIRST_FORWARD_RULE="$("${IPTABLES[@]}" -S FORWARD | grep '^-A FORWARD ' | sed -n '1p')"
+    [[ "${FIRST_FORWARD_RULE}" == *"--comment amneziawg-forward-jump"* ]]
+    [[ "${FIRST_FORWARD_RULE}" == *"-j ${FORWARD_CHAIN}"* ]]
+    ACTIVE_POLICY="$("${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null \
+      | awk '$1 == "-A" && $3 == "-j" && $4 ~ /^AMNEZIAWG-POLICY-[AB]$/ { print $4; exit }')"
+    [ -n "${ACTIVE_POLICY}" ]
+    "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \
+      -i awg0 -o awg0 -s "${VPN_CIDR}" -d "${VPN_CIDR}" \
+      -m comment --comment amneziawg-client-isolation -j DROP
+    "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \
+      -i awg0 -o "${EXT_IF}" -s "${VPN_CIDR}" \
+      -m conntrack --ctstate NEW,ESTABLISHED,RELATED \
+      -m comment --comment amneziawg-internet-out -j ACCEPT
+    "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \
+      -i "${EXT_IF}" -o awg0 -d "${VPN_CIDR}" \
+      -m conntrack --ctstate ESTABLISHED,RELATED \
+      -m comment --comment amneziawg-internet-return -j ACCEPT
 
     "${IPTABLES[@]}" -t nat -C POSTROUTING -s "${VPN_CIDR}" -o "${EXT_IF}" -m comment --comment amneziawg-masquerade -j MASQUERADE
+    [ "$(sysctl -n net.ipv4.conf.awg0.send_redirects)" = "0" ]
     awg show awg0 >/dev/null 2>&1
 
   run.sh: |
@@ -189,6 +298,8 @@ data:
 
     SERVER_CONFIG="/etc/amnezia/server/awg0.conf"
     CLIENTS_DIR="${AMNEZIAWG_CLIENTS_DIR:-/run/amnezia/clients}"
+    POLICY_FILE="${AMNEZIAWG_POLICY_FILE:-/run/amnezia/policy/policy.conf}"
+    CONFIG_GENERATION="${AMNEZIAWG_CONFIG_GENERATION:-/run/amnezia/config-generation}"
     RUNTIME_CONFIG="/run/amnezia/awg0.conf"
     SYNC_CONFIG="/run/amnezia/awg0.sync.conf"
     STATUS_FILE="/run/amnezia/reload-status"
@@ -223,13 +334,12 @@ data:
       chmod 0600 "${RUNTIME_CONFIG}"
     }
 
-    client_config_hash() {
-      {
-        for client_config in "${CLIENTS_DIR}"/*; do
-          [ -f "${client_config}" ] || continue
-          sha256sum "${client_config}"
-        done
-      } | sha256sum | awk '{print $1}'
+    runtime_config_hash() {
+      if [ -f "${CONFIG_GENERATION}" ]; then
+        sha256sum "${CONFIG_GENERATION}" | awk '{print $1}'
+      else
+        printf 'missing\n'
+      fi
     }
 
     write_reload_status() {
@@ -250,10 +360,14 @@ data:
     }
 
     apply_live_config() {
+      # Fail closed during a peer/key and policy transition. /dev/null produces
+      # an Internet-only policy with all awg0-to-awg0 traffic isolated.
+      /scripts/firewall-up.sh 5847 10.8.0.0/16 /dev/null
       render_config
       awg-quick strip "${RUNTIME_CONFIG}" > "${SYNC_CONFIG}"
       chmod 0600 "${SYNC_CONFIG}"
       awg syncconf awg0 "${SYNC_CONFIG}"
+      /scripts/firewall-up.sh 5847 10.8.0.0/16 "${POLICY_FILE}"
     }
 
     watch_client_config() {
@@ -263,7 +377,7 @@ data:
         wait "$!" || return 0
 
         local current_hash
-        current_hash="$(client_config_hash)"
+        current_hash="$(runtime_config_hash)"
         if [ "${current_hash}" = "${last_hash}" ]; then
           continue
         fi
@@ -283,7 +397,7 @@ data:
     trap cleanup EXIT
     trap 'exit 0' TERM INT
 
-    initial_hash="$(client_config_hash)"
+    initial_hash="$(runtime_config_hash)"
     render_config
     cleanup
     awg-quick up "${RUNTIME_CONFIG}"
@@ -299,6 +413,8 @@ data:
     CLIENT_SECRET_KEY="${AMNEZIAWG_CLIENT_SECRET_KEY:-peers.conf}"
     CLIENTS_DIR="${AMNEZIAWG_CLIENTS_DIR:-/run/amnezia/clients}"
     PEERS_FILE="${CLIENTS_DIR}/peers.conf"
+    POLICY_FILE="${AMNEZIAWG_POLICY_FILE:-/run/amnezia/policy/policy.conf}"
+    CONFIG_GENERATION="${AMNEZIAWG_CONFIG_GENERATION:-/run/amnezia/config-generation}"
     SYNC_INTERVAL="${AMNEZIAWG_CLIENT_SECRET_SYNC_INTERVAL:-5}"
     NAMESPACE="${POD_NAMESPACE:-$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)}"
 
@@ -308,33 +424,66 @@ data:
         : > "${PEERS_FILE}"
         chmod 0600 "${PEERS_FILE}"
       fi
+      mkdir -p "$(dirname "${POLICY_FILE}")"
+      if [ ! -f "${POLICY_FILE}" ]; then
+        : > "${POLICY_FILE}"
+        chmod 0600 "${POLICY_FILE}"
+      fi
+      {
+        sha256sum "${PEERS_FILE}"
+        sha256sum "${POLICY_FILE}"
+      } | sha256sum | awk '{print $1}' > "${CONFIG_GENERATION}"
     }
 
     sync_once() {
       mkdir -p "${CLIENTS_DIR}"
-      local tmp_file="${PEERS_FILE}.tmp"
-      local encoded=""
+      mkdir -p "$(dirname "${POLICY_FILE}")"
+      local snapshot="${CONFIG_GENERATION}.snapshot"
+      local peers_tmp="${PEERS_FILE}.tmp"
+      local policy_tmp="${POLICY_FILE}.tmp"
+      local peers_encoded policy_encoded generation
 
-      if ! encoded="$(kubectl get secret "${CLIENT_SECRET}" -n "${NAMESPACE}" -o "go-template={{ index .data \"${CLIENT_SECRET_KEY}\" }}" 2>/dev/null)"; then
+      if ! kubectl get secret "${CLIENT_SECRET}" -n "${NAMESPACE}" \
+        -o "go-template={{ with index .data \"${CLIENT_SECRET_KEY}\" }}{{ . }}{{ end }}{{ \"\n\" }}{{ with index .data \"policy.conf\" }}{{ . }}{{ end }}{{ \"\n\" }}" \
+        > "${snapshot}" 2>/dev/null; then
         echo "WARN: failed to read Secret ${NAMESPACE}/${CLIENT_SECRET}; keeping current peers" >&2
+        rm -f "${snapshot}"
         write_empty_once
         return 0
       fi
 
-      if [ -n "${encoded}" ]; then
-        printf '%s' "${encoded}" | base64 -d > "${tmp_file}"
-      else
-        : > "${tmp_file}"
-      fi
-      chmod 0600 "${tmp_file}"
+      peers_encoded="$(sed -n '1p' "${snapshot}")"
+      policy_encoded="$(sed -n '2p' "${snapshot}")"
+      rm -f "${snapshot}"
 
-      if [ -f "${PEERS_FILE}" ] && cmp -s "${tmp_file}" "${PEERS_FILE}"; then
-        rm -f "${tmp_file}"
+      if [ -n "${peers_encoded}" ]; then
+        printf '%s' "${peers_encoded}" | base64 -d > "${peers_tmp}"
+      else
+        : > "${peers_tmp}"
+      fi
+      if [ -n "${policy_encoded}" ]; then
+        printf '%s' "${policy_encoded}" | base64 -d > "${policy_tmp}"
+      else
+        # Missing policy is fail-closed: no client-to-client pairs are allowed.
+        : > "${policy_tmp}"
+      fi
+      chmod 0600 "${peers_tmp}" "${policy_tmp}"
+
+      generation="$({
+        sha256sum "${peers_tmp}"
+        sha256sum "${policy_tmp}"
+      } | sha256sum | awk '{print $1}')"
+      if [ -f "${CONFIG_GENERATION}" ] \
+        && [ "$(sed -n '1p' "${CONFIG_GENERATION}")" = "${generation}" ]; then
+        rm -f "${peers_tmp}" "${policy_tmp}"
         return 0
       fi
 
-      mv "${tmp_file}" "${PEERS_FILE}"
-      echo "Synced AmneziaWG client peers from Secret ${NAMESPACE}/${CLIENT_SECRET}:${CLIENT_SECRET_KEY}"
+      mv "${policy_tmp}" "${POLICY_FILE}"
+      mv "${peers_tmp}" "${PEERS_FILE}"
+      printf '%s\n' "${generation}" > "${CONFIG_GENERATION}.tmp"
+      mv "${CONFIG_GENERATION}.tmp" "${CONFIG_GENERATION}"
+      echo "Synced AmneziaWG peers and policy from Secret ${NAMESPACE}/${CLIENT_SECRET}"
     }
 
     if [ "${1:-}" = "once" ]; then

k8s/apps/gitea/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   - external-secrets.yaml
   - deployment.yaml
   - runner-rbac.yaml
+  - runner-rebalance.yaml
   - user-unban-cronjob.yaml
   - service.yaml
   - ingress.yaml

k8s/apps/gitea/runner-rebalance.yaml

@@ -0,0 +1,127 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner-rebalancer
+  namespace: gitea
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner-rebalancer
+  namespace: gitea
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-rebalancer
+  namespace: gitea
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gitea-runner-rebalancer
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner-rebalancer
+    namespace: gitea
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gitea-runner-rebalancer
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    resourceNames:
+      - ai.tail2fe2d.ts.net
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: gitea-runner-rebalancer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gitea-runner-rebalancer
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner-rebalancer
+    namespace: gitea
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-runner-rebalancer
+  namespace: gitea
+spec:
+  schedule: "*/2 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 300
+      template:
+        metadata:
+          labels:
+            app: gitea-runner-rebalancer
+        spec:
+          serviceAccountName: gitea-runner-rebalancer
+          restartPolicy: Never
+          containers:
+            - name: rebalance
+              image: bitnami/kubectl:latest
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -ec
+                - |
+                  target_node="ai.tail2fe2d.ts.net"
+                  ready="$(
+                    kubectl get node "${target_node}" \
+                      -o jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
+                  )"
+
+                  if [ "${ready}" != "True" ]; then
+                    echo "${target_node} is not Ready; keeping the runner on its current node"
+                    exit 0
+                  fi
+
+                  runner_pod="$(
+                    kubectl -n gitea get pods \
+                      -l app=gitea-runner \
+                      --field-selector=status.phase=Running \
+                      -o jsonpath='{.items[0].metadata.name}'
+                  )"
+                  runner_node="$(
+                    kubectl -n gitea get pods \
+                      -l app=gitea-runner \
+                      --field-selector=status.phase=Running \
+                      -o jsonpath='{.items[0].spec.nodeName}'
+                  )"
+
+                  if [ -z "${runner_pod}" ] || [ -z "${runner_node}" ]; then
+                    echo "No running Gitea runner found"
+                    exit 0
+                  fi
+
+                  if [ "${runner_node}" = "${target_node}" ]; then
+                    echo "${runner_pod} already runs on ${target_node}"
+                    exit 0
+                  fi
+
+                  echo "Moving ${runner_pod} from ${runner_node} to preferred node ${target_node}"
+                  kubectl -n gitea delete pod "${runner_pod}" --wait=false