From 80b792ce8c1f4353d8877ffc29e0acdec21896a8 Mon Sep 17 00:00:00 2001 From: Ultradesu Date: Thu, 30 Jul 2026 23:16:36 +0100 Subject: [PATCH] Fixed gitea runner scheduling. Bump amnezia --- k8s/apps/amnezia/configmap-scripts.yaml | 217 ++++++++++++++++++++---- k8s/apps/gitea/kustomization.yaml | 1 + k8s/apps/gitea/runner-rebalance.yaml | 127 ++++++++++++++ 3 files changed, 311 insertions(+), 34 deletions(-) create mode 100644 k8s/apps/gitea/runner-rebalance.yaml diff --git a/k8s/apps/amnezia/configmap-scripts.yaml b/k8s/apps/amnezia/configmap-scripts.yaml index b171c83..35d721b 100644 --- a/k8s/apps/amnezia/configmap-scripts.yaml +++ b/k8s/apps/amnezia/configmap-scripts.yaml @@ -10,7 +10,10 @@ data: PORT="${1:-5847}" VPN_CIDR="${2:-10.8.0.0/16}" + POLICY_FILE="${3:-/run/amnezia/policy/policy.conf}" FORWARD_CHAIN="AMNEZIAWG-FORWARD" + POLICY_CHAIN_A="AMNEZIAWG-POLICY-A" + POLICY_CHAIN_B="AMNEZIAWG-POLICY-B" IPTABLES=(iptables -w 30) external_interface() { @@ -49,10 +52,84 @@ data: done } - reset_chain() { - local chain="$1" - "${IPTABLES[@]}" -N "${chain}" >/dev/null 2>&1 || true - "${IPTABLES[@]}" -F "${chain}" + ensure_chain() { + "${IPTABLES[@]}" -N "$1" >/dev/null 2>&1 || true + } + + valid_ipv4_32() { + local value="$1" + [[ "${value}" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/32$ ]] || return 1 + local address="${value%/32}" + local octet + IFS=. read -r -a octets <<< "${address}" + [ "${#octets[@]}" -eq 4 ] || return 1 + for octet in "${octets[@]}"; do + [[ "${octet}" =~ ^[0-9]+$ ]] || return 1 + [ "${octet}" -le 255 ] || return 1 + done + } + + routed_through_awg0() { + ip -4 route get "${1%/32}" 2>/dev/null | grep -Eq '(^|[[:space:]])dev awg0([[:space:]]|$)' + } + + active_policy_chain() { + "${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null \ + | awk '$1 == "-A" && $3 == "-j" && $4 ~ /^AMNEZIAWG-POLICY-[AB]$/ { print $4; exit }' + } + + apply_policy() { + local active inactive source destination extra + active="$(active_policy_chain || true)" + if [ "${active}" = "${POLICY_CHAIN_A}" ]; then + inactive="${POLICY_CHAIN_B}" + else + inactive="${POLICY_CHAIN_A}" + fi + + ensure_chain "${inactive}" + "${IPTABLES[@]}" -F "${inactive}" + + if [ -f "${POLICY_FILE}" ]; then + while read -r source destination extra; do + [ -n "${source:-}" ] || continue + [[ "${source}" == \#* ]] && continue + if [ -n "${extra:-}" ] \ + || ! valid_ipv4_32 "${source}" \ + || ! valid_ipv4_32 "${destination:-}" \ + || ! routed_through_awg0 "${source}" \ + || ! routed_through_awg0 "${destination}"; then + echo "Invalid or out-of-tunnel policy line: ${source:-} ${destination:-} ${extra:-}" >&2 + "${IPTABLES[@]}" -F "${inactive}" + return 1 + fi + "${IPTABLES[@]}" -A "${inactive}" \ + -i awg0 -o awg0 -s "${source}" -d "${destination}" \ + -m comment --comment amneziawg-group-pair -j ACCEPT + done < "${POLICY_FILE}" + fi + + # This DROP is deliberately restricted to packets entering and leaving + # awg0. It cannot match Kubernetes, CNI, kubelet, Tailscale, or host traffic. + "${IPTABLES[@]}" -A "${inactive}" \ + -i awg0 -o awg0 -s "${VPN_CIDR}" -d "${VPN_CIDR}" \ + -m comment --comment amneziawg-client-isolation -j DROP + "${IPTABLES[@]}" -A "${inactive}" \ + -i awg0 -o "${EXT_IF}" -s "${VPN_CIDR}" \ + -m conntrack --ctstate NEW,ESTABLISHED,RELATED \ + -m comment --comment amneziawg-internet-out -j ACCEPT + "${IPTABLES[@]}" -A "${inactive}" \ + -i "${EXT_IF}" -o awg0 -d "${VPN_CIDR}" \ + -m conntrack --ctstate ESTABLISHED,RELATED \ + -m comment --comment amneziawg-internet-return -j ACCEPT + "${IPTABLES[@]}" -A "${inactive}" -j RETURN + + # Insert the complete new policy before removing the old jump. There is + # never a window with a partially built or absent active policy. + "${IPTABLES[@]}" -I "${FORWARD_CHAIN}" 1 -j "${inactive}" + if [ -n "${active}" ]; then + "${IPTABLES[@]}" -D "${FORWARD_CHAIN}" -j "${active}" + fi } EXT_IF="$(external_interface || true)" @@ -65,6 +142,7 @@ data: fi sysctl -w net.ipv4.ip_forward=1 + sysctl -w net.ipv4.conf.awg0.send_redirects=0 # Remove rules written by older revisions. In particular, the unqualified # tailscale UDP DROP also blocks Flannel VXLAN (UDP/8472). @@ -79,11 +157,21 @@ data: "${IPTABLES[@]}" -F AMNEZIAWG-INPUT >/dev/null 2>&1 || true "${IPTABLES[@]}" -X AMNEZIAWG-INPUT >/dev/null 2>&1 || true - reset_chain "${FORWARD_CHAIN}" - "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT - "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT - "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -j RETURN - ensure_rule -A FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}" + ensure_chain "${FORWARD_CHAIN}" + ensure_chain "${POLICY_CHAIN_A}" + ensure_chain "${POLICY_CHAIN_B}" + if ! "${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null | grep -q -- '-j RETURN'; then + "${IPTABLES[@]}" -A "${FORWARD_CHAIN}" -j RETURN + fi + apply_policy + delete_rule "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT + delete_rule "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT + + # The jump is first, but non-awg traffic immediately RETURNs to the original + # host FORWARD chain without an ACCEPT or DROP decision. + delete_rule FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}" + "${IPTABLES[@]}" -I FORWARD 1 \ + -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}" ensure_rule -A -t nat POSTROUTING -s "${VPN_CIDR}" -o "${EXT_IF}" -m comment --comment amneziawg-masquerade -j MASQUERADE @@ -96,6 +184,8 @@ data: PORT="${1:-5847}" VPN_CIDR="${2:-10.8.0.0/16}" FORWARD_CHAIN="AMNEZIAWG-FORWARD" + POLICY_CHAIN_A="AMNEZIAWG-POLICY-A" + POLICY_CHAIN_B="AMNEZIAWG-POLICY-B" IPTABLES=(iptables -w 30) external_interface() { @@ -139,6 +229,10 @@ data: "${IPTABLES[@]}" -F AMNEZIAWG-INPUT >/dev/null 2>&1 || true "${IPTABLES[@]}" -X AMNEZIAWG-INPUT >/dev/null 2>&1 || true "${IPTABLES[@]}" -F "${FORWARD_CHAIN}" >/dev/null 2>&1 || true + "${IPTABLES[@]}" -F "${POLICY_CHAIN_A}" >/dev/null 2>&1 || true + "${IPTABLES[@]}" -F "${POLICY_CHAIN_B}" >/dev/null 2>&1 || true + "${IPTABLES[@]}" -X "${POLICY_CHAIN_A}" >/dev/null 2>&1 || true + "${IPTABLES[@]}" -X "${POLICY_CHAIN_B}" >/dev/null 2>&1 || true "${IPTABLES[@]}" -X "${FORWARD_CHAIN}" >/dev/null 2>&1 || true firewall-check.sh: | @@ -176,11 +270,26 @@ data: exit 1 fi - "${IPTABLES[@]}" -C FORWARD -m comment --comment amneziawg-forward-jump -j "${FORWARD_CHAIN}" - "${IPTABLES[@]}" -C "${FORWARD_CHAIN}" -i awg0 -m comment --comment amneziawg-forward-in -j ACCEPT - "${IPTABLES[@]}" -C "${FORWARD_CHAIN}" -o awg0 -m comment --comment amneziawg-forward-out -j ACCEPT + FIRST_FORWARD_RULE="$("${IPTABLES[@]}" -S FORWARD | grep '^-A FORWARD ' | sed -n '1p')" + [[ "${FIRST_FORWARD_RULE}" == *"--comment amneziawg-forward-jump"* ]] + [[ "${FIRST_FORWARD_RULE}" == *"-j ${FORWARD_CHAIN}"* ]] + ACTIVE_POLICY="$("${IPTABLES[@]}" -S "${FORWARD_CHAIN}" 2>/dev/null \ + | awk '$1 == "-A" && $3 == "-j" && $4 ~ /^AMNEZIAWG-POLICY-[AB]$/ { print $4; exit }')" + [ -n "${ACTIVE_POLICY}" ] + "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \ + -i awg0 -o awg0 -s "${VPN_CIDR}" -d "${VPN_CIDR}" \ + -m comment --comment amneziawg-client-isolation -j DROP + "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \ + -i awg0 -o "${EXT_IF}" -s "${VPN_CIDR}" \ + -m conntrack --ctstate NEW,ESTABLISHED,RELATED \ + -m comment --comment amneziawg-internet-out -j ACCEPT + "${IPTABLES[@]}" -C "${ACTIVE_POLICY}" \ + -i "${EXT_IF}" -o awg0 -d "${VPN_CIDR}" \ + -m conntrack --ctstate ESTABLISHED,RELATED \ + -m comment --comment amneziawg-internet-return -j ACCEPT "${IPTABLES[@]}" -t nat -C POSTROUTING -s "${VPN_CIDR}" -o "${EXT_IF}" -m comment --comment amneziawg-masquerade -j MASQUERADE + [ "$(sysctl -n net.ipv4.conf.awg0.send_redirects)" = "0" ] awg show awg0 >/dev/null 2>&1 run.sh: | @@ -189,6 +298,8 @@ data: SERVER_CONFIG="/etc/amnezia/server/awg0.conf" CLIENTS_DIR="${AMNEZIAWG_CLIENTS_DIR:-/run/amnezia/clients}" + POLICY_FILE="${AMNEZIAWG_POLICY_FILE:-/run/amnezia/policy/policy.conf}" + CONFIG_GENERATION="${AMNEZIAWG_CONFIG_GENERATION:-/run/amnezia/config-generation}" RUNTIME_CONFIG="/run/amnezia/awg0.conf" SYNC_CONFIG="/run/amnezia/awg0.sync.conf" STATUS_FILE="/run/amnezia/reload-status" @@ -223,13 +334,12 @@ data: chmod 0600 "${RUNTIME_CONFIG}" } - client_config_hash() { - { - for client_config in "${CLIENTS_DIR}"/*; do - [ -f "${client_config}" ] || continue - sha256sum "${client_config}" - done - } | sha256sum | awk '{print $1}' + runtime_config_hash() { + if [ -f "${CONFIG_GENERATION}" ]; then + sha256sum "${CONFIG_GENERATION}" | awk '{print $1}' + else + printf 'missing\n' + fi } write_reload_status() { @@ -250,10 +360,14 @@ data: } apply_live_config() { + # Fail closed during a peer/key and policy transition. /dev/null produces + # an Internet-only policy with all awg0-to-awg0 traffic isolated. + /scripts/firewall-up.sh 5847 10.8.0.0/16 /dev/null render_config awg-quick strip "${RUNTIME_CONFIG}" > "${SYNC_CONFIG}" chmod 0600 "${SYNC_CONFIG}" awg syncconf awg0 "${SYNC_CONFIG}" + /scripts/firewall-up.sh 5847 10.8.0.0/16 "${POLICY_FILE}" } watch_client_config() { @@ -263,7 +377,7 @@ data: wait "$!" || return 0 local current_hash - current_hash="$(client_config_hash)" + current_hash="$(runtime_config_hash)" if [ "${current_hash}" = "${last_hash}" ]; then continue fi @@ -283,7 +397,7 @@ data: trap cleanup EXIT trap 'exit 0' TERM INT - initial_hash="$(client_config_hash)" + initial_hash="$(runtime_config_hash)" render_config cleanup awg-quick up "${RUNTIME_CONFIG}" @@ -299,6 +413,8 @@ data: CLIENT_SECRET_KEY="${AMNEZIAWG_CLIENT_SECRET_KEY:-peers.conf}" CLIENTS_DIR="${AMNEZIAWG_CLIENTS_DIR:-/run/amnezia/clients}" PEERS_FILE="${CLIENTS_DIR}/peers.conf" + POLICY_FILE="${AMNEZIAWG_POLICY_FILE:-/run/amnezia/policy/policy.conf}" + CONFIG_GENERATION="${AMNEZIAWG_CONFIG_GENERATION:-/run/amnezia/config-generation}" SYNC_INTERVAL="${AMNEZIAWG_CLIENT_SECRET_SYNC_INTERVAL:-5}" NAMESPACE="${POD_NAMESPACE:-$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)}" @@ -308,33 +424,66 @@ data: : > "${PEERS_FILE}" chmod 0600 "${PEERS_FILE}" fi + mkdir -p "$(dirname "${POLICY_FILE}")" + if [ ! -f "${POLICY_FILE}" ]; then + : > "${POLICY_FILE}" + chmod 0600 "${POLICY_FILE}" + fi + { + sha256sum "${PEERS_FILE}" + sha256sum "${POLICY_FILE}" + } | sha256sum | awk '{print $1}' > "${CONFIG_GENERATION}" } sync_once() { mkdir -p "${CLIENTS_DIR}" - local tmp_file="${PEERS_FILE}.tmp" - local encoded="" + mkdir -p "$(dirname "${POLICY_FILE}")" + local snapshot="${CONFIG_GENERATION}.snapshot" + local peers_tmp="${PEERS_FILE}.tmp" + local policy_tmp="${POLICY_FILE}.tmp" + local peers_encoded policy_encoded generation - if ! encoded="$(kubectl get secret "${CLIENT_SECRET}" -n "${NAMESPACE}" -o "go-template={{ index .data \"${CLIENT_SECRET_KEY}\" }}" 2>/dev/null)"; then + if ! kubectl get secret "${CLIENT_SECRET}" -n "${NAMESPACE}" \ + -o "go-template={{ with index .data \"${CLIENT_SECRET_KEY}\" }}{{ . }}{{ end }}{{ \"\n\" }}{{ with index .data \"policy.conf\" }}{{ . }}{{ end }}{{ \"\n\" }}" \ + > "${snapshot}" 2>/dev/null; then echo "WARN: failed to read Secret ${NAMESPACE}/${CLIENT_SECRET}; keeping current peers" >&2 + rm -f "${snapshot}" write_empty_once return 0 fi - if [ -n "${encoded}" ]; then - printf '%s' "${encoded}" | base64 -d > "${tmp_file}" - else - : > "${tmp_file}" - fi - chmod 0600 "${tmp_file}" + peers_encoded="$(sed -n '1p' "${snapshot}")" + policy_encoded="$(sed -n '2p' "${snapshot}")" + rm -f "${snapshot}" - if [ -f "${PEERS_FILE}" ] && cmp -s "${tmp_file}" "${PEERS_FILE}"; then - rm -f "${tmp_file}" + if [ -n "${peers_encoded}" ]; then + printf '%s' "${peers_encoded}" | base64 -d > "${peers_tmp}" + else + : > "${peers_tmp}" + fi + if [ -n "${policy_encoded}" ]; then + printf '%s' "${policy_encoded}" | base64 -d > "${policy_tmp}" + else + # Missing policy is fail-closed: no client-to-client pairs are allowed. + : > "${policy_tmp}" + fi + chmod 0600 "${peers_tmp}" "${policy_tmp}" + + generation="$({ + sha256sum "${peers_tmp}" + sha256sum "${policy_tmp}" + } | sha256sum | awk '{print $1}')" + if [ -f "${CONFIG_GENERATION}" ] \ + && [ "$(sed -n '1p' "${CONFIG_GENERATION}")" = "${generation}" ]; then + rm -f "${peers_tmp}" "${policy_tmp}" return 0 fi - mv "${tmp_file}" "${PEERS_FILE}" - echo "Synced AmneziaWG client peers from Secret ${NAMESPACE}/${CLIENT_SECRET}:${CLIENT_SECRET_KEY}" + mv "${policy_tmp}" "${POLICY_FILE}" + mv "${peers_tmp}" "${PEERS_FILE}" + printf '%s\n' "${generation}" > "${CONFIG_GENERATION}.tmp" + mv "${CONFIG_GENERATION}.tmp" "${CONFIG_GENERATION}" + echo "Synced AmneziaWG peers and policy from Secret ${NAMESPACE}/${CLIENT_SECRET}" } if [ "${1:-}" = "once" ]; then diff --git a/k8s/apps/gitea/kustomization.yaml b/k8s/apps/gitea/kustomization.yaml index cadf3bb..119411f 100644 --- a/k8s/apps/gitea/kustomization.yaml +++ b/k8s/apps/gitea/kustomization.yaml @@ -6,6 +6,7 @@ resources: - external-secrets.yaml - deployment.yaml - runner-rbac.yaml + - runner-rebalance.yaml - user-unban-cronjob.yaml - service.yaml - ingress.yaml diff --git a/k8s/apps/gitea/runner-rebalance.yaml b/k8s/apps/gitea/runner-rebalance.yaml new file mode 100644 index 0000000..77a049b --- /dev/null +++ b/k8s/apps/gitea/runner-rebalance.yaml @@ -0,0 +1,127 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-runner-rebalancer + namespace: gitea +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitea-runner-rebalancer + namespace: gitea +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitea-runner-rebalancer + namespace: gitea +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitea-runner-rebalancer +subjects: + - kind: ServiceAccount + name: gitea-runner-rebalancer + namespace: gitea +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitea-runner-rebalancer +rules: + - apiGroups: + - "" + resources: + - nodes + resourceNames: + - ai.tail2fe2d.ts.net + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitea-runner-rebalancer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gitea-runner-rebalancer +subjects: + - kind: ServiceAccount + name: gitea-runner-rebalancer + namespace: gitea +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gitea-runner-rebalancer + namespace: gitea +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + app: gitea-runner-rebalancer + spec: + serviceAccountName: gitea-runner-rebalancer + restartPolicy: Never + containers: + - name: rebalance + image: bitnami/kubectl:latest + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + - | + target_node="ai.tail2fe2d.ts.net" + ready="$( + kubectl get node "${target_node}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}' + )" + + if [ "${ready}" != "True" ]; then + echo "${target_node} is not Ready; keeping the runner on its current node" + exit 0 + fi + + runner_pod="$( + kubectl -n gitea get pods \ + -l app=gitea-runner \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}' + )" + runner_node="$( + kubectl -n gitea get pods \ + -l app=gitea-runner \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].spec.nodeName}' + )" + + if [ -z "${runner_pod}" ] || [ -z "${runner_node}" ]; then + echo "No running Gitea runner found" + exit 0 + fi + + if [ "${runner_node}" = "${target_node}" ]; then + echo "${runner_pod} already runs on ${target_node}" + exit 0 + fi + + echo "Moving ${runner_pod} from ${runner_node} to preferred node ${target_node}" + kubectl -n gitea delete pod "${runner_pod}" --wait=false