diff --git a/k8s-ng/core-state/values.yaml b/k8s-ng/core-state/values.yaml index 548e07c..eb3f451 100644 --- a/k8s-ng/core-state/values.yaml +++ b/k8s-ng/core-state/values.yaml @@ -81,3 +81,9 @@ apps: namespace: external-secrets wave: 4 # reads the Vaultwarden vault, so it comes after vaultwarden skipDryRunOnMissingResource: true # ClusterSecretStore CRD comes from the external-secrets app + - name: gitea + path: k8s-ng/core/gitea + namespace: gitea + wave: 5 # after secret-stores (its secrets come from the vault); pinned to the music node; the runner to the ai node + prune: false # never delete the instance because of a manifest change + skipDryRunOnMissingResource: true diff --git a/k8s-ng/core/gitea/deployment.yaml b/k8s-ng/core/gitea/deployment.yaml new file mode 100644 index 0000000..ee33f4e --- /dev/null +++ b/k8s-ng/core/gitea/deployment.yaml @@ -0,0 +1,70 @@ +# Gitea with SQLite; everything (repositories, database, SSH host keys) lives in +# /var/lib/gitea on the music node. +# `replicas` is deliberately not set: Argo CD does not manage it, so the instance +# can be scaled to 0 with kubectl during a data migration without being healed back. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea + labels: + app: gitea +spec: + selector: + matchLabels: + app: gitea + strategy: + type: Recreate # two instances must never open the same SQLite database + template: + metadata: + labels: + app: gitea + spec: + nodeSelector: + kubernetes.io/hostname: music + volumes: + - name: storage + hostPath: + path: /var/lib/gitea + type: DirectoryOrCreate + containers: + - name: gitea + image: gitea/gitea:28.0.0 + resources: + requests: + memory: 512Mi + cpu: 200m + limits: + memory: 2Gi + cpu: 1500m + env: + - name: GITEA__service__REGISTER_MANUAL_CONFIRM + value: "true" + - name: GITEA__service__ENABLE_CAPTCHA + value: "true" + - name: GITEA__service__REQUIRE_CAPTCHA_FOR_LOGIN + value: "false" + - name: GITEA__service__REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA + value: "true" + - name: GITEA__service__CAPTCHA_TYPE + value: cfturnstile + - name: GITEA__security__ALLOWED_HOST_LIST + value: "*" + envFrom: + - secretRef: + name: gitea-runner-act-runner-secrets + ports: + - name: http + containerPort: 3000 + protocol: TCP + - name: ssh + containerPort: 22 + protocol: TCP + readinessProbe: + httpGet: + path: /api/healthz + port: http + periodSeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: storage + mountPath: /data diff --git a/k8s-ng/core/gitea/external-secrets.yaml b/k8s-ng/core/gitea/external-secrets.yaml new file mode 100644 index 0000000..f7ad74e --- /dev/null +++ b/k8s-ng/core/gitea/external-secrets.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: gitea-runner-token +spec: + refreshInterval: 1m + target: + name: gitea-runner-act-runner-secrets + deletionPolicy: Delete + template: + type: Opaque + data: + token: |- + {{ .password }} + GITEA__service__CF_TURNSTILE_SITEKEY: |- + {{ .CF_TURNSTILE_SITEKEY }} + GITEA__service__CF_TURNSTILE_SECRET: |- + {{ .CF_TURNSTILE_SECRET }} + data: + - secretKey: password + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: e475b5ab-ea3c-48a5-bb4c-a6bc552fc064 + property: login.password + - secretKey: CF_TURNSTILE_SITEKEY + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: e475b5ab-ea3c-48a5-bb4c-a6bc552fc064 + property: fields[0].value + - secretKey: CF_TURNSTILE_SECRET + sourceRef: + storeRef: + name: vaultwarden-login + kind: ClusterSecretStore + remoteRef: + key: e475b5ab-ea3c-48a5-bb4c-a6bc552fc064 + property: fields[1].value diff --git a/k8s-ng/core/gitea/ingress.yaml b/k8s-ng/core/gitea/ingress.yaml new file mode 100644 index 0000000..78d351f --- /dev/null +++ b/k8s-ng/core/gitea/ingress.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gitea + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure +spec: + rules: + - host: gt.hexor.cy + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gitea + port: + number: 80 + tls: + - secretName: gitea-tls + hosts: + - gt.hexor.cy diff --git a/k8s-ng/core/gitea/kustomization.yaml b/k8s-ng/core/gitea/kustomization.yaml new file mode 100644 index 0000000..787e952 --- /dev/null +++ b/k8s-ng/core/gitea/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - external-secrets.yaml + - deployment.yaml + - service.yaml + - ingress.yaml + - runner.yaml + - runner-rbac.yaml + - user-unban-cronjob.yaml diff --git a/k8s-ng/core/gitea/runner-rbac.yaml b/k8s-ng/core/gitea/runner-rbac.yaml new file mode 100644 index 0000000..6a20c91 --- /dev/null +++ b/k8s-ng/core/gitea/runner-rbac.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-runner +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-runner-kubernetes-token + annotations: + kubernetes.io/service-account.name: gitea-runner +type: kubernetes.io/service-account-token +--- +# CI may edit the routes of the auth proxy (the ConfigMap lives in the keycloak namespace here). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: auth-proxy-routes-manager + namespace: keycloak +rules: + - apiGroups: [""] + resources: [configmaps] + resourceNames: [auth-proxy-routes] + verbs: [get, update, patch] + - apiGroups: [""] + resources: [configmaps] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitea-runner-auth-proxy-routes + namespace: keycloak +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: auth-proxy-routes-manager +subjects: + - kind: ServiceAccount + name: gitea-runner + namespace: gitea diff --git a/k8s-ng/core/gitea/runner.yaml b/k8s-ng/core/gitea/runner.yaml new file mode 100644 index 0000000..d2c5d02 --- /dev/null +++ b/k8s-ng/core/gitea/runner.yaml @@ -0,0 +1,84 @@ +# Gitea Actions runner on the ai node. Jobs run as containers in the host's Docker +# (virtualisation.docker in hosts/ai of the nix repo) through the mounted socket. +# `replicas` is not set (see deployment.yaml): scale it with kubectl when needed. +apiVersion: v1 +kind: ConfigMap +metadata: + name: gitea-runner-config +data: + config.yaml: | + runner: + capacity: 8 # jobs in parallel; the ai node has 32 threads and 124 GiB +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea-runner + labels: + app: gitea-runner + annotations: + reloader.stakater.com/auto: "true" +spec: + selector: + matchLabels: + app: gitea-runner + strategy: + type: Recreate + template: + metadata: + labels: + app: gitea-runner + spec: + serviceAccountName: gitea-runner + dnsConfig: + options: + - name: ndots + value: "2" + nodeSelector: + workload/ai: "true" + tolerations: + - key: workload + operator: Equal + value: ai + effect: NoSchedule + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + type: Socket + - name: runner-data + emptyDir: + sizeLimit: 30Gi + - name: config + configMap: + name: gitea-runner-config + containers: + - name: gitea-runner + image: gitea/act_runner:nightly + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + - name: runner-data + mountPath: /data + - name: config + mountPath: /config.yaml + subPath: config.yaml + env: + - name: CONFIG_FILE + value: /config.yaml + - name: GITEA_INSTANCE_URL + # Job containers run outside the cluster network, so they need the public URL. + value: https://gt.hexor.cy + - name: GITEA_RUNNER_REGISTRATION_TOKEN + valueFrom: + secretKeyRef: + name: gitea-runner-act-runner-secrets + key: token + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: GITEA_RUNNER_NAME + value: $(NODE_NAME) + - name: GITEA_RUNNER_LABELS + value: ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04,ubuntu-20.04:docker://ghcr.io/catthehacker/ubuntu:act-20.04 diff --git a/k8s-ng/core/gitea/service.yaml b/k8s-ng/core/gitea/service.yaml new file mode 100644 index 0000000..6683559 --- /dev/null +++ b/k8s-ng/core/gitea/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea +spec: + selector: + app: gitea + ports: + - protocol: TCP + port: 80 + targetPort: 3000 +--- +# Git over SSH on every node (servicelb); the node firewall opens 30022 (nix repo, k3s-base.nix). +apiVersion: v1 +kind: Service +metadata: + name: gitea-ssh +spec: + type: LoadBalancer + ports: + - port: 30022 + targetPort: 22 + selector: + app: gitea diff --git a/k8s-ng/core/gitea/user-unban-cronjob.yaml b/k8s-ng/core/gitea/user-unban-cronjob.yaml new file mode 100644 index 0000000..672320f --- /dev/null +++ b/k8s-ng/core/gitea/user-unban-cronjob.yaml @@ -0,0 +1,59 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gitea-user-unban + labels: + app: gitea-user-unban +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + app: gitea-user-unban + spec: + restartPolicy: OnFailure + nodeSelector: + kubernetes.io/hostname: music + volumes: + - name: storage + hostPath: + path: /var/lib/gitea + type: Directory + containers: + - name: sqlite-unban + image: gitea/gitea:28.0.0 + imagePullPolicy: IfNotPresent + resources: + requests: + memory: 32Mi + cpu: 10m + limits: + memory: 128Mi + cpu: 100m + command: + - /bin/sh + - -ec + - | + sqlite3 -cmd ".timeout 30000" /data/gitea/gitea.db " + UPDATE \"user\" + SET is_active = 1, + prohibit_login = 0, + updated_unix = unixepoch() + WHERE lower(email) = lower('ab@hexor.cy') + AND (is_active <> 1 OR prohibit_login <> 0); + + SELECT printf( + 'gitea user watchdog: id=%d login=%s email=%s is_active=%d prohibit_login=%d updated_unix=%d', + id, lower_name, email, is_active, prohibit_login, updated_unix + ) + FROM \"user\" + WHERE lower(email) = lower('ab@hexor.cy'); + " + volumeMounts: + - name: storage + mountPath: /data