diff --git a/k8s/core/system-upgrade/app.yaml b/k8s/core/system-upgrade/app.yaml new file mode 100644 index 0000000..4d5f70d --- /dev/null +++ b/k8s/core/system-upgrade/app.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: k3s-system-upgrade + namespace: argocd +spec: + project: core + destination: + namespace: system-upgrade + server: https://kubernetes.default.svc + source: + repoURL: ssh://git@gt.hexor.cy:30022/ab/homelab.git + targetRevision: HEAD + path: k8s/core/system-upgrade + syncPolicy: + automated: + selfHeal: true + prune: true + syncOptions: + - CreateNamespace=true + diff --git a/k8s/core/system-upgrade/crd.yaml b/k8s/core/system-upgrade/crd.yaml new file mode 100644 index 0000000..927a8ca --- /dev/null +++ b/k8s/core/system-upgrade/crd.yaml @@ -0,0 +1,684 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: plans.upgrade.cattle.io +spec: + group: upgrade.cattle.io + names: + categories: + - upgrade + kind: Plan + plural: plans + singular: plan + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.upgrade.image + name: Image + type: string + - jsonPath: .spec.channel + name: Channel + type: string + - jsonPath: .spec.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + channel: + nullable: true + type: string + concurrency: + type: integer + cordon: + type: boolean + drain: + nullable: true + properties: + deleteEmptydirData: + nullable: true + type: boolean + deleteLocalData: + nullable: true + type: boolean + disableEviction: + type: boolean + force: + type: boolean + gracePeriod: + nullable: true + type: integer + ignoreDaemonSets: + nullable: true + type: boolean + podSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + skipWaitForDeleteTimeout: + type: integer + timeout: + nullable: true + type: integer + type: object + exclusive: + type: boolean + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + jobActiveDeadlineSecs: + type: integer + nodeSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + postCompleteDelay: + nullable: true + type: string + prepare: + nullable: true + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + envs: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + appArmorProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + volumes: + items: + properties: + destination: + nullable: true + type: string + name: + nullable: true + type: string + source: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + secrets: + items: + properties: + ignoreUpdates: + type: boolean + name: + nullable: true + type: string + path: + nullable: true + type: string + type: object + nullable: true + type: array + serviceAccountName: + nullable: true + type: string + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + upgrade: + nullable: true + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + envs: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + appArmorProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + volumes: + items: + properties: + destination: + nullable: true + type: string + name: + nullable: true + type: string + source: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + version: + nullable: true + type: string + window: + nullable: true + properties: + days: + items: + nullable: true + type: string + nullable: true + type: array + endTime: + nullable: true + type: string + startTime: + nullable: true + type: string + timeZone: + nullable: true + type: string + type: object + required: + - upgrade + type: object + status: + properties: + applying: + items: + nullable: true + type: string + nullable: true + type: array + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + latestHash: + nullable: true + type: string + latestVersion: + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/k8s/core/system-upgrade/kustomization.yaml b/k8s/core/system-upgrade/kustomization.yaml new file mode 100644 index 0000000..77f59e5 --- /dev/null +++ b/k8s/core/system-upgrade/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - app.yaml + - crd.yaml + - system-upgrade-controller.yaml + - plan.yaml + diff --git a/k8s/core/system-upgrade/plan.yaml b/k8s/core/system-upgrade/plan.yaml new file mode 100644 index 0000000..a15910b --- /dev/null +++ b/k8s/core/system-upgrade/plan.yaml @@ -0,0 +1,43 @@ +# Server plan +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: server-plan + namespace: system-upgrade +spec: + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "true" + serviceAccountName: system-upgrade + upgrade: + image: rancher/k3s-upgrade + version: v1.32.3+k3s1 +--- +# Agent plan +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: agent-plan + namespace: system-upgrade +spec: + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + prepare: + args: + - prepare + - server-plan + image: rancher/k3s-upgrade + serviceAccountName: system-upgrade + upgrade: + image: rancher/k3s-upgrade + version: v1.32.3+k3s1 + diff --git a/k8s/core/system-upgrade/system-upgrade-controller.yaml b/k8s/core/system-upgrade/system-upgrade-controller.yaml new file mode 100644 index 0000000..d8da40c --- /dev/null +++ b/k8s/core/system-upgrade/system-upgrade-controller.yaml @@ -0,0 +1,322 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/enforce: privileged + name: system-upgrade +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system-upgrade-controller + namespace: system-upgrade +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - deletecollection + - patch + - update + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-upgrade-controller +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - patch + - update +- apiGroups: + - "" + resources: + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - update +- apiGroups: + - "" + resources: + - events + verbs: + - get + - create + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - system-upgrade-controller + resources: + - leases + verbs: + - get + - update +- apiGroups: + - upgrade.cattle.io + resources: + - plans + - plans/status + verbs: + - get + - list + - watch + - create + - patch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-upgrade-controller-drainer +rules: +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - replicasets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-upgrade + namespace: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system-upgrade-controller +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-upgrade-controller +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade-drainer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-upgrade-controller-drainer +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: v1 +data: + SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false" + SYSTEM_UPGRADE_CONTROLLER_LEADER_ELECT: "true" + SYSTEM_UPGRADE_CONTROLLER_THREADS: "2" + SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900" + SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "99" + SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always + SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.30.3 + SYSTEM_UPGRADE_JOB_PRIVILEGED: "true" + SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900" + SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m +kind: ConfigMap +metadata: + name: default-controller-env + namespace: system-upgrade +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: system-upgrade-controller + namespace: system-upgrade +spec: + selector: + matchLabels: + upgrade.cattle.io/controller: system-upgrade-controller + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: system-upgrade-controller + upgrade.cattle.io/controller: system-upgrade-controller + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - system-upgrade-controller + topologyKey: kubernetes.io/hostname + containers: + - env: + - name: SYSTEM_UPGRADE_CONTROLLER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['upgrade.cattle.io/controller'] + - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SYSTEM_UPGRADE_CONTROLLER_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: + - configMapRef: + name: default-controller-env + image: rancher/system-upgrade-controller:v0.15.2 + imagePullPolicy: IfNotPresent + name: system-upgrade-controller + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /etc/ssl + name: etc-ssl + readOnly: true + - mountPath: /etc/pki + name: etc-pki + readOnly: true + - mountPath: /etc/ca-certificates + name: etc-ca-certificates + readOnly: true + - mountPath: /tmp + name: tmp + serviceAccountName: system-upgrade + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/controlplane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoExecute + key: node-role.kubernetes.io/etcd + operator: Exists + volumes: + - hostPath: + path: /etc/ssl + type: DirectoryOrCreate + name: etc-ssl + - hostPath: + path: /etc/pki + type: DirectoryOrCreate + name: etc-pki + - hostPath: + path: /etc/ca-certificates + type: DirectoryOrCreate + name: etc-ca-certificates + - emptyDir: {} + name: tmp