NG: llamacpp on the ai node (Qwen3.8-27B) with the AMD GPU device plugin

Co-Authored-By: Claude <noreply@anthropic.com>

k8s-ng/apps/llamacpp/configmap.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llamacpp-config
+data:
+  LLAMA_CACHE: /models
+  LLAMA_ARG_HOST: 0.0.0.0
+  LLAMA_ARG_PORT: "8080"
+  # The model is chosen here: <hf-user>/<repo>:<quant>. It is downloaded into
+  # /models on first start; changing it restarts the pod (reloader).
+  LLAMA_ARG_HF_REPO: "unsloth/Qwen3.8-27B-GGUF:UD-Q6_K"
+  LLAMA_ARG_CTX_SIZE: "131072"
+  LLAMA_ARG_N_GPU_LAYERS: all
+  LLAMA_ARG_FLASH_ATTN: "on"
+  LLAMA_ARG_FIT: "on"
+  # Strix Halo: mmap-ed weights hang or corrupt on ROCm; read them directly.
+  LLAMA_ARG_LOAD_MODE: dio

k8s-ng/apps/llamacpp/deployment.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llamacpp
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: llamacpp
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate # one GPU, one pod
+  selector:
+    matchLabels:
+      app: llamacpp
+  template:
+    metadata:
+      labels:
+        app: llamacpp
+    spec:
+      dnsPolicy: Default
+      nodeSelector:
+        workload/ai: "true"
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: ai
+          effect: NoSchedule
+      containers:
+        - name: llamacpp
+          image: ghcr.io/ggml-org/llama.cpp:server-rocm-b11434
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: llamacpp-config
+          env:
+            - name: HF_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: llamacpp-hf-token
+                  key: token
+                  optional: true
+            # hipBLASLt gives the biggest prompt-processing gain on ROCm 7.x.
+            - name: ROCBLAS_USE_HIPBLASLT
+              value: "1"
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          resources:
+            limits:
+              amd.com/gpu: 1
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 180 # first start downloads the model
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 3
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      volumes:
+        - name: models
+          hostPath:
+            path: /var/lib/llamacpp/models
+            type: DirectoryOrCreate

k8s-ng/apps/llamacpp/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml

k8s-ng/apps/llamacpp/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: llamacpp
+  labels:
+    app: llamacpp
+spec:
+  type: ClusterIP
+  selector:
+    app: llamacpp
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+      protocol: TCP

k8s-ng/core-state/values.yaml

@@ -28,6 +28,10 @@ apps:
     path: k8s-ng/core/reloader
     namespace: reloader
     wave: 1
+  - name: gpu-system
+    path: k8s-ng/core/gpu
+    namespace: gpu-system
+    wave: 1
   - name: cert-manager
     path: k8s-ng/core/cert-manager
     namespace: cert-manager
@@ -66,6 +70,12 @@ apps:
     namespace: web-petting
     wave: 5
     skipDryRunOnMissingResource: true
+  - name: llamacpp
+    project: apps
+    path: k8s-ng/apps/llamacpp
+    namespace: llamacpp
+    wave: 5 # pinned to the ai node (GPU); must not hold back the other waves
+    skipDryRunOnMissingResource: true
   - name: secret-stores
     path: k8s-ng/core/secret-stores
     namespace: external-secrets

k8s-ng/core/gpu/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+helmCharts:
+  - name: amd-gpu
+    repo: https://rocm.github.io/k8s-device-plugin/
+    version: 0.22.0
+    releaseName: amd-gpu-device-plugin
+    namespace: gpu-system
+    valuesFile: values.yaml
+    includeCRDs: true

k8s-ng/core/gpu/values.yaml

@@ -0,0 +1,35 @@
+# AMD GPU device plugin: exposes the Strix Halo iGPU of the "ai" node as the
+# amd.com/gpu resource (and injects /dev/kfd and /dev/dri into the pod).
+nfd:
+  enabled: false
+
+labeller:
+  enabled: false
+
+dp:
+  image:
+    repository: docker.io/rocm/k8s-device-plugin
+    tag: "1.31.0.9"
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+
+# The ai node carries the label workload/ai=true and the taint workload=ai:NoSchedule
+# (hosts/ai in the nix repo).
+tolerations:
+  - key: workload
+    operator: Equal
+    value: ai
+    effect: NoSchedule
+
+node_selector_enabled: true
+node_selector:
+  kubernetes.io/arch: amd64
+  workload/ai: "true"