Aleksandr BogomiakovandClaude
373ce77ac8
Update Kubernetes Services Wiki / Generate and Update K8s Wiki (push) Successful in 44s
Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
1.9 KiB
YAML
75 lines
1.9 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: llamacpp
|
|
annotations:
|
|
reloader.stakater.com/auto: "true"
|
|
labels:
|
|
app: llamacpp
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate # one GPU, one pod
|
|
selector:
|
|
matchLabels:
|
|
app: llamacpp
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: llamacpp
|
|
spec:
|
|
dnsPolicy: Default
|
|
nodeSelector:
|
|
workload/ai: "true"
|
|
tolerations:
|
|
- key: workload
|
|
operator: Equal
|
|
value: ai
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: llamacpp
|
|
image: ghcr.io/ggml-org/llama.cpp:server-rocm-b11434
|
|
imagePullPolicy: IfNotPresent
|
|
envFrom:
|
|
- configMapRef:
|
|
name: llamacpp-config
|
|
env:
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: llamacpp-hf-token
|
|
key: token
|
|
optional: true
|
|
# hipBLASLt gives the biggest prompt-processing gain on ROCm 7.x.
|
|
- name: ROCBLAS_USE_HIPBLASLT
|
|
value: "1"
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
protocol: TCP
|
|
resources:
|
|
limits:
|
|
amd.com/gpu: 1
|
|
startupProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: http
|
|
failureThreshold: 180 # first start downloads the model
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: http
|
|
failureThreshold: 3
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /models
|
|
volumes:
|
|
- name: models
|
|
hostPath:
|
|
path: /var/lib/llamacpp/models
|
|
type: DirectoryOrCreate
|