Skip to content

Commit 8b91ece

Browse files
yosefmihyosefmih
authored
Merge pull request #1286 from porter-dev/ym/llm_models
adding llm helm charts
2 parents 7efaf50 + ed4606f commit 8b91ece

8 files changed

Lines changed: 190 additions & 0 deletions

File tree

‎Tiltfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ local_resource(
7575
helm cm-push addons/postgresql-managed local && \
7676
helm cm-push addons/redis-managed local && \
7777
helm cm-push addons/deepgram local && \
78+
helm cm-push addons/hf-llm-models local && \
7879
helm repo update local
7980
''',
8081
deps=[

‎addons/hf-llm-models/.helmignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎addons/hf-llm-models/Chart.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: v2
2+
name: hf-llm-models
3+
description: A helm chart that will be used to deploy LLM models from hugging face in porter
4+
5+
type: application
6+
7+
version: 0.1.0
8+
9+
appVersion: "0.1.0"

‎addons/hf-llm-models/templates/deployment.yaml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
llm-model: {{ .Release.Name }}
6+
annotations:
7+
porter.run/hf-llm-model-version: "{{ .Chart.Version }}"
8+
name: {{ .Release.Name }}-workload
9+
spec:
10+
replicas: 1
11+
strategy:
12+
type: Recreate
13+
selector:
14+
matchLabels:
15+
llm-model: {{ .Release.Name }}
16+
template:
17+
metadata:
18+
labels:
19+
llm-model: {{ .Release.Name }}
20+
spec:
21+
tolerations:
22+
- key: "removable"
23+
operator: "Equal"
24+
value: "true"
25+
effect: "NoSchedule"
26+
{{- if .Values.tolerations }}
27+
{{- toYaml .Values.tolerations | nindent 8 }}
28+
{{- end }}
29+
containers:
30+
- command:
31+
- python3
32+
- -m
33+
- vllm.entrypoints.openai.api_server
34+
- --model
35+
- {{ .Values.modelID }}
36+
{{- if .Values.quantization }}
37+
- --quantization
38+
- {{ .Values.quantization }}
39+
{{- end }}
40+
- --download-dir
41+
- {{ .Values.modelDir }}
42+
- --gpu-memory-utilization={{ printf "%.2f" .Values.gpuMemoryUtilization }}
43+
{{- if .Values.dType}}
44+
- --dtype
45+
- {{ .Values.dType }}
46+
{{- end }}
47+
- --tensor-parallel-size={{ .Values.tensorParallelSize }}
48+
{{- if .Values.maxModelLen }}
49+
- --max-model-len={{ .Values.maxModelLen }}
50+
{{- end }}
51+
image: {{ .Values.vllmImage }}
52+
imagePullPolicy: Always
53+
env:
54+
- name: HF_TOKEN
55+
value: {{ .Values.huggingFaceToken }}
56+
ports:
57+
- containerPort: 8000
58+
protocol: TCP
59+
name: https
60+
resources:
61+
requests:
62+
{{- if .Values.resources.requests.cpu }}
63+
cpu: {{ .Values.resources.requests.cpu }}
64+
{{- end }}
65+
{{- if .Values.resources.requests.memory }}
66+
memory: {{ .Values.resources.requests.memory }}
67+
{{- end }}
68+
{{- if .Values.resources.requests.nvidiaGpu }}
69+
nvidia.com/gpu: {{ .Values.resources.requests.nvidiaGpu }}
70+
{{- end }}
71+
limits:
72+
{{- if .Values.resources.limits.cpu }}
73+
cpu: {{ .Values.resources.limits.cpu }}
74+
{{- end }}
75+
{{- if .Values.resources.limits.memory }}
76+
memory: {{ .Values.resources.limits.memory }}
77+
{{- end }}
78+
{{- if .Values.resources.limits.nvidiaGpu }}
79+
nvidia.com/gpu: {{ .Values.resources.limits.nvidiaGpu }}
80+
{{- end }}
81+
volumeMounts:
82+
- name: model-volume
83+
mountPath: {{ .Values.modelDir }}
84+
name: vllm
85+
securityContext:
86+
allowPrivilegeEscalation: false
87+
terminationGracePeriodSeconds: 10
88+
volumes:
89+
- name: model-volume
90+
persistentVolumeClaim:
91+
claimName: "{{ .Release.Name }}-model-pvc"

‎addons/hf-llm-models/templates/pvc.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
kind: PersistentVolumeClaim
3+
metadata:
4+
name: {{ .Release.Name }}-model-pvc
5+
spec:
6+
accessModes:
7+
- ReadWriteMany
8+
storageClassName: efs-{{ .Release.Name }}
9+
resources:
10+
requests:
11+
storage: 20Gi

‎addons/hf-llm-models/templates/service.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
labels:
5+
llm-model: {{ .Release.Name }}
6+
name: {{ .Release.Name }}
7+
spec:
8+
ports:
9+
- name: https
10+
port: 8000
11+
targetPort: https
12+
selector:
13+
llm-model: {{ .Release.Name }}

‎addons/hf-llm-models/templates/storageclass.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: storage.k8s.io/v1
2+
kind: StorageClass
3+
metadata:
4+
name: efs-{{ .Release.Name }}
5+
provisioner: efs.csi.aws.com
6+
parameters:
7+
provisioningMode: efs-ap
8+
directoryPerms: "700"
9+
fileSystemId: {{ .Values.efsFileSystemId }}
10+
reuseAccessPoint: "true"

‎addons/hf-llm-models/values.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
vllmImage: "docker.io/vllm/vllm-openai:latest"
2+
3+
huggingFaceToken: # a hugging face token if it is required by the model
4+
5+
modelDir: "/model"
6+
7+
modelID: "mistralai/Mistral-7B-Instruct-v0.2"
8+
9+
gpuMemoryUtilization: 0.98
10+
11+
maxModelLen: 26016
12+
13+
dType: half
14+
15+
efsFileSystemId: # efs file system id created in the clients cluster, required
16+
17+
tensorParallelSize: 1
18+
19+
resources:
20+
requests:
21+
cpu: 2
22+
memory: 14000Mi
23+
nvidiaGpu: "1"
24+
limits:
25+
cpu: 4
26+
memory: 40000Mi
27+
nvidiaGpu: "1"
28+
29+
tolerations:
30+
- key: "nvidia.com/gpu"
31+
operator: "Exists"
32+
effect: "NoSchedule"

0 commit comments

Comments
 (0)