From db09e2a91ac76ee48fe6fe9ded5ae7fabfec0c0c Mon Sep 17 00:00:00 2001 From: devpramod Date: Tue, 1 Jul 2025 23:02:15 +0000 Subject: [PATCH 1/6] Enable docsum to connect with external llm endpoints via helm Signed-off-by: devpramod --- helm-charts/docsum/Chart.yaml | 3 +++ helm-charts/docsum/README.md | 5 ++++- helm-charts/docsum/external-llm-values.yaml | 18 ++++++++++++++++++ helm-charts/docsum/templates/deployment.yaml | 16 ++++++++++++++++ helm-charts/docsum/values.yaml | 13 +++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 helm-charts/docsum/external-llm-values.yaml diff --git a/helm-charts/docsum/Chart.yaml b/helm-charts/docsum/Chart.yaml index 4d02035f8..c7527baca 100644 --- a/helm-charts/docsum/Chart.yaml +++ b/helm-charts/docsum/Chart.yaml @@ -21,13 +21,16 @@ dependencies: - name: llm-uservice version: 0-latest repository: "file://../common/llm-uservice" + condition: llm-uservice.enabled - name: whisper version: 0-latest repository: "file://../common/whisper" + condition: whisper.enabled - name: ui version: 0-latest repository: "file://../common/ui" alias: docsum-ui + condition: docsum-ui.enabled - name: nginx version: 0-latest repository: "file://../common/nginx" diff --git a/helm-charts/docsum/README.md b/helm-charts/docsum/README.md index cd03cdd4d..39ce4a3a8 100644 --- a/helm-charts/docsum/README.md +++ b/helm-charts/docsum/README.md @@ -26,7 +26,10 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-values.yaml # To use AMD ROCm device with TGI # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml - +# To use with OPEA KubeAI models installed to same cluster +# helm install docsum docsum -f docsum/variant_external-llm-values.yaml --set externalLLM.LLM_MODEL="some-KubeAI-model" externalLLM.LLM_SERVICE_HOST_IP="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVICE_PORT="" --set externalLLM.OPENAI_API_KEY="" +# To use with other external OpenAI compatible LLM endpoints +# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key" ``` ## Verify diff --git a/helm-charts/docsum/external-llm-values.yaml b/helm-charts/docsum/external-llm-values.yaml new file mode 100644 index 000000000..0d744b2ad --- /dev/null +++ b/helm-charts/docsum/external-llm-values.yaml @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# External LLM configuration override +externalLLM: + enabled: true # Enable external LLM service + LLM_SERVICE_HOST_IP: "http://your-llm-server" # External LLM service host + LLM_MODEL: "your-model" # LLM model to use + OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication + LLM_SERVER_PORT: "80" # Port for the external LLM service + +# Disable internal LLM service when using external LLM +llm-uservice: + enabled: false +vllm: + enabled: false +tgi: + enabled: false diff --git a/helm-charts/docsum/templates/deployment.yaml b/helm-charts/docsum/templates/deployment.yaml index 486820715..da8b83216 100644 --- a/helm-charts/docsum/templates/deployment.yaml +++ b/helm-charts/docsum/templates/deployment.yaml @@ -35,13 +35,29 @@ spec: - name: {{ .Release.Name }} env: - name: LLM_SERVICE_HOST_IP + {{- if .Values.externalLLM.enabled }} + value: {{ .Values.externalLLM.LLM_SERVICE_HOST_IP }} + {{- else }} value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") }} + {{- end }} - name: LLM_SERVICE_PORT + {{- if .Values.externalLLM.enabled }} + value: {{ .Values.externalLLM.LLM_SERVICE_PORT | default "80" | quote }} + {{- else }} value: {{ index .Values "llm-uservice" "service" "port" | quote }} + {{- end }} + {{- if .Values.whisper.enabled }} - name: ASR_SERVICE_HOST_IP value: {{ include "whisper.fullname" (index .Subcharts "whisper") }} - name: ASR_SERVICE_PORT value: {{ index .Values "whisper" "service" "port" | quote }} + {{- end }} + {{- if .Values.externalLLM.enabled }} + - name: LLM_MODEL + value: {{ .Values.externalLLM.LLM_MODEL }} + - name: OPENAI_API_KEY + value: {{ .Values.externalLLM.OPENAI_API_KEY }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/helm-charts/docsum/values.yaml b/helm-charts/docsum/values.yaml index 9cca28e4c..6119cc0fb 100644 --- a/helm-charts/docsum/values.yaml +++ b/helm-charts/docsum/values.yaml @@ -58,6 +58,7 @@ affinity: {} # To override values in subchart llm-uservice llm-uservice: + enabled: true image: repository: opea/llm-docsum DOCSUM_BACKEND: "vLLM" @@ -79,6 +80,7 @@ vllm: nginx: enabled: false docsum-ui: + enabled: true image: repository: opea/docsum-gradio-ui tag: "latest" @@ -101,8 +103,19 @@ docsum-ui: # type: ClusterIP dashboard: + enabled: true prefix: "OPEA DocSum" +whisper: + enabled: true + +# External LLM configuration +externalLLM: + enabled: false + LLM_SERVICE_HOST_IP: "http://your-llm-server" + LLM_MODEL: "your-model" + OPENAI_API_KEY: "your-api-key" + global: http_proxy: "" https_proxy: "" From bee11e0743196f14a4859faedb9b21729ef61bf1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 14 Jul 2025 13:08:02 +0000 Subject: [PATCH 2/6] modify values yaml name Signed-off-by: Ubuntu --- ...{external-llm-values.yaml => variant_external-llm-values.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename helm-charts/docsum/{external-llm-values.yaml => variant_external-llm-values.yaml} (100%) diff --git a/helm-charts/docsum/external-llm-values.yaml b/helm-charts/docsum/variant_external-llm-values.yaml similarity index 100% rename from helm-charts/docsum/external-llm-values.yaml rename to helm-charts/docsum/variant_external-llm-values.yaml From 250c5740bec405aa32de3176c7a4fe0b9351bd2c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 14 Jul 2025 13:10:59 +0000 Subject: [PATCH 3/6] addressed PR comment Signed-off-by: Ubuntu --- helm-charts/docsum/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/docsum/README.md b/helm-charts/docsum/README.md index 39ce4a3a8..fa645a653 100644 --- a/helm-charts/docsum/README.md +++ b/helm-charts/docsum/README.md @@ -27,7 +27,7 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs # To use AMD ROCm device with TGI # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml # To use with OPEA KubeAI models installed to same cluster -# helm install docsum docsum -f docsum/variant_external-llm-values.yaml --set externalLLM.LLM_MODEL="some-KubeAI-model" externalLLM.LLM_SERVICE_HOST_IP="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVICE_PORT="" --set externalLLM.OPENAI_API_KEY="" +# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.LLM_MODEL="some-KubeAI-model" --set externalLLM.OPENAI_API_KEY="" # To use with other external OpenAI compatible LLM endpoints # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key" ``` From 19e1030321bcde33c40d091dfd82b54f593c8707 Mon Sep 17 00:00:00 2001 From: devpramod Date: Thu, 31 Jul 2025 14:40:46 +0000 Subject: [PATCH 4/6] change host and port names for external llm env variables Signed-off-by: devpramod --- helm-charts/docsum/templates/deployment.yaml | 4 ++-- helm-charts/docsum/values.yaml | 4 +++- helm-charts/docsum/variant_external-llm-values.yaml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/helm-charts/docsum/templates/deployment.yaml b/helm-charts/docsum/templates/deployment.yaml index da8b83216..3df4ab617 100644 --- a/helm-charts/docsum/templates/deployment.yaml +++ b/helm-charts/docsum/templates/deployment.yaml @@ -36,13 +36,13 @@ spec: env: - name: LLM_SERVICE_HOST_IP {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVICE_HOST_IP }} + value: {{ .Values.externalLLM.LLM_SERVER_HOST }} {{- else }} value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") }} {{- end }} - name: LLM_SERVICE_PORT {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVICE_PORT | default "80" | quote }} + value: {{ .Values.externalLLM.LLM_SERVER_PORT | default "80" | quote }} {{- else }} value: {{ index .Values "llm-uservice" "service" "port" | quote }} {{- end }} diff --git a/helm-charts/docsum/values.yaml b/helm-charts/docsum/values.yaml index 6119cc0fb..0a931152a 100644 --- a/helm-charts/docsum/values.yaml +++ b/helm-charts/docsum/values.yaml @@ -80,6 +80,7 @@ vllm: nginx: enabled: false docsum-ui: + # if false, set also nginx.enabled=false enabled: true image: repository: opea/docsum-gradio-ui @@ -112,7 +113,8 @@ whisper: # External LLM configuration externalLLM: enabled: false - LLM_SERVICE_HOST_IP: "http://your-llm-server" + LLM_SERVER_HOST: "http://your-llm-server" + LLM_SERVER_PORT: "80" LLM_MODEL: "your-model" OPENAI_API_KEY: "your-api-key" diff --git a/helm-charts/docsum/variant_external-llm-values.yaml b/helm-charts/docsum/variant_external-llm-values.yaml index 0d744b2ad..1e1a9819b 100644 --- a/helm-charts/docsum/variant_external-llm-values.yaml +++ b/helm-charts/docsum/variant_external-llm-values.yaml @@ -4,10 +4,10 @@ # External LLM configuration override externalLLM: enabled: true # Enable external LLM service - LLM_SERVICE_HOST_IP: "http://your-llm-server" # External LLM service host + LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host + LLM_SERVER_PORT: "80" # Port for the external LLM service LLM_MODEL: "your-model" # LLM model to use OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication - LLM_SERVER_PORT: "80" # Port for the external LLM service # Disable internal LLM service when using external LLM llm-uservice: From d60651fdb82a05e43a18c10d76cc8a76ecad9dd7 Mon Sep 17 00:00:00 2001 From: devpramod Date: Fri, 15 Aug 2025 15:18:02 +0000 Subject: [PATCH 5/6] enable llm-uservice to point to external llm endpoints Signed-off-by: devpramod --- helm-charts/docsum/README.md | 9 ++++---- helm-charts/docsum/templates/deployment.yaml | 14 ------------- .../docsum/variant_external-llm-values.yaml | 21 ++++++++++--------- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/helm-charts/docsum/README.md b/helm-charts/docsum/README.md index fa645a653..2841a88b8 100644 --- a/helm-charts/docsum/README.md +++ b/helm-charts/docsum/README.md @@ -26,10 +26,11 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-values.yaml # To use AMD ROCm device with TGI # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml -# To use with OPEA KubeAI models installed to same cluster -# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST="http://kubeai.kubeai/openai" --set externalLLM.LLM_SERVER_PORT="" --set externalLLM.LLM_MODEL="some-KubeAI-model" --set externalLLM.OPENAI_API_KEY="" -# To use with other external OpenAI compatible LLM endpoints -# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set externalLLM.LLM_SERVER_HOST="http://your-llm-server" --set externalLLM.LLM_MODEL="your-model" --set externalLLM.OPENAI_API_KEY="your-api-key" +# To use with external OpenAI-compatible LLM endpoints (OpenAI, vLLM, TGI, etc.) +# This configures the llm-uservice to connect to external LLM providers while maintaining DocSum compatibility +# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set llm-uservice.env.OPENAI_API_KEY="your-api-key" --set llm-uservice.env.LLM_MODEL_ID="gpt-4-turbo" +# For vLLM/TGI endpoints: +# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set llm-uservice.env.LLM_ENDPOINT="http://your-vllm-server/v1" --set llm-uservice.env.LLM_MODEL_ID="your-model" ``` ## Verify diff --git a/helm-charts/docsum/templates/deployment.yaml b/helm-charts/docsum/templates/deployment.yaml index 3df4ab617..deb1fb4a9 100644 --- a/helm-charts/docsum/templates/deployment.yaml +++ b/helm-charts/docsum/templates/deployment.yaml @@ -35,29 +35,15 @@ spec: - name: {{ .Release.Name }} env: - name: LLM_SERVICE_HOST_IP - {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVER_HOST }} - {{- else }} value: {{ include "llm-uservice.fullname" (index .Subcharts "llm-uservice") }} - {{- end }} - name: LLM_SERVICE_PORT - {{- if .Values.externalLLM.enabled }} - value: {{ .Values.externalLLM.LLM_SERVER_PORT | default "80" | quote }} - {{- else }} value: {{ index .Values "llm-uservice" "service" "port" | quote }} - {{- end }} {{- if .Values.whisper.enabled }} - name: ASR_SERVICE_HOST_IP value: {{ include "whisper.fullname" (index .Subcharts "whisper") }} - name: ASR_SERVICE_PORT value: {{ index .Values "whisper" "service" "port" | quote }} {{- end }} - {{- if .Values.externalLLM.enabled }} - - name: LLM_MODEL - value: {{ .Values.externalLLM.LLM_MODEL }} - - name: OPENAI_API_KEY - value: {{ .Values.externalLLM.OPENAI_API_KEY }} - {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/helm-charts/docsum/variant_external-llm-values.yaml b/helm-charts/docsum/variant_external-llm-values.yaml index 1e1a9819b..c2f292f40 100644 --- a/helm-charts/docsum/variant_external-llm-values.yaml +++ b/helm-charts/docsum/variant_external-llm-values.yaml @@ -1,17 +1,18 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# External LLM configuration override -externalLLM: - enabled: true # Enable external LLM service - LLM_SERVER_HOST: "http://your-llm-server" # External LLM service host - LLM_SERVER_PORT: "80" # Port for the external LLM service - LLM_MODEL: "your-model" # LLM model to use - OPENAI_API_KEY: "your-api-key" # OpenAI API key for authentication - -# Disable internal LLM service when using external LLM +# External LLM configuration - configures llm-uservice to use external LLM providers +# This keeps the llm-uservice wrapper (required for /v1/docsum endpoint) but connects it to external LLMs llm-uservice: - enabled: false + enabled: true # Keep the wrapper service for DocSum compatibility + env: + # Configure llm-uservice to use external OpenAI-compatible endpoints + LLM_ENDPOINT: "https://api.openai.com/v1" # External LLM API endpoint (OpenAI, vLLM, TGI, etc.) + OPENAI_API_KEY: "${OPENAI_API_KEY}" # API key for authentication + LLM_MODEL_ID: "gpt-4-turbo" # Model to use + TEXTGEN_BACKEND: "openai" # Backend type for OpenAI-compatible endpoints + +# Disable local inference services since we're using external LLMs vllm: enabled: false tgi: From 20db664134632dfb98869f0f774f9f890d1d5711 Mon Sep 17 00:00:00 2001 From: devpramod Date: Wed, 20 Aug 2025 22:18:08 +0000 Subject: [PATCH 6/6] add env variable for api key in llm-uservice configmap Signed-off-by: devpramod --- .../common/llm-uservice/templates/configmap.yaml | 3 +++ helm-charts/docsum/README.md | 7 ++++--- helm-charts/docsum/values.yaml | 14 ++++++-------- .../docsum/variant_external-llm-values.yaml | 10 ++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/helm-charts/common/llm-uservice/templates/configmap.yaml b/helm-charts/common/llm-uservice/templates/configmap.yaml index fc5f0ea8f..4e8f9d068 100644 --- a/helm-charts/common/llm-uservice/templates/configmap.yaml +++ b/helm-charts/common/llm-uservice/templates/configmap.yaml @@ -70,6 +70,9 @@ data: {{- if .Values.LLM_MODEL_ID }} LLM_MODEL_ID: {{ .Values.LLM_MODEL_ID | quote }} {{- end }} + {{- if .Values.OPENAI_API_KEY }} + OPENAI_API_KEY: {{ .Values.OPENAI_API_KEY | quote }} + {{- end }} HF_HOME: "/tmp/.cache/huggingface" {{- if not .Values.global.offline }} HF_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | default .Values.global.HF_TOKEN | quote }} diff --git a/helm-charts/docsum/README.md b/helm-charts/docsum/README.md index 2841a88b8..90d9cf908 100644 --- a/helm-charts/docsum/README.md +++ b/helm-charts/docsum/README.md @@ -28,9 +28,10 @@ helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --set global.modelUs # helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/rocm-tgi-values.yaml # To use with external OpenAI-compatible LLM endpoints (OpenAI, vLLM, TGI, etc.) # This configures the llm-uservice to connect to external LLM providers while maintaining DocSum compatibility -# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set llm-uservice.env.OPENAI_API_KEY="your-api-key" --set llm-uservice.env.LLM_MODEL_ID="gpt-4-turbo" -# For vLLM/TGI endpoints: -# helm install docsum docsum --set global.HF_TOKEN=${HFTOKEN} --values docsum/variant_external-llm-values.yaml --set llm-uservice.env.LLM_ENDPOINT="http://your-vllm-server/v1" --set llm-uservice.env.LLM_MODEL_ID="your-model" +# For OpenAI: +# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.OPENAI_API_KEY="your-api-key" --set llm-uservice.LLM_ENDPOINT="https://api.openai.com" --set llm-uservice.LLM_MODEL_ID="gpt-4-turbo" +# For vLLM/TGI or other OpenAI-compatible endpoints: +# helm install docsum docsum --values docsum/variant_external-llm-values.yaml --set llm-uservice.LLM_ENDPOINT="http://your-server-url" --set llm-uservice.LLM_MODEL_ID="your-model" ``` ## Verify diff --git a/helm-charts/docsum/values.yaml b/helm-charts/docsum/values.yaml index 0a931152a..7a0b9273a 100644 --- a/helm-charts/docsum/values.yaml +++ b/helm-charts/docsum/values.yaml @@ -57,6 +57,12 @@ tolerations: [] affinity: {} # To override values in subchart llm-uservice +# For external LLM configuration, you can override these values here or use variant_external-llm-values.yaml +# Example for external OpenAI-compatible endpoints: +# DOCSUM_BACKEND: "vLLM" # Use vLLM backend for OpenAI-compatible APIs +# LLM_ENDPOINT: "https://api.openai.com" # External endpoint (omit /v1 suffix) +# LLM_MODEL_ID: "gpt-4-turbo" # Model to use +# OPENAI_API_KEY: "your-api-key" # API key for authentication llm-uservice: enabled: true image: @@ -110,14 +116,6 @@ dashboard: whisper: enabled: true -# External LLM configuration -externalLLM: - enabled: false - LLM_SERVER_HOST: "http://your-llm-server" - LLM_SERVER_PORT: "80" - LLM_MODEL: "your-model" - OPENAI_API_KEY: "your-api-key" - global: http_proxy: "" https_proxy: "" diff --git a/helm-charts/docsum/variant_external-llm-values.yaml b/helm-charts/docsum/variant_external-llm-values.yaml index c2f292f40..34a4f99a3 100644 --- a/helm-charts/docsum/variant_external-llm-values.yaml +++ b/helm-charts/docsum/variant_external-llm-values.yaml @@ -5,12 +5,10 @@ # This keeps the llm-uservice wrapper (required for /v1/docsum endpoint) but connects it to external LLMs llm-uservice: enabled: true # Keep the wrapper service for DocSum compatibility - env: - # Configure llm-uservice to use external OpenAI-compatible endpoints - LLM_ENDPOINT: "https://api.openai.com/v1" # External LLM API endpoint (OpenAI, vLLM, TGI, etc.) - OPENAI_API_KEY: "${OPENAI_API_KEY}" # API key for authentication - LLM_MODEL_ID: "gpt-4-turbo" # Model to use - TEXTGEN_BACKEND: "openai" # Backend type for OpenAI-compatible endpoints + DOCSUM_BACKEND: "vLLM" # Use vLLM backend for OpenAI-compatible APIs + LLM_ENDPOINT: "https://api.openai.com" # External LLM API endpoint (omit /v1 suffix) + OPENAI_API_KEY: "${OPENAI_API_KEY}" # API key for authentication + LLM_MODEL_ID: "gpt-4-turbo" # Model to use # Disable local inference services since we're using external LLMs vllm: