From c3f14f322d39da85905111ce831b37fc5e7e71fa Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Fri, 8 Aug 2025 15:50:40 -0600 Subject: [PATCH 1/8] fix: fixes EROFS from Yarn + Node 22 --- sdk-scheme-adapter/chart-service/templates/deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdk-scheme-adapter/chart-service/templates/deployment.yaml b/sdk-scheme-adapter/chart-service/templates/deployment.yaml index 444df5ca3..d00c65033 100644 --- a/sdk-scheme-adapter/chart-service/templates/deployment.yaml +++ b/sdk-scheme-adapter/chart-service/templates/deployment.yaml @@ -69,6 +69,8 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | nindent 8 }} {{- end }} volumes: + - name: writable-tmp + emptyDir: {} {{- if .Values.env.JWS_SIGN }} - name: jws-private-key secret: @@ -146,6 +148,8 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.livenessProbe "context" $) | nindent 12 }} {{- end }} env: + - name: TMPDIR + value: /writable-tmp {{- if not (hasKey .Values.env "BACKEND_ENDPOINT") }} - name: BACKEND_ENDPOINT value: {{ printf "%s-backend:3000" (include "common.names.fullname" . ) }} @@ -177,3 +181,5 @@ spec: - name: jws-public-keys mountPath: "/jwsVerificationKeys/" {{- end }} + - name: writable-tmp + mountPath: /writable-tmp From ac051e8eeabbaf62eea69e5ec214f0dfe348d951 Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Mon, 11 Aug 2025 21:01:56 -0600 Subject: [PATCH 2/8] fix: ml-testing-toolkit backend config files --- ml-testing-toolkit/chart-backend/templates/statefulset.yaml | 5 +---- ml-testing-toolkit/chart-backend/values.yaml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index dae753215..d019db32b 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -127,11 +127,8 @@ spec: mountPath: /opt/app/examples/environments/{{ $k }} subPath: {{ $k }} {{- end }} - {{- range $k, $v := .Values.config_files }} - name: {{ $serviceFullName }}-volume-config-default - mountPath: /opt/app/spec_files/{{ $k | replace "__" "/" }} - subPath: {{ $k }} - {{- end }} + mountPath: /opt/default_config - name: release-cd-rc mountPath: /home/ml-user/.release_cd volumes: diff --git a/ml-testing-toolkit/chart-backend/values.yaml b/ml-testing-toolkit/chart-backend/values.yaml index d122d065a..4c7f36755 100644 --- a/ml-testing-toolkit/chart-backend/values.yaml +++ b/ml-testing-toolkit/chart-backend/values.yaml @@ -28,7 +28,7 @@ replicaCount: 1 # .Values.persistence.enabled = true commandPersistence: '["sh", "-c", "if [ -d /opt/ttk-data ]; then if [ ! -d /opt/ttk-data/spec_files ]; then cp -pR spec_files /opt/ttk-data/spec_files; fi; mv spec_files spec_files_bkp; ln -s /opt/ttk-data/spec_files spec_files; fi; npm run start;"]' # .Values.persistence.enabled = false -command: '["sh", "-c", "npm run start;"]' +command: '["sh", "-c", "cd /opt/default_config; for FILE in *; do newFileName=/opt/app/spec_files/${FILE//__/\\/}; mkdir -p ${newFileName%/*}; firstLine=$(head -n 1 $FILE | tr -d \\\"); regex=\"^(http|https|ftp)://\"; if [[ \"$firstLine\" =~ $regex ]]; then wget $firstLine -O $newFileName; else cp $FILE $newFileName; fi; done; cd /opt/app; npm run start;"]' ## Pod scheduling preferences. ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity From 11c137579ecf4688745b380eff7939508af8596a Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Tue, 12 Aug 2025 16:54:17 -0600 Subject: [PATCH 3/8] fix: ml-testing-toolkit backend config files --- .../chart-backend/templates/statefulset.yaml | 2 ++ ml-testing-toolkit/chart-backend/values.yaml | 31 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index d019db32b..85bc32dae 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -156,6 +156,8 @@ spec: configMap: name: release-cd-rc optional: true + - name: spec-files + emptyDir: {} {{- if .Values.persistence.enabled }} volumeClaimTemplates: diff --git a/ml-testing-toolkit/chart-backend/values.yaml b/ml-testing-toolkit/chart-backend/values.yaml index 4c7f36755..1bce2f600 100644 --- a/ml-testing-toolkit/chart-backend/values.yaml +++ b/ml-testing-toolkit/chart-backend/values.yaml @@ -115,7 +115,7 @@ parameters: {} ## imagePullPolicy: Always ## command: ['sh', '-c', 'echo "hello world"'] ## -initContainers: [] +initContainers: ## Use the following initContainers if you are using a MongoDB as a dependency # initContainers: | # - name: wait-for-mongodb @@ -152,6 +152,35 @@ initContainers: [] # - name: DB_DATABASE # value: '{{ .Values.config.mongodb.database }}' + - name: add-extra-specs + image: busybox:1.36 + command: + [ + "sh", + "-c", + " + set -e; + cd /opt/default_config; + for FILE in *; do + CONTENT=$(cat \"$FILE\"); + DEST=\"/opt/app/spec_files/${FILE//__/\/}\"; + mkdir -p \"$(dirname \"$DEST\")\"; + if echo \"$CONTENT\" | grep -q '^\"http'; then + URL=$(echo \"$CONTENT\" | tr -d '\"'); + echo \"Downloading $URL to $DEST\"; + wget -q -O \"$DEST\" \"$URL\"; + else + cp \"$FILE\" \"$DEST\"; + fi + done + " + ] + volumeMounts: + - name: spec-files + mountPath: /opt/app/spec_files + - name: ml-testing-toolkit-backend-volume-config-default + mountPath: /opt/default_config + ## @param master.podLabels Extra labels for pod(s) ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## From 32409bad671d4f705562d580613e3030d81ae992 Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Tue, 12 Aug 2025 17:25:44 -0600 Subject: [PATCH 4/8] fix: ml-testing-toolkit backend config files --- ml-testing-toolkit/chart-backend/values.yaml | 39 +++++++++----------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/ml-testing-toolkit/chart-backend/values.yaml b/ml-testing-toolkit/chart-backend/values.yaml index 1bce2f600..34db6f0f0 100644 --- a/ml-testing-toolkit/chart-backend/values.yaml +++ b/ml-testing-toolkit/chart-backend/values.yaml @@ -28,7 +28,7 @@ replicaCount: 1 # .Values.persistence.enabled = true commandPersistence: '["sh", "-c", "if [ -d /opt/ttk-data ]; then if [ ! -d /opt/ttk-data/spec_files ]; then cp -pR spec_files /opt/ttk-data/spec_files; fi; mv spec_files spec_files_bkp; ln -s /opt/ttk-data/spec_files spec_files; fi; npm run start;"]' # .Values.persistence.enabled = false -command: '["sh", "-c", "cd /opt/default_config; for FILE in *; do newFileName=/opt/app/spec_files/${FILE//__/\\/}; mkdir -p ${newFileName%/*}; firstLine=$(head -n 1 $FILE | tr -d \\\"); regex=\"^(http|https|ftp)://\"; if [[ \"$firstLine\" =~ $regex ]]; then wget $firstLine -O $newFileName; else cp $FILE $newFileName; fi; done; cd /opt/app; npm run start;"]' +command: '["sh", "-c", "npm run start;"]' ## Pod scheduling preferences. ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity @@ -155,26 +155,23 @@ initContainers: - name: add-extra-specs image: busybox:1.36 command: - [ - "sh", - "-c", - " - set -e; - cd /opt/default_config; - for FILE in *; do - CONTENT=$(cat \"$FILE\"); - DEST=\"/opt/app/spec_files/${FILE//__/\/}\"; - mkdir -p \"$(dirname \"$DEST\")\"; - if echo \"$CONTENT\" | grep -q '^\"http'; then - URL=$(echo \"$CONTENT\" | tr -d '\"'); - echo \"Downloading $URL to $DEST\"; - wget -q -O \"$DEST\" \"$URL\"; - else - cp \"$FILE\" \"$DEST\"; - fi - done - " - ] + - sh + - -c + - | + set -e + cd /opt/default_config + for FILE in *; do + CONTENT=$(cat "$FILE") + DEST="/opt/app/spec_files/${FILE//__/\/}" + mkdir -p "$(dirname "$DEST")" + if echo "$CONTENT" | grep -q '^"http'; then + URL=$(echo "$CONTENT" | tr -d '"') + echo "Downloading $URL to $DEST" + wget -q -O "$DEST" "$URL" + else + cp "$FILE" "$DEST" + fi + done volumeMounts: - name: spec-files mountPath: /opt/app/spec_files From 69743133ef86b3688f36ca2bf86a177be56f6147 Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Tue, 12 Aug 2025 21:34:35 -0600 Subject: [PATCH 5/8] fix: ml-testing-toolkit backend config files --- .../chart-backend/templates/statefulset.yaml | 47 ++++++++++++++++++- ml-testing-toolkit/chart-backend/values.yaml | 28 +---------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index 85bc32dae..83b4d597d 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -63,6 +63,48 @@ spec: {{- if .Values.initContainers }} {{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | nindent 8 }} {{- end }} + - name: copy-original-files + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - | + cp -r /opt/app/spec_files/* /opt/app/spec_files/.* /target/ 2>/dev/null || true + volumeMounts: + - name: spec-files + mountPath: /target + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }} + - name: add-extra-specs + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - | + set -e + cd /opt/default_config + for FILE in *; do + CONTENT=$(cat "$FILE") + DEST="/opt/app/spec_files/${FILE//__/\/}" + mkdir -p "$(dirname "$DEST")" + if echo "$CONTENT" | grep -q '^"http'; then + URL=$(echo "$CONTENT" | tr -d '"') + echo "Downloading $URL to $DEST" + wget -q -O "$DEST" "$URL" + else + echo "Copying $FILE to $DEST" + cp "$FILE" "$DEST" + fi + done + volumeMounts: + - name: spec-files + mountPath: /opt/app/spec_files + - name: {{ include "ml-testing-toolkit-backend.fullname" . }}-volume-config-default + mountPath: /opt/default_config {{- if .Values.image.pullSecrets }} {{- include "common.images.renderPullSecrets" (dict "images" (list .Values.image) "context" $) | nindent 6 }} {{- end }} @@ -127,10 +169,11 @@ spec: mountPath: /opt/app/examples/environments/{{ $k }} subPath: {{ $k }} {{- end }} - - name: {{ $serviceFullName }}-volume-config-default - mountPath: /opt/default_config - name: release-cd-rc mountPath: /home/ml-user/.release_cd + - name: spec-files # Added to mount merged files + mountPath: /opt/app/spec_files + readOnly: true volumes: {{- if .Values.persistence.enabled }} - name: ttk-data diff --git a/ml-testing-toolkit/chart-backend/values.yaml b/ml-testing-toolkit/chart-backend/values.yaml index 34db6f0f0..d122d065a 100644 --- a/ml-testing-toolkit/chart-backend/values.yaml +++ b/ml-testing-toolkit/chart-backend/values.yaml @@ -115,7 +115,7 @@ parameters: {} ## imagePullPolicy: Always ## command: ['sh', '-c', 'echo "hello world"'] ## -initContainers: +initContainers: [] ## Use the following initContainers if you are using a MongoDB as a dependency # initContainers: | # - name: wait-for-mongodb @@ -152,32 +152,6 @@ initContainers: # - name: DB_DATABASE # value: '{{ .Values.config.mongodb.database }}' - - name: add-extra-specs - image: busybox:1.36 - command: - - sh - - -c - - | - set -e - cd /opt/default_config - for FILE in *; do - CONTENT=$(cat "$FILE") - DEST="/opt/app/spec_files/${FILE//__/\/}" - mkdir -p "$(dirname "$DEST")" - if echo "$CONTENT" | grep -q '^"http'; then - URL=$(echo "$CONTENT" | tr -d '"') - echo "Downloading $URL to $DEST" - wget -q -O "$DEST" "$URL" - else - cp "$FILE" "$DEST" - fi - done - volumeMounts: - - name: spec-files - mountPath: /opt/app/spec_files - - name: ml-testing-toolkit-backend-volume-config-default - mountPath: /opt/default_config - ## @param master.podLabels Extra labels for pod(s) ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ ## From 7faecaa17af7bec54095848dae267e7511e1ff2a Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Tue, 12 Aug 2025 22:09:19 -0600 Subject: [PATCH 6/8] fix: ml-testing-toolkit backend config files --- ml-testing-toolkit/chart-backend/templates/statefulset.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index 83b4d597d..1c570b980 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -77,7 +77,6 @@ spec: resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File - imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }} - name: add-extra-specs image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} From 19a4ae7ab8d949d063ca85f0199c0714e138f20b Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Wed, 13 Aug 2025 19:26:25 -0600 Subject: [PATCH 7/8] fix: ml-testing-toolkit backend config files --- .../chart-backend/templates/statefulset.yaml | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index ab744f8dc..dcd74dcdb 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -70,7 +70,15 @@ spec: - sh - -c - | - cp -r /opt/app/spec_files/* /opt/app/spec_files/.* /target/ 2>/dev/null || true + set -e + echo "Copying files to /tmp/stage" + mkdir -p /tmp/stage + cp -r -v /opt/app/spec_files/. /tmp/stage/ || { echo "Copy to /tmp/stage failed"; exit 1; } + echo "Setting permissions in /tmp/stage:" + chmod -R 644 /tmp/stage/* 2>/dev/null || true + find /tmp/stage -type d -exec chmod 755 {} \; 2>/dev/null || true + echo "Copying from /tmp/stage to /target" + cp -r -v /tmp/stage/. /target/ || { echo "Copy to /target failed"; exit 1; } volumeMounts: - name: spec-files mountPath: /target @@ -86,17 +94,24 @@ spec: - | set -e cd /opt/default_config + echo "Listing ConfigMap files:" + ls -l for FILE in *; do CONTENT=$(cat "$FILE") DEST="/opt/app/spec_files/${FILE//__/\/}" mkdir -p "$(dirname "$DEST")" - if echo "$CONTENT" | grep -q '^"http'; then - URL=$(echo "$CONTENT" | tr -d '"') - echo "Downloading $URL to $DEST" - wget -q -O "$DEST" "$URL" + if [ ! -f "$DEST" ]; then + if echo "$CONTENT" | grep -q '^"http'; then + URL=$(echo "$CONTENT" | tr -d '"') + echo "Downloading $URL to $DEST" + wget -q --tries=3 --timeout=10 -O "$DEST" "$URL" || echo "Failed to download $URL" + else + echo "Copying $FILE to $DEST" + cp "$FILE" "$DEST" + fi + chmod 644 "$DEST" 2>/dev/null || true else - echo "Copying $FILE to $DEST" - cp "$FILE" "$DEST" + echo "Skipping $FILE, already exists at $DEST" fi done volumeMounts: From 6cf5fa4508f58258421476ed5bfcc097a30991a4 Mon Sep 17 00:00:00 2001 From: Shashikant Hirugade Date: Wed, 13 Aug 2025 20:01:47 -0600 Subject: [PATCH 8/8] fix: ml-testing-toolkit backend config files --- .../chart-backend/templates/statefulset.yaml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml index dcd74dcdb..039ed34b5 100644 --- a/ml-testing-toolkit/chart-backend/templates/statefulset.yaml +++ b/ml-testing-toolkit/chart-backend/templates/statefulset.yaml @@ -94,24 +94,17 @@ spec: - | set -e cd /opt/default_config - echo "Listing ConfigMap files:" - ls -l for FILE in *; do CONTENT=$(cat "$FILE") DEST="/opt/app/spec_files/${FILE//__/\/}" mkdir -p "$(dirname "$DEST")" - if [ ! -f "$DEST" ]; then - if echo "$CONTENT" | grep -q '^"http'; then - URL=$(echo "$CONTENT" | tr -d '"') - echo "Downloading $URL to $DEST" - wget -q --tries=3 --timeout=10 -O "$DEST" "$URL" || echo "Failed to download $URL" - else - echo "Copying $FILE to $DEST" - cp "$FILE" "$DEST" - fi - chmod 644 "$DEST" 2>/dev/null || true + if echo "$CONTENT" | grep -q '^"http'; then + URL=$(echo "$CONTENT" | tr -d '"') + echo "Downloading $URL to $DEST" + wget -q -O "$DEST" "$URL" else - echo "Skipping $FILE, already exists at $DEST" + echo "Copying $FILE to $DEST" + cp "$FILE" "$DEST" fi done volumeMounts: