test: test deepspeed ssh key

Author: amirialy

charts/kubeflow-trainer/templates/runtimes/deepspeed-distributed.yaml

@@ -44,11 +44,14 @@ spec:
             spec:
               template:
                 spec:
+                  securityContext:
+                    runAsUser: 1000
+                    runAsGroup: 1000
+                    fsGroup: 1000
+
                   containers:
                     - name: node
                       image: {{ include "trainer.runtimeImage" (list .Values.runtimes.deepspeedDistributed.image .) }}
-                      securityContext:
-                        runAsUser: 1000
                       command:
                         - /bin/bash
                         - -c
@@ -69,8 +72,14 @@ spec:
                             echo "===== PRIVATE KEY INFO ====="
                             openssl pkey -in /home/mpiuser/.ssh/id_rsa -noout -text || true
 
-                            echo "===== FILE PERMISSIONS ====="
-                            stat /home/mpiuser/.ssh/* || true
+                            echo "===== REAL SECRET FILES ====="
+                            ls -lna /home/mpiuser/.ssh/..data || true
+
+                            echo "===== ID_RSA ====="
+                            ls -lna /home/mpiuser/.ssh/..data/id_rsa || true
+
+                            echo "===== NAMEI ====="
+                            namei -l /home/mpiuser/.ssh/id_rsa || true
 
                             echo "===== SSHD DEBUG ====="
                             exec /usr/sbin/sshd -ddd -f /home/mpiuser/.sshd_config
@@ -91,9 +100,12 @@ spec:
             spec:
               template:
                 spec:
+                  securityContext:
+                    runAsUser: 1000
+                    runAsGroup: 1000
+                    fsGroup: 1000
+
                   containers:
                     - name: node
                       image: {{ include "trainer.runtimeImage" (list .Values.runtimes.deepspeedDistributed.image .) }}
-                      securityContext:
-                        runAsUser: 1000
 {{- end }}

manifests/base/runtimes/deepspeed_distributed.yaml

@@ -26,11 +26,14 @@ spec:
             spec:
               template:
                 spec:
+                  securityContext:
+                    runAsUser: 1000
+                    runAsGroup: 1000
+                    fsGroup: 1000
+
                   containers:
                     - name: node
                       image: ghcr.io/kubeflow/trainer/deepspeed-runtime
-                      securityContext:
-                        runAsUser: 1000
                       command:
                         - /bin/bash
                         - -c
@@ -51,8 +54,14 @@ spec:
                             echo "===== PRIVATE KEY INFO ====="
                             openssl pkey -in /home/mpiuser/.ssh/id_rsa -noout -text || true
 
-                            echo "===== FILE PERMISSIONS ====="
-                            stat /home/mpiuser/.ssh/* || true
+                            echo "===== REAL SECRET FILES ====="
+                            ls -lna /home/mpiuser/.ssh/..data || true
+
+                            echo "===== ID_RSA ====="
+                            ls -lna /home/mpiuser/.ssh/..data/id_rsa || true
+
+                            echo "===== NAMEI ====="
+                            namei -l /home/mpiuser/.ssh/id_rsa || true
 
                             echo "===== SSHD DEBUG ====="
                             exec /usr/sbin/sshd -ddd -f /home/mpiuser/.sshd_config
@@ -73,8 +82,11 @@ spec:
             spec:
               template:
                 spec:
+                  securityContext:
+                    runAsUser: 1000
+                    runAsGroup: 1000
+                    fsGroup: 1000
+
                   containers:
                     - name: node
                       image: ghcr.io/kubeflow/trainer/deepspeed-runtime
-                      securityContext:
-                        runAsUser: 1000