Startup time limits and error propagation

.github/workflows/startup-fault-tests.yml

@@ -0,0 +1,238 @@
+name: startup-fault-tests
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: read
+
+concurrency:
+  group: startup-fault-tests-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  startup-fault-serial-ubuntu_x86:
+    name: Startup fault tests (Serial, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@54042514f505b273907334ae2b9cdbb9a0213c1a
+        with:
+          version: latest
+
+      - name: Build spindle-serial-ubuntu image
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose --progress=plain build
+
+      - name: Bring spindle-serial-ubuntu up
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose up -d
+
+      - name: Verify munge works in spindle-serial-ubuntu
+        run: |
+          docker exec spindlenode bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-serial-ubuntu startup fault tests
+        run: |
+          docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_startup_fault_tests.sh --launcher=serial'
+
+      - name: Bring spindle-serial-ubuntu down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose down
+
+  startup-fault-serial-ubuntu_arm64:
+    name: Startup fault tests (Serial, Ubuntu, arm64)
+    environment: Spindle CI
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 35
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@54042514f505b273907334ae2b9cdbb9a0213c1a
+        with:
+          version: latest
+
+      - name: Build spindle-serial-ubuntu image
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose --progress=plain build
+
+      - name: Bring spindle-serial-ubuntu up
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose up -d
+
+      - name: Verify munge works in spindle-serial-ubuntu
+        run: |
+          docker exec spindlenode bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-serial-ubuntu startup fault tests
+        run: |
+          docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_startup_fault_tests.sh --launcher=serial'
+
+      - name: Bring spindle-serial-ubuntu down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-serial-ubuntu
+          docker compose down
+
+  startup-fault-flux-ubuntu:
+    name: Startup fault tests (Flux, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@54042514f505b273907334ae2b9cdbb9a0213c1a
+        with:
+          version: latest
+
+      - name: Build spindle-flux-ubuntu image
+        run: |
+          cd containers/spindle-flux-ubuntu
+          docker compose --progress=plain build
+
+      - name: Bring spindle-flux-ubuntu up
+        run: |
+          cd containers/spindle-flux-ubuntu
+          docker compose up -d --wait --wait-timeout 60
+
+      - name: Verify munge works in spindle-flux-ubuntu
+        run: |
+          docker exec node-1 bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-flux-ubuntu startup fault tests
+        run: |
+          docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_startup_fault_tests.sh --launcher=flux --nodes=${workers}'
+
+      - name: Run spindle-flux-ubuntu session startup fault tests
+        run: |
+          docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_startup_fault_tests.sh --launcher=flux --nodes=${workers} --session'
+
+      - name: Bring spindle-flux-ubuntu down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-flux-ubuntu
+          docker compose down
+
+  startup-fault-slurm-srun-ubuntu:
+    name: Startup fault tests (Slurm srun, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@54042514f505b273907334ae2b9cdbb9a0213c1a
+        with:
+          version: latest
+
+      - name: Login to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate MariaDB configuration
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-srun
+          ./generate_config.sh
+
+      - name: Build spindle-slurm-srun-ubuntu image
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-srun
+          docker compose --progress=plain build
+
+      - name: Bring spindle-slurm-srun-ubuntu up
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-srun
+          docker compose up -d --wait --wait-timeout 120
+
+      - name: Verify munge works in spindle-slurm-srun-ubuntu
+        run: |
+          docker exec slurm-srun-head bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-slurm-srun-ubuntu startup fault tests
+        run: |
+          docker exec slurm-srun-head bash -c 'cd Spindle-build/testsuite && ./run_startup_fault_tests.sh --launcher=slurm --nodes=${workers}'
+
+      - name: Bring spindle-slurm-srun-ubuntu down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-srun
+          docker compose down
+
+  startup-fault-slurm-plugin-ubuntu:
+    name: Startup fault tests (Slurm Plugin, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@54042514f505b273907334ae2b9cdbb9a0213c1a
+        with:
+          version: latest
+
+      - name: Login to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate MariaDB configuration
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          ./generate_config.sh
+
+      - name: Build spindle-slurm-plugin-ubuntu image
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose --progress=plain build
+
+      - name: Bring spindle-slurm-plugin-ubuntu up
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose up -d --wait --wait-timeout 120
+
+      - name: Verify munge works in spindle-slurm-plugin-ubuntu
+        run: |
+          docker exec slurm-plugin-head bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-slurm-plugin-ubuntu startup fault tests
+        run: |
+          docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_startup_fault_tests.sh --launcher=slurm-plugin --nodes=${workers}'
+
+      - name: Bring spindle-slurm-plugin-ubuntu down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing-plugin
+          docker compose down

config.h.in

@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Default startup and connection time limit in seconds */
+#undef COBO_CONNECT_TIMELIMIT
+
 /* Colon-separated list of candidate paths for back-end communication and
    housekeeping */
 #undef COMMPATHS
@@ -157,6 +160,10 @@
 /* The default port for Spindle */
 #undef SPINDLE_PORT
 
+/* Default time limit in seconds for clients to wait for the server ready file
+   */
+#undef SPINDLE_READY_TIMELIMIT
+
 /* Defined if GLIBC has static TLS allocation bug */
 #undef STATIC_TLS_ALLOC_BUG
 

configure

@@ -853,6 +853,8 @@ with_cachepath
 with_commpaths
 with_commpath
 with_default_local_prefix
+with_default_connect_timelimit
+with_default_ready_timelimit
 with_testrm
 with_rm
 enable_slurm
@@ -1607,6 +1609,12 @@ Optional Packages:
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
+  --with-default-connect-timelimit=SECS
+                          Time limit for Spindle startup connections; default
+                          for COBO_CONNECT_TIMELIMIT
+  --with-default-ready-timelimit=SECS
+                          Time limit for applications to wait for the Spindle
+                          server to start; default for SPINDLE_READY_TIMELIMIT
   --with-testrm=ARG
   --with-rm=ARG           Default resource manager (valid options are 'slurm',
                           'slurm-plugin', 'flux', and 'serial')
@@ -16629,6 +16637,7 @@ SPINDLE_SOURCE_ROOT=`realpath ${srcdir}`
 DEFAULT_PORT=21940
 DEFAULT_LOC='$TMPDIR'
 DEFAULT_NUM_COBO_PORTS=250
+DEFAULT_CONNECT_TIMELIMIT=60
 DEFAULT_LOCAL_PREFIX='/var/:/tmp/:/proc/:/dev/:/sys/'
 
 
@@ -16717,6 +16726,45 @@ else
 fi
 
 
+# Check whether --with-default-connect-timelimit was given.
+if test "${with_default_connect_timelimit+set}" = set; then :
+  withval=$with_default_connect_timelimit; COBO_CONNECT_TIMELIMIT=${withval}
+else
+  COBO_CONNECT_TIMELIMIT=$DEFAULT_CONNECT_TIMELIMIT
+fi
+
+
+# Check whether --with-default-ready-timelimit was given.
+if test "${with_default_ready_timelimit+set}" = set; then :
+  withval=$with_default_ready_timelimit; SPINDLE_READY_TIMELIMIT=${withval}
+else
+  SPINDLE_READY_TIMELIMIT=
+fi
+
+case $COBO_CONNECT_TIMELIMIT in #(
+  ''|*[!0-9]*|0*) :
+    as_fn_error $? "--with-default-connect-timelimit must be a positive integer number of seconds" "$LINENO" 5 ;; #(
+  *) :
+     ;;
+esac
+case $SPINDLE_READY_TIMELIMIT in #(
+  '') :
+     ;; #(
+  *[!0-9]*|0*) :
+    as_fn_error $? "--with-default-ready-timelimit must be a positive integer number of seconds" "$LINENO" 5 ;; #(
+  *) :
+
+cat >>confdefs.h <<_ACEOF
+#define SPINDLE_READY_TIMELIMIT $SPINDLE_READY_TIMELIMIT
+_ACEOF
+ ;;
+esac
+
+cat >>confdefs.h <<_ACEOF
+#define COBO_CONNECT_TIMELIMIT $COBO_CONNECT_TIMELIMIT
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_PORT $SPINDLE_PORT
 _ACEOF

configure.common.ac

@@ -3,6 +3,7 @@
 DEFAULT_PORT=21940
 DEFAULT_LOC='$TMPDIR'
 DEFAULT_NUM_COBO_PORTS=250
+DEFAULT_CONNECT_TIMELIMIT=60
 DEFAULT_LOCAL_PREFIX='/var/:/tmp/:/proc/:/dev/:/sys/'
 
 AM_MAINTAINER_MODE([disable])
@@ -41,6 +42,21 @@ AC_ARG_WITH(default-local-prefix,
             [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])],
             [SPINDLE_LOCAL_PREFIX=${withval}],
             [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATHS"])
+AC_ARG_WITH(default-connect-timelimit,
+            [AS_HELP_STRING([--with-default-connect-timelimit=SECS],[Time limit for Spindle startup connections; default for COBO_CONNECT_TIMELIMIT])],
+            [COBO_CONNECT_TIMELIMIT=${withval}],
+            [COBO_CONNECT_TIMELIMIT=$DEFAULT_CONNECT_TIMELIMIT])
+AC_ARG_WITH(default-ready-timelimit,
+            [AS_HELP_STRING([--with-default-ready-timelimit=SECS],[Time limit for applications to wait for the Spindle server to start; default for SPINDLE_READY_TIMELIMIT])],
+            [SPINDLE_READY_TIMELIMIT=${withval}],
+            [SPINDLE_READY_TIMELIMIT=])
+AS_CASE([$COBO_CONNECT_TIMELIMIT],
+        [''|*[[!0-9]]*|0*], [AC_MSG_ERROR([--with-default-connect-timelimit must be a positive integer number of seconds])])
+AS_CASE([$SPINDLE_READY_TIMELIMIT],
+        [''], [],
+        [*[[!0-9]]*|0*], [AC_MSG_ERROR([--with-default-ready-timelimit must be a positive integer number of seconds])],
+        [AC_DEFINE_UNQUOTED([SPINDLE_READY_TIMELIMIT],[$SPINDLE_READY_TIMELIMIT],[Default time limit in seconds for clients to wait for the server ready file])])
+AC_DEFINE_UNQUOTED([COBO_CONNECT_TIMELIMIT],[$COBO_CONNECT_TIMELIMIT],[Default startup and connection time limit in seconds])
 AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle])
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])

src/client/beboot/spindle_bootstrap.c

@@ -78,7 +78,10 @@ static int establish_connection()
       return -1;
 
    send_pid(ldcsid);
-   send_rankinfo_query(ldcsid, &rankinfo[0], &rankinfo[1], &rankinfo[2], &rankinfo[3]);      
+   if (send_rankinfo_query(ldcsid, &rankinfo[0], &rankinfo[1], &rankinfo[2], &rankinfo[3]) == -1) {
+      err_printf("Failed to get rank info from server\n");
+      return -1;
+   }
    if (opts & OPT_NUMA)
       send_cpu(ldcsid, get_cur_cpu());
 
@@ -406,7 +409,10 @@ int main(int argc, char *argv[])
    /**
     * Exec the user's application.
     **/
-   setup_environment();
+   if (setup_environment() != 0) {
+      err_printf("spindle_bootstrap could not get the cache path from the server\n");
+      return -1;
+   }
    execvp(executable, cmdline);