Release candidate: v1.93.0 (GoogleCloudPlatform#5759)

Author: sudheer-quad

cmd/job/submit.go

@@ -111,6 +111,8 @@ func init() {
 	SubmitCmd.Flags().StringVarP(&dryRunManifest, "dry-run-out", "o", "", "Path to output the generated Kubernetes manifest instead of applying it.")
 	SubmitCmd.Flags().StringVarP(&platform, "platform", "f", "linux/amd64", "Target platform for the image build (e.g., 'linux/amd64', 'linux/arm64'). Used with --base-image.")
 
+	SubmitCmd.Flags().StringSliceVar(&volumeStr, "mount", nil, "Volumes to mount (format: <src>:<dest>[:<mode>], mode can be 'ro' or 'rw', default 'ro').")
+
 	SubmitCmd.Flags().StringVarP(&workloadName, "name", "n", "", "Name of the workload to create. Required.")
 	SubmitCmd.Flags().StringVarP(&kueueQueueName, "queue", "q", "", "Name of the Kueue LocalQueue to submit the workload to. If empty, it will be auto-discovered.")
 	SubmitCmd.Flags().IntVar(&numNodes, "num-nodes", 1, "The number of nodes to use per group/slice. Defaults to 1 for CPU/GPU, or auto-calculated for TPUs.")
@@ -131,9 +133,9 @@ func init() {
 	SubmitCmd.Flags().BoolVar(&awaitJobCompletion, "await-job-completion", false, "If true, gcluster will wait for the submitted job to complete.")
 	SubmitCmd.Flags().StringVar(&timeoutStr, "timeout", "-1s", "Time to wait for job in seconds or string format (e.g. 1h, 10m). Default is max timeout (-1s).")
 	SubmitCmd.Flags().StringVar(&priorityClassName, "priority", "medium", "A priority, one of `very-low`, `low`, `medium`, `high` or `very-high`. Defaults to `medium`.")
+	SubmitCmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose logging for the workload (TPUs and GPUs).")
 
 	SubmitCmd.Flags().BoolVar(&isPathwaysJob, "pathways", false, "If present, gcluster will generate a manifest for a Pathways job.")
-	SubmitCmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose logging for the workload (TPUs and GPUs).")
 	SubmitCmd.Flags().StringVar(&pathways.ProxyServerImage, "pathways-proxy-server-image", "", "The image for the Pathways proxy server.")
 	SubmitCmd.Flags().StringVar(&pathways.ServerImage, "pathways-server-image", "", "The image for the Pathways server.")
 	SubmitCmd.Flags().StringVar(&pathways.WorkerImage, "pathways-worker-image", "", "The image for the Pathways worker.")
@@ -147,8 +149,6 @@ func init() {
 	SubmitCmd.Flags().StringVar(&pathways.ColocatedPythonSidecarImage, "pathways-colocated-python-sidecar-image", "", "Image for an optional Python-based sidecar container to run alongside the Pathways head components.")
 	SubmitCmd.Flags().StringVar(&pathways.HeadNodePool, "pathways-head-np", "", "The node pool to use for the Pathways head job. If empty, it will be auto-detected (looking for 'cpu-np' or 'pathways-np').")
 
-	SubmitCmd.Flags().StringSliceVar(&volumeStr, "mount", nil, "Volumes to mount (format: <src>:<dest>[:<mode>], mode can be 'ro' or 'rw', default 'ro').")
-
 	_ = SubmitCmd.MarkFlagRequired("command")
 	_ = SubmitCmd.MarkFlagRequired("name")
 	_ = SubmitCmd.MarkFlagRequired("compute-type")

community/examples/AMD/hpc-amd-slurm.yaml

@@ -171,7 +171,9 @@ deployment_groups:
         # these images must match the images used by Slurm modules below because
         # we are building OpenMPI with PMI support in libraries contained in
         # Slurm installation
-        family: slurm-gcp-6-12-hpc-rocky-linux-8
+        # Rocky 8: slurm-gcp-6-12-hpc-rocky-linux-8
+        # Rocky 9: slurm-gcp-6-12-hpc-rocky-linux-9
+        family: slurm-gcp-6-12-hpc-rocky-linux-9
         project: schedmd-slurm-public
 
   - id: low_cost_nodeset

community/examples/hpc-build-slurm-image.yaml

@@ -24,7 +24,9 @@ vars:
   zone: us-central1-a
 
   image_build_machine_type: n2d-standard-16
-  build_from_image_family: hpc-rocky-linux-8
+  # Rocky 8: hpc-rocky-linux-8
+  # Rocky 9: hpc-rocky-linux-9
+  build_from_image_family: hpc-rocky-linux-9
   build_from_image_project: cloud-hpc-image-public
   build_from_git_ref: 6.12.1
   built_image_family: my-custom-slurm
@@ -49,15 +51,17 @@ deployment_groups:
         content: |
           #!/bin/bash
           set -e -o pipefail
-          # Slurm build on Rocky8 will upgrade to python3.12 as part of build
-          dnf install -y python3.12 python3.12-pip
-          alternatives --set python3 /usr/bin/python3.12
-          python3 -m pip install pip --upgrade
-          python3 -m pip install ansible==8.7.0
-          python3 -m pip install selinux
+          # Slurm build will install and set python3.12 as the default python3
+          dnf install -y python3.12 python3.12-pip perl-ExtUtils-MakeMaker perl-devel
+          python3.12 -m pip install pip --upgrade
+          python3.12 -m pip install ansible==8.7.0
           export PATH=/usr/local/bin:$PATH
           ansible --version
-          ansible-galaxy role install googlecloudplatform.google_cloud_ops_agents
+          n=0
+          until [ "$n" -ge 3 ] || ansible-galaxy role install googlecloudplatform.google_cloud_ops_agents; do
+            ((n=n+1))
+            sleep 5
+          done
       - type: data
         destination: /var/tmp/slurm_vars.json
         content: |
@@ -67,7 +71,8 @@ deployment_groups:
             "nvidia_version": "latest",
             "install_ompi": true,
             "install_lustre": false,
-            "install_gcsfuse": true
+            "install_gcsfuse": true,
+            "monitoring_agent": "cloud-ops"
           }
       - type: shell
         destination: install_slurm.sh

community/examples/hpc-slurm-ha.yaml

@@ -0,0 +1,200 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+blueprint_name: hpcslurmha
+
+vars:
+  project_id: # Set Project ID
+  deployment_name: hpcslurmha
+  region: us-central1
+  zone: us-central1-a
+
+terraform_backend_defaults:
+  type: gcs
+  configuration:
+    bucket: # Set Bucket Name Here
+
+deployment_groups:
+- group: primary
+  modules:
+  # Service Accounts
+  - id: controller_sa
+    source: modules/project/service-account
+    kind: terraform
+    settings:
+      deployment_name: $(vars.deployment_name)
+      project_id: $(vars.project_id)
+      name: controller-sa
+      description: Controller Service Account
+      display_name: Controller SA
+      project_roles:
+      - compute.instanceAdmin.v1
+      - iam.serviceAccountUser
+      - storage.objectViewer
+      - logging.logWriter
+      - monitoring.metricWriter
+
+  - id: login_sa
+    source: modules/project/service-account
+    kind: terraform
+    settings:
+      deployment_name: $(vars.deployment_name)
+      project_id: $(vars.project_id)
+      name: login-sa
+      description: Login Node Service Account
+      display_name: Login SA
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - storage.objectViewer
+
+  - id: compute_sa
+    source: modules/project/service-account
+    kind: terraform
+    settings:
+      deployment_name: $(vars.deployment_name)
+      project_id: $(vars.project_id)
+      name: compute-sa
+      description: Compute Node Service Account
+      display_name: Compute SA
+      project_roles:
+      - logging.logWriter
+      - monitoring.metricWriter
+      - storage.objectViewer
+
+  # Network
+  - id: network
+    source: modules/network/vpc
+    settings:
+      network_name: $(vars.deployment_name)-net
+      subnetworks:
+      - subnet_name: $(vars.deployment_name)-primary-subnet
+        subnet_region: $(vars.region)
+        subnet_ip: 10.80.0.0/24
+
+  # Filestore (Shared /home)
+  - id: home_fs
+    source: modules/file-system/filestore
+    settings:
+      network_id: $(network.network_id)
+      size_gb: 1024
+      zone: $(vars.zone)
+
+  # Debug Logic (Define Login Node properties, does not create VM directly)
+  - id: debug_nodes
+    source: community/modules/scheduler/schedmd-slurm-gcp-v6-login
+    settings:
+      name_prefix: debug
+      num_instances: 1
+      machine_type: n1-standard-2
+      service_account_email: $(login_sa.service_account_email)
+      service_account_scopes: ["https://www.googleapis.com/auth/cloud-platform"]
+      enable_oslogin: true
+      subnetwork_self_link: $(network.subnetwork_self_link)
+      enable_login_public_ips: false # NO EXTERNAL IP
+
+  # Nodeset (Compute)
+  - id: compute_nodes
+    source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
+    settings:
+      name: a2-nodes
+      node_count_dynamic_max: 4
+      node_count_static: 0
+      machine_type: n1-standard-2
+      service_account_email: $(compute_sa.service_account_email)
+      service_account_scopes: ["https://www.googleapis.com/auth/cloud-platform"]
+      enable_oslogin: true
+      subnetwork_self_link: $(network.subnetwork_self_link)
+      enable_placement: false
+      network_storage:
+      - server_ip: $(home_fs.network_storage.server_ip)
+        remote_mount: $(home_fs.network_storage.remote_mount)
+        local_mount: /home
+        fs_type: nfs
+        mount_options: _netdev,hard,intr
+
+  # Partition
+  - id: debug_partition
+    source: community/modules/compute/schedmd-slurm-gcp-v6-partition
+    use: [compute_nodes]
+    settings:
+      partition_name: debug
+      is_default: true
+
+  # Filestore (Dedicated State for Controller)
+  - id: controller_fs
+    source: modules/file-system/filestore
+    settings:
+      network_id: $(network.network_id)
+      size_gb: 1024
+      zone: $(vars.zone)
+
+  # Cloud SQL (External DB)
+  - id: slurm_db
+    source: community/modules/database/slurm-cloudsql-federation
+    settings:
+      deployment_name: $(vars.deployment_name)
+      project_id: $(vars.project_id)
+      region: $(vars.region)
+      tier: db-f1-micro
+      sql_instance_name: $(vars.deployment_name)-db
+      network_id: $(network.network_id)
+      use_psc_connection: true
+      subnetwork_self_link: $(network.subnetwork_self_link)
+
+  # Controller (Custom HA Module)
+  - id: slurm_controller
+    source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
+    use: [debug_partition]
+    settings:
+      enable_backup_controller: true
+      slurm_cluster_name: hpcslurmha
+      backup_zone: us-central1-f
+      # Disable PD state disk, rely on /var/spool/slurm filestore share
+      controller_state_disk: null
+      cloudsql:
+        server_ip: $(slurm_db.cloudsql.server_ip)
+        user: $(slurm_db.cloudsql.user)
+        password: $(slurm_db.cloudsql.password)
+        db_name: $(slurm_db.cloudsql.db_name)
+      enable_controller_public_ips: false # NO EXTERNAL IP
+      machine_type: n1-standard-2
+      zone: $(vars.zone)
+      service_account_email: $(controller_sa.service_account_email)
+      service_account_scopes: ["https://www.googleapis.com/auth/cloud-platform"]
+      enable_oslogin: true
+      subnetwork_self_link: $(network.subnetwork_self_link)
+      login_nodes: $(debug_nodes.login_nodes)
+      network_storage:
+      - server_ip: $(home_fs.network_storage.server_ip)
+        remote_mount: $(home_fs.network_storage.remote_mount)
+        local_mount: /home
+        fs_type: nfs
+        mount_options: _netdev,hard,intr
+      - server_ip: $(controller_fs.network_storage.server_ip)
+        remote_mount: $(controller_fs.network_storage.remote_mount)
+        local_mount: /var/spool/slurm
+        fs_type: nfs
+        mount_options: _netdev,hard,intr
+      login_network_storage:
+      - server_ip: $(home_fs.network_storage.server_ip)
+        remote_mount: $(home_fs.network_storage.remote_mount)
+        local_mount: /home
+        fs_type: nfs
+        mount_options: _netdev,hard,intr
+      static_ips:
+      - "10.80.0.10"
+      - "10.80.0.11"

community/examples/hpc-slurm6-apptainer.yaml

@@ -63,7 +63,9 @@ deployment_groups:
     settings:
       source_image_project_id: [schedmd-slurm-public]
       # see latest in https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/docs/images.md#published-image-family
-      source_image_family: slurm-gcp-6-12-hpc-rocky-linux-8
+      # Rocky 8: slurm-gcp-6-12-hpc-rocky-linux-8
+      # Rocky 9: slurm-gcp-6-12-hpc-rocky-linux-9
+      source_image_family: slurm-gcp-6-12-hpc-rocky-linux-9
       # You can find size of source image by using following command
       # gcloud compute images describe-from-family <source_image_family> --project schedmd-slurm-public
       disk_size: $(vars.disk_size)