Add GitHub Actions matrix build for Spark 3 and 4

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ jobs:
   main:
     name: Run spark-measure end-to-end tests
     runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        spark_image_tag: [ '3.5.6-scala2.12-java17-python3-ubuntu', '4.0.0-scala2.13-java17-python3-ubuntu' ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -25,7 +28,7 @@ jobs:
           ./e2e/prereq.sh
       - name: Build spark-measure image
         run: |
-          ./e2e/build.sh
+          ./e2e/build.sh -i ${{ matrix.spark_image_tag }}
       - name: Load spark-measure image into k8s/kind cluster
         run: |
           ./e2e/push-image.sh -k -d

Dockerfile

@@ -1,27 +1,33 @@
-# Use a Scala SBT base image with Java 17 for building sparkMeasure jar files
+ARG spark_image_tag=4.0.0-scala2.13-java17-python3-ubuntu
+ARG scala_version
+
 FROM sbtscala/scala-sbt:eclipse-temurin-alpine-17.0.15_6_1.11.3_2.13.16 AS builder
+ARG scala_version
 
 # Set the working directory
 WORKDIR /app
 
 # Copy the SBT configuration file
 COPY build.sbt ./
+
 # Copy the SBT project configuration directory
 COPY project ./project
 
 # Copy the application source code
 COPY src ./src
 
-# Compile the project with Scala 2.12.18
-ENV SCALA_VERSION=2.12.18
-RUN sbt ++${SCALA_VERSION} package
+# Compile the project with Scala
+ENV SCALA_VERSION_LONG=${scala_version}.8
+RUN sbt ++${SCALA_VERSION_LONG} package
 
-# Use the official Spark image with Scala 2.12, Java 17, and Python 3 for runtime
-FROM docker.io/library/spark:3.5.6-scala2.12-java17-python3-ubuntu
+# Build the spark image container sparkMeasure
+FROM docker.io/library/spark:${spark_image_tag}
+ARG scala_version
 
 USER root
 
-# Set up the Prometheus JMX exporter to expose metrics to Prometheus
+# Setup for the Prometheus JMX exporter.
+# Add the Prometheus JMX exporter Java agent jar for exposing metrics sent to the JmxSink to Prometheus.
 ENV JMX_EXPORTER_AGENT_VERSION=1.1.0
 ADD https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_AGENT_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_AGENT_VERSION}.jar /opt/spark/jars
 RUN chmod 644 /opt/spark/jars/jmx_prometheus_javaagent-${JMX_EXPORTER_AGENT_VERSION}.jar
@@ -35,8 +41,8 @@ RUN pip install /opt/src/python
 # Add rootfs filesystem, it contains python scripts for runnning end-to-end tests
 ADD e2e/rootfs/ /
 
-# Copy the compiled jar from the build stage
-COPY --from=builder /app/target/scala-2.12/*.jar /opt/spark/jars/
+# Copy the sparkMeasure compiled jar from the build stage
+COPY --from=builder /app/target/scala-${scala_version}/*.jar /opt/spark/jars/
 
 # Set the Spark user
 ARG spark_uid=185
@@ -47,4 +53,4 @@ USER ${spark_uid}
 EXPOSE 4040
 
 # Set the default entrypoint
-CMD ["/bin/bash"]
+CMD ["/bin/bash"]

e2e/argocd.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Install pre-requisite for fink ci
+# Run integration tests for sparkMeasure over Kubernetes
 
 # @author  Fabrice Jammes
 
@@ -9,6 +9,28 @@ set -euxo pipefail
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 PROJECT_DIR=$(cd "$DIR/.."; pwd -P)
 
+usage() {
+  cat << EOD
+
+Usage: `basename $0` [options]
+
+  Available options:
+    -h        this message
+
+Run integration tests for sparkMeasure over Kubernetes
+
+EOD
+}
+
+# get the options
+while getopts hi: c ; do
+    case $c in
+	    h) usage ; exit 0 ;;
+	    \?) usage ; exit 2 ;;
+    esac
+done
+shift `expr $OPTIND - 1`
+
 ciux ignite --selector itest "$PROJECT_DIR"
 
 # Run the CD pipeline
@@ -27,7 +49,7 @@ if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then
   # During a PR, the git repository containing the CI code is the forked one
   git_repo=$(jq -r '.pull_request.head.repo.full_name' "$GITHUB_EVENT_PATH")
 else
-  revision="${SPARKMEASURE_WORKBRANCH:-main}"
+  revision="${SPARKMEASURE_WORKBRANCH}"
   git_repo="${GITHUB_REPOSITORY}"
 fi
 git_repo="https://github.com/${git_repo}"
@@ -45,4 +67,4 @@ argocd app set spark-jobs -p image.tag="$CIUX_IMAGE_TAG"
 argocd app sync -l app.kubernetes.io/part-of=$app_name,app.kubernetes.io/component=operator
 argocd app wait -l app.kubernetes.io/part-of=$app_name,app.kubernetes.io/component=operator
 
-argocd app sync -l app.kubernetes.io/part-of=$app_name
+argocd app sync -l app.kubernetes.io/part-of=$app_name

e2e/build.sh

@@ -1,31 +1,31 @@
 #!/bin/bash
-
-
 # Build image containing integration tests for sparkMeasure over Kubernetes
-
 # @author  Fabrice Jammes
 
 set -euxo pipefail
 
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 PROJECT_DIR=$(cd "$DIR/.."; pwd -P)
+spark_image_tag="3.5.6-scala2.12-java17-python3-ubuntu"
 
 usage() {
   cat << EOD
 
 Usage: `basename $0` [options]
 
   Available options:
-    -h          this message
+    -h        this message
+    -i        spark image tag to use as base image (default: 3.5.6-scala2.12-java17-python3-ubuntu)
 
 Build image containing integration tests for sparkMeasure over Kubernetes
 EOD
 }
 
 # get the options
-while getopts h c ; do
+while getopts hi: c ; do
     case $c in
 	    h) usage ; exit 0 ;;
+	    i) spark_image_tag="$OPTARG" ;;
 	    \?) usage ; exit 2 ;;
     esac
 done
@@ -34,8 +34,13 @@ shift `expr $OPTIND - 1`
 ciux ignite --selector build $PROJECT_DIR
 . $PROJECT_DIR/.ciux.d/ciux_build.sh
 
+scala_version=$(echo "$spark_image_tag" | grep -oP 'scala\K[0-9]+\.[0-9]+')
+
 echo "Building Docker image for sparkMeasure integration tests"
-docker image build --tag "$CIUX_IMAGE_URL" "$PROJECT_DIR"
+docker image build  \
+  --build-arg "spark_image_tag=$spark_image_tag" \
+  --build-arg "scala_version=$scala_version" \
+  --tag "$CIUX_IMAGE_URL" "$PROJECT_DIR"
 
 echo "Build successful"
 

e2e/charts/spark-jobs/templates/spark-pi.yaml

@@ -10,7 +10,6 @@ spec:
   mainApplicationFile: local:///opt/spark/examples/spark-pi.py
   arguments:
     - "10"
-  sparkVersion: "3.5.6"
   driver:
     cores: 1
     coreLimit: "1200m"
@@ -19,4 +18,4 @@ spec:
   executor:
     cores: 1
     instances: 1
-    memory: "512m"
+    memory: "512m"

e2e/charts/spark-jobs/templates/spark-sql.yaml

@@ -8,7 +8,6 @@ spec:
   image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
   imagePullPolicy: "{{ .Values.image.pullPolicy }}"
   mainApplicationFile: local:///opt/spark/examples/spark-sql.py
-  sparkVersion: "3.5.6"
   driver:
     cores: 1
     coreLimit: "1200m"