diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5875e6b..7a5242f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -8,6 +8,9 @@ jobs: main: name: Run spark-measure end-to-end tests runs-on: ubuntu-24.04 + strategy: + matrix: + spark_image_tag: [ '3.5.6-scala2.12-java17-python3-ubuntu', '4.0.0-scala2.13-java17-python3-ubuntu' ] steps: - name: Checkout code uses: actions/checkout@v2 @@ -25,7 +28,7 @@ jobs: ./e2e/prereq.sh - name: Build spark-measure image run: | - ./e2e/build.sh + ./e2e/build.sh -i ${{ matrix.spark_image_tag }} - name: Load spark-measure image into k8s/kind cluster run: | ./e2e/push-image.sh -k -d diff --git a/Dockerfile b/Dockerfile index 3035efd..ee30124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,33 @@ -# Use a Scala SBT base image with Java 17 for building sparkMeasure jar files +ARG spark_image_tag=4.0.0-scala2.13-java17-python3-ubuntu +ARG scala_version + FROM sbtscala/scala-sbt:eclipse-temurin-alpine-17.0.15_6_1.11.3_2.13.16 AS builder +ARG scala_version # Set the working directory WORKDIR /app # Copy the SBT configuration file COPY build.sbt ./ + # Copy the SBT project configuration directory COPY project ./project # Copy the application source code COPY src ./src -# Compile the project with Scala 2.12.18 -ENV SCALA_VERSION=2.12.18 -RUN sbt ++${SCALA_VERSION} package +# Compile the project with Scala +ENV SCALA_VERSION_LONG=${scala_version}.8 +RUN sbt ++${SCALA_VERSION_LONG} package -# Use the official Spark image with Scala 2.12, Java 17, and Python 3 for runtime -FROM docker.io/library/spark:3.5.6-scala2.12-java17-python3-ubuntu +# Build the spark image container sparkMeasure +FROM docker.io/library/spark:${spark_image_tag} +ARG scala_version USER root -# Set up the Prometheus JMX exporter to expose metrics to Prometheus +# Setup for the Prometheus JMX exporter. +# Add the Prometheus JMX exporter Java agent jar for exposing metrics sent to the JmxSink to Prometheus. ENV JMX_EXPORTER_AGENT_VERSION=1.1.0 ADD https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_AGENT_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_AGENT_VERSION}.jar /opt/spark/jars RUN chmod 644 /opt/spark/jars/jmx_prometheus_javaagent-${JMX_EXPORTER_AGENT_VERSION}.jar @@ -35,8 +41,8 @@ RUN pip install /opt/src/python # Add rootfs filesystem, it contains python scripts for runnning end-to-end tests ADD e2e/rootfs/ / -# Copy the compiled jar from the build stage -COPY --from=builder /app/target/scala-2.12/*.jar /opt/spark/jars/ +# Copy the sparkMeasure compiled jar from the build stage +COPY --from=builder /app/target/scala-${scala_version}/*.jar /opt/spark/jars/ # Set the Spark user ARG spark_uid=185 @@ -47,4 +53,4 @@ USER ${spark_uid} EXPOSE 4040 # Set the default entrypoint -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/e2e/argocd.sh b/e2e/argocd.sh index 657d7a4..28f0956 100755 --- a/e2e/argocd.sh +++ b/e2e/argocd.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Install pre-requisite for fink ci +# Run integration tests for sparkMeasure over Kubernetes # @author Fabrice Jammes @@ -9,6 +9,28 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) PROJECT_DIR=$(cd "$DIR/.."; pwd -P) +usage() { + cat << EOD + +Usage: `basename $0` [options] + + Available options: + -h this message + +Run integration tests for sparkMeasure over Kubernetes + +EOD +} + +# get the options +while getopts hi: c ; do + case $c in + h) usage ; exit 0 ;; + \?) usage ; exit 2 ;; + esac +done +shift `expr $OPTIND - 1` + ciux ignite --selector itest "$PROJECT_DIR" # Run the CD pipeline @@ -27,7 +49,7 @@ if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then # During a PR, the git repository containing the CI code is the forked one git_repo=$(jq -r '.pull_request.head.repo.full_name' "$GITHUB_EVENT_PATH") else - revision="${SPARKMEASURE_WORKBRANCH:-main}" + revision="${SPARKMEASURE_WORKBRANCH}" git_repo="${GITHUB_REPOSITORY}" fi git_repo="https://github.com/${git_repo}" @@ -45,4 +67,4 @@ argocd app set spark-jobs -p image.tag="$CIUX_IMAGE_TAG" argocd app sync -l app.kubernetes.io/part-of=$app_name,app.kubernetes.io/component=operator argocd app wait -l app.kubernetes.io/part-of=$app_name,app.kubernetes.io/component=operator -argocd app sync -l app.kubernetes.io/part-of=$app_name \ No newline at end of file +argocd app sync -l app.kubernetes.io/part-of=$app_name diff --git a/e2e/build.sh b/e2e/build.sh index f1fe8a2..06b5661 100755 --- a/e2e/build.sh +++ b/e2e/build.sh @@ -1,14 +1,12 @@ #!/bin/bash - - # Build image containing integration tests for sparkMeasure over Kubernetes - # @author Fabrice Jammes set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) PROJECT_DIR=$(cd "$DIR/.."; pwd -P) +spark_image_tag="3.5.6-scala2.12-java17-python3-ubuntu" usage() { cat << EOD @@ -16,16 +14,18 @@ usage() { Usage: `basename $0` [options] Available options: - -h this message + -h this message + -i spark image tag to use as base image (default: 3.5.6-scala2.12-java17-python3-ubuntu) Build image containing integration tests for sparkMeasure over Kubernetes EOD } # get the options -while getopts h c ; do +while getopts hi: c ; do case $c in h) usage ; exit 0 ;; + i) spark_image_tag="$OPTARG" ;; \?) usage ; exit 2 ;; esac done @@ -34,8 +34,13 @@ shift `expr $OPTIND - 1` ciux ignite --selector build $PROJECT_DIR . $PROJECT_DIR/.ciux.d/ciux_build.sh +scala_version=$(echo "$spark_image_tag" | grep -oP 'scala\K[0-9]+\.[0-9]+') + echo "Building Docker image for sparkMeasure integration tests" -docker image build --tag "$CIUX_IMAGE_URL" "$PROJECT_DIR" +docker image build \ + --build-arg "spark_image_tag=$spark_image_tag" \ + --build-arg "scala_version=$scala_version" \ + --tag "$CIUX_IMAGE_URL" "$PROJECT_DIR" echo "Build successful" diff --git a/e2e/charts/spark-jobs/templates/spark-pi.yaml b/e2e/charts/spark-jobs/templates/spark-pi.yaml index 8487942..8eede14 100644 --- a/e2e/charts/spark-jobs/templates/spark-pi.yaml +++ b/e2e/charts/spark-jobs/templates/spark-pi.yaml @@ -10,7 +10,6 @@ spec: mainApplicationFile: local:///opt/spark/examples/spark-pi.py arguments: - "10" - sparkVersion: "3.5.6" driver: cores: 1 coreLimit: "1200m" @@ -19,4 +18,4 @@ spec: executor: cores: 1 instances: 1 - memory: "512m" \ No newline at end of file + memory: "512m" diff --git a/e2e/charts/spark-jobs/templates/spark-sql.yaml b/e2e/charts/spark-jobs/templates/spark-sql.yaml index 422bc3b..7ebd0c1 100644 --- a/e2e/charts/spark-jobs/templates/spark-sql.yaml +++ b/e2e/charts/spark-jobs/templates/spark-sql.yaml @@ -8,7 +8,6 @@ spec: image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" mainApplicationFile: local:///opt/spark/examples/spark-sql.py - sparkVersion: "3.5.6" driver: cores: 1 coreLimit: "1200m"