diff --git a/.github/workflows/build-kafka.yaml b/.github/workflows/build-kafka.yaml index e9badf75e5..b09d2c1b2a 100644 --- a/.github/workflows/build-kafka.yaml +++ b/.github/workflows/build-kafka.yaml @@ -15,8 +15,42 @@ permissions: packages: write jobs: - build: + java: runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to Registry + uses: docker/login-action@v4 + with: + username: "${{ github.repository_owner }}" + password: "${{ github.token }}" + registry: ghcr.io + + - name: Extract environment + run: |- + solution/kafka_build_vars.sh >> $GITHUB_ENV + + - name: Build and push java + uses: docker/build-push-action@v7 + with: + push: true + context: ./solution/java + tags: "${{ env.JAVA_IMAGE }}:${{ env.JAVA_TAG }}-${{ env.JAVA_BUILD_TREE_HASH }}" + cache-from: | + type=registry,ref=${{ env.JAVA_IMAGE }}:${{ env.JAVA_TAG }}-${{ env.JAVA_BUILD_TREE_HASH }} + type=gha,scope=java-${{ env.JAVA_TAG }} + cache-to: type=gha,mode=max,scope=java-${{ env.JAVA_TAG }} + + kafka: + runs-on: ubuntu-24.04 + needs: java steps: - name: Checkout uses: actions/checkout@v6 @@ -47,6 +81,7 @@ jobs: push: true context: ./solution/kafka build-args: |- + BASE_IMAGE=${{ env.JAVA_IMAGE }}:${{ env.JAVA_TAG }}-${{ env.JAVA_BUILD_TREE_HASH }} scala_version=${{ env.scala_version }} kafka_version=${{ env.kafka_version }} tags: "${{ env.KAFKA_IMAGE }}:${{ env.KAFKA_TAG }}-${{ env.BUILD_TREE_HASH }}" @@ -74,6 +109,7 @@ jobs: cruise-control: runs-on: ubuntu-24.04 + needs: java steps: - name: Checkout uses: actions/checkout@v7 @@ -100,6 +136,7 @@ jobs: push: true context: ./solution/cruise-control build-args: |- + BASE_IMAGE=${{ env.JAVA_IMAGE }}:${{ env.JAVA_TAG }}-${{ env.JAVA_BUILD_TREE_HASH }} CC_VERSION=${{ env.KAFKA_CRUISECONTROL_TAG }} tags: "${{ env.KAFKA_CRUISECONTROL_IMAGE }}:${{ env.KAFKA_CRUISECONTROL_TAG }}-${{ env.CRUISECONTROL_BUILD_TREE_HASH }}" cache-from: | @@ -109,6 +146,7 @@ jobs: zookeeper: runs-on: ubuntu-24.04 + needs: java steps: - name: Checkout uses: actions/checkout@v7 @@ -135,6 +173,7 @@ jobs: push: true context: ./solution/zookeeper build-args: |- + BASE_IMAGE=${{ env.JAVA_IMAGE }}:${{ env.JAVA_TAG }}-${{ env.JAVA_BUILD_TREE_HASH }} ZK_VERSION=${{ env.ZK_VERSION }} ZK_OPERATOR_VERSION=${{ env.ZK_OPERATOR_VERSION }} tags: "${{ env.ZOOKEEPER_IMAGE }}:${{ env.ZOOKEEPER_TAG }}-${{ env.ZOOKEEPER_BUILD_TREE_HASH }}" diff --git a/solution/build.sh b/solution/build.sh index bbe9252304..c77738153b 100755 --- a/solution/build.sh +++ b/solution/build.sh @@ -87,7 +87,8 @@ function flatten_source_images() { source <( ${REPOSITORY_DIR}/solution/kafka_build_vars.sh ) - yq eval '.* | select(.image) | (.sourceRegistry // "docker.io") + "/" + .image + ":" + .tag' deps.yaml | + yq eval '.* | select(.image) | select(.buildOnly != true) | + (.sourceRegistry // "docker.io") + "/" + .image + ":" + .tag' deps.yaml | sed '/ghcr.io\/scality\/zenko\/kafka/ s/$/-'"${BUILD_TREE_HASH}"'/' | sed '/ghcr.io\/scality\/zenko\/cruise-control/ s/$/-'"${CRUISECONTROL_BUILD_TREE_HASH}"'/' | sed '/ghcr.io\/scality\/zenko\/zookeeper/ s/$/-'"${ZOOKEEPER_BUILD_TREE_HASH}"'/' @@ -100,14 +101,14 @@ function zenko_operator_tag() function dependencies_versions_env() { - yq eval '.[] | select(.image) | .envsubst + "=" + .image | sub("_TAG=", "_IMAGE=")' deps.yaml + yq eval '.[] | select(.image) | select(.envsubst) | .envsubst + "=" + .image | sub("_TAG=", "_IMAGE=")' deps.yaml yq eval '.[] | select(.dashboard) | .envsubst + "=" + .dashboard | sub("_TAG=.*/", "_DASHBOARD=")' deps.yaml yq eval '.[] | select(.policy) | .envsubst + "=" + .policy | sub("_TAG=.*/", "_POLICY=")' deps.yaml find ${REPOSITORY_DIR}/monitoring/ -mindepth 1 -maxdepth 1 -type d -print0 | while IFS= read -r -d '' folder ; do local dashboard="${folder##*/}" echo "$(tr a-z- A-Z_ <<< $dashboard)_DASHBOARD=${dashboard}-dashboard" done - yq eval '.[] | select(.tag) | .envsubst + "=" + .tag' deps.yaml + yq eval '.[] | select(.tag) | select(.envsubst) | .envsubst + "=" + .tag' deps.yaml echo ZENKO_VERSION_NAME=${VERSION_FULL} source <( "${REPOSITORY_DIR}/solution/kafka_build_vars.sh" ) diff --git a/solution/cruise-control/Dockerfile b/solution/cruise-control/Dockerfile index d4c771e336..cba82ec199 100644 --- a/solution/cruise-control/Dockerfile +++ b/solution/cruise-control/Dockerfile @@ -10,12 +10,14 @@ # invocation and jar layout come from: # https://github.com/adobe/cruise-control/blob/master/docker/Dockerfile +ARG JDK_IMAGE=eclipse-temurin:21.0.12_8-jdk-noble +ARG BASE_IMAGE=ghcr.io/scality/zenko/java:21.0.12_8 ARG CC_REPO=https://github.com/adobe/cruise-control.git ARG CC_VERSION=3.0.3-adbe-20260722 ARG CC_COMMIT=c668b8316df9260dc0865e3c3f2274ebc60c5ce8 # The JDK only: the build drives Gradle through the project's own wrapper. -FROM eclipse-temurin:21.0.12_8-jdk-noble AS build +FROM ${JDK_IMAGE} AS build ARG CC_REPO ARG CC_VERSION @@ -41,7 +43,7 @@ RUN ./gradlew --console=plain --no-daemon clean jar copyDependantLibs && \ cruise-control/build/dependant-libs/cruise-control-metrics-reporter.jar #################################################################################################### -FROM eclipse-temurin:21.0.12_8-jre-noble +FROM ${BASE_IMAGE} # CruiseControl reads webserver.ui.diskpath unconditionally, so the directory has to exist # even though no web UI is shipped. diff --git a/solution/cruise-control/README.md b/solution/cruise-control/README.md index 7945abe19a..e68275c2b9 100644 --- a/solution/cruise-control/README.md +++ b/solution/cruise-control/README.md @@ -1,7 +1,7 @@ # CruiseControl Docker Image Docker file for building [CruiseControl](https://github.com/adobe/cruise-control) from Adobe's fork, -running on Eclipse Temurin. +running on an Eclipse Temurin runtime trimmed with `jlink`. We build it ourselves because the published `adobe/cruise-control` images ship Oracle JDK under the No-Fee Terms and Conditions, which does not allow redistribution as part of a commercial product. diff --git a/solution/deps.yaml b/solution/deps.yaml index d8628fb4dd..dc5431d8cb 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -43,6 +43,11 @@ jmx-javaagent: image: jmx-javaagent tag: 1.5.0 envsubst: JMX_JAVAAGENT_TAG +java: # shared base for our java images; built, never deployed + sourceRegistry: ghcr.io/scality/zenko + image: java + tag: 21.0.12_8 + buildOnly: true kaf: # CLI Kafka client all-in-one tag: 0.2.8 envsubst: KAF_VERSION diff --git a/solution/java/Dockerfile b/solution/java/Dockerfile new file mode 100644 index 0000000000..4491c74b2d --- /dev/null +++ b/solution/java/Dockerfile @@ -0,0 +1,45 @@ +# syntax=docker/dockerfile:1 + +# Shared Java runtime for our kafka, kafka-connect, cruise-control and zookeeper images. +# +# Having one base rather than repeating these stages per image matters for size: the runtime +# and the patched OS end up in layers all four images share, instead of four copies that only +# look identical. Measured across the three service images, this is the difference between +# saving 21MB and saving 135MB. + +ARG JDK_IMAGE=eclipse-temurin:21.0.12_8-jdk-noble +ARG BASE_IMAGE=debian:trixie-20260824-slim + +#################################################################################################### +# Minimal Java runtime: ~100MB, against ~300MB for the full JDK. +# +# ALL-MODULE-PATH keeps every platform module. Netty, the JMX agent, SASL and TLS reach modules +# through reflection and ServiceLoader, which jdeps cannot see, so trimming the list trades ~25MB +# for failures that only appear at runtime. --release-info keeps the vendor metadata jlink drops. +FROM ${JDK_IMAGE} AS jre_build + +RUN jlink --add-modules ALL-MODULE-PATH \ + --strip-debug --no-man-pages --no-header-files --compress=zip-6 \ + --release-info ${JAVA_HOME}/release \ + --output /javaruntime + +#################################################################################################### +FROM ${BASE_IMAGE} + +ENV JAVA_HOME=/opt/java/openjdk \ + LANG=C.UTF-8 +ENV PATH=${JAVA_HOME}/bin:${PATH} + +COPY --from=jre_build /javaruntime ${JAVA_HOME} + +# jlink does not generate a class data sharing archive, and the stock Temurin images ship one. +# Without it the JVM starts roughly 2.5x slower, since every run re-parses the class metadata +# that CDS would otherwise map straight into memory. +RUN ${JAVA_HOME}/bin/java -Xshare:dump + +# Debian's base image is only rebuilt periodically, so patch it: we pin it, so nothing else will. +# curl is used by the operators' probes, ca-certificates by anything talking TLS. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl && \ + apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/solution/java/README.md b/solution/java/README.md new file mode 100644 index 0000000000..929b8c1323 --- /dev/null +++ b/solution/java/README.md @@ -0,0 +1,26 @@ +# Shared Java Base Image + +Base image for our Java services: kafka, kafka-connect, cruise-control and zookeeper. It is built +and pushed, but never deployed on its own. + +It carries an [Eclipse Temurin](https://adoptium.net) runtime trimmed with `jlink`, on a pinned +Debian base whose packages are upgraded at build time. + +## Why a shared image rather than repeating the stages + +The same stages inlined in each Dockerfile produce layers that only *look* identical: the `jlink` +output does dedupe, but each `apt` layer is built separately and does not. Across the three service +images that is the difference between saving 21MB and saving 135MB. + +Pinning our own base also means nobody else patches it -- Temurin's images are rebuilt +continuously, ours are not -- hence the `apt-get upgrade`. Without it the images scanned three +critical and ten high CVEs worse than the Temurin-based ones they replace. + +## Bumping + +Update `java.tag` in `solution/deps.yaml` and `JDK_IMAGE` in the Dockerfile together; the tag +is the Temurin version so that the two stay legible side by side. `BASE_IMAGE` (Debian) is pinned to +a dated tag and wants refreshing periodically. + +Every image that builds `FROM` this one folds `solution/java` into its own build tree hash, so +bumping this image republishes all of them under new tags. diff --git a/solution/kafka/Dockerfile b/solution/kafka/Dockerfile index fe82d6dba9..31d3c38302 100644 --- a/solution/kafka/Dockerfile +++ b/solution/kafka/Dockerfile @@ -1,3 +1,5 @@ +ARG BASE_IMAGE=ghcr.io/scality/zenko/java:21.0.12_8 + FROM alpine:3.24.2 AS kafka_dist ARG scala_version=2.13 @@ -22,7 +24,7 @@ RUN tar -xzf $kafka_distro RUN rm -r kafka_$scala_version-$kafka_version/bin/windows #################################################################################################### -FROM eclipse-temurin:21.0.12_8-jre-noble +FROM ${BASE_IMAGE} ARG scala_version=2.13 ARG kafka_version=3.9.1 @@ -34,7 +36,7 @@ ENV KAFKA_VERSION=$kafka_version \ ENV PATH=${PATH}:${KAFKA_HOME}/bin -RUN mkdir ${KAFKA_HOME} && apt-get update && apt-get install curl -y && apt-get clean +RUN mkdir ${KAFKA_HOME} COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$kafka_version ${KAFKA_HOME} COPY log4j.properties ${KAFKA_HOME}/config/ diff --git a/solution/kafka/README.md b/solution/kafka/README.md index 115fe9d432..7b8dab8afc 100644 --- a/solution/kafka/README.md +++ b/solution/kafka/README.md @@ -1,5 +1,5 @@ # Apache Kafka Docker Image Docker file for building docker image for [Apache Kafka](https://kafka.apache.org) from official -[Apache Kafka Distros](https://www.apache.org/dyn/closer.cgi?path=/kafka/), running on Eclipse -Temurin. +[Apache Kafka Distros](https://www.apache.org/dyn/closer.cgi?path=/kafka/), running on an Eclipse +Temurin runtime trimmed with `jlink`. diff --git a/solution/kafka_build_vars.sh b/solution/kafka_build_vars.sh index ee15fa9c2d..2da5398e1e 100755 --- a/solution/kafka_build_vars.sh +++ b/solution/kafka_build_vars.sh @@ -11,9 +11,17 @@ get_image_from_deps() { } cd "${CURRENT_DIR}" + +# Our java images all build FROM the java image, so its tree has to feed their tree hashes: otherwise +# a change to the base would republish different content under an already-published tag. +JAVA_TREE=$(git rev-parse HEAD:solution/java) + cat <-`, and