diff --git a/.github/workflows/publish-snapshot.yaml b/.github/workflows/publish-snapshot.yaml new file mode 100644 index 000000000..4ceba841d --- /dev/null +++ b/.github/workflows/publish-snapshot.yaml @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Publishes a SNAPSHOT build to the ASF Nexus snapshot repository after every +# trunk commit that passes CI. +name: Publish Snapshot + +on: + workflow_run: + workflows: [ "Test" ] + types: [ completed ] + branches: [ trunk ] + workflow_dispatch: + +# If several commits land in quick succession, cancel a stale in-progress +# publish in favor of the latest. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + publish-common: + name: Publish snapshot (common artifacts) + runs-on: ubuntu-latest + # Only publish after a successful build of a push to trunk (never a PR, and + # never a fork's Test run, which would otherwise have access to our secrets). + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_repository.full_name == github.repository) + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - name: Setup JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + - name: Cache Maven repository + uses: actions/cache/restore@v4 + with: + path: ~/.m2 + key: maven-repo-jdk11-${{ github.event.workflow_run.head_sha || github.sha }} + - name: Cache workspace + id: cache-build-restore + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }} + key: build-jdk11-${{ github.event.workflow_run.head_sha || github.sha }} + - name: Build dependencies (cache miss fallback) + if: steps.cache-build-restore.outputs.cache-hit != 'true' + run: | + sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done' + for i in {2..20} + do + sudo ip addr add "127.0.0.${i}" dev lo + sudo route add -host "127.0.0.${i}" dev lo; + done + + export JDK_VERSION="11" + export SPARK_VERSION="3" + export SCALA_VERSION="2.12" + export CASSANDRA_USE_JDK11="true" + + ./scripts/build-dependencies.sh + - name: Publish common artifacts + env: + ARTIFACT_TYPE: common + SCALA_VERSION: "2.13" + SPARK_VERSION: "3" + JDK_VERSION: "11" + MAVEN_USERNAME: ${{ secrets.NEXUS_USER }} + MAVEN_PASSWORD: ${{ secrets.NEXUS_PW }} + run: ./scripts/publish-snapshot.sh + + publish-spark: + name: Publish snapshot (spark artifacts) - Scala ${{ matrix.scala }} Spark${{ matrix.spark }} JDK${{ matrix.jdk }} + runs-on: ubuntu-latest + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_repository.full_name == github.repository) + strategy: + matrix: + include: + - scala: '2.12' + spark: '3' + jdk: '11' + - scala: '2.13' + spark: '3' + jdk: '11' + - scala: '2.13' + spark: '4' + jdk: '17' + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - name: Setup JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk }} + - name: Cache Maven repository + uses: actions/cache/restore@v4 + with: + path: ~/.m2 + key: maven-repo-jdk${{ matrix.jdk }}-${{ github.event.workflow_run.head_sha || github.sha }} + - name: Cache workspace + id: cache-build-restore + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }} + key: build-jdk${{ matrix.jdk }}-${{ github.event.workflow_run.head_sha || github.sha }} + - name: Build dependencies (cache miss fallback) + if: steps.cache-build-restore.outputs.cache-hit != 'true' + run: | + sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done' + for i in {2..20} + do + sudo ip addr add "127.0.0.${i}" dev lo + sudo route add -host "127.0.0.${i}" dev lo; + done + + export JDK_VERSION="${{ matrix.jdk }}" + export SPARK_VERSION="${{ matrix.spark }}" + export SCALA_VERSION="${{ matrix.scala }}" + if [ "${{ matrix.jdk }}" = "11" ]; then + export CASSANDRA_USE_JDK11=true + fi + if [ "${{ matrix.jdk }}" = "17" ]; then + # JDK17 leg only targets Cassandra 5.0+; skip 4.0 / 4.1 dtest jar builds. + export BRANCHES="cassandra-5.0" + fi + + ./scripts/build-dependencies.sh + - name: Publish spark artifacts + env: + ARTIFACT_TYPE: spark + SCALA_VERSION: ${{ matrix.scala }} + SPARK_VERSION: ${{ matrix.spark }} + JDK_VERSION: ${{ matrix.jdk }} + MAVEN_USERNAME: ${{ secrets.NEXUS_USER }} + MAVEN_PASSWORD: ${{ secrets.NEXUS_PW }} + run: ./scripts/publish-snapshot.sh diff --git a/DEV-README.md b/DEV-README.md index b877188e5..a973174c7 100644 --- a/DEV-README.md +++ b/DEV-README.md @@ -70,6 +70,27 @@ SCALA_VERSION=2.13 SPARK_VERSION=3 ./gradlew clean assemble -PartifactType=commo SCALA_VERSION=2.13 SPARK_VERSION=3 ./gradlew assemble -PartifactType=spark ``` +## Publishing snapshots + +On every trunk commit that passes CI, GitHub Actions (`.github/workflows/publish-snapshot.yaml`) +publishes a `SNAPSHOT` build to the ASF Nexus snapshot repository, so downstream projects (e.g. +`cassandra-sidecar`) can build against analytics trunk without waiting on a formal release vote. + +To publish manually, e.g. for testing, run `scripts/publish-snapshot.sh` with: + +```shell +ARTIFACT_TYPE=common SCALA_VERSION=2.13 SPARK_VERSION=3 JDK_VERSION=11 \ +MAVEN_USERNAME= MAVEN_PASSWORD= \ +./scripts/publish-snapshot.sh + +ARTIFACT_TYPE=spark SCALA_VERSION=2.13 SPARK_VERSION=3 JDK_VERSION=11 \ +MAVEN_USERNAME= MAVEN_PASSWORD= \ +./scripts/publish-snapshot.sh +``` + +The script refuses to run unless `gradle.properties`' `version` ends in `-SNAPSHOT`, and never +signs artifacts, matching the SNAPSHOT-only signing behavior already built into `build.gradle`. + ### Git hooks (optional) To enable git hooks, run the following command at project root. diff --git a/gradle/common/publishing-spark.gradle b/gradle/common/publishing-spark.gradle index 03f717f34..0e421b6bd 100644 --- a/gradle/common/publishing-spark.gradle +++ b/gradle/common/publishing-spark.gradle @@ -17,9 +17,7 @@ */ -if ("${project.rootProject.ext.sparkLabel}" == '3') { - archivesBaseName = "${archivesBaseName}_spark3" -} +archivesBaseName = "${archivesBaseName}_spark${project.rootProject.ext.sparkLabel}" archivesBaseName = "${archivesBaseName}_${scalaMajorVersion}" apply from: "$rootDir/gradle/common/publishing-non-spark.gradle" diff --git a/scripts/publish-snapshot.sh b/scripts/publish-snapshot.sh new file mode 100755 index 000000000..3ebf1acef --- /dev/null +++ b/scripts/publish-snapshot.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Publishes a SNAPSHOT build of one artifact type ("common" or "spark") to the +# ASF Nexus snapshot repository. +set -e -o pipefail + +SCRIPT_DIR=$( dirname -- "$( readlink -f -- "$0"; )"; ) +ROOT_DIR=$( dirname -- "${SCRIPT_DIR}"; ) + +ARTIFACT_TYPE="${ARTIFACT_TYPE:-}" +MAVEN_REPOSITORY_URL="${MAVEN_REPOSITORY_URL:-https://repository.apache.org/content/repositories/snapshots}" +MAVEN_USERNAME="${MAVEN_USERNAME:-}" +MAVEN_PASSWORD="${MAVEN_PASSWORD:-}" + +if [[ "${ARTIFACT_TYPE}" != "common" && "${ARTIFACT_TYPE}" != "spark" ]]; then + echo "ARTIFACT_TYPE must be 'common' or 'spark' (got: '${ARTIFACT_TYPE}')" + exit 1 +fi + +if [[ -z "${MAVEN_USERNAME}" || -z "${MAVEN_PASSWORD}" ]]; then + echo "MAVEN_USERNAME and MAVEN_PASSWORD must both be set" + exit 1 +fi + +version=$("${ROOT_DIR}/code_version.sh") + +if [[ "${version}" != *-SNAPSHOT ]]; then + echo "Refusing to publish: version '${version}' does not end in -SNAPSHOT." + echo "This script only publishes development snapshots, never release versions." + exit 1 +fi + +echo "Publishing ${ARTIFACT_TYPE} artifacts for version ${version} to ${MAVEN_REPOSITORY_URL}" + +"${ROOT_DIR}/gradlew" --no-daemon \ + -PartifactType="${ARTIFACT_TYPE}" \ + -PskipSigning \ + -Pmaven.repository.url="${MAVEN_REPOSITORY_URL}" \ + -Pmaven.username="${MAVEN_USERNAME}" \ + -Pmaven.password="${MAVEN_PASSWORD}" \ + publish --stacktrace