Updating spark-k8s dependency to hadoop 3.4.3

Author: Maleware

hadoop/hadoop/Dockerfile

@@ -10,6 +10,7 @@ ARG PROTOBUF_VERSION
 ARG AWS_JAVA_SDK_BUNDLE_VERSION
 ARG AZURE_STORAGE_VERSION
 ARG AZURE_KEYVAULT_CORE_VERSION
+ARG ANALYTICSACCELERATOR_S3_VERSION
 ARG STACKABLE_USER_UID
 
 WORKDIR /stackable
@@ -166,6 +167,9 @@ else
     cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
     # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more.
     curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
+    # Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize.
+    # TODO: switch to https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar once mirrored.
+    curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo1.maven.org/maven2/software/amazon/s3/analyticsaccelerator/analyticsaccelerator-s3/${ANALYTICSACCELERATOR_S3_VERSION}/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar"
     cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
     cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
     cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/

hadoop/hadoop/boil-config.toml

@@ -30,8 +30,7 @@ protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release
 aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
 azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
 azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-
-
+analyticsaccelerator-s3-version = "1.3.1" # Pinned in hadoop-project-3.4.3.pom via amazon-s3-analyticsaccelerator-s3.version.
 
 [versions."3.5.0".local-images]
 java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
@@ -42,3 +41,4 @@ protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-
 aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0
 azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0
 azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+analyticsaccelerator-s3-version = "1.3.1" # Pinned in hadoop-project-3.5.0.pom via amazon-s3-analyticsaccelerator-s3.version.

spark-k8s/boil-config.toml

@@ -6,7 +6,7 @@
 containerfile = "Dockerfile.3"
 
 [versions."3.5.8".local-images]
-"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
+"hadoop/hadoop" = "3.4.3"
 java-base = "17"
 java-devel = "17"
 hbase = "2.6.3"
@@ -27,7 +27,7 @@ hbase-connector-version = "1.0.1_3.5.8"
 containerfile = "Dockerfile.4"
 
 [versions."4.1.1".local-images]
-"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
+"hadoop/hadoop" = "3.4.3"
 java-base = "21"
 java-devel = "21"