feat: Add Hadoop 3.5.0 and 3.4.3

.scripts/upload_new_boost_version.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# This script mirrors the boostorg/boost source bundle for the given version to Nexus.
+# The boost source bundle is architecture independent.
+# It contains its own build system (b2) which is also built from source before building boost itself, so we don't need to worry about architecture specific builds.
+# This artifact is used by the hadoop/boost local image.
+
+
+set -euo pipefail
+
+VERSION=${1:?"Missing version number argument (arg 1)"}
+NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
+
+read -r -s -p "Nexus Password: " NEXUS_PASSWORD
+echo ""
+
+# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory
+# Find the directory name of the script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# the temp directory used, within $DIR
+WORK_DIR=$(mktemp -d -p "$DIR")
+
+# check if tmp dir was created
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+  echo "Could not create temp dir"
+  exit 1
+fi
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+cd "$WORK_DIR" || exit
+
+# boost does not currently publish signatures or SBOMs
+BOOST_UNDERSCORE="$(echo "${VERSION}" | tr '.' '_')"
+BOOST_TARBALL="boost_${BOOST_UNDERSCORE}.tar.bz2"
+DOWNLOAD_URL="https://archives.boost.io/release/$VERSION/source/$BOOST_TARBALL"
+
+echo "Downloading boost"
+if ! curl --fail -Ls -O "$DOWNLOAD_URL"; then
+  echo "Failed to download from $DOWNLOAD_URL"
+  exit 1
+fi
+
+FILE_NAME=$(basename "$DOWNLOAD_URL")
+
+echo "Uploading boost to Nexus"
+if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/boost/'; then
+  echo "Failed to upload boost to Nexus"
+  exit 1
+fi
+
+echo "Successfully uploaded new version of boost ($VERSION) to Nexus"
+echo "https://repo.stackable.tech/service/rest/repository/browse/packages/boost/"

druid/boil-config.toml

@@ -15,7 +15,7 @@ authorizer-version = "0.7.0"
 # https://druid.apache.org/docs/34.0.0/operations/java/
 java-base = "17"
 java-devel = "17"
-"hadoop/hadoop" = "3.4.2"
+"hadoop/hadoop" = "3.4.3"
 
 # Deprecated since 26.3
 [versions."34.0.0".build-arguments]
@@ -26,7 +26,7 @@ authorizer-version = "0.7.0"
 # https://druid.apache.org/docs/35.0.1/operations/java/
 java-base = "21"
 java-devel = "21"
-"hadoop/hadoop" = "3.4.2"
+"hadoop/hadoop" = "3.4.3"
 
 [versions."35.0.1".build-arguments]
 authorizer-version = "0.7.0"

hadoop/boil-config.toml

@@ -20,4 +20,24 @@ java-devel = "11"
 [versions."3.4.2".build-arguments]
 async-profiler-version = "2.9"
 jmx-exporter-version = "1.3.0"
-hdfs-utils-version = "0.5.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.4.3".local-images]
+"hadoop/hadoop" = "3.4.3"
+java-base = "11"
+java-devel = "11"
+
+[versions."3.4.3".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.5.0".local-images]
+"hadoop/hadoop" = "3.5.0"
+java-base = "17"
+java-devel = "17"
+
+[versions."3.5.0".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"

hadoop/hadoop/Dockerfile

@@ -1,11 +1,15 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
+FROM local-image/shared/boost AS boost-builder
 FROM local-image/java-devel AS hadoop-builder
 
 ARG PRODUCT_VERSION
 ARG RELEASE_VERSION
 ARG PROTOBUF_VERSION
+ARG AWS_JAVA_SDK_BUNDLE_VERSION
+ARG AZURE_STORAGE_VERSION
+ARG AZURE_KEYVAULT_CORE_VERSION
 ARG STACKABLE_USER_UID
 
 WORKDIR /stackable
@@ -14,17 +18,22 @@ COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable
 COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF_VERSION} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF_VERSION}
 
 RUN <<EOF
-rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 microdnf update
-# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
-# automake and libtool are required to build protobuf
-microdnf install boost1.78-devel automake libtool
+# libstdc++ is a runtime dependency for boost,
+# automake and libtool are needed to build protobuf
+microdnf install libstdc++ automake libtool
 microdnf clean all
 rm -rf /var/cache/yum
 mkdir /opt/protobuf
 chown ${STACKABLE_USER_UID}:0 /opt/protobuf
 EOF
 
+COPY --chown=${STACKABLE_USER_UID}:0 --from=boost-builder /stackable/boost /stackable/boost
+
+ENV BOOST_ROOT=/stackable/boost
+ENV LD_LIBRARY_PATH=/stackable/boost/lib
+ENV CPATH=/stackable/boost/include
+
 USER ${STACKABLE_USER_UID}
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
@@ -35,10 +44,18 @@ RUN <<EOF
     # Create snapshot of the source code including custom patches
     tar -czf /stackable/protobuf-${PROTOBUF_VERSION}-src.tar.gz .
 
-    ./autogen.sh
-    ./configure --prefix=/opt/protobuf
-    make "-j$(nproc)"
-    make install
+    if [ "$PROTOBUF_VERSION" == "3.7.1" ] || [ "$PROTOBUF_VERSION" == "3.21.12" ]; then
+        ./autogen.sh
+        ./configure --prefix=/opt/protobuf
+        make "-j$(nproc)"
+        make install
+    else
+        # protobuf > 3.21 bundles abseil-cpp and utf8_range as git submodules
+        git submodule update --init --recursive
+        cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+        cmake --build build --parallel "$(nproc)"
+        cmake --install build --prefix /opt/protobuf
+    fi
     (cd .. && rm -r ${PROTOBUF_VERSION})
 EOF
 
@@ -126,3 +143,31 @@ rm -rf /stackable/.m2
 # Set correct groups; make sure only required artifacts for the final image are located in /stackable
 chmod -R g=u /stackable
 EOF
+
+RUN <<EOF
+NEW_VERSION=${PRODUCT_VERSION}-stackable${RELEASE_VERSION}
+
+# Place cloud libraries (for s3a:// and abfs://) in the /stackable/hadoop-cloud-libraries/ folder to
+# have one central place for dependant images to pull them from
+mkdir -p /stackable/hadoop-cloud-libraries/
+
+if [[ "$PRODUCT_VERSION" == "3.3.6" || "$PRODUCT_VERSION" == "3.4.2" ]]; then
+    cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    # For some reason it was renamed from "aws-java-sdk-bundle" to "bundle"
+    if [[ "$PRODUCT_VERSION" == "3.3.6" ]]; then
+        cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar
+    else
+        cp /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar
+    fi
+    cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+else
+    cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more.
+    curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
+    cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+fi
+EOF

hadoop/hadoop/boil-config.toml

@@ -1,11 +1,44 @@
 [versions."3.3.6".local-images]
 java-devel = "11"
+"shared/boost" = "1.72.0" # I could not find a documented recommended version
 
 [versions."3.3.6".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.7.1" # https://github.com/apache/hadoop/blob/rel/release-3.3.6/BUILDING.txt
+aws-java-sdk-bundle-version = "1.12.367"  # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
 
 [versions."3.4.2".local-images]
-java-devel = "11"
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
 
 [versions."3.4.2".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
+aws-java-sdk-bundle-version = "2.29.52"   # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
+
+
+[versions."3.4.3".local-images]
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+
+[versions."3.4.3".build-arguments]
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
+
+
+[versions."3.5.0".local-images]
+java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.86.0" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
+
+[versions."3.5.0".build-arguments]
+protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
+aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1

hadoop/hadoop/stackable/patches/3.4.3/0001-YARN-11527-Update-node.js.patch

@@ -0,0 +1,22 @@
+From 7ad31a922a9fbcecd884b4bdf5c416f6b0ea539e Mon Sep 17 00:00:00 2001
+From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
+Date: Tue, 26 May 2026 15:40:40 +0200
+Subject: YARN-11527-Update-node.js
+
+---
+ hadoop-project/pom.xml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
+index 0813904f98a..f837b1f5201 100644
+--- a/hadoop-project/pom.xml
++++ b/hadoop-project/pom.xml
+@@ -236,7 +236,7 @@
+     <woodstox.version>5.4.0</woodstox.version>
+     <nimbus-jose-jwt.version>10.4</nimbus-jose-jwt.version>
+     <jcip-annotations.version>1.0-1</jcip-annotations.version>
+-    <nodejs.version>v12.22.1</nodejs.version>
++    <nodejs.version>v14.17.0</nodejs.version>
+     <yarnpkg.version>v1.22.5</yarnpkg.version>
+     <apache-ant.version>1.10.13</apache-ant.version>
+     <jmh.version>1.20</jmh.version>