HDDS-11234. Manage Netty native memory consumption (apache#10354)

Author: smengcl

hadoop-hdds/common/src/main/conf/ozone-env.sh

@@ -248,6 +248,21 @@ export OZONE_OS_TYPE=${OZONE_OS_TYPE:-$(uname -s)}
 # Java property: hadoop.log.dir
 # export OZONE_SECURE_LOG_DIR=${OZONE_LOG_DIR}
 
+###
+# Netty native (direct) memory caps (HDDS-11234)
+###
+# Both unshaded io.netty and the Ratis-shaded copy default their pooled
+# direct-memory ceiling to MaxDirectMemorySize (≈ -Xmx) per JVM, which
+# can let the resident size of a busy DataNode or S3 Gateway grow well
+# beyond the heap. To put a hard cap on each pool, export one or both
+# of the following before starting Ozone daemons. Values are raw byte
+# counts (suffixes like "m" or "g" are NOT supported by Netty's
+# property parser); for example, 536870912 = 512 MiB.
+#
+# For example, to cap each pool at 4 GiB on a DataNode with -Xmx16g:
+# export OZONE_NETTY_MAX_DIRECT_MEMORY=4294967296
+# export OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY=4294967296
+
 ###
 # Ozone Manager specific parameters
 ###

hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh

@@ -1420,6 +1420,17 @@ function ozone_java_setup
     RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${RATIS_OPTS}"
   fi
 
+  # Opt-in caps on Netty's pooled direct-memory arena (HDDS-11234). Two
+  # properties are needed because Ozone runs both the unshaded io.netty
+  # *and* the Ratis-shaded copy in the same JVM, each with its own
+  # independent ceiling.
+  if [[ -n "${OZONE_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+    OZONE_OPTS="-Dio.netty.maxDirectMemory=${OZONE_NETTY_MAX_DIRECT_MEMORY} ${OZONE_OPTS}"
+  fi
+  if [[ -n "${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+    RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.maxDirectMemory=${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY} ${RATIS_OPTS}"
+  fi
+
   ozone_set_module_access_args
 }