diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index 6cadd2e96d..9973931922 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import java.util.zip.Inflater; +import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.ATSConstants; import org.apache.tez.common.Preconditions; @@ -505,11 +506,16 @@ public static Map convertEdgeManagerPluginDescriptor( public static Map convertConfigurationToATSMap(Configuration conf) { // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500 - Iterator> iter = new Configuration(conf).iterator(); + Configuration snapshot = new Configuration(conf); + // The AM config holds every hadoop site file loaded at startup, so + // publishing it as is would expose credentials to anyone with timeline + // read access. Mask via hadoop.security.sensitive-config-keys. + ConfigRedactor redactor = new ConfigRedactor(snapshot); + Iterator> iter = snapshot.iterator(); Map atsConf = new TreeMap(); while (iter.hasNext()) { Entry entry = iter.next(); - atsConf.put(entry.getKey(), entry.getValue()); + atsConf.put(entry.getKey(), redactor.redact(entry.getKey(), entry.getValue())); } return atsConf; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java index 2bf3746bbb..488f993f38 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.tez.client.CallerContext; @@ -222,4 +223,48 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { } } + @Test + public void testConvertConfigurationToATSMapRedactsSecrets() { + // Defaults must load: ConfigRedactor's fallback pattern is narrower than + // core-default.xml and misses fs.s3a.session.token. + Configuration conf = new Configuration(); + conf.set("tez.am.dag.scheduler.class", + "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder"); + conf.set("mapreduce.job.name", "normal-job"); + conf.set("fs.s3a.secret.key", "fake-s3a-secret-key"); + conf.set("fs.s3a.session.token", "fake-s3a-session-token"); + conf.set("ssl.server.keystore.password", "fake-keystore-password"); + conf.set("hadoop.security.credential.provider.password", "fake-credential-password"); + // An access key id is not a secret; the default pattern omits it. + conf.set("fs.s3a.access.key", "fake-s3a-access-key-id"); + + Map ats = DAGUtils.convertConfigurationToATSMap(conf); + + assertEquals("org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder", + ats.get("tez.am.dag.scheduler.class")); + assertEquals("normal-job", ats.get("mapreduce.job.name")); + assertEquals("fake-s3a-access-key-id", ats.get("fs.s3a.access.key")); + for (String key : new String[] {"fs.s3a.secret.key", "fs.s3a.session.token", + "ssl.server.keystore.password", "hadoop.security.credential.provider.password"}) { + assertFalse(ats.get(key).contains("fake"), + key + " must be redacted, got: " + ats.get(key)); + } + } + + @Test + public void testConvertConfigurationToATSMapHonoursConfiguredPattern() { + // A configured pattern replaces the defaults and must be honoured. + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS, + "my\\.site\\.internal\\.id"); + conf.set("my.site.internal.id", "fake-internal-id"); + conf.set("mapreduce.job.name", "normal-job"); + + Map ats = DAGUtils.convertConfigurationToATSMap(conf); + + assertFalse(ats.get("my.site.internal.id").contains("fake"), + "configured sensitive key must be redacted, got: " + ats.get("my.site.internal.id")); + assertEquals("normal-job", ats.get("mapreduce.job.name")); + } + }