Skip to content

Commit f0f194c

Browse files
avinashkollururish691
avinashkolluru
and
rish691
authored
Add support for filtering root spans matching a criteria (#231)
 * Add support for dropping/processing root exit spans with exclusion criteria Co-authored-by: rish691 <rishabh@traceable.ai>
1 parent 35fdfb3 commit f0f194c

4 files changed

Lines changed: 262 additions & 35 deletions

File tree

‎span-normalizer/helm/templates/span-normalizer-config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ data:
5656
{{- if hasKey .Values.spanNormalizerConfig.processor "spanDropCriterion" }}
5757
spanDropCriterion = {{ .Values.spanNormalizerConfig.processor.spanDropCriterion | toJson }}
5858
{{- end }}
59+
60+
{{- if hasKey .Values.spanNormalizerConfig.processor "rootExitSpanDropCriterion" }}
61+
rootExitSpanDropCriterion = {{ .Values.spanNormalizerConfig.processor.rootExitSpanDropCriterion | toJson }}
62+
{{- end }}
5963
}
6064
{{- end }}
6165

‎span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessor.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,10 @@
2222
public class JaegerSpanPreProcessor
2323
implements Transformer<byte[], Span, KeyValue<byte[], PreProcessedSpan>> {
2424

25+
static final String SPANS_COUNTER = "hypertrace.reported.spans";
2526
private static final Logger LOG = LoggerFactory.getLogger(JaegerSpanPreProcessor.class);
26-
2727
private static final ConcurrentMap<String, Counter> statusToSpansCounter =
2828
new ConcurrentHashMap<>();
29-
static final String SPANS_COUNTER = "hypertrace.reported.spans";
30-
3129
private TenantIdHandler tenantIdHandler;
3230
private SpanFilter spanFilter;
3331

@@ -83,23 +81,23 @@ public KeyValue<byte[], PreProcessedSpan> transform(byte[] key, Span value) {
8381
}
8482

8583
@VisibleForTesting
86-
PreProcessedSpan preProcessSpan(Span value) {
84+
PreProcessedSpan preProcessSpan(Span span) {
8785
Map<String, JaegerSpanInternalModel.KeyValue> tags =
88-
value.getTagsList().stream()
86+
span.getTagsList().stream()
8987
.collect(Collectors.toMap(t -> t.getKey().toLowerCase(), t -> t, (v1, v2) -> v2));
9088

91-
Optional<String> maybeTenantId = tenantIdHandler.getAllowedTenantId(value, tags);
89+
Optional<String> maybeTenantId = tenantIdHandler.getAllowedTenantId(span, tags);
9290
if (maybeTenantId.isEmpty()) {
9391
return null;
9492
}
9593

9694
String tenantId = maybeTenantId.get();
9795

98-
if (spanFilter.shouldDropSpan(tags)) {
96+
if (spanFilter.shouldDropSpan(span, tags)) {
9997
return null;
10098
}
10199

102-
return new PreProcessedSpan(tenantId, value);
100+
return new PreProcessedSpan(tenantId, span);
103101
}
104102

105103
@Override

‎span-normalizer/span-normalizer/src/main/java/org/hypertrace/core/spannormalizer/jaeger/SpanFilter.java‎

Lines changed: 77 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.hypertrace.core.spannormalizer.jaeger;
22

33
import com.typesafe.config.Config;
4-
import io.jaegertracing.api_v2.JaegerSpanInternalModel.KeyValue;
4+
import io.jaegertracing.api_v2.JaegerSpanInternalModel;
55
import java.util.Arrays;
66
import java.util.Collections;
77
import java.util.List;
@@ -11,12 +11,16 @@
1111
import javax.annotation.Nullable;
1212
import org.apache.commons.lang3.StringUtils;
1313
import org.apache.commons.lang3.tuple.Pair;
14+
import org.hypertrace.core.span.constants.RawSpanConstants;
15+
import org.hypertrace.core.span.constants.v1.SpanAttribute;
1416
import org.slf4j.Logger;
1517
import org.slf4j.LoggerFactory;
1618

1719
public class SpanFilter {
18-
1920
private static final Logger LOG = LoggerFactory.getLogger(SpanFilter.class);
21+
private static final String SPAN_KIND_TAG =
22+
RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND);
23+
private static final String SPAN_KIND_CLIENT = "client";
2024
/**
2125
* Config key using which a list of criterion can be specified to drop the matching spans. Any
2226
* span matching any one of the criterion is dropped. Each criteria is a comma separated list of
@@ -28,34 +32,69 @@ public class SpanFilter {
2832
*/
2933
private static final String SPAN_DROP_CRITERION_CONFIG = "processor.spanDropCriterion";
3034

35+
public static final String ROOT_SPAN_DROP_CRITERION_CONFIG =
36+
"processor.rootExitSpanDropCriterion";
37+
private static final String ROOT_SPAN_ALWAYS_DROP = "alwaysDrop";
38+
private static final String ROOT_SPAN_DROP_EXCLUSIONS = "exclusionsMatchCriterion";
39+
3140
private static final String COMMA = ",";
3241
private static final String COLON = ":";
3342

34-
private final List<List<Pair<String, String>>> spanDropCriterion;
43+
private List<List<Pair<String, String>>> spanDropCriterion = Collections.emptyList();
44+
private boolean alwaysDropRootSpan = false;
45+
private List<List<Pair<String, String>>> rootSpanDropExclusionCriterion = Collections.emptyList();
3546

3647
public SpanFilter(Config config) {
48+
if (config.hasPath(SPAN_DROP_CRITERION_CONFIG)) {
49+
List<String> criterion = config.getStringList(SPAN_DROP_CRITERION_CONFIG);
50+
LOG.info("Span drop criterion: {}", criterion);
51+
// Parse the config to see if there is any criteria to drop spans.
52+
this.spanDropCriterion = parseStringList(criterion);
53+
}
3754

38-
List<String> criterion =
39-
config.hasPath(SPAN_DROP_CRITERION_CONFIG)
40-
? config.getStringList(SPAN_DROP_CRITERION_CONFIG)
41-
: Collections.emptyList();
55+
if (config.hasPath(ROOT_SPAN_DROP_CRITERION_CONFIG)) {
56+
Config rootSpanDropCriterionConfig = config.getConfig(ROOT_SPAN_DROP_CRITERION_CONFIG);
57+
LOG.info("Root Span drop criterion: {}", rootSpanDropCriterionConfig);
58+
this.alwaysDropRootSpan =
59+
rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_ALWAYS_DROP)
60+
&& rootSpanDropCriterionConfig.getBoolean(ROOT_SPAN_ALWAYS_DROP);
61+
List<String> exclusionList =
62+
rootSpanDropCriterionConfig.hasPath(ROOT_SPAN_DROP_EXCLUSIONS)
63+
? rootSpanDropCriterionConfig.getStringList(ROOT_SPAN_DROP_EXCLUSIONS)
64+
: Collections.emptyList();
65+
// Parse the config to see if there is any criteria to drop spans.
66+
this.rootSpanDropExclusionCriterion = parseStringList(exclusionList);
67+
}
68+
}
4269

43-
// Parse the config to see if there is any criteria to drop spans.
44-
this.spanDropCriterion =
45-
criterion.stream()
46-
// Split each criteria based on comma
47-
.map(s -> s.split(COMMA))
48-
.map(
49-
a ->
50-
Arrays.stream(a)
51-
.map(this::convertToPair)
52-
.filter(Objects::nonNull)
53-
.collect(Collectors.toList()))
54-
.collect(Collectors.toList());
70+
private List<List<Pair<String, String>>> parseStringList(List<String> stringList) {
71+
return stringList.stream()
72+
// Split each criteria based on comma
73+
.map(s -> s.split(COMMA))
74+
.map(
75+
a ->
76+
Arrays.stream(a)
77+
.map(this::convertToPair)
78+
.filter(Objects::nonNull)
79+
.collect(Collectors.toList()))
80+
.collect(Collectors.toList());
81+
}
5582

56-
if (!this.spanDropCriterion.isEmpty()) {
57-
LOG.info("Span drop criterion: {}", this.spanDropCriterion);
83+
/**
84+
* Method to check if the given span attributes match any of the drop criterion. Returns true if
85+
* the span should be dropped, false otherwise.
86+
*/
87+
public boolean shouldDropSpan(
88+
JaegerSpanInternalModel.Span span, Map<String, JaegerSpanInternalModel.KeyValue> tags) {
89+
if (anyCriteriaMatch(tags, spanDropCriterion)) {
90+
return true;
5891
}
92+
93+
if (isRootExitSpan(span, tags)) {
94+
boolean anyCriteriaMatch = anyCriteriaMatch(tags, rootSpanDropExclusionCriterion);
95+
return (alwaysDropRootSpan && !anyCriteriaMatch) || (!alwaysDropRootSpan && anyCriteriaMatch);
96+
}
97+
return false;
5998
}
6099

61100
@Nullable
@@ -69,12 +108,10 @@ private Pair<String, String> convertToPair(String s) {
69108
return null;
70109
}
71110

72-
/**
73-
* Method to check if the given span attributes match any of the drop criterion. Returns true if
74-
* the span should be dropped, false otherwise.
75-
*/
76-
public boolean shouldDropSpan(Map<String, KeyValue> tags) {
77-
return this.spanDropCriterion.stream()
111+
private boolean anyCriteriaMatch(
112+
Map<String, JaegerSpanInternalModel.KeyValue> tags,
113+
List<List<Pair<String, String>>> criteriaList) {
114+
return criteriaList.stream()
78115
.anyMatch(
79116
l ->
80117
l.stream()
@@ -84,4 +121,17 @@ public boolean shouldDropSpan(Map<String, KeyValue> tags) {
84121
&& StringUtils.equals(
85122
tags.get(p.getLeft()).getVStr(), p.getRight())));
86123
}
124+
125+
private boolean isRootExitSpan(
126+
JaegerSpanInternalModel.Span span, Map<String, JaegerSpanInternalModel.KeyValue> tags) {
127+
if (!span.getReferencesList().isEmpty()) {
128+
return false;
129+
}
130+
JaegerSpanInternalModel.KeyValue spanKindKeyValue = tags.get(SPAN_KIND_TAG);
131+
if (null == spanKindKeyValue) {
132+
return false;
133+
}
134+
135+
return SPAN_KIND_CLIENT.equals(spanKindKeyValue.getVStr());
136+
}
87137
}

‎span-normalizer/span-normalizer/src/test/java/org/hypertrace/core/spannormalizer/jaeger/JaegerSpanPreProcessorTest.java‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import java.util.List;
99
import java.util.Map;
1010
import java.util.Random;
11+
import org.hypertrace.core.span.constants.RawSpanConstants;
12+
import org.hypertrace.core.span.constants.v1.SpanAttribute;
1113
import org.junit.jupiter.api.Assertions;
1214
import org.junit.jupiter.api.Test;
1315

@@ -179,6 +181,179 @@ public void testDropSpanWithEmptyCriterion() {
179181
Assertions.assertNotNull(preProcessedSpan);
180182
}
181183

184+
@Test
185+
public void testDropSpan_RootSpan_EmptyExclusionList() {
186+
String tenantId = "tenant-" + random.nextLong();
187+
Map<String, Object> configs = new HashMap<>(getCommonConfig());
188+
configs.putAll(
189+
Map.of(
190+
"processor",
191+
Map.of(
192+
"tenantIdTagKey", "tenant-key",
193+
"rootExitSpanDropCriterion.alwaysDrop", "true")));
194+
195+
JaegerSpanPreProcessor jaegerSpanPreProcessor =
196+
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
197+
Process process = Process.newBuilder().setServiceName("testService").build();
198+
199+
// root exit span
200+
Span span =
201+
Span.newBuilder()
202+
.setProcess(process)
203+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
204+
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
205+
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
206+
.addTags(
207+
KeyValue.newBuilder()
208+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
209+
.setVStr("client")
210+
.build())
211+
.build();
212+
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
213+
Assertions.assertNull(preProcessedSpan);
214+
215+
span =
216+
Span.newBuilder()
217+
.setProcess(process)
218+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
219+
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
220+
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
221+
.addTags(
222+
KeyValue.newBuilder()
223+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
224+
.setVStr("server")
225+
.build())
226+
.build();
227+
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
228+
Assertions.assertNotNull(preProcessedSpan);
229+
}
230+
231+
/** Always drop except when there is a match from the exclusion list */
232+
@Test
233+
public void testDropSpan_RootSpan_AlwaysDrop_ExclusionList() {
234+
String tenantId = "tenant-" + random.nextLong();
235+
Map<String, Object> configs = new HashMap<>(getCommonConfig());
236+
configs.putAll(
237+
Map.of(
238+
"processor",
239+
Map.of(
240+
"tenantIdTagKey", "tenant-key",
241+
"rootExitSpanDropCriterion.alwaysDrop", "true",
242+
"rootExitSpanDropCriterion.exclusionsMatchCriterion",
243+
List.of("foo:bar,k1:v1", "k2:v2"))));
244+
245+
JaegerSpanPreProcessor jaegerSpanPreProcessor =
246+
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
247+
Process process = Process.newBuilder().setServiceName("testService").build();
248+
249+
// root exit span
250+
Span span =
251+
Span.newBuilder()
252+
.setProcess(process)
253+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
254+
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
255+
.addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build())
256+
.addTags(
257+
KeyValue.newBuilder()
258+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
259+
.setVStr("client")
260+
.build())
261+
.build();
262+
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
263+
Assertions.assertNotNull(preProcessedSpan);
264+
265+
span =
266+
Span.newBuilder()
267+
.setProcess(process)
268+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
269+
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
270+
.addTags(
271+
KeyValue.newBuilder()
272+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
273+
.setVStr("client")
274+
.build())
275+
.build();
276+
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
277+
Assertions.assertNotNull(preProcessedSpan);
278+
279+
span =
280+
Span.newBuilder()
281+
.setProcess(process)
282+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
283+
.addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build())
284+
.addTags(
285+
KeyValue.newBuilder()
286+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
287+
.setVStr("client")
288+
.build())
289+
.build();
290+
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
291+
Assertions.assertNull(preProcessedSpan);
292+
}
293+
294+
/** Always keep except when there is a match from the exclusion list */
295+
@Test
296+
public void testDropSpan_RootSpan_NotAlwaysDrop_ExclusionList() {
297+
String tenantId = "tenant-" + random.nextLong();
298+
Map<String, Object> configs = new HashMap<>(getCommonConfig());
299+
configs.putAll(
300+
Map.of(
301+
"processor",
302+
Map.of(
303+
"tenantIdTagKey", "tenant-key",
304+
"rootExitSpanDropCriterion.alwaysDrop", "false",
305+
"rootExitSpanDropCriterion.exclusionsMatchCriterion",
306+
List.of("foo:bar,k1:v1", "k2:v2"))));
307+
308+
JaegerSpanPreProcessor jaegerSpanPreProcessor =
309+
new JaegerSpanPreProcessor(ConfigFactory.parseMap(configs));
310+
Process process = Process.newBuilder().setServiceName("testService").build();
311+
312+
// root exit span
313+
Span span =
314+
Span.newBuilder()
315+
.setProcess(process)
316+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
317+
.addTags(KeyValue.newBuilder().setKey("foo").setVStr("bar").build())
318+
.addTags(KeyValue.newBuilder().setKey("k1").setVStr("v1").build())
319+
.addTags(
320+
KeyValue.newBuilder()
321+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
322+
.setVStr("client")
323+
.build())
324+
.build();
325+
PreProcessedSpan preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
326+
Assertions.assertNull(preProcessedSpan);
327+
328+
span =
329+
Span.newBuilder()
330+
.setProcess(process)
331+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
332+
.addTags(KeyValue.newBuilder().setKey("k2").setVStr("v2").build())
333+
.addTags(
334+
KeyValue.newBuilder()
335+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
336+
.setVStr("client")
337+
.build())
338+
.build();
339+
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
340+
Assertions.assertNull(preProcessedSpan);
341+
342+
span =
343+
Span.newBuilder()
344+
.setProcess(process)
345+
.addTags(KeyValue.newBuilder().setKey("tenant-key").setVStr(tenantId).build())
346+
.addTags(KeyValue.newBuilder().setKey("k3").setVStr("v3").build())
347+
.addTags(
348+
KeyValue.newBuilder()
349+
.setKey(RawSpanConstants.getValue(SpanAttribute.SPAN_ATTRIBUTE_SPAN_KIND))
350+
.setVStr("client")
351+
.build())
352+
.build();
353+
preProcessedSpan = jaegerSpanPreProcessor.preProcessSpan(span);
354+
Assertions.assertNotNull(preProcessedSpan);
355+
}
356+
182357
private Map<String, Object> getCommonConfig() {
183358
return Map.of(
184359
"span.type",

0 commit comments

Comments
 (0)