Skip to content
This repository was archived by the owner on Jun 26, 2024. It is now read-only.

Commit c613c1e

Browse files
saxenakshitizsaxenakshitiz
authored
chore: use filter in entities query for selections as well (#201)
1 parent 77592de commit c613c1e

12 files changed

Lines changed: 709 additions & 84 deletions

File tree

‎gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.hypertrace.gateway.service.common;
22

3+
import static java.util.Collections.emptyMap;
34
import static java.util.function.Predicate.not;
5+
import static org.hypertrace.core.attribute.service.v1.AttributeSource.QS;
46

57
import com.google.common.collect.ImmutableMap;
68
import com.google.common.collect.Sets;
@@ -58,6 +60,7 @@ public class ExpressionContext {
5860
private ImmutableMap<String, List<Expression>> sourceToFilterExpressionMap;
5961
private ImmutableMap<String, Set<String>> sourceToFilterAttributeMap;
6062
private ImmutableMap<String, Set<String>> filterAttributeToSourceMap;
63+
private Map<AttributeSource, Filter> sourceToFilterMap;
6164

6265
// and filter
6366
private boolean isAndFilter;
@@ -92,6 +95,9 @@ public ExpressionContext(
9295
buildSourceToGroupByExpressionMaps();
9396

9497
this.isAndFilter = gatewayServiceConfig.isEntityAndFilterEnabled() && isAndFilter(filter);
98+
// build source to filter map only if we only have AND filter
99+
this.sourceToFilterMap =
100+
isAndFilter(filter) ? buildSourceToAndFilterMap(filter) : Collections.emptyMap();
95101
}
96102

97103
public Map<String, List<Expression>> getSourceToSelectionExpressionMap() {
@@ -106,6 +112,10 @@ public void setSourceToSelectionExpressionMap(
106112
.build();
107113
}
108114

115+
public Map<AttributeSource, Filter> getSourceToFilterMap() {
116+
return sourceToFilterMap;
117+
}
118+
109119
public Map<String, Set<String>> getSourceToSelectionAttributeMap() {
110120
return sourceToSelectionAttributeMap;
111121
}
@@ -620,6 +630,45 @@ private static Set<String> getIntersectingSourceSets(
620630
.orElse(Collections.emptySet());
621631
}
622632

633+
private Map<AttributeSource, Filter> buildSourceToAndFilterMap(Filter filter) {
634+
Operator operator = filter.getOperator();
635+
if (operator == Operator.AND) {
636+
return filter.getChildFilterList().stream()
637+
.map(this::buildSourceToAndFilterMap)
638+
.flatMap(map -> map.entrySet().stream())
639+
.collect(
640+
Collectors.toUnmodifiableMap(
641+
Map.Entry::getKey,
642+
Map.Entry::getValue,
643+
(value1, value2) ->
644+
Filter.newBuilder()
645+
.setOperator(Operator.AND)
646+
.addChildFilter(value1)
647+
.addChildFilter(value2)
648+
.build()));
649+
650+
} else if (operator == Operator.OR) {
651+
return Collections.emptyMap();
652+
} else {
653+
List<AttributeSource> attributeSources = getAttributeSources(filter.getLhs());
654+
if (attributeSources.isEmpty()) {
655+
return emptyMap();
656+
}
657+
658+
return attributeSources.contains(QS)
659+
? Map.of(QS, filter)
660+
: Map.of(attributeSources.get(0), filter);
661+
}
662+
}
663+
664+
public List<AttributeSource> getAttributeSources(Expression expression) {
665+
Set<String> attributeIds = ExpressionReader.extractAttributeIds(expression);
666+
return attributeIds.stream()
667+
.map(attributeId -> attributeMetadataMap.get(attributeId).getSourcesList())
668+
.flatMap(Collection::stream)
669+
.collect(Collectors.toUnmodifiableList());
670+
}
671+
623672
@Override
624673
public String toString() {
625674
return "ExpressionContext{"

‎gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java‎

Lines changed: 12 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,26 @@
11
package org.hypertrace.gateway.service.entity.query;
22

33
import static java.util.Collections.emptyList;
4-
import static java.util.Collections.emptyMap;
54
import static java.util.Collections.unmodifiableList;
65
import static org.hypertrace.core.attribute.service.v1.AttributeSource.EDS;
76
import static org.hypertrace.core.attribute.service.v1.AttributeSource.QS;
87

98
import com.google.common.annotations.VisibleForTesting;
109
import java.util.ArrayList;
1110
import java.util.Collection;
12-
import java.util.Collections;
1311
import java.util.HashMap;
1412
import java.util.List;
1513
import java.util.Map;
16-
import java.util.Map.Entry;
1714
import java.util.Optional;
1815
import java.util.Set;
1916
import java.util.stream.Collectors;
2017
import java.util.stream.Stream;
21-
import org.hypertrace.core.attribute.service.v1.AttributeMetadata;
2218
import org.hypertrace.core.attribute.service.v1.AttributeSource;
2319
import org.hypertrace.gateway.service.common.ExpressionContext;
24-
import org.hypertrace.gateway.service.common.util.ExpressionReader;
2520
import org.hypertrace.gateway.service.common.util.TimeRangeFilterUtil;
2621
import org.hypertrace.gateway.service.entity.query.visitor.ExecutionContextBuilderVisitor;
2722
import org.hypertrace.gateway.service.entity.query.visitor.FilterOptimizingVisitor;
2823
import org.hypertrace.gateway.service.entity.query.visitor.PrintVisitor;
29-
import org.hypertrace.gateway.service.v1.common.Expression;
3024
import org.hypertrace.gateway.service.v1.common.Filter;
3125
import org.hypertrace.gateway.service.v1.common.Operator;
3226
import org.hypertrace.gateway.service.v1.common.OrderByExpression;
@@ -39,18 +33,11 @@ public class ExecutionTreeBuilder {
3933

4034
private static final Logger LOG = LoggerFactory.getLogger(ExecutionTreeBuilder.class);
4135

42-
private final Map<String, AttributeMetadata> attributeMetadataMap;
4336
private final EntityExecutionContext executionContext;
4437
private final Set<String> sourceSetsIfFilterAndOrderByAreFromSameSourceSets;
4538

4639
public ExecutionTreeBuilder(EntityExecutionContext executionContext) {
4740
this.executionContext = executionContext;
48-
this.attributeMetadataMap =
49-
executionContext
50-
.getAttributeMetadataProvider()
51-
.getAttributesMetadata(
52-
executionContext.getEntitiesRequestContext(),
53-
executionContext.getEntitiesRequest().getEntityType());
5441

5542
this.sourceSetsIfFilterAndOrderByAreFromSameSourceSets =
5643
ExpressionContext.getSourceSetsIfFilterAndOrderByAreFromSameSourceSets(
@@ -132,7 +119,7 @@ public QueryNode build() {
132119

133120
ExecutionTreeUtils.removeDuplicateSelectionAttributes(executionContext, QS.name());
134121

135-
QueryNode filterTree = buildFilterTree(executionContext, entitiesRequest.getFilter());
122+
QueryNode filterTree = buildFilterTreeNode(executionContext, entitiesRequest.getFilter());
136123
if (LOG.isDebugEnabled()) {
137124
LOG.debug("Filter Tree:{}", filterTree.acceptVisitor(new PrintVisitor()));
138125
}
@@ -268,8 +255,7 @@ QueryNode buildExecutionTree(EntityExecutionContext executionContext, QueryNode
268255
return rootNode;
269256
}
270257

271-
@VisibleForTesting
272-
QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) {
258+
QueryNode buildFilterTreeNode(EntityExecutionContext context, Filter filter) {
273259
EntitiesRequest entitiesRequest = executionContext.getEntitiesRequest();
274260
// Convert the time range into a filter and set it on the request so that all downstream
275261
// components needn't treat it specially
@@ -281,29 +267,29 @@ QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) {
281267
entitiesRequest.getEndTimeMillis());
282268

283269
boolean isAndFilter = executionContext.getExpressionContext().isAndFilter();
284-
return isAndFilter
285-
? buildAndFilterTree(entitiesRequest)
286-
: buildFilterTree(entitiesRequest, timeRangeFilter);
270+
return isAndFilter ? buildAndFilterTree(context) : buildFilterTree(context, timeRangeFilter);
287271
}
288272

289273
@VisibleForTesting
290-
QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) {
274+
QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) {
275+
EntitiesRequest entitiesRequest = context.getEntitiesRequest();
291276
if (filter.equals(Filter.getDefaultInstance())) {
292277
return new NoOpNode();
293278
}
294279
Operator operator = filter.getOperator();
295280
if (operator == Operator.AND) {
296281
return new AndNode(
297282
filter.getChildFilterList().stream()
298-
.map(childFilter -> buildFilterTree(entitiesRequest, childFilter))
283+
.map(childFilter -> buildFilterTree(context, childFilter))
299284
.collect(Collectors.toList()));
300285
} else if (operator == Operator.OR) {
301286
return new OrNode(
302287
filter.getChildFilterList().stream()
303-
.map(childFilter -> buildFilterTree(entitiesRequest, childFilter))
288+
.map(childFilter -> buildFilterTree(context, childFilter))
304289
.collect(Collectors.toList()));
305290
} else {
306-
List<AttributeSource> sources = getAttributeSources(filter.getLhs());
291+
List<AttributeSource> sources =
292+
context.getExpressionContext().getAttributeSources(filter.getLhs());
307293
// if the filter by and order by are from QS, pagination can be pushed down to QS
308294

309295
// There will always be a DataFetcherNode for QS, because the results are always fetched
@@ -319,7 +305,8 @@ QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) {
319305
}
320306

321307
// filters and order by on QS, but you can still have selection on EDS
322-
QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) {
308+
QueryNode buildAndFilterTree(EntityExecutionContext context) {
309+
EntitiesRequest entitiesRequest = context.getEntitiesRequest();
323310
// If the filter by and order by are from QS (and selections are on other sources), pagination
324311
// can be pushed down to QS
325312
// Since the filter and order by are from QS, there won't be any filter on other
@@ -330,7 +317,7 @@ QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) {
330317
}
331318

332319
Map<AttributeSource, Filter> sourceToAndFilterMap =
333-
new HashMap<>(buildSourceToAndFilterMap(entitiesRequest.getFilter()));
320+
new HashMap<>(context.getExpressionContext().getSourceToFilterMap());
334321

335322
// qs node as the pivot node to fetch time range data
336323
QueryNode qsNode =
@@ -387,37 +374,6 @@ QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) {
387374
}
388375
}
389376

390-
private Map<AttributeSource, Filter> buildSourceToAndFilterMap(Filter filter) {
391-
Operator operator = filter.getOperator();
392-
if (operator == Operator.AND) {
393-
return filter.getChildFilterList().stream()
394-
.map(this::buildSourceToAndFilterMap)
395-
.flatMap(map -> map.entrySet().stream())
396-
.collect(
397-
Collectors.toUnmodifiableMap(
398-
Entry::getKey,
399-
Entry::getValue,
400-
(value1, value2) ->
401-
Filter.newBuilder()
402-
.setOperator(Operator.AND)
403-
.addChildFilter(value1)
404-
.addChildFilter(value2)
405-
.build()));
406-
407-
} else if (operator == Operator.OR) {
408-
return Collections.emptyMap();
409-
} else {
410-
List<AttributeSource> attributeSources = getAttributeSources(filter.getLhs());
411-
if (attributeSources.isEmpty()) {
412-
return emptyMap();
413-
}
414-
415-
return attributeSources.contains(QS)
416-
? Map.of(QS, filter)
417-
: Map.of(attributeSources.get(0), filter);
418-
}
419-
}
420-
421377
private QueryNode checkAndAddSortAndPaginationNode(
422378
QueryNode childNode, EntityExecutionContext executionContext) {
423379
EntitiesRequest entitiesRequest = executionContext.getEntitiesRequest();
@@ -479,12 +435,4 @@ private QueryNode createQsDataFetcherNodeWithLimitAndOffset(EntitiesRequest enti
479435
private QueryNode createPaginateOnlyNode(QueryNode queryNode, EntitiesRequest entitiesRequest) {
480436
return new PaginateOnlyNode(queryNode, entitiesRequest.getLimit(), entitiesRequest.getOffset());
481437
}
482-
483-
public List<AttributeSource> getAttributeSources(Expression expression) {
484-
Set<String> attributeIds = ExpressionReader.extractAttributeIds(expression);
485-
return attributeIds.stream()
486-
.map(attributeId -> attributeMetadataMap.get(attributeId).getSourcesList())
487-
.flatMap(Collection::stream)
488-
.collect(Collectors.toUnmodifiableList());
489-
}
490438
}

‎gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/visitor/ExecutionVisitor.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
import java.util.LinkedList;
1111
import java.util.List;
1212
import java.util.Map;
13+
import java.util.Optional;
1314
import java.util.Set;
1415
import java.util.concurrent.CompletableFuture;
1516
import java.util.concurrent.ExecutorService;
1617
import java.util.stream.Collectors;
1718
import java.util.stream.IntStream;
19+
import org.hypertrace.core.attribute.service.v1.AttributeSource;
1820
import org.hypertrace.gateway.service.common.datafetcher.EntityFetcherResponse;
1921
import org.hypertrace.gateway.service.common.datafetcher.EntityResponse;
2022
import org.hypertrace.gateway.service.common.datafetcher.IEntityFetcher;
@@ -265,7 +267,7 @@ public EntityResponse visit(SelectionNode selectionNode) {
265267
.getExpressionContext()
266268
.getSourceToSelectionExpressionMap()
267269
.get(source))
268-
.setFilter(filter)
270+
.setFilter(addSourceFilters(executionContext, source, filter))
269271
.build();
270272
IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source);
271273
EntitiesRequestContext context =
@@ -295,7 +297,7 @@ public EntityResponse visit(SelectionNode selectionNode) {
295297
.getExpressionContext()
296298
.getSourceToMetricExpressionMap()
297299
.get(source))
298-
.setFilter(filter)
300+
.setFilter(addSourceFilters(executionContext, source, filter))
299301
.build();
300302
IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source);
301303
EntitiesRequestContext context =
@@ -325,7 +327,7 @@ public EntityResponse visit(SelectionNode selectionNode) {
325327
.getExpressionContext()
326328
.getSourceToTimeAggregationMap()
327329
.get(source))
328-
.setFilter(filter)
330+
.setFilter(addSourceFilters(executionContext, source, filter))
329331
.build();
330332
IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source);
331333
EntitiesRequestContext requestContext =
@@ -355,6 +357,25 @@ public EntityResponse visit(SelectionNode selectionNode) {
355357
}
356358
}
357359

360+
private Filter addSourceFilters(
361+
EntityExecutionContext executionContext, String source, Filter filter) {
362+
Optional<Filter> sourceFilterOptional =
363+
Optional.ofNullable(
364+
executionContext
365+
.getExpressionContext()
366+
.getSourceToFilterMap()
367+
.get(AttributeSource.valueOf(source)));
368+
return sourceFilterOptional
369+
.map(
370+
sourceFilter ->
371+
Filter.newBuilder()
372+
.setOperator(Operator.AND)
373+
.addChildFilter(filter)
374+
.addChildFilter(sourceFilter)
375+
.build())
376+
.orElse(filter);
377+
}
378+
358379
Filter constructFilterFromChildNodesResult(EntityFetcherResponse result) {
359380
if (result.isEmpty()) {
360381
return Filter.getDefaultInstance();

‎gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/AbstractServiceTest.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import java.io.Reader;
2020
import java.util.ArrayList;
2121
import java.util.Arrays;
22-
import java.util.Collections;
2322
import java.util.HashMap;
2423
import java.util.HashSet;
2524
import java.util.Iterator;
@@ -39,6 +38,7 @@
3938
import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs;
4039
import org.hypertrace.gateway.service.common.util.QueryServiceClient;
4140
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
41+
import org.hypertrace.gateway.service.entity.config.LogConfig;
4242
import org.junit.jupiter.api.Assertions;
4343
import org.junit.jupiter.api.BeforeAll;
4444
import org.junit.jupiter.params.ParameterizedTest;
@@ -82,14 +82,19 @@ public static void setUp() throws IOException {
8282
+ " }\n"
8383
+ " ]\n"
8484
+ " }\n"
85-
+ "]";
85+
+ "]\n"
86+
+ "entity.service.log.config = {\n"
87+
+ " query.threshold.millis = 1500\n"
88+
+ "}\n";
8689
Config config = ConfigFactory.parseString(scopeFiltersConfig);
8790
scopeFilterConfigs = new ScopeFilterConfigs(config);
88-
entityIdColumnsConfig = new EntityIdColumnsConfig(Collections.emptyMap());
91+
entityIdColumnsConfig = new EntityIdColumnsConfig(Map.of("BACKEND", "id"));
8992
gatewayServiceConfig = mock(GatewayServiceConfig.class);
9093
when(gatewayServiceConfig.getEntityIdColumnsConfig()).thenReturn(entityIdColumnsConfig);
9194
when(gatewayServiceConfig.getScopeFilterConfigs()).thenReturn(scopeFilterConfigs);
9295
entityTypesProvider = mock(EntityTypesProvider.class);
96+
LogConfig logConfig = new LogConfig(config);
97+
when(gatewayServiceConfig.getLogConfig()).thenReturn(logConfig);
9398
}
9499

95100
private static Reader readResourceFile(String fileName) {

‎gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/converter/EntityServiceAndGatewayServiceConverterTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@
2424
import org.hypertrace.gateway.service.v1.entity.EntitiesRequest;
2525
import org.junit.jupiter.api.Test;
2626

27-
public class EntityServiceAndGatewayServiceConverterTest extends AbstractGatewayServiceTest {
27+
class EntityServiceAndGatewayServiceConverterTest extends AbstractGatewayServiceTest {
2828

2929
@Test
30-
public void testAddBetweenFilter() {
30+
void testAddBetweenFilter() {
3131
int startTimeMillis = 1;
3232
int endTimeMillis = 2;
33-
String timestamp = "lastActivity";
33+
String timestamp = "startTime";
3434
String timestampAttributeName = BACKEND.name() + "." + timestamp;
3535

3636
Expression.Builder expectedStartTimeConstant =

0 commit comments

Comments
 (0)