Skip to content

Add lite runtime to conformance test suite #1367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Oct 21, 2026
Merged

Add lite runtime to conformance test suite #1367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 100 additions & 2 deletions conformance/src/test/java/dev/cel/conformance/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
load("@rules_java//java:defs.bzl", "java_library")
load("//conformance/src/test/java/dev/cel/conformance:conformance_test.bzl", "MODE", "conformance_test")
load("//conformance/src/test/java/dev/cel/conformance:conformance_test.bzl", "MODE", "RUNTIME", "conformance_test")

package(default_applicable_licenses = [
"//:license",
Expand All @@ -26,17 +26,33 @@ java_library(
"//common/ast:cel_block",
"//common/types",
"//common/types:cel_proto_types",
"//common/types:message_type_provider",
"//common/types:type_providers",
"//common/values:cel_value_provider",
"//common/values:proto_message_lite_value_provider",
"//compiler",
"//compiler:compiler_builder",
"//extensions",
"//extensions:bindings",
"//extensions:comprehensions_runtime_library",
"//extensions:encoders_runtime_library",
"//extensions:lists_runtime_library",
"//extensions:math_runtime_library",
"//extensions:optional_library",
"//extensions:sets_runtime_library",
"//extensions:strings_runtime_library",
"//parser:macro",
"//parser:parser_builder",
"//parser:parser_factory",
"//runtime",
"//runtime:lite_runtime",
"//runtime:lite_runtime_factory",
"//runtime:program",
"//runtime:runtime_planner_impl",
"//runtime:standard_functions",
"//testing:expr_value_utils",
"//testing/protos:test_all_types_cel_java_proto2",
"//testing/protos:test_all_types_cel_java_proto3",
"@cel_spec//proto/cel/expr:expr_java_proto",
"@cel_spec//proto/cel/expr/conformance/proto2:test_all_types_java_proto",
"@cel_spec//proto/cel/expr/conformance/proto3:test_all_types_java_proto",
Expand All @@ -63,10 +79,15 @@ java_library(
tags = ["conformance_maven"],
deps = MAVEN_JAR_DEPS + [
"//:java_truth",
"//common/values:proto_message_lite_value_provider",
"//compiler:compiler_builder",
"//parser:parser_factory",
"//runtime:lite_runtime",
"//runtime:lite_runtime_factory",
"//runtime:runtime_planner_impl",
"//testing:expr_value_utils",
"//testing/protos:test_all_types_cel_java_proto2",
"//testing/protos:test_all_types_cel_java_proto3",
"@cel_spec//proto/cel/expr:expr_java_proto",
"@cel_spec//proto/cel/expr/conformance/proto2:test_all_types_java_proto",
"@cel_spec//proto/cel/expr/conformance/proto3:test_all_types_java_proto",
Expand Down Expand Up @@ -168,6 +189,76 @@ _TESTS_TO_SKIP_PLANNER = [
"enums/strong_proto3",
]

_TESTS_TO_SKIP_LITE = _TESTS_TO_SKIP_PLANNER + [
# Optionals are not supported in CelLiteRuntime.
"optionals",
"block_ext/basic/optional_list,optional_map,optional_map_chained,optional_message",

# Proto2 extensions are not supported in CelLiteRuntime.
"proto2_ext",
"proto2/extensions_has",
"proto2/extensions_get",

# Custom proto2 default values on unset fields require SelectOptimization in CelLiteRuntime.
"proto2/empty_field/scalar_with_default",

# TODO: Proto equality after dyn() unwrapping is not yet supported in
# CelLiteRuntime.
"comparisons/eq_literal/not_eq_dyn_proto2_msg_null,not_eq_dyn_proto3_msg_null",
"comparisons/eq_wrapper/eq_proto_different_types",
"comparisons/ne_literal/ne_proto_different_types",

# TODO: google.protobuf.Any unpacking is not yet supported in CelLiteRuntime.
"dynamic/any/literal_empty,var",

# TODO: Message creation with prepopulated fields is not yet supported in
# CelLiteRuntime.
"comparisons/eq_wrapper/eq_bool,eq_bytes,eq_double,eq_float,eq_int32,eq_int64,eq_string,eq_uint32,eq_uint64",
"comparisons/eq_wrapper/eq_proto2,eq_proto3,eq_proto2_missing_fields_neq,eq_proto3_missing_fields_neq,eq_proto_nan_equal",
"comparisons/eq_wrapper/eq_proto2_any_unpack_equal,eq_proto2_any_unpack_not_equal,eq_proto2_any_unpack_bytewise_fallback_not_equal,eq_proto2_any_unpack_bytewise_fallback_equal,eq_proto3_any_unpack_equal,eq_proto3_any_unpack_not_equal,eq_proto3_any_unpack_bytewise_fallback_not_equal,eq_proto3_any_unpack_bytewise_fallback_equal",
"comparisons/ne_literal/ne_proto2,ne_proto3,ne_proto2_missing_fields_neq,ne_proto3_missing_fields_neq,ne_proto_nan_not_equal",
"comparisons/ne_literal/ne_proto2_any_unpack,ne_proto2_any_unpack_bytewise_fallback,ne_proto3_any_unpack,ne_proto3_any_unpack_bytewise_fallback",
"dynamic/int32/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto2_max,field_assign_proto2_min,field_assign_proto2_range,field_read_proto2,field_read_proto2_zero,field_assign_proto3,field_assign_proto3_zero,field_assign_proto3_max,field_assign_proto3_min,field_assign_proto3_range,field_read_proto3,field_read_proto3_zero",
"dynamic/int64/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto3,field_assign_proto3_zero",
"dynamic/uint32/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto2_max,field_assign_proto2_range,field_assign_proto3,field_assign_proto3_zero,field_assign_proto3_max,field_assign_proto3_range,field_read_proto2,field_read_proto2_zero",
"dynamic/uint64/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto3,field_assign_proto3_zero,field_read_proto2,field_read_proto2_zero",
"dynamic/float/literal,literal_not_double,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto2_subnorm,field_assign_proto2_round_to_zero,field_assign_proto2_range,field_read_proto2,field_read_proto2_zero,field_assign_proto3,field_assign_proto3_zero,field_assign_proto3_subnorm,field_assign_proto3_round_to_zero,field_assign_proto3_range,field_read_proto3,field_read_proto3_zero",
"dynamic/double/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_zero,field_assign_proto2_range,field_read_proto2,field_read_proto2_zero,field_assign_proto3,field_assign_proto3_zero,field_assign_proto3_range,field_read_proto3,field_read_proto3_zero",
"dynamic/bool/literal,literal_no_field_access,field_assign_proto2,field_assign_proto2_false,field_assign_proto3,field_assign_proto3_false",
"dynamic/string/literal,literal_no_field_access,literal_unicode,field_assign_proto2,field_assign_proto2_empty,field_assign_proto3,field_assign_proto3_empty",
"dynamic/bytes/literal,literal_no_field_access,literal_unicode,field_assign_proto2,field_assign_proto2_empty,field_assign_proto3,field_assign_proto3_empty",
"dynamic/list/literal,literal_no_field_access,literal_empty,field_assign_proto2,field_assign_proto2_empty,field_read_proto2,field_read_proto2_empty,field_assign_proto3,field_assign_proto3_empty,field_read_proto3,field_read_proto3_empty",
"dynamic/struct/literal,literal_no_field_access,literal_empty,field_assign_proto2,field_assign_proto2_empty,field_assign_proto2_bad,field_read_proto2,field_read_proto2_empty,field_assign_proto3,field_assign_proto3_empty,field_assign_proto3_bad,field_read_proto3,field_read_proto3_empty",
"dynamic/value_null/literal,literal_no_field_access,field_assign_proto2,field_read_proto2,field_assign_proto3,field_read_proto3",
"dynamic/value_number/literal,literal_no_field_access,literal_zero,field_assign_proto2,field_assign_proto2_zero,field_read_proto2,field_read_proto2_zero,field_assign_proto3,field_assign_proto3_zero,field_read_proto3,field_read_proto3_zero",
"dynamic/value_string/literal,literal_no_field_access,literal_empty,field_assign_proto2,field_assign_proto2_empty,field_read_proto2,field_read_proto2_zero,field_assign_proto3,field_assign_proto3_empty,field_read_proto3,field_read_proto3_zero",
"dynamic/value_bool/literal,literal_no_field_access,literal_false,field_assign_proto2,field_assign_proto2_false,field_read_proto2,field_read_proto2_false,field_assign_proto3,field_assign_proto3_false,field_read_proto3,field_read_proto3_false",
"dynamic/value_struct/literal,literal_no_field_access,literal_empty,field_assign_proto2,field_assign_proto2_empty,field_read_proto2,field_read_proto2_empty,field_assign_proto3,field_assign_proto3_empty,field_read_proto3,field_read_proto3_empty",
"dynamic/value_list/literal,literal_no_field_access,literal_empty,field_assign_proto2,field_assign_proto2_empty,field_read_proto2,field_read_proto2_empty,field_assign_proto3,field_assign_proto3_empty,field_read_proto3,field_read_proto3_empty",
"dynamic/any/literal,literal_no_field_access,field_assign_proto2,field_read_proto2,field_assign_proto3,field_read_proto3",
"dynamic/complex",
"enums/legacy_proto2/assign_standalone_name,assign_standalone_int,assign_standalone_int_too_big,assign_standalone_int_too_neg,assign_repeated_enum,list_enum_as_list_int,enum_as_int",
"enums/legacy_proto3/assign_standalone_name,assign_standalone_int,assign_standalone_int_big,assign_standalone_int_neg,assign_standalone_int_too_big,assign_standalone_int_too_neg,assign_repeated_enum,list_enum_as_list_int,enum_as_int",
"parse/nest/message_literal",
"parse/repeat/message_literal",
"parse/whitespace",
"parse/comments",
"parse/struct_field_names",
"proto2/literal_singular",
"proto2/literal_wellknown",
"proto2/set_null",
"proto2/quoted_fields",
"proto2/has/repeated_none_explicit,repeated_one,repeated_many,map_none_explicit,map_one_default,map_one,map_many,required,optional_set_no_default,optional_set_with_default,optional_set_to_default,optional_message_set,optional_enum_set,optional_enum_set_zero,oneof_other_set,oneof_set,oneof_set_default",
"proto3/literal_singular",
"proto3/literal_wellknown",
"proto3/set_null",
"proto3/quoted_fields",
"proto3/has/repeated_none_explicit,repeated_one,repeated_many,map_none_explicit,map_one_default,map_one,map_many,single_set,single_set_to_default,single_message_set,single_message_set_to_default,single_enum_set,single_enum_set_zero,oneof_other_set,oneof_set,oneof_set_default",
"type_deductions/complex_initializers/struct",
"type_deductions/field_access/int_field",
"wrappers",
]

conformance_test(
name = "conformance",
data = _ALL_TESTS,
Expand All @@ -190,6 +281,13 @@ conformance_test(
conformance_test(
name = "conformance_planner",
data = _ALL_TESTS,
runtime = RUNTIME.PLANNER,
skip_tests = _TESTS_TO_SKIP_PLANNER,
use_planner = True,
)

conformance_test(
name = "conformance_lite",
data = _ALL_TESTS,
runtime = RUNTIME.LITE,
skip_tests = _TESTS_TO_SKIP_LITE,
)
135 changes: 95 additions & 40 deletions conformance/src/test/java/dev/cel/conformance/ConformanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package dev.cel.conformance;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
import static dev.cel.testing.utils.ExprValueUtils.fromValue;
Expand All @@ -24,9 +25,9 @@
import dev.cel.expr.MapValue;
import dev.cel.expr.Type;
import dev.cel.expr.Value;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.Descriptors.FileDescriptor;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.TypeRegistry;
import dev.cel.checker.CelChecker;
Expand All @@ -44,13 +45,23 @@
import dev.cel.common.ast.CelConstant;
import dev.cel.common.ast.CelExpr;
import dev.cel.common.types.CelProtoTypes;
import dev.cel.common.types.CelTypeProvider;
import dev.cel.common.types.ProtoMessageTypeProvider;
import dev.cel.common.types.SimpleType;
import dev.cel.common.values.CelValueProvider;
import dev.cel.common.values.ProtoMessageLiteValueProvider;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.compiler.CelCompilerLibrary;
import dev.cel.expr.conformance.test.SimpleTest;
import dev.cel.extensions.CelBindingsExtensions;
import dev.cel.extensions.CelComprehensionsRuntimeLibrary;
import dev.cel.extensions.CelEncoderRuntimeLibrary;
import dev.cel.extensions.CelExtensions;
import dev.cel.extensions.CelListsRuntimeLibrary;
import dev.cel.extensions.CelMathRuntimeLibrary;
import dev.cel.extensions.CelOptionalLibrary;
import dev.cel.extensions.CelSetsRuntimeLibrary;
import dev.cel.extensions.CelStringRuntimeLibrary;
import dev.cel.parser.CelMacro;
import dev.cel.parser.CelMacroExpander;
import dev.cel.parser.CelMacroExprFactory;
Expand All @@ -59,12 +70,14 @@
import dev.cel.parser.CelParserFactory;
import dev.cel.parser.CelStandardMacro;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntime.Program;
import dev.cel.runtime.CelLiteRuntimeFactory;
import dev.cel.runtime.CelLiteRuntimeLibrary;
import dev.cel.runtime.CelRuntimeBuilder;
import dev.cel.runtime.CelRuntimeFactory;
import dev.cel.runtime.CelRuntimeImpl;
import dev.cel.runtime.CelRuntimeLibrary;
import dev.cel.runtime.CelStandardFunctions;
import dev.cel.runtime.Program;
import java.util.Map;
import java.util.Optional;
import org.junit.runners.model.Statement;
Expand Down Expand Up @@ -104,6 +117,40 @@ public final class ConformanceTest extends Statement {
CelExtensions.strings(),
CelOptionalLibrary.INSTANCE);

private static final ImmutableList<CelLiteRuntimeLibrary> CANONICAL_LITE_RUNTIME_EXTENSIONS =
ImmutableList.of(
CelComprehensionsRuntimeLibrary.comprehensions(),
CelEncoderRuntimeLibrary.encoders(),
CelListsRuntimeLibrary.lists(),
CelMathRuntimeLibrary.math(),
CelSetsRuntimeLibrary.sets(),
CelStringRuntimeLibrary.strings());

private static final CelValueProvider LITE_VALUE_PROVIDER =
ProtoMessageLiteValueProvider.newInstance(
dev.cel.expr.conformance.proto2.TestAllTypesCelDescriptor.getDescriptor(),
dev.cel.expr.conformance.proto2.NestedTestAllTypesCelDescriptor.getDescriptor(),
dev.cel.expr.conformance.proto2.TestRequiredCelDescriptor.getDescriptor(),
dev.cel.expr.conformance.proto3.TestAllTypesCelDescriptor.getDescriptor(),
dev.cel.expr.conformance.proto3.NestedTestAllTypesCelDescriptor.getDescriptor());

private static final ImmutableList<FileDescriptor> CONFORMANCE_FILE_DESCRIPTORS =
ImmutableList.of(
dev.cel.expr.conformance.proto2.TestAllTypes.getDescriptor().getFile(),
dev.cel.expr.conformance.proto3.TestAllTypes.getDescriptor().getFile(),
dev.cel.expr.conformance.proto2.TestAllTypesExtensions.getDescriptor());

private static final CelTypeProvider LITE_TYPE_PROVIDER =
ProtoMessageTypeProvider.newBuilder()
.addFileDescriptors(CONFORMANCE_FILE_DESCRIPTORS)
.build();

enum RuntimeType {
LEGACY,
PLANNER,
LITE
}

static final TypeRegistry CONFORMANCE_TYPE_REGISTRY =
TypeRegistry.newBuilder()
.add(dev.cel.expr.conformance.proto2.TestAllTypes.getDescriptor())
Expand Down Expand Up @@ -147,33 +194,11 @@ private static CelChecker getChecker(SimpleTest test) throws Exception {
.setOptions(OPTIONS)
.setContainer(CelContainer.ofName(test.getContainer()))
.addDeclarations(decls.build())
.addFileTypes(dev.cel.expr.conformance.proto2.TestAllTypesExtensions.getDescriptor())
.addFileTypes(CONFORMANCE_FILE_DESCRIPTORS)
.addLibraries(CANONICAL_COMPILER_EXTENSIONS)
.addMessageTypes(dev.cel.expr.conformance.proto2.TestAllTypes.getDescriptor())
.addMessageTypes(dev.cel.expr.conformance.proto3.TestAllTypes.getDescriptor())
.build();
}

private static CelRuntime getRuntime(SimpleTest test, boolean usePlanner) {
CelRuntimeBuilder builder =
usePlanner ? CelRuntimeImpl.newBuilder() : CelRuntimeFactory.standardCelRuntimeBuilder();

builder
// CEL-Internal-2
.setOptions(OPTIONS)
.addLibraries(CANONICAL_RUNTIME_EXTENSIONS)
.setExtensionRegistry(CONFORMANCE_EXTENSION_REGISTRY)
.addMessageTypes(dev.cel.expr.conformance.proto2.TestAllTypes.getDescriptor())
.addMessageTypes(dev.cel.expr.conformance.proto3.TestAllTypes.getDescriptor())
.addFileTypes(dev.cel.expr.conformance.proto2.TestAllTypesExtensions.getDescriptor());

if (usePlanner) {
builder.setContainer(CelContainer.ofName(test.getContainer()));
}

return builder.build();
}

private static ImmutableMap<String, Object> getBindings(SimpleTest test) throws Exception {
ImmutableMap.Builder<String, Object> bindings =
ImmutableMap.builderWithExpectedSize(test.getBindingsCount());
Expand Down Expand Up @@ -204,16 +229,7 @@ private static SimpleTest defaultTestMatcherToTrueIfUnset(SimpleTest test) {
private final String name;
private final SimpleTest test;
private final boolean skip;
private final boolean usePlanner;

public ConformanceTest(String name, SimpleTest test, boolean skip, boolean usePlanner) {
this.name = Preconditions.checkNotNull(name);
this.test =
Preconditions.checkNotNull(
defaultTestMatcherToTrueIfUnset(Preconditions.checkNotNull(test)));
this.skip = skip;
this.usePlanner = usePlanner;
}
private final RuntimeType runtimeType;

public String getName() {
return name;
Expand Down Expand Up @@ -251,16 +267,15 @@ public void evaluate() throws Throwable {
return;
}

if (!usePlanner && test.getDisableCheck()) {
// Only planner supports parsed-only evaluation
if (runtimeType.equals(RuntimeType.LEGACY) && test.getDisableCheck()) {
// Only planner and lite runtime support parsed-only evaluation
return;
}

CelRuntime runtime = getRuntime(test, usePlanner);
ExprValue result = null;
CelEvaluationException error = null;
try {
Program program = runtime.createProgram(ast);
Program program = createProgram(ast);
result = toExprValue(program.eval(getBindings(test)), ast.getResultType());
} catch (CelEvaluationException e) {
error = e;
Expand Down Expand Up @@ -295,6 +310,46 @@ public void evaluate() throws Throwable {
}
}

@SuppressWarnings("deprecation") // Intentionally testing the legacy interpreter runtime.
private Program createProgram(CelAbstractSyntaxTree ast) throws CelEvaluationException {
if (runtimeType.equals(RuntimeType.LITE)) {
return CelLiteRuntimeFactory.newLiteRuntimeBuilder()
.setOptions(OPTIONS)
.setStandardFunctions(CelStandardFunctions.ALL_STANDARD_FUNCTIONS)
.addLibraries(CANONICAL_LITE_RUNTIME_EXTENSIONS)
.setValueProvider(LITE_VALUE_PROVIDER)
.setTypeProvider(LITE_TYPE_PROVIDER)
.setContainer(CelContainer.ofName(test.getContainer()))
.build()
.createProgram(ast);
}

CelRuntimeBuilder builder =
runtimeType.equals(RuntimeType.PLANNER)
? CelRuntimeImpl.newBuilder()
: CelRuntimeFactory.standardCelRuntimeBuilder();

builder
// CEL-Internal-2
.setOptions(OPTIONS)
.addLibraries(CANONICAL_RUNTIME_EXTENSIONS)
.setExtensionRegistry(CONFORMANCE_EXTENSION_REGISTRY)
.addFileTypes(CONFORMANCE_FILE_DESCRIPTORS);

if (runtimeType.equals(RuntimeType.PLANNER)) {
builder.setContainer(CelContainer.ofName(test.getContainer()));
}

return builder.build().createProgram(ast);
}

ConformanceTest(String name, SimpleTest test, boolean skip, RuntimeType runtimeType) {
this.name = checkNotNull(name);
this.test = defaultTestMatcherToTrueIfUnset(checkNotNull(test));
this.skip = skip;
this.runtimeType = checkNotNull(runtimeType);
}

/**
* Conformance-only library providing macros for the {@code block_ext} test suite.
*
Expand Down
Loading
Loading