diff --git a/common/src/main/java/dev/cel/common/values/RawProtoMessageLiteValue.java b/common/src/main/java/dev/cel/common/values/RawProtoMessageLiteValue.java index 2a3bdf940..349356ca8 100644 --- a/common/src/main/java/dev/cel/common/values/RawProtoMessageLiteValue.java +++ b/common/src/main/java/dev/cel/common/values/RawProtoMessageLiteValue.java @@ -193,14 +193,29 @@ private static Object decodeWireField( fieldDescriptor != null ? fieldDescriptor.getEncodingType() == EncodingType.LIST : field.defaultValue() instanceof List; - String protoTypeName = - fieldDescriptor != null - ? fieldDescriptor.getFieldProtoTypeName() - : UNKNOWN_MESSAGE_TYPE_NAME; + String protoTypeName = resolveProtoTypeName(field, fieldDescriptor); return decodeWireEntries(unknowns, typeCode, protoTypeName, isRepeated, converter); } + /** + * Resolves the protobuf message type name for a field. + * + *

Prefers the descriptor's message type name when present. Under runtime version skew (where + * the descriptor is omitted), falls back to the type name preserved in {@link SelectField} from + * the optimizer, or {@link #UNKNOWN_MESSAGE_TYPE_NAME} if unspecified. + */ + private static String resolveProtoTypeName( + SelectField field, @Nullable FieldLiteDescriptor fieldDescriptor) { + if (fieldDescriptor != null) { + return fieldDescriptor.getFieldProtoTypeName(); + } + if (!field.protoTypeName().isEmpty()) { + return field.protoTypeName(); + } + return UNKNOWN_MESSAGE_TYPE_NAME; + } + private static Object resolveDefault( SelectField field, @Nullable FieldLiteDescriptor fieldDescriptor, @@ -211,7 +226,8 @@ private static Object resolveDefault( if (fieldDescriptor == null) { if (field.typeCode() == FieldLiteDescriptor.Type.MESSAGE.getNumber()) { - return create(ByteString.EMPTY, UNKNOWN_MESSAGE_TYPE_NAME, converter); + return create( + ByteString.EMPTY, resolveProtoTypeName(field, /* fieldDescriptor= */ null), converter); } throw CelAttributeNotFoundException.forFieldResolution(field.fieldName()); } @@ -249,6 +265,9 @@ static boolean isPresentInWire( ? fieldDescriptor.getProtoFieldType().getNumber() : field.typeCode(); + // In protobuf wire format, a zero-length entry for a singular field (e.g. empty string, + // bytes, or empty submessage) represents explicit presence on the wire. Only packed repeated + // fields with empty payload represent an empty/absent collection. if (!isRepeated) { return true; } diff --git a/common/src/main/java/dev/cel/common/values/SelectField.java b/common/src/main/java/dev/cel/common/values/SelectField.java index 8bee5a0d7..c89d3bf1c 100644 --- a/common/src/main/java/dev/cel/common/values/SelectField.java +++ b/common/src/main/java/dev/cel/common/values/SelectField.java @@ -69,6 +69,12 @@ public abstract class SelectField { */ public abstract @Nullable Object defaultValue(); + /** + * Protobuf message type name of this hop (e.g. {@code "google.protobuf.Duration"}, {@code + * "com.example.User"}), or empty string if unknown or not a message field. + */ + public abstract String protoTypeName(); + /** * Creates a presence-test qualifier hop. * @@ -83,7 +89,11 @@ public static SelectField create(long fieldNumber, String fieldName) { fieldNumber); checkNotNull(fieldName); return new AutoValue_SelectField( - (int) fieldNumber, fieldName, NO_TYPE_CODE, /* defaultValue= */ null); + (int) fieldNumber, + fieldName, + NO_TYPE_CODE, + /* defaultValue= */ null, + /* protoTypeName= */ ""); } /** @@ -98,13 +108,36 @@ public static SelectField create(long fieldNumber, String fieldName) { */ public static SelectField create( long fieldNumber, String fieldName, long typeCode, @Nullable Object defaultValue) { + return create(fieldNumber, fieldName, typeCode, defaultValue, /* protoTypeName= */ ""); + } + + /** + * Creates a fully-specified field selection hop with type code, optional default value, and + * protobuf type name. + * + * @param fieldNumber Protobuf field number. Takes {@code long} for compatibility with CEL's int64 + * constant representations. + * @param fieldName Protobuf field name. + * @param typeCode Protobuf wire type code or {@link #CEL_MAP_TYPE_CODE}. Takes {@code long} for + * compatibility with CEL's int64 constant representations. + * @param defaultValue Default value for the field, or null if unspecified. + * @param protoTypeName Protobuf message type name, or empty string if unspecified. + */ + public static SelectField create( + long fieldNumber, + String fieldName, + long typeCode, + @Nullable Object defaultValue, + String protoTypeName) { checkArgument( fieldNumber >= 1 && fieldNumber <= MAX_FIELD_NUMBER, "Field number out of protobuf range: %s", fieldNumber); checkNotNull(fieldName); checkArgument(isSupportedTypeCode(typeCode), "Invalid protobuf type code: %s", typeCode); - return new AutoValue_SelectField((int) fieldNumber, fieldName, (int) typeCode, defaultValue); + checkNotNull(protoTypeName); + return new AutoValue_SelectField( + (int) fieldNumber, fieldName, (int) typeCode, defaultValue, protoTypeName); } /** diff --git a/common/src/test/java/dev/cel/common/values/RawProtoMessageLiteValueTest.java b/common/src/test/java/dev/cel/common/values/RawProtoMessageLiteValueTest.java index 9a4079a4c..c1abcc97c 100644 --- a/common/src/test/java/dev/cel/common/values/RawProtoMessageLiteValueTest.java +++ b/common/src/test/java/dev/cel/common/values/RawProtoMessageLiteValueTest.java @@ -40,6 +40,7 @@ import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor; import java.io.ByteArrayOutputStream; import java.time.Duration; +import java.time.Instant; import java.util.NoSuchElementException; import java.util.Optional; import org.junit.Test; @@ -1260,4 +1261,332 @@ public void selectByFieldNumber_absentMessageFieldWithoutDescriptor_returnsUnkno .hasMessageThat() .contains("Decoding unknown map field from wire bytes is unsupported"); } + + @SuppressWarnings("ImmutableEnumChecker") // Test only + private enum WellKnownFieldWithoutDescriptorTestCase { + DURATION( + TestAllTypes.newBuilder() + .setSingleDuration(ProtoTimeUtils.toProtoDuration(Duration.ofSeconds(120L, 500L))) + .build(), + SelectField.create( + TestAllTypes.SINGLE_DURATION_FIELD_NUMBER, + "single_duration", + FieldLiteDescriptor.Type.MESSAGE.getNumber(), + Duration.ZERO, + "google.protobuf.Duration"), + Duration.ofSeconds(120L, 500L), + Duration.ZERO), + TIMESTAMP( + TestAllTypes.newBuilder() + .setSingleTimestamp( + ProtoTimeUtils.toProtoTimestamp(Instant.ofEpochSecond(1700000000L, 123456789L))) + .build(), + SelectField.create( + TestAllTypes.SINGLE_TIMESTAMP_FIELD_NUMBER, + "single_timestamp", + FieldLiteDescriptor.Type.MESSAGE.getNumber(), + Instant.EPOCH, + "google.protobuf.Timestamp"), + Instant.ofEpochSecond(1700000000L, 123456789L), + Instant.EPOCH); + + private final TestAllTypes populatedProto; + private final SelectField selectField; + private final Object expectedPopulatedValue; + private final Object expectedDefaultValue; + + WellKnownFieldWithoutDescriptorTestCase( + TestAllTypes populatedProto, + SelectField selectField, + Object expectedPopulatedValue, + Object expectedDefaultValue) { + this.populatedProto = populatedProto; + this.selectField = selectField; + this.expectedPopulatedValue = expectedPopulatedValue; + this.expectedDefaultValue = expectedDefaultValue; + } + } + + @Test + public void selectByFieldNumber_wellKnownFieldWithoutDescriptor_decodesOrReturnsDefault( + @TestParameter WellKnownFieldWithoutDescriptorTestCase testCase) { + RawProtoMessageLiteValue populatedRaw = + RawProtoMessageLiteValue.create( + testCase.populatedProto.toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + RawProtoMessageLiteValue emptyRaw = + RawProtoMessageLiteValue.create( + TestAllTypes.getDefaultInstance().toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + + Object populatedSelected = populatedRaw.selectByFieldNumber(testCase.selectField); + Object emptySelected = emptyRaw.selectByFieldNumber(testCase.selectField); + + assertThat(populatedSelected).isEqualTo(testCase.expectedPopulatedValue); + assertThat(emptySelected).isEqualTo(testCase.expectedDefaultValue); + } + + @Test + public void findByFieldNumber_wellKnownFieldWithoutDescriptor_returnsOptionalValue( + @TestParameter WellKnownFieldWithoutDescriptorTestCase testCase) { + RawProtoMessageLiteValue populatedRaw = + RawProtoMessageLiteValue.create( + testCase.populatedProto.toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + RawProtoMessageLiteValue emptyRaw = + RawProtoMessageLiteValue.create( + TestAllTypes.getDefaultInstance().toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + + Optional populatedFound = populatedRaw.findByFieldNumber(testCase.selectField); + Optional emptyFound = emptyRaw.findByFieldNumber(testCase.selectField); + + assertThat(populatedFound).hasValue(testCase.expectedPopulatedValue); + assertThat(emptyFound).isEmpty(); + } + + @Test + public void selectByFieldNumber_negativeInt32Varint_decodesSignedIntCorrectly() { + TestAllTypes proto = TestAllTypes.newBuilder().setSingleInt32(-42).build(); + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + proto.toByteString(), "cel.expr.conformance.proto3.TestAllTypes", EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.SINGLE_INT32_FIELD_NUMBER, + "single_int32", + FieldLiteDescriptor.Type.INT32.getNumber(), + 0L); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isEqualTo(-42L); + } + + @Test + public void selectByFieldNumber_negativeInt32MinValue_decodesSignedIntCorrectly() { + TestAllTypes proto = TestAllTypes.newBuilder().setSingleInt32(Integer.MIN_VALUE).build(); + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + proto.toByteString(), "cel.expr.conformance.proto3.TestAllTypes", EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.SINGLE_INT32_FIELD_NUMBER, + "single_int32", + FieldLiteDescriptor.Type.INT32.getNumber(), + 0L); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isEqualTo((long) Integer.MIN_VALUE); + } + + @Test + public void selectByFieldNumber_unpackedAndPackedRepeatedInt64_decodeIdentically() + throws Exception { + ByteArrayOutputStream unpackedBaos = new ByteArrayOutputStream(); + CodedOutputStream unpackedCos = CodedOutputStream.newInstance(unpackedBaos); + unpackedCos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 10L); + unpackedCos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 20L); + unpackedCos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 30L); + unpackedCos.flush(); + RawProtoMessageLiteValue unpackedRaw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(unpackedBaos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + TestAllTypes packedProto = + TestAllTypes.newBuilder() + .addRepeatedInt64(10L) + .addRepeatedInt64(20L) + .addRepeatedInt64(30L) + .build(); + RawProtoMessageLiteValue packedRaw = + RawProtoMessageLiteValue.create( + packedProto.toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.REPEATED_INT64_FIELD_NUMBER, + "repeated_int64", + FieldLiteDescriptor.Type.INT64.getNumber(), + ImmutableList.of()); + + Object unpackedResult = unpackedRaw.selectByFieldNumber(field); + Object packedResult = packedRaw.selectByFieldNumber(field); + + assertThat(unpackedResult).isEqualTo(ImmutableList.of(10L, 20L, 30L)); + assertThat(packedResult).isEqualTo(ImmutableList.of(10L, 20L, 30L)); + } + + @Test + public void hasFieldByNumber_unpackedAndPackedRepeatedInt64_returnsTrue() throws Exception { + ByteArrayOutputStream unpackedBaos = new ByteArrayOutputStream(); + CodedOutputStream unpackedCos = CodedOutputStream.newInstance(unpackedBaos); + unpackedCos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 10L); + unpackedCos.flush(); + RawProtoMessageLiteValue unpackedRaw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(unpackedBaos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + TestAllTypes packedProto = TestAllTypes.newBuilder().addRepeatedInt64(10L).build(); + RawProtoMessageLiteValue packedRaw = + RawProtoMessageLiteValue.create( + packedProto.toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField hasField = + SelectField.create(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, "repeated_int64"); + + boolean unpackedPresent = unpackedRaw.hasFieldByNumber(hasField); + boolean packedPresent = packedRaw.hasFieldByNumber(hasField); + + assertThat(unpackedPresent).isTrue(); + assertThat(packedPresent).isTrue(); + } + + @Test + public void findByFieldNumber_unpackedAndPackedRepeatedInt64_returnsExpectedValue() + throws Exception { + ByteArrayOutputStream unpackedBaos = new ByteArrayOutputStream(); + CodedOutputStream unpackedCos = CodedOutputStream.newInstance(unpackedBaos); + unpackedCos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 10L); + unpackedCos.flush(); + RawProtoMessageLiteValue unpackedRaw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(unpackedBaos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + TestAllTypes packedProto = TestAllTypes.newBuilder().addRepeatedInt64(10L).build(); + RawProtoMessageLiteValue packedRaw = + RawProtoMessageLiteValue.create( + packedProto.toByteString(), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.REPEATED_INT64_FIELD_NUMBER, + "repeated_int64", + FieldLiteDescriptor.Type.INT64.getNumber(), + ImmutableList.of()); + + Optional unpackedFound = unpackedRaw.findByFieldNumber(field); + Optional packedFound = packedRaw.findByFieldNumber(field); + + assertThat(unpackedFound).hasValue(ImmutableList.of(10L)); + assertThat(packedFound).hasValue(ImmutableList.of(10L)); + } + + @Test + public void selectByFieldNumber_mixedUnpackedAndPackedRepeatedInt64_concatenatesInOrder() + throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + CodedOutputStream cos = CodedOutputStream.newInstance(baos); + cos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 10L); + ByteArrayOutputStream packedChunk = new ByteArrayOutputStream(); + CodedOutputStream packedCos = CodedOutputStream.newInstance(packedChunk); + packedCos.writeInt64NoTag(20L); + packedCos.writeInt64NoTag(30L); + packedCos.flush(); + cos.writeByteArray(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, packedChunk.toByteArray()); + cos.writeInt64(TestAllTypes.REPEATED_INT64_FIELD_NUMBER, 40L); + cos.flush(); + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(baos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.REPEATED_INT64_FIELD_NUMBER, + "repeated_int64", + FieldLiteDescriptor.Type.INT64.getNumber(), + ImmutableList.of()); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isEqualTo(ImmutableList.of(10L, 20L, 30L, 40L)); + } + + @Test + public void selectByFieldNumber_fiveByteUnsignedInt32Varint_signExtendsCorrectly() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write((TestAllTypes.SINGLE_INT32_FIELD_NUMBER << 3)); + // 5-byte varint encoding of 0xFFFFFFD6 (-42 in 32-bit two's complement). + // Verifies that 32-bit sign extension correctly yields -42L rather than +4294967254L. + baos.write(0xD6); + baos.write(0xFF); + baos.write(0xFF); + baos.write(0xFF); + baos.write(0x0F); + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(baos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField field = + SelectField.create( + TestAllTypes.SINGLE_INT32_FIELD_NUMBER, + "single_int32", + FieldLiteDescriptor.Type.INT32.getNumber(), + 0L); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isEqualTo(-42L); + } + + @Test + public void selectByFieldNumber_unsetSubmessageWithProtoTypeName_returnsDefaultWithTypeName() { + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + ByteString.EMPTY, "cel.expr.conformance.proto3.TestAllTypes", EMPTY_CONVERTER); + SelectField field = + SelectField.create( + 999L, + "custom_msg", + FieldLiteDescriptor.Type.MESSAGE.getNumber(), + null, + "test.CustomMessage"); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isInstanceOf(RawProtoMessageLiteValue.class); + assertThat(((RawProtoMessageLiteValue) result).celType().name()) + .isEqualTo("test.CustomMessage"); + } + + @Test + public void selectByFieldNumber_wireSubmessageWithProtoTypeName_decodesWithTypeName() + throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + CodedOutputStream cos = CodedOutputStream.newInstance(baos); + cos.writeTag(999, WireFormat.WIRETYPE_LENGTH_DELIMITED); + cos.writeByteArrayNoTag(new byte[] {0x08, 0x2A}); + cos.flush(); + RawProtoMessageLiteValue raw = + RawProtoMessageLiteValue.create( + ByteString.copyFrom(baos.toByteArray()), + "cel.expr.conformance.proto3.TestAllTypes", + EMPTY_CONVERTER); + SelectField field = + SelectField.create( + 999L, + "custom_msg", + FieldLiteDescriptor.Type.MESSAGE.getNumber(), + null, + "test.CustomMessage"); + + Object result = raw.selectByFieldNumber(field); + + assertThat(result).isInstanceOf(RawProtoMessageLiteValue.class); + assertThat(((RawProtoMessageLiteValue) result).celType().name()) + .isEqualTo("test.CustomMessage"); + } } diff --git a/common/src/test/java/dev/cel/common/values/SelectFieldTest.java b/common/src/test/java/dev/cel/common/values/SelectFieldTest.java index ba9dc7008..a64698e13 100644 --- a/common/src/test/java/dev/cel/common/values/SelectFieldTest.java +++ b/common/src/test/java/dev/cel/common/values/SelectFieldTest.java @@ -33,6 +33,7 @@ public void create_twoArguments_success() { assertThat(field.fieldName()).isEqualTo("foo"); assertThat(field.typeCode()).isEqualTo(SelectField.NO_TYPE_CODE); assertThat(field.defaultValue()).isNull(); + assertThat(field.protoTypeName()).isEmpty(); } @Test @@ -43,6 +44,23 @@ public void create_fourArguments_success() { assertThat(field.fieldName()).isEqualTo("bar"); assertThat(field.typeCode()).isEqualTo(9); assertThat(field.defaultValue()).isEqualTo("default_str"); + assertThat(field.protoTypeName()).isEmpty(); + } + + @Test + public void create_fiveArguments_success() { + SelectField field = SelectField.create(2L, "bar", 11, null, "google.protobuf.Duration"); + + assertThat(field.fieldNumber()).isEqualTo(2); + assertThat(field.fieldName()).isEqualTo("bar"); + assertThat(field.typeCode()).isEqualTo(11); + assertThat(field.defaultValue()).isNull(); + assertThat(field.protoTypeName()).isEqualTo("google.protobuf.Duration"); + } + + @Test + public void create_fiveArgNullProtoTypeName_throwsNullPointerException() { + assertThrows(NullPointerException.class, () -> SelectField.create(1L, "foo", 11, null, null)); } @Test @@ -128,6 +146,10 @@ public void equalsAndHashCode_testedProperly() { SelectField.create(1L, "foo", 9, "default"), SelectField.create(1L, "foo", 9, "default")) .addEqualityGroup(SelectField.create(1L, "foo", 9, "other_default")) + .addEqualityGroup( + SelectField.create(1L, "foo", 11, null, "google.protobuf.Duration"), + SelectField.create(1L, "foo", 11, null, "google.protobuf.Duration")) + .addEqualityGroup(SelectField.create(1L, "foo", 11, null, "google.protobuf.Timestamp")) .testEquals(); } } diff --git a/runtime/src/main/java/dev/cel/runtime/planner/OptimizedSelectPlanner.java b/runtime/src/main/java/dev/cel/runtime/planner/OptimizedSelectPlanner.java index 9459d86ef..349b57bf7 100644 --- a/runtime/src/main/java/dev/cel/runtime/planner/OptimizedSelectPlanner.java +++ b/runtime/src/main/java/dev/cel/runtime/planner/OptimizedSelectPlanner.java @@ -143,10 +143,23 @@ private static ImmutableList unpackAttributeFields( rawTypeCode); Object defaultValue = (hopElements.size() == 4) ? resolveDefaultValue(hopElements.get(3)) : null; + String protoTypeName = ""; if (isLeaf) { validateLeafTypeIdent((int) rawTypeCode, defaultValue, typeIdent); + // Preserve the full protobuf message name from typeIdent so descriptorless runtime + // evaluation (version skew) can identify well-known types (e.g. Duration, Timestamp) and + // populate RawProtoMessageLiteValue's type. + // Repeated message fields also use MESSAGE_TYPE_CODE (11) for their element wire type, but + // their typeIdent is "list" rather than the proto message name (map fields use + // CEL_MAP_TYPE_CODE (100) instead of 11, so they are already excluded). + boolean isSingularMessage = + rawTypeCode == SelectField.MESSAGE_TYPE_CODE && !typeIdent.equals(LIST_TYPE_IDENT); + if (isSingularMessage) { + protoTypeName = typeIdent; + } } - fieldsBuilder.add(SelectField.create(fieldNumber, fieldName, rawTypeCode, defaultValue)); + fieldsBuilder.add( + SelectField.create(fieldNumber, fieldName, rawTypeCode, defaultValue, protoTypeName)); } return fieldsBuilder.build(); }