fix(isthmus)!: convert precision_time and precision_timestamp up to nanoseconds

isthmus/src/main/java/io/substrait/isthmus/SubstraitTypeSystem.java

@@ -106,13 +106,14 @@ public int getMaxPrecision(final SqlTypeName typeName) {
       case BINARY:
       case VARBINARY:
         return Integer.MAX_VALUE;
+      case TIME:
+      case TIMESTAMP:
+      case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+        return 9;
       case INTERVAL_DAY:
       case INTERVAL_YEAR:
       case INTERVAL_YEAR_MONTH:
-      case TIME:
       case TIME_WITH_LOCAL_TIME_ZONE:
-      case TIMESTAMP:
-      case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
         return 6;
       case DECIMAL:
         return 38;

isthmus/src/main/java/io/substrait/isthmus/expression/LiteralConverter.java

@@ -296,8 +296,11 @@ public Expression.Literal convert(RexLiteral literal, RelDataType resultType) {
               LocalDateTime.parse(timestamp.toString(), CALCITE_LOCAL_DATETIME_FORMATTER);
           int precision = TypeConverter.precisionOf(resultType);
           long value =
-              localDateTime.toEpochSecond(ZoneOffset.UTC) * LongMath.pow(10, precision)
-                  + rescaleNanos(localDateTime.getNano(), precision);
+              epochUnits(
+                  localDateTime.toEpochSecond(ZoneOffset.UTC),
+                  rescaleNanos(localDateTime.getNano(), precision),
+                  precision,
+                  timestamp);
           // toEpochSecond floors, and the nanosecond part it leaves behind is always positive, so
           // a pre-epoch timestamp narrowed to a coarser precision moves back in time rather than
           // towards the epoch. That is what a timestamp wants — 1969-12-31 23:59:59.5 at second
@@ -400,6 +403,40 @@ public static byte[] padRightIfNeeded(
     return padRightIfNeeded(bytes.getBytes(), length);
   }
 
+  /**
+   * Returns a timestamp as a count of 10^-precision seconds since the epoch.
+   *
+   * <p>A Substrait temporal value is a 64-bit count of its own unit, and the finer the unit the
+   * narrower the range it spans: nanoseconds reach only to the year 2262, where Calcite's own
+   * {@code TimestampString} reaches 9999. A timestamp outside the range is reported rather than
+   * wrapped into a different instant.
+   *
+   * @param epochSeconds whole seconds since the epoch, floored
+   * @param subSecondUnits the sub-second part, already in units of 10^-precision seconds
+   * @param precision the fractional-second precision
+   * @param timestamp the timestamp being converted, for the failure message
+   * @return the value of the Substrait literal
+   * @throws IllegalArgumentException if the value does not fit in 64 bits
+   */
+  private static long epochUnits(
+      long epochSeconds, long subSecondUnits, int precision, TimestampString timestamp) {
+    long unitsPerSecond = LongMath.pow(10, precision);
+    try {
+      // The floored seconds of the range's first second overflow on their own although the value
+      // fits once the sub-second part is added back, so a negative value borrows that second.
+      return epochSeconds < 0 && subSecondUnits > 0
+          ? Math.addExact(
+              Math.multiplyExact(epochSeconds + 1, unitsPerSecond), subSecondUnits - unitsPerSecond)
+          : Math.addExact(Math.multiplyExact(epochSeconds, unitsPerSecond), subSecondUnits);
+    } catch (ArithmeticException e) {
+      throw new IllegalArgumentException(
+          String.format(
+              "timestamp %s does not fit in a 64-bit count of 10^-%d seconds",
+              timestamp, precision),
+          e);
+    }
+  }
+
   /**
    * Rescales a nanosecond count to the fractional-second unit a Substrait temporal literal of the
    * given precision is expressed in.

isthmus/src/test/java/io/substrait/isthmus/CalciteLiteralTest.java

@@ -143,9 +143,64 @@ void tTimeWithMicroSecond() {
 
   @Test
   void tTimeWithNanoSecond() {
+    bitest(
+        ExpressionCreator.precisionTime(
+            false, (14L * 60 * 60 + 22 * 60 + 47) * 1_000_000_000L + 123_456_789, 9),
+        rex.makeTimeLiteral(new TimeString("14:22:47.123456789"), 9));
+  }
+
+  /**
+   * A Substrait temporal value is a 64-bit count of its own unit, so the finer the unit the
+   * narrower the range: nanoseconds run out in 2262, where a Calcite TimestampString reaches 9999.
+   * A timestamp past that is reported rather than wrapped into a different instant.
+   */
+  @Test
+  void aTimestampTooLargeForItsPrecisionIsReported() {
+    RexLiteral literal =
+        rex.makeTimestampLiteral(new TimestampString("9999-12-31 23:59:59.999999999"), 9);
+
+    IllegalArgumentException error =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new LiteralConverter(TypeConverter.DEFAULT).convert(literal));
+
+    assertTrue(error.getMessage().contains("does not fit in a 64-bit count of 10^-9 seconds"));
+  }
+
+  /**
+   * Both ends of the nanosecond range convert, and one nanosecond past either is reported. The
+   * lower end is the one a floored split can get wrong: its seconds alone do not fit in 64 bits.
+   */
+  @ParameterizedTest
+  @CsvSource({
+    "1677-09-21 00:12:43.145224192, -9223372036854775808",
+    "2262-04-11 23:47:16.854775807,  9223372036854775807",
+  })
+  void theEndsOfTheNanosecondRangeConvert(String timestamp, long nanos) {
     assertEquals(
-        rex.makeTimeLiteral(new TimeString("14:22:47.123456789"), 9),
-        rex.makeTimeLiteral(new TimeString("14:22:47.123456"), 6));
+        ExpressionCreator.precisionTimestamp(false, nanos, 9),
+        new LiteralConverter(TypeConverter.DEFAULT)
+            .convert(rex.makeTimestampLiteral(new TimestampString(timestamp), 9)));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"1677-09-21 00:12:43.145224191", "2262-04-11 23:47:16.854775808"})
+  void oneNanosecondPastTheRangeIsReported(String timestamp) {
+    RexLiteral literal = rex.makeTimestampLiteral(new TimestampString(timestamp), 9);
+
+    IllegalArgumentException error =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> new LiteralConverter(TypeConverter.DEFAULT).convert(literal));
+
+    assertTrue(error.getMessage().contains("does not fit in a 64-bit count of 10^-9 seconds"));
+  }
+
+  @Test
+  void tPrecisionTimestampAtNanosecondPrecision() {
+    bitest(
+        ExpressionCreator.precisionTimestamp(false, 1_704_067_200_123_456_789L, 9),
+        rex.makeTimestampLiteral(new TimestampString("2024-01-01 00:00:00.123456789"), 9));
   }
 
   @Test

isthmus/src/test/java/io/substrait/isthmus/CalciteTypeTest.java

@@ -113,7 +113,7 @@ void time(boolean nullable) {
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   void precisionTimeStamp(boolean nullable) {
-    for (int precision : new int[] {0, 3, 6}) {
+    for (int precision : new int[] {0, 3, 6, 9}) {
       testType(
           Type.withNullability(nullable).precisionTimestamp(precision),
           SqlTypeName.TIMESTAMP,
@@ -125,7 +125,7 @@ void precisionTimeStamp(boolean nullable) {
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   void precisionTimestamptz(boolean nullable) {
-    for (int precision : new int[] {0, 3, 6}) {
+    for (int precision : new int[] {0, 3, 6, 9}) {
       testType(
           Type.withNullability(nullable).precisionTimestampTZ(precision),
           SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE,
@@ -134,6 +134,23 @@ void precisionTimestamptz(boolean nullable) {
     }
   }
 
+  /**
+   * Substrait allows 0 to 12; Calcite carries nanoseconds, and builds a type at 12 as one at 9
+   * rather than reporting that it cannot. A precision past what it can hold is refused, naming the
+   * bound, instead of being narrowed in silence.
+   */
+  @ParameterizedTest
+  @ValueSource(ints = {10, 12})
+  void aPrecisionFinerThanNanosecondsIsRefused(int precision) {
+    IllegalArgumentException error =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                TypeConverter.DEFAULT.toCalcite(
+                    type, TypeCreator.REQUIRED.precisionTimestamp(precision), null));
+    assertTrue(error.getMessage().contains("max precision in Calcite type system is set to 9"));
+  }
+
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   void intervalYear(boolean nullable) {

isthmus/src/test/java/io/substrait/isthmus/ExpressionConvertabilityTest.java

@@ -152,6 +152,9 @@ void supportedPrecisionForPrecisionTimestampLiteral() {
     assertPrecisionTimestampLiteral(4);
     assertPrecisionTimestampLiteral(5);
     assertPrecisionTimestampLiteral(6);
+    assertPrecisionTimestampLiteral(7);
+    assertPrecisionTimestampLiteral(8);
+    assertPrecisionTimestampLiteral(9);
   }
 
   void assertPrecisionTimestampLiteral(int precision) {
@@ -166,7 +169,7 @@ void assertPrecisionTimestampLiteral(int precision) {
 
   @Test
   void supportedPrecisionForPrecisionTimestampTZLiteral() {
-    // The same set the non-TZ literal supports: SubstraitTypeSystem configures a maximum of 6 for
+    // The same set the non-TZ literal supports: SubstraitTypeSystem configures a maximum of 9 for
     // TIMESTAMP_WITH_LOCAL_TIME_ZONE, and the TZ literal is checked against that name too.
     assertPrecisionTimestampTZLiteral(0);
     assertPrecisionTimestampTZLiteral(1);
@@ -175,6 +178,9 @@ void supportedPrecisionForPrecisionTimestampTZLiteral() {
     assertPrecisionTimestampTZLiteral(4);
     assertPrecisionTimestampTZLiteral(5);
     assertPrecisionTimestampTZLiteral(6);
+    assertPrecisionTimestampTZLiteral(7);
+    assertPrecisionTimestampTZLiteral(8);
+    assertPrecisionTimestampTZLiteral(9);
   }
 
   void assertPrecisionTimestampTZLiteral(int precision) {
@@ -192,12 +198,7 @@ void unsupportedPrecisionForPrecisionTimestampLiteral() {
     // test different edge case precision values
     assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(-1);
 
-    assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(7);
-    assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(8);
-
-    // this would be nanoseconds which are supported in Substrait but not in Calcite
-    assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(9);
-
+    // finer than a nanosecond, which is the finest unit a Calcite TimestampString carries
     assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(10);
     assertThrowsUnsupportedPrecisionPrecisionTimestampLiteral(11);
 
@@ -218,12 +219,7 @@ void unsupportedPrecisionPrecisionTimestampTZLiteral() {
     // test different edge case precision values
     assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(-1);
 
-    assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(7);
-    assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(8);
-
-    // this would be nanoseconds which are supported in Substrait but not in Calcite
-    assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(9);
-
+    // finer than a nanosecond, which is the finest unit a Calcite TimestampString carries
     assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(10);
     assertThrowsUnsupportedPrecisionPrecisionTimestampTZLiteral(11);
 

isthmus/src/test/java/io/substrait/isthmus/SubstraitTypeSystemTest.java

@@ -45,12 +45,16 @@ void decimalDefaultScale() {
 
   @Test
   void timestampMaxPrecision() {
-    assertEquals(6, typeSystem.getMaxPrecision(SqlTypeName.TIMESTAMP));
+    // Nanoseconds: the finest unit a Calcite TimestampString carries, and what the type factory
+    // builds a TIMESTAMP at once the ceiling allows it. Picoseconds clamp rather than throw, so
+    // Substrait's 10 to 12 stay out.
+    assertEquals(9, typeSystem.getMaxPrecision(SqlTypeName.TIMESTAMP));
+    assertEquals(9, typeSystem.getMaxPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
   }
 
   @Test
   void timeMaxPrecision() {
-    assertEquals(6, typeSystem.getMaxPrecision(SqlTypeName.TIME));
+    assertEquals(9, typeSystem.getMaxPrecision(SqlTypeName.TIME));
   }
 
   @Test