Upgrade to DataFusion 54 and arrow 58.3 (#62)

adriangb · friendlymatthew · claude · web-flow · commit 9e1c84698150 · 2026-06-10T08:54:55.000+02:00
* Remove as_any from ScalarUDFImpl impls (removed in pydantic DF fork)

* Upgrade to DataFusion 54 and arrow 58.3

DataFusion 54 removed `as_any` from `ScalarUDFImpl` and bumped arrow to 58.3.
Bump the dependency requirements accordingly.

Co-Authored-By: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;

---------

Co-authored-by: Matthew Kim &lt;38759997+friendlymatthew@users.noreply.github.com&gt;
Co-authored-by: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -7,20 +7,20 @@ license = "Apache-2.0"
 authors = ["Matthew Kim <matthew@pydantic.dev>"]
 
 [dependencies]
-datafusion = "53"
-arrow = "58.1"
-arrow-schema = "58.1"
-parquet-variant-compute = "58.1"
-parquet-variant-json = "58.1"
-parquet-variant = "58.1"
+datafusion = "54"
+arrow = "58.3"
+arrow-schema = "58.3"
+parquet-variant-compute = "58.3"
+parquet-variant-json = "58.3"
+parquet-variant = "58.3"
 
 [dev-dependencies]
 anyhow = "1.0.100"
-arrow-cast = "58.1"
+arrow-cast = "58.3"
 serde_json = "1.0.145"
 flate2 = "1.0"
 tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] }
-datafusion-sqllogictest = "53"
+datafusion-sqllogictest = "54"
 sqllogictest = "0.29"
 indicatif = "0.18"
 env_logger = "0.11"
diff --git a/src/cast_to_variant.rs b/src/cast_to_variant.rs
@@ -157,9 +157,6 @@ impl CastToVariantUdf {
 }
 
 impl ScalarUDFImpl for CastToVariantUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "cast_to_variant"
diff --git a/src/impl_variant_get.rs b/src/impl_variant_get.rs
@@ -23,9 +23,6 @@ macro_rules! impl_variant_get_typed {
         }
 
         impl ScalarUDFImpl for $struct_name {
-            fn as_any(&self) -> &dyn std::any::Any {
-                self
-            }
 
             fn name(&self) -> &str {
                 $func_name
diff --git a/src/is_variant_null.rs b/src/is_variant_null.rs
@@ -27,9 +27,6 @@ impl Default for IsVariantNullUdf {
 }
 
 impl ScalarUDFImpl for IsVariantNullUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "is_variant_null"
diff --git a/src/json_to_variant.rs b/src/json_to_variant.rs
@@ -38,9 +38,6 @@ impl Default for JsonToVariantUdf {
 }
 
 impl ScalarUDFImpl for JsonToVariantUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "json_to_variant"
diff --git a/src/variant_contains.rs b/src/variant_contains.rs
@@ -31,10 +31,6 @@ fn variant_contains(variant: Option<&Variant<'_, '_>>, path: &VariantPath<'_>) -
 }
 
 impl ScalarUDFImpl for VariantContainsUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn name(&self) -> &str {
         "variant_contains"
     }
diff --git a/src/variant_get.rs b/src/variant_get.rs
@@ -335,9 +335,6 @@ impl Default for VariantGetUdf {
 }
 
 impl ScalarUDFImpl for VariantGetUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_get"
@@ -392,9 +389,6 @@ impl Default for VariantGetFieldUdf {
 }
 
 impl ScalarUDFImpl for VariantGetFieldUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_get_field"
diff --git a/src/variant_list_construct.rs b/src/variant_list_construct.rs
@@ -28,9 +28,6 @@ impl Default for VariantListConstruct {
 }
 
 impl ScalarUDFImpl for VariantListConstruct {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_list_construct"
diff --git a/src/variant_list_delete.rs b/src/variant_list_delete.rs
@@ -67,9 +67,6 @@ fn delete_list_element(variant_list: Variant, index: usize) -> Result<(Vec<u8>,
 }
 
 impl ScalarUDFImpl for VariantListDelete {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_list_delete"
diff --git a/src/variant_list_insert.rs b/src/variant_list_insert.rs
@@ -29,9 +29,6 @@ impl Default for VariantListInsert {
 }
 
 impl ScalarUDFImpl for VariantListInsert {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_list_insert"
diff --git a/src/variant_normalize.rs b/src/variant_normalize.rs
@@ -62,9 +62,6 @@ impl Default for VariantNormalizeUdf {
 }
 
 impl ScalarUDFImpl for VariantNormalizeUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_normalize"
diff --git a/src/variant_object_construct.rs b/src/variant_object_construct.rs
@@ -29,9 +29,6 @@ impl Default for VariantObjectConstruct {
 }
 
 impl ScalarUDFImpl for VariantObjectConstruct {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_object_construct"
diff --git a/src/variant_object_delete.rs b/src/variant_object_delete.rs
@@ -29,9 +29,6 @@ impl Default for VariantObjectDelete {
 }
 
 impl ScalarUDFImpl for VariantObjectDelete {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_object_delete"
diff --git a/src/variant_object_insert.rs b/src/variant_object_insert.rs
@@ -29,9 +29,6 @@ impl Default for VariantObjectInsert {
 }
 
 impl ScalarUDFImpl for VariantObjectInsert {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_object_insert"
diff --git a/src/variant_object_keys.rs b/src/variant_object_keys.rs
@@ -45,9 +45,6 @@ fn append_keys_from_variant(v_opt: Option<Variant>, builder: &mut ListBuilder<St
 }
 
 impl ScalarUDFImpl for VariantObjectKeys {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_object_keys"
diff --git a/src/variant_pretty.rs b/src/variant_pretty.rs
@@ -28,9 +28,6 @@ impl Default for VariantPretty {
 }
 
 impl ScalarUDFImpl for VariantPretty {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_pretty"
diff --git a/src/variant_to_json.rs b/src/variant_to_json.rs
@@ -39,9 +39,6 @@ impl Default for VariantToJsonUdf {
 }
 
 impl ScalarUDFImpl for VariantToJsonUdf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
 
     fn name(&self) -> &str {
         "variant_to_json"

Original file line number	Diff line number	Diff line change
`@@ -157,9 +157,6 @@ impl CastToVariantUdf {`
`157`	`157`	`}`
`158`	`158`
`159`	`159`	`impl ScalarUDFImpl for CastToVariantUdf {`
`160`		`- fn as_any(&self) -> &dyn std::any::Any {`
`161`		`- self`
`162`		`- }`
`163`	`160`
`164`	`161`	`fn name(&self) -> &str {`
`165`	`162`	`"cast_to_variant"`
Original file line number	Diff line number	Diff line change
`@@ -23,9 +23,6 @@ macro_rules! impl_variant_get_typed {`
`23`	`23`	`}`
`24`	`24`
`25`	`25`	`impl ScalarUDFImpl for $struct_name {`
`26`		`- fn as_any(&self) -> &dyn std::any::Any {`
`27`		`- self`
`28`		`- }`
`29`	`26`
`30`	`27`	`fn name(&self) -> &str {`
`31`	`28`	`$func_name`
Original file line number	Diff line number	Diff line change
`@@ -27,9 +27,6 @@ impl Default for IsVariantNullUdf {`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`impl ScalarUDFImpl for IsVariantNullUdf {`
`30`		`- fn as_any(&self) -> &dyn std::any::Any {`
`31`		`- self`
`32`		`- }`
`33`	`30`
`34`	`31`	`fn name(&self) -> &str {`
`35`	`32`	`"is_variant_null"`
Original file line number	Diff line number	Diff line change
`@@ -38,9 +38,6 @@ impl Default for JsonToVariantUdf {`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`impl ScalarUDFImpl for JsonToVariantUdf {`
`41`		`- fn as_any(&self) -> &dyn std::any::Any {`
`42`		`- self`
`43`		`- }`
`44`	`41`
`45`	`42`	`fn name(&self) -> &str {`
`46`	`43`	`"json_to_variant"`
Original file line number	Diff line number	Diff line change
`@@ -31,10 +31,6 @@ fn variant_contains(variant: Option<&Variant<'_, '_>>, path: &VariantPath<'_>) -`
`31`	`31`	`}`
`32`	`32`
`33`	`33`	`impl ScalarUDFImpl for VariantContainsUdf {`
`34`		`- fn as_any(&self) -> &dyn std::any::Any {`
`35`		`- self`
`36`		`- }`
`37`		`-`
`38`	`34`	`fn name(&self) -> &str {`
`39`	`35`	`"variant_contains"`
`40`	`36`	`}`
Original file line number	Diff line number	Diff line change
`@@ -28,9 +28,6 @@ impl Default for VariantListConstruct {`
`28`	`28`	`}`
`29`	`29`
`30`	`30`	`impl ScalarUDFImpl for VariantListConstruct {`
`31`		`- fn as_any(&self) -> &dyn std::any::Any {`
`32`		`- self`
`33`		`- }`
`34`	`31`
`35`	`32`	`fn name(&self) -> &str {`
`36`	`33`	`"variant_list_construct"`
Original file line number	Diff line number	Diff line change
`@@ -67,9 +67,6 @@ fn delete_list_element(variant_list: Variant, index: usize) -> Result<(Vec<u8>,`
`67`	`67`	`}`
`68`	`68`
`69`	`69`	`impl ScalarUDFImpl for VariantListDelete {`
`70`		`- fn as_any(&self) -> &dyn std::any::Any {`
`71`		`- self`
`72`		`- }`
`73`	`70`
`74`	`71`	`fn name(&self) -> &str {`
`75`	`72`	`"variant_list_delete"`