chore: upgrade DataFusion family

Signed-off-by: luofucong <luofc@foxmail.com>

Author: MichaelScofield

Cargo.lock

@@ -100,13 +100,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
 ahash = { version = "0.8", features = ["compile-time-rng"] }
 aquamarine = "0.6"
-arrow = { version = "57.0", features = ["prettyprint"] }
-arrow-array = { version = "57.0", default-features = false, features = ["chrono-tz"] }
-arrow-buffer = "57.0"
-arrow-cast = "57.0"
-arrow-flight = "57.0"
-arrow-ipc = { version = "57.0", default-features = false, features = ["lz4", "zstd"] }
-arrow-schema = { version = "57.0", features = ["serde"] }
+arrow = { version = "57.2", features = ["prettyprint"] }
+arrow-array = { version = "57.2", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = "57.2"
+arrow-cast = "57.2"
+arrow-flight = "57.2"
+arrow-ipc = { version = "57.2", default-features = false, features = ["lz4", "zstd"] }
+arrow-schema = { version = "57.2", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 # Remember to update axum-extra, axum-macros when updating axum
@@ -126,19 +126,19 @@ config = "0.13.0"
 const_format = "0.2"
 crossbeam-utils = "0.8"
 dashmap = "6.1"
-datafusion = "51.0"
-datafusion-common = "51.0"
-datafusion-datasource = "51.0"
-datafusion-expr = "51.0"
-datafusion-functions = "51.0"
-datafusion-functions-aggregate-common = "51.0"
-datafusion-optimizer = "51.0"
-datafusion-orc = { git = "https://github.com/GreptimeTeam/datafusion-orc.git", rev = "35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e" }
-datafusion-pg-catalog = "0.13"
-datafusion-physical-expr = "51.0"
-datafusion-physical-plan = "51.0"
-datafusion-sql = "51.0"
-datafusion-substrait = "51.0"
+datafusion = "52.0"
+datafusion-common = "52.0"
+datafusion-datasource = "52.0"
+datafusion-expr = "52.0"
+datafusion-functions = "52.0"
+datafusion-functions-aggregate-common = "52.0"
+datafusion-optimizer = "52.0"
+datafusion-orc = "0.7"
+datafusion-pg-catalog = "0.14"
+datafusion-physical-expr = "52.0"
+datafusion-physical-plan = "52.0"
+datafusion-sql = "52.0"
+datafusion-substrait = "52.0"
 deadpool = "0.12"
 deadpool-postgres = "0.14"
 derive_builder = "0.20"
@@ -185,7 +185,7 @@ otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5
     "server",
 ] }
 parking_lot = "0.12"
-parquet = { version = "57.0", default-features = false, features = ["arrow", "async", "object_store"] }
+parquet = { version = "57.2", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
 pin-project = "1.0"
 pretty_assertions = "1.4.0"
@@ -224,7 +224,7 @@ simd-json = "0.15"
 similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
-sqlparser = { version = "0.59.0", default-features = false, features = ["std", "visitor", "serde"] }
+sqlparser = { version = "0.60.0", default-features = false, features = ["std", "visitor", "serde"] }
 sqlx = { version = "0.8", default-features = false, features = ["any", "macros", "json", "runtime-tokio-rustls"] }
 strum = { version = "0.27", features = ["derive"] }
 sysinfo = "0.33"
@@ -323,20 +323,19 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
 rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
 
 [patch.crates-io]
-datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "74ac8e2806be6de91ff192b97f64735392539d16" }
-datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" }
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "d7d95a44889e099e32d78e9bad9bc00598faef28" }                           # on branch v0.59.x
+datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" }
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "e8a3ac44693768965abf7030cd38ebcff53a4d29" }                           # on branch v0.60.x
 
 [profile.release]
 debug = 1

Cargo.toml

@@ -10,7 +10,7 @@ workspace = true
 [dependencies]
 arrow.workspace = true
 arrow-schema.workspace = true
-async-compression = { version = "0.3", features = [
+async-compression = { version = "0.4", features = [
     "bzip2",
     "gzip",
     "xz",

src/common/datasource/Cargo.toml

@@ -203,8 +203,8 @@ pub enum Error {
         error: parquet::errors::ParquetError,
     },
 
-    #[snafu(display("Failed to build file stream"))]
-    BuildFileStream {
+    #[snafu(transparent)]
+    DataFusion {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
@@ -241,8 +241,9 @@ impl ErrorExt for Error {
             | ReadRecordBatch { .. }
             | WriteRecordBatch { .. }
             | EncodeRecordBatch { .. }
-            | BuildFileStream { .. }
             | OrcReader { .. } => StatusCode::Unexpected,
+
+            DataFusion { .. } => StatusCode::Internal,
         }
     }
 

src/common/datasource/src/error.rs

@@ -313,16 +313,13 @@ pub async fn file_to_stream(
                 filename.to_string(),
                 0,
             )]))
-            .with_projection_indices(projection)
+            .with_projection_indices(projection)?
             .with_file_compression_type(df_compression)
             .build();
 
     let store = Arc::new(OpendalStore::new(store.clone()));
-    let file_opener = file_source
-        .with_projection(&config)
-        .create_file_opener(store, &config, 0);
-    let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())
-        .context(error::BuildFileStreamSnafu)?;
+    let file_opener = file_source.create_file_opener(store, &config, 0)?;
+    let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?;
 
     Ok(Box::pin(stream))
 }

src/common/datasource/src/file_format.rs

@@ -46,7 +46,10 @@ struct Test<'a> {
 impl Test<'_> {
     async fn run(self, store: &ObjectStore) {
         let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
-        let file_opener = self.file_source.create_file_opener(store, &self.config, 0);
+        let file_opener = self
+            .file_source
+            .create_file_opener(store, &self.config, 0)
+            .unwrap();
 
         let result = FileStream::new(
             &self.config,

src/common/datasource/src/file_format/tests.rs

@@ -154,11 +154,13 @@ pub async fn setup_stream_to_csv_test(
     let config = scan_config(None, origin_path, csv_source.clone());
     let size = store.read(origin_path).await.unwrap().len();
 
-    let csv_opener = csv_source.create_file_opener(
-        Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
-        &config,
-        0,
-    );
+    let csv_opener = csv_source
+        .create_file_opener(
+            Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
+            &config,
+            0,
+        )
+        .unwrap();
     let stream = FileStream::new(&config, 0, csv_opener, &ExecutionPlanMetricsSet::new()).unwrap();
 
     let (tmp_store, dir) = test_tmp_store("test_stream_to_csv");

src/common/datasource/src/test_util.rs

@@ -428,7 +428,7 @@ impl Accumulator for CountHashAccumulator {
             &self.random_state,
             &mut self.batch_hashes,
         )?;
-        for hash in hashes.as_slice() {
+        for hash in hashes {
             self.values.insert(*hash);
         }
         Ok(())

src/common/function/src/aggrs/count_hash.rs

@@ -48,7 +48,7 @@ pub fn rename_logical_plan_columns(
             plan.schema().qualified_field_from_column(&old_column)?;
 
         for (qualifier, field) in plan.schema().iter() {
-            if qualifier.eq(&qualifier_rename) && field.as_ref() == field_rename {
+            if qualifier.eq(&qualifier_rename) && field == field_rename {
                 projection.push(col(Column::from((qualifier, field))).alias(new_name));
             }
         }

src/common/query/src/logical_plan.rs

@@ -178,8 +178,8 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to extract column from filter"))]
-    ExtractColumnFromFilter {
+    #[snafu(transparent)]
+    DataFusion {
         #[snafu(source)]
         error: DataFusionError,
         #[snafu(implicit)]
@@ -225,8 +225,9 @@ impl ErrorExt for Error {
             | ManifestExists { .. }
             | BuildStream { .. }
             | ParquetScanPlan { .. }
-            | UnexpectedEngine { .. }
-            | ExtractColumnFromFilter { .. } => StatusCode::Unexpected,
+            | UnexpectedEngine { .. } => StatusCode::Unexpected,
+
+            DataFusion { .. } => StatusCode::Internal,
         }
     }