Merge branch 'main' into experiment_roaring_bitmap_for_int32_anti_semi_joins

Author: coderfender

Cargo.toml

@@ -104,8 +104,11 @@ arrow-data = { version = "58.1.0", default-features = false }
 arrow-flight = { version = "58.1.0", features = [
     "flight-sql-experimental",
 ] }
+# Both codecs are required here to make sure that code paths like
+# file-spilling have access to all compression codecs.
 arrow-ipc = { version = "58.1.0", default-features = false, features = [
     "lz4",
+    "zstd",
 ] }
 arrow-ord = { version = "58.1.0", default-features = false }
 arrow-schema = { version = "58.1.0", default-features = false }

datafusion/datasource-arrow/Cargo.toml

@@ -62,6 +62,6 @@ name = "datafusion_datasource_arrow"
 path = "src/mod.rs"
 
 [features]
-compression = [
-    "arrow-ipc/zstd",
-]
+# This feature is deprecated, as core functionality in the SpillManager requires all features
+# it enabled, and will be removed in a future version.
+compression = []

datafusion/datasource-arrow/src/mod.rs

@@ -21,6 +21,10 @@
 #![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
 
 //! [`ArrowFormat`]: Apache Arrow file format abstractions
+//!
+//! Note: As of DataFusion 54.0.0, the `compression` feature of this crate
+//! is a no-op, only kept for backwards compatibility purposes, and it will
+//! be removed in a future release.
 
 pub mod file_format;
 pub mod source;