fix: Enable arrow-ipc/zstd in datasource-arrow to make test_spill_compression pass in every config (#21504)

## Which issue does this PR close?

- Closes #21503.

## Rationale for this change

The spill manager assumes that all available compressions are actually
available, which currently relies on feature unification with
`datafusion-datasource-arrow`.

## What changes are included in this PR?

Move `arrow-ipc/zstd` feature from `datafusion-datasource-arrow` to the
workspace dependency, like lz4.

## Are these changes tested?

Existing tests cover the functionality, tested individual crates
locally.

## Are there any user-facing changes?

Shifts some features around which might change behavior for users for
DataFusion, not sure how the project reasons about this level of
changes.

Author: AdamGS

Cargo.toml

@@ -104,8 +104,11 @@ arrow-data = { version = "58.1.0", default-features = false }
 arrow-flight = { version = "58.1.0", features = [
     "flight-sql-experimental",
 ] }
+# Both codecs are required here to make sure that code paths like
+# file-spilling have access to all compression codecs.
 arrow-ipc = { version = "58.1.0", default-features = false, features = [
     "lz4",
+    "zstd",
 ] }
 arrow-ord = { version = "58.1.0", default-features = false }
 arrow-schema = { version = "58.1.0", default-features = false }

datafusion/datasource-arrow/Cargo.toml

@@ -62,6 +62,6 @@ name = "datafusion_datasource_arrow"
 path = "src/mod.rs"
 
 [features]
-compression = [
-    "arrow-ipc/zstd",
-]
+# This feature is deprecated, as core functionality in the SpillManager requires all features
+# it enabled, and will be removed in a future version.
+compression = []

datafusion/datasource-arrow/src/mod.rs

@@ -21,6 +21,10 @@
 #![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
 
 //! [`ArrowFormat`]: Apache Arrow file format abstractions
+//!
+//! Note: As of DataFusion 54.0.0, the `compression` feature of this crate
+//! is a no-op, only kept for backwards compatibility purposes, and it will
+//! be removed in a future release.
 
 pub mod file_format;
 pub mod source;