refactor: move from pyarrow to arro3

Signed-off-by: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com>

Author: ion-elgreco

.github/scripts/retry_integration_test.sh

@@ -5,7 +5,7 @@ MAX_RETRIES=$2
 RETRY_DELAY=$3
 ATTEMPT=1
 run_command() {
-  uv run --no-sync pytest -m "($TEST_NAME and integration)" --doctest-modules 2>&1
+  uv run --no-sync pytest -m "($TEST_NAME and integration and pyarrow)" --doctest-modules 2>&1
 }
 until [ $ATTEMPT -gt $MAX_RETRIES ]
 do

.github/workflows/python_build.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Check Python
         run: |
-          uv sync --only-group dev --no-install-project
+          uv sync --no-install-project
           make check-python
 
       - name: Check Rust
@@ -49,7 +49,7 @@ jobs:
         run: make unit-test
 
   test:
-    name: Python Build (Python 3.10 PyArrow latest)
+    name: Python Build (Python 3.10 Optional latest pyarrow)
     runs-on: ubuntu-latest
     env:
       SCCACHE_GHA_ENABLED: "true"
@@ -70,13 +70,18 @@ jobs:
         run: make develop
 
       - name: Run tests
-        run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
+        run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark and pyarrow' --doctest-modules
 
       - name: Test without pandas
         run: |
           uv pip uninstall pandas
-          uv run --no-sync pytest -m "not pandas and not integration and not benchmark"
-          uv pip install pandas
+          uv run --no-sync pytest -m "pyarrow and not pandas and not integration and not benchmark"
+
+      - name: Test without pyarrow and without pandas
+        run: |
+          uv pip uninstall pyarrow
+          uv run --no-sync pytest -m "not pyarrow and not pandas and not integration and not benchmark and no_pyarrow"
+
 
   test-lakefs:
     name: Python Build (Python 3.10 LakeFS Integration tests)
@@ -189,4 +194,19 @@ jobs:
 
       - name: Run deltalake
         run: |
-          uv run python -c 'import deltalake'
+          uv run --no-sync python -c 'import deltalake'
+
+      - name: Run deltalake without pyarrow
+        run: |
+          uv pip uninstall pyarrow
+          uv run --no-sync python -c 'import deltalake'
+
+      - name: Run deltalake without pyarrow pandas
+        run: |
+          uv pip uninstall pyarrow pandas
+          uv run --no-sync python -c 'import deltalake'
+
+      - name: Run deltalake without pandas
+        run: |
+          uv pip install pyarrow
+          uv run --no-sync python -c 'import deltalake'

docs/usage/examining-table.md

@@ -57,7 +57,7 @@ the table will be loaded into.
     let schema = table.get_schema()?;
     println!("schema: {:?}", schema);
     ```
-These schemas have a JSON representation that can be retrieved. 
+These schemas have a JSON representation that can be retrieved.
 
 === "Python"
     To reconstruct from json, use [DeltaTable.schema.to_json()][deltalake.schema.Schema.to_json].
@@ -73,10 +73,10 @@ These schemas have a JSON representation that can be retrieved.
 It is also possible to retrieve the Arrow schema:
 === "Python"
 
-    Use [DeltaTable.schema.to_pyarrow()][deltalake.schema.Schema.to_pyarrow] to retrieve the PyArrow schema:
+    Use [DeltaTable.schema.to_arrow()][deltalake.schema.Schema.to_arrow] to retrieve the Arro3 schema:
 
     ``` python
-    >>> dt.schema().to_pyarrow()
+    >>> dt.schema().to_arrow()
     id: int64
     ```
 === "Rust"
@@ -161,4 +161,4 @@ This works even with past versions of the table:
     table.load_version(0).await?;
     let actions = table.snapshot()?.add_actions_table(true)?;
     println!("{}", pretty_format_batches(&vec![actions])?);
-    ```
+    ```

docs/usage/querying-delta-tables.md

@@ -18,7 +18,7 @@ To load into Pandas or a PyArrow table use the `DeltaTable.to_pandas` and `Delta
 ``` python
 >>> from deltalake import DeltaTable
 >>> dt = DeltaTable("../rust/tests/data/delta-0.8.0-partitioned")
->>> dt.schema().to_pyarrow()
+>>> dt.schema().to_arrow()
 value: string
 year: string
 month: string
@@ -117,7 +117,7 @@ Dask Name: read-parquet, 6 tasks
 1     7  2021    12  20
 ```
 
-When working with the Rust API, Apache Datafusion can be used to query data from a delta table. 
+When working with the Rust API, Apache Datafusion can be used to query data from a delta table.
 
 ```rust
 let table = deltalake::open_table("../rust/tests/data/delta-0.8.0-partitioned").await?;
@@ -134,4 +134,4 @@ let ctx = SessionContext::new();
 let dataframe = ctx.read_table( Arc::new(table.clone()))?;
 let df = dataframe.filter(col("year").eq(lit(2021)))?.select(vec![col("value")])?;
 df.show().await?;
-```
+```

python/Cargo.toml

@@ -17,6 +17,8 @@ doc = false
 [dependencies]
 delta_kernel.workspace = true
 
+pyo3-arrow = { version = "*", default-features = false}
+
 # arrow
 arrow-schema = { workspace = true, features = ["serde"] }
 

python/Makefile

@@ -67,7 +67,7 @@ test-cov: ## Create coverage report
 
 .PHONY: test-pyspark
 test-pyspark:
-	uv run --no-sync pytest -m 'pyspark and integration'
+	uv run --no-sync pytest -m 'pyarrow and pyspark and integration'
 
 .PHONY: build-documentation
 build-documentation: ## Build documentation with Sphinx