Test

Author: matthewmturner

.github/workflows/test.yml

@@ -196,7 +196,31 @@ jobs:
         uses: ./.github/actions/setup-rust
       - name: Run Deltalake tests
         run: |
-          cargo test --features=deltalake extension_cases::deltalake
+          cargo test --features=deltalake extension_cases::deltalake::test_deltalake
+  test-deltalake-s3:
+    name: Extension / Deltalake-S3
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust Toolchain
+        uses: ./.github/actions/setup-rust
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@v0.2.3
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+          configuration: DEBUG=1
+      - name: Upload Delta Lake data to LocalStack
+        run: |
+          awslocal s3 mb s3://test
+          awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table
+          echo "Test Execution complete!"
+      - name: Run Deltalake S3 tests
+        run: |
+          cargo test --features="deltalake s3" extension_cases::deltalake::test_deltalake_s3
   test-udfs-wasm:
     name: Extension / UDFs-WASM
     runs-on: ubuntu-latest

CLAUDE.md

@@ -52,6 +52,7 @@ cargo test --features=flightsql extension_cases::flightsql -- --test-threads=1
 cargo test --features=s3 extension_cases::s3
 cargo test --features=functions-json extension_cases::functions_json
 cargo test --features=deltalake extension_cases::deltalake
+cargo test --features="deltalake s3" extension_cases::deltalake::test_deltalake_s3  # Requires LocalStack
 cargo test --features=udfs-wasm extension_cases::udfs_wasm
 cargo test --features=vortex extension_cases::vortex
 cargo test --features=vortex cli_cases::basic::test_output_vortex
@@ -189,16 +190,20 @@ The main runtime in `src/main.rs` uses a single-threaded Tokio runtime optimized
 
 ### Testing Against LocalStack
 
-Some tests (S3, TUI, CLI) require LocalStack for S3 testing. The CI workflow shows the setup:
+Some tests (S3, TUI, CLI, Delta Lake + S3) require LocalStack for S3 testing. The CI workflow shows the setup:
 
 ```bash
 # Start LocalStack
 localstack start -d
-awslocal s3api create-bucket --bucket tmp --acl public-read
-awslocal s3 mv data/aggregate_test_100.csv s3://tmp/
+awslocal s3api create-bucket --bucket test --acl public-read
+awslocal s3 mv data/aggregate_test_100.csv s3://test/
 
-# Run tests
+# Run S3 tests
 cargo test --features=s3 extension_cases::s3
+
+# For Delta Lake + S3 tests, also sync the delta lake data
+awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table
+cargo test --features="deltalake s3" extension_cases::deltalake::test_deltalake_s3
 ```
 
 ### Benchmarking

tests/extension_cases/deltalake.rs

@@ -42,3 +42,60 @@ async fn test_deltalake() {
         vec!["+----+", "| id |", "+----+", "| 5  |", "| 7  |", "| 9  |", "+----+"]
     );
 }
+
+#[cfg(feature = "s3")]
+#[tokio::test(flavor = "multi_thread")]
+async fn test_deltalake_s3() {
+    use assert_cmd::Command;
+    use std::io::Write;
+
+    use crate::{cli_cases::contains_str, config::TestConfigBuilder};
+
+    let tempdir = tempfile::tempdir().unwrap();
+    let ddl_path = tempdir.path().join("my_ddl.sql");
+    let mut file = std::fs::File::create(ddl_path.clone()).unwrap();
+    let ddl = "CREATE EXTERNAL TABLE delta_tbl STORED AS DELTATABLE LOCATION 's3://test/deltalake/simple_table';";
+    file.write_all(ddl.as_bytes()).unwrap();
+    file.flush().unwrap();
+
+    // Create a temp db directory to avoid conflicts with existing db files
+    let db_dir = tempdir.path().join("db");
+    std::fs::create_dir(&db_dir).unwrap();
+
+    let mut config_builder = TestConfigBuilder::default();
+    config_builder.with_ddl_path("cli", ddl_path);
+    config_builder.with_db_path(&format!("file://{}", db_dir.display()));
+    config_builder.with_s3_object_store(
+        "cli",
+        "s3",
+        "test",
+        "s3://test",
+        "http://localhost:4566",
+        "LSIAQAAAAAAVNCBMPNSG",
+        "5555555555555555555555555555555555555555",
+        true,
+    );
+    let config = config_builder.build("my_config.toml");
+
+    let assert = Command::cargo_bin("dft")
+        .unwrap()
+        .arg("--config")
+        .arg(config.path)
+        .arg("--run-ddl")
+        .arg("-c")
+        .arg("SELECT id FROM delta_tbl ORDER BY id")
+        .assert()
+        .success();
+
+    let expected = r#"
++----+
+| id |
++----+
+| 5  |
+| 7  |
+| 9  |
++----+
+"#;
+
+    assert.stdout(contains_str(expected));
+}