diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index c4f0fa8..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(cargo test:*)", - "WebFetch(domain:crates.io)", - "WebFetch(domain:docs.rs)", - "Bash(cargo tree:*)", - "Bash(cargo check:*)", - "Bash(cargo update:*)", - "Bash(cargo install:*)", - "Bash(cargo metadata:*)", - "Bash(cargo clean:*)", - "WebSearch", - "Bash(cargo search:*)", - "Bash(cargo info:*)", - "Bash(cargo build:*)", - "Bash(cargo doc:*)", - "Bash(RUST_BACKTRACE=1 cargo test:*)", - "Bash(cargo run:*)", - "Bash(RUST_LOG=debug cargo run:*)", - "Bash(cat:*)", - "Bash(cargo clippy:*)", - "Bash(find:*)", - "WebFetch(domain:arrow.apache.org)", - "Bash(timeout 90 cargo test:*)", - "Bash(test:*)", - "WebFetch(domain:github.com)", - "Bash(ls:*)", - "Bash(grep:*)", - "Bash(cargo fmt:*)", - "Bash(./target/release/dft:*)", - "Bash(/Users/matth/OpenSource/datafusion-tui/target/debug/dft:*)", - "Bash(./target/debug/dft:*)" - ], - "deny": [], - "ask": [] - } -} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7729f45..5c57d3b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,6 +85,14 @@ jobs: test-cli: name: App / CLI runs-on: ubuntu-latest + services: + rustfs: + image: rustfs/rustfs:latest + ports: + - 9000:9000 + env: + RUSTFS_ACCESS_KEY: LSIAQAAAAAAVNCBMPNSG + RUSTFS_SECRET_KEY: "5555555555555555555555555555555555555555" strategy: matrix: arch: [amd64] @@ -107,17 +115,17 @@ jobs: target-${{ runner.os }}- - name: Setup Rust Toolchain uses: ./.github/actions/setup-rust - - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.3 - with: - image-tag: 'latest' - install-awslocal: 'true' - configuration: DEBUG=1 - - name: Run Tests against LocalStack + - name: Wait for RustFS + run: timeout 60 bash -c 'until nc -z localhost 9000; do sleep 2; done' + - name: Setup S3 buckets + env: + AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG + AWS_SECRET_ACCESS_KEY: "5555555555555555555555555555555555555555" + AWS_DEFAULT_REGION: us-east-1 run: | - awslocal s3 mb s3://test - awslocal s3 mv data/aggregate_test_100.csv s3://test/ - awslocal s3 mb s3://tpch-db + aws --endpoint-url http://localhost:9000 s3 mb s3://test + aws --endpoint-url http://localhost:9000 s3 cp data/aggregate_test_100.csv s3://test/ + aws --endpoint-url http://localhost:9000 s3 mb s3://tpch-db echo "Test Execution complete!" - name: Run CLI tests run: | @@ -125,6 +133,14 @@ jobs: test-tui: name: App / TUI runs-on: ubuntu-latest + services: + rustfs: + image: rustfs/rustfs:latest + ports: + - 9000:9000 + env: + RUSTFS_ACCESS_KEY: LSIAQAAAAAAVNCBMPNSG + RUSTFS_SECRET_KEY: "5555555555555555555555555555555555555555" strategy: matrix: arch: [amd64] @@ -147,17 +163,17 @@ jobs: target-${{ runner.os }}- - name: Setup Rust Toolchain uses: ./.github/actions/setup-rust - - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.3 - with: - image-tag: 'latest' - install-awslocal: 'true' - configuration: DEBUG=1 - - name: Run Tests against LocalStack + - name: Wait for RustFS + run: timeout 60 bash -c 'until nc -z localhost 9000; do sleep 2; done' + - name: Setup S3 buckets + env: + AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG + AWS_SECRET_ACCESS_KEY: "5555555555555555555555555555555555555555" + AWS_DEFAULT_REGION: us-east-1 run: | - awslocal s3 mb s3://test - awslocal s3 mv data/aggregate_test_100.csv s3://test/ - awslocal s3 mb s3://tpch-db + aws --endpoint-url http://localhost:9000 s3 mb s3://test + aws --endpoint-url http://localhost:9000 s3 cp data/aggregate_test_100.csv s3://test/ + aws --endpoint-url http://localhost:9000 s3 mb s3://tpch-db echo "Test Execution complete!" - name: Run TUI tests run: | @@ -166,6 +182,14 @@ jobs: test-s3: name: Extension / S3 runs-on: ubuntu-latest + services: + rustfs: + image: rustfs/rustfs:latest + ports: + - 9000:9000 + env: + RUSTFS_ACCESS_KEY: LSIAQAAAAAAVNCBMPNSG + RUSTFS_SECRET_KEY: "5555555555555555555555555555555555555555" strategy: matrix: arch: [amd64] @@ -188,20 +212,20 @@ jobs: target-${{ runner.os }}- - name: Setup Rust Toolchain uses: ./.github/actions/setup-rust - - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.3 - with: - image-tag: 'latest' - install-awslocal: 'true' - configuration: DEBUG=1 - - name: Run Tests against LocalStack + - name: Wait for RustFS + run: timeout 60 bash -c 'until nc -z localhost 9000; do sleep 2; done' + - name: Setup S3 buckets + env: + AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG + AWS_SECRET_ACCESS_KEY: "5555555555555555555555555555555555555555" + AWS_DEFAULT_REGION: us-east-1 run: | - awslocal s3 mb s3://test - awslocal s3 mv data/aggregate_test_100.csv s3://test/ + aws --endpoint-url http://localhost:9000 s3 mb s3://test + aws --endpoint-url http://localhost:9000 s3 cp data/aggregate_test_100.csv s3://test/ echo "Test Execution complete!" - name: Run S3 tests run: | - cargo test --features=s3 extension_cases::s3 + cargo test --features=s3 extension_cases::s3 -- --test-threads=1 test-functions-json: name: Extension / Functions-JSON runs-on: ubuntu-latest @@ -235,6 +259,14 @@ jobs: test-deltalake: name: Extension / Deltalake runs-on: ubuntu-latest + services: + rustfs: + image: rustfs/rustfs:latest + ports: + - 9000:9000 + env: + RUSTFS_ACCESS_KEY: LSIAQAAAAAAVNCBMPNSG + RUSTFS_SECRET_KEY: "5555555555555555555555555555555555555555" strategy: matrix: arch: [amd64] @@ -257,26 +289,20 @@ jobs: target-${{ runner.os }}- - name: Setup Rust Toolchain uses: ./.github/actions/setup-rust - - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.3 - with: - image-tag: 'latest' - install-awslocal: 'true' - configuration: DEBUG=1 - - name: Upload Delta Lake data to LocalStack + - name: Wait for RustFS + run: timeout 60 bash -c 'until nc -z localhost 9000; do sleep 2; done' + - name: Upload Delta Lake data to RustFS + env: + AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG + AWS_SECRET_ACCESS_KEY: "5555555555555555555555555555555555555555" + AWS_DEFAULT_REGION: us-east-1 run: | - awslocal s3 mb s3://test - awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table - echo "Delta Lake data uploaded to LocalStack" + aws --endpoint-url http://localhost:9000 s3 mb s3://test + aws --endpoint-url http://localhost:9000 s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table + echo "Delta Lake data uploaded to RustFS" - name: Run Deltalake tests run: | cargo test --features="deltalake s3" extension_cases::deltalake - env: - AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG - AWS_SECRET_ACCESS_KEY: 5555555555555555555555555555555555555555 - AWS_ENDPOINT_URL: http://localhost:4566 - AWS_REGION: us-east-1 - AWS_ALLOW_HTTP: true test-udfs-wasm: name: Extension / UDFs-WASM runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 54a6f42..47cd74c 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ bench/url_files/* # Tags tags + +.claude/settings.local.json diff --git a/Cargo.lock b/Cargo.lock index a4e101c..5fe2494 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,7 +390,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" dependencies = [ - "bitflags", + "bitflags 2.11.0", "serde", "serde_core", "serde_json", @@ -633,6 +633,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1205,12 +1214,33 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit-vec" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -1437,12 +1467,6 @@ dependencies = [ "winx", ] -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "castaway" version = "0.2.4" @@ -1508,7 +1532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1608,16 +1632,16 @@ version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "crossterm 0.29.0", + "crossterm", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width", ] [[package]] name = "compact_str" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -1645,7 +1669,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.0", + "unicode-width", "windows-sys 0.59.0", ] @@ -1690,6 +1714,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1908,34 +1941,22 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags", - "crossterm_winapi", - "futures-core", - "mio", - "parking_lot", - "rustix 0.38.44", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crossterm_winapi", + "derive_more", "document-features", + "futures-core", + "mio", "parking_lot", "rustix 1.1.3", + "signal-hook", + "signal-hook-mio", "winapi", ] @@ -1964,6 +1985,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "csscolorparser" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" +dependencies = [ + "lab", + "phf 0.11.3", +] + [[package]] name = "csv" version = "1.4.0" @@ -2410,7 +2441,7 @@ dependencies = [ "axum 0.7.9", "clap", "color-eyre", - "crossterm 0.28.1", + "crossterm", "datafusion", "datafusion-app", "datafusion-udfs-wasm", @@ -2434,6 +2465,7 @@ dependencies = [ "predicates", "prost", "ratatui", + "ratatui-textarea", "reqwest", "serde", "serde_json", @@ -2452,7 +2484,6 @@ dependencies = [ "tracing", "tracing-subscriber", "tui-logger", - "tui-textarea", "url", "uuid", "vortex", @@ -2957,6 +2988,12 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "deltae" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" + [[package]] name = "deltalake" version = "0.30.2" @@ -3056,7 +3093,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3963d9fe965af7b1dea433271389e1e39c6a97ffdbc2e81d808f5b329e4577b3" dependencies = [ - "convert_case", + "convert_case 0.9.0", "itertools 0.14.0", "proc-macro2", "quote", @@ -3072,6 +3109,28 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.115", +] + [[package]] name = "difflib" version = "0.4.0" @@ -3321,6 +3380,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -3397,6 +3465,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastlanes" version = "0.5.0" @@ -3427,12 +3505,35 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -3445,7 +3546,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags", + "bitflags 2.11.0", "rustc_version", ] @@ -3663,7 +3764,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" dependencies = [ - "bitflags", + "bitflags 2.11.0", "debugid", "fxhash", "serde", @@ -3811,8 +3912,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.1.5", "serde", ] @@ -4197,7 +4296,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width", "web-time", ] @@ -4428,6 +4527,23 @@ dependencies = [ "lock_api", ] +[[package]] +name = "kasuari" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" +dependencies = [ + "hashbrown 0.16.1", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + [[package]] name = "lazy_static" version = "1.5.0" @@ -4562,10 +4678,19 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", ] +[[package]] +name = "line-clipping" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f50e8f47623268b5407192d26876c4d7f89d686ca130fdc53bced4814cd29f8" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -4607,11 +4732,11 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru" -version = "0.12.5" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -4640,6 +4765,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "mach2" version = "0.4.3" @@ -4717,6 +4852,12 @@ dependencies = [ "rustix 1.1.3", ] +[[package]] +name = "memmem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" + [[package]] name = "memoffset" version = "0.9.1" @@ -4779,6 +4920,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4866,6 +5013,29 @@ version = "6.6.666" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6" +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -4927,6 +5097,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.115", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4977,6 +5158,15 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -5110,7 +5300,7 @@ version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -5163,6 +5353,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.1.0" @@ -5280,25 +5479,120 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3637c05577168127568a64e9dc5a6887da720efef07b3d9472d45f63ab191166" +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.115", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + [[package]] name = "petgraph" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "hashbrown 0.15.5", "indexmap", "serde", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.115", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", ] [[package]] @@ -5834,23 +6128,100 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +checksum = "d1ce67fb8ba4446454d1c8dbaeda0557ff5e94d39d5e5ed7f10a65eb4c8266bc" dependencies = [ - "bitflags", - "cassowary", + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-macros", + "ratatui-termwiz", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef8dea09a92caaf73bff7adb70b76162e5937524058a7e5bff37869cbbec293" +dependencies = [ + "bitflags 2.11.0", "compact_str", - "crossterm 0.28.1", + "hashbrown 0.16.1", "indoc", - "instability", - "itertools 0.13.0", + "itertools 0.14.0", + "kasuari", "lru", - "paste", - "strum 0.26.3", + "strum 0.27.2", + "thiserror 2.0.18", "unicode-segmentation", "unicode-truncate", - "unicode-width 0.2.0", + "unicode-width", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c9b9f652b4c121fb25c6a391dd06406d3b092ba68827e6d2f09550edc54b3" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f1342a13e83e4bb9d0b793d0ea762be633f9582048c892ae9041ef39c936f4" +dependencies = [ + "ratatui-core", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-termwiz" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f76fe0bd0ed4295f0321b1676732e2454024c15a35d01904ddb315afd3d545c" +dependencies = [ + "ratatui-core", + "termwiz", +] + +[[package]] +name = "ratatui-textarea" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de236b7cc74b3f7dea227b3fbad97bf459cddf552b6503d888fb9a106eda59ab" +dependencies = [ + "ratatui-core", + "ratatui-crossterm", + "ratatui-widgets", + "regex", + "unicode-width", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dbfa023cd4e604c2553483820c5fe8aa9d71a42eea5aa77c6e7f35756612db" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.16.1", + "indoc", + "instability", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "strum 0.27.2", + "time", + "unicode-segmentation", + "unicode-width", ] [[package]] @@ -5859,7 +6230,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -5908,7 +6279,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -6116,7 +6487,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -6129,7 +6500,7 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys 0.11.0", @@ -6246,7 +6617,7 @@ version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -6674,7 +7045,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -6695,7 +7066,7 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cap-fs-ext", "cap-std", "fd-lock", @@ -6751,12 +7122,75 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminfo" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" +dependencies = [ + "fnv", + "nom", + "phf 0.11.3", + "phf_codegen", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + [[package]] name = "termtree" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "termwiz" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bitflags 2.11.0", + "fancy-regex", + "filedescriptor", + "finl_unicode", + "fixedbitset 0.4.2", + "hex", + "lazy_static", + "libc", + "log", + "memmem", + "nix", + "num-derive", + "num-traits", + "ordered-float 4.6.0", + "pest", + "pest_derive", + "phf 0.11.3", + "sha2", + "signal-hook", + "siphasher", + "terminfo", + "termios", + "thiserror 1.0.69", + "ucd-trie", + "unicode-segmentation", + "vtparse", + "wezterm-bidi", + "wezterm-blob-leases", + "wezterm-color-types", + "wezterm-dynamic", + "wezterm-input-types", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -6824,7 +7258,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "libc", "num-conv", + "num_threads", "powerfmt", "serde_core", "time-core", @@ -7103,7 +7539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "base64 0.22.1", - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http 1.4.0", @@ -7237,9 +7673,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tui-logger" -version = "0.17.4" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382b7ea88082dbe2236ed1e942552b1bfc59e98fdc5d0599f11a627aae9ee2be" +checksum = "9384df20a5244a6ab204bc4b6959b41f37f0ee7b5e0f2feb7a8a78f58e684d06" dependencies = [ "chrono", "env_filter 0.1.4", @@ -7252,18 +7688,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "tui-textarea" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5318dd619ed73c52a9417ad19046724effc1287fb75cdcc4eca1d6ac1acbae" -dependencies = [ - "crossterm 0.28.1", - "ratatui", - "regex", - "unicode-width 0.2.0", -] - [[package]] name = "twox-hash" version = "2.1.2" @@ -7296,6 +7720,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicode-ident" version = "1.0.23" @@ -7310,21 +7740,15 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-truncate" -version = "1.1.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "unicode-segmentation", - "unicode-width 0.1.14", + "unicode-width", ] -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.0" @@ -7386,6 +7810,7 @@ version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ + "atomic", "getrandom 0.4.1", "js-sys", "rand 0.9.2", @@ -8022,7 +8447,7 @@ checksum = "e60af71eee0bea4c909c66596ab978e835d0a2985efc27805362b35bd29da24f" dependencies = [ "arrow-array", "arrow-schema", - "bit-vec", + "bit-vec 0.8.0", "futures", "itertools 0.14.0", "parking_lot", @@ -8151,6 +8576,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "vtparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9b2acfb050df409c972a37d3b8e08cdea3bddb0c09db9d53137e504cfabed0" +dependencies = [ + "utf8parse", +] + [[package]] name = "wait-timeout" version = "0.2.1" @@ -8192,7 +8626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe3101bd34deeb64225431f8b1b1793c87e7cad94383464878b3f90da6995977" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "cap-fs-ext", "cap-rand", "cap-std", @@ -8348,7 +8782,7 @@ version = "0.221.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d06bfa36ab3ac2be0dee563380147a5b81ba10dd8885d7fbbc9eb574be67d185" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -8361,7 +8795,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -8373,7 +8807,7 @@ version = "0.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f08c9adee0428b7bddf3890fc27e015ac4b761cc608c822667102b8bfd6995e" dependencies = [ - "bitflags", + "bitflags 2.11.0", "indexmap", "semver", ] @@ -8398,7 +8832,7 @@ dependencies = [ "addr2line 0.24.2", "anyhow", "async-trait", - "bitflags", + "bitflags 2.11.0", "bumpalo", "cc", "cfg-if", @@ -8660,7 +9094,7 @@ dependencies = [ "bumpalo", "leb128fmt", "memchr", - "unicode-width 0.2.0", + "unicode-width", "wasm-encoder 0.245.1", ] @@ -8702,6 +9136,78 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "wezterm-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" +dependencies = [ + "log", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-blob-leases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" +dependencies = [ + "getrandom 0.3.4", + "mac_address", + "sha2", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "wezterm-color-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de81ef35c9010270d63772bebef2f2d6d1f2d20a983d27505ac850b8c4b4296" +dependencies = [ + "csscolorparser", + "deltae", + "lazy_static", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" +dependencies = [ + "log", + "ordered-float 4.6.0", + "strsim", + "thiserror 1.0.69", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c0cf2d539c645b448eaffec9ec494b8b19bd5077d9e58cb1ae7efece8d575b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wezterm-input-types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" +dependencies = [ + "bitflags 1.3.2", + "euclid", + "lazy_static", + "serde", + "wezterm-dynamic", +] + [[package]] name = "wiggle" version = "29.0.1" @@ -8710,7 +9216,7 @@ checksum = "4b9af35bc9629c52c261465320a9a07959164928b4241980ba1cf923b9e6751d" dependencies = [ "anyhow", "async-trait", - "bitflags", + "bitflags 2.11.0", "thiserror 1.0.69", "tracing", "wasmtime", @@ -9109,7 +9615,7 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags", + "bitflags 2.11.0", "windows-sys 0.59.0", ] @@ -9171,7 +9677,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "indexmap", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index d914b1c..2762c78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ arrow-flight = { features = [ axum = { features = ["macros"], optional = true, version = "0.7.9" } clap = { features = ["derive"], version = "4.5.27" } color-eyre = "0.6.3" -crossterm = { features = ["event-stream"], optional = true, version = "0.28.1" } +crossterm = { features = ["event-stream"], optional = true, version = "0.29" } datafusion = { version = "51" } datafusion-app = { path = "crates/datafusion-app", version = "0.1.0" } directories = "5.0.1" @@ -39,9 +39,10 @@ object_store = "0.12" parquet = "57" pin-project-lite = { version = "0.2.14" } prost = "0.14" -ratatui = { optional = true, version = "0.29.0" } +ratatui = { optional = true, version = "0.30" } +ratatui-textarea = { features = ["search"], optional = true, version = "0.8" } serde = { features = ["derive"], version = "1.0.197" } -strum = "0.26.2" +strum = { features = ["derive"], version = "0.26.2" } tokio = { features = [ "macros", "process", @@ -64,8 +65,7 @@ tracing = { features = ["log"], version = "0.1.41" } tracing-subscriber = { features = ["env-filter"], version = "0.3.19" } tui-logger = { features = [ "tracing-support", -], optional = true, version = "0.17" } -tui-textarea = { features = ["search"], optional = true, version = "0.7.0" } +], optional = true, version = "0.18" } url = { features = ["serde"], version = "2.5.2" } uuid = { optional = true, version = "1.10.0" } vortex = { optional = true, version = "0.58" } @@ -115,7 +115,7 @@ http = [ ] huggingface = ["datafusion-app/huggingface"] s3 = ["datafusion-app/s3"] -tui = ["dep:crossterm", "dep:ratatui", "dep:tui-logger", "dep:tui-textarea"] +tui = ["dep:crossterm", "dep:ratatui", "dep:ratatui-textarea", "dep:tui-logger"] udfs-wasm = ["datafusion-app/udfs-wasm"] vortex = [ "datafusion-app/vortex", diff --git a/crates/datafusion-app/src/extensions/deltalake.rs b/crates/datafusion-app/src/extensions/deltalake.rs index 562448a..294306f 100644 --- a/crates/datafusion-app/src/extensions/deltalake.rs +++ b/crates/datafusion-app/src/extensions/deltalake.rs @@ -19,7 +19,11 @@ use crate::config::ExecutionConfig; use crate::extensions::{DftSessionStateBuilder, Extension}; -use deltalake::delta_datafusion::DeltaTableFactory; +use datafusion::catalog::{Session, TableProviderFactory}; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::logical_plan::CreateExternalTable; +use deltalake::table::builder::ensure_table_uri; +use std::collections::HashMap; use std::sync::Arc; #[derive(Debug, Default)] @@ -31,6 +35,59 @@ impl DeltaLakeExtension { } } +/// A wrapper around delta-rs's table factory that normalizes option keys passed +/// through DataFusion's `CREATE EXTERNAL TABLE ... OPTIONS (...)` clause. +/// +/// DataFusion's `parse_options_map` prepends `"format."` to any option key that +/// does not contain a dot (e.g. `'aws_endpoint' '...'` becomes `"format.aws_endpoint"` +/// in `cmd.options`). delta-rs's `S3ObjectStoreFactory` parses these keys via +/// `AmazonS3ConfigKey::from_str`, which does not recognise the `"format."` prefix, +/// so credentials and endpoint settings are silently ignored and the AWS SDK +/// credential chain is invoked instead. +/// +/// This factory strips the leading `"format."` prefix before forwarding the options +/// to `open_table_with_storage_options`, allowing callers to supply S3 credentials +/// directly in the DDL without needing ambient environment variables. +#[derive(Debug, Default)] +struct DeltaTableFactory {} + +#[async_trait::async_trait] +impl TableProviderFactory for DeltaTableFactory { + async fn create( + &self, + _ctx: &dyn Session, + cmd: &CreateExternalTable, + ) -> datafusion::error::Result> { + let table_url = + ensure_table_uri(&cmd.location).map_err(|e| DataFusionError::External(Box::new(e)))?; + + let provider = if cmd.options.is_empty() { + deltalake::open_table(table_url) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))? + } else { + // DataFusion prepends "format." to option keys that don't contain a dot. + // Strip that prefix so delta-rs can recognise the keys. + let options: HashMap = cmd + .options + .iter() + .map(|(k, v)| { + let key = k + .strip_prefix("format.") + .map(str::to_string) + .unwrap_or_else(|| k.clone()); + (key, v.clone()) + }) + .collect(); + deltalake::open_table_with_storage_options(table_url, options) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))? + }; + + Ok(Arc::new(provider)) + } +} + #[async_trait::async_trait] impl Extension for DeltaLakeExtension { async fn register( @@ -38,13 +95,6 @@ impl Extension for DeltaLakeExtension { _config: ExecutionConfig, builder: &mut DftSessionStateBuilder, ) -> datafusion::common::Result<()> { - // Register S3 handlers if s3 feature is enabled - // This is required for Delta Lake to recognize s3:// URLs - #[cfg(feature = "s3")] - { - deltalake::aws::register_handlers(None); - } - builder.add_table_factory("DELTATABLE", Arc::new(DeltaTableFactory {})); Ok(()) } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index d72668b..d6d3381 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.91.0" +channel = "1.91.1" diff --git a/src/tui/state/tabs/flightsql.rs b/src/tui/state/tabs/flightsql.rs index 9f3e695..0f5cfe7 100644 --- a/src/tui/state/tabs/flightsql.rs +++ b/src/tui/state/tabs/flightsql.rs @@ -25,8 +25,8 @@ use ratatui::crossterm::event::KeyEvent; use ratatui::style::palette::tailwind; use ratatui::style::Style; use ratatui::widgets::TableState; +use ratatui_textarea::TextArea; use tokio::task::JoinHandle; -use tui_textarea::TextArea; use crate::config::AppConfig; use crate::tui::pagination::{extract_page, has_sufficient_rows, PAGE_SIZE}; @@ -117,7 +117,7 @@ impl FlightSQLTabState<'_> { let content = lines.join(""); if content == default { self.editor - .move_cursor(tui_textarea::CursorMove::Jump(0, 0)); + .move_cursor(ratatui_textarea::CursorMove::Jump(0, 0)); self.editor.delete_str(default.len()); } } @@ -151,12 +151,13 @@ impl FlightSQLTabState<'_> { // TODO: Create Editor struct and move this there pub fn next_word(&mut self) { self.editor - .move_cursor(tui_textarea::CursorMove::WordForward) + .move_cursor(ratatui_textarea::CursorMove::WordForward) } // TODO: Create Editor struct and move this there pub fn previous_word(&mut self) { - self.editor.move_cursor(tui_textarea::CursorMove::WordBack) + self.editor + .move_cursor(ratatui_textarea::CursorMove::WordBack) } pub fn delete_word(&mut self) { diff --git a/src/tui/state/tabs/sql.rs b/src/tui/state/tabs/sql.rs index 39fb048..5c12bc6 100644 --- a/src/tui/state/tabs/sql.rs +++ b/src/tui/state/tabs/sql.rs @@ -25,8 +25,8 @@ use ratatui::crossterm::event::KeyEvent; use ratatui::style::palette::tailwind; use ratatui::style::{Modifier, Style}; use ratatui::widgets::TableState; +use ratatui_textarea::TextArea; use tokio::task::JoinHandle; -use tui_textarea::TextArea; use crate::config::AppConfig; use crate::tui::pagination::{extract_page, has_sufficient_rows, PAGE_SIZE}; @@ -151,7 +151,7 @@ impl SQLTabState<'_> { let content = lines.join(""); if content == default { self.editor - .move_cursor(tui_textarea::CursorMove::Jump(0, 0)); + .move_cursor(ratatui_textarea::CursorMove::Jump(0, 0)); self.editor.delete_str(default.len()); } } @@ -210,20 +210,22 @@ impl SQLTabState<'_> { match self.mode { SQLTabMode::Normal => self .editor - .move_cursor(tui_textarea::CursorMove::WordForward), + .move_cursor(ratatui_textarea::CursorMove::WordForward), SQLTabMode::DDL => self .ddl_editor - .move_cursor(tui_textarea::CursorMove::WordForward), + .move_cursor(ratatui_textarea::CursorMove::WordForward), } } // TODO: Create Editor struct and move this there pub fn previous_word(&mut self) { match self.mode { - SQLTabMode::Normal => self.editor.move_cursor(tui_textarea::CursorMove::WordBack), + SQLTabMode::Normal => self + .editor + .move_cursor(ratatui_textarea::CursorMove::WordBack), SQLTabMode::DDL => self .ddl_editor - .move_cursor(tui_textarea::CursorMove::WordBack), + .move_cursor(ratatui_textarea::CursorMove::WordBack), } } diff --git a/tests/cli_cases/tpch.rs b/tests/cli_cases/tpch.rs index a4c10f2..9b8124b 100644 --- a/tests/cli_cases/tpch.rs +++ b/tests/cli_cases/tpch.rs @@ -57,7 +57,7 @@ async fn test_custom_config_with_s3() { let mut config_builder = TestConfigBuilder::default(); config_builder.with_db_path("s3://tpch-db/db/"); let bucket = "tpch-db"; - let endpoint = "http://localhost:4566"; + let endpoint = "http://localhost:9000"; let access_key = "LSIAQAAAAAAVNCBMPNSG"; let secret = "5555555555555555555555555555555555555555"; let allow_http = true; diff --git a/tests/extension_cases/deltalake.rs b/tests/extension_cases/deltalake.rs index 6335b24..640e143 100644 --- a/tests/extension_cases/deltalake.rs +++ b/tests/extension_cases/deltalake.rs @@ -54,7 +54,12 @@ async fn test_deltalake_s3() { let tempdir = tempfile::tempdir().unwrap(); let ddl_path = tempdir.path().join("my_ddl.sql"); let mut file = std::fs::File::create(ddl_path.clone()).unwrap(); - let ddl = "CREATE EXTERNAL TABLE delta_tbl STORED AS DELTATABLE LOCATION 's3://test/deltalake/simple_table';"; + let ddl = "CREATE EXTERNAL TABLE delta_tbl STORED AS DELTATABLE LOCATION 's3://test/deltalake/simple_table' \ + OPTIONS ('aws_endpoint' 'http://localhost:9000', \ + 'aws_access_key_id' 'LSIAQAAAAAAVNCBMPNSG', \ + 'aws_secret_access_key' '5555555555555555555555555555555555555555', \ + 'aws_allow_http' 'true', \ + 'aws_region' 'us-east-1');"; file.write_all(ddl.as_bytes()).unwrap(); file.flush().unwrap(); @@ -70,7 +75,7 @@ async fn test_deltalake_s3() { "s3", "test", "s3://test", - "http://localhost:4566", + "http://localhost:9000", "LSIAQAAAAAAVNCBMPNSG", "5555555555555555555555555555555555555555", true, diff --git a/tests/extension_cases/s3.rs b/tests/extension_cases/s3.rs index d69e1e9..f98a72e 100644 --- a/tests/extension_cases/s3.rs +++ b/tests/extension_cases/s3.rs @@ -36,7 +36,7 @@ async fn test_s3_basic() { "s3", "test", "s3://test", - "http://localhost:4566", + "http://localhost:9000", "LSIAQAAAAAAVNCBMPNSG", "5555555555555555555555555555555555555555", true, @@ -72,7 +72,7 @@ async fn test_s3_credential_chain_env_vars() { "AWS_SECRET_ACCESS_KEY", "5555555555555555555555555555555555555555", ); - std::env::set_var("AWS_ENDPOINT_URL_S3", "http://localhost:4566"); + std::env::set_var("AWS_ENDPOINT_URL_S3", "http://localhost:9000"); let tempdir = tempfile::tempdir().unwrap(); let ddl_path = tempdir.path().join("my_ddl.sql"); @@ -88,7 +88,7 @@ async fn test_s3_credential_chain_env_vars() { "s3", "test", "s3://test", - Some("http://localhost:4566"), + Some("http://localhost:9000"), true, ); let config = config_builder.build("my_config.toml"); @@ -124,7 +124,7 @@ async fn test_s3_credential_chain_with_static_override() { // Set wrong credentials in environment std::env::set_var("AWS_ACCESS_KEY_ID", "WRONG_KEY"); std::env::set_var("AWS_SECRET_ACCESS_KEY", "WRONG_SECRET"); - std::env::set_var("AWS_ENDPOINT_URL_S3", "http://localhost:4566"); + std::env::set_var("AWS_ENDPOINT_URL_S3", "http://localhost:9000"); let tempdir = tempfile::tempdir().unwrap(); let ddl_path = tempdir.path().join("my_ddl.sql"); @@ -143,7 +143,7 @@ async fn test_s3_credential_chain_with_static_override() { "s3", "test", "s3://test", - "http://localhost:4566", + "http://localhost:9000", "LSIAQAAAAAAVNCBMPNSG", "5555555555555555555555555555555555555555", true,