Page index udf

Author: matthewmturner

crates/datafusion-app/src/local.rs

@@ -115,6 +115,10 @@ impl ExecutionContext {
             "parquet_metadata",
             Arc::new(datafusion_functions_parquet::ParquetMetadataFunc {}),
         );
+        session_ctx.register_udtf(
+            "parquet_page_index",
+            Arc::new(datafusion_functions_parquet::ParquetPageIndexFunc {}),
+        );
 
         let catalog = create_app_catalog(config, app_name, app_version)?;
         session_ctx.register_catalog(&config.catalog.name, catalog);

crates/datafusion-functions-parquet/Cargo.toml

@@ -15,3 +15,8 @@ arrow = { version = "57" }
 async-trait = "0.1.41"
 datafusion = { version = "51" }
 parquet = { default-features = false, version = "57" }
+
+[dev-dependencies]
+parquet = { version = "57", features = ["arrow"] }
+tempfile = "3"
+tokio = { version = "1", features = ["macros", "rt"] }

crates/datafusion-functions-parquet/src/lib.rs

@@ -37,6 +37,9 @@ use parquet::file::statistics::Statistics;
 use std::fs::File;
 use std::sync::Arc;
 
+mod page_index;
+pub use page_index::ParquetPageIndexFunc;
+
 // Copied from https://github.com/apache/datafusion/blob/main/datafusion-cli/src/functions.rs
 /// PARQUET_META table function
 #[derive(Debug)]
@@ -219,7 +222,7 @@ impl TableFunctionImpl for ParquetMetadataFunc {
                 column_id_arr.push(col_idx as i64);
                 file_offset_arr.push(column.file_offset());
                 num_values_arr.push(column.num_values());
-                path_in_schema_arr.push(column.column_path().to_string());
+                path_in_schema_arr.push(column.column_path().string());
                 type_arr.push(column.column_type().to_string());
                 logical_type_arr.push(
                     column