Always quote CSVs coming out of lambda-to-csv

Digging deeper into load failures I discovered that while
arrow-csv and other Rust CSV parsers can handle some characters
properly, some RDMS and C-based CSV parsers (older) cannot, without
quoting.

This relies on a backport of apache/datafusion#20813 to the 52.4.0
release in order to support this. It also requires arrow-csv 57.2.0 or
later since that release is the first that `QuoteStyle` exists.

Author: rtyler

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 resolver = "3"
 
 [workspace.package]
-version = "1.8.7"
+version = "1.8.8"
 edition = "2024"
 keywords = ["deltalake", "parquet", "lambda", "delta", "sqs"]
 homepage = "https://github.com/buoyant-data/oxbow"
@@ -22,6 +22,7 @@ aws_lambda_events = { version = "0.15", default-features = false, features = ["s
 # The datafusion feature is required to support invariants which may be in error, but is required as of currently released 0.18.2
 deltalake = { version = "0.31.1", features = ["s3", "json", "datafusion"] }
 #deltalake = { git = "https://github.com/ethan-tyler/delta-rs", branch = "fix/meta-only-count-stats", features = ["s3", "json", "datafusion"]}
+#deltalake = { git = "https://github.com/ethan-tyler/delta-rs", branch = "fix/meta-only-count-stats", features = ["s3", "json", "datafusion"]}
 #deltalake = { path = "../../delta-io/delta-rs/crates/deltalake", features = ["s3", "json", "datafusion"]}
 futures = { version = "0.3" }
 tokio = { version = "=1", features = ["macros"] }
@@ -32,6 +33,14 @@ tracing = { version = "0.1", features = ["log"] }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] }
 url = { version = "2.3", features = ["serde"] }
 
+[patch.crates-io]
+datafusion = { git = "https://github.com/buoyant-data/datafusion", branch = "buoyant/release-52.4.1" }
+datafusion-datasource = { git = "https://github.com/buoyant-data/datafusion", branch = "buoyant/release-52.4.1" }
+datafusion-physical-expr-adapter = { git = "https://github.com/buoyant-data/datafusion", branch = "buoyant/release-52.4.1" }
+datafusion-ffi = { git = "https://github.com/buoyant-data/datafusion", branch = "buoyant/release-52.4.1" }
+datafusion-proto = { git = "https://github.com/buoyant-data/datafusion", branch = "buoyant/release-52.4.1" }
+
+
 [profile.release]
 panic = "abort"
 lto = true

lambdas/cdf-to-csv/src/main.rs

@@ -3,6 +3,7 @@
 //! produce CSV files to ingest into Aurora for Change Data Feeds
 
 use aws_lambda_events::event::sqs::SqsEvent;
+use deltalake::datafusion::common::parsers::CsvQuoteStyle;
 use deltalake::datafusion::config::CsvOptions;
 use deltalake::datafusion::dataframe::DataFrameWriteOptions;
 use deltalake::datafusion::prelude::*;
@@ -215,6 +216,7 @@ fn escape_dataframe(input: DataFrame) -> DeltaResult<DataFrame> {
 fn csv_options() -> CsvOptions {
     CsvOptions {
         null_value: std::env::var("CSV_NULL_CHARACTER").ok(),
+        quote_style: CsvQuoteStyle::Always,
         timestamp_format: std::env::var("CSV_TIMESTAMP_FORMAT").ok(),
         timestamp_tz_format: std::env::var("CSV_TIMESTAMP_TZ_FORMAT").ok(),
         ..Default::default()
@@ -384,7 +386,6 @@ mod tests {
             )
             .await?;
         let written = df.clone();
-        let written_schema = written.schema();
         df = escape_dataframe(df)?;
 
         df.write_csv(

lambdas/cdf-to-csv/tests/data/row.json

@@ -1,3 +1,3 @@
-{"id": 209, "other": 149, "rating" : 1.00, "boo" : true, "r" : "TRUST ME BRO\the sources are \"my friend once...\" \"my friend's daughter...\"\nbook full of chicken! as a pilot, break dancing was simply awfull\n\nper quanto" }
+{"id": 209, "other": 149, "r" : "TRUST ME BRO\the sources are \"my friend once...\" \"my friend's daughter...\"\nbook full of chicken! as a pilot, break dancing was simply awfull\n\nper quanto :-\\", "rating" : 1.00, "boo" : true }
 {"id": 209, "other": 149, "rating" : 1.00, "boo" : false, "r" : "TRUST ME BRO\the sources are \"my friend once...\" \"my friend's daughter...\"\nbook full of chicken! as a pilot, break dancing was simply awfull\n\nper quanto" }
 {"id": 209, "other": 149, "rating" : 1.00, "boo" : null, "r" : "TRUST ME BRO\the sources are \"my friend once...\" \"my friend's daughter...\"\nbook full of chicken! as a pilot, break dancing was simply awfull\n\nper quanto" }