Skip to content

Commit 19f19db

Browse files
rtylerrtyler
committed
chore: upgrade the aws dependencies in deltalake-aws
With the recent MSRV change for the newest datafusion release, finally time to upgrade some of these. In the course of doing this, I decided to get rid of the unnecessary `maplit` crate dependency here. Making HashMaps isn't that hard 😄 Signed-off-by: R. Tyler Croy <rtyler@brokenco.de>
1 parent 5b9b642 commit 19f19db

6 files changed

Lines changed: 292 additions & 144 deletions

File tree

‎crates/aws/Cargo.toml‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,34 +29,33 @@ uuid = { workspace = true, features = ["serde", "v4"] }
2929
url = { workspace = true }
3030

3131
# crates.io dependencies
32-
aws-smithy-runtime-api = { version = "1.7" }
33-
aws-smithy-runtime = { version = "1.7", optional = true }
32+
aws-smithy-runtime-api = { version = "1.8" }
33+
aws-smithy-runtime = { version = "1.8", optional = true }
3434
aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
35-
aws-config = { version = "1.5", default-features = false, features = [
35+
aws-config = { version = "1.8", default-features = false, features = [
3636
"behavior-version-latest",
3737
"rt-tokio",
3838
"credentials-process",
3939
] }
40-
aws-sdk-dynamodb = { version = "=1.79.0", default-features = false, features = [
40+
aws-sdk-dynamodb = { version = "1.93.0", default-features = false, features = [
4141
"behavior-version-latest",
4242
"rt-tokio",
4343
] }
44-
aws-sdk-sts = { version = "=1.73.0", default-features = false, features = [
44+
aws-sdk-sts = { version = "1.86.0", default-features = false, features = [
4545
"behavior-version-latest",
4646
"rt-tokio",
4747
] }
4848
backon = { version = "1", default-features = false, features = ["tokio-sleep"] }
4949
hyper-tls = { version = "0.5", optional = true }
50-
maplit = "1"
5150

5251
[dev-dependencies]
53-
deltalake-core = { path = "../core" }
5452
chrono = { workspace = true }
55-
serial_test = "3"
53+
deltalake-core = { path = "../core" }
5654
deltalake-test = { path = "../test" }
5755
pretty_env_logger = "0.5.0"
5856
rand = "0.8"
5957
serde_json = { workspace = true }
58+
serial_test = "3"
6059

6160
[features]
6261
default = ["rustls"]

‎crates/aws/src/credentials.rs‎

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ impl ProvideCredentials for OptionsCredentialsProvider {
166166
#[cfg(test)]
167167
mod options_tests {
168168
use super::*;
169-
use maplit::hashmap;
170169

171170
#[test]
172171
fn test_empty_options_error() {
@@ -181,10 +180,10 @@ mod options_tests {
181180

182181
#[test]
183182
fn test_uppercase_options_resolve() {
184-
let options = hashmap! {
185-
"AWS_ACCESS_KEY_ID".into() => "key".into(),
186-
"AWS_SECRET_ACCESS_KEY".into() => "secret".into(),
187-
};
183+
let options = HashMap::from([
184+
("AWS_ACCESS_KEY_ID".into(), "key".into()),
185+
("AWS_SECRET_ACCESS_KEY".into(), "secret".into()),
186+
]);
188187
let provider = OptionsCredentialsProvider { options };
189188
let result = provider.credentials();
190189
assert!(result.is_ok(), "StorageOptions with at least AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should resolve");
@@ -195,10 +194,10 @@ mod options_tests {
195194

196195
#[test]
197196
fn test_lowercase_options_resolve() {
198-
let options = hashmap! {
199-
"aws_access_key_id".into() => "key".into(),
200-
"aws_secret_access_key".into() => "secret".into(),
201-
};
197+
let options = HashMap::from([
198+
("AWS_ACCESS_KEY_ID".into(), "key".into()),
199+
("AWS_SECRET_ACCESS_KEY".into(), "secret".into()),
200+
]);
202201
let provider = OptionsCredentialsProvider { options };
203202
let result = provider.credentials();
204203
assert!(result.is_ok(), "StorageOptions with at least AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should resolve");
@@ -295,17 +294,21 @@ pub async fn resolve_credentials(options: &HashMap<String, String>) -> DeltaResu
295294
#[cfg(test)]
296295
mod tests {
297296
use super::*;
298-
use crate::constants;
299-
use maplit::hashmap;
300297
use serial_test::serial;
301298

302299
#[tokio::test]
303300
#[serial]
304301
async fn test_options_credentials_provider() {
305-
let options = hashmap! {
306-
constants::AWS_ACCESS_KEY_ID.to_string() => "test_id".to_string(),
307-
constants::AWS_SECRET_ACCESS_KEY.to_string() => "test_secret".to_string(),
308-
};
302+
let options = HashMap::from([
303+
(
304+
constants::AWS_ACCESS_KEY_ID.to_string(),
305+
"test_id".to_string(),
306+
),
307+
(
308+
constants::AWS_SECRET_ACCESS_KEY.to_string(),
309+
"test_secret".to_string(),
310+
),
311+
]);
309312

310313
let config = resolve_credentials(&options).await;
311314
assert!(config.is_ok(), "{config:?}");
@@ -334,11 +337,20 @@ mod tests {
334337
#[tokio::test]
335338
#[serial]
336339
async fn test_options_credentials_provider_session_token() {
337-
let options = hashmap! {
338-
constants::AWS_ACCESS_KEY_ID.to_string() => "test_id".to_string(),
339-
constants::AWS_SECRET_ACCESS_KEY.to_string() => "test_secret".to_string(),
340-
constants::AWS_SESSION_TOKEN.to_string() => "test_token".to_string(),
341-
};
340+
let options = HashMap::from([
341+
(
342+
constants::AWS_ACCESS_KEY_ID.to_string(),
343+
"test_id".to_string(),
344+
),
345+
(
346+
constants::AWS_SECRET_ACCESS_KEY.to_string(),
347+
"test_secret".to_string(),
348+
),
349+
(
350+
constants::AWS_SESSION_TOKEN.to_string(),
351+
"test_token".to_string(),
352+
),
353+
]);
342354

343355
let config = resolve_credentials(&options)
344356
.await
@@ -362,10 +374,16 @@ mod tests {
362374
#[tokio::test]
363375
#[serial]
364376
async fn test_object_store_credential_provider() -> DeltaResult<()> {
365-
let options = hashmap! {
366-
constants::AWS_ACCESS_KEY_ID.to_string() => "test_id".to_string(),
367-
constants::AWS_SECRET_ACCESS_KEY.to_string() => "test_secret".to_string(),
368-
};
377+
let options = HashMap::from([
378+
(
379+
constants::AWS_ACCESS_KEY_ID.to_string(),
380+
"test_id".to_string(),
381+
),
382+
(
383+
constants::AWS_SECRET_ACCESS_KEY.to_string(),
384+
"test_secret".to_string(),
385+
),
386+
]);
369387
let sdk_config = resolve_credentials(&options)
370388
.await
371389
.expect("Failed to resolve credentials for the test");
@@ -386,10 +404,16 @@ mod tests {
386404
#[tokio::test]
387405
#[serial]
388406
async fn test_object_store_credential_provider_consistency() -> DeltaResult<()> {
389-
let options = hashmap! {
390-
constants::AWS_ACCESS_KEY_ID.to_string() => "test_id".to_string(),
391-
constants::AWS_SECRET_ACCESS_KEY.to_string() => "test_secret".to_string(),
392-
};
407+
let options = HashMap::from([
408+
(
409+
constants::AWS_ACCESS_KEY_ID.to_string(),
410+
"test_id".to_string(),
411+
),
412+
(
413+
constants::AWS_SECRET_ACCESS_KEY.to_string(),
414+
"test_secret".to_string(),
415+
),
416+
]);
393417
let sdk_config = resolve_credentials(&options)
394418
.await
395419
.expect("Failed to resolve credentijals for the test");

‎crates/aws/src/lib.rs‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -324,10 +324,16 @@ impl DynamoDbLockClient {
324324
}
325325

326326
fn get_primary_key(&self, version: i64, table_path: &str) -> HashMap<String, AttributeValue> {
327-
maplit::hashmap! {
328-
constants::ATTR_TABLE_PATH.to_owned() => string_attr(table_path),
329-
constants::ATTR_FILE_NAME.to_owned() => string_attr(format!("{version:020}.json")),
330-
}
327+
HashMap::from([
328+
(
329+
constants::ATTR_TABLE_PATH.to_owned(),
330+
string_attr(table_path),
331+
),
332+
(
333+
constants::ATTR_FILE_NAME.to_owned(),
334+
string_attr(format!("{version:020}.json")),
335+
),
336+
])
331337
}
332338

333339
/// Read a log entry from DynamoDb.
@@ -434,9 +440,10 @@ impl DynamoDbLockClient {
434440
.limit(limit.try_into().unwrap_or(i32::MAX))
435441
.scan_index_forward(false)
436442
.key_condition_expression(format!("{} = :tn", constants::ATTR_TABLE_PATH))
437-
.set_expression_attribute_values(Some(
438-
maplit::hashmap!(":tn".into() => string_attr(table_path)),
439-
))
443+
.set_expression_attribute_values(Some(HashMap::from([(
444+
":tn".into(),
445+
string_attr(table_path),
446+
)])))
440447
.send()
441448
.await
442449
},
@@ -483,11 +490,11 @@ impl DynamoDbLockClient {
483490
.table_name(self.get_lock_table_name())
484491
.set_key(Some(self.get_primary_key(version, table_path)))
485492
.update_expression("SET complete = :c, expireTime = :e".to_owned())
486-
.set_expression_attribute_values(Some(maplit::hashmap! {
487-
":c".to_owned() => string_attr("true"),
488-
":e".to_owned() => num_attr(seconds_since_epoch),
489-
":f".into() => string_attr("false"),
490-
}))
493+
.set_expression_attribute_values(Some(HashMap::from([
494+
(":c".to_owned(), string_attr("true")),
495+
(":e".to_owned(), num_attr(seconds_since_epoch)),
496+
(":f".into(), string_attr("false")),
497+
])))
491498
.condition_expression(constants::CONDITION_UPDATE_INCOMPLETE)
492499
.send()
493500
.await?;
@@ -529,9 +536,10 @@ impl DynamoDbLockClient {
529536
.delete_item()
530537
.table_name(self.get_lock_table_name())
531538
.set_key(Some(self.get_primary_key(version, table_path)))
532-
.set_expression_attribute_values(Some(maplit::hashmap! {
533-
":f".into() => string_attr("false"),
534-
}))
539+
.set_expression_attribute_values(Some(HashMap::from([(
540+
":f".into(),
541+
string_attr("false"),
542+
)])))
535543
.condition_expression(constants::CONDITION_DELETE_INCOMPLETE.as_str())
536544
.send()
537545
.await?;
@@ -627,12 +635,25 @@ fn create_value_map(
627635
) -> HashMap<String, AttributeValue> {
628636
// cut off `_delta_log` part: temp_path in DynamoDb is relative to `_delta_log` not table root.
629637
let temp_path = Path::from_iter(commit_entry.temp_path.parts().skip(1));
630-
let mut value_map = maplit::hashmap! {
631-
constants::ATTR_TABLE_PATH.to_owned() => string_attr(table_path),
632-
constants::ATTR_FILE_NAME.to_owned() => string_attr(format!("{:020}.json", commit_entry.version)),
633-
constants::ATTR_TEMP_PATH.to_owned() => string_attr(temp_path),
634-
constants::ATTR_COMPLETE.to_owned() => string_attr(if commit_entry.complete { "true" } else { "false" }),
635-
};
638+
let mut value_map = HashMap::from([
639+
(
640+
constants::ATTR_TABLE_PATH.to_owned(),
641+
string_attr(table_path),
642+
),
643+
(
644+
constants::ATTR_FILE_NAME.to_owned(),
645+
string_attr(format!("{:020}.json", commit_entry.version)),
646+
),
647+
(constants::ATTR_TEMP_PATH.to_owned(), string_attr(temp_path)),
648+
(
649+
constants::ATTR_COMPLETE.to_owned(),
650+
string_attr(if commit_entry.complete {
651+
"true"
652+
} else {
653+
"false"
654+
}),
655+
),
656+
]);
636657
commit_entry.expire_time.as_ref().map(|t| {
637658
value_map.insert(
638659
constants::ATTR_EXPIRE_TIME.to_owned(),

0 commit comments

Comments
 (0)
⚡