fix: handle trailing slash in URLs.

Author: peterhuene

src/backend/azure.rs

@@ -618,7 +618,9 @@ impl StorageBackend for AzureBlobStorageBackend {
         }
 
         // The prefix should end with `/` to signify a directory.
-        prefix.push('/');
+        if !prefix.ends_with('/') {
+            prefix.push('/');
+        }
 
         {
             // Append the operation and block id to the URL
@@ -675,10 +677,8 @@ impl StorageBackend for AzureBlobStorageBackend {
             }
 
             paths.extend(results.blobs.items.into_iter().map(|b| {
-                b.name
-                    .strip_prefix(&prefix)
-                    .map(Into::into)
-                    .unwrap_or(b.name)
+                let name = b.name.strip_prefix(&prefix).unwrap_or(&b.name);
+                name.strip_prefix('/').unwrap_or(name).into()
             }));
 
             next = results.next.unwrap_or_default();

src/backend/google.rs

@@ -677,9 +677,11 @@ impl StorageBackend for GoogleStorageBackend {
 
         let (bucket, path) = url.bucket_and_path();
 
-        // The prefix should end with `/` to signify a directory.
         let mut prefix = path.strip_prefix('/').unwrap_or(path).to_string();
-        prefix.push('/');
+        // The prefix should end with `/` to signify a directory.
+        if !prefix.ends_with('/') {
+            prefix.push('/');
+        }
 
         // Format the request to always use the virtual-host style URL
         url.set_host(Some(&format!("{bucket}.{GOOGLE_ROOT_DOMAIN}")))
@@ -745,12 +747,10 @@ impl StorageBackend for GoogleStorageBackend {
                 return Ok(paths);
             }
 
-            paths.extend(
-                results
-                    .contents
-                    .into_iter()
-                    .map(|c| c.key.strip_prefix(&prefix).map(Into::into).unwrap_or(c.key)),
-            );
+            paths.extend(results.contents.into_iter().map(|c| {
+                let key = c.key.strip_prefix(&prefix).unwrap_or(&c.key);
+                key.strip_prefix('/').unwrap_or(key).into()
+            }));
 
             token = results.token.unwrap_or_default();
             if token.is_empty() {

src/backend/s3.rs

@@ -759,9 +759,11 @@ impl StorageBackend for S3StorageBackend {
 
         let (bucket, path) = url.bucket_and_path();
 
-        // The prefix should end with `/` to signify a directory.
         let mut prefix = path.strip_prefix('/').unwrap_or(path).to_string();
-        prefix.push('/');
+        // The prefix should end with `/` to signify a directory.
+        if !prefix.ends_with('/') {
+            prefix.push('/');
+        }
 
         // Format the request to always use the virtual-host style URL
         let domain = url.domain().expect("URL should have domain");
@@ -829,12 +831,10 @@ impl StorageBackend for S3StorageBackend {
                 return Ok(paths);
             }
 
-            paths.extend(
-                results
-                    .contents
-                    .into_iter()
-                    .map(|c| c.key.strip_prefix(&prefix).map(Into::into).unwrap_or(c.key)),
-            );
+            paths.extend(results.contents.into_iter().map(|c| {
+                let key = c.key.strip_prefix(&prefix).unwrap_or(&c.key);
+                key.strip_prefix('/').unwrap_or(key).into()
+            }));
 
             token = results.token.unwrap_or_default();
             if token.is_empty() {

tests/copy.rs

@@ -749,10 +749,14 @@ async fn walk() -> Result<()> {
             .context("failed to upload file")?;
         }
 
-        let files = cloud_copy::walk(config.clone(), client.clone(), url)
+        let files = cloud_copy::walk(config.clone(), client.clone(), url.clone())
             .await
             .expect("should walk");
-        assert_eq!(files, &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]);
+        assert_eq!(
+            files,
+            &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
+            "unexpected walk output for URL `{url}`"
+        );
     }
 
     Ok(())