Merge pull request #8691 from thibaultmg/delete-s3-directory

compact: delete directory markers in block.Delete

Author: GiedriusS

CHANGELOG.md

@@ -20,6 +20,8 @@ It is recommend to upgrade the storage components first (Receive, Store, etc.) a
 
 ### Added
 
+- [#8691](https://github.com/thanos-io/thanos/pull/8691): Compactor: remove the directory marker objects for some s3 compatible object stores
+
 ### Changed
 
 - [#8670](https://github.com/thanos-io/thanos/pull/8670): Receive: *breaking :warning:* removed `--shipper.ignore-unequal-block-size`. TSDB now delays compaction until blocks have been uploaded by the shipper, allowing compaction while uploading without risking data loss.

pkg/block/block.go

@@ -240,6 +240,19 @@ func Delete(ctx context.Context, logger log.Logger, bkt objstore.Bucket, id ulid
 		level.Debug(logger).Log("msg", "deleted file", "file", deletionMarkFile, "bucket", bkt.Name())
 	}
 
+	// Some object storages represent directories as explicit empty objects.
+	// We try to delete the directory marker objects themselves after all their contents are removed.
+	directoryMarkerPaths := []string{
+		path.Join(id.String(), ChunksDirname) + objstore.DirDelim,
+		id.String() + objstore.DirDelim,
+	}
+
+	for _, p := range directoryMarkerPaths {
+		if err := bkt.Delete(ctx, p); err != nil && !bkt.IsObjNotFoundErr(err) {
+			level.Debug(logger).Log("msg", "failed to delete directory marker object", "dir", p, "err", err)
+		}
+	}
+
 	return nil
 }
 

pkg/block/block_test.go

@@ -257,6 +257,22 @@ func TestDelete(t *testing.T) {
 		testutil.Ok(t, Delete(ctx, log.NewNopLogger(), bkt, b2))
 		testutil.Equals(t, 0, len(bkt.Objects()))
 	}
+	{
+		b3 := ulid.MustNew(3, nil)
+
+		// Upload explicit directory markers.
+		dirName := b3.String() + objstore.DirDelim
+		testutil.Ok(t, bkt.Upload(ctx, dirName, bytes.NewReader([]byte{})))
+
+		chunksDirName := path.Join(b3.String(), ChunksDirname) + objstore.DirDelim
+		testutil.Ok(t, bkt.Upload(ctx, chunksDirName, bytes.NewReader([]byte{})))
+
+		testutil.Equals(t, 2, len(bkt.Objects()))
+
+		// Check if delete also deletes the directory markers.
+		testutil.Ok(t, Delete(ctx, log.NewNopLogger(), bkt, b3))
+		testutil.Equals(t, 0, len(bkt.Objects()))
+	}
 }
 
 func TestMarkForDeletion(t *testing.T) {