Merge pull request #698 from stephenplusplus/spp--storage-deleteFiles

storage: add deleteFiles()

Author: stephenplusplus

docs/site/components/docs/docs.html

@@ -83,7 +83,10 @@ <h4 ng-show="method.params">Parameters</h4>
         <tbody>
           <tr
             ng-repeat="param in method.params"
-            ng-class="{ 'param-optional': param.optional }">
+            ng-class="{
+              'param-optional': param.optional,
+              'param-nullable': param.nullable
+            }">
             <th scope="row" class="param">
               <span ng-if="param.subparam" class="param-parent">
                 <div>{{param.name.split('.').slice(0, -1).join('.')}}</div>

docs/site/components/docs/docs.js

@@ -110,9 +110,19 @@ angular
           'bucket',
           'file',
           'job',
-          'table'
+          'table',
+          'index',
+          'document',
+          'field',
+          'topic',
+          'subscription'
         ];
-        type = type.replace('=', '');
+
+        var arrayRegex = /Array\.<([^>]+)>/g;
+        type = type.replace('=', '')
+          .replace('?', '')
+          .replace(arrayRegex, '$1[]');
+
         if (CUSTOM_TYPES.indexOf(type.toLowerCase()) > -1) {
           if (types[index - 1]) {
             type = types[index - 1] + '/' + type;

docs/site/css/main.css

@@ -856,17 +856,27 @@ ul {
   padding-top: 0.7em;
 }
 
-.param-optional .param-types {
+.param-optional .param-types,
+.param-nullable .param-types {
   font-style: italic;
 }
 
-.param-optional .param-types:after {
-  content: " (optional)";
+.param-optional .param-types:after,
+.param-nullable .param-types:after {
+  display: block;
   color: #aaa;
   font-style: italic;
   font-size: 85%;
 }
 
+.param-optional .param-types:after {
+  content: " (optional)";
+}
+
+.param-nullable .param-types:after {
+  content: " (may be null)";
+}
+
 .method-heading {
     position: relative;
 }

lib/storage/bucket.js

@@ -310,6 +310,124 @@ Bucket.prototype.delete = function(callback) {
   this.makeReq_('DELETE', '', null, true, callback);
 };
 
+/**
+ * Iterate over the bucket's files, calling `file.delete()` on each.
+ *
+ * <strong>This is not an atomic request.</strong> A delete attempt will be made
+ * for each file individually. Any one can fail, in which case only a portion of
+ * the files you intended to be deleted would have.
+ *
+ * Operations are performed in parallel, up to 10 at once. The first error
+ * breaks the loop and will execute the provided callback with it. Specify
+ * `{ force: true }` to suppress the errors until all files have had a chance to
+ * be processed.
+ *
+ * The `query` object passed as the first argument will also be passed to
+ * {module:storage/bucket#getFiles}.
+ *
+ * @param {object=} query - Query object. See {module:storage/bucket#getFiles}
+ *     for all of the supported properties.
+ * @param {boolean} query.force - Supress errors until all files have been
+ *     processed.
+ * @param {function} callback - The callback function.
+ * @param {?error|?error[]} callback.err - An API error or array of errors from
+ *     files that were not able to be deleted.
+ *
+ * @example
+ * //-
+ * // Delete all of the files in the bucket.
+ * //-
+ * bucket.deleteFiles(function(err) {});
+ *
+ * //-
+ * // By default, if a file cannot be deleted, this method will stop deleting
+ * // files from your bucket. You can override this setting with `force: true`.
+ * //-
+ * bucket.deleteFiles({
+ *   force: true
+ * }, function(errors) {
+ *   // `errors`:
+ *   //    Array of errors if any occurred, otherwise null.
+ * });
+ *
+ * //-
+ * // The first argument to this method acts as a query to
+ * // {module:storage/bucket#getFiles}. As an example, you can delete files
+ * // which match a prefix.
+ * //-
+ * bucket.deleteFiles({
+ *   prefix: 'images/'
+ * }, function(err) {
+ *   if (!err) {
+ *     // All files in the `images` directory have been deleted.
+ *   }
+ * });
+ */
+Bucket.prototype.deleteFiles = function(query, callback) {
+  if (util.is(query, 'function')) {
+    callback = query;
+    query = {};
+  }
+
+  query = query || {};
+
+  var self = this;
+
+  var MAX_PARALLEL_LIMIT = 10;
+  var errors = [];
+
+  // Start deleting files, iteratively fetching more as necessary.
+  deleteFiles(query, function(err) {
+    if (err || errors.length > 0) {
+      callback(err || errors);
+      return;
+    }
+
+    callback(null);
+  });
+
+  function deleteFiles(query, callback) {
+    self.getFiles(query, function(err, files, nextQuery) {
+      if (err) {
+        callback(err);
+        return;
+      }
+
+      // Iterate through each file and attempt to delete it.
+      async.eachLimit(files, MAX_PARALLEL_LIMIT, deleteFile, function(err) {
+        if (err) {
+          callback(err);
+          return;
+        }
+
+        if (nextQuery) {
+          deleteFiles(nextQuery, callback);
+          return;
+        }
+
+        callback();
+      });
+    });
+  }
+
+  function deleteFile(file, callback) {
+    file.delete(function(err) {
+      if (err) {
+        if (query.force) {
+          errors.push(err);
+          callback();
+          return;
+        }
+
+        callback(err);
+        return;
+      }
+
+      callback();
+    });
+  }
+};
+
 /**
  * Create a File object. See {module:storage/file} to see how to handle
  * the different use cases you may have.
@@ -849,7 +967,7 @@ Bucket.prototype.makeAllFilesPublicPrivate_ = function(options, callback) {
   var updatedFiles = [];
 
   // Start processing files, iteratively fetching more as necessary.
-  processFiles({}, function (err) {
+  processFiles({}, function(err) {
     if (err || errors.length > 0) {
       callback(err || errors, updatedFiles);
       return;

system-test/storage.js

@@ -42,27 +42,6 @@ var files = {
   }
 };
 
-function deleteVersionedFiles(bucket, callback) {
-  bucket.getFiles({ versions: true }, function(err, files) {
-    if (err) {
-      callback(err);
-      return;
-    }
-
-    async.each(files, deleteFile, callback);
-  });
-}
-
-function deleteFiles(bucket, callback) {
-  bucket.getFiles(function(err, files) {
-    if (err) {
-      callback(err);
-      return;
-    }
-    async.map(files, deleteFile, callback);
-  });
-}
-
 function deleteFile(file, callback) {
   file.delete(callback);
 }
@@ -100,7 +79,7 @@ describe('storage', function() {
   });
 
   after(function(done) {
-    deleteFiles(bucket, function(err) {
+    bucket.deleteFiles(function(err) {
       assert.ifError(err);
       bucket.delete(done);
     });
@@ -221,7 +200,7 @@ describe('storage', function() {
                     bucket.acl.default.delete({ entity: 'allUsers' }, next);
                   },
                   function(next) {
-                    deleteFiles(bucket, next);
+                    bucket.deleteFiles(next);
                   }
                 ], done);
               });
@@ -278,7 +257,7 @@ describe('storage', function() {
               async.each(files, isFilePrivate, function(err) {
                 assert.ifError(err);
 
-                deleteFiles(bucket, done);
+                bucket.deleteFiles(done);
               });
             });
           });
@@ -686,7 +665,7 @@ describe('storage', function() {
     var filenames = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3'];
 
     before(function(done) {
-      deleteFiles(bucket, function(err) {
+      bucket.deleteFiles(function(err) {
         assert.ifError(err);
 
         var file = bucket.file(filenames[0]);
@@ -750,7 +729,7 @@ describe('storage', function() {
     });
 
     afterEach(function(done) {
-      deleteVersionedFiles(versionedBucket, done);
+      versionedBucket.deleteFiles({ versions: true }, done);
     });
 
     after(function(done) {

test/storage/bucket.js

@@ -340,6 +340,125 @@ describe('Bucket', function() {
     });
   });
 
+  describe('deleteFiles', function() {
+    it('should get files from the bucket', function(done) {
+      var query = { a: 'b', c: 'd' };
+
+      bucket.getFiles = function(query_) {
+        assert.deepEqual(query_, query);
+        done();
+      };
+
+      bucket.deleteFiles(query, assert.ifError);
+    });
+
+    it('should process 10 files at a time', function(done) {
+      eachLimit_Override = function(arr, limit) {
+        assert.equal(limit, 10);
+        done();
+      };
+
+      bucket.getFiles = function(query, callback) {
+        callback(null, []);
+      };
+
+      bucket.deleteFiles({}, assert.ifError);
+    });
+
+    it('should delete the files', function(done) {
+      var timesCalled = 0;
+
+      var files = [
+        bucket.file('1'),
+        bucket.file('2')
+      ].map(util.propAssign('delete', function(callback) {
+        timesCalled++;
+        callback();
+      }));
+
+      bucket.getFiles = function(query, callback) {
+        callback(null, files);
+      };
+
+      bucket.deleteFiles({}, function(err) {
+        assert.ifError(err);
+        assert.equal(timesCalled, files.length);
+        done();
+      });
+    });
+
+    it('should get more files if more exist', function(done) {
+      var fakeNextQuery = { a: 'b', c: 'd' };
+
+      bucket.getFiles = function(query, callback) {
+        if (Object.keys(query).length === 0) {
+          // First time through, return a `nextQuery` value.
+          callback(null, [], fakeNextQuery);
+        } else {
+          // Second time through.
+          assert.deepEqual(query, fakeNextQuery);
+          done();
+        }
+      };
+
+      bucket.deleteFiles({}, assert.ifError);
+    });
+
+    it('should execute callback with error from getting files', function(done) {
+      var error = new Error('Error.');
+
+      bucket.getFiles = function(query, callback) {
+        callback(error);
+      };
+
+      bucket.deleteFiles({}, function(err) {
+        assert.strictEqual(err, error);
+        done();
+      });
+    });
+
+    it('should execute callback with error from deleting file', function(done) {
+      var error = new Error('Error.');
+
+      var files = [
+        bucket.file('1'),
+        bucket.file('2')
+      ].map(util.propAssign('delete', function(callback) {
+        callback(error);
+      }));
+
+      bucket.getFiles = function(query, callback) {
+        callback(null, files);
+      };
+
+      bucket.deleteFiles({}, function(err) {
+        assert.strictEqual(err, error);
+        done();
+      });
+    });
+
+    it('should execute callback with queued errors', function(done) {
+      var error = new Error('Error.');
+
+      var files = [
+        bucket.file('1'),
+        bucket.file('2')
+      ].map(util.propAssign('delete', function(callback) {
+        callback(error);
+      }));
+
+      bucket.getFiles = function(query, callback) {
+        callback(null, files);
+      };
+
+      bucket.deleteFiles({ force: true }, function(errs) {
+        assert.strictEqual(errs[0], error);
+        assert.strictEqual(errs[1], error);
+        done();
+      });
+    });
+  });
+
   describe('file', function() {
     var FILE_NAME = 'remote-file-name.jpg';
     var file;