docs(samples): sample for upload directory (googleapis#770)

Author: AVaksman

handwritten/storage/samples/files.js

@@ -155,6 +155,84 @@ async function uploadFileWithKmsKey(bucketName, filename, kmsKeyName) {
   // [END storage_upload_with_kms_key]
 }
 
+// sample-metadata:
+//   title: Upload a diretory to a bucket.
+//   description: Uploads full hierarchy of a local directory to a bucket.
+//   usage: node files.js upload-directory <bucketName> <directoryPath>
+
+async function uploadDirectory(bucketName, directoryPath) {
+  // [START upload_directory]
+  // Imports the Google Cloud client library
+  const {Storage} = require('@google-cloud/storage');
+  const fs = require('fs');
+  const path = require('path');
+
+  // Creates a client
+  const storage = new Storage();
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // const bucketName = 'Name of a bucket, e.g. my-bucket';
+  // const directoryPath = 'Local directory to upload, e.g. ./local/path/to/direcotry';
+
+  // get the list of files from the specified directory
+  const fileList = [];
+  let dirCtr = 1;
+  let itemCtr = 0;
+  const pathDirName = path.dirname(directoryPath);
+
+  getFiles(directoryPath);
+
+  function getFiles(directory) {
+    fs.readdir(directory, (err, items) => {
+      dirCtr--;
+      itemCtr += items.length;
+      items.forEach(item => {
+        const fullPath = path.join(directory, item);
+        fs.stat(fullPath, (err, stat) => {
+          itemCtr--;
+          if (stat.isFile()) {
+            fileList.push(fullPath);
+          } else if (stat.isDirectory()) {
+            dirCtr++;
+            getFiles(fullPath);
+          }
+          if (dirCtr === 0 && itemCtr === 0) {
+            onComplete();
+          }
+        });
+      });
+    });
+  }
+
+  async function onComplete() {
+    const resp = await Promise.all(
+      fileList.map(filePath => {
+        let destination = path.relative(pathDirName, filePath);
+        // If running on Windows
+        if (process.platform === 'win32') {
+          destination = destination.replace(/\\/g, '/');
+        }
+        return storage
+          .bucket(bucketName)
+          .upload(filePath, {destination})
+          .then(
+            uploadResp => ({fileName: destination, status: uploadResp[0]}),
+            err => ({fileName: destination, response: err})
+          );
+      })
+    );
+
+    const successfulUploads =
+      fileList.length - resp.filter(r => r.status instanceof Error).length;
+    console.log(
+      `${successfulUploads} files uploaded to ${bucketName} successfully.`
+    );
+  }
+  // [END upload_directory]
+}
+
 async function downloadFile(bucketName, srcFilename, destFilename) {
   // [START storage_download_file]
   // Imports the Google Cloud client library
@@ -474,6 +552,12 @@ require(`yargs`)
     opts =>
       uploadFileWithKmsKey(opts.bucketName, opts.srcFileName, opts.kmsKeyName)
   )
+  .command(
+    `upload-directory <bucketName> <srcDirectoryPath>`,
+    `Uploads full hierarchy of a local directory to a bucket.`,
+    {},
+    opts => uploadDirectory(opts.bucketName, opts.srcDirectoryPath)
+  )
   .command(
     `download <bucketName> <srcFileName> <destFileName>`,
     `Downloads a file from a bucket.`,
@@ -547,6 +631,10 @@ require(`yargs`)
     `node $0 upload-with-kms-key my-bucket ./file.txt my-key`,
     `Uploads "./file.txt" to "my-bucket" using "my-key".`
   )
+  .example(
+    `node $0 upload-directory my-bucket ./my-folder`,
+    `Uploads full hierarchy of "./my-folder directory to "my-bucket.`
+  )
   .example(
     `node $0 download my-bucket file.txt ./file.txt`,
     `Downloads "gs://my-bucket/file.txt" to "./file.txt".`

handwritten/storage/samples/resources/resourcesSub1/testSub1.txt

@@ -0,0 +1,2 @@
+Sub1
+Hello World!

handwritten/storage/samples/system-test/files.test.js

@@ -37,6 +37,7 @@ const copiedFileName = 'test3.txt';
 const signedFileName = 'signed-upload.txt';
 const kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US;
 const filePath = path.join(cwd, 'resources', fileName);
+const folderPath = path.join(cwd, 'resources');
 const downloadFilePath = path.join(cwd, 'downloaded.txt');
 const cmd = `node files.js`;
 
@@ -73,6 +74,49 @@ it('should upload a file with a kms key', async () => {
   assert.strictEqual(exists, true);
 });
 
+it('should upload a local directory', done => {
+  const output = execSync(
+    `${cmd} upload-directory ${bucketName} ${folderPath}`
+  );
+
+  const fileList = [];
+  getFileList(folderPath);
+
+  function getFileList(directory) {
+    const items = fs.readdirSync(directory);
+    items.forEach(item => {
+      const fullPath = path.join(directory, item);
+      const stat = fs.lstatSync(fullPath);
+      if (stat.isFile()) {
+        fileList.push(fullPath);
+      } else {
+        getFileList(fullPath);
+      }
+    });
+  }
+
+  assert.match(
+    output,
+    new RegExp(
+      `${fileList.length} files uploaded to ${bucketName} successfully.`
+    )
+  );
+
+  Promise.all(
+    fileList.map(file =>
+      bucket
+        .file(path.relative(path.dirname(folderPath), file).replace(/\\/g, '/'))
+        .exists()
+    )
+  ).then(resps => {
+    const ctr = resps.reduce((acc, cur) => {
+      return acc + cur[0];
+    }, 0);
+    assert.strictEqual(ctr, fileList.length);
+    done();
+  }, assert.ifError);
+});
+
 it('should download a file', () => {
   const output = execSync(
     `${cmd} download ${bucketName} ${fileName} ${downloadFilePath}`