test: added conformance testing scenario 7 (#2035)

* test: add conformance scenario 7

* handle a 200 response code when checking server offset

* add 201 status code

* fixed bucketUploadResumable test

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* deleted commented lines

Co-authored-by: Denis DelGrosso <ddelgrosso@google.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: 3 people

conformance-test/conformanceCommon.ts

@@ -28,6 +28,7 @@ interface RetryCase {
 interface Method {
   name: String;
   resources: String[];
+  group?: String;
 }
 
 export interface RetryTestCase {
@@ -68,7 +69,10 @@ export function executeScenario(testCase: RetryTestCase) {
   ) {
     const instructionSet: RetryCase = testCase.cases[instructionNumber];
     testCase.methods.forEach(async jsonMethod => {
-      const functionList = methodMap.get(jsonMethod?.name);
+      const functionList =
+        jsonMethod?.group !== undefined
+          ? methodMap.get(jsonMethod?.group)
+          : methodMap.get(jsonMethod?.name);
       functionList?.forEach(storageMethodString => {
         const storageMethodObject =
           libraryMethods[storageMethodString as keyof LibraryMethodsModuleType];

conformance-test/libraryMethods.ts

@@ -15,6 +15,15 @@
 import {Bucket, File, Notification, Storage, HmacKey, Policy} from '../src';
 import * as path from 'path';
 import {ApiError} from '../src/nodejs-common';
+import {
+  createTestBuffer,
+  createTestFileFromBuffer,
+  deleteTestFile,
+} from './testBenchUtil';
+import * as uuid from 'uuid';
+
+const FILE_SIZE_BYTES = 9 * 1024 * 1024;
+const CHUNK_SIZE_BYTES = 2 * 1024 * 1024;
 
 export interface ConformanceTestOptions {
   bucket?: Bucket;
@@ -375,35 +384,38 @@ export async function bucketSetStorageClass(options: ConformanceTestOptions) {
 export async function bucketUploadResumableInstancePrecondition(
   options: ConformanceTestOptions
 ) {
+  const filePath = path.join(__dirname, `test-data/tmp-${uuid.v4()}.txt`);
+  createTestFileFromBuffer(FILE_SIZE_BYTES, filePath);
   if (options.bucket!.instancePreconditionOpts) {
     options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0;
+    delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch;
   }
-  await options.bucket!.upload(
-    path.join(
-      __dirname,
-      '../../conformance-test/test-data/retryStrategyTestData.json'
-    ),
-    {resumable: true}
-  );
+  await options.bucket!.upload(filePath, {
+    resumable: true,
+    chunkSize: CHUNK_SIZE_BYTES,
+    metadata: {contentLength: FILE_SIZE_BYTES},
+  });
+  deleteTestFile(filePath);
 }
 
 export async function bucketUploadResumable(options: ConformanceTestOptions) {
+  const filePath = path.join(__dirname, `test-data/tmp-${uuid.v4()}.txt`);
+  createTestFileFromBuffer(FILE_SIZE_BYTES, filePath);
   if (options.preconditionRequired) {
-    await options.bucket!.upload(
-      path.join(
-        __dirname,
-        '../../conformance-test/test-data/retryStrategyTestData.json'
-      ),
-      {preconditionOpts: {ifMetagenerationMatch: 2, ifGenerationMatch: 0}}
-    );
+    await options.bucket!.upload(filePath, {
+      resumable: true,
+      chunkSize: CHUNK_SIZE_BYTES,
+      metadata: {contentLength: FILE_SIZE_BYTES},
+      preconditionOpts: {ifGenerationMatch: 0},
+    });
   } else {
-    await options.bucket!.upload(
-      path.join(
-        __dirname,
-        '../../conformance-test/test-data/retryStrategyTestData.json'
-      )
-    );
+    await options.bucket!.upload(filePath, {
+      resumable: true,
+      chunkSize: CHUNK_SIZE_BYTES,
+      metadata: {contentLength: FILE_SIZE_BYTES},
+    });
   }
+  deleteTestFile(filePath);
 }
 
 export async function bucketUploadMultipartInstancePrecondition(
@@ -590,21 +602,31 @@ export async function rotateEncryptionKey(options: ConformanceTestOptions) {
 export async function saveResumableInstancePrecondition(
   options: ConformanceTestOptions
 ) {
-  await options.file!.save('testdata', {resumable: true});
+  const buf = createTestBuffer(FILE_SIZE_BYTES);
+  await options.file!.save(buf, {
+    resumable: true,
+    chunkSize: CHUNK_SIZE_BYTES,
+    metadata: {contentLength: FILE_SIZE_BYTES},
+  });
 }
 
 export async function saveResumable(options: ConformanceTestOptions) {
+  const buf = createTestBuffer(FILE_SIZE_BYTES);
   if (options.preconditionRequired) {
-    await options.file!.save('testdata', {
+    await options.file!.save(buf, {
       resumable: true,
+      chunkSize: CHUNK_SIZE_BYTES,
+      metadata: {contentLength: FILE_SIZE_BYTES},
       preconditionOpts: {
         ifGenerationMatch: options.file!.metadata.generation,
         ifMetagenerationMatch: options.file!.metadata.metageneration,
       },
     });
   } else {
-    await options.file!.save('testdata', {
+    await options.file!.save(buf, {
       resumable: true,
+      chunkSize: CHUNK_SIZE_BYTES,
+      metadata: {contentLength: FILE_SIZE_BYTES},
     });
   }
 }

conformance-test/scenarios/scenarioSeven.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright 2022 Google LLC. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as testFile from '../test-data/retryStrategyTestData.json';
+import {executeScenario, RetryTestCase} from '../conformanceCommon';
+import * as assert from 'assert';
+
+const SCENARIO_NUMBER_TO_TEST = 7;
+const retryTestCase: RetryTestCase | undefined =
+  testFile.retryStrategyTests.find(test => test.id === SCENARIO_NUMBER_TO_TEST);
+
+describe(`Scenario ${SCENARIO_NUMBER_TO_TEST}`, () => {
+  assert(retryTestCase);
+  executeScenario(retryTestCase);
+});

conformance-test/test-data/retryInvocationMap.json

@@ -78,6 +78,12 @@
         "saveMultipart",
         "createResumableUpload"
     ],
+    "storage.resumable.upload": [
+        "bucketUploadResumableInstancePrecondition",
+        "saveResumableInstancePrecondition",
+        "bucketUploadResumable",
+        "saveResumable"
+    ],
     "storage.objects.delete": [
         "fileDeleteInstancePrecondition",
         "fileDelete"

conformance-test/test-data/retryStrategyTestData.json

@@ -402,6 +402,26 @@
       ],
       "preconditionProvided": true,
       "expectSuccess": false
+    },
+    {
+      "id": 7,
+      "description": "resumable_uploads_handle_complex_retries",
+      "cases": [
+        {
+          "instructions": ["return-reset-connection", "return-503"]
+        },
+        {
+          "instructions": ["return-503-after-256K"]
+        },
+        {
+          "instructions": ["return-503-after-8192K"]
+        }
+      ],
+      "methods": [
+        {"name": "storage.objects.insert", "group": "storage.resumable.upload", "resources": ["BUCKET"]}
+      ],
+      "preconditionProvided": true,
+      "expectSuccess": true
     }
   ]
 }

conformance-test/testBenchUtil.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import {execSync} from 'child_process';
+import {unlinkSync, writeFileSync} from 'fs';
 import {URL} from 'url';
 
 const HOST = process.env.STORAGE_EMULATOR_HOST || 'http://localhost:9000';
@@ -38,3 +39,16 @@ export async function runTestBenchDockerImage(): Promise<Buffer> {
 export async function stopTestBenchDockerImage(): Promise<Buffer> {
   return execSync(STOP_CMD);
 }
+
+export function createTestBuffer(sizeInBytes: number): Buffer {
+  return Buffer.alloc(sizeInBytes, 'testdata');
+}
+
+export function createTestFileFromBuffer(sizeInMb: number, path: string): void {
+  const buf = createTestBuffer(sizeInMb);
+  writeFileSync(path, buf);
+}
+
+export function deleteTestFile(path: string): void {
+  unlinkSync(path);
+}

src/resumable-upload.ts

@@ -524,7 +524,20 @@ export class Upload extends Writable {
   }
 
   protected async createURIAsync(): Promise<string> {
-    const metadata = this.metadata;
+    const metadata = {...this.metadata};
+    const headers: gaxios.Headers = {};
+
+    // Delete content length and content type from metadata if they exist.
+    // These are headers and should not be sent as part of the metadata.
+    if (metadata.contentLength) {
+      headers['X-Upload-Content-Length'] = metadata.contentLength.toString();
+      delete metadata.contentLength;
+    }
+
+    if (metadata.contentType) {
+      headers!['X-Upload-Content-Type'] = metadata.contentType;
+      delete metadata.contentType;
+    }
 
     // Check if headers already exist before creating new ones
     const reqOpts: GaxiosOptions = {
@@ -540,6 +553,7 @@ export class Upload extends Writable {
       data: metadata,
       headers: {
         'x-goog-api-client': `gl-node/${process.versions.node} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.uri}`,
+        ...headers,
       },
     };
 

test/resumable-upload.ts

@@ -800,7 +800,10 @@ describe('resumable-upload', () => {
           ifGenerationMatch: GENERATION,
           ifMetagenerationNotMatch: PARAMS.ifMetagenerationNotMatch,
         });
-        assert.strictEqual(reqOpts.data, up.metadata);
+        const metadataNoHeaders = {...up.metadata};
+        delete metadataNoHeaders.contentLength;
+        delete metadataNoHeaders.contentType;
+        assert.deepStrictEqual(reqOpts.data, metadataNoHeaders);
         done();
         return {headers: {location: '/foo'}};
       };