feat(middleware-flexible-checksums): allow custom checksum algorithm implementations (#7746)

Author: kuhe

.prettierignore

@@ -6,5 +6,6 @@ CHANGELOG.md
 clients/**/src
 clients/**/README.md
 private/**
+!private/aws-client-api-test/src/**/*
 packages/nested-clients/src/submodules/*/endpoint/ruleset.ts
 **/*.java

packages-internal/middleware-flexible-checksums/src/configuration.ts

@@ -1,4 +1,4 @@
-import {
+import type {
   BodyLengthCalculator,
   ChecksumConstructor,
   Encoder,
@@ -9,7 +9,8 @@ import {
   StreamHasher,
 } from "@smithy/types";
 
-import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
+import type { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
+import type { FlexibleChecksumsInputConfig } from "./resolveFlexibleChecksumsConfig";
 
 /**
  * @internal
@@ -74,4 +75,6 @@ export interface PreviouslyResolved {
    * Minimum bytes from a stream to buffer into a chunk before passing to chunked encoding.
    */
   requestStreamBufferSize: number;
+
+  checksumAlgorithms?: FlexibleChecksumsInputConfig["checksumAlgorithms"];
 }

packages-internal/middleware-flexible-checksums/src/getChecksumAlgorithmForRequest.spec.ts

@@ -55,7 +55,7 @@ describe(getChecksumAlgorithmForRequest.name, () => {
     });
   });
 
-  it("throws error if input[requestAlgorithmMember] if not supported by client", () => {
+  it("does not throw an error if an unknown algo is selected", () => {
     const unsupportedAlgo = "unsupportedAlgo";
     const mockInput = { [mockRequestAlgorithmMember]: unsupportedAlgo };
     const mockOptions = {
@@ -65,10 +65,8 @@ describe(getChecksumAlgorithmForRequest.name, () => {
     };
     expect(() => {
       getChecksumAlgorithmForRequest(mockInput, mockOptions);
-    }).toThrowError(
-      `The checksum algorithm "${unsupportedAlgo}" is not supported by the client.` +
-        ` Select one of ${CLIENT_SUPPORTED_ALGORITHMS}.`
-    );
+    }).not.toThrowError();
+    // throwing an error is deferred to the selectChecksumAlgorithmFunction step.
   });
 
   describe("returns input[requestAlgorithmMember] if supported by client", () => {

packages-internal/middleware-flexible-checksums/src/getChecksumAlgorithmForRequest.ts

@@ -1,5 +1,5 @@
-import { ChecksumAlgorithm, DEFAULT_CHECKSUM_ALGORITHM, RequestChecksumCalculation } from "./constants";
-import { CLIENT_SUPPORTED_ALGORITHMS } from "./types";
+import type { ChecksumAlgorithm } from "./constants";
+import { DEFAULT_CHECKSUM_ALGORITHM, RequestChecksumCalculation } from "./constants";
 
 export interface GetChecksumAlgorithmForRequestOptions {
   /**
@@ -26,7 +26,7 @@ export interface GetChecksumAlgorithmForRequestOptions {
 export const getChecksumAlgorithmForRequest = (
   input: any,
   { requestChecksumRequired, requestAlgorithmMember, requestChecksumCalculation }: GetChecksumAlgorithmForRequestOptions
-): ChecksumAlgorithm | undefined => {
+): ChecksumAlgorithm | string | undefined => {
   // The Operation input member that is used to configure request checksum behavior is not set.
   if (!requestAlgorithmMember) {
     // Select an algorithm only if request checksum calculation is supported
@@ -41,13 +41,6 @@ export const getChecksumAlgorithmForRequest = (
   }
 
   const checksumAlgorithm = input[requestAlgorithmMember];
-  // Validate that at least one algorithm from customer preference is supported by the SDK.
-  if (!CLIENT_SUPPORTED_ALGORITHMS.includes(checksumAlgorithm)) {
-    throw new Error(
-      `The checksum algorithm "${checksumAlgorithm}" is not supported by the client.` +
-        ` Select one of ${CLIENT_SUPPORTED_ALGORITHMS}.`
-    );
-  }
 
-  return checksumAlgorithm as ChecksumAlgorithm;
+  return checksumAlgorithm as ChecksumAlgorithm | string;
 };

packages-internal/middleware-flexible-checksums/src/getChecksumLocationName.ts

@@ -3,5 +3,5 @@ import { ChecksumAlgorithm } from "./constants";
 /**
  * Returns location (header/trailer) name to use to populate checksum in.
  */
-export const getChecksumLocationName = (algorithm: ChecksumAlgorithm): string =>
+export const getChecksumLocationName = (algorithm: ChecksumAlgorithm | string): string =>
   algorithm === ChecksumAlgorithm.MD5 ? "content-md5" : `x-amz-checksum-${algorithm.toLowerCase()}`;

packages-internal/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.spec.ts

@@ -31,6 +31,7 @@ describe(resolveFlexibleChecksumsConfig.name, () => {
       requestChecksumCalculation: DEFAULT_REQUEST_CHECKSUM_CALCULATION,
       responseChecksumValidation: DEFAULT_RESPONSE_CHECKSUM_VALIDATION,
       requestStreamBufferSize: 0,
+      checksumAlgorithms: {},
     });
     expect(normalizeProvider).toHaveBeenCalledTimes(2);
   });

packages-internal/middleware-flexible-checksums/src/resolveFlexibleChecksumsConfig.ts

@@ -1,12 +1,8 @@
-import { Provider } from "@smithy/types";
+import type { ChecksumConstructor, Provider } from "@smithy/types";
 import { normalizeProvider } from "@smithy/util-middleware";
 
-import {
-  DEFAULT_REQUEST_CHECKSUM_CALCULATION,
-  DEFAULT_RESPONSE_CHECKSUM_VALIDATION,
-  RequestChecksumCalculation,
-  ResponseChecksumValidation,
-} from "./constants";
+import type { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
+import { DEFAULT_REQUEST_CHECKSUM_CALCULATION, DEFAULT_RESPONSE_CHECKSUM_VALIDATION } from "./constants";
 
 /**
  * @public
@@ -38,6 +34,20 @@ export interface FlexibleChecksumsInputConfig {
    * of 8kb or greater.
    */
   requestStreamBufferSize?: number | false;
+
+  /**
+   * Optional implementations of checksum algorithms adhering to the
+   * Checksum interface.
+   */
+  checksumAlgorithms?: {
+    CRC32?: ChecksumConstructor;
+    CRC32C?: ChecksumConstructor;
+    CRC64NVME?: ChecksumConstructor;
+    SHA1?: ChecksumConstructor;
+    SHA256?: ChecksumConstructor;
+  } & {
+    [algorithmId: string]: ChecksumConstructor;
+  };
 }
 
 /**
@@ -47,6 +57,7 @@ export interface FlexibleChecksumsResolvedConfig {
   requestChecksumCalculation: Provider<RequestChecksumCalculation>;
   responseChecksumValidation: Provider<ResponseChecksumValidation>;
   requestStreamBufferSize: number;
+  checksumAlgorithms?: FlexibleChecksumsInputConfig["checksumAlgorithms"];
 }
 
 /**
@@ -60,5 +71,6 @@ export const resolveFlexibleChecksumsConfig = <T>(
     requestChecksumCalculation: normalizeProvider(requestChecksumCalculation ?? DEFAULT_REQUEST_CHECKSUM_CALCULATION),
     responseChecksumValidation: normalizeProvider(responseChecksumValidation ?? DEFAULT_RESPONSE_CHECKSUM_VALIDATION),
     requestStreamBufferSize: Number(requestStreamBufferSize ?? 0),
+    checksumAlgorithms: input.checksumAlgorithms ?? {},
   });
 };

packages-internal/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts

@@ -11,6 +11,9 @@ describe(selectChecksumAlgorithmFunction.name, () => {
     md5: vi.fn(),
     sha1: vi.fn(),
     sha256: vi.fn(),
+    checksumAlgorithms: {
+      novel: class {},
+    },
   };
 
   it.each([
@@ -24,7 +27,11 @@ describe(selectChecksumAlgorithmFunction.name, () => {
   });
 
   it("throws an error for unsupported checksum algorithm", () => {
-    expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED" as any, mockConfig as any)).toThrow();
+    expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED", mockConfig as any)).toThrow();
+  });
+
+  it("doesn't throw an error if the unknown algo's implementation was provided", () => {
+    expect(() => selectChecksumAlgorithmFunction("novel", mockConfig as any)).not.toThrow();
   });
 
   it("returns Crc64Nvme if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => {

packages-internal/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts

@@ -1,35 +1,44 @@
 import { AwsCrc32c } from "@aws-crypto/crc32c";
 import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
-import { ChecksumConstructor, HashConstructor } from "@smithy/types";
+import type { ChecksumConstructor, HashConstructor } from "@smithy/types";
 
-import { PreviouslyResolved } from "./configuration";
+import type { PreviouslyResolved } from "./configuration";
 import { ChecksumAlgorithm } from "./constants";
 import { getCrc32ChecksumAlgorithmFunction } from "./getCrc32ChecksumAlgorithmFunction";
+import { CLIENT_SUPPORTED_ALGORITHMS } from "./types";
 
 /**
  * Returns the function that will compute the checksum for the given {@link ChecksumAlgorithm}.
  */
 export const selectChecksumAlgorithmFunction = (
-  checksumAlgorithm: ChecksumAlgorithm,
+  checksumAlgorithm: ChecksumAlgorithm | string,
   config: PreviouslyResolved
 ): ChecksumConstructor | HashConstructor => {
+  const { checksumAlgorithms = {} } = config;
   switch (checksumAlgorithm) {
     case ChecksumAlgorithm.MD5:
-      return config.md5;
+      return checksumAlgorithms?.MD5 ?? config.md5;
     case ChecksumAlgorithm.CRC32:
-      return getCrc32ChecksumAlgorithmFunction();
+      return checksumAlgorithms?.CRC32 ?? getCrc32ChecksumAlgorithmFunction();
     case ChecksumAlgorithm.CRC32C:
-      return AwsCrc32c;
+      return checksumAlgorithms?.CRC32C ?? AwsCrc32c;
     case ChecksumAlgorithm.CRC64NVME:
       if (typeof crc64NvmeCrtContainer.CrtCrc64Nvme !== "function") {
-        return Crc64Nvme;
+        return checksumAlgorithms?.CRC64NVME ?? Crc64Nvme;
       }
-      return crc64NvmeCrtContainer.CrtCrc64Nvme;
+      return checksumAlgorithms?.CRC64NVME ?? crc64NvmeCrtContainer.CrtCrc64Nvme;
     case ChecksumAlgorithm.SHA1:
-      return config.sha1;
+      return checksumAlgorithms?.SHA1 ?? config.sha1;
     case ChecksumAlgorithm.SHA256:
-      return config.sha256;
+      return checksumAlgorithms?.SHA256 ?? config.sha256;
     default:
-      throw new Error(`Unsupported checksum algorithm: ${checksumAlgorithm}`);
+      if (checksumAlgorithms?.[checksumAlgorithm]) {
+        return checksumAlgorithms[checksumAlgorithm];
+      }
+      throw new Error(
+        `The checksum algorithm "${checksumAlgorithm}" is not supported by the client.` +
+          ` Select one of ${CLIENT_SUPPORTED_ALGORITHMS}, or provide an implementation to ` +
+          ` the client constructor checksums field.`
+      );
   }
 };

private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts

@@ -1,4 +1,5 @@
-import { S3Client, S3ClientConfigType } from "@aws-sdk/client-s3";
+import type { S3ClientConfigType } from "@aws-sdk/client-s3";
+import { S3Client } from "@aws-sdk/client-s3";
 import { AwsRestXmlProtocol } from "@aws-sdk/core/protocols";
 import { defaultProvider as credentialDefaultProvider, defaultProvider } from "@aws-sdk/credential-provider-node";
 import { NODE_USE_ARN_REGION_CONFIG_OPTIONS } from "@aws-sdk/middleware-bucket-endpoint";
@@ -22,7 +23,7 @@ import { NODE_MAX_ATTEMPT_CONFIG_OPTIONS, NODE_RETRY_MODE_CONFIG_OPTIONS } from
 import { loadConfig as loadNodeConfig } from "@smithy/node-config-provider";
 import { NodeHttpHandler, streamCollector } from "@smithy/node-http-handler";
 import { loadConfigsForDefaultMode } from "@smithy/smithy-client";
-import { EndpointV2, HttpAuthSchemeProvider } from "@smithy/types";
+import type { EndpointV2, HttpAuthSchemeProvider } from "@smithy/types";
 import { parseUrl } from "@smithy/url-parser";
 import { fromBase64, toBase64 } from "@smithy/util-base64";
 import { calculateBodyLength } from "@smithy/util-body-length-node";
@@ -144,6 +145,29 @@ export const initializeWithMaximalConfiguration = () => {
     bucketEndpoint: false,
     requestChecksumCalculation: DEFAULT_REQUEST_CHECKSUM_CALCULATION,
     responseChecksumValidation: DEFAULT_RESPONSE_CHECKSUM_VALIDATION,
+    checksumAlgorithms: {
+      SHA256: class {
+        update() {}
+        reset() {}
+        async digest() {
+          return new Uint8Array();
+        }
+      },
+      CRC64NVME: class {
+        update() {}
+        reset() {}
+        async digest() {
+          return new Uint8Array();
+        }
+      },
+      unknown: class {
+        update() {}
+        reset() {}
+        async digest() {
+          return new Uint8Array();
+        }
+      },
+    },
     requestStreamBufferSize: 8 * 1024,
     expectContinueHeader: 8 * 1024 * 1024,
   };