[Multiple Datasource] Support Amazon OpenSearch Serverless (#3957)

* [Multiple Datasource]Support Amazon OpenSearch Serverless in SigV4
* remove experimental text in yml
* Refactor create data source form for authentication

Signed-off-by: Su <szhongna@amazon.com>

Author: zhongnansu

CHANGELOG.md

@@ -85,7 +85,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add satisfaction survey link to help menu ([#3676] (https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3676))
 - [Vis Builder] Add persistence to visualizations inner state ([#3751](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3751))
 - [Table Visualization] Move format table, consolidate types and add unit tests ([#3397](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3397))
-
+- [Multiple Datasource] Support Amazon OpenSearch Serverless ([#3957](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3957))
 ### 🐛 Bug Fixes
 
 - [Vis Builder] Fixes auto bounds for timeseries bar chart visualization ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))

config/opensearch_dashboards.yml

@@ -229,8 +229,7 @@
 # functionality in Visualization.
 # vis_builder.enabled: false
 
-# Set the value of this setting to true to enable the experimental multiple data source
-# support feature. Use with caution.
+# Set the value of this setting to true to enable multiple data source feature.
 #data_source.enabled: false
 # Set the value of these settings to customize crypto materials to encryption saved credentials
 # in data sources.

package.json

@@ -136,7 +136,7 @@
     "@hapi/podium": "^4.1.3",
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
-    "@opensearch-project/opensearch": "^2.1.0",
+    "@opensearch-project/opensearch": "^2.2.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",
@@ -169,7 +169,7 @@
     "dns-sync": "^0.2.1",
     "elastic-apm-node": "^3.7.0",
     "elasticsearch": "^16.7.0",
-    "http-aws-es": "6.0.0",
+    "http-aws-es": "npm:@zhongnansu/http-aws-es@6.0.1",
     "execa": "^4.0.2",
     "expiry-js": "0.1.7",
     "fast-deep-equal": "^3.1.1",

packages/osd-opensearch/package.json

@@ -12,7 +12,7 @@
     "osd:watch": "../../scripts/use_node scripts/build --watch"
   },
   "dependencies": {
-    "@opensearch-project/opensearch": "^2.1.0",
+    "@opensearch-project/opensearch": "^2.2.0",
     "@osd/dev-utils": "1.0.0",
     "abort-controller": "^3.0.0",
     "chalk": "^4.1.0",

src/plugins/data_source/common/data_sources/types.ts

@@ -25,6 +25,7 @@ export interface SigV4Content extends SavedObjectAttributes {
   accessKey: string;
   secretKey: string;
   region: string;
+  service?: SigV4ServiceName;
 }
 
 export interface UsernamePasswordTypedContent extends SavedObjectAttributes {
@@ -37,3 +38,8 @@ export enum AuthType {
   UsernamePasswordType = 'username_password',
   SigV4 = 'sigv4',
 }
+
+export enum SigV4ServiceName {
+  OpenSearch = 'es',
+  OpenSearchServerless = 'aoss',
+}

src/plugins/data_source/opensearch_dashboards.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": [],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["common/data_sources"]
 }

src/plugins/data_source/server/client/configure_client.test.ts

@@ -167,6 +167,30 @@ describe('configureClient', () => {
     expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(2);
   });
 
+  test('configure client with auth.type == sigv4, service == aoss, should successfully call new Client()', async () => {
+    savedObjectsMock.get.mockReset().mockResolvedValueOnce({
+      id: DATA_SOURCE_ID,
+      type: DATA_SOURCE_SAVED_OBJECT_TYPE,
+      attributes: {
+        ...dataSourceAttr,
+        auth: {
+          type: AuthType.SigV4,
+          credentials: { ...sigV4AuthContent, service: 'aoss' },
+        },
+      },
+      references: [],
+    });
+
+    jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
+      decryptedText: 'accessKey',
+      encryptionContext: { endpoint: 'http://localhost' },
+    });
+
+    await configureClient(dataSourceClientParams, clientPoolSetup, config, logger);
+
+    expect(ClientMock).toHaveBeenCalledTimes(1);
+  });
+
   test('configure test client for non-exist datasource should not call saved object api, nor decode any credential', async () => {
     const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
       decryptedText: 'password',

src/plugins/data_source/server/client/configure_client.ts

@@ -160,7 +160,7 @@ const getBasicAuthClient = (
 };
 
 const getAWSClient = (credential: SigV4Content, clientOptions: ClientOptions): Client => {
-  const { accessKey, secretKey, region } = credential;
+  const { accessKey, secretKey, region, service } = credential;
 
   const credentialProvider = (): Promise<Credentials> => {
     return new Promise((resolve) => {
@@ -172,6 +172,7 @@ const getAWSClient = (credential: SigV4Content, clientOptions: ClientOptions): C
     ...AwsSigv4Signer({
       region,
       getCredentials: credentialProvider,
+      service,
     }),
     ...clientOptions,
   });

src/plugins/data_source/server/client/configure_client_utils.ts

@@ -91,7 +91,7 @@ export const getAWSCredential = async (
   cryptography: CryptographyServiceSetup
 ): Promise<SigV4Content> => {
   const { endpoint } = dataSource;
-  const { accessKey, secretKey, region } = dataSource.auth.credentials! as SigV4Content;
+  const { accessKey, secretKey, region, service } = dataSource.auth.credentials! as SigV4Content;
 
   const {
     decryptedText: accessKeyText,
@@ -122,6 +122,7 @@ export const getAWSCredential = async (
     region,
     accessKey: accessKeyText,
     secretKey: secretKeyText,
+    service,
   };
 
   return credential;

src/plugins/data_source/server/legacy/configure_legacy_client.test.ts

@@ -6,7 +6,7 @@
 import { SavedObjectsClientContract } from '../../../../core/server';
 import { loggingSystemMock, savedObjectsClientMock } from '../../../../core/server/mocks';
 import { DATA_SOURCE_SAVED_OBJECT_TYPE } from '../../common';
-import { AuthType, DataSourceAttributes } from '../../common/data_sources';
+import { AuthType, DataSourceAttributes, SigV4Content } from '../../common/data_sources';
 import { DataSourcePluginConfigType } from '../../config';
 import { cryptographyServiceSetupMock } from '../cryptography_service.mocks';
 import { CryptographyServiceSetup } from '../cryptography_service';
@@ -27,6 +27,7 @@ describe('configureLegacyClient', () => {
   let clientPoolSetup: OpenSearchClientPoolSetup;
   let configOptions: ConfigOptions;
   let dataSourceAttr: DataSourceAttributes;
+  let sigV4AuthContent: SigV4Content;
 
   let mockOpenSearchClientInstance: {
     close: jest.Mock;
@@ -71,6 +72,12 @@ describe('configureLegacyClient', () => {
       },
     } as DataSourceAttributes;
 
+    sigV4AuthContent = {
+      region: 'us-east-1',
+      accessKey: 'accessKey',
+      secretKey: 'secretKey',
+    };
+
     clientPoolSetup = {
       getClientFromPool: jest.fn(),
       addClientToPool: jest.fn(),
@@ -157,6 +164,42 @@ describe('configureLegacyClient', () => {
     expect(mockResult).toBeDefined();
   });
 
+  test('configure client with auth.type == sigv4 and service param, should call new Client() with service param', async () => {
+    savedObjectsMock.get.mockReset().mockResolvedValueOnce({
+      id: DATA_SOURCE_ID,
+      type: DATA_SOURCE_SAVED_OBJECT_TYPE,
+      attributes: {
+        ...dataSourceAttr,
+        auth: {
+          type: AuthType.SigV4,
+          credentials: { ...sigV4AuthContent, service: 'aoss' },
+        },
+      },
+      references: [],
+    });
+
+    parseClientOptionsMock.mockReturnValue(configOptions);
+
+    jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({
+      decryptedText: 'accessKey',
+      encryptionContext: { endpoint: 'http://localhost' },
+    });
+
+    await configureLegacyClient(
+      dataSourceClientParams,
+      callApiParams,
+      clientPoolSetup,
+      config,
+      logger
+    );
+
+    expect(parseClientOptionsMock).toHaveBeenCalled();
+    expect(ClientMock).toHaveBeenCalledTimes(1);
+    expect(ClientMock).toHaveBeenCalledWith(expect.objectContaining({ service: 'aoss' }));
+
+    expect(savedObjectsMock.get).toHaveBeenCalledTimes(1);
+  });
+
   test('configure client with auth.type == username_password and password contaminated', async () => {
     const decodeAndDecryptSpy = jest
       .spyOn(cryptographyMock, 'decodeAndDecrypt')