diff --git a/package-lock.json b/package-lock.json index 39f9d1048e..ed1c751e4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,8 @@ "dependencies": { "@aws-sdk/client-s3": "3.840.0", "@aws-sdk/client-sts": "3.840.0", + "@aws-sdk/credential-providers": "3.840.0", + "@aws-sdk/s3-request-presigner": "3.840.0", "@azure/identity": "4.10.1", "@azure/monitor-query": "1.3.2", "@azure/storage-blob": "12.27.0", @@ -300,6 +302,72 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-cognito-identity": { + "version": "3.840.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.840.0.tgz", + "integrity": "sha512-0sn/X63Xqqh5D1FYmdSHiS9SkDzTitoGO++/8IFik4xf/jpn4ZQkIoDPvpxFZcLvebMuUa6jAQs4ap4RusKGkg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.840.0", + "@aws-sdk/credential-provider-node": "3.840.0", + "@aws-sdk/middleware-host-header": "3.840.0", + "@aws-sdk/middleware-logger": "3.840.0", + "@aws-sdk/middleware-recursion-detection": "3.840.0", + "@aws-sdk/middleware-user-agent": "3.840.0", + "@aws-sdk/region-config-resolver": "3.840.0", + "@aws-sdk/types": "3.840.0", + "@aws-sdk/util-endpoints": "3.840.0", + "@aws-sdk/util-user-agent-browser": "3.840.0", + "@aws-sdk/util-user-agent-node": "3.840.0", + "@smithy/config-resolver": "^4.1.4", + "@smithy/core": "^3.6.0", + "@smithy/fetch-http-handler": "^5.0.4", + "@smithy/hash-node": "^4.0.4", + "@smithy/invalid-dependency": "^4.0.4", + "@smithy/middleware-content-length": "^4.0.4", + "@smithy/middleware-endpoint": "^4.1.13", + "@smithy/middleware-retry": "^4.1.14", + "@smithy/middleware-serde": "^4.0.8", + "@smithy/middleware-stack": "^4.0.4", + "@smithy/node-config-provider": "^4.1.3", + "@smithy/node-http-handler": "^4.0.6", + "@smithy/protocol-http": "^5.1.2", + "@smithy/smithy-client": "^4.4.5", + "@smithy/types": "^4.3.1", + "@smithy/url-parser": "^4.0.4", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.21", + "@smithy/util-defaults-mode-node": "^4.0.21", + "@smithy/util-endpoints": "^3.0.6", + "@smithy/util-middleware": "^4.0.4", + "@smithy/util-retry": "^4.0.6", + "@smithy/util-utf8": "^4.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-cognito-identity/node_modules/@smithy/node-http-handler": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.1.0.tgz", + "integrity": "sha512-vqfSiHz2v8b3TTTrdXi03vNz1KLYYS3bhHCDv36FYDqxT7jvTll1mMnCrkD+gOvgwybuunh/2VmvOMqwBegxEg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/abort-controller": "^4.0.4", + "@smithy/protocol-http": "^5.1.2", + "@smithy/querystring-builder": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-iam": { "version": "3.840.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-iam/-/client-iam-3.840.0.tgz", @@ -611,6 +679,22 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/credential-provider-cognito-identity": { + "version": "3.840.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.840.0.tgz", + "integrity": "sha512-p1RaMVd6+6ruYjKsWRCZT/jWhrYfDKbXY+/ScIYTvcaOOf9ArMtVnhFk3egewrC7kPXFGRYhg2GPmxRotNYMng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.840.0", + "@aws-sdk/types": "3.840.0", + "@smithy/property-provider": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/credential-provider-env": { "version": "3.840.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.840.0.tgz", @@ -764,6 +848,36 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/credential-providers": { + "version": "3.840.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.840.0.tgz", + "integrity": "sha512-+CxYdGd+uM4NZ9VUvFTU1c/H61qhDB4q362k8xKU+bz24g//LDQ5Mpwksv8OUD1en44v4fUwgZ4SthPZMs+eFQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.840.0", + "@aws-sdk/core": "3.840.0", + "@aws-sdk/credential-provider-cognito-identity": "3.840.0", + "@aws-sdk/credential-provider-env": "3.840.0", + "@aws-sdk/credential-provider-http": "3.840.0", + "@aws-sdk/credential-provider-ini": "3.840.0", + "@aws-sdk/credential-provider-node": "3.840.0", + "@aws-sdk/credential-provider-process": "3.840.0", + "@aws-sdk/credential-provider-sso": "3.840.0", + "@aws-sdk/credential-provider-web-identity": "3.840.0", + "@aws-sdk/nested-clients": "3.840.0", + "@aws-sdk/types": "3.840.0", + "@smithy/config-resolver": "^4.1.4", + "@smithy/core": "^3.6.0", + "@smithy/credential-provider-imds": "^4.0.6", + "@smithy/node-config-provider": "^4.1.3", + "@smithy/property-provider": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/lib-storage": { "version": "3.840.0", "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.840.0.tgz", @@ -1040,6 +1154,25 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/s3-request-presigner": { + "version": "3.840.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.840.0.tgz", + "integrity": "sha512-1jcrhVoSZjiAQJGNswI0RGR36/+OG6yTV42wQamHdNHk+/68dn9MGTUVr+58AEFOyEAPE/EvkiYRD6n5WkUjMg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/signature-v4-multi-region": "3.840.0", + "@aws-sdk/types": "3.840.0", + "@aws-sdk/util-format-url": "3.840.0", + "@smithy/middleware-endpoint": "^4.1.13", + "@smithy/protocol-http": "^5.1.2", + "@smithy/smithy-client": "^4.4.5", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/signature-v4-multi-region": { "version": "3.840.0", "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.840.0.tgz", @@ -1115,6 +1248,21 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/util-format-url": { + "version": "3.840.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.840.0.tgz", + "integrity": "sha512-VB1PWyI1TQPiPvg4w7tgUGGQER1xxXPNUqfh3baxUSFi1Oh8wHrDnFywkxLm3NMmgDmnLnSZ5Q326qAoyqKLSg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.840.0", + "@smithy/querystring-builder": "^4.0.4", + "@smithy/types": "^4.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/util-locate-window": { "version": "3.804.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.804.0.tgz", diff --git a/package.json b/package.json index 65e0059f89..0cf0ba59e4 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,8 @@ "dependencies": { "@aws-sdk/client-s3": "3.840.0", "@aws-sdk/client-sts": "3.840.0", + "@aws-sdk/credential-providers": "3.840.0", + "@aws-sdk/s3-request-presigner": "3.840.0", "@azure/identity": "4.10.1", "@azure/monitor-query": "1.3.2", "@azure/storage-blob": "12.27.0", diff --git a/src/agent/block_store_services/block_store_s3.js b/src/agent/block_store_services/block_store_s3.js index 3d60ad84cc..572a9b7ac6 100644 --- a/src/agent/block_store_services/block_store_s3.js +++ b/src/agent/block_store_services/block_store_s3.js @@ -2,7 +2,7 @@ 'use strict'; const _ = require('lodash'); -const AWS = require('aws-sdk'); +const { S3 } = require('@aws-sdk/client-s3'); const config = require('../../../config'); const P = require('../../util/promise'); @@ -12,6 +12,7 @@ const cloud_utils = require('../../util/cloud_utils'); const size_utils = require('../../util/size_utils'); const BlockStoreBase = require('./block_store_base').BlockStoreBase; const { RpcError } = require('../../rpc'); +const { NodeHttpHandler } = require("@smithy/node-http-handler"); const DEFAULT_REGION = 'us-east-1'; @@ -39,16 +40,17 @@ class BlockStoreS3 extends BlockStoreBase { RoleSessionName: 'block_store_operations' }; } else { - this.s3cloud = new AWS.S3({ + this.s3cloud = new S3({ endpoint: endpoint, - accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), - secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), - s3ForcePathStyle: true, - signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method), + credentials: { + accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), + secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), + }, + forcePathStyle: true, region: DEFAULT_REGION, - httpOptions: { - agent: http_utils.get_default_agent(endpoint) - } + requestHandler: new NodeHttpHandler({ + httpsAgent: http_utils.get_default_agent(endpoint) + }), }); } } else { @@ -56,16 +58,17 @@ class BlockStoreS3 extends BlockStoreBase { config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION.DEFAULT; this.disable_metadata = config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA[this.cloud_info.endpoint_type] || config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA.DEFAULT; - this.s3cloud = new AWS.S3({ + this.s3cloud = new S3({ endpoint: endpoint, - s3ForcePathStyle: true, - accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), - secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), - signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(endpoint, this.cloud_info.auth_method), - s3DisableBodySigning: cloud_utils.disable_s3_compatible_bodysigning(endpoint), - httpOptions: { - agent: http_utils.get_unsecured_agent(endpoint) - } + forcePathStyle: true, + credentials: { + accessKeyId: this.cloud_info.access_keys.access_key.unwrap(), + secretAccessKey: this.cloud_info.access_keys.secret_key.unwrap(), + }, + applyChecksum: cloud_utils.disable_s3_compatible_bodysigning(endpoint), + requestHandler: new NodeHttpHandler({ + httpsAgent: http_utils.get_unsecured_agent(endpoint) + }), }); } @@ -74,12 +77,12 @@ class BlockStoreS3 extends BlockStoreBase { async init() { try { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const res = await this.s3cloud.getObject({ Bucket: this.cloud_info.target_bucket, Key: this.usage_path, - }).promise(); + }); const usage_data = this.disable_metadata ? res.Body.toString() : @@ -101,12 +104,12 @@ class BlockStoreS3 extends BlockStoreBase { async _read_block_md(block_md) { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const res = await this.s3cloud.headObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_md.id), - }).promise(); + }); return { block_md: this._get_store_block_md(block_md, res), store_md5: res.ETag.toUpperCase(), @@ -159,12 +162,12 @@ class BlockStoreS3 extends BlockStoreBase { async _read_block(block_md) { try { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const res = await this.s3cloud.getObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_md.id), - }).promise(); + }); return { data: res.Body, block_md: this._get_store_block_md(block_md, res), @@ -186,14 +189,14 @@ class BlockStoreS3 extends BlockStoreBase { const encoded_md = this.disable_metadata ? '' : this._encode_block_md(block_md); dbg.log3('writing block id to cloud:', block_key); if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } await this.s3cloud.putObject({ Bucket: this.cloud_info.target_bucket, Key: block_key, Body: data, Metadata: this.disable_metadata ? undefined : { noobaablockmd: encoded_md }, - }).promise(); + }); if (options && options.ignore_usage) return; // return usage count for the object return this._update_usage({ @@ -233,7 +236,7 @@ class BlockStoreS3 extends BlockStoreBase { async _write_usage_internal() { const usage_data = this._encode_block_md(this._usage); if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const res = await this.s3cloud.putObject({ Bucket: this.cloud_info.target_bucket, @@ -242,7 +245,7 @@ class BlockStoreS3 extends BlockStoreBase { Metadata: this.disable_metadata ? undefined : { [this.usage_md_key]: usage_data }, - }).promise(); + }); // if our target bucket returns version ids that means versioning is enabled // and for the usage file that we keep replacing we want to keep only the latest // so we delete the past versions of the usage file. @@ -256,18 +259,18 @@ class BlockStoreS3 extends BlockStoreBase { const endpoint = this.cloud_info.endpoint; if (cloud_utils.is_aws_endpoint(endpoint)) { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } // in s3 there is no error for non-existing object await this.s3cloud.deleteObjectTagging({ Bucket: this.cloud_info.target_bucket, Key: block_key - }).promise(); + }); } else { await this.s3cloud.deleteObject({ Bucket: this.cloud_info.target_bucket, Key: block_key - }).promise(); + }); } } catch (err) { // NoSuchKey is expected @@ -300,7 +303,7 @@ class BlockStoreS3 extends BlockStoreBase { let key_marker; let version_marker; if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } while (is_truncated) { const res = await this.s3cloud.listObjectVersions({ @@ -309,7 +312,7 @@ class BlockStoreS3 extends BlockStoreBase { Delimiter: '/', KeyMarker: key_marker, VersionIdMarker: version_marker, - }).promise(); + }); is_truncated = res.IsTruncated; key_marker = res.NextKeyMarker; version_marker = res.NextVersionIdMarker; @@ -322,7 +325,7 @@ class BlockStoreS3 extends BlockStoreBase { await this.s3cloud.deleteObjects({ Bucket: this.cloud_info.target_bucket, Delete: { Objects: delete_list }, - }).promise(); + }); } } } @@ -336,7 +339,7 @@ class BlockStoreS3 extends BlockStoreBase { let version_marker; dbg.log0(`cleaning up all objects with prefix ${this.base_path}`); if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } while (!done) { const list_res = await this.s3cloud.listObjectVersions({ @@ -344,7 +347,7 @@ class BlockStoreS3 extends BlockStoreBase { Bucket: this.cloud_info.target_bucket, KeyMarker: key_marker, VersionIdMarker: version_marker - }).promise(); + }); const del_objs = list_res.Versions.map(ver => ({ Key: ver.Key, VersionId: ver.VersionId })); if (del_objs.length > 0) { await this.s3cloud.deleteObjects({ @@ -352,7 +355,7 @@ class BlockStoreS3 extends BlockStoreBase { Delete: { Objects: del_objs, } - }).promise(); + }); total += del_objs.length; } @@ -378,7 +381,7 @@ class BlockStoreS3 extends BlockStoreBase { // Todo: Assuming that all requested blocks were deleted, which a bit naive try { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const usage = await this._get_blocks_usage(block_ids); deleted_storage.size -= usage.size; @@ -390,7 +393,7 @@ class BlockStoreS3 extends BlockStoreBase { Key: this._block_key(block_id) })) } - }).promise(); + }); if (res.Errors) { for (const delete_error of res.Errors) { const block_id = this._block_id_from_key(delete_error.Key); @@ -416,12 +419,12 @@ class BlockStoreS3 extends BlockStoreBase { await P.map_with_concurrency(10, block_ids, async block_id => { try { if (this.cloud_info.aws_sts_arn) { - this.s3cloud = await cloud_utils.createSTSS3Client(this.cloud_info, this.additionalS3Params); + this.s3cloud = await cloud_utils.createSTSS3SDKv3Client(this.cloud_info, this.additionalS3Params); } const res = await this.s3cloud.headObject({ Bucket: this.cloud_info.target_bucket, Key: this._block_key(block_id), - }).promise(); + }); const noobaablockmd = res.Metadata.noobaablockmd || res.Metadata.noobaa_block_md; const md_size = (noobaablockmd && noobaablockmd.length) || 0; usage.size += Number(res.ContentLength) + md_size; diff --git a/src/test/unit_tests/util_functions_tests/test_cloud_utils.js b/src/test/unit_tests/util_functions_tests/test_cloud_utils.js index 67d5a179fc..e3e1d9ff80 100644 --- a/src/test/unit_tests/util_functions_tests/test_cloud_utils.js +++ b/src/test/unit_tests/util_functions_tests/test_cloud_utils.js @@ -7,42 +7,49 @@ const sinon = require('sinon'); const AWS = require('aws-sdk'); const cloud_utils = require('../../../util/cloud_utils'); const dbg = require('../../../util/debug_module')(__filename); -const fs = require("fs"); +const { STSClient } = require('@aws-sdk/client-sts'); +const fs = require('fs'); + const projectedServiceAccountToken = "/var/run/secrets/openshift/serviceaccount/token"; const fakeAccessKeyId = "fakeAccessKeyId"; const fakeSecretAccessKey = "fakeSecretAccessKey"; const fakeSessionToken = "fakeSessionToken"; const roleArn = "arn:aws:iam::261532230807:role/noobaa_s3_sts"; const defaultSTSCredsValidity = 3600; +const REGION = "us-east-1"; const expectedParams = [{ RoleArn: roleArn, RoleSessionName: 'testSession', WebIdentityToken: 'web-identity-token', DurationSeconds: defaultSTSCredsValidity, }]; + mocha.describe('AWS STS tests', function() { let STSStub; let stsFake; mocha.before('Creating STS stub', function() { + sinon.stub(fs.promises, "readFile") .withArgs(projectedServiceAccountToken) .returns("web-identity-token"); + stsFake = { - assumeRoleWithWebIdentity: sinon.stub().returnsThis(), - promise: sinon.stub() - .resolves({ - Credentials: { - AccessKeyId: fakeAccessKeyId, - SecretAccessKey: fakeSecretAccessKey, - SessionToken: fakeSessionToken - } - }), - }; + assumeRoleWithWebIdentity: sinon.stub().returnsThis(), + promise: sinon.stub() + .resolves({ + Credentials: { + AccessKeyId: fakeAccessKeyId, + SecretAccessKey: fakeSecretAccessKey, + SessionToken: fakeSessionToken + } + }), + }; STSStub = sinon.stub(AWS, 'STS') .callsFake(() => stsFake); }); mocha.after('Restoring STS stub', function() { STSStub.restore(); + fs.promises.readFile.restore?.(); }); mocha.it('should generate aws sts creds', async function() { const params = { @@ -73,3 +80,53 @@ mocha.describe('AWS STS tests', function() { assert.equal(s3.config.region, 'us-east-1'); }); }); + +mocha.describe('AWS STS SDK V3 tests', function() { + let sts_v3_stub; + mocha.before('Creating STS stub', function() { + sinon.stub(fs.promises, "readFile") + .withArgs(projectedServiceAccountToken) + .returns("web-identity-token"); + sts_v3_stub = sinon.stub(STSClient.prototype, 'send') + .callsFake(() => Promise.resolve({ + Credentials: { + AccessKeyId: fakeAccessKeyId, + SecretAccessKey: fakeSecretAccessKey, + SessionToken: fakeSessionToken + } + })); + }); + mocha.after('Restoring STS v3 stub', function() { + sts_v3_stub.restore(); + fs.promises.readFile.restore?.(); + }); + mocha.it('should generate aws sts creds', async function() { + const params = { + aws_sts_arn: roleArn, + region: REGION, + }; + const roleSessionName = "testSession"; + const json = await cloud_utils.generate_aws_sdkv3_sts_creds(params, roleSessionName); + sinon.assert.calledOnce(sts_v3_stub); + assert.equal(json.accessKeyId, fakeAccessKeyId); + assert.equal(json.secretAccessKey, fakeSecretAccessKey); + assert.equal(json.sessionToken, fakeSessionToken); + dbg.log0('test.aws.sts.assumeRoleWithWebIdentity: ', json); + }); + + mocha.it('should generate an STS S3 client', async function() { + const params = { + aws_sts_arn: roleArn, + region: 'us-east-1' + }; + const additionalParams = { + RoleSessionName: 'testSession' + }; + const s3 = await cloud_utils.createSTSS3SDKv3Client(params, additionalParams); + dbg.log0('test.aws.sts.createSTSS3Client: ', (await s3.config.credentials())); + assert.equal((await s3.config.credentials()).accessKeyId, fakeAccessKeyId); + assert.equal((await s3.config.credentials()).secretAccessKey, fakeSecretAccessKey); + assert.equal((await s3.config.credentials()).sessionToken, fakeSessionToken); + assert.equal((await s3.config.region()), 'us-east-1'); + }); +}); diff --git a/src/util/cloud_utils.js b/src/util/cloud_utils.js index ec6b5c465f..59cbe3fd9d 100644 --- a/src/util/cloud_utils.js +++ b/src/util/cloud_utils.js @@ -10,6 +10,8 @@ const { S3 } = require('@aws-sdk/client-s3'); const url = require('url'); const _ = require('lodash'); const SensitiveString = require('./sensitive_string'); +const { STSClient, AssumeRoleWithWebIdentityCommand } = require('@aws-sdk/client-sts'); +const { NodeHttpHandler } = require('@smithy/node-http-handler'); const config = require('../../config'); const noobaa_s3_client = require('../sdk/noobaa_s3_client/noobaa_s3_client'); @@ -61,6 +63,41 @@ async function generate_aws_sts_creds(params, roleSessionName) { ); } +async function createSTSS3SDKv3Client(params, additionalParams) { + const creds = await generate_aws_sdkv3_sts_creds(params, additionalParams.RoleSessionName); + return new S3({ + credentials: creds, + region: params.region || config.DEFAULT_REGION, + endpoint: additionalParams.endpoint, + requestHandler: new NodeHttpHandler({ + httpsAgent: additionalParams.httpOptions + }), + forcePathStyle: additionalParams.s3ForcePathStyle + }); +} + +async function generate_aws_sdkv3_sts_creds(params, roleSessionName) { + + const sts_client = new STSClient({ region: params.region || config.DEFAULT_REGION }); + const input = { + DurationSeconds: defaultSTSCredsValidity, + RoleArn: params.aws_sts_arn, + RoleSessionName: roleSessionName || defaultRoleSessionName, + WebIdentityToken: (await fs.promises.readFile(projectedServiceAccountToken)).toString(), + }; + const command = new AssumeRoleWithWebIdentityCommand(input); + const response = await sts_client.send(command); + if (_.isEmpty(response) || _.isEmpty(response.Credentials)) { + dbg.error(`AWS STS empty creds ${params.RoleArn}, RolesessionName: ${params.RoleSessionName},Projected service Account Token Path : ${projectedServiceAccountToken}`); + throw new RpcError('AWS_STS_ERROR', 'Empty AWS STS creds retrieved for Role "' + params.RoleArn + '"'); + } + return { + accessKeyId: response.Credentials.AccessKeyId, + secretAccessKey: response.Credentials.SecretAccessKey, + sessionToken: response.Credentials.SessionToken, + }; +} + function get_signed_url(params, expiry = 604800, custom_operation = 'getObject') { const op = custom_operation; const s3 = new AWS.S3({ @@ -213,3 +250,5 @@ exports.set_noobaa_s3_connection = set_noobaa_s3_connection; exports.createSTSS3Client = createSTSS3Client; exports.generate_aws_sts_creds = generate_aws_sts_creds; exports.generate_access_keys = generate_access_keys; +exports.createSTSS3SDKv3Client = createSTSS3SDKv3Client; +exports.generate_aws_sdkv3_sts_creds = generate_aws_sdkv3_sts_creds;