feat: Increase STS Sample Coverage (#42)

* test: Prepare for multiple samples and tests

* feat: Increase STS Sample Coverage

* 🦉 Updates from OwlBot

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* style: lint

* ci: Add `pre-samples-test.sh` for AWS integration testing

* style: Streamline error handling

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: danielbankhead

storagetransfer/aws-request.js

@@ -0,0 +1,110 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(
+  projectId,
+  description,
+  awsSourceBucket,
+  awsAccessKeyId,
+  awsSecretAccessKey,
+  gcsSinkBucket
+) {
+  // [START storagetransfer_transfer_from_aws]
+
+  // Imports the Google Cloud client library
+  const {
+    StorageTransferServiceClient,
+  } = require('@google-cloud/storage-transfer');
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of the Google Cloud Platform Project that owns the job
+  // projectId = 'my-project-id'
+
+  // A useful description for your transfer job
+  // description = 'My transfer job'
+
+  // AWS S3 source bucket name
+  // awsSourceBucket = 'my-s3-source-bucket'
+
+  // AWS Access Key ID
+  // awsAccessKeyId = 'AKIA...'
+
+  // AWS Secret Access Key
+  // awsSecretAccessKey = 'HEAoMK2.../...ku8'
+
+  // Google Cloud Storage destination bucket name
+  // gcsSinkBucket = 'my-gcs-destination-bucket'
+
+  // Creates a client
+  const client = new StorageTransferServiceClient();
+
+  /**
+   * Creates a one-time transfer job from Amazon S3 to Google Cloud Storage.
+   */
+  async function transferFromS3() {
+    // Setting the start date and the end date as the same time creates a
+    // one-time transfer
+    const now = new Date();
+    const oneTimeSchedule = {
+      day: now.getDate(),
+      month: now.getMonth() + 1,
+      year: now.getFullYear(),
+    };
+
+    // Runs the request and creates the job
+    const [transferJob] = await client.createTransferJob({
+      transferJob: {
+        projectId,
+        description,
+        status: 'ENABLED',
+        schedule: {
+          scheduleStartDate: oneTimeSchedule,
+          scheduleEndDate: oneTimeSchedule,
+        },
+        transferSpec: {
+          awsS3DataSource: {
+            bucketName: awsSourceBucket,
+            awsAccessKey: {
+              accessKeyId: awsAccessKeyId,
+              secretAccessKey: awsSecretAccessKey,
+            },
+          },
+          gcsDataSink: {
+            bucketName: gcsSinkBucket,
+          },
+        },
+      },
+    });
+
+    console.log(
+      `Created and ran a transfer job from '${awsSourceBucket}' to '${gcsSinkBucket}' with name ${transferJob.name}`
+    );
+  }
+
+  transferFromS3();
+  // [END storagetransfer_transfer_from_aws]
+}
+
+main(...process.argv.slice(2));
+
+process.on('unhandledRejection', err => {
+  console.error(err);
+  process.exitCode = 1;
+});

storagetransfer/check-latest-transfer-operation.js

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(projectId, jobName) {
+  // [START storagetransfer_get_latest_transfer_operation]
+
+  // Imports the Google Cloud client library
+  const {
+    StorageTransferServiceClient,
+  } = require('@google-cloud/storage-transfer');
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of the Google Cloud Platform Project that owns the job
+  // projectId = 'my-project-id'
+
+  // Storage Transfer Service job name
+  // jobName = 'transferJobs/1234567890'
+
+  // Creates a client
+  const client = new StorageTransferServiceClient();
+
+  /**
+   * Checks the latest transfer operation for a given transfer job.
+   */
+  async function checkLatestTransferOperation() {
+    const [transferJob] = await client.getTransferJob({projectId, jobName});
+
+    if (transferJob.latestOperationName) {
+      const [transferOperation] = await client.operationsClient.getOperation({
+        name: transferJob.latestOperationName,
+      });
+
+      const operation = JSON.stringify(transferOperation, null, 2);
+
+      console.log(`Latest transfer operation for '${jobName}': ${operation}`);
+    } else {
+      console.log(`Transfer job '${jobName}' has not ran yet.`);
+    }
+  }
+
+  checkLatestTransferOperation();
+  // [END storagetransfer_get_latest_transfer_operation]
+}
+
+main(...process.argv.slice(2));
+
+process.on('unhandledRejection', err => {
+  console.error(err);
+  process.exitCode = 1;
+});

storagetransfer/get-transfer-job-with-retries.js

@@ -0,0 +1,79 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(projectId, jobName, maxRetryDelayMillis) {
+  // [START storagetransfer_create_retry_handler]
+
+  // Imports the Google Cloud client library
+  const {
+    StorageTransferServiceClient,
+  } = require('@google-cloud/storage-transfer');
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of the Google Cloud Platform Project that owns the job
+  // projectId = 'my-project-id'
+
+  // Storage Transfer Service job name
+  // jobName = 'transferJobs/1234567890'
+
+  // The maximum delay time, in milliseconds, between requests
+  // maxRetryDelayMillis = 60000
+
+  // Creates a client
+  const client = new StorageTransferServiceClient();
+
+  /**
+   * Check the latest transfer operation associated with a transfer job
+   * with retries.
+   */
+  async function getTransferJobWithRetries() {
+    // Setting the start date and the end date as the same time creates a
+    // one-time transfer
+
+    const options = {
+      retry: {
+        backoffSettings: {
+          maxRetryDelayMillis,
+        },
+      },
+    };
+
+    const [transferJob] = await client.getTransferJob(
+      {projectId, jobName},
+      options
+    );
+
+    console.log(
+      `Fetched transfer job: ${transferJob.name} with a maximum of ${maxRetryDelayMillis}ms delay time between requests`
+    );
+  }
+
+  getTransferJobWithRetries();
+  // [END storagetransfer_create_retry_handler]
+}
+
+const [projectId, jobName, maxRetryDelayMillis] = [...process.argv.slice(2)];
+
+main(projectId, jobName, Number.parseInt(maxRetryDelayMillis));
+
+process.on('unhandledRejection', err => {
+  console.error(err);
+  process.exitCode = 1;
+});

storagetransfer/nearline-request.js

@@ -0,0 +1,113 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(
+  projectId,
+  description,
+  gcsSourceBucket,
+  gcsSinkBucket,
+  startDate = new Date()
+) {
+  // [START storagetransfer_transfer_to_nearline]
+
+  // Imports the Google Cloud client library
+  const {
+    StorageTransferServiceClient,
+  } = require('@google-cloud/storage-transfer');
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of the Google Cloud Platform Project that owns the job
+  // projectId = 'my-project-id'
+
+  // A useful description for your transfer job
+  // description = 'My transfer job'
+
+  // Google Cloud Storage source bucket name
+  // gcsSourceBucket = 'my-gcs-source-bucket'
+
+  // Google Cloud Storage destination bucket name
+  // gcsSinkBucket = 'my-gcs-destination-bucket'
+
+  // Date to start daily migration
+  // startDate = new Date()
+
+  // Creates a client
+  const client = new StorageTransferServiceClient();
+
+  /**
+   * Create a daily migration from a GCS bucket to another GCS bucket for
+   * objects untouched for 30+ days.
+   */
+  async function createDailyNearline30DayMigration() {
+    // Runs the request and creates the job
+    const [transferJob] = await client.createTransferJob({
+      transferJob: {
+        projectId,
+        description,
+        status: 'ENABLED',
+        schedule: {
+          scheduleStartDate: {
+            day: startDate.getDate(),
+            month: startDate.getMonth() + 1,
+            year: startDate.getFullYear(),
+          },
+        },
+        transferSpec: {
+          gcsDataSource: {
+            bucketName: gcsSourceBucket,
+          },
+          gcsDataSink: {
+            bucketName: gcsSinkBucket,
+          },
+          objectConditions: {
+            minTimeElapsedSinceLastModification: {
+              seconds: 2592000, // 30 days
+            },
+          },
+          transferOptions: {
+            deleteObjectsFromSourceAfterTransfer: true,
+          },
+        },
+      },
+    });
+
+    console.log(`Created transferJob: ${transferJob.name}`);
+  }
+
+  createDailyNearline30DayMigration();
+  // [END storagetransfer_transfer_to_nearline]
+}
+
+const [projectId, description, gcsSourceBucket, gcsSinkBucket, startDate] = [
+  ...process.argv.slice(2),
+];
+
+main(
+  projectId,
+  description,
+  gcsSourceBucket,
+  gcsSinkBucket,
+  new Date(startDate)
+);
+
+process.on('unhandledRejection', err => {
+  console.error(err);
+  process.exitCode = 1;
+});

storagetransfer/package.json

@@ -10,16 +10,17 @@
     "*.js"
   ],
   "scripts": {
-    "test": "c8 mocha --timeout 600000 test/*.js"
+    "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js"
   },
   "dependencies": {
     "@google-cloud/storage-transfer": "^1.1.1"
   },
   "devDependencies": {
-    "chai": "^4.2.0",
+    "@google-cloud/storage": "^5.15.3",
+    "aws-sdk": "^2.1073.0",
     "c8": "^7.1.0",
+    "chai": "^4.2.0",
     "mocha": "^9.0.0",
-    "uuid": "^8.0.0",
-    "@google-cloud/storage": "^5.15.3"
+    "uuid": "^8.0.0"
   }
 }