Skip to content

Commit 1cdb592

Browse files
JesseLovelaceJesseLovelace
authored
(feat) Storage Transfer Service: add gapic samples and refactor test suite (#6120)
* add gapic samples and refactor test suite * Add quickstart, move to own directory, run formatter * Add cleanup to tests * move client creation up in quickstart * Fix import * fix copyrights
1 parent 150aa2d commit 1cdb592

File tree

18 files changed

+897
-329
lines changed

18 files changed

+897
-329
lines changed

‎storage/storage-transfer/pom.xml‎ renamed to ‎storage-transfer/pom.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@
5151
</exclusion>
5252
</exclusions>
5353
</dependency>
54+
<dependency>
55+
<groupId>com.google.cloud</groupId>
56+
<artifactId>google-cloud-storage-transfer</artifactId>
57+
<version>0.1.0</version>
58+
</dependency>
5459
<dependency>
5560
<groupId>com.google.auth</groupId>
5661
<artifactId>google-auth-library-oauth2-http</artifactId>
@@ -84,5 +89,19 @@
8489
<scope>test</scope>
8590
</dependency>
8691

92+
<dependency>
93+
<groupId>com.google.cloud</groupId>
94+
<artifactId>google-cloud-storage</artifactId>
95+
<version>2.1.5</version>
96+
<scope>test</scope>
97+
</dependency>
98+
99+
<dependency>
100+
<groupId>com.amazonaws</groupId>
101+
<artifactId>aws-java-sdk-s3</artifactId>
102+
<version>1.12.77</version>
103+
<scope>test</scope>
104+
</dependency>
105+
87106
</dependencies>
88107
</project>

‎storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/CheckLatestTransferOperation.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Copyright 2021 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
// [START storagetransfer_get_latest_transfer_operation]
20+
21+
import com.google.longrunning.Operation;
22+
import com.google.storagetransfer.v1.proto.StorageTransferServiceClient;
23+
import com.google.storagetransfer.v1.proto.TransferProto.GetTransferJobRequest;
24+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob;
25+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferOperation;
26+
import java.io.IOException;
27+
28+
public class CheckLatestTransferOperation {
29+
30+
// Gets the requested transfer job and checks its latest operation
31+
public static void checkLatestTransferOperation(String projectId, String jobName)
32+
throws IOException {
33+
// Your Google Cloud Project ID
34+
// String projectId = "your-project-id";
35+
36+
// The name of the job to check
37+
// String jobName = "myJob/1234567890";
38+
39+
StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create();
40+
41+
// Get transfer job and check latest operation
42+
TransferJob transferJob =
43+
storageTransfer.getTransferJob(
44+
GetTransferJobRequest.newBuilder().setJobName(jobName).setProjectId(projectId).build());
45+
String latestOperationName = transferJob.getLatestOperationName();
46+
47+
if (!latestOperationName.isEmpty()) {
48+
Operation operation = storageTransfer.getOperationsClient().getOperation(latestOperationName);
49+
TransferOperation latestOperation =
50+
TransferOperation.parseFrom(operation.getMetadata().getValue());
51+
52+
System.out.println("The latest operation for transfer job " + jobName + " is:");
53+
System.out.println(latestOperation.toString());
54+
55+
} else {
56+
System.out.println(
57+
"Transfer job "
58+
+ jobName
59+
+ " hasn't run yet,"
60+
+ " try again once the job starts running.");
61+
}
62+
}
63+
}
64+
// [END storagetransfer_get_latest_transfer_operation]

‎storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/QuickstartSample.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright 2021 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
// [START storagetransfer_quickstart]
20+
import com.google.storagetransfer.v1.proto.StorageTransferServiceClient;
21+
import com.google.storagetransfer.v1.proto.TransferProto.CreateTransferJobRequest;
22+
import com.google.storagetransfer.v1.proto.TransferProto.RunTransferJobRequest;
23+
import com.google.storagetransfer.v1.proto.TransferTypes.GcsData;
24+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob;
25+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferSpec;
26+
27+
public class QuickstartSample {
28+
/** Quickstart sample using transfer service to transfer from one GCS bucket to another. */
29+
public static void quickStartSample(
30+
String projectId, String gcsSourceBucket, String gcsSinkBucket) throws Exception {
31+
// Your Google Cloud Project ID
32+
// String projectId = "your-project-id";
33+
34+
// The name of the source GCS bucket to transfer objects from
35+
// String gcsSourceBucket = "your-source-gcs-source-bucket";
36+
37+
// The name of the GCS bucket to transfer objects to
38+
// String gcsSinkBucket = "your-sink-gcs-bucket";
39+
StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create();
40+
41+
TransferJob transferJob =
42+
TransferJob.newBuilder()
43+
.setProjectId(projectId)
44+
.setTransferSpec(
45+
TransferSpec.newBuilder()
46+
.setGcsDataSource(GcsData.newBuilder().setBucketName(gcsSourceBucket))
47+
.setGcsDataSink(GcsData.newBuilder().setBucketName(gcsSinkBucket)))
48+
.setStatus(TransferJob.Status.ENABLED)
49+
.build();
50+
51+
TransferJob response =
52+
storageTransfer.createTransferJob(
53+
CreateTransferJobRequest.newBuilder().setTransferJob(transferJob).build());
54+
55+
storageTransfer.runTransferJobAsync(
56+
RunTransferJobRequest.newBuilder()
57+
.setProjectId(projectId)
58+
.setJobName(response.getName())
59+
.build())
60+
.get();
61+
System.out.println(
62+
"Created and ran transfer job between two GCS buckets with name " + response.getName());
63+
}
64+
}
65+
// [END storagetransfer_quickstart]

‎storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferFromAws.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* Copyright 2021 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
// [START storagetransfer_transfer_from_aws]
20+
21+
import com.google.storagetransfer.v1.proto.StorageTransferServiceClient;
22+
import com.google.storagetransfer.v1.proto.TransferProto.CreateTransferJobRequest;
23+
import com.google.storagetransfer.v1.proto.TransferTypes.AwsAccessKey;
24+
import com.google.storagetransfer.v1.proto.TransferTypes.AwsS3Data;
25+
import com.google.storagetransfer.v1.proto.TransferTypes.GcsData;
26+
import com.google.storagetransfer.v1.proto.TransferTypes.Schedule;
27+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob;
28+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob.Status;
29+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferSpec;
30+
import com.google.type.Date;
31+
import com.google.type.TimeOfDay;
32+
import java.io.IOException;
33+
import java.util.Calendar;
34+
35+
public class TransferFromAws {
36+
37+
// Creates a one-off transfer job from Amazon S3 to Google Cloud Storage.
38+
public static void transferFromAws(
39+
String projectId,
40+
String jobDescription,
41+
String awsSourceBucket,
42+
String gcsSinkBucket,
43+
long startDateTime,
44+
String awsAccessKeyId,
45+
String awsSecretAccessKey)
46+
throws IOException {
47+
48+
// Your Google Cloud Project ID
49+
// String projectId = "your-project-id";
50+
51+
// A short description of this job
52+
// String jobDescription = "Sample transfer job from S3 to GCS.";
53+
54+
// The name of the source AWS bucket to transfer data from
55+
// String awsSourceBucket = "yourAwsSourceBucket";
56+
57+
// The name of the GCS bucket to transfer data to
58+
// String gcsSinkBucket = "your-gcs-bucket";
59+
60+
// What day and time in UTC to start the transfer, expressed as an epoch date timestamp.
61+
// If this is in the past relative to when the job is created, it will run the next day.
62+
// long startDateTime =
63+
// new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2000-01-01 00:00:00").getTime();
64+
65+
// The ID used to access your AWS account. Should be accessed via environment variable.
66+
// String awsAccessKeyId = System.getenv("AWS_ACCESS_KEY_ID");
67+
68+
// The Secret Key used to access your AWS account. Should be accessed via environment variable.
69+
// String awsSecretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY");
70+
71+
// Set up source and sink
72+
TransferSpec transferSpec =
73+
TransferSpec.newBuilder()
74+
.setAwsS3DataSource(
75+
AwsS3Data.newBuilder()
76+
.setBucketName(awsSourceBucket)
77+
.setAwsAccessKey(
78+
AwsAccessKey.newBuilder()
79+
.setAccessKeyId(awsAccessKeyId)
80+
.setSecretAccessKey(awsSecretAccessKey)))
81+
.setGcsDataSink(GcsData.newBuilder().setBucketName(gcsSinkBucket))
82+
.build();
83+
84+
// Parse epoch timestamp into the model classes
85+
Calendar startCalendar = Calendar.getInstance();
86+
startCalendar.setTimeInMillis(startDateTime);
87+
// Note that this is a Date from the model class package, not a java.util.Date
88+
Date startDate =
89+
Date.newBuilder()
90+
.setYear(startCalendar.get(Calendar.YEAR))
91+
.setMonth(startCalendar.get(Calendar.MONTH) + 1)
92+
.setDay(startCalendar.get(Calendar.DAY_OF_MONTH))
93+
.build();
94+
TimeOfDay startTime =
95+
TimeOfDay.newBuilder()
96+
.setHours(startCalendar.get(Calendar.HOUR_OF_DAY))
97+
.setMinutes(startCalendar.get(Calendar.MINUTE))
98+
.setSeconds(startCalendar.get(Calendar.SECOND))
99+
.build();
100+
Schedule schedule =
101+
Schedule.newBuilder()
102+
.setScheduleStartDate(startDate)
103+
.setScheduleEndDate(startDate)
104+
.setStartTimeOfDay(startTime)
105+
.build();
106+
107+
// Set up the transfer job
108+
TransferJob transferJob =
109+
TransferJob.newBuilder()
110+
.setDescription(jobDescription)
111+
.setProjectId(projectId)
112+
.setTransferSpec(transferSpec)
113+
.setSchedule(schedule)
114+
.setStatus(Status.ENABLED)
115+
.build();
116+
117+
// Create a Transfer Service client
118+
StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create();
119+
120+
// Create the transfer job
121+
TransferJob response =
122+
storageTransfer.createTransferJob(
123+
CreateTransferJobRequest.newBuilder().setTransferJob(transferJob).build());
124+
125+
System.out.println("Created transfer job from AWS to GCS:");
126+
System.out.println(response.toString());
127+
}
128+
}
129+
// [END storagetransfer_transfer_from_aws]

0 commit comments

Comments
 (0)