Develop Reference

Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain

I'm getting the following exception when trying to read a file to AWS S3 Error:
Unable to load AWS credentials from any provider in the chain.
I have generated a public bucket and also generated an AWS IAM role with full S3 bucket access and textract access.
I am trying to read an image from S3 bucket and run AWS Textract service.
Java code:
package com.textract;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.textract.TextractClient;
import software.amazon.awssdk.services.textract.model.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class App {
public static Map<String, String> getRelationships(Map<String, Block> blockMap, Map<String, Block> keyMap,Map<String, Block> valueMap) {
Map<String, String> result = new LinkedHashMap<>();
for(Map.Entry<String, Block> itr : keyMap.entrySet()) {
Block valueBlock = findValue(itr.getValue(), valueMap);
String key = getText(itr.getValue(), blockMap);
String value = getText(valueBlock, blockMap);
result.put(key, value);
}
return result;
}
public static Block findValue(Block keyBlock, Map<String, Block> valueMap) {
Block b = null;
for(Relationship relationship : keyBlock.relationships()) {
if(relationship.type().toString().equals("VALUE")) {
for(String id : relationship.ids()) {
b = valueMap.get(id);
}
}
}
return b;
}
public static String getText(Block result, Map<String, Block> blockMap) {
StringBuilder stringBuilder = new StringBuilder();
for(Relationship relationship : result.relationships()) {
if(relationship.type().toString().equals("CHILD")) {
for(String id : relationship.ids()) {
Block b = blockMap.get(id);
if(b.blockTypeAsString().equals("WORD")) {
stringBuilder.append(b.text()).append(" ");
}
}
}
}
return stringBuilder.toString();
}
public static void main(String[] args) {
BasicAWSCredentials creds = new BasicAWSCredentials("<Secret Key>", "<Access Key>");
AmazonS3 s3client = AmazonS3Client.builder()
.withRegion("ap-southeast-1")
.withCredentials(new AWSStaticCredentialsProvider(creds))
.build();
// AmazonS3 s3client = AmazonS3ClientBuilder.standard().build();
S3Object s3Object = s3client.getObject("<S3 Bucket>", "image.jpg");
S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent();
SdkBytes bytes = SdkBytes.fromInputStream(s3ObjectInputStream);
Document doc = Document.builder().bytes(bytes).build();
List<FeatureType> list = new ArrayList<>();
list.add(FeatureType.FORMS);
AnalyzeDocumentRequest request = AnalyzeDocumentRequest.builder().featureTypes(list).document(doc).build();
TextractClient textractClient = TextractClient.builder().region(Region.US_WEST_2).build();
AnalyzeDocumentResponse response = textractClient.analyzeDocument(request);
List<Block> blocks = response.blocks();
Map<String, Block> blockMap = new LinkedHashMap<>();
Map<String, Block> keyMap = new LinkedHashMap<>();
Map<String, Block> valueMap = new LinkedHashMap<>();
for (Block b : blocks) {
String block_id = b.id();
blockMap.put(block_id, b);
if(b.blockTypeAsString().equals("KEY_VALUE_SET")) {
for(EntityType entityType : b.entityTypes()) {
if(entityType.toString().equals("KEY")) {
keyMap.put(block_id, b);
} else {
valueMap.put(block_id, b);
}
}
}
}
System.out.println(getRelationships(blockMap, keyMap, valueMap));
textractClient.close();
}
}
Console error:
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
Exception in thread "main" software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(credentialsProviders=[SystemPropertyCredentialsProvider(), EnvironmentVariableCredentialsProvider(), WebIdentityTokenCredentialsProvider(), ProfileCredentialsProvider(), ContainerCredentialsProvider(), InstanceProfileCredentialsProvider()]) : [SystemPropertyCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., EnvironmentVariableCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., WebIdentityTokenCredentialsProvider(): Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set., ProfileCredentialsProvider(): Profile file contained no credentials for profile 'default': ProfileFile(profiles=[]), ContainerCredentialsProvider(): Cannot fetch credentials from container - neither AWS_CONTAINER_CREDENTIALS_FULL_URI or AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variables are set., InstanceProfileCredentialsProvider(): Unable to load credentials from service endpoint.]
at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:98)
at software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain.resolveCredentials(AwsCredentialsProviderChain.java:112)
at software.amazon.awssdk.auth.credentials.internal.LazyAwsCredentialsProvider.resolveCredentials(LazyAwsCredentialsProvider.java:45)
at software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider.resolveCredentials(DefaultCredentialsProvider.java:104)
at software.amazon.awssdk.awscore.client.handler.AwsClientHandlerUtils.createExecutionContext(AwsClientHandlerUtils.java:76)
at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.createExecutionContext(AwsSyncClientHandler.java:68)
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:97)
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:167)
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:94)
at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:55)
at software.amazon.awssdk.services.textract.DefaultTextractClient.analyzeDocument(DefaultTextractClient.java:215)
at com.textract.App.main(App.java:83)

You are only passing the credentials to the S3 client. You aren't passing them to the Textract client. If you are creating the credentials provider manually like that, you will need to pass it to each client instance that is created.
I highly suggest using the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to provide credentials, instead of setting them directly in your code like that, as it will be much more flexible when you run this code in other environments, and will allow the AWS SDK to pick up the credentials automatically without the need for you to create a credentials provider at all.

I finally got it working with the below code with the below article:
Calling Amazon Textract Synchronous Operations
String document="input.png";
String bucket="bucket";
AmazonTextract client = AmazonTextractClientBuilder.defaultClient();
DetectDocumentTextRequest request = new DetectDocumentTextRequest()
.withDocument(new Document()
.withS3Object(new S3Object()
.withName(document)
.withBucket(bucket)));
DetectDocumentTextResult result = client.detectDocumentText(request);
Working Code
package aws.cloud.work;
import java.io.IOException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.textract.AmazonTextract;
import com.amazonaws.services.textract.AmazonTextractClientBuilder;
import com.amazonaws.services.textract.model.DetectDocumentTextRequest;
import com.amazonaws.services.textract.model.DetectDocumentTextResult;
import com.amazonaws.services.textract.model.Document;
import com.amazonaws.services.textract.model.S3Object;
public class TextractOriginalMaster3 {
static AmazonTextractClientBuilder clientBuilder = AmazonTextractClientBuilder.standard().withRegion(Regions.AP_SOUTHEAST_1);
public static void main(String[] args) throws IOException {
//Set AWS Credentials to use Textract
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(new
BasicAWSCredentials("Accesskey", "Secretkey")));
//**Getting document from S3 Bucket Path
String document = "image.png";
String bucket = "Bucket Name";
//Calling AWS Textract Client
AmazonTextract client = clientBuilder.build();
DetectDocumentTextRequest request = new DetectDocumentTextRequest()
.withDocument(new Document()
.withS3Object(new S3Object()
.withName(document)
.withBucket(bucket)));
DetectDocumentTextResult result = client.detectDocumentText(request);
System.out.println(result);
result.getBlocks().forEach(block -> {
if (block.getBlockType().equals("LINE"))
System.out.println("text is " + block.getText() + " confidence is " + block.getConfidence());
});
}
}

To answer this, we can load the credentials using StaticCredentialsProvider
Please refer the below code.
TextractClient textractClient = TextractClient.builder().region(Region.of(region)).credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials
.create(accessKey,secretKey))).build();

Google Drive API unable to List Files (v3 Java)

I'm using Google API in Java and have the following code snippets.
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.GeneralSecurityException;
import java.util.Collections;
import java.util.List;
import java.lang.Object;
import java.util.ArrayList;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.util.store.FileDataStoreFactory;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.FileList;
public class DriveQuickstart {
private static final String APPLICATION_NAME = "DriveQuickstart";
private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
private static final String TOKENS_DIRECTORY_PATH = "tokens";
// * Global instance of the scopes required by this quickstart.
// * If modifying these scopes, delete your previously saved tokens/ folder.
private static final List<String> SCOPES = Collections.singletonList(DriveScopes.DRIVE_METADATA_READONLY);
private static final String CREDENTIALS_FILE_PATH = "C:/Quickstart/src/main/resources/client_secret.json";
// * Creates an authorized Credential object.
// * #param HTTP_TRANSPORT The network HTTP Transport.
// * #return An authorized Credential object.
// * #throws IOException If the client_secret.json file cannot be found.
// */
private static Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException {
// Load client secrets.
InputStream in = new FileInputStream(CREDENTIALS_FILE_PATH);
if (in == null) {
throw new FileNotFoundException("Resource not found: " + CREDENTIALS_FILE_PATH);
}
GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in));
// Build flow and trigger user authorization request.
GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder(
HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES)
.setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH)))
.setAccessType("offline")
.build();
LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8888).build();
return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user");
}
public static void main(String[] args) throws IOException, GeneralSecurityException {
try
{
// Build a new authorized API client service.
System.out.println("This is a test line");
//declare final variable NetHttpTransport and GoogleNetHttpTransport from import statements
final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport();
System.out.println("This is the second test line");
Drive service = new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT))
.setApplicationName(APPLICATION_NAME)
.build();
System.out.println("This is the third test line");
// Print the names and IDs for files.
FileList result = service.files().list()
.setPageSize(10)
.setFields("nextPageToken, files(id, name)")
.execute();
List<File> files = result.getFiles();
if (files == null || files.isEmpty()) {
System.out.println("No files found.");
} else {
System.out.println("Files:");
for (File file : files) {
System.out.printf("%s (%s)\n", file.getName(), file.getId());
}
}
Drive.Files var1 = service.files();
System.out.println("var1 = " + var1.toString());
Drive.Files.List var2 = var1.list();
if (var2 == null)
System.out.println("null");
else {
System.out.printf("Var2 size is: %d\n", var2.size() );
}
//iterate through map and print results
}
catch(Exception e){
System.out.println("error");
}
System.out.println("This is the fourth test line");
}
}
}
To go more into detail. I'm using a somewhat unconventional method to try it. I'm using only command line to run (no IDEs, no gradle).
Here is a link to an ss with my output in command prompt.
cmd output
Assuming that I am actually accessing the metadata in Google Drive, and that the Drive.Files is nested inside an Abstract Map (java.util.AbstractMap), I should be able to return the size of the map to get an idea of where it it searching by how many files it is returning.
I'm not sure what is going on. First time I compile and run, I get the tab opening in chrome asking for permission to gain access to metadata to drive.
Second time, it gives the "WARNING: unable to change permissions for everybody: " errors.
On top of that, var1 ('set to service.files()') returns com.google.api.services.drive.Drive$Files#458ad742 when using toString().
The subclass service.files().list() assigned to var2 should return a list of files in my Google Drive, but apparently it is not doing so.
I may be misunderstanding a lot of stuff, so bear with me. Thanks for the help.
In case people ask, I have tried the tutorial here
https://developers.google.com/drive/api/v3/quickstart/java
It opened the tab to ask for metadata access to google drive on whichever google account I selected.
storedcredential
So from that alone, I (think) connected to Google Drive.
However, the default code that is supposed to list the names and ids of 10 files doesn't do anything.
How am I running the program:
Here is the batch file that I'm using to compile and run the code in cmd.
jar cfe DriveQuickstart.jar Quickstart DriveQuickstart.class
javac -cp ./* DriveQuickstart.java
java -cp ./* DriveQuickstart
For some reason I don't know/understand, this was the only way that wouldn't return methods from the imported jar files not existing.

Familiarize yourself with the Java Quickstart to see how to get the results of the API calls.
Problems
1 - setPermissionsToOwnerOnly
I'm not sure what is going on. First time I compile and run, I get the tab opening in chrome asking for permission to gain access to metadata to drive. Second time, it gives the "WARNING: unable to change permissions for everybody: " errors.
This seems to be related to this issue. It could mean that the folder you are using to run your code from has different permissions than expected.
Does this happen if you place your code on your Desktop (as example) and then run it from there?
2 - toString()
On top of that, var1 ('set to service.files()') returns com.google.api.services.drive.Drive$Files#458ad742 when using toString(). The subclass service.files().list() assigned to var2 should return a list of files in my Google Drive, but apparently it is not doing so.
This weird looking string is how Java prints out an instance of a class.
com.google.api.services.drive.Drive$Files#458ad742
<NAMESPACE>$<CLASS>#<HASHCODE>
The fact that the toString method prints this out means that this method wasn't overwritten for this specific Class, so its calling the general Object.toString() method.
To print out what you want to see, try the nested methods.

AWS Sagemaker and Java SDK

I have an AWS Sagemaker endpoint and I am trying to call it using the Java SDK. However I am getting this error:
Exception in thread "main" java.lang.NoSuchMethodError: com.amazonaws.services.sagemakerruntime.AmazonSageMakerRuntimeClient.beforeClientExecution(Lcom/amazonaws/AmazonWebServiceRequest;)Lcom/amazonaws/AmazonWebServiceRequest;
at com.amazonaws.services.sagemakerruntime.AmazonSageMakerRuntimeClient.invokeEndpoint(AmazonSageMakerRuntimeClient.java:150)
at TestAmazon.main(TestAmazon.java:72)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
When I run the invoke-endpoint on the CLI it works as expected.
The code:
InvokeEndpointRequest invokeEndpointRequest = new InvokeEndpointRequest();
invokeEndpointRequest.setContentType("application/x-image");
ByteBuffer buf = ByteBuffer.wrap(read_buf);
invokeEndpointRequest.setBody(buf);
invokeEndpointRequest.setEndpointName("imageclassification-ep--2018-04-17-19-47-00");
invokeEndpointRequest.setAccept("application/json");
AmazonSageMakerRuntime amazonSageMaker = AmazonSageMakerRuntimeClientBuilder.defaultClient();
InvokeEndpointResult invokeEndpointResult = amazonSageMaker.invokeEndpoint(invokeEndpointRequest);
Any help would be much appreciated.

So I found the issue to my problem it was that I was using:
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>1.11.83</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Instead of the latest version 1.11.313

Anyone reading this thread and wanting to use SageMaker and Java V2 (not V1), see this GitHub location:
https://github.com/aws-doc-sdk-examples/tree/master/javav2/example_code/sagemaker
There are many examples of how to work with this API. For example, the following code example demonstrates how to start a model training job for Amazon SageMaker.
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.sagemaker.SageMakerClient;
import software.amazon.awssdk.services.sagemaker.model.S3DataSource;
import software.amazon.awssdk.services.sagemaker.model.DataSource;
import software.amazon.awssdk.services.sagemaker.model.Channel;
import software.amazon.awssdk.services.sagemaker.model.ResourceConfig;
import software.amazon.awssdk.services.sagemaker.model.TrainingInstanceType;
import software.amazon.awssdk.services.sagemaker.model.CheckpointConfig;
import software.amazon.awssdk.services.sagemaker.model.OutputDataConfig;
import software.amazon.awssdk.services.sagemaker.model.StoppingCondition;
import software.amazon.awssdk.services.sagemaker.model.AlgorithmSpecification;
import software.amazon.awssdk.services.sagemaker.model.TrainingInputMode;
import software.amazon.awssdk.services.sagemaker.model.CreateTrainingJobRequest;
import software.amazon.awssdk.services.sagemaker.model.CreateTrainingJobResponse;
import software.amazon.awssdk.services.sagemaker.model.SageMakerException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
//snippet-end:[sagemaker.java2.train_job.import]
/**
* To setup the model data and other requirements to make this Java V2 example work, follow this AWS tutorial prior to running this Java code example.
* https://aws.amazon.com/getting-started/hands-on/build-train-deploy-machine-learning-model-sagemaker/
*/
public class CreateTrainingJob {
public static void main(String[] args) {
final String USAGE = "\n" +
"Usage:\n" +
" CreateTrainingJob <s3UriData><s3Uri><trainingJobName><roleArn><s3OutputPath><channelName><trainingImage>\n\n" +
"Where:\n" +
" s3UriData - The location where the training data is located (ie, s3://trainbucket/train.csv).\n\n" +
" s3Uri - The S3 path where you want Amazon SageMaker to store checkpoints (ie, s3://trainbucket).\n\n" +
" trainingJobName - The name of the training job. \n\n" +
" roleArn - The Amazon Resource Name (ARN) of the IAM role that Amazon SageMaker uses.\n\n" +
" s3OutputPath - The output path located in a S3 bucket (i.e., s3://trainbucket/sagemaker).\n\n" +
" channelName - The channel name \n\n" +
" trainingImage - The training image.";
if (args.length < 7) {
System.out.println(USAGE);
System.exit(1);
}
/* Read the name from command args */
String s3UriData = args[0];
String s3Uri = args[1];
String trainingJobName = args[2];
String roleArn = args[3];
String s3OutputPath = args[4];
String channelName = args[5];
String trainingImage = args[6];
Region region = Region.US_WEST_2;
SageMakerClient sageMakerClient = SageMakerClient.builder()
.region(region)
.build();
trainJob(sageMakerClient, s3UriData, s3Uri, trainingJobName, roleArn, s3OutputPath, channelName, trainingImage);
}
//snippet-start:[sagemaker.java2.train_job.main]
public static void trainJob(SageMakerClient sageMakerClient,
String s3UriData,
String s3Uri,
String trainingJobName,
String roleArn,
String s3OutputPath,
String channelName,
String trainingImage) {
try {
S3DataSource s3DataSource = S3DataSource.builder()
.s3Uri(s3UriData)
.s3DataType("S3Prefix")
.s3DataDistributionType("FullyReplicated")
.build();
DataSource dataSource = DataSource.builder()
.s3DataSource(s3DataSource)
.build();
Channel channel = Channel.builder()
.channelName(channelName)
.contentType("csv")
.dataSource(dataSource)
.build();
// Build a LIST of CHannels
List<Channel> myChannel = new ArrayList();
myChannel.add(channel);
ResourceConfig resourceConfig = ResourceConfig.builder()
.instanceType(TrainingInstanceType.ML_M5_2_XLARGE) // ml.c5.2xlarge
.instanceCount(10)
.volumeSizeInGB(1)
.build();
CheckpointConfig checkpointConfig = CheckpointConfig.builder()
.s3Uri(s3Uri)
.build();
OutputDataConfig outputDataConfig = OutputDataConfig.builder()
.s3OutputPath(s3OutputPath)
.build();
StoppingCondition stoppingCondition = StoppingCondition.builder()
.maxRuntimeInSeconds(1200)
.build();
AlgorithmSpecification algorithmSpecification = AlgorithmSpecification.builder()
.trainingImage(trainingImage)
.trainingInputMode(TrainingInputMode.FILE)
.build();
// Set hyper parameters
Map<String,String> hyperParameters = new HashMap<String, String>();
hyperParameters.put("num_round", "100");
hyperParameters.put("eta", "0.2");
hyperParameters.put("gamma", "4");
hyperParameters.put("max_depth", "5");
hyperParameters.put("min_child_weight", "6");
hyperParameters.put("objective", "binary:logistic");
hyperParameters.put("silent", "0");
hyperParameters.put("subsample", "0.8");
CreateTrainingJobRequest trainingJobRequest = CreateTrainingJobRequest.builder()
.trainingJobName(trainingJobName)
.algorithmSpecification(algorithmSpecification)
.roleArn(roleArn)
.resourceConfig(resourceConfig)
.checkpointConfig(checkpointConfig)
.inputDataConfig(myChannel)
.outputDataConfig(outputDataConfig)
.stoppingCondition(stoppingCondition)
.hyperParameters(hyperParameters)
.build();
CreateTrainingJobResponse jobResponse = sageMakerClient.createTrainingJob(trainingJobRequest);
System.out.println("The Amazon Resource Name (ARN) of the training job is "+ jobResponse.trainingJobArn());
} catch (SageMakerException e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
}
//snippet-end:[sagemaker.java2.train_job.main]
}

401 Unauthorized error when using a server account to impersonate a user in order to access their Google Drive

I am writing a back-end process in Java that will impersonate a user and add/remove documents on their Google Drive.
The server account seems to authenticate correctly but when I try to impersonate a user, I get a 401 Unauthorized error. Please see below for details.
Configuration
I have configured the server account as follows:
Created a project under Google APIs and enabled Google Drive API
Created a service account called anothertest#yyyyyyyyy.iam.gserviceaccount.com, set the role as Service Account Actor and given it domain-wide delegation. It has Client ID 110xxxxxxxxx342
I have download the P12 key file
I have configured the domain using the Manage API client access screen to authorize 110xxxxxxxxx342 to have the scope: https://www.googleapis.com/auth/drive.
Google Support have looked at my configuration and have given it the thumbs up.
My code then looks as follows:
package com.dcm.sharingdocuments;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.Map.Entry;
import com.google.api.client.auth.oauth2.TokenErrorResponse;
import com.google.api.client.auth.oauth2.TokenResponseException;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.FileList;
public class SharingDocumentsTest3 {
private static final String SERVICE_ACCOUNT_EMAIL = " anothertest#yyyyyyyyy.iam.gserviceaccount.com";
public static Drive getDriveService(String userEmail) throws Exception {
File keyFile = new File("E:\\Projects\\Workspace\\Sharing Documents\\authentication\\AnotherTestKeyFile.p12");
HttpTransport httpTransport = new NetHttpTransport();
JacksonFactory jsonFactory = new JacksonFactory();
List<String> SCOPES = Arrays.asList(DriveScopes.DRIVE_METADATA_READONLY);
GoogleCredential credential = null;
if (userEmail == null) {
credential = new GoogleCredential.Builder().setTransport(httpTransport).setJsonFactory(jsonFactory)
.setServiceAccountId(SERVICE_ACCOUNT_EMAIL).setServiceAccountScopes(SCOPES)
.setServiceAccountPrivateKeyFromP12File(keyFile).build();
credential.refreshToken();
} else {
credential = new GoogleCredential.Builder().setTransport(httpTransport).setJsonFactory(jsonFactory)
.setServiceAccountId(SERVICE_ACCOUNT_EMAIL).setServiceAccountScopes(SCOPES)
.setServiceAccountPrivateKeyFromP12File(keyFile).setServiceAccountUser(userEmail).build();
credential.refreshToken();
}
Drive service = new Drive.Builder(httpTransport, jsonFactory, null).setHttpRequestInitializer(credential)
.build();
return service;
}
public static void main(String[] args) {
SharingDocumentsTest3 sdt3 = new SharingDocumentsTest3();
sdt3.execute();
}
private void execute() {
try {
Drive service = getDriveService(null);
Drive services = getDriveService("anzzzze#zzzzz.me.uk");
displayFiles(services);
} catch (Exception e) {
e.printStackTrace();
}
}
private void displayFiles(Drive service) throws Exception {
FileList result = service.files().list().setPageSize(10).execute();
List<com.google.api.services.drive.model.File> files = result.getFiles();
if (files == null || files.size() == 0) {
System.out.println("No files found.");
} else {
System.out.println("Files:");
for (com.google.api.services.drive.model.File file : files) {
Set<Entry<String, Object>> entries = file.entrySet();
Iterator<Entry<String, Object>> it = entries.iterator();
while (it.hasNext()) {
Entry<String, Object> entry = it.next();
String key = entry.getKey();
Object value = entry.getValue();
if (value instanceof String) {
System.out.println("\tKey = " + key + ", Value = " + (String) value);
} else {
System.out.println("\tKey = " + key + ", Value = " + value.toString());
}
}
System.out.printf("%s (%s)\n", file.getName(), file.getId());
}
}
}
}
When I run the code as is above, I get the error:
Mar 29, 2017 9:55:27 AM com.google.api.client.googleapis.services.AbstractGoogleClient <init>
WARNING: Application name is not set. Call Builder#setApplicationName.
com.google.api.client.auth.oauth2.TokenResponseException: 401 Unauthorized
at com.google.api.client.auth.oauth2.TokenResponseException.from(TokenResponseException.java:105)
at com.google.api.client.auth.oauth2.TokenRequest.executeUnparsed(TokenRequest.java:287)
at com.google.api.client.auth.oauth2.TokenRequest.execute(TokenRequest.java:307)
at com.google.api.client.googleapis.auth.oauth2.GoogleCredential.executeRefreshToken(GoogleCredential.java:384)
at com.google.api.client.auth.oauth2.Credential.refreshToken(Credential.java:489)
at com.dcm.sharingdocuments.SharingDocumentsTest3.getDriveService(SharingDocumentsTest3.java:50)
at com.dcm.sharingdocuments.SharingDocumentsTest3.execute(SharingDocumentsTest3.java:75)
at com.dcm.sharingdocuments.SharingDocumentsTest3.main(SharingDocumentsTest3.java:65)
So the code fails at credential.refreshToken() when I set the setServiceAccountUser. It appears to have successfully refreshed the token when I do not. I have tried various combinations of this code – e.g. commented out the refreshToken() lines, commented out the getDriveService(null) line – but whenever I try to use/refresh the credential obtained for the impersonated user I get the 401 Unauthorized error.
If I modify the code so that the drive obtained by getDriveService(null) is passed to DisplayFiles(...), then I get one file listed called “Getting Started”. So it seems that the service account authorization is working and Google have added their default file to the Drive for the server account.
I am using google-*1.22.0.jar files and Java 1.8 to the run the above code
The problem I think is in the way I have configured the domain or the way I am trying to impersonate the user but my code looks as many examples on the web do and Google Support appear to say that I have configured the domain correctly.
Anything you can suggest as a resolution or next step would be much appreciated!

I have been stuck on this problem for a long time and I finally found my problem. There is definitely a bug in the "Manage API client access" Admin console...
You must put the "Client ID" (e.g. 110xxxxxxxxx342) for the client name and NOT the "Service Account ID" (the one that looks like an email). Now, their documentation is correct, and they do say in the documentation to use the Client ID, I have to give them that.
So here is the bug. When I arrived to the Manage API screen, I saw "Example: www.example.com". I typed in the Service Account ID there, thinking that the email address format matched "www.example.com" better than the Client ID. I pressed "Authorize", and the entry had clearly been accepted and everything was good. The result looks like this:
It even generated the Client ID from the Service ID! Great! Except my code gets a 401 error every time I try to connect with setServiceUser().
If I return to the Manage API Client Access console and if I remove the previous entry and perform the same actions except use the Client ID instead of the Service ID. The result is this:
Exactly the same, but now I don't get a 401 error. So there is NO WAY to look at the console, and know if you have it successfully configured it or not. I tested this 3 times to make sure I wasn't losing my mind...

ETrade Java API issue - previewEquityOrder and previewOptionOrder throw an ETWSException

I am working with the ETrade Java API. I was able to use most of the functions but I am having trouble with the previewEquityOrder and the previewOptionOrder functions. Here are the error messages/ exceptions I get when I call these functions:
URL : https://etwssandbox.etrade.com/order/sandbox/rest/previewequityorder
? Java exception occurred:
com.etrade.etws.sdk.common.ETWSException
at com.etrade.etws.sdk.common.ETWSUtil.constructException(ETWSUtil.java:9)
at com.etrade.etws.sdk.core.ConnectionUtils.invoke(ConnectionUtils.java:90)
at com.etrade.etws.sdk.core.ConnectionUtils.invoke(ConnectionUtils.java:32)
at com.etrade.etws.sdk.client.OrderClient.previewEquityOrder(OrderClient.java:145)
For the previewOptionOrder:
URL : https://etwssandbox.etrade.com/order/sandbox/rest/previewoptionorder
? Java exception occurred:
com.etrade.etws.sdk.common.ETWSException
at com.etrade.etws.sdk.common.ETWSUtil.constructException(ETWSUtil.java:9)
at com.etrade.etws.sdk.core.ConnectionUtils.invoke(ConnectionUtils.java:90)
at com.etrade.etws.sdk.core.ConnectionUtils.invoke(ConnectionUtils.java:32)
at com.etrade.etws.sdk.client.OrderClient.previewOptionOrder(OrderClient.java:167)
The following Java code can reproduce the problem. You can compile this code on a Mac using the following command. On windows machine, replace the " : " with " ; " as the separator.
javac -classpath "./commons-codec-1.3.jar:./commons-httpclient-3.1.jar:./commons-httpclient-contrib-ssl-3.1.jar:./commons-lang-2.4-javadoc.jar:./commons-lang-2.4-sources.jar:./commons-lang-2.4.jar:./commons-logging-api.jar:./commons-logging.jar:./etws-accounts-sdk-1.0.jar:./etws-common-connections-1.0.jar:./etws-market-sdk-1.0.jar:./etws-oauth-sdk-1.0.jar:./etws-order-sdk-1.0.jar:./log4j-1.2.15.jar:./xstream-1.3.1.jar:" test.java
You can run the compiled class from command line using the following command:
java -classpath "./commons-codec-1.3.jar:./commons-httpclient-3.1.jar:./commons-httpclient-contrib-ssl-3.1.jar:./commons-lang-2.4-javadoc.jar:./commons-lang-2.4-sources.jar:./commons-lang-2.4.jar:./commons-logging-api.jar:./commons-logging.jar:./etws-accounts-sdk-1.0.jar:./etws-common-connections-1.0.jar:./etws-market-sdk-1.0.jar:./etws-oauth-sdk-1.0.jar:./etws-order-sdk-1.0.jar:./log4j-1.2.15.jar:./xstream-1.3.1.jar:" test <consumer_key> <consumer_secret>
You will need to pass in the ETrade consumer key and consumer secret as command line arguments to run this.
Notice that the authentication part works which is verified by getting the accounts list.
import com.etrade.etws.account.Account;
import com.etrade.etws.account.AccountListResponse;
import com.etrade.etws.oauth.sdk.client.IOAuthClient;
import com.etrade.etws.oauth.sdk.client.OAuthClientImpl;
import com.etrade.etws.oauth.sdk.common.Token;
import com.etrade.etws.sdk.client.ClientRequest;
import com.etrade.etws.sdk.client.Environment;
import com.etrade.etws.sdk.common.ETWSException;
import com.etrade.etws.sdk.client.AccountsClient;
import com.etrade.*;
import com.etrade.etws.order.PreviewEquityOrder;
import com.etrade.etws.order.PreviewEquityOrderResponse;
import com.etrade.etws.order.EquityOrderRequest;
import com.etrade.etws.order.EquityOrderTerm;
import com.etrade.etws.order.EquityOrderAction;
import com.etrade.etws.order.MarketSession;
import com.etrade.etws.order.EquityPriceType;
import com.etrade.etws.order.EquityOrderRoutingDestination;
import com.etrade.etws.sdk.client.OrderClient;
import java.math.BigInteger;
import java.awt.Desktop;
import java.net.URI;
import java.*;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
public class test
{
public static void main(String[] args) throws IOException, ETWSException
{
//Variables
if(args.length<2){
System.out.println("Class test needs two input argument as follows:");
System.out.println("test <consumer_key> <consumer_secret>");
return;
}
String oauth_consumer_key = args[0]; // Your consumer key
String oauth_consumer_secret = args[1]; // Your consumer secret
String oauth_request_token = null; // Request token
String oauth_request_token_secret = null; // Request token secret
String oauth_verify_code = null;
String oauth_access_token = null;
String oauth_access_token_secret = null;
ClientRequest request = new ClientRequest();
System.out.println("HERE");
IOAuthClient client = OAuthClientImpl.getInstance(); // Instantiate IOAUthClient
// Instantiate ClientRequest
request.setEnv(Environment.SANDBOX); // Use sandbox environment
request.setConsumerKey(oauth_consumer_key); //Set consumer key
request.setConsumerSecret(oauth_consumer_secret);
Token token = client.getRequestToken(request); // Get request-token object
oauth_request_token = token.getToken(); // Get token string
oauth_request_token_secret = token.getSecret(); // Get token secret
request.setToken(oauth_request_token);
request.setTokenSecret(oauth_request_token_secret);
String authorizeURL = null;
authorizeURL = client.getAuthorizeUrl(request);
System.out.println(authorizeURL);
System.out.println("Copy the URL into your browser. Get the verification code and type here");
oauth_verify_code = get_verification_code();
//oauth_verify_code = Verification(client,request);
request.setVerifierCode(oauth_verify_code);
token = client.getAccessToken(request);
oauth_access_token = token.getToken();
oauth_access_token_secret = token.getSecret();
request.setToken(oauth_access_token);
request.setTokenSecret(oauth_access_token_secret);
// Get Account List
AccountsClient account_client = new AccountsClient(request);
AccountListResponse response = account_client.getAccountList();
List<Account> alist = response.getResponse();
Iterator<Account> al = alist.iterator();
while (al.hasNext()) {
Account a = al.next();
System.out.println("===================");
System.out.println("Account: " + a.getAccountId());
System.out.println("===================");
}
// Preview Equity Order
OrderClient order_client = new OrderClient(request);
PreviewEquityOrder orderRequest = new PreviewEquityOrder();
EquityOrderRequest eor = new EquityOrderRequest();
eor.setAccountId("83405188"); // sample values
eor.setSymbol("AAPL");
eor.setAllOrNone("FALSE");
eor.setClientOrderId("asdf1234");
eor.setOrderTerm(EquityOrderTerm.GOOD_FOR_DAY);
eor.setOrderAction(EquityOrderAction.BUY);
eor.setMarketSession(MarketSession.REGULAR);
eor.setPriceType(EquityPriceType.MARKET);
eor.setQuantity(new BigInteger("100"));
eor.setRoutingDestination(EquityOrderRoutingDestination.AUTO.value());
eor.setReserveOrder("TRUE");
orderRequest.setEquityOrderRequest(eor);
PreviewEquityOrderResponse order_response = order_client.previewEquityOrder(orderRequest);
}
public static String get_verification_code() {
try{
BufferedReader br =
new BufferedReader(new InputStreamReader(System.in));
String input;
input=br.readLine();
return input;
}catch(IOException io){
io.printStackTrace();
return "";
}
}
}
I posted this on the ETrade community forum but that forum is not very active. I also sent a request to ETrade and haven't gotten a reply yet. If I get a solution from them, I will come back and post it here. In the mean time any help is greatly appreciated.

After debugging my above code, I figured out that the problem is that I was setting the ReserveOrder to TRUE but I wasn't providing the required ReserveOrderQuantity. I got the above code from the Java code snippet in the ETrade Developer Platform Guide. This is clearly a bug in their documentation.