Develop Reference

How to read from an Amazon S3 Bucket and call AWS services

I am able to call AWS Textract to read an image from my local path. How can I integrate this textract code to read the image uploaded onto a created S3 bucket with the S3 bucket codes.
Working Textract Code to textract images from local path
package aws.cloud.work;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.io.InputStream;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.textract.AmazonTextract;
import com.amazonaws.services.textract.AmazonTextractClientBuilder;
import com.amazonaws.services.textract.model.DetectDocumentTextRequest;
import com.amazonaws.services.textract.model.DetectDocumentTextResult;
import com.amazonaws.services.textract.model.Document;
import com.amazonaws.util.IOUtils;
public class TextractDemo {
static AmazonTextractClientBuilder clientBuilder = AmazonTextractClientBuilder.standard()
.withRegion(Regions.US_EAST_1);
private static FileWriter file;
public static void main(String[] args) throws IOException {
//AWS Credentials to access AWS Textract services
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials("Access Key", "Secret key")));
//Set the path of the image to be textract. Can be configured to use from S3
String document="C:\\Users\\image-local-path\\sampleTT.jpg";
ByteBuffer imageBytes;
//Code to use AWS Textract services
try (InputStream inputStream = new FileInputStream(new File(document))) {
imageBytes = ByteBuffer.wrap(IOUtils.toByteArray(inputStream));
}
AmazonTextract client = clientBuilder.build();
DetectDocumentTextRequest request = new DetectDocumentTextRequest()
.withDocument(new Document().withBytes(imageBytes));
/*
* DetectDocumentTextResult result = client.detectDocumentText(request);
* System.out.println(result); result.getBlocks().forEach(block ->{
* if(block.getBlockType().equals("LINE")) System.out.println("text is "+
* block.getText() + " confidence is "+ block.getConfidence());
*/
//
DetectDocumentTextResult result = client.detectDocumentText(request);
System.out.println(result);
JSONObject obj = new JSONObject();
result.getBlocks().forEach(block -> {
if (block.getBlockType().equals("LINE"))
System.out.println("text is " + block.getText() + " confidence is " + block.getConfidence());
JSONArray fields = new JSONArray();
fields.add(block.getText() + " , " + block.getConfidence());
obj.put(block.getText(), fields);
});
//To import the results into JSON file and output the console output as sample.txt
try {
file = new FileWriter("/Users/output-path/sample.txt");
file.write(obj.toJSONString());
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
file.flush();
file.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
This is an example of the console out where the "text" and corresponding "confidence scores" are returned
S3 bucket code integration I managed to find from the docs:
String document = "sampleTT.jpg";
String bucket = "textract-images";
AmazonS3 s3client = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(
new EndpointConfiguration("https://s3.amazonaws.com","us-east-1"))
.build();
// Get the document from S3
com.amazonaws.services.s3.model.S3Object s3object = s3client.getObject(bucket, document);
S3ObjectInputStream inputStream = s3object.getObjectContent();
BufferedImage image = ImageIO.read(inputStream);
(Edited) - Thanks #smac2020, I currently have a working Rekognition Code that reads from my AWS console S3 bucket and runs the Rekognition services that I am referencing to. However, I am unable to modify and merge it with
the Textract source code
package com.amazonaws.samples;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.rekognition.AmazonRekognition;
import com.amazonaws.services.rekognition.AmazonRekognitionClientBuilder;
import com.amazonaws.services.rekognition.model.AmazonRekognitionException;
import com.amazonaws.services.rekognition.model.DetectLabelsRequest;
import com.amazonaws.services.rekognition.model.DetectLabelsResult;
import com.amazonaws.services.rekognition.model.Image;
import com.amazonaws.services.rekognition.model.Label;
import com.amazonaws.services.rekognition.model.S3Object;
import java.util.List;
public class DetectLabels {
public static void main(String[] args) throws Exception {
String photo = "sampleTT.jpg";
String bucket = "Textract-bucket";
// AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withRegion("ap-southeast-1").build();
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider (new BasicAWSCredentials("Access Key", "Secret Key"));
AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withCredentials(credentialsProvider).withRegion("ap-southeast-1").build();
DetectLabelsRequest request = new DetectLabelsRequest()
.withImage(new Image()
.withS3Object(new S3Object()
.withName(photo).withBucket(bucket)))
.withMaxLabels(10)
.withMinConfidence(75F);
try {
DetectLabelsResult result = rekognitionClient.detectLabels(request);
List <Label> labels = result.getLabels();
System.out.println("Detected labels for " + photo);
for (Label label: labels) {
System.out.println(label.getName() + ": " + label.getConfidence().toString());
}
} catch(AmazonRekognitionException e) {
e.printStackTrace();
}
}
}

Looks like you are trying to read an Amazon S3 object from a Spring boot app and then pass that byte array to DetectDocumentTextRequest.
There is a tutorial that shows a very similar use case where a Spring BOOT app reads the bytes from an Amazon S3 object and passes it to the Amazon Rekognition service (instead of Textract).
The Java code is:
// Get the byte[] from this AWS S3 object.
public byte[] getObjectBytes (String bucketName, String keyName) {
s3 = getClient();
try {
GetObjectRequest objectRequest = GetObjectRequest
.builder()
.key(keyName)
.bucket(bucketName)
.build();
ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
byte[] data = objectBytes.asByteArray();
return data;
} catch (S3Exception e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
return null;
}
See this AWS development article to see how to build a Spring BOOT app that has this functionality.
Creating an example AWS photo analyzer application using the AWS SDK for Java
This example uses the AWS SDK For Java V2. If you are not familiar with working with the latest SDK version, I recommend that you start here:
Get started with the AWS SDK for Java 2.x

Error while converting image file from AWS S3 Object to Base64

I am using the below code for parsing S3 object into Base64 string.
InputStream is = s3Object.getObjectContent().getDelegateStream();
byte[] bytes = IOUtils.toByteArray(is);
String base64Data = Base64.getEncoder().encodeToString(bytes); //Java Util lib.
This code works fine for text and pdf file. But while converting image files I am getting error MIME type not supported.
I also tried to use AWS and Apache Commons Base64 lib. still it is not working.
Base64.encodeAsString(bytes) //AWS
Base64.encodeBase64String(bytes) //Apache Commons

AWS Java SDK 2
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import java.io.IOException;
public class S3ReadImg {
public static void main(String[] args) throws IOException {
S3Client s3 = S3Client.builder()
.region(Region.EU_WEST_1) //
.build(); // Might change for you
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket("my_bucket_name")
.key("four.bmp") // also tested for image types png, jpg, gif
.build();
byte[] objectResponse = s3.getObject(getObjectRequest).readAllBytes();
String encodedfile = new String(java.util.Base64.getEncoder().encode(objectResponse), java.nio.charset.StandardCharsets.UTF_8);
}
}

GetObjectRequest request = GetObjectRequest.builder()
.bucket("my_bucket_name")
.key("four.bmp")
.build();
try {
byte[] objectResponse = s3Client.getObject(request, ResponseTransformer.toBytes())
.asByteArray();
return Base64.getEncoder().encodeToString(objectResponse);
} catch (NoSuchKeyException e) {
log.error(e.getMessage());
throw e;
}

Found out issue was not in the code. API Gateway was not configured properly.
I referred this link
https://medium.com/swlh/upload-binary-files-to-s3-using-aws-api-gateway-with-aws-lambda-2b4ba8c70b8e

AmazonS3Exception: request signature error only in linux remote server?

I wrote a Java program to list all the buckets and to upload a file in S3 compatible Object storage service.
The program is working fine in Windows my local machine but when I (after changing the path of the file to be uploaded of course ) transfer the runnable jar in the remote linux server and execute it I'm getting the following error-
> Exception in thread "main"
> com.amazonaws.services.s3.model.AmazonS3Exception: The request
> signature we calculated does not match the signature you provided.
> Check your AWS Secret Access Key and signing method. For more
> information, see REST Authentication and SOAP Authentication for
> details. (Service: Amazon S3; Status Code: 403; Error Code:
> SignatureDoesNotMatch; Request ID:
> 4e271b5b-d7f5-42b3-a4ad-886988bcb785; S3 Extended Request ID: null),
> S3 Extended Request ID: null
The issue seems to be in the 2nd half of the program as the list of buckets are returning in linux env. as well but during the file upload it is throwing error.
import java.io.File;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.model.Bucket;
/**
* List your Amazon S3 buckets.
*/
public class ListBuckets
{
private static void listObjects(AmazonS3 s3) {
List<Bucket> buckets = s3.listBuckets();
System.out.println("Your Amazon S3 buckets are:");
for (Bucket b : buckets) {
System.out.println("* " + b.getName());
}
}
private static void putObject(AmazonS3 s3, String bucketName, String objectName, String pathName) throws Exception
{
s3.putObject(bucketName, objectName, new File(pathName));
}
private static void time(String t) {
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
Date date = new Date();
System.out.println(t+"-->"+dateFormat.format(date));
}
public static void main(String[] args) throws Exception
{
final String accessKey = "XXXXXXXXXXXXXX";
final String secretKey = "XXXXXXXXXXXXXXXXXXXXXXXX";
BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
#SuppressWarnings("deprecation")
final AmazonS3 s3 = new AmazonS3Client(credentials);
S3ClientOptions opts = new S3ClientOptions().withPathStyleAccess(true);
s3.setS3ClientOptions(opts);
s3.setEndpoint("https://XXXXXX.com");
ListBuckets.time("startTime");
ListBuckets.listObjects(s3);
//String pathName = "C:\\Users\\XXXXXX\\Documents\\New folder\\New Text Document - Copy.txt";
String pathName = "/home/abcd/XXXXX/objectStorage/CHANGELOG.mdown";
ListBuckets.putObject(s3, "snap-shot/sample-aws-ex", pathName, pathName);
ListBuckets.time("end time");
}
}`

Unbelievable!
You know what the issue was in linux? The object name and path name are two different things.
putObject(AmazonS3 s3, String bucketName, String objectName, String pathName)
where the pathName is the path of your file i.e.
String pathName = "/home/abc/xxxxx/objectStorage/errorlog.txt";
Notice it starts with forward slash, whereas object name should not start with / i.e.
String objectName = "home/abc/xxxxxx/objectStorage/errorlog.txt";
I wish the exception thrown would have given better clarity on what was wrong. The exception thrown only made me deviate from the root cause.

How to upload a file to AWS S3 Bucket?

I am creating a simple application where I want to upload a file to my AWS S3 bucket. Here is my code:
import java.io.File;
import java.io.IOException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.fasterxml.jackson.*;
public class UploadFileInBucket {
public static void main(String[] args) throws IOException {
String clientRegion = "<myRegion>";
String bucketName = "<myBucketName>";
String stringObjKeyName = "testobject";
String fileObjKeyName = "testfileobject";
String fileName = "D:\\Attachments\\LICENSE";
try {
BasicAWSCredentials awsCreds = new BasicAWSCredentials("<myAccessKey>", "<mySecretKey>");
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withRegion(clientRegion)
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.build();
// Upload a text string as a new object.
s3Client.putObject(bucketName, stringObjKeyName, "Uploaded String Object");
// Upload a file as a new object with ContentType and title specified.
PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("plain/text");
metadata.addUserMetadata("x-amz-meta-title", "someTitle");
request.setMetadata(metadata);
s3Client.putObject(request);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
}
catch(SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
}
}
I am unable to upload a file and getting an error as:
Exception in thread "main" java.lang.NoSuchFieldError:
ALLOW_FINAL_FIELDS_AS_MUTATORS
at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:52)
at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:64)
at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:52)
at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:105)
at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:249)
at com.amazonaws.client.builder.AwsClientBuilder.withRegion(AwsClientBuilder.java:238)
at UploadFileInBucket.main(UploadFileInBucket.java:28)
I have added required AWS bucket credentials, permissions and dependencies to execute this code.
What changes I should made in the code to get my file uploaded to desired bucket?

It looks as though you either have the wrong version of the Jackson libraries or are somehow linking with multiple versions of them.
The AWS for Java SDK distribution contains a third-party/lib directory which contains all of the (correct versions of) the libraries that version of the SDK should be built with. Depending on which features of the SDK you are using you may not need all of them, but those are the specific 3rd party libraries you should be using.

You need to add Jackson to your classpath. Its classes are missing.
I don't know which version you need, but you can download them from their gitpage: https://github.com/FasterXML/jackson/

How to read uploaded file that triggered lambda execution directly?

My current AWS setup is a lambda function that is being triggered whenever I put an object into a S3 bucket. I implemented the lambda's handler function in Java. What I want to do is simply accessing the file that was uploaded and triggered the execution of the lambda function. E.g., if I upload sample.json to the bucket, I want to access the contents of this file in my handler function.
I know I can do something like this:
public Void handleRequest(S3Event input, Context context) {
for (S3EventNotificationRecord record : input.getRecords()) {
String key = record.getS3().getObject().getKey();
String bucket = record.getS3().getBucket().getName();
AmazonS3 s3Client = new AmazonS3Client(credentials);
try {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));
InputStream input = s3Object.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
while (true) {
String line = reader.readLine();
if (line == null) break;
// Do something with line...
}
// ...
The problem is that I am not allowed to use access keys. Thus, I cannot create an s3Client to download the file with. In other words, I have to get the object from the argument that my handler method takes, i.e., S3Event input. How would I do that?

If your Lambda function is configured with an appropriate IAM role (that allows s3:GetObject of the relevant S3 object), then you don't need to explicitly provide credentials in your code.
Here's sample Java code to get a object in response to an object uploaded Lambda event:
public class S3GetTextBody implements RequestHandler<S3Event, String> {
public String handleRequest(S3Event s3event, Context context) {
try {
S3EventNotificationRecord record = s3event.getRecords().get(0);
// Retrieve the bucket & key for the uploaded S3 object that
// caused this Lambda function to be triggered
String bkt = record.getS3().getBucket().getName();
String key = record.getS3().getObject().getKey().replace('+', ' ');
key = URLDecoder.decode(key, "UTF-8");
// Read the source file as text
AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();
String body = s3Client.getObjectAsString(bkt, key);
System.out.println("Body: " + body);
return "ok";
} catch (Exception e) {
System.err.println("Exception: " + e);
return "error";
}
}
}
Imports:
import java.net.URLDecoder;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.S3Event;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord;