Develop Reference

How to read from an Amazon S3 Bucket and call AWS services

I am able to call AWS Textract to read an image from my local path. How can I integrate this textract code to read the image uploaded onto a created S3 bucket with the S3 bucket codes.
Working Textract Code to textract images from local path
package aws.cloud.work;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.io.InputStream;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.textract.AmazonTextract;
import com.amazonaws.services.textract.AmazonTextractClientBuilder;
import com.amazonaws.services.textract.model.DetectDocumentTextRequest;
import com.amazonaws.services.textract.model.DetectDocumentTextResult;
import com.amazonaws.services.textract.model.Document;
import com.amazonaws.util.IOUtils;
public class TextractDemo {
static AmazonTextractClientBuilder clientBuilder = AmazonTextractClientBuilder.standard()
.withRegion(Regions.US_EAST_1);
private static FileWriter file;
public static void main(String[] args) throws IOException {
//AWS Credentials to access AWS Textract services
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials("Access Key", "Secret key")));
//Set the path of the image to be textract. Can be configured to use from S3
String document="C:\\Users\\image-local-path\\sampleTT.jpg";
ByteBuffer imageBytes;
//Code to use AWS Textract services
try (InputStream inputStream = new FileInputStream(new File(document))) {
imageBytes = ByteBuffer.wrap(IOUtils.toByteArray(inputStream));
}
AmazonTextract client = clientBuilder.build();
DetectDocumentTextRequest request = new DetectDocumentTextRequest()
.withDocument(new Document().withBytes(imageBytes));
/*
* DetectDocumentTextResult result = client.detectDocumentText(request);
* System.out.println(result); result.getBlocks().forEach(block ->{
* if(block.getBlockType().equals("LINE")) System.out.println("text is "+
* block.getText() + " confidence is "+ block.getConfidence());
*/
//
DetectDocumentTextResult result = client.detectDocumentText(request);
System.out.println(result);
JSONObject obj = new JSONObject();
result.getBlocks().forEach(block -> {
if (block.getBlockType().equals("LINE"))
System.out.println("text is " + block.getText() + " confidence is " + block.getConfidence());
JSONArray fields = new JSONArray();
fields.add(block.getText() + " , " + block.getConfidence());
obj.put(block.getText(), fields);
});
//To import the results into JSON file and output the console output as sample.txt
try {
file = new FileWriter("/Users/output-path/sample.txt");
file.write(obj.toJSONString());
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
file.flush();
file.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
This is an example of the console out where the "text" and corresponding "confidence scores" are returned
S3 bucket code integration I managed to find from the docs:
String document = "sampleTT.jpg";
String bucket = "textract-images";
AmazonS3 s3client = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(
new EndpointConfiguration("https://s3.amazonaws.com","us-east-1"))
.build();
// Get the document from S3
com.amazonaws.services.s3.model.S3Object s3object = s3client.getObject(bucket, document);
S3ObjectInputStream inputStream = s3object.getObjectContent();
BufferedImage image = ImageIO.read(inputStream);
(Edited) - Thanks #smac2020, I currently have a working Rekognition Code that reads from my AWS console S3 bucket and runs the Rekognition services that I am referencing to. However, I am unable to modify and merge it with
the Textract source code
package com.amazonaws.samples;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.rekognition.AmazonRekognition;
import com.amazonaws.services.rekognition.AmazonRekognitionClientBuilder;
import com.amazonaws.services.rekognition.model.AmazonRekognitionException;
import com.amazonaws.services.rekognition.model.DetectLabelsRequest;
import com.amazonaws.services.rekognition.model.DetectLabelsResult;
import com.amazonaws.services.rekognition.model.Image;
import com.amazonaws.services.rekognition.model.Label;
import com.amazonaws.services.rekognition.model.S3Object;
import java.util.List;
public class DetectLabels {
public static void main(String[] args) throws Exception {
String photo = "sampleTT.jpg";
String bucket = "Textract-bucket";
// AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withRegion("ap-southeast-1").build();
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider (new BasicAWSCredentials("Access Key", "Secret Key"));
AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withCredentials(credentialsProvider).withRegion("ap-southeast-1").build();
DetectLabelsRequest request = new DetectLabelsRequest()
.withImage(new Image()
.withS3Object(new S3Object()
.withName(photo).withBucket(bucket)))
.withMaxLabels(10)
.withMinConfidence(75F);
try {
DetectLabelsResult result = rekognitionClient.detectLabels(request);
List <Label> labels = result.getLabels();
System.out.println("Detected labels for " + photo);
for (Label label: labels) {
System.out.println(label.getName() + ": " + label.getConfidence().toString());
}
} catch(AmazonRekognitionException e) {
e.printStackTrace();
}
}
}

Looks like you are trying to read an Amazon S3 object from a Spring boot app and then pass that byte array to DetectDocumentTextRequest.
There is a tutorial that shows a very similar use case where a Spring BOOT app reads the bytes from an Amazon S3 object and passes it to the Amazon Rekognition service (instead of Textract).
The Java code is:
// Get the byte[] from this AWS S3 object.
public byte[] getObjectBytes (String bucketName, String keyName) {
s3 = getClient();
try {
GetObjectRequest objectRequest = GetObjectRequest
.builder()
.key(keyName)
.bucket(bucketName)
.build();
ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
byte[] data = objectBytes.asByteArray();
return data;
} catch (S3Exception e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
return null;
}
See this AWS development article to see how to build a Spring BOOT app that has this functionality.
Creating an example AWS photo analyzer application using the AWS SDK for Java
This example uses the AWS SDK For Java V2. If you are not familiar with working with the latest SDK version, I recommend that you start here:
Get started with the AWS SDK for Java 2.x

Error while converting image file from AWS S3 Object to Base64

I am using the below code for parsing S3 object into Base64 string.
InputStream is = s3Object.getObjectContent().getDelegateStream();
byte[] bytes = IOUtils.toByteArray(is);
String base64Data = Base64.getEncoder().encodeToString(bytes); //Java Util lib.
This code works fine for text and pdf file. But while converting image files I am getting error MIME type not supported.
I also tried to use AWS and Apache Commons Base64 lib. still it is not working.
Base64.encodeAsString(bytes) //AWS
Base64.encodeBase64String(bytes) //Apache Commons

AWS Java SDK 2
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import java.io.IOException;
public class S3ReadImg {
public static void main(String[] args) throws IOException {
S3Client s3 = S3Client.builder()
.region(Region.EU_WEST_1) //
.build(); // Might change for you
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket("my_bucket_name")
.key("four.bmp") // also tested for image types png, jpg, gif
.build();
byte[] objectResponse = s3.getObject(getObjectRequest).readAllBytes();
String encodedfile = new String(java.util.Base64.getEncoder().encode(objectResponse), java.nio.charset.StandardCharsets.UTF_8);
}
}

GetObjectRequest request = GetObjectRequest.builder()
.bucket("my_bucket_name")
.key("four.bmp")
.build();
try {
byte[] objectResponse = s3Client.getObject(request, ResponseTransformer.toBytes())
.asByteArray();
return Base64.getEncoder().encodeToString(objectResponse);
} catch (NoSuchKeyException e) {
log.error(e.getMessage());
throw e;
}

Found out issue was not in the code. API Gateway was not configured properly.
I referred this link
https://medium.com/swlh/upload-binary-files-to-s3-using-aws-api-gateway-with-aws-lambda-2b4ba8c70b8e

How to implement azure-storage into vaadin framework

I am trying to upload a file into Azure Blob.
and I am trying to achieve through Upload in vaadin framework. Vaadin Version : 6.7.8
I am able to develop a code for uploading the file into azure blob.
My Problem Statement Lies below :
I have written a class UploadToBlob.java to upload a file into azure blob.
If I run the class UploadToBlob.java indivually (ran from eclipse run as java application), I am able to upload the file into azure blob.
If I create a object of the UploadToBlob class in my other class[ModifyComplaintComponent.java], storageAccount = CloudStorageAccount.parse(storageConnectionString); is not getting execute.
Below is the UploadToBlob.java code:
package com.---.trs.scms.ui.components;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
public class UploadToBlob {
public static void main(String[] args) {
try {
final String storageConnectionString = "DefaultEndpointsProtocol=https;AccountName=abcd;AccountKey=bmiA7+****==;EndpointSuffix=core.windows.net";
System.out.println("---I am getting called Main-1 ");
CloudStorageAccount storageAccount;
storageAccount = CloudStorageAccount.parse(storageConnectionString);
com.microsoft.azure.storage.blob.CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
CloudBlobContainer container = blobClient.getContainerReference("container2");
container.createIfNotExists();
String filePath = "C:\\Users\\----\\Desktop\\Timesheet - 19th Aug,2019.pdf";
com.microsoft.azure.storage.blob.CloudBlockBlob blob = container.getBlockBlobReference("Timesheet.pdf");
java.io.File source = new java.io.File(filePath);
java.io.FileInputStream fileInputStream = new java.io.FileInputStream(source);
blob.upload(fileInputStream, source.length());
} catch (Exception e) {
e.printStackTrace();
}
}
}
For now , I am passing manual file PATH as above to upload in azure blob, as I told above , this class is getting called till the line of code System.out.println("---I am getting called Main-1 ");
Here is the ModifyComplaintComponent code from where I am calling UploadToBlob.java:
import com.vaadin.ui.HorizontalLayout;
import com.vaadin.ui.Upload;
public class ModifyComplaintComponent extends CustomComponent {
//other component code which I haven't pasted here
private Upload uploadnew;
try {
System.out.println("------Inside try block-----------");
UploadToBlob fileReceiver= new UploadToBlob ();
uploadnew = new Upload("Upload a file", fileReceiver);
uploadnew.setReceiver(fileReceiver);
uploadnew.addListener(fileReceiver);
System.out.println("------end of try block-----------");
} catch (Exception e) {
System.out.println("------catch block-----------");
e.printStackTrace();
}
HorizontalLayout hlayout = new HorizontalLayout();
hlayout.setSpacing(true);
hlayout.addComponent(uploadnew);
}
The Reason why I have given a manual file path in my UploadToBlob code is because I firstly wanted to make this code called from ModifyComplaintComponent class.
Secondly when I try to browse the file , and file gets selected but when I click on upload , I get NullPointerException On Vaadin Upload UI Part and even if i selected the file , UI says "no file choosen"
The challenge I am facing is If I run the Upload.java file individually I am able to upload static file into azure blob , but I wanted to browse and upload a file in vaadin framework into azure blob storage.

Firstly, Upload is a Component of Vaadin. You should not create your own Upload class.
Secondly, the public static main method is an entrance where your program starts. If you want to use a method of a class, you need to explicitly invoke it.
TheClassName.MethodName(...) // For static method
new TheClassName(...).MethodName(...) //For non-static method
Thirdly, I did some tests, the following is a successful sample. Two classes will be created:
Class UploadReceiver
This class implements the Receiver interface and some listeners.
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.vaadin.ui.Upload;
import org.springframework.stereotype.Component;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
#Component
public class UploadReceiver implements Upload.Receiver, Upload.StartedListener, Upload.SucceededListener, Upload.ProgressListener {
// Storage account connection string.
public static String conn = "DefaultEndpointsProtocol=https;AccountName=stora***789;AccountKey=G3***w==;EndpointSuffix=core.windows.net";
#Override
public OutputStream receiveUpload(String filename, String mimeType) {
System.out.println("Uploading -> " + mimeType + " ; File name -> " + filename);
return GetOutputStream("vaadin",filename);
}
#Override
public void uploadStarted(Upload.StartedEvent startedEvent) {
System.out.println("Upload started!");
}
#Override
public void uploadSucceeded(Upload.SucceededEvent succeededEvent) {
System.out.println("Upload succeeded!");
}
public OutputStream GetOutputStream(String container, String blob){
OutputStream outputStream = null;
try{
CloudStorageAccount storageAccount = CloudStorageAccount.parse(conn);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.getContainerReference(container);
CloudBlockBlob cloudBlockBlob = blobContainer.getBlockBlobReference(blob);
outputStream = cloudBlockBlob.openOutputStream();
} catch (StorageException e) {
e.printStackTrace();
} catch (InvalidKeyException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
return outputStream;
}
#Override
public void updateProgress(long readBytes, long contentLength) {
System.out.println("Progress: readBytes -> " + readBytes + " ; contentLength -> " + contentLength);
}
}
Class MainUI
This is the UI page. I just add an upload component.
import com.vaadin.server.VaadinRequest;
import com.vaadin.spring.annotation.SpringUI;
import com.vaadin.ui.Alignment;
import com.vaadin.ui.UI;
import com.vaadin.ui.Upload;
import com.vaadin.ui.VerticalLayout;
import org.springframework.beans.factory.annotation.Autowired;
#SpringUI
public class MainUI extends UI {
private VerticalLayout layout;
private Upload upload;
private UploadReceiver uploadReceiver;
#Autowired
public MainUI(UploadReceiver uploadReceiver){
this.uploadReceiver = uploadReceiver;
}
#Override
protected void init(VaadinRequest vaadinRequest) {
// Set layout
layout = new VerticalLayout();
layout.setDefaultComponentAlignment(Alignment.MIDDLE_CENTER);
setContent(layout);
// Add upload
upload = new Upload("Upload a file", uploadReceiver);
upload.addStartedListener(uploadReceiver);
upload.addSucceededListener(uploadReceiver);
upload.addProgressListener(uploadReceiver);
layout.addComponent(upload);
}
}
Result:
After I clicked the upload button and chose a file to upload, I could get the following outputs from console:
And, by checking the storage account with Storage Explorer, I could see that the file was successfully uploaded:
Update:
This is how the upload works:
I do not know how your code passed the compiling. To construct an Upload object, you need to pass a caption string and a receiver which implements Upload.Receiver interface.
public Upload(String caption, Receiver uploadReceiver)
And to implement the Upload.Receiver interface, you have to override the receiveUpload method.
OutputStream receiveUpload(String filename, String mimeType)
The receiveUpload will return an output stream, where vaadin will finally write contents to.
That's all. Give vaadin an output stream, and it will write all the contents to the stream.
The input file is sent from your browser and handled by vaadin. I did not find a way to manually set the input content in vaadin. Sorry.

How to upload a file to AWS S3 Bucket?

I am creating a simple application where I want to upload a file to my AWS S3 bucket. Here is my code:
import java.io.File;
import java.io.IOException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.fasterxml.jackson.*;
public class UploadFileInBucket {
public static void main(String[] args) throws IOException {
String clientRegion = "<myRegion>";
String bucketName = "<myBucketName>";
String stringObjKeyName = "testobject";
String fileObjKeyName = "testfileobject";
String fileName = "D:\\Attachments\\LICENSE";
try {
BasicAWSCredentials awsCreds = new BasicAWSCredentials("<myAccessKey>", "<mySecretKey>");
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withRegion(clientRegion)
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.build();
// Upload a text string as a new object.
s3Client.putObject(bucketName, stringObjKeyName, "Uploaded String Object");
// Upload a file as a new object with ContentType and title specified.
PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("plain/text");
metadata.addUserMetadata("x-amz-meta-title", "someTitle");
request.setMetadata(metadata);
s3Client.putObject(request);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
}
catch(SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
}
}
I am unable to upload a file and getting an error as:
Exception in thread "main" java.lang.NoSuchFieldError:
ALLOW_FINAL_FIELDS_AS_MUTATORS
at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:52)
at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:64)
at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:52)
at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:105)
at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:249)
at com.amazonaws.client.builder.AwsClientBuilder.withRegion(AwsClientBuilder.java:238)
at UploadFileInBucket.main(UploadFileInBucket.java:28)
I have added required AWS bucket credentials, permissions and dependencies to execute this code.
What changes I should made in the code to get my file uploaded to desired bucket?

It looks as though you either have the wrong version of the Jackson libraries or are somehow linking with multiple versions of them.
The AWS for Java SDK distribution contains a third-party/lib directory which contains all of the (correct versions of) the libraries that version of the SDK should be built with. Depending on which features of the SDK you are using you may not need all of them, but those are the specific 3rd party libraries you should be using.

You need to add Jackson to your classpath. Its classes are missing.
I don't know which version you need, but you can download them from their gitpage: https://github.com/FasterXML/jackson/

How to read uploaded file that triggered lambda execution directly?

My current AWS setup is a lambda function that is being triggered whenever I put an object into a S3 bucket. I implemented the lambda's handler function in Java. What I want to do is simply accessing the file that was uploaded and triggered the execution of the lambda function. E.g., if I upload sample.json to the bucket, I want to access the contents of this file in my handler function.
I know I can do something like this:
public Void handleRequest(S3Event input, Context context) {
for (S3EventNotificationRecord record : input.getRecords()) {
String key = record.getS3().getObject().getKey();
String bucket = record.getS3().getBucket().getName();
AmazonS3 s3Client = new AmazonS3Client(credentials);
try {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));
InputStream input = s3Object.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
while (true) {
String line = reader.readLine();
if (line == null) break;
// Do something with line...
}
// ...
The problem is that I am not allowed to use access keys. Thus, I cannot create an s3Client to download the file with. In other words, I have to get the object from the argument that my handler method takes, i.e., S3Event input. How would I do that?

If your Lambda function is configured with an appropriate IAM role (that allows s3:GetObject of the relevant S3 object), then you don't need to explicitly provide credentials in your code.
Here's sample Java code to get a object in response to an object uploaded Lambda event:
public class S3GetTextBody implements RequestHandler<S3Event, String> {
public String handleRequest(S3Event s3event, Context context) {
try {
S3EventNotificationRecord record = s3event.getRecords().get(0);
// Retrieve the bucket & key for the uploaded S3 object that
// caused this Lambda function to be triggered
String bkt = record.getS3().getBucket().getName();
String key = record.getS3().getObject().getKey().replace('+', ' ');
key = URLDecoder.decode(key, "UTF-8");
// Read the source file as text
AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();
String body = s3Client.getObjectAsString(bkt, key);
System.out.println("Body: " + body);
return "ok";
} catch (Exception e) {
System.err.println("Exception: " + e);
return "error";
}
}
}
Imports:
import java.net.URLDecoder;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.S3Event;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord;