Develop Reference

How to read from an Amazon S3 Bucket and call AWS services

I am able to call AWS Textract to read an image from my local path. How can I integrate this textract code to read the image uploaded onto a created S3 bucket with the S3 bucket codes.
Working Textract Code to textract images from local path
package aws.cloud.work;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.io.InputStream;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.textract.AmazonTextract;
import com.amazonaws.services.textract.AmazonTextractClientBuilder;
import com.amazonaws.services.textract.model.DetectDocumentTextRequest;
import com.amazonaws.services.textract.model.DetectDocumentTextResult;
import com.amazonaws.services.textract.model.Document;
import com.amazonaws.util.IOUtils;
public class TextractDemo {
static AmazonTextractClientBuilder clientBuilder = AmazonTextractClientBuilder.standard()
.withRegion(Regions.US_EAST_1);
private static FileWriter file;
public static void main(String[] args) throws IOException {
//AWS Credentials to access AWS Textract services
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials("Access Key", "Secret key")));
//Set the path of the image to be textract. Can be configured to use from S3
String document="C:\\Users\\image-local-path\\sampleTT.jpg";
ByteBuffer imageBytes;
//Code to use AWS Textract services
try (InputStream inputStream = new FileInputStream(new File(document))) {
imageBytes = ByteBuffer.wrap(IOUtils.toByteArray(inputStream));
}
AmazonTextract client = clientBuilder.build();
DetectDocumentTextRequest request = new DetectDocumentTextRequest()
.withDocument(new Document().withBytes(imageBytes));
/*
* DetectDocumentTextResult result = client.detectDocumentText(request);
* System.out.println(result); result.getBlocks().forEach(block ->{
* if(block.getBlockType().equals("LINE")) System.out.println("text is "+
* block.getText() + " confidence is "+ block.getConfidence());
*/
//
DetectDocumentTextResult result = client.detectDocumentText(request);
System.out.println(result);
JSONObject obj = new JSONObject();
result.getBlocks().forEach(block -> {
if (block.getBlockType().equals("LINE"))
System.out.println("text is " + block.getText() + " confidence is " + block.getConfidence());
JSONArray fields = new JSONArray();
fields.add(block.getText() + " , " + block.getConfidence());
obj.put(block.getText(), fields);
});
//To import the results into JSON file and output the console output as sample.txt
try {
file = new FileWriter("/Users/output-path/sample.txt");
file.write(obj.toJSONString());
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
file.flush();
file.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
This is an example of the console out where the "text" and corresponding "confidence scores" are returned
S3 bucket code integration I managed to find from the docs:
String document = "sampleTT.jpg";
String bucket = "textract-images";
AmazonS3 s3client = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(
new EndpointConfiguration("https://s3.amazonaws.com","us-east-1"))
.build();
// Get the document from S3
com.amazonaws.services.s3.model.S3Object s3object = s3client.getObject(bucket, document);
S3ObjectInputStream inputStream = s3object.getObjectContent();
BufferedImage image = ImageIO.read(inputStream);
(Edited) - Thanks #smac2020, I currently have a working Rekognition Code that reads from my AWS console S3 bucket and runs the Rekognition services that I am referencing to. However, I am unable to modify and merge it with
the Textract source code
package com.amazonaws.samples;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.rekognition.AmazonRekognition;
import com.amazonaws.services.rekognition.AmazonRekognitionClientBuilder;
import com.amazonaws.services.rekognition.model.AmazonRekognitionException;
import com.amazonaws.services.rekognition.model.DetectLabelsRequest;
import com.amazonaws.services.rekognition.model.DetectLabelsResult;
import com.amazonaws.services.rekognition.model.Image;
import com.amazonaws.services.rekognition.model.Label;
import com.amazonaws.services.rekognition.model.S3Object;
import java.util.List;
public class DetectLabels {
public static void main(String[] args) throws Exception {
String photo = "sampleTT.jpg";
String bucket = "Textract-bucket";
// AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withRegion("ap-southeast-1").build();
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider (new BasicAWSCredentials("Access Key", "Secret Key"));
AmazonRekognition rekognitionClient = AmazonRekognitionClientBuilder.standard().withCredentials(credentialsProvider).withRegion("ap-southeast-1").build();
DetectLabelsRequest request = new DetectLabelsRequest()
.withImage(new Image()
.withS3Object(new S3Object()
.withName(photo).withBucket(bucket)))
.withMaxLabels(10)
.withMinConfidence(75F);
try {
DetectLabelsResult result = rekognitionClient.detectLabels(request);
List <Label> labels = result.getLabels();
System.out.println("Detected labels for " + photo);
for (Label label: labels) {
System.out.println(label.getName() + ": " + label.getConfidence().toString());
}
} catch(AmazonRekognitionException e) {
e.printStackTrace();
}
}
}

Looks like you are trying to read an Amazon S3 object from a Spring boot app and then pass that byte array to DetectDocumentTextRequest.
There is a tutorial that shows a very similar use case where a Spring BOOT app reads the bytes from an Amazon S3 object and passes it to the Amazon Rekognition service (instead of Textract).
The Java code is:
// Get the byte[] from this AWS S3 object.
public byte[] getObjectBytes (String bucketName, String keyName) {
s3 = getClient();
try {
GetObjectRequest objectRequest = GetObjectRequest
.builder()
.key(keyName)
.bucket(bucketName)
.build();
ResponseBytes<GetObjectResponse> objectBytes = s3.getObjectAsBytes(objectRequest);
byte[] data = objectBytes.asByteArray();
return data;
} catch (S3Exception e) {
System.err.println(e.awsErrorDetails().errorMessage());
System.exit(1);
}
return null;
}
See this AWS development article to see how to build a Spring BOOT app that has this functionality.
Creating an example AWS photo analyzer application using the AWS SDK for Java
This example uses the AWS SDK For Java V2. If you are not familiar with working with the latest SDK version, I recommend that you start here:
Get started with the AWS SDK for Java 2.x

Unable to move file from one folder to another in S3

I want to move the file inside the S3 folder to another folder which is in the same s3 Bucket. I tried the below code
CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName,
srcFolder+"/"+Filename, bucketName,
targetFolder+"/"+Filename);
s3Client.copyObject(copyObjRequest);
DeleteObjectRequest deleteObjRequest = new DeleteObjectRequest(bucketName,
srcFolder+"/"+Filename);
s3Client.deleteObject(deleteObjRequest);
The folder may contain multiple file, i want to move only the selected file. Above code is not showing any error, but nothing happens. Can anyone please suggest me the right solution for it.

It would be a good initial stab just to run the following code and check what the output is, without and deletions.
Also worth checking the ACL and bucket policy on the object.
This is the format expected
CopyObjectRequest(java.lang.String sourceBucketName, java.lang.String sourceKey, java.lang.String destinationBucketName, java.lang.String destinationKey)
If you want a copy of the object in the same bucket
CopyObjectRequest copyObjRequest = new CopyObjectRequest("myBucket", "myObject.txt", "myBucket", "myNewObject.txt");
s3Client.copyObject(copyObjRequest);
If you want a copy of the object in a different bucket
CopyObjectRequest copyObjRequest = new CopyObjectRequest("myBucket", "myObject.txt", "myOtherBucket", "myNewObject.txt");
s3Client.copyObject(copyObjRequest);
Sample code for testing
import java.io.IOException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CopyObjectRequest;
public class CopyObjectSingleOperation {
public static void main(String[] args) throws IOException {
String clientRegion = "*** Client region ***";
String bucketName = "*** Bucket name ***";
String sourceKey = "*** Source object key *** ";
String destinationKey = "*** Destination object key ***";
try {
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new ProfileCredentialsProvider())
.withRegion(clientRegion)
.build();
// Copy the object into a new object in the same bucket.
CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, sourceKey, bucketName, destinationKey);
s3Client.copyObject(copyObjRequest);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
}
catch(SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
}
}

How to upload a file to AWS S3 Bucket?

I am creating a simple application where I want to upload a file to my AWS S3 bucket. Here is my code:
import java.io.File;
import java.io.IOException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.fasterxml.jackson.*;
public class UploadFileInBucket {
public static void main(String[] args) throws IOException {
String clientRegion = "<myRegion>";
String bucketName = "<myBucketName>";
String stringObjKeyName = "testobject";
String fileObjKeyName = "testfileobject";
String fileName = "D:\\Attachments\\LICENSE";
try {
BasicAWSCredentials awsCreds = new BasicAWSCredentials("<myAccessKey>", "<mySecretKey>");
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withRegion(clientRegion)
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.build();
// Upload a text string as a new object.
s3Client.putObject(bucketName, stringObjKeyName, "Uploaded String Object");
// Upload a file as a new object with ContentType and title specified.
PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("plain/text");
metadata.addUserMetadata("x-amz-meta-title", "someTitle");
request.setMetadata(metadata);
s3Client.putObject(request);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
}
catch(SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
}
}
I am unable to upload a file and getting an error as:
Exception in thread "main" java.lang.NoSuchFieldError:
ALLOW_FINAL_FIELDS_AS_MUTATORS
at com.amazonaws.partitions.PartitionsLoader.<clinit>(PartitionsLoader.java:52)
at com.amazonaws.regions.RegionMetadataFactory.create(RegionMetadataFactory.java:30)
at com.amazonaws.regions.RegionUtils.initialize(RegionUtils.java:64)
at com.amazonaws.regions.RegionUtils.getRegionMetadata(RegionUtils.java:52)
at com.amazonaws.regions.RegionUtils.getRegion(RegionUtils.java:105)
at com.amazonaws.client.builder.AwsClientBuilder.getRegionObject(AwsClientBuilder.java:249)
at com.amazonaws.client.builder.AwsClientBuilder.withRegion(AwsClientBuilder.java:238)
at UploadFileInBucket.main(UploadFileInBucket.java:28)
I have added required AWS bucket credentials, permissions and dependencies to execute this code.
What changes I should made in the code to get my file uploaded to desired bucket?

It looks as though you either have the wrong version of the Jackson libraries or are somehow linking with multiple versions of them.
The AWS for Java SDK distribution contains a third-party/lib directory which contains all of the (correct versions of) the libraries that version of the SDK should be built with. Depending on which features of the SDK you are using you may not need all of them, but those are the specific 3rd party libraries you should be using.

You need to add Jackson to your classpath. Its classes are missing.
I don't know which version you need, but you can download them from their gitpage: https://github.com/FasterXML/jackson/

AWS S3 Requester Pays Does not work with TransferManager

I am trying to use TransferManager class in my java program to download a file from a Requester Pays Buckets.
I am getting "Status Code: 403" exception from amazon AWS.
I wrote a simple program to test this feature and compare it single connection way of downloading a file. Here is my code:
import java.io.*;
import com.amazonaws.auth.*;
import com.amazonaws.services.s3.*;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.*;
import com.amazonaws.util.IOUtils;
public class RequesterPaysTest {
final static AWSCredentials awsCredentials = new BasicAWSCredentials(MY IAMAccessKey,MY IAMSecretKey);
final static String bucketName = "7268982505fe.mixnode.com";
final static String fileName = "5379-7268982505fe-0-1496081968663.warc.gz";
final static AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
final static AmazonS3 s3client = AmazonS3ClientBuilder.standard().withCredentials(awsCredentialsProvider).withRegion(MY bucketRegion).build();
final static GetObjectRequest getRequest = new GetObjectRequest(bucketName, fileName, true);
static void testSimpleRequesterPays() {
try {
S3Object object = s3client.getObject(getRequest);
InputStream objectData = object.getObjectContent();
FileOutputStream out = new FileOutputStream (new File(fileName));
IOUtils.copy(objectData, out);
out.close();
System.out.println(" Simple RequesterPays successful");
} catch (Exception e) {
System.out.println(" Simple RequesterPays unsuccessful: " + e.getMessage());
}
}
static void testTransferManagerRequesterPays() {
try {
TransferManager tx = TransferManagerBuilder.standard().withS3Client(s3client).build();
Download download = tx.download(getRequest, new File(fileName));
while (download.isDone() == false)
Thread.sleep(10);
System.out.println(" TransferManager RequesterPays successful");
} catch (Exception e) {
System.out.println(" TransferManager RequesterPays unsuccessful: " + e.getMessage());
}
}
public static void main(String[] args) throws IOException {
testSimpleRequesterPays();
testTransferManagerRequesterPays();
}
}
And here is the output:
Simple RequesterPays successful
TransferManager RequesterPays unsuccessful: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 77D5EBF5EE195A7A)
As you can see, the same file can be downloaded using simple method but not with TransferManager. I tried to create my own bucket and played around with permissions but it did not work.
I was wondering if I miss anything in my code? Or whether AWS s3 does not have support for using TransferManager on a requester pays bucket?

TransferManager internally using http HEAD method instead of GET for downloading objects(file(s)). You can find it by debugging TransferManager SDK code. If you don't have HEAD method configuration in your aws api gateway, you have to create HEAD method configuration to match with TransferManager SDK code. By the way if resource not found it should give 404 but you are getting 403. reason is here https://forums.aws.amazon.com/thread.jspa?threadID=216684

How can I send a PNG of a QR-code in a HTTP response body (with Spark)?

I want to generate a QR-code image, convert it to PNG and return it as a HTTP response to my client.
To generate the QR-code I use ZXing. I already tested the conversion part by writing using a FileOutputStream with MatrixToImageWriter.writeToStream(...). That works like a charm.
The web framework I am currently using is Spark (Version 1.1.1). The return of the handle(...)-method is set as the response body. What am I doing wrong here?
With the current solution I get The image "http://localhost:4567/qrcode" cannot be displayed because it contains errors when performing the GET-request with Firefox.
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import static spark.Spark.get;
import spark.Request;
import spark.Response;
import spark.Route;
import com.google.gson.Gson;
import com.google.common.io.BaseEncoding;
import com.google.zxing.BarcodeFormat;
import com.google.zxing.client.j2se.MatrixToImageWriter;
import com.google.zxing.common.BitMatrix;
import com.google.zxing.qrcode.QRCodeWriter;
public class App {
public static void main(String[] args) {
get(new Route("/qrcode") {
#Override
public Object handle(Request request, Response response) {
// Test data
QrData data = new QrData("test");
// Data is wrapped in JSON
String json = new Gson().toJson(data);
// Transform JSON to QR-code PNG byte string
String qrString = "";
try {
qrString = generatePngQrCode(json);
} catch (Exception e) {
e.printStackTrace();
}
// Set response parameters
response.type("image/png");
response.status(200);
// Return response body
return qrString;
}
});
}
public String generatePngQrCode(String content) throws Exception {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// ZXing QR-code encoding
BitMatrix bitMatrix = new QRCodeWriter().encode(content, BarcodeFormat.QR_CODE, 400, 400);
// Convert to PNG image and write to stream
MatrixToImageWriter.writeToStream(bitMatrix, "png", outputStream);
// Encode to Base 64
return BaseEncoding.base64().encode(outputStream.toByteArray());
}
}

Just went through this. You can write any file/binary data/output stream using the following code:
byte[] bytes = Files.readAllBytes(Paths.get(filePath));
HttpServletResponse raw = res.raw();
raw.getOutputStream().write(bytes);
raw.getOutputStream().flush();
raw.getOutputStream().close();
return res.raw();

Use response.getRaw to obtain an OutputStream that should be used to write the PNG to (using MatrixToImageWriter).