Develop Reference

Create Tar archive from directory on S3 using AWS Lambda

I need to extract a bunch of zip files stored on s3 and add them to a tar archive and store that archive on s3. it is likely that that the sum of the zip files will greater than the 512mb local storage allowed from lambda functions. I have a partial souldtion that gets the objects from s3 extracts them and puts them in a s3 object without using the lambda local storage.
Extract object Thread
public class ExtractObject implements Runnable{
private String objectName;
private String uuid;
private final byte[] buffer = new byte[1024];
public ExtractAdvert(String name, String uuid) {
this.objectName= name;
this.uuid= uuid;
}
#Override
public void run() {
final String srcBucket = "my-bucket-name";
final AmazonS3 s3Client = new AmazonS3Client();
try {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(srcBucket, objectName));
ZipInputStream zis = new ZipInputStream(s3Object.getObjectContent());
ZipEntry entry = zis.getNextEntry();
while(entry != null) {
String fileName = entry.getName();
String mimeType = FileMimeType.fromExtension(FilenameUtils.getExtension(fileName)).mimeType();
System.out.println("Extracting " + fileName + ", compressed: " + entry.getCompressedSize() + " bytes, extracted: " + entry.getSize() + " bytes, mimetype: " + mimeType);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
int len;
while ((len = zis.read(buffer)) > 0) {
outputStream.write(buffer, 0, len);
}
InputStream is = new ByteArrayInputStream(outputStream.toByteArray());
ObjectMetadata meta = new ObjectMetadata();
meta.setContentLength(outputStream.size());
meta.setContentType(mimeType);
System.out.println("##### " + srcBucket + ", " + FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName);
// Add this to tar archive instead of putting back to s3
s3Client.putObject(srcBucket, FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName, is, meta);
is.close();
outputStream.close();
entry = zis.getNextEntry();
}
zis.closeEntry();
zis.close();
} catch (IOException ioe) {
System.out.println(ioe.getMessage());
}
}
}
this runs for each object that needs to be extracted and saves them in a s3 object in the structure needed for the tar file.
I think what i need is instead of putting the object back to s3 is to keep it in memory and add it to a tar archive. and upload that but after a lot of looking around and trial and error i have not created a successful tar file.
The main issue is i can't use the tmp directory in lambda.
Edit
should i be creating the tar file as i go instead of putting objects to s3? (see comment // Add this to tar archive instead of putting back to s3)
if so how do i create a tar stream without a storing it locally?
EDIT 2: Attempt at taring the files
ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result result;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);
do {
result = s3Client.listObjectsV2(req);
for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
if(objectSummary.getKey().startsWith("tmp/") ) {
System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
InputStream is = s3Object.getObjectContent();
System.out.println("Pre Create entry");
TarArchiveEntry archiveEntry = new TarArchiveEntry(IOUtils.toByteArray(is));
// Getting following exception above
// IllegalArgumentException: Invalid byte 111 at offset 7 in ' positio' len=8
System.out.println("Pre put entry");
tarOut.putArchiveEntry(archiveEntry);
System.out.println("Post put entry");
}
}
String token = result.getNextContinuationToken();
System.out.println("Next Continuation Token: " + token);
req.setContinuationToken(token);
} while (result.isTruncated());
ObjectMetadata metadata = new ObjectMetadata();
InputStream is = new ByteArrayInputStream(baos.toByteArray());
s3Client.putObject(new PutObjectRequest(bucketName, bucketFolder + "tar-file", is, metadata));

I have found a solution to this and it very similar to my attempt in Edit 2 above.
private final String bucketName = "bucket-name";
private final String bucketFolder = "tmp/";
private final String tarKey = "tar-dir/tared-file.tar";
private void createTar() throws IOException, ArchiveException {
ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result result;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);
do {
result = s3Client.listObjectsV2(req);
for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
if (objectSummary.getKey().startsWith(bucketFolder)) {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
InputStream is = s3Object.getObjectContent();
String s3Key = objectSummary.getKey();
String tarPath = s3Key.substring(s3Key.indexOf('/') + 1, s3Key.length());
s3Key.lastIndexOf('.'));
byte[] ba = IOUtils.toByteArray(is);
TarArchiveEntry archiveEntry = new TarArchiveEntry(tarPath);
archiveEntry.setSize(ba.length);
tarOut.putArchiveEntry(archiveEntry);
tarOut.write(ba);
tarOut.closeArchiveEntry();
}
}
String token = result.getNextContinuationToken();
System.out.println("Next Continuation Token: " + token);
req.setContinuationToken(token);
} while (result.isTruncated());
ObjectMetadata metadata = new ObjectMetadata();
InputStream is = baos.toInputStream();
metadata.setContentLength(baos.size());
s3Client.putObject(new PutObjectRequest(bucketName, tarKey, is, metadata));
}

Kraken API - Invalid signature

I'm trying to change a working example from C sharp to Java but i'm struggling and don't know where the problem is. I contacted Kraken and they advised me that I have the wrong signature... The response is:
{"error":["EAPI:Invalid signature"]}
Here's the C sharp version:
private JsonObject QueryPrivate(string a_sMethod, string props = null)
{
// generate a 64 bit nonce using a timestamp at tick resolution
Int64 nonce = DateTime.Now.Ticks;
props = "nonce=" + nonce + props;
string path = string.Format("/{0}/private/{1}", _version, a_sMethod);
string address = _url + path;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(address);
webRequest.ContentType = "application/x-www-form-urlencoded";
webRequest.Method = "POST";
webRequest.Headers.Add("API-Key", _key);
byte[] base64DecodedSecred = Convert.FromBase64String(_secret);
var np = nonce + Convert.ToChar(0) + props;
var pathBytes = Encoding.UTF8.GetBytes(path);
var hash256Bytes = sha256_hash(np);
var z = new byte[pathBytes.Count() + hash256Bytes.Count()];
pathBytes.CopyTo(z, 0);
hash256Bytes.CopyTo(z, pathBytes.Count());
var signature = getHash(base64DecodedSecred, z);
webRequest.Headers.Add("API-Sign", Convert.ToBase64String(signature));
if (props != null)
{
using (var writer = new StreamWriter(webRequest.GetRequestStream()))
{
writer.Write(props);
}
}
//Make the request
try
{
using (WebResponse webResponse = webRequest.GetResponse())
{
using (Stream str = webResponse.GetResponseStream())
{
using (StreamReader sr = new StreamReader(str))
{
return (JsonObject)JsonConvert.Import(sr);
}
}
}
}
}
The full code is here at:
https://bitbucket.org/arrivets/krakenapi/src/cff138b017c38efde2db1a080fb765790a6d04c8/KrakenClient/KrakenClient.cs?at=master&fileviewer=file-view-default
Here is my Java version:
private void fetch() throws UnsupportedEncodingException, IOException, NoSuchAlgorithmException {
String version = "0";
String key = ".....6";
String secret = "....g==";
long nonce = System.currentTimeMillis();
String props = null;
props = "nonce=" + nonce + props; // I've tried this with and without the 'null' on the end
// url
String url = "https://api.kraken.com";
String path = "/" + version + "/private/" + "Balance";
String address = url + path;
// post req
HttpPost httpPost = new HttpPost(address);
// headers
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
httpPost.setHeader("API-Key", key);
// decode buffer
BASE64Decoder decoder = new BASE64Decoder();
byte[] base64DecodedSecred = decoder.decodeBuffer(secret);
// nonce & props
String np = nonce + (char) 0 + props;
// create byte array
byte[] pathBytes = path.getBytes("UTF-8");
byte[] hash256Bytes = sha256(np);
byte[] z = new byte[pathBytes.length + hash256Bytes.length];
System.arraycopy(pathBytes, 0, z, 0, pathBytes.length);
System.arraycopy(hash256Bytes, 0, z, pathBytes.length, hash256Bytes.length);
// encrypt signature
byte[] signature = hmacEncrypt(z, base64DecodedSecred); // my hmacEncrypt is message, secret (opposite to the c sharp)
BASE64Encoder encoder = new BASE64Encoder();
httpPost.setHeader("API-Sign", encoder.encode(signature));
// Post
List<NameValuePair> nvps = new ArrayList<>();
nvps.add(new BasicNameValuePair("nonce", String.valueOf(nonce)));
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
// Client & Response
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
// parse
JsonParser jp = new JsonParser();
JsonElement root = jp.parse(EntityUtils.toString(entity));
System.out.println(root); // {"error":["EAPI:Invalid signature"]}
// close client
httpClient.close();
}
I'm sorry to have posted a large bits of code, any help would be appreciated. Thank you!

there is no cchmac 512 encryption in your code. that is why you get this EAPI:Invalid signature"

Jetty throws exception : 'Buffer size exceeded: no line terminator' when using getParts

I am trying to upload a file. In the servlet when I call getParts() on request, I am getting the below error :
java.io.IOException: Buffer size exceeded: no line terminator
at org.eclipse.jetty.util.ReadLineInputStream.readLine(ReadLineInputStream.java:55)
at org.eclipse.jetty.util.MultiPartInputStreamParser.parse(MultiPartInputStreamParser.java:487)
at org.eclipse.jetty.util.MultiPartInputStreamParser.getParts(MultiPartInputStreamParser.java:405)
at org.eclipse.jetty.server.Request.getParts(Request.java:2311)
at org.eclipse.jetty.server.Request.getParts(Request.java:2290)
Any idea how to resolve this error ? I researched a lot and also implemented a lot of solutions, but nothing is working.
Annotated servlet with following annotation:
#MultipartConfig
When starting Jetty, added following configurations to server: (seems like TagLibConfiguration is not available in version 9.3.6.v20151106)
WebAppContext root = new WebAppContext();
root.setConfigurations(new Configuration[] {
new AnnotationConfiguration(), new WebXmlConfiguration(),
new WebInfConfiguration(), //new TagLibConfiguration(),
new PlusConfiguration(), new MetaInfConfiguration(),
new JettyWebXmlConfiguration(),
new FragmentConfiguration(), new EnvConfiguration() });
....
....
ServletHolder clientServletHolder = new ServletHolder(new ClientServlet(lwServer, lwServer.getSecureAddress()
.getPort()));
clientServletHolder.getRegistration().setMultipartConfig(new MultipartConfigElement(System.getProperty("java.io.tmpdir")));
....
....
The file that I am uploading is a binary file.
If I try below
String content = IOUtils.toString(req.getInputStream(), parameters.get("charset"));
byte[] byteArray = content.getBytes();
System.out.println("content len : " + content.length() + " byteArray size : " + byteArray.length);
The output is
content len : 46934 byteArray size : 52864
And when I do a file comparison the content does not match.
The contents don't match for below either. The charset I believe is 'UTF-8'.
File saveFile = new File("C:\\test\\downloadedFile.bin");
// opens input stream of the request for reading data
InputStream inputStream = req.getInputStream();
// opens an output stream for writing file
FileOutputStream outputStream = new FileOutputStream(saveFile);
byte[] buffer = new byte[100000];
int bytesRead = -1;
System.out.println("Receiving data...");
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
System.out.println("Data received.");
outputStream.close();
inputStream.close();
AngularJS code is as follows:
$('#uploadFileSubmit').click(function(e) {
e.preventDefault();
var value = $('#uploadFileInputValue').val();
var f = document.getElementById('uploadFileInputValue').files[0];
var r = new FileReader();
var fileContent;
r.onloadend = function(e){
fileContent = e.target.result;
};
r.readAsBinaryString(f);
alert("hi !!")
if(value) {
$('#uploadFileModal').modal('hide');
var rsc = {};
rsc["id"] = scope.resource.def.id;
value = lwResources.getTypedValue(fileContent, scope.resource.def.type);
rsc["value"] = value;
$http({method: 'PUT', url: "api/clients/" + $routeParams.clientId + scope.resource.path, data: rsc, timeout:300000, headers:{'Content-Type': 'multipart/form-data'}})
.success(function(data, status, headers, config) { .... }
.error(function(data, status, headers, config) {....});;
But nothing seems to resolve the error. Any idea how to resolve this error ?

Http File upload using Apache MultipartEntityBuilder

Below is the Http post method for file upload in c#. What is the equivalent for this code in java which uses apache library. How to add contentDisposition in java and pass byte array value in it. Providing some online reference is much appreciated.
C# Code
byte[] date = //file in byte format
var fileContent = new StreamContent(new MemoryStream(data));
fileContent.Headers.ContentDisposition = new ContentDispositionHeaderValue("form-data")
{
Name = "\"files\"",
FileName = "\"" + filename + "\""
}; // the extra quotes are key here
fileContent.Headers.ContentType = new MediaTypeHeaderValue(contentType);
var content = new MultipartFormDataContent();
content.Add(fileContent);
HttpResponseMessage response = null;
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, _url + uri);
request.Content = content;
My Java Code
StringBody name = new StringBody("\"files\"", ContentType.MULTIPART_FORM_DATA);
StringBody file = new StringBody("\"" + filename + "\"", ContentType.MULTIPART_FORM_DATA);
HttpEntity entity = MultipartEntityBuilder.create()
.addPart("Name", name)
.addPart("FileName", file)
.addBinaryBody("file", data)
.build();
Postmethod = new HttpPost(_url + uri);
Postmethod.addHeader(useragent);
Postmethod.addHeader(Accesstoken);
Postmethod.setEntity(entity);
Postmethod.addHeader("content-type", contentType);
response = httpClient.execute(Postmethod);
The response status code is 400 .Where did I go wrong?
Thanks in Advance..

file upload using java servlet as a service without a web browser

I am very new to java and servlet programming.
I am not sure whether it is possible to write a servlet which when passed a URL from the local client machine, uploads the file to the server.
basically on the client machine we have a C# program and on the server side we have Apache-tomcat installed. I need to upload file(s) to the server using C# program on client machine.
Should I provide any more information (?)
Thanks in Advance

Note this code illustrates the general idea and not guaranteed to work without modification.
The C# file upload part
// this code shows you how the browsers wrap the file upload request, you still can fine a way simpler code to do the same thing.
public void PostMultipleFiles(string url, string[] files)
{
string boundary = "----------------------------" + DateTime.Now.Ticks.ToString("x");
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
httpWebRequest.ContentType = "multipart/form-data; boundary=" + boundary;
httpWebRequest.Method = "POST";
httpWebRequest.KeepAlive = true;
httpWebRequest.Credentials = System.Net.CredentialCache.DefaultCredentials;
Stream memStream = new System.IO.MemoryStream();
byte[] boundarybytes =System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary +"\r\n");
string formdataTemplate = "\r\n--" + boundary + "\r\nContent-Disposition: form-data; name=\"{0}\";\r\n\r\n{1}";
string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n Content-Type: application/octet-stream\r\n\r\n";
memStream.Write(boundarybytes, 0, boundarybytes.Length);
for (int i = 0; i < files.Length; i++)
{
string header = string.Format(headerTemplate, "file" + i, files[i]);
//string header = string.Format(headerTemplate, "uplTheFile", files[i]);
byte[] headerbytes = System.Text.Encoding.UTF8.GetBytes(header);
memStream.Write(headerbytes, 0, headerbytes.Length);
FileStream fileStream = new FileStream(files[i], FileMode.Open,
FileAccess.Read);
byte[] buffer = new byte[1024];
int bytesRead = 0;
while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) != 0)
{
memStream.Write(buffer, 0, bytesRead);
}
memStream.Write(boundarybytes, 0, boundarybytes.Length);
fileStream.Close();
}
httpWebRequest.ContentLength = memStream.Length;
Stream requestStream = httpWebRequest.GetRequestStream();
memStream.Position = 0;
byte[] tempBuffer = new byte[memStream.Length];
memStream.Read(tempBuffer, 0, tempBuffer.Length);
memStream.Close();
requestStream.Write(tempBuffer, 0, tempBuffer.Length);
requestStream.Close();
try
{
WebResponse webResponse = httpWebRequest.GetResponse();
Stream stream = webResponse.GetResponseStream();
StreamReader reader = new StreamReader(stream);
string var = reader.ReadToEnd();
}
catch (Exception ex)
{
response.InnerHtml = ex.Message;
}
httpWebRequest = null;
}
and to understand how the above code was written you might wanna take a look at How does HTTP file upload work?
POST /upload?upload_progress_id=12344 HTTP/1.1
Host: localhost:3000
Content-Length: 1325
Origin: http://localhost:3000
... other headers ...
Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryePkpFF7tjBAqx29L
------WebKitFormBoundaryePkpFF7tjBAqx29L
Content-Disposition: form-data; name="MAX_FILE_SIZE"
100000
------WebKitFormBoundaryePkpFF7tjBAqx29L
Content-Disposition: form-data; name="uploadedfile"; filename="hello.o"
Content-Type: application/x-object
... contents of file goes here ...
------WebKitFormBoundaryePkpFF7tjBAqx29L--
and finally all you have to do is to implement a servlet that can handle the file upload request, then you do whatever that you want to do with the file, take a look at this file upload tutorial
protected void processRequest(HttpServletRequest request,
HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html;charset=UTF-8");
// Create path components to save the file
final String path = request.getParameter("destination");
final Part filePart = request.getPart("file");
final String fileName = getFileName(filePart);
OutputStream out = null;
InputStream filecontent = null;
final PrintWriter writer = response.getWriter();
try {
out = new FileOutputStream(new File(path + File.separator
+ fileName));
filecontent = filePart.getInputStream();
int read = 0;
final byte[] bytes = new byte[1024];
while ((read = filecontent.read(bytes)) != -1) {
out.write(bytes, 0, read);
}
writer.println("New file " + fileName + " created at " + path);
LOGGER.log(Level.INFO, "File{0}being uploaded to {1}",
new Object[]{fileName, path});
} catch (FileNotFoundException fne) {
writer.println("You either did not specify a file to upload or are "
+ "trying to upload a file to a protected or nonexistent "
+ "location.");
writer.println("<br/> ERROR: " + fne.getMessage());
LOGGER.log(Level.SEVERE, "Problems during file upload. Error: {0}",
new Object[]{fne.getMessage()});
} finally {
if (out != null) {
out.close();
}
if (filecontent != null) {
filecontent.close();
}
if (writer != null) {
writer.close();
}
}
}
private String getFileName(final Part part) {
final String partHeader = part.getHeader("content-disposition");
LOGGER.log(Level.INFO, "Part Header = {0}", partHeader);
for (String content : part.getHeader("content-disposition").split(";")) {
if (content.trim().startsWith("filename")) {
return content.substring(
content.indexOf('=') + 1).trim().replace("\"", "");
}
}
return null;
}