I'm working on a springboot backend server to provide an audio interface to the android frontend. I expose the audio directory to the outside world directly in the application.yaml file. But in this way, I found that the audio playback has a delay of about 5s before it can be played when I use docker to deploy the .jar to the cloud server. But when I went to test the network interface of some music web, I found that they were very fast and could start playing almost instantly. I wonder how this is done? Can this be achieved if springboot is used as the backend?
this is a part of application.yaml, where web.sound-path is the folder path where I store the audio on cloud server.
spring:
web:
resources:
static-locations: classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${web.sound-path}
The android frontend directly accesses http://ip:port/audioname.mp3 to play audio.
I am a novice in the development of java network, I do not know how to further improve, I hope someone can give detailed other solutions
my springboot version is 2.7.5
And my cloud server has 10Mbps bandwidth, quad-core CPU and 8G memory.
I've looked for a lot of information online but nothing is of much help, I'd really appreciate if someone could give a good answer.
I've tried two other methods on the web.
One is to use a NonStaticResourceHttpRequestHandler.
the NonStaticResourceHttpRequestHandler is :
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.resource.ResourceHttpRequestHandler;
import javax.servlet.http.HttpServletRequest;
import java.nio.file.Path;
#Component
public class NonStaticResourceHttpRequestHandler extends ResourceHttpRequestHandler {
public final static String ATTR_FILE = "NON-STATIC-FILE";
#Override
protected Resource getResource(HttpServletRequest request) {
final Path filePath = (Path) request.getAttribute(ATTR_FILE);
return new FileSystemResource(filePath);
}
}
and the method in controller is :
#GetMapping("/get/{name}")
public void getSound(#ApiParam("name") #PathVariable("name") String name, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
Path path = Paths.get(soundPath + name);
if (Files.exists(path)) {
String mimeType = Files.probeContentType(path);
if (!StringUtils.isEmpty(mimeType)) {
response.setContentType(mimeType);
}
request.setAttribute(NonStaticResourceHttpRequestHandler.ATTR_FILE, path);
nonStaticResourceHttpRequestHandler.handleRequest(request, response);
} else {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
response.setCharacterEncoding(StandardCharsets.UTF_8.toString());
}
}
another solution I have tried is: (The code in it may not be very beautiful, because I rewritten it based on the existing code on the Internet.)
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
#RestController
#RequestMapping("/audio")
public class AudioVideoController {
#Value("${web.sound-path}")
public String AUDIO_PATH;
public static final int BYTE_RANGE = 128; // increase the byterange from here
#GetMapping("/play/{fileName}")
public ResponseEntity<byte[]> streamAudio(#RequestHeader(value = "Range", required = false) String httpRangeList,
#PathVariable("fileName") String fileName) {
return getContent(AUDIO_PATH, fileName, httpRangeList, "audio");
}
private ResponseEntity<byte[]> getContent(String location, String fileName, String range, String contentTypePrefix) {
long rangeStart = 0;
long rangeEnd;
byte[] data;
Long fileSize;
String fileType = fileName.substring(fileName.lastIndexOf(".") + 1);
try {
fileSize = Optional.ofNullable(fileName)
.map(file -> Paths.get(getFilePath(location), file))
.map(this::sizeFromFile)
.orElse(0L);
if (range == null) {
return ResponseEntity.status(HttpStatus.OK)
.header("Content-Type", contentTypePrefix + "/" + fileType)
.header("Content-Length", String.valueOf(fileSize))
.body(readByteRange(location, fileName, rangeStart, fileSize - 1));
}
String[] ranges = range.split("-");
rangeStart = Long.parseLong(ranges[0].substring(6));
if (ranges.length > 1) {
rangeEnd = Long.parseLong(ranges[1]);
} else {
rangeEnd = fileSize - 1;
}
if (fileSize < rangeEnd) {
rangeEnd = fileSize - 1;
}
data = readByteRange(location, fileName, rangeStart, rangeEnd);
} catch (IOException e) {
e.printStackTrace();
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
String contentLength = String.valueOf((rangeEnd - rangeStart) + 1);
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT)
.header("Content-Type", contentTypePrefix +
"/" + fileType)
.header("Accept-Ranges", "bytes")
.header("Content-Length", contentLength)
.header("Content-Range", "bytes" + " " + rangeStart + "-" + rangeEnd + "/" + fileSize)
.body(data);
}
public byte[] readByteRange(String location, String filename, long start, long end) throws IOException {
Path path = Paths.get(getFilePath(location), filename);
try (InputStream inputStream = (Files.newInputStream(path));
ByteArrayOutputStream bufferedOutputStream = new ByteArrayOutputStream()) {
byte[] data = new byte[BYTE_RANGE];
int nRead;
while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
bufferedOutputStream.write(data, 0, nRead);
}
bufferedOutputStream.flush();
byte[] result = new byte[(int) (end - start) + 1];
System.arraycopy(bufferedOutputStream.toByteArray(), (int) start, result, 0, result.length);
return result;
}
}
private String getFilePath(String location) {
return location;
}
private Long sizeFromFile(Path path) {
try {
return Files.size(path);
} catch (IOException ex) {
ex.printStackTrace();
}
return 0L;
}
}
I have tried these two methods, and the audio can be obtained normally, but there is still a delay of a few seconds from the time the http request is sent to the audio starts playing.
I was hoping there was a way to get audio files quickly. The ideal situation is to enter an http request, press the Enter key, and then immediately start playing audio.
Related
I want to implement video Player in Angular using Spring Boot Rest API. I can play the video but I can't make video seeking. Every time the video starts over and over again when I use Chrome or Edge.
I tried this endpoint:
#RequestMapping(value = "/play_video/{video_id}", method = RequestMethod.GET)
#ResponseBody public ResponseEntity<byte[]> getPreview1(#PathVariable("video_id") String video_id, HttpServletResponse response) {
ResponseEntity<byte[]> result = null;
try {
String file = "/opt/videos/" + video_id + ".mp4";
Path path = Paths.get(file);
byte[] image = Files.readAllBytes(path);
response.setStatus(HttpStatus.OK.value());
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
headers.setContentLength(image.length);
result = new ResponseEntity<byte[]>(image, headers, HttpStatus.OK);
} catch (java.nio.file.NoSuchFileException e) {
response.setStatus(HttpStatus.NOT_FOUND.value());
} catch (Exception e) {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
return result;
}
I found this post which gives some ides: How to Implement HTTP byte-range requests in Spring MVC
But currently it's not working. Video is again playing from the start when I try to shift the position.
I use this player: https://github.com/smnbbrv/ngx-plyr
I have configured it this way:
<div class="media">
<div
class="class-video mr-3 mb-1"
plyr
[plyrPlaysInline]="true"
[plyrSources]="gymClass.video"
(plyrInit)="player = $event"
(plyrPlay)="played($event)">
</div>
<div class="media-body">
{{ gymClass.description }}
</div>
</div>
Do you know how I can fix this issue?
First solution: Using FileSystemResource
FileSystemResource internally handles byte-range header support, reading and writing the appropriate headers.
Two problems with this approach.
It uses FileInputStream internally for reading files. This is fine for small files, but not for large files served through byte-range requests. FileInputStream will read the file from the beginning and discard the not needed content until it reches the requested start offset. This can cause slowdowns with larger files.
It sets "application/json" as the "Content-Type" response header. So, I provide my own "Content-Type" header. See this thread
import java.nio.file.Path;
import java.nio.file.Paths;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.ResponseBody;
#Controller
public class Stream3 {
#GetMapping(value = "/play_video/{video_id}")
#ResponseBody
public ResponseEntity<FileSystemResource> stream(#PathVariable("video_id") String video_id) {
String filePathString = "/opt/videos/" + video_id + ".mp4";
final HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.add("Content-Type", "video/mp4");
return new ResponseEntity<>(new FileSystemResource(filePathString), responseHeaders, HttpStatus.OK);
}
}
Second solution: Using HttpServletResponse and RandomAccessFile
With RandomAccessFile you can implement support for byte-range requests. The advantage over FileInputStream, is that you don't need to read the file from the beginning every time there is a new range request, making this method usable also for larger files. RandomAccessFile has a method called seek(long) which calls the C method fseek(), which directly moves the pointer for the file to the requested offset.
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.servlet.http.HttpServletResponse;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.ResponseBody;
#Controller
public class Stream {
#GetMapping(value = "/play_video/{video_id}")
#ResponseBody
public void stream(
#PathVariable("video_id") String video_id,
#RequestHeader(value = "Range", required = false) String rangeHeader,
HttpServletResponse response) {
try {
OutputStream os = response.getOutputStream();
long rangeStart = 0;
long rangeEnd;
String filePathString = "/opt/videos/" + video_id + ".mp4";
Path filePath = Paths.get(filePathString);
Long fileSize = Files.size(filePath);
byte[] buffer = new byte[1024];
RandomAccessFile file = new RandomAccessFile(filePathString, "r");
try (file) {
if (rangeHeader == null) {
response.setHeader("Content-Type", "video/mp4");
response.setHeader("Content-Length", fileSize.toString());
response.setStatus(HttpStatus.OK.value());
long pos = rangeStart;
file.seek(pos);
while (pos < fileSize - 1) {
file.read(buffer);
os.write(buffer);
pos += buffer.length;
}
os.flush();
return;
}
String[] ranges = rangeHeader.split("-");
rangeStart = Long.parseLong(ranges[0].substring(6));
if (ranges.length > 1) {
rangeEnd = Long.parseLong(ranges[1]);
} else {
rangeEnd = fileSize - 1;
}
if (fileSize < rangeEnd) {
rangeEnd = fileSize - 1;
}
String contentLength = String.valueOf((rangeEnd - rangeStart) + 1);
response.setHeader("Content-Type", "video/mp4");
response.setHeader("Content-Length", contentLength);
response.setHeader("Accept-Ranges", "bytes");
response.setHeader("Content-Range", "bytes" + " " + rangeStart + "-" + rangeEnd + "/" + fileSize);
response.setStatus(HttpStatus.PARTIAL_CONTENT.value());
long pos = rangeStart;
file.seek(pos);
while (pos < rangeEnd) {
file.read(buffer);
os.write(buffer);
pos += buffer.length;
}
os.flush();
}
} catch (FileNotFoundException e) {
response.setStatus(HttpStatus.NOT_FOUND.value());
} catch (IOException e) {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
}
}
}
Third solution: Also using RandomAccessFile, but StreamingResponseBody instead of HttpServletResponse
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
#Controller
public class Stream2 {
#GetMapping(value = "/play_video/{video_id}")
#ResponseBody
public ResponseEntity<StreamingResponseBody> stream(
#PathVariable("video_id") String video_id,
#RequestHeader(value = "Range", required = false) String rangeHeader) {
try {
StreamingResponseBody responseStream;
String filePathString = "/opt/videos/" + video_id + ".mp4";
Path filePath = Paths.get(filePathString);
Long fileSize = Files.size(filePath);
byte[] buffer = new byte[1024];
final HttpHeaders responseHeaders = new HttpHeaders();
if (rangeHeader == null) {
responseHeaders.add("Content-Type", "video/mp4");
responseHeaders.add("Content-Length", fileSize.toString());
responseStream = os -> {
RandomAccessFile file = new RandomAccessFile(filePathString, "r");
try (file) {
long pos = 0;
file.seek(pos);
while (pos < fileSize - 1) {
file.read(buffer);
os.write(buffer);
pos += buffer.length;
}
os.flush();
} catch (Exception e) {}
};
return new ResponseEntity<>(responseStream, responseHeaders, HttpStatus.OK);
}
String[] ranges = rangeHeader.split("-");
Long rangeStart = Long.parseLong(ranges[0].substring(6));
Long rangeEnd;
if (ranges.length > 1) {
rangeEnd = Long.parseLong(ranges[1]);
} else {
rangeEnd = fileSize - 1;
}
if (fileSize < rangeEnd) {
rangeEnd = fileSize - 1;
}
String contentLength = String.valueOf((rangeEnd - rangeStart) + 1);
responseHeaders.add("Content-Type", "video/mp4");
responseHeaders.add("Content-Length", contentLength);
responseHeaders.add("Accept-Ranges", "bytes");
responseHeaders.add("Content-Range", "bytes" + " " + rangeStart + "-" + rangeEnd + "/" + fileSize);
final Long _rangeEnd = rangeEnd;
responseStream = os -> {
RandomAccessFile file = new RandomAccessFile(filePathString, "r");
try (file) {
long pos = rangeStart;
file.seek(pos);
while (pos < _rangeEnd) {
file.read(buffer);
os.write(buffer);
pos += buffer.length;
}
os.flush();
} catch (Exception e) {}
};
return new ResponseEntity<>(responseStream, responseHeaders, HttpStatus.PARTIAL_CONTENT);
} catch (FileNotFoundException e) {
return new ResponseEntity<>(HttpStatus.NOT_FOUND);
} catch (IOException e) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}
In your component.ts:
You can change the currently displaying video with playVideoFile()
export class AppComponent implements OnInit {
videoSources: Plyr.Source[];
ngOnInit(): void {
const fileName = 'sample';
this.playVideoFile(fileName);
}
playVideoFile(fileName: string) {
this.videoSources = [
{
src: `http://localhost:8080/play_video/${fileName}`,
},
];
}
}
And the html:
<div
#plyr
plyr
[plyrPlaysInline]="false"
[plyrSources]="videoSources"
></div>
If you are using the <video /> element in Chrome, seeking only works if the endpoint implements partial content requests by honouring requests with a Range header and respond with a 206 Partial Content response.
When downloading bigger (specially videos) files using Spring-boot and java nio package only download part of the file. But smaller files such as images, pdf ect get downloaded properly and usable.
For example : Let say video size is 3.5MB but when downloaded it only show 160KB and cannot play it in any player(that is because, probably partially downloaded)
Following is the controller
package com.filedownloader_with_nio_package.controllers;
import javax.servlet.ServletContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import com.filedownloader_with_nio_package.model.FileDetails;
import com.filedownloader_with_nio_package.services.FileDownloadService;
#RestController
public class FileDownloadController {
#Autowired
FileDownloadService fileDownloadService;
#RequestMapping(method = RequestMethod.POST, value = "/filedownload")
public String downloadFile(#RequestBody FileDetails fileDetails){
return fileDownloadService.downloadFile(fileDetails);
}
}
Following is Service
package com.filedownloader_with_nio_package.services;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import org.springframework.stereotype.Service;
import com.filedownloader_with_nio_package.exceptions.FileNotDownloadedCorrectlyException;
import com.filedownloader_with_nio_package.model.FileDetails;
import com.filedownloader_with_nio_package.utils.Constants;
#Service
public class FileDownloadService {
public String downloadFile(FileDetails fileDetails) {
try {
URL url = new URL(fileDetails.getFileUrl());
ReadableByteChannel readableByteChannel = Channels.newChannel(url
.openStream());
String downloadedFile = fileDetails.getFileDownloadLocation() + "/"
+ fileDetails.getFileName() + "."
+ fileDetails.getFileType();
FileOutputStream fileOutputStream = new FileOutputStream(
downloadedFile);
WritableByteChannel writableByteChannel = fileOutputStream
.getChannel();
//
//
ByteBuffer buffer = ByteBuffer.allocate(1024);
while (readableByteChannel.read(buffer) != -1) {
buffer.flip();
while (buffer.hasRemaining()) {
writableByteChannel.write(buffer);
}
buffer.clear();
}
//
//
fileOutputStream.flush();
fileOutputStream.close();
return downloadedFile;
} catch (MalformedURLException e) {
throw new FileNotDownloadedCorrectlyException(
Constants.FILE_NOT_DOWNLOADED_CORRECTLY, e);
} catch (IOException e) {
throw new FileNotDownloadedCorrectlyException(
Constants.FILE_NOT_DOWNLOADED_CORRECTLY, e);
}
}
}
Following is FileDetails model
package com.filedownloader_with_nio_package.model;
public class FileDetails {
private String fileName;
private String fileUrl;
private String fileType;
private String fileDownloadLocation;
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getFileUrl() {
return fileUrl;
}
public void setFileUrl(String fileUrl) {
this.fileUrl = fileUrl;
}
public String getFileType() {
return fileType;
}
public void setFileType(String fileType) {
this.fileType = fileType;
}
public String getFileDownloadLocation() {
return fileDownloadLocation;
}
public void setFileDownloadLocation(String fileDownloadLocation) {
this.fileDownloadLocation = fileDownloadLocation;
}
}
This is request body
{
"fileName": "SB2",
"fileUrl": "https://drive.google.com/open?id=1_gkQK8sAlgTslzfRGOvNtbEAwtoPeyJv",
"fileType":"mp4",
"fileDownloadLocation": "C:/DownloadedFiles"
}
I went trough the related questions and answers but I could not find a proper solution for this.
Can any one help to sort out the issue ? or any idea about this welcome.
you need a url with extension to download large files (videos) when using nio packages.
URL url = new URL("https://elpvideo.s3-us-west-2.amazonaws.com/4445/4445.mp4");
If your hosting service provides an API to download files with a particular token then nio packages don't work properly.
URL url = new URL("https://fv2-1.failiem.lv/down.php?i=sk2j3yuya");
tip
if you are downloading a large size of the file then make sure to set timeout if not then it will be disconnected in the middle.
URL url = new URL("https://elpvideo.s3-us-west-2.amazonaws.com/4445/4445.mp4");
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setReadTimeout((1000*30)); //30 mins
ReadableByteChannel readableByteChannel = Channels.newChannel(urlConnection.getInputStream());
I am trying to read large file into chunks from S3 without cutting any line for parallel processing.
Let me explain by example:
There is file of size 1G on S3. I want to divide this file into chucks of 64 MB. It is easy I can do it like :
S3Object s3object = s3.getObject(new GetObjectRequest(bucketName, key));
InputStream stream = s3object.getObjectContent();
byte[] content = new byte[64*1024*1024];
while (stream.read(content) != -1) {
//process content here
}
but problem with chunk is it may have 100 complete line and one incomplete. but I can not process incomplete line and don't want to discard it.
Is any way to handle this situations ? means all chucks have no partial line.
My usual approach (InputStream -> BufferedReader.lines() -> batches of lines -> CompletableFuture) won't work here because the underlying S3ObjectInputStream times out eventually for huge files.
So I created a new class S3InputStream, which doesn't care how long it's open for and reads byte blocks on demand using short-lived AWS SDK calls. You provide a byte[] that will be reused. new byte[1 << 24] (16Mb) appears to work well.
package org.harrison;
import java.io.IOException;
import java.io.InputStream;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GetObjectRequest;
/**
* An {#link InputStream} for S3 files that does not care how big the file is.
*
* #author stephen harrison
*/
public class S3InputStream extends InputStream {
private static class LazyHolder {
private static final AmazonS3 S3 = AmazonS3ClientBuilder.defaultClient();
}
private final String bucket;
private final String file;
private final byte[] buffer;
private long lastByteOffset;
private long offset = 0;
private int next = 0;
private int length = 0;
public S3InputStream(final String bucket, final String file, final byte[] buffer) {
this.bucket = bucket;
this.file = file;
this.buffer = buffer;
this.lastByteOffset = LazyHolder.S3.getObjectMetadata(bucket, file).getContentLength() - 1;
}
#Override
public int read() throws IOException {
if (next >= length) {
fill();
if (length <= 0) {
return -1;
}
next = 0;
}
if (next >= length) {
return -1;
}
return buffer[this.next++];
}
public void fill() throws IOException {
if (offset >= lastByteOffset) {
length = -1;
} else {
try (final InputStream inputStream = s3Object()) {
length = 0;
int b;
while ((b = inputStream.read()) != -1) {
buffer[length++] = (byte) b;
}
if (length > 0) {
offset += length;
}
}
}
}
private InputStream s3Object() {
final GetObjectRequest request = new GetObjectRequest(bucket, file).withRange(offset,
offset + buffer.length - 1);
return LazyHolder.S3.getObject(request).getObjectContent();
}
}
The aws-java-sdk already provides streaming functionality for your S3 objects. You have to call "getObject" and the result will be an InputStream.
1) AmazonS3Client.getObject(GetObjectRequest getObjectRequest) -> S3Object
2) S3Object.getObjectContent()
Note: The method is a simple getter and does not actually create a
stream. If you retrieve an S3Object, you should close this input
stream as soon as possible, because the object contents aren't
buffered in memory and stream directly from Amazon S3. Further,
failure to close this stream can cause the request pool to become
blocked.
aws java docs
100 complete line and one incomplete
do you mean you need to read the stream line by line? If so, instead of using a an InputStream try to read the s3 object stream by using BufferedReader so that you can read the stream line by line but I think this will make a little slower than by chunk.
S3Object s3object = s3.getObject(new GetObjectRequest(bucketName, key));
BufferedReader in = new BufferedReader(new InputStreamReader(s3object.getObjectContent()));
String line;
while ((line = in.readLine()) != null) {
//process line here
}
You can read all the files in the bucket with checking the tokens. And you can read files with other java libs.. i.e. Pdf.
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import javax.swing.JTextArea;
import java.io.FileWriter;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import org.joda.time.DateTime;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import java.io.File;
//..
// in your main class
private static AWSCredentials credentials = null;
private static AmazonS3 amazonS3Client = null;
public static void intializeAmazonObjects() {
credentials = new BasicAWSCredentials(ACCESS_KEY, SECRET_ACCESS_KEY);
amazonS3Client = new AmazonS3Client(credentials);
}
public void mainMethod() throws IOException, AmazonS3Exception{
// connect to aws
intializeAmazonObjects();
ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result listObjectsResult;
do {
listObjectsResult = amazonS3Client.listObjectsV2(req);
int count = 0;
for (S3ObjectSummary objectSummary : listObjectsResult.getObjectSummaries()) {
System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());
// Date lastModifiedDate = objectSummary.getLastModified();
// String bucket = objectSummary.getBucketName();
String key = objectSummary.getKey();
String newKey = "";
String newBucket = "";
String resultText = "";
// only try to read pdf files
if (!key.contains(".pdf")) {
continue;
}
// Read the source file as text
String pdfFileInText = readAwsFile(objectSummary.getBucketName(), objectSummary.getKey());
if (pdfFileInText.isEmpty())
continue;
}//end of current bulk
// If there are more than maxKeys(in this case 999 default) keys in the bucket,
// get a continuation token
// and list the next objects.
String token = listObjectsResult.getNextContinuationToken();
System.out.println("Next Continuation Token: " + token);
req.setContinuationToken(token);
} while (listObjectsResult.isTruncated());
}
public String readAwsFile(String bucketName, String keyName) {
S3Object object;
String pdfFileInText = "";
try {
// AmazonS3 s3client = getAmazonS3ClientObject();
object = amazonS3Client.getObject(new GetObjectRequest(bucketName, keyName));
InputStream objectData = object.getObjectContent();
PDDocument document = PDDocument.load(objectData);
document.getClass();
if (!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
pdfFileInText = tStripper.getText(document);
}
} catch (Exception e) {
e.printStackTrace();
}
return pdfFileInText;
}
The #stephen-harrison answer works well. I updated it for v2 of the sdk. I made a couple of tweaks: mainly the connection can now be authorized and the LazyHolder class is no longer static -- I couldn't figure out how to authorize the connection and still keep the class static.
For another approach using Scala, see https://alexwlchan.net/2019/09/streaming-large-s3-objects/
package foo.whatever;
import java.io.IOException;
import java.io.InputStream;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
/**
* Adapted for aws Java sdk v2 by jomofrodo#gmail.com
*
* An {#link InputStream} for S3 files that does not care how big the file is.
*
* #author stephen harrison
*/
public class S3InputStreamV2 extends InputStream {
private class LazyHolder {
String appID;
String secretKey;
Region region = Region.US_WEST_1;
public S3Client S3 = null;
public void connect() {
AwsBasicCredentials awsCreds = AwsBasicCredentials.create(appID, secretKey);
S3 = S3Client.builder().region(region).credentialsProvider(StaticCredentialsProvider.create(awsCreds))
.build();
}
private HeadObjectResponse getHead(String keyName, String bucketName) {
HeadObjectRequest objectRequest = HeadObjectRequest.builder().key(keyName).bucket(bucketName).build();
HeadObjectResponse objectHead = S3.headObject(objectRequest);
return objectHead;
}
// public static final AmazonS3 S3 = AmazonS3ClientBuilder.defaultClient();
}
private LazyHolder lazyHolder = new LazyHolder();
private final String bucket;
private final String file;
private final byte[] buffer;
private long lastByteOffset;
private long offset = 0;
private int next = 0;
private int length = 0;
public S3InputStreamV2(final String bucket, final String file, final byte[] buffer, String appID, String secret) {
this.bucket = bucket;
this.file = file;
this.buffer = buffer;
lazyHolder.appID = appID;
lazyHolder.secretKey = secret;
lazyHolder.connect();
this.lastByteOffset = lazyHolder.getHead(file, bucket).contentLength();
}
#Override
public int read() throws IOException {
if (next >= length || (next == buffer.length && length == buffer.length)) {
fill();
if (length <= 0) {
return -1;
}
next = 0;
}
if (next >= length) {
return -1;
}
return buffer[this.next++] & 0xFF;
}
public void fill() throws IOException {
if (offset >= lastByteOffset) {
length = -1;
} else {
try (final InputStream inputStream = s3Object()) {
length = 0;
int b;
while ((b = inputStream.read()) != -1) {
buffer[length++] = (byte) b;
}
if (length > 0) {
offset += length;
}
}
}
}
private InputStream s3Object() {
final Long rangeEnd = offset + buffer.length - 1;
final String rangeString = "bytes=" + offset + "-" + rangeEnd;
final GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucket).key(file).range(rangeString)
.build();
return lazyHolder.S3.getObject(getObjectRequest);
}
}
Got puzzled while we were migrating from AWS Sdk V1 to V2 and realised in V2 SDK its not the same way to define the range
With AWS V1 SDK
S3Object currentS3Obj = client.getObject(new GetObjectRequest(bucket, key).withRange(start, end));
return currentS3Obj.getObjectContent();
With AWS V2 SDK
var range = String.format("bytes=%d-%d", start, end);
ResponseBytes<GetObjectResponse> currentS3Obj = client.getObjectAsBytes(GetObjectRequest.builder().bucket(bucket).key(key).range(range).build());
return currentS3Obj.asInputStream();
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 7 years ago.
Improve this question
I'm programming a multiple file downloader in Java with JavaFx, but i have some problems with threading.
The problem i have is with the threading part.
I want to start multiple downloads (different urls / files) at the same time, for example two. If i start this two downloadthreads (I think) a race condition happens, because the filename and filesize for both threads are the same and there is also only one file on the HDD, not two as expected.
I am sure it's a race condition problem, but how can i solve it?
Main.java
package de.minimal.program;
import java.util.ArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import de.minimal.program.model.Download;
import de.minimal.program.util.Dl;
import javafx.application.Application;
import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.stage.Stage;
public class Main extends Application {
private ObservableList<Download> downloadData = FXCollections.observableArrayList();
private int i = 0;
public ObservableList<Download> getDownloadData(){
return downloadData;
}
#Override
public void start(Stage primaryStage) {
downloadData.add(new Download("http://mirror.de.leaseweb.net/videolan/vlc/2.2.1/win32/vlc-2.2.1-win32.exe"));
downloadData.add(new Download("http://releases.ubuntu.com/15.10/ubuntu-15.10-desktop-amd64.iso"));
ArrayList<Thread> t = new ArrayList<Thread>();
ExecutorService executor = Executors.newFixedThreadPool(2, new ThreadFactory() {
#Override
public Thread newThread(Runnable r) {
Thread a = new Thread(r);
a.setName("Thread " + i);
i++;
t.add(a);
return a;
}
});
for(Download dl : downloadData){
Dl d = new Dl(dl);
executor.execute(d);
}
}
public static void main(String[] args) {
launch(args);
}
}
DL.java
package de.minimal.program.util;
import java.util.List;
import de.minimal.program.httpconnection.HttpConnection;
import de.minimal.program.model.Download;
import javafx.concurrent.Task;
public class Dl extends Task<List<Download>> implements Runnable{
private Download download;
private HttpConnection connection;
public Dl(Download download){
this.download = download;
}
#Override
protected synchronized List<Download> call() throws Exception {
connection = new HttpConnection(download);
connection.downloadFile();
return null;
}
}
HTTPConnection.java
package de.minimal.program.httpconnection;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import de.minimal.program.model.Download;
public class HttpConnection {
private static String url;
private Download download;
private static final int BUFFER_SIZE = 4096;
public HttpConnection(Download download){
this.download = download;
}
public void downloadFile() throws IOException{
String saveDir = download.getDownloadSavePath();
url = download.getDownloadUrl();
URL obj = new URL(url);
HttpURLConnection connection = (HttpURLConnection) obj.openConnection();
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
// Forbid redirects for file resuming reasons
connection.setInstanceFollowRedirects(false);
int responseCode = connection.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_PARTIAL) {
String fileName = "";
String disposition = connection.getHeaderField("Content-Disposition");
long contentLength = connection.getContentLengthLong();
boolean appendToFile = false;
if(responseCode == HttpURLConnection.HTTP_PARTIAL)
appendToFile = true;
if(download.getFilesize() == 0){
download.setFilesize(contentLength);
}
if (disposition != null) {
// extracts file name from header field
int index = disposition.indexOf("filename=");
if (index > 0) {
fileName = disposition.substring(index + 10,
disposition.length() - 1);
}
} else {
// extracts file name from URL
fileName = url.substring(url.lastIndexOf("/") + 1, url.length());
}
download.setFilename(fileName);
// opens input stream from the HTTP connection
InputStream inputStream = connection.getInputStream();
String saveFilePath = saveDir + File.separator + fileName;
// opens an output stream to save into file
FileOutputStream outputStream = new FileOutputStream(saveFilePath, appendToFile);
int bytesRead = -1;
long downloadedBytes = download.getTransferedBytes();
long start = System.currentTimeMillis();
byte[] buffer = new byte[BUFFER_SIZE];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloadedBytes += bytesRead;
if(System.currentTimeMillis() - start >= 2000){
download.setTransferedBytes(downloadedBytes);
start = System.currentTimeMillis();
}
}
outputStream.close();
inputStream.close();
System.out.println("Thread " + Thread.currentThread().getName() + " Filedownload " + fileName + " finished");
} else {
System.out.println("No file to download. Server replied HTTP code: " + responseCode);
}
connection.disconnect();
}
}
Download.java
package de.minimal.program.model;
import javafx.beans.property.LongProperty;
import javafx.beans.property.SimpleLongProperty;
import javafx.beans.property.SimpleStringProperty;
import javafx.beans.property.StringProperty;
public class Download {
private final StringProperty filename;
private final StringProperty filepath;
private final LongProperty filesize;
private final LongProperty transferedBytes;
private String downloadUrl;
private String downloadSavePath = "SET PATH ";
public Download(){
this("");
}
public Download(String downloadUrl){
this.downloadUrl = downloadUrl;
this.filename = new SimpleStringProperty(downloadUrl);
this.filepath = new SimpleStringProperty(downloadSavePath);
this.filesize = new SimpleLongProperty(0);
this.transferedBytes = new SimpleLongProperty(0);
}
// Filename
public synchronized String getFilename(){
return filename.get();
}
public synchronized void setFilename(String filename){
System.out.println("Thread " + Thread.currentThread().getName() + " Set filename: " + filename);
this.filename.set(filename);
}
public synchronized StringProperty filenameProperty(){
return filename;
}
// Filepath
public String getFilepath(){
return filepath.get();
}
public void setFilepath(String filepath){
System.out.println("Set filepath: " + filepath);
this.filepath.set(filepath);
}
public StringProperty filepathProperty(){
return filepath;
}
// Filesize
public Long getFilesize(){
return filesize.get();
}
public void setFilesize(Long filesize){
System.out.println("Thread " + Thread.currentThread().getName() + " Set filesize: " + filesize);
this.filesize.set(filesize);
}
public LongProperty filesizeProperty(){
return filesize;
}
// TransferedBytes
public Long getTransferedBytes(){
return transferedBytes.get();
}
public void setTransferedBytes(Long transferedBytes){
System.out.println("Thread " + Thread.currentThread().getName() + " bytes transfered " + transferedBytes);
this.transferedBytes.set(transferedBytes);
}
public LongProperty transferedBytesProperty(){
return transferedBytes;
}
// URL
public String getDownloadUrl(){
return downloadUrl;
}
public void setDownloadUrl(String downloadUrl){
this.downloadUrl = downloadUrl;
}
// SavePath
public String getDownloadSavePath(){
return downloadSavePath;
}
public void setDownloadSavePath(String downloadSavePath){
this.downloadSavePath = downloadSavePath;
}
}
EDIT:
This is the minimal code.
You can add links, start and stop downloads, and change the number of simultaneous concurrent downloads.
EDIT 2:
Minified it again. Hope this time its better.
Adds two downloadlinks and starts them immediately. Reproduces the mentioned problem.
EDIT 3:
Solved it.
The problem was the
private static String url;
I remember that my professor told once that static variables are not thread safe. So more information can found here
10 points about Static in Java point 2
Is writing in a single file a real constraint ? What you could do is to write in separate files, then once the files are complete, merge them into a single one.
Alternatively, if the results from the requests are not so big and could fit in memory, you could directly return the files from the downloading threads, and then write them into a single file.
I'm trying to set up a server on aws with simple http server and save each http post request headers & payload.
It works locally.
My steps after connection via ssh to the ec2 server:
javac Server.java
sudo nohup java Server
It saves the headers to log file but not the payload and it doesn't returns 204 response.
Server.java
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Executors;
public class Server {
private static final int PORT = 80;
private static final String FILE_PATH = "/home/ec2-user/logs/";
private static final String UTF8 = "UTF-8";
private static final String DELIMITER = "|||";
private static final String LINE_BREAK = "\n";
private static final String FILE_PREFIX = "dd_MM_YYYY_HH";
private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(FILE_PREFIX);
private static final String FILE_TYPE = ".txt";
public static void main(String[] args) {
try {
HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
server.createContext("/", new HttpHandler() {
#Override
public void handle(HttpExchange t) throws IOException {
System.out.println("Req\t" + t.getRemoteAddress());
InputStream initialStream = t.getRequestBody();
byte[] buffer = new byte[initialStream.available()];
initialStream.read(buffer);
File targetFile = new File(FILE_PATH + simpleDateFormat.format(new Date()) + FILE_TYPE);
OutputStream outStream = new FileOutputStream(targetFile, true);
String prefix = LINE_BREAK + t.getRequestHeaders().entrySet().toString() + LINE_BREAK + System.currentTimeMillis() + DELIMITER;
outStream.write(prefix.getBytes());
Map<String, String> queryPairs = new HashMap<>();
String params = new String(buffer);
String[] pairs = params.split("&");
for (String pair : pairs) {
int idx = pair.indexOf("=");
String key = pair.substring(0, idx);
String val = pair.substring(idx + 1);
String decodedKey = URLDecoder.decode(key, UTF8);
String decodeVal = URLDecoder.decode(val, UTF8);
queryPairs.put(decodedKey, decodeVal);
}
outStream.write(queryPairs.toString().getBytes());
t.sendResponseHeaders(204, -1);
t.close();
}
});
server.setExecutor(Executors.newCachedThreadPool());
server.start();
} catch (Exception e) {
System.out.println("Exception: " + e.getMessage());
e.printStackTrace();
}
}
}
Consider these changes to your handle method. As a starting point, two things are changed:
It reads the complete input and copies that into your file (initialStream.available() might not be the full truth)
catch, log and rethrow IOExceptions (you didn't see your 204 after all)
Consider redirecting your output into files, so you can check what happend on server later:
sudo nohup java Server > server.log 2> server.err &
If you described in more detail the desired target file structure we could figure something out there as well I guess.
#Override
public void handle(HttpExchange t) throws IOException {
try {
System.out.println("Req\t" + t.getRemoteAddress());
InputStream initialStream = t.getRequestBody();
File targetFile = new File(FILE_PATH + simpleDateFormat.format(new Date()) + FILE_TYPE);
OutputStream outStream = new FileOutputStream(targetFile, true);
// This will copy ENTIRE input stream into your target file
IOUtils.copy(initialStream, outStream);
outStream.close();
t.sendResponseHeaders(204, -1);
t.close();
} catch(IOException e) {
e.printStackTrace();
throw e;
}
}