Develop Reference

Downloading images from AWS gives me corrupt files

I'm trying to download images that are hosted on Amazon Web Services. My methods work fine on any other host, but downloading an image off this url for example http://s3-eu-west-1.amazonaws.com/static.melkweg.nl/uploads/images/scaled/event_header/18226 is giving me trouble. It does download, but the file is only 49kb big and cannot be opened.
I've tried different methods such as Apache's FileUtils copyURLToFile, BufferedInputStream, ImageIO, etc. Some throw errors, most just download a corrupt file.
Here are the methods I've tried:
public static void downloadApache(String imageurl, String target)
{
try
{
File file = new File(target);
URL url = new URL(imageurl);
FileUtils.copyURLToFile(url, file);
}
catch(Exception e)
{
System.err.println("[3]Something went wrong.");
}
}
public static void downloadImage(String imageurl, String name)
{
try
{
URL url = new URL(imageurl);
InputStream in = new BufferedInputStream(url.openStream());
OutputStream out = new BufferedOutputStream(new FileOutputStream(name));
for ( int i; (i = in.read()) != -1; ) {
out.write(i);
}
in.close();
out.close();
}
catch(Exception e)
{
e.printStackTrace();
System.err.println("[0]Something went wrong.");
}
}
public static void downloadImageIO(String imageurl, String target)
{
try
{
URL url = new URL(imageurl);
BufferedImage image = ImageIO.read(url);
ImageIO.write(image, "jpg", new File(target));
}
catch(Exception e)
{
e.printStackTrace();
System.err.println("[1]Something went wrong.");
}
}
public static void downloadImageCopy(String imageurl, String target)
{
try
{
try (InputStream in = new URL(imageurl).openStream()) {
Files.copy(in, Paths.get(target), StandardCopyOption.REPLACE_EXISTING);
}
}
catch(Exception e)
{
e.printStackTrace();
System.err.println("[2]Something went wrong.");
}
}
And here's the main method if that is of any interest
public static void main(String[] args)
{
String imageurl = "http://s3-eu-west-1.amazonaws.com/static.melkweg.nl/uploads/images/scaled/event_header/18226";
String name = "downloaded_image.jpg";
String target = "C:/Users/Robotic/Downloads/" + name;
Download.downloadImage(imageurl, name);
Download.downloadImageCopy(imageurl, target);
Download.downloadImageIO(imageurl, target);
Download.downloadApache(imageurl, target);
}
Thanks in advance.

The file that you are getting from S3 is gzip compressed, you need to decompress it before trying to read it.
$ wget http://s3-eu-west-1.amazonaws.com/static.melkweg.nl/uploads/images/scaled/event_header/18226
$ file 18226
18226: gzip compressed data, from Unix

As pointed out in the earlier answer, it is in gzip format.
You can use the following method and get the file unzipped
public static void downloadApache(String imageurl, String target) {
try {
File file = new File(target+".gzip");
URL url = new URL(imageurl);
FileUtils.copyURLToFile(url, file);
byte[] buffer = new byte[1024];
try {
java.util.zip.GZIPInputStream gzis = new java.util.zip.GZIPInputStream(new FileInputStream(file));
FileOutputStream out = new FileOutputStream(target);
int len;
while ((len = gzis.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
gzis.close();
out.close();
} catch (IOException ex) {
ex.printStackTrace();
}
} catch (Exception e) {
System.err.println("[3]Something went wrong.");
}
}

FTP File upload Exception 'Software caused connection abort: recv failed' -JAVA

I am trying to upload multiple directories to FTP but after 4-5 minutes of uploading stared I am getting 'Software caused connection abort: recv failed' exception only a few files got uploaded. I checked the FTP Time out settings it is set to 10 minutes and also tried both enterLocalPassiveMode() and sendNoOp() but didn't work please help me to fix this.
public class FTPUpload {
public void initFtpupload(String localparentDir,String batchno) //not in use
{
try {
readProperty();
CreateFile();
localDirPath=localparentDir;
lotno=batchno;
FTPUpload ftpobj = new FTPUpload(ftpIp, ftpPort, ftpUname, ftpPassword);
System.out.println("Local Dir PAth:"+localparentDir);
System.out.println("Local dir path: "+localDirPath);
ftp.changeWorkingDirectory(ftpRootDir);
if(true)
{
IterateDirectory(localDirPath);
}
ftpobj.disconnect();
} catch (Exception e) {
String error=stackTraceToString(e);
genLog("Exception in InitFtpUpload: "+e);
System.out.println("Exception:" + e);
genLog("Exception Trace:" + error);
}
}
// Constructor to connect to the FTP Server
public FTPUpload(String host, int port, String username, String password) throws Exception {
java.util.Date dtCurDate = new java.util.Date();
SimpleDateFormat sdfDate = new SimpleDateFormat("dd_MM_yyyy_hh_mm_ss");
String strCurDate = sdfDate.format(dtCurDate);
String strLogPath = System.getProperty("user.dir") + File.separator + "Log";
String strLogFilePath = strLogPath + File.separator + "FTPConnectionLog_" + strCurDate + ".log";
File logfile=new File(strLogFilePath);
ftp = new FTPClient();
//ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out)));
ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(logfile)));
int reply;
ftp.connect(host, port);
System.out.println("FTP URL is:" + ftp.getDefaultPort());
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
genLog("Exception in connecting to FTP Server");
throw new Exception("Exception in connecting to FTP Server");
}
ftp.login(username, password);
ftp.setFileType(FTP.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
genLog("Connected");
System.out.println("Connected ");
}
// list the files in a specified directory on the FTP
public static boolean IterateDirectory(String localRootDirectoryPath) throws IOException {
// lists files and directories in the current working directory
//for test
ftp.changeWorkingDirectory(ftpRootDir);
System.out.println("Inside IterateDirectory ");
FTPFile dirs[]=ftp.listFiles();
for(FTPFile f:dirs)
{
System.out.println("####"+f);
}
//upto this...
boolean verificationFilename = false;
String newFtpDirectoy = "";
File localfile = new File(localRootDirectoryPath);
File files[] = localfile.listFiles();
for (File file : files) {
String details = file.getName();
if (file.isDirectory()) {
IterateDirectory(file.getPath());
iflag=false;
} else {
if (iflag == false) {
localRootDirectoryPath = file.getParent();
newFtpDirectoy = localRootDirectoryPath.substring(localRootDirectoryPath.indexOf("HUB"));
newFtpDirectoy = newFtpDirectoy.replaceAll("\\\\", "/");
if (true) {
makeDirectories(ftp, newFtpDirectoy);
String localParenDir = file.getParent();
genLog("ftpDirPath: "+ftpDirPath);
genLog("New FTP Dir: "+newFtpDirectoy);
genLog("Local parent dir: "+localParenDir);
System.out.println("ftpDirPath: "+ftpDirPath);
System.out.println("New FTP Dir: "+newFtpDirectoy);
System.out.println("Local parent dir: "+localParenDir);
uploadDirectory(ftp, ftpDirPath, localParenDir, newFtpDirectoy);
mailist.add(localRootDirectoryPath);
File f=new File(localRootDirectoryPath);
String finalPath=f.getParent();
if(!mailistlocal.contains(finalPath)){
mailistlocal.add(finalPath);
}
}
iflag = true;
}
}
}
return true;
}
// Disconnect the connection to FTP
public void disconnect() {
if (this.ftp.isConnected()) {
try {
this.ftp.logout();
this.ftp.disconnect();
} catch (IOException f) {
// do nothing as file is already saved to server
}
System.out.println("FTP Disconnected successfully");
genLog("FTP Disconnected successfully");
}
}
}
}
}
}

You might want to consider using an existing library rather than creating your own. For example: https://commons.apache.org/proper/commons-net/
https://commons.apache.org/proper/commons-net/javadocs/api-1.4.1/org/apache/commons/net/ftp/FTPClient.html
EDIT (Added working example):
This code worked for me using the jar file found here: https://commons.apache.org/proper/commons-net/download_net.cgi
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import org.apache.commons.net.tftp.TFTP;
import org.apache.commons.net.tftp.TFTPClient;
public class FtpExample {
public static void main(String[] args) throws Exception {
String fileName = "C:\\TEMP\\MyTestFile.txt";
String hostName = "speedtest.tele2.net";
String remoteFileName = "upload/MyTestFile.txt";
File file = new File(fileName);
System.out.println("Sending file: " + file.getName() + "(exists: " + file.exists() + ")");
sendFile(file, hostName, remoteFileName);
System.out.println("Done!");
}
private static void sendFile(File file, String hostName, String remoteFileName) {
TFTPClient tftp = null;
InputStream inputStream = null;
try {
inputStream = new FileInputStream(file);
tftp = new TFTPClient();
tftp.open();
tftp.sendFile(remoteFileName, TFTP.BINARY_MODE, inputStream, hostName);
} catch (Exception exp) {
throw new RuntimeException(exp);
} finally {
try {
if (tftp != null) {
tftp.close();
}
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception exp) {
throw new RuntimeException(exp);
}
}
}
}
}
}

I got the same error
Software caused connection abort
From a FTP client perspective, the server was closing my connection after successful authentication, because my ip was no whitelisted in the ftp server cloud provider.
After whitelisting my public IP, I could connect to FTP server succesfully.

JAVA Commons-Net FTPClient, downloading files are corrupted

I am using commans.net api in order to perform tasks e.g uploading and
downloading files from the ftp server. I can perform those tasks
successfully but the downloaded files are corrupted. i couldn't open
those files in usual manner. please help me..
my simple code look like
public class FTPConnect {
public static void main(String[] args) {
startServer();
connectClient();
}
private static void startServer() {
FtpServerFactory serverFactory = new FtpServerFactory();
ListenerFactory factory = new ListenerFactory();
factory.setPort(21);// set the port of the listener (choose your desired
// port, not 1234)
System.out.println("port has been set to 21");
serverFactory.addListener("default", factory.createListener());
PropertiesUserManagerFactory userManagerFactory = new PropertiesUserManagerFactory();
userManagerFactory.setFile(new File("lib/users.properties"));
System.out.println("users.properties has been set..");
userManagerFactory.setPasswordEncryptor(new PasswordEncryptor() {
#Override
public String encrypt(String password) {
return password;
}
#Override
public boolean matches(String passwordToCheck,
String storedPassword) {
return passwordToCheck.equals(storedPassword);
}
});
System.out.println("password has been encrypted...");
BaseUser user = new BaseUser();
user.setName("java");
user.setPassword("shiva.dave");
System.out.println("password has been set..to java and shiva.dave");
user.setHomeDirectory("lib");
System.out.println("home directory has been set...");
List<Authority> authorities = new ArrayList<Authority>();
authorities.add(new WritePermission());
user.setAuthorities(authorities);
UserManager um = userManagerFactory.createUserManager();
try {
um.save(user);// Save the user to the user list on the filesystem
System.out.println("user has been set to the filesystem..");
} catch (FtpException e1) {
e1.printStackTrace();
}
serverFactory.setUserManager(um);
FtpServer server = serverFactory.createServer();
try
{
server.start();//Your FTP server starts listening for incoming FTP-connections, using the configuration options previously set
System.out.println("Server has been started.....");
}
catch (FtpException ex)
{
ex.printStackTrace();
}
}
private static void connectClient() {
FTPClient client = new FTPClient();
try{
client.connect(InetAddress.getLocalHost(), 21);
String loging_success = client.login("java", "shiva.dave") == true ? "success" : "failed";
System.out.println("login: " + loging_success);
FTPFile[] clients = client.listFiles("/");
System.out.println("Listed " + clients.length + " files.");
for (FTPFile file : clients) {
System.out.println(file.getName());
}
for (FTPFile ftpFile : clients) {
String remoteFile2 = ftpFile.getName();
File downloadFile2 = new File("D:/test2/"+ftpFile.getName());
OutputStream outputStream2 = new BufferedOutputStream(new FileOutputStream(downloadFile2));
InputStream inputStream = client.retrieveFileStream(remoteFile2);
byte[] bytesArray = new byte[4096];
int bytesRead = -1;
while ((bytesRead = inputStream.read(bytesArray)) != -1) {
outputStream2.write(bytesArray, 0, bytesRead);
}
boolean success = client.completePendingCommand();
if (success) {
System.out.println("File #2 has been downloaded successfully.");
}
outputStream2.close();
inputStream.close();
}
}
catch(Exception e){
e.printStackTrace();
}
finally{
try{
client.logout();
client.disconnect();
}
catch(Exception e){
e.printStackTrace();
}
}
}
}

I founded a solution and want to put that here..
first of all when you try to download files from the ftp server you must need to specify the file type and transfer mode in order to download file successfully.
here is the code you need to insert.
client.setFileType(FTP.BINARY_FILE_TYPE, FTP.BINARY_FILE_TYPE);
client.setFileTransferMode(FTP.BINARY_FILE_TYPE);
"WATCH IT GUYS"
you must need to insert these two lines of code after successfully logged in otherwise you wouldn't be able to download files successfully...
Hope this answer will help you to download files without being corrupted..

How can I list all image files and upload them to FTP?

How can I easy do that:
I have correctly saved all file.directories here: ArrayList fileList = new ArrayList();
And how can I upload them using this method?
public static FTPClient con = null;
public static void upload(String path, String fileName)
{
try
{
con = new FTPClient();
con.connect("server");
if (con.login("login", "pass"))
{
con.enterLocalPassiveMode(); // important!
con.setFileType(FTP.BINARY_FILE_TYPE);
String data = path;
FileInputStream in = new FileInputStream(new File(data));
boolean result = con.storeFile("/Test/" + fileName, in);
in.close();
con.logout();
con.disconnect();
}
}
catch (Exception e)
{
e.printStackTrace();
}
}

How to download a text file via FTP and read the contents to a string

This is re-worded from a previous question (which was probably a bit unclear).
I want to download a text file via FTP from a remote server, read the contents of the text file into a string and then discard the file. I don't need to actually save the file.
I am using the Apache Commons library so I have:
import org.apache.commons.net.ftp.FTPClient;
Can anyone help please, without simply redirecting me to a page with lots of possible answers on?

Not going to do the work for you, but once you have your connection established, you can call retrieveFile and pass it an OutputStream. You can google around and find the rest...
FTPClient ftp = new FTPClient();
...
ByteArrayOutputStream myVar = new ByteArrayOutputStream();
ftp.retrieveFile("remoteFileName.txt", myVar);
ByteArrayOutputStream
retrieveFile

Normally I'd leave a comment asking 'What have you tried?'. But now I'm feeling more generous :-)
Here you go:
private void ftpDownload() {
FTPClient ftp = null;
try {
ftp = new FTPClient();
ftp.connect(mServer);
try {
int reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
throw new Exception("Connect failed: " + ftp.getReplyString());
}
if (!ftp.login(mUser, mPassword)) {
throw new Exception("Login failed: " + ftp.getReplyString());
}
try {
ftp.enterLocalPassiveMode();
if (!ftp.setFileType(FTP.BINARY_FILE_TYPE)) {
Log.e(TAG, "Setting binary file type failed.");
}
transferFile(ftp);
} catch(Exception e) {
handleThrowable(e);
} finally {
if (!ftp.logout()) {
Log.e(TAG, "Logout failed.");
}
}
} catch(Exception e) {
handleThrowable(e);
} finally {
ftp.disconnect();
}
} catch(Exception e) {
handleThrowable(e);
}
}
private void transferFile(FTPClient ftp) throws Exception {
long fileSize = getFileSize(ftp, mFilePath);
InputStream is = retrieveFileStream(ftp, mFilePath);
downloadFile(is, buffer, fileSize);
is.close();
if (!ftp.completePendingCommand()) {
throw new Exception("Pending command failed: " + ftp.getReplyString());
}
}
private InputStream retrieveFileStream(FTPClient ftp, String filePath)
throws Exception {
InputStream is = ftp.retrieveFileStream(filePath);
int reply = ftp.getReplyCode();
if (is == null
|| (!FTPReply.isPositivePreliminary(reply)
&& !FTPReply.isPositiveCompletion(reply))) {
throw new Exception(ftp.getReplyString());
}
return is;
}
private byte[] downloadFile(InputStream is, long fileSize)
throws Exception {
byte[] buffer = new byte[fileSize];
if (is.read(buffer, 0, buffer.length)) == -1) {
return null;
}
return buffer; // <-- Here is your file's contents !!!
}
private long getFileSize(FTPClient ftp, String filePath) throws Exception {
long fileSize = 0;
FTPFile[] files = ftp.listFiles(filePath);
if (files.length == 1 && files[0].isFile()) {
fileSize = files[0].getSize();
}
Log.i(TAG, "File size = " + fileSize);
return fileSize;
}

You can just skip the download to local filesystem part and do:
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(server, port);
ftpClient.login(user, pass);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
InputStream inputStream = ftpClient.retrieveFileStream("/folder/file.dat");
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "Cp1252"));
while(reader.ready()) {
System.out.println(reader.readLine()); // Or whatever
}
inputStream.close();
} catch (IOException ex) {
ex.printStackTrace();
} finally {
try {
if (ftpClient.isConnected()) {
ftpClient.logout();
ftpClient.disconnect();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}