Develop Reference

fetch data from ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/ using java

how can I fetch movie.list.gz folder from the link
1: ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/, I have used a code to access the FTP server for fetching data but it asks for port no. which I don't know. Can anyone please tell how to fetch data from the above link in java. I am developing a Web Application which will display all the movie names present in IMDB database.
CODE:
String server = "ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com";
int port = 21;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(server);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
// APPROACH #1: using retrieveFile(String, OutputStream)
String remoteFile1 = "/pub/movie.list.gz";
File downloadFile1 = new File("F:/softwares/mov.list.gz");
OutputStream outputStream1 = new BufferedOutputStream(new FileOutputStream(downloadFile1));
boolean success = ftpClient.retrieveFile(remoteFile1, outputStream1);
outputStream1.close();
if (success) {
System.out.println("File #1 has been downloaded successfully.");
}
} catch (IOException ex) {
System.out.println("Error: " + ex.getMessage());
ex.printStackTrace();
} finally {
try {
if (ftpClient.isConnected()) {
ftpClient.logout();
ftpClient.disconnect();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
}

Java FTP Upload Completed but not returning

Good day. I have imported org.apache.commons.net.ftp.FTP to my project and I faced some issues when upload the zip file to FTP.
When it uploads small file(<100MB), it works fine. But when it comes to larger file (>500MB), it will stops there, without returning and execute the remaining operation, even it the file is transferred to the server.
And how the function / FTP acknowledge when the transfer is done? Is there anyway to check it?
public class zipFTP
{
public boolean uploadFile(String ht, String usr, String ps, String fpath, String uploadloc)
{
FTPClient client = new FTPClient();
FileInputStream fis = null;
writeLog nLog = new writeLog();
String channel = "FTP";
boolean completed = false;
//client.setBufferSize(1048576);
client.setControlKeepAliveReplyTimeout(300);
//System.out.println(client.getBufferSize());
try
{
nLog.writeToLog(dumptoFTP.filename, channel, "Uploading...");
client.connect(ht);
client.login(usr, ps);
client.enterLocalPassiveMode();
client.setFileType(FTP.BINARY_FILE_TYPE);
fis = new FileInputStream(fpath);
completed = client.storeFile(uploadloc, fis);
//client.completePendingCommand(); //to complete the transaction entirely
if(completed)
{
if (dumptoFTP.isSysLog)
System.out.println("File uploaded");
nLog.writeToLog(dumptoFTP.filename, channel, "File uploaded");
}
fis.close();
}
catch (IOException e)
{
e.printStackTrace();
if (dumptoFTP.isSysLog)
System.out.println("Upload failed");
nLog.writeToLog(dumptoFTP.filename, channel, "Upload failed");
nLog.writeToLog(dumptoFTP.filename, channel, e.toString());
}
finally
{
try
{
if(client.isConnected())
{
client.logout();
client.disconnect();
}
}
catch (IOException ex)
{
ex.printStackTrace();
}
}
return completed;
}
}

Ended up, setBufferSize() does the trick

FTP File upload Exception 'Software caused connection abort: recv failed' -JAVA

I am trying to upload multiple directories to FTP but after 4-5 minutes of uploading stared I am getting 'Software caused connection abort: recv failed' exception only a few files got uploaded. I checked the FTP Time out settings it is set to 10 minutes and also tried both enterLocalPassiveMode() and sendNoOp() but didn't work please help me to fix this.
public class FTPUpload {
public void initFtpupload(String localparentDir,String batchno) //not in use
{
try {
readProperty();
CreateFile();
localDirPath=localparentDir;
lotno=batchno;
FTPUpload ftpobj = new FTPUpload(ftpIp, ftpPort, ftpUname, ftpPassword);
System.out.println("Local Dir PAth:"+localparentDir);
System.out.println("Local dir path: "+localDirPath);
ftp.changeWorkingDirectory(ftpRootDir);
if(true)
{
IterateDirectory(localDirPath);
}
ftpobj.disconnect();
} catch (Exception e) {
String error=stackTraceToString(e);
genLog("Exception in InitFtpUpload: "+e);
System.out.println("Exception:" + e);
genLog("Exception Trace:" + error);
}
}
// Constructor to connect to the FTP Server
public FTPUpload(String host, int port, String username, String password) throws Exception {
java.util.Date dtCurDate = new java.util.Date();
SimpleDateFormat sdfDate = new SimpleDateFormat("dd_MM_yyyy_hh_mm_ss");
String strCurDate = sdfDate.format(dtCurDate);
String strLogPath = System.getProperty("user.dir") + File.separator + "Log";
String strLogFilePath = strLogPath + File.separator + "FTPConnectionLog_" + strCurDate + ".log";
File logfile=new File(strLogFilePath);
ftp = new FTPClient();
//ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out)));
ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(logfile)));
int reply;
ftp.connect(host, port);
System.out.println("FTP URL is:" + ftp.getDefaultPort());
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
genLog("Exception in connecting to FTP Server");
throw new Exception("Exception in connecting to FTP Server");
}
ftp.login(username, password);
ftp.setFileType(FTP.BINARY_FILE_TYPE);
ftp.enterLocalPassiveMode();
genLog("Connected");
System.out.println("Connected ");
}
// list the files in a specified directory on the FTP
public static boolean IterateDirectory(String localRootDirectoryPath) throws IOException {
// lists files and directories in the current working directory
//for test
ftp.changeWorkingDirectory(ftpRootDir);
System.out.println("Inside IterateDirectory ");
FTPFile dirs[]=ftp.listFiles();
for(FTPFile f:dirs)
{
System.out.println("####"+f);
}
//upto this...
boolean verificationFilename = false;
String newFtpDirectoy = "";
File localfile = new File(localRootDirectoryPath);
File files[] = localfile.listFiles();
for (File file : files) {
String details = file.getName();
if (file.isDirectory()) {
IterateDirectory(file.getPath());
iflag=false;
} else {
if (iflag == false) {
localRootDirectoryPath = file.getParent();
newFtpDirectoy = localRootDirectoryPath.substring(localRootDirectoryPath.indexOf("HUB"));
newFtpDirectoy = newFtpDirectoy.replaceAll("\\\\", "/");
if (true) {
makeDirectories(ftp, newFtpDirectoy);
String localParenDir = file.getParent();
genLog("ftpDirPath: "+ftpDirPath);
genLog("New FTP Dir: "+newFtpDirectoy);
genLog("Local parent dir: "+localParenDir);
System.out.println("ftpDirPath: "+ftpDirPath);
System.out.println("New FTP Dir: "+newFtpDirectoy);
System.out.println("Local parent dir: "+localParenDir);
uploadDirectory(ftp, ftpDirPath, localParenDir, newFtpDirectoy);
mailist.add(localRootDirectoryPath);
File f=new File(localRootDirectoryPath);
String finalPath=f.getParent();
if(!mailistlocal.contains(finalPath)){
mailistlocal.add(finalPath);
}
}
iflag = true;
}
}
}
return true;
}
// Disconnect the connection to FTP
public void disconnect() {
if (this.ftp.isConnected()) {
try {
this.ftp.logout();
this.ftp.disconnect();
} catch (IOException f) {
// do nothing as file is already saved to server
}
System.out.println("FTP Disconnected successfully");
genLog("FTP Disconnected successfully");
}
}
}
}
}
}

You might want to consider using an existing library rather than creating your own. For example: https://commons.apache.org/proper/commons-net/
https://commons.apache.org/proper/commons-net/javadocs/api-1.4.1/org/apache/commons/net/ftp/FTPClient.html
EDIT (Added working example):
This code worked for me using the jar file found here: https://commons.apache.org/proper/commons-net/download_net.cgi
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import org.apache.commons.net.tftp.TFTP;
import org.apache.commons.net.tftp.TFTPClient;
public class FtpExample {
public static void main(String[] args) throws Exception {
String fileName = "C:\\TEMP\\MyTestFile.txt";
String hostName = "speedtest.tele2.net";
String remoteFileName = "upload/MyTestFile.txt";
File file = new File(fileName);
System.out.println("Sending file: " + file.getName() + "(exists: " + file.exists() + ")");
sendFile(file, hostName, remoteFileName);
System.out.println("Done!");
}
private static void sendFile(File file, String hostName, String remoteFileName) {
TFTPClient tftp = null;
InputStream inputStream = null;
try {
inputStream = new FileInputStream(file);
tftp = new TFTPClient();
tftp.open();
tftp.sendFile(remoteFileName, TFTP.BINARY_MODE, inputStream, hostName);
} catch (Exception exp) {
throw new RuntimeException(exp);
} finally {
try {
if (tftp != null) {
tftp.close();
}
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception exp) {
throw new RuntimeException(exp);
}
}
}
}
}
}

I got the same error
Software caused connection abort
From a FTP client perspective, the server was closing my connection after successful authentication, because my ip was no whitelisted in the ftp server cloud provider.
After whitelisting my public IP, I could connect to FTP server succesfully.

Socket programming, multithreaded programming. Error: IOException: java.net.ConnectException: Connection timed out: connect

I'm a beginner at java and also for socket programming.
This program I've written is aimed to allow p2p file synchronisation between client and server.
When a peer establishes a TCP connection with another peer, both of them will compare the lists of files they possess and proceed to exchange files so that they both have exactly the
same files. A TCP connection must remain open until the peer process is manually terminated at either end of the connection.
I've looked at online tutorials and similar programs to write the code for this program. However, after running the server, the following error is shown on the console when I run the client.
IOException: java.net.ConnectException: Connection timed out: connect
Please advice me on how I could solve the error. Thanks so much in advance!
Here is my code for reference.
Server:
package bdn;
import java.net.*;
import java.io.*;
import java.util.*;
public class MultipleSocketServer extends Thread{
// private Socket connection;
private String TimeStamp;
private int ID;
static int port = 6789;
static ArrayList<File> currentfiles = new ArrayList<File>();
static ArrayList<File> missingsourcefiles = new ArrayList<File>();
public static void main(String[] args)
{
File allFiles = new File("src/bdn/files");
if (!allFiles.exists()) {
if (allFiles.mkdir()) {
System.out.println("Directory is created.");
}
}
//Load the list of files in the directory
listOfFiles(allFiles);
try {
new MultipleSocketServer().startServer();
} catch (Exception e) {
System.out.println("I/O failure: " + e.getMessage());
e.printStackTrace();
}
}
public static void listOfFiles(final File folder){
for (final File fileEntry : folder.listFiles()) {
if (fileEntry.isDirectory()) {
listOfFiles(fileEntry);
} else {
//System.out.println(fileEntry.getName());
currentfiles.add(fileEntry.getAbsoluteFile());
}
}
}
public void startServer() throws Exception {
ServerSocket serverSocket = null;
boolean listening = true;
try {
serverSocket = new ServerSocket(port);
} catch (IOException e) {
System.err.println("Could not listen on port: " + port);
System.exit(-1);
}
while (listening) {
handleClientRequest(serverSocket);
}
//serverSocket.close();
}
private void handleClientRequest(ServerSocket serverSocket) {
try {
new ConnectionRequestHandler(serverSocket.accept()).run();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Handles client connection requests.
*/
public class ConnectionRequestHandler implements Runnable{
private Socket _socket = null;
public ConnectionRequestHandler(Socket socket) {
_socket = socket;
}
public void run() {
System.out.println("Client connected to socket: " + _socket.toString());
try {
ObjectOutputStream oos = new ObjectOutputStream (_socket.getOutputStream());
oos.flush();
oos.writeObject(currentfiles); //sending to client side.
oos.flush();
} catch (IOException e) {
e.printStackTrace();
}
syncMissingFiles();
}
#SuppressWarnings("unchecked")
public void syncMissingFiles(){
try {
ObjectInputStream ois = new ObjectInputStream(_socket.getInputStream());
System.out.println("Is the socket connected="+_socket.isConnected());
missingsourcefiles= (ArrayList<File>) ois.readObject();
System.out.println(missingsourcefiles);
//add missing files to current file list.
ListIterator<File> iter = missingsourcefiles.listIterator();
File temp_file;
while (iter.hasNext()) {
// System.out.println(iter.next());
temp_file = iter.next();
currentfiles.add(temp_file);
}
} catch (IOException e) {
e.printStackTrace();
} catch ( ClassNotFoundException e) {
e.printStackTrace();
}
}
}
}
Client:
package bdn;
import java.net.*;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.ListIterator;
/* The java.io package contains the basics needed for IO operations. */
import java.io.*;
public class SocketClient {
/** Define a port */
static int port = 2000;
static Socket peer_socket = null;
static String peerAddress;
static ArrayList<File> currentfiles = new ArrayList<File>();
static ArrayList<File> sourcefiles = new ArrayList<File>();
static ArrayList<File> missingsourcefiles = new ArrayList<File>();
static ArrayList<File> missingcurrentfiles = new ArrayList<File>();
SocketClient(){}
public static void listOfFiles(final File folder) {
for (final File fileEntry : folder.listFiles()) {
if (fileEntry.isDirectory()) {
listOfFiles(fileEntry);
} else {
//System.out.println(fileEntry.getName());
currentfiles.add(fileEntry.getAbsoluteFile());
}
}
}
#SuppressWarnings("unchecked")
public static void getListfromPeer() {
try {
ObjectInputStream inList =new ObjectInputStream(peer_socket.getInputStream());
sourcefiles = (ArrayList<File>) inList.readObject();
} catch (ClassNotFoundException e) {
System.err.println("Error in data type.");
}catch (IOException classNot){
System.err.println("Error in data type.");
}
}
public static void compareList1() {
//Compare the source files and current files. If not in current files, add the files to "missingcurrentfiles".
ListIterator<File> iter = sourcefiles.listIterator();
File temp_file;
while (iter.hasNext()) {
// System.out.println(iter.next());
temp_file = iter.next();
if (!currentfiles.contains(temp_file)) //file cannot be found
{
missingcurrentfiles.add(temp_file);
}
}
}
public static void compareList2() {
//Compare the source files and current files. If not in current files, add the files to "missingsourcefiles".
ListIterator<File> iter = currentfiles.listIterator();
File temp_file;
while (iter.hasNext()) {
// System.out.println(iter.next());
temp_file = iter.next();
if (!sourcefiles.contains(temp_file)) //file cannot be found
{
missingsourcefiles.add(temp_file);
}
}
}
public static void main(String[] args) {
//Make file lists of the directory in here.
File allFiles = new File("src/bdn/files");
if (!allFiles.exists()) {
if (allFiles.mkdir()) {
System.out.println("Directory is created.");
}
}
/*Get the list of files in that directory and store the names in array*/
listOfFiles(allFiles);
/*Connect to peer*/
try {
new SocketClient().transfer();
//new TcpClient().checkForInput();
} catch (Exception e) {
System.out.println("Failed at main" + e.getMessage());
e.printStackTrace();
}
}
public void transfer(){
getPeerAddress();
// StringBuffer instr = new StringBuffer();
// String TimeStamp;
try {
/** Obtain an address object of the server */
// InetAddress address = InetAddress.getByName(host);
// System.out.println("Address of connected host:"+ address);
/** Establish a socket connection */
Socket connection = new Socket(peerAddress, port);
System.out.println("New SocketClient initialized");
getListfromPeer();
compareList1();
compareList2();
System.out.println(missingcurrentfiles);
System.out.println(missingsourcefiles);
/** Instantiate a BufferedOutputStream object */
// BufferedOutputStream bos = new BufferedOutputStream(connection.getOutputStream());
/** Instantiate an ObjectOutputStream*/
ObjectOutputStream oos = new ObjectOutputStream(connection.getOutputStream());
oos.flush();
/*
TimeStamp = new java.util.Date().toString();
String process = "Calling the Socket Server on "+ host + " port " + port +
" at " + TimeStamp + (char) 13;
*//** Write across the socket connection and flush the buffer *//*
oos.writeObject(process);
oos.flush();*/
oos.writeObject(missingsourcefiles);
oos.flush();
System.out.println("Missing files in source sent back.");
}
catch (IOException f) {
System.out.println("IOException: " + f);
}
catch (Exception g) {
System.out.println("Exception: " + g);
}
}
public void syncMissingFiles(){
//add missing files to current file list.
ListIterator<File> iter = missingcurrentfiles.listIterator();
File temp_file;
while (iter.hasNext()) {
// System.out.println(iter.next());
temp_file = iter.next();
currentfiles.add(temp_file);
}
}
public static void getPeerAddress() {
// prompt the user to enter the client's address
System.out.print("Please enter the IP address of the client :");
// open up standard input
BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
// read the IP address from the command-line
try {
peerAddress = br.readLine();
} catch (IOException ioe) {
System.out.println("IO error trying to read client's IP address!");
System.exit(1);
}
System.out.println("Thanks for the client's IP address, " + peerAddress);
}
}

From what I've noticed, and I'm not an expert, you set your server to listen on port 6789 while your client connect to port 2000.
They should both work on the same port.
Try do the following in your client constructor.
SocketClient(String servername){
//while port is same as server port
peer_socket = new Socket(servername, port);
}
You can set servername in your main to be "localhost" in case you are working locally
Also, I would recommand to verify the port you are working on is not blocked by any firewall existing on your machine.

How to download a text file via FTP and read the contents to a string

This is re-worded from a previous question (which was probably a bit unclear).
I want to download a text file via FTP from a remote server, read the contents of the text file into a string and then discard the file. I don't need to actually save the file.
I am using the Apache Commons library so I have:
import org.apache.commons.net.ftp.FTPClient;
Can anyone help please, without simply redirecting me to a page with lots of possible answers on?

Not going to do the work for you, but once you have your connection established, you can call retrieveFile and pass it an OutputStream. You can google around and find the rest...
FTPClient ftp = new FTPClient();
...
ByteArrayOutputStream myVar = new ByteArrayOutputStream();
ftp.retrieveFile("remoteFileName.txt", myVar);
ByteArrayOutputStream
retrieveFile

Normally I'd leave a comment asking 'What have you tried?'. But now I'm feeling more generous :-)
Here you go:
private void ftpDownload() {
FTPClient ftp = null;
try {
ftp = new FTPClient();
ftp.connect(mServer);
try {
int reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
throw new Exception("Connect failed: " + ftp.getReplyString());
}
if (!ftp.login(mUser, mPassword)) {
throw new Exception("Login failed: " + ftp.getReplyString());
}
try {
ftp.enterLocalPassiveMode();
if (!ftp.setFileType(FTP.BINARY_FILE_TYPE)) {
Log.e(TAG, "Setting binary file type failed.");
}
transferFile(ftp);
} catch(Exception e) {
handleThrowable(e);
} finally {
if (!ftp.logout()) {
Log.e(TAG, "Logout failed.");
}
}
} catch(Exception e) {
handleThrowable(e);
} finally {
ftp.disconnect();
}
} catch(Exception e) {
handleThrowable(e);
}
}
private void transferFile(FTPClient ftp) throws Exception {
long fileSize = getFileSize(ftp, mFilePath);
InputStream is = retrieveFileStream(ftp, mFilePath);
downloadFile(is, buffer, fileSize);
is.close();
if (!ftp.completePendingCommand()) {
throw new Exception("Pending command failed: " + ftp.getReplyString());
}
}
private InputStream retrieveFileStream(FTPClient ftp, String filePath)
throws Exception {
InputStream is = ftp.retrieveFileStream(filePath);
int reply = ftp.getReplyCode();
if (is == null
|| (!FTPReply.isPositivePreliminary(reply)
&& !FTPReply.isPositiveCompletion(reply))) {
throw new Exception(ftp.getReplyString());
}
return is;
}
private byte[] downloadFile(InputStream is, long fileSize)
throws Exception {
byte[] buffer = new byte[fileSize];
if (is.read(buffer, 0, buffer.length)) == -1) {
return null;
}
return buffer; // <-- Here is your file's contents !!!
}
private long getFileSize(FTPClient ftp, String filePath) throws Exception {
long fileSize = 0;
FTPFile[] files = ftp.listFiles(filePath);
if (files.length == 1 && files[0].isFile()) {
fileSize = files[0].getSize();
}
Log.i(TAG, "File size = " + fileSize);
return fileSize;
}

You can just skip the download to local filesystem part and do:
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(server, port);
ftpClient.login(user, pass);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
InputStream inputStream = ftpClient.retrieveFileStream("/folder/file.dat");
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "Cp1252"));
while(reader.ready()) {
System.out.println(reader.readLine()); // Or whatever
}
inputStream.close();
} catch (IOException ex) {
ex.printStackTrace();
} finally {
try {
if (ftpClient.isConnected()) {
ftpClient.logout();
ftpClient.disconnect();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}