Develop Reference

fetch data from ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/ using java

how can I fetch movie.list.gz folder from the link
1: ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/, I have used a code to access the FTP server for fetching data but it asks for port no. which I don't know. Can anyone please tell how to fetch data from the above link in java. I am developing a Web Application which will display all the movie names present in IMDB database.
CODE:
String server = "ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com";
int port = 21;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(server);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
// APPROACH #1: using retrieveFile(String, OutputStream)
String remoteFile1 = "/pub/movie.list.gz";
File downloadFile1 = new File("F:/softwares/mov.list.gz");
OutputStream outputStream1 = new BufferedOutputStream(new FileOutputStream(downloadFile1));
boolean success = ftpClient.retrieveFile(remoteFile1, outputStream1);
outputStream1.close();
if (success) {
System.out.println("File #1 has been downloaded successfully.");
}
} catch (IOException ex) {
System.out.println("Error: " + ex.getMessage());
ex.printStackTrace();
} finally {
try {
if (ftpClient.isConnected()) {
ftpClient.logout();
ftpClient.disconnect();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
}

FTP file successfully transferred but is not true

Here is my simple code:
public boolean sendCSV() throws IOException {
System.out.println("Translating from xls to csv...");
File xls = new File("file.xls");
File csvFile;
csvFile = CSVManager.xlsToCsv(xls);
csvFile.renameTo(new File("file.csv"));
FTPClient client = new FTPClient();
FileInputStream fis = null;
try {
System.out.println("Establishing connection...");
client.connect(ftpHost, ftpPort);
System.out.print(client.getReplyString());
System.out.println("Connection ok.");
if (client.login(ftpUser, ftpPass)) {
System.out.println("Login ok");
System.out.print(client.getReplyString());
System.out.println("Setting PASV");
client.enterLocalPassiveMode();
System.out.print(client.getReplyString());
if (client.changeWorkingDirectory("/MYDIR")) {
System.out.println("Dir changed");
System.out.print(client.getReplyString());
} else {
System.out.println("Error changing dir");
System.out.print(client.getReplyString());
}
String filename = csvFile.getName();
TrayIconManager.SwitchTrayToFull();
fis = new FileInputStream(filename);
if (client.storeFile(filename, fis)) {
System.out.println("File sent");
System.out.print(client.getReplyString());
} else {
System.out.println("Error during sending file");
System.out.print(client.getReplyString());
}
}
if (client.logout()) {
System.out.println("Logout ok");
System.out.print(client.getReplyString());
return true;
} else {
System.out.println("Logout problem");
System.out.print(client.getReplyString());
return false;
}
} catch (IOException e) {
e.printStackTrace();
return false;
} finally {
try {
if (fis != null) {
fis.close();
}
client.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
}
}
What is the problem? The output says that
File sent
226-29 Kbytes used (0%) - authorized: 512000 Kb
226-File successfully transferred
226 0.343 seconds (measured here), 30.59 Kbytes per second
but is not true, because once I check on the ftp server the file is not there!
Why Kbytes used is 0%? Maybe is that the error?
The code seems to be ok, also the output.
So what is the real problem?

Client-server Java programm using sockets stucks while runnig, but works while debbuging

I have created a simple utility, that implements server and client. Server waits for incomming connection, client connects and if there are any files in folder on server - they are sending to client.
I used DataInput/OutputStreams - writeUTF/readUTF for sending messages between client and server, writeLong/readLong - to send file size, write/read to send data.
The problem is that when I run both client and server - working stucks after sending one or two files. But if I trace the client and server in IDE step-by-step it works great, and it also works if I transfer files with buffer size in one byte - so it works really slowly, but it does not stuck.
I use Eclipse IDE, jdk 1.7 on client side and 1.8 on server side (tried 1.7 also), Ubuntu linux and Windows 2003 Server on two instances of VMWare players.
I also tried to add Thread.sleep(10000) and flush() anything might be flushed, even added socket.setTcpNoDelay(true); but no result.
It looks like server sends the next file name and waits for echo() from client, but client does not get it and stucks on readUTF(). But, if I do it in manual mode step-by-step it works fine.
Any ideas? Thank you!
Client source:
/**
* Method receives a file from server
* #param outWriter
* #param inBufferedReader
* #param inStream
*
*/
private void receiveFileFromServer(DataOutputStream out, DataInputStream in) {
try {
String fileName = in.readUTF(); ///<----- stucks here *************
if (awlConnectionLogger.isInfoEnabled()) {
awlConnectionLogger.info("Receiving file "+fileName);
}
out.writeUTF(fileName);
out.flush();
Long fileSize = in.readLong();
if (awlConnectionLogger.isInfoEnabled()) {
awlConnectionLogger.info("Filesize "+fileSize);
}
out.writeUTF(fileSize.toString());
out.flush();
File localFile = new File(fileName);
FileOutputStream fileOS = new FileOutputStream(localFile);
byte[] buffer = new byte [2048];
int bytesRead = 0;
try {
for (int i = 0; i<fileSize/buffer.length; i++) {
bytesRead = in.read(buffer, 0, buffer.length);
fileOS.write(buffer,0, bytesRead);
}
bytesRead = in.read(buffer, 0, (int)(fileSize%buffer.length));
fileOS.write(buffer,0, bytesRead);
System.out.println("----------------------- "+bytesRead);
} catch (IOException e) {
awlConnectionLogger.error("Error reading file "+fileName);
};
fileOS.close();
if (awlConnectionLogger.isInfoEnabled()) {
awlConnectionLogger.info("File received.");
}
} catch (IOException e) {
awlConnectionLogger.error("Error reading Stream or socket closed");
}
}
/**
* Method sends a #param command to server, waits for answer, then if answer is not null and equals #param answer it logs #param messageOk else #param messageError
* returns true if everything is ok, else returns false
* #param outWriter
* #param inBufferedReader
* #throws IOException
*/
private boolean sendCommandReceiveAnswer(String command, String answer, String messageOk,String messageError,
DataOutputStream out, DataInputStream in) throws IOException {
out.writeUTF(command); //Hello handshake
out.flush();
String data = in.readUTF();
if ((data!=null)&&(data.equals(answer))) {
if (awlConnectionLogger.isInfoEnabled()) {
awlConnectionLogger.info(messageOk);
}
return true;
} else {
awlConnectionLogger.error(messageError);
return false;
}
}
/**
* Try to establish connection with the awl-server according to detected RDP session
* #param serverIP - ip address of server
* #param user - username
*/
private void establishConnection(String serverIP, String user) {
if (awlConnectionLogger.isInfoEnabled()) {
awlConnectionLogger.info("Trying to establish awl-connection to "+serverIP+" as "+user);
}
Integer remoteAwlPort = Integer.parseInt(Config.awlPort);
try (Socket awlServerSocket = new Socket(serverIP,remoteAwlPort)) {//Creating autocloseable socket
this.awlServerSocket = awlServerSocket;
DataOutputStream out = new DataOutputStream(awlServerSocket.getOutputStream());
DataInputStream in = new DataInputStream(awlServerSocket.getInputStream());
if (!sendCommandReceiveAnswer("Hello-awl-client", "Hello-awl-server", "Server hello OK.",
"Unknown server type. Closing thread.", out, in)) {
return;
}; //Hello handshake - return if not successful
if (!sendCommandReceiveAnswer(user, "User-ok", "Username sent.", "Error sending username. Closing thread.", out, in)) {
return;
} //Sending username
String hostName = InetAddress.getLocalHost().getHostName(); //Getting client local hostname
if (!sendCommandReceiveAnswer(hostName, "Hostname-ok", "Local hostname "+hostName+" sent. Ready to receive files.",
"Error sending hostname. Closing thread.", out, in)) {
return;
} //Sending hostname
awlServerSocket.setTcpNoDelay(true);
while (isActive) {
receiveFileFromServer(out, in); //receiving files
out.writeUTF("Ready");
out.flush();
}
} catch (UnknownHostException e) {
awlConnectionLogger.error("Error in server IP adress");
} catch (SocketException e) {
awlConnectionLogger.error("Problem accessing or creating Socket.");
} catch (IOException e) {
awlConnectionLogger.error("General IO Error or Socket closed");
};
}
Server source:
void Echo (DataInputStream in) {
String echo = null;
try {
echo = in.readUTF();
System.out.println("Echo:"+echo);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void sendToClient (File file, DataOutputStream out, DataInputStream in) {
System.out.println("Sending file name "+file.getName());
try {
out.writeUTF(file.getName());
out.flush();
Echo (in); //<--------- stucks here *******************
out.writeLong(file.length());
out.flush();
Echo (in);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Sending file");
byte [] buffer = new byte [2048];
FileInputStream fileIS = null;
try {
fileIS = new FileInputStream(file);
} catch (FileNotFoundException e) {
System.out.println("Error opening file "+file);
e.printStackTrace();
}
try {
Integer bytesRead;
int i = 0;
while((bytesRead=fileIS.read(buffer))>0) {
out.write(buffer, 0, bytesRead);
}
out.flush();
} catch (IOException e) {
System.out.println("Error reading file "+file);
e.printStackTrace();
};
System.out.println("File sent.");
try {
fileIS.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void run () {
DataOutputStream out = null;
DataInputStream in = null;
String data = null;
String user = null;
String clientHostName = null;
try {
socket.setTcpNoDelay(true);
out = new DataOutputStream(socket.getOutputStream());
in = new DataInputStream(socket.getInputStream());
} catch (IOException e1) {
e1.printStackTrace();
}
System.out.println("New incoming connection thread starded.");
try {
data = in.readUTF();
if (data.equals("Hello-awl-client")) {
System.out.println("Client hello OK.");
out.writeUTF("Hello-awl-server");
out.flush();
user = in.readUTF();
System.out.println("User: "+user);
out.writeUTF("User-ok");
out.flush();
clientHostName = in.readUTF();
System.out.println("Client hostname: "+clientHostName);
System.out.println("Server hostname: "+InetAddress.getLocalHost().getHostName());
out.writeUTF("Hostname-ok");
out.flush();
System.out.println("Ready to send files.");
while (isActive) {
File file = new File(ServerConfig.pdfFolder+"//"+user);
if (!file.isDirectory()) {
System.out.println("Error in path to user directory. Exiting thread");
return;
}
File [] files = file.listFiles();
for (File fileItem:files) {
while (!fileItem.renameTo(fileItem)) {
Thread.sleep(1000);
}; //waits while file is not busy to send it
if (socket.isOutputShutdown()) {
out = new DataOutputStream(socket.getOutputStream());
}
sendToClient (fileItem, out, in);
if (fileItem.delete()) {
System.out.println("File deleted.");
} else {
System.out.println("File not deleted.");
}
data = in.readUTF();
System.out.println(data);
}
}
} else {
System.out.println("Unknown connection type. Closing thread.");
return;
}
} catch (IOException | InterruptedException e) {
System.out.println("Connection error or can't sleep thread. Closing thread.");
return;
}
}

JAVA Commons-Net FTPClient, downloading files are corrupted

I am using commans.net api in order to perform tasks e.g uploading and
downloading files from the ftp server. I can perform those tasks
successfully but the downloaded files are corrupted. i couldn't open
those files in usual manner. please help me..
my simple code look like
public class FTPConnect {
public static void main(String[] args) {
startServer();
connectClient();
}
private static void startServer() {
FtpServerFactory serverFactory = new FtpServerFactory();
ListenerFactory factory = new ListenerFactory();
factory.setPort(21);// set the port of the listener (choose your desired
// port, not 1234)
System.out.println("port has been set to 21");
serverFactory.addListener("default", factory.createListener());
PropertiesUserManagerFactory userManagerFactory = new PropertiesUserManagerFactory();
userManagerFactory.setFile(new File("lib/users.properties"));
System.out.println("users.properties has been set..");
userManagerFactory.setPasswordEncryptor(new PasswordEncryptor() {
#Override
public String encrypt(String password) {
return password;
}
#Override
public boolean matches(String passwordToCheck,
String storedPassword) {
return passwordToCheck.equals(storedPassword);
}
});
System.out.println("password has been encrypted...");
BaseUser user = new BaseUser();
user.setName("java");
user.setPassword("shiva.dave");
System.out.println("password has been set..to java and shiva.dave");
user.setHomeDirectory("lib");
System.out.println("home directory has been set...");
List<Authority> authorities = new ArrayList<Authority>();
authorities.add(new WritePermission());
user.setAuthorities(authorities);
UserManager um = userManagerFactory.createUserManager();
try {
um.save(user);// Save the user to the user list on the filesystem
System.out.println("user has been set to the filesystem..");
} catch (FtpException e1) {
e1.printStackTrace();
}
serverFactory.setUserManager(um);
FtpServer server = serverFactory.createServer();
try
{
server.start();//Your FTP server starts listening for incoming FTP-connections, using the configuration options previously set
System.out.println("Server has been started.....");
}
catch (FtpException ex)
{
ex.printStackTrace();
}
}
private static void connectClient() {
FTPClient client = new FTPClient();
try{
client.connect(InetAddress.getLocalHost(), 21);
String loging_success = client.login("java", "shiva.dave") == true ? "success" : "failed";
System.out.println("login: " + loging_success);
FTPFile[] clients = client.listFiles("/");
System.out.println("Listed " + clients.length + " files.");
for (FTPFile file : clients) {
System.out.println(file.getName());
}
for (FTPFile ftpFile : clients) {
String remoteFile2 = ftpFile.getName();
File downloadFile2 = new File("D:/test2/"+ftpFile.getName());
OutputStream outputStream2 = new BufferedOutputStream(new FileOutputStream(downloadFile2));
InputStream inputStream = client.retrieveFileStream(remoteFile2);
byte[] bytesArray = new byte[4096];
int bytesRead = -1;
while ((bytesRead = inputStream.read(bytesArray)) != -1) {
outputStream2.write(bytesArray, 0, bytesRead);
}
boolean success = client.completePendingCommand();
if (success) {
System.out.println("File #2 has been downloaded successfully.");
}
outputStream2.close();
inputStream.close();
}
}
catch(Exception e){
e.printStackTrace();
}
finally{
try{
client.logout();
client.disconnect();
}
catch(Exception e){
e.printStackTrace();
}
}
}
}

I founded a solution and want to put that here..
first of all when you try to download files from the ftp server you must need to specify the file type and transfer mode in order to download file successfully.
here is the code you need to insert.
client.setFileType(FTP.BINARY_FILE_TYPE, FTP.BINARY_FILE_TYPE);
client.setFileTransferMode(FTP.BINARY_FILE_TYPE);
"WATCH IT GUYS"
you must need to insert these two lines of code after successfully logged in otherwise you wouldn't be able to download files successfully...
Hope this answer will help you to download files without being corrupted..

How to download a text file via FTP and read the contents to a string

This is re-worded from a previous question (which was probably a bit unclear).
I want to download a text file via FTP from a remote server, read the contents of the text file into a string and then discard the file. I don't need to actually save the file.
I am using the Apache Commons library so I have:
import org.apache.commons.net.ftp.FTPClient;
Can anyone help please, without simply redirecting me to a page with lots of possible answers on?

Not going to do the work for you, but once you have your connection established, you can call retrieveFile and pass it an OutputStream. You can google around and find the rest...
FTPClient ftp = new FTPClient();
...
ByteArrayOutputStream myVar = new ByteArrayOutputStream();
ftp.retrieveFile("remoteFileName.txt", myVar);
ByteArrayOutputStream
retrieveFile

Normally I'd leave a comment asking 'What have you tried?'. But now I'm feeling more generous :-)
Here you go:
private void ftpDownload() {
FTPClient ftp = null;
try {
ftp = new FTPClient();
ftp.connect(mServer);
try {
int reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
throw new Exception("Connect failed: " + ftp.getReplyString());
}
if (!ftp.login(mUser, mPassword)) {
throw new Exception("Login failed: " + ftp.getReplyString());
}
try {
ftp.enterLocalPassiveMode();
if (!ftp.setFileType(FTP.BINARY_FILE_TYPE)) {
Log.e(TAG, "Setting binary file type failed.");
}
transferFile(ftp);
} catch(Exception e) {
handleThrowable(e);
} finally {
if (!ftp.logout()) {
Log.e(TAG, "Logout failed.");
}
}
} catch(Exception e) {
handleThrowable(e);
} finally {
ftp.disconnect();
}
} catch(Exception e) {
handleThrowable(e);
}
}
private void transferFile(FTPClient ftp) throws Exception {
long fileSize = getFileSize(ftp, mFilePath);
InputStream is = retrieveFileStream(ftp, mFilePath);
downloadFile(is, buffer, fileSize);
is.close();
if (!ftp.completePendingCommand()) {
throw new Exception("Pending command failed: " + ftp.getReplyString());
}
}
private InputStream retrieveFileStream(FTPClient ftp, String filePath)
throws Exception {
InputStream is = ftp.retrieveFileStream(filePath);
int reply = ftp.getReplyCode();
if (is == null
|| (!FTPReply.isPositivePreliminary(reply)
&& !FTPReply.isPositiveCompletion(reply))) {
throw new Exception(ftp.getReplyString());
}
return is;
}
private byte[] downloadFile(InputStream is, long fileSize)
throws Exception {
byte[] buffer = new byte[fileSize];
if (is.read(buffer, 0, buffer.length)) == -1) {
return null;
}
return buffer; // <-- Here is your file's contents !!!
}
private long getFileSize(FTPClient ftp, String filePath) throws Exception {
long fileSize = 0;
FTPFile[] files = ftp.listFiles(filePath);
if (files.length == 1 && files[0].isFile()) {
fileSize = files[0].getSize();
}
Log.i(TAG, "File size = " + fileSize);
return fileSize;
}

You can just skip the download to local filesystem part and do:
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(server, port);
ftpClient.login(user, pass);
ftpClient.enterLocalPassiveMode();
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
InputStream inputStream = ftpClient.retrieveFileStream("/folder/file.dat");
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "Cp1252"));
while(reader.ready()) {
System.out.println(reader.readLine()); // Or whatever
}
inputStream.close();
} catch (IOException ex) {
ex.printStackTrace();
} finally {
try {
if (ftpClient.isConnected()) {
ftpClient.logout();
ftpClient.disconnect();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}