private final String host;
private final String userAccount;
private final String keyDir;
private ChannelSftp sftpChannel;
private Session session;
private Channel channel
public void send(List<Path> filesToSend, String destination) throws SftpException, IOException {
if (sftpChannel == null) {
logger.error("Failed to create SFTP channel");
}
for (Path file : filesToSend) {
send(file, destination);
}
//summary of sent files over sftpchannel
}
public void send(Path file, String destination) throws SftpException, IOException {
if (sftpChannel == null) {
logger.error("Failed to create SFTP channel");
}
sftpChannel.put(Files.newInputStream(file), destination + File.separator + file.getFileName());
} //end send
}//end class
After all the files have been sent, can somebody show me how I can get a count of how many number of files have successfully been sent. Not important but also if any failed or any kind of monitoring. How can I do this with Jsch library.
I would like something in my log such as :
Preparing to send [14] files
Number of files sent is [14]...
public void send(List<Path> filesToSend, String destination) {
logger.debug("Preparing to send ["+filesToSend.size()+"] files");
if (sftpChannel == null) {
logger.error("Failed to create SFTP channel");
}
int successCount = 0;
int failedCount = 0;
for (Path file : filesToSend) {
try {
send(file, destination);
successCount++;
} catch (Exception e) {
failedCount++;
}
}
//summary of sent files over sftpchannel
logger.debug("Successfully sent " + successCount);
logger.debug("Failed to sent " + failedCount);
}
Given that jsch throws exceptions when an error occurs, why don't you just implement your own counter (just an int would work)?
Related
I have Java application connecting to linux over SFTP. I'm using jsch as a framework. Now, let's say I need to rename a file.
public boolean rename(String name) {
boolean result = false;
channelSftp = (ChannelSftp)channel;
LsEntry currentFile = //here I have LsEntry object, pointing to specific record;
logger.info("Renaming CRC file " + currentFile.getFilename() + " to " + name);
try {
//For the first parameter I get current position in the directory, then I append the filename of the currently processed file.
//For the first parameter I get current position in the directory, then I append the new name.
channel.rename(channel.pwd() + currentFile .getFilename(), channel.pwd() + name);
result = true;
} catch (Exception e) {
logger.error("Error renaming crc file to " + name, e);
result = false;
}
return result;
}
Now after renaming the file on the filesystem, I also need to rename the file in the current LsEntry object I'm working with. The problem is that LsEntry doesn't provide such method, so I have to load it again. Now how do I look for it? I need to find specific file so I can use it as updated LsEntry object for later use. Is that possible?
EDIT1:
The LsEntry object, which represents entry on the filesystem has to be created somehow, I do that by casting vector object into it. Like so:
System.out.println("searching for files in following directory" + directory);
channelSftp.cd(directory);
Vector foundFiles = channelSftp.ls(directory);
for(int i=2; i<foundFiles.size();i++){
LsEntry files = (LsEntry) foundFiles.get(i);
System.out.println("found file: " + files.getFilename());
System.out.println("Found file with details : " + files.getLongname());
System.out.println("Found file on path: " + channelSftp.pwd());
channelSftp.rename(channelSftp.pwd() + files.getFilename(), channelSftp.pwd() + "picovina");
//LsEntry has now old name.
public class SftpClient {
private final JSch jSch;
private Session session;
private ChannelSftp channelSftp;
private boolean connected;
public SftpClient() { this.jSch = new JSch(); }
public void connect(final ConnectionDetails details) throws ConnectionException {
try {
if (details.usesDefaultPort()) {
session = jSch.getSession(details.getUserName(), details.getHost());
} else {
session = jSch.getSession(details.getUserName(), details.getHost(), details.getPort());
}
channelSftp = createSftp(session, details.getPassword());
channelSftp.connect();
connected = session.isConnected();
} catch (JSchException e) {
throw new ConnectionException(e.getMessage());
}
}
public void disconnect() {
if (connected) {
channelSftp.disconnect();
session.disconnect();
}
}
public void cd(final String path) throws FileActionException {
try {
channelSftp.cd(path);
} catch (SftpException e) {
throw new FileActionException(e.getMessage());
}
}
public List<FileWrapper> list() throws FileActionException {
try {
return collectToWrapperList(channelSftp.ls("*"));
} catch (SftpException e) {
throw new FileActionException(e.getMessage());
}
}
public String pwd() throws FileActionException {
try {
return channelSftp.pwd();
} catch (SftpException e) {
throw new FileActionException(e.getMessage());
}
}
public boolean rename(final FileWrapper wrapper, final String newFileName) throws FileActionException {
try {
String currentPath = channelSftp.pwd();
channelSftp.rename(currentPath + wrapper.getFileName(), currentPath + newFileName);
} catch (SftpException e) {
throw new FileActionException(e.getMessage());
}
return true;
}
private List<FileWrapper> collectToWrapperList(Vector<ChannelSftp.LsEntry> entries) {
return entries.stream()
.filter(entry -> !entry.getAttrs().isDir())
.map(entry -> FileWrapper.from(entry.getAttrs().getMtimeString(), entry.getFilename(), entry.getAttrs().getSize()))
.collect(Collectors.toList());
}
private ChannelSftp createSftp(final Session session, final String password) throws JSchException {
session.setPassword(password);
Properties properties = new Properties();
properties.setProperty("StrictHostKeyChecking", "no");
session.setConfig(properties);
session.connect();
return (ChannelSftp) session.openChannel("sftp");
}
}
Note here that the list method effectively returns a list of FileWrapper objects instead of LsEntry objects.
public class FileWrapper {
private static final String TIME_FORMAT = "EEE MMM dd HH:mm:ss zzz yyyy";
private Date timeStamp;
public Date getTimeStamp() { return timeStamp; }
private String fileName;
public String getFileName() { return fileName; }
private Long fileSize;
public Long getFileSize() { return fileSize; }
private FileWrapper(String timeStamp, String fileName, Long fileSize) throws ParseException {
this.timeStamp = new SimpleDateFormat(TIME_FORMAT).parse(timeStamp);
this.fileName = fileName;
this.fileSize = fileSize;
}
public static FileWrapper from(final String timeStamp, final String fileName, final Long fileSize) {
try {
return new FileWrapper(timeStamp, fileName, fileSize);
} catch (ParseException e) {
e.printStackTrace();
}
return null;
}
}
With this, you can easily list the remote directory and get all the files' attributes.
With that on hand you can simply invoke SftpClient#rename and rename the file you want.
I know that you want to avoid refactoring, but given the very tight nature or LsEntry as well as the fact that the library still uses Vector and such, I suppose that this is the best way to go (you'll avoid headaches in the future).
I know that this may not be 100% the answer you expect, but I think it's going to be helpful for you.
When download file from ftp server via FTPClient class files comes corrupted check for initial file (img1) and downloaded file (img2)
Source:
public class FtpClient extends FTPClient {
private String host;
private int port;
private String username;
private String password;
private boolean connected;
public FtpClient(String host, int port, String username, String password) {
this.host = host;
this.port = port;
this.username = username;
this.password = password;
connected = connect();
}
private void verifyConnection() throws ConnectionException {
if(!connected){
ConnectionException ex = new ConnectionException();
log.error(ex.getMessage());
throw ex;
}
}
private boolean connect(){
try {
//try to connect
connect(host, port);
if(!login(username, password)){
logout();
return false;
}
int reply = getReplyCode();
if(!FTPReply.isPositiveCompletion(reply)){
disconnect();
return false;
}
enterRemotePassiveMode();
setFileTransferMode(BINARY_FILE_TYPE);
setFileType(BINARY_FILE_TYPE);
log.debug("Remote system is " + getSystemType());
log.debug("Current directory is " + printWorkingDirectory());
return true;
} catch (IOException e) {
log.error(e.getMessage());
e.printStackTrace();
return false;
}
}
public void get(String batchName, String fileName) throws IOException, ConnectionException {
verifyConnection();
File file = new File("/tmp/" + batchName);
if (!file.exists()){
boolean mkdir = file.mkdir();
log.info("Create batch directory '{}' result: {}",batchName, mkdir);
}
OutputStream outputStream = new FileOutputStream("/tmp/" + batchName + "/" + fileName);
boolean result = retrieveFile(batchName + "/" + fileName, outputStream);
log.debug("Retrieve file {}, result: {}", batchName + "/" + fileName, result);
outputStream.close();
}
}
img1
img2
What is going wrong?
You are using an incorrect CONSTANT for FTPClient.setFileTransferMode.
setFileTransferMode(BINARY_FILE_TYPE);
Like the doc says :
mode - The new transfer mode to use (one of the FTP class _TRANSFER_MODE constants).
You can use one of those :
FTP.BLOCK_TRANSFER_MODE
FTP.COMPRESSED_TRANSFER_MODE
FTP.STREAM_TRANSFER_MODE
This will probably correct your problem of transfer, since your are using a value that could be anything ...
You are passing a value of 2 where it is expecting one of [10|11|12]. So It probably keep the default STREAM_TRANSFER_MODE value.
I used FTP and FTPClient in package 'org.apache.commons.net.ftp' to download files from FTP server.
Here is my total example code
public class FtpInput {
private static final Logger LOG = Logger.getLogger(FtpInput.class);
private static final int TIMEOUT = 120000;
private static final String SIZE_COMMAND_REPLY_CODE = "213 ";
/**
* FTPClient
*/
private FTPClient ftpClient;
/**
* FTP size
*/
private long completeFileSize = 0;
protected String ip = "";
protected int port = 21;
protected String user = "";
protected String passwd = "";
protected String path = "";
protected String fileName = "";
/**
* count input bytes
*/
private CountingInputStream is;
/**
* the bytes already processed
*/
private long processedBytesNum;
private byte[] inputBuffer = new byte[1024];
/**
* connect to ftp server and fetch inputStream
*/
public void connect() {
this.ftpClient = new FTPClient();
ftpClient.setRemoteVerificationEnabled(false);
try {
ftpClient.connect(ip, port);
if (!ftpClient.login(user, passwd)) {
throw new IOException("ftp login failed!");
}
if (StringUtils.isNotBlank(path)) {
if (!ftpClient.changeWorkingDirectory(path)) {
ftpClient.mkd(path);
if (!ftpClient.changeWorkingDirectory(path)) {
throw new IOException("ftp change working dir failed! path:" + path);
}
}
}
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
ftpClient.setSoTimeout(TIMEOUT);
ftpClient.setConnectTimeout(TIMEOUT);
ftpClient.setDataTimeout(TIMEOUT);
ftpClient.enterLocalPassiveMode();
// keep control channel keep-alive when download large file
ftpClient.setControlKeepAliveTimeout(120);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException("ftp login failed!", e);
}
// get complete ftp size
completeFileSize = getFtpFileSize();
LOG.info(String.format("ftp file size: %d", completeFileSize));
try {
InputStream ftpis = this.ftpClient.retrieveFileStream(this.fileName);
if (ftpis == null) {
LOG.error("cannot fetch source file.");
}
this.is = new CountingInputStream(ftpis);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
/**
* readBytes
*
* #return
*/
public byte[] readBytes() {
byte[] bytes = readBytesFromStream(is, inputBuffer);
// the bytes processed
processedBytesNum = is.getCount();
return bytes;
}
/**
* readBytesFromStream
*
* #param stream
* #param inputBuffer
* #return
*/
protected byte[] readBytesFromStream(InputStream stream, byte[] inputBuffer) {
Preconditions.checkNotNull(stream != null, "InputStream has not been inited yet.");
Preconditions.checkArgument(inputBuffer != null && inputBuffer.length > 0);
int readBytes;
try {
readBytes = stream.read(inputBuffer);
} catch (IOException e) {
throw new RuntimeException(e);
}
if (readBytes == inputBuffer.length) {
// inputBuffer is filled full.
return inputBuffer;
} else if (readBytes > 0 && readBytes < inputBuffer.length) {
// inputBuffer is not filled full.
byte[] tmpBytes = new byte[readBytes];
System.arraycopy(inputBuffer, 0, tmpBytes, 0, readBytes);
return tmpBytes;
} else if (readBytes == -1) {
// Read end.
return null;
} else {
// may other situation happens?
throw new RuntimeException(String.format("readBytesFromStream: readBytes=%s inputBuffer.length=%s",
readBytes, inputBuffer.length));
}
}
/**
* fetch the byte size of remote file size
*/
private long getFtpFileSize() {
try {
ftpClient.sendCommand("SIZE", this.fileName);
String reply = ftpClient.getReplyString().trim();
LOG.info(String.format("ftp file %s size reply : %s", fileName, reply));
Preconditions.checkArgument(reply.startsWith(SIZE_COMMAND_REPLY_CODE),
"ftp file size reply: %s is not success", reply);
String sizeSubStr = reply.substring(SIZE_COMMAND_REPLY_CODE.length());
long actualFtpSize = Long.parseLong(sizeSubStr);
return actualFtpSize;
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
public void close() {
try {
if (is != null) {
LOG.info(String.format("already read %d bytes from ftp file %s", is.getCount(), fileName));
is.close();
}
if (ftpClient != null) {
// Must call completePendingCommand() to finish command.
boolean isSuccessTransfer = ftpClient.completePendingCommand();
if (!isSuccessTransfer) {
LOG.error("error happened when complete transfer of ftp");
}
ftpClient.logout();
ftpClient.disconnect();
}
} catch (Throwable e) {
e.printStackTrace();
LOG.error(String.format("Close ftp input failed:%s,%s", e.getMessage(), e.getCause()));
} finally {
is = null;
ftpClient = null;
}
}
public void validInputComplete() {
Preconditions.checkArgument(processedBytesNum == completeFileSize, "ftp file transfer is not complete");
}
/**
* main
*
* #param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String ip = "***.***.***.****";
int port = 21;
String user = "***";
String passwd = "***";
String path = "/home/work";
String fileName = "b.txt";
FtpInput input = new FtpInput();
try {
input.fileName = fileName;
input.path = path;
input.ip = ip;
input.port = port;
input.user = user;
input.passwd = passwd;
// connect to FTP server
input.connect();
while (true) {
// read bytes
byte[] bytes = input.readBytes();
if (bytes == null) {
break;
}
LOG.info("read " + bytes.length + " bytes at :" + new Date(System.currentTimeMillis()));
// Attention: this is used for simulating the process of writing data into hive table
// it maybe consume more than 1 minute;
Thread.sleep(3000);
}
input.validInputComplete();
} catch (Exception e) {
e.printStackTrace();
} finally {
input.close();
}
}
}
here is the exception message:
java.net.SocketTimeoutException: Read timed out
or
java.net.SocketException: Connection reset
at stream.readBytes in method readBytesFromStream
At first, i think it probably caused by writing into hive table slowly, and then the FTP Server closed the connection.
But actually, the speed of writing into hive table is fast enough.
Now, i need your help, how can i fix this problem.
From your comments, it looks like it can take hours before you finish downloading the file.
You cannot reasonably expect an FTP server to wait for you for hours to finish the transfer. Particularly if you are not transferring anything most of the time. You waste server resources and most servers will protect themselves against such abuse.
Your design is flawed.
You should redesign your application to first fully download the file; and import the file only after the download finishes.
In case of file upload fails then how to do multiple attempts while doing SFTP by using JSCH API?
How to ensure file is uploaded successfully?
How to create thread-safe file upload utility?
Create a common static utility method which can be invoked from the external class. This method has a map argument to persist the values of sFTPUser, sFTPHost, sFTPPort, sFTPPwd, destinationLocation and uploadedFileName :
public static void doSFTP(Map<String, String> ftpParameters) {
if (ftpParameters.get("ID_NAME").equals(
NAPSCommonConstants.MFT_NAPSGPCS_INTF)) {
// do sftp for NAPS GPCS Interface.
uploadUsingSFTP(ftpParameters);
}
}
Use synchronized method to ensure thread safety:
private static synchronized void uploadUsingSFTP(
Map<String, String> ftpPrameterList) {
new SFTPUtility().uploadFileMFT(ftpPrameterList.get("sFTPUser"),
ftpPrameterList.get("sFTPHost"), new Integer(ftpPrameterList
.get("sFTPPort")), ftpPrameterList.get("sFTPPwd"),
ftpPrameterList.get("sourceLocation"), ftpPrameterList
.get("destinationLocation"), ftpPrameterList
.get("uploadedFileName"));
}
Responsible method to upload files using SFTP with 5 attempts:
private void uploadFileMFT(String sFTPUser, String sFTPHost, int sFTPPort,
String sFTPPwd, String sourceLocation, String destinationLocation,
String uploadedFileName) {
LOG.info("Inside uploadFileMFT to upload and verify the file.");
JSch jsch = new JSch();
Vector<String> fileList = null;
/** 5 re-attempt logic to get session */
int attempts = 0;
boolean successfulConnect;
do {
try {
successfulConnect = true;
session = jsch.getSession(sFTPUser, sFTPHost, sFTPPort);
LOG.debug("session connected ...");
session.setPassword(sFTPPwd);
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
LOG.debug("Sftp Session connected ...");
channel = session.openChannel("sftp");
LOG.debug("Sftp Channel opened ...");
channelSftp = (ChannelSftp) channel;
channelSftp.connect();
LOG.info(" Sftp channel opened and connected ...");
channelSftp.put(sourceLocation, destinationLocation);
fileList = channelSftp.ls(destinationLocation);
} catch (JSchException e) {
++attempts;
successfulConnect = false;
LOG.error(e);
} catch (SftpException e) {
++attempts;
successfulConnect = false;
LOG.error(e);
} finally {
if (null != channelSftp) {
channelSftp.exit();
LOG.debug(" sftp Channel exited.");
}
if (null != channel) {
channel.disconnect();
LOG.debug(" Channel disconnected.");
}
if (null != session) {
session.disconnect();
LOG.debug(" Host Session disconnected.");
}
}
} while (attempts < 5 && successfulConnect == false);
fileUploadValidation(fileList, uploadedFileName);
LOG.info("Exiting from method - uploadFileMFT ...");
}
Finally uploaded file can be validated:
private void fileUploadValidation (Vector<String> fileList, String uploadedFileName){
boolean isFileExistis = false;
Object[] objArr = fileList.toArray();
for (int i = 0; i < objArr.length; i++) {
String fileName = objArr[i].toString();
isFileExistis = fileName.contains(uploadedFileName);
if (isFileExistis) {
LOG.info("Uploaded file '" + uploadedFileName + "' was transferred successfull ...");
break;
}else if(i >= objArr.length){
LOG.info("Uploaded file '" + uploadedFileName + "' was failed to transfer ...");
}
}
}
I use the JSch library to develop a SFTP client.
The problem is both get and put methods' status is -1.
Here is my code:
class SftpClient {
private static final Logger LOG = Logger.getLogger(SftpClient.class);
/** Connection port number */
public static final int PORT = 22;
/** SECURED protocol name */
private static final String PROTOCOL = "sftp";
/** Connection time out in milliseconds */
public static final int TIME_OUT = 5000;
/** This class serves as a central configuration point, and as a factory for Session objects configured with these settings */
private JSch _client;
/** A session represents a connection to a SSH server */
private Session _session;
/** Channel connected to a SECURED server (as a subsystem of the SSH server) */
private ChannelSftp _channelSftp;
/**
* Value returned by the last executed command.
*/
private int _exitValue;
/**
* Computer contains the url, the login and the password to connect.
*/
private Computer _computer;
/**
* Initialize a SECURED client
* #param target - Machine we want to connect to
*/
public SftpClient(Computer target) {
_client = new JSch();
_computer = target;
}
protected void connect() throws Exception {
try {
if (_client == null) {
_client = new JSch();
}
if (_session == null) {
_session = _client.getSession(_computer.getLogin(), _computer.getUrl(), PORT);
Properties props = new Properties();
props.put("StrictHostKeyChecking", "no");
props.put("compression.s2c", "zlib,none");
props.put("compression.c2s", "zlib,none");
_session.setConfig(props);
_session.setPassword(_computer.getPassword());
if (LOG.isDebugEnabled()) {
LOG.debug("Connecting to "+_computer.getUrl()+" with login "+_computer.getLogin()+"...");
}
}
if (!_session.isConnected()) {
_session.connect(TIME_OUT);
}
// disconnect previous channel if it has not been killed properly
if (_channelSftp != null && _channelSftp.isConnected()) {
_channelSftp.disconnect();
}
_channelSftp = (ChannelSftp) _session.openChannel(PROTOCOL);
_channelSftp.connect();
if (LOG.isInfoEnabled()) {
LOG.info("Connected to "+_computer.getUrl()+" with login "+_computer.getLogin());
}
} catch(JSchException e) {
LOG.error("Auth failed", e);
throw e;
}
}
protected void connect(String path) throws Exception {
connect();
if (_channelSftp != null && _channelSftp.isConnected()) {
_channelSftp.cd(path);
}
}
public boolean get(final String remoteDirectory, final String remoteFile, final String localDirectory) throws Exception {
boolean res = false;
if (LOG.isInfoEnabled()) {
LOG.info("Download file "+remoteDirectory+"/"+remoteFile+" from "+_computer+" in "+localDirectory);
}
if (remoteDirectory != null && remoteFile != null && !remoteFile.isEmpty() && localDirectory != null) {
try {
// connect to the server and change directory
connect(remoteDirectory);
// change local directory
_channelSftp.lcd(localDirectory);
// download the file, keeping the same name
_channelSftp.get(remoteFile, remoteFile);
// update exit value
_exitValue = _channelSftp.getExitStatus();
if (_exitValue == 0) {
res = true;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Exit status is: "+_exitValue);
}
} catch(SftpException e){
LOG.error("Auth failed", e);
throw e;
} finally {
if (_channelSftp != null && _channelSftp.isConnected()) {
_channelSftp.disconnect();
_channelSftp.exit();
}
}
} else {
LOG.warn("Check remoteDirectory ('"+remoteDirectory+"') or remoteFile ('"+remoteFile+"') or localDirectory ('"+localDirectory+"').");
}
return res;
}
public void put(final File localFile, final String destPath) throws Exception {
if (LOG.isInfoEnabled()) {
LOG.info("Send file "+localFile+" to "+_computer+" in "+destPath);
}
if (localFile == null) {
_exitValue = -1;
LOG.error("The given local file is null. Aborting tranfer.");
return;
}
if (!localFile.exists()) {
_exitValue = -1;
LOG.error("The given local file '"+localFile+"' does not exist. Aborting tranfer.");
return;
}
final InputStream input = new FileInputStream(localFile);
if (input == null || input.available() <= 0) {
_exitValue = -1;
LOG.error("Cannot read file "+localFile);
return;
}
try {
connect(destPath);
_channelSftp.put(input, localFile.getName());
_exitValue = _channelSftp.getExitStatus();
if (LOG.isDebugEnabled()) {
LOG.debug("Exit status is: "+_exitValue);
}
} catch(SftpException e){
LOG.error("Auth failed", e);
throw e;
} finally {
if (_channelSftp != null && _channelSftp.isConnected()) {
_channelSftp.disconnect();
_channelSftp.exit();
}
IOUtils.closeQuietly(input);
}
}
public void disconnect() {
if (_channelSftp != null && _channelSftp.isConnected()) {
_channelSftp.disconnect();
_channelSftp.exit();
}
if (_session != null && _session.isConnected()) {
_session.disconnect();
if (LOG.isInfoEnabled()) {
LOG.info("SECURED FTP disconnected");
}
}
}
}
No exception is thrown.
By the way, the files I want to upload and download are tar files. Does Jsch have a FTP binary mode?
Files are well transferred. I can use them properly but I am worried about the exit status.
I had a look to the Jsch API and it says:
The exit status is only available for certain types of channels, and only after the channel was closed (more exactly, just before the channel is closed).
How can I know if the exit status is available for ChannelSftp?
Looking at the source code it looks ExitStatus is not implemented for SFTP
channel.setExitStatus(reason_code);
is implemented for only below two cases in Session class
case SSH_MSG_CHANNEL_OPEN_FAILURE:
case SSH_MSG_CHANNEL_REQUEST:
and both the cases are not called atleast for successful scenarios that I tested.