Develop Reference

Android - Zip Path Traversal in play store

I am uploading my App on play store but get me bellow error:
Zip Path Traversal Your app contains an unsafe unzipping pattern that
may lead to a Path Traversal vulnerability. Please see this Google
Help Center article to learn how to fix the issue.
org.apache.cordova.Zip.unzipSync
I edited my source code like this LINK, but get me error.
Here is my source code changed:
public class Zip extends CordovaPlugin {
private static final String LOG_TAG = "Zip";
// Can't use DataInputStream because it has the wrong endian-ness.
private static int readInt(InputStream is) throws IOException {
int a = is.read();
int b = is.read();
int c = is.read();
int d = is.read();
return a | b << 8 | c << 16 | d << 24;
}
#Override
public boolean execute(String action, CordovaArgs args, final CallbackContext callbackContext) throws JSONException {
if ("unzip".equals(action)) {
unzip(args, callbackContext);
return true;
}
return false;
}
private void unzip(final CordovaArgs args, final CallbackContext callbackContext) {
this.cordova.getThreadPool().execute(new Runnable() {
public void run() {
unzipSync(args, callbackContext);
}
});
}
private void unzipSync(CordovaArgs args, CallbackContext callbackContext) {
InputStream inputStream = null;
try {
String zipFileName = args.getString(0);
String outputDirectory = args.getString(1);
// Since Cordova 3.3.0 and release of File plugins, files are accessed via cdvfile://
// Accept a path or a URI for the source zip.
Uri zipUri = getUriForArg(zipFileName);
Uri outputUri = getUriForArg(outputDirectory);
CordovaResourceApi resourceApi = webView.getResourceApi();
File tempFile = resourceApi.mapUriToFile(zipUri);
if (tempFile == null || !tempFile.exists()) {
String errorMessage = "Zip file does not exist";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage);
return;
}
File outputDir = resourceApi.mapUriToFile(outputUri);
outputDirectory = outputDir.getAbsolutePath();
outputDirectory += outputDirectory.endsWith(File.separator) ? "" : File.separator;
if (outputDir == null || (!outputDir.exists() && !outputDir.mkdirs())) {
String errorMessage = "Could not create output directory";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage);
return;
}
OpenForReadResult zipFile = resourceApi.openForRead(zipUri);
ProgressEvent progress = new ProgressEvent();
progress.setTotal(zipFile.length);
inputStream = new BufferedInputStream(zipFile.inputStream);
inputStream.mark(10);
int magic = readInt(inputStream);
if (magic != 875721283) { // CRX identifier
inputStream.reset();
} else {
// CRX files contain a header. This header consists of:
// * 4 bytes of magic number
// * 4 bytes of CRX format version,
// * 4 bytes of public key length
// * 4 bytes of signature length
// * the public key
// * the signature
// and then the ordinary zip data follows. We skip over the header before creating the ZipInputStream.
readInt(inputStream); // version == 2.
int pubkeyLength = readInt(inputStream);
int signatureLength = readInt(inputStream);
inputStream.skip(pubkeyLength + signatureLength);
progress.setLoaded(16 + pubkeyLength + signatureLength);
}
// The inputstream is now pointing at the start of the actual zip file content.
ZipInputStream zis = new ZipInputStream(inputStream);
inputStream = zis;
ZipEntry ze;
byte[] buffer = new byte[32 * 1024];
boolean anyEntries = false;
while ((ze = zis.getNextEntry()) != null) {
try {
anyEntries = true;
String compressedName = ze.getName();
if (ze.isDirectory()) {
try {
File dir = new File(outputDirectory + compressedName);
File f = new File(dir, ze.getName());
String canonicalPath = f.getCanonicalPath();
if (!canonicalPath.startsWith(dir.toString())){
dir.mkdirs();
}else {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
}
}
}
} catch (Exception e) {
String errorMessage = "An error occurred while unzipping.";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage, e);
}
} else {
File file = new File(outputDirectory + compressedName);
File f = new File(file, ze.getName());
String canonicalPath = f.getCanonicalPath();
if (!canonicalPath.startsWith(file.toString())) {
file.getParentFile().mkdirs();
if (file.exists() || file.createNewFile()) {
try {
Log.w("Zip", "extracting: " + file.getPath());
FileOutputStream fout = new FileOutputStream(file);
int count;
while ((count = zis.read(buffer)) != -1) {
fout.write(buffer, 0, count);
}
fout.close();
} catch (Exception e) {
String errorMessage = "An error occurred while unzipping.";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage, e);
}
}
}else {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
}
}
}
}
progress.addLoaded(ze.getCompressedSize());
updateProgress(callbackContext, progress);
zis.closeEntry();
} catch (Exception e) {
String errorMessage = "An error occurred while unzipping.";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage, e);
}
}
// final progress = 100%
progress.setLoaded(progress.getTotal());
updateProgress(callbackContext, progress);
if (anyEntries)
callbackContext.success();
else
callbackContext.error("Bad zip file");
} catch (Exception e) {
String errorMessage = "An error occurred while unzipping.";
callbackContext.error(errorMessage);
Log.e(LOG_TAG, errorMessage, e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
}
}
}
}
private void updateProgress(CallbackContext callbackContext, ProgressEvent progress) throws JSONException {
PluginResult pluginResult = new PluginResult(PluginResult.Status.OK, progress.toJSONObject());
pluginResult.setKeepCallback(true);
callbackContext.sendPluginResult(pluginResult);
}
private Uri getUriForArg(String arg) {
CordovaResourceApi resourceApi = webView.getResourceApi();
Uri tmpTarget = Uri.parse(arg);
return resourceApi.remapUri(
tmpTarget.getScheme() != null ? tmpTarget : Uri.fromFile(new File(arg)));
}
private static class ProgressEvent {
private long loaded;
private long total;
public long getLoaded() {
return loaded;
}
public void setLoaded(long loaded) {
this.loaded = loaded;
}
public void addLoaded(long add) {
this.loaded += add;
}
public long getTotal() {
return total;
}
public void setTotal(long total) {
this.total = total;
}
public JSONObject toJSONObject() throws JSONException {
return new JSONObject(
"{loaded:" + loaded +
",total:" + total + "}");
}
}
}

Validator for an upload file

I need a validator for an upload file, for the moment I can upload all the files but I need a check that file is less than 10 MB and only text format such as ms word, txt, ppt, excel (not executable, might be harmful).
Do I have to use and libraries of java for that, or I don't know what, cause I am a junior. If anyone has any ideas that will be very nice.
I have seen some other similar question and i try out but none that can help me.
Ps: I am working on java spring.
Here is my code is compiled but not working is possible edit and also to check for the length.
class FileUploader implements Receiver, SucceededListener, FailedListener, ProgressListener {
private static final long serialVersionUID = 1L;
public File file;
public String filename;
#Override
public void updateProgress(long readBytes, long contentLength) {
UI ui = UI.getCurrent();
ui.access(() -> {
progressBar.setCaption("Uploaded: " + (float) readBytes / (float) contentLength * 100 + "%");
progressBar.setValue((float) readBytes / (float) contentLength);
progressBar.setVisible(true);
});
}
#Override
public void uploadFailed(FailedEvent event) {
UIHelper.showErrorNotification("File could not be uploaded");
}
#Override
public void uploadSucceeded(SucceededEvent event) {
try {
String savePath = "/var/ccpt_work_files/";
Path filePath = Paths.get(savePath);
if (Files.exists(filePath)) {
copyFiles("/tmp/" + event.getFilename(), savePath + event.getFilename(), savePath + event.getFilename());
} else {
File targetFile = new File(savePath);
if (!targetFile.mkdirs()) {
UIHelper.showErrorNotification("Couldn't create dir: " + targetFile);
} else {
copyFiles("/tmp/" + event.getFilename(), savePath + event.getFilename(), savePath + event.getFilename());
}
}
} catch (IOException e) {
UIHelper.showErrorNotification("File could not be uploaded");
}
UIHelper.showInformationNotification("File successfully uploaded");
}
private void copyFiles(String from, String to, String finalPath) throws IOException {
com.google.common.io.Files.copy(new File(from), new File(to));
uploadedFilePath = finalPath;
}
#Override
public OutputStream receiveUpload(String filename, String mimeType) {
this.filename = filename;
FileOutputStream fos = null;
try {
file = new File("/tmp/" + filename);
fos = new FileOutputStream(file);
if (!file.exists()) {
file.createNewFile();
}
} catch (final IOException e) {
UIHelper.showErrorNotification("File could not be stored in server");
return null;
}
return fos;
}
};

If you already have the File object of type java.io.File, you can just check for file size and mime type
boolean hasValidFileSize(File file, double maxFileSize) {
double bytes = file.length();
double megabytes = (kilobytes / 1024) / 1024;
if (megabytes > maxFileSize) {
return false;
}
return true;
}
For non harmful files, you can just check for the mime type. Look for ways on how to get the mime types for the files that you needed to be allowed and compare it with your file's mime type.

You can use this method to get the fileSize of a file.
static String getFileSizeMegaBytes(File file) {
return (double) file.length() / (1024 * 1024) + " mb";
}
Refer the post to get the file type.
File tyle extension

Android adding AAC ADTS to Mediarecorder PCM

My Mediarecorder gives me a PCM File as an output when I record the phone's microphone. Now when trying to listen to this File that it created all I hear is static and I think, if I have understood correctly, I get a PCM file from Mediarecorder not AAC and I need to add ADTS header to the PCM to be able to listen to it.
I have seen threads with custom Encoders but I can not seem to figure out where and what I need to do with them.
I make an output File from microphone recoridng like this:
private static final int CHANNEL = AudioFormat.CHANNEL_IN_MONO;
private static final int AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT;
private static final int SAMPLE_RATE = 44100; //44.1kHz
private static final int BUFFER_SIZE = 2048;
public Status status = Status.IDLE;
private AudioRecordingHandler arh;
private File outputFile;
private Context context;
/**
* Starts script for running. Needs output file to work!
*/
public void start() {
if (outputFile == null) { return; }
System.out.println("Start reading stream...");
aacEncoder = new AACEncoder(SAMPLE_RATE, micOutputPCM);
new Thread(new Runnable() {
#Override
public void run() {
record.startRecording();
byte[] data = new byte[BUFFER_SIZE];
float[] audioFloatBuffer = new float[BUFFER_SIZE/2];
Yin y = new Yin(SAMPLE_RATE, BUFFER_SIZE/2);
while(status == Status.RECORDING) {
record.read(data, 0, BUFFER_SIZE);
audioFloatBuffer = ConverterUtil.toFloatArray(data, 0, audioFloatBuffer,
0, audioFloatBuffer.length);
PitchDetectionResult pdr = y.getPitch(audioFloatBuffer);
aacEncoder.writeIntoOutputfile(data);
arh.handlePitch(pdr.getPitch());
}
aacEncoder.stopEncoding();
}
}).start();
}
/**
* Stops script
*/
public void stop() {
status = Status.IDLE;
record.stop();
arh.finishedRecording(micOutputPCM);
}
Here is how I get the byte[] from the File and where I try to encode the ADTS header to them.
public static File addHeaderToAac(File micOutputPCM, File output) throws IOException {
byte[] pcmFile = fullyReadFileToBytes(micOutputPCM);
int bufferSize = 2048;
//addADTSHeader to byte[] and return a File object
return fileWithADTSHeader;
}
public static byte[] fullyReadFileToBytes(File f) throws IOException {
int size = (int) f.length();
byte bytes[] = new byte[size];
byte tmpBuff[] = new byte[size];
FileInputStream fis= new FileInputStream(f);;
try {
int read = fis.read(bytes, 0, size);
if (read < size) {
int remain = size - read;
while (remain > 0) {
read = fis.read(tmpBuff, 0, remain);
System.arraycopy(tmpBuff, 0, bytes, size - remain, read);
remain -= read;
}
}
} catch (IOException e){
throw e;
} finally {
fis.close();
}
return bytes;
}
My question is, does anyone have an Encoder that can accept a File or byte[] or ByteStream as an input and return a File.
Because ultimately I want to make a mp4parser AACTrackImpl, which can be found here : https://github.com/sannies/mp4parser
AACTrackImpl aacTrack2 = new MP3TrackImpl(new FileDataSourceImpl(micOutputPCM));
Also If I am missing some important details about how to convert and what I should do to be able to play it then that information will also be useful.
If I need provide more information in order to answer this question, then I will gladly do so.
Edit:
I've been trying to make an encoder that would do what I need, but so far I have had no success.
public static File addHeaderToAac(File pcmFile1, File output, Context context) throws IOException {
byte[] pcmFile = fullyReadFileToBytes(pcmFile1);
int bufferSize = 2048;
AACEncoder encoder = new AACEncoder(44100, output);
encoder.encodeAudioFrameToAAC(pcmFile);
return output;
}
I am trying to encode the PCM to AAC with this encoder, but this encoder writes the output file to memory, but I need an object. And when I give it my byte[] it also gives me an error :
W/System.err: at java.nio.ByteBuffer.put(ByteBuffer.java:642)
And the error is coming from this line :
inputBuf.put(frameData);
Finally, my encoder:
public class AACEncoder {
final String TAG = "UEncoder Processor";
final int sampleRate;
File outputFile;
FileOutputStream fos;
final int TIMEOUT_USEC = 10000 ;
MediaCodec encoder;
boolean isEncoderRunning = false;
boolean outputDone = false;
MediaCodec.BufferInfo info;
public AACEncoder(final int sampleRate, File outputFile) {
this.sampleRate = sampleRate;
this.info = new MediaCodec.BufferInfo();
this.outputFile = outputFile;
openFileStream();
initEncoder();
}
/**
* Initializes CrappyEncoder for AAC-LC (Low complexity)
* #throws Exception
*/
public void initEncoder() {
try {
encoder = MediaCodec.createEncoderByType("audio/mp4a-latm");
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
format.setInteger(MediaFormat.KEY_BIT_RATE, 128000);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (IOException ex) {
Log.e(TAG, "Failed to create CrappyEncoder");
ex.printStackTrace();
}
}
int generateIndex = 0;
public void encodeAudioFrameToAAC(byte[] frameData) {
if (encoder == null) return;
if (!isEncoderRunning) {
encoder.start();
isEncoderRunning = true;
}
ByteBuffer[] encoderInputBuffers = encoder.getInputBuffers();
if (fos != null) {
int inputBufIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
long ptsUsec = (System.currentTimeMillis() * 1000) / 10000;
if (outputDone) {
encoder.queueInputBuffer(inputBufIndex, 0, 0, ptsUsec,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
ByteBuffer inputBuf = encoderInputBuffers[inputBufIndex];
inputBuf.clear();
inputBuf.put(frameData);
encoder.queueInputBuffer(inputBufIndex, 0, frameData.length, ptsUsec, 0);
}
generateIndex++;
}
tryEncodeOutputBuffer();
}
checkIfOutputDone();
}
/**
* Gets data from output buffer and encodes it to
* AAC-LC encoding with ADTS header attached before every frame
*/
private void tryEncodeOutputBuffer() {
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
//If >= 0 then valid response
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus >= 0) {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size + 7);
byte[] data = new byte[info.size + 7];
addADTStoPacket(data, info.size + 7);
encodedData.get(data, 7, info.size);
encodedData.position(info.offset);
writeIntoOutputfile(data);
encoder.releaseOutputBuffer(encoderStatus, false);
}
}
private void checkIfOutputDone() {
if (outputDone) {
if (fos != null) {
try {
fos.close();
} catch (IOException ioe) {
Log.w(TAG, "failed closing debug file");
throw new RuntimeException(ioe);
}
fos = null;
}
}
}
/**
* Add ADTS header at the beginning of each and every AAC packet.
* This is needed as MediaCodec CrappyEncoder generates a packet of raw
* AAC data.
*
* Note the packetLen must count in the ADTS header itself.
**/
private void addADTStoPacket(byte[] packet, int packetLen) {
int profile = 2; //AAC LC
//39=MediaCodecInfo.CodecProfileLevel.AACObjectELD;
int freqIdx = 4; //44.1KHz
int chanCfg = 2; //CPE
// fill in ADTS data
packet[0] = (byte)0xFF;
packet[1] = (byte)0xF9;
packet[2] = (byte)(((profile-1)<<6) + (freqIdx<<2) +(chanCfg>>2));
packet[3] = (byte)(((chanCfg&3)<<6) + (packetLen>>11));
packet[4] = (byte)((packetLen&0x7FF) >> 3);
packet[5] = (byte)(((packetLen&7)<<5) + 0x1F);
packet[6] = (byte)0xFC;
}
private void openFileStream() {
fos = null;
try {
fos = new FileOutputStream(outputFile, false);
} catch (FileNotFoundException e) {
Log.e("AudioRecorder", e.getMessage());
}
}
/**
* Writes data into file
* #param data
*/
public void writeIntoOutputfile(byte[] data) {
try {
fos.write(data);
} catch (IOException ioe) {
Log.w(TAG, "failed writing debug data to file");
throw new RuntimeException(ioe);
}
}
public void stopEncoding() {
isEncoderRunning = false;
encoder.stop();
closeStream();
}
private void closeStream() {
try {
if (fos != null) {
fos.close();
}
} catch (IOException e) {
Log.e("AudioRecorder", e.getMessage());
}
}
}

Byte encryption for text file

I have an app in which I have to read a .txt file so that I can store some values and keep them. This is working pretty well, except for the fact that I want to make those values non-readable or "non-understandable" for external users.
My idea was to convert the file content into Hex or Binary and, in the reading process, change it back to Char. The thing is that I don't have access to methods such as String.Format due to my compiler.
Here's how I'm currently reading and keeping the values:
byte[] buffer = new byte[1024];
int len = myFile.read(buffer);
String data = null;
int i=0;
data = new String(buffer,0,len);
Class to open and manipulate the file:
public class File {
private boolean debug = false;
private FileConnection fc = null;
private OutputStream os = null;
private InputStream is = null;
private String fileName = "example.txt";
private String pathName = "logs/";
final String rootName = "file:///a:/";
public File(String fileName, String pathName) {
super();
this.fileName = fileName;
this.pathName = pathName;
if (!pathName.endsWith("/")) {
this.pathName += "/"; // add a slash
}
}
public boolean isDebug() {
return debug;
}
public void setDebug(boolean debug) {
this.debug = debug;
}
public void write(String text) throws IOException {
write(text.getBytes());
}
public void write(byte[] bytes) throws IOException {
if (debug)
System.out.println(new String(bytes));
os.write(bytes);
}
private FileConnection getFileConnection() throws IOException {
// check if subfolder exists
fc = (FileConnection) Connector.open(rootName + pathName);
if (!fc.exists() || !fc.isDirectory()) {
fc.mkdir();
if (debug)
System.out.println("Dir created");
}
// open file
fc = (FileConnection) Connector.open(rootName + pathName + fileName);
if (!fc.exists())
fc.create();
return fc;
}
/**
* release resources
*/
public void close() {
if (is != null)
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
is = null;
if (os != null)
try {
os.close();
} catch (IOException e) {
e.printStackTrace();
}
os = null;
if (fc != null)
try {
fc.close();
} catch (IOException e) {
e.printStackTrace();
}
fc = null;
}
public void open(boolean writeAppend) throws IOException {
fc = getFileConnection();
if (!writeAppend)
fc.truncate(0);
is = fc.openInputStream();
os = fc.openOutputStream(fc.fileSize());
}
public int read(byte[] buffer) throws IOException {
return is.read(buffer);
}
public void delete() throws IOException {
close();
fc = (FileConnection) Connector.open(rootName + pathName + fileName);
if (fc.exists())
fc.delete();
}
}
I would like to know a simple way on how to read this content. Binary or Hex, both would work for me.

So, with some understanding of the question, I believe you're really looking for a form of obfuscation? As mentioned in the comments, the easiest way to do this is likely a form of cipher.
Consider this example implementation of a shift cipher:
Common
int shift = 11;
Writing
// Get the data to be wrote to file.
String data = ...
// cipher the data.
char[] chars = data.toCharArray();
for (int i = 0; i < chars.length; ++i) {
chars[i] = (char)(chars[i] + shift);
}
String cipher = new String(chars);
// Write the data to the cipher file.
...
Reading
// Read the cipher file.
String data = ...
// Decipher the data.
char[] chars = data.toCharArray();
for (int i = 0; i < chars.length; ++i) {
chars[i] = (char)(chars[i] - shift);
}
String decipher = new String(chars);
// Use data as required.
...
Here's an example implementation on Ideone. The output:
Data : I can read this IP 192.168.0.1
Cipher : T+nly+}plo+st~+T[+<D=9<AC9;9<
Decipher: I can read this IP 192.168.0.1
I tried to keep this as low level as possible in order to satisfy the Java 3 requirement.
Note that this is NOT secure by any means. Shift ciphers (like most ciphers in a bubble) are trivial to break by malicious entities. Please do not use this if security is an actual concern.

Your solution is too complex. With java 8, you can try :
String fileName = "configFile.txt";
try (Stream<String> stream = Files.lines(Paths.get(fileName))) {
//TO-DO .Ex
stream.forEach(System.out::println);
} catch (IOException e) {
e.printStackTrace();
}

Discontinuous FTP download throws "Read timed out" or "Connection reset"

I used FTP and FTPClient in package 'org.apache.commons.net.ftp' to download files from FTP server.
Here is my total example code
public class FtpInput {
private static final Logger LOG = Logger.getLogger(FtpInput.class);
private static final int TIMEOUT = 120000;
private static final String SIZE_COMMAND_REPLY_CODE = "213 ";
/**
* FTPClient
*/
private FTPClient ftpClient;
/**
* FTP size
*/
private long completeFileSize = 0;
protected String ip = "";
protected int port = 21;
protected String user = "";
protected String passwd = "";
protected String path = "";
protected String fileName = "";
/**
* count input bytes
*/
private CountingInputStream is;
/**
* the bytes already processed
*/
private long processedBytesNum;
private byte[] inputBuffer = new byte[1024];
/**
* connect to ftp server and fetch inputStream
*/
public void connect() {
this.ftpClient = new FTPClient();
ftpClient.setRemoteVerificationEnabled(false);
try {
ftpClient.connect(ip, port);
if (!ftpClient.login(user, passwd)) {
throw new IOException("ftp login failed!");
}
if (StringUtils.isNotBlank(path)) {
if (!ftpClient.changeWorkingDirectory(path)) {
ftpClient.mkd(path);
if (!ftpClient.changeWorkingDirectory(path)) {
throw new IOException("ftp change working dir failed! path:" + path);
}
}
}
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
ftpClient.setSoTimeout(TIMEOUT);
ftpClient.setConnectTimeout(TIMEOUT);
ftpClient.setDataTimeout(TIMEOUT);
ftpClient.enterLocalPassiveMode();
// keep control channel keep-alive when download large file
ftpClient.setControlKeepAliveTimeout(120);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException("ftp login failed!", e);
}
// get complete ftp size
completeFileSize = getFtpFileSize();
LOG.info(String.format("ftp file size: %d", completeFileSize));
try {
InputStream ftpis = this.ftpClient.retrieveFileStream(this.fileName);
if (ftpis == null) {
LOG.error("cannot fetch source file.");
}
this.is = new CountingInputStream(ftpis);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
/**
* readBytes
*
* #return
*/
public byte[] readBytes() {
byte[] bytes = readBytesFromStream(is, inputBuffer);
// the bytes processed
processedBytesNum = is.getCount();
return bytes;
}
/**
* readBytesFromStream
*
* #param stream
* #param inputBuffer
* #return
*/
protected byte[] readBytesFromStream(InputStream stream, byte[] inputBuffer) {
Preconditions.checkNotNull(stream != null, "InputStream has not been inited yet.");
Preconditions.checkArgument(inputBuffer != null && inputBuffer.length > 0);
int readBytes;
try {
readBytes = stream.read(inputBuffer);
} catch (IOException e) {
throw new RuntimeException(e);
}
if (readBytes == inputBuffer.length) {
// inputBuffer is filled full.
return inputBuffer;
} else if (readBytes > 0 && readBytes < inputBuffer.length) {
// inputBuffer is not filled full.
byte[] tmpBytes = new byte[readBytes];
System.arraycopy(inputBuffer, 0, tmpBytes, 0, readBytes);
return tmpBytes;
} else if (readBytes == -1) {
// Read end.
return null;
} else {
// may other situation happens?
throw new RuntimeException(String.format("readBytesFromStream: readBytes=%s inputBuffer.length=%s",
readBytes, inputBuffer.length));
}
}
/**
* fetch the byte size of remote file size
*/
private long getFtpFileSize() {
try {
ftpClient.sendCommand("SIZE", this.fileName);
String reply = ftpClient.getReplyString().trim();
LOG.info(String.format("ftp file %s size reply : %s", fileName, reply));
Preconditions.checkArgument(reply.startsWith(SIZE_COMMAND_REPLY_CODE),
"ftp file size reply: %s is not success", reply);
String sizeSubStr = reply.substring(SIZE_COMMAND_REPLY_CODE.length());
long actualFtpSize = Long.parseLong(sizeSubStr);
return actualFtpSize;
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
public void close() {
try {
if (is != null) {
LOG.info(String.format("already read %d bytes from ftp file %s", is.getCount(), fileName));
is.close();
}
if (ftpClient != null) {
// Must call completePendingCommand() to finish command.
boolean isSuccessTransfer = ftpClient.completePendingCommand();
if (!isSuccessTransfer) {
LOG.error("error happened when complete transfer of ftp");
}
ftpClient.logout();
ftpClient.disconnect();
}
} catch (Throwable e) {
e.printStackTrace();
LOG.error(String.format("Close ftp input failed:%s,%s", e.getMessage(), e.getCause()));
} finally {
is = null;
ftpClient = null;
}
}
public void validInputComplete() {
Preconditions.checkArgument(processedBytesNum == completeFileSize, "ftp file transfer is not complete");
}
/**
* main
*
* #param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String ip = "***.***.***.****";
int port = 21;
String user = "***";
String passwd = "***";
String path = "/home/work";
String fileName = "b.txt";
FtpInput input = new FtpInput();
try {
input.fileName = fileName;
input.path = path;
input.ip = ip;
input.port = port;
input.user = user;
input.passwd = passwd;
// connect to FTP server
input.connect();
while (true) {
// read bytes
byte[] bytes = input.readBytes();
if (bytes == null) {
break;
}
LOG.info("read " + bytes.length + " bytes at :" + new Date(System.currentTimeMillis()));
// Attention: this is used for simulating the process of writing data into hive table
// it maybe consume more than 1 minute;
Thread.sleep(3000);
}
input.validInputComplete();
} catch (Exception e) {
e.printStackTrace();
} finally {
input.close();
}
}
}
here is the exception message:
java.net.SocketTimeoutException: Read timed out
or
java.net.SocketException: Connection reset
at stream.readBytes in method readBytesFromStream
At first, i think it probably caused by writing into hive table slowly, and then the FTP Server closed the connection.
But actually, the speed of writing into hive table is fast enough.
Now, i need your help, how can i fix this problem.

From your comments, it looks like it can take hours before you finish downloading the file.
You cannot reasonably expect an FTP server to wait for you for hours to finish the transfer. Particularly if you are not transferring anything most of the time. You waste server resources and most servers will protect themselves against such abuse.
Your design is flawed.
You should redesign your application to first fully download the file; and import the file only after the download finishes.