i'm trying to ensure an output File integrity in case of disk out of space , network problem ,or any anyException that might occur during the streaming to file process .
is there a way to precalculate the FileStream checkSum before writing to disk then check if the file was written properly.
it sounds a bit nonsensical for me , that a system validates the integrity of its own exported XML through checkSum , normaly it's the job of the other end to verify if the the consumed file lives up to the file produced by the other system .
but it's a requirement i have to implement.
her's the stream i write as a file :
String xmlTransfer ="";
File testFile = new File("testFile.xml");
InputStream in = new ByteArrayInputStream(xmlTransfer.getBytes("utf-8"));
FileOutputStream out = new FileOutputStream(testFile)
byte[] buffer = new byte[2048];
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
out.close();
in.close();
No, you can't figure out how much data will come from a stream in advance. That's simply not how streams are meant to work.
What you could do, if you are writing both ends of the code, is to first calculate the file size on the sending end and send that before sending the file contents itself.
The best way is to catch exception. If something go wrong an exception will be launched and you could remove the partially written file in this case.
A second way is to have a in-memory stream before writing down to the filesystem but it consumes memory.
A third way is to ensure the destination disk capacity (new File(path).getFreeSpace())
The MD5 check sounds too slow for me in regards of the question.
try this :
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.security.MessageDigest;
public class CheckSumFileTest
{
private File buildChecksumFile(File fileToCheck, String filePrefix, String checksumAlgorithm) throws Exception
{
String checksum = null;
File checksumFile = null;
String tempDir = System.getProperty("java.io.tmpdir");
try {
checksumFile = new File(tempDir, filePrefix+"."+ checksumAlgorithm.toLowerCase());
checksumFile.createNewFile();
checksumFile.deleteOnExit();
} catch (Exception e1) {
e1.printStackTrace();
throw e1;
}
FileWriter fw = null;
try {
checksum = checkSum(fileToCheck,checksumAlgorithm);
fw = new FileWriter(checksumFile);
fw.write(checksum);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
finally
{
if(fw !=null)
fw.close();
}
return checksumFile;
}
private static String checkSum(File file, String checksumAlgorithm) throws Exception
{
MessageDigest digest = MessageDigest.getInstance(checksumAlgorithm);
InputStream input = null;
StringBuffer sb = new StringBuffer();
try{
input = new FileInputStream(file);
byte[] buffer = new byte[8192];
do {
int read = input.read(buffer);
if(read <= 0)
break;
digest.update(buffer, 0, read);
} while(true);
byte[] sum = digest.digest();
for (int i = 0; i < sum.length; i++) {
sb.append(Integer.toString((sum[i] & 0xff) + 0x100, 16).substring(1));
}
}catch(IOException io)
{
}finally{
if(input != null)
input.close();
}
return sb.toString();
}
private static String checkSumInStream(InputStream stream, String checksumAlgorithm) throws Exception
{
MessageDigest digest = MessageDigest.getInstance(checksumAlgorithm);
InputStream input = null;
StringBuffer sb = new StringBuffer();
try{
input = stream;
byte[] buffer = new byte[8192];
do {
int read = input.read(buffer);
if(read <= 0)
break;
digest.update(buffer, 0, read);
} while(true);
byte[] sum = digest.digest();
for (int i = 0; i < sum.length; i++) {
sb.append(Integer.toString((sum[i] & 0xff) + 0x100, 16).substring(1));
}
}catch(IOException io)
{
}finally{
if(input != null)
input.close();
}
return sb.toString();
}
private boolean checkIntegrity(String targetFileName, String checksumFileName, String checksumAlgorithm) throws Exception
{
FileInputStream stream = null;
BufferedReader br = null;
InputStreamReader ipsr = null;
File checksumFile = null;
String checksumString="";
File targetFile = new File(targetFileName);
try{
checksumFile = new File(checksumFileName);
stream = new FileInputStream(checksumFile);
ipsr = new InputStreamReader(stream);
br = new BufferedReader(ipsr);
//In checksum file : only one line to read
checksumString = br.readLine();
}finally
{
if(br != null)
br.close();
if(ipsr != null)
ipsr.close();
if(stream != null)
stream.close();
}
if(checksumString.equals(checkSum(targetFile,checksumAlgorithm)))
{
return true;
}
else
{
return false;
}
}
/**
* #param args
*/
public static void main(String[] args)
{
String str = "Amine";
InputStream stream = new ByteArrayInputStream(str.getBytes());
//step1
try {
System.out.println(checkSumInStream(stream,"MD5"));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//step2
File file = new File("c:/test.txt");
// if file doesnt exists, then create it
if (!file.exists()) {
try {
file.createNewFile();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
FileWriter fw;
BufferedWriter bw;
try {
fw = new FileWriter(file.getAbsoluteFile());
bw = new BufferedWriter(fw);
bw.write(str);
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
System.out.println(checkSum(file, "MD5"));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Done");
}
}
You should check by MD5, not file size
You can calculate your MD5 while you're reading the stream.
See https://stackoverflow.com/a/304350/3230038
Then, after saving the file, you can generate the md5 again and compare
UPDATE - here's my more detailed idea for this. I am assuming that you just want to calculate the MD5 without having to bring the whole byte[] into memory. In this case, I think you have 2 options
calculate MD5 on the fly, as you're saving, then after saving, check md5 again (if you're on linux you can just use md5sum)
calculate MD5 in a first pass, then save the file in a second pass.
for example
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;
public class MD5OnTheFly {
/**
* #param args
* #throws NoSuchAlgorithmException
* #throws IOException
*/
public static void main(String[] args) throws NoSuchAlgorithmException, IOException {
long ini = System.currentTimeMillis();
File file = new File("/home/leoks/Downloads/VirtualBox-4.3.0.tar");
System.out.println("size:"+file.length());
InputStream is = new FileInputStream(file);
MessageDigest md = MessageDigest.getInstance("MD5");
DigestInputStream dis = new DigestInputStream(is, md);
IOUtils.copy(dis, new NullOutputStream());
byte[] digest = md.digest();
StringBuffer hexString = new StringBuffer();
for (int i = 0; i < digest.length; i++) {
String hex = Integer.toHexString(0xff & digest[i]);
if (hex.length() == 1)
hexString.append('0');
hexString.append(hex);
}
System.out.println(hexString);
long end = System.currentTimeMillis();
System.out.println(end-ini+" millis");
}
}
returns
410859520
dda81aea75a83b1489662c6bcd0677e4
1413 millis
and then
[leoks#home ~]$ md5sum /home/leoks/Downloads/VirtualBox-4.3.0.tar
dda81aea75a83b1489662c6bcd0677e4 /home/leoks/Downloads/VirtualBox-4.3.0.tar
[leoks#home ~]$
Related
I have written a client-socket "system" that is supposed to upload a file.
Although, when I attempt to upload, content duplicates.
I'm pretty sure that it is because the program doesn't recognise the eof.
I've found something like "Object stream", but I don't fancy importing new classes. I reckon that I don't really require that. But I wanna know how what the problem precisely is and how to hanle it.
package client;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.Socket;
import java.net.UnknownHostException;
public class Client {
private Socket socket;
private DataInputStream in;
private DataOutputStream out;
public static void main(String[] args) {
new Client();
}
public Client()
{
try {
socket = new Socket("127.0.0.1", 5010);
in = new DataInputStream(socket.getInputStream());
out = new DataOutputStream(socket.getOutputStream());
this.sendFile("./some.txt");
in.close();
out.close();
socket.close();
}
catch(UnknownHostException ex)
{
System.out.println("unknown host");
}
catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void sendFile(String path)
{
int bytes = 0;
File file = new File(path);
FileInputStream input;
try {
input = new FileInputStream(file);
long size = file.length();
//long size = file.getTotalSpace();
System.out.println(size);
// send a file's size
out.writeLong(size);
byte[] buffer = new byte[1024];
int i = 0, r=0;
//while((bytes = input.read(buffer,0,buffer.length))!=-1)
while(size > 0 && (bytes = input.read(buffer,0,(int)Math.min(buffer.length, size)))!=-1)
{
System.out.println("\n -------------"+(++i));
for (byte b : buffer)
try
{
if ((char)b == '\n' || r == 0)
System.out.print("\n" + (++r));
System.out.print((char)b);
}
catch(NullPointerException ex)
{
}
out.write(buffer, 0, bytes);
out.flush();
size -= bytes;
}
input.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
package server;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
public class Server {
private ServerSocket ss;
private Socket cs;
private DataInputStream in;
private DataOutputStream out;
public static void main(String[] args) {
new Server();
}
public Server()
{
try {
ss = new ServerSocket(5010);
cs = ss.accept();
in = new DataInputStream(cs.getInputStream());
out = new DataOutputStream(cs.getOutputStream());
this.receiveFile("./uploaded.txt");
in.close();
out.close();
cs.close();
ss.close();
}
catch (IOException e) {
e.printStackTrace();
}
}
private void receiveFile(String path)
{
int bytes = 0;
try {
File file = new File(path);
file.createNewFile();
FileOutputStream output = new FileOutputStream(file);
long size = in.readLong();
byte[] buffer = new byte[1024];
int i = 0;
while(size>0 && (bytes = in.read(buffer, 0, (int)Math.min(buffer.length, size))) != -1)
{
System.out.println("\n -------------"+(++i));
for (byte b : buffer)
try
{
System.out.print((char)b);
}
catch(NullPointerException ex)
{
}
output.write(buffer, 0, bytes);
size -= bytes;
}
output.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
The problem was that I didn't check if the size were 0 on the client side.
That try catch should NOT be in the for loop !!!! It only needs single code use by wrapping. Also use counting metering conventionally the number of bytes with a conventional numeric "for" loop, Not for(byte b : buffer). Note: byte is not strictly numeric, it will only reach to 255 in the for counter! It depends the quantity bytes required iterated over and should be as many as are packaged to it over the connection.
In the read you need to obtain the number of bytes sent andmark that into the write length to take from the array, so it would be better to instantiate the array based on the number of bytes or maximum bytes the the client sender has sent or negotiated as the maximum allowed (see the api docs for the stream.
NB ObjectStream does not apply , it's for RMI)
Of counting into the byte[] buffer array, you should remove it from the method and put it as a global. In the method , instantiate a new "buffer" array on the global variable each iteration of the loop according to the number of bytes read as it's new length.
The code does not appear to be particularly safe or debugable. You might try carefully constructing it starting again from scratch.
I am trying to to extract files out of a nested zip archive and process them in memory.
What this question is not about:
How to read a zip file in Java: NO, the question is how to read a zip file within a zip file within a zip and so on and so forth (as in nested zip files).
Write temporary results on disk: NO, I'm asking about doing it all in memory. I found many answers using the not-so-efficient technique of writing results temporarily to disk, but that's not what I want to do.
Example:
Zipfile -> Zipfile1 -> Zipfile2 -> Zipfile3
Goal: extract the data found in each of the nested zip files, all in memory and using Java.
ZipFile is the answer, you say? NO, it is not, it works for the first iteration, that is for:
Zipfile -> Zipfile1
But once you get to Zipfile2, and perform a:
ZipInputStream z = new ZipInputStream(zipFile.getInputStream( zipEntry) ) ;
you will get a NullPointerException.
My code:
public class ZipHandler {
String findings = new String();
ZipFile zipFile = null;
public void init(String fileName) throws AppException{
try {
//read file into stream
zipFile = new ZipFile(fileName);
Enumeration<?> enu = zipFile.entries();
exctractInfoFromZip(enu);
zipFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
//The idea was recursively extract entries using ZipFile
public void exctractInfoFromZip(Enumeration<?> enu) throws IOException, AppException{
try {
while (enu.hasMoreElements()) {
ZipEntry zipEntry = (ZipEntry) enu.nextElement();
String name = zipEntry.getName();
long size = zipEntry.getSize();
long compressedSize = zipEntry.getCompressedSize();
System.out.printf("name: %-20s | size: %6d | compressed size: %6d\n",
name, size, compressedSize);
// directory ?
if (zipEntry.isDirectory()) {
System.out.println("dir found:" + name);
findings+=", " + name;
continue;
}
if (name.toUpperCase().endsWith(".ZIP") || name.toUpperCase().endsWith(".GZ")) {
String fileType = name.substring(
name.lastIndexOf(".")+1, name.length());
System.out.println("File type:" + fileType);
System.out.println("zipEntry: " + zipEntry);
if (fileType.equalsIgnoreCase("ZIP")) {
//ZipFile here returns a NULL pointer when you try to get the first nested zip
ZipInputStream z = new ZipInputStream(zipFile.getInputStream(zipEntry) ) ;
System.out.println("Opening ZIP as stream: " + name);
findings+=", " + name;
exctractInfoFromZip(zipInputStreamToEnum(z));
} else if (fileType.equalsIgnoreCase("GZ")) {
//ZipFile here returns a NULL pointer when you try to get the first nested zip
GZIPInputStream z = new GZIPInputStream(zipFile.getInputStream(zipEntry) ) ;
System.out.println("Opening ZIP as stream: " + name);
findings+=", " + name;
exctractInfoFromZip(gZipInputStreamToEnum(z));
} else
throw new AppException("extension not recognized!");
} else {
System.out.println(name);
findings+=", " + name;
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Findings " + findings);
}
public Enumeration<?> zipInputStreamToEnum(ZipInputStream zStream) throws IOException{
List<ZipEntry> list = new ArrayList<ZipEntry>();
while (zStream.available() != 0) {
list.add(zStream.getNextEntry());
}
return Collections.enumeration(list);
}
I have not tried it but using ZipInputStream you can read any InputStream that contains a ZIP file as data. Iterate through the entries and when you found the correct entry use the ZipInputStreamto create another nestedZipInputStream`.
The following code demonstrates this. Imagine we have a readme.txt inside 0.zip which is again zipped in 1.zip which is zipped in 2.zip. Now we read some text from readme.txt:
try (FileInputStream fin = new FileInputStream("D:/2.zip")) {
ZipInputStream firstZip = new ZipInputStream(fin);
ZipInputStream zippedZip = new ZipInputStream(findEntry(firstZip, "1.zip"));
ZipInputStream zippedZippedZip = new ZipInputStream(findEntry(zippedZip, "0.zip"));
ZipInputStream zippedZippedZippedReadme = findEntry(zippedZippedZip, "readme.txt");
InputStreamReader reader = new InputStreamReader(zippedZippedZippedReadme);
char[] cbuf = new char[1024];
int read = reader.read(cbuf);
System.out.println(new String(cbuf, 0, read));
.....
public static ZipInputStream findEntry(ZipInputStream in, String name) throws IOException {
ZipEntry entry = null;
while ((entry = in.getNextEntry()) != null) {
if (entry.getName().equals(name)) {
return in;
}
}
return null;
}
Note the code is really ugly and does not close anything nor does it checks for errors. It is just a minimized version that demonstrates how it works.
Theoretically there is no limit how many ZipInputStreams you cascade into another. The data is never written into a temporary file. The decryption is only performed on-demand when you read each InputStream.
this is the way I found to unzip file in memory:
The code is not clean AT ALL, but i understand the rules are to post something working, so i have this hopefully to help so
What I do is use a recursive method to navigate the complex ZIP file and extract
folder
other inner zips
files
and save the results in memory to later work with them.
Main things I found I want to share with you:
1 ZipFile is useless if you have nested zip files
2 You have to use the basic Zip InputStream and Outputstream
3 I only use recursive programming to unzip nested zips
package course.hernan;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.IOUtils;
public class FileReader {
private static final int BUFFER_SIZE = 2048;
public static void main(String[] args) {
try {
File f = new File("DIR/inputs.zip");
FileInputStream fis = new FileInputStream(f);
BufferedInputStream bis = new BufferedInputStream(fis);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
BufferedOutputStream bos = new BufferedOutputStream(baos);
byte[] buffer = new byte[BUFFER_SIZE];
while (bis.read(buffer, 0, BUFFER_SIZE) != -1) {
bos.write(buffer);
}
bos.flush();
bos.close();
bis.close();
//This STACK has the output byte array information
Deque<Map<Integer, Object[]>> outputDataStack = ZipHandler1.unzip(baos);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
package course.hernan;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.lang3.StringUtils;
public class ZipHandler1 {
private static final int BUFFER_SIZE = 2048;
private static final String ZIP_EXTENSION = ".zip";
public static final Integer FOLDER = 1;
public static final Integer ZIP = 2;
public static final Integer FILE = 3;
public static Deque<Map<Integer, Object[]>> unzip(ByteArrayOutputStream zippedOutputFile) {
try {
ZipInputStream inputStream = new ZipInputStream(
new BufferedInputStream(new ByteArrayInputStream(
zippedOutputFile.toByteArray())));
ZipEntry entry;
Deque<Map<Integer, Object[]>> result = new ArrayDeque<Map<Integer, Object[]>>();
while ((entry = inputStream.getNextEntry()) != null) {
LinkedHashMap<Integer, Object[]> map = new LinkedHashMap<Integer, Object[]>();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
System.out.println("\tExtracting entry: " + entry);
int count;
byte[] data = new byte[BUFFER_SIZE];
if (!entry.isDirectory()) {
BufferedOutputStream out = new BufferedOutputStream(
outputStream, BUFFER_SIZE);
while ((count = inputStream.read(data, 0, BUFFER_SIZE)) != -1) {
out.write(data, 0, count);
}
out.flush();
out.close();
// recursively unzip files
if (entry.getName().toUpperCase().endsWith(ZIP_EXTENSION.toUpperCase())) {
map.put(ZIP, new Object[] {entry.getName(), unzip(outputStream)});
result.add(map);
//result.addAll();
} else {
map.put(FILE, new Object[] {entry.getName(), outputStream});
result.add(map);
}
} else {
map.put(FOLDER, new Object[] {entry.getName(), unzip(outputStream)});
result.add(map);
}
}
inputStream.close();
return result;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
Thanks to JMax.
In my case, The result of reading the pdf file is different from the expected result, It becomes bigger and cannot be opened.
Finally I found that I had made a mistake, The buffer may not be full，
The following is the error code.
while((n = zippedZippedZippedReadme.read(buffer)) != -1) {
fos.write(buffer);
}
Here is the correct code,
try (FileInputStream fin = new FileInputStream("1.zip")) {
ZipInputStream firstZip = new ZipInputStream(fin);
ZipInputStream zippedZip = new ZipInputStream(findEntry(firstZip, "0.zip"));
ZipInputStream zippedZippedZippedReadme = findEntry(zippedZip, "test.pdf");
long startTime = System.currentTimeMillis();
byte[] buffer = new byte[4096];
File outputFile = new File("test.pdf");
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
int n;
while((n = zippedZippedZippedReadme.read(buffer)) != -1) {
fos.write(buffer, 0 ,n);
}
fos.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("time consuming:" + (System.currentTimeMillis() - startTime)/1000.0);
}
hope to be helpful!
I am trying to read a file that is encoded using Shift_JIS and then convert it into UTF-8. When i use java.nio CharsetDecoder.decode it throws the following error. I am not able to pinpoint the actual cause of this issue.
java.nio.charset.UnmappableCharacterException: Input length = 2
java.nio.charset.UnmappableCharacterException: Input length = 2
at java.nio.charset.CoderResult.throwException(CoderResult.java:278)
at java.nio.charset.CharsetDecoder.decode(CharsetDecoder.java:798)
at CharacterSetConversionUtility.getString(CharacterSetConversionUtility.java:23)
at CharacterSetConversionUtility.convertBetweenEncodings(CharacterSetConversionUtility.java:39)
at CharacterSetConversionUtility.main(CharacterSetConversionUtility.java:94
Below is the code snippet
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.charset.CharsetDecoder;
import org.mozilla.universalchardet.UniversalDetector;
public class CharacterSetConversionUtility
{
public static String getString(String charSet, byte[] bytes) throws CharacterCodingException
{
ByteBuffer buffer = ByteBuffer.wrap(bytes);
Charset charset = Charset.forName(charSet);
CharsetDecoder decoder = charset.newDecoder();
CharBuffer output = decoder.decode(buffer);
return output.toString();
}
public static byte[] convertToEncoding(String charSet, String input) throws CharacterCodingException
{
CharBuffer buffer = CharBuffer.wrap(input);
Charset charset = Charset.forName(charSet);
CharsetEncoder encoder = charset.newEncoder();
ByteBuffer output = encoder.encode(buffer);
return output.array();
}
public static byte[] convertBetweenEncodings(byte[] originalBytes, String sourceCharSet, String destCharSet)
throws CharacterCodingException
{
String unicodeString = getString(sourceCharSet, originalBytes);
byte[] output = convertToEncoding(destCharSet, unicodeString);
return output;
}
/** Utility Method to detect character encoding in a byte stream **/
public static String getCharacterEncoding(String fileName){
byte[] buf = new byte[4096];
String encoding = null;
try {
java.io.FileInputStream fis = new java.io.FileInputStream(fileName);
// (1)
UniversalDetector detector = new UniversalDetector(null);
// (2)
int nread;
while ((nread = fis.read(buf)) > 0 && !detector.isDone()) {
detector.handleData(buf, 0, nread);
}
// (3)
detector.dataEnd();
// (4)
encoding = detector.getDetectedCharset();
if (encoding != null) {
System.out.println("Detected encoding = " + encoding);
} else {
System.out.println("No encoding detected.");
}
// (5)
detector.reset();
//
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return encoding;
}
public static void main(String[] args) {
Path path = Paths.get("E:/Encoding Issue/SJISFile");
try {
byte[] inputdata = Files.readAllBytes(path);
//Detect the character encoding of the input data
String inputCharEncoding = getCharacterEncoding("E:/Encoding Issue/SJISFile");
//Perform a character set conversion
byte[] outputdata =convertBetweenEncodings(inputdata,inputCharEncoding,"UTF-8");
FileOutputStream fos = new FileOutputStream("E:/Encoding Issue/convertedutf8.txt");
fos.write(outputdata);
fos.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
So, I don't have a definitive answer to the cause of the failure you experienced, although I suspect it lies somewhere in the conversion between String/char[]/byte[].
That said, I'd like to offer a more simple and compact working solution to the problem at hand, which uses this method instead, i.e. the conversion functionality offered by the String class itself as opposed to using en/decoders.
This will work for the Shift_JIS charset or any other one. Also, nothing wrong with the use of UniversalDetector, but I omitted it for simplicity's sake and hard-coded the source character set instead. Finally this version is JavaSE 1.6 compatible.
Hope it helps :)
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
public class JapaneseCharsetTest {
public static final int CHAR_LENGTH_TO_READ = 8192;
public static void main(String[] args) {
JapaneseCharsetTest test = new JapaneseCharsetTest();
test.doIt();
}
public void doIt() {
InputStreamReader reader = null;
FileOutputStream fos = null;
try {
FileInputStream stream = new FileInputStream(new File("C:/Path/To/My/ShiftJISFile.txt"));
reader = new InputStreamReader(stream, Charset.forName("Shift_JIS"));
fos = new FileOutputStream("C:/Path/To/My/UTF8TargetFile.txt");
char[] inputdata = new char[CHAR_LENGTH_TO_READ];
int len = 0;
while ((len = reader.read(inputdata)) != -1) {
convert(len, inputdata, fos);
}
}
catch (IOException e) {
e.printStackTrace();
}
finally {
try {if (reader != null) reader.close();} catch (IOException ignored) {}
try {if (fos != null) fos.close();} catch (IOException ignored){}
}
}
private void convert(int len, char[] inputData, FileOutputStream fos) throws IOException {
char[] charsToWrite = inputData;
if (len < CHAR_LENGTH_TO_READ) {
// Last chunk of data - cut it to size
charsToWrite = new char[len];
CharBuffer.wrap(inputData).get(charsToWrite, 0, len);
}
// Convert initial charset (here Shift_JIS) to target (here UTF-8)
byte[] utf8 = new String(charsToWrite).getBytes("UTF-8");
fos.write(utf8);
}
}
I am trying to make a ftp downloader which would restart from the position where the file was lastly read.
I will be storing some meta-data for this . But while testing i am kicking out the client and also disconnecting the server . But the handle is not getting into the exceptional as indicated in the code:
package fileresumes;
/**
*
* #author agarwall
*/
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Calendar;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPConnectionClosedException;
import org.apache.commons.net.ftp.FTPFile;
public class FileRes {
public static void main(String[] args) {
FTPClient client = new FTPClient();
FileOutputStream fos = null;
int totalBytesRead = 0;
try {
client.connect("localhost");
client.login("anonymous", "");
// The remote filename to be downloaded.
String filename = "testing.txt";
fos = new FileOutputStream(filename);
boolean append = false;
int offset = 0;
long last_modified = 0;
int size = 0;
//long ro = client.getRestartOffset();
//ro = client.getRestartOffset();
//Download file from FTP server;
final File file = new File("C:/users/deadman/Desktop/", "testing.txt");
if (file.exists()) {
last_modified = file.lastModified(); // lastModified() returns the milliseconds since 1970-01-01
append = true;
// Read offset from meta-data file
size = (int) file.length();
offset = size;
}
//Setting the offset to resume download
client.setRestartOffset(offset);
InputStream inputFileStream;
inputFileStream = client.retrieveFileStream("/large.txt");
int bytesRead = 0;
try {
OutputStream out = new FileOutputStream(file, append);
final byte[] bytes = new byte[1024];
while ((bytesRead = inputFileStream.read(bytes)) != -1) {
out.write(bytes, 0, bytesRead);
totalBytesRead += bytesRead;
}
inputFileStream.close();
out.flush();
int get_reply_code = client.getReplyCode();
System.out.println(get_reply_code);
} catch (IOException e) {
// I want my metadata to be updated here .
System.out.println("IOException");
} catch (RuntimeException e) {
// I want my metadata to be updated here .
System.out.println("Runtime Exception ");
} finally {
try {
int get_reply_code = client.getReplyCode();
System.out.println(get_reply_code);
if (fos != null) {
fos.close();
}
client.disconnect();
System.out.println("finish");
} catch (IOException e) {
}
}
} catch (Exception e) {
}
}
}
Can anyone help me in case of broken connection how can we handle the exception here.
I have been trying to do something similar, and so far the only thing I have been able to do is to attempt another command. I haven't gotten to any transfers in coding yet, but for me if I login and there is either a timeout, get kicked, etc, if I try to change folders (or something similar) it throws a FTPConnectionClosedException. I have looked around Google, and have posted a question on here to see if there is a better way, but so far have not heard anything.
Anytime I try to serialize a file I get the error: FileNotFound. Not sure why. Here is my FileHelper code:
package org.stocktwits.helper;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import org.stocktwits.model.Quote;
public class FileHelper {
// Returns the contents of the file in a byte array.
public static byte[] getBytesFromFile(File file) throws IOException {
InputStream is = new FileInputStream(file);
// Get the size of the file
long length = file.length();
// You cannot create an array using a long type.
// It needs to be an int type.
// Before converting to an int type, check
// to ensure that file is not larger than Integer.MAX_VALUE.
if (length > Integer.MAX_VALUE) {
// File is too large
}
// Create the byte array to hold the data
byte[] bytes = new byte[(int)length];
// Read in the bytes
int offset = 0;
int numRead = 0;
while (offset < bytes.length
&& (numRead=is.read(bytes, offset, bytes.length-offset)) >= 0) {
offset += numRead;
}
// Ensure all the bytes have been read in
if (offset < bytes.length) {
throw new IOException("Could not completely read file "+file.getName());
}
// Close the input stream and return bytes
is.close();
return bytes;
}
public static void serializeQuotes(ArrayList<Quote> quotes){
try {
// Serialize to a file
ObjectOutput out = new ObjectOutputStream(new FileOutputStream("quotes.ser"));
out.writeObject(quotes);
out.close();
// Serialize to a byte array
ByteArrayOutputStream bos = new ByteArrayOutputStream() ;
out = new ObjectOutputStream(bos) ;
out.writeObject(quotes);
out.close();
// Get the bytes of the serialized object
//byte[] buf = bos.toByteArray();
} catch (IOException e) {
System.out.println(e);
}
}
#SuppressWarnings("unchecked")
public static void deserializeQuotes(ArrayList<Quote> quotes){
try {
// Deserialize from a file
File file = new File("quotes.ser");
ObjectInputStream in = new ObjectInputStream(new FileInputStream(file));
// Deserialize the object
quotes = (ArrayList<Quote>) in.readObject();
in.close();
// Get some byte array data
byte[] bytes = FileHelper.getBytesFromFile(file);
// see Reading a File into a Byte Array for the implementation of this method
// Deserialize from a byte array
in = new ObjectInputStream(new ByteArrayInputStream(bytes));
quotes = (ArrayList<Quote>) in.readObject();
in.close();
} catch (ClassNotFoundException e) {
System.out.println(e);
} catch (IOException e) {
System.out.println(e);
}
}
}
private void serializeQuotes(){
FileOutputStream fos;
try {
fos = openFileOutput(Constants.FILENAME, Context.MODE_PRIVATE);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(quotes);
oos.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
}catch(IOException e){
e.printStackTrace();
}
}
#SuppressWarnings("unchecked")
private void deserializeQuotes(){
try{
FileInputStream fis = openFileInput(Constants.FILENAME);
ObjectInputStream ois = new ObjectInputStream(fis);
quotes = (ArrayList<Quote>) ois.readObject();
} catch (FileNotFoundException e) {
e.printStackTrace();
}catch(IOException e){
e.printStackTrace();
}catch(ClassNotFoundException e){
e.printStackTrace();
}
}