Develop Reference

Best way to close the stream during bulk upload

I'm uploading bulk files (say 50000) to Box through Box Java API's. To do that we call the uploadFile method 50000 times. The method also creates and close the file input stream.
Is there a way to keep the stream open until the bulk load is done? Even if I close the stream in finally block, it will close it every time I call the method.
private static String uploadFile(String pathFileName, BoxAPIConnection api, BoxFolder folder) {
boolean fileExists = false;
String fileId = null;
FileInputStream stream = null;
log.debug("Invoked uploadFileAsBoxAppUser-uploadFile :" + pathFileName + ":" + api + ":" + folder);
try {
String fileName = pathFileName.substring(pathFileName.lastIndexOf("\\") + 1, pathFileName.length());
log.debug("fileName :" + fileName);
for (BoxItem.Info itemInfo : folder) {
if (itemInfo instanceof BoxFile.Info) {
BoxFile.Info fileInfo = (BoxFile.Info) itemInfo;
if (fileName.equals(fileInfo.getName())) {
fileExists = true;
fileId = fileInfo.getID();
log.debug("fileExists in Destination box Folder fileID " + fileId);
}
}
}
if (!fileExists) {
log.debug("uploading new file: " + fileName);
stream = new FileInputStream(pathFileName);
BoxFile.Info boxInfo = folder.uploadFile(stream, pathFileName);
fileId = boxInfo.getID();
} else {
log.debug("uploading new version of file: " + fileName);
BoxFile file = new BoxFile(api, fileId);
stream = new FileInputStream(pathFileName);
file.uploadVersion(stream);
}
} catch (IOException e) {
log.debug("Exception in uploadFileAsBoxAppUser :" + e);
}
finally {
if (stream != null)
try {
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return fileId;
}

Downloading corrupted files with OkHttp

The method I wrote to download files always produce corrupted files.
public static String okDownloadToFileSync(final String link, final String fileName, final boolean temp, DownloadStatusManager statusManager, ErrorDisplayerInterface errorDisplayerInterface) {
Request request = new Request.Builder()
.url(link)
.build();
OkHttpClient client = Api.getInstance().getOkHttpClient();
OutputStream output = null;
InputStream input = null;
try {
Response response = client.newCall(request).execute();
//Add the file length to the statusManager
final int contentLength = Integer.parseInt(response.header("Content-Length"));
if (statusManager != null) {
statusManager.add(Hash.md5(link), contentLength);
}
//Get content type to know extension
final String contentType = response.header("Content-Type");
final String ext = contentTypeMap.get(contentType);
Log.i(TAG, link + "\n --> contentType = " + contentType + "\n --> ext = " + ext);
if (ext == null) {
Log.e(TAG, "-----------\next is null, seems like there is a problem with that url : \n " + link + "\n----------");
return null;
} else if (ext.equals("json")) {
Log.e(TAG, "-----------\ndownloadable file seems to be a json, seems like there is a problem with that url : \n " + link + "\n----------");
return null;
}
//Check if file already exists
if (!temp && fileName != null) {
File test = new File(M360Application.getContext().getFilesDir(), fileName + "." + ext);
if (test.exists()) {
Log.i(TAG, "File exists ! : " + test.getPath());
test.delete();
//return test.getAbsolutePath();
}
}
// expect HTTP 200 OK, so we don't mistakenly save error report instead of the file
if (!response.isSuccessful()) {
errorDisplayerInterface.popWarn(null, "Error while downloading " + link, "connection.getResponseCode() != HttpURLConnection.HTTP_OK");
return null;
}
input = response.body().byteStream();
File file;
if (temp) {
file = File.createTempFile(UUID.randomUUID().toString(), ext, M360Application.getContext().getCacheDir());
} else {
file = new File(M360Application.getContext().getFilesDir(), fileName + "." + ext);
}
output = new FileOutputStream(file);
output.write(response.body().bytes());
// byte data[] = new byte[4096];
// long total = 0;
// int count;
// while ((count = input.read(data)) != -1) {
// output.write(data, 0, count);
// total++;
//
// if (statusManager != null) {
// statusManager.update(Hash.md5(link), contentLength - total);
// }
// }
return file.getAbsolutePath();
} catch (IOException e) {
e.printStackTrace();
errorDisplayerInterface.popError(null, e);
} finally {
if (statusManager != null) {
statusManager.finish(Hash.md5(link));
}
try {
if (output != null)
output.close();
if (input != null)
input.close();
} catch (IOException ignored) {
ignored.printStackTrace();
}
}
return null;
}
I access these file via adb, transfer them to my sccard, and there I see that they seem to have the proper size, but has no type according to for instance Linux file command.
Would you know what is missing and how to fix it?
Thank you.
Edit
Simpler version of the code ( but the bug is the same )
public static String okioDownloadToFileSync(final String link, final String fileName) throws IOException {
Request request = new Request.Builder()
.url(link)
.build();
OkHttpClient client = Api.getInstance().getOkHttpClient();
Response response = client.newCall(request).execute();
final int contentLength = Integer.parseInt(response.header("Content-Length"));
//Get content type to know extension
final String contentType = response.header("Content-Type");
final String ext = contentTypeMap.get(contentType);
// expect HTTP 200 OK, so we don't mistakenly save error report instead of the file
if (!response.isSuccessful()) {
return null;
}
File file = new File(M360Application.getContext().getFilesDir(), fileName + "." + ext);
BufferedSink sink = Okio.buffer(Okio.sink(file));
sink.writeAll(response.body().source());
sink.close();
Log.i(TAG, "file.length : " + file.length() + " | contentLength : " + contentLength);
return file.getAbsolutePath();
}
The log : file.length : 2485394 | contentLength : 1399242
Solution
The problem was that I was getting the OkHttpClient from my API singleton, which was used by retrofit and had multiples interceptors. Those interceptors were polluting the response.
So I OkHttpClient client = Api.getInstance().getOkHttpClient(); became OkHttpClient client = new OkHttpClient.Builder().build(); and everything is now ok !
Thanks a lot. I'm dividing the method into smaller pieces right now.

Instead of
output.write(response.body().bytes());
try something like this
byte[] buff = new byte[1024 * 4];
while (true) {
int byteCount = input.read(buff);
if (byteCount == -1) {
break;
}
output.write(buff, 0, byteCount);
}

losing data while writing through asynchronousFileChannel in java

I am trying to use asynchronousFileChannel to write the date into a text file. I made 3 jar file of the program with the AsynchronousFileChannel and compiled all 3 jars simultaneously through command prompt to read 3 different text files and output to one common temporary file
I have 2000 records in my test files(3) to be read,but the output in the common temporary file is missing some of the records,the output should have 6000 records but it shows only 5366 or 5666 or sometimes less than that.
I am not able to figure out why some data is lost as it is the functionality of a asynchronousFileChannel.
Here is the code for the java program using asynchronousfilechannel.
class Writer(){
public void writeOut(ReadableData fileData)
throws InterruptedException {
Path file = null;
AsynchronousFileChannel asynchFileChannel = null;
String filePath = tempFileName;
try {
file = Paths.get(filePath);
asynchFileChannel = AsynchronousFileChannel.open(file,
StandardOpenOption.WRITE, StandardOpenOption.CREATE);
CompletionHandler<Integer, Object> handler = new CompletionHandler<Integer, Object>() {
#Override
public void completed(Integer result, Object attachment) {
if (result == Integer.MAX_VALUE) {
log.debug("Attachment: " + attachment + " " + result
+ " bytes written");
log.debug("CompletionHandler Thread ID: "
+ Thread.currentThread().getId());
}
result++;
}
#Override
public void failed(Throwable e, Object attachment) {
try {
throw e;
} catch (Throwable e1) {
e1.printStackTrace();
}
log.debug("File Write Failed Exception:");
e.printStackTrace();
}
};
String printData = fileData.getId() + "|"
+ fileData.getName() + "|" + fileData.getEmpId()
+ "|" + fileData.getServieId() + "|" + "\n";
asynchFileChannel.write(ByteBuffer.wrap(printData.getBytes()),
asynchFileChannel.size(), "file write", handler);
log.debug(printData);
}
}
catch (IOException e) {
e.printStackTrace();
log.error(e.getMessage());
} finally {
}
}
}
}
and this is my class to read data from 3 files:
public class FileReader1 {
static Logger log = Logger.getLogger(FileHandlerNorthBoundMain.class
.getName());
Writer wrO=new Writer();
public static void main(String[] args) throws IOException,
IllegalFileFormatException, InterruptedException {
String filePath = "C:\\Users\\Public\\testdata1.csv"; //"C:\\Users\\Public\\testdata2.csv"; "C:\\Users\\Public\\testdata3.csv";
File file = new File(filePath);
log.info("Fetching data.... from: " + filePath);
ArrayList<ReadableData> list = new ArrayList<ReadableData>();
FileInputStream fs = null;
BufferedReader reader = null;
String Name;
int Id, EmpId, ServiceId;
ReadableData readableData = null;
int count = 0;
fs = new FileInputStream(file);
reader = new BufferedReader(new InputStreamReader(fs));
String line = reader.readLine();
while (line != null) {
StringTokenizer st = new StringTokenizer(line, "\\|");
while (st.hasMoreTokens()) {
try {
Id = Integer.parseInt(st.nextToken());
Name = st.nextToken();
EmpId = Integer.parseInt(st.nextToken());
ServiceId = Integer.parseInt(st.nextToken());
readableData = new ReadableData(Id,
, Name, EmpId,ServiceId);
wrO.writeOut(readableData);
list.add(count, readableData);
count = count++;
} catch (Exception ex) {
log.error("Illegal File Format");
throw new IllegalFileFormatException("Illegal File Format");
}
}
line = reader.readLine();
}
reader.close();
}

Modify your Writer class with the following code part with asynchronousFileChannel lock()
byte[] test = printData.getBytes();
Future<FileLock> featureLock = asynchFileChannel.lock();
log.info("Waiting for the file to be locked ...");
FileLock lock = featureLock.get();
if (lock.isValid()) {
log.debug(printData);
Future<Integer> featureWrite = asynchFileChannel.write(
ByteBuffer.wrap(test), asynchFileChannel.size());
log.info("Waiting for the bytes to be written ...");
int written = featureWrite.get();
log.info("I’ve written " + written + " bytes into "
+ file.getFileName() + " locked file!");
lock.release();
}

This might be because asynchronousFileChannel is thread safe but Bytebuffer is not,care should be taken to ensure that the buffer is not accessed until after the operation has completed.
check the documentation http://openjdk.java.net/projects/nio/javadoc/java/nio/channels/AsynchronousFileChannel.html

Why all files in my application are written to jboss tmp directory?

I have function like this in my Bean:
public String uploadFile(UploadedFile uploadedFile) {
logger.info("Enter: uploadFile(UploadedFile uploadedFile).");
String name = uploadedFile.getName();
String extension = name.substring(name.length() - 3);
if (extension.contentEquals("peg")) {
extension = "jpeg";
}
RandomString rs = new RandomString(RANDOM_PHOTO_NAME_LENGTH);
this.randomPhotoName = rs.nextString();
String fileName = this.randomPhotoName + "." + extension;
logger.info("File name: " + name + ". Extension: " + extension + ". New fileName: " + fileName);
ServletContext sc = (ServletContext) FacesContext.getCurrentInstance()
.getExternalContext().getContext();
File f = new File(
sc.getRealPath(Constant.USER_FILE_PATH));
if (!f.exists()) {
logger.info("Folder "
+ Constant.USER_FILE_PATH
+ " nie istniej. Tworze nowy.");
f.mkdirs();
}
File backupFile = new File(
sc.getRealPath(Constant.USER_FILE_PATH
+ fileName));
InputStream in = null;
OutputStream out = null;
try {
in = new BufferedInputStream(uploadedFile.getInputStream());
out = new FileOutputStream(backupFile);
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) > 0)
out.write(buf, 0, len);
} catch (IOException ioe) {
FacesContext context = FacesContext.getCurrentInstance();
FacesMessage msg = null;
msg = new FacesMessage(
"Pojawił się nieoczekiwany błąd przy uploadowaniu pliku.");
context.addMessage(null, msg);
logger.error(
"Pojawił się nieoczekiwany błąd przy uploadowaniu pliku.",
ioe);
} finally {
try {
in.close();
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
logger.info("Exit: uploadFile(UploadedFile uploadedFile).");
return fileName;
}
And all files are saved in tmp directory eg:
\jboss-5.1.0.GA\server\default\tmp\a006-czo4uq-gzzu4l42-1-gzzuk7xr-a2\TupTus.war\media\img\user-gallery\6u2fpgu3tkzniwg.JPG

Because all your files are built as:
File backupFile = new File(
sc.getRealPath(Constant.USER_FILE_PATH
+ fileName));
Sounds like sc.getRealPath() returns working directory that JBoss allocates for your application.
So, the real question is to you: where do you want to write the files? If not there, so where? If you prefer user temporary directory use new File(System.getProperty("java.io.tmpdir"), fileName) and write there.
If you want to be able to configure the path out of the box you can store this path either in DB or configuration file or pass it via your custom system properties when you are running JBoss using command line switch -D.

Create a file and then delete in java

My requirement is there is an export button,onclick the data in the database is loaded and then converted into a .csv ,.doc or .html file ,which can be opened or saved somehere, this needs to be converted into a file , save it in the local path and uploaded to the SFTP server and deleted from the loacl path. My code goes like this .....
public String a(String tableName, String format,String jsondataAsString,String jsonKeyName,String userName,
String password, String hostName,String remotePath) throws IOException{
userName="a";
password="a";
hostName="10.100.10.100";
remotePath="/tmp";
System.out.println("TableName-->" +tableName);
System.out.println("Format-->" +format);
System.out.println("JsonData-->" +jsondataAsString);
System.out.println("userName-->" +userName);
System.out.println("password-->" +password);
System.out.println("hostname-->" +hostName);
System.out.println("RemotePath-->" +remotePath);
String mimeType = null;
String fileName = null;
String seperator = null;
String lineSeperator = null;
boolean isFileTransferComplete=true;
OutputStream f1 =null;
if (format.equalsIgnoreCase("CSV")) {
fileName = tableName + ".csv";
mimeType = "application/CSV";
seperator = ",";
lineSeperator = "\n";
} else if (format.equalsIgnoreCase("Word")) {
fileName = tableName + ".doc";
mimeType = "application/msword";
seperator = " ";
lineSeperator = "\n\n";
} else {
fileName = tableName + ".html";
mimeType = "application/html";
seperator = " ";
lineSeperator = "<br><br>";
}
String localfilePath="D:/aaa/" +fileName;
String error="";
try{
String data = convertJsonToString(jsondataAsString, seperator, jsonKeyName,lineSeperator,format);
if (data == null || data.length() == 0) {
data = "[BLANK]";
}
boolean isFileTobeDeleted=true;
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
System.out.println("Buffer-->" +buffer);
buffer.flush();
buffer.write(data.getBytes());
f1 = new FileOutputStream(localfilePath);
buffer.writeTo(f1);
isFileTransferComplete = new SFTPHandler(hostName,
PicoEmsGuiConstants.SFTP_Port, userName, password)
.uploadFile(remotePath,localfilePath);
System.out.println("FileTransfer" +isFileTransferComplete);
File target = new File(localfilePath);
target.delete();
}
System.out.println("isFileTobeDeleted" +isFileTobeDeleted);
}catch(Exception e){
System.out.println("Exception :::>" +e.getMessage());
}finally{
f1.close();
}
return isFileTransferComplete+"--"+ remotePath;
}
I am able to create file but after the completion of uploading unable to delete form the loacl path...Can anyone tell me where i am goin wrong

Don't you have to close the stream and then attempt to delete it?
finally {
f1.close();
if(file != null && file.exists()) {
file.delete();
}
}