Develop Reference

Java - Clipboard modification : Works in Eclipse, but not as JAR

I created a software based on two sources (stated in the code below) which detect HTML copied to the clipboard and change local images to base 64 ones.
This code works perfectly when I run it in Eclipse, but not using a JAR.
Initially, I was not using the method getHtmlDataFlavor, but I added it when I tried the software as a JAR. Then, I had to ensure in HtmlSelection.getTransferData to have if (flavor.getRepresentationClass() == java.io.Reader.class) otherwise it would crash. But using the JAR, I'm only getting the plain text version! Though, it stills works when ran in Eclipse.
Does someone have an idea ?
I am running on Windows 10.
Executing in command line using : java -jar ClipboardImageToBase64-1.0.0-jar-with-dependencies.jar
GitHub project :
https://github.com/djon2003/ClipboardImageToBase64
/**
* Library from HTML parsing : https://jsoup.org
*
* Code based on :
* - http://stackoverflow.com/a/14226456/214898
* - http://elliotth.blogspot.ca/2005/01/copying-html-to-clipboard-from-java.html
*/
import java.awt.Toolkit;
import java.awt.datatransfer.Clipboard;
import java.awt.datatransfer.ClipboardOwner;
import java.awt.datatransfer.DataFlavor;
import java.awt.datatransfer.Transferable;
import java.awt.datatransfer.UnsupportedFlavorException;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class ClipBoardListener extends Thread implements ClipboardOwner {
Clipboard sysClip = Toolkit.getDefaultToolkit().getSystemClipboard();
private static DataFlavor HTML_FLAVOR = new DataFlavor("text/html;class=java.io.Reader", "HTML");
private int nbImagesConverted = 0;
private Transferable currentTransferable;
#Override
public void run() {
Transferable trans = sysClip.getContents(this);
TakeOwnership(trans);
}
#Override
public void lostOwnership(Clipboard c, Transferable t) {
System.out.println("Copy to clipboard detected");
try {
ClipBoardListener.sleep(250); // waiting e.g for loading huge
// elements like word's etc.
} catch (Exception e) {
System.out.println("Exception: " + e);
}
Transferable contents = sysClip.getContents(this);
try {
process_clipboard(contents, c);
} catch (Exception ex) {
Logger.getLogger(ClipBoardListener.class.getName()).log(Level.SEVERE, null, ex);
}
TakeOwnership(currentTransferable);
}
void TakeOwnership(Transferable t) {
sysClip.setContents(t, this);
}
private void getHtmlDataFlavor(Transferable t) {
DataFlavor df = null;
for (DataFlavor tDf : t.getTransferDataFlavors()) {
if (tDf.getMimeType().contains("text/html")) {
if (tDf.getRepresentationClass() == java.io.Reader.class) {
df = tDf;
break;
}
}
}
HTML_FLAVOR = df;
}
public void process_clipboard(Transferable t, Clipboard c) {
String tempText = "";
Transferable trans = t;
currentTransferable = t;
getHtmlDataFlavor(t);
if (HTML_FLAVOR == null) {
System.out.println("No HTML flavor detected");
return;
}
nbImagesConverted = 0;
try {
if (trans != null ? trans.isDataFlavorSupported(HTML_FLAVOR) : false) {
if (trans.isDataFlavorSupported(DataFlavor.stringFlavor)) {
tempText = (String) trans.getTransferData(DataFlavor.stringFlavor);
}
java.io.Reader r = (java.io.Reader) trans.getTransferData(HTML_FLAVOR);
StringBuilder content = getReaderContent(r);
String newHtml = changeImages(content);
currentTransferable = new HtmlSelection(newHtml, tempText);
System.out.println("Converted " + nbImagesConverted + " images");
} else {
System.out.println("Not converted:" + trans.isDataFlavorSupported(HTML_FLAVOR));
System.out.println(trans.getTransferData(HTML_FLAVOR));
/*
for (DataFlavor tt : trans.getTransferDataFlavors()) {
if (tt.getMimeType().contains("text/html")) {
System.out.println("-------");
System.out.println(tt.toString());
}
}
*/
}
} catch (Exception e) {
currentTransferable = t;
System.out.println("Conversion error");
e.printStackTrace();
}
}
private String changeImages(StringBuilder content) throws RuntimeException, IOException {
Document doc = Jsoup.parse(content.toString());
Elements imgs = doc.select("img");
for (Element img : imgs) {
String filePath = img.attr("src");
filePath = filePath.replace("file:///", "");
filePath = filePath.replace("file://", "");
File file = new File(filePath);
if (file.exists()) {
String encoded = Base64.encodeBase64String(FileUtils.readFileToByteArray(file));
String extension = file.getName();
extension = extension.substring(extension.lastIndexOf(".") + 1);
String dataURL = "data:image/" + extension + ";base64," + encoded;
img.attr("src", dataURL); // or whatever
nbImagesConverted++;
}
}
String html = doc.outerHtml();
html = html.replaceAll("(?s)<!--.*?-->", ""); //Remove html comments
return html; // returns the modified HTML
}
private StringBuilder getReaderContent(java.io.Reader r) throws IOException {
char[] arr = new char[8 * 1024];
StringBuilder buffer = new StringBuilder();
int numCharsRead;
while ((numCharsRead = r.read(arr, 0, arr.length)) != -1) {
buffer.append(arr, 0, numCharsRead);
}
r.close();
return buffer;
}
private static class HtmlSelection implements Transferable {
private String html;
private String plainText;
public HtmlSelection(String html, String plainText) {
this.html = html;
this.plainText = plainText;
}
public DataFlavor[] getTransferDataFlavors() {
DataFlavor[] dfs = {HTML_FLAVOR, DataFlavor.stringFlavor};
return dfs;
}
public boolean isDataFlavorSupported(DataFlavor flavor) {
return flavor.getMimeType().contains("text/html") || flavor.getMimeType().contains("text/plain");
}
public Object getTransferData(DataFlavor flavor) throws UnsupportedFlavorException {
if (flavor.getMimeType().contains("text/html")) {
if (flavor.getRepresentationClass() == java.io.Reader.class) {
return new StringReader(html);
} else {
return html;
}
} else {
return plainText;
}
//throw new UnsupportedFlavorException(flavor);
}
}
}

I finally fixed all my problems. I created a GitHub project for those who are interested.
https://github.com/djon2003/ClipboardImageToBase64

Java application not completing

Was trying to get a JFrame added to see if it would help with using launch4j to convert a small jar file to an .exe. I wrote a short program to help sort HPLC data at work and want to make it just a simple point and click.
It works when I run it from the command line java KFile and the JFileChooser lets me choose directories for the script to work on. When I converted it to the .exe, the JFileChooser never rendered and the .exe closes.
I read that I might need a JFrame parent and so I created a JFrame, but now the script hangs before completion as if waiting for the frame to close. I'm pretty new to java, so I'm not sure how I to resolve this issue.
import java.io.*;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.FileVisitResult;
import java.nio.MappedByteBuffer;
import javax.swing.JFileChooser;
import javax.swing.JFrame;
import java.util.Collection;
import java.util.ArrayList;
import java.nio.file.SimpleFileVisitor;
public class KFile extends SimpleFileVisitor<Path> {
public static void main(String[] args) {
Path currPath = Paths.get("");
String currDir = currPath.toAbsolutePath().toString();
System.out.println(currDir);
File dataDir = chooseDir("open");
File destDir = chooseDir("save");
if(!destDir.exists()) {
try {
destDir.mkdir();
}
catch (SecurityException se) {
System.out.println("Couldn't make directory!");
}
}
int n = 0;
if(dataDir.exists()) {
Collection<Path> allDir = new ArrayList<Path>();
try {
addTree(dataDir.toPath(),allDir);
}
catch (IOException e) {
System.out.println("Error with scanning");
}
for( Path thisPath : allDir ) {
if(thisPath.toString().contains("Report.pdf")) {
Path thisDir = thisPath.getParent();
File f = new File(thisDir.toString(), "\\Report.txt");
n = n + 1;
String fileName = "Report " + n + ".pdf";
try {
fileName = parseName(f);
System.out.println(fileName);
} catch (IOException e) {
e.printStackTrace();
}
File thisFile = new File(destDir + "\\" + fileName);
try {
copyFile(thisPath.toFile(),thisFile);
} catch ( IOException e) {
e.printStackTrace();
}
}
}
}
}
public static boolean copyFile(File sourceFile, File destFile) throws IOException {
//create file if it doesn't exist.
if(!destFile.exists()) {
destFile.createNewFile();
}
FileChannel source = null;
FileChannel destination = null;
try {
source = new FileInputStream(sourceFile).getChannel();
destination = new FileOutputStream(destFile).getChannel();
destination.transferFrom(source, 0, source.size());
}
finally {
if(source != null) {
source.close();
}
if(destination != null) {
destination.close();
return true;
}
return false;
}
}
public static File chooseDir(String s) {
JFrame myFrame = new JFrame("HPLC Data Transfer");
myFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
myFrame.pack();
myFrame.setVisible(true);
JFileChooser chooser = new JFileChooser();
File currDir = new File(System.getProperty("user.home") + "\\Documents");
chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
chooser.setCurrentDirectory(currDir);
int choice = 0;
if (s.equals("save")) {
choice = chooser.showSaveDialog(myFrame);
} else {
choice = chooser.showOpenDialog(myFrame);
}
myFrame.setVisible(false);
myFrame.removeAll();
if(choice == JFileChooser.APPROVE_OPTION) {
System.out.println("You chose to open: " + chooser.getSelectedFile().getName());
return chooser.getSelectedFile();
}
return new File("");
}
static String parseName(File f) throws IOException {
BufferedReader textReader = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-16"));
int lnCnt = 32;
String[] fileData = new String[lnCnt];
for (int i = 0; i < lnCnt; i++) {
fileData[i] = textReader.readLine();
}
fileData[1] = fileData[1].replace("\uFEFF","");
String name = fileData[1].substring(13) + ".pdf";
textReader.close();
return name;
}
static void addTree(Path directory, final Collection<Path> all)
throws IOException {
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
all.add(file);
return FileVisitResult.CONTINUE;
}
});
}
}

You could try changing
myFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
to
myFrame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
and then call
myFrame.dispose();
to terminate the JFrame.
Since javadocs says EXIT_ON_CLOSE terminates the whole program using System.exit(); I'm not sure if that's the problem that is stopping your application but I hope it helps :)

It looks like you just called setVisible(false) when dealing with your JFrame. That just hides your JFrame, it doesn't get rid of it. If you want to get rid of your frame entirely (and all of its resources), call myFrame.dispose();

Print md5 checksum values using hashmap and strings

i have following code for comparing the md5 hash values for two folder but i need to show the list of files and the hash value of each file. can anyone please help me out with this. i just need to get hash value for one folder only.
package com.example;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
public class Compare
{
//This can be any folder locations which you want to compare
File dir1 = new File("/Users/Samip/Desktop/crypto");
File dir2 = new File("/Users/Samip/Desktop/crypto1");
public static void main(String ...args)
{
Compare compare = new Compare();
try
{
compare.getDiff(compare.dir1,compare.dir2);
}
catch(IOException ie)
{
ie.printStackTrace();
}
}
public void getDiff(File dirA, File dirB) throws IOException
{
File[] fileList1 = dirA.listFiles();
File[] fileList2 = dirB.listFiles();
Arrays.sort(fileList1);
Arrays.sort(fileList2);
HashMap<String, File> map1;
if(fileList1.length < fileList2.length)
{
map1 = new HashMap<String, File>();
for(int i=0;i<fileList1.length;i++)
{
map1.put(fileList1[i].getName(),fileList1[i]);
}
compareNow(fileList2, map1);
}
else
{
map1 = new HashMap<String, File>();
for(int i=0;i<fileList2.length;i++)
{
map1.put(fileList2[i].getName(),fileList2[i]);
}
compareNow(fileList1, map1);
}
}
public void compareNow(File[] fileArr, HashMap<String, File> map) throws IOException
{
for(int i=0;i<fileArr.length;i++)
{
String fName = fileArr[i].getName();
File fComp = map.get(fName);
map.remove(fName);
if(fComp!=null)
{
if(fComp.isDirectory())
{
getDiff(fileArr[i], fComp);
}
else
{
String cSum1 = checksum(fileArr[i]);
String cSum2 = checksum(fComp);
if(!cSum1.equals(cSum2))
{
System.out.println(fileArr[i].getName()+"\t\t"+ "different");
}
else
{
System.out.println(fileArr[i].getName()+"\t\t"+"identical");
}
}
}
else
{
if(fileArr[i].isDirectory())
{
traverseDirectory(fileArr[i]);
}
else
{
System.out.println(fileArr[i].getName()+"\t\t"+"only in "+fileArr[i].getParent());
}
}
}
Set<String> set = map.keySet();
Iterator<String> it = set.iterator();
while(it.hasNext())
{
String n = it.next();
File fileFrmMap = map.get(n);
map.remove(n);
if(fileFrmMap.isDirectory())
{
traverseDirectory(fileFrmMap);
}
else
{
System.out.println(fileFrmMap.getName() +"\t\t"+"only in "+ fileFrmMap.getParent());
}
}
}
public void traverseDirectory(File dir)
{
File[] list = dir.listFiles();
for(int k=0;k<list.length;k++)
{
if(list[k].isDirectory())
{
traverseDirectory(list[k]);
}
else
{
System.out.println(list[k].getName() +"\t\t"+"only in "+ list[k].getParent());
}
}
}
public String checksum(File file)
{
try
{
InputStream fin = new FileInputStream(file);
java.security.MessageDigest md5er = MessageDigest.getInstance("MD5");
byte[] buffer = new byte[1024];
int read;
do
{
read = fin.read(buffer);
if (read > 0)
md5er.update(buffer, 0, read);
} while (read != -1);
fin.close();
byte[] digest = md5er.digest();
if (digest == null)
return null;
String strDigest = "0x";
for (int i = 0; i < digest.length; i++)
{
strDigest += Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1).toUpperCase();
}
return strDigest;
}
catch (Exception e)
{
return null;
}
}
}

In you main method, instead using Compare.getDiff(dir1, dir2) you want to
Get a file listing of your target directory
Invoke Compare.checksum(file) on each file and print the result
Looks like you have all the code, you just need to reshape it a little.

Consider this example. The hash-generating code has been taken from your previous question - same goes for the file-iteration code. You just replace that folder to match your.
import java.io.*;
import java.security.MessageDigest;
public class PrintChecksums {
public static void main(String[] args) {
String sourceDir = "/Users/Jan/Desktop/Folder1";
try {
new PrintChecksums().printHashs(new File(sourceDir));
} catch (Exception e) {
e.printStackTrace();
}
}
private void printHashs(File sourceDir) throws Exception {
for (File f : sourceDir.listFiles()) {
String hash = createHash(f); // That you almost have
System.out.println(f.getAbsolutePath() + " / Hashvalue: " + hash);
}
}
public String createHash(File datafile) throws Exception {
// SNIP - YOUR CODE BEGINS
MessageDigest md = MessageDigest.getInstance("SHA1");
FileInputStream fis = new FileInputStream(datafile);
byte[] dataBytes = new byte[1024];
int nread = 0;
while ((nread = fis.read(dataBytes)) != -1) {
md.update(dataBytes, 0, nread);
}
byte[] mdbytes = md.digest();
// convert the byte to hex format
StringBuffer sb = new StringBuffer("");
for (int i = 0; i < mdbytes.length; i++) {
sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
}
// SNAP - YOUR CODE ENDS
return sb.toString();
}
}

Please have a look at the below code. I have added a function printCheckSum() which iterates though directory, scans each file and prints its hash value.
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
public class Compare
{
//This can be any folder locations which you want to compare
File dir1 = new File("D:\\dir1");
File dir2 = new File("D:\\dir2");
public static void main(String ...args)
{
Compare compare = new Compare();
try
{
compare.printCheckSum(compare.dir1);
}
catch(IOException ie)
{
ie.printStackTrace();
}
}
public void getDiff(File dirA, File dirB) throws IOException
{
File[] fileList1 = dirA.listFiles();
File[] fileList2 = dirB.listFiles();
Arrays.sort(fileList1);
Arrays.sort(fileList2);
HashMap<String, File> map1;
if(fileList1.length < fileList2.length)
{
map1 = new HashMap<String, File>();
for(int i=0;i<fileList1.length;i++)
{
map1.put(fileList1[i].getName(),fileList1[i]);
}
compareNow(fileList2, map1);
}
else
{
map1 = new HashMap<String, File>();
for(int i=0;i<fileList2.length;i++)
{
map1.put(fileList2[i].getName(),fileList2[i]);
}
compareNow(fileList1, map1);
}
}
public void compareNow(File[] fileArr, HashMap<String, File> map) throws IOException
{
for(int i=0;i<fileArr.length;i++)
{
String fName = fileArr[i].getName();
File fComp = map.get(fName);
map.remove(fName);
if(fComp!=null)
{
if(fComp.isDirectory())
{
getDiff(fileArr[i], fComp);
}
else
{
String cSum1 = checksum(fileArr[i]);
String cSum2 = checksum(fComp);
if(!cSum1.equals(cSum2))
{
System.out.println(fileArr[i].getName()+"\t\t"+ "different");
}
else
{
System.out.println(fileArr[i].getName()+"\t\t"+"identical");
}
}
}
else
{
if(fileArr[i].isDirectory())
{
traverseDirectory(fileArr[i]);
}
else
{
System.out.println(fileArr[i].getName()+"\t\t"+"only in "+fileArr[i].getParent());
}
}
}
Set<String> set = map.keySet();
Iterator<String> it = set.iterator();
while(it.hasNext())
{
String n = it.next();
File fileFrmMap = map.get(n);
map.remove(n);
if(fileFrmMap.isDirectory())
{
traverseDirectory(fileFrmMap);
}
else
{
System.out.println(fileFrmMap.getName() +"\t\t"+"only in "+ fileFrmMap.getParent());
}
}
}
public void traverseDirectory(File dir)
{
File[] list = dir.listFiles();
for(int k=0;k<list.length;k++)
{
if(list[k].isDirectory())
{
traverseDirectory(list[k]);
}
else
{
System.out.println(list[k].getName() +"\t\t"+"only in "+ list[k].getParent());
}
}
}
public String checksum(File file)
{
try
{
InputStream fin = new FileInputStream(file);
java.security.MessageDigest md5er = MessageDigest.getInstance("MD5");
byte[] buffer = new byte[1024];
int read;
do
{
read = fin.read(buffer);
if (read > 0)
md5er.update(buffer, 0, read);
} while (read != -1);
fin.close();
byte[] digest = md5er.digest();
if (digest == null)
return null;
String strDigest = "0x";
for (int i = 0; i < digest.length; i++)
{
strDigest += Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1).toUpperCase();
}
return strDigest;
}
catch (Exception e)
{
return null;
}
}
public void printCheckSum(File dir) throws IOException{
File[] fileList = dir.listFiles();
for(File file : fileList){
if(file.isDirectory()){
printCheckSum(file);
}else
System.out.println(file.getName() +"\t :: \t" + checksum(file));
}
}
}
Hope this helps. Cheers!

How to download All the URLs one by one and keep in different folders

I have one html file where I have kept all the URLs(Download link for CSV files).I want a tool/program that has to go through each url one by one and download the file, Then keep the file in the specified folder which will be written in the same html file itself.
html file is a table with 3 columns
File name,File location and download URL
Url will download the CSV file after opening a new window (target=_blank).Also after download it will close the child window automatically if there is no error.
I have tried the automation(Selenium using java)
But there are some challenges as follows.
It should wait until the download completes
Sometimes the url may show error,in that case it should close the child window and return to parent window
I have resolved the 1st case by keeping a watcher which will check whether the file is downloaded or not each second(by counting the number of csv files in the folder)
I can switch to child window and check whether there is any error but if there is no error my driver is got stuck over there.
How to resolve this
Code to check whether error is there in child window
public boolean foundError(FirefoxDriver driver) {
System.out.println(browser.getWindowHandle() + "Parent" + parentHandle);
String child = "";
int numberOfWindows = 0;
//return true;
if (driver.getWindowHandles().size() > 1) {
for (String winHandle : driver.getWindowHandles()) {
numberOfWindows++;
if (!parentHandle.equals(winHandle)) {
child = winHandle;
System.out.println("Child" + winHandle);
}
}
}
if (numberOfWindows > 1) {
System.out.println("tostring1" + driver.toString());
if (!parentHandle.equals(child)) {
driver.switchTo().window(child);
}
System.out.println("Switched to child");
Set set = driver.getWindowHandles();
System.out.println("Number of windows=" + set.size());
// System.out.println("Number of windows="+set.size()+"driver url"+driver.getCurrentUrl());
// System.out.println("tostring2"+driver.toString());
try {
// WebDriverWait wait1 = new WebDriverWait(driver, 5);
System.out.println("Body text" + driver.findElementByTagName("body").getText());/////////////////////////////Here driver will get stuck
//System.out.println("text"+driver.findElementByClassName("body").toString());
// List<WebElement> elements=driver.findElementsByClassName("ErrorBody");elements.size()>0
if (!driver.findElementByTagName("body").getText().equals("")) {
driver.close();
driver.switchTo().window(parentHandle);
return true;
}
System.out.println("No error");
driver.switchTo().window(parentHandle);
System.out.println("Switched to parent");
} catch (Exception e) {
System.out.println("Error Catch block page time out:" + e);
driver.switchTo().window(parentHandle);
return false;
// driver.switchTo().window(parentHandle);
}
}
return false;
}

I used different method
using Jsoup to parse the html file and downloading
import java.io.File;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class URLDownload {
private Document doc;
String url = "", folder, file;
private final File sourceFile;
int i = 1;
int r = 1, c = 1;
int anchorCol = 3;
Library lib;
URLDownload() {
lib = new Library();
sourceFile = new File("Download.html");
try {
doc = Jsoup.parse(sourceFile, "UTF-8");
} catch (IOException ex) {
Logger.getLogger(URLDownload.class.getName()).log(Level.SEVERE, null, ex);
}
//Elements links = doc.select("a[href]");
Elements rows = doc.select("tr");
System.out.println("Size=" + rows.size());
for (Element row : rows) {
Elements cols = row.getElementsByTag("td");
c = 1;
for (Element col : cols) {
System.out.println("Row"+r);
if (c == 1) {
file = col.text();//System.out.println("File in main"+file);
} else if (c == 2) {
folder = col.text();//System.out.println("Folder in main"+folder);
} else {
try {
url = col.getElementsByTag("a").attr("href");
} catch (Exception e) {
System.out.print("-");
}
}
c++;
}
if (!url.equals("")) {
lib.setLocation(file,folder);
lib.downloadFile(url);
}
url = "";
i++;
r++;
}
}
public static void main(String arg[]) {
new URLDownload();
}
}
and following is the Library class file
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class Library {
boolean downloaded = false;
Thread t;
int waitTime = 0;
String baseLoc = "";
int size = 1024, ByteWritten = 0;
URL url;
URLConnection uCon = null;
String folderLoc = "", file = "firstFile.csv";
File loc;
private OutputStream outStream;
private InputStream is=null;
private byte[] buf;
private int ByteRead;
private int FolderInUrl = 4;
private boolean rootFolder = true;
private File resultFile;
private FileOutputStream fileResult;
private XSSFWorkbook workbookResult;
private XSSFSheet sheetResult;
private int updateExcelRowNum = -1;
private int updateExcelColNum = -1;
String date;
private int waitLimit = 900000;
Library() {
/*System.out.print(Calendar.getInstance().toString());
Date d=new Date();
String date=d.toString();
System.out.println(date);*/
//t = new Thread(this);
// t.start();
date = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(Calendar.getInstance().getTime());
System.out.print(date);
baseLoc = date + "/";
WriteDataToExcel();
baseLoc += "Business Reports/";
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Report Name");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Path");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Status");
updateExcel();
}
public void setLocation(String a, String b) {
file = a;
file += ".csv";
folderLoc = baseLoc + getFolderPath(b);
// System.out.println("File Name: "+file);
// System.out.println("Folder loc: "+folderLoc);
}
public String getFolderPath(String b) {
String path = "";
try {
System.out.println("path" + b);
path = b;
// path = java.net.URLDecoder.decode(b, "UTF-8");
String p[] = path.split("/");
path = "";
for (int i = FolderInUrl; i < p.length - 1; i++) {
rootFolder = false;
p[i] = removeSpacesAtEnd(p[i]);
path = path + p[i] + "/";
}
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
return path;
}
public void downloadFile(String urlString) {
// System.out.println("Started");
try {
url = new URL(urlString);
} catch (MalformedURLException ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
try {
loc = new File(folderLoc);
if (!loc.exists()) {
loc.mkdirs();
}
outStream = new BufferedOutputStream(new FileOutputStream(folderLoc + file));
uCon = url.openConnection();
uCon.setReadTimeout(waitLimit);
is = uCon.getInputStream();
downloaded=true;
buf = new byte[size];
while ((ByteRead = is.read(buf)) != -1) {
System.out.println("while executing" + ByteRead);
outStream.write(buf, 0, ByteRead);
ByteWritten += ByteRead;
}
//System.out.println("Downloaded" + ByteWritten);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
if (ByteWritten < 1000) {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
} else {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
}
updateExcel();
} catch (Exception e) {
System.out.println("error catch" + e);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Rejected the Download after waiting " + (waitLimit / 60000) + " minutes");
updateExcel();
waitTime = 0;
} finally {
try {
System.out.println("Error in streams");
if(downloaded)
is.close();
outStream.close();
downloaded= false;
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void moveToFolder(String reportName, String path) {
try {
File repo = new File(folderLoc + "/" + reportName + ".csv");
path = folderLoc + "/" + path;
File pathFolder = new File(path);
if (!pathFolder.exists()) {
pathFolder.mkdirs();
}
pathFolder = new File(path + reportName + ".csv");
System.out.println("Path=" + pathFolder.getAbsolutePath() + "\nReport path=" + repo.getAbsolutePath());
System.out.println("Source" + repo.getAbsolutePath());
//System.out.println("Status" + repo.renameTo(new File(pathFolder.getAbsolutePath())));
System.out.println("Status" + Files.move(repo.toPath(), new File(pathFolder.getAbsolutePath()).toPath(), REPLACE_EXISTING));
//Files.
} catch (Exception e) {
System.out.println("error while moving" + e);
}
}
public String changeSpecialCharacters(String report) {
report = report.replaceAll(":", "_");
return report;
}
public String removeSpacesAtEnd(String inputPath) {
for (int i = inputPath.length() - 1; i >= 0; i--) {
if (inputPath.charAt(i) != ' ') {
break;
} else {
System.out.println("Before string is" + inputPath);
inputPath = inputPath.substring(0, i);
System.out.println("AFter string is" + inputPath);
}
}
return inputPath;
}
public void WriteDataToExcel() {
try {
// file = new FileInputStream(new File("config.xlsx"));
// File resultFolder = new File("Results");
// if (resultFolder.exists()) {
// deleteDirectory(resultFolder);
// }
// resultFolder.mkdirs();
if (!new File(baseLoc).exists()) {
new File(baseLoc).mkdirs();
}
resultFile = new File(baseLoc + "Reports info " + date + ".xlsx");
System.out.println("Path" + resultFile.getAbsolutePath());
resultFile.createNewFile();
// rFilePath = resultFile.getAbsolutePath();
fileResult = new FileOutputStream(resultFile);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//Get the workbook instance for XLS file
// System.out.println("file success");
XSSFWorkbook workbook = null;
try {
workbookResult = new XSSFWorkbook();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Opening the browser");
//Get first sheet from the workbook
sheetResult = workbookResult.createSheet();
//sheetResult.set
//Get iterator to all the rows in current sheet
//Get iterator to all cells of current row
//ar.add(folderLocation);
// ar.add(firefoxProfileLocation);
}
public void updateExcel() {
try {
//fileResult.close();
fileResult = new FileOutputStream(resultFile);
workbookResult.write(fileResult);
fileResult.close();
} catch (Exception e) {
System.out.println(e);
}
}
public void createRowExcel(int num) {
updateExcelRowNum++;
num = updateExcelRowNum;
sheetResult.createRow(num);
}
public void updateRowColExcel(int rnum, int cnum, String value) {
updateExcelColNum++;
cnum = updateExcelColNum;
sheetResult.getRow(rnum).createCell(cnum);
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void updateColumn(int rnum, int cnum, String value) {
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void resetCounters() {
updateExcelColNum = -1;
}
/* #Override
public void run() {
while (true) {
if (true) {
waitTime += 1000;
System.out.println(waitTime);
if (waitTime > waitLimit) {
try {
is.close();
outStream.close();
//downloaded=false;
// cancelDownload=true;
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
try {
Thread.sleep(1000);
} catch (Exception e) {
}
}
}*/
}

FileNotFoundException when using Hadoop distributed cache

this time someone should please relpy
i am struggling with running my code using distributed cahe. i have already the files on hdfs but when i run this code :
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.awt.image.Raster;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URISyntaxException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.imageio.ImageIO;
import org.apache.hadoop.filecache.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import java.lang.String;
import java.lang.Runtime;
import java.net.URI;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
public class blur2 {
public static class BlurMapper extends MapReduceBase implements Mapper<Text, BytesWritable, LongWritable, BytesWritable>
{
OutputCollector<LongWritable, BytesWritable> goutput;
int IMAGE_HEIGHT = 240;
int IMAGE_WIDTH = 320;
public BytesWritable Gmiu;
public BytesWritable Gsigma;
public BytesWritable w;
byte[] bytes = new byte[IMAGE_HEIGHT*IMAGE_WIDTH*3];
public BytesWritable emit = new BytesWritable(bytes);
int count = 0;
int initVar = 125;
public LongWritable l = new LongWritable(1);
byte[] byte1 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH];
byte[] byte2 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH];
byte[] byte3 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH];
public void map(Text key, BytesWritable file,OutputCollector<LongWritable, BytesWritable> output, Reporter reporter) throws IOException {
goutput = output;
BufferedImage img = ImageIO.read(new ByteArrayInputStream(file.getBytes()));
Raster ras=img.getData();
DataBufferByte db= (DataBufferByte)ras.getDataBuffer();
byte[] data = db.getData();
if(count==0){
for(int i=0;i<IMAGE_HEIGHT*IMAGE_WIDTH;i++)
{
byte1[i]=20;
byte2[i]=125;
}
Gmiu = new BytesWritable(data);
Gsigma = new BytesWritable(byte1);
w = new BytesWritable(byte2);
count++;
}
else{
byte1 = Gmiu.getBytes();
byte2 = Gsigma.getBytes();
byte3 = w.getBytes();
for(int i=0;i<IMAGE_HEIGHT*IMAGE_WIDTH;i++)
{
byte pixel = data[i];
Double tempmiu=new Double(0.0);
Double tempsig=new Double(0.0);
double temp1=0.0; double alpha = 0.05;
tempmiu = (1-alpha)*byte1[i] + alpha*pixel;
temp1=temp1+(pixel-byte1[i])*(pixel-byte1[i]);
tempsig=(1-alpha)*byte2[i]+ alpha*temp1;
byte1[i] = tempmiu.byteValue();
byte2[i]= tempsig.byteValue();
Double w1=new Double((1-alpha)*byte3[i]+alpha*100);
byte3[i] = w1.byteValue();
}
Gmiu.set(byte1,0,IMAGE_HEIGHT*IMAGE_WIDTH);
Gsigma.set(byte2,0,IMAGE_HEIGHT*IMAGE_WIDTH);
w.set(byte3,0,IMAGE_HEIGHT*IMAGE_WIDTH);
}
byte1 = Gsigma.getBytes();
for(int i=0;i<IMAGE_HEIGHT*IMAGE_WIDTH;i++)
{
bytes[i]=byte1[i];
}
byte1 = Gsigma.getBytes();
for(int i=0;i<IMAGE_HEIGHT*IMAGE_WIDTH;i++)
{
bytes[IMAGE_HEIGHT*IMAGE_WIDTH+i]=byte1[i];
}
byte1 = w.getBytes();
for(int i=0;i<IMAGE_HEIGHT*IMAGE_WIDTH;i++)
{
bytes[2*IMAGE_HEIGHT*IMAGE_WIDTH+i]=byte1[i];
}
emit.set(bytes,0,3*IMAGE_HEIGHT*IMAGE_WIDTH);
}
#Override
public void close(){
try{
goutput.collect(l, emit);
}
catch(Exception e){
e.printStackTrace();
System.exit(-1);
}
}
}
//end of first job , this is running perfectly
public static void main(String[] args) throws URISyntaxException {
if(args.length!=3) {
System.err.println("Usage: blurvideo input output");
System.exit(-1);
}
JobClient client = new JobClient();
JobConf conf = new JobConf(blur2.class);
conf.setOutputValueClass(BytesWritable.class);
conf.setInputFormat(SequenceFileInputFormat.class);
//conf.setNumMapTasks(n)
SequenceFileInputFormat.addInputPath(conf, new Path(args[0]));
TextOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setMapperClass(BlurMapper.class);
conf.setNumReduceTasks(0);
//conf.setReducerClass(org.apache.hadoop.mapred.lib.IdentityReducer.class);
client.setConf(conf);
try {
JobClient.runJob(conf);
} catch (Exception e) {
e.printStackTrace();
}
// exec("jar cf /home/hmobile/hadoop-0.19.2/imag /home/hmobile/hadoop-0.19.2/output");
JobClient client2 = new JobClient();
JobConf conf2 = new JobConf(blur2.class);
conf2.setOutputValueClass(BytesWritable.class);
conf2.setInputFormat(SequenceFileInputFormat.class);
//conf.setNumMapTasks(n)
SequenceFileInputFormat.addInputPath(conf2, new Path(args[0]));
SequenceFileOutputFormat.setOutputPath(conf2, new Path(args[2]));
conf2.setMapperClass(BlurMapper2.class);
conf2.setNumReduceTasks(0);
DistributedCache.addCacheFile(new URI("~/ayush/output/part-00000"), conf2);// these files are already on the hdfs
DistributedCache.addCacheFile(new URI("~/ayush/output/part-00001"), conf2);
client2.setConf(conf2);
try {
JobClient.runJob(conf2);
} catch (Exception e) {
e.printStackTrace();
}
}
public static class BlurMapper2 extends MapReduceBase implements Mapper<Text, BytesWritable, LongWritable, BytesWritable>
{
int IMAGE_HEIGHT = 240;
int T =60;
int IMAGE_WIDTH = 320;
public BytesWritable Gmiu;
public BytesWritable Gsigma;
public BytesWritable w;
byte[] bytes = new byte[IMAGE_HEIGHT*IMAGE_WIDTH];
public BytesWritable emit = new BytesWritable(bytes);
int initVar = 125;int gg=0;
int K=64;int k=0,k1=0,k2=0;
public LongWritable l = new LongWritable(1);
byte[] Gmiu1 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH*K];
byte[] Gsigma1 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH*K];
byte[] w1 = new byte[IMAGE_HEIGHT*IMAGE_WIDTH*K];
public Path[] localFiles=new Path[2];
private FileSystem fs;
#Override
public void configure(JobConf conf2)
{
try {
fs = FileSystem.getLocal(new Configuration());
localFiles = DistributedCache.getLocalCacheFiles(conf2);
//System.out.println(localFiles[0].getName());
} catch (IOException ex) {
Logger.getLogger(blur2.class.getName()).log(Level.SEVERE, null, ex);
}
}
public void map(Text key, BytesWritable file,OutputCollector<LongWritable, BytesWritable> output, Reporter reporter) throws IOException
{
if(gg==0){
//System.out.println(localFiles[0].getName());
String wrd; String line;
for(Path f:localFiles)
{
if(!f.getName().endsWith("crc"))
{
// FSDataInputStream localFile = fs.open(f);
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(fs.open(f)));
int c = 0;
try {
while ((line = br.readLine()) != null) {
StringTokenizer itr = new StringTokenizer(line, " ");
while (itr.hasMoreTokens()) {
wrd = itr.nextToken();
c++;
int i = Integer.parseInt(wrd, 16);
Integer I = new Integer(i);
byte b = I.byteValue();
if (c < IMAGE_HEIGHT * IMAGE_WIDTH) {
Gmiu1[k] = b;k++;
} else {
if ((c >= IMAGE_HEIGHT * IMAGE_WIDTH) && (c < 2 * IMAGE_HEIGHT * IMAGE_WIDTH)) {
Gsigma1[k] = b;k1++;
} else {
w1[k] = b;k2++;
}
}
}
}
} catch (IOException ex) {
Logger.getLogger(blur2.class.getName()).log(Level.SEVERE, null, ex);
}
} catch (FileNotFoundException ex) {
Logger.getLogger(blur2.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
br.close();
} catch (IOException ex) {
Logger.getLogger(blur2.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
gg++;
}
}
}
}
tackled a lot with this, can anyone please tell why i am getting this error:
java.io.FileNotFoundException: File does not exist: ~/ayush/output/part-00000
at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:394)
at org.apache.hadoop.filecache.DistributedCache.getTimestamp(DistributedCache.java:475)
at org.apache.hadoop.mapred.JobClient.configureCommandLineOptions(JobClient.java:676)
at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:774)
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1127)
at blur2.main(blur2.java:175)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:165)
at org.apache.hadoop.mapred.JobShell.run(JobShell.java:54)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:79)
at org.apache.hadoop.mapred.JobShell.main(JobShell.java:68)

The problem is with the filename you are using "~/ayush/output/part-00000" relies on Unix shell (sh, bash, ksh) tilde expansion to replace the "~" with the pathname of your home directory.
Java (and C, and C++, and most other programming languages) don't do tilde expansion. You need to provide the pathname as "/home/ayush/output/part-00000" ... or whatever absolute pathname it is that the tilded form expands to.
Strictly speaking, the URI should be created as follows:
new File("/home/ayush/output/part-00000").toURI()
not as
new URI("/home/ayush/output/part-00000")
The latter creates a URI without a "protocol", and that could be problematic.