Develop Reference

Java Folder/File watch

I am trying to build a application that watch a folder and its sub folders to detect file creation or modification. Total files to watch will be growing day by day.
I had tried with java nio WatchService and apache common FileAlterationObserver. WatchService sometimes missing event when file creation/modification happens after WatchKey is taken and before reset. Since FileAlterationObserver is based on polling, when file count is increasing performance is also degrading.
What will be the best approach to build such an application?

Thank you #DuncG. After going through the sample mentioned, I found my solution to my problem.
Adding this sample code if someone facing the same problem.
Here in the example I am adding all the events to a set (this will remove the duplicate events) and process the saved events once the WatchKey is empty. New directories will be registered to WatchService while processing the saved events.
package com.filewatcher;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardWatchEventKinds;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WatchService implements Runnable {
private static final long POLL_DELAY = 3;
private static final Logger LOGGER = LoggerFactory.getLogger(WatchService.class);
private final WatchService watcher;
private final Map<WatchKey, Path> keys;
private final Set<Path> events = new HashSet<Path>();
public WatchService(Path dir) throws IOException {
this.watcher = FileSystems.getDefault().newWatchService();
this.keys = new HashMap<WatchKey, Path>();
walkAndRegisterDirectories(dir);
}
#Override
public void run() {
while (true) {
try {
WatchKey key;
try {
key = watcher.poll(POLL_DELAY, TimeUnit.SECONDS);
} catch (InterruptedException x) {
return;
}
if (key != null) {
Path root = keys.get(key);
for (WatchEvent<?> event : key.pollEvents()) {
Path eventPath = (Path) event.context();
if (eventPath == null) {
System.out.println(event.kind());
continue;
}
Path fullPath = root.resolve(eventPath);
events.add(fullPath);
}
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
}
} else {
if (events.size() > 0) {
processEvents(events);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
/**
* Process events and register new directory with watch service
* #param events
* #throws IOException
*/
private void processEvents(Set<Path> events) throws IOException {
for (Path path : events) {
// register directory with watch service if its not already registered
if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS) && !this.keys.containsValue(path)) {
registerDirectory(path);
// Since new directory was not registered, get all files inside the directory.
// new/modified files after this will get notified by watch service
File[] files = path.toFile().listFiles();
for (File file : files) {
LOGGER.info(file.getAbsolutePath());
}
} else {
LOGGER.info(path.toString());
}
}
// clear events once processed
events.clear();
}
/**
* Register a directory and its sub directories with watch service
* #param root folder
* #throws IOException
*/
private void walkAndRegisterDirectories(final Path root) throws IOException
{
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
registerDirectory(dir);
return FileVisitResult.CONTINUE;
}
});
}
/**
* Register a directory with watch service
* #param directory
* #throws IOException
*/
private void registerDirectory(Path dir) throws IOException {
WatchKey key = dir.register(this.watcher, StandardWatchEventKinds.ENTRY_CREATE,
StandardWatchEventKinds.ENTRY_MODIFY);
this.keys.put(key, dir);
}
}
public class FileWatcherApplication implements CommandLineRunner {
#Value("${filewatch.folder}")
private String rootPath;
public static void main(String[] args) {
SpringApplication.run(FileWatcherApplication.class, args);
}
#Override
public void run(String... args) throws Exception {
File rootFolder = new File(rootPath);
if (!rootFolder.exists()) {
rootFolder.mkdirs();
}
new Thread(new WatchService(Paths.get(rootPath)), "WatchThread").start();
}
}

Java copy files matching pattern from many folders to another folder

Please take a look at the code I have so far and if possible explain what I'm doing wrong. I'm trying to learn.
I made a little program to search for a type of file in a directory and all its sub-directories and copy them into another folder.
Code
import java.util.ArrayList;
import java.util.List;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
public class FandFandLoop {
public static void main(String[] args) {
final File folder = new File("C:/Users/ina/src");
List<String> result = new ArrayList<>();
search(".*\\.txt", folder, result);
File to = new File("C:/Users/ina/dest");
for (String s : result) {
System.out.println(s);
File from = new File(s);
try {
copyDir(from.toPath(), to.toPath());
System.out.println("done");
}
catch (IOException ex) {
ex.printStackTrace();
}
}
}
public static void copyDir(Path src, Path dest) throws IOException {
Files.walk(src)
.forEach(source -> {
try {
Files.copy(source, dest.resolve(src.relativize(source)),
StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
e.printStackTrace();
}
});
}
public static void search(final String pattern, final File folder, List<String> result) {
for (final File f : folder.listFiles()) {
if (f.isDirectory()) {
search(pattern, f, result);
}
if (f.isFile()) {
if (f.getName().matches(pattern)) {
result.add(f.getAbsolutePath());
}
}
}
}
}
It works, but what it actually does is to take my .txt files and write them into another file named dest without extension.
And at some point, it deletes the folder dest.
The deletion happens because of StandardCopyOption.REPLACE_EXISTING, if I understand this, but what I would have liked to obtain was that if several files had the same name then only one copy of it should be kept.

There is no need to call Files.walk on the matched source files.
You can improve this code by switching completely to using java.nio.file.Path and not mixing string paths and File objects. Additionally instead of calling File.listFiles() recursively you can use Files.walk or even better Files.find.
So you could instead use the following:
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Objects;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
public class CopyFiles {
public static void copyFiles(Path src, Path dest, PathMatcher matcher, CopyOption... copyOptions) throws IOException {
// Argument validation
if (!Files.isDirectory(src)) {
throw new IllegalArgumentException("Source '" + src + "' is not a directory");
}
if (!Files.isDirectory(dest)) {
throw new IllegalArgumentException("Destination '" + dest + "' is not a directory");
}
Objects.requireNonNull(matcher);
Objects.requireNonNull(copyOptions);
BiPredicate<Path, BasicFileAttributes> filter = (path, attributes) -> attributes.isRegularFile() && matcher.matches(path);
// Use try-with-resources to close stream as soon as it is not longer needed
try (Stream<Path> files = Files.find(src, Integer.MAX_VALUE, filter)) {
files.forEach(file -> {
Path destFile = dest.resolve(src.relativize(file));
try {
copyFile(file, destFile, copyOptions);
}
// Stream methods do not allow checked exceptions, have to wrap it
catch (IOException ioException) {
throw new UncheckedIOException(ioException);
}
});
}
// Wrap UncheckedIOException; cannot unwrap it to get actual IOException
// because then information about the location where the exception was wrapped
// will get lost, see Files.find doc
catch (UncheckedIOException uncheckedIoException) {
throw new IOException(uncheckedIoException);
}
}
private static void copyFile(Path srcFile, Path destFile, CopyOption... copyOptions) throws IOException {
Path destParent = destFile.getParent();
// Parent might be null if dest is empty path
if (destParent != null) {
// Create parent directories before copying file
Files.createDirectories(destParent);
}
Files.copy(srcFile, destFile, copyOptions);
}
public static void main(String[] args) throws IOException {
Path srcDir = Paths.get("path/to/src");
Path destDir = Paths.get("path/to/dest");
// Could also use FileSystem.getPathMatcher
PathMatcher matcher = file -> file.getFileName().toString().endsWith(".txt");
copyFiles(srcDir, destDir, matcher);
}
}

Update : Move files from one folder to another based on file extension

Situation:
I am doing an automation where I have to download only CSV files from a set of files. And now i want to move only CSV files from one folder to another.
Question:
Can you please provide me code to delete files from the source folder as soon as the files are moved?
This is my code so far:
public class MyFilteredFileList {
public static void main(String a[])
{
try {
File source = new File("C:\\Users\\sh370472\\Downloads");
File dest = new File("E:\\Query\\");
FileUtils.copyDirectory(source, dest, new FileFilter() {
#Override
public boolean accept(File pathname)
{
boolean source=pathname.getName().toLowerCase().endsWith(".csv");
if (source)
return true;
return false;
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
}
Edit : But I am still facing a problem. It's downloading all the files at once and then deleting. But my requirement is something like - It should download first file -> move the downloaded file into another folder -> delete first downloaded file from download folder -> download second folder... and the process repeats

Just add pathname.deleteOnExit(); on accept method
#Override
public boolean accept(File pathname) {
boolean source = pathname.getName().toLowerCase().endsWith(".csv");
if (source) {
pathname.deleteOnExit();
return true;
}
return false;
}
whole code :
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
/**
* Created by Lenovo on 02/12/2018.
*/
public class FileMove {
public static void main(String a[])
{
try {
File source = new File("C:\\Users\\sh370472\\Downloads");
File dest = new File("E:\\Query\\");
FileUtils.copyDirectory(source, dest, new FileFilter() {
#Override
public boolean accept(File pathname) {
boolean source = pathname.getName().toLowerCase().endsWith(".csv");
if (source) {
pathname.deleteOnExit();
return true;
}
return false;
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
}

You can simply move the file this way.
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class MyFilteredFileList {
public static void main(String a[]){
try {
Files.move (Paths.get("C:\\Users\\sh370472\\Downloads\\file.txt"),
Paths.get("E:\\Query\\file.txt"));
} catch (IOException e) {
e.printStackTrace();
}
}
}

Copy this class and run :) Try setting the correct source and destination path if it doesn't work.
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
public class MoveFile {
private boolean moveFile(String source, String destination) throws IOException {
Path srcPath = Paths.get(source);
Path destPath = Paths.get(destination);
System.out.println("Moving processed file to archive");
Files.move(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING);
return true;
}
private File[] getTheFilesWithExtension(String filePath, String ext) {
File fileDir = new File(filePath);
File[] files = fileDir.listFiles((dir, name) -> name.toLowerCase().endsWith(ext));
return files;
}
public static void main(String... args){
{
MoveFile moveFile = new MoveFile();
String sourcePath = "C:\\\\Users\\\\sh370472\\\\Downloads";
String destPath = "E:\\Query\\";
String extension = ".csv";
try {
File[] files = moveFile.getTheFilesWithExtension(sourcePath, extension);
for (File file : files){
moveFile.moveFile(file.getPath(),destPath+file.getName());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
Note: Its for Java 8 and above , for lower java version, you can update the lambda expression in getTheFilesWithExtension with the below code
private File[] getTheFilesWithExtension(String filePath, String ext) {
File fileDir = new File(filePath);
File[] files = fileDir.listFiles(
new FilenameFilter() {
#Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(ext);
}
}
);
return files;
}

Access restriction: The type Player is not accessible due to restriction on required library C:\Program Files (x86)\Java\jre7\lib\ext\jmf.jar

I'm trying LiveAudioStreaming in java. For livestreming through JMF. I'm getting following error.
Access restriction: The type Player is not accessible due to restriction on required library C:\Program Files (x86)\Java\jre7\lib\ext\jmf.jar.
Please help. I used code from this link
Here is the code:
package com.simpleaudioplayer.simpleaudioplayer;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import javax.media.*; // import JMF classes
import javax.swing.JFileChooser;
public class Main {
private Player audioPlayer = null;
public SimpleAudioPlayer(URL url) {
try {
//MediaLocator ml=new MediaLocator(url);
audioPlayer = Manager.createPlayer(url);
} catch (Exception ex) {
System.out.println(ex);
}
}
public SimpleAudioPlayer(File file) throws MalformedURLException {
this(file.toURI().toURL());
}
public void play() {
audioPlayer.start(); // start playing
}
public void stop() {
audioPlayer.stop(); //stop playing
audioPlayer.close();
}
public static void main(String[] args) {
try {
// TODO code application logic here
JFileChooser fc = new JFileChooser();
fc.showOpenDialog(null);
File file = fc.getSelectedFile();
SimpleAduioPlayer sap = new SimpleAduioPlayer(file);
sap.play();
//sap.stop();
} catch (MalformedURLException ex) {
System.out.println(ex);
}
}
}

List pdf files recursively

I've created a tool that checks pdf files for errors. The tool selects with a filechooser a directory, checks if there are pdf files and scans them. But I want that the tool checks recursively the directory. If I try this code:
public class RecursiveFileDisplay {
public static void main(String[] args) {
File currentDir = new File("C://Users//Tommy//Desktop"); // current directory
displayDirectoryContents(currentDir);
}
public static void displayDirectoryContents(File dir) {
try {
File[] files = dir.listFiles();
for (File file : files) {
if (file.isDirectory()) {
displayDirectoryContents(file);
} else {
if (file.getName().endsWith((".pdf"))) {
System.out.println(" file:" + file.getCanonicalPath());
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
the tool will show me for each directory on a Desktop a new list and doesn't sum up all the results in one list...:

If you want your program to create a whole list with all the files processed, then you have to modify your displayDirectoryContent method as following:
public static List<File> displayDirectoryContents(File dir) {
ArrayList<File> rtnFiles = new ArrayList<File>();
try {
File[] files = dir.listFiles();
for (File file : files) {
if (file.isDirectory()) {
rtnFiles.addAll(displayDirectoryContents(file));
} else {
if (file.getName().endsWith((".pdf"))) {
System.out.println(" file:" + file.getCanonicalPath());
rtnFiles.add(file);
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return rtnFiles;
}
This way, it will return you with all the files found/processed. If you want this method only return you with failed files, then you will have to check not only the file is a .pdf file, but it is ok/failed, and then add to the list.
If you want to controll sum, then you can create a Java Class with two Integers one for pdf found files, and other for pdf failed found files. Recursively you can cum it up in the same way I build the list.
Hope it helps.

You can use the java.nio.file.Files.walkFileTree
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
public class FolderRecurs {
public Path pathToStart = Paths.get("The_URI_OF_THE_ROOT_DIRECTORY");
public List<File> listPDF = new ArrayList<>();
public FolderRecurs(Path pathToStart) {
this.pathToStart = pathToStart;
}
public void goRecurs() throws IOException{
Files.walkFileTree(pathToStart, new SimpleFileVisitor<Path>() {
//for folders
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs){
boolean error = false;
//your code
return error?FileVisitResult.TERMINATE : FileVisitResult.CONTINUE;
}
//for files
#Override
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs){
boolean error = false;
//Convert path to file
File file = filePath.toFile();
if (file.getName().endsWith((".pdf"))){
listPDF.add(file);
}
//your code
return error?FileVisitResult.TERMINATE : FileVisitResult.CONTINUE;
}
});
}
}