Develop Reference

java watchservice - running multiple job on cluster

I am using java watchservice to watch a specific folder when a specific file is written from another program.
I am running multiple jobs using this code and each watching each a specific folder that is different from each other.
This seems to work for the first job. But for the subsequent jobs, the watch service doesn't seem to be able get the write event even after the file it is looking for is written. It just kept idle as if the file has not yet been written.
Does anyone has experience with this?
Here is part of the code:
import java.util.*;
import java.nio.file.*;
import java.io.File;
private void run_code() {
WatchEvent.Kind watchType = StandardWatchEventKinds.ENTRY_CREATE;
while (i <= max_loop)
{
// Look out for output from python
String py_java_file = "Py2java_" + Integer.toString(i) + ".txt";
lookOutForFile(work_dir, py_java_file, watchType);
// Write finish file
try {
Files.write(Paths.get(work_dir+"java2Py_" + Integer.toString(i) + ".txt"), "run completed".getBytes());
}
catch (Exception e) {
print("Error writing file. ");
}
// Increment counter
i = i + 1;
}
}
private void lookOutForFile(String watchPath, String watchFile, WatchEvent.Kind watchType){
Path path = Paths.get(watchPath);
try (WatchService watchService = FileSystems.getDefault().newWatchService()) {
WatchKey key = path.register(watchService,
StandardWatchEventKinds.ENTRY_CREATE,
StandardWatchEventKinds.ENTRY_MODIFY,
StandardWatchEventKinds.ENTRY_DELETE);
startListening(watchService, watchPath, watchFile, watchType);
print("Found watch file");
} catch (Exception e) {
e.printStackTrace();
}
}
private void startListening (WatchService watchService, String watchPath, String watchFile, WatchEvent.Kind watchType)
throws InterruptedException {
boolean foundEvent = false;
while (!foundEvent) {
WatchKey queuedKey = watchService.take();
for (WatchEvent<?> watchEvent : queuedKey.pollEvents()) {
String sf1=String.format("kind=%s, count=%d, context=%s Context type=%s%n ", watchEvent.kind(), watchEvent.count(), watchEvent.context(), ((Path) watchEvent.context()).getClass());
if (watchEvent.kind() == watchType) {
String fileCreated = String.format("%s", watchEvent.context());
if (fileCreated.equals(watchFile))
{
foundEvent = true;
}
}
if (!queuedKey.reset()) {
break;
}
}
}
}

loop to check if a file exists

I have a folder that must contain always one file config8, and if a new file is created in this folder the old file is deleted and replaced by the new file with the same name config8.
I write this code
File file1 = new File("/home/olfa/Bureau/config/config8");
File file2 = new File("/home/olfa/Bureau/config/config9");
while (file2.exists())
{
file1.delete();
file2.renameTo(file1); }
}
serverConnection = new ServerConnection("/home/olfa/Bureau/config/config8");
I need to add a loop to check everytime if config9 is created.

Instead of a loop try a WatchService.
Basically you would be watching a particular directory for change and then you can react on this change.
https://docs.oracle.com/javase/tutorial/essential/io/notification.html
For example :
import static java.nio.file.StandardWatchEventKinds.*;
WatchService watcher = FileSystems.getDefault().newWatchService();
Path dir = ...;
try {
WatchKey key = dir.register(watcher,
ENTRY_CREATE,
ENTRY_DELETE,
ENTRY_MODIFY);
} catch (IOException x) {
System.err.println(x);
}
Then you can process your key events.

If you have to solve this task with Java 1.6, you can use https://commons.apache.org/proper/commons-vfs/, version 2.1.
Here is an example for moving all incoming config files to "config8":
import org.apache.commons.vfs2.*;
import org.apache.commons.vfs2.impl.DefaultFileMonitor;
import java.io.File;
public class ConfigWatcher {
private static final String configDirName = "target/config";
private static final String configName = "config8";
private static final String absoluteConfigName = new File(configDirName + File.separator + configName).getAbsolutePath();
private FileSystemManager manager = null;
FileObject configDir = null;
private FileObject configFile = null;
private FileChangeEvent lastEvent = null;
public void watchConfig() throws Exception {
manager = VFS.getManager();
DefaultFileMonitor fm = new DefaultFileMonitor(new ConfigFileListener());
configFile = manager.resolveFile(absoluteConfigName);
configDir = manager.resolveFile(new File(configDirName).getAbsolutePath());
fm.setDelay(1000);
fm.addFile(configDir);
fm.start();
}
class ConfigFileListener implements FileListener {
public void fileCreated(FileChangeEvent fileChangeEvent) throws Exception {
FileObject latestConfigFile = fileChangeEvent.getFile();
String fileBaseName = fileChangeEvent.getFile().getName().getBaseName();
if (!configName.endsWith(fileBaseName) && !fileChangeEvent.equals(lastEvent)) {
System.out.println("new config detected - move config");
latestConfigFile.moveTo(configFile);
}
lastEvent = fileChangeEvent;
}
public void fileChanged(FileChangeEvent fileChangeEvent) {
}
public void fileDeleted(FileChangeEvent fileChangeEvent) {
}
}
public static void main(String[] args) throws Exception {
final ConfigWatcher configWatcher = new ConfigWatcher();
configWatcher.watchConfig();
while (true) {
Thread.sleep(1000);
}
}
}

LOG4J2 Use multiple config files using java

can log4j2 use multiple config files. I wanna run my project and load one default config file - logger.xml and after that to check if there is a second configuration from another file logger_1.xml and to add it and not to override the first one.
Here is some dummy code. In short I wanna fill up the arrayList with file paths and then to load all of them.
public class LoggerConfiguratorManager
{
public static final String LOG4J_PATH = "etc/confs/logger.xml";
private static LoggerContext context = null;
private static final ConfigurationFactory factory = XmlConfigurationFactory.getInstance();
private static ConfigurationSource configurationSource = null;
private static Configuration configuration = null;
private static final ArrayList<String> registred_logger = new ArrayList<>();
private static void loadLoggerConfig(String logger_path)
{
InputStream is = null;
try
{
if(logger_path.endsWith(".xml"))
is = new FileInputStream(logger_path);
else
{
final ZipFile archive = new ZipFile(logger_path);
final ZipEntry logger_entry = archive.getEntry(LOG4J_PATH);
if(logger_entry == null) throw new IOException("Cannot find 'logger.xml' in " + logger_path);
is = archive.getInputStream(logger_entry);
}
configurationSource = new ConfigurationSource(is);
configuration = factory.getConfiguration(configurationSource);
}
catch(IOException ex)
{
System.err.println("=============================================================================");
System.err.println("=============================== LOGGER CONFIG ===============================");
System.err.println("=============================================================================");
System.err.println("=== [ERROR] " + ex);
}
finally
{
if (configurationSource != null)
{
context = Configurator.initialize(null, configurationSource);
context.start(configuration);
try { is.close(); } catch(IOException ex) { }
}
}
}
public static void load()
{
registred_logger.add(Globals.getClassLocation(LoggerConfiguratorManager.class));
for(final String conf : registred_logger)
loadLoggerConfig(conf);
}
public static void regLoggerConf(String conf_path) { registred_logger.add(conf_path); }

I would suggest doing instead:
public class LoggerConfiguratorManager {
private static final String LOG4J_PATH = "etc/confs/log4j2.xml";
private static final StringBuffer paths = new StringBuffer(LOG4J_PATH);
public static void registerConfiguration(String confPath) {
paths.append(",").append(confPath);
}
public static void initLog4j() {
Configurator.initializer("My Config", null, paths.toString(), null);
}
}
For a full working example please see https://github.com/rgoers/CompositeConfigurationExample.

The WatchEvent's context doesn't reflect renaming of the directory

I'm trying to implement a recursive directory watching functionality. But I found the API very hard to understand.
Let's say I have an empty directory which is the root of the directory tree I would like to watch. Now I create a new directory within this root (it's default name is 'New folder' in Windows 7) and I immediately rename it to something like 'xxx'.
The problem is when I copy a file in the new created 'xxx' directory. The WatchEvent's context is 'New folder' instead of the 'xxx'.
Here is my sscce:
public class Test {
private static final String SRC_DIR = "D:/test";
private final WatchService watcherService;
public Test() throws IOException {
watcherService = FileSystems.getDefault().newWatchService();
registerDirectoryTree(Paths.get(SRC_DIR));
startWatching();
}
private void startWatching() throws IOException {
while (true) {
WatchKey watchKey;
try {
watchKey = watcherService.take();
} catch (InterruptedException e) {
break;
}
Path directory = null;
try {
directory = (Path) watchKey.watchable();
for (WatchEvent<?> event : watchKey.pollEvents()) {
if (event.kind() == StandardWatchEventKinds.ENTRY_CREATE) {
Path fileName = (Path) event.context();
Path filePath = directory.resolve(fileName);
if (Files.isDirectory(filePath, LinkOption.NOFOLLOW_LINKS)) {
registerDirectoryTree(filePath);
} else if (Files.isRegularFile(filePath, LinkOption.NOFOLLOW_LINKS)) {
System.out.println("Processing file. Path: " + filePath);
} else {
System.out.println("Unknown path type. Path: " + filePath);
}
}
}
} finally {
boolean valid = watchKey.reset();
if (!valid) {
System.out.println("Watch key is not valid. Directory: " + directory);
}
}
}
}
private void registerDirectoryTree(Path sourceDir) throws IOException {
Files.walkFileTree(sourceDir, new SimpleFileVisitor<Path>() {
#Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
registerDirectory(dir);
return FileVisitResult.CONTINUE;
}
});
}
private void registerDirectory(Path path) throws IOException {
System.out.println("Registering watch service on " + path);
path.register(watcherService, StandardWatchEventKinds.ENTRY_CREATE);
}
public static void main(String[] args) throws IOException {
new Test();
}
}

Directory Scanner in Java

Scan a set of directories continuously for a set of file name filters.
For each file name filter arrived, process the file and repeat the steps for all
What can be the recommended design for this in jdk 1.5 , possibly using java.concurrent.Executor and Future

I have done a similar task with the web crawler.Just a few changes had to be made... It is a concurrent implementation with newly found directories getting scanned by the thread pool in the Executor Framework.It uses concurrent collections for Queue and List to hold the indexed files. The indexer picks up the files from the queue and does something with them.
here is the FileFilter implementation
public class ImageFileFilter implements FileFilter
{
private final String[] okFileExtensions =
new String[] {"jpg", "png", "gif"};
public boolean accept(File file)
{
for (String extension : okFileExtensions)
{
if (file.getName().toLowerCase().endsWith(extension))
{
return true;
}
}
return false;
}
}
here is the Class with the main method...
public class FileFilterTest {
public static void main(String[] args) {
File dir = new File("D:\\dev\\css-templates\\cms-admin");
BlockingQueue blockingQueue = new ArrayBlockingQueue(5);
FileCrawler fileCrawler = new FileCrawler(blockingQueue,
new ImageFileFilter(), dir);
new Thread(fileCrawler).start();
FileIndexer indexer = new FileIndexer(blockingQueue);
new Thread(indexer).start();
}
}
Here is the file crawler thread
public class FileCrawler implements Runnable {
private final BlockingQueue fileQueue;
private ConcurrentSkipListSet indexedFiles = new ConcurrentSkipListSet();
private final FileFilter fileFilter;
private final File root;
private final ExecutorService exec = Executors.newCachedThreadPool();
public FileCrawler(BlockingQueue fileQueue,
final FileFilter fileFilter,
File root) {
this.fileQueue = fileQueue;
this.root = root;
this.fileFilter = new FileFilter() {
public boolean accept(File f) {
return f.isDirectory() || fileFilter.accept(f);
}
};
}
public void run() {
submitCrawlTask(root);
}
private void submitCrawlTask(File f) {
CrawlTask crawlTask = new CrawlTask(f);
exec.execute(crawlTask);
}
private class CrawlTask implements Runnable {
private final File file;
CrawlTask(File file ) {
this.file= file;
}
public void run() {
if(Thread.currentThread().isInterrupted())
return;
File[] entries = file.listFiles(fileFilter);
if (entries != null) {
for (File entry : entries)
if (entry.isDirectory())
submitCrawlTask(entry);
else if (entry !=null && !indexedFiles.contains(entry)){
indexedFiles.add(entry);
try {
fileQueue.put(entry);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
}
}
Here is the file indexer thread
public class FileIndexer implements Runnable {
private final BlockingQueue queue;
public FileIndexer(BlockingQueue queue) {
this.queue = queue;
}
public void run() {
try {
while (true) {
indexFile(queue.take());
}
} catch (InterruptedException e) {
System.out.println("Indexer Interrupted");
Thread.currentThread().interrupt();
}
}
public void indexFile(File file) {
// do something with the file...
System.out.println("Indexing File : " + file.getAbsolutePath() + " " + file.getName());
};
}

I guess this is what you are trying to do:
You have a set of dirs:
dir1
dir2
dir3
And you need to place a "watch" on these 3 directories for a specific file name pattern.
Example: If a new file is added with name: watchme_9192.log, then you java logic should kick in and process that file.
So, based on that assumption, you can try: jnotify
JNotify is a java library that allow java application to listen to
file system events, such as:
File created
File modified
File renamed
File deleted
Also, related: best practice for directory polling