Develop Reference

Java copy files matching pattern from many folders to another folder

Please take a look at the code I have so far and if possible explain what I'm doing wrong. I'm trying to learn.
I made a little program to search for a type of file in a directory and all its sub-directories and copy them into another folder.
Code
import java.util.ArrayList;
import java.util.List;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
public class FandFandLoop {
public static void main(String[] args) {
final File folder = new File("C:/Users/ina/src");
List<String> result = new ArrayList<>();
search(".*\\.txt", folder, result);
File to = new File("C:/Users/ina/dest");
for (String s : result) {
System.out.println(s);
File from = new File(s);
try {
copyDir(from.toPath(), to.toPath());
System.out.println("done");
}
catch (IOException ex) {
ex.printStackTrace();
}
}
}
public static void copyDir(Path src, Path dest) throws IOException {
Files.walk(src)
.forEach(source -> {
try {
Files.copy(source, dest.resolve(src.relativize(source)),
StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
e.printStackTrace();
}
});
}
public static void search(final String pattern, final File folder, List<String> result) {
for (final File f : folder.listFiles()) {
if (f.isDirectory()) {
search(pattern, f, result);
}
if (f.isFile()) {
if (f.getName().matches(pattern)) {
result.add(f.getAbsolutePath());
}
}
}
}
}
It works, but what it actually does is to take my .txt files and write them into another file named dest without extension.
And at some point, it deletes the folder dest.
The deletion happens because of StandardCopyOption.REPLACE_EXISTING, if I understand this, but what I would have liked to obtain was that if several files had the same name then only one copy of it should be kept.

There is no need to call Files.walk on the matched source files.
You can improve this code by switching completely to using java.nio.file.Path and not mixing string paths and File objects. Additionally instead of calling File.listFiles() recursively you can use Files.walk or even better Files.find.
So you could instead use the following:
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Objects;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
public class CopyFiles {
public static void copyFiles(Path src, Path dest, PathMatcher matcher, CopyOption... copyOptions) throws IOException {
// Argument validation
if (!Files.isDirectory(src)) {
throw new IllegalArgumentException("Source '" + src + "' is not a directory");
}
if (!Files.isDirectory(dest)) {
throw new IllegalArgumentException("Destination '" + dest + "' is not a directory");
}
Objects.requireNonNull(matcher);
Objects.requireNonNull(copyOptions);
BiPredicate<Path, BasicFileAttributes> filter = (path, attributes) -> attributes.isRegularFile() && matcher.matches(path);
// Use try-with-resources to close stream as soon as it is not longer needed
try (Stream<Path> files = Files.find(src, Integer.MAX_VALUE, filter)) {
files.forEach(file -> {
Path destFile = dest.resolve(src.relativize(file));
try {
copyFile(file, destFile, copyOptions);
}
// Stream methods do not allow checked exceptions, have to wrap it
catch (IOException ioException) {
throw new UncheckedIOException(ioException);
}
});
}
// Wrap UncheckedIOException; cannot unwrap it to get actual IOException
// because then information about the location where the exception was wrapped
// will get lost, see Files.find doc
catch (UncheckedIOException uncheckedIoException) {
throw new IOException(uncheckedIoException);
}
}
private static void copyFile(Path srcFile, Path destFile, CopyOption... copyOptions) throws IOException {
Path destParent = destFile.getParent();
// Parent might be null if dest is empty path
if (destParent != null) {
// Create parent directories before copying file
Files.createDirectories(destParent);
}
Files.copy(srcFile, destFile, copyOptions);
}
public static void main(String[] args) throws IOException {
Path srcDir = Paths.get("path/to/src");
Path destDir = Paths.get("path/to/dest");
// Could also use FileSystem.getPathMatcher
PathMatcher matcher = file -> file.getFileName().toString().endsWith(".txt");
copyFiles(srcDir, destDir, matcher);
}
}

Compile Java code in-memory [duplicate]

This question already has answers here:
Compile code fully in memory with javax.tools.JavaCompiler [duplicate]
(7 answers)
Closed 6 years ago.
I want to treat a String as a Java file then compile and run it. In other words, use Java as a script language.
To get better performance, we should avoid writing .class files to disk.

This answer is from one of my blogs, Compile and Run Java Source Code in Memory.
Here are the three source code files.
MemoryJavaCompiler.java
package me.soulmachine.compiler;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.tools.*;
/**
* Simple interface to Java compiler using JSR 199 Compiler API.
*/
public class MemoryJavaCompiler {
private javax.tools.JavaCompiler tool;
private StandardJavaFileManager stdManager;
public MemoryJavaCompiler() {
tool = ToolProvider.getSystemJavaCompiler();
if (tool == null) {
throw new RuntimeException("Could not get Java compiler. Please, ensure that JDK is used instead of JRE.");
}
stdManager = tool.getStandardFileManager(null, null, null);
}
/**
* Compile a single static method.
*/
public Method compileStaticMethod(final String methodName, final String className,
final String source)
throws ClassNotFoundException {
final Map<String, byte[]> classBytes = compile(className + ".java", source);
final MemoryClassLoader classLoader = new MemoryClassLoader(classBytes);
final Class clazz = classLoader.loadClass(className);
final Method[] methods = clazz.getDeclaredMethods();
for (final Method method : methods) {
if (method.getName().equals(methodName)) {
if (!method.isAccessible()) method.setAccessible(true);
return method;
}
}
throw new NoSuchMethodError(methodName);
}
public Map<String, byte[]> compile(String fileName, String source) {
return compile(fileName, source, new PrintWriter(System.err), null, null);
}
/**
* compile given String source and return bytecodes as a Map.
*
* #param fileName source fileName to be used for error messages etc.
* #param source Java source as String
* #param err error writer where diagnostic messages are written
* #param sourcePath location of additional .java source files
* #param classPath location of additional .class files
*/
private Map<String, byte[]> compile(String fileName, String source,
Writer err, String sourcePath, String classPath) {
// to collect errors, warnings etc.
DiagnosticCollector<JavaFileObject> diagnostics =
new DiagnosticCollector<JavaFileObject>();
// create a new memory JavaFileManager
MemoryJavaFileManager fileManager = new MemoryJavaFileManager(stdManager);
// prepare the compilation unit
List<JavaFileObject> compUnits = new ArrayList<JavaFileObject>(1);
compUnits.add(fileManager.makeStringSource(fileName, source));
return compile(compUnits, fileManager, err, sourcePath, classPath);
}
private Map<String, byte[]> compile(final List<JavaFileObject> compUnits,
final MemoryJavaFileManager fileManager,
Writer err, String sourcePath, String classPath) {
// to collect errors, warnings etc.
DiagnosticCollector<JavaFileObject> diagnostics =
new DiagnosticCollector<JavaFileObject>();
// javac options
List<String> options = new ArrayList<String>();
options.add("-Xlint:all");
// options.add("-g:none");
options.add("-deprecation");
if (sourcePath != null) {
options.add("-sourcepath");
options.add(sourcePath);
}
if (classPath != null) {
options.add("-classpath");
options.add(classPath);
}
// create a compilation task
javax.tools.JavaCompiler.CompilationTask task =
tool.getTask(err, fileManager, diagnostics,
options, null, compUnits);
if (task.call() == false) {
PrintWriter perr = new PrintWriter(err);
for (Diagnostic diagnostic : diagnostics.getDiagnostics()) {
perr.println(diagnostic);
}
perr.flush();
return null;
}
Map<String, byte[]> classBytes = fileManager.getClassBytes();
try {
fileManager.close();
} catch (IOException exp) {
}
return classBytes;
}
}
MemoryJavaFileManager.java
package me.soulmachine.compiler;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.Map;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;
/**
* JavaFileManager that keeps compiled .class bytes in memory.
*/
#SuppressWarnings("unchecked")
final class MemoryJavaFileManager extends ForwardingJavaFileManager {
/** Java source file extension. */
private final static String EXT = ".java";
private Map<String, byte[]> classBytes;
public MemoryJavaFileManager(JavaFileManager fileManager) {
super(fileManager);
classBytes = new HashMap<>();
}
public Map<String, byte[]> getClassBytes() {
return classBytes;
}
public void close() throws IOException {
classBytes = null;
}
public void flush() throws IOException {
}
/**
* A file object used to represent Java source coming from a string.
*/
private static class StringInputBuffer extends SimpleJavaFileObject {
final String code;
StringInputBuffer(String fileName, String code) {
super(toURI(fileName), Kind.SOURCE);
this.code = code;
}
public CharBuffer getCharContent(boolean ignoreEncodingErrors) {
return CharBuffer.wrap(code);
}
}
/**
* A file object that stores Java bytecode into the classBytes map.
*/
private class ClassOutputBuffer extends SimpleJavaFileObject {
private String name;
ClassOutputBuffer(String name) {
super(toURI(name), Kind.CLASS);
this.name = name;
}
public OutputStream openOutputStream() {
return new FilterOutputStream(new ByteArrayOutputStream()) {
public void close() throws IOException {
out.close();
ByteArrayOutputStream bos = (ByteArrayOutputStream)out;
classBytes.put(name, bos.toByteArray());
}
};
}
}
public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location,
String className,
Kind kind,
FileObject sibling) throws IOException {
if (kind == Kind.CLASS) {
return new ClassOutputBuffer(className);
} else {
return super.getJavaFileForOutput(location, className, kind, sibling);
}
}
static JavaFileObject makeStringSource(String fileName, String code) {
return new StringInputBuffer(fileName, code);
}
static URI toURI(String name) {
File file = new File(name);
if (file.exists()) {
return file.toURI();
} else {
try {
final StringBuilder newUri = new StringBuilder();
newUri.append("mfm:///");
newUri.append(name.replace('.', '/'));
if(name.endsWith(EXT)) newUri.replace(newUri.length() - EXT.length(), newUri.length(), EXT);
return URI.create(newUri.toString());
} catch (Exception exp) {
return URI.create("mfm:///com/sun/script/java/java_source");
}
}
}
}
MemoryClassLoader.java
package me.soulmachine.compiler;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
/**
* ClassLoader that loads .class bytes from memory.
*/
final class MemoryClassLoader extends URLClassLoader {
private Map<String, byte[]> classBytes;
public MemoryClassLoader(Map<String, byte[]> classBytes,
String classPath, ClassLoader parent) {
super(toURLs(classPath), parent);
this.classBytes = classBytes;
}
public MemoryClassLoader(Map<String, byte[]> classBytes, String classPath) {
this(classBytes, classPath, ClassLoader.getSystemClassLoader());
}
public MemoryClassLoader(Map<String, byte[]> classBytes) {
this(classBytes, null, ClassLoader.getSystemClassLoader());
}
public Class load(String className) throws ClassNotFoundException {
return loadClass(className);
}
public Iterable<Class> loadAll() throws ClassNotFoundException {
List<Class> classes = new ArrayList<Class>(classBytes.size());
for (String name : classBytes.keySet()) {
classes.add(loadClass(name));
}
return classes;
}
protected Class findClass(String className) throws ClassNotFoundException {
byte[] buf = classBytes.get(className);
if (buf != null) {
// clear the bytes in map -- we don't need it anymore
classBytes.put(className, null);
return defineClass(className, buf, 0, buf.length);
} else {
return super.findClass(className);
}
}
private static URL[] toURLs(String classPath) {
if (classPath == null) {
return new URL[0];
}
List<URL> list = new ArrayList<URL>();
StringTokenizer st = new StringTokenizer(classPath, File.pathSeparator);
while (st.hasMoreTokens()) {
String token = st.nextToken();
File file = new File(token);
if (file.exists()) {
try {
list.add(file.toURI().toURL());
} catch (MalformedURLException mue) {}
} else {
try {
list.add(new URL(token));
} catch (MalformedURLException mue) {}
}
}
URL[] res = new URL[list.size()];
list.toArray(res);
return res;
}
}
Explanations:
In order to represent a Java source file in memory instead of disk, I defined a StringInputBuffer class in the MemoryJavaFileManager.java.
To save the compiled .class files in memory, I implemented a class MemoryJavaFileManager. The main idea is to override the function getJavaFileForOutput() to store bytecodes into a map.
To load the bytecodes in memory, I have to implement a customized classloader MemoryClassLoader, which reads bytecodes in the map and turn them into classes.
Here is a unite test.
package me.soulmachine.compiler;
import org.junit.Test;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import static org.junit.Assert.assertEquals;
public class MemoryJavaCompilerTest {
private final static MemoryJavaCompiler compiler = new MemoryJavaCompiler();
#Test public void compileStaticMethodTest()
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException {
final String source = "public final class Solution {\n"
+ "public static String greeting(String name) {\n"
+ "\treturn \"Hello \" + name;\n" + "}\n}\n";
final Method greeting = compiler.compileStaticMethod("greeting", "Solution", source);
final Object result = greeting.invoke(null, "soulmachine");
assertEquals("Hello soulmachine", result.toString());
}
}
Reference
JavaCompiler.java from Cloudera Morphlines
How to create an object from a string in Java (how to eval a string)?
InMemoryJavaCompiler
Java-Runtime-Compiler
动态的Java - 无废话JavaCompilerAPI中文指南

Catch UncheckedIOException in Java 8 stream

EDIT: This does not seem to be possible, see https://bugs.openjdk.java.net/browse/JDK-8039910.
I have a helper class that provides a Stream<Path>. This code just wraps Files.walk and sorts the output:
public Stream<Path> getPaths(Path path) {
return Files.walk(path, FOLLOW_LINKS).sorted();
}
As symlinks are followed, in case of loops in the filesystem (e.g. a symlink x -> .) the code used in Files.walk throws an UncheckedIOException wrapping an instance of FileSystemLoopException.
In my code I would like to catch such exceptions and, for example, just log a helpful message. The resulting stream could/should just stop providing entries as soon as this happens.
I tried adding .map(this::catchException) and .peek(this::catchException) to my code, but the exception is not caught in this stage.
Path checkException(Path path) {
try {
logger.info("path.toString() {}", path.toString());
return path;
} catch (UncheckedIOException exception) {
logger.error("YEAH");
return null;
}
}
How, if at all, can I catch an UncheckedIOException in my code giving out a Stream<Path>, so that consumers of the path do not encounter this exception?
As an example, the following code should never encounter the exception:
List<Path> paths = getPaths().collect(toList());
Right now, the exception is triggered by code invoking collect (and I could catch the exception there):
java.io.UncheckedIOException: java.nio.file.FileSystemLoopException: /tmp/junit5844257414812733938/selfloop
at java.nio.file.FileTreeIterator.fetchNextIfNeeded(FileTreeIterator.java:88)
at java.nio.file.FileTreeIterator.hasNext(FileTreeIterator.java:104)
at java.util.Iterator.forEachRemaining(Iterator.java:115)
at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
at ...
EDIT: I provided a simple JUnit test class. In this question I ask you to fix the test by just modifying the code in provideStream.
package somewhere;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.nio.file.FileVisitOption.FOLLOW_LINKS;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.fail;
public class StreamTest {
#Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
#Test
public void test() throws Exception {
Path rootPath = Paths.get(temporaryFolder.getRoot().getPath());
createSelfloop();
Stream<Path> stream = provideStream(rootPath);
assertThat(stream.collect(Collectors.toList()), is(not(nullValue())));
}
private Stream<Path> provideStream(Path rootPath) throws IOException {
return Files.walk(rootPath, FOLLOW_LINKS).sorted();
}
private void createSelfloop() throws IOException {
String root = temporaryFolder.getRoot().getPath();
try {
Path symlink = Paths.get(root, "selfloop");
Path target = Paths.get(root);
Files.createSymbolicLink(symlink, target);
} catch (UnsupportedOperationException x) {
// Some file systems do not support symbolic links
fail();
}
}
}

You can make your own walking stream factory:
public class FileTree {
public static Stream<Path> walk(Path p) {
Stream<Path> s=Stream.of(p);
if(Files.isDirectory(p)) try {
DirectoryStream<Path> ds = Files.newDirectoryStream(p);
s=Stream.concat(s, StreamSupport.stream(ds.spliterator(), false)
.flatMap(FileTree::walk)
.onClose(()->{ try { ds.close(); } catch(IOException ex) {} }));
} catch(IOException ex) {}
return s;
}
// in case you don’t want to ignore exceprions silently
public static Stream<Path> walk(Path p, BiConsumer<Path,IOException> handler) {
Stream<Path> s=Stream.of(p);
if(Files.isDirectory(p)) try {
DirectoryStream<Path> ds = Files.newDirectoryStream(p);
s=Stream.concat(s, StreamSupport.stream(ds.spliterator(), false)
.flatMap(sub -> walk(sub, handler))
.onClose(()->{ try { ds.close(); }
catch(IOException ex) { handler.accept(p, ex); } }));
} catch(IOException ex) { handler.accept(p, ex); }
return s;
}
// and with depth limit
public static Stream<Path> walk(
Path p, int maxDepth, BiConsumer<Path,IOException> handler) {
Stream<Path> s=Stream.of(p);
if(maxDepth>0 && Files.isDirectory(p)) try {
DirectoryStream<Path> ds = Files.newDirectoryStream(p);
s=Stream.concat(s, StreamSupport.stream(ds.spliterator(), false)
.flatMap(sub -> walk(sub, maxDepth-1, handler))
.onClose(()->{ try { ds.close(); }
catch(IOException ex) { handler.accept(p, ex); } }));
} catch(IOException ex) { handler.accept(p, ex); }
return s;
}
}

Logger handler wont work in different classes java

I'm pretty new in Java but I have to write a simple logger.
My problem is that my logger won't transfer its handler.
My first class is:
public class LoggerClass {
static final Logger logger = Logger.getLogger("myLogger");
FileHandler fh;
public LoggerClass() {
try {
this.fh = new FileHandler("C:/Users/Administrator/Desktop/Logging%u.txt");
} catch (IOException | SecurityException ex) {
Logger.getLogger(LoggerClass.class.getName()).log(Level.SEVERE, null, ex);
}
logger.addHandler(fh);
SimpleFormatter formatter = new SimpleFormatter();
fh.setFormatter(formatter);
}
}
and i want to transfer the logger to another class by doing this:
package Logger;
import static Logger.LoggerClass.logger;
import java.io.IOException;
import java.util.logging.Level;
public class LoggingTest {
public static void main(String[] args) throws IOException {
try
{
((Object) null).toString();
}
catch ( Exception e )
{
logger.log( Level.SEVERE, "oh oh", e );
}
logger.info( "Hat funktioniert" );
}
}
i tried nearly everything that i've found, and the logger functions but it only gives its output on console and not how it should be in a file

In your test you never configured your logger. Try this:
package Logger;
import static Logger.LoggerClass.logger;
import java.io.IOException;
import java.util.logging.Level;
//There are better ways to do this.
static {
new LoggerClass();
}
public class LoggingTest {
public static void main(String[] args) throws IOException {
try
{
((Object) null).toString();
}
catch ( Exception e )
{
logger.log( Level.SEVERE, "oh oh", e );
}
logger.info( "Hat funktioniert" );
}
}

change this:
static final Logger logger = Logger.getLogger("myLogger");
To
public static final Logger logger = Logger.getLogger("myLogger");
Reason you have given it a default access in your class where you defined Logger and using defail access you can access it in package only. While where you are using is you have another package and hence error.
Apart form above use logger like:
LoggerInterface.logger.info( "Hat funktioniert" );
Since you defined it static so use classname.instance.method(parameter)

kryo serializing of class (task object) in apache spark returns null while de-serialization

I am using java spark API to write some test application . I am using a class which doesn't extends serializable interface . So to make the application work I am using kryo serializer to serialize the class . But the problem which I observed while debugging was that during the de-serialization the returned class object becomes null and in turn throws a null pointer exception . It seems to be closure problem where things are going wrong but not sure.Since I am new to this kind of serialization I don't know where to start digging.
Here is the code I am testing :
package org.apache.spark.examples;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
/**
* Spark application to test the Serialization issue in spark
*/
public class Test {
static PrintWriter outputFileWriter;
static FileWriter file;
static JavaSparkContext ssc;
public static void main(String[] args) {
String inputFile = "/home/incubator-spark/examples/src/main/scala/org/apache/spark/examples/InputFile.txt";
String master = "local";
String jobName = "TestSerialization";
String sparkHome = "/home/test/Spark_Installation/spark-0.7.0";
String sparkJar = "/home/test/TestSerializationIssesInSpark/TestSparkSerIssueApp/target/TestSparkSerIssueApp-0.0.1-SNAPSHOT.jar";
SparkConf conf = new SparkConf();
conf.set("spark.closure.serializer","org.apache.spark.serializer.KryoSerializer");
conf.set("spark.kryo.registrator", "org.apache.spark.examples.MyRegistrator");
// create the Spark context
if(master.equals("local")){
ssc = new JavaSparkContext("local", jobName,conf);
//ssc = new JavaSparkContext("local", jobName);
} else {
ssc = new JavaSparkContext(master, jobName, sparkHome, sparkJar);
}
JavaRDD<String> testData = ssc.textFile(inputFile).cache();
final NotSerializableJavaClass notSerializableTestObject= new NotSerializableJavaClass("Hi ");
#SuppressWarnings({ "serial", "unchecked"})
JavaRDD<String> classificationResults = testData.map(
new Function<String, String>() {
#Override
public String call(String inputRecord) throws Exception {
if(!inputRecord.isEmpty()) {
//String[] pointDimensions = inputRecord.split(",");
String result = "";
try {
FileWriter file = new FileWriter("/home/test/TestSerializationIssesInSpark/results/test_result_" + (int) (Math.random() * 100));
PrintWriter outputFile = new PrintWriter(file);
InetAddress ip;
ip = InetAddress.getLocalHost();
outputFile.println("IP of the server: " + ip);
result = notSerializableTestObject.testMethod(inputRecord);
outputFile.println("Result: " + result);
outputFile.flush();
outputFile.close();
file.close();
} catch (UnknownHostException e) {
e.printStackTrace();
}
catch (IOException e1) {
e1.printStackTrace();
}
return result;
} else {
System.out.println("End of elements in the stream.");
String result = "End of elements in the input data";
return result;
}
}
}).cache();
long processedRecords = classificationResults.count();
ssc.stop();
System.out.println("sssssssssss"+processedRecords);
}
}
Here is the KryoRegistrator class
package org.apache.spark.examples;
import org.apache.spark.serializer.KryoRegistrator;
import com.esotericsoftware.kryo.Kryo;
public class MyRegistrator implements KryoRegistrator {
public void registerClasses(Kryo kryo) {
kryo.register(NotSerializableJavaClass.class);
}
}
Here is the class I am serializing :
package org.apache.spark.examples;
public class NotSerializableJavaClass {
public String testVariable;
public NotSerializableJavaClass(String testVariable) {
super();
this.testVariable = testVariable;
}
public String testMethod(String vartoAppend){
return this.testVariable + vartoAppend;
}
}

This is because spark.closure.serializer only supports the Java serializer. See http://spark.apache.org/docs/latest/configuration.html about spark.closure.serializer