Develop Reference

Add custom folders to classpath in bazel java tests

I'm trying to migrate a large codebase from maven to bazel and I've found that some of the tests write to target/classes and target/test-classes and the production code reads it as resources on the classpath. This is because maven surefire/failsafe run by default from the module directory and add target/classes and target/test-classes to the classpath.
For me to migrate this large codebase the only reasonable solution is to create target, target/classes and target/test-classes folders and add the last two to the classpath of the tests.
Any ideas on how this can be achieved?
Thanks

Another line of approach. Instead of generating a test suite, create a custom javaagent and a custom class loader. Use jvm_flags to setup and configure it.
The javaagent has a premain method. This sounds like a natural place to do things that happen before the regular main method, even if they don't have anything to do with class instrumentation, debugging, coverage gathering, or any other usual uses of javaagents.
The custom javaagent reads system property extra.dirs and creates directories specified there. It then reads property extra.link.path and creates the symbolic links as specified there, so I can place resources where the tests expect them, without having to copy them.
Classloader is needed so that we can amend the classpath at runtime without hacks. Great advantage is that this solution works on Java 10.
The custom classloader reads system property extra.class.path and (in effect) prepends it before what is in java.class.path.
Doing things this way means that standard bazel rules can be used.
BUILD
runtime_classgen_dirs = ":".join([
"target/classes",
"target/test-classes",
])
java_test(
...,
jvm_flags = [
# agent
"-javaagent:$(location //tools:test-agent_deploy.jar)",
"-Dextra.dirs=" + runtime_classgen_dirs,
# classloader
"-Djava.system.class.loader=ResourceJavaAgent",
"-Dextra.class.path=" + runtime_classgen_dirs,
],
,,,,
deps = [
# not runtime_deps, cause https://github.com/bazelbuild/bazel/issues/1566
"//tools:test-agent_deploy.jartest-agent_deploy.jar"
],
...,
)
tools/BUILD
java_binary(
name = "test-agent",
testonly = True,
srcs = ["ResourceJavaAgent.java"],
deploy_manifest_lines = ["Premain-Class: ResourceJavaAgent"],
main_class = "ResourceJavaAgent",
visibility = ["//visibility:public"],
)
tools/ResourceJavaAgent.java
import java.io.File;
import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
// https://stackoverflow.com/questions/60764/how-should-i-load-jars-dynamically-at-runtime
public class ResourceJavaAgent extends URLClassLoader {
private final ClassLoader parent;
public ResourceJavaAgent(ClassLoader parent) throws MalformedURLException {
super(buildClassPath(), null);
this.parent = parent; // I need the parent as backup for SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
System.out.println("initializing url classloader");
}
private static URL[] buildClassPath() throws MalformedURLException {
final String JAVA_CLASS_PATH = "java.class.path";
final String EXTRA_CLASS_PATH = "extra.class.path";
List<String> paths = new LinkedList<>();
paths.addAll(Arrays.asList(System.getProperty(EXTRA_CLASS_PATH, "").split(File.pathSeparator)));
paths.addAll(Arrays.asList(System.getProperty(JAVA_CLASS_PATH, "").split(File.pathSeparator)));
URL[] urls = new URL[paths.size()];
for (int i = 0; i < paths.size(); i++) {
urls[i] = Paths.get(paths.get(i)).toUri().toURL(); // important only for resource url, really: this url must be absolute, to pass getClass().getResource("/users.properties").toURI()) with uri that isOpaque == false.
// System.out.println(urls[i]);
}
// this is for spawnVM functionality in tests
System.setProperty(JAVA_CLASS_PATH, System.getProperty(EXTRA_CLASS_PATH, "") + File.pathSeparator + System.getProperty(JAVA_CLASS_PATH));
return urls;
}
#Override
public Class<?> loadClass(String s) throws ClassNotFoundException {
try {
return super.loadClass(s);
} catch (ClassNotFoundException e) {
return parent.loadClass(s); // we search parent second, not first, as the default URLClassLoader would
}
}
private static void createRequestedDirs() {
for (String path : System.getProperty("extra.dirs", "").split(File.pathSeparator)) {
new File(path).mkdirs();
}
}
private static void createRequestedLinks() {
String linkPaths = System.getProperty("extra.link.path", null);
if (linkPaths == null) {
return;
}
for (String linkPath : linkPaths.split(",")) {
String[] fromTo = linkPath.split(":");
Path from = Paths.get(fromTo[0]);
Path to = Paths.get(fromTo[1]);
try {
Files.createSymbolicLink(from.toAbsolutePath(), to.toAbsolutePath());
} catch (IOException e) {
throw new IllegalArgumentException("Unable to create link " + linkPath, e);
}
}
}
public static void premain(String args, Instrumentation instrumentation) throws Exception {
createRequestedDirs();
createRequestedLinks();
}
}

If you could tell the tests where to write these files (in case target/classes and target/test-classes are hardcoded), and then turn the test run into a genrule, then you can specify the genrule's outputs as data for the production binary's *_binary rule.

I solved the first part, creating the directories. I still don't know how to add the latter two to classpath.
Starting from https://gerrit.googlesource.com/bazlets/+/master/tools/junit.bzl, I modified it to read
_OUTPUT = """import org.junit.runners.Suite;
import org.junit.runner.RunWith;
import org.junit.BeforeClass;
import java.io.File;
#RunWith(Suite.class)
#Suite.SuiteClasses({%s})
public class %s {
#BeforeClass
public static void setUp() throws Exception {
new File("./target").mkdir();
}
}
"""
_PREFIXES = ("org", "com", "edu")
# ...
I added the #BeforeClass setUp method.
I stored this as junit.bzl into third_party directory in my project.
Then in a BUILD file,
load("//third_party:junit.bzl", "junit_tests")
junit_tests(
name = "my_bundled_test",
srcs = glob(["src/test/java/**/*.java"]),
data = glob(["src/test/resources/**"]),
resources = glob(["src/test/resources/**"]),
tags = [
# ...
],
runtime_deps = [
# ...
],
],
deps = [
# ...
],
)
Now the test itself is wrapped with a setUp method which will create a directory for me. I am not deleting them afterwards, which is probably a sound idea to do.
The reason I need test resources in a directory (as opposed to in a jar file, which bazel gives by default) is that my test passes the URI to new FileInputStream(new File(uri)). If the file resides in a JAR, the URI will be file:/path/to/my.jar!/my.file and the rest of the test cannot work with such URI.

ClassNotFoundException when running outside IDE

I have a java application which was developed in Eclipse.
There is a folder on the system which contains a lot of ".java" files. These ".java" files are classes which some user has written. I wish to load all these java classes and compile them inside my java application.
Another property of all the ".java" files are that all the classes written inside extend a class which is inside my original application.
I used the following to read and compile all the classes.
File parentFile = new File(rulesDir + "\\");
String fileName = rulesDir + "\\" + ruleName + ".java";
File ruleFile = new File(fileName);
// Compile source file.
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
compiler.run(null, null, null, ruleFile.getPath());
// Load and instantiate compiled class.
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] { parentFile.toURI().toURL() });
Class<? extends AbstractRule> cls = (Class<? extends AbstractRule>)Class.forName(ruleName, true, classLoader);
If I run the above code inside Eclipse, it works fine. When I run the application as a jar from elsewhere, it throws an ClassNotFoundException for the line
Class<? extends AbstractRule> cls = (Class<? extends AbstractRule>)Class.forName(ruleName, true, classLoader);
Why is this happening? What is different that it executes in Eclipse and doesn't via command line?

From the documentation for Class.forName
name - fully qualified name of the desired class
So, in order to get that fully qualified class name, you need to manipulate your rulesDir variable to replace the backslashes with periods, then prepend that to your ruleName variable, combined with another period, to get the fully qualified class name. Then you'll be able to use the ClassLoader to load the class. The fully qualified name is required so that the ClassLoader can find your resource from the root of your classpath.
NB I make the assumption that your rulesDir is a relative path from the base of your classpath. If it is not, then you'll have extra manipulation to do here
See code manipulation below:
import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import javax.tools.JavaCompiler;
import javax.tools.ToolProvider;
import rules.AbstractRule;
public class MainApp {
public static void main(String[] args) {
try {
System.out.println("Test");
// NB Amended here to now take project root, relative path to rules directory and class name. So that compilation can take place based on the absolute path and class loading from the relative one.
compile("C:\\Media\\Code\\manodestra_java\\src\\tmp", "rules", "TestRule");
} catch (Exception e) {
e.printStackTrace();
}
}
private static void compile(String projectRoot, String rulesDir, String ruleName) throws Exception {
File parentFile = new File(projectRoot + "\\" + rulesDir + "\\");
System.out.println(parentFile);
String fileName = parentFile.getCanonicalPath() + "\\" + ruleName + ".java";
File ruleFile = new File(fileName);
System.out.println(ruleFile);
// Compile source file.
System.out.println("Compiling...");
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
compiler.run(null, null, null, ruleFile.getPath());
// Load and instantiate compiled class.
System.out.println("Loading class...");
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] { parentFile.toURI().toURL() });
System.out.println("Class Loader: " + classLoader);
ruleName = rulesDir.replace("\\", ".") + "." + ruleName;
Class<? extends AbstractRule> clazz = (Class<? extends AbstractRule>)Class.forName(ruleName, true, classLoader);
System.out.println(clazz);
}
}
For the sake of my testing, this class was defined in the default package and I created a rules directory below that level to contain my subclasses of AbstractRule. So, rules.TestRule was my fully qualified path to my class name. But, yours could be...
com.example.testapplication.rules.TestRule, etc.
And that's what would be required in Class.forName. There's a path to your classpath root, then the relative path from there to your java files (which is equivalent to the package of your classes), then the actual class names under that package path.

Java make class available from JAR

I am making a program that operates as multiple JAR file dependencies. Basically, the thing loops through the .class files in a JAR file and gets a Class object for each of them. Each JAR has a Plugin.class file that I don't want to be available, but I want all the Classes to be accessible by other JAR dependencies as well as the main program. For example, in one JAR I have the class something.somethingelse.SomeClass, and from a second one (I made sure it is loaded second) I want to be able to import (at execution because it's in a separate JARfile) something.somethingelse.SomeClass and use it. I Have tried this after loading it into a Class object but it gives me ClassNotFound errors. I am using the newest java update and the newest version of eclipse IDE. I have three projects, "main", "aaa", and "aab". I have aaa and aab exported to JARs of which the contents are loaded into Class objects by main. aaa is loaded before aab, and I want aab to be able to access the classes from aaa through import aaa.Class. How can I (from main) make the classes of both jarfiles available to each other?
Here is my load plugin function:
public static void load(File file) throws Exception
{
JarFile jarFile = new JarFile(file);
Enumeration e = jarFile.entries();
URL[] urls = new URL[] { file.toURI().toURL() };
ClassLoader cl = new URLClassLoader(urls);
while (e.hasMoreElements()) {
JarEntry je = (JarEntry) e.nextElement();
if(je.isDirectory() || !je.getName().endsWith(".class") || je.getName() == "Plugin.class"){
continue;
}
// -6 because of .class
String className = je.getName().substring(0,je.getName().length()-6);
className = className.replace('/', '.');
Class c = cl.loadClass(className);
}
ClassLoader loader = new URLClassLoader(urls);
Class c = loader.loadClass("Plugin");
Object cobj = c.newInstance();
Method[] allMethods = c.getDeclaredMethods();
Method method = null;
boolean found = false;
for (Method m : allMethods) {
String mname = m.getName();
if (mname == "startPlugin"){
method = m;
found = true;
}
}
if(found)
{
method.invoke(cobj);
}
else
{
//skip class
}
}
And then my first JAR (aaa.jar) declares a class called hlfl.ui.UserInterface.
My second JAR's Plugin class is as follows:
import hlfl.ui.*;
public class Plugin {
//THIS DEPENDENCY EXPORTS TO: aab.jar
public void startPlugin()
{
System.out.println("Plugin Loading Interface Loaded [AAB]");
UserInterface c = new UserInterface();
}
}
But when I run it it gives me the following:
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun. reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at sf.htmlguy.hlaunch.PluginLoader.load(PluginLoader.java:58)
at sf.htmlguy.hlaunch.PluginLoader.loadAll(PluginLoader.java:22)
at sf.htmlguy.hlaunch.HLaunch.main(HLaunch.java:14)
Caused by: java.lang.NoClassDefFoundError: hlfl/ui/UserInterface
at Plugin.startPlugin(Plugin.java:7)
... 7 more
Caused by: java.lang.ClassNotFoundException: hlfl.ui.UserInterface
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 8 more
Just in case, the code is on SourceForge (the three projects are in subdirectories, "hlaunch for linux" is the main one.):
https://sourceforge.net/p/hlaunch/code

As far as I can tell your load method is creating a URLClassLoader containing just one JAR file. So you're going to end up with a classloader structure like this
main
/ \
/ \
UCL with aaa.jar UCL with aab.jar
thus classes in aaa and in aab can both see classes in main, but aaa and aab cannot see each other. If you want each plugin to be able to see the classes of those plugins that were loaded before it, then you need to arrange things so that each plugin you load uses the classloader of the previous plugin as its parent
main
|
UCL with aaa.jar
|
UCL with aab.jar
To do this you'd have to cache the loader you create when you load one plugin, and then pass that as a parameter when you create the next plugin's classloader.
private static ClassLoader lastPluginClassLoader = null;
public static void load(File file) throws Exception {
//...
ClassLoader loader = null;
if(lastPluginClassLoader == null) {
loader = new URLClassLoader(urls);
} else {
loader = new URLClassLoader(urls, lastPluginClassLoader);
}
lastPluginClassLoader = loader;
// ...
}
But all this (a) is not thread safe unless synchronized and (b) makes the behaviour critically dependent on the order in which the plugins are loaded. To do things properly you'd need some way to declare which plugins depend on which other plugins, and set up the classloader tree appropriately, etc. etc.
... and if you go too far down that road you've just re-invented OSGi.

Method to dynamically load java class files

What would be a good way to dynamically load java class files so that a program compiled into a jar can read all the class files in a directory and use them, and how can one write the files so that they have the necessary package name in relation to the jar?

I believe it's a ClassLoader you're after.
I suggest you start by looking at the example below which loads class files that are not on the class path.
// Create a File object on the root of the directory containing the class file
File file = new File("c:\\myclasses\\");
try {
// Convert File to a URL
URL url = file.toURI().toURL(); // file:/c:/myclasses/
URL[] urls = new URL[]{url};
// Create a new class loader with the directory
ClassLoader cl = new URLClassLoader(urls);
// Load in the class; MyClass.class should be located in
// the directory file:/c:/myclasses/com/mycompany
Class cls = cl.loadClass("com.mycompany.MyClass");
} catch (MalformedURLException e) {
} catch (ClassNotFoundException e) {
}

Class myclass = ClassLoader.getSystemClassLoader().loadClass("package.MyClass");
or
Class myclass = Class.forName("package.MyClass");
or loading the class from different folder which is not in the classpath:
File f = new File("C:/dir");
URL[] cp = {f.toURI().toURL()};
URLClassLoader urlcl = new URLClassLoader(cp);
Class myclass = urlcl.loadClass("package.MyClass");
For further usage of the loaded Class you can use Reflection if the loaded Class is not in your classpath and you can not import and cast it. For example if you want to call a static method with the name "main":
Method m = myclass.getMethod("main", String[].class);
String[] args = new String[0];
m.invoke(null, args); // invoke the method

If you add a directory to your class path, you can add classes after the application starts and those classes can be loaded as soon as they have been written to the directory.

Can I dynamically discover xml files in the classpath inside an EJB 3 container?

Background:
One of the components of our project operates using spring. Some SQL code is dynamically generated, based on a given XML spring configuration.
At first it was fine to store all the XML configurations in the same package on the classpath, (and then load it as a resource when the service is called) but over time we ended up with a large number of configurations. It came time to separate the configurations into different namespaces.
The Goal
What I want is, given a starting package on the classpath, to recursively walk the directory structure and discover any spring XML files dynamically. (So that as new configurations / packages are added, the files will still be found by the service).
The Problem
I was able to accomplish my goal fine when running outside an EJB container by using Thread.getContextClassloader().getResource(myBasePackage), then getting a File object and using it to walk the tree on the filesystem. Clunky, I know, but it was still classpath relative and it worked.
However, you cannot do this inside an EJB container (you can't interact with the filesystem at all), so I had to use the rather annoying workaround in which I maintain a list of hardcoded packages to search.
The Question
Is there a way (running inside an EJB container) to dynamically walk the classpath (from a given starting location) searching for arbitrary resources?

Short answer: Not while staying in compliance with the EJB spec. Because the spec envisions containers running in all kinds of non-standard situations, it does not make this possible.
Longer answer: Since you are not creating these resources dynamically, I would write a routine that gives you a list of all of the resources at build time and puts them in a dynamically generated file that your EJB knows how to reference. So you basically create a directory listing of packages and files that you can load in the EJB that are referenced in one master file.
Spring answer: Spring supports finding resources on the classpath, although I have no idea how well this works in the EJB context (and I doubt its EJB compliant, but I haven't checked). Some details here.

DISCLAIMER: As already pointed out, creating resources in the classpath is not recommended and depending on the EJB container explicitly forbidden. This may cause you a lot of problems because containers may explode your resources into another folder or even replicate the resources throughout the cluster (if thats the case). In order to create resources dynamically you have to create a custom classloader. So, I would never do it. It is better to access the filesystem directly than the classpath. It is less ugly and eventually cluster-safe if you use a remote filesystem + file locks.
If even after all I explained you still want to play with the classpath, you can try to do something like: get the classloader via
ClassLoader cld = Thread.currentThread().getContextClassLoader();
Starting from a base package enumerate all occurrences
Enumeration<URL> basePackageUrls = cld.getResources(basePackagePath);
Each URL is generally either a file link (file:///home/scott/.../MyResource.properties) or a jar link (file:///lib.jar!/com/domain/MyResource.properties). You have to check the pattern in the URL. Using that, enumerate the contents of the folder using the normal java API and find the subpackages. Proceed until you have scanned all packages.
See the class below (will be released with an open-source project of mine soon). It implemens a classpath scanner that you can pass in a selector. It works like a visitor. It my work for you, if not, get ideas from it. See the sample annotation selector at the end.
public class ClasspathScanner
{
private static final Log log = LogFactory.getLog(ClasspathScanner.class);
private static final String JAR_FILE_PATTERN = ".jar!";
private ClassSelector selector;
private Set<Class<?>> classes;
// PUBLIC METHODS ------------------------------------------------------------------------------
public synchronized Set<Class<?>> scanPackage(String basePackage, ClassSelector selector)
throws Exception
{
if (selector == null)
{
throw new NullPointerException("Selector cannot be NULL");
}
this.selector = selector;
this.classes = new HashSet<Class<?>>();
Set<Class<?>> aux;
try
{
scanClasses0(basePackage);
aux = this.classes;
}
finally
{
this.selector = null;
this.classes = null;
}
return aux;
}
// HELPER CLASSES ------------------------------------------------------------------------------
private void scanClasses0(String basePackage)
throws IOException, ClassNotFoundException, FileNotFoundException
{
File packageDirectory = null;
ClassLoader cld = getLoader();
String basePackagePath = basePackage.replace('.', '/');
Enumeration<URL> basePackageUrls = cld.getResources(basePackagePath);
if (basePackageUrls == null || !basePackageUrls.hasMoreElements())
{
throw new ClassNotFoundException("Base package path not found: [" + basePackagePath
+ "]");
}
while (basePackageUrls.hasMoreElements())
{
String packagePath = basePackageUrls.nextElement().getFile();
if (packagePath.contains(JAR_FILE_PATTERN))
{
scanJarFile(basePackagePath, packagePath);
}
else
{
packageDirectory = new File(packagePath);
scanDirectory(basePackage, packageDirectory);
}
}
}
private void scanDirectory(String packageName, File packagePath)
throws ClassNotFoundException, FileNotFoundException
{
if (packagePath.exists())
{
File[] packageFiles = packagePath.listFiles();
for (File file : packageFiles)
{
if (file.isFile() && file.getName().endsWith(".class"))
{
String fullFileName = packageName + '.' + file.getName();
checkClass(fullFileName);
}
else if (file.isDirectory())
{
scanDirectory(packageName + "." + file.getName(), file);
}
}
}
else
{
throw new FileNotFoundException(packagePath.getPath());
}
}
private void scanJarFile(String basePackagePath, String jarFileUrl)
throws IOException, ClassNotFoundException
{
String jarFilePath = jarFileUrl.substring("file:".length(), jarFileUrl
.indexOf(JAR_FILE_PATTERN)
+ JAR_FILE_PATTERN.length() - 1);
log.debug("URL JAR file path: [" + jarFilePath + "]");
jarFilePath = URLDecoder.decode(jarFilePath, "UTF-8");
log.debug("Decoded JAR file path: [" + jarFilePath + "]");
JarFile jar = new JarFile(new File(jarFilePath));
for (Enumeration<JarEntry> jarFiles = jar.entries(); jarFiles.hasMoreElements();)
{
JarEntry file = jarFiles.nextElement();
String fileName = file.getName();
if (!file.isDirectory() && fileName.endsWith(".class")
&& fileName.startsWith(basePackagePath))
{
String className = fileName.replace('/', '.');
checkClass(className);
}
}
}
private void checkClass(String fullFilePath) throws ClassNotFoundException
{
String className = fullFilePath.substring(0, fullFilePath.length() - 6);
Class<?> c = getLoader().loadClass(className);
if (selector.select(c))
{
classes.add(c);
}
}
private ClassLoader getLoader()
{
ClassLoader loader = Thread.currentThread().getContextClassLoader();
if (loader == null)
{
loader = getClass().getClassLoader();
}
return loader;
}
// INNER CLASSES -------------------------------------------------------------------------------
public interface ClassSelector
{
boolean select(Class<?> clazz);
}
public static class AnnotatedClassSelector implements ClassSelector
{
private final Class<? extends Annotation>[] annotations;
public AnnotatedClassSelector(Class<? extends Annotation>... annotations)
{
this.annotations = annotations;
}
public boolean select(Class<?> clazz)
{
for (Class<? extends Annotation> ac : annotations)
{
if (clazz.isAnnotationPresent(ac))
{
return true;
}
}
return false;
}
}
}