Develop Reference

How to investigate data on Apache Camel Route?

My project is working on getting data from one system to another. We are using Apache Camel Routes to send the data between JBoss EAP v7 servers. My question is, is there a way to investigate what the content of the packages are as they come across different routes?
We have tried upping the logging but our files/console just get flooded. We have also tried to use Hawtio on the server to see the messages coming across the routes but have had no success identifying where our message is getting "stuck".
Any help is appreciated!

You can use unit tests to test your routes locally and then either log contents of the exchange at specific points using adviceWith and weave methods.
With unit tests you can easily debug your routes in your favourite IDE even if you're running camel in something like Karaf or Red Hat fuse.
package com.example;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.RoutesBuilder;
import org.apache.camel.builder.AdviceWithRouteBuilder;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.mock.MockEndpoint;
import org.apache.camel.model.dataformat.JsonLibrary;
import org.apache.camel.test.junit4.CamelTestSupport;
import org.junit.Test;
public class ExampleRouteTests extends CamelTestSupport {
#Test
public void exampleTest() throws Exception
{
ContractDetails testDetails = new ContractDetails(1512, 1215);
mockJDBCEndpoints();
context.getRouteDefinition("exampleRoute")
.adviceWith(context, new AdviceWithRouteBuilder(){
#Override
public void configure() throws Exception {
replaceFromWith("direct:start");
weaveByToUri("direct:getDetailsFromAPI")
.replace()
.to("log:testLogger?showAll=true")
.to("mock:api")
.setBody(constant(testDetails));
weaveByToUri("direct:saveToDatabase")
.replace()
.to("log:testLogger?showAll=true")
.to("mock:db");
}
});
MockEndpoint apiMockEndpoint = getMockEndpoint("mock:api");
apiMockEndpoint.expectedMessageCount(1);
MockEndpoint dbMockEndpoint = getMockEndpoint("mock:db");
dbMockEndpoint.expectedMessageCount(1);
context.start();
String body = "{\"name\":\"Bob\",\"age\":10}";
template.sendBody("direct:start", body);
apiMockEndpoint.assertIsSatisfied();
dbMockEndpoint.assertIsSatisfied();
}
#Override
protected RoutesBuilder createRouteBuilder() throws Exception {
return new RouteBuilder(){
#Override
public void configure() throws Exception {
from("amqp:queue:example")
.routeId("exampleRoute")
.unmarshal().json(JsonLibrary.Jackson,
Person.class)
.to("direct:getDetailsFromAPI")
.process(new SomeProcessor())
.to("direct:saveToDatabase");
from("direct:saveToDatabase")
.routeId("saveToDatabaseRoute")
.to("velocity:sql/insertQueryTemplate.vt")
.to("jdbc:exampleDatabase");
from("direct:getDetailsFromAPI")
.removeHeaders("*")
.toD("http4:someAPI?name=${body.getName()}")
.unmarshal().json(JsonLibrary.Jackson,
ContractDetails.class);
}
};
}
void mockJDBCEndpoints() throws Exception {
context.getRouteDefinition("saveToDatabaseRoute")
.adviceWith(context, new AdviceWithRouteBuilder(){
#Override
public void configure() throws Exception {
weaveByToUri("jdbc:*")
.replace()
.to("mock:db");
}
});
}
#Override
public boolean isUseAdviceWith() {
return true;
}
}
Now for troubleshooting problems that do not occur with unit tests you can configure generic or route specific exception handling with onException and use Dead letter channel to process and and store information about the failed exchange. Alternatively you can just use stream or file component to save information about the exception and failed exchange in to a separate file to avoid flooding logs.

Adding REST route to an existing Jetty endpoint in Camel at runtime

I have been inventing a way how to work around the problem of adding consumers to a jetty endpoint (it does not allow multiple consumers). The way we do it in our company is to build our own router and a broadcasting endpoint which consumes from jetty and routes requests to underlying "subscriptions". Only one of them will eventually process the request. It kind of works but it's not completely ok, since recently when updating to latest Camel we have found our custom built component to leak memory and in general I consider using built-in functionality over custom hacks.
I started investigating the Camel REST API and found it very nice and pretty much replacing our home-grown component apart from one thing - you cannot re-configure it at runtime - you have to stop the context basically for this to work. Below I include my unit test with a happy path and the path that fails. Frankly I think is a bug, but if there is a legitimate way to achieve what I want, I'd like to hear sound advice:
package com.anydoby.camel;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
/**
* Test tries to add/remove routes at runtime.
*/
public class RoutesTest {
private DefaultCamelContext ctx;
#Before
public void pre() throws Exception {
ctx = new DefaultCamelContext();
new RouteBuilder(ctx) {
#Override
public void configure() throws Exception {
restConfiguration("jetty").host("localhost").port(8080);
rest("/")
.get("/issues/{isin}").route().id("issues")
.process(e -> e.getOut().setBody("Here's your issue " + e.getIn().getHeader("isin"))).endRest()
.get("/listings").route().id("listings").process(e -> e.getOut().setBody("some listings"));
}
}.addRoutesToCamelContext(ctx);
ctx.start();
}
#Test
public void test() throws IOException {
{
InputStream stream = new URL("http://localhost:8080/issues/35").openStream();
assertEquals("Here's your issue 35", IOUtils.toString(stream));
}
{
InputStream stream = new URL("http://localhost:8080/listings").openStream();
assertEquals("some listings", IOUtils.toString(stream));
}
}
#Test
public void disableRoute() throws Exception {
ctx.stopRoute("issues");
ctx.removeRoute("issues");
try (InputStream stream = new URL("http://localhost:8080/issues/35").openStream()) {
fail();
} catch (Exception e) {
}
new RouteBuilder(ctx) {
#Override
public void configure() throws Exception {
rest().get("/issues/{isin}/{sedol}").route().id("issues")
.process(e -> e.getOut()
.setBody("Here's your issue " + e.getIn().getHeader("isin") + ":" + e.getIn().getHeader("sedol")))
.endRest();
}
}.addRoutesToCamelContext(ctx);
{
InputStream stream = new URL("http://localhost:8080/issues/35/65").openStream();
assertEquals("Here's your issue 35:65", IOUtils.toString(stream));
}
}
}
The disableRoute() test fails since I cannot add another consumer to an existing endpoint.
So my question is - "is there a way to add a new URL mapping to a restful camel-jetty endpoint"? If you do it during first configuration it works fine, but when later you want to reconfigure one of the routes the error is:
org.apache.camel.FailedToStartRouteException: Failed to start route because of Multiple consumers for the same endpoint is not allowed: jetty:http://localhost:8080/issues/%7Bisin%7D/%7Bsedol%7D?httpMethodRestrict=GET

How-to catch "log4j:WARN No appenders could be found for logger"?

I want to handle this error to set up an alternative configuration log4j. Is that possible?
I don't have any stacktrace, only log's warnings
log4j:WARN No appenders could be found for logger (agent.Test1Agent.JavaAgent).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

There is no way to do what you want correctly because log4j doesn't throw any exceptions or notifies somehow in case of misconfiguration. But it is possible.
See PoC below
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.helpers.Loader;
import org.apache.log4j.spi.Configurator;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URL;
public class Log4jAlternativeConfig {
private static class PrintStreamCallbackSupportDecorator extends PrintStream {
public PrintStreamCallbackSupportDecorator(OutputStream out, Callback callback) {
super(out);
this.callback = callback;
}
public interface Callback {
public void onPrintln(String x);
}
private Callback callback;
#Override
public void println(String x) {
callback.onPrintln(x);
super.println(x);
}
}
public static void main(String[] args) {
PrintStreamCallbackSupportDecorator.Callback callback = new PrintStreamCallbackSupportDecorator.Callback() {
#Override
public void onPrintln(String x) {
if (x.startsWith("log4j:WARN No appenders could be found for logger")) {
Configurator configurator = new PropertyConfigurator();
URL url = Loader.getResource("log4j_alternative.properties");
configurator.doConfigure(url, LogManager.getLoggerRepository());
}
}
};
System.setErr(new PrintStreamCallbackSupportDecorator(System.err, callback));
Logger log = LogManager.getLogger(Log4jAlternativeConfig.class);
//causes "No appenders could be found for logger" warning
log.error("test");
//should be logged as configured in log4j_alternative.properties
log.error("test 2");
}
}
The solution isn't perfect but it works.

You can setup an alternative log4j configuration by providing a default configuration file on your classpath. Since log4j uses Thread.getContextClassLoader().getResource() to locate the default configuration files and does not directly check the file system, it should be included in your jar. So that if no external configuration provided it will fall back on the default configuration.
See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info

How to use custom PolicySpi

I'm trying to implement a custom java.security.Permission type, which should be checked at runtime (so no policy file, but in code). This checking is done by a java.security.Policy. I understood I should implement my own java.security.PolicySpi for this.
I cannot find any explanation on how to initialise and use a PolicySpi, or is there a better way to do this?

Checking permissions
In your question you stated that you then want to check the permission with java.security.Policy, but without using a spi.policy file.
From the PolicySpi API, you can see that a PolicySpi object features 4 methods:
engineGetPermissions(CodeSource codesource)
engineGetPermissions(ProtectionDomain domain)
engineImplies(ProtectionDomain domain, Permission permission)
engineRefresh()
However, you might not need PolicySpi as there are easier solutions to check permissions.
See:
Security Manager vs Access Controller
AccessController usage
Since you haven't specified what kind of permission you will grant, I will assume it is a permission concerning a java.security.CodeSource object.
To check all current permissions for a file:
public static void main(String[] args) {
CodeSource source;
try {
source = new CodeSource(new URL("file:/c:/*"), (java.security.cert.Certificate[]) null);
Policy policy = Policy.getPolicy();
System.out.println(policy.getPermissions(source));
} catch (IOException e) {
e.printStackTrace();
}
}
A nice example for the SecurityManager checkPermission() is this tutorial.
For checking specific FilePermissions, you can use:
FilePermission perm = new FilePermission("path/file", "read");
AccessController.checkPermission(perm);
Granting permissions
Granting permissions at runtime can be done with java.lang.RuntimePermission.
For other examples of how to grant permissions to a file, I suggest you read the following:
Access Control Mechanisms and Algorithms
Configuring spi.policy files
Security Managers and Permissions
That should bring you a long way! Good luck!

The previous answer lists alternatives to using PolicySpi (and more generally custom Policy implementations ). This answer will instead give a simplistic example on how a PolicySpi implementation can actually be used as a replacement of the system-default Policy.
Author a JCA Provider.
package com.example;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.security.Provider;
public final class TestProvider extends Provider {
private static final long serialVersionUID = 5544432861418770903L;
public TestProvider() {
super("TestProvider", 1, "TestProvider 1.0");
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
putService(new TestPolicySpiService(this));
return null;
});
}
}
Author the sole Service descriptor encapsulated by the provider.
package com.example;
import java.security.Policy.Parameters;
import java.security.PolicySpi;
import java.security.Provider;
import java.security.Provider.Service;
import java.util.Collections;
final class TestPolicySpiService extends Service {
TestPolicySpiService(Provider p) {
super(p, "Policy", "TestPolicy", PolicySpi.class.getName(), Collections.emptyList(), Collections.emptyMap());
}
#Override
public PolicySpi newInstance(Object constructorParameter) {
Parameters policyParams = null;
if (constructorParameter instanceof Parameters) {
policyParams = (Parameters) constructorParameter;
}
return new TestPolicySpi(policyParams);
}
#Override
public boolean supportsParameter(Object parameter) {
return parameter instanceof Parameters;
}
}
Author the actual service (the PolicySpi implementation in this case) that the service descriptor produces.
package com.example;
import java.security.Permission;
import java.security.Policy.Parameters;
import java.security.PolicySpi;
import java.security.ProtectionDomain;
final class TestPolicySpi extends PolicySpi {
TestPolicySpi(Parameters policyParams) {}
#Override
protected boolean engineImplies(ProtectionDomain domain, Permission permission) {
// deny unconditionally
return false;
}
}
Register the provider either statically, by modifying the security.provider.n properties in JAVA_HOME/lib/security/java.security, or programmatically, via java.security.Security.addProvider(Provider) / java.security.Security.insertProviderAt(Provider, int).
Replace the default Policy.
package com.example;
import java.security.NoSuchAlgorithmException;
import java.security.Policy;
public class Main {
public static void main(String... args) throws NoSuchAlgorithmException {
// the following assumes that the provider has been statically registered
Policy.setPolicy(Policy.getInstance("TestPolicy", null));
System.setSecurityManager(new SecurityManager());
// test
System.out.println(System.getProperty("user.home")); // should raise AccessControlException
}
}
Is there a better way to do this?
There certainly is a less involved way, as long as the consequent tight coupling between application and policy does not irk you too badly: Just subclass Policy directly and pass an instance of your implementation to Policy.setPolicy(Policy).
Further reading:
Java Cryptography Architecture (JCA) Reference Guide
How to Implement a Provider in the Java Cryptography Architecture
Standard Algorithm Name Documentation for JDK 8
Troubleshooting Security

As of Java 6, the default implementation for PolicySpi is sun.security.provider.PolicySpiFile. You can get inspired from the source code of PolicySpiFile:
package sun.security.provider;
import java.security.CodeSource;
import java.security.Permission;
import java.security.PermissionCollection;
import java.security.Policy;
import java.security.PolicySpi;
import java.security.ProtectionDomain;
import java.security.URIParameter;
import java.net.MalformedURLException;
/**
* This class wraps the PolicyFile subclass implementation of Policy
* inside a PolicySpi implementation that is available from the SUN provider
* via the Policy.getInstance calls.
*
*/
public final class PolicySpiFile extends PolicySpi {
private PolicyFile pf;
public PolicySpiFile(Policy.Parameters params) {
if (params == null) {
pf = new PolicyFile();
} else {
if (!(params instanceof URIParameter)) {
throw new IllegalArgumentException
("Unrecognized policy parameter: " + params);
}
URIParameter uriParam = (URIParameter)params;
try {
pf = new PolicyFile(uriParam.getURI().toURL());
} catch (MalformedURLException mue) {
throw new IllegalArgumentException("Invalid URIParameter", mue);
}
}
}
protected PermissionCollection engineGetPermissions(CodeSource codesource) {
return pf.getPermissions(codesource);
}
protected PermissionCollection engineGetPermissions(ProtectionDomain d) {
return pf.getPermissions(d);
}
protected boolean engineImplies(ProtectionDomain d, Permission p) {
return pf.implies(d, p);
}
protected void engineRefresh() {
pf.refresh();
}
}

Redirect slf4j to string

How can I configure slf4j to redirect all logged information to a Java string?
This is sometimes useful in unit tests, e.g. to test no warnings are printed when loading a servlet, or to make sure a forbidden SQL table is never used.

A bit late, but still...
As logging configurations should be easy to replace when unit testing, you could just configure to log over stdout and then capture that prior to executing the logging subject.
Then set the logger to be silent for all but the subject under test.
#Test
public void test()
{
String log = captureStdOut(() -> {
// ... invoke method that shouldn't log
});
assertThat(log, is(emptyString()));
}
public static String captureStdOut(Runnable r)
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream out = System.out;
try {
System.setOut(new PrintStream(baos, true, StandardCharsets.UTF_8.name()));
r.run();
return new String(baos.toByteArray(), StandardCharsets.UTF_8);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException("End of the world, Java doesn't recognise UTF-8");
} finally {
System.setOut(out);
}
}
And if using slf4j over log4j in tests, a simple log4j.properties:
log4j.rootLogger=OFF, out
log4j.category.com.acme.YourServlet=INFO, out
log4j.appender.out=org.apache.log4j.ConsoleAppender
log4j.appender.out.layout=org.apache.log4j.PatternLayout
log4j.appender.out.layout.ConversionPattern=%-5p %c{1}:%L - %m%n
Or if you loath configuration as an external dependencies in unit tests, then programmatically configure log4j:
//...
static final String CONSOLE_APPENDER_NAME = "console.appender";
private String pattern = "%d [%p|%c|%C{1}] %m%n";
private Level threshold = Level.ALL;
private Level defaultLevel = Level.OFF;
//...
public void configure()
{
configureRootLogger();
configureConsoleAppender();
configureCustomLevels();
}
private void configureConsoleAppender()
{
ConsoleAppender console = new ConsoleAppender();
console.setName(CONSOLE_APPENDER_NAME);
console.setLayout(new PatternLayout(pattern));
console.setThreshold(threshold);
console.activateOptions();
Logger.getRootLogger().addAppender(console);
}
private void configureRootLogger()
{
Logger.getRootLogger().getLoggerRepository().resetConfiguration();
Logger.getRootLogger().setLevel(defaultLevel);
}

As I see it you have two options.
First you could implement a custom Appender (depending on which slf4j implementation you're using) which simply appends each logged statement to a StringBuffer. In this case you probably have to hold a static reference to your StringBuffer so your test classes can access it.
Second you could write your own implementation of ILoggerFactory and Logger. Again your Logger would just append all the messages to internal StringBuffers, although in this case you'd probably have multiple buffers, one for each log level. If you did it this way you'd have an easy way of retrieving the Logger instances since you'd own the factory that was distributing them.

Shouldn't make sense to redirect all the logs to watch to a separate log file? That way you have the control you want (you can delete the log file before running the test and checking if the file has been create at any moment) without losing the benefits of logging (redirecting your output to a String can cause memory leaks and is less performant)

This is a simple way to log to the console:
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.BasicConfigurator;
import ch.qos.logback.classic.LoggerContext;
private void LogToConsole() {
BasicConfigurator bc = new BasicConfigurator();
LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
lc.reset();
bc.configure(lc);
}

Not quite exactly what you're doing, but I've written a LogInterceptingTestHarness which enables assertion of specific log statements. You could similarly use it (or something like it) to assert nothing has been logged at a certain level.
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.List;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.Logger;
import org.junit.After;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
import lombok.Getter;
/**
* Use this class to intercept logs for the purposes of unit testing log output.
* <p>
* On {#link Before} of the unit test, call {#link #initHarness(Class, Level)} or {#link #initHarness(Class, Level, String)} to get a new harness and hold onto reference to it in a class-level
* variable of your unit test
* <p>
* On {#link After} of the unit test, you MUST call {#link #teardown()} in order to remove the mocked {#link #appender}
*
* #author jeff.nelson
*
*/
#Getter
public class LogInterceptingTestHarness {
private final Appender appender;
private final ArgumentCaptor<LogEvent> logEventCaptor;
private final Logger itsLogger;
private LogInterceptingTestHarness(Class<?> classInterceptLogsFor, Level logLevel, String appenderName) {
logEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
appender = mock(Appender.class);
doReturn("testAppender").when(appender).getName();
doReturn(true).when(appender).isStarted();
itsLogger = (Logger) LogManager.getLogger(classInterceptLogsFor);
itsLogger.addAppender(appender);
itsLogger.setLevel(logLevel);
}
public void teardown() {
itsLogger.removeAppender(appender);
}
public List<LogEvent> verifyNumLogEvents(int numEvents) {
verify(appender, times(numEvents)).append(logEventCaptor.capture());
return logEventCaptor.getAllValues();
}
public LogEvent verifyOneLogEvent() {
return verifyNumLogEvents(1).get(0);
}
public void assertLoggedMessage(String message) {
assertLogMessage(message, logEventCaptor.getValue());
}
public void assertLoggedMessage(String message, int messageIndex) {
assertLogMessage(message, logEventCaptor.getAllValues().get(messageIndex));
}
public static void assertLogMessage(String message, LogEvent event) {
assertEquals(message, event.getMessage().getFormattedMessage());
}
public static LogInterceptingTestHarness initHarness(Class<?> classInterceptLogsFor, Level logLevel) {
return initHarness(classInterceptLogsFor, logLevel, "testAppender");
}
public static LogInterceptingTestHarness initHarness(Class<?> classInterceptLogsFor, Level logLevel, String appenderName) {
return new LogInterceptingTestHarness(classInterceptLogsFor, logLevel, appenderName);
}
}