Develop Reference

MDC logging for SSHD server with custom, per-company file-system

We are using Apache-Mina SSHD 1.7 to expose a SFTP server that uses a custom file-system implementation which creates a file system per company. So users of the same company (or more precisely for the same connector) will access the same file system while a users of an other company will access a filesystem unique to their company. The file-system is moreover just a view on a MySQL database and will write uploaded files after some conversions directly into the DB and read files on download from the DB.
The setup of the server looks like the excerpt below
void init() {
server = MessageSftpServer.setUpDefaultServer();
server.setPort(port);
LOG.debug("Server is configured for port {}", port);
File pemFile = new File(pemLocation);
FileKeyPairProvider provider = new FileKeyPairProvider(pemFile.toPath());
validateKeyPairProvider(provider.loadKeys(), publicKeyList);
server.setKeyPairProvider(provider);
server.setCommandFactory(new ScpCommandFactory());
server.setPasswordAuthenticator(
(String username, String password, ServerSession session) -> {
...
});
PropertyResolverUtils.updateProperty(server, ServerAuthenticationManager.MAX_AUTH_REQUESTS, 3);
SftpSubsystemFactory sftpFactory = new SftpSubsystemFactory.Builder()
.withShutdownOnExit(false)
.withUnsupportedAttributePolicy(UnsupportedAttributePolicy.Warn)
.build();
server.setSubsystemFactories(Collections.singletonList(sftpFactory));
// add our custom virtual file system to trick the user into believing she is operating against
// a true file system instead of just operating against a backing database
server.setFileSystemFactory(
new DBFileSystemFactory(connectorService, companyService, mmService, template));
// filter connection attempts based on remote IPs defined in connectors
server.addSessionListener(whitelistSessionListener);
}
Within the file system factory we basically just create the URI for the file system provider and pass it to the respective method of it
#Override
public FileSystem createFileSystem(Session session) throws IOException {
SFTPServerConnectorEntity connector =
connectorService.getSFTPServerConnectorForUser(session.getUsername());
if (null == connector) {
throw new IOException("No SFTP Server connector found for user " + session.getUsername());
}
String ip = CommonUtils.getIPforSession(session);
URI fsUri = URI.create("dbfs://" + session.getUsername() + "#" + ip + "/" + connector.getUuid());
LOG.debug("Checking whether to create file system for user {} connected via IP {}",
session.getUsername(), ip);
Map<String, Object> env = new HashMap<>();
env.put("UserAgent", session.getClientVersion());
try {
return fileSystemProvider.newFileSystem(fsUri, env);
} catch (FileSystemAlreadyExistsException fsaeEx) {
LOG.debug("Reusing existing filesystem for connector {}", connector.getUuid());
return fileSystemProvider.getFileSystem(fsUri);
}
}
and within the provider we simply parse the values from the provided URI and environment variables to create the final filesystem if none was yet available within the cache
#Override
public DBFileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException {
LOG.trace("newFileSystem({}, {}))", uri, env);
ConnectionInfo ci = ConnectionInfo.fromSchemeSpecificPart(uri.getSchemeSpecificPart());
String cacheKey = generateCacheKey(ci);
synchronized (fileSystems) {
if (fileSystems.containsKey(cacheKey)) {
throw new FileSystemAlreadyExistsException(
"A filesystem for connector " + ci.getConnectorUuid()
+ " connected from IP " + ci.getIp() + " already exists");
}
}
SFTPServerConnectorEntity connector =
connectorService.get(SFTPServerConnectorEntity.class, ci.getConnectorUuid());
List<CompanyEntity> companies = companyService.getCompaniesForConnector(connector);
if (companies.size() < 1) {
throw new IOException("No company for connector " + connector.getUuid() + " found");
}
DBFileSystem fileSystem = null;
synchronized (fileSystems) {
if (!fileSystems.containsKey(cacheKey)) {
LOG.info("Created new filesystem for connector {} (Remote IP: {}, User: {}, UserAgent: {})",
ci.getConnectorUuid(), ci.getIp(), ci.getUser(), env.get("UserAgent"));
fileSystem = new DBFileSystem(this, connector.getUsername(), companies, connector,
template, ci.getIp(), (String) env.get("UserAgent"));
Pair<DBFileSystem, AtomicInteger> sessions = Pair.of(fileSystem, new AtomicInteger(1));
fileSystems.put(cacheKey, sessions);
}
}
if (null == fileSystem) {
throw new FileSystemAlreadyExistsException(
"A filesystem for connector " + ci.getConnectorUuid()
+ " connected from IP " + ci.getIp() + " already exists");
}
return fileSystem;
}
#Override
public DBFileSystem getFileSystem(URI uri) {
LOG.trace("getFileSystem({}))", uri);
String schemeSpecificPart = uri.getSchemeSpecificPart();
if (!schemeSpecificPart.startsWith("//")) {
throw new IllegalArgumentException(
"Invalid URI provided. URI must have a form of 'dbfs://ip:port/connector-uuid' where "
+ "'ip' is the IP address of the connected user, 'port' is the remote port of the user and "
+ "'connector-uuid' is a UUID string identifying the connector the filesystem was created for");
}
ConnectionInfo ci = ConnectionInfo.fromSchemeSpecificPart(schemeSpecificPart);
String cacheKey = generateCacheKey(ci);
if (!fileSystems.containsKey(cacheKey)) {
throw new FileSystemNotFoundException(
"No filesystem found for connector " + ci.getConnectorUuid() + " with connection from IP "
+ ci.getIp());
}
Pair<DBFileSystem, AtomicInteger> sessions = fileSystems.get(cacheKey);
if (!sessions.getKey().isOpen()) {
throw new FileSystemNotFoundException(
"Filesystem for connector " + ci.getConnectorUuid() + " with connection from IP " + ci
.getIp() + " was closed already");
}
int curSessions = sessions.getValue().incrementAndGet();
LOG.info("Added further session to filesystem for connector {}. Current connected sessions: {} (Remote IP: {}, User: {})",
ci.getConnectorUuid(), curSessions, ci.getIp(), ci.getUser());
return sessions.getKey();
}
private String generateCacheKey(String user, String ip, String connectorUuid) {
return connectorUuid + "_" + ip + "_" + user;
}
private String generateCacheKey(ConnectionInfo ci) {
return generateCacheKey(ci.getUser(), ci.getIp(), ci.getConnectorUuid());
}
This works out really well, however, as more and more users get added to the SFTP server the monitoring of the performed actions is suffering a bit due to the lack of propper MDC logging. Simply adding MDC logging isn't working cleanly as Mina or SSHD in particular share the threads among connected users which lead to the MDC context printing the wrong information at times which further lead to confusion on analyzing the log. As a temporary solution we removed it currently from the project.
We also tried to customize Nio2Session (and a couple of other classes) in order to intervene into the threading creation, though this classes were obviously not designed for inheritance which later lead to problems down the road.
Is there a better strategy to include propper MDC logging in our particular scenario where not one file system is used but a filesystem per company approach?

RMI and JMX Socket Factories

I'm trying to start an embedded JMX server in my java app. I want to use the same port for the RMI Registry and for the actual RMI traffic (or JMX traffic if you like). Apparently this is possible since the RMI Registry is merely a Remote Object itself.
The added difficulty is that I need to use Socket Factories because I need to bind to a specific NIC.
I start off by:
int registryPort = 3012;
int jmxPort = 3012; // use the same port
and here's my server socket factory. Pretty straight-forward stuff:
public class MyRMIServerSocketFactory implements RMIServerSocketFactory {
private final InetAddress inetAddress;
public MyRMIServerSocketFactory(InetAddress inetAddress) {
this.inetAddress = inetAddress;
}
#Override
public ServerSocket createServerSocket(int port) throws IOException {
return new ServerSocket(port, 0, inetAddress);
}
#Override
public int hashCode() {
int hash = 5;
hash = 97 * hash + (this.inetAddress != null ? this.inetAddress.hashCode() : 0);
return hash;
}
#Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final FlexibleRMIServerSocketFactory other = (FlexibleRMIServerSocketFactory) obj;
if (this.inetAddress != other.inetAddress && (this.inetAddress == null || !this.inetAddress.equals(other.inetAddress))) {
return false;
}
return true;
}
}
(the equals() and hashCode() are auto-generated by my IDE, don't get stuck on them)
I create the RMI Registry like this:
serverSocketFactory = new MyRMIServerSocketFactory(inetAddressBind);
LocateRegistry.createRegistry(
registryPort,
RMISocketFactory.getDefaultSocketFactory(), // client socket factory
serverSocketFactory // server socket factory
);
and then on to creating the JMXConnectorServer:
JMXServiceURL url = new JMXServiceURL(
"service:jmx:rmi://localhost:" + jmxPort +
"/jndi/rmi://:" + registryPort + "/jmxrmi");
Map env = new HashMap();
env.put(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE, serverSocketFactory);
connector = JMXConnectorServerFactory.newJMXConnectorServer(
url,
env,
ManagementFactory.getPlatformMBeanServer());
connector.start();
This results in a bind error on the connector.start() saying that the address is already in use.
If I skip using Socket Factories altogether:
LocateRegistry.createRegistry(registryPort);
and
JMXServiceURL url = new JMXServiceURL(
"service:jmx:rmi://localhost:" + jmxPort +
"/jndi/rmi://:" + registryPort + "/jmxrmi");
connector = JMXConnectorServerFactory.newJMXConnectorServer(
url,
null,
ManagementFactory.getPlatformMBeanServer());
connector.start();
it works as expected, i.e. only a single port will be opened and no errors.
Question: How to make the 'single-listening-port-scenario' work with Socket Factories ?
UPDATE - FINAL SOLUTION
It works if you create the Registry with a null client socket factory, i.e.
LocateRegistry.createRegistry(
registryPort,
null, // client socket factory (let it default to whatever RMI lib wants)
serverSocketFactory // server socket factory
);
I also had to set the java.rmi.server.hostname System Property which I guess will often be the case in a scenario like mine.

This should work: you have a correct-looking equals() method in your ServerSocketFactory, which is the important bit. RMI does call that. However the same doesn't apply currently to your client socket factory. You need to pass null as the client socket factory, not RMISocketFactory.getDefaultSocketFactory(), as that gives you a sun.rmi.transport.proxy.RMIMasterSocketFactory, which doesn't implement equals() for some reason. Or else your own implementation of RMIClientSocketFactory with a plausible equals() method.
So what is happening here is that RMI is comparing the CSFs first, and they are coming out unequal, so it doesn't even bother comparing the SSFs:
csf1.equals(csf2) && ssf1.equals(ssf2)
so it tries to create a new ServerSocket on the port you specifed, which is the same as the first port, so it fails.
You could add a shortcut at the beginning of equals that returns true if this == that.

You should search for the JMXMP protocol, and for the jmxremote_optional.jar that contains it. This is a more controllable, more efficient protocol for JMX.

Identify server on Tomcat (HttpServletRequest.getLocalAddr() fails)

With Tomcat setup behind Apache, how can an id (IP address ideally) of the server be easily determined?
The specific situation is that multiple servers are setup behind a load balancer, thus the incoming request host name is non-unique and insufficient to identify a particular server for logging purposes. Using HttpServletRequest.getLocalAddr() is unfortunately returning the same hostname instead of the IP address as would be expected (I am assuming this is related to this very old issue here: https://issues.apache.org/bugzilla/show_bug.cgi?id=46082).
Is there a way to make getLocalAddr() perform as documented, or are other methods required to query the IP address of the server?

On our project, we use JMX to get all the config information.
It takes a few steps, because it is like navigating down the server.xml file
This link has some info: http://oss.wxnet.org/mbeans.html
It is probably overkill if all you want is the IP, but I thought I'd throw it out there.
MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
Set<ObjectName> theConnectors = mbeanServer.queryNames(
new ObjectName("Catalina:type=Connector,*"),
null);
if (theConnectors != null)
{
for (ObjectName nextConnectorName : theConnectors)
{
InetAddress theInetAddress = (InetAddress) mbeanServer.getAttribute(
nextConnectorName,
"address");
if (theInetAddress != null)
{
ipAddress = theInetAddress.getHostAddress();
}
if (!StringUtil.isEmpty(ipAddress))
{
// found the IP address
break;
}
}
}

For my situation, the solution was to get the IP address of the server directly instead of attempting to get the local address via HttpServleRequest.
I cached the IP for use in my filter via:
private static final String serverIp;
static {
String addressString = null;
try
{
InetAddress address = InetAddress.getLocalHost();
addressString = address.getHostAddress();
} catch (Exception e)
{
logger.error("Exception while attempting to determine local ip address",e);
}
if (addressString != null) serverIp = addressString;
else serverIp = "unknown";
}

I had a similar issue recently (a few years after the original question) and found this question and answers. The issue in my case was that the ServletRequest#getLocalAddr() implementation was returning the remote address instead of the local address. The issue was caused by a regression in Tomcat v9.0.22. It was fixed in v9.0.23. See the question and answer here:
https://stackoverflow.com/a/57725039/9602527

How to identify the URL of an Java web application from within?

My Java web application contains a startup servlet. Its init() method is invoked, when the web application server (Tomcat) is started. Within this method I need the URL of my web application. Since there is no HttpServletRequest, how to get this information?

You can't. Because there is no "URL of an Java web application" as seen "from within". A servlet is not tied to an URL, that is done from the outside. (Perhaps you have a Apache server that connects to a Tomcat - Tomcat can't know about it)
It makes sense to ask a HttpServletRequest for its url, because we are speaking of the information of a event (the URL that was actually used to generate this request), it does not make sense to ask for a configuration URL.

A workaround could be to perform the initialization lazy when the first request arrives. You can implement a filter that do that once, e.g. by storing a boolean flag in a static variable and synchronizing access to the flag correctly. But it implies a little overhead because each subsequent request will go through the filter which then bypass the initialization. It was just a thought.

There is nothing in the servlet API that provides this information, plus any given resource may be bound to multiple URL's.
What you CAN do, is to inspect the servlet context when you receive an actual request and see what URL was used.

Here is how it works for me and probably for most configurations:
public static String getWebappUrl(ServletConfig servletConfig, boolean ssl) {
String protocol = ssl ? "https" : "http";
String host = getHostName();
String context = servletConfig.getServletContext().getServletContextName();
return protocol + "://" + host + "/" + context;
}
public static String getHostName() {
String[] hostnames = getHostNames();
if (hostnames.length == 0) return "localhost";
if (hostnames.length == 1) return hostnames[0];
for (int i = 0; i < hostnames.length; i++) {
if (!"localhost".equals(hostnames[i])) return hostnames[i];
}
return hostnames[0];
}
public static String[] getHostNames() {
String localhostName;
try {
localhostName = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException ex) {
return new String[] {"localhost"};
}
InetAddress ia[];
try {
ia = InetAddress.getAllByName(localhostName);
} catch (UnknownHostException ex) {
return new String[] {localhostName};
}
String[] sa = new String[ia.length];
for (int i = 0; i < ia.length; i++) {
sa[i] = ia[i].getHostName();
}
return sa;
}

Someone knows a mail (SMTP) delivery library for Java?

I'd like to send mail without bothering with the SMTP-Server which is used for delivery.
So JavaMail API doesn't work for me because I have to specify a SMTP server to connect to.
I'd like the library to find out on its own which SMTP server is responsible for which email address by querying the MX record of the mail address domain.
I'm looking for something like Aspirin. Unfortunately I can't use Aspirin itself because the development stopped 2004 and the library fails to communicate with modern spam hardened servers correctly.
An embeddable version of James would do the task. But I haven't found documentation concerning whether this is possible.
Or does anyone know about other libraries I could use?

One possible solution: get the MX record on your own and use JavaMail API.
You can get the MX record using the dnsjava project:
Maven2 dependency:
<dependency>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
<version>2.0.1</version>
</dependency>
Method for MX record retrieval:
public static String getMXRecordsForEmailAddress(String eMailAddress) {
String returnValue = null;
try {
String hostName = getHostNameFromEmailAddress(eMailAddress);
Record[] records = new Lookup(hostName, Type.MX).run();
if (records == null) { throw new RuntimeException("No MX records found for domain " + hostName + "."); }
if (log.isTraceEnabled()) {
// log found entries for debugging purposes
for (int i = 0; i < records.length; i++) {
MXRecord mx = (MXRecord) records[i];
String targetString = mx.getTarget().toString();
log.trace("MX-Record for '" + hostName + "':" + targetString);
}
}
// return first entry (not the best solution)
if (records.length > 0) {
MXRecord mx = (MXRecord) records[0];
returnValue = mx.getTarget().toString();
}
} catch (TextParseException e) {
throw new RuntimeException(e);
}
if (log.isTraceEnabled()) {
log.trace("Using: " + returnValue);
}
return returnValue;
}
private static String getHostNameFromEmailAddress(String mailAddress) throws TextParseException {
String parts[] = mailAddress.split("#");
if (parts.length != 2) throw new TextParseException("Cannot parse E-Mail-Address: '" + mailAddress + "'");
return parts[1];
}
Sending mail via JavaMail code:
public static void sendMail(String toAddress, String fromAddress, String subject, String body) throws AddressException, MessagingException {
String smtpServer = getMXRecordsForEmailAddress(toAddress);
// create session
Properties props = new Properties();
props.put("mail.smtp.host", smtpServer);
Session session = Session.getDefaultInstance(props);
// create message
Message msg = new MimeMessage(session);
msg.setFrom(new InternetAddress(fromAddress));
msg.setRecipient(Message.RecipientType.TO, new InternetAddress(toAddress));
msg.setSubject(subject);
msg.setText(body);
// send message
Transport.send(msg);
}

This is completely the wrong way to handle this.
Anyone connected to the internet will have some kind of "legit" SMTP server available to them to take the submission of email -- your ISP, your office, etc.
You WANT to leverage because they do several things for you.
1) they take your message and the responsibility to handle that message. After you drop it off, it's not your problem anymore.
2) Any mail de-spamming technologies are handled by the server. Even better, when/if those technologies change (Domain keys anyone?), the server handles it, not your code.
3) You, as a client of that sending mail system, already have whatever credentials you need to talk to that server. Main SMTP servers are locked down via authentication, IP range, etc.
4) You're not reinventing the wheel. Leverage the infrastructure you have. Are you writing an application or a mail server? Setting up mail server is an every day task that is typically simple to do. All of those casual "dumb" users on the internet have managed to get email set up.

Don't.
Sending email is much more complex than it seems. Email servers excel at (or should excel at) reliable delivery.
Set up a separate email server if you need to- that will be essentially the same as implementing one in Java (I doubt you will find libraries for this task- they would be essentially complete mail servers), but much more simpler.