Develop Reference

Infinispan drops connection to MYSQL 5.7 database

We have Keycloak in HA, which we have configured with a external Infinispan cluster for sessions, clientSessions & authenticationSessions.
Everything works under containers in a similar approach like the one performed under https://github.com/albertoSoto/keycloak-infinispan-cluster
The project runs KC 15.0.2 with Wildfly (migration to quarkus to be done), and in that case, uses Infinispan 11.0.9 to perform the external data persistence to Mysql 5.7. The driver used is the latest one, using https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar as suggested by Oracle. The connection driver is com.mysql.cj.jdbc.Driver.
The project starts nice, but after random time, mysql drops the connection and the Infinispan cluster can't reconnect.
In a trial to make it work, I have been able to use Agroal configuration, based in a properties file like it's at the bottom of this message.
The content of that agroal property file, that overrides JPA behavior in the project is the following:
org.infinispan.agroal.metricsEnabled=false
org.infinispan.agroal.minSize=10
org.infinispan.agroal.maxSize=100
org.infinispan.agroal.initialSize=20
org.infinispan.agroal.acquisitionTimeout_s=1
org.infinispan.agroal.validationTimeout_m=1
org.infinispan.agroal.leakTimeout_s=10
org.infinispan.agroal.reapTimeout_m=10
org.infinispan.agroal.maxLifetime_m=10
org.infinispan.agroal.autoCommit=true
org.infinispan.agroal.jdbcTransactionIsolation=READ_COMMITTED
org.infinispan.agroal.jdbcUrl=jdbc:mysql://mysql:3306/infinispan
org.infinispan.agroal.driverClassName=com.mysql.cj.jdbc.Driver
org.infinispan.agroal.principal=keycloak
org.infinispan.agroal.credential=password
The error shown after the connection is closed from the db is the following:
[1;31m21:55:31,052 ERROR (jgroups-319,vi-infinispan-1-5379) [org.infinispan.interceptors.impl.InvocationContextInterceptor] ISPN000136: Error executing command RemoveCommand on Cache 'clientSessions', writing keys [WrappedByteArray{bytes=0304090000000E\j\a\v\a\.\u\t\i\l\.\U\U\I\DBC9903F798\m85\/000000020000000C\l\e\a\s\t\S\i\g\B\i\t\s\$000000000B\m\o\s\t\S\i\g\B\i\t\s\$00168D0C\z8AB49FBA9B\C118A06A0DB\D82... (85 bytes), hashCode=73644551}] org.infinispan.remoting.RemoteException: ISPN000217: Received exception from vi-infinispan-0-53111, see cause for remote stack trace
at org.infinispan.remoting.transport.ResponseCollectors.wrapRemoteException(ResponseCollectors.java:25)
at org.infinispan.remoting.transport.ValidSingleResponseCollector.withException(ValidSingleResponseCollector.java:37)
at org.infinispan.remoting.transport.ValidSingleResponseCollector.addResponse(ValidSingleResponseCollector.java:21)
at org.infinispan.remoting.transport.impl.SingleTargetRequest.addResponse(SingleTargetRequest.java:73)
at org.infinispan.remoting.transport.impl.SingleTargetRequest.onResponse(SingleTargetRequest.java:43)
at org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1402)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1305)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:131)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1445)
at org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:913)
at org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:876)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1049)
at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:772)
at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:753)
at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:405)
at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:592)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1396)
at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.infinispan.persistence.spi.PersistenceException: Error while removing string keys from database
at org.infinispan.marshall.exts.ThrowableExternalizer.readObject(ThrowableExternalizer.java:234)
at org.infinispan.marshall.exts.ThrowableExternalizer.readObject(ThrowableExternalizer.java:42)
at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
at org.infinispan.remoting.responses.ExceptionResponse$Externalizer.readObject(ExceptionResponse.java:49)
at org.infinispan.remoting.responses.ExceptionResponse$Externalizer.readObject(ExceptionResponse.java:41)
at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:192)
at org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:221)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1394)
... 25 more
Caused by: java.sql.SQLNonTransientConnectionException: No operations allowed after connection closed.
We do use JDBC_PING for the cluster connection and 2 nodes are active. They register themselves properly and everything works like a charm, until the timeout is set.
The base configuration that I have placed is the following:
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:11.0 https://infinispan.org/schemas/infinispan-config-11.0.xsd
urn:infinispan:server:11.0 https://infinispan.org/schemas/infinispan-server-11.0.xsd"
xmlns="urn:infinispan:config:11.0"
xmlns:server="urn:infinispan:server:11.0">
<!--
Generic XML definition located under
https://docs.jboss.org/infinispan/11.0/configdocs/
-->
<jgroups>
<stack-file name="default-udp" path="default-jgroups.xml"/>
<stack-file name="default-tcp" path="default-jgroups-tcp.xml"/>
<stack-file name="gce" path="default-jgroups-google.xml"/>
<stack-file name="k8s" path="default-jgroups-kubernetes.xml"/>
<stack-file name="kc-udp" path="default-keycloak-jgroups-udp.xml"/>
<stack-file name="custom-k8s-jdbc" path="custom-jgroups-kubernetes-jdbc.xml"/>
<stack-file name="custom-tcp-jdbc" path="custom-jgroups-tcp-jdbc.xml"/>
</jgroups>
<cache-container name="default" statistics="${env.INFINISPAN_CACHE_STATISTICS:false}">
<serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
<white-list>
<class>java.util.UUID</class>
<regex>org.keycloak.models.sessions.infinispan.*</regex>
</white-list>
</serialization>
<serialization marshaller="org.infinispan.commons.marshall.JavaSerializationMarshaller">
<white-list>
<class>java.util.UUID</class>
<regex>org.keycloak.models.sessions.infinispan.*</regex>
</white-list>
</serialization>
<transport cluster="${infinispan.cluster.name:cluster}" stack="${infinispan.cluster.stack:default-udp}"
node-name="${infinispan.node.name:}"/>
<replicated-cache-configuration name="sessions-cfg" mode="SYNC" start="EAGER"
statistics="${env.INFINISPAN_CACHE_STATISTICS:false}">
<state-transfer timeout="${infinispan.statetransfer.timeout:600000}"/>
<encoding media-type="application/x-jboss-marshalling"/>
<expiration lifespan="900000000000000000"/>
</replicated-cache-configuration>
<distributed-cache-configuration name="distributed-cache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
<expiration lifespan="900000000000000000"/>
<persistence passivation="false">
<string-keyed-jdbc-store shared="true" xmlns="urn:infinispan:config:store:jdbc:11.0">
<connection-pool properties-file="${env.PROPERTIES_FILE:/opt/infinispan/server/conf/connection-pool.properties}" />
<string-keyed-table drop-on-exit="false"
prefix="ISPN">
<id-column name="ID_COLUMN" type="VARCHAR(255)"/>
<!-- Blob generates error on KC. We increase it to a safe max size (65K per row)
<data-column name="DATA_COLUMN" type="BLOB" />
-->
<data-column name="DATA_COLUMN" type="VARBINARY(50000)"/>
<timestamp-column name="TIMESTAMP_COLUMN" type="BIGINT"/>
<segment-column name="SEGMENT_COLUMN" type="INT"/>
</string-keyed-table>
</string-keyed-jdbc-store>
</persistence>
<state-transfer timeout="${infinispan.statetransfer.timeout:600000}"/>
</distributed-cache-configuration>
<!--https://infinispan.org/docs/stable/titles/configuring/configuring.html#distributed-caches_clustered-caches-->
<!--https://infinispan.org/docs/stable/titles/configuring/configuring.html#configuring-jdbc-cache-stores_persistence-->
<distributed-cache name="sessions" owners="2" configuration="distributed-cache-cfg">
</distributed-cache>
<distributed-cache name="clientSessions" owners="2" configuration="distributed-cache-cfg">
</distributed-cache>
<distributed-cache name="authenticationSessions" owners="2" configuration="distributed-cache-cfg">
</distributed-cache>
</cache-container>
<!-- Original at v11 - -->
<server xmlns="urn:infinispan:server:11.0">
<interfaces>
<interface name="public">
<inet-address value="${infinispan.bind.address:0.0.0.0}"/>
</interface>
</interfaces>
<socket-bindings default-interface="public" port-offset="0">
<socket-binding name="default" port="11222"/>
</socket-bindings>
<security>
<security-realms>
<security-realm name="default">
<properties-realm groups-attribute="Roles">
<user-properties path="users.properties" relative-to="infinispan.server.config.path"
plain-text="true"/>
<group-properties path="groups.properties" relative-to="infinispan.server.config.path"/>
</properties-realm>
</security-realm>
</security-realms>
</security>
<endpoints socket-binding="default" security-realm="default">
<hotrod-connector name="hotrod">
<authentication>
<sasl mechanisms="SCRAM-SHA-512 SCRAM-SHA-384 SCRAM-SHA-256 SCRAM-SHA-1 DIGEST-SHA-512 DIGEST-SHA-384 DIGEST-SHA-256 DIGEST-SHA DIGEST-MD5 PLAIN"
qop="auth" server-name="infinispan"/>
</authentication>
</hotrod-connector>
<rest-connector name="rest">
<authentication mechanisms="DIGEST BASIC"/>
</rest-connector>
</endpoints>
</server>
</infinispan>
The thing is... what are am I doing wrong?
There is not too much information about it. Can anyone help?
Thank you!

Unfortunately I think you have stumbled across a bug with the persistence availability check that prevents stores from reconnecting if an exception is thrown ISPN-13863. I have just created a PR, however the fix will only be available in the Infinispan 14.x stream.

Getting Uncaught init() exception created by servlet [JAX-RS Servlet] in application [ABC_WSv2.1] in LIberty 21.0.0.6

we are getting below exception in liberty running in linux box for jax-rs based web service application. But while running in local windows version Liberty (21.0.0.6) , it works fine and no exception thrown. Also we tried various option available in stack overflow and google, nothing worked. can anyone help us on this
E SRVE0271E: Uncaught init() exception created by servlet [JAX-RS Servlet] in application [ABC_WSv2.1]: java.lang.ClassCastException: com.abc.occws.SearchingApplication incompatible with javax.ws.rs.core.Application
server.xml configuration below
<?xml version="1.0" encoding="UTF-8"?>
<server description="new server">
<!-- Enable features -->
<featureManager>
<feature>webProfile-8.0</feature>
</featureManager>
<!-- To access this server from a remote client add a host attribute to the following element, e.g. host="*" -->
<httpEndpoint httpPort="9080" httpsPort="9443" id="defaultHttpEndpoint" />
<dataSource id="ABCDataSource" jndiName="jdbc/ABCDS" queryTimeout="20m" type="javax.sql.DataSource">
<connectionManager agedTimeout="1000m" minPoolSize="1" numConnectionsPerThreadLocal="10"/>
<jdbcDriver libraryRef="OracleLib"/>
<properties.oracle URL="jdbc:oracle:thin:#//10.135.157.5:1565/DOABC_APP" password="xxx" user="xxxx"/>
</dataSource>
<!-- Define JDBC Drivers -->
<library id="OracleLib">
<file name="C:/dev/Liberty/ojdbc8-19.3.0.0.jar"/>
</library>
<applicationMonitor updateTrigger="mbean" />
<application id="ABC_WSv2.1" location="C:\Dev\Liberty\ABC_WSv2.1.ear" name="ABC_WSv2.1" type="ear" context-root="/abc-ws">
<classloader commonLibraryRef="OracleLib"/>
</application>
<webContainer uppressHtmlRecursiveErrorOutput="true"/>
</server>

On the surface this looks like this problem is classloader related. On Windows I suspect you are running this application is isolation whereas on Linux it is running in concert with other things and likely Parent-Last classloading is involved.
I would suggest collecting Liberty classloader trace to see where the affected classes are being loaded from. Here are some instructions for doing that: Liberty Classloader Mustgather
Note: This is assuming that you are running the same version of your application on both Windows and Linux and in both cases your application class extends javax.ws.rs.core.Application. If that is not the case then that is certainly causing the problem.

Tomcat 8.5 - 403 Access Denied

I've created a virtual machine with Ubuntu 16.04 LTS and, following this guide, I've installed Tomcat 8.5 and configured as follows:
/opt/tomcat/webapps/host-manager/META-INF/context.xml:
<Context antiResourceLocking="false" privileged="true">
<Valve className="org.apache.catalina.valves.RemoteAddrValve"
allow="192\.168\.1\.124|127\.\d+\.\d+\.\d+|::1|0:0:0:0:0:0:0:1" />
</Context>
<!-- my local ip is 192.168.1.124 so I've added it -->
/opt/tomcat/conf/tomcat-users.xml
<role rolename="manager-gui" />
<role rolename="admin-gui" />
<user username="myuser" password="MyPass123#" roles="manager-gui,admin-gui" />
No other configuration files has been modified.
The problem comes out when I try to access to the manager interface from my browser, it doesn't ask me any kind of credentials and throws me a 403 (Access Denied) error.
How can I fix this problem?

This is worked for me. Please check
Find the CATALINA_HOME/webapps/manager/META-INF/context.xml file and add the comment markers around the Valve.
<Context antiResourceLocking="false" privileged="true" >
<!--
<Valve className="org.apache.catalina.valves.RemoteAddrValve" allow="127\.\d+\.\d+\.\d+|::1|0:0:0:0:0:0:0:1" />
-->
</Context>
Tomcat8 Documentation.

java web app and Postgres on websphere-liberty

I'm deploying a working Tomcat webapp on websphere-liberty on docker. The webapp connects to a postgres data source on docker. In websphere when I try to get connection with
DataSource ds = (DataSource)ctx.lookup("java:comp/env/jdbc/postgres");
Connection conn = ds.getConnection();
my web.xml is setted as:
<resource-ref>
<description>postgreSQL Connection</description>
<res-ref-name>jdbc/postgres</res-ref-name>
<res-type>javax.sql.XADataSource</res-type>
<res-auth>Container</res-auth>
</resource-ref>
I get the following error
javax.naming.NamingException: CWNEN1001E: The object referenced by the java:comp/env/jdbc/postgres JNDI name could not be instantiated.
If the reference name maps to a JNDI name in the deployment descriptor bindings for the application
performing the JNDI lookup, make sure that the JNDI name mapping in the deployment descriptor binding is correct.
If the JNDI name mapping is correct, make sure the target resource
can be resolved with the specified name relative to the default initial context.
[Root exception is com.ibm.wsspi.injectionengine.InjectionException: CWNEN0030E: The
server was unable to obtain an object instance for the java:comp/env/jdbc/postgres reference.
The exception message was: CWNEN1004E: The server was unable to find the jdbc/postgres default binding with the javax.sql.XADataSource type for
the java:comp/env/jdbc/postgres reference.]
I exlude that is a problem with docker network. What I have setted wrong? Is something in my websphere-liberty?
server.xml is
<server description="Default server">
<!-- Enable features -->
<featureManager>
<feature>webProfile-7.0</feature>
<feature>adminCenter-1.0</feature>
<feature>jdbc-4.0</feature>
</featureManager>
<quickStartSecurity userName="admin" userPassword="password"/>
<!-- Define the host name for use by the collective.
If the host name needs to be changed, the server should be
removed from the collective and re-joined. -->
<!-- <variable name="defaultHostName" value="localhost" /> -->
<!-- Define an Administrator and non-Administrator -->
<basicRegistry id="basic">
<user name="admin" password="admin" />
<user name="nonadmin" password="nonadminpwd" />
</basicRegistry>
<!-- Assign 'admin' to Administrator -->
<administrator-role>
<user>admin</user>
</administrator-role>
<!-- <keyStore id="defaultKeyStore" password="Liberty" /> -->
<httpEndpoint id="defaultHttpEndpoint"
host="*"
httpPort="9080"
httpsPort="9443" />
<remoteFileAccess>
<writeDir>${server.config.dir}</writeDir>
</remoteFileAccess>
<library id="postgres-lib">
<fileset dir="/arturial/project/" includes="postgresql-9.4.1208.jre6.jar"/>
</library>
<dataSource id="jdbc-Prima_WA_db" jndiName="jdbc/postgres" type="javax.sql.DataSource">
<jdbcDriver libraryRef="postgres-lib"/>
<connectionManager numConnectionsPerThreadLocal="10" id="connectionManager" minPoolSize="1"/>
<!-- <properties.oracle user="postgres" password="postgres" -
url="jdbc:postgres://172.17.0.3:5432/Prima_WA_db"/> -->
</dataSource>
<!--
<applicationManager updateTrigger="disabled"/>
<application id="primawebapp" name="primawebapp" location="war/primawebapp" type="war">
<classLoader delegation="parentLast" commonLibraryRef="postgres-lib"/>
</application>
-->
</server>

Try this;
DataSource ds = (DataSource)ctx.lookup("jdbc/postgres");
Also, a different way.

Your <datasource> must have properties configured under it, otherwise there is no way for Liberty to know how to connect to your database.
For postgresql, try the following configuration:
<dataSource id="jdbc-Prima_WA_db" jndiName="jdbc/postgres">
<jdbcDriver libraryRef="postgres-lib"/>
<properties serverName="172.17.0.3" portNumber="5432" databaseName="Prima_WA_db"
user="postgres" password="postgres"/>
</dataSource>
Of course, using the proper values that correspond to the postgresql instance that you are running on.

Looks like your datasource definition might be incomplete, try updating it with the following:
<dataSource type="javax.sql.XADataSource" ...
<jdbcDriver javax.sql.XADataSource="org.postgresql.xa.PGXADataSource" ...

connect java metro client to wcf service with sessions

I need to connect a java metro client to a wcf service for a project I'm cuurently working on. I used the tutorial at click.
Everything seems to work properly until I want to add sessions to my service. When I add <reliableSession enabled="true" /> to my config file, the client isn't able to reload the wcf service.
config file:
<system.serviceModel>
<diagnostics>
<messageLogging logEntireMessage="false" logMalformedMessages="false"
logMessagesAtServiceLevel="false" logMessagesAtTransportLevel="false" />
</diagnostics>
<services>
<service name="Service" behaviorConfiguration="ServiceBehavior">
<!-- Service Endpoints -->
<endpoint address="" bindingConfiguration="interopBinding" binding="metroBinding" contract="IService">
<identity>
<dns value="localhost" />
</identity>
</endpoint>
<endpoint address="mex" binding="mexHttpBinding" contract="IMetadataExchange"/>
</service>
</services>
<behaviors>
<serviceBehaviors>
<behavior name="ServiceBehavior">
<!-- To avoid disclosing metadata information, set the value below to false and remove the metadata endpoint above before deployment -->
<serviceMetadata httpGetEnabled="true"/>
<!-- To receive exception details in faults for debugging purposes, set the value below to true. Set to false before deployment to avoid disclosing exception information -->
<serviceDebug includeExceptionDetailInFaults="false"/>
<!-- To configure the service certificate -->
<serviceCredentials>
<serviceCertificate storeLocation="LocalMachine" storeName="Root" x509FindType="FindBySubjectDistinguishedName" findValue="CN=go2.openscrolling" />
<clientCertificate>
<authentication certificateValidationMode="PeerOrChainTrust" />
</clientCertificate>
</serviceCredentials>
</behavior>
</serviceBehaviors>
</behaviors>
<bindings>
<metroBinding>
<binding name="interopBinding" messageEncoding="Text">
<security mode="MutualCertificate" establishSecurityContext="false" algorithmSuite="Basic128"/>
<!--<reliableSession enabled="true" />-->
</binding>
</metroBinding>
</bindings>
<extensions>
<bindingExtensions>
<add name="metroBinding" type="Microsoft.ServiceModel.Interop.Metro.Configuration.MetroBindingCollectionElement, Microsoft.ServiceModel.Interop, Version=1.0.0.0, Culture=neutral, PublicKeyToken=4fc38efee625237e"/>
</bindingExtensions>
</extensions>
</system.serviceModel>
<system.webServer>
<modules runAllManagedModulesForAllRequests="true"/>
</system.webServer>
</configuration>