Develop Reference

Saving a Chunk of Data into DB from Java Application

I have a Java Batch Program which is triggered from a BPEL process used for inserting set of Record into DB.
Size of record might vary but at an average around 20,000 to 40,000.
My custom java batch picks up the data from an excel uploaded from a third party app into server location.
For performing the DML operation I have gone with Hibernate/Spring JDBC framework.
Refer to my spring configuration file as below::
<context:component-scan base-package="mybasepackage" />
<bean id="myProps"
class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<list>
<value>classpath*:myDB.properties</value>
</list>
</property>
<property name="ignoreUnresolvablePlaceholders" value="true" />
</bean>
<bean id="dataSource"
class="org.springframework.jdbc.datasource.DriverManagerDataSource">
<property name="driverClassName" value="${DB_DRIVER}" />
<property name="url" value="${DB_URL}" />
<property name="username" value="${DB_USER}" />
<property name="password" value="${DB_PASSWORD}" />
</bean>
<bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="dataSource" />
</bean>
<bean id="sessionFactory"
class="org.springframework.orm.hibernate5.LocalSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="packagesToScan" value="mypackage" />
<property name="hibernateProperties">
<props>
<prop key="hibernate.dialect">org.hibernate.dialect.Oracle12cDialect</prop>
<prop key="hibernate.show_sql">true</prop>
<prop key="hibernate.jdbc.batch_size">20</prop>
<prop key="hibernate.c3p0.min_size">5</prop>
<prop key="hibernate.c3p0.max_size">10</prop>
<prop key="hibernate.c3p0.max_size">7000</prop>
</props>
</property>
</bean>
<bean id="myBatchDao"
class="mypackage.MyBatchDaoImpl">
<property name="jdbcTemplate" ref="jdbcTemplate" />
<property name="sessionFactory" ref="sessionFactory" />
</bean>
<bean id="myBatchService"
class="mypackage.MyBatchServiceImpl">
<property name="myBatchDao" ref="myBatchDao" />
</bean>
<bean id="transactionManager"
class="org.springframework.orm.hibernate5.HibernateTransactionManager">
<property name="sessionFactory" ref="sessionFactory" />
</bean>
<tx:annotation-driven transaction-manager="transactionManager" />
I am already using c3p0 for ConnectionPooling and also visited the following links:
link1
link2
But still I am getting following exception::
<org.hibernate.engine.jdbc.spi.SqlExceptionHelper> <BEA-000000> <IO Error: Got minus one from a read call, connect lapse 6370 ms., Authentication lapse 0 ms.>
org.springframework.transaction.CannotCreateTransactionException: Could not open Hibernate Session for transaction;nested exception is org.hibernate.exception.JDBCConnectionException: Unable to acquire JDBC Connection
at org.springframework.orm.hibernate5.HibernateTransactionManager.doBegin(HibernateTransactionManager.java:564)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:371)at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:335)

Camel creating too many tcp connections (over 15000) in TIME_WAIT status to connect ActiveMQ

I am using Apache Camel(with Spring) and ActiveMQ in project. Here are the settings related to JMS/ActiveMQ:
Camel version: activemq-camel-5.15.3.jar (all ActiveMQ related jars)
ActiveMQ version : 5.15.0
<!-- language: lang-xml -->
<bean id="defaultActiveMQRedeliveryPolicy" class="org.apache.activemq.RedeliveryPolicy">
</bean>
<util:list id="redeliveryPolicyEntries">
<bean id="activeMQRedeliveryPolicy1" class="org.apache.activemq.RedeliveryPolicy">
<property name="queue" value="inbox"></property>
</bean>
</util:list>
<bean id="amqRedeliveryPolicyMap"
class="org.apache.activemq.broker.region.policy.RedeliveryPolicyMap">
<property name="defaultEntry" ref="defaultActiveMQRedeliveryPolicy"></property>
<property name="redeliveryPolicyEntries" ref="redeliveryPolicyEntries"></property>
</bean>
<bean id="amqPrefetchPolicy" class="org.apache.activemq.ActiveMQPrefetchPolicy">
</bean>
<bean id="pooledConnectionFactory" class="org.apache.activemq.pool.PooledConnectionFactory" init-method="start" destroy-method="stop">
<property name="maxConnections" value="20" />
<property name="maximumActiveSessionPerConnection" value="40" />
<property name="connectionFactory" ref="jmsConnectionFactory">
</property>
</bean>
<bean id="jmsConnectionFactory" class="org.apache.activemq.ActiveMQConnectionFactory">
<property name="brokerURL" value="${jmsConnectionFactory.brokerURL}" />
<property name="userName" value="admin" />
<property name="password" value="admin" />
<property name="prefetchPolicy" ref="amqPrefetchPolicy" />
<property name="redeliveryPolicyMap" ref="amqRedeliveryPolicyMap" />
</bean>
<bean id="jmsConfig" class="org.apache.camel.component.jms.JmsConfiguration">
<property name="connectionFactory" ref="pooledConnectionFactory" />
<property name="concurrentConsumers" value="15" />
<property name="maxConcurrentConsumers" value="30" />
<property name="asyncConsumer" value="false" />
<property name="cacheLevelName" value="CACHE_CONSUMER" />
</bean>
<!-- this bean actually represents a jms component to be used in our camel-integration
setup.make endpoints by using name(id) of this bean. -->
<bean id="activemq" class="org.apache.activemq.camel.component.ActiveMQComponent">
<property name="configuration" ref="jmsConfig" />
<property name="transacted" value="false" />
<property name="transactionManager">
<bean class="org.springframework.jms.connection.JmsTransactionManager">
<property name="connectionFactory" ref="jmsConnectionFactory" />
</bean>
</property>
</bean>
As you see I am using PooledConnectionFactory so I am expecting a fixed no of connections to connect with ActiveMQ. But unexpectedly I see a large no of TCP connections being opened in TIME_WAIT even when my application is idle and no messages are being produced/consumed at time. I confirmed this situation with infra team that confirmed all the Operating System level configuration are fine.
Here I tried debugging the doReceiveAndExecute method in AbstractPollingMessageListenerContainer- sessionToUse is not null, consumerToUse is also not null and code flows through receiveMessage(line number 304).I could not find the problem in debug trace as attached in debug screenshots:
and
and my actual problem
Is it a problem with MessageListenerContainer or with ConnectionFactory?? Am I missing some configuration which would prevent this from happening or is this an existing issue? If so is there a workaround?

Just spotted in your configuration that you configured the jmsConnectionFactory (not the pooled factory) in your transaction manager. Not sure if this could raise the issue because the pooled factory is simply not used.
<property name="transactionManager">
<bean class="org.springframework.jms.connection.JmsTransactionManager">
<property name="connectionFactory" ref="jmsConnectionFactory" />
</bean>
</property>

Hazelcast-client configuration on AWS

My question regards hazelcast-client configuration.
From what I understand, when configured properly the client is supposed to fetch hazelcast server nodes IPs automatically from the AWS api , the problem is doesn't even try connect.
Here's some log I found.
Caused by: java.lang.IllegalStateException: Unable to connect to any address in the config! The following addresses were tried: []
at com.hazelcast.client.spi.impl.ClusterListenerSupport.connectToCluster(ClusterListenerSupport.java:178)
at com.hazelcast.client.spi.impl.ClientClusterServiceImpl.start(ClientClusterServiceImpl.java:189)
also here's client conifig
<bean id="hazelcastInstance" class="com.hazelcast.client.HazelcastClient" factory-method="newHazelcastClient">
<constructor-arg>
<bean class="com.hazelcast.client.config.ClientConfig">
<property name="groupConfig">
<bean class="com.hazelcast.config.GroupConfig">
<property name="name" value="dev"/>
</bean>
</property>
<property name="properties">
<props>
<prop key="hazelcast.icmp.enabled">true</prop>
</props>
</property>
<property name="networkConfig">
<bean class="com.hazelcast.client.config.ClientNetworkConfig">
<property name="awsConfig">
<bean class="com.hazelcast.client.config.ClientAwsConfig">
<property name="insideAws" value="true" />
<property name="enabled" value="${hazelcast.aws.enabled:false}" />
<property name="region" value="${hazelcast.aws.region:set-me}" />
<property name="accessKey"value="key" />
<property name="secretKey" value="secret"/>
<property name="hostHeader" value="ec2.amazonaws.com"/>
<property name="iamRole" value="${hazelcast.aws.iam.role:#{null}}"/>
<!-- <property name="securityGroupName" value="${hazelcast.aws.securityGroupName:#{null}}" /> -->
<property name="tagKey" value="${hazelcast.aws.tagKey:hazelcast-cluster}" />
<property name="tagValue" value="${hazelcast.aws.tagValue:#{null}}" />
<property name="connectionTimeoutSeconds" value="${hazelcast.aws.connectionTimeout:15}" />
</bean>
</property>
</bean>
</property>
</bean>
</constructor-arg>
</bean>
also tried to force server ip, that works. I need the client to discover server ips automatically. any clue ?

You need to use the discovery plugin mechanism if you want the client to discover clusters on AWS (or any other cloud). The old AWS discovery was for members only.
Please see https://github.com/hazelcast/hazelcast-aws

Spring4 with Hibernate4 fails in standalone process

I'm failling to use Spring4 togather with Hibernate4 in a standalone process (no container like tomcat, WAS, ...)
How can I use Hibernate4, Spring4 and Spring data repositories togather in a standalone process?
However I confiugre Spring I allways get the same exception:
Caused by: java.lang.NullPointerException
at org.hibernate.engine.transaction.internal.jta.JtaStatusHelper.getStatus(JtaStatusHelper.java:76)
at org.hibernate.engine.transaction.internal.jta.JtaStatusHelper.isActive(JtaStatusHelper.java:118)
at org.hibernate.engine.transaction.internal.jta.CMTTransaction.join(CMTTransaction.java:149)
When googling for this, I get pointet to some information about hibernate.transaction.jta.platform and the docu for Hibernate 4.3 is here http://docs.jboss.org/hibernate/orm/4.3/devguide/en-US/html_single/#services-JtaPlatform
But the only option I see for my case would be org.hibernate.engine.transaction.jta.platform.internal.NoJtaPlatform but this still leads to the same error.
Here is my Spring config:
<context:component-scan base-package="com.xx.yy" />
<jpa:repositories base-package="com.xx.zz.respositories"></jpa:repositories>
<bean name="dataSource" class="org.springframework.jdbc.datasource.DriverManagerDataSource">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://localhost:3306/culture" />
<property name="username" value="root" />
<property name="password" value="" />
</bean>
<tx:annotation-driven transaction-manager="transactionManager" />
<bean class="org.springframework.orm.jpa.JpaTransactionManager" id="transactionManager">
<property name="entityManagerFactory" ref="entityManagerFactory" />
<property name="jpaDialect">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaDialect" />
</property>
</bean>
<bean id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="jtaDataSource" ref="dataSource" />
<property name="packagesToScan" value="culture.matching.index.model" />
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="generateDdl" value="true" />
<property name="showSql" value="false" />
<property name="databasePlatform" value="org.hibernate.dialect.MySQLDialect" />
<property name="database" value="MYSQL" />
</bean>
</property>
<property name="jpaProperties">
<value>
hibernate.transaction.jta.platform=org.hibernate.engine.transaction.jta.platform.internal.NoJtaPlatform
</value>
</property>
</bean>

Answer by #geoand helped a lot: https://github.com/spring-projects/spring-boot/tree/master/spring-boot-samples/spring-boot-sample-simple
I therefore moved from XML to Java config

How to disable JMS spring listener in test evnvironment

I am using spring JMSTemplate to connect to a queue on a different weblogic domain.
I wired all the components like jndiTemplate, connectionfactory, destination etc in the Spring config.
Messages are handled by a simple implementation of MessageListener.
It all works fine. But when I try to deploy this application where the destination is not available, deployment fails. this is very important for me as we dont have JMS infrastructure in some of our environments.
I tried with lookupOnStartup=false on all the components and tried injecting them into DMLC based on a environment variable. But the issue is, the listener doesnt seem to be starting up. All the messages are left in the queue.
Any help is highly appreciated.
Spring-config:
<bean id="jndiTemplate" class="org.springframework.jndi.JndiTemplate">
<property name="environment">
<props>
<prop key="java.naming.factory.initial">weblogic.jndi.WLInitialContextFactory</prop>
<prop key="java.naming.provider.url">t3://wp2128.xyz.int:7001</prop>
<prop key="java.naming.security.authentication">none</prop>
<prop key="java.naming.security.principal"></prop>
</props>
</property>
</bean>
<bean id="connectionFactory" class="org.springframework.jndi.JndiObjectFactoryBean" >
<property name="jndiTemplate" ref="jndiTemplate" />
<property name="jndiName" value="jms/connectionFactory" />
</bean>
<bean id="destination" class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiTemplate" ref="jndiTemplate" />
<property name="jndiName" value="jms/testQueue" />
</bean>
<bean id="simpleConsumer"
class="org.springframework.jms.listener.DefaultMessageListenerContainer">
<property name="concurrentConsumers" value="5" />
<property name="connectionFactory" ref="connectionFactory" />
<property name="destination" ref="destination" />
<property name="messageListener" ref="simpleMessageListener" />
</bean>

You can set autoStartup to false. This property value can be externalized in a properties files.
<bean id="simpleConsumer"
class="org.springframework.jms.listener.DefaultMessageListenerContainer">
<property name="concurrentConsumers" value="5" />
<property name="connectionFactory" ref="connectionFactory" />
<property name="destination" ref="destination" />
<property name="messageListener" ref="simpleMessageListener" />
<property name="autoStartup" value="false"/>
</bean>