Spring Batch execution time with In-Memory Metadata - java

I am new to Spring batch and I am trying to use Spring Batch with restartable feature. I am using MapJobRepositoryFactoryBean as Job repository. Everything looks fine. But when I run the same job multiple times I could see the execution time increasing considerably. I guess some memory leak is happening.If no job is running, I am cleaning the repository as well. But no luck. How do I know whats happening exactly. After running the same job for 4-5 times, the execution time is going to 2-3 times of the first execution.
<jpa:repositories base-package=".reference.data.repository"/>
<context:annotation-config />
<tx:annotation-driven transaction-manager="transactionManager" />
<bean id="validator"
class="org.springframework.validation.beanvalidation.LocalValidatorFactoryBean" />
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource">
<property name="driverClassName" value="com.microsoft.sqlserver.jdbc.SQLServerDriver" />
<property name="url"
value="jdbc:sqlserver://${reference-data-manager.database.hostname}:
${reference-data-manager.database.port};
database=${reference-data-manager.database.name};
user=${reference-data-manager.database.username};
password=${reference-data-manager.database.password}" />
</bean>
<bean id="simpleJobConfiguration" class="reference.job.SimpleJobConfiguration">
</bean>
<bean id="emf"
class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="dataSource" />
</bean>
<bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="dataSource" />
</bean>
<bean class="org.springframework.orm.jpa.support.PersistenceAnnotationBeanPostProcessor" />
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
<property name="entityManagerFactory" ref="emf" />
</bean>
<bean id="importJob" class="org.springframework.batch.core.job.SimpleJob" scope="prototype">
<property name="jobRepository" ref="jobRepository"></property>
</bean>
<batch:step id="importCodesStep">
<batch:tasklet allow-start-if-complete="true">
<batch:chunk reader="codeMappingReader" writer="codeMappingWriter"
processor="codeMappingProcessor" commit-interval="${reference-data-manager.batch.size}"
skip-policy="reasonRemarkAssnSkipPolicy" skip-limit="${reference-data-manager.skip.limit}">
<batch:skippable-exception-classes>
<batch:include class="org.springframework.batch.item.ParseException"/>
</batch:skippable-exception-classes>
</batch:chunk>
</batch:tasklet>
<batch:listeners>
<batch:listener ref="reasonRemarkAssnStepListener"/>
<batch:listener ref="reasonRemarkAssnSkipListener"/>
</batch:listeners>
</batch:step>
<bean id="reasonRemarkAssnStepListener" class="reference.listeners.ReasonRemarkAssnStepListener">
</bean>
<bean id="reasonRemarkAssnSkipListener" class="reference.listeners.ReasonRemarkAssnSkipListener">
</bean>
<bean id="reasonRemarkAssnSkipPolicy" class="reference.listeners.ReasonRemarkAssnSkipPolicy">
<!-- <property name="skipLimit" value="5"/> -->
</bean>
<bean id="codeMappingWriter" class="reference.writer.ReasonRemarkAssnWriter" scope="step">
<property name="entityManagerFactory" ref="emf" />
</bean>
<bean id="codeMappingProcessor" class="reference.processors.ReasonRemarkAssnProcessor" scope="step">
<property name="userId" value="#{jobParameters['USER_ID']}" />
<property name="clientMnemonic" value="#{jobParameters['CLIENT_MENOMONIC']}" />
</bean>
<bean id="codeMappingReader" class="reference.readers.ReasonRemarkAssnReader" scope="step">
<property name="org" value="#{jobParameters['ORG']}"/>
<property name="resource" value="" />
<property name="linesToSkip" value="1" />
<property name="lineMapper">
<bean class="reference.mapper.ReasonRemarkAassnLineMapper">
<property name="lineTokenizer">
<bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="names" value="Reason Code,Remark Code,Category,Category Priority,Ignore Insight Processing,Active,Comment" />
</bean>
</property>
<property name="fieldSetMapper" ref="reasonRemarkAssnMapper"/>
</bean>
</property>
</bean>
<bean id="reasonRemarkAssnMapper" class="reference.mapper.ReasonRemarkAssnMapper">
<property name="codeGroups" value="${reference-data-manager.code.groups}"></property>
</bean>
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
</bean>
<bean id="jobLauncher"
class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository" />
</bean>
<bean class="CryptoPropertyPlaceholderConfigurer">
<property name="configUtil" ref="configUtil" />
<property name="location"
value="file:config/reference-data-manager.conf" />
<property name="ignoreResourceNotFound" value="true" />
<property name="ignoreUnresolvablePlaceholders" value="true" />
</bean>
<bean name="configUtil" class="CoreConfigUtil">
<property name="crypto" ref="crypto" />
</bean>
<bean id="referenceDataManager" class="reference.service.impl.ReferenceDataManagerImpl">
<property name="step" ref="importCodesStep"></property>
</bean>
Here is my job invocation...
#RestfulServiceAddress("/reference")
public class ReferenceDataManagerImpl implements ReferenceDataManager {
#Autowired
private JobLauncher jobLauncher;
#Autowired
IDataAccessService dataAccessService;
#Autowired
private IConfigurationServiceFactory configurationServiceFactory;
#Autowired
private SimpleJob job;
#Autowired
private TaskletStep step;
#Autowired
private SimpleJobConfiguration jobConfiguration;
#Autowired
private MapJobRepositoryFactoryBean jobRepository;
#Override
public Response importData(MultipartBody input, String org,
String userId, MessageContext mc) throws ServiceFault {
Response.ResponseBuilder builder = null;
ReasonRemarkAssnResponse responseObj = null;
boolean isSuccess = false;
UserInfo userInfo= dataAccessService.userInfoFindByUserId(userId);
JobParametersBuilder jobParametersBuilder = new JobParametersBuilder();
List<Attachment> attachments = input.getAllAttachments();
DataHandler dataHandler = attachments.get(0).getDataHandler();
byte[] bFile = null;
if(null != dataHandler){
try {
InputStream is = dataHandler.getInputStream();
bFile = new byte[is.available()];
is.read(bFile);
is.close();
} catch (IOException e) {
//TODO
}
}
SimpleJobConfiguration.customStorage.put(org, bFile);
jobParametersBuilder.addLong(ReferenceConstants.JOB_PARAMS_USERID, userInfo.getId());
jobParametersBuilder.addString(ReferenceConstants.JOB_PARAMS_ORG, org);
jobParametersBuilder.addString(ReferenceConstants.JOB_PARAMS_TIME, Calendar.getInstance().getTime().toString());
String response = "{error:Error occured while importing the file}";
job.setName(ReferenceConstants.JOB_PARAMS_JOB_NAME_PREFIX+org.toUpperCase());
job.addStep(step);
job.setRestartable(true);
JobExecution execution = null;
try {
execution = jobLauncher.run(job, jobParametersBuilder.toJobParameters());
isSuccess = true;
} catch (JobExecutionAlreadyRunningException e) {
//TODO
}catch (JobRestartException e) {
//TODO
}catch (JobInstanceAlreadyCompleteException e) {
//TODO
}catch (JobParametersInvalidException e) {
//TODO
}
response = prepareResponse(responseObj);
synchronized (SimpleJobConfiguration.customStorage){
if(null != SimpleJobConfiguration.customStorage.get(org)){
SimpleJobConfiguration.customStorage.remove(org);
}
if(SimpleJobConfiguration.customStorage.isEmpty()){
jobRepository.clear();
}
}
builder = Response.status(Response.Status.OK);
builder.type(MediaType.APPLICATION_JSON);
builder.entity(response);
return builder.build();
}
}

Don't use the MapJobRepositoryFactoryBean unless it's for testing. That's all it's intended for.
You shouldn't be building a new Job instance with every call to the controller. If you have stateful components within a step, declare them as step scoped so that you get a new instance per step execution.
The Map based JobRepository keeps everything in memory. That's the idea. So as you execute more and more jobs, the repository is going to grow...eating up your space in memory.
For launching a job in a controller, give something like this a try:
#RestController
public class JobLaunchingController {
#Autowired
private JobLauncher jobLauncher;
#Autowired
private Job job;
#RequestMapping(value = "/", method = RequestMethod.POST)
#ResponseStatus(HttpStatus.ACCEPTED)
public void launch(#RequestParam("name") String name) throws Exception {
JobParameters jobParameters =
new JobParametersBuilder()
.addString("name", name)
.toJobParameters();
this.jobLauncher.run(job, jobParameters);
}
}

Related

Shiro gets different sessions for every getSession()

I want to do the session attributes validation recently.I made an interceptor that when requests come it will get the shiro session to set some attributes in the session.But I found that I always got the different shiro sessions when use the getSession() for the same user who has logged in the system.
Here is the shiro configuration xml
<?xml version="1.0" encoding="UTF-8"?>
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd"
default-lazy-init="true">
<description>Shiro Configuration</description>
<bean id="sessionIdGenerator" class="org.apache.shiro.session.mgt.eis.JavaUuidSessionIdGenerator"/>
<bean id="sessionDAO" class="org.apache.shiro.session.mgt.eis.EnterpriseCacheSessionDAO">
<property name="activeSessionsCacheName" value="shiro-activeSessionCache"/>
<property name="sessionIdGenerator" ref="sessionIdGenerator"/>
</bean>
<bean id="sessionManager" class="org.apache.shiro.session.mgt.DefaultSessionManager">
<property name="globalSessionTimeout" value="3600000"/>
<property name="deleteInvalidSessions" value="true"/>
<property name="sessionValidationSchedulerEnabled" value="true"/>
<property name="sessionValidationScheduler" ref="sessionValidationScheduler"/>
<property name="sessionDAO" ref="sessionDAO"/>
</bean>
<!--<bean id="sessionValidationScheduler" class="org.apache.shiro.session.mgt.quartz.ExecutorServiceSessionValidationScheduler ">
<property name="interval" value="60000"/>
<property name="sessionManager" ref="sessionManager"/>
</bean>-->
<bean id="sessionValidationScheduler" class="org.apache.shiro.session.mgt.ExecutorServiceSessionValidationScheduler">
<constructor-arg name="sessionManager" ref="sessionManager"/>
<property name="interval" value="3600000"/>
</bean>
<bean id="securityManager" class="org.apache.shiro.web.mgt.DefaultWebSecurityManager">
<property name="realm" ref="shiroDbRealm"/>
<property name="cacheManager" ref="cacheManager"/>
<property name="sessionManager" ref="sessionManager"/>
</bean>
<bean id="shiroDbRealm" class="com.aspire.cms.service.impl.ShiroDbRealm">
</bean>
<bean id="systemLogoutFilter" class="com.aspire.cms.filter.SystemLogoutFilter"></bean>
<!-- Shiro Filter -->
<bean id="shiroFilter" class="org.apache.shiro.spring.web.ShiroFilterFactoryBean">
<property name="securityManager" ref="securityManager"/>
<property name="loginUrl" value="/login"/>
<property name="successUrl" value="/main"/>
<property name="unauthorizedUrl" value="/login"/>
<property name="filters">
<map>
<entry key="logout" value-ref="systemLogoutFilter" />
</map>
</property>
<property name="filterChainDefinitions">
<value>
/login = authc
/ajaxLogin = anon
/logout = logout
/static/** = anon
/** = user
</value>
</property>
</bean>
<bean id="cacheManager" class="org.apache.shiro.cache.MemoryConstrainedCacheManager"/>
<bean id="lifecycleBeanPostProcessor" class="org.apache.shiro.spring.LifecycleBeanPostProcessor"/>
<bean class="org.springframework.aop.framework.autoproxy.DefaultAdvisorAutoProxyCreator"
depends-on="lifecycleBeanPostProcessor">
<property name="proxyTargetClass" value="true"/>
</bean>
<bean class="org.apache.shiro.spring.security.interceptor.AuthorizationAttributeSourceAdvisor">
<property name="securityManager" ref="securityManager"/>
</bean>
<bean class="org.springframework.beans.factory.config.MethodInvokingFactoryBean">
<property name="staticMethod" value="org.apache.shiro.SecurityUtils.setSecurityManager"/>
<property name="arguments" ref="securityManager"/>
</bean>
Here is the code to get shiro session
public class CsrfInterceptor extends HandlerInterceptorAdapter {
private static final Logger logger = LogManager.getLogger(CsrfInterceptor.class);
#Override
public boolean preHandle(HttpServletRequest request,
HttpServletResponse response, Object handler) throws Exception {
Session session = SecurityUtils.getSubject().getSession();
System.out.println("==========sessionId:"+session.getId());
String sessionCsrfToken = (String) session.getAttribute(CsrfTokenManager.CSRF_TOKEN_FOR_SESSION_ATTR_NAME);
if ("POST".equalsIgnoreCase(request.getMethod())) {
String CsrfToken = CsrfTokenManager.getTokenFromRequest(request);
if (CsrfToken == null || !CsrfToken.equals(sessionCsrfToken)) {
String reLoginUrl = "/login?backurl="
+ URLEncoder.encode(getCurrentUrl(request), "utf-8");
response.sendRedirect(reLoginUrl);
return false;
}
}
return true;
}

spring Quartz scheduler not accessing database

here is my 'GenericQuartzJob' class
public class GenericQuartzJob extends QuartzJobBean
{
private String batchProcessorName;
public String getBatchProcessorName() {
return batchProcessorName;
}
public void setBatchProcessorName(String name) {
// System.out.println("jo"+name);
this.batchProcessorName = name;
}
protected void executeInternal(JobExecutionContext jobCtx) throws JobExecutionException
{
try {
// System.out.println("jo");
SchedulerContext schedCtx = jobCtx.getScheduler().getContext();
ApplicationContext appCtx =
(ApplicationContext) schedCtx.get("applicationContext");
java.lang.Runnable proc = (java.lang.Runnable) appCtx.getBean(batchProcessorName);
proc.run();
}
catch (Exception ex) {
// ex.printStackTrace();
throw new JobExecutionException("Unable to execute batch job: " + batchProcessorName, ex);
}
}
}
Here is my MyRunnableJob Class
#Configuration
#ComponentScan("com.ehydromet.service")
public class MyRunnableJob implements Runnable {
//#Resource private SpringService myService;
#Autowired
public UserService service;
#Override
public void run() {
// System.out.println("hu");
try{
service.getAllAdminUser();
}catch(Exception ex){
System.out.println(ex.getMessage()+" MyRunnableJob");
}MqttImplement mqtt=new MqttImplement();
mqtt.publishMessage();
//Run the Job. This code will run in the Spring context so you can use injected dependencies here.
}
}
Below is my MqttImplement class
public class MqttImplement implements MqttCallback {
#Autowired
private static UserService service;
#RequestMapping("/publish")
public void publishData(ModelMap map){
MyTask.map=map;
pubTopic="tmsdata";
susTopic="tmsack";
if(initializeMqTTParameters() ){
if(suscribeForAck()){
// new ClassPathXmlApplicationContext("spring-quartz.xml");
AbstractApplicationContext context = new AnnotationConfigApplicationContext(AppConfig.class);
}
}
// initializeMqTTParameters();
// suscribeForAck();
// publishMessage();
}
public void publishMessage(){
try{
// System.out.print("sending ");
Received received=service.getReceivedDataToPublishTMS();
if(received!=null){
System.out.print("sending you not null");
String publisgMSG=createMessageToPublish(received);
pubMessage="lava," ;
publishMessageTms();
}else{
System.out.println("null hora");
}
}catch(Exception ex){
System.out.println("hora"+ex.getLocalizedMessage()+" "+ex.toString());
}
}
}
and here is xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd">
<!-- <bean id="exampleBusinessObject" class="com.ehydromet"/> -->
<!-- Old JobDetail Definition
<bean id="jobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
<property name="targetObject" ref="exampleBusinessObject"/>
<property name="targetMethod" value="doIt"/>
</bean>
-->
<!-- Runnable Job: this will be used to call my actual job. Must implement runnable -->
<bean name="myRunnableJob" class="com.ehydromet.controller.MyRunnableJob"></bean>
<!-- New Clusterable Definition -->
<bean id="jobDetail" class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="jobClass" value="com.ehydromet.schedule.GenericQuartzJob" />
<property name="jobDataAsMap">
<map>
<entry key="batchProcessorName" value="myRunnableJob" />
</map>
</property>
</bean>
<bean id="simpleTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
<!-- see the example of method invoking job above -->
<property name="jobDetail" ref="jobDetail"/>
<!-- 10 seconds -->
<property name="startDelay" value="5000"/>
<!-- repeat every 50 seconds -->
<property name="repeatInterval" value="5000"/>
</bean>
<context:component-scan base-package="com.ehydromet.service"/>
<context:component-scan base-package="com.ehydromet.dao"/>
<context:component-scan base-package="com.ehydromet.service.impl"/>
<context:component-scan base-package="com.ehydromet.dao.impl"/>
<bean id="sessionFactory"
class="org.springframework.orm.hibernate3.annotation.AnnotationSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="annotatedClasses">
<list>
<value>com.ehydromet.entity.Alarms</value>
<value>com.ehydromet.entity.UserData</value>
<value>com.ehydromet.entity.ModemInfo</value>
<value>com.ehydromet.entity.Received</value>
<value>com.ehydromet.entity.Modems</value>
<value>com.ehydromet.entity.AdminLogin</value>
<value>com.ehydromet.entity.UserPolicy</value>
</list>
</property>
<property name="hibernateProperties">
<props>
<prop key="hibernate.dialect">org.hibernate.dialect.MySQLDialect</prop>
<prop key="hibernate.hbm2ddl.auto">validate</prop>
</props>
</property>
</bean>
<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="sessionFactory">
<ref bean="sessionFactory"/>
</property>
<property name="triggers">
<list>
<ref bean="simpleTrigger" />
</list>
</property>
<!-- Add this to make the Spring Context Available -->
<property name="applicationContextSchedulerContextKey"><value>applicationContext</value></property>
</bean>
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://localhost:3306/ejava" />
<property name="username" value="root" />
<property name="password" value="" />
</bean>
<bean id="transactionManager"
class="org.springframework.orm.hibernate3.HibernateTransactionManager">
<property name="sessionFactory" ref="sessionFactory" />
</bean>
<bean id="hibernateInterceptor" class="org.springframework.orm.hibernate3.HibernateInterceptor">
<property name="sessionFactory">
<ref bean="sessionFactory"/>
</property>
</bean>
</beans>
Dao classes are #Transactional and #Repository.
in MqttImplement class service object is null i think autowired is not working with Quartz scheduler. Any suggestion????
i think autowired is kind of resolved. but
error is-- org.springframework.beans.NotWritablePropertyException: Invalid property 'sessionFactory' of bean class [org.springframework.scheduling.quartz.SchedulerFactoryBean]: Bean property 'sessionFactory' is not writable or has an invalid setter method. Does the parameter type of the setter match the return type of the getter?
i am new to it. so may be some foolish mistakes i have done, please have a look.
Have you tried #Autowired without static? Seems weird for me...
From other side you do this:
MqttImplement mqttPublishTms=new MqttImplement();
................
mqttPublishTms.publishMessage();
So, it's new not Dependency Injection. There is no surprise that the service property there is null.

JdbcTemplate not work when use #Transactional annotation

When use spring data and jdbctemplate in same function
spring data work fine but jdbc update not work .
this is service class.
jdbc update then not error occurred but database not updated
#Service
public class SampleService {
private static final Logger logger = LoggerFactory.getLogger(SampleService.class);
#Inject
private SampleRepository sampleRepository;
#Inject
private JpaTransactionManager transactionManager;
private JdbcTemplate jdbcTemplate() {
return new JdbcTemplate(transactionManager.getDataSource());
}
#Transactional(propagation=Propagation.REQUIRED)
public void execute(){
sampleRepository.deleteAll();
sampleRepository.save(new Sample("sample"));
logger.info("data count (after jpa insert) : {}", sampleRepository.count());//1
int count = jdbcTemplate().update("insert into Sample (sample) values ( ? )", "sample");
logger.info("jdbc insert count: {}", count);//1
logger.info("data count (after jdbc insert) : {}", sampleRepository.count());//expected: 2 , actual:1
count = jdbcTemplate().queryForObject("select count(id) from Sample", Integer.class);
logger.info("data count (after jdbc insert) : {}", count); //jdbc count query result also return 1
}
when not use #Transactional annotation then work fine.
public void execute2(){
DefaultTransactionDefinition def = new DefaultTransactionDefinition();
def.setName("testTx");
def.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
TransactionStatus status = transactionManager.getTransaction(def);
... this method work fine ...
transactionManager.commit(status);
}
}
and this is data access configuration.
<context:annotation-config />
<tx:jta-transaction-manager />
<tx:annotation-driven transaction-manager="transactionManager" />
<jpa:repositories base-package="example" />
<context:property-placeholder location="classpath:config.properties" />
<bean class="org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor" />
<bean id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="persistenceUnitName" value="DefaultUnit" />
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="showSql" value="${db.showsql}" />
<property name="generateDdl" value="${db.generateDDL}" />
<property name="database" value="${db.database}" /><!-- H2 -->
</bean>
</property>
</bean>
<bean id="dataSource" class="org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy">
<constructor-arg ref="basicDataSource" />
</bean>
<bean id="basicDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<property name="driverClassName" value="${db.driver}" />
<property name="url" value="${db.url}" />
<property name="username" value="${db.user}" />
<property name="password" value="${db.pass}" />
<property name="defaultAutoCommit" value="false" />
</bean>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
<property name="entityManagerFactory" ref="entityManagerFactory" />
</bean>

Setting up Tomcat JDBC connection pool with the Spring JDBCTemplate

I'm trying to setup a Tomcat connection pool (to MySQL) in my Java web app, while using the Spring JDBCTemplate.
This is the Java class to create the connection pool:
#Configuration
public class DataAccessConfiguration {
#Bean(destroyMethod = "close")
public javax.sql.DataSource datasource() {
org.apache.tomcat.jdbc.pool.DataSource ds = new org.apache.tomcat.jdbc.pool.DataSource();
ds.setDriverClassName("org.h2.Driver");
ds.setUrl("jdbc:h2:java-config");
ds.setUsername("sa");
ds.setPassword("");
ds.setInitialSize(5);
ds.setMaxActive(10);
ds.setMaxIdle(5);
ds.setMinIdle(2);
return ds;
}
#Bean public JdbcOperations tpl() {
return new JdbcTemplate(datasource());
}
}
This is how I get the ApplicationContext (in the main method for example):
public static void main(String[] args) {
ApplicationContext context =
new ClassPathXmlApplicationContext("Beans.xml");
How should I define the DataAccessConfiguration class in the Beans.xml file so Spring knows to use it?
**
Update:
**
This is the actual configuration method:
public javax.sql.DataSource datasource() {
org.apache.tomcat.jdbc.pool.DataSource ds = new org.apache.tomcat.jdbc.pool.DataSource();
PoolProperties p = new PoolProperties();
p.setUrl("jdbc:mysql://localhost:3306/mysql");
p.setDriverClassName("com.mysql.jdbc.Driver");
p.setUsername("root");
p.setPassword("");
p.setJmxEnabled(true);
p.setTestWhileIdle(false);
p.setTestOnBorrow(true);
p.setValidationQuery("SELECT 1");
p.setTestOnReturn(false);
p.setValidationInterval(30000);
p.setTimeBetweenEvictionRunsMillis(30000);
p.setMaxActive(100);
p.setInitialSize(10);
p.setMaxWait(10000);
p.setRemoveAbandonedTimeout(60);
p.setMinEvictableIdleTimeMillis(30000);
p.setMinIdle(10);
p.setLogAbandoned(true);
p.setRemoveAbandoned(true);
p.setJdbcInterceptors(
"org.apache.tomcat.jdbc.pool.interceptor.ConnectionState;"+
"org.apache.tomcat.jdbc.pool.interceptor.StatementFinalizer");
ds.setPoolProperties(p);
return ds;
}
Can you please help rewrite in the Beans.xml?
For the values passed in Properties setter methods exist. Therefore XML-based bean definition goes like:
<bean name="dataSource" class="org.apache.tomcat.jdbc.pool.DataSource">
<property name="url" value="jdbc:mysql://localhost:3306/mysql"/>
<property name="driverClassName" value="com.mysql.jdbc.Driver"/>
....
</bean>
<bean name="tpl" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="dataSource"/>
</bean>
<bean id="dataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource"
destroy-method="close">
<property name="jdbcUrl" value="#{config['db.url']}" />
<property name="driverClass" value="#{config['db.driver']}" />
<property name="user" value="#{config['db.username']}" />
<property name="password" value="#{config['db.password']}" />
<property name="acquireIncrement" value="1" />
<property name="idleConnectionTestPeriod" value="300" />
<property name="minPoolSize" value="1" />
<property name="maxPoolSize" value="20" />
</bean>
<bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="dataSource" />
</bean>

Activemq - Exceeded the maximum number of allowed client connections

My acitvemq server always print error below :
2014-07-12 16:14:27,820 | ERROR | Could not accept connection :
org.apache.activemq.transport.tcp.ExceededMaximumConnectionsException:
Exceeded the maximum number of allowed client connections.
See the 'maximumConnections' property on the TCP transport configuration URI
in the ActiveMQ configuration file (e.g., activemq.xml)
| org.apache.activemq.broker.TransportConnector
| ActiveMQ Transport Server Thread Handler:
tcp://0.0.0.0:61616?maximumConnections=1000&wireformat.maxFrameSize=104857600
When I restart the server it will be ok. But after a few days the error come out again.
I don't why the connections always increase to 1000.
My server config:
<!-- activeMQ -->
<bean id="jmsConnectionFactory" class="org.apache.activemq.ActiveMQConnectionFactory">
<property name="brokerURL" value="${jms.brokerURL}"></property>
</bean>
<!-- Spring Caching -->
<bean id="cachingConnectionFactory"
class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="jmsConnectionFactory" />
<property name="sessionCacheSize" value="10" />
</bean>
<!-- Spring JMS Template -->
<bean id="jmsTemplate" class="org.springframework.jms.core.JmsTemplate">
<property name="connectionFactory" ref="cachingConnectionFactory" />
<property name="explicitQosEnabled" value="true" />
<property name="priority" value="4" />
</bean>
<bean id="scoreQueue" class="org.apache.activemq.command.ActiveMQQueue">
<constructor-arg value="SCORE" />
</bean>
<bean id="scoreMessage" class="com.tt.score.mq.server.ScoreMessage"></bean>
<bean id="scoreListener"
class="org.springframework.jms.listener.DefaultMessageListenerContainer">
<property name="connectionFactory" ref="jmsConnectionFactory"></property>
<property name="destination" ref="scoreQueue"></property>
<property name="messageListener" ref="scoreMessage"></property>
<property name="concurrentConsumers" value="10" />
<property name="maxConcurrentConsumers" value="100" />
<property name="sessionAcknowledgeModeName" value="CLIENT_ACKNOWLEDGE" />
</bean>
My client config xml:
<!-- Spring Caching -->
<bean id="cachingConnectionFactory"
class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="jmsConnectionFactory" />
<property name="sessionCacheSize" value="10" />
</bean>
<!-- Spring JMS Template -->
<bean id="jmsTemplate" class="org.springframework.jms.core.JmsTemplate">
<property name="connectionFactory" ref="cachingConnectionFactory" />
<property name="explicitQosEnabled" value="true" />
<property name="priority" value="4" />
</bean>
<bean id="messageProducer" class="com.tt.score.mq.client.MessageProducer">
<property name="jmsTemplate" ref="jmsTemplate" />
<property name="scoreQueue" ref="scoreQueue" />
</bean>
<bean id="scoreQueue" class="org.apache.activemq.command.ActiveMQQueue">
<constructor-arg value="SCORE" />
</bean>
Other info:
acitvemq server : 5.8.0
client acitvemq : 5.4.2
spring : 3.0.7
spring-jms : 3.0.7
We use transactionManager so the DefaultMessageListenerContainer's cachelevel will be set to none.
---update add dao config----------------------------------
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close">
<property name="driverClassName"><value>${jdbc.driverClass}</value></property>
<property name="username"><value>${jdbc.user}</value></property>
<property name="url"><value>${jdbc.jdbcUrl}</value></property>
<property name="password">
<bean class="com.tongbanjie.commons.util.EncryptDBPasswordFactory">
<property name="password" value="${jdbc.password}" />
</bean>
</property>
<property name="maxActive"><value>${jdbc.maxActive}</value></property>
<property name="initialSize"><value>${jdbc.initialSize}</value></property>
<property name="maxWait"><value>60000</value></property>
<property name="maxIdle"><value>${jdbc.maxIdle}</value></property>
<property name="minIdle"><value>5</value></property>
<property name="removeAbandoned"><value>true</value></property>
<property name="removeAbandonedTimeout"><value>180</value></property>
<property name="timeBetweenEvictionRunsMillis"><value>60000</value></property>
<property name="minEvictableIdleTimeMillis"><value>1800000</value></property>
<property name="defaultAutoCommit" value="false" />
<property name="connectionProperties">
<value>bigStringTryClob=true;clientEncoding=UTF-8;defaultRowPrefetch=50;serverEncoding=ISO-8859-1</value>
</property>
</bean>
<bean class="org.springframework.aop.framework.autoproxy.DefaultAdvisorAutoProxyCreator" />
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource" />
</bean>
<tx:annotation-driven transaction-manager="transactionManager" />
<bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource">
<ref bean="dataSource"/>
</property>
</bean>
<!-- myBatis -->
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="configLocation" value="classpath:META-INF/mybatis/score-configuration.xml" />
<property name="mapperLocations" value="classpath*:META-INF/mybatis/mapper/*.xml" />
<property name="dataSource" ref="dataSource" />
</bean>
<bean id="commonSqlSessionDao" abstract="true">
<property name="sqlSessionFactory">
<ref bean="sqlSessionFactory" />
</property>
</bean>
-----post the code that we how to use the template now
the jmsTemplate wrapped in a class
public class MessageProducer {
private JmsTemplate jmsTemplate;
private ActiveMQQueue scoreQueue;
public void sendScoreQueue(Map<String, String> userMap) {
sendMessage(this.scoreQueue, userMap);
}
private void sendMessage(Destination destination, final Map<String, String> map) {
this.jmsTemplate.send(destination, new MessageCreator() {
public Message createMessage(Session session) throws JMSException {
MapMessage message = session.createMapMessage();
for (String key : map.keySet()) {
message.setStringProperty(key, (String) map.get(key));
}
return message;
}
});
}
}
And we use a thead to send call the MessageProducer class's sendScoreQueue method.
As follows:
//the code is old and ugly.that is the original position we call the mq.
ThreadUtils.execute(new Thread(new SendMsgThread(dycc, ScoreMQSendType.SEND_TYPE_SCORE)));
///
public class ThreadUtils {
protected static ThreadPoolExecutor executor = null;
public static Properties Props = null;
public static void execute(Thread thread) {
executor.execute(thread);
}
static {
if (executor == null)
Integer corePoolSize = 5;
Integer maximumPoolSize = 10;
Integer keepAliveTime = 3000;
executor = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, TimeUnit.MINUTES,
new LinkedBlockingQueue());
}
}
public class SendMsgThread implements Runnable {
private Log log = LogFactory.getLog(SendMsgThread.class);
private Map<String, String> map;
private String type;
private static MessageProducer producer = null;
public SendMsgThread(Map<String, String> map, String type){
this.type = type;
this.map = map;
}
public void run() {
try {
if(type.equals(ScoreMQSendType.SEND_TYPE_SCORE) || type.equals(ScoreMQSendType.SEND_TYPE_REGISTER)) {
producer.sendMessage(map);
}
} catch (Exception e) {
this.log.error("sendMsgThread sendScoreQueue error.", e);
}
}
static {
if (producer == null) producer =
(MessageProducer )SpringContextHolder.getBean(MessageProducer .class);
}
}
For this scenario you should use PooledConnectionFactory instead of cachingConnectionFactory.
More information can be found here.The difference between them can be found here
I had the same problem recurring with same error, and the solution was obtained as a result of trial and error, use jms call to concurrent consumers max to 101, instead of 100, and see if results repeat, adding more to the value results the code executed further on debugger, you will reach a value when code works , Also the solution appears to be using a connection pool factory.
Try this, hope it works, rest my implementation is same as yours in bean file

Categories

Resources