I have multiple CSV files to read. I want the processing to be done one file at a time. Rather than reading all the records till it reaches commit level.
I have put together a job which uses partition but on running the job I see that there are two entries for every row. As if the job is running twice.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:context="http://www.springframework.org/schema/context" xmlns:p="http://www.springframework.org/schema/p"
xmlns:batch="http://www.springframework.org/schema/batch" xmlns:mvc="http://www.springframework.org/schema/mvc"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-4.0.xsd
http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-4.0.xsd
http://www.springframework.org/schema/batch
http://www.springframework.org/schema/batch/spring-batch-2.2.xsd">
<import resource="classpath:/database.xml" />
<bean id="asyncTaskExecutor" class="org.springframework.core.task.SimpleAsyncTaskExecutor" >
<property name="concurrencyLimit" value="1"></property>
</bean>
<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="5" />
</bean>
<bean id="partitioner" class="org.springframework.batch.core.partition.support.MultiResourcePartitioner" scope="step">
<property name="resources" value="file:#{jobParameters[filePath]}/*.dat" />
</bean>
<bean id="multiResourceReader"
class="org.springframework.batch.item.file.MultiResourceItemReader"
scope="step">
<property name="resources" value="file:#{jobParameters[filePath]}/*.dat"></property>
<property name="delegate" ref="logItFileReader"></property>
</bean>
<batch:job id="remediationJob">
<batch:step id="partitionedStep" >
<batch:partition step="readWriteContactsPartitionedStep" partitioner="partitioner">
<batch:handler task-executor="asyncTaskExecutor" />
</batch:partition>
</batch:step>
</batch:job>
<batch:step id="readWriteContactsPartitionedStep">
<batch:tasklet>
<batch:transaction-attributes isolation="READ_UNCOMMITTED"/>
<batch:chunk reader="multiResourceReader" writer="rawItemDatabaseWriter" commit-interval="10" skip-policy="pdwUploadSkipPolicy"/>
<batch:listeners>
<batch:listener ref="customItemReaderListener"></batch:listener>
<batch:listener ref="csvLineSkipListener"></batch:listener>
<batch:listener ref="getCurrentResourceChunkListener"></batch:listener>
</batch:listeners>
</batch:tasklet>
</batch:step>
<bean id="logItFileReader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step">
<!-- Read a csv file -->
<property name="strict" value="false"></property>
<property name="lineMapper">
<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
<!-- split it -->
<property name="lineTokenizer">
<bean
class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
<property name="delimiter" value="####" />
<property name="strict" value="true" />
</bean>
</property>
<property name="fieldSetMapper">
<!-- map to an object -->
<bean class="org.kp.oppr.remediation.batch.vo.CSVDataVOFieldMapper">
</bean>
</property>
</bean>
</property>
</bean>
<bean id="rawItemDatabaseWriter" class="org.kp.oppr.remediation.batch.csv.RawItemDatabaseWriter"
scope="step">
</bean>
<bean id="pdwUploadSkipPolicy"
class="org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy" />
<bean id="csvDataVO" class="org.kp.oppr.remediation.batch.vo.CSVDataVO"
scope="prototype"></bean>
<!-- BATCH LISTENERS -->
<bean id="pdwFileMoverListener"
class="org.kp.oppr.remediation.batch.listener.PdwFileMoverListener"
scope="step">
</bean>
<bean id="csvLineSkipListener"
class="org.kp.oppr.remediation.batch.listener.CSVLineSkipListener"
scope="step">
</bean>
<bean id="customItemReaderListener"
class="org.kp.oppr.remediation.batch.listener.CustomItemReaderListener"></bean>
<bean id="getCurrentResourceChunkListener"
class="org.kp.oppr.remediation.batch.listener.GetCurrentResourceChunkListener">
<property name="proxy" ref ="multiResourceReader" />
</bean>
<!--
<bean id="stepListener" class="org.kp.oppr.remediation.batch.listener.ExampleStepExecutionListener">
<property name="resources" ref="multiResourceReader"/>
</bean>
-->
<!-- Skip Policies -->
</beans>
Is there something I am missing here ?
Well You have 2 questions there
1 - "I want the processing to be done one file at a time. Rather than reading all the records till it reaches commit level."
Set Commit-Interval to 1 - It will read an item, process it and the writer will wait until it has 1 item to write.
2 - As if the job is running twice.
Looks like it will run as many times as the number of files you have.
You should not use a MultiResourceItemReader for this step.
The partitioner splits the resources into multiple and create separate execution contexts.
Your MultiResourceItemReader is again considering all the files because of the resources property being set.
Related
This is my example-default.xml,
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<bean abstract="true" id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<!-- Set to true to enable distributed class loading for examples, default is false. -->
<property name="peerClassLoadingEnabled" value="true"/>
<!-- Enable task execution events for examples. -->
<property name="includeEventTypes">
<list>
<!--Task execution events-->
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_STARTED"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_TIMEDOUT"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_SESSION_ATTR_SET"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_REDUCED"/>
<!--Cache events-->
<util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ"/>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED"/>
</list>
</property>
<property name="CacheExpiryPolicy">
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="expiryPolicyFactory">
<bean class="javax.cache.expiry.CreatedExpiryPolicy" factory-method="factoryOf">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="MINUTES"/>
<constructor-arg value="5"/>
</bean>
</constructor-arg>
</bean>
</property>
</bean>
</property>
But the above gives Bean property 'CacheExpiryPolicy' is not writable or has an invalid setter method. Does the parameter type of the setter match the return type of the getter?
How I can solve this?
I found an apache-ignite-users forum question on this.
Please refer to this here
So, the final updated xml according to that forum reference is :
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<!-- Added cache expiry policy -->
<bean id="cacheExpiryPolicy"
class="javax.cache.configuration.FactoryBuilder$SingletonFactory">
<constructor-arg>
<bean class="javax.cache.expiry.CreatedExpiryPolicy">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="MINUTES" />
<constructor-arg value="5" />
</bean>
</constructor-arg>
</bean>
</constructor-arg>
</bean>
<bean abstract="true" id="ignite.cfg"
class="org.apache.ignite.configuration.IgniteConfiguration">
<!-- Set to true to enable distributed class loading for examples, default
is false. -->
<property name="peerClassLoadingEnabled" value="true" />
<!-- Enable task execution events for examples. -->
<property name="includeEventTypes">
<list>
<!--Task execution events -->
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_STARTED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_TIMEDOUT" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_SESSION_ATTR_SET" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_REDUCED" />
<!--Cache events -->
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED" />
</list>
</property>
<!-- set the cacheConfiguration property -->
<property name="cacheConfiguration">
<list>
<bean
class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="default" />
<property name="atomicityMode" value="ATOMIC" />
<property name="expiryPolicyFactory">
<bean parent="cacheExpiryPolicy" />
</property>
</bean>
</list>
</property>
</bean>
</beans>
Here is the example from the documentation:
<property name="cacheConfiguration">
<list>
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="cacheWithExpiryPolicy"/>
<property name="expiryPolicyFactory">
<bean class="javax.cache.expiry.CreatedExpiryPolicy" factory-method="factoryOf">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="MINUTES"/>
<constructor-arg value="5"/>
</bean>
</constructor-arg>
</bean>
</property>
</bean>
</list>
</property>
I have the following spring batch job with a partitioned step that creates 3600 partitions for a partitioned step. I use a ThreadPoolTaskExecutor with a max pool size 100 and a queue capacity of 100 (although it seems to make no difference for speed). Im using Visual VM to monitor the threads and I notice the taskExecutor threads don't start until > 5 minutes after starting the job.
Oddly enough, If I limit the number of partitions to 100 the threads start fairly quickly and finish in about a minute.
Another issue I notice is that there doesn't seem to be more than one database connection ever as seen in the VisualVM thread visualization
Can someone please review my batch job below and tell me if I am missing something that would limit the number of database connections to 1? Also, why would adding more partitions affect the performance if my ThreadPoolTaskExecutor parameters don't change? Shouldn't the jobs just sit in a queue until there is a thread able to service them?
--- Spring batch job ---
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:batch="http://www.springframework.org/schema/batch"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:lang="http://www.springframework.org/schema/lang"
xsi:schemaLocation="http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-3.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-4.0.xsd">
<context:property-placeholder location="${ext.properties.dataManipulation.Properties}"/>
<import resource="${ext.properties.dataManipulation.Connection}"/>
<import resource="flatFileLineProperties.xml"/>
<!-- JobRepository and JobLauncher are configuration/setup classes -->
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean"/>
<bean id="transactionManager"
class="org.springframework.batch.support.transaction.ResourcelessTransactionManager"/>
<bean id="jobLauncher"
class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
<property name="jobRepository" ref="jobRepository"/>
</bean>
<util:map id="keys" map-class="java.util.HashMap">
<entry key="SAILING_ID" value="ASCENDING" value-type="org.springframework.batch.item.database.Order"/>
<entry key="RES_ID" value="ASCENDING" value-type="org.springframework.batch.item.database.Order" />
</util:map>
<!-- Here is my partioned step -->
<bean id="reservationsItemReader" class="org.springframework.batch.item.database.JdbcPagingItemReader" scope="step">
<property name="dataSource" ref="dataSource" />
<property name="queryProvider">
<bean class="org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="selectClause">
<value>
<![CDATA[
GUEST_ID,
FIRST_NAME,
LAST_NAME,
TITLE,
HOUSEHOLD_NAME,
SAILING_ID,
RES_ID
]]>
</value>
</property>
<property name="fromClause" value="FROM RESERVATION "/>
<property name="whereClause" >
<value>
<![CDATA[ AND SAIL_ID = :sailId
]]>
</value>
</property>
<!--<property name="sortKey" value="SAILING_ID" />-->
<property name="sortKeys" ref="keys"/>
<!--<property name="sortKeys" ref="sortKeys"/>-->
</bean>
</property>
<property name="parameterValues">
<map>
<!--<entry key="shipCode" value="#{stepExecutionContext[shipCode]}" />-->
<entry key="sailId" value="#{stepExecutionContext[sailId]}" />
</map>
</property>
<!--property name="pageSize" value="500000" /-->
<property name="pageSize" value="40000" />
<property name="rowMapper">
<bean class="com.ncl.endeca.mapper.ColumnToHashMapper" />
</property>
</bean>
<bean id="sortKeys" class="java.util.HashMap" scope="prototype" >
<constructor-arg>
<map key-type="java.lang.String" value-type="org.springframework.batch.item.database.Order">
<entry key="SAILING_ID" value="ASCENDING" />
<entry key="RES_ID" value="ASCENDING" />
</map>
</constructor-arg>
</bean>
<util:list id="client_fields" value-type="java.lang.String">
<value>FIRST_NAME</value>
<value>LAST_NAME</value>
<value>TITLE</value>
<value>HOUSEHOLD_NAME</value>
</util:list>
<bean id="reservationsItemWriter" class="com.ncl.endeca.writer.ReservationWriter" scope="step">
<property name="guestFields" ref="client_fields" />
<property name="outPrefix" value="${file.out.prefix}" />
<property name="shipCode" value="#{stepExecutionContext[shipCode]}" />
<property name="sailId" value="#{stepExecutionContext[sailId]}" />
<property name="soldOutSailings" ref="soldOutSailingsList" />
</bean>
<bean id="yearsAgo" class="java.lang.Integer">
<constructor-arg>
<value>${yearsAgo}</value>
</constructor-arg>
</bean>
<bean id="yearsAhead" class="java.lang.Integer">
<constructor-arg>
<value>${yearsAhead}</value>
</constructor-arg>
</bean>
<bean id="resPartitioner" class="com.ncl.endeca.partition.ReservationPartitioner">
<property name="yearsAgo" ref="yearsAgo" />
<property name="yearsAhead" ref="yearsAhead" />
<property name="batchLimit" value="${batch.limit}" />
<property name="dataSource" ref="dataSource"/>
</bean>
<bean id="taskExecutor"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="${batch.corePoolSize}" />
<property name="maxPoolSize" value="${batch.maxPoolSize}" />
<property name="queueCapacity" value="${batch.queueCapacity}" />
</bean>
<!--<bean id="taskExecutor" class="org.springframework.core.task.SimpleAsyncTaskExecutor"/>-->
<!-- each thread will run this job, with different stepExecutionContext values. -->
<step id="slave" xmlns="http://www.springframework.org/schema/batch" >
<flow parent="readReservations"/>
</step>
<!--<bean id="countrySpecificCompletionPolicy" class="org.springframework.batch.core.resource.StepExecutionSimpleCompletionPolicy">-->
<!--<property name="keyName" value="sailId"/>-->
<!--</bean>-->
<batch:flow id="readReservations">
<batch:step id="reservations" xmlns="http://www.springframework.org/schema/batch" >
<tasklet throttle-limit="${batch.corePoolSize}">
<chunk reader="reservationsItemReader" writer="reservationsItemWriter" commit-interval="50000" />
</tasklet>
</batch:step>
</batch:flow>
<!-- Actual Job -->
<batch:job id="dataManipulationJob">
<batch:step id="masterStep">
<batch:partition step="slave" partitioner="resPartitioner">
<batch:handler grid-size="100" task-executor="taskExecutor" />
</batch:partition>
</batch:step>
</batch:job>
I have tried BasicDataSource and Hikari connections but the pool sizes have no affect when I monitor VisualVM
---- connection.xml ----
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
xmlns:batch="http://www.springframework.org/schema/batch"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:lang="http://www.springframework.org/schema/lang"
xsi:schemaLocation="http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-3.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-4.0.xsd">
<bean id="hikariConfig" class="com.zaxxer.hikari.HikariConfig">
<property name="poolName" value="springHikariCP" />
<property name="connectionTestQuery" value="SELECT 10 from dual" />
<property name="dataSourceClassName" value="${hibernate.dataSourceClassName}" />
<property name="maximumPoolSize" value="${batch.maxPoolSize}" />
<property name="idleTimeout" value="${hibernate.hikari.idleTimeout}" />
<property name="dataSourceProperties">
<props>
<prop key="url">${dataSource.url}</prop>
<prop key="user">${dataSource.username}</prop>
<prop key="password">${dataSource.password}</prop>
</props>
</property>
</bean>
<!-- HikariCP configuration -->
<!--<bean id="dataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">-->
<!--<constructor-arg ref="hikariConfig" />-->
<!--</bean>-->
<!--<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close" scope="step">-->
<!--<property name="driverClassName" value="${hibernate.dataSourceClassName}" />-->
<!--<property name="url" value="${dataSource.url}" />-->
<!--<property name="username" value="${dataSource.username}" />-->
<!--<property name="password" value="${dataSource.password}" />-->
<!--<property name="testWhileIdle" value="false"/>-->
<!--<property name="maxActive" value="${batch.corePoolSize}"/>-->
<!--</bean>-->
<!-- connect to database -->
<bean id="dataSource"
class="org.springframework.jdbc.datasource.DriverManagerDataSource" >
<property name="driverClassName" value="oracle.jdbc.OracleDriver" />
<property name="url" value="jdbc:oracle:thin:#******" />
<property name="username" value="****" />
<property name="password" value="****" />
</bean>
The time to create partitions depends on the performance of your ReservationPartitioner as well as the number of partitions. Creating 3600 partitions means creating 3600 StepExecution/ExecutionContext objects and persisting them in the corresponding tables. This can take time for such a high number of partitions.
In regards to the database connection, you are using the MapJobRepositoryFactoryBean with a ResourcelessTransactionManager so there are no interactions with a database for Spring Batch meta-data. The only component that interacts with the database according to your configuration is the JdbcPagingItemReader (I don't know what is the type of your ReservationWriter) so it is not surprising to see a single database connection.
I am trying to use mysql database instead of default HSQL in spring batch admin. For that as per documentation
http://docs.spring.io/spring-batch-admin/reference/reference.xhtml and Using jndi datasource with spring batch admin
I copied env-context.xml to src/main/resources/META-INF/batch/override/manager/env-context.xml and changed its configuration value from
<value>classpath:batch-${ENVIRONMENT:hsql}.properties</value>
to
<value>classpath:batch-mysql.properties</value>
Below is my full configuration.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<!-- Use this to set additional properties on beans at run time -->
<bean id="placeholderProperties" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<list>
<value>classpath:/org/springframework/batch/admin/bootstrap/batch.properties</value>
<value>classpath:batch-default.properties</value>
<value>classpath:batch-mysql.properties</value>
</list>
</property>
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE" />
<property name="ignoreResourceNotFound" value="true" />
<property name="ignoreUnresolvablePlaceholders" value="false" />
<property name="order" value="1" />
</bean>
</beans>
I also tried coping data-source-context.xml to same folder and changing its configurations to mysql
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
xsi:schemaLocation="http://www.springframework.org/schema/jdbc http://www.springframework.org/schema/jdbc/spring-jdbc-3.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="dataSource"
class="org.apache.commons.dbcp.BasicDataSource">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://localhost/batch" />
<property name="username" value="root" />
<property name="password" value="root" />
<property name="testWhileIdle" value="true"/>
<property name="validationQuery" value="SELECT 1"/>
</bean>
<bean id="transactionManager"
class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="dataSource" />
</bean>
<!-- Initialise the database if enabled: -->
<jdbc:initialize-database data-source="dataSource" enabled="false" ignore-failures="DROPS">
<jdbc:script location="classpath*:/org/springframework/batch/core/schema-drop-mysql.sql"/>
<jdbc:script location="classpath:/org/springframework/batch/core/schema-mysql.sql"/>
<jdbc:script location="classpath:/business-schema-mysql.sql"/>
</jdbc:initialize-database>
</beans>
But it still using hsql database? How to override default configuration to use mysql database ?
You shouldn't replace the <value>classpath:batch-${ENVIRONMENT:hsql}.properties</value>. Instead, pass in an environment variable ENVIRONMENT set to mysql. That should cause all the appropriate components to pick up the correct database. You can read more about that feature here: http://docs.spring.io/spring-batch-admin/reference/infrastructure.html#Environment_Settings
If you want to try using just annotations without any xml configurations - try this
HibernateJpaVendorAdapter jpaVendorAdapter = new HibernateJpaVendorAdapter();
jpaVendorAdapter.setDatabase(Database.MYSQL);
It is working. My code is available here - http://github.com/sidnan/spring-batch-example.
I was able to get the connection working with above approach with below steps
First, I copied env-context.xml to src/main/resources/META-INF/batch/override/manager/env-context.xml:
<?xml version="1.0" encoding="UTF-8"?>`
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">`
<!-- Use this to set additional properties on beans at run time -->
<bean id="placeholderProperties" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<list>
<value>classpath:/org/springframework/batch/admin/bootstrap/batch.properties</value>
<value>classpath:batch-default.properties</value>
<value>classpath:batch-${ENVIRONMENT:sqlserver}.properties</value>
</list>
</property>
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_OVERRIDE" />
<property name="ignoreResourceNotFound" value="true" />
<property name="ignoreUnresolvablePlaceholders" value="false" />
<property name="order" value="1" />
</bean>
</beans>
After that, put the following entries in batch-sqlserver.properties of sql server under resources as
# Default placeholders for database platform independent features
batch.remote.base.url=http://localhost:8080/spring-batch-admin-sample
# Non-platform dependent settings that you might like to change
batch.jdbc.driver=com.microsoft.sqlserver.jdbc.SQLServerDriver
batch.jdbc.url=jdbc:sqlserver://localhost;databaseName=batchrepo
batch.jdbc.user=la
batch.jdbc.password=la
atch.jdbc.testWhileIdle=true
batch.data.source.init=false
batch.jdbc.validationQuery=
batch.database.incrementer.class=org.springframework.jdbc.support.incrementer.SqlServerMaxValueIncrementer
batch.lob.handler.class=org.springframework.jdbc.support.lob.DefaultLobHandler
batch.database.incrementer.parent=columnIncrementerParent
batch.grid.size=2
batch.jdbc.pool.size=6
batch.verify.cursor.position=true
batch.isolationlevel=ISOLATION_SERIALIZABLE
batch.initializer.enabled=false
Since my tables were already created in the database, I skipped these entries:
#batch.drop.script=/org/springframework/batch/core/schema-drop-sqlserver.sql
#batch.schema.script=/org/springframework/batch/core/schema-sqlserver.sql
#batch.business.schema.script=business-schema-sqlserver.sql
Finally, with batch.initializer.enabled=false, I was finally able to make the connection.
I can monitor the job as well as lunch the new jobs in my admin application. These launched jobs also appear in the DB.
We are working on a Spring based Web application where the key to the business is high availability. Hibernate is the ORM and MySQL is the DB that is used. Our architecture forces us to have the following mechanism.
The Webapp first tries to connect to the primary MySQL server.
If that fails, it connects to the Secondary MySQL server, which is mostly out of sync with the data.
The webapp needs to know which MySQL Server it is connected to, since we want to notify the user when he is using the secondary server.
As soon as the connection re-establishes with the primary, the connected has to be switched from secondary to primary.
I am stuck at the very first phase. I am unable find out how to direct Spring/Hibernate to use multiple DB Servers.
Here is the current config file (removing the unwanted stuff):
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:jee="http://www.springframework.org/schema/jee" xmlns:lang="http://www.springframework.org/schema/lang"
xmlns:p="http://www.springframework.org/schema/p" xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/jee http://www.springframework.org/schema/jee/spring-jee.xsd
http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<context:annotation-config />
<context:component-scan base-package="com.smartshop" />
<bean
class="org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor" />
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://localhost:3306/primarydb" />
<property name="username" value="username" />
<property name="password" value="password" />
<property name="maxIdle" value="10" />
<property name="maxActive" value="100" />
<property name="maxWait" value="10000" />
<property name="validationQuery" value="select 1" />
<property name="testOnBorrow" value="false" />
<property name="testWhileIdle" value="true" />
<property name="timeBetweenEvictionRunsMillis" value="1200000" />
<property name="minEvictableIdleTimeMillis" value="1800000" />
<property name="numTestsPerEvictionRun" value="5" />
<property name="defaultAutoCommit" value="false" />
</bean>
<bean id="sessionFactory"
class="org.springframework.orm.hibernate3.LocalSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
<property name="configLocation">
<value>/WEB-INF/hibernate.cfg.xml</value>
</property>
<property name="configurationClass">
<value>org.hibernate.cfg.AnnotationConfiguration</value>
</property>
<property name="hibernateProperties">
<props>
<prop key="hibernate.dialect">org.hibernate.dialect.MySQLDialect</prop>
<prop key="hibernate.show_sql">false</prop>
</props>
</property>
</bean>
<tx:annotation-driven />
<bean id="transactionManager"
class="org.springframework.orm.hibernate3.HibernateTransactionManager">
<property name="sessionFactory" ref="sessionFactory" />
</bean>
<bean
class="org.springframework.orm.hibernate3.support.OpenSessionInViewInterceptor"
name="openSessionInViewInterceptor">
<property name="sessionFactory" ref="sessionFactory"></property>
<property name="flushMode">
<bean
id="org.springframework.orm.hibernate3.HibernateAccessor.FLUSH_AUTO"
class="org.springframework.beans.factory.config.FieldRetrievingFactoryBean" />
</property>
</bean>
<bean id="handlerMapping"
class="org.springframework.web.servlet.mvc.annotation.DefaultAnnotationHandlerMapping">
<property name="interceptors">
<list>
<ref bean="localeChangeInterceptor" />
<ref bean="openSessionInViewInterceptor" />
</list>
</property>
</bean>
Is there a way to define Spring to connect to a backup datasource when the primary datasource is inaccessible?
if you configure your datasource as a jndi datasource you can use the following configuration
<bean id="dataSource"
class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiName" ref="datasourceJNDIName" />
<property name="defaultObject" ref="fallBackDataSource" />
</bean>
<!-- fall back datasource if JNDI look up of main datasource fails -->
<bean id="fallBackDataSource"
class="org.springframework.jndi.JndiObjectFactoryBean">
<property name="jndiName" ref="datasourceJNDIName-2" />
</bean>
This kind of tricks have to be done on MySQL side not on the webapp. MySQL cluster for instance can handle this.
More infos here:
-http://www.mysql.com/products/cluster/
-http://en.wikipedia.org/wiki/MySQL_Cluster
UPDATE:
Ok, so, have a look here -> org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource . Build a custom implementation that override the method getConnection() and getConnection(String username, String password). Surroud them by catching SQLException, and if occurs, choose the other datasource.
I know we can use spring's PropertyPlaceholderConfigurer bean in spring xml file which reads specified properties file and use values in xml file. Like wise is there a way where we can use this mechanism in my persistence.xml file.
Can i use org.eclipse.persistence.jpa.PersistenceProvider in datasource bean like this in spring xml file?
<bean id="dataSource"
class="org.eclipse.persistence.jpa.PersistenceProvider">
<property name="javax.persistence.jdbc.driver" value="${datasource.driverClassName}" />
<property name="javax.persistence.jdbc.url" value="${datasource.url}" />
<property name="javax.persistence.jdbc.user" value="${datasource.username}" />
<property name="javax.persistence.jdbc.password" value="${datasource.password}" />
</bean>
<bean id="entityManager"
class="org.springframework.orm.jpa.LocalEntityManagerFactoryBean">
<property name="persistenceXmlLocation" value="classpath:./META-INF/persistence.xml"/>
<property name="persistenceUnitName" value="JPAService"/>
<property name="dataSource" ref="dataSource"/>
</bean>
Thanks in Advance.
Like I said in my comment, the first part is not possible, check this SO question
Concerning the second part: yes, that'll work. We use a separate datasource.xml file though and import it into the application context for better modularity.
spring-context.xml:
<import resource="classpath:datasouce.xml" />
datasource.xml:
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:task="http://www.springframework.org/schema/task"
xmlns:jee="http://www.springframework.org/schema/jee"
xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.0.xsd
http://www.springframework.org/schema/jee http://www.springframework.org/schema/jee/spring-jee-3.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd"
default-autowire="byName">
<bean id="myDatasource" class="org.springframework.jdbc.datasource.DriverManagerDataSource">
<property name="username" value="..." />
<property name="password" value="..." />
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://localhost:3306/myTestDB" />
</bean>
<bean id="entityManagerFactory"
class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
<property name="dataSource" ref="myDatasource"/>
<property name="jpaVendorAdapter">
<bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
<property name="showSql" value="true" />
<property name="generateDdl" value="true" />
<property name="databasePlatform" value="org.hibernate.dialect.MySQL5InnoDBDialect" />
</bean>
</property>
</bean>
<tx:annotation-driven transaction-manager="transactionManager"/>
<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager" />
</beans>
Rather than using your build to create a prod or dev version of your persistence.xml, just move all property settings to your spring content.
read the original post by emeraldjava loading .properties in spring-context.xml and persistence.xml