Develop Reference

Return List of Objects from Spring Batch Custom ItemReader

I have a requirement to process thousands of Payments..So I used VtdXml instead of StaxEventItemReader in Spring Batch, I have created Custom Item Reader for the same. In order to read huge xml with Multi Threading, I have created partition with 10 Threads. I am splitting huge xml file into 10 files and assigning to each Thread in Partition. Once I read the xml and I will convert into list of Objects and send to Writer. After received in Writer, I will customize the List of Objects and Merge into the final List. Whenever Am returning list of Objects read called again and it is never ending. How do I pass the list of Objects to the Writer and merge into the final List?
public class VtdWholeItemReader<T> implements ResourceAwareItemReaderItemStream<T> {
private Resource resource;
private boolean noInput;
private boolean strict = true;
private InputStream inputStream;
private int index = 0;
#Override
public void open(ExecutionContext executionContext) {
Assert.notNull(resource, "The Resource must not be null.");
noInput = true;
if (!resource.exists()) {
if (strict) {
throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode)");
}
log.warn("Input resource does not exist " + resource.getDescription());
return;
}
if (!resource.isReadable()) {
if (strict) {
throw new IllegalStateException("Input resource must be readable (reader is in 'strict' mode)");
}
log.warn("Input resource is not readable " + resource.getDescription());
return;
}
noInput = false;
}
#Override
public void update(ExecutionContext executionContext) {
}
#Override
public void close() {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
inputStream = null;
}
}
#Override
public void setResource(Resource resource) {
this.resource = resource;
}
#Override
public T read()
throws java.lang.Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
if (noInput) {
return null;
}
List<Payment> paymentList = new ArrayList<Payment>();
try {
VTDGen vg = new VTDGen();
VTDGen vgHen = new VTDGen();
boolean headercheck = true;
if (vg.parseFile("src/main/resources/input/partitioner/" + resource.getFilename(), false)) {
VTDNav vn = vg.getNav();
AutoPilot ap = new AutoPilot(vn);
ap.selectXPath("/root/Payment");
// flb contains all the offset and length of the segments to be skipped
FastLongBuffer flb = new FastLongBuffer(4);
int i;
byte[] xml = vn.getXML().getBytes();
while ((i = ap.evalXPath()) != -1) {
flb.append(vn.getElementFragment());
}
int size = flb.size();
log.info("Payment Size {}", size);
if (size != 0) {
for (int k = 0; k < size; k++) {
String message = new String(xml, flb.lower32At(k), flb.upper32At(k), StandardCharsets.UTF_8);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Payment payment = objectMapper
.readValue(message, Payment.class);
paymentList.add(pcPayment);
index = pcPaymentList.size() + 1;
}
}
log.info("Payment List:: {}", paymentList.size());
log.info("Index::{}", index);
return index > paymentList .size() ? null : (T) paymentList;
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
SpringBatch ConfigClass
private final Logger logger = LoggerFactory.getLogger(SpringBatchConfig.class);
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Autowired
ResourcePatternResolver resoursePatternResolver;
#Bean
public Job job() {
return jobBuilderFactory.get("job").start(readpayment()).build();
}
#Bean
public JobLauncher jobLauncher() throws Exception {
SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
jobLauncher.setJobRepository(jobRepository());
jobLauncher.afterPropertiesSet();
return jobLauncher;
}
#Bean
public JobRepository jobRepository() throws Exception {
MapJobRepositoryFactoryBean factory = new MapJobRepositoryFactoryBean();
factory.setTransactionManager(new ResourcelessTransactionManager());
return (JobRepository) factory.getObject();
}
#Bean
protected Step readpayment() {
return stepBuilderFactory.get("readpayment").partitioner("paymentStep", partitioner(null))
.step(paymentStep()).taskExecutor(taskExecutor()).build();
}
#Bean
protected Step paymentStep() {
return stepBuilderFactory.get("paymentStep")
.<Payment,Payment>chunk(10)
.reader(xmlFileItemReader(null))
.writer(writer()).build();
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(10);
taskExecutor.setCorePoolSize(10);
taskExecutor.setQueueCapacity(10);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
#Bean
#StepScope
ItemReader<Payment> xmlFileItemReader(#Value("#{stepExecutionContext[fileName]}") String filename) {
VtdWholeItemReader<Payment> xmlFileReader = new VtdWholeItemReader<>();
xmlFileReader.setResource(new ClassPathResource("input/partitioner/" + filename));
return xmlFileReader;
}
#Bean
#StepScope
public CustomMultiResourcePartitioner partitioner(#Value("#{jobParameters['fileName']}") String fileName) {
logger.info("fileName {}", fileName);
CustomMultiResourcePartitioner partitioner = new CustomMultiResourcePartitioner();
Resource[] resources;
try {
resources = resoursePatternResolver.getResources("file:src/main/resources/input/partitioner/*.xml");
} catch (IOException e) {
throw new RuntimeException("I/O problems when resolving the input file pattern.", e);
}
partitioner.setResources(resources);
return partitioner;
}
#Bean
public ItemWriter<Payment> writer() {
return new PaymentItemWriter();
}
PaymentItemWriter
#Override
public void write(List<? extends List<Payment>> items) throws Exception {
log.info("Items {}", items.size());
}

May be try making ItemWriter bean as step scope ["#StepScope"] in Spring batch configuration class

Make your xmlFileItemReader method return the actual type VtdWholeItemReader instead of the interface type ItemReader:
#Bean
#StepScope
VtdWholeItemReader<Payment> xmlFileItemReader(#Value("#{stepExecutionContext[fileName]}") String filename) {
VtdWholeItemReader<Payment> xmlFileReader = new VtdWholeItemReader<>();
xmlFileReader.setResource(new ClassPathResource("input/partitioner/" + filename));
return xmlFileReader;
}
This way, Spring will correctly proxy your reader as an ItemStreamReader (and not ItemReader) and honor the contract of calling open/update/close methods.

Encountered an error executing step masterStep Execption Loading data to mysql Spring Batch

I am trying to load set of csv file into mysql using spring batch.
My code is given below. But when I run with 2 or 3 files the data is loading fine. but when I try with 100 or more Its trowing an execption aftre loading 4 to 8 files data.
Exception:
ERROR 11633 --- [ main] o.s.batch.core.step.AbstractStep : Encountered an error executing step masterStep in job importUserJob
org.springframework.batch.core.JobExecutionException: Partition handler returned an unsuccessful step
at org.springframework.batch.core.partition.support.PartitionStep.doExecute(PartitionStep.java:112) ~[spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:200) ~[spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.SimpleStepHandler.handleStep(SimpleStepHandler.java:148) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.flow.JobFlowExecutor.executeStep(JobFlowExecutor.java:66) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.flow.support.state.StepState.handle(StepState.java:67) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.flow.support.SimpleFlow.resume(SimpleFlow.java:169) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.flow.support.SimpleFlow.start(SimpleFlow.java:144) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.flow.FlowJob.doExecute(FlowJob.java:136) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.job.AbstractJob.execute(AbstractJob.java:308) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.batch.core.launch.support.SimpleJobLauncher$1.run(SimpleJobLauncher.java:141) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) [spring-core-5.0.5.RELEASE.jar!/:5.0.5.RELEASE]
at org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:134) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.execute(JobLauncherCommandLineRunner.java:163) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.executeLocalJobs(JobLauncherCommandLineRunner.java:179) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.launchJobFromProperties(JobLauncherCommandLineRunner.java:134) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.run(JobLauncherCommandLineRunner.java:128) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:797) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:781) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:335) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:137) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
I don't know whats happening behind, Any help appreciated.
Code:
#Configuration
#EnableBatchProcessing
public class ImportJobConfig {
#Autowired
private JobBuilderFactory jobBuilderFactory;
#Autowired
private StepBuilderFactory stepBuilderFactory;
#Autowired
private DataSource dataSource;
#Autowired
private JdbcBatchItemWriter<Person> writer;
#Autowired
private FlatFileItemReader<Person> personItemReader;
#Bean
public DataSource dataSource() {
final DriverManagerDataSource dataSource = new DriverManagerDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl("URL");
dataSource.setUsername("USERNAME");
dataSource.setPassword("PASSWORD");
return dataSource;
}
#Bean
public ThreadPoolTaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(4);
taskExecutor.setCorePoolSize(4);
taskExecutor.setQueueCapacity(4);
taskExecutor.afterPropertiesSet();
taskExecutor.setAllowCoreThreadTimeOut(true);
return taskExecutor;
}
#Bean
#Qualifier("masterStep")
public Step masterStep() {
return stepBuilderFactory.get("masterStep").partitioner("step1", partitioner()).step(step1())
.taskExecutor(taskExecutor()).build();
}
#Bean("partitioner")
#StepScope
public Partitioner partitioner() {
System.out.println("In Partitioner");
MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
String filePath = "//data//person//*.csv";
Resource[] resources = null;
try {
resources = resolver.getResources("file:" + filePath);
} catch (IOException e) {
e.printStackTrace();
}
partitioner.setResources(resources);
partitioner.partition(4);
return partitioner;
}
#Bean
public APSUploadFileItemProcessor processor() {
return new APSUploadFileItemProcessor();
}
#Bean
public JdbcBatchItemWriter<Person> writer() {
JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<>();
try {
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
writer.setSql("INSERT INTO person(name,business,phone) VALUES(name, :business, :phone)");
writer.setDataSource(dataSource);
} catch (Exception e) {
e.printStackTrace();
}
return writer;
}
#Bean
public Job importUserJob(JobCompletionNotificationListener listener, Step step1) {
return jobBuilderFactory.get("importUserJob").incrementer(new RunIdIncrementer()).listener(listener)
.flow(masterStep()).end().build();
}
#Bean
public Step step1() {
return stepBuilderFactory.get("step1").<Person, Person>chunk(10000).processor(processor())
.writer(writer).reader(personItemReader).build();
}
#Bean
#StepScope
#Qualifier("personItemReader")
#DependsOn("partitioner")
public FlatFileItemReader<Person> personItemReader(
#Value("#{stepExecutionContext['fileName']}") String filename) throws MalformedURLException {
return new FlatFileItemReaderBuilder<Person>().name("personItemReader").delimited().delimiter("|")
.names(new String[] { "name","business","phone" })
.fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {
{
setTargetType(Person.class);
}
}).resource(new UrlResource(filename)).build();
}
}

Spring Integration FTP remove local files after use (Spring Boot)

I am trying to write a program that can take a file from one server via ftp and place it on another server via ftp. However, I am having issues deleting the local file after it has been written. Being able to save it locally is not an issue as long as it is temporary. I have tried using an ExpressionEvaluatingRequestHandlerAdvice with an OnSuccessExpression and I could not get it to actually use the expression. The code is here:
#Configuration
#EnableConfigurationProperties(FTPConnectionProperties.class)
public class FTPConfiguration {
private FTPConnectionProperties ftpConnectionProperties;
public FTPConfiguration(FTPConnectionProperties ftpConnectionProperties) {
this.ftpConnectionProperties = ftpConnectionProperties;
}
#Bean
public SessionFactory<FTPFile> ftpInputSessionFactory() {
DefaultFtpSessionFactory sf = new DefaultFtpSessionFactory();
sf.setHost(ftpConnectionProperties.getInputServer());
sf.setUsername(ftpConnectionProperties.getInputFtpUser());
sf.setPassword(ftpConnectionProperties.getInputFtpPassword());
return new CachingSessionFactory<>(sf);
}
#Bean
public SessionFactory<FTPFile> ftpOutputSessionFactory() {
DefaultFtpSessionFactory sf = new DefaultFtpSessionFactory();
sf.setHost(ftpConnectionProperties.getOutputServer());
sf.setUsername(ftpConnectionProperties.getOutputFtpUser());
sf.setPassword(ftpConnectionProperties.getOutputFtpPassword());
return new CachingSessionFactory<>(sf);
}
#Bean
public FtpInboundFileSynchronizer ftpInboundFileSynchronizer() {
FtpInboundFileSynchronizer fileSynchronizer = new FtpInboundFileSynchronizer(ftpInputSessionFactory());
fileSynchronizer.setDeleteRemoteFiles(true);
fileSynchronizer.setRemoteDirectory(ftpConnectionProperties.getInputDirectory());
fileSynchronizer.setFilter(new FtpSimplePatternFileListFilter("*.TIF"));
return fileSynchronizer;
}
#Bean
#InboundChannelAdapter(channel = "input", poller = #Poller(fixedDelay = "5000"))
public MessageSource<File> ftpMessageSource() {
FtpInboundFileSynchronizingMessageSource source = new FtpInboundFileSynchronizingMessageSource(ftpInboundFileSynchronizer());
source.setLocalDirectory(new File("ftp-inbound"));
source.setAutoCreateLocalDirectory(true);
source.setLocalFilter(new FileSystemPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), ""));
return source;
}
#Bean
#ServiceActivator(inputChannel = "input")
public MessageHandler handler() {
FtpMessageHandler handler = new FtpMessageHandler(ftpOutputSessionFactory());
handler.setRemoteDirectoryExpression(new LiteralExpression(ftpConnectionProperties.getOutputDirectory()));
handler.setFileNameGenerator(message -> {
if (message.getPayload() instanceof File) {
return ((File) message.getPayload()).getName();
} else {
throw new IllegalArgumentException("File expected as payload.");
}
});
return handler;
}
}
It is handling the remote files exactly as expected, deleting the remote file from the source and putting into the output, but not removing the local file after use.

I would suggest you to make that input channel as a PublishSubscribeChannel and add one more simple subscriber:
#Bean
public PublishSubscribeChannel input() {
return new PublishSubscribeChannel();
}
#Bean
#ServiceActivator(inputChannel = "input")
public MessageHandler handler() {
...
}
#Bean
#ServiceActivator(inputChannel = "input")
public MessageHandler deleteLocalFileService() {
return m -> ((File) message.getPayload()).delete();
}
This way the same message with the File payload is going to be sent first to your FtpMessageHandler and only after that to this new deleteLocalFileService for removing the local file based on the payload.

Simple Solution to fetch file from SFTP server and then move that file to other folder with different name.
#Bean
public SessionFactory<ChannelSftp.LsEntry> sftpSessionFactory() {
DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(true);
if (sftpServerProperties.getSftpPrivateKey() != null) {
factory.setPrivateKey(sftpServerProperties.getSftpPrivateKey());
factory.setPrivateKeyPassphrase(sftpServerProperties.getSftpPrivateKeyPassphrase());
} else {
factory.setPassword(sftpServerProperties.getPassword());
}
factory.setHost(sftpServerProperties.getSftpHost());
factory.setPort(sftpServerProperties.getSftpPort());
factory.setUser(sftpServerProperties.getSftpUser());
factory.setAllowUnknownKeys(true);
return new CachingSessionFactory<>(factory);
}
#Bean
public SftpInboundFileSynchronizer sftpInboundFileSynchronizer() {
SftpInboundFileSynchronizer fileSynchronizer = new SftpInboundFileSynchronizer(sftpSessionFactory());
fileSynchronizer.setDeleteRemoteFiles(false);
fileSynchronizer.setRemoteDirectory(sftpServerProperties.getSftpRemoteDirectoryDownload());
fileSynchronizer.setFilter(new SftpSimplePatternFileListFilter(sftpServerProperties.getSftpRemoteDirectoryDownloadFilter()));
return fileSynchronizer;
}
#Bean
#InboundChannelAdapter(channel = "fromSftpChannel", poller = #Poller(cron = "*/10 * * * * *"))
public MessageSource<File> sftpMessageSource() {
SftpInboundFileSynchronizingMessageSource source = new SftpInboundFileSynchronizingMessageSource(
sftpInboundFileSynchronizer());
source.setLocalDirectory(util.createDirectory(Constants.FILES_DIRECTORY));
source.setAutoCreateLocalDirectory(true);
return source;
}
#Bean
#ServiceActivator(inputChannel = "fromSftpChannel")
public MessageHandler resultFileHandler() {
return (Message<?> message) -> {
String csvFilePath = util.getDirectory(Constants.FILES_DIRECTORY) + Constants.INSIDE + message.getHeaders().get("file_name");
util.readCSVFile(csvFilePath, String.valueOf(message.getHeaders().get("file_name")));
File file = (File) message.getPayload();
File newFile = new File(file.getPath() + System.currentTimeMillis());
try {
FileUtils.copyFile(file, newFile);
sftpGateway.sendToSftp(newFile);
} catch (Exception e) {
e.printStackTrace();
}
if (file.exists()) {
file.delete();
}
if (newFile.exists()) {
newFile.delete();
}
};
}
#Bean
#ServiceActivator(inputChannel = "toSftpChannelDest")
public MessageHandler handlerOrderBackUp() {
SftpMessageHandler handler = new SftpMessageHandler(sftpSessionFactory());
handler.setAutoCreateDirectory(true);
handler.setRemoteDirectoryExpression(new LiteralExpression(sftpServerProperties.getSftpRemoteBackupDirectory()));
return handler;
}
#MessagingGateway
public interface SFTPGateway {
#Gateway(requestChannel = "toSftpChannelDest")
void sendToSftp(File file);
}

How to mock WebFluxRequestExecutingMessageHandler with MockIntegrationContext.substituteMessageHandlerFor

I have implemented an IntegrationFlow where I want to do to following tasks:
Poll for files from a directory
Transform the file content to a string
Send the string via WebFluxRequestExecutingMessageHandler to a REST-Endpoint and use an AdviceChain to handle success and error responses
Implementation
#Configuration
#Slf4j
public class JsonToRestIntegration {
#Autowired
private LoadBalancerExchangeFilterFunction lbFunction;
#Value("${json_folder}")
private String jsonPath;
#Value("${json_success_folder}")
private String jsonSuccessPath;
#Value("${json_error_folder}")
private String jsonErrorPath;
#Value("${rest-service-url}")
private String restServiceUrl;
#Bean
public DirectChannel httpResponseChannel() {
return new DirectChannel();
}
#Bean
public MessageChannel successChannel() {
return new DirectChannel();
}
#Bean
public MessageChannel failureChannel() {
return new DirectChannel();
}
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller() {
return Pollers.fixedDelay(1000).get();
}
#Bean
public IntegrationFlow jsonFileToRestFlow() {
return IntegrationFlows
.from(fileReadingMessageSource(), e -> e.id("fileReadingEndpoint"))
.transform(org.springframework.integration.file.dsl.Files.toStringTransformer())
.enrichHeaders(s -> s.header("Content-Type", "application/json; charset=utf8"))
.handle(reactiveOutbound())
.log()
.channel(httpResponseChannel())
.get();
}
#Bean
public FileReadingMessageSource fileReadingMessageSource() {
FileReadingMessageSource source = new FileReadingMessageSource();
source.setDirectory(new File(jsonPath));
source.setFilter(new SimplePatternFileListFilter("*.json"));
source.setUseWatchService(true);
source.setWatchEvents(FileReadingMessageSource.WatchEventType.CREATE);
return source;
}
#Bean
public MessageHandler reactiveOutbound() {
WebClient webClient = WebClient.builder()
.baseUrl("http://jsonservice")
.filter(lbFunction)
.build();
WebFluxRequestExecutingMessageHandler handler = new WebFluxRequestExecutingMessageHandler(restServiceUrl, webClient);
handler.setHttpMethod(HttpMethod.POST);
handler.setCharset(StandardCharsets.UTF_8.displayName());
handler.setOutputChannel(httpResponseChannel());
handler.setExpectedResponseType(String.class);
handler.setAdviceChain(singletonList(expressionAdvice()));
return handler;
}
public Advice expressionAdvice() {
ExpressionEvaluatingRequestHandlerAdvice advice = new ExpressionEvaluatingRequestHandlerAdvice();
advice.setTrapException(true);
advice.setSuccessChannel(successChannel());
advice.setOnSuccessExpressionString("payload + ' war erfolgreich'");
advice.setFailureChannel(failureChannel());
advice.setOnFailureExpressionString("payload + ' war nicht erfolgreich'");
return advice;
}
#Bean
public IntegrationFlow loggingFlow() {
return IntegrationFlows.from(httpResponseChannel())
.handle(message -> {
String originalFileName = (String) message.getHeaders().get(FileHeaders.FILENAME);
log.info("some log");
})
.get();
}
#Bean
public IntegrationFlow successFlow() {
return IntegrationFlows.from(successChannel())
.handle(message -> {
MessageHeaders messageHeaders = ((AdviceMessage) message).getInputMessage().getHeaders();
File originalFile = (File) messageHeaders.get(ORIGINAL_FILE);
String originalFileName = (String) messageHeaders.get(FILENAME);
if (originalFile != null && originalFileName != null) {
File jsonSuccessFolder = new File(jsonSuccessPath);
File jsonSuccessFile = new File(jsonSuccessFolder, originalFileName);
try {
Files.move(originalFile.toPath(), jsonSuccessFile.toPath());
} catch (IOException e) {
log.error("some log", e);
}
}
})
.get();
}
#Bean
public IntegrationFlow failureFlow() {
return IntegrationFlows.from(failureChannel())
.handle(message -> {
Message<?> failedMessage = ((MessagingException) message.getPayload()).getFailedMessage();
if (failedMessage != null) {
File originalFile = (File) failedMessage.getHeaders().get(FileHeaders.ORIGINAL_FILE);
String originalFileName = (String) failedMessage.getHeaders().get(FileHeaders.FILENAME);
if (originalFile != null && originalFileName != null) {
File jsonErrorFolder = new File(tonisJsonErrorPath);
File jsonErrorFile = new File(jsonErrorFolder, originalFileName);
try {
Files.move(originalFile.toPath(), jsonErrorFile.toPath());
} catch (IOException e) {
log.error("some log", e);
}
}
}
})
.get();
}
}
So far it seems to work in production. In the test I want to do the following steps:
Copy JSON-Files to the input directory
Start the polling for the json files
Do assertions on the HTTP-Response from the WebFluxRequestExecutingMessageHandler which are routed through my advice chain
But I'm struggling in the test with the following tasks:
Mocking the WebFluxRequestExecutingMessageHandler with the MockIntegrationContext.substituteMessageHandlerFor()-method
Manually start the polling of the json files
Test
#RunWith(SpringRunner.class)
#SpringIntegrationTest()
#Import({JsonToRestIntegration.class})
#JsonTest
public class JsonToRestIntegrationTest {
#Autowired
public DirectChannel httpResponseChannel;
#Value("${json_folder}")
private String jsonPath;
#Value("${json_success_folder}")
private String jsonSuccessPath;
#Value("${json_error_folder}")
private String jsonErrorPath;
#Autowired
private MockIntegrationContext mockIntegrationContext;
#Autowired
private MessageHandler reactiveOutbound;
#Before
public void setUp() throws Exception {
Files.createDirectories(Paths.get(jsonPath));
Files.createDirectories(Paths.get(jsonSuccessPath));
Files.createDirectories(Paths.get(jsonErrorPath));
}
#After
public void tearDown() throws Exception {
FileUtils.deleteDirectory(new File(jsonPath));
FileUtils.deleteDirectory(new File(jsonSuccessPath));
FileUtils.deleteDirectory(new File(jsonErrorPath));
}
#Test
public void shouldSendJsonToRestEndpointAndReceiveOK() throws Exception {
File jsonFile = new ClassPathResource("/test.json").getFile();
Path targetFilePath = Paths.get(jsonPath + "/" + jsonFile.getName());
Files.copy(jsonFile.toPath(), targetFilePath);
httpResponseChannel.subscribe(httpResponseHandler());
this.mockIntegrationContext.substituteMessageHandlerFor("", reactiveOutbound);
}
private MessageHandler httpResponseHandler() {
return message -> Assert.assertThat(message.getPayload(), is(notNullValue()));
}
#Configuration
#Import({JsonToRestIntegration.class})
public static class JsonToRestIntegrationTest {
#Autowired
public MessageChannel httpResponseChannel;
#Bean
public MessageHandler reactiveOutbound() {
ArgumentCaptor<Message<?>> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
MockMessageHandler mockMessageHandler = mockMessageHandler(messageArgumentCaptor).handleNextAndReply(m -> m);
mockMessageHandler.setOutputChannel(httpResponseChannel);
return mockMessageHandler;
}
}
}
Updated Working Example with mocked WebFluX web client:
Implementation
public class JsonToRestIntegration {
private final LoadBalancerExchangeFilterFunction lbFunction;
private final BatchConfigurationProperties batchConfigurationProperties;
#Bean
public DirectChannel httpResponseChannel() {
return new DirectChannel();
}
#Bean
public DirectChannel errorChannel() {
return new DirectChannel();
}
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller() {
return Pollers.fixedDelay(100, TimeUnit.MILLISECONDS).get();
}
#Bean
public IntegrationFlow jsonFileToRestFlow() {
return IntegrationFlows
.from(fileReadingMessageSource(), e -> e.id("fileReadingEndpoint"))
.transform(org.springframework.integration.file.dsl.Files.toStringTransformer("UTF-8"))
.enrichHeaders(s -> s.header("Content-Type", "application/json; charset=utf8"))
.handle(reactiveOutbound())
.channel(httpResponseChannel())
.get();
}
#Bean
public FileReadingMessageSource fileReadingMessageSource() {
FileReadingMessageSource source = new FileReadingMessageSource();
source.setDirectory(new File(batchConfigurationProperties.getJsonImportFolder()));
source.setFilter(new SimplePatternFileListFilter("*.json"));
source.setUseWatchService(true);
source.setWatchEvents(FileReadingMessageSource.WatchEventType.CREATE);
return source;
}
#Bean
public WebFluxRequestExecutingMessageHandler reactiveOutbound() {
WebClient webClient = WebClient.builder()
.baseUrl("http://service")
.filter(lbFunction)
.build();
WebFluxRequestExecutingMessageHandler handler = new WebFluxRequestExecutingMessageHandler(batchConfigurationProperties.getServiceUrl(), webClient);
handler.setHttpMethod(HttpMethod.POST);
handler.setCharset(StandardCharsets.UTF_8.displayName());
handler.setOutputChannel(httpResponseChannel());
handler.setExpectedResponseType(String.class);
handler.setAdviceChain(singletonList(expressionAdvice()));
return handler;
}
public Advice expressionAdvice() {
ExpressionEvaluatingRequestHandlerAdvice advice = new ExpressionEvaluatingRequestHandlerAdvice();
advice.setTrapException(true);
advice.setFailureChannel(errorChannel());
return advice;
}
#Bean
public IntegrationFlow responseFlow() {
return IntegrationFlows.from(httpResponseChannel())
.handle(message -> {
MessageHeaders messageHeaders = message.getHeaders();
File originalFile = (File) messageHeaders.get(ORIGINAL_FILE);
String originalFileName = (String) messageHeaders.get(FILENAME);
if (originalFile != null && originalFileName != null) {
File jsonSuccessFolder = new File(batchConfigurationProperties.getJsonSuccessFolder());
File jsonSuccessFile = new File(jsonSuccessFolder, originalFileName);
try {
Files.move(originalFile.toPath(), jsonSuccessFile.toPath());
} catch (IOException e) {
log.error("Could not move file", e);
}
}
})
.get();
}
#Bean
public IntegrationFlow failureFlow() {
return IntegrationFlows.from(errorChannel())
.handle(message -> {
Message<?> failedMessage = ((MessagingException) message.getPayload()).getFailedMessage();
if (failedMessage != null) {
File originalFile = (File) failedMessage.getHeaders().get(ORIGINAL_FILE);
String originalFileName = (String) failedMessage.getHeaders().get(FILENAME);
if (originalFile != null && originalFileName != null) {
File jsonErrorFolder = new File(batchConfigurationProperties.getJsonErrorFolder());
File jsonErrorFile = new File(jsonErrorFolder, originalFileName);
try {
Files.move(originalFile.toPath(), jsonErrorFile.toPath());
} catch (IOException e) {
log.error("Could not move file", originalFileName, e);
}
}
}
})
.get();
}
}
Test
#RunWith(SpringRunner.class)
#SpringIntegrationTest(noAutoStartup = "fileReadingEndpoint")
#Import({JsonToRestIntegration.class, BatchConfigurationProperties.class})
#JsonTest
#DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class JsonToRestIntegrationIT {
private static final FilenameFilter JSON_FILENAME_FILTER = (dir, name) -> name.endsWith(".json");
#Autowired
private BatchConfigurationProperties batchConfigurationProperties;
#Autowired
private ObjectMapper om;
#Autowired
private MessageHandler reactiveOutbound;
#Autowired
private DirectChannel httpResponseChannel;
#Autowired
private DirectChannel errorChannel;
#Autowired
private FileReadingMessageSource fileReadingMessageSource;
#Autowired
private SourcePollingChannelAdapter fileReadingEndpoint;
#MockBean
private LoadBalancerExchangeFilterFunction lbFunction;
private String jsonImportPath;
private String jsonSuccessPath;
private String jsonErrorPath;
#Before
public void setUp() throws Exception {
jsonImportPath = batchConfigurationProperties.getJsonImportFolder();
jsonSuccessPath = batchConfigurationProperties.getJsonSuccessFolder();
jsonErrorPath = batchConfigurationProperties.getJsonErrorFolder();
Files.createDirectories(Paths.get(jsonImportPath));
Files.createDirectories(Paths.get(jsonSuccessPath));
Files.createDirectories(Paths.get(jsonErrorPath));
}
#After
public void tearDown() throws Exception {
FileUtils.deleteDirectory(new File(jsonImportPath));
FileUtils.deleteDirectory(new File(jsonSuccessPath));
FileUtils.deleteDirectory(new File(jsonErrorPath));
}
#Test
public void shouldMoveJsonFileToSuccessFolderWhenHttpResponseIsOk() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
httpResponseChannel.addInterceptor(new ChannelInterceptorAdapter() {
#Override
public void postSend(Message<?> message, MessageChannel channel, boolean sent) {
latch.countDown();
super.postSend(message, channel, sent);
}
});
errorChannel.addInterceptor(new ChannelInterceptorAdapter() {
#Override
public void postSend(Message<?> message, MessageChannel channel, boolean sent) {
fail();
}
});
ClientHttpConnector httpConnector = new HttpHandlerConnector((request, response) -> {
response.setStatusCode(HttpStatus.OK);
response.getHeaders().setContentType(MediaType.APPLICATION_JSON_UTF8);
DataBufferFactory bufferFactory = response.bufferFactory();
String valueAsString = null;
try {
valueAsString = om.writeValueAsString(new ResponseDto("1"));
} catch (JsonProcessingException e) {
fail();
}
return response.writeWith(Mono.just(bufferFactory.wrap(valueAsString.getBytes())))
.then(Mono.defer(response::setComplete));
});
WebClient webClient = WebClient.builder()
.clientConnector(httpConnector)
.build();
new DirectFieldAccessor(this.reactiveOutbound)
.setPropertyValue("webClient", webClient);
File jsonFile = new ClassPathResource("/test.json").getFile();
Path targetFilePath = Paths.get(jsonImportPath + "/" + jsonFile.getName());
Files.copy(jsonFile.toPath(), targetFilePath);
fileReadingEndpoint.start();
assertThat(latch.await(12, TimeUnit.SECONDS), is(true));
File[] jsonImportFolder = new File(jsonImportPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonImportFolder, is(notNullValue()));
assertThat(filesInJsonImportFolder.length, is(0));
File[] filesInJsonSuccessFolder = new File(jsonSuccessPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonSuccessFolder, is(notNullValue()));
assertThat(filesInJsonSuccessFolder.length, is(1));
File[] filesInJsonErrorFolder = new File(jsonErrorPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonErrorFolder, is(notNullValue()));
assertThat(filesInJsonErrorFolder.length, is(0));
}
#Test
public void shouldMoveJsonFileToErrorFolderWhenHttpResponseIsNotOk() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
errorChannel.addInterceptor(new ChannelInterceptorAdapter() {
#Override
public void postSend(Message<?> message, MessageChannel channel, boolean sent) {
latch.countDown();
super.postSend(message, channel, sent);
}
});
httpResponseChannel.addInterceptor(new ChannelInterceptorAdapter() {
#Override
public void postSend(Message<?> message, MessageChannel channel, boolean sent) {
fail();
}
});
ClientHttpConnector httpConnector = new HttpHandlerConnector((request, response) -> {
response.setStatusCode(HttpStatus.BAD_REQUEST);
response.getHeaders().setContentType(MediaType.APPLICATION_JSON_UTF8);
DataBufferFactory bufferFactory = response.bufferFactory();
return response.writeWith(Mono.just(bufferFactory.wrap("SOME BAD REQUEST".getBytes())))
.then(Mono.defer(response::setComplete));
});
WebClient webClient = WebClient.builder()
.clientConnector(httpConnector)
.build();
new DirectFieldAccessor(this.reactiveOutbound)
.setPropertyValue("webClient", webClient);
File jsonFile = new ClassPathResource("/error.json").getFile();
Path targetFilePath = Paths.get(jsonImportPath + "/" + jsonFile.getName());
Files.copy(jsonFile.toPath(), targetFilePath);
fileReadingEndpoint.start();
assertThat(latch.await(11, TimeUnit.SECONDS), is(true));
File[] filesInJsonImportFolder = new File(jsonImportPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonImportFolder, is(notNullValue()));
assertThat(filesInJsonImportFolder.length, is(0));
File[] filesInJsonSuccessFolder = new File(jsonSuccessPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonSuccessFolder, is(notNullValue()));
assertThat(filesInJsonSuccessFolder.length, is(0));
File[] filesInJsonErrorFolder = new File(jsonErrorPath).listFiles(JSON_FILENAME_FILTER);
assertThat(filesInJsonErrorFolder, is(notNullValue()));
assertThat(filesInJsonErrorFolder.length, is(1));
}
}

this.mockIntegrationContext.substituteMessageHandlerFor("", reactiveOutbound);
The first parameter of this method is an endpoint id. (I guess we are just missing Javadocs there on those methods...).
So, what you need is something like this:
.handle(reactiveOutbound(), e -> e.id("webFluxEndpoint"))
And then in that test-case you do:
this.mockIntegrationContext.substituteMessageHandlerFor("webFluxEndpoint", reactiveOutbound);
You don't need to override bean in the test class config. The MockMessageHandler can be just used in the test method body.
You poll files via .from(fileReadingMessageSource()). To do a manual control you need to have it stopped in the beginning. For this purpose you add an endpoint id again:
.from(fileReadingMessageSource(), e -> e.id("fileReadingEndpoint"))
And then in the test configuration you do this:
#SpringIntegrationTest(noAutoStartup = "fileReadingEndpoint")
Another approach for the WebFlux would be via customized WebClient to mock request to the server. For example:
ClientHttpConnector httpConnector = new HttpHandlerConnector((request, response) -> {
response.setStatusCode(HttpStatus.OK);
response.getHeaders().setContentType(MediaType.TEXT_PLAIN);
DataBufferFactory bufferFactory = response.bufferFactory();
return response.writeWith(Mono.just(bufferFactory.wrap("FOO\nBAR\n".getBytes())))
.then(Mono.defer(response::setComplete));
});
WebClient webClient = WebClient.builder()
.clientConnector(httpConnector)
.build();
new DirectFieldAccessor(this.reactiveOutbound)
.setPropertyValue("webClient", webClient);

springbatch excel does not work for the second time

I am using an api to call the springbatch job. when i call for the first time it works fine. second time gives the below error.
Caused by: java.lang.IllegalArgumentException: Sheet index (3) is out of range (0..2).
Below is my code.-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Config:
#Configuration
#EnableBatchProcessing
#PropertySource("classpath:batch.properties")
public class ExcelFileToDatabaseJobConfig {
#Autowired
public JobBuilderFactory jobBuilderFactory;
#Autowired
public StepBuilderFactory stepBuilderFactory;
#Bean
ItemReader<StudentDTO> excelStudentReader() throws Exception {
System.out.println("inside excelStudentReader");
PoiItemReader<StudentDTO> reader = new PoiItemReader<>();
reader.setLinesToSkip(1);
//reader.setResource(new FileSystemResource("C:/Users/lenovo/IdeaProjects/test/out/production/classes/Price Change.xlsx"));
reader.setResource(new UrlResource("file:///C:/Users/lenovo/IdeaProjects/test/out/production/classes/inserttest.xlsx"));
reader.setRowMapper(excelRowMapper());
return reader;
}
private RowMapper<StudentDTO> excelRowMapper() {
System.out.println("inside excelRowMapper");
BeanWrapperRowMapper<StudentDTO> rowMapper = new BeanWrapperRowMapper<>();
rowMapper.setTargetType(StudentDTO.class);
return rowMapper;
}
#Bean
public JdbcBatchItemWriter<StudentDTO> writer(DataSource dataSource) {
System.out.println("inside writer");
return new JdbcBatchItemWriterBuilder<StudentDTO>()
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
.sql("INSERT INTO tbl_item_master (mrp_rate,price,item_code) VALUES (:mrp_rate,:rate,:u_item_code)")
//.sql("update ignore tbl_item_master set mrp_rate=:mrp_rate,price=:rate where item_code=:u_item_code")
.dataSource(dataSource)
.build();
}
#Bean
public Job importUserJob(JobCompletionNotificationListener listener, Step step1) {
System.out.println("inside importUserJob");
return jobBuilderFactory.get("importUserJob")
.incrementer(new RunIdIncrementer())
.listener(listener)
.flow(step1)
.end()
.build();
}
#Bean
public Step step1(JdbcBatchItemWriter<StudentDTO> writer,CustomChunkListener customChunkListener,
CustomItemReaderListener customItemReaderListener,CustomItemWriterListener customItemWriterListener,
CustomStepListener customStepListener,CustomSkipListener customSkipListener) throws Exception {
System.out.println("inside step1");
return stepBuilderFactory.get("step1")
.<StudentDTO, StudentDTO>chunk(3000)
.reader(excelStudentReader())
//.processor(processor())
.writer(writer)
.faultTolerant()
.skip(Exception.class)
.noRetry(Exception.class)
.noRollback(Exception.class)
.skipLimit(100)
.listener(customItemWriterListener)
.listener(customSkipListener)
.build();
}
#Bean
public CustomStepListener customStepListener() {
return new CustomStepListener();
}
#Bean
public CustomItemWriterListener itemWriterListener() {
return new CustomItemWriterListener();
}
#Bean
public CustomItemReaderListener itemReaderListener() {
return new CustomItemReaderListener();
}
#Bean
public CustomSkipListener customSkipListener() {
return new CustomSkipListener();
}
}
Controller:
#Controller
public class FileController1 {
#Autowired
private JobLauncher jobLauncher;
#Autowired
private Job importUserJob;
#RequestMapping(value = "/echofile", method = RequestMethod.POST, produces = {"application/json"})
public #ResponseBody String echoFile(MultipartHttpServletRequest request,
HttpServletResponse response) throws Exception {
MultipartFile multipartFile = request.getFile("file");
String path = new ClassPathResource("/").getURL().getPath();
File fileToImport = new File(path + multipartFile.getOriginalFilename());
//filePath = fileToImport.getAbsolutePath();
OutputStream outputStream = new FileOutputStream(fileToImport);
IOUtils.copy(multipartFile.getInputStream(), outputStream);
outputStream.flush();
outputStream.close();
//Launch the Batch Job
JobExecution jobExecution = jobLauncher.run(importUserJob,new JobParametersBuilder()
.addString("fullPathFileName", fileToImport.getAbsolutePath()).addLong("time",System.currentTimeMillis()).toJobParameters());
return "something";
}
}
batch.properties:
spring.batch.job.enabled=false