I'm working at implementing a buffer, in order to execute various jobs.
I'm working on a Spring-based project.
I decided to use Spring Integration to accomplish my aim. I went through a Cafè sample project in order to understand how SI works.
To demonstrate Spring Integration I implemented a table where I dynamically insert jobs to be executed.
This table is the "gateway". Then I configured a router and various channels.
What I don't fully understand are the poller elements which have to check if new jobs in the "gateway" are present.
Is this correct?
If so, how can I configure the poller?
Thanks in advance!
Here the xml code:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-feed="http://www.springframework.org/schema/integration/feed"
xmlns:int-stream="http://www.springframework.org/schema/integration/stream"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration
http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/stream
http://www.springframework.org/schema/integration/stream/spring-integration-stream.xsd">
<int:gateway id="usersToSend" service-interface="it.stasbranger.spidly.rss.UsersToSend" />
<int:channel id="channel" />
<int:channel id="providers"/>
<int:router input-channel="providers" ref="providerRouter" method="resolveProviderChannel"/>
<int:channel id="twitterChannel">
<int:queue capacity="10"/>
</int:channel>
<int:service-activator input-channel="twitterChannel" ref="twitterService" method="updateStatusByProfile"/>
<int:channel id="facebookChannel">
<int:queue capacity="10"/>
</int:channel>
<int:service-activator input-channel="facebookChannel" ref="facebookService" method="updateStatusByProfile"/>
<int:channel id="linkedinChannel">
<int:queue capacity="10"/>
</int:channel>
<int:service-activator input-channel="linkedinChannel" ref="linkedinService" method="writeSlogan2Linkedin"/>
<bean id="twitterService" class="it.social.TwitterService"/>
<bean id="facebookService" class="it.social.FacebookService"/>
<bean id="linkedinService" class="it.social.LinkedinService"/>
<int:poller id="poller" default="true">
</int:poller>
FB
<gateway/>s are not polled, they are "message-driven" in that the caller "sends" a message into the flow using the gateway.
For a polling scenario, use an <int:inbound-channel-adapter/> which polls a method (on the poller's schedule) looking for work to do.
If the method returns null, the poller goes back to sleep (until the next trigger). If the method returns a value, the message is sent to the channel.
Related
In my Spring boot application, I have written below to poll Data from DB .
The data is polled when the Spring boot application is started..
How can I start polling the data on some event?
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-jdbc="http://www.springframework.org/schema/integration/jdbc"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/jdbc http://www.springframework.org/schema/integration/jdbc/spring-integration-jdbc.xsd
http://www.springframework.org/schema/jdbc http://www.springframework.org/schema/jdbc/spring-jdbc-3.0.xsd">
<int:channel id="fromdb"/>
<int:channel id="outChannel">
<int:dispatcher task-executor="executorPool"/>
</int:channel/>
<task:executor id="executorPool" pool-size="10"/>
<int-jdbc:inbound-channel-adapter
channel="fromdb" data-source="dataSource"
max-rows-per-poll="100"
query="SELECT * FROM Items WHERE INVENTORY_STATUS = 0"
update="UPDATE Items SET INVENTORY_STATUS = 1">
<int:poller fixed-delay="4000" />
</int-jdbc:inbound-channel-adapter>
<int:bridge input-channel="fromdb" output-channel="outChannel"/>
<int:service-activator input-channel="outChannel"
ref="jdbcMessageHandler" method="onMessage" />
</beans>
Above configuration start polling when app starts. However, I want polling to when below API is invoked
#RestController
public class FileConsumerController {
private final Logger logger = LoggerFactory.getLogger(FileConsumerController.class);
#GetMapping(value = "/inventoryStatus")
public ResponseEntity<String> getInventoryStatus(#RequestParam("accId") String accId){
// ONLY after this API is invoked, I want above int-jdbc:inbound-channel-adapter to start polling
}
}
ONLY after this API is invoked, I want above int-jdbc:inbound-channel-adapter to start polling. How can this be achieved?
See an answer to this question: How to invoke Spring channel adapter from rest api?
The inbound channel adapter does its logic on a schedule and that happens in a cycle independently of the rest application logic. It really sounds like you'd need to get data from DB when that getInventoryStatus is called. So, a JDBC outbound Gateway with request-reply capabilities is what you need.
any inbound channel adapter is an automatic reaction to data changes in the target store: it does nothing with human initiated events.
See more info in docs:
https://www.enterpriseintegrationpatterns.com/patterns/messaging/PollingConsumer.html
https://docs.spring.io/spring-integration/reference/html/core.html#pollable-message-source
Basically, I am not aware of port before hand. I will make a REST request to a host to send the port number. I am getting the port number, but not able to move ahead as I am not able to set connection factory into inBoundClient with received port number. Please have a look at the code below, to understand the problem.
I have tcp connections defined as below:
<?xml version="1.0" encoding="UTF-8"?>
<context:component-scan base-package="com.tcpclient" />
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-ip="http://www.springframework.org/schema/integration/ip"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/ip http://www.springframework.org/schema/integration/ip/spring-integration-ip.xsd">
<context:annotation-config />
<!--Deserializer for incoming data on the socket -->
<bean
class="org.springframework.integration.ip.tcp.serializer.ByteArraySingleTerminatorSerializer"
id="serializeAndDeserializer">
<constructor-arg type="byte" value="0" />
</bean>
<!-- TCP Client configuration -->
<!-- Channels for communication -->
<int:channel id="tcp-client-input" />
<int:channel id="message" />
<int:channel id="message-serviceActivator" />
<int:gateway id="gateway" service-interface="com.tcpclient.ClientGateway"
default-request-channel="tcp-client-input" default-reply-channel="message" />
<int-ip:tcp-outbound-channel-adapter
id="outBoundClient" channel="tcp-client-input"
retry-interval="60000" auto-startup="false" />
<int-ip:tcp-inbound-channel-adapter
id="inBoundClient" channel="message"
client-mode="true" auto-startup="false" retry-interval="60000" />
<int:object-to-string-transformer
input-channel="message" output-channel="message-serviceActivator" />
<int:service-activator input-channel="message-serviceActivator"
method="onRtlsMessageArrival">
<bean class="com.tcpclient.HandleMessage" />
</int:service-activator>
</beans>
Config manager
In my class, I am trying below:
#component
public class ConfigManager {
#Autowired
#Qualifier("clientFactory")
TcpConnectionFactoryFactoryBean connFactory;
#Autowired
#Qualifier("inBoundClient")
SmartLifecycle inboundClient;
#Autowired
#Qualifier("outBoundClient")
SmartLifecycle outBoundClient;
public void initialize(Boolean canStart) {
try {
if (canStart) {
String portResponse = restTemplate.postForObject(SYSTEM_OPEN_SOCK_URI, openSockeEntity,
String.class);
int portNumber = parsePortFromJson(portResponse);
connFactory.setPort(portNumber);
TcpReceivingChannelAdapter receiver = (TcpReceivingChannelAdapter) inboundClient;
receiver.setConnectionFactory(connFactory);
receiver.start();
EventDrivenConsumer sender = (EventDrivenConsumer) outBoundClient;
sender.start();
}
} catch (Exception e) {
logger.error("Error occured while fetching data.");
}
}
}
But, at the line receiver.setConnectionFactory(connFactory);,
I have compiler error,
The method setConnectionFactory(AbstractConnectionFactory) in the type TcpReceivingChannelAdapter is not applicable for the arguments (TcpConnectionFactoryFactoryBean).
Is there anyway I could set port dynamically.
EDIT: As per Gary's suggestion,
#Autowired
#Qualifier("outboundClient.handler")
TcpSendingMessageHandler outBoundClient;`,
outBoundClient.setConnectionFactory(connFactory);
outBoundClient.setRetryInterval(60000);
outBoundClient.afterPropertiesSet();
outBoundClient.start();
But, I have this below error:
Dispatcher has no subscribers for channel 'application.tcp-client-input'.; nested exception is org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers
when I try gateway.send("ACK"); one of my beans. but I have my outBoundClient having channel="tcp-client-input"
Since the port is set in a constructor, you can't change it after the bean is created.
You also can't change the port on a connection factory bean after is has already created its connection factory.
You need to create the connection factory yourself (new TcpNetClientConnectionFactory(...)) rather than have Spring create it - be sure to call afterPropertiesSet() after creating and configuring it and before you add it to the adapter.
Scenario : poll the database every 'n' seconds and retrieve list. Do some internal business validations.If validations are success only, send the retrieved list to an external system.Receive an acknowledgment response from that system(After receiving this,poller should send next list).Then perform some business operations like calling other systems where polling is not needed.
can any one tell me how this scenario can be handled?
poller.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
xmlns:jdbc="http://www.springframework.org/schema/jdbc" xmlns:int-jdbc="http://www.springframework.org/schema/integration/jdbc"
xmlns:context="http://www.springframework.org/schema/context" xmlns:p="http://www.springframework.org/schema/p"
xmlns:task="http://www.springframework.org/schema/task" xmlns:int-http="http://www.springframework.org/schema/integration/http"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-4.1.xsd
http://www.springframework.org/schema/integration
http://www.springframework.org/schema/integration/spring-integration-4.1.xsd
http://www.springframework.org/schema/integration/http
http://www.springframework.org/schema/integration/http/spring-integration-http-4.1.xsd
http://www.springframework.org/schema/integration/jdbc
http://www.springframework.org/schema/integration/jdbc/spring-integration-jdbc-4.1.xsd
http://www.springframework.org/schema/jdbc
http://www.springframework.org/schema/jdbc/spring-jdbc-4.1.xsd">
<import resource="persistence-config.xml" />
<int:channel id="inchannel">
</int:channel>
<int:channel id="outchannel">
<int:dispatcher task-executor="taskExecutor"/>
</int:channel>
<task:executor id="taskExecutor" pool-size="2"/>
<bean id="poller" class="main.java.com.as.poller.PollerService" />
<int:service-activator input-channel="inchannel"
output-channel="outchannel" ref="poller" method="sendMessage" />
<int-jdbc:inbound-channel-adapter
query="select loyalty_id from TBL_RECEIPT where receipt_status=0"
channel="inchannel" data-source="dataSource" max-rows-per-poll="1">
<int:poller fixed-rate="5000">
</int:poller>
</int-jdbc:inbound-channel-adapter>
</beans>
ackage main.java.com.as.poller;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.stereotype.Component;
#Component
public class PollerService{
public void sendMessage()
{
ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext(
"AS-servlet.xml");
DirectChannel directChannel = (DirectChannel) context.getBean("inchannel");
}
}
I added a direct channel to poll the DB.Also added an executer channel..But i am getting the exception
Error creating bean with name 'org.springframework.integration.config.ServiceActivatorFactoryBean#0': Cannot resolve reference to bean 'executerChannel' while setting bean property 'outputChannel'; nested exception is org.springframework.beans.factory.NoSuchBeanDefinitionException: No bean named 'executerChannel' is defined
rg.springframework.beans.factory.support.BeanDefinitionValueResolver.resolveReference(BeanDefinitionValueResolver.java:359)
at org.springframework.beans.factory.support.BeanDefinitionValueResolver.resolveValueIfNecessary(BeanDefinitionValueResolver.java:108)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyPropertyValues(AbstractAutowireCapableBeanFactory.java:1477)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.populateBean(AbstractAutowireCapableBeanFactory.java:1222)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:537)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:476)
Can anyone please help with this...I have created only PollerService.java and dont have any idea to implement executer channel and task executer
The <poller> does that for you by default if you use fixed-delay option and your downstream flow is single threaded (only DirectChannels). In that case the next polling task (in your case reading DB using JPA) won't be started until the finish of previous one and some time like fixed-delay. See PeriodicTrigger, BTW.
Regarding your "Then perform some business operations". You should just make the next channel after "Receive an acknowledgment response" as an ExecutorChannel to free the polling Thread for the next poll.
You should really read more in the Spring Integration Reference Manual and especially about the Poller.
I am new to RabbitMQ, I just went through Rabbitmq docs (Routing). I am quite confused between Exchange with routing keys. My requirement is , I want to create multiple queues dynamically. Please refer below diagram.
Ex. Lets say If producer create message for consumer c3, then it should go to Exchange and route to Queue 3 only and consume by C3 only.
At present I only require 3 Queues in future this count may increase. so how to deal with this situation too.
Note : I refer this blog Exchange
I have used Spring hibernate along with Rabbitmq.
below code shows Rabbit MQ Listener configuration File.
<?xml version="1.0" encoding="UTF-8" ?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rabbit="http://www.springframework.org/schema/rabbit"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://www.springframework.org/schema/rabbit
http://www.springframework.org/schema/rabbit/spring-rabbit-1.0.xsd">
<rabbit:connection-factory id="connectionFactory" host="xx.xx.10.181" username="admin" password="admin" />
<rabbit:admin connection-factory="connectionFactory" />
<!-- Create Queues -->
<rabbit:queue id="q1" name="q1.queue" durable="true" />
<rabbit:queue id="q2" name="q2.queue" durable="true" />
<rabbit:queue id="q3" name="q3.queue" durable="true" />
<!--create Exchange here -->
<rabbit:direct-exchange id="myExchange" name="MY Exchange">
<rabbit:bindings>
<rabbit:binding queue="q1" key="my.queue.q1" />
<rabbit:binding queue="q2" key="my.queue.q2" />
<rabbit:binding queue="q3" key="my.queue.q3" />
</rabbit:bindings>
</rabbit:direct-exchange>
<!-- instantiate Listeners -->
<bean id="q1Listener" class="in.my.brocker.amqp.Q1Listener" />
<bean id="q2Listener" class="in.my.brocker.amqp.Q2Listener" />
<bean id="q3Listener" class="in.my.brocker.amqp.Q3Listener" />
<!-- glue the listener and myAnonymousQueue to the container-->
<rabbit:listener-container id="myListenerContainer" connection-factory="connectionFactory">
<rabbit:listener ref="q1Listener" queues="q1" />
<rabbit:listener ref="q2Listener" queues="q2" />
<rabbit:listener ref="q3Listener" queues="q3" />
</rabbit:listener-container>
</beans>
In above rabbit-listener-context.xml, I have created 3 Queues along with 3 Listener Classes. My aim is to Queue should be access by defined routing keys. I want to create nth no of queues this dynamically? How can we achieve this?
Im having 2 outbound gateways with similar but not equal data and format. I'd want to use one as alternative in case 1st doesn't reply or reply too long. Also in future I want to have option to choose which one to get data from.(So Id like to have it as separate outbound gateway, I saw examples with retry advices and would like to avoid this variant)
Atm at very simplified schema I use error channel as incoming channel of alternative gateway. Works perfect until alternative gateway fails too, then I receive:
org.springframework.integration.MessagingException: failure occurred in error-handling flow
There's a real reason of this (connection timeout or any) in stacktrace, but it seems not satisfying for me, since i'd like to handle some exception types differently. I'd like to get error message (and exception) from alternative gateway, not wrapped one.
Is there any way to get it right or Im totally wrong in architecture of this?
Looks like you want to achieve a failover pattern. For this purpose you can simply subscribe both endpoints to the same channel, but without load-balancing:
<channel id="input">
<dispatcher load-balancer="none"/>
</channel>
<service-activator input-channel="input" ref="service1"/>
<service-activator input-channel="input" ref="service2"/>
In this case, if the first <service-activator> will fail by some reason, the Message will be delivered to the second one.
And there is no reason to use retry, or try to reestablish flow from error handling.
UPDATE:
To skip some exceptions and do failover to the next subscriber you can use some custom RequestHandlerAdvice(Expression Evaluating Advice):
class ExceptionProviderRequestHandlerAdvice extends ExpressionEvaluatingRequestHandlerAdvice {
#Override
protected Object doInvoke(AbstractRequestHandlerAdvice.ExecutionCallback callback, Object target, Message<?> message) throws Exception {
def result = super.doInvoke(callback, target, message)
if (result in Exception) {
throw result
}
return result
}
}
And your onFailureExpression should decide: to return exception for rethrow or not:
<int:request-handler-advice-chain>
<beans:bean class="com.cs.otppr.core.aop.ExceptionProviderRequestHandlerAdvice">
<beans:property name="returnFailureExpressionResult" value="true"/>
<beans:property name="onFailureExpression"
value="#exception instanceof T (com.my.proj.MyException) ? #exception : null"/>
</beans:bean>
</int:request-handler-advice-chain>
Or, of course, you can try to find solution on your own...
found a solution that suits me, not sure its proper one, but for now it satisfies all suggested future requirements. I separated my outbound gateways to 2 segments with independent error channels, where error channel of first is input of second. I can process errors from both outbounds, route exceptions and as seems to me easily add some logic futher.
In case of both outbound gateway fails there will be only last gateway exception recieved but that suits me since i dont need to aggregate those errors.
More suggestions are gladly accepted of course.
app-config.xml looks like this
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:elasticsearch="http://www.pilato.fr/schema/elasticsearch"
xmlns:int-http="http://www.springframework.org/schema/integration/http"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:cache="http://www.springframework.org/schema/cache"
xsi:schemaLocation="http://www.springframework.org/schema/integration/http http://www.springframework.org/schema/integration/http/spring-integration-http-3.0.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration-3.0.xsd
http://www.pilato.fr/schema/elasticsearch http://www.pilato.fr/schema/elasticsearch/elasticsearch-0.2.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd
http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache-3.2.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd">
<int-http:inbound-gateway id="inboundController"
request-channel="requests"
reply-channel="replies-f"
path="/item_info"
supported-methods="GET"
payload-expression="#requestParams"
>
<int-http:request-mapping
params="id"/>
</int-http:inbound-gateway>
<int-http:outbound-gateway id="simpleGateway"
request-channel="outboundRequests"
reply-channel="replies-f"
http-method="GET"
expected-response-type="com.dph.iteminfo.transformer.ItemInfoDTO"
url="http://10.5.4.134:8080/ess/iteminfo?id={itemId}&group_size={groupSize}"
>
<int-http:uri-variable name="itemId" expression="payload.get('id')[0]"/>
<int-http:uri-variable name="groupSize" expression="payload.containsKey('group_size') ? payload.get('group_size')[0] : 5"/>
</int-http:outbound-gateway>
<int-http:outbound-gateway id="alternativeSource"
request-channel="altOutboundChannel"
reply-channel="replies-f"
http-method="GET"
expected-response-type="java.lang.String"
url="http://10.5.4.134:8080/ess/iteminfo?id={itemId}&len={groupSize}"
>
<int-http:uri-variable name="itemId" expression="payload.get('id')[0]"/>
<int-http:uri-variable name="groupSize" expression="payload.containsKey('group_size') ? payload.get('group_size')[0] : 5"/>
</int-http:outbound-gateway>
<int:service-activator input-channel="requests" ref="outboundMain"/>
<int:gateway id="outboundMain" default-request-channel="outboundRequests" error-channel="altChannel" />
<int:exception-type-router default-output-channel="replies-f" input-channel="altChannel">
<int:mapping exception-type="org.springframework.web.client.HttpServerErrorException" channel="resendableMessagesChannel"/>
<int:mapping exception-type="org.springframework.web.client.ResourceAccessException" channel="resendableMessagesChannel"/>
</int:exception-type-router>
<int:service-activator input-channel="resendableMessagesChannel" ref="resender"/>
<int:gateway id="resender" default-request-channel="processedErrorsChannel" error-channel="finalErrors"/>
<int:transformer input-channel="processedErrorsChannel" output-channel="altOutboundChannel" expression="payload.getFailedMessage().getPayload()" />
<int:bridge input-channel="finalErrors" output-channel="replies-f"/>
<!-- will be error processing here -->
<int:channel id="finalErrors" />
<int:channel id="requests" />
<int:channel id="replies-f" />
</beans>