Develop Reference

Kinesis 2.2.11 java unable to create consumer

I need to migrate to Kinesis library to version 2.2.11 so I followed the tutorial: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration.html
I need to run multiple instances of my consumer app, so every one of them needs to have an unique application name in order to have a separate lease table in DynamoDb.
When initializing the consumer Kinesis runs DynamoDBLeaseRefresher.createLeaseTableIfNotExists which checks if a new table needs to be created for this application name and creates one if it cannot be found.
So 2 operations are performed:
DescribeTable - it returns the table info or throws a ResourceNotFoundExecption,
if needed - CreateTable.
The problem for me is with the DescribeTable method. When I am looking for an existing table it returns it with no problem. But when I am looking for a non-existent table it throws the ResourceNotFoundExecption -> so far so good. Unfortunately it then gets wrapped and is now:
java.util.concurrent.CompletionException: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: software.amazon.awssdk.awscore.exception.AwsServiceException$Builder.extendedRequestId(Ljava/lang/String;)Lsoftware/amazon/awssdk/awscore/exception/AwsServiceException$Builder;
and the app expecting ResourceNotFoundException gets something different instead and crashes.
The wrapped exception message is a bit misleading: "Unable to execute HTTP request" since the request was performed and returned the proper message: "Resource not found".
Funny thing is that it sometimes works, the exception does not get wrapped, the CreateTable operation is performed and the consumer starts properly.
I have made a workaround for it for now where I just create the table before the initialization of the LeaseCoordinator, so it always gets the existing table.
here is my code:
public KinesisStreamReaderService(String streamName, String applicationName, String regionName) {
KinesisAsyncClient kinesisClient = KinesisAsyncClient.builder()
.credentialsProvider(EnvironmentVariableCredentialsProvider.create())
.region(Region.of(connectionProperties.getRegion()))
.httpClientBuilder(createHttpClientBuilder())
.build();
DynamoDbAsyncClient dynamoClient = DynamoDbAsyncClient.builder().region(Region.of(regionName)).build();
CloudWatchAsyncClient cloudWatchClient = CloudWatchAsyncClient.builder().region(Region.of(regionName)).build();
// if(!dynamoDbTableExists(dynamoClient, applicationName)) {
// createDynamoDbTable(dynamoClient, applicationName);
// }
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisClient,
dynamoClient, cloudWatchClient, workerId(), KinesisReaderProcessor::new);
configsBuilder.retrievalConfig().initialPositionInStreamExtended(
InitialPositionInStreamExtended.newInitialPosition(
InitialPositionInStream.LATEST));
scheduler = new Scheduler(
configsBuilder.checkpointConfig(),
configsBuilder.coordinatorConfig(),
configsBuilder.leaseManagementConfig(),
configsBuilder.lifecycleConfig(),
configsBuilder.metricsConfig(),
configsBuilder.processorConfig(),
configsBuilder.retrievalConfig().retrievalSpecificConfig(new PollingConfig(streamName, kinesisClient))
);
}
private void createDynamoDbTable(DynamoDbAsyncClient dynamoClient, String applicationName) {
log.info("Creating new lease table: {}", applicationName);
CompletableFuture<CreateTableResponse> createTableFuture = dynamoClient
.createTable(CreateTableRequest.builder()
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build())
.tableName(applicationName)
.keySchema(KeySchemaElement.builder().attributeName("leaseKey").keyType(KeyType.HASH).build())
.attributeDefinitions(AttributeDefinition.builder().attributeName("leaseKey").attributeType(
ScalarAttributeType.S).build())
.build());
try {
CreateTableResponse createTableResponse = createTableFuture.get();
log.debug("Created new lease table: {}", createTableResponse.tableDescription().tableName());
} catch (InterruptedException | ExecutionException e) {
throw new DataStreamException(e.getMessage(), e);
}
}
private boolean dynamoDbTableExists(DynamoDbAsyncClient dynamoClient, String tableName) {
CompletableFuture<DescribeTableResponse> describeTableResponseCompletableFutureNew = dynamoClient
.describeTable(DescribeTableRequest.builder()
.tableName(tableName).build());
try {
DescribeTableResponse describeTableResponseNew = describeTableResponseCompletableFutureNew
.get();
return nonNull(describeTableResponseNew);
} catch (InterruptedException | ExecutionException e) {
log.info(e.getMessage(), e);
}
return false;
}
private static String workerId() {
String workerId;
try {
workerId = format("%s_%s", getLocalHost().getCanonicalHostName(), randomUUID().toString());
} catch (UnknownHostException e) {
workerId = randomUUID().toString();
}
return workerId;
}
#Override
public void read(Consumer<String> consumer) {
this.consumer = consumer;
scheduler.run();
}
private class KinesisReaderProcessor implements ShardRecordProcessor {
private String shardId;
#Override
public void initialize(InitializationInput initializationInput) {
this.shardId = initializationInput.shardId();
log.info("Initializing record processor for shard: {}", shardId);
}
#Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
log.debug("Checking shard {} for new records", shardId);
List<KinesisClientRecord> records = processRecordsInput.records();
if (!records.isEmpty()) {
log.debug("Processing {} records from kinesis stream shard {}", records.size(), shardId);
records.forEach(record -> {
String json = UTF_8.decode(record.data()).toString();
log.info(json);
consumer.accept(json);
});
}
}
#Override
public void leaseLost(LeaseLostInput leaseLostInput) {
log.info("Record processor has lost lease, terminating");
}
#Override
public void shardEnded(ShardEndedInput shardEndedInput) {
try {
shardEndedInput.checkpointer().checkpoint();
} catch (ShutdownException | InvalidStateException e) {
log.error(e.getMessage(), e);
}
}
#Override
public void shutdownRequested(ShutdownRequestedInput shutdownRequestedInput) {
try {
shutdownRequestedInput.checkpointer().checkpoint();
} catch (ShutdownException | InvalidStateException e) {
log.error(e.getMessage(), e);
}
}
}
}
Am I missing some configuration for the scheduler or something? Why is it sometimes working?
Thanks
Edit:
The problem is this block of code in DynamoDBLeaseRefresher.tableStatus() is invoked to check if the table exists:
DescribeTableResponse result;
try {
try {
result =
(DescribeTableResponse)FutureUtils.resolveOrCancelFuture(this.dynamoDBClient.describeTable(request), this.dynamoDbRequestTimeout);
} catch (ExecutionException var5) {
throw exceptionManager.apply(var5.getCause());
} catch (InterruptedException var6) {
throw new DependencyException(var6);
}
} catch (ResourceNotFoundException var7) {
log.debug("Got ResourceNotFoundException for table {} in leaseTableExists, returning false.", this.table);
return null;
}
and in my case it should get ResourceNotFoundException if the table is not found, but as I said the expection gets wrapped to CompletionException before it reaches the appropriate catch block and is caught in the code here:
catch (ExecutionException var5) {
throw exceptionManager.apply(var5.getCause());
This is happening 20 times in the loop while trying to Initialize the LeaseCoordinator and then just stops trying to initialize the connection. (As mentioned above it works occasionally, but that makes it even stranger to me)
With my workaround it only needs 1 try to get initialized

You don't need to create a lease table manually - DynamoDBLeaseCoordinator will create one if not exists on initialization and wait until it exists:
#Override
public void initialize() throws ProvisionedThroughputException, DependencyException, IllegalStateException {
final boolean newTableCreated =
leaseRefresher.createLeaseTableIfNotExists(initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity);
if (newTableCreated) {
log.info("Created new lease table for coordinator with initial read capacity of {} and write capacity of {}.",
initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity);
}
// Need to wait for table in active state.
final long secondsBetweenPolls = 10L;
final long timeoutSeconds = 600L;
final boolean isTableActive = leaseRefresher.waitUntilLeaseTableExists(secondsBetweenPolls, timeoutSeconds);
if (!isTableActive) {
throw new DependencyException(new IllegalStateException("Creating table timeout"));
}
}
The issue in your case, I think, is that it's eventually created and you probably should periodically check until table appears - like DynamoDBLeaseCoordinator#initialize() does.

StateMachineService.acquireStateMachine shouldn't start stateMachine

On acquiring a state machine with stateMachineService the machine is started, but I passed 'false' as a second parameter.
stateMachine = stateMachineService.acquireStateMachine(id, false)
According to console output 'acquireStateMachine' starts the machine.
I'm using DefaultStateMachineService
#Bean
public StateMachineService<BookingItemState, BookingItemEvent> stateMachineService(
StateMachineFactory<BookingItemState, BookingItemEvent> stateMachineFactory,
StateMachineRuntimePersister<BookingItemState, BookingItemEvent, String> stateMachineRuntimePersister) {
return new DefaultStateMachineService<>(stateMachineFactory, stateMachineRuntimePersister);
}

The issue is in DefaultStateMachineService class. I suppose that you have configured SM as below enabling autoStartap property:
#Override
public void configure(StateMachineConfigurationConfigurer<String, String> config) throws Exception {
config
.withConfiguration()
.autoStartup(true);
}
If you call acquireStateMachine the DefaultStateMachineService creates a new SM using stateMachineFactory (but you SM has enabled autoStartup) it starts a new SM and stores it to DB.
Let's consider the metnod:
public StateMachine<S, E> acquireStateMachine(String machineId, boolean start) {
log.info("Acquiring machine with id " + machineId);
StateMachine<S, E> stateMachine;
// naive sync to handle concurrency with release
synchronized (machines) {
stateMachine = machines.get(machineId);
if (stateMachine == null) {
log.info("Getting new machine from factory with id " + machineId);
stateMachine = stateMachineFactory.getStateMachine(machineId);
if (stateMachinePersist != null) {
try {
StateMachineContext<S, E> stateMachineContext = stateMachinePersist.read(machineId);
stateMachine = restoreStateMachine(stateMachine, stateMachineContext);
} catch (Exception e) {
log.error("Error handling context", e);
throw new StateMachineException("Unable to read context from store", e);
}
}
machines.put(machineId, stateMachine);
}
}
// handle start outside of sync as it might take some time and would block other machines acquire
return handleStart(stateMachine, start);
}
To avoid this issue you may disable autoStartup option or implement you custom StateMachineService. But then you have to explicitly call stateMachine.start().

getJobDataMap in Quartz gives Null Pointer exception

I am trying to put some data into Quartz job data map and access them in the class which implements the Job class. But it gives me the Null Pointer exception. When the application is run without the code which access the Job Data Map, it runs fine.
I use a Cron trigger to execute a scheduled job. In this example case, I configured it to run in each 20 seconds.
#Bean
public Trigger simpleJobTrigger(#Qualifier("simpleJobDetail") JobDetail jobDetail) {
CronTriggerFactoryBean factoryBean = new CronTriggerFactoryBean();
factoryBean.setJobDetail(jobDetail);
factoryBean.setStartDelay(0L);
factoryBean.setName("test-trigger");
factoryBean.setStartTime(LocalDateTime.now().toDate());
factoryBean.setCronExpression("0/20 * * * * ?");
factoryBean.setMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW);
try {
factoryBean.afterPropertiesSet();
} catch (ParseException e) {
e.printStackTrace();
}
return factoryBean.getObject();
}
Following is my simpleJobDetail bean.
#Bean
public JobDetailFactoryBean simpleJobDetail() {
JobDetailFactoryBean factoryBean = new JobDetailFactoryBean();
factoryBean.setJobClass(Executor.class);
factoryBean.setDurability(true);
factoryBean.setName("test-job");
factoryBean.getJobDataMap().put("caller", "James");
return factoryBean;
}
This is my execute method.
public class Executor implements Job {
#Autowired
ScheduledTaskService scheduledTaskService;
#Override
public void execute(JobExecutionContext jobExecutionContext) {
JobDataMap jobDataMap = null;
try {
jobDataMap = jobExecutionContext.getTrigger().getJobDataMap();
String caller = jobDataMap.get("caller").toString();
System.out.println("This is called by the user "+caller);
} catch (Exception e) {
//e.printStackTrace();
System.out.println("UNABLE TO ACCESS THE JOB DATA MAP "+e);
}
scheduledTaskService.doThePayment();
}
}
When I run the application, it prints the log given in the catch clause.
UNABLE TO ACCESS THE JOB DATA MAP java.lang.NullPointerException
Why execute method fails to access the JobDataMap ? Is there any configuration or a property I should set? What is the reason for job map to be not available at this point?
How can I get this resolved?

Found the issue in my code.
When accessing the JobDataMap, I have accessed it in following way.
jobDataMap = jobExecutionContext.getTrigger().getJobDataMap();
instead, in my case, I should access it from JobDetails, not from the trigger.
jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();

Insert data after completing job in Quartz scheduler using spring

I'm working on a task in which I need to save job status in database but when I tried to do this it gives me java.lang.nullpointerexception. I think it happen because whenever I tried to select/save/update any record from the database only at that time it gives me error similar to this.
Here is my code
public class PostFBJob implements Job {
private SchedulerService schedulerService;
private SubCampaignService subCampaignService;
#SuppressWarnings("unchecked")
public void execute(JobExecutionContext context) throws JobExecutionException {
JobDetail jobDetail = context.getJobDetail();
JobDataMap jobDataMap = jobDetail.getJobDataMap();
schedulerService = (SchedulerService) jobDataMap.get("schedulerService");
SubCampaign subCampaign = (SubCampaign) jobDataMap.get("subCampaign");
if (prStreamItem.getName().equalsIgnoreCase("Facebook") && StringUtils.isNotBlank(branch.getFbAccessToken())) {
FacebookService facebookService = FacebookService.getSingleton();
try {
subCampaign.setStatus("Completed");
subCampaign.setMessage("Completed");
subCampaignService.updateSubCampaign(subCampaign);
} catch (Exception e) {
log.error("", e);
}
}
}
}
Exception
java.lang.NullPointerException
at com.ace.Job.SubCampaignJob.execute(SubCampaignJob.java:147)
at org.quartz.core.JobRunShell.run(JobRunShell.java:213)
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557)
Error: java.lang.NullPointerException
Please help me to solve this issue. I'm new to Spring and Quartz.
Thanks in advance

This exception due to null object of subCampaignService.
Injection work only those objects which are managed by spring.
To scan your all packages where you want to use Service.
Annotation based configuration
#ComponentScan({"com.ace"})
XML based configuration
<context:component-scan base-package="com.ace"/>
And also check null before update like below:
try {
subCampaign.setStatus("Completed");
subCampaign.setMessage("Completed");
if(subCampaign != null && subCampaignService != null) //Check is not null to subCampaign before update
subCampaignService.updateSubCampaign(subCampaign);
} catch (Exception e) {
log.error("", e);
}

I think the answer you are looking for is here
Please check below link
Correct way to persist Quartz triggers in database
And you can also check this link
Using Hibernate session with quartz

Spring batch restrict single instance of job only

I have one spring batch job which can be kicked of by rest URL. I want to make sure only one job instance is allowed to run. and if another instance already running then don't start another. even if the parameters are different.
I searched and found nothing out of box solution. thinking of extending SimpleJobLauncher. to check if any instance of the job running or not.

You could try to intercept the job execution, implementing the JobExecutionListener interface:
public class MyJobExecutionListener extends JobExecutionListener {
//active JobExecution, used as a lock.
private JobExecution _active;
public void beforeJob(JobExecution jobExecution) {
//create a lock
synchronized(jobExecution) {
if(_active!=null && _active.isRunning()) {
jobExecution.stop();
} else {
_active=jobExecution;
}
}
}
public void afterJob(JobExecution jobExecution) {
//release the lock
synchronized(jobExecution) {
if(jobExecution==_active) {
_active=null;
}
}
}
}
And then, inject to the Job definition:
<job id="myJobConfig">
<listeners>
<listener ref="myListener"/>
</listeners>
</job>

I solved this by creating an JobExecutionListner and with the help of JobExplorer I checked if any other instance is running if running then stop current job.I created listener so that it can be plugged in to any job that requires this kind of scenario.
Set<JobExecution> jobExecutions = ((SimpleJobExplorer) jobExplorer.getObject()).findRunningJobExecutions(jobExecution.getJobInstance().getJobName());
if(jobExecutions.size()>1){
Long currentTime = (new Date()).getTime();
for(JobExecution execution : jobExecutions ){
if(execution.getJobInstance().getId().compareTo(jobExecution.getJobInstance().getId())!=0 && (currentTime - execution.getStartTime().getTime()) <lockOverideTime){
jobExecution.stop();
throw new IllegalStateException("Another instance of the job running job name : " +jobExecution.getJobInstance().getJobName() );
}
}
}

Or, in response to REST URL, check using JobExplorer if your job is running using job's specifics business rules

I think a simple method like the following might do the trick:
#Autowire
private JobExplorer jobExplorer;
private boolean isJobRunning(Job job) {
Set<JobExecution> jobExecutions = jobExplorer.findRunningJobExecutions(job.getName());
return !jobExecutions.isEmpty();
}
Then, prior to executing your job make the check:
private void executeJob(Job job, #Nonnull JobParameters params) {
if (isJobRunning(job)) {
return;
}
try {
jobLauncher.run(job, params);
} catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException | JobParametersInvalidException e) {
log.error("could not run job " + jobIdentifier, e);
}
}