Develop Reference

Apache Kafka: Configuring `ListSerde` for `TopologyTestDriver`

I am writing a unit test for a custom suppress. The class is called SupressProcessor. The unit test looks like:
import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig
import no.statnett.observations.grid.flow.CorridorFlow.avro.v1.CorridorFlow
import my.stream.SuppressProcessor
import my.stream.TopicSerdeConfig
import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.streams.StreamsConfig
import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.processor.api.ProcessorSupplier
import org.apache.kafka.streams.state.Stores
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.TestInstance
import testUtils.RecordFactory
import java.time.Duration
import java.util.Properties
#TestInstance(TestInstance.Lifecycle.PER_CLASS)
class SuppressProcessorTests {
// Following https://www.ru-rocker.com/2020/12/07/how-to-unit-test-kafka-streams-application-part-2-processor-api/
private lateinit var topology: Topology
private lateinit var testDriver: TopologyTestDriver
private val schemaRegistryScope = SuppressProcessorTests::class.java.name
private val schemaRegistryUrl = "mock://$schemaRegistryScope"
private val inputTopicName = "input_topic"
private val outputTopicName = "output_topic"
private val topicSerde = TopicSerdeConfig(inputTopicName, outputTopicName, schemaRegistryUrl)
private val sourceName = "source"
private val processorName = "suppress-processor"
private val stateStoreName = "suppress-processor-store"
private val recordFactory = RecordFactory()
#BeforeEach
fun setup() {
val config = Properties()
config[StreamsConfig.APPLICATION_ID_CONFIG] = "suppress-processor-test"
config[StreamsConfig.BOOTSTRAP_SERVERS_CONFIG] = ""
config[StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG] = topicSerde.stringKeySerde::class.java
config[StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG] = "org.apache.kafka.common.serialization.Serdes.ListSerde"
config[CommonClientConfigs.DEFAULT_LIST_VALUE_SERDE_INNER_CLASS] = topicSerde.inputValueSerde.javaClass
config[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = schemaRegistryUrl
val storeBuilder = Stores
.timestampedKeyValueStoreBuilder(
Stores.inMemoryKeyValueStore(stateStoreName),
topicSerde.stringKeySerde, topicSerde.corridorFlowListSerde
)
.withLoggingDisabled()
topology = Topology()
topology.addSource(sourceName, topicSerde.stringKeySerde.deserializer(), topicSerde.corridorFlowListSerde.deserializer(), inputTopicName)
topology.addProcessor(processorName, ProcessorSupplier { SuppressProcessor(stateStoreName) },sourceName)
topology.addStateStore(storeBuilder, processorName)
topology.addSink("sink", outputTopicName, processorName)
testDriver = TopologyTestDriver(topology, config)
}
#Test
fun `Should schedule as expected`(){
val inputTopic = testDriver.createInputTopic(
inputTopicName,
topicSerde.stringKeySerde.serializer(),
topicSerde.corridorFlowListSerde.serializer()
)
val outputTopic = testDriver.createOutputTopic(
outputTopicName,
topicSerde.stringKeySerde.deserializer(),
topicSerde.corridorFlowListSerde.deserializer()
)
val corridorFlow = recordFactory.generateCorridorFlow()
val corridorFlows = listOf<CorridorFlow>(corridorFlow, corridorFlow)
inputTopic.pipeInput(corridorFlow.mrid, corridorFlows)
assertThat(outputTopic.isEmpty).isTrue
testDriver.advanceWallClockTime(Duration.ofSeconds(3))
assertThat(outputTopic.isEmpty).isFalse
}
}
The SupressProcessor looks like:
import net.logstash.logback.marker.Markers
import my.App
import my.avro.CorridorFlow
import org.apache.kafka.streams.processor.PunctuationType
import org.apache.kafka.streams.processor.api.ContextualProcessor
import org.apache.kafka.streams.processor.api.ProcessorContext
import org.apache.kafka.streams.processor.api.Record
import org.apache.kafka.streams.state.TimestampedKeyValueStore
import org.apache.kafka.streams.state.ValueAndTimestamp
import java.time.Duration
import java.time.Instant
class SuppressProcessor(private val stateStoreName: String) :
ContextualProcessor<String, MutableList<CorridorFlow>, String, MutableList<CorridorFlow>>() {
private lateinit var store: TimestampedKeyValueStore<String, MutableList<CorridorFlow>>
override fun init(context: ProcessorContext<String, MutableList<CorridorFlow>>?) {
super.init(context)
this.store = context().getStateStore(stateStoreName)
context().schedule(Duration.ofSeconds(1), PunctuationType.WALL_CLOCK_TIME) { schedule() }
}
override fun process(record: Record<String, MutableList<CorridorFlow>>) {
App.log.debug(
Markers.append("SuppressedValue", record.value()),
"Suppress received key=${record.key()} receivedAt=${context().currentSystemTimeMs()}"
)
store.put(record.key(), ValueAndTimestamp.make(record.value(), getExpirationTimestamp()))
}
private fun getExpirationTimestamp(timestamp: Long = context().currentSystemTimeMs()): Long =
timestamp + Duration.ofSeconds(2).toMillis()
private fun schedule(currentTimestamp: Long = context().currentSystemTimeMs()) {
val expiredKeys = mutableListOf<String>()
val iterator = store.all()
iterator.forEach { aggregate ->
App.log.info(aggregate.toString())
App.log.info("currentTimestamp: $currentTimestamp")
App.log.info("aggregateTimestamp: ${aggregate.value.timestamp()}")
if (aggregate.value.timestamp() >= currentTimestamp) return#forEach
App.log.info(aggregate.toString())
App.log.debug(
Markers.append("Suppress", aggregate.value),
"Suppress release key=${aggregate.key} releasedAt=${currentTimestamp}"
)
val recordTimestamp = Instant.parse(aggregate.value.value().first().timeInterval.end).toEpochMilli()
context().forward(Record(aggregate.key, aggregate.value.value(), recordTimestamp))
expiredKeys.add(aggregate.key)
}
iterator.close()
context().commit()
expiredKeys.forEach { store.delete(it) }
}
}
And the Serde configuration file looks like:
import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig;
import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde;
import my.avro.BiddingZoneFlow;
import my.avro.CorridorFlow;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class TopicSerdeConfig {
public final String inputTopic;
public final String outputTopic;
public final String schemaRegistryUrl;
public final Serde<String> stringKeySerde;
public final SpecificAvroSerde<CorridorFlow> inputValueSerde;
public final Serde<List<CorridorFlow>> corridorFlowListSerde;
public final SpecificAvroSerde<BiddingZoneFlow> outputValueSerde;
public TopicSerdeConfig(String inputTopic, String outputTopic, String schemaRegistryUrl) {
this.inputTopic = inputTopic;
this.outputTopic = outputTopic;
this.schemaRegistryUrl = schemaRegistryUrl;
var serdeConfig = Collections.singletonMap(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, schemaRegistryUrl);
// Configure input serdes
stringKeySerde = Serdes.String();
stringKeySerde.configure(serdeConfig, true);
inputValueSerde = new SpecificAvroSerde<>();
inputValueSerde.configure(serdeConfig, false);
// Configure aggregation serdes
corridorFlowListSerde = Serdes.ListSerde(ArrayList.class, inputValueSerde);
// Configure output serdes
outputValueSerde = new SpecificAvroSerde<>();
outputValueSerde.configure(serdeConfig, false);
}
}
Note that this file is written in Java as opposed to Kotlin which is used in the other files.
When running the unit test, the following error message occurs:
rg.apache.kafka.common.config.ConfigException: Invalid value org.apache.kafka.common.serialization.Serdes.ListSerde for configuration default.value.serde: Class org.apache.kafka.common.serialization.Serdes.ListSerde could not be found.
at org.apache.kafka.common.config.ConfigDef.parseType(ConfigDef.java:744)
at org.apache.kafka.common.config.ConfigDef.parseValue(ConfigDef.java:490)
at org.apache.kafka.common.config.ConfigDef.parse(ConfigDef.java:483)
at org.apache.kafka.common.config.AbstractConfig.<init>(AbstractConfig.java:113)
at org.apache.kafka.common.config.AbstractConfig.<init>(AbstractConfig.java:146)
at org.apache.kafka.streams.StreamsConfig.<init>(StreamsConfig.java:1235)
at org.apache.kafka.streams.processor.internals.ClientUtils$QuietStreamsConfig.<init>(ClientUtils.java:55)
at org.apache.kafka.streams.TopologyTestDriver.<init>(TopologyTestDriver.java:320)
at org.apache.kafka.streams.TopologyTestDriver.<init>(TopologyTestDriver.java:299)
at org.apache.kafka.streams.TopologyTestDriver.<init>(TopologyTestDriver.java:275)
at my.SuppressProcessorTests.setup(SuppressProcessorTests.kt:66)
Any idea on how to configure ListSerde for unit tests using TopologyTestDriver?
EDIT: Added pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>parent-poms</groupId>
<artifactId>parent</artifactId>
<version>7.0.1</version>
<relativePath/> <!-- http://maven.apache.org/ref/3.0.3/maven-model/maven.html#class_parent -->
</parent>
<groupId>application</groupId>
<artifactId>flow-application</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<name>flow-application</name>
<description>flow-application</description>
<properties>
<ktor.version>2.2.1</ktor.version>
<confluent.version>7.3.0</confluent.version>
<kafka.version>3.3.1</kafka.version>
<prometeus.version>1.10.2</prometeus.version>
<avro.version>1.11.1</avro.version>
<observations-avro.version>2.0.12</observations-avro.version>
<logback-classic.version>1.4.5</logback-classic.version>
<logstash-logback-encoder.version>7.2</logstash-logback-encoder.version>
<slf4j.version>2.0.6</slf4j.version>
<!-- Test -->
<junit-jupiter.version>5.9.1</junit-jupiter.version>
<kotlinx-coroutines-debug.version>1.6.4</kotlinx-coroutines-debug.version>
<assertj-core.version>3.23.1</assertj-core.version>
</properties>
<dependencies>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-schema-registry-client</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-streams-avro-serde</artifactId>
<version>${confluent.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>${prometeus.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>${prometeus.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
</dependency>
<dependency>
<groupId>no.statnett.schemas.business</groupId>
<artifactId>observations-avro</artifactId>
<version>${observations-avro.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-core-jvm</artifactId>
<version>${ktor.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-netty-jvm</artifactId>
<version>${ktor.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-metrics-jvm</artifactId>
<version>${ktor.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-content-negotiation-jvm</artifactId>
<version>${ktor.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-serialization-gson-jvm</artifactId>
<version>${ktor.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<!-- Logging -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback-classic.version}</version>
</dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>${logstash-logback-encoder.version}</version>
</dependency>
<!-- Testing -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit-jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>${junit-jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams-test-utils</artifactId>
<version>${kafka.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-tests-jvm</artifactId>
<version>${ktor.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlinx</groupId>
<artifactId>kotlinx-coroutines-debug</artifactId>
<version>${kotlinx-coroutines-debug.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj-core.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

The solution was to change the config of the TopologyTestDriver to the following
val config = Properties()
config[StreamsConfig.APPLICATION_ID_CONFIG] = "suppress-processor-test"
config[StreamsConfig.BOOTSTRAP_SERVERS_CONFIG] = ""
config[StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG] = topicSerde.stringKeySerde::class.java
config[StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG] = topicSerde.corridorFlowListSerde.javaClass
config[CommonClientConfigs.DEFAULT_LIST_VALUE_SERDE_TYPE_CLASS] = topicSerde.inputValueSerde.javaClass
config[CommonClientConfigs.DEFAULT_LIST_VALUE_SERDE_INNER_CLASS] = topicSerde.inputValueSerde.javaClass
config[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = schemaRegistryUrl

"mvn test" 0 tests run - spring + cucumber + junit5

I have a test project where I'm trying to setup e2e api tests using rest-assured. Tests run fine if I run them from the feature files, however, when I try to run them with maven, 0 tests run. I believe there is something funky with my pom.xml but I can't figure it out...
My project structure looks like:this
My pom.xml:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.0</version>
</parent>
<artifactId>qa-automation-example</artifactId>
<version>0.0.1-SNAPSHOT</version>
<groupId>com</groupId>
<packaging>jar</packaging>
<properties>
<java.version>17</java.version>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-bom</artifactId>
<version>7.2.3</version>
<scope>import</scope>
<type>pom</type>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-suite</artifactId>
<version>1.8.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit-platform-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-spring</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>json-path</artifactId>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>xml-path</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<properties>
<configurationParameters>
cucumber.junit-platform.naming-strategy=long
</configurationParameters>
</properties>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
My Application.java
package com;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.PropertySource;
import org.springframework.context.annotation.PropertySources;
#PropertySources({
#PropertySource("classpath:application.properties")
})
#SpringBootApplication
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}
My CucumberSpringConfiguration.class
import io.cucumber.spring.CucumberContextConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import com.Application;
#CucumberContextConfiguration
#SpringBootTest(classes = Application.class)
public class CucumberSpringConfiguration {
}
My CucumberTest.java
import org.junit.platform.suite.api.ConfigurationParameter;
import org.junit.platform.suite.api.IncludeEngines;
import org.junit.platform.suite.api.SelectClasspathResource;
import org.junit.platform.suite.api.Suite;
import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME;
#Suite
#IncludeEngines("cucumber")
#SelectClasspathResource("src/test/resources/example")
#ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "com.example")
public class CucumberTest {
}
I'm not really familiar with Spring though so I'm pretty sure I'm not using it correctly in my ApiTestStepDef.java
package com.example;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
import io.cucumber.java.en.Given;
import io.restassured.response.Response;
import io.restassured.specification.RequestSpecification;
import org.junit.jupiter.api.Assertions;
import org.springframework.beans.factory.annotation.Autowired;
import com.client.RestAssuredClient;
import com.model.User;
import com.utils.Helper;
import static io.restassured.RestAssured.given;
public class ApiTestStepDef {
private Response response;
private RequestSpecification request;
private User user;
private User responseBody;
#Autowired
private RestAssuredClient restAssuredClient;
#Given("{string} endpoint")
public void setBaseUsersURI(String url){
request =
given().log().all().
spec(restAssuredClient.createReqSpec(url));
}
#When("user posts request with details {string} {string} {string}")
public void sendRequest(String name, String gender, String status){
user = new User(name, gender, Helper.createRandomEmail(), status);
response =
request.given().log().all().
body(user).
when().
post().
then().log().all().
extract().response();
}
#Then("response status code is {int} and response contains correct user details")
public void checkResponseStatusCode(int statusCode){
response.then().spec(restAssuredClient.createResSpec(statusCode));
responseBody = response.getBody().as(User.class);
Assertions.assertEquals(user.getGender(), responseBody.getGender());
Assertions.assertEquals(user.getStatus(), responseBody.getStatus());
Assertions.assertEquals(user.getEmail(), responseBody.getEmail());
Assertions.assertEquals(user.getName(), responseBody.getName());
}
}
And RestAssuredClient.java
package com.client;
import io.restassured.builder.RequestSpecBuilder;
import io.restassured.builder.ResponseSpecBuilder;
import io.restassured.http.ContentType;
import io.restassured.specification.RequestSpecification;
import io.restassured.specification.ResponseSpecification;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import static io.restassured.RestAssured.oauth2;
#Component
public class RestAssuredClient {
#Value("${access.token}")
private String accessToken;
#Value("${base.uri}")
private String baseUri;
public ResponseSpecification createResSpec(int statusCode){
return
new ResponseSpecBuilder()
.expectStatusCode(statusCode)
.expectContentType(ContentType.JSON)
.build();
}
public RequestSpecification createReqSpec(String url){
return new RequestSpecBuilder()
.setBaseUri(baseUri)
.setContentType(ContentType.JSON)
.setAuth(oauth2(accessToken))
.setBasePath(url)
.build();
}
}

#SelectClasspathResource("src/test/resources/example")
Typically src/test/resources is not part of the classpath.
After running mvn test have a look at target/test-classes to understand the structure of what is on the classpath.

Unable to run JUnit5 tests with Maven

I'm running into an issue where I can't run JUnit5 tests using Maven. Running them in the IDE works just fine but using "mvn test" produces the following output:
T E S T S
[INFO] -------------------------------------------------------
[INFO]
[INFO] Results:
[INFO]
[INFO] Tests run: 0, Failures: 0, Errors: 0, Skipped: 0
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
Here are my test classes :
package com.example.spstream.controllers.events;
import com.example.spstream.entities.Event;
import com.example.spstream.repositories.EventRepository;
import com.example.spstream.repositories.UserRepository;
import com.example.spstream.util.Mapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.dao.DataAccessException;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import java.time.LocalDateTime;
import static org.hamcrest.Matchers.containsString;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
#SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
#AutoConfigureMockMvc
#EnableAutoConfiguration(exclude = SecurityAutoConfiguration.class)
public class EventsCreationTest {
private static final String MISSING_TITLE_ERROR_MESSAGE = "title is missing";
private static final String MISSING_ACTIVITY_ERROR_MESSAGE = "activity is missing";
private static final String MISSING_LOCALISATION_ERROR_MESSAGE = "localisation is missing";
private static final String INVALID_ORGANISER_ID_ERROR_MESSAGE = "user %s does not exist";
private static final String MISSING_ORGANISER_ID_ERROR_MESSAGE = "organiser id is missing";
#Autowired
private MockMvc mockMvc;
#MockBean
private UserRepository userRepository;
#SpyBean
private EventRepository eventRepository;
private static final String DATE_IN_PAST_ERROR_MESSAGE = "date is in the past";
#BeforeEach
public void reset(){
Mockito.reset(userRepository);
Mockito.when(userRepository.existsById("123456")).thenReturn(true);
}
//prevents hardcoded events from failing tests due to date in the past
public void setEventDateToTomorrow(Event event) {
event.setDateTime(LocalDateTime.now().plusDays(1));
}
public void setEventDateToYesterday(Event event) {
event.setDateTime(LocalDateTime.now().minusDays(1));
}
public void testCorrectEventCreationWithEvent(Event event) throws Exception {
mockMvc.perform(MockMvcRequestBuilders.post("/events")
.content(Mapper.writeObjectToJson(event))
.contentType(MediaType.APPLICATION_JSON)
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().isCreated())
.andExpect(jsonPath("$.id").exists())
.andExpect(jsonPath("$.id").isString());
}
public void testIncorrectEventCreationWithEvent(Event event, String errorMessagePattern) throws Exception {
mockMvc.perform(MockMvcRequestBuilders.post("/events")
.content(Mapper.writeObjectToJson(event))
.contentType(MediaType.APPLICATION_JSON)
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().isBadRequest())
.andExpect(content().string(containsString(String.format(errorMessagePattern, event.getOrganiserId()))));
}
/**
* correct data
**/
#Test
public void testMinimalCorrectEvent() throws Exception {
Event minimalEvent = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
setEventDateToTomorrow(minimalEvent);
testCorrectEventCreationWithEvent(minimalEvent);
}
#Test
public void testMaximalCorrectEvent() throws Exception {
Event maximalEvent = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/maximal_event.json"), Event.class);
setEventDateToTomorrow(maximalEvent);
testCorrectEventCreationWithEvent(maximalEvent);
}
/**
* missing data
**/
#Test
public void testIncorrectEventTitleMissing() throws Exception {
Event eventTitleMissing = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
setEventDateToTomorrow(eventTitleMissing);
eventTitleMissing.setTitle(null);
testIncorrectEventCreationWithEvent(eventTitleMissing, MISSING_TITLE_ERROR_MESSAGE);
}
#Test
public void testIncorrectEventActivityMissing() throws Exception {
Event eventActivityMissing = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
eventActivityMissing.setActivity(null);
setEventDateToTomorrow(eventActivityMissing);
testIncorrectEventCreationWithEvent(eventActivityMissing, MISSING_ACTIVITY_ERROR_MESSAGE);
}
#Test
public void testIncorrectEventLocalisationMissing() throws Exception {
Event eventLocalisationMissing = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
eventLocalisationMissing.setLocalisation(null);
setEventDateToTomorrow(eventLocalisationMissing);
testIncorrectEventCreationWithEvent(eventLocalisationMissing, MISSING_LOCALISATION_ERROR_MESSAGE);
}
#Test
public void testIncorrectEventMissingUserId() throws Exception {
Event eventOrganiserIdMissing = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/incorrect/missing_user_id.json"), Event.class);
setEventDateToTomorrow(eventOrganiserIdMissing);
testIncorrectEventCreationWithEvent(eventOrganiserIdMissing, MISSING_ORGANISER_ID_ERROR_MESSAGE);
}
/**
* invalid data
**/
#Test
public void testIncorrectEventInvalidOrganiserId() throws Exception {
Mockito.when(userRepository.existsById(Mockito.any())).thenReturn(false);
Event eventInvalidOrganiserId = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
setEventDateToTomorrow(eventInvalidOrganiserId);
testIncorrectEventCreationWithEvent(eventInvalidOrganiserId, INVALID_ORGANISER_ID_ERROR_MESSAGE);
}
#Test
public void testIncorrectEventDateInThePast() throws Exception {
Event eventInPast = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
setEventDateToYesterday(eventInPast);
testIncorrectEventCreationWithEvent(eventInPast, DATE_IN_PAST_ERROR_MESSAGE);
}
/**
* internal database issue
**/
#Test
public void testCorrectEventServerError() throws Exception {
Event eventInvalidOrganiserId = Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/create/correct/minimal_event.json"), Event.class);
setEventDateToTomorrow(eventInvalidOrganiserId);
Mockito.when(eventRepository.save(eventInvalidOrganiserId)).thenThrow(Mockito.mock(DataAccessException.class));
mockMvc.perform(MockMvcRequestBuilders.post("/events")
.content(Mapper.writeObjectToJson(eventInvalidOrganiserId))
.contentType(MediaType.APPLICATION_JSON)
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().is5xxServerError());
System.out.println("whatever");
}
}
package com.example.spstream.controllers.events;
import com.example.spstream.entities.Event;
import com.example.spstream.repositories.EventRepository;
import com.example.spstream.util.Mapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.dao.DataAccessException;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import java.util.List;
import java.util.Optional;
import static com.example.spstream.util.Mapper.readJsonFromFile;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
#SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
#AutoConfigureMockMvc
#EnableAutoConfiguration(exclude = SecurityAutoConfiguration.class)
public class EventsAccessTest {
#MockBean
EventRepository mockEventRepository;
#Autowired
MockMvc mockMvc;
#BeforeEach
public void reset(){
Mockito.reset(mockEventRepository);
}
#Test
public void testFindAll() throws Exception{
List<Event> events = Mapper.readObjectListFromJson(readJsonFromFile("controllers/events/access/all_events.json"), Event.class);
Mockito.when(mockEventRepository.findAll()).thenReturn(events);
mockMvc.perform(MockMvcRequestBuilders.get("/events")
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(content().json(Mapper.readJsonFromFile("controllers/events/access/all_events.json")));
}
#Test
public void testFindEventWhichExists() throws Exception{
Mockito.when(mockEventRepository.findById("123456")).thenReturn(Optional.of(Mapper.readObjectFromJson(Mapper.readJsonFromFile("controllers/events/access/final_event.json"),Event.class)));
mockMvc.perform(MockMvcRequestBuilders.get("/events/123456")
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(content().json(Mapper.readJsonFromFile("controllers/events/access/final_event.json")));
}
#Test
public void testFindEventWhichDoesntExist() throws Exception {
Mockito.when(mockEventRepository.findById("7891011")).thenReturn(Optional.empty());
mockMvc.perform(MockMvcRequestBuilders.get("/events/7891011")
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().isNotFound())
.andExpect(content().contentType(MediaType.APPLICATION_JSON));
}
#Test
public void testFindEventDatabaseError() throws Exception {
Mockito.when(mockEventRepository.findById("123456")).thenThrow(Mockito.mock(DataAccessException.class));
mockMvc.perform(MockMvcRequestBuilders.get("/events/123456")
.accept(MediaType.APPLICATION_JSON))
.andExpect(status().is5xxServerError());
}
}
The pom :
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.2</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>spstream</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>spstream</name>
<description>spstream</description>
<properties>
<java.version>17</java.version>
<testcontainers.version>1.16.2</testcontainers.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock-jre8</artifactId>
<version>2.32.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.8.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${testcontainers.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>3.0.0-M5</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
</project>
I have done some research and I figured it might have something to do mith mixing up JUnit4 and JUnit5 features which leads to maven surefire plugin not running tests. However I can't find where those leftover JUnit4 features might be.
I'd appreciate any help.

As pointed out by other comments and answers I had residual JUnit4 dependencies due to test containers. I was able to fix the issue by explicitly setting JUnit5 as a dependency for maven surefire plugin like so :
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.8.2</version>
</dependency>
</dependencies>
</plugin>

The Junit4 is available because of Testcontainer dependency.
Testcontainers have a dependency on Junit4 and have it available by default.
You might also encounter the following issue in few cases:
IDE's detects the test cases written in Junit4 format but in sometime in case you make the test classes and methods package-private, they don't detect it.
I am not sure if they would be removing it in further releases but they do have Junit5 support which should resolve the issue
https://www.testcontainers.org/test_framework_integration/junit_5/

Kafka Spark Streaming LocationStrategies java class def not found exception

I am trying to integrate the kafka message broker and spark and facing an issue saying
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/streaming/kafka010/LocationStrategies
Below is the java spark code
package com.test.spell;
import java.util.Arrays;
/**
* Hello world!
*
*/
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.spark.api.java.function.*;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.spark.SparkConf;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaInputDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka010.ConsumerStrategies;
import org.apache.spark.streaming.kafka010.KafkaUtils;
import org.apache.spark.streaming.kafka010.LocationStrategies;
import scala.Tuple2;
public class App
{
private static final Pattern SPACE = Pattern.compile(" ");
public static void main( String[] args )
{
String brokers = "localhost:9092";
String topics = "spark-topic";
// Create context with a 2 seconds batch interval
SparkConf sparkConf = new SparkConf().setAppName("JavaDirectKafkaWordCount");
JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.seconds(2));
Set<String> topicsSet = new HashSet<>(Arrays.asList(topics.split(",")));
Map<String, Object> kafkaParams = new HashMap<>();
kafkaParams.put("metadata.broker.list", brokers);
// Create direct kafka stream with brokers and topics
JavaInputDStream<ConsumerRecord<String, String>> messages = KafkaUtils.createDirectStream(
jssc,
LocationStrategies.PreferConsistent(),
ConsumerStrategies.Subscribe(topicsSet, kafkaParams));
System.out.println("In programn");
// Get the lines, split them into words, count the words and print
JavaDStream<String> lines = messages.map(new Function<ConsumerRecord<String,String>, String>() {
#Override
public String call(ConsumerRecord<String, String> kafkaRecord) throws Exception {
return kafkaRecord.value();
}
});
JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
#Override
public Iterator<String> call(String line) throws Exception {
System.out.println(line);
return Arrays.asList(line.split(" ")).iterator();
}
});
/* JavaPairDStream<String,Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() {
#Override
public Tuple2<String, Integer> call(String word) throws Exception {
return new Tuple2<>(word,1);
}
});*/
// wordCounts.print();
// Start the computation
jssc.start();
jssc.awaitTermination();
}
}
Below is my pom.xml
I have tried many jar file versions couldn't find the right one.
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.spark-project.spark</groupId>
<artifactId>unused</artifactId>
<version>1.0.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.8.2-beta</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
<version>0.9.0-incubating</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.10</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka_2.10</artifactId>
<version>1.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.3.1</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.10.0.0</version>
</dependency>
</dependencies>
</project>
I am running my spark job as follows:
./bin/spark-submit --class com.test.spell.spark.App \
--master local \
--driver-memory 4g \
--executor-memory 2g \
--executor-cores 1 \
--queue default \
/home/cwadmin/spark_programs/spell/target/spell-0.0.1-SNAPSHOT.jar
I feel that the above problem is rising due to wrong jar files usage. Can someone help me out in fixing this. I would like to know what are the right jar files that should be used here. Also it would be grateful if someone shares some valuable resources regarding these programs like integration of Spark and Kafka.
I am trying to fix this issue since 2 days and unable to solve this
Thanks in advance.

First, you need to use the same version of Spark dependencies - I see that you're using 2.1.0 for spark-core, 2.3.1 for spark-streaming, 2.0.0 for spark-streaming-kafka, etc.
Second - you need to use the same version of Scala for these dependencies, and it should to version of Scala that was used to compile your build of Spark.
Third - you don't need to explicitly specify dependencies for Kafka libraries.
You need to build a fat-jar of your application, that will include necessary libraries (except spark-core that should be marked as provided). The easiest way to do this is to use Maven Assembly plugin.

Spring boot application is not running due to failure of initialization of 'requestMappingHandlerMapping'

It is a spring boot 2.0.0 application which was running earlier, but this error started after I added dependencies for spring-cloud-stream.I tried removing spring-fox dependency but it only removed the nested exception.
The complete error after removing spring-fox dependency:
ConfigServletWebServerApplicationContext : Exception encountered during context initialization - cancelling refresh attempt: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'requestMappingHandlerMapping' defined in class path resource [org/springframework/boot/autoconfigure/web/servlet/WebMvcAutoConfiguration$EnableWebMvcConfiguration.class]: Invocation of init method failed; nested exception is java.lang.ArrayStoreException: sun.reflect.annotation.TypeNotPresentExceptionProxy
The complete pom.xml is:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.foo.bar.application</groupId>
<artifactId>application-reviews-manager</artifactId>
<version>1.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>generic-name</name>
<description>Manager</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.0.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<application.utils.version>1.0.0</application.utils.version>
<application.utils.build.number>SNAPSHOT</application.utils.build.number>
<application.rnr.repositories.version>1.0.0</application.rnr.repositories.version>
<application.rnr.repositories.build.number>SNAPSHOT</application.rnr.repositories.build.number>
<dcp.config.version>0.0.1</dcp.config.version>
<dcp.config.build.number>SNAPSHOT</dcp.config.build.number>
<aspectj.version>1.8.9</aspectj.version>
</properties>
<dependencies>
<dependency>
<groupId>com.foo.bar.application</groupId>
<artifactId>application-rnr-repositories</artifactId>
<version>${application.rnr.repositories.version}-${application.rnr.repositories.build.number}</version>
</dependency>
<dependency>
<groupId>com.foo.bar.application</groupId>
<artifactId>application-utils</artifactId>
<version>${application.utils.version}-${application.utils.build.number}</version>
</dependency>
<dependency>
<groupId>com.foo.dcp.commons.config</groupId>
<artifactId>dcp-config-client</artifactId>
<version>${dcp.config.version}-${dcp.config.build.number}</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>5.6.3</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>5.6.3</version>
</dependency>
<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock</artifactId>
<version>1.58</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch.test</groupId>
<artifactId>framework</artifactId>
<version>5.0.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
<version>1.3.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream</artifactId>
<version>1.3.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
I'm just a starter in spring-cloud-stream, so any help would be appreciated.
Edit:
We found out that the mistake is in the way or place the annotation
#EnableBinding is used.
The relevant pieces of codes are:
Application.java
package com.foo.bar;
import com.foo.bar.configuration.FooStreams;
import com.foo.bar.configuration.BarStreams;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.context.annotation.Bean;
import com.externalpackage.config.client.AppConfiguration;
import com.foo.bar.storage.repository.CustomElasticRestClient;
#SpringBootApplication
#EnableBinding({FooStreams.class, BarStreams.class})
public class FooApplication {
public static void main(String[] args) {
SpringApplication.run(FooApplication.class, args);
}
#Bean
public AppConfiguration appConfiguration() {
return AppConfiguration.instance();
}
#Bean
public CustomElasticRestClient highLevelElasticClient(AppConfiguration appConfiguration) throws Exception {
return new CustomElasticRestClient(appConfiguration);
}
}
FooStreams.java
package com.foo.bar.configuration;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.SubscribableChannel;
public interface FooStreams {
String OUTPUT = "foo-out";
#Output(value = OUTPUT)
MessageChannel postFooToOutChanel();
}
BarStreams.java
package com.tesco.foo.bar.configuration;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
public interface BarStreams {
String OUTPUT = "bar-out";
#Output(value = OUTPUT)
MessageChannel postBarToOutChanel();
}
KafkaPublisherService.java:
package com.foo.bar.service;
import com.foo.bar.configuration.FooStreams;
import com.foo.bar.configuration.BarStreams;
import com.foo.bar.domain.Foo;
import com.foo.bar.domain.Bar;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHeaders;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.stereotype.Service;
import org.springframework.util.MimeTypeUtils;
#Service
#Slf4j
public class KafkaPublisherService {
#Autowired
private BarStreams barStreams;
#Autowired
private FooStreams fooStreams;
public void sendBar(final Bar bar) {
log.info("Publishing bar to kafka {}", bar);
MessageChannel messageChannel = barStreams.postBarToOutChanel();
messageChannel.send(MessageBuilder
.withPayload(bar)
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON)
.build());
}
public void sendFoo(final Foo foo) {
log.info("Publishing foo to kafka {}", foo);
MessageChannel messageChannel = fooStreams.postFooToOutChanel();
messageChannel.send(MessageBuilder
.withPayload(foo)
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON)
.build());
}
}
application.properties
spring.cloud.stream.kafka.binder.brokers=localhost:9092
spring.cloud.stream.bindings.foo-out.destination=feedback
spring.cloud.stream.bindings.foo-out.contentType=application/json
spring.cloud.stream.bindings.bar-out.destination=review
spring.cloud.stream.bindings.bar-out.contentType=application/json