Develop mqtt connector for Kafka using spring.
Using the mqtt library provided by spring, messages are collected as follows.
message handler
#Bean
#ServiceActivator(inputChannel = "mqttInputChannel")
public MessageHandler handler() {
return new MessageHandler() {
#Override
public void handleMessage(Message<?> message) throws MessagingException {
String topic = message.getHeaders().get(MqttHeaders.RECEIVED_TOPIC).toString();
if(topic.equals("myTopic")) {
System.out.println("Mqtt data pub");
}
System.out.println(message.getPayload());
if(topic==null) {
topic = "mqttdata";
}
String tag = "test/vib";
String name = null;
if(name==null) {
name = KafkaMessageService.MQTT_PRODUCER;
}
HashMap<String, Object> datalist = new HashMap<String, Object>();
try {
datalist =convertJSONstringToMap(message.getPayload().toString());
System.out.println(datalist.get("mac"));
counts = kafkaMessageService.publish(topic, name, tag, (HashMap<String,Object>[] datalist);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
public static HashMap<String,Object> convertJSONstringToMap(String json) throws Exception {
ObjectMapper mapper = new ObjectMapper();
HashMap<String, Object> map = new HashMap<String, Object>();
map = mapper.readValue(json, new TypeReference<HashMap<String, Object>>() {});
return map;
}
publish method
public int publish(String topic,String producerName,String tag,HashMap<String,Object>[] datalist) throws NotMatchedProducerException,KafkaPubFailureException{
KafkaProducerAdaptor adaptor = searchProducerAdaptor(producerName);
if(adaptor==null) {
throw new NotMatchedProducerException();
}
KafkaTemplate<String,Object> kafkaTemplate = adaptor.getKafkaTemplate();
LocalDateTime currentDateTime = LocalDateTime.now();
String receivedTime = currentDateTime.toString();
ObjectMapper objectMapper = new ObjectMapper();
String key = adaptor.getName();
int counts = 0;
for(HashMap<String,Object> data : datalist) {
Map<String,Object> messagePacket = new HashMap<String,Object>();
messagePacket.put("tag", tag);
messagePacket.put("data", data);
messagePacket.put("receivedtime", receivedTime);
try {
kafkaTemplate.send(topic,key,objectMapper.valueToTree(messagePacket)).get();
logger.info("Sent message : topic=["+topic+"],key=["+key+"] value=["+messagePacket+"]");
} catch(Exception e) {
logger.info("Unable to send message : topic=["+topic+"],key=["+key+"] message=["+messagePacket+"] / due to : "+e.getMessage());
throw new KafkaPubFailureException(e);
}
counts++;
}
return counts;
}
I don't know how to declare a hashmap <String, object> [] as an instance and how to use it.
The above source was taken from spring support as it is, and some modifications were made.
I want one JSONObject response after passing URI through RESTTemplate
Test case is passing but the code coverage is still 0%
I have to return accountDetails object in JSON format
how do we Pass URI which takes account ID and given response entity in JSON Format this is what I have to figure out.
Test method:
void scheduleOnDemand() throws Exception {
AccountDTO accountDTO = new AccountDTO();
accountDTO.setId(1);
accountDTO.setTimeZone("Asia/Kolkata");
accountDTO.setPlatformType("AZURE");
accountDTO.setEnvironmentName("test");
accountDTO.setName("azureAccount");
accountDTO.setNextScheduleDate("2021-09-13");
accountDTO.setEnvironmentId(1);
HashMap<String, Object> accountDetails = new HashMap<>();
accountDetails.put("account_Id", "1");
accountDetails.put("TimeZone", "Asia/Kolkata");
accountDetails.put("AgentStatus", "Initiated");
accountDetails.put("Account_Platform", "AZURE");
accountDetails.put("Schedule_Time", "13:30:50.000");
accountDetails.put("Environment_Name", "test");
accountDetails.put("Account_Name", "azureAccount");
accountDetails.put("History_Id", "109");
accountDetails.put("Schedule_Date", "2021-09-13");
accountDetails.put("Environment_Id", "1");
Mockito.when(restTemplate.postForEntity("configure/accountDetails?Account_Id=1",null, JSONObject.class))
.thenReturn(new ResponseEntity((accountDetails), HttpStatus.OK));
}
Actual Method:
#Override
public JSONObject scheduleOnDemand(String accountId) throws Exception {
JSONObject object = null;
// PlatformHistoryDetails phistory = null;
HashMap<String, Object> accountDetails = new HashMap<>();
accountDetails = utilService.getAccountDetails(Integer.parseInt(accountId));
if (((String) accountDetails.get("scantype")).equalsIgnoreCase("infra")||((String) accountDetails.get("scantype")).equalsIgnoreCase("all")) {
URI postUri = UriComponentsBuilder.fromPath("/").pathSegment("api/scheduleOnDemand")
.queryParam("requestId", MDC.get("requestId")).queryParam("service", "scan")
.queryParam("Account_Id", accountId).build().toUri();
PlatformHistoryDetails phistory = modelMapper.map(apiClient.postOperation(postUri, Object.class),
PlatformHistoryDetails.class);
phistory.getHistory().setUser("admin");
object = utilService.processOneAccount(phistory);
} else {
throw new Exception("Account is not of type INFRA or ALL but of type " + accountDetails.get("scantype"));
}
return object;
}
accountDetails Implementation:
#Override
#SuppressWarnings({ "unchecked", "rawtypes" })
public HashMap<String, Object> getAccountDetails(int accountId) {
URI getUri = UriComponentsBuilder.fromPath("/").pathSegment("configure/accountDetails")
.queryParam("Account_Id", accountId).build().toUri();
HashMap<String, Object> account = (LinkedHashMap) apiClient.getAccountDetails(getUri, Object.class);
return account;
}
I have a List<Map<String, Object>> input like below:
[{
CURRENCY = USD,
STATUS = NEW,
PUBLISH_REGION = DEL,
SOURCE = ALADDIN,
RECON_STATUS = null,
JOB_ID_COUNT = 783
}, {
CURRENCY = USD,
STATUS = IN_PROGRESS,
PUBLISH_REGION = DEL,
SOURCE = ALADDIN,
RECON_STATUS = null,
JOB_ID_COUNT = 462
}, {
CURRENCY = USD,
STATUS = NEW,
PUBLISH_REGION = DEL,
SOURCE = GROUP,
RECON_STATUS = null,
JOB_ID_COUNT = 4
}]
I am trying to create another List<Map<String, Object>> by grouping on CURRENCY, PUBLISH_REGION, SOURCE and RECON_STATUS columns. And add all unique STATUS values as pivot to the output map and use JOB_ID_COUNT to summarize/aggregate the count.
List<String> groups = new ArrayList<>(asList("SOURCE", "RECON_STATUS", "PUBLISH_REGION", "CURRENCY"));
List<Map<String, Object>> = input.stream()
.collect(groupingBy(row -> row.get(groups.get(0)), mapping(map -> map.get(groups.get(0)), toList())));
I am expecting below response:
Output:
[{
CURRENCY = USD,
PUBLISH_REGION = DEL,
SOURCE = ALADDIN,
RECON_STATUS = null,
NEW = 783,
IN_PROGRESS = 462
}, {
CURRENCY = USD,
PUBLISH_REGION = DEL,
SOURCE = GROUP,
RECON_STATUS = null,
NEW = 4,
IN_PROGRESS = 0
}]
I am getting compile time error when trying to group by multiple map fields. Single field groupingBy is working fine. Any help is greatly appriciated.
Without Using Custom Class
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class MultipleFieldSorting2 {
private static Map<String, Object> map, map1, map2;
private static List<Map<String, Object>> lst = new ArrayList<>();
static {
map = new HashMap<>();
map.put("CURRENCY", "USD");
map.put("STATUS", "NEW");
map.put("PUBLISH_REGION", "DEL");
map.put("SOURCE", "ALADDIN");
map.put("RECON_STATUS", null);
map.put("JOB_ID_COUNT", "783");
map1 = new HashMap<>();
map1.put("CURRENCY", "USD");
map1.put("STATUS", "IN_PROGRESS");
map1.put("PUBLISH_REGION", "DEL");
map1.put("SOURCE", "ALADDIN");
map1.put("RECON_STATUS", null);
map1.put("JOB_ID_COUNT", "462");
map2 = new HashMap<>();
map2.put("CURRENCY", "USD");
map2.put("STATUS", "NEW");
map2.put("PUBLISH_REGION", "DEL");
map2.put("SOURCE", "GROUP");
map2.put("RECON_STATUS", null);
map2.put("JOB_ID_COUNT", "4");
lst.add(map);
lst.add(map1);
lst.add(map2);
}
public static Map<String, Object> mapper(Map<String, Object> e){
String key = e.get("CURRENCY") + "-" + e.get("PUBLISH_REGION") + "-" + e.get("SOURCE") + "-" + e.get("RECON_STATUS");
Map<String, Object> groupedValue = res.get(key);
if(groupedValue!=null){
groupedValue.put((String) e.get("STATUS"), groupedValue.get("STATUS")!=null ? groupedValue.get("STATUS")+","+e.get("JOB_ID_COUNT") : e.get("JOB_ID_COUNT"));
if(groupedValue.get("NEW")==null){
groupedValue.put("NEW", 0);
}
if(groupedValue.get("IN_PROGRESS")==null){
groupedValue.put("IN_PROGRESS", 0);
}
}else{
groupedValue = new HashMap<>();
res.put(key, groupedValue);
groupedValue.put("CURRENCY", e.get("CURRENCY"));
groupedValue.put("PUBLISH_REGION", e.get("PUBLISH_REGION"));
groupedValue.put("SOURCE", e.get("SOURCE"));
groupedValue.put("RECON_STATUS", e.get("RECON_STATUS"));
groupedValue.put((String) e.get("STATUS"), e.get("JOB_ID_COUNT"));
}
return groupedValue;
}
static Map<String, Map<String, Object>> res = new HashMap<>();
public static void main(String[] args) {
List<Map<String, Object>> finalResult = new ArrayList<>();
lst.stream()
.map(MultipleFieldSorting2::mapper)
.forEach(result -> {
if(!finalResult.contains(result))
finalResult.add(result);
});
System.out.println(finalResult);
}
}
Tried this solution and it is working
Stream the source List
Map each value of map in the list to Class MapWrapper(a pojo where each key is a field)
GroupBy using the groupByKey defined in MapWrapper(uses CURRENCY, PUBLISH_REGION, SOURCE and RECON_STATUS columns)
3.a The result is a Map<String, List<MapWrapper>>
4.Stream through the entry set
map - and get the value alone from (Map<String, List<MapWrapper>>)
Map - convert from List<MapWrapper> to Map<String, Object> using MapWrapper::map
Collect to a list
In Short the solution is
List<Map<String, Object>> value = lst.stream()
.map(map -> new MapWrapper(map))
.collect(groupingBy(MapWrapper::groupByKey))
.entrySet()
.stream()
.map(e -> e.getValue())
.map(MapWrapper::map).collect(toList());
Working Code
public class MultipleFieldSorting {
private static Map<String, Object> map, map1, map2;
private static List<Map<String, Object>> lst = new ArrayList<>();
static {
map = new HashMap<>();
map.put("CURRENCY", "USD");
map.put("STATUS", "NEW");
map.put("PUBLISH_REGION", "DEL");
map.put("SOURCE", "ALADDIN");
map.put("RECON_STATUS", null);
map.put("JOB_ID_COUNT", "783");
map1 = new HashMap<>();
map1.put("CURRENCY", "USD");
map1.put("STATUS", "IN_PROGRESS");
map1.put("PUBLISH_REGION", "DEL");
map1.put("SOURCE", "ALADDIN");
map1.put("RECON_STATUS", null);
map1.put("JOB_ID_COUNT", "462");
map2 = new HashMap<>();
map2.put("CURRENCY", "USD");
map2.put("STATUS", "NEW");
map2.put("PUBLISH_REGION", "DEL");
map2.put("SOURCE", "GROUP");
map2.put("RECON_STATUS", null);
map2.put("JOB_ID_COUNT", "4");
lst.add(map);
lst.add(map1);
lst.add(map2);
}
public static void main(String[] args) {
List<Map<String, Object>> value = lst.stream()
.map(map -> new MapWrapper(map))
.collect(groupingBy(MapWrapper::groupByKey))
.entrySet()
.stream()
.map(e -> e.getValue())
.map(MapWrapper::map).collect(toList());
System.out.println(value);
}
}
class MapWrapper {
private String currency;
private String status;
private String publish;
private String source;
private String recon_status;
private String job_id;
public MapWrapper(Map<String, Object> map) {
this.currency = (String) map.get("CURRENCY");
this.status = (String) map.get("STATUS");
this.publish = (String) map.get("PUBLISH_REGION");
this.source = (String) map.get("SOURCE");
this.recon_status = (String) map.get("RECON_STATUS");
this.job_id = (String) map.get("JOB_ID_COUNT");
}
String groupByKey() {
return new StringBuilder().append(this.getCurrency()).append("-").append(this.publish).append("-")
.append(this.source).append("-").append(this.recon_status).toString();
}
public static Map<String, Object> map(List<MapWrapper> lst){
Map<String, Object> res = new HashMap<>();
res.put("CURRENCY",lst.get(0).getCurrency());
res.put("PUBLISH_REGION",lst.get(0).getPublish());
res.put("SOURCE",lst.get(0).getSource());
res.put("RECON_STATUS",lst.get(0).getRecon_status());
for(MapWrapper m : lst){
res.put(m.getStatus(), m.getJob_id());
}
if(res.get("NEW")==null){
res.put("NEW", 0);
}
if(res.get("IN_PROGRESS")==null){
res.put("IN_PROGRESS", 0);
}
return res;
}
String getCurrency() {
return currency;
}
void setCurrency(String currency) {
this.currency = currency;
}
String getStatus() {
return status;
}
void setStatus(String status) {
this.status = status;
}
String getPublish() {
return publish;
}
void setPublish(String publish) {
this.publish = publish;
}
String getSource() {
return source;
}
void setSource(String source) {
this.source = source;
}
String getJob_id() {
return job_id;
}
void setJob_id(String job_id) {
this.job_id = job_id;
}
String getRecon_status() {
return recon_status;
}
void setRecon_status(String recon_status) {
this.recon_status = recon_status;
}
}
I have a case to transform a response from
Dogs API
to a different structure like this :
[
{
"breed": "pug",
"sub_breed": []
},
{
"breed": "ridgeback",
"sub_breed": [
{
"breed": "rhodesian",
"sub_breed": []
}
]
},
{
"breed": "doberman",
"sub_breed": []
},
{
"breed": "hound",
"sub_breed": [
{
"breed": "Ibizan",
"sub_breed": []
},
{
"breed": "afghan",
"sub_breed": []
}
]
}
]
I am confused after getting the response and don't know how to transform it.
Here is what I do until getting the response
public List<DogResponse> getDogs() {
List<DogResponse> response = new ArrayList<DogResponse>();
try {
String url = "https://dog.ceo/api/breeds/list/all";
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> result = restTemplate.getForEntity(url, String.class);
ObjectMapper mapper = new ObjectMapper();
Map<String, String> map = mapper.readValue(result.getBody().toString(), Map.class);
System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(map.get("message")));
for (Entry<String, String> entry : map.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
System.out.println("key : "+key);
System.out.println("val : "+value);
}
} catch (Exception e) {
// TODO: handle exception
}
return response;
}
DogResponse
public class DogResponse {
private String breed;
private DogResponse sub_breed;
public String getBreed() {
return breed;
}
public void setBreed(String breed) {
this.breed = breed;
}
public DogResponse getSub_breed() {
return sub_breed;
}
public void setSub_breed(DogResponse sub_breed) {
this.sub_breed = sub_breed;
}
}
I am trying using Map but failed when I want to print the key and value, it's showing nothing.
You should map the response to List of DogResponse you may have an issue because of circular dependency.
List<DogResponse> dogs = mapper.readValue(jsonString, new TypeReference<List<DogResponse>>() {});
You can try this.
public List<DogResponse> getDogs() {
List<DogResponse> response = new ArrayList<DogResponse>();
try {
String url = "https://dog.ceo/api/breeds/list/all";
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> result = restTemplate.getForEntity(url, String.class);
ObjectMapper mapper = new ObjectMapper();
Map<String, Map<String, List<String>>> map = mapper.readValue(result.getBody().toString(), Map.class);
System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(map.get("message")));
Map<String, List<String>> innerMap = map.get("message");
for (Entry<String, List<String>> entry : innerMap.entrySet()) {
String key = entry.getKey();
List<String> value = entry.getValue();
System.out.println("key : " + key);
System.out.println("val : " + value);
}
} catch (Exception e) {
// TODO: handle exception
}
return response;
}
ResponseEntity result = restTemplate.getForEntity(url, DogResponse.class);
This should work.
I have the client that makes a rest call to a server like this:
Entity<RequestObject> entity = Entity.json(new RequestObject(Arrays.asList(1, 2, 3), Arrays.asList("xColumn", "yColumn")));
ClientConfig config = new ClientConfig();
config.register(JacksonJsonProvider.class);
Client client = ClientBuilder.newClient(config);
Response response = client.target("http://172.18.0.1:10000/getProductsData")
.request(MediaType.APPLICATION_JSON_TYPE)
.post(Entity.json(entity.getEntity()), Response.class);
List<WarehouseProductData> dwsData = response.readEntity(new GenericType<List<WarehouseProductData>>(){});
client.close();
The the other server is responding from the following method:
#PostMapping("/getProductsData")
public List<WarehouseProductData> greetingPost1(#RequestBody String json) {
WarehouseProductData wd = new WarehouseProductData(85654865);
wd.addData("xColumn", "value of product 85654865 for x column");
wd.addData("yColumn", "value of product 85654865 for y column");
wd.addData("the me column", "value of product 85654865 for 'the me column' column");
wd.addData("response for x", "1");
wd.addData("response for x", "2");
return Collections.singletonList(wd);
The WarehauseProductData class is like this:
public class WarehouseProductData {
private int agoId;
private Map<String, String> data;
public WarehouseProductData(int agoId, Map<String, String> data) {
this.agoId = agoId;
this.data = data;
}
public WarehouseProductData(int agoId) {
this.agoId = agoId;
this.data = new HashMap<>();
}
public int getAgoId() {
return agoId;
}
public void setAgoId(int agoId) {
this.agoId = agoId;
}
public Map<String, String> getData() {
return data;
}
public void setData(Map<String, String> data) {
this.data = data;
}
public void addData(String columnName, String value) {
data.put(columnName, value);
}
}
The problem is that when the
List<WarehouseProductData> dwsData = response.readEntity(new GenericType<List<WarehouseProductData>>(){});
is ran the dwsData is like this:
dwsData size = 1
|
--0 {WarehouseProductData}
|
-- agoId = 85654865
|
-- data = null (!!!which is incorect!!!)
So how to parse the response in order to get the data field correctly?