Develop Reference

Why do I get duplicated data from flink?

I'm new to flink, and I'm trying to read a stream from kafka, however I'm getting duplicate data processed, and I'm wondering why ?
I know that's the problem came from flink because when I wrote a simple consumer in java I got no duplicate data
flink-connector-kafka_2.11 version 1.10.0
flink version 1.11
is there any issue to check if flink is processing only once the data provided by kafka ?
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
KafkaConsumer consumer = new KafkaConsumer("fashion","172.16.3.241:9092","fashion","org.apache.kafka.common.serialization.ByteBufferDeserializer");
FlinkKafkaConsumer<JsonNode> stream_consumer = new FlinkKafkaConsumer<>(consumer.getTopic(), new DeserializationSchema<JsonNode>() {
private final ObjectMapper objMapper = new ObjectMapper();
#Override
public JsonNode deserialize(byte[] bytes) throws IOException {
return objMapper.readValue(bytes,JsonNode.class);
}
#Override
public boolean isEndOfStream(JsonNode jsonNode) {
return false;
}
#Override
public TypeInformation<JsonNode> getProducedType() {
return TypeExtractor.getForClass(JsonNode.class);
}
}, consumer.getProperties());
DataStream<JsonNode> tweets = env.addSource(stream_consumer);
tweets.flatMap(new getTweetSchema());
env.execute("Flink Streaming Java API Skeleton");
}
private static class getTweetSchema implements FlatMapFunction<JsonNode, Tweet>{
private static final long serialVersionUID = -6867736771747690202L;
private JSONObject objTweet;
public void flatMap(JsonNode tweet, Collector<Tweet> out) throws JSONException, ParseException{
try{
if (objTweet == null){
objTweet = new JSONObject(tweet.asText());
}
HashSet<String> hashtag = new HashSet<>();
String text = objTweet.get("text").toString();
DateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss Z yyyy", Locale.ENGLISH );
Date created_at = dateFormat.parse(objTweet.get("created_at").toString());
String source = objTweet.get("source").toString();
source = source.substring(source.length() - 11).replaceAll("</a>","");
String lang = objTweet.get("lang").toString();
Boolean isRT = text.matches("^RT.*");
Long id = Long.parseLong(objTweet.get("id").toString());
if (objTweet.has("extended_tweet")){
JSONArray arr = objTweet.getJSONObject("extended_tweet").getJSONObject("entities").getJSONArray("hashtags");
if(!(arr.isEmpty())){
for(int i = 0; i< arr.length();i++){
hashtag.add(arr.getJSONObject(i).get("text").toString());
}
System.out.println(arr);
}
}
out.collect(new Tweet(id, text,created_at,source,lang,isRT,hashtag));
}catch (JSONException | ParseException e){
System.out.println("e");
throw e;
}
}
}

JsonGenerationException when serializing nested object using custom serializer in Jackson

Here is the class that I want to serialize.
public class ItemRow<T> {
private String id;
private List<T> items;
}
There are two variations that are allowed.
ItemRow<String>, ItemRow<ItemRow>.
In the latter case, it will be nested.
eg:
ItemRow item1 = new ItemRow("abc", Arrays.asList("item1", "item2", "item3"));
String result = mapper.writeValueAsString(item1);
System.out.println(result);
should give
{
"abc":["item1","item2","item3"]
}
Now, the latter case
ItemRow item2 = new ItemRow("cde", Arrays.asList("item4, item5"));
ItemRow item = new ItemRow("combined", Arrays.asList(item1,item2));
result = mapper.writeValueAsString(item);
System.out.println(result);
should give
{
"combined": {
"abc": ["item1", "item2", "item3"],
"cde": ["item4", "item5"]
}
}
But I get exception while serializing the latter. The first one works as expected. so I believe the recursive serialization is failing, but I am unable to find out why
Here is exception
com.fasterxml.jackson.core.JsonGenerationException: Can not start an object, expecting field name (context: Object)
at com.fasterxml.jackson.core.JsonGenerator._reportError(JsonGenerator.java:1961)
at com.fasterxml.jackson.core.json.JsonGeneratorImpl._reportCantWriteValueExpectName(JsonGeneratorImpl.java:244)
at com.fasterxml.jackson.core.json.WriterBasedJsonGenerator._verifyValueWrite(WriterBasedJsonGenerator.java:866)
at com.fasterxml.jackson.core.json.WriterBasedJsonGenerator.writeStartObject(WriterBasedJsonGenerator.java:279)
at hello.ItemRowSerializer.serialize(ItemRow.java:58)
at hello.ItemRowSerializer.serialize(ItemRow.java:42)
at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider._serialize(DefaultSerializerProvider.java:480)
at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:319)
at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2655)
at com.fasterxml.jackson.core.base.GeneratorBase.writeObject(GeneratorBase.java:381)
at hello.ItemRowSerializer.serialize(ItemRow.java:67)
at hello.ItemRowSerializer.serialize(ItemRow.java:42)
Serializer implementation
class ItemRowSerializer extends JsonSerializer<ItemRow> {
#Override
public void serialize(ItemRow itemRow, JsonGenerator jgen, SerializerProvider serializerProvider) throws IOException {
String id = itemRow.getId();
List<Object> items = itemRow.getItems();
if (items.isEmpty()) {
jgen.writeStartObject();
jgen.writeFieldName(id);
jgen.writeStartArray();
jgen.writeEndArray();
jgen.writeEndObject();
}
else {
jgen.writeStartObject();
Object item = items.get(0);
jgen.writeFieldName(id);
if (item instanceof ItemRow){
for (Object i : items) {
//ItemRow temp = (ItemRow) i;
//jgen.writeObjectField(temp.getId(), temp);
//jgen.writeObjectField(id, i);
jgen.writeStartObject();
jgen.writeObject(i);
jgen.writeEndObject();
}
}
else {
//jgen.writeFieldName(id);
jgen.writeStartArray();
for (Object arg : items) {
jgen.writeString(arg.toString());
}
jgen.writeEndArray();
}
}
jgen.writeEndObject();
}
}

Your serializer algoritihm is incorrect. The code is down above. You do not need to start object when you are directly deserializing an object. I removed this steps and minimized the code.
Example Test;
#Test
public void serializeTest() throws JsonProcessingException
{
ObjectMapper mapper = new ObjectMapper();
SimpleModule module = new SimpleModule();
module.addSerializer(ItemRow.class, new ItemRowSerializer());
mapper.registerModule(module);
ItemRow item1 = new ItemRow("abc", Arrays.asList("item1", "item2", "item3"));
String result = mapper.writeValueAsString(item1);
System.out.println(result);
ItemRow item2 = new ItemRow("cde", Arrays.asList("item4", "item5"));
ItemRow item6 = new ItemRow("deeper-1", Arrays.asList("item6", "item7"));
ItemRow item7 = new ItemRow("deeper-2", Arrays.asList("item6", "item7"));
ItemRow item8 = new ItemRow("deeper", Arrays.asList(item6, item7));
ItemRow item3 = new ItemRow("inner-1", Arrays.asList("item6", "item7"));
ItemRow item4 = new ItemRow("inner-2", Arrays.asList("item6", "item7"));
ItemRow item5 = new ItemRow("inner", Arrays.asList(item3, item4, item8));
ItemRow item = new ItemRow("combined", Arrays.asList(item1,item2,item5));
result = mapper.writeValueAsString(item);
System.out.println(result);
}
Algorithm;
public class ItemRowSerializer extends JsonSerializer<ItemRow>
{
#Override
public void serialize(ItemRow itemRow, JsonGenerator jgen, SerializerProvider serializerProvider) throws IOException
{
jgen.writeStartObject();
writeInnerObject(jgen, itemRow);
jgen.writeEndObject();
}
private void writeStringArr(JsonGenerator jgen, List items) throws IOException
{
jgen.writeStartArray();
for (Object arg : items)
{
jgen.writeString(arg.toString());
}
jgen.writeEndArray();
}
private void writeInnerObject(JsonGenerator jgen, ItemRow row) throws IOException
{
jgen.writeFieldName(row.getId());
if (row.getItems().size() > 0 && row.getItems().get(0) instanceof ItemRow)
{
jgen.writeStartObject();
for (int i = 0; i < row.getItems().size(); i++)
{
ItemRow innerRow = (ItemRow) row.getItems().get(i);
if( innerRow.getItems().size() > 0 && innerRow.getItems().get(0) instanceof ItemRow )
{
writeInnerObject(jgen, innerRow);
}
else
{
jgen.writeFieldName(innerRow.getId());
writeStringArr(jgen, innerRow.getItems());
}
}
jgen.writeEndObject();
}
else
{
writeStringArr(jgen, row.getItems());
}
}
}

RestTemplate & Jackson - Custom JSON deserializing?

The webservice returns an empty string instead of NULL which causes Jackson to crash.
So I created a custom parser, and I'm trying to parse it manually? Any idea How I could achieve this?
What Am I doing wrong here? All I'm trying to do is to parse JSON to object as I normally would. The field names are added to my properties using #JsonProperty so the parser should know how to convert it.
public class InsertReplyDeserializer extends JsonDeserializer<ListingReply> {
#Override
public ListingReply deserialize(JsonParser jsonParser, DeserializationContext arg1)
throws IOException, JsonProcessingException {
ObjectCodec oc = jsonParser.getCodec();
JsonNode node = oc.readTree(jsonParser);
// If service returns "" instead of null return a NULL object and don't try to parse
if (node.getValueAsText() == "")
return null;
ObjectMapper objectMapper = new ObjectMapper();
ListingReply listingReply = objectMapper.readValue(node, ListingReply.class);
return listingReply;
}
}

Here is how I resolved it
#Override
public MyObject deserialize(JsonParser jsonParser, DeserializationContext arg1)
throws IOException, JsonProcessingException {
ObjectCodec oc = jsonParser.getCodec();
JsonNode node = oc.readTree(jsonParser);
if (node.getValueAsText() == "")
return null;
MyObject myObject = new MyObject();
myObject.setMyStirng(node.get("myString").getTextValue());
JsonNode childNode = node.get("childObject");
ObjectMapper objectMapper = new ObjectMapper();
ChildObject childObject = objectMapper.readValue(childNode,
ChildObject.class);
myObject.setChildObject(childObject);
return myObject;
}

I am not sure you need to manually parse response. You solution would work but seems sub-optimal in my opinion. Since it looks like that you are using RestTemplate, you should rather write (or move your parser code to) your own message converter. Then add this converter to your rest template object which will internally deserialize the value for you. Something along the lines,
public class CustomHttpmsgConverter extends AbstractHttpMessageConverter<Object> {
private ObjectMapper objectMapper = new ObjectMapper();
#Override
protected Object readInternal(Class clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException {
InputStream istream = inputMessage.getBody();
String responseString = IOUtils.toString(istream);
if(responseString.isEmpty()) //if your response is empty
return null;
JavaType javaType = getJavaType(clazz);
try {
return this.objectMapper.readValue(responseString, javaType);
} catch (Exception ex) {
throw new HttpMessageNotReadableException(responseString);
}
}
//add this converter to your resttemplate
RestTemplate template = new RestTemplate();
List<HttpMessageConverter<?>> converters = new ArrayList<HttpMessageConverter<?>>();
converters.add(new CustomHttpmsgConverter());
template.setMessageConverters(converters);

Jackson custom deserializers creating empty pojo when reading list

I have the following json
{
"id":null,
"name":"Myapp",
"description":"application",
"myListA":["java.util.ArrayList",[{
"id":50,
"name":"nameA1",
"myListB":{
"id":48,
"name":"nameB1",
"myListC":["java.util.ArrayList",[{
"id":1250,
"name":"nameC1",
"description":"nameC1_desc",
"myReferenceObject":{
"code":"someCodeA"
}
},{
"id":1251,
"name":"nameC2",
"description":"nameC1_desc",
"myReferenceObject":{
"code":"someCodeB"
}
and so on.
I i wish to replace myReferenceObject with an item from persistence layer.
I followed JacksonHowToCustomDeserializers
My deserializer is as follows:
public class MyReferenceObjectCodeDeserializer extends JsonDeserializer<MyReferenceObjectBean> {
#Override
public ColumnReferenceBean deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
while (jp.nextToken() != JsonToken.END_OBJECT) {
String fieldname = jp.getCurrentName();
jp.nextToken();
if ("code".equalsIgnoreCase(fieldname)) {
MyReferenceObjectBean b = MyReferenceObjectServiceImpl.retrieveByCode(jp.getText());
logger.info("returning " +b.toString());
return b;
}
}
logger.info("returning null");
return null;
}
}
And I attache the module like so:
ObjectMapper mapper = new ObjectMapper();
mapper.enableDefaultTyping();
SimpleModule module = new SimpleModule("myModule", new Version(1, 0, 0, null));
module.addDeserializer(MyReferenceObjectBean.class, new MyReferenceObjectCodeDeserializer());
mapper.registerModule(module);
try {
return mapper.readValue(serializedJsonString, MyMainObjectBean.class);
} catch (IOException e) {
logger.error("Unable to parse=" + serializedJsonString, e);
}
everything debugs correctly however the resulting myListC list has double the amount of objects with even numbers holding the correct objects along with the correct myReferenceObject out of persistence (deserialized correctly using my module) and the odd elements holding empty Pojos, that is an object with null values for all variables.
Through debug, It never reaches return null in my custom deserializer, for it works properly each time its executed. The issue seems to be further up stream where it inserts blank myListC objects.
Any help would greeatly be appreciated.
Thanks!

There is a logic problem in your code.
You want to loop until you reach the end of the object but break your loop with return b (if block). This means that you will not read the object stream until its end.
Try something like this (didn't try it but should work).
public class MyReferenceObjectCodeDeserializer extends JsonDeserializer<MyReferenceObjectBean> {
#Override
public ColumnReferenceBean deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
MyReferenceObjectBean b = null;
while (jp.nextToken() != JsonToken.END_OBJECT) {
String fieldname = jp.getCurrentName();
jp.nextToken();
if ("code".equalsIgnoreCase(fieldname)) {
b = MyReferenceObjectServiceImpl.retrieveByCode(jp.getText());
logger.info("returning " +b.toString());
}
}
if (b==null) logger.info("returning null");
return b;
}
}
You can also have a look at Genson http://code.google.com/p/genson/ if you can change from jackson. In addition of some other features it is supposed to be easier to use. Here is how you can solve your problem with genson (for this example it is quite similar to jackson):
public class MyReferenceObjectCodeDeserializer implements Deserializer<MyReferenceObjectBean> {
public MyReferenceObjectBeandeserialize(ObjectReader reader, Context ctx) throws TransformationException, IOException {
MyReferenceObjectBean b = null;
reader.beginObject();
while (reader.hasNext()) {
reader.next();
if ("code".equalsIgnoreCase(reader.name()))
b = MyReferenceObjectServiceImpl.retrieveByCode(reader.valueAsString());
}
reader.endObject();
return b;
}
}
// register it
Genson genson = new Genson.Builder().withDeserializers(new MyReferenceObjectCodeDeserializer()).create();
MyClass myClass = genson.deserialize(json, MyClass.class);

jackson - json encoding of doubles with controlled precision

I'm encoding a complex Map structure with arrays of double values.
High precision is not important and output size is, so I'm trying to get the JSON tool (Jackson in this case) to serialize the double values using a provided DecimalFormat.
The following is my best shot, but this fails as the serializer is not picked by the object mapper to encode the array:
class MyTest
{
public class MyDoubleSerializer extends JsonSerializer<double[]>
{
public void serialize(double[] value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException
{
for (double d : value)
{
jgen.writeStartArray();
jgen.writeRaw( df.format( d ) );
jgen.writeEndArray();
}
}
}
#Test
public void test1() throws Exception
{
ObjectMapper mapper = new ObjectMapper();
SimpleModule module = new SimpleModule("MyModule", new Version(0, 1, 0, "alpha"));
module.addSerializer(double[].class, new MyDoubleSerializer());
mapper.registerModule(module);
Map<String, Object> data = new HashMap<String, Object>();
double[] doubleList = { 1.1111111111D, (double) (System.currentTimeMillis()) };
data.put( "test", doubleList );
System.out.print( mapper.writeValueAsString( data ));
}
}
The output is:
{"test":[1.1111111111,1.315143204964E12}
What I was looking for:
{"test":[1.32E12, 1.11E0]}
Any ideas?
Also, I don't like having to generate a String and write is as raw - is there I could feed a StringBuffer into into DecimalFormat to do this?
Thanks

Managed to resolve this, by borrowing from the built-in serializer for Double.
It's a bit of a hack, because writeRaw() doesn't care about the context and doesn't write a comma between array members, so I'm casting the Json writer and calling its writeValue() method to handle this.
Strangely enough, this does not work on the example in the question (again doesn't get called for serializing these doubles), but does work on my real-world object which is more complex.
Enjoy...
public class JacksonDoubleArrayTest
{
private DecimalFormat df = new DecimalFormat( "0.##E0" );
public class MyDoubleSerializer extends org.codehaus.jackson.map.ser.ScalarSerializerBase<Double>
{
protected MyDoubleSerializer()
{
super( Double.class );
}
#Override
public final void serializeWithType( Double value, JsonGenerator jgen, SerializerProvider provider, TypeSerializer typeSer ) throws IOException,
JsonGenerationException
{
serialize( value, jgen, provider );
}
#Override
public void serialize( Double value, JsonGenerator jgen, SerializerProvider provider ) throws IOException, JsonGenerationException
{
if ( Double.isNaN( value ) || Double.isInfinite( value ) )
{
jgen.writeNumber( 0 ); // For lack of a better alternative in JSON
return;
}
String x = df.format( value );
if ( x.endsWith( "E0" ) )
{
x = x.substring( 0, x.length() - 2 );
}
else if ( x.endsWith( "E1" ) && x.length() == 6 )
{
x = "" + x.charAt( 0 ) + x.charAt( 2 ) + '.' + x.charAt( 3 );
}
JsonWriteContext ctx = (JsonWriteContext)jgen.getOutputContext();
ctx.writeValue();
if ( jgen.getOutputContext().getCurrentIndex() > 0 )
{
x = "," + x;
}
jgen.writeRaw( x );
}
#Override
public JsonNode getSchema( SerializerProvider provider, Type typeHint )
{
return createSchemaNode( "number", true );
}
}
#SuppressWarnings("unchecked")
private static Map<String, Object> load() throws JsonParseException, JsonMappingException, IOException
{
ObjectMapper loader = new ObjectMapper();
return (Map<String, Object>)loader.readValue( new File( "x.json" ), Map.class );
}
#Test
public void test1() throws JsonGenerationException, JsonMappingException, IOException
{
ObjectMapper mapper = new ObjectMapper();
SimpleModule module = new SimpleModule( "StatsModule", new Version( 0, 1, 0, "alpha" ) );
module.addSerializer( Double.class, new MyDoubleSerializer() );
mapper.registerModule( module );
String out = mapper.writeValueAsString( load() );
// System.out.println( out.length() );
}
}