Develop Reference

Flink Collector issue when Collection Object with Map of Object class

I am facing a issue where when i collecting object from flink flatmap collector than i am not getting value collected correctly. I am getting object reference and its not giving me actual value.
dataStream.filter(new FilterFunction<GenericRecord>() {
#Override
public boolean filter(GenericRecord record) throws Exception {
if (record.get("user_id") != null) {
return true;
}
return false;
}
}).flatMap(new ProfileEventAggregateFlatMapFunction(aggConfig))
.map(new MapFunction<ProfileEventAggregateEmittedTuple, String>() {
#Override
public String map(
ProfileEventAggregateEmittedTuple profileEventAggregateEmittedTupleNew)
throws Exception {
String res=null;
try {
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
res= mapper.writeValueAsString(profileEventAggregateEmittedTupleNew);
} catch (Exception e) {
e.printStackTrace();
}
return res;
}
}).print();
public class ProfileEventAggregateFlatMapFunction extends
RichFlatMapFunction<GenericRecord, ProfileEventAggregateEmittedTuple> {
private final ProfileEventAggregateTupleEmitter aggregator;
ObjectMapper mapper = ObjectMapperPool.getInstance().get();
public ProfileEventAggregateFlatMapFunction(String config) throws IOException {
this.aggregator = new ProfileEventAggregateTupleEmitter(config);
}
#Override
public void flatMap(GenericRecord event,
Collector<ProfileEventAggregateEmittedTuple> collector) throws Exception {
try {
List<ProfileEventAggregateEmittedTuple> aggregateTuples = aggregator.runAggregates(event);
for (ProfileEventAggregateEmittedTuple tuple : aggregateTuples) {
collector.collect(tuple);
}
}}
Debug Results:
tuple that i am collecting in collector
tuple = {ProfileEventAggregateEmittedTuple#7880}
profileType = "userprofile"
key = "1152473"
businessType = "keyless"
name = "consumer"
aggregates = {ArrayList#7886} size = 1
0 = {ProfileEventAggregate#7888} "geo_id {geo_id=1} {keyless_select_destination_cnt=1, total_estimated_distance=12.5}"
entityType = "geo_id"
dimension = {LinkedHashMap#7891} size = 1
"geo_id" -> {Integer#7897} 1
key = "geo_id"
value = {Integer#7897} 1
metrics = {LinkedHashMap#7892} size = 2
"keyless_select_destination_cnt" -> {Long#7773} 1
key = "keyless_select_destination_cnt"
value = {Long#7773} 1
"total_estimated_distance" -> {Double#7904} 12.5
key = "total_estimated_distance"
value = {Double#7904} 12.5
This i get in my map function .map(new MapFunction<ProfileEventAggregateEmittedTuple, String>()
profileEventAggregateEmittedTuple = {ProfileEventAggregateEmittedTuple#7935}
profileType = "userprofile"
key = "1152473"
businessType = "keyless"
name = "consumer"
aggregates = {GenericData$Array#7948} size = 1
0 = {ProfileEventAggregate#7950} "geo_id {geo_id=java.lang.Object#863dce2} {keyless_select_destination_cnt=java.lang.Object#7cdb4bfc, total_estimated_distance=java.lang.Object#52e81f57}"
entityType = "geo_id"
dimension = {HashMap#7952} size = 1
"geo_id" -> {Object#7957}
key = "geo_id"
value = {Object#7957}
Class has no fields
metrics = {HashMap#7953} size = 2
"keyless_select_destination_cnt" -> {Object#7962}
key = "keyless_select_destination_cnt"
value = {Object#7962}
Class has no fields
"total_estimated_distance" -> {Object#7963}
Please help me to understand what is happening why i am not getting correct data.
public class ProfileEventAggregateEmittedTuple implements Cloneable, Serializable {
private String profileType;
private String key;
private String businessType;
private String name;
private List<ProfileEventAggregate> aggregates = new ArrayList<ProfileEventAggregate>();
private long startTime;
private long endTime;
public String getProfileType() {
return profileType;
}
public void setProfileType(String profileType) {
this.profileType = profileType;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public String getBusinessType() {
return businessType;
}
public void setBusinessType(String businessType) {
this.businessType = businessType;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<ProfileEventAggregate> getAggregates() {
return aggregates;
}
public void addAggregate(ProfileEventAggregate aggregate) {
this.aggregates.add(aggregate);
}
public void setAggregates(List<ProfileEventAggregate> aggregates) {
this.aggregates = aggregates;
}
public long getStartTime() {
return startTime;
}
public void setStartTime(long startTime) {
this.startTime = startTime;
}
public long getEndTime() {
return endTime;
}
public void setEndTime(long endTime) {
this.endTime = endTime;
}
#Override
public ProfileEventAggregateEmittedTuple clone() {
ProfileEventAggregateEmittedTuple clone = new ProfileEventAggregateEmittedTuple();
clone.setProfileType(this.profileType);
clone.setKey(this.key);
clone.setBusinessType(this.businessType);
clone.setName(this.name);
for (ProfileEventAggregate aggregate : this.aggregates) {
clone.addAggregate(aggregate.clone());
}
return clone;
}
public class ProfileEventAggregate implements Cloneable, Serializable {
private String entityType;
private Map<String, Object> dimension =new LinkedHashMap<String, Object>();
private Map<String, Object> metrics = new LinkedHashMap<String, Object>();
public Map<String, Object> getDimension() {
return dimension;
}
public void setDimension(Map<String, Object> dimension) {
this.dimension.putAll(dimension);
}
public void addDimension(String dimensionKey, Object dimensionValue) {
this.dimension.put(dimensionKey, dimensionValue);
}
public Map<String, Object> getMetrics() {
return metrics;
}
public void addMetric(String metricKey, Object metricValue) {
this.metrics.put(metricKey, metricValue);
}
public void setMetrics(Map<String, Object> metrics) {
this.metrics.putAll(metrics);
}
public String getEntityType() {
return entityType;
}
public void setEntityType(String entityType) {
this.entityType = entityType;
}
#Override
public ProfileEventAggregate clone() {
ProfileEventAggregate clone = new ProfileEventAggregate();
clone.setEntityType(this.entityType);
clone.getDimension().putAll(this.getDimension());
clone.getMetrics().putAll(this.metrics);
return clone;
}

When you don't enableObjectReuse, objects are copied with your configured serializer (seems to be Avro?).
In your case, you use Map<String, Object> where you cannot infer a plausible schema.
The easiest fix would be to enableObjectReuse. Else make sure your serializer matches your data. So you could add a unit test where you use AvroSerializer#copy and make sure your POJO is properly annotated if you want to stick with Avro reflect or even better go with a schema first approach, where you generate your Java POJO with a Avro schema and use specific Avro.
Let's discuss some alternatives:
Use GenericRecord. Instead of converting it to a Java type, directly access GenericRecord. This is usually the only way when the full record is flexible (e.g. your job takes any input and writes it out to S3).
Denormalize schema. Instead of having some class Event { int id; Map<String, Object> data; } you would use class EventInformation { int id; String predicate; Object value; }. You would need to group all information for processing. However, you will run into the same type issues with Avro.
Use wide-schema. Looking at the previous approach, if the different predicates are known beforehand, then you can use that to craft a wide-schema class Event { int id; Long predicate1; Integer predicate2; ... String predicateN; } where all oft he entries are nullable and most of them are indeed null. To encode null is very cheap.
Ditch Avro. Avro is fully typed. You may want to use something more dynamic. Protobuf has Any to support arbitrary submessages.
Use Kryo. Kryo can serialize arbitrary object trees at the cost of being slower and having more overhead.
If you want to write the data, you also need to think about a solution where the type information is added for proper deserialization. For an example, check out this JSON question. But there are more ways to implement it.

javax.ws.rs.core.Response with a collection as entity

I have the following code:
#GET
#Path("v1/entity")
#ApiOperation(
value = "List",
notes = "Enables you to List.",
tags = { "List" })
#ApiImplicitParams(
{
#ApiImplicitParam(name = "pageSize",
value = "Page Size",
dataType = "int",
paramType = "formData",
example = "50"),
#ApiImplicitParam(name = "sortAscending",
value = "Sort Ascending",
dataType = "boolean",
paramType = "formData",
example = "false")
})
public Response list(#ApiParam(hidden = true) Integer pageSize,
#ApiParam(hidden = true) Boolean sortAscending) {
Collection<EntityData> dataCollection;
if (pageSize == null || sortAscending == null || pageSize <= 0) {
dataCollection = storeController.list();
} else {
SortDirection sortDirection = sortAscending ? SortDirection.ASC : SortDirection.DESC;
dataCollection= storeController.list(pageSize, sortDirection);
}
logger.info("List contains {} elements", dataCollection.size());
GenericEntity<Collection<EntityData>> response = new GenericEntity<Collection<EntityData>>(dataCollection){};
return Response.ok(response).build();
//return ResponseEntity.ok(dataCollection);
}
Both methods from storeController object only return an ArrayList of EntityData, whose structure is below:
public class EntityData implements Serializable {
private String document;
private String validTo;
private String validFrom;
private String entityAlias;
public String getDocument() {
return document;
}
public void setDocument(String document) {
this.document= document;
}
public String getValidTo() {
return validTo;
}
public void setValidTo(String validTo) {
this.validTo = validTo;
}
public String getValidFrom() {
return validFrom;
}
public void setValidFrom(String validFrom) {
this.validFrom = validFrom;
}
public String getEntityAlias() {
return entityAlias;
}
public void setEntityAlias(String entityAlias) {
this.entityAlias = entityAlias;
}
}
When I call the API, I get this error:
No message body writer has been found for class java.util.ArrayList, ContentType: */*
I used this as a reference, but it didn't work. I've also tried to, instead of using Response to retrieve the contents, using ResponseEntity<Collection<EntityData>>, but the error persists. Just as using directly the "ArrayList" in the Response (without wrapping on GenericEntity).
If I change the last line to return Response.ok().build(); it works, but I need the Collection to be retrieved... The logger indicates that the Collection has 9 elements when I run.
I know I am missing something here, but I can't see it. Can you help me?

Make sure your resource method (or the entire resource class) is annotated either with:
#Produces("applicaton/json") or;
#Produces(MediaType.APPLICATION_JSON).
Then ensure you have a JSON provider, such as Jackson, in the classpath:
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson.version}</version>
</dependency>
Depending on how the JAX-RS implementation is configured, adding the jackson-jaxrs-json-provider artifact to your classpath is enough to use Jackson as a JSON provider. Otherwise you need to register JacksonJsonProvider.

How to store a List<String> from spring JPA to text[] array in database

I am trying to write an Java- Array List into postgres database.　But i am not able to DO, i don't get error as well. I am trying to post data
"skill":["java,dotnet"]
using postman tool but on Post nothing happens and i don't see any data in db.I also don't get any error in my console.please help me out on this i m wondering how to do this
I am able to read Array data from database but cannot write
/*
* Spring Entity:
*/
#Column(name="skill" , columnDefinition = "text[]")
#Convert(converter =ListToStringConverter.class)
private List<String> skill=new ArrayList<>();
ublic List<String> getSkill() {
return skill;
}
public void setSkill(List<String> skill) {
this.skill= skill;
}
#Converter
public class ListToStringConverter implements AttributeConverter<List<String>, String> {
#Override
public String convertToDatabaseColumn(List<String> attribute) {
if (attribute == null || attribute.isEmpty()) {
return "";
}
return StringUtils.join(attribute, ",");
}
#Override
public List<String> convertToEntityAttribute(String dbData) {
if (dbData == null || dbData.trim().length() == 0) {
return new ArrayList<String>();
}
String[] data = dbData.split(",");
return Arrays.asList(data);
}
}
Controller Code:
This is my Controller code to create a object in database and i use a interface service that is call from controller to Save to db-Postgres which is a JPA repo
#RequestMapping(value = "/reqCreate", method = RequestMethod.POST)
public ResponseEntity<Requirement> addRequirement(#RequestBody Requirement requirement) {
reqService.save(requirement);
return new ResponseEntity<Requirement>(requirement, HttpStatus.CREATED);
}
Service:
public void save(Requirement r) {
reqRepo.save(r);
}

The columnDefinition seems to be wrong.
You want to convert Collection into String, but you still declare column of array type.
Replace "text[]" with "text" and check if it works?

Setting Default value to a variable when deserializing using gson

I am trying to convert JSON to Java object. When a certain value of a pair is null, it should be set with some default value.
Here is my POJO:
public class Student {
String rollNo;
String name;
String contact;
String school;
public String getRollNo() {
return rollNo;
}
public void setRollNo(String rollNo) {
this.rollNo = rollNo;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getSchool() {
return school;
}
public void setSchool(String school) {
this.school = school;
}
}
Example JSON object:
{
"rollNo":"123", "name":"Tony", "school":null
}
So if school is null, I should make this into a default value, such as "school":"XXX". How can I configure this with Gson while deserializing the objects?

If the null is in the JSON, Gson is going to override any defaults you might set in the POJO. You could go to the trouble of creating a custom deserializer, but that might be overkill in this case.
I think the easiest (and, arguably best given your use case) thing to do is the equivalent of Lazy Loading. For example:
private static final String DEFAULT_SCHOOL = "ABC Elementary";
public String getSchool() {
if (school == null) school == DEFAULT_SCHOOL;
return school;
}
public void setSchool(String school) {
if (school == null) this.school = DEFAULT_SCHOOL;
else this.school = school;
}
Note: The big problem with this solution is that in order to change the Defaults, you have to change the code. If you want the default value to be customizable, you should go with the custom deserializer as linked above.

I think that the way to do this is to either write your no-args constructor to fill in default values, or use a custom instance creator. The deserializer should then replace the default values for all attributes in the JSON object being deserialized.

I was having the same issue, until I found this great solution.
For reference, you can create a post-processing class:
interface PostProcessable {
fun gsonPostProcess()
}
class PostProcessingEnabler : TypeAdapterFactory {
fun <T> create(gson: Gson, type: TypeToken<T>): TypeAdapter<T> {
val delegate = gson.getDelegateAdapter(this, type)
return object : TypeAdapter<T>() {
#Throws(IOException::class)
fun write(out: JsonWriter, value: T) {
delegate.write(out, value)
}
#Throws(IOException::class)
fun read(`in`: JsonReader): T {
val obj = delegate.read(`in`)
if (obj is PostProcessable) {
(obj as PostProcessable).gsonPostProcess()
}
return obj
}
}
}
}
Register it like this:
GsonBuilder().registerTypeAdapterFactory(PostProcessingEnabler())
Implement it on your model:
class MyClass : Serializable, PostProcessable {
// All your variable data
override fun gsonPostProcess() {
// All your post processing logic you like on your object
// set default value for example
}
}
And finally use it when converting json string:
var myObject = myGson.fromJson(myObjectJson, MyClass::class)
Or using retrofit2:
val api = Retrofit.Builder()
.baseUrl(BASE_URL)
.addConverterFactory(
GsonConverterFactory.create(
GsonBuilder().registerTypeAdapterFactory(
GsonPostProcessingEnabler()
).create()
)
)
.client(OkHttpClient.Builder().build())
.build()
.create(AccountApi::class.java)

You can simply make a universal function that checks for null
model.SchoolName= stringNullChecker(model.SchoolName);
public static String stringNullChecker(String val) {
if (null == val) val = "";
return val;
}

Parsing JSON to Enum Fields

in my Android application, I am returned a simple JSON object with simple key value pairs, eg:
{"username" : "billySmith", "gender" : 1}
And an enum with the respective field names username and gender (String and int, respectively).
I am trying to use Gson to parse the JSON object and populate the enum fields with the json values. I am a little uncertain of how to use GSON with enums. I am familiar with the concept that an instance of an object should be set equal to gson.fromJson(jsonObect, instanceType.class);.
To add more detail, I am using Enums so that the values can be retrieved from anywhere in my android project.
if (response.getStatusLine().getStatusCode() == 200 && result != "")
{
GlobalEnum globalEnum = GlobalEnum.getInstance();
Gson gson = new GsonBuilder().create();
globalEnum = gson.fromJson(result, GlobalEnum.class);
}
where "result" is the string representation of an HTTP Response's entity
GlobalEnum snippet:
public enum GlobalEnum
{
INSTANCE;
private String username;
private int gender;
public static GlobalEnum getInstance()
{
return INSTANCE;
}
public int getGender()
{
return gender;
}
public void setGender(int gender)
{
this.gender = gender;
}
}
*Edit:
Reworded: I have an enum, and I have a jsonObject. Both the enum and JSON object have "username" and "gender". using Gson, I would like to parse the JSON object so that the Values from the JSONobject will be assigned to the respective fields in the Enum.

You may have misunderstood the meaning of an Enum in Java. They usually shouldn't be opened to modifications on runtime like this.
I guess the following logic would serve you better, saving you from this kind of trouble when parsing JSON into Enums.
First, a UserInformation Java Bean class, wrapping the username and gender fields:
public class UserInformation
{
private String username;
private int gender;
public UserInformation(String username, int gender)
{
this.username = username;
this.gender = gender;
}
public String getUsername()
{
return username;
}
public void setUsername(String username)
{
this.username = username;
}
public int getGender()
{
return gender;
}
public void setGender(int gender)
{
this.gender = gender;
}
}
Then your GlobalEnum class, renamed to GlobalValues and modified to work as a value container:
public abstract class GlobalValues
{
// You can also create get/set methods for encapsulation if you want
public static UserInformation userInformation;
}
And then the logic on which you are parsing your JSON String into your UserInformation object, and then storing it on your GlobalValues class.
String jsonStr = "{\"username\" : \"billySmith\", \"gender\" : 1}";
Gson gson = new Gson();
GlobalValues.userInformation = gson.fromJson(jsonStr, UserInformation.class);

As I said in the comment you need create a type adapter to be able to get your enum during json parsing. This is an example of what i have done for my purposes.
In your enum create TypeAdapterFactory gsonTypeAdaptor like so:
public static TypeAdapterFactory gsonTypeAdaptor = new TypeAdapterFactory() {
#Override
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
Class<T> rawType = (Class<T>) type.getRawType();
if (!(rawType.isEnum() && Predicates.assignableFrom(rawType).apply(<your enum>.class))) {
return null;
}
return new TypeAdapter<T>() {
public void write(JsonWriter out, T value) throws IOException {
if (value == null) {
out.nullValue();
} else {
out.value(((<your enum>)value).name);
}
}
public T read(JsonReader reader) throws IOException {
if (reader.peek() == JsonToken.NULL) {
reader.nextNull();
return null;
} else {
return (T) <get your enum by using reader.nextString()>;
}
}
};
}
};
one adapter is in place, register it with your gson builder like so:
builder.registerTypeAdapterFactory(<your enum>.gsonTypeAdaptor);
Let me know if this was useful.