Hadoop facebook mutual friends using mapreduce - java

I was trying a mapreduce program in hadoop (Java version) , to find the mutual friends list from a json file . The json file content has the following pattern :
{"name":"abc","id":123} [{"name":"xyz","id":124},{"name":"def","id":125},{"name":"cxf","id":155}]
{"name":"cxf","id":155} [{"name":"xyz","id":124},{"name":"abc","id":123},{"name":"yyy","id":129}]
Pattern to be interpreted as follows :
friend json tab separated by array of related friends json's
Hence abc has xyz , def and cxf as friends
cxf has xyz abc and yyy as friends .
Given the above the mutual friends between abc and cxf is xyz .
tried to implement the same using mapreduce by creating custom writables , with the mapper emitting following key values , key being pair of friends and value being related friends of the first friend in the key (ie , pair of friends)
K->V
(abc,xyz) -> [xyz,def,cxf]
(abc,def) -> [xyz,def,cxf]
(abc,cxf) -> [xyz,def,cxf]
(cxf,xyz) -> [xyz,abc,yyy]
(cxf,abc) -> [xyz,abc,yyy]
(cxf,yyy) -> [xyz,abc,yyy]
The key here is actually a Custom writable , created a class which extends WritableComparable and i have overridden the compareTo method so that both these pairs (a,b) and (b,a) are same . But the problem i am facing is that the compareTo method is not invoked for all combinations of pairs and hence the reducer logic is failing.
Based on the above example , there are 6 K, V pairs emitted by the mapper . But compareTo is invoked only 5 times key1.compareTo(key2) , key2.compareTo(key3), key3.compareTo(key4),key4.compareTo(key5),,key5.compareTo(key6) .
Any idea why this is happening ?
Below is the code as per the logic suggested by f11ler
Driver class :
package com.facebook.updated;
import java.io.IOException;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger;
public class FacebookMain extends Configured implements Tool
{
Logger logger = Logger.getLogger(FacebookMain.class);
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(new FacebookMain(), args));
}
#Override
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
logger.info("Running======>");
Job job = Job.getInstance();
job.setJarByClass(FacebookMain.class);
job.setJobName("FBApp");
job.setMapOutputKeyClass(Friend.class);
job.setMapOutputValueClass(Friend.class);
job.setOutputKeyClass(FriendPair.class);
job.setOutputValueClass(Friend.class);
job.setMapperClass(FacebookMapper.class);
job.setReducerClass(FacebookReducer.class);
job.setInputFormatClass(org.apache.hadoop.mapreduce.lib.input.TextInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
boolean val = job.waitForCompletion(true);
return val ? 0 : 1;
}
}
The customWritables (used to represent a friend and friendpair)
package com.facebook.updated;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import lombok.Getter;
import lombok.Setter;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.log4j.Logger;
#Getter
#Setter
public class Friend implements WritableComparable<Friend> {
Logger logger = Logger.getLogger(Friend.class);
private IntWritable id;
private Text name;
public Friend() {
this.id = new IntWritable();
this.name = new Text();
}
#Override
public int compareTo(Friend arg0) {
int val = getId().compareTo(arg0.getId());
logger.info("compareTo Friend ======> " + arg0 + " and " + this + " compare is " + val);
return val;
}
#Override
public void readFields(DataInput in) throws IOException {
id.readFields(in);
name.readFields(in);
}
#Override
public void write(DataOutput out) throws IOException {
id.write(out);
name.write(out);
}
#Override
public boolean equals(Object obj) {
Friend f2 = (Friend) obj;
boolean val = this.getId().equals(f2.getId());
//logger.info("equals Friend ======> " + obj + " and " + this);
return val;
}
#Override
public String toString() {
return id + ":" + name + " ";
}
}
package com.facebook.updated;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import lombok.Getter;
import lombok.Setter;
import org.apache.hadoop.io.WritableComparable;
import org.apache.log4j.Logger;
#Getter
#Setter
public class FriendPair implements WritableComparable<FriendPair> {
Logger logger = Logger.getLogger(FriendPair.class);
private Friend first;
private Friend second;
public FriendPair() {
this.first = new Friend();
this.second = new Friend();
}
public FriendPair(Friend f1, Friend f2) {
this.first = f1;
this.second = f2;
}
#Override
public int compareTo(FriendPair o) {
logger.info("compareTo FriendPair ======> " + o + " and " + this);
FriendPair pair2 = o;
int cmp = -1;
if (getFirst().compareTo(pair2.getFirst()) == 0 || getFirst().compareTo(pair2.getSecond()) == 0) {
cmp = 0;
}
if (cmp != 0) {
// logger.info("compareTo FriendPair ======> " + o + " and " + this
// + " comparison is " + cmp);
return cmp;
}
cmp = -1;
if (getSecond().compareTo(pair2.getFirst()) == 0 || getSecond().compareTo(pair2.getSecond()) == 0) {
cmp = 0;
}
// logger.info("compareTo FriendPair ======> " + o + " and " + this +
// " comparison is " + cmp);
// logger.info("getFirst() " + getFirst());
// logger.info("pair2.getFirst() " + pair2.getFirst());
// logger.info("getFirst().compareTo(pair2.getFirst()) " +
// getFirst().compareTo(pair2.getFirst()));
// logger.info("getFirst().compareTo(pair2.getSecond()) " +
// getFirst().compareTo(pair2.getSecond()));
// logger.info("getSecond().compareTo(pair2.getFirst()) " +
// getSecond().compareTo(pair2.getFirst()));
// logger.info("getSecond().compareTo(pair2.getSecond()) " +
// getSecond().compareTo(pair2.getSecond()));
// logger.info("pair2.getSecond() " + pair2.getSecond());
// logger.info("getSecond() " + getSecond());
// logger.info("pair2.getFirst() " + pair2.getFirst());
// logger.info("pair2.getSecond() " + pair2.getSecond());
return cmp;
}
#Override
public boolean equals(Object obj) {
FriendPair pair1 = this;
FriendPair pair2 = (FriendPair) obj;
boolean eq = false;
logger.info("equals FriendPair ======> " + obj + " and " + this);
if (pair1.getFirst().equals(pair2.getFirst()) || pair1.getFirst().equals(pair2.getSecond()))
eq = true;
if (!eq) {
// logger.info("equals FriendPair ======> " + obj + " and " + this +
// " equality is " + eq);
return false;
}
if (pair1.getSecond().equals(pair2.getFirst()) || pair1.getSecond().equals(pair2.getSecond()))
eq = true;
// logger.info("equals FriendPair ======> " + obj + " and " + this +
// " equality is " + eq);
return eq;
}
#Override
public void readFields(DataInput in) throws IOException {
first.readFields(in);
second.readFields(in);
}
#Override
public void write(DataOutput out) throws IOException {
first.write(out);
second.write(out);
}
#Override
public String toString() {
return "[" + first + ";" + second + "]";
}
#Override
public int hashCode() {
logger.info("hashCode FriendPair ======> " + this);
return first.getId().hashCode() + second.getId().hashCode();
}
}
Mapper and Reducer
package com.facebook.updated;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.log4j.Logger;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.util.JSON;
public class FacebookMapper extends Mapper<LongWritable, Text, Friend, Friend> {
Logger log = Logger.getLogger(FacebookMapper.class);
#Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Friend, Friend>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer st = new StringTokenizer(line, "\t");
String person = st.nextToken();
String friends = st.nextToken();
BasicDBObject personObj = (BasicDBObject) JSON.parse(person);
BasicDBList friendsList = (BasicDBList) JSON.parse(friends);
List<Friend> frndJavaList = new ArrayList<>();
for (Object frndObj : friendsList) {
frndJavaList.add(getFriend((BasicDBObject) frndObj));
}
Friend frnd = getFriend(personObj);
Friend[] array = frndJavaList.toArray(new Friend[frndJavaList.size()]);
for (Friend f : array) {
log.info("Map output is " + f + " and " + frnd);
context.write(f, frnd);
}
}
private static Friend getFriend(BasicDBObject personObj) {
Friend frnd = new Friend();
frnd.setId(new IntWritable(personObj.getInt("id")));
frnd.setName(new Text(personObj.getString("name")));
frnd.setHomeTown(new Text(personObj.getString("homeTown")));
return frnd;
}
}
package com.facebook.updated;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.log4j.Logger;
public class FacebookReducer extends Reducer<Friend, Friend, FriendPair, Friend> {
Logger log = Logger.getLogger(FacebookReducer.class);
#Override
protected void reduce(Friend friend, Iterable<Friend> vals,
Reducer<Friend, Friend, FriendPair, Friend>.Context context) throws IOException, InterruptedException {
List<Friend> friends = new ArrayList<>();
for (Friend frnd : vals) {
friends.add(frnd);
}
log.info("Reducer output is " + friend + " and values are " + friends);
if (friends.size() == 2) {
FriendPair key = new FriendPair(friends.get(0), friends.get(1));
context.write(key, friend);
} else {
//log.info("Size of friends is not 2 key is " + friend + " and values are " + friends);
}
}
}
Input json file containing 2 lines
{"name":"abc","id":123} [{"name":"xyz","id":124},{"name":"def","id":125},{"name":"cxf","id":155}]
{"name":"cxf","id":155} [{"name":"xyz","id":124},{"name":"abc","id":123},{"name":"yyy","id":129}]
Output of reducer
(abc,abc)->xyz

compareTo method is required for sorting, this relation should be transitive. This mean that if a > b and b > c then a > c. Probably this is not true for your implementation.
Why you generate this kind of records in mapper?
If "being a friend" is a symmetric relation you can simply do a mapper-only job with this logic (pseudo-code):
for(int i = 0; i < values.length; ++i)
for(int j = 0; j < values.length; ++j)
if (i ==j)
continue
emmit (values[i], values[j]), key
Update:
If this is not symmetric (which means that "xyz has friend abc" not follows from "abc has friend xyz") then we need reverse records:
Mapper:
for(int i = 0; i < values.length; ++i)
emmit values[i], key
Reducer (same as mapper before):
for(int i = 0; i < values.length; ++i)
for(int j = 0; j < values.length; ++j)
if (i ==j)
continue
emmit (values[i], values[j]), key
Update2:
Lets see how this algorithm works with your example:
The result of mapper:
xyz -> abc
def -> abc
cxf -> abc
xyz -> cxf
abc -> cxf
yyy -> cxf
Mapreduce wiil group this values by key, so the input of reducer:
xyz -> [abc,cxf]
def -> [abc]
cxf -> [abc]
abc -> [cxf]
yyy -> [cxf]
In reducer we do a nested loop by values, but skip comparing with self. Result:
(abc, cxf) -> xyz
This is what we want to get.

Related

Is Java VM defective or why does repeated use of CompilationTask and Reflections cause a software defect?

ISSUE RESOLVED: The code below remains defective in Java 1.8.232 but has correct behavior in Java 13.0.1. Perhaps someone at OpenJDK could look into this issue because Java 8 is still supported? It is unknown what version resolves this issue between Java 8 and Java 13.
Is Java VM defective? Or why does this repeated use of Java CompilationTask, ToolProvider.getSystemJavaCompiler, and Reflections cause a software defect?
The following code has been reduce from the original code baseline (https://github.com/peytonc), and makes use of run-time code generation, run-time compilation, and reflections API.
In the software below, a defect occurs with repeated calls to the Java compiler (to compile dynamic code at runtime) and reflections (to invoke the dynamic code). At first, all calls perform the correct calculation, but after an extended period of time, the calculations become incorrect. When the software is behaving as expected, the JVM only keeps 1 instance of each unique package/class (which seems correct to me). However, when the defect occurs, the JVM has two or more copies of the same package/class (which seems incorrect to me).
On my computer [Intel NUC, Fedora 31, OpenJDK Runtime Environment (build 1.8.0_232-b09)], the code below produces the results below. Iterations 0-913 all show correct behavior, whereas iteration 914 shows the first defective behavior.
INFO: Iteration #0: (c0) (i0) (c1) (i1) (c2) (i2) (c4) (i4) (c3) (i3) (c5) (i5) (c6) (i6) (c7) (i7) (c8) (i8) (c9) (i9)
INFO: Iteration #1: (c0) (i0) (c1) (i1) (c4) (i4) (c5) (c6) (i5) (i6) (c7) (i7) (c8) (i8) (c9) (i9) (c3) (i3) (c2) (i2)
INFO: Iteration #2: (c3) (i3) (c1) (i1) (c2) (i2) (c0) (c4) (i4) (i0) (c5) (i5) (c7) (c8) (i7) (c9) (i9) (c6) (i6) (i8)
...
INFO: Iteration #913: (c0) (i0) (c2) (i2) (c3) (i3) (c4) (c5) (i4) (i5) (c1) (c6) (i1) (i6) (c7) (i7) (c8) (i8) (c9) (i9)
INFO: Iteration #914: (c0) (c1) (c3) (i3) (i0) (c2) (i2) (i1)
ERROR: On iteration #914 id #4, actualResultsFromProgram != expectedResultFromProgram, 4!=5
ERROR: On iteration #914 id #5, actualResultsFromProgram != expectedResultFromProgram, 5!=6
ERROR: On iteration #914 id #6, actualResultsFromProgram != expectedResultFromProgram, 6!=7
ERROR: On iteration #914 id #7, actualResultsFromProgram != expectedResultFromProgram, 7!=8
ERROR: On iteration #914 id #8, actualResultsFromProgram != expectedResultFromProgram, 8!=9
ERROR: On iteration #914 id #9, actualResultsFromProgram != expectedResultFromProgram, 9!=10
package minijava;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
import javax.tools.JavaCompiler.CompilationTask;
public class Programs {
public int species = 1;
private static final JavaCompiler JAVA_COMPILER = ToolProvider.getSystemJavaCompiler();
private static final int MAX_POPULATION = 10;
private List<Program> listProgram = new ArrayList<Program>(MAX_POPULATION);
private static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors();
private long iteration = 0;
// The compute function will increment the first array value by one
private static final String sourceCodeTemplate =
"package species0.id0; \n" +
"import java.util.ArrayList; \n" +
"public class GeneticProgram { \n" +
" public static void compute(ArrayList<Long> values00) { \n" +
" long value = values00.get(0); \n" +
" System.out.print(\" (c\" + value + \") \"); \n" +
" values00.set(0, value + 1); \n" +
" } \n" +
"} \n";
public Programs() {
System.out.println(sourceCodeTemplate);
int errorCode = 0;
while(errorCode == 0) {
System.out.print("\nINFO: Iteration #" + iteration + ":");
errorCode = createPrograms();
if(errorCode == 0) {
compilePrograms();
}
if(errorCode == 0) {
executePrograms();
}
iteration++;
}
}
public int createPrograms() {
listProgram.clear();
for(int index=0; index<MAX_POPULATION; index++) {
String simulatedRandomSourceCode = replacePackage(sourceCodeTemplate, species, index);
Program program = new Program(simulatedRandomSourceCode, species, index);
program.vectors.add(new Long(index)); // this number will be incremented by 1 upon execution of program
listProgram.add(program);
}
return 0;
}
public int compilePrograms() {
DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<JavaFileObject>();
for(Program program : listProgram) {
try (StandardJavaFileManager standardJavaFileManager = JAVA_COMPILER.getStandardFileManager(diagnostics, Locale.ENGLISH, null)) {
Iterable<Program> javaFileObject = Arrays.asList(program);
ProgramClassSimpleJavaFileObject programClassSimpleJavaFileObject = null;
try {
programClassSimpleJavaFileObject = new ProgramClassSimpleJavaFileObject(Program.PACKAGE_SPECIES + species + "." + Program.PACKAGE_ID + program.ID + "." + Program.PROGRAM_CLASS);
} catch (Exception e) {
e.printStackTrace();
return -1;
}
ProgramForwardingJavaFileManager programForwardingJavaFileManager = new ProgramForwardingJavaFileManager(standardJavaFileManager, programClassSimpleJavaFileObject, program.programClassLoader);
CompilationTask compilerTask = JAVA_COMPILER.getTask(null, programForwardingJavaFileManager, diagnostics, null, null, javaFileObject);
if (!compilerTask.call()) {
System.out.println("\nERROR: compilePrograms compilerTask.call()");
return -1;
}
} catch (IOException e) {
e.printStackTrace();
return -1;
}
}
return 0;
}
public int executePrograms() {
List<CallableMiniJava> listCallable = new ArrayList<CallableMiniJava>(MAX_POPULATION);
ExecutorService executorService = Executors.newFixedThreadPool(AVAILABLE_PROCESSORS);
try {
for(Program program : listProgram) {
listCallable.add(new CallableMiniJava(program));
}
for(CallableMiniJava callableMiniJava : listCallable) {
executorService.execute(callableMiniJava);
}
executorService.shutdown();
if(!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
executorService.shutdownNow();
int milliseconds = 1000;
while(!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
milliseconds += 1000;
System.out.println("\nINFO: Runaway program for " + milliseconds + " milliseconds");
}
}
for (Program program : listProgram) {
long actualResultsFromProgram = program.vectors.get(0);
long expectedResultFromProgram = program.ID + 1;
if(actualResultsFromProgram != expectedResultFromProgram) {
System.out.println("\nERROR: On iteration #" + iteration + " id #" + program.ID + ", actualResultsFromProgram != expectedResultFromProgram, " + actualResultsFromProgram + "!=" + expectedResultFromProgram);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
return -1;
}
return 0;
}
public static String replacePackage(String source, int species, int packageNumber) {
return source.replaceFirst("package species[0-9][^;]*;", "package " + Program.PACKAGE_SPECIES + species + "." + Program.PACKAGE_ID + packageNumber + ";");
}
public static void main(String[] args) {
Programs programs = new Programs();
}
}
package minijava;
import java.net.URI;
import java.util.ArrayList;
import javax.tools.SimpleJavaFileObject;
public class Program extends SimpleJavaFileObject {
public static final String PROGRAM_CLASS = new String("GeneticProgram");
public static final String PACKAGE_SPECIES = new String("species");
public static final String PACKAGE_ID = new String("id");
public String source;
public ArrayList<Long> vectors;
public int species;
public int ID;
public ProgramClassLoader programClassLoader = null;
Program(String source, int species, int ID) {
super(URI.create("string:///" + PACKAGE_SPECIES + species + '/' + PACKAGE_ID + ID + '/' + PROGRAM_CLASS + Kind.SOURCE.extension), Kind.SOURCE);
this.source = new String(source);
this.species = species;
this.ID = ID;
vectors = new ArrayList<Long>(1);
programClassLoader = new ProgramClassLoader(ClassLoader.getSystemClassLoader());
}
#Override
public CharSequence getCharContent(boolean ignoreEncodingErrors) {
return source;
}
}
package minijava;
import java.lang.reflect.Method;
import java.util.ArrayList;
public class CallableMiniJava implements Runnable {
private Program program = null;
private Class<?> cls = null;
private Method method = null;
public CallableMiniJava(Program program) {
if(program.vectors != null) {
this.program = program;
try {
cls = program.programClassLoader.loadClass(Program.PACKAGE_SPECIES + program.species + "." + Program.PACKAGE_ID + program.ID + "." + Program.PROGRAM_CLASS);
} catch (ClassNotFoundException e1) {
e1.printStackTrace();
}
try {
method = cls.getMethod("compute", ArrayList.class);
} catch (NoSuchMethodException e1) {
e1.printStackTrace();
} catch (SecurityException e1) {
e1.printStackTrace();
}
}
}
#Override
public void run() {
try {
method.invoke(null, program.vectors);
System.out.print(" (i" + program.ID + ") ");
} catch(Exception e) {
e.printStackTrace();
}
}
}
package minijava;
import java.util.HashMap;
import java.util.Map;
public class ProgramClassLoader extends ClassLoader {
public Map<String, ProgramClassSimpleJavaFileObject> mapProgramClass = new HashMap<>();
public ProgramClassLoader(ClassLoader classLoader) {
super(classLoader);
}
#Override
protected Class<?> findClass(String className) throws ClassNotFoundException {
ProgramClassSimpleJavaFileObject programClassSimpleJavaFileObject = mapProgramClass.get(className);
if(programClassSimpleJavaFileObject != null) {
byte[] byteCode = programClassSimpleJavaFileObject.byteArrayOutputStream.toByteArray();
return defineClass(className, byteCode, 0, byteCode.length);
} else {
return super.findClass(className);
}
}
}
package minijava;
import javax.tools.SimpleJavaFileObject;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
public class ProgramClassSimpleJavaFileObject extends SimpleJavaFileObject {
public ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
public ProgramClassSimpleJavaFileObject(String className) throws Exception {
super(new URI(className), Kind.CLASS);
}
#Override
public OutputStream openOutputStream() throws IOException {
return byteArrayOutputStream;
}
}
package minijava;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import java.io.IOException;
public class ProgramForwardingJavaFileManager extends ForwardingJavaFileManager<JavaFileManager> {
private ProgramClassLoader programClassLoader;
private ProgramClassSimpleJavaFileObject programClassSimpleJavaFileObject;
protected ProgramForwardingJavaFileManager(JavaFileManager javaFileManager, ProgramClassSimpleJavaFileObject programClassSimpleJavaFileObject, ProgramClassLoader programClassLoader) {
super(javaFileManager);
this.programClassSimpleJavaFileObject = programClassSimpleJavaFileObject;
this.programClassLoader = programClassLoader;
this.programClassLoader.mapProgramClass.put(programClassSimpleJavaFileObject.getName(), programClassSimpleJavaFileObject);
}
#Override
public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, Kind kind, FileObject fileObject) throws IOException {
return programClassSimpleJavaFileObject;
}
#Override
public ClassLoader getClassLoader(Location location) {
return programClassLoader;
}
}

How to send message to a particular session using web socket in java?

Here below is my client side script for WebSockets. In this code I have defined a WebSockets object with IP address and port.
Client Script:
var webSocket =
new WebSocket('ws://localhost:8080/Spring4JSONHandling/websocket');
webSocket.onmessage = function processMessage(message)
{
var jsonData = JSON.parse(message.data);
jsonforlogout = jsonData;
//var user=JSON.parse(username.data);
console.log(jsonData); //to print
if (jsonData.message != null){
var msgAdd = JSON.stringify(jsonData.message);
msgAdd = msgAdd.replace(/[&\/\\#,+()$~%.'"*?<>{}]/g, '');
alert("3"+msgAdd);
var xyz = msgAdd.split(" : ");
var logged = '${me}';
alert("logged" + logged);
alert("xyz[0]" + xyz[0]);
if (logged == xyz[0]){
alert("1");
$("#chat" + xyz[0]).chatbox("option", "boxManager").addMsg(xyz[0], xyz[1]);
} else{
alert("2");
$("#chat" + xyz[0]).chatbox("option", "boxManager").addMsg(xyz[0], xyz[1]);
}
}
/* if(jsonData.message=='close'){websocket.close();} */
if (jsonData.users != null)
{
document.getElementById("chatusers").innerHTML = "";
var loggedInUser = '${me}';
var i = 0;
while (i < jsonData.users.length)
{
var onlineAdd = JSON.stringify(jsonData.users[i]);
onlineAdd = onlineAdd.replace(/[&\/\\#,+()$~%.'":*?<>{}]/g, '');
if (loggedInUser != onlineAdd){
var clone = "<a href='#' onclick=\"clickFun('" + onlineAdd.trim() + "');\" class='usr' data-chat-id='chat" + onlineAdd + "' data-chat-fname='" + onlineAdd + "' data-chat-lname='" + onlineAdd + "'"
+ " data-chat-status='online' data-chat-alertmsg='' data-chat-alertshow='true' data-rel='popover-hover' data-placement='right' data-html='true'"
+ " data-content=\"<div class='usr-card'>"
+ "<img src='Spring4JSONHandling/resources/img/1.png' alt='Jessica Dolof'>"
+ "<div class='usr-card-content'>"
+ "<h3>Jessica Dolof</h3>"
+ "<p>Sales Administrator</p>"
+ "</div>"
+ "</div>\"><i></i>" + onlineAdd + "</a>";
$('#chatusers').append(clone);
}
i++;
}
}
//or(i = 0; i < responselist.data.length; i++) {
/* var i=0;
while(i<jsonData.users.length)
{
var comboitem = document.createElement("option");
comboitem.text = jsonData.users[i];//"utsav";//
comboitem.value = jsonData.users[i];//"10";
usercombo.options.add(comboitem);
i++;
} */
}
function sendMessage(txt) {
//alert(messageText.value+ " h1");
webSocket.send(txt);
// messageText.value = "";
}
window.onbeforeunload = function () {
webSocket.onclose = function() {};
webSocket.close();
};
Here is the code for the server end point. In this code I have defined a WebSockets object with a server endpoint.
Server Code:
package com.outbottle.controllers;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import javax.json.Json;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObject;
import javax.json.JsonWriter;
import javax.servlet.http.HttpSession;
import javax.websocket.EndpointConfig;
import javax.websocket.OnClose;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.Session;
import javax.websocket.server.ServerEndpoint;
#ServerEndpoint("/websocket")
public class WebSocketTest {
static Set<Session> chatroomUsers= Collections.synchronizedSet(new HashSet<Session>());
private final static HashMap<String, WebSocketTest> sockets = new HashMap<>();
private String myUniqueId;
private String getMyUniqueId() {
// unique ID from this class' hash code
return Integer.toHexString(this.hashCode());
}
#OnOpen
public void handleOpen(Session userSession) throws IOException {
chatroomUsers.add(userSession);
System.out.println("user added"); //user added
this.myUniqueId = this.getMyUniqueId();
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"+ myUniqueId);
WebSocketTest.sockets.put(this.myUniqueId, this);
System.out.println("###################################"+sockets);
}
#OnMessage
public void handleMessage(String message, Session userSession) throws IOException{
String username= (String) userSession.getUserProperties().get("username");
Iterator<Session> itr=chatroomUsers.iterator();
if(username==null){
userSession.getUserProperties().put("username", message);
userSession.getBasicRemote().sendText(buildJsonMessageData("System","You are now connected as "+message));
while (itr.hasNext()) (itr.next()).getBasicRemote().sendText(buildJsonUsersData()); }
else {
while (itr.hasNext()) {
itr.next().getBasicRemote().sendText(buildJsonMessageData(username,message));
}
}
}
#OnClose
public void handleClose(Session userSession) throws IOException {
// TODO Auto-generated method stub
System.out.println("user logout");
chatroomUsers.remove(userSession);
Iterator<Session> itr = chatroomUsers.iterator();
while (itr.hasNext()) (itr.next()).getBasicRemote().sendText(buildJsonUsersData());
}
private String buildJsonUsersData() {
Iterator<String> itr= getUserNames().iterator();
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
while (itr.hasNext()) jsonArrayBuilder.add((String)itr.next());
return Json.createObjectBuilder().add("users", jsonArrayBuilder).build().toString();
}
private String buildJsonMessageData(String username, String message)
{
JsonObject jsonObject=Json.createObjectBuilder().add("message", username+" : "+ message).build();
StringWriter stringWriter= new StringWriter();
try (JsonWriter jsonWriter = Json.createWriter(stringWriter)){
jsonWriter.write(jsonObject);
}
return stringWriter.toString();
}
private Set<String> getUserNames()
{
HashSet<String> returnSet = new HashSet<String>();
Iterator<Session> itr= chatroomUsers.iterator();
System.out.println("######################");
while (itr.hasNext())
{
returnSet.add(itr.next().getUserProperties().get("username").toString());
}
return returnSet;
}
}
The problem is when I send a message to the user on a different IP, all users with the same name get the message. I need to send the message to a particular session only.
It looks like you are iterating through all the sessions chatroomUsers in the handleMessage function.
If you only want to send the message to a particular user/session, do you not need to check for your username property here?

How to obtain the cosine similarity with Lucene

Currently I try to get the cosine similarity between two document with Lucene (4.10.4).
I already read this answer about cosine similarity with Lucene , and I used this example to understand how it works with Lucene.
But when I tested with 2 same words per each document (ex: "Hello world"), I've got a cosine of similarity at 0.9999999999999998
My code look like that:
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.math3.linear.ArrayRealVector;
import org.apache.commons.math3.linear.RealVector;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
public class CosineSimeTest {
static String indexName = "/tmp/CosineExample";
public static final String CONTENT = "field";
public static final int N = 2;
private final Set<String> terms = new HashSet<>();
private final RealVector v1;
private final RealVector v2;
public static void main(String[] args) {
try {
CosineSimeTest cosSim = new CosineSimeTest("hello world", "hello world");
System.out.println(cosSim.getCosineSimilarity());
} catch (IOException e) {
e.printStackTrace();
}
}
public CosineSimeTest(String s1, String s2) throws IOException {
Directory directory = createIndex(s1, s2);
IndexReader reader = DirectoryReader.open(directory);
Map<String, Double> f1 = getWieghts(reader, 0);
Map<String, Double> f2 = getWieghts(reader, 1);
reader.close();
v1 = toRealVector(f1);
System.out.println("V1: " + v1);
v2 = toRealVector(f2);
System.out.println("V2: " + v2);
}
public Directory createIndex(String s1, String s2) throws IOException {
File f = new File(indexName);
if (f.exists()) {
FileUtils.deleteDirectory(f);
}
Directory directory = FSDirectory.open(new File(indexName));
StandardAnalyzer analyzer = new StandardAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(null, analyzer);
IndexWriter writer = new IndexWriter(directory, iwc);
addDocument(writer, s1);
addDocument(writer, s2);
writer.close();
return directory;
}
public void addDocument(IndexWriter writer, String data) throws IOException {
Document doc = new Document();
FieldType type = new FieldType();
type.setIndexed(true);
type.setStoreTermVectors(true);
type.setStoreTermVectorPositions(true);
type.freeze();
Field field = new Field(CONTENT, data, type);
doc.add(field);
writer.addDocument(doc);
}
public double getCosineSimilarity() {
double dotProduct = v1.dotProduct(v2);
System.out.println("Dot: " + dotProduct);
System.out.println("V1_norm: " + v1.getNorm() + ", V2_norm: " + v2.getNorm());
double normalization = (v1.getNorm() * v2.getNorm());
System.out.println("Norm: " + normalization);
return dotProduct / normalization;
}
public Map<String, Double> getWieghts(IndexReader reader, int docId) throws IOException {
Terms vector = reader.getTermVector(docId, CONTENT);
Map<String, Integer> docFrequencies = new HashMap<>();
Map<String, Integer> termFrequencies = new HashMap<>();
Map<String, Double> tf_Idf_Weights = new HashMap<>();
TermsEnum termsEnum = null;
DocsEnum docsEnum = null;
termsEnum = vector.iterator(termsEnum);
BytesRef text = null;
while ((text = termsEnum.next()) != null) {
String term = text.utf8ToString();
docFrequencies.put(term, reader.docFreq(new Term(CONTENT, term)));
docsEnum = termsEnum.docs(null, null);
while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
termFrequencies.put(term, docsEnum.freq());
}
terms.add(term);
}
for (String term : docFrequencies.keySet()) {
int tf = termFrequencies.get(term);
int df = docFrequencies.get(term);
double idf = (1 + Math.log(N) - Math.log(df));
double w = tf * idf;
tf_Idf_Weights.put(term, w);
}
// System.out.println("Printing docFrequencies:");
// printMap(docFrequencies);
//
// System.out.println("Printing termFrequencies:");
// printMap(termFrequencies);
//
// System.out.println("Printing if/idf weights:");
// printMapDouble(tf_Idf_Weights);
return tf_Idf_Weights;
}
public RealVector toRealVector(Map<String, Double> map) {
RealVector vector = new ArrayRealVector(terms.size());
int i = 0;
double value = 0;
for (String term : terms) {
if (map.containsKey(term)) {
value = map.get(term);
} else {
value = 0;
}
vector.setEntry(i++, value);
}
return vector;
}
public static void printMap(Map<String, Integer> map) {
for (String key : map.keySet()) {
System.out.println("Term: " + key + ", value: " + map.get(key));
}
}
public static void printMapDouble(Map<String, Double> map) {
for (String key : map.keySet()) {
System.out.println("Term: " + key + ", value: " + map.get(key));
}
}
public void getVersionOfLucene(StandardAnalyzer analyzer) {
System.out.println("version : " + analyzer.getVersion());
}
}
What is the problem ? How to fix this ?
Thanks in advance.

Bukkit NullPointer Plugin reference

I am making a bukkit plugin that uses a config to store data, but when I use plugin.getConfig() I get a NullPointer. I think is it because of the reference to plugin, but how can I fix that?
The error is in the Storage class where I use the plugin. Instance
Main: http://pastebin.com/d3bFXbiR
import org.bukkit.ChatColor;
import org.bukkit.Location;
import org.bukkit.block.Block;
import org.bukkit.command.Command;
import org.bukkit.command.CommandSender;
import org.bukkit.configuration.file.FileConfiguration;
import org.bukkit.entity.LivingEntity;
import org.bukkit.entity.Player;
import org.bukkit.event.EventHandler;
import org.bukkit.event.block.Action;
import org.bukkit.event.player.PlayerInteractEvent;
import org.bukkit.plugin.java.JavaPlugin;
public class Main extends JavaPlugin {
public void onEnable() {
final FileConfiguration config = this.getConfig();
config.options().copyDefaults(true);
saveConfig();
}
#Override
public boolean onCommand(CommandSender sender, Command cmd, String label, String[] args) {
if (cmd.getName().equalsIgnoreCase("BlockCmd")) {
if (sender.isOp()) {
if (args.length < 1) {
sender.sendMessage(ChatColor.RED + "/BlockCmd [command] | Kijk naar het blok dat je wilt cmd'en");
} else {
Block block = ((LivingEntity) sender).getTargetBlock(null, 100);
Location bl = block.getLocation();
StringBuilder sb = new StringBuilder();
for (int i = 1; i < args.length; i++) {
sb.append(args[i]).append(" ");
}
String allArgs = sb.toString().trim();
Storage.addClickCmd(bl, allArgs);
sender.sendMessage(ChatColor.GRAY + "[BlockCommand] " + ChatColor.BLUE + "Successfully added a command to the block");
sender.sendMessage(ChatColor.GRAY + "[BlockCommand] " + ChatColor.BLUE + "Command: " + ChatColor.GREEN + allArgs);
}
} else {
sender.sendMessage(ChatColor.RED + "Dit is alleen voor operators");
}
}
return true;
}
#EventHandler
public void onRightClick(PlayerInteractEvent event) {
Player p = event.getPlayer();
if ((event.getAction() == Action.RIGHT_CLICK_BLOCK) || (event.getAction() == Action.LEFT_CLICK_BLOCK)) {
Location loc = event.getClickedBlock().getLocation();
int x = loc.getBlockX();
int y = loc.getBlockY();
int z = loc.getBlockZ();
String w = p.getWorld().getName();
String cCc = "Click.Cmd." + w + "." + x + "." + y + "." + z;
if (Storage.getClickCmd(w, x, y, z) != null) {
String cCc2 = Storage.getString(cCc);
p.performCommand(cCc2);
}
}
}
}
Storage: http://pastebin.com/wvQS3n57
import org.bukkit.Location;
import org.bukkit.event.Listener;
public class Storage implements Listener {
static Main plugin;
public Storage(Main instance) {
plugin = instance;
}
public static void addClickCmd(Location loc, String text) {
int x = loc.getBlockX();
int y = loc.getBlockY();
int z = loc.getBlockZ();
String w = loc.getWorld().getName();
if (plugin != null && plugin.getConfig() != null) {
System.out.println("Check");
}
//if(plugin.getConfig() !=null){}
//plugin.getConfig().set("Click.Cmd." + w + "." + x + "." + y + "." + z, text);
}
public static String getClickCmd(String w, int x, int y, int z) {
return plugin.getConfig().getString("Click.Cmd." + w + "." + x + "." + y + "." + z);
}
public static String getString(String path) {
return plugin.getConfig().getString(path);
}
}
You use static Main plugin; but it is never initialized, since you never instantiate a Storage object, only use its static functions.
In your plugin class create a Storage object, and initialize it in the onEnable method, passing the plugin for its constructor. For example:
public class Main extends JavaPlugin {
Storage myStorage = null;
public void onEnable() {
final FileConfiguration config = this.getConfig();
config.options().copyDefaults(true);
saveConfig();
myStorage = new Storage(this);
}
Later in your plugin class use this object instead - and best to make your static methods in Storage (as well as the plugin member) non-static, since you will always need to create an instance to use the plugin set in the constructor.
I think this article gives a nice, basic overview about class members.

Display all the JAC security providers and their service type with Algorithm

i want to create a simple java code that display all the security Providers with :
Name
info
service Type
Algorithm
Main Activity.java
import java.io.ObjectInputStream.GetField;
import java.security.Provider;
import java.security.Provider.Service;
import java.security.Security;
public class MainActivity {
public static void main(String[] args) {
System.out.println("Availble Providers are:");
Provider[] providerList = Security.getProviders();
for (int i = 0; i < providerList.length; i++) {
System.out.println("[" + (i + 1) + "] - Name: "
+ providerList[i].getName());
System.out.println("Information:\n" + providerList[i].getInfo());
System.out
.print("Here are all providers with types of service and algorithm provided:\n");
}
for (int i = 0; i < serviceList.length; i++) {
System.out.println("- Name: " + providerList[i].getName() + "\n");
System.out.print("Service Type: " + serviceList[i].getType()
+ "Algorithm: " + serviceList[i].getAlgorithm());
}
}
}
before the second for loop i need to initialize the services to be able to call the service type and Algorithm.
You need your second loop to be inside the first loop as you need the current provider so you can call its getServices method. Something like:
import java.security.Provider;
import java.security.Provider.Service;
import java.security.Security;
import java.util.Set;
public class MainActivity
{
public static void main(String[] args)
{
Provider [] providerList = Security.getProviders();
for (Provider provider : providerList)
{
System.out.println("Name: " + provider.getName());
System.out.println("Information:\n" + provider.getInfo());
Set<Service> serviceList = provider.getServices();
for (Service service : serviceList)
{
System.out.println("Service Type: " + service.getType() + " Algorithm " + service.getAlgorithm());
}
}
}
}

Categories

Resources