Develop Reference

Java Hadoop wierd join behaviour

Aim
I have two csv files trying to make a join between them. One containing movieId, title and the other containing userId, movieId, comment-tag. I want to find out how many comments-tags each movie has, by printing title, comment_count. So my code:
Driver
public class Driver
{
public Driver(String[] args)
{
if (args.length < 3) {
System.err.println("input path ");
}
try {
Job job = Job.getInstance();
job.setJobName("movie tag count");
// set file input/output path
MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, TagMapper.class);
MultipleInputs.addInputPath(job, new Path(args[2]), TextInputFormat.class, MovieMapper.class);
FileOutputFormat.setOutputPath(job, new Path(args[3]));
// set jar class name
job.setJarByClass(Driver.class);
// set mapper and reducer to job
job.setReducerClass(Reducer.class);
// set output key class
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
int returnValue = job.waitForCompletion(true) ? 0 : 1;
System.out.println(job.isSuccessful());
System.exit(returnValue);
} catch (IOException | ClassNotFoundException | InterruptedException e) {
e.printStackTrace();
}
}
}
MovieMapper
public class MovieMapper extends org.apache.hadoop.mapreduce.Mapper<Object, Text, Text, Text>
{
#Override
protected void map(Object key, Text value, Context context) throws IOException, InterruptedException
{
String line = value.toString();
String[] items = line.split("(?!\\B\"[^\"]*),(?![^\"]*\"\\B)"); //comma not in quotes
String movieId = items[0].trim();
if(tryParseInt(movieId))
{
context.write(new Text(movieId), new Text(items[1].trim()));
}
}
private boolean tryParseInt(String s)
{
try {
Integer.parseInt(s);
return true;
} catch (NumberFormatException e) {
return false;
}
}
}
TagMapper
public class TagMapper extends org.apache.hadoop.mapreduce.Mapper<Object, Text, Text, Text>
{
#Override
protected void map(Object key, Text value, Context context) throws IOException, InterruptedException
{
String line = value.toString();
String[] items = line.split("(?!\\B\"[^\"]*),(?![^\"]*\"\\B)");
String movieId = items[1].trim();
if(tryParseInt(movieId))
{
context.write(new Text(movieId), new Text("_"));
}
}
private boolean tryParseInt(String s)
{
try {
Integer.parseInt(s);
return true;
} catch (NumberFormatException e) {
return false;
}
}
}
Reducer
public class Reducer extends org.apache.hadoop.mapreduce.Reducer<Text, Text, Text, IntWritable>
{
#Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException
{
int noOfFrequency = 0;
Text movieTitle = new Text();
for (Text o : values)
{
if(o.toString().trim().equals("_"))
{
noOfFrequency++;
}
else
{
System.out.println(o.toString());
movieTitle = o;
}
}
context.write(movieTitle, new IntWritable(noOfFrequency));
}
}
The problem
The result I get is something like this:
title, count
_, count
title, count
title, count
_, count
title, count
_, count
How does this _ gets to be the key? I can't understand it. There is an if statment checking if there is an _ count it and don't put it as the title. Is there something wrong with the toString() method and the equals operation fails? Any ideas?

it is not weird because you iterate through values and o is a pointer to elements of values which is here are Text. at some point in time you make movieTitle to points to where o points movieTitle = o. in next iterations o points to "_" and also movieTitle points to "_".
if you change your code like this every thing works fine:
int noOfFrequency = 0;
Text movieTitle = null;
for (Text o : values)
{
if(o.toString().trim().equals("_"))
{
noOfFrequency++;
}
else
{
movieTitle = new Text(o.toString());
}
}
context.write(movieTitle, new IntWritable(noOfFrequency));

Hadoop MapReduce reducer does not start

The map phase runs and then just quits without bothering with the reducer. The job alternately prints "Hello from mapper." and "Writing CellWithTotalAmount" and that's it. The output directory it creates is empty.
I've checked at least a dozen of other "reducer won't start" questions and have not found an answer. I've checked that the output of map is the same as input into reduce, that reduce uses Iterable, that correct output classes have been set, etc.
Job config
public class HoursJob {
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: HoursJob <input path> <output path>");
System.exit(-1);
}
Job job = Job.getInstance();
job.setJarByClass(HoursJob.class);
job.setJobName("Hours job");
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(HoursMapper.class);
job.setReducerClass(HoursReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(CellWithTotalAmount.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
int ret = job.waitForCompletion(true) ? 0 : 1;
System.exit(ret);
}
}
Mapper
public class HoursMapper
extends Mapper<LongWritable, Text, IntWritable, CellWithTotalAmount> {
static double BEGIN_LONG = -74.913585;
static double BEGIN_LAT = 41.474937;
static double GRID_LENGTH = 0.011972;
static double GRID_HEIGHT = 0.008983112;
#Override
public void map(LongWritable key, Text value, Mapper.Context context)
throws IOException, InterruptedException {
System.out.println("Hello from mapper.");
String recordString = value.toString();
try {
DEBSFullRecord record = new DEBSFullRecord(recordString);
Date pickupDate = record.getPickup();
Calendar calendar = GregorianCalendar.getInstance();
calendar.setTime(pickupDate);
int pickupHour = calendar.get(Calendar.HOUR_OF_DAY);
int cellX = (int)
((record.getPickupLongitude() - BEGIN_LONG) / GRID_LENGTH) + 1;
int cellY = (int)
((BEGIN_LAT - record.getPickupLatitude()) / GRID_HEIGHT) + 1;
CellWithTotalAmount hourInfo =
new CellWithTotalAmount(cellX, cellY, record.getTotal());
context.write(new IntWritable(pickupHour), hourInfo);
} catch (Exception ex) {
System.out.println(
"Cannot parse: " + recordString + "due to the " + ex);
}
}
}
Reducer
public class HoursReducer
extends Reducer<IntWritable, CellWithTotalAmount, Text, NullWritable> {
#Override
public void reduce(IntWritable key, Iterable<CellWithTotalAmount> values,
Context context) throws IOException, InterruptedException {
System.out.println("Hello from reducer.");
int[][] cellRideCounters = getCellRideCounters(values);
CellWithRideCount cellWithMostRides =
getCellWithMostRides(cellRideCounters);
int[][] cellTotals = getCellTotals(values);
CellWithTotalAmount cellWithGreatestTotal =
getCellWithGreatestTotal(cellTotals);
String output = key + " "
+ cellWithMostRides.toString() + " "
+ cellWithGreatestTotal.toString();
context.write(new Text(output), NullWritable.get());
}
//omitted for brevity
}
Custom writable class
public class CellWithTotalAmount implements Writable {
public int cellX;
public int cellY;
public double totalAmount;
public CellWithTotalAmount(int cellX, int cellY, double totalAmount) {
this.cellX = cellX;
this.cellY = cellY;
this.totalAmount = totalAmount;
}
#Override
public void write(DataOutput out) throws IOException {
System.out.println("Writing CellWithTotalAmount");
out.writeInt(cellX);
out.writeInt(cellY);
out.writeDouble(totalAmount);
}
#Override
public void readFields(DataInput in) throws IOException {
System.out.println("Reading CellWithTotalAmount");
cellX = in.readInt();
cellY = in.readInt();
totalAmount = in.readDouble();
}
#Override
public String toString() {
return cellX + " " + cellY + " " + totalAmount;
}
}

I think there is a lot of exception in reduce function so Framework can not complete the job properly
public class HoursReducer
extends Reducer<IntWritable, CellWithTotalAmount, Text, NullWritable> {
#Override
public void reduce(IntWritable key, Iterable<CellWithTotalAmount> values,
Context context) throws IOException, InterruptedException {
System.out.println("Hello from reducer.");
try{
int[][] cellRideCounters = getCellRideCounters(values);
if(cellRideCounter[0].length>0){ // control it before executing it. more explanation is above
CellWithRideCount cellWithMostRides =
getCellWithMostRides(cellRideCounters);
int[][] cellTotals = getCellTotals(values);
CellWithTotalAmount cellWithGreatestTotal =
getCellWithGreatestTotal(cellTotals);
String output = key + " "
+ cellWithMostRides.toString() + " "
+ cellWithGreatestTotal.toString();
context.write(new Text(output), NullWritable.get());
}
}catch(Exception e)
e.printstack();
return;
{
}
}
add try-catch to get exceptions in reduce function
. Return from function in catch
. Also add an if statement before calling getCellWithMostRiders(..) I think the issue is in here. Fill the if statement as you want I made a guess and fill it according to my guess change it however you want if it is not proper for you

Java - Collection

I'm working on my first java project and I have a question. The question should be quiet simple (though the the code is not that short, but there's no reason to be intimidated :) ). I create a basic roleplaying-game and I have an abstract class "Character" that defines each character. Among its subclasses you can find Mage, who has a Spellbook (Map). Spellbook class offers methods like addToSpellbook, that works fine. In addition I have an Inventory class that has addToInventory method, which is quit identical to addToSpellbook.
My question is as follows - why can I use In the main method addToSpellbook and can't use AddToInventory?
I guess the reason is that Map doesn't have AddToInventory, so I should override put, but still, how can I use addToSpellbook ?
public class Game {
public static void main(String[] args) throws IOException {
CharacterCreator heroCreator = new CharacterCreator();
CharacterCreator.showAllClasses();
Scanner sc = new Scanner(System.in);
int scan = sc.nextInt();
String chosenClass = CharacterCreator.getCharacterClass(scan);
Character hero = CharacterCreator.createCharacter(chosenClass);
try {
hero.displayCharacter();
}catch (Exception e){
System.out.println("Problem displaying character data");
}
hero.getInventory().addToInventory("Long sword");
CharacterCreator heroCreator2 = new CharacterCreator();
CharacterCreator.showAllClasses();
Scanner sc2 = new Scanner(System.in);
int scan2 = sc.nextInt();
String chosenClass2 = CharacterCreator.getCharacterClass(scan2);
Character hero2 = CharacterCreator.createCharacter(chosenClass2);
try {
hero2.displayCharacter();
}catch (Exception e){
System.out.println("Wrong input");
}
if(hero instanceof Mage) {
((Mage)hero).getSpellBook().addToSpellBook("Magic Missiles");
((Mage)hero).getSpellBook().addToSpellBook("Fireball");
((Mage)hero).getSpellBook().addToSpellBook("Mage Armor");
((Mage)hero).getSpellBook().showSpellBook();
((Mage)hero).getSpellBook().getSpellFromSpellbook("Fireball").castSpell(hero, hero2);
((Mage)hero).getSpellBook().getSpellFromSpellbook("Magic Missiles").castSpell(hero, hero2);
((Mage)hero).getSpellBook().getSpellFromSpellbook("Mage Armor").castSpell(hero, hero);
}
}
}
abstract public class Character {
private Equipment equipment;
private Map<String, Integer> inventory;
protected Character(String name){
equipment = new Equipment();
inventory = new HashMap<String, Integer>();
}
protected Character(String name, int lvl){
equipment = new Equipment();
inventory = new HashMap<String, Integer>();
}
}
public Equipment getEquipment() { return equipment; }
public Map getInventory() { return inventory; }
}
public class Inventory {
private Map<String,Integer> inventory;
Inventory() {
inventory = new HashMap<String, Integer>();
}
public void addToInventory(String item) {
boolean found = false;
try {
for (Iterator<Map.Entry<String, Integer>> iter = inventory.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<String, Integer> newItem = iter.next();
if (newItem.getKey() == item) {
inventory.put(item, inventory.get(newItem) + 1);
break;
}
}
}catch (Exception e) {
System.out.println(item + " : adding failed");
}
if (!found) {
inventory.put(item,1);
}
}
public void showInventory() {
System.out.println("Show Inventory: ");
for (Map.Entry<String,Integer> entry: inventory.entrySet()) {
System.out.println( entry.getKey() + ", quantity: " + entry.getValue() );
}
System.out.println("");
}
}
public class Mage extends Character {
private SpellBook spellBook;
public Mage(String name) {
super(name);
SpellBook spellbook = new SpellBook();
}
protected Mage(String name, int lvl){
super(name, lvl);
spellBook = new SpellBook();
}
public SpellBook getSpellBook() { return spellBook; }
}
}
public class SpellBook {
private Map<String, Spell> spellBook;
SpellBook() {
spellBook = new HashMap<String, Spell>();
}
public Map getSpellBook() { return spellBook; }
public void addToSpellBook(String spellName) {
Spell newSpell = null;
try {
if (DamageSpell.getSpell(spellName) != null) {
newSpell = DamageSpell.getSpell(spellName);
} else if (ChangeStatSpell.getSpell(spellName) != null) {
newSpell = ChangeStatSpell.getSpell(spellName);
}
System.out.println(newSpell.getSpellName() + " has been added to the spellbook");
spellBook.put(newSpell.getSpellName(), newSpell);
} catch (Exception e){
System.out.println("Adding " + spellName +"to spellbook has failed");
}
}
public void showSpellBook() {
System.out.println("Show spellbook: ");
for (Iterator<String> iter = spellBook.keySet().iterator(); iter.hasNext(); ) {
String spell = iter.next();
System.out.println(spell);
}
System.out.println("");
}
public Spell getSpellFromSpellbook(String spellName) {
Spell spl = null;
//Spell splGet = spellBook.get(spellName); /* straight forward implementation*/
// System.out.println("The spell " + splGet.getSpellName() + " has been retrived from the spellbook by using get method");
try {
for (Iterator<Map.Entry<String, Spell>> iter = spellBook.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<String, Spell> spell = iter.next();
if (spell.getKey() == spellName) {
spl = spell.getValue();
}
}
}catch (Exception e) {
System.out.println(spellName + " : no such spell in spellbook");
}
return spl;
}
}

getInventory() returns a Map and Map doesn't have the addToInventory() method.
getInventory() should add an Inventory instance.

Reduce method in Reducer class is not executing

In the below code,inside reducer class reduce method is not executing. please help me.In my reduce method i want to write output in multiple files. so i have used multipleoutputs.
public class DataValidation {
public static class Map extends Mapper<LongWritable, Text, Text, Text> {
int flag = 1;
boolean result;
private HashMap<String, FileConfig> fileConfigMaps = new HashMap<String, FileConfig>();
private HashMap<String, List<LineValidator>> mapOfValidators = new HashMap<String, List<LineValidator>>();
private HashMap<String, List<Processor>> mapOfProcessors = new HashMap<String, List<Processor>>();
protected void setup(Context context) throws IOException {
System.out.println("configure inside map class");
ConfigurationParser parser = new ConfigurationParser();
Config config = parser.parse(new Configuration());
List<FileConfig> file = config.getFiles();
for (FileConfig f : file) {
try {
fileConfigMaps.put(f.getName(), f);
System.out.println("quotes in" + f.isQuotes());
System.out.println("file from xml : " + f.getName());
ValidationBuilder builder = new ValidationBuilder();
// ProcessorBuilder constructor = new ProcessorBuilder();
List<LineValidator> validators;
validators = builder.build(f);
// List<Processor> processors = constructor.build(f);
mapOfValidators.put(f.getName(), validators);
// mapOfProcessors.put(f.getName(),processors);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// String filename = ((FileSplit) context.getInputSplit()).getPath()
// .getName();
FileSplit fs = (FileSplit) context.getInputSplit();
String fileName = fs.getPath().getName();
System.out.println("filename : " + fileName);
String line = value.toString();
String[] csvDataArray = null;
List<LineValidator> lvs = mapOfValidators.get(fileName);
flag = 1;
csvDataArray = line.split(",", -1);
FileConfig fc = fileConfigMaps.get(fileName);
System.out.println("filename inside fileconfig " + fc.getName());
System.out.println("quote values" + fc.isQuotes());
if (fc.isQuotes()) {
for (int i = 0; i < csvDataArray.length; i++) {
csvDataArray[i] = csvDataArray[i].replaceAll("\"", "");
}
}
for (LineValidator lv : lvs) {
if (flag == 1) {
result = lv.validate(csvDataArray, fileName);
if (result == false) {
String write = line + "," + lv.getFailureDesc();
System.out.println("write" + write);
System.out.println("key" + new Text(fileName));
// output.collect(new Text(filename), new Text(write));
context.write(new Text(fileName), new Text(write));
flag = 0;
if (lv.stopValidation(csvDataArray) == true) {
break;
}
}
}
}
}
protected void cleanup(Context context) {
System.out.println("clean up in mapper");
}
}
public static class Reduce extends Reducer<Text, Text, NullWritable, Text> {
protected void reduce(Text key, Iterator<Text> values, Context context)
throws IOException, InterruptedException {
System.out.println("inside reduce method");
while (values.hasNext()) {
System.out.println(" Nullwritable value" + NullWritable.get());
System.out.println("key inside reduce method" + key.toString());
context.write(NullWritable.get(), values.next());
// out.write(NullWritable.get(), values.next(), "/user/hadoop/"
// + context.getJobID() + "/" + key.toString() + "/part-");
}
}
}
public static void main(String[] args) throws Exception {
System.out.println("hello");
Configuration configuration = getConf();
Job job = Job.getInstance(configuration);
job.setJarByClass(DataValidation.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
private static Configuration getConf() {
return new Configuration();
}
}

You have not properly over-ridden reduce method. Use this:
public void reduce(Key key, Iterable values,
Context context) throws IOException, InterruptedException

Getting 0 output records in hadoop MapReduce

I have the following code in hadoop where the mapper and reducer are as follows:
public static class Map2 extends Mapper<LongWritable, Text, NullWritable, Text>
{
TreeMap<Text, Text> top10 = new TreeMap<Text, Text>();
HashMap<String, String> userInfo = new HashMap<String, String>();
public void setup(Context context) throws IOException, InterruptedException
{
try
{
URI[] uris = DistributedCache.getCacheFiles(context.getConfiguration());
FileSystem fs = FileSystem.get(context.getConfiguration());
if (uris == null || uris.length == 0)
{
throw new IOException("Error reading file from distributed cache. No URIs found.");
}
String path = "./users.dat";
fs.copyToLocalFile(new Path(uris[0]), new Path(path));
BufferedReader br = new BufferedReader(new FileReader(path));
String line = null;
while((line = br.readLine()) != null)
{
String split[] = line.split("\\::");
String age = split[2];
String gender = split[1];
userInfo.put(split[0], gender + "\t" + age);
}
br.close();
}
catch(Exception e)
{
}
}
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException
{
try
{
String line = value.toString();
int sum = Integer.parseInt(line.split("\\t")[1]);
String userID = line.split("\\t")[0];
String newKey = sum + " " + userID;
if(userInfo.containsKey(userID))
{
String record = userInfo.get(userID);
String val = userID + "\t" + record + "\t" + sum;
top10.put(new Text(newKey), new Text(val));
if (top10.size() > 10)
{
top10.remove(top10.firstKey());
}
}
}
catch(Exception e)
{
}
}
protected void cleanup(Context context) throws IOException, InterruptedException
{
try
{
for (Text s1 : top10.descendingMap().values())
{
context.write(NullWritable.get(), s1);
}
}
catch(Exception e)
{
}
}
}
public static class Reduce2 extends Reducer<NullWritable, Text, NullWritable, Text>
{
private TreeMap<Text, Text> top10 = new TreeMap<Text, Text>();
public void reduce(NullWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException
{
try
{
String line = values.toString();
String sum = line.split("\\t")[3];
String userID = line.split("\\t")[0];
String gender = line.split("\\t")[1];
String age = line.split("\\t")[2];
String newKey = sum + " " + userID;
String val = userID + "\t" + gender + "\t" + age + "\t" + sum;
top10.put(new Text(newKey), new Text(val));
if(top10.size() > 10)
{
top10.remove(top10.firstKey());
}
}
catch(Exception e)
{
}
}
protected void cleanup(Context context) throws IOException, InterruptedException
{
try
{
for (Text s1 : top10.descendingMap().values())
{
context.write(NullWritable.get(), s1);
}
}
catch(Exception e)
{
}
}
}
The driver method is as follows:
Configuration conf2 = new Configuration();
DistributedCache.addCacheFile(new Path("/Spring2014_HW-1/input_HW-1/users.dat").toUri(), conf2);
Job job2 = new Job(conf2, "Phase2");
job2.setOutputKeyClass(NullWritable.class);
job2.setOutputValueClass(Text.class);
job2.setJarByClass(MapSideJoin.class);
job2.setMapperClass(Map2.class);
job2.setReducerClass(Reduce2.class);
job2.setInputFormatClass(TextInputFormat.class);
job2.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job2, new Path(args[1]));
FileOutputFormat.setOutputPath(job2, new Path(args[2]));
//job2.setNumReduceTasks(1);
job2.waitForCompletion(true);
I get the message as map output records = 10 and reduce output records = 0, even though I've emit output from the reducer? Where does this output from the reducer disappear?
Thanks.