Program to illustrate FOF Map Reduce:
import java.io.IOException;
import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class FriendCommon {
public static class FriendMapper extends Mapper<Object, Text, Text, Text> {
private Text pair = new Text();
private Text user = new Text();
public void map(Object key, Text value, Context context) throws IOException,
InterruptedException {
String[] line = value.toString().split(" -> ");
if (line.length == 2) {
String[] friends = line[1].split(" ");
for (int i = 0; i < friends.length; i++) {
for (int j = i + 1; j < friends.length; j++) {
String friendPair = friends[i].trim() + "," + friends[j].trim();
pair.set(friendPair);
user.set(line[0].trim());
context.write(pair, user);
}
}
}
}
}
public static class FriendReducer extends Reducer<Text, Text, Text, Text> {
private Text commonFriends = new Text();
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException,
InterruptedException {
Set<String> userList = new HashSet<>();
for (Text value : values) {
userList.add(value.toString());
}
if (userList.size() > 1) {List<String> sortedUsers = new ArrayList<>(userList);
Collections.sort(sortedUsers);
String friends = String.join(" ", sortedUsers);
int count = sortedUsers.size();
commonFriends.set(friends + "\t" + count);
context.write(key, commonFriends);
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "FriendCommon");
job.setJarByClass(FriendCommon.class);
job.setMapperClass(FriendMapper.class);
job.setReducerClass(FriendReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}



No comments:
Post a Comment