├── README.md ├── Hadoop - Assignment.pdf ├── LICENSE ├── 1_WordFrequency.java ├── 2_AverageMarks.java └── 3_4_5_PartitionerCombiner.java /README.md: -------------------------------------------------------------------------------- 1 | # hadoop-programs 2 | Hadoop Programs for Hadoop/CUDA Lab at MANIT, Bhopal 3 | -------------------------------------------------------------------------------- /Hadoop - Assignment.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jishanshaikh4/hadoop-programs/HEAD/Hadoop - Assignment.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Jishan Shaikh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /1_WordFrequency.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import org.apache.hadoop.conf.Configuration; 3 | import org.apache.hadoop.fs.Path; 4 | import org.apache.hadoop.io.IntWritable; 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.Text; 7 | import org.apache.hadoop.mapreduce.Job; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 11 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 12 | import org.apache.hadoop.util.GenericOptionsParser; 13 | public class WordFrequency { 14 | public static void main(String [] args) throws Exception 15 | { 16 | 17 | Configuration conf=new Configuration(); 18 | Job job=new Job(conf,"wordcount"); 19 | job.setJarByClass(WordFrequency.class); 20 | job.setMapperClass(MapClass.class); 21 | job.setReducerClass(ReduceClass.class); 22 | job.setOutputKeyClass(Text.class); 23 | job.setOutputValueClass(IntWritable.class); 24 | FileInputFormat.addInputPath(job, new Path(args[0])); 25 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 26 | System.exit(job.waitForCompletion(true)?0:1); 27 | 28 | } 29 | 30 | public static class MapClass extends Mapper{ 31 | 32 | public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 33 | { 34 | String line = value.toString(); 35 | String[] words=line.split(" "); 36 | for(String word: words) 37 | { 38 | context.write(new Text(word.trim()), new IntWritable(1)); 39 | } 40 | } 41 | } 42 | 43 | public static class ReduceClass extends Reducer 44 | { 45 | public void reduce(Text word, Iterable values, Context context) throws IOException, InterruptedException 46 | { 47 | int sum = 0; 48 | for(IntWritable value : values) 49 | { 50 | sum += value.get(); 51 | } 52 | 53 | context.write(word, new IntWritable(sum)); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /2_AverageMarks.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import org.apache.hadoop.conf.Configuration; 3 | import org.apache.hadoop.fs.Path; 4 | import org.apache.hadoop.io.IntWritable; 5 | import org.apache.hadoop.io.FloatWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 13 | 14 | public class AverageMarks { 15 | public static void main(String [] args) throws Exception 16 | 17 | { 18 | 19 | Configuration conf=new Configuration(); 20 | Job job=new Job(conf,"Average marks "); 21 | job.setJarByClass(AverageMarks.class); 22 | job.setMapperClass(MapClass.class); 23 | job.setReducerClass(ReduceClass.class); 24 | job.setOutputKeyClass(Text.class); 25 | job.setMapOutputValueClass(IntWritable.class); 26 | job.setOutputValueClass(FloatWritable.class); 27 | FileInputFormat.addInputPath(job, new Path(args[0])); 28 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 29 | System.exit(job.waitForCompletion(true)?0:1); 30 | } 31 | 32 | public static class MapClass extends Mapper{ 33 | 34 | public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 35 | { 36 | String line = value.toString(); 37 | String[] Words=line.split(","); 38 | if(Words.length>1) 39 | { 40 | context.write(new Text(Words[0]), new IntWritable(Integer.parseInt(Words[1]))); 41 | } 42 | 43 | } 44 | } 45 | 46 | public static class ReduceClass extends Reducer 47 | { 48 | public void reduce(Text word, Iterable values, Context context) throws IOException, InterruptedException 49 | { 50 | int total= 0; 51 | int count=0; 52 | for(IntWritable value : values) 53 | { 54 | total += value.get(); 55 | count+=1; 56 | } 57 | 58 | context.write(word, new FloatWritable(total/count)); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /3_4_5_PartitionerCombiner.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import org.apache.hadoop.conf.Configuration; 3 | import org.apache.hadoop.fs.Path; 4 | import org.apache.hadoop.io.IntWritable; 5 | import org.apache.hadoop.io.FloatWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Job; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | import org.apache.hadoop.mapreduce.Partitioner; 12 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 13 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 14 | 15 | public class Driver{ 16 | public static void main(String [] args) throws Exception{ 17 | 18 | Configuration conf =new Configuration(); 19 | Job job=new Job(conf,"Implement Partitioner"); 20 | job.setJarByClass(Driver.class); 21 | job.setMapperClass(MapClass.class); 22 | job.setPartitionerClass(PartitionClass.class); 23 | job.setReducerClass(ReduceClass.class); 24 | job.setNumReduceTasks(6); 25 | job.setOutputKeyClass(Text.class); 26 | job.setOutputValueClass(Text.class); 27 | FileInputFormat.addInputPath(job, new Path(args[0])); 28 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 29 | System.exit(job.waitForCompletion(true)?0:1); 30 | } 31 | 32 | public static class MapClass extends Mapper{ 33 | 34 | public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 35 | { 36 | String line = value.toString(); 37 | String[] Words=line.split(","); 38 | if(Words.length>1) 39 | { 40 | context.write(new Text(Words[0]), new Text(Words[1])); 41 | } 42 | } 43 | } 44 | 45 | public static class PartitionClass extends Partitioner{ 46 | public int getPartition(Text key,Text value,int numReduceTask) 47 | { 48 | if(numReduceTask==0) 49 | return 0; 50 | if(key.equals(new Text("Cse1"))) 51 | return 0; 52 | else if(key.equals(new Text("Mech1"))) 53 | return 1%numReduceTask; 54 | else if(key.equals(new Text("Cse2"))) 55 | return 2%numReduceTask; 56 | else if(key.equals(new Text("Mech2"))) 57 | return 3%numReduceTask; 58 | else if (key.equals(new Text("Civil"))) 59 | return 4%numReduceTask; 60 | else 61 | return 5%numReduceTask; 62 | } 63 | } 64 | 65 | public static class ReduceClass extends Reducer{ 66 | public void reduce(Text key,Iterable Values,Context context)throws IOException, InterruptedException{ 67 | int sum=0; 68 | for(Text val:Values) 69 | { 70 | context.write(key,val); 71 | } 72 | } 73 | } 74 | } 75 | --------------------------------------------------------------------------------