hadoop Java API 比较python 下的hadoo
2019-04-17 本文已影响0人
sadamu0912
java api 来运行mapreduce程序
1 首先需要搭建一个hadoop集群。
2 配置环境变量
export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath):$CLASSPATH
[root@master workspace]# $HADOOP_HOME/bin/hadoop classpath
/root/software/hadoop/hadoop-2.6.1/etc/hadoop:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/common/lib/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/common/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/hdfs:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/hdfs/lib/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/hdfs/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/yarn/lib/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/yarn/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/mapreduce/lib/*:
/root/software/hadoop/hadoop-2.6.1/share/hadoop/mapreduce/*:
/root/software/hadoop/hadoop-2.6.1/contrib/capacity-scheduler/*.jar
3 代码
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
4 编译生成jar包 jar cf
$ javac WordCount.java
$ jar cf wc.jar WordCount*.class
5 在hdfs上建立相应的目录。然后上传数据到hdfs上。
hdfs dfs -put xxx /input/wordcount/
6 用hadoop jar 执行mapreduce程序 （注意在执行之前只有/output目录，并没有/output/wordcount目录）
hadoop jar xx.jar WordCount /input/wordcount /output/wordcount
7 查看结果
[root@master workspace]# hdfs dfs -text /output/wordcount/part-r-00000 | head -n 20
19/04/16 23:03:32 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
(Baynes 1
(Dartie 1
(Dartie’s 1
(Down-by-the-starn) 2
(Down-by-the-starn), 1
(He 1
(I 1
(James) 1
(L500) 1
(Louisa 1
(Mrs. 1
(Roger 1
(Roger’s 1
(Soames 1
(Soames) 1
用python 和hadoop streaming 来运行mapreduce程序
run.sh 代码：
#!/bin/bash
HADOOP_CMD="/root/software/hadoop/hadoop-2.6.1/bin/hadoop"
# 在shell当中获取当前目录 $(pwd)
STREAM_JAR_PATH=$(pwd)/hadoop-streaming-2.6.1.jar
INPUT_FILE_PATH="/input/wordcount/article.txt"
OUTPUT_PATH="/output/wordcount"
$HADOOP_CMD fs -rmr -skipTrash $OUTPUT_PATH
# Step 1.
$HADOOP_CMD jar $STREAM_JAR_PATH \
-input $INPUT_FILE_PATH \
-output $OUTPUT_PATH \
-mapper "python map.py" \
-reducer "python reduce.py" \
# 指定要分发到计算节点的文件。因为hadoop 是datalocality 所以需要分发计算任务到数据节点。
-file ./map.py \
-file ./reduce.py
map.py 代码：
#!/usr/local/bin/python
import sys
import time
for line in sys.stdin:
ss = line.strip().split(' ')
for s in ss:
#time.sleep(100000)
if s.strip() != "":
print "%s\t%s" % (s, 1)
reduce.py 代码：
import sys
import re
cur_word = None
sum = 0
for line in sys.stdin:
ss = line.strip().split('\t')
if len(ss) != 2:
continue
word, cnt = ss
# 正则匹配特殊的字符，去除数字，？。--——等特殊字符
if(re.search(r'\.|\?|:|-|_|__|"|\d',word)):
continue
if cur_word == None:
cur_word = word
if cur_word != word:
print '\t'.join([cur_word, str(sum)])
cur_word = word
sum = 0
sum += int(cnt)
print '\t'.join([cur_word, str(sum)])
我们先本地调试一波：
[root@master python]# cat data/The_Man_of_Property.txt |python map.py | sort -k1 |python reduce.py| sort -t $'\t' -k2 -rn |head -n 20
the 5144
of 3407
to 2782
and 2573
a 2543
he 2139
his 1912
was 1702
in 1694
had 1526
that 1273
with 1029
her 1020
— 931
at 815
for 765
not 723
she 711
He 695
it 689
发现the 频率最高。然后放集群上跑。
直接sh run.sh
然后：
hdfs dfs -text /output/wordcount/part-00000 >result.data
然后cat result.data| sort -t $'\t' -k2 -rn |head -n 20
[root@master python]# cat result.data |sort -t $'\t' -k2 -rn | head -n 20
the 5144
of 3407
to 2782
and 2573
a 2543
he 2139
his 1912
was 1702
in 1694
had 1526
that 1273
with 1029
her 1020
— 931
at 815
for 765
not 723
she 711
He 695
it 689
结果是一样的.
