利用MapReduce从hdfs写入hbase

a-c-dream 发布于

源数据

路径

1
/student/input/student.txt

内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
95002,刘晨,女,19,IS
95017,王风娟,女,18,IS
95018,王一,女,19,IS
95013,冯伟,男,21,CS
95014,王小丽,女,19,CS
95019,邢小丽,女,19,IS
95020,赵钱,男,21,IS
95003,王敏,女,22,MA
95004,张立,男,19,IS
95012,孙花,女,20,CS
95010,孔小涛,男,19,CS
95005,刘刚,男,18,MA
95006,孙庆,男,23,CS
95007,易思玲,女,19,MA
95008,李娜,女,18,CS
95021,周二,男,17,MA
95022,郑明,男,20,MA
95001,李勇,男,20,CS
95011,包小柏,男,18,MA
95009,梦圆圆,女,18,MA
95015,王君,男,18,MA

java程序：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
package hbase_hdfs_;

import java.io.IOException;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class HdfsToHbase extends Configured implements Tool{
	private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum";
	private static final String ZK_CONNECT_VALUE = "hadoop01:2181,hadoop02:2181,hadoop03:2181";
	public static void main(String[] args) throws Exception
	{
		int isDone =ToolRunner.run(new HdfsToHbase(), args);
		System.exit(isDone);
	}
	@Override
	public int run(String[] arg0) throws Exception {

		Configuration conf = new Configuration();
		conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE);
		conf.set("fs.defaultFS", "hdfs://hadoop01:9000");
		System.setProperty("HADOOP_USER_NAME", "root");
		
		Job job = Job.getInstance(conf,"hdfstohbase");
		job.setJarByClass(HdfsToHbase.class);
		job.setMapperClass(H2H_Mapper.class);
		job.setMapOutputValueClass(NullWritable.class);
		job.setMapOutputKeyClass(Text.class);
		job.setInputFormatClass(TextInputFormat.class);
		TableMapReduceUtil.initTableReducerJob("users", H2H_Reducer.class, job, null,null, null,null,false);
		job.setOutputKeyClass(NullWritable.class);
		job.setMapOutputValueClass(Put.class);
		FileInputFormat.addInputPath(job, new Path("/student/input"));
		boolean isDone = job.waitForCompletion(true);
		
		return isDone?0:1;
	}
	public static class H2H_Mapper extends Mapper<LongWritable,Text,Text,NullWritable>
	{
		@Override
		protected void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException
		{
			context.write(value, NullWritable.get());
		}
	}
	public static class H2H_Reducer extends TableReducer<Text, NullWritable, NullWritable>
	{
		@Override
		protected void reduce (Text key,Iterable<NullWritable> values,Context context) throws IOException, InterruptedException
		{
			String [] lines = key.toString().split(",");
			Put put = new Put(lines[0].getBytes());
			put.addColumn("info".getBytes(), "name".getBytes(), lines[1].getBytes());
			put.addColumn("info".getBytes(), "gender".getBytes(), lines[2].getBytes());
			put.addColumn("info".getBytes(), "department".getBytes(), lines[3].getBytes());
			context.write(NullWritable.get(), put);
		}
	}
}

scan结果：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
hbase:002:0> scan 'users'
ROW                         COLUMN+CELL                                                                    
 95001                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=20                   
 95001                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95001                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95001                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE6\x9D\x8E\xE5\x8B
                            \x87                                                                           
 95002                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95002                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95002                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95002                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\x88\x98\xE6\x99
                            \xA8                                                                           
 95003                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=22                   
 95003                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95003                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95003                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE7\x8E\x8B\xE6\x95
                            \x8F                                                                           
 95004                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95004                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95004                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95004                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\xBC\xA0\xE7\xAB
                            \x8B                                                                           
 95005                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95005                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95005                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95005                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\x88\x98\xE5\x88
                            \x9A                                                                           
 95006                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=23                   
 95006                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95006                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95006                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\xAD\x99\xE5\xBA
                            \x86                                                                           
 95007                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95007                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95007                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95007                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE6\x98\x93\xE6\x80
                            \x9D\xE7\x8E\xB2                                                               
 95008                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95008                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95008                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95008                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE6\x9D\x8E\xE5\xA8
                            \x9C                                                                           
 95009                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95009                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95009                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95009                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE6\xA2\xA6\xE5\x9C
                            \x86\xE5\x9C\x86                                                               
 95010                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95010                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95010                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95010                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\xAD\x94\xE5\xB0
                            \x8F\xE6\xB6\x9B                                                               
 95011                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95011                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95011                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95011                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\x8C\x85\xE5\xB0
                            \x8F\xE6\x9F\x8F                                                               
 95012                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=20                   
 95012                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95012                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95012                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\xAD\x99\xE8\x8A
                            \xB1                                                                           
 95013                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=21                   
 95013                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95013                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95013                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\x86\xAF\xE4\xBC
                            \x9F                                                                           
 95014                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95014                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=CS            
 95014                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95014                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE7\x8E\x8B\xE5\xB0
                            \x8F\xE4\xB8\xBD                                                               
 95015                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95015                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95015                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95015                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE7\x8E\x8B\xE5\x90
                            \x9B                                                                           
 95017                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=18                   
 95017                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95017                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95017                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE7\x8E\x8B\xE9\xA3
                            \x8E\xE5\xA8\x9F                                                               
 95018                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95018                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95018                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95018                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE7\x8E\x8B\xE4\xB8
                            \x80                                                                           
 95019                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=19                   
 95019                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95019                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE5\xA5\xB3      
 95019                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE9\x82\xA2\xE5\xB0
                            \x8F\xE4\xB8\xBD                                                               
 95020                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=21                   
 95020                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=IS            
 95020                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95020                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE8\xB5\xB5\xE9\x92
                            \xB1                                                                           
 95021                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=17                   
 95021                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95021                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95021                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE5\x91\xA8\xE4\xBA
                            \x8C                                                                           
 95022                      column=info:age, timestamp=2021-10-19T18:17:14.637, value=20                   
 95022                      column=info:department, timestamp=2021-10-19T18:17:14.637, value=MA            
 95022                      column=info:gender, timestamp=2021-10-19T18:17:14.637, value=\xE7\x94\xB7      
 95022                      column=info:name, timestamp=2021-10-19T18:17:14.637, value=\xE9\x83\x91\xE6\x98
                            \x8E                                                                           
21 row(s)
Took 0.8146 seconds
a-c-dream

a-c-dream

贵在坚持

93
13
0
CATEGORYS
TAGS