修改core-site.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<!-- 指定hdfs的nameservice为ns1 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns1</value>
</property>
<!-- 指定hadoop临时目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/root/export/servers/hadoop-2.7.4/tmp</value>
</property>
<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>node-01:2181,node-02:2181,node-03:2181</value>
</property>
</configuration>修改hdfs-site.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89<!-- 设置副本个数 -->
<porperty>
<name>dfs.replication</name>
<value>2</value>
</porperty>
<!-- 设置namenode.name目录 -->
<porperty>
<name>dfs.namenode.name.dir</name>
<value>file:/export/data/hadoop/name</value>
</porperty>
<!-- 设置namenode.data目录 -->
<porperty>
<name>dfs.datanode.data.dir</name>
<value>file:/export/data/hadoop/data</value>
</porperty>
<!-- 指定hdfs的nameservice为ns1,需要和core-site.xml中的保持一致 -->
<porperty>
<name>dfs.nameservices</name>
<value>ns1</value>
</porperty>
<!-- ns1下面有2个NameNode，分别是nn1,nn2 -->
<porperty>
<name>dfs.ha.namenodes.ns1</name>
<value>nn1,nn2</value>
</porperty>
<!-- nn1的RPC通信地址 -->
<porperty>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>node-01:9000</value>
</porperty>
<!-- nn1的http通信地址 -->
<porperty>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>node-01:50070</value>
</porperty>
<!-- nn2的RPC通信地址 -->
<porperty>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>node-02:9000</value>
</porperty>
<!-- nn2的http通信地址 -->
<porperty>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>node-02:50070</value>
</porperty>
<!-- 指定NameNode的元数据在journalnode上的存放位置 -->
<porperty>
<name>dfs.namenode.shared.edits.dir</name>
<value>
qjournal://node-01:8485;node-02:8485;node-03:8485/ns1
</value>
</porperty>
<!-- 指定journalnode在本地磁盘存放数据的位置 -->
<porperty>
<name>dfs.journalnode.edits.dir</name>
<value>/export/data/hadoop/journaldata</value>
</porperty>
<!-- 开启NameNode失败自动重启 -->
<porperty>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</porperty>
<!-- 配置失败自动切换的实现方式 -->
<porperty>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>
org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
</value>
</porperty>
<!-- 配置隔离机制方法，多个机制用换行分隔，即每个机制暂用一行 -->
<porperty>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</porperty>
<!-- 使用sshfence隔离机制时需要ssh免登陆 -->
<porperty>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</porperty>
<!-- 配置sshfence隔离机制超时时间 -->
<porperty>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</porperty>修改yarn-site.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46<porperty>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>2048</value>
</porperty>
<porperty>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>2048</value>
</porperty>
<porperty>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>1</value>
</porperty>
<!-- 开启RM高可用 -->
<porperty>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</porperty>
<!-- 指定RM的cluster id -->
<porperty>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</porperty>
<!-- 指定RM的名字 -->
<porperty>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</porperty>
<!-- 分别指定RM的地址 -->
<porperty>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>node-01</value>
</porperty>
<porperty>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>node-02</value>
</porperty>
<!-- 指定zk集群地址 -->
<porperty>
<name>yarn.resourcemanager.zk-address</name>
<value>node-01:2181,node-02:2181,node-03:2181</value>
</porperty>
<porperty>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</porperty>启动zookeeper
启动journalnode
格式化NameNode（node-01,node=02）
1
hadoop namenode –format在node-01节点上格式化ZKFC
1
hdfs zkfc -formatZK
a-c-dream
贵在坚持
93
13
0