Eddie昌的博客
Eddie昌
2024-04-25
目录

搭建一套源生hadoop、Spark、Flink集群2-Hadoop安装原创

# 搭建一套源生Hadoop、Spark、Flink集群

本文记录如何搭建原生集群。

# 一、安装Hadoop

用eddie用户登录

# 1.基本配置

# 解压
tar -zxvf /opt/softwares/hadoop-3.3.6.tar.gz -C /opt/apache/
# 配置HADOOP_HOME
su root 
vim /etc/profile.d/my_env.sh
---------------文件内容--------------
# HADOOP 配置
export HADOOP_VERSION=3.3.6
export HADOOP_HOME=/opt/apache/hadoop-3.3.6
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_COMMON_HOME=${HADOOP_HOME}
export HADOOP_HDFS_HOME=${HADOOP_HOME}
export HADOOP_MAPRED_HOME=${HADOOP_HOME}
export HADOOP_YARN_HOME=${HADOOP_HOME}
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
-----------------------------
xsync /etc/profile.d/my_env.sh
# 切回eddie
su eddie
xcall source /etc/profile.d/my_env.sh
# 分发软件包
xsync /opt/apache/hadoop-3.3.6/
# 修改配置文件
cd $HADOOP_CONF_DIR
vim $HADOOP_CONF_DIR/core-site.xml
------------------------------
 <configuration>
   <!-- 指定NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://bd181:8020</value>
    </property>

    <!-- 指定hadoop数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/apache_data/hdfs</value>
    </property>

    <!-- 配置HDFS网页登录使用的静态用户为eddie -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>eddie</value>
    </property>

         <!--配置root(超级用户)允许通过代理访问的主机节点-->
        <property>
                <name>hadoop.proxyuser.root.hosts</name>
                <value>*</value>
        </property>

        <!--配置root(超级用户)允许通过代理用户所属组-->
        <property>
                <name>hadoop.proxyuser.root.groups</name>
                <value>*</value>
        </property>

        <!--配置root(超级用户)允许通过代理的用户-->
        <property>
                <name>hadoop.proxyuser.root.user</name>
                <value>*</value>
        </property>
        <!--配置eddie(超级用户)允许通过代理访问的主机节点-->
        <property>
                <name>hadoop.proxyuser.eddie.hosts</name>
                <value>*</value>
        </property>

        <!--配置eddie(超级用户)允许通过代理用户所属组-->
        <property>
                <name>hadoop.proxyuser.eddie.groups</name>
                <value>*</value>
        </property>

        <!--配置eddie(超级用户)允许通过代理的用户-->
        <property>
                <name>hadoop.proxyuser.eddie.user</name>
                <value>*</value>
        </property>
</configuration>
------------------------------
vim $HADOOP_CONF_DIR/hdfs-site.xml
------------------------------
<configuration>
        <!-- nn web端访问地址-->
        <property>
                <name>dfs.namenode.http-address</name>
                <value>bd181:9870</value>
        </property>
        <!-- 2nn web端访问地址-->
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>bd182:9868</value>
        </property>

         <property>
                <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
                <value>false</value>
        </property>
</configuration>
------------------------------
vim $HADOOP_CONF_DIR/yarn-site.xml
------------------------------
<configuration>

<!-- Site specific YARN configuration properties -->
<!-- 指定MR走shuffle -->
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>

<!-- 指定ResourceManager的地址-->
<property>
    <name>yarn.resourcemanager.hostname</name>
    <value>bd182</value>
</property>

<!-- 环境变量的继承 -->
<property>
    <name>yarn.nodemanager.env-whitelist</name>
   <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
<!-- 开启日志聚集功能 -->
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>
<!-- 设置日志聚集服务器地址 -->
<property>
    <name>yarn.log.server.url</name>
    <value>http://bd182:19888/jobhistory/logs</value>
</property>
<!-- 设置日志保留时间为7天 -->
<property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
</property>

<!-- 设置Yarn可用内存 -->
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>16384</value>
</property>
<!-- 设置单个任务最大内存 -->
<property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>16384</value>
</property>
<!--是否启动一个线程检查每个任务正使用的物理内存量，如果任务超出分配值，则直接将其杀掉，默认是true -->
<property>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>false</value>
</property>
<!--是否启动一个线程检查每个任务正使用的虚拟内存量，如果任务超出分配值，则直接将其杀掉，默认是true -->
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
</configuration>

------------------------------
vim $HADOOP_CONF_DIR/mapred-site.xml
------------------------------
<configuration>

<!-- 指定MapReduce程序运行在Yarn上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
<!-- 历史服务器端地址 -->
<property>
    <name>mapreduce.jobhistory.address</name>
    <value>bd182:10020</value>
</property>

<!-- 历史服务器web端地址 -->
<property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>bd182:19888</value>
</property>

</configuration>
------------------------------
# 配置hadoop环境
vim $HADOOP_CONF_DIR/hadoop-env.sh
------------------------------
export JAVA_HOME=/opt/apache/jdk1.8.0_212
------------------------------
# 分发配置文件
xsync /opt/apache/hadoop-3.3.6/
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194

# 2.集群配置

vim $HADOOP_CONF_DIR/workers
-------------------------------
bd181
bd182
bd183
bd184
bd185
-------------------------------
xsync $HADOOP_CONF_DIR/workers

#初始化 namenode:bd181 snn bd182 rm跟jobhistory:bd182
# 在bd181运行
hdfs namenode -format #仅第一次执行
sh $HADOOP_HOME/sbin/start-dfs.sh
# 在bd182运行
# yarn
sh $HADOOP_HOME/sbin/start-yarn.sh
# jobHistory 历史服务器
mapred --daemon start historyserver
# 访问nameNode:http://bd181:9870/
# 访问resourceManager:http://bd182:8088
# 访问jobHistory:http://bd182:19888/jobhistory
# 测试
hadoop fs -mkdir /tmp
hadoop fs -put $HADOOP_CONF_DIR/core-site.xml /tmp
hadoop fs -cat /t,p/core-site.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# 3.集群启动命令

# 建立脚本
vim /bin/myhadoop.sh
-----------------------------------------------------------------------------
#!/bin/bash
if [ $# -lt 1 ]
then
echo "No Args Input..."
exit ;
fi
case $1 in
"start")
echo " =================== 启动 hadoop 集群 ==================="
echo " --------------- 启动 hdfs ---------------"
ssh bd181 "/opt/apache/hadoop-3.3.6/sbin/start-dfs.sh"
echo " --------------- 启动 yarn ---------------"

ssh bd182 "/opt/apache/hadoop-3.3.6/sbin/start-yarn.sh"
echo " --------------- 启动 historyserver ---------------"
ssh bd182 "/opt/apache/hadoop-3.3.6/bin/mapred --daemon start historyserver"
;;
"stop")
echo " =================== 关闭 hadoop 集群 ==================="
echo " --------------- 关闭 historyserver ---------------"
ssh bd182 "/opt/apache/hadoop-3.3.6/bin/mapred --daemon stop historyserver"
echo " --------------- 关闭 yarn ---------------"
ssh bd182 "/opt/apache/hadoop-3.3.6/sbin/stop-yarn.sh"
echo " --------------- 关闭 hdfs ---------------"
ssh bd181 "/opt/apache/hadoop-3.3.6/sbin/stop-dfs.sh"
;;
*)
echo "Input Args Error..."
;;
esac
-----------------------------------------------------------------------------
chmod +x /bin/myhadoop.sh
xsync /bin/myhadoop.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# 二、问题

# 问题一：ERROR: You must be a privileged user in order to run a secure service

ERROR: You must be a privileged user in order to run a secure service

官方文档：https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Secure_DataNode

# 先切换到root用户
vim $HADOOP_CONF_DIR/hadoop-env.sh
# 把下面变量替换成root
export HDFS_DATANODE_USER=root
export HADOOP_SECURE_DN_USER=root
export HDFS_DATANODE_SECURE_USER=root
export HDFS_NAMENODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
export YARN_PROXYSERVER_USER=root
# 修改环境变量
vim /etc/profile.d/my_env.sh
# 把下面变量替换成root
export HDFS_DATANODE_USER=root
export HADOOP_SECURE_DN_USER=root
export HDFS_DATANODE_SECURE_USER=root
export HDFS_NAMENODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
export YARN_PROXYSERVER_USER=root
# 文件同步
xsync /etc/profile.d/my_env.sh
xsync $HADOOP_CONF_DIR/hadoop-env.sh
xcall source /etc/profile.d/my_env.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# 问题二： ERROR: Cannot set priority of datanode process

不报错后，又报了 ERROR: Cannot set priority of datanode process错误，查看报错日志：

vim privileged-root-datanode-bd181.out
JSVC_HOME is not set or set incorrectly. jsvc is required to run secure
or privileged daemons. Please download and install jsvc from
http://archive.apache.org/dist/commons/daemon/binaries/
1
2
3
4
5

处理方法：

vim $HADOOP_CONF_DIR/hadoop-env.sh
vim /etc/profile.d/my_env.sh
# 把下面的注释掉
# export HDFS_DATANODE_USER=root
# export HADOOP_SECURE_DN_USER=root
# export HDFS_DATANODE_SECURE_USER=root
# export HDFS_NAMENODE_USER=root
# export HDFS_SECONDARYNAMENODE_USER=root
# export YARN_RESOURCEMANAGER_USER=root
# export YARN_NODEMANAGER_USER=root
# export YARN_PROXYSERVER_USER=root
# 分发
xsync $HADOOP_CONF_DIR/hadoop-env.sh
xsync /etc/profile.d/my_env.sh
xcall source /etc/profile.d/my_env.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

用root用户启动成功：

start-dfs.sh
xcall jps
--------- bd185 ----------
4325 Jps
4236 DataNode
--------- bd184 ----------
4240 DataNode
4330 Jps
--------- bd183 ----------
4338 Jps
4249 DataNode
--------- bd182 ----------
7015 Jps
6830 DataNode
--------- bd181 ----------
18981 Jps
18519 NameNode
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

切换到eddie用户，同样启动成功。

# 问题3：ERROR: Cannot write historyserver pid /tmp/hadoop-eddie-historyserver.pid

# 把/tmp文件夹修改为777
sudo chmod -R 777 /tmp
1
2

# 问题4：启动后找不到secondarynamenode进程

# 进入log目录
cd /opt/apache/hadoop-3.3.6/logs
vim hadoop-eddie-secondarynamenode-bd182.log
# 找到最后面发现报错
-----------------------------------------------------------------------------
java.io.FileNotFoundException: /opt/apache_data/hdfs/dfs/namesecondary/current/VERSION (权限不够)
-----------------------------------------------------------------------------
# 原因是第一次启动hdfs的时候用的是root用户，导致文件是root创建的
ls -l /opt/apache_data/hdfs/dfs/namesecondary/current/VERSION
# 发现所属是root
-rw-r--r-- 1 root root 217 7月  20 21:47 /opt/apache_data/hdfs/dfs/namesecondary/current/VERSION
# 修改整个目录的属组
xcall sudo chown eddie:eddie /opt/apache_data/ -R
1
2
3
4
5
6
7
8
9
10
11
12
13
#Hadoop#Linux
搭建一套源生Hadoop、Spark、Flink集群1-环境准备
06.搭建一套源生hadoop、Spark、Flink集群3-Hive安装

← 06.搭建一套源生hadoop、Spark、Flink集群3-Hive安装→

最近更新
01
06.搭建一套源生hadoop、Spark、Flink集群3-Hive安装 原创
04-26
02
搭建一套源生Hadoop、Spark、Flink集群1-环境准备 原创
04-25
03
求股票波峰波谷 原创
04-08
更多文章>
Theme by Vdoing | Copyright © 2019-2024 Evan Xu | MIT License | 粤ICP备2023070487号-1 | 粤公网安备44200102445447