Hadoop环境搭建

<!--解压Java安装包-->
tar -zxvf /opt/software/jdk-8u221-linux-x64.tar.gz -C /opt/module

<!--解压Java安装包-->
tar -zxvf /opt/software/hadoop-3.2.1.tar.gz -C /opt/module

<!--编辑并添加内容到环境配置文件-->
vi /etc/prolfile

<!--添加内容-->
#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_221
export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop-3.2.1
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

<!--让配置生效-->
source /etc/profile

<!--验证java-->
java -version
#版本内容如下：
java version "1.8.0_221"
Java(TM) SE Runtime Environment (build 1.8.0_221-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.221-b11, mixed mode)

<!--验证hadoop-->
hadoop version
#版本内容如下：
Hadoop 3.2.1
Source code repository https://gitbox.apache.org/repos/asf/hadoop.git -r b3cbbb467e22ea829b3808f4b7b01d07e0bf3842
Compiled by rohithsharmaks on 2019-09-10T15:56Z
Compiled with protoc 2.5.0
From source with checksum 776eaf9eee9c0ffc370bcbc1888737
This command was run using /opt/module/hadoop-3.2.1/share/hadoop/common/hadoop-common-3.2.1.jar

<!--编辑hadoop-env.sh>
vi etc/hadoop/hadoop-env.sh
<!--添加内容-->
# set to the root of your Java installation
export JAVA_HOME=/opt/module/jdk1.8.0_221

<!--运行例子-->  
mkdir input
cp etc/hadoop/*.xml input
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.2.1.jar grep input output ‘dfs[a-z.]+‘
cat output/*

<!--指定HDFS中NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop100:9000</value>
    </property>
<!--指定Hadoop运行时产生文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-3.2.1/data/tmp</value>
    </property>

<!--指定hdfs副本的数量-->
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

bin/hdfs namenode -format

sbin/hadoop-daemon.sh start namenode

sbin/hadoop-daemon.sh start datanode

jps
16624 NameNode
19045 DataNode

<!--Reducer获取数据的方式-->
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<!--指定YARN的ResourceManager的地址-->
<property>
    <name>yarn.nodemanager.hostname</name>
    <value>hadoop100</value>
</property>

<!--指定MR运行在YARN上-->
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>

sbin/yarn-daemon.sh start resourcemanager

jps
16624 NameNode
19045 DataNode
17093 Jps
13740 ResourceManager

<!--添加内容-->
export JAVA_HOME=/opt/module/jdk1.8.0_221

<!--指定HDFS中NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop102:9000</value>
    </property>
<!--指定Hadoop运行时产生文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-3.2.1/data/tmp</value>
    </property>

<!--hdfs副本的数量-->
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
<!--指定hadoop辅助名称节点主机配置-->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop103:80090</value>
    </property>

<!--指定MR运行在YARN上-->
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>

<!--Reducer获取数据的方式-->
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<!--指定YARN的ResourceManager的地址-->
<property>
    <name>yarn.nodemanager.hostname</name>
    <value>hadoop103</value>
</property>

<!--101-->
bin/hdfs namenode -format
sbin/hadoop-daemon.sh start namenode
sbin/hadoop-daemon.sh start datanode
sbin/yarn-daemon.sh start nodemanager

<!--102-->
sbin/hadoop-daemon.sh start datanode
sbin/yarn-daemon.sh start nodemanager
sbin/yarn-daemon.sh start resourcemanager

<!--103--> 
sbin/hadoop-daemon.sh start datanode
sbin/yarn-daemon.sh start nodemanager