一、说明
节点(CentOS7.5) Server || Agent CPU
node11 Server || Agent 4G
node12 Agent 2G
node13 Agent 2G
#下载mysql-connector-java
二、系统环境搭建
1、网络配置(所有节点)
1．设置主机名和IP
#修改主机名
hostnamectl set-hostname node11
hostnamectl set-hostname node12
hostnamectl set-hostname node13
2．配置/etc/hosts IP和主机名的映射
#所有节点执行
cat >> /etc/hosts <<EOF 192.168.92.23 node11 192.168.92.24 node12 192.168.92.25 node13
EOF
2、SSH免密码登录
#一直回车直即可
ssh-keygen -t rsa
ssh-copy-id -i node11
ssh-copy-id -i node12
ssh-copy-id -i node13
ssh root@node11 date
ssh root@node12 date
ssh root@node13 date
3、关闭防火墙
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
#查看防火墙状态
firewall-cmd --state
systemctl status firewalld
4、关闭SELINUX
#查看selinux状态
sestatus -v
vi /etc/selinux/config
SELINUX=disabled
5、安装JDK
#jdk-8u271这个版本太高
mkdir -p /software
rpm -qa | grep java
#卸载自带open jdk
rpm -e --nodeps java-1.7.0-openjdk-headless-1.7.0.141-2.6.10.1.el7_3.x86_64
rpm -e --nodeps java-1.7.0-openjdk-1.7.0.141-2.6.10.1.el7_3.x86_64
rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.131-3.b12.el7_3.x86_64
rpm -e --nodeps java-1.8.0-openjdk-1.8.0.131-3.b12.el7_3.x86_64
cd /software
scp jdk-8u191-linux-x64-rpm.zip root@node12:/software/
scp jdk-8u191-linux-x64-rpm.zip root@node13:/software/
unzip jdk-8u191-linux-x64-rpm.zip
rpm -ivh jdk-8u191-linux-x64.rpm
ln -s /usr/java/jdk1.8.0_191-amd64/ /usr/java/jdk1.8.0_191
cat >> /etc/profile <<EOF
export JAVA_HOME=/usr/java/jdk1.8.0_191
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/sbin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
EOF
source /etc/profile
java -version
6、设置NTP
yum -y install ntp
#注释掉所有server *.*.*的指向，新添加一条可连接的ntp服务器
#在其他节点上把ntp指向master服务器地址即可(/etc/ntp.conf下) server node11 iburst
vi /etc/ntp.conf
server 192.168.92.23 iburst
systemctl start ntpd
systemctl enable ntpd
systemctl status ntpd
7、安装配置MySql
#主节点安装
#卸载mariadb
rpm -qa|grep mariadb
rpm -e --nodeps mariadb-libs-5.5.52-1.el7.x86_64
rpm -qa|grep mariadb
#上传MySQL包到：/software
ll
mysql-5.7.24-linux-glibc2.12-x86_64.tar.gz
#mysql-connector-java-5.1.39.tar.gz
3．安装MySQL
groupadd mysql
useradd -r -g mysql -s /sbin/nologin mysql
mv mysql-5.7.18-linux-glibc2.5-x86_64.tar.gz /usr/local/
cd /usr/local/
tar zxvf mysql-5.7.18-linux-glibc2.5-x86_64.tar.gz -C /usr/local/
ln -s mysql-5.7.18-linux-glibc2.5-x86_64/ mysql
cd mysql
mkdir mysql-files
ll
chown -R mysql.mysql .
ll
#op).e)osp8dF
bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data
cp -rf /etc/my.cnf /etc/my.cnf.old
cat >> /etc/my.cnf <<EOF
[mysqld]
basedir=/usr/local/mysql
datadir=/usr/local/mysql/data
EOF
#启动mysql skip-grant-tables
bin/mysqld_safe --user=mysql &
cp support-files/mysql.server /etc/init.d/mysql
chmod a+x /etc/init.d/mysql
chkconfig --add mysql
chkconfig mysql on
service mysql status
#设置root的密码
#设置mysql环境变量
echo "export PATH=$PATH:/usr/local/mysql/bin" >> /etc/profile
source /etc/profile
mysql -uroot -p
#use mysql;
#update user set authentication_string=PASSWORD(‘123456‘) where User=‘root‘;
#以上修改密码不成功，可以set
SET PASSWORD = PASSWORD(‘123456‘);
#或者
#alter user root@‘localhost‘ identified by ‘123456‘;
flush privileges;
#重新初始化
exit
killall mysqld
rm -rf /usr/local/mysql/data/
chown -R mysql.mysql /usr/local/mysql
#Z,hBoE:fh20>
bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data
bin/mysql_ssl_rsa_setup
service mysql start
mysql -uroot -p
SET PASSWORD = PASSWORD(‘123456‘);
flush privileges;
exit
三、Cloudera Manager Server&Agent安装
1、安装Cloudera Manager Server&Agent
#三台机器都要安装
cd /software/
scp cloudera-manager-centos7-cm5.15.0_x86_64.tar.gz root@node12:/software/
scp cloudera-manager-centos7-cm5.15.0_x86_64.tar.gz root@node13:/software/
tar xvzf cloudera-manager-centos7-cm5.15.0_x86_64.tar.gz -C /opt
2、配置CM Agent
1.cm的安装包的server和agent都是同一个包，启动的时候启动命令指定为server还是agent；
2.cm有一个工作目录，需要我们来创建；创建一个用户；修改agent的配置，给它指定server的位置；那个端口7182是server和agent通信的端口；
3.server_host配置为server的名字,三台机器
vi /opt/cm-5.15.0/etc/cloudera-scm-agent/config.ini
3、创建用户cloudera-scm（所有节点）
useradd --system --home=/opt/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
4、创建Parcel目录
创建parcel目录，这个目录是server和agent用来接收和发送数据的目录，server端的parcel-repo这个目录会把所有的安装文件全部下载到此目录，而agent也需要安装包，parcels就是用来存储指定的安装包的，当然需要有权限能操作这些目录；
#Server节点
mkdir -p /opt/cloudera/parcel-repo
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
#Agent节点
mkdir -p /opt/cloudera/parcels
chown cloudera-scm:cloudera-scm /opt/cloudera/parcels
#把CDH安装包移到cloudera/parcel-repo下面,并修改其中后缀为sha1的文件为sha,如下
mv CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel /opt/cloudera/parcel-repo
mv CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel.sha1 /opt/cloudera/parcel-repo
cd /opt/cloudera/parcel-repo
mv CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel.sha1 CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel.sha
5、配置CM Server的数据库
cd /software
tar zxvf mysql-connector-java-5.1.39.tar.gz
cd mysql-connector-java-5.1.39
cp mysql-connector-java-5.1.39-bin.jar mysql-connector-java.jar
cp mysql-connector-java.jar /opt/cm-5.15.0/share/cmf/lib/
mysql -u root -p123456 use mysql; grant all on *.* to ‘root‘@‘%‘ identified by ‘123456‘ with grant option; flush privileges;
exit
#初始化cm数据库:
cd /opt/cm-5.15.0/share/cmf/schema/ #格式：数据库类型、数据库、数据库服务器、用户名、密码、cm server服务器 ./scm_prepare_database.sh mysql cm -h node11 -uroot -p123456 --scm-host node11 scm scm scm
JAVA_HOME=/usr/java/jdk1.8.0_191
Verifying that we can write to /opt/cm-5.15.0/etc/cloudera-scm-server
log4j:ERROR Could not find value for key log4j.appender.A
log4j:ERROR Could not instantiate appender named "A".
Creating SCM configuration file in /opt/cm-5.15.0/etc/cloudera-scm-server
Executing: /usr/java/jdk1.8.0_271/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cm-5.15.0/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /opt/cm-5.15.0/etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
log4j:ERROR Could not find value for key log4j.appender.A
log4j:ERROR Could not instantiate appender named "A".
[2021-01-10 21:12:06,901] INFO 0[main] - com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java) - Successfully connected to database.
All done, your SCM database is configured correctly!
#创建其他组件需要的数据库
mysql -uroot -p123456
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
exit
6、启动CM Manager&Agent服务
#先确保启动了mysql
service mysql status
#node11:执行
/opt/cm-5.15.0/etc/init.d/cloudera-scm-server start
#node11,12,13: 执行
/opt/cm-5.15.0/etc/init.d/cloudera-scm-agent start
#service服务启动需要几分钟，取决于服务器的性能，此时我们可以使用
netstat -apn|grep 7180
#查询端口被占用后，就可以用浏览器打开:用户名与密码分别为admin/admin
7、Server&Agent开机自启动服务
#配置server主节点node11
cp /opt/cm-5.15.0/etc/init.d/cloudera-scm-server /etc/init.d/cloudera-scm-server
chkconfig --add cloudera-scm-server
chkconfig cloudera-scm-server on
vi /etc/init.d/cloudera-scm-server
#CMF_DEFAULTS=${CMF_DEFAULTS:-/etc/default} 修改为如下
CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.15.0/etc/default}
chkconfig --list cloudera-scm-server
service cloudera-scm-server start & systemctl start cloudera-scm-server.service
service cloudera-scm-server status
systemctl status cloudera-scm-server.service
#配置agent所有节点,node11,node12,node13
cp /opt/cm-5.15.0/etc/init.d/cloudera-scm-agent /etc/init.d/cloudera-scm-agent
chkconfig --add cloudera-scm-agent
chkconfig cloudera-scm-agent on
vi /etc/init.d/cloudera-scm-agent
#CMF_DEFAULTS=${CMF_DEFAULTS:-/etc/default} 修改为
CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.15.0/etc/default}
#添加：export JAVA_HOME=/usr/local/java/jdk1.8
service cloudera-scm-agent start & systemctl start cloudera-scm-agent.service
service cloudera-scm-agent status
systemctl status cloudera-scm-agent.service
四、CDH5安装
下一步,选择免费版本
继续,下一步,直到如下界面,可以在当前管理的主机列表中看到对应的节点。选择要安装的节点，这里我选择全部节点，继续。
选择自己下载的CDH版本,点击继续
接下来是安装Parcel,服务器检查，
#更改交换空间:
echo 10 > /proc/sys/vm/swappiness
#如下禁用此设置，然后将同一命令添加到 /etc/rc.local 等初始化脚本中，以便在系统重启时予以设置。以下主机将受到影响:
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
cat >> /etc/rc.local <<EOF
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
EOF
cat /etc/rc.local
重新运行检测,
全部完成,点击完成,这里选择自定义安装,不要选择：Hue,Isilon,Kafka,Kudu
服务配置，一般情况下保持默认就可以了，也可以自己手动根据节点情况进行分配，点击继续
测试如下数据库连接
#这里启动hive报错找不到mysql连接驱动包,拷贝一份 mysql-connector-java.jar 到 /opt/cloudera/parcels/CDH-5.15.0-1.cdh5.15.0.p0.21/lib/hive/lib/下面
cp /opt/cm-5.15.0/share/cmf/lib/mysql-connector-java.jar /opt/cloudera/parcels/CDH-5.15.0-1.cdh5.15.0.p0.21/lib/hive/lib/
点击继续,默认集群设置,直到安装成功.
CDH搭建Hadoop集群_Centos7
原文：https://www.cnblogs.com/buffercache/p/14261605.html
