kubernetes(十一)--kubeadm部署k8s高可用集群

一、系统初始化

系统初始化参考：https://www.cnblogs.com/hujinzhong/p/12251486.html

节点准备：

以一台master节点为例：

1）主机名及hosts解析

[root@master01 ~]# hostnamectl set-hostname k8s-master01
[root@master01 ~]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.0.0.11  master01
10.0.0.12  master02
10.0.0.13  master03
10.0.0.21  node01
10.0.0.22  node02
10.0.0.100 k8s-vip

2）安装依赖包

[root@master01 ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@master01 ~]# yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git

3）防火墙/swap/selinux

#防火墙
[root@master01 ~]# systemctl stop firewalld && systemctl disable firewalld
[root@master01 ~]# yum install -y iptables-services && systemctl start iptables && systemctl enable iptables &&iptables -F && service iptables save

#selinix
[root@master01 ~]# setenforce 0 && sed -i ‘s#^SELINUX=.*#SELINUX=disabled#g‘ /etc/selinux/config
setenforce: SELinux is disabled
[root@master01 ~]# getenforce
Disabled

#关闭swap
[root@master01 ~]# swapoff -a && sed -r -i ‘/swap/s@(.*)@#\1@g‘ /etc/fstab

4）内核升级

[root@master01 ~]# rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
[root@master01 ~]# yum --enablerepo=elrepo-kernel install -y kernel-lt
[root@master01 ~]# grub2-set-default "CentOS Linux (4.4.182-1.el7.elrepo.x86_64) 7 (Core)"
[root@master01 ~]# reboot
[root@k8s-master01 ~]# uname -r
4.4.213-1.el7.elrepo.x86_64

5）关闭NUMA

[root@k8s-master01 ~]# cp /etc/default/grub{,.bak}
[root@k8s-master01 ~]# vim /etc/default/grub # 在 GRUB_CMDLINE_LINUX 一行添加 `numa=off` 参数
GRUB_CMDLINE_LINUX="biosdevname=0 net.ifnames=0 rhgb quiet numa=off"
[root@k8s-master01 ~]# cp /boot/grub2/grub.cfg{,.bak}
[root@k8s-master01 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg

6）内核参数调整

cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

[root@k8s-master01 ~]# sysctl -p /etc/sysctl.d/kubernetes.conf

7）时区调整

[root@k8s-master01 ~]# timedatectl set-timezone Asia/Shanghai
[root@k8s-master01 ~]# timedatectl set-local-rtc 0
[root@k8s-master01 ~]# systemctl restart rsyslog && systemctl restart crond

8）关闭不必要服务

[root@k8s-master01 ~]# systemctl stop postfix && systemctl disable postfix

9）设置 rsyslogd 和 systemd journald

[root@k8s-master01 ~]# mkdir /var/log/journal
[root@k8s-master01 ~]# mkdir /etc/systemd/journald.conf.d

cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
#持久化保存到磁盘
Storage=persistent
# 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空间10G
SystemMaxUse=10G
# 单日志文件最大200M
SystemMaxFileSize=200M
# 日志保存时间 2 周
MaxRetentionSec=2week
# 不将日志转发到 syslog
ForwardToSyslog=no
EOF

[root@master03 ~]# systemctl restart systemd-journald

10）kube-proxy开启ipvs的前提设置

[root@k8s-master01 ~]# modprobe br_netfilter

cat  >  /etc/sysconfig/modules/ipvs.modules  <<EOF
#!/bin/bash
modprobe  --  ip_vs
modprobe  --  ip_vs_rr
modprobe  --  ip_vs_wrr
modprobe  --  ip_vs_sh
modprobe  --  nf_conntrack_ipv4
EOF

[root@k8s-master01 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules &&lsmod | grep -e ip_vs -e nf_conntrack_ipv4

11）docker安装

[root@k8s-master01 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@k8s-master01 ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo[root@k8s-master01 ~]# yum update -y && yum install -y docker-ce

[root@k8s-master01 ~]# mkdir /etc/docker -p
[root@k8s-master01 ~]# mkdir -p  /etc/systemd/system/docker.service.d

cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
    "max-size": "100m"
    }
}
EOF

[root@master03 ~]# systemctl daemon-reload && systemctl start docker && systemctl enable docker

12）导入相关镜像

[root@k8s-master01 ~]# mkdir /root/k8s -p
[root@k8s-master01 ~]# cd k8s/
[root@k8s-master01 k8s]# docker load -i keepalived.tar
[root@k8s-master01 k8s]# docker load -i haproxy.tar
[root@k8s-master01 k8s]# docker images
REPOSITORY                           TAG                 IMAGE ID            CREATED             SIZE
k8s.gcr.io/kube-apiserver            v1.15.1             68c3eb07bfc3        6 months ago        207MB
k8s.gcr.io/kube-scheduler            v1.15.1             b0b3c4c404da        6 months ago        81.1MB
k8s.gcr.io/kube-proxy                v1.15.1             89a062da739d        6 months ago        82.4MB
k8s.gcr.io/kube-controller-manager   v1.15.1             d75082f1d121        6 months ago        159MB
k8s.gcr.io/coredns                   1.3.1               eb516548c180        12 months ago       40.3MB
k8s.gcr.io/etcd                      3.3.10              2c4adeb21b4f        14 months ago       258MB
wise2c/keepalived-k8s                latest              0ba6a7862982        20 months ago       14MB
wise2c/haproxy-k8s                   latest              fde31577093d        20 months ago       71.1MB
k8s.gcr.io/pause                     3.1                 da86e6ba6ca1        2 years ago         742kB

二、启动haproxy容器

[root@k8s-master01 lb]# cat start-haproxy.sh 
#!/bin/bash
MasterIP1=10.0.0.11
MasterIP2=10.0.0.12
MasterIP3=10.0.0.13
MasterPort=6443

docker run -d --restart=always --name HAProxy-K8S -p 6444:6444         -e MasterIP1=$MasterIP1         -e MasterIP2=$MasterIP2         -e MasterIP3=$MasterIP3         -e MasterPort=$MasterPort         -v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg         wise2c/haproxy-k8s

[root@k8s-master01 lb]# cat /data/lb/etc/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    retries 3
    option redispatch
    timeout connect  5000
    timeout client  50000
    timeout server  50000

frontend stats-front
  bind *:8081
  mode http
  default_backend stats-back

frontend fe_k8s_6444
  bind *:6444
  mode tcp
  timeout client 1h
  log global
  option tcplog
  default_backend be_k8s_6443
  acl is_websocket hdr(Upgrade) -i WebSocket
  acl is_websocket hdr_beg(Host) -i ws

backend stats-back
  mode http
  balance roundrobin
  stats uri /haproxy/stats
  stats auth pxcstats:secret

backend be_k8s_6443
  mode tcp
  timeout queue 1h
  timeout server 1h
  timeout connect 1h
  log global
  balance roundrobin
  server rancher01 10.0.0.11:6443

[root@k8s-master01 lb]# ./start-haproxy.sh
[root@k8s-master01 lb]# netstat -lntp|grep 6444
tcp6       0      0 :::6444                 :::*                    LISTEN      40073/docker-proxy

三、启动keepalive容器

[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=10.0.0.100
INTERFACE=eth0
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18

docker run -itd --restart=always --name=Keepalived-K8S         --net=host --cap-add=NET_ADMIN         -e VIRTUAL_IP=$VIRTUAL_IP         -e INTERFACE=$INTERFACE         -e CHECK_PORT=$CHECK_PORT         -e RID=$RID         -e VRID=$VRID         -e NETMASK_BIT=$NETMASK_BIT         -e MCAST_GROUP=$MCAST_GROUP         wise2c/keepalived-k8s

[root@k8s-master01 lb]# ./start-keepalived.sh 
3792352f22407eed1962a213ec82b1f00935b55b951704064c86142998bc4594

[root@k8s-master01 lb]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:97:61:29 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.11/24 brd 10.0.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 10.0.0.100/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
3: docker0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default 
    link/ether 02:42:9f:ca:92:e2 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
5: veth91153da@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master docker0 state UP group default 
    link/ether 92:fb:b5:91:09:bf brd ff:ff:ff:ff:ff:ff link-netnsid 0

kubernetes(十一)--kubeadm部署k8s高可用集群

原文：https://www.cnblogs.com/hujinzhong/p/12274210.html