环境准备

主机名 WanIP LanIP 角色 应用 配置
master01 10.0.0.200 172.16.1.200 master kubectl、apiserver、 scheduler、controller、 etcd、kubelet、docker、 kube-proxy 1h2g
node01 10.0.0.201 172.16.1.201 node kubelet、docker、kubeproxy、proxy 1h2g
node02 10.0.0.202 172.16.1.202 node kubelet、docker、kubeproxy、proxy 1h2g
node03 10.0.0.203 172.16.1.203 node kubelet、docker、kubeproxy、proxy 2h4g

IP规划

三种Service IP
PodIP 10.2.0.0
ClusterIP 10.1.0.0
NodeIP 10.0.0.0

基础环境优化

注意：四个节点都操作

1.配置kubelet配置文件，使用系统自带的Cgroup驱动和禁用swap

1
2
3
4
5
6
7
8
9
10
[root@master ~]# cat >/etc/sysconfig/kubelet <<EOF
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
KUBELET_EXTRA_ARGS="--fail-swap-on=false"
EOF

`注释`
## 让kubelet使用系统自带的Cgroup驱动
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
## 让kubelet禁止使用swap（虚拟内存）
KUBELET_EXTRA_ARGS="--fail-swap-on=false"

2.内核参数调优

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@master ~]#  cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
fs,file-max=52706963
fs.nr_open=52706963
EOF

`注释`
## 开启iptables功能 ipv6
net.bridge.bridge-nf-call-ip6tables=1
## 开启iptables功能 ipv4
net.bridge.bridge-nf-call-iptables=1
## 开启内核转发
net.ipv4.ip_forward=1
## 内核参数禁用swap
vm.swappiness=0
## 文件描述符文件最大值
fs,file-max=52706963
## 文件描述符开启数量
fs.nr_open=52706963

3.更改docker源

1
2
3
4
5
6
7
8
9
10
11
12
#添加docekr源
[root@master ~]#  cat > /etc/yum.repos.d/docker-ce.repo <<"EOF"
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg
EOF

#替换为华为源
[root@master ~]# sed -i 's+download.docker.com+mirrors.huaweicloud.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo

4.安装时间同步服务

1
2
3
[root@master ~]# yum install -y chrony
[root@master ~]# systemctl start chronyd
[root@master ~]# systemctl enable chronyd

5.关闭swap

1
2
3
4
5
6
7
8
9
#临时关闭swap
[root@master ~]#  swapoff -a && sysctl -w vm.swappiness=0
#永久关闭
[root@master ~]#  sed -i '/swap/d' /etc/fstab
#查看是否关闭
[root@master ~]# free -h
              total        used        free      shared  buff/cache   available
Mem:           1.9G         95M        1.6G        9.5M        283M        1.7G
Swap:            0B          0B          0B

6.加载ipvs模块

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
[root@master ~]# cat  > /etc/sysconfig/modules/ipvs.modules <<EOF
#! /bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

#授权
[root@master ~]# chmod +x /etc/sysconfig/modules/ipvs.modules
#source使生效
[root@master ~]# source /etc/sysconfig/modules/ipvs.modules
#查看是否生效
[root@master ~]# lsmod|grep -e 'ip_vs' -e 'nf_conntrack_ipv'
nf_conntrack_ipv4      15053  0 
nf_defrag_ipv4         12729  1 nf_conntrack_ipv4
ip_vs_sh               12688  0 
ip_vs_wrr              12697  0 
ip_vs_rr               12600  0 
ip_vs                 145497  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          133095  2 ip_vs,nf_conntrack_ipv4
libcrc32c              12644  3 xfs,ip_vs,nf_conntrack

#了解下IPVS  LVS:ipvsadm

安装docker

注意：所有节点执行

1.安装指定版本docekr和容器运行时

1
2
3
[root@master ~]# yum install -y docker-ce-19.03.15 docker-ce-cli-19.03.15 containerd.io

`容器运行时，Docker、containerd、CRI-O`

2.启动和开机自启

1
2
[root@master ~]# systemctl start docker
[root@master ~]# systemctl enable docker

3.添加docker镜像加速

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[root@master ~]# cat > /etc/docker/daemon.json <<EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": [
    "https://docker.1panel.live",
    "https://dockercf.jsdelivr.fyi",
    "https://docker-cf.registry.cyou",
    "https://docker.chenby.cn",
    "https://docker.jsdelivr.fyi",
    "https://docker.m.daocloud.io",
    "https://docker.m.daocloud.io",
    "https://docker.mirrors.sjtug.sjtu.edu.cn",
    "https://docker.mirrors.ustc.edu.cn",
    "https://docker.nju.edu.cn",
    "https://dockerproxy.com",
    "https://docker.rainbond.cc",
    "https://docker.registry.cyou",
    "https://dockertest.jsdelivr.fyi",
    "https://hub-mirror.c.163.com",
    "https://hub.rat.dev/",
    "https://mirror.aliyuncs.com",
    "https://mirror.baidubce.com",
    "https://mirror.iscas.ac.cn",
    "https://registry.docker-cn.com"
  ]
}
EOF

`"exec-opts": ["native.cgroupdriver=systemd"], `#镜像加速使用系统Cgroup驱动

4.重启docerk

1
2
[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart docker

安装kubeadm

注意：所有节点执行

1.更换kubernetes的yum源

1
2
3
4
5
6
7
8
9
cat <<"EOF" > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.huaweicloud.com/kubernetes/yum/repos/kubernetes-el7-$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://mirrors.huaweicloud.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.huaweicloud.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

2.下载指定版本kubernetes

1
2
3
4
5
[root@master ~]# yum install kubelet-1.19.3 kubeadm-1.19.3  kubectl-1.19.3  ipvsadm -y

`kubelet`: node节点容器运行时的控制器 
`kubeadm`: 做k8s集群 kubectl k8s的命令客户端
`ipvsadm`: 加载ipvs模块

3.启动并加入开机自启

1
2
[root@master ~]# systemctl start kubelet
[root@master ~]# systemctl enable kubelet

初始化集群

:dango:注意：仅master节点执行

步骤1-3都是master节点执行

1.初始化集群

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[root@master ~]# kubeadm init \
--apiserver-advertise-address=10.0.0.200 \
--image-repository registry.aliyuncs.com/google_containers  \
--kubernetes-version=v1.19.3 \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.2.0.0/16 \
--service-dns-domain=cluster.local \
--ignore-preflight-errors=Swap \
--ignore-preflight-errors=NumCPU

`注释`
`--apiserver-advertise-address=10.0.0.200`  #当前master主机ip
`--service-cidr=10.1.0.0/16`							 #规划ClusterIP网段
`--pod-network-cidr=10.2.0.0/16`					 #规划的podIP
`--service-dns-domain=cluster.local`			 #使用本地DNS集群，k8会自己装Coredns
`--ignore-preflight-errors=Swap`					#忽略Swap报错
`--ignore-preflight-errors=NumCPU`				#忽略CPU报错

`ClusterIP`: 用于构建容器化应用的内部通信网络，作为POD的负载均衡

###初始化时间比较长，没有报错等待即可

保存token

1
2
3
4
5
6
7
8
9
10
11
12
13
14
`初始化成功后会输入如下，你的和我的不一样，注意保存,24h后会消失`
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
`#这个命令也保存，等一下直接复制执行`
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.0.0.200:6443 --token ao4unc.i520imoy7frmmucj \
    --discovery-token-ca-cert-hash sha256:77e1fdf379200e1e377304865da17a4f35212955eaf331d2df736c452a2e82f8

!!! 到这一步前面没报错就可以打快照，万一后面出错方便回滚，建议所有节点都可以打一次，如果报错可以回滚到同一起始点

!! 如果时间太久忘记加入令牌的命令，执行命令

kubeadm token create --print-join-command

获取新的join令牌

2.创建配置文件目录

1
2
3
4
[root@master ~]#   mkdir -p $HOME/.kube
[root@master ~]#   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master ~]#   sudo chown $(id -u):$(id -g) $HOME/.kube/config
`直接复制上面的三条输出`

3.查看k8s集群节点

1
2
3
[root@master ~]# kubectl get nodes
NAME     STATUS     ROLES    AGE   VERSION
master   NotReady   master   11m   v1.19.3

4.其他node节点加入集群

注意：这里是3个node节点执行 复制自己的，含有token的命令，不要复制我的

1
2
3
4
5
6
7
8
9
10
[root@node01 ~]# kubeadm join 10.0.0.200:6443 --token ao4unc.i520imoy7frmmucj \
    --discovery-token-ca-cert-hash sha256:77e1fdf379200e1e377304865da17a4f35212955eaf331d2df736c452a2e82f8 
    
[root@node02 ~]# kubeadm join 10.0.0.200:6443 --token ao4unc.i520imoy7frmmucj \
    --discovery-token-ca-cert-hash sha256:77e1fdf379200e1e377304865da17a4f35212955eaf331d2df736c452a2e82f8 

[root@node03 ~]# kubeadm join 10.0.0.200:6443 --token ao4unc.i520imoy7frmmucj \
    --discovery-token-ca-cert-hash sha256:77e1fdf379200e1e377304865da17a4f35212955eaf331d2df736c452a2e82f8 
···
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

5.查看k8s集群

去master节点看

1
2
3
4
5
6
7
[root@master ~]# kubectl get nodes
NAME     STATUS     ROLES    AGE     VERSION
master   NotReady   master   17m     v1.19.3
node01   NotReady   <none>   3m12s   v1.19.3
node02   NotReady   <none>   3m32s   v1.19.3
node03   NotReady   <none>   3m27s   v1.19.3
`看到四个节点NotReady即OK，因为还未配置网络，不能跨主机通信`

6.设置Kube-proxy使用ipvs模式

1
2
3
4
5
`k8s默认使用的是iptables防火墙，可以修改成性能更高的ipvs模式，该模式LVS也在使用`
[root@master01 ~]# kubectl edit cm kube-proxy -n kube-system
mode: "" -->  mode: "ipvs"

#搜mode，添上ipvs即可

7.查看指定名称空间中pod相信信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
[root@master ~]# kubectl get pod -n kube-system
NAME                             READY   STATUS    RESTARTS   AGE
coredns-6d56c8448f-spslt         0/1     Pending   0          26m
coredns-6d56c8448f-x5lqj         0/1     Pending   0          26m
etcd-master                      1/1     Running   0          26m
kube-apiserver-master            1/1     Running   0          26m
kube-controller-manager-master   1/1     Running   0          26m
kube-proxy-qmbtd                 1/1     Running   0          26m
kube-proxy-sqnbb                 1/1     Running   0          12m
kube-proxy-t7hcw                 1/1     Running   0          12m
kube-proxy-xzv7f                 1/1     Running   0          12m
kube-scheduler-master            1/1     Running   0          26m

`状态没异常即可，除Pending和Running之外的状态都是不正常的，检查是否步骤出错`

`coredns-6d56c8448f-spslt`			#dns服务器
`coredns-6d56c8448f-x5lqj`			#dns服务器
`etcd-master`									 #数据库所有节点的存储
`kube-apiserver-master`					#集群的控制中心--'司令部'
`kube-controller-manager-master` #维护集群的状态--'控制器'
`kube-proxy-qmbtd`							#负责将网络流量转发到正确的Pod、端口映射
`kube-proxy-sqnbb`
`kube-proxy-t7hcw`
`kube-proxy-xzv7f`
`kube-scheduler-master`					#调度器

8.查看指定名称空间中pod详细信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@master ~]# kubectl get pod -n kube-system -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP        
coredns-6d56c8448f-spslt         0/1     Pending   0          42m   <none>    
coredns-6d56c8448f-x5lqj         0/1     Pending   0          42m   <none>    
etcd-master                      1/1     Running   0          42m   10.0.0.200
kube-apiserver-master            1/1     Running   0          42m   10.0.0.200
kube-controller-manager-master   1/1     Running   0          42m   10.0.0.200
kube-proxy-qmbtd                 1/1     Running   0          42m   10.0.0.200
kube-proxy-sqnbb                 1/1     Running   0          28m   10.0.0.203
kube-proxy-t7hcw                 1/1     Running   0          28m   10.0.0.201
kube-proxy-xzv7f                 1/1     Running   0          28m   10.0.0.202
kube-scheduler-master            1/1     Running   0          42m   10.0.0.200

`检查状态是否都是Running，比对下是否步骤有出入`

9.查看名称空间

1
2
3
4
5
6
7
8
9
[root@master ~]# kubectl get namespace
NAME              STATUS   AGE
default           Active   45m
kube-node-lease   Active   45m
kube-public       Active   45m
kube-system       Active   45m

`namespace 可以简写为 ns`
[root@master ~]# kubectl get ns

10.重启kube-proxy

1
2
3
4
5
6
7
`删除所有proxy`
kubectl get pod -n kube-system|grep 'kube-proxy'|awk '{print "kubectl delete pod -n kube-system "$1}'|bash

`#删除单个proxy`
kubectl delete pod -n kube-system kube-proxy名字 `例如kube-proxy-qmbtd`

`#因为k8会自动拉起停掉的pod，所以删除即可重启`

11.再次检查状态

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@master ~]# kubectl get pod -n kube-system -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP        
coredns-6d56c8448f-spslt         0/1     Pending   0          52m   <none>    
coredns-6d56c8448f-x5lqj         0/1     Pending   0          52m   <none>    
etcd-master                      1/1     Running   0          52m   10.0.0.200
kube-apiserver-master            1/1     Running   0          52m   10.0.0.200
kube-controller-manager-master   1/1     Running   0          52m   10.0.0.200
kube-proxy-62xz4                 1/1     Running   0          47s   10.0.0.203
kube-proxy-gxz9p                 1/1     Running   0          35s   10.0.0.202
kube-proxy-sfzzf                 1/1     Running   0          37s   10.0.0.201
kube-proxy-vjgc5                 1/1     Running   0          49s   10.0.0.200
kube-scheduler-master            1/1     Running   0          52m   10.0.0.200

`检查状态是否异常，如有异常，比对之前步骤，也可查看系统日志报错`

配置flannel

1.修改flannel资源清单

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
`kube-flannel.yml`
#下载链接 https://github.com/flannel-io/flannel/blob/master/Documentation/kube-flannel.yml

[root@master ~]# vim kube-flannel.yml
#1.将Network网段改为10.2.0.0/16
  net-conf.json: |
    {
      "Network": "10.2.0.0/16",
      "EnableNFTables": false,
      "Backend": {
        "Type": "vxlan"
      }
    }

#2.将容器绑定网卡eth0   在`- --kube-subnet-mgr`  后面加上`- --iface=eth0` 如下
containers:
	- args:
		- --ip-masq
		- --kube-subnet-mgr
		- --iface=eth0

#3.删除如下三行（意思是，只有标签是app的节点才能下载flannel）
  selector:
    matchLabels:
      app: flannel

2.应用flannel资源清单

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# 应用flannel资源清单
[root@master ~]# kubectl apply -f kube-flannel.yml


#检查flannel的pod是否成功启动
[root@master ~]# kubectl get pod -n kube-flannel
NAME                    READY   STATUS                  RESTARTS   AGE
kube-flannel-ds-g9drp   0/1     Running                 0          6m9s
kube-flannel-ds-rcwkv   1/1     Running                 0          6m9s
kube-flannel-ds-txcn2   1/1     Running                 0          6m9s
kube-flannel-ds-whd7p   1/1     Running                 0          6m9s

#检查k8s集群节点状态
[root@master ~]# kubectl get nodes
NAME     STATUS   ROLES    AGE   VERSION
master   Ready    master   17m   v1.19.3
node01   Ready    <none>   14m   v1.19.3
node02   Ready    <none>   14m   v1.19.3
node03   Ready    <none>   14m   v1.19.3

## 检查coredns是否正常运行
[root@master ~]# kubectl get pod -n kube-system
NAME                             READY   STATUS    RESTARTS   AGE
coredns-6d56c8448f-hxpjc         1/1     Running   0          17m
coredns-6d56c8448f-lb5m4         1/1     Running   0          17m
etcd-master                      1/1     Running   0          17m
kube-apiserver-master            1/1     Running   0          17m
kube-controller-manager-master   1/1     Running   0          17m
kube-proxy-46jqn                 1/1     Running   0          12m
kube-proxy-4bnmp                 1/1     Running   0          12m
kube-proxy-5lc7l                 1/1     Running   0          12m
kube-proxy-r8cll                 1/1     Running   0          12m
kube-scheduler-master            1/1     Running   0          17m

k8s命令补全

1
2
3
4
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
kubectl completion bash > /etc/bash_completion.d/kubectl