kubernetes 1.20 二进制方式高可用部署之部署Etcd集群（2）

三 ，部署Etcd集群

3.1 准备cfssl证书生成工具

cfssl是一个开源的证书管理工具，使用json文件生成证书，相比openssl更方便使用。
可以是任意一台机器，但是要注意IP或域名的变动
这里选择 192.168.1.23 k8s-master2 这台机器

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 #下载

chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64 #赋权
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo #移动

3.2 生成Etcd证书

master1节点

• 自签证书颁发机构（CA）

#创建目录
[root@k8s-master2 ~]# mkdir -p ca/etcd
[root@k8s-master2 ~]# cd ca/etcd
#自签CA
[root@k8s-master2 etcd]# vim ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
[root@k8s-master1 etcd]# vim ca-csr.json
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
#生成证书
[root@k8s-master1 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

• 使用自签CA签发Etcd HTTPS证书

#创建证书申请文件
[root@k8s-master1 etcd]# vim server-csr.json
{
    "CN": "etcd",
    "hosts": [
    "192.168.1.20",
    "192.168.1.21",
    "192.168.1.22"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}


#生成证书
[root@k8s-master1 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

注：上述文件hosts字段中IP为所有etcd节点的集群内部通信IP，一个都不能少！为了方便后期扩容可以多写几个预留的IP

3.3 部署Etcd集群

master1节点

#创建工作目录并解压二进制包
[root@k8s-master1 ~]# mkdir -p /opt/etcd/{bin,cfg,ssl}
[root@k8s-master1 ~]# tar zxvf etcd-v3.4.9-linux-amd64.tar.gz
[root@k8s-master1 ~]# mv etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

#创建etcd配置文件
[root@k8s-master1 ~]# vim /opt/etcd/cfg/etcd.conf
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.20:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.20:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.20:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.20:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.20:2380,etcd-2=https://192.168.1.21:2380,etcd-3=https://192.168.1.22:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

*参数解释
• ETCD_NAME：节点名称，集群中唯一
• ETCD_DATA_DIR：数据目录
• ETCD_LISTEN_PEER_URLS：集群通信监听地址
• ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
• ETCD_INITIAL_ADVERTISE_PEERURLS：集群通告地址
• ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
• ETCD_INITIAL_CLUSTER：集群节点地址
• ETCD_INITIALCLUSTER_TOKEN：集群Token
• ETCD_INITIALCLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群

systemd管理etcd

[root@k8s-master1 ~]# vim /usr/lib/systemd/system/etcd.service

###############etcd-v3.4.9
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--logger=zap                                  ##########etcd-v3.2.12删除这句
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target


#拷贝刚才生成的证书
[root@k8s-master1 ~]# cp ~/ca/etcd/*.pem /opt/etcd/ssl


#将上面Master节点所有生成的文件拷贝到节点Node2和节点Node3上
[root@k8s-master1 ~]# scp -r /opt/etcd/ k8s-node1:/opt/
[root@k8s-master1 ~]# scp /usr/lib/systemd/system/etcd.service k8s-node1:/usr/lib/systemd/system/
[root@k8s-master1 ~]# scp -r /opt/etcd/ k8s-node2:/opt/
[root@k8s-master1 ~]# scp /usr/lib/systemd/system/etcd.service k8s-node2:/usr/lib/systemd/system/


#在节点Node2和节点Node3分别修改etcd.conf配置文件
[root@k8s-node1 ~]# vim /opt/etcd/cfg/etcd.conf
#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.21:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.21:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.21:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.21:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.20:2380,etcd-2=https://192.168.1.21:2380,etcd-3=https://192.168.1.22:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

[root@k8s-node2 ~]# vim /opt/etcd/cfg/etcd.conf
#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.22:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.22:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.22:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.22:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.20:2380,etcd-2=https://192.168.1.21:2380,etcd-3=https://192.168.1.22:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

master1、node1、node2节点

#启动并设置开机启动
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd

*注：master如果先启动了，会出现start悬停等待的现象，这时候可以先把node1、node2的etcd启动，随后master的etcd会正常启动。

#查看集群状态
[root@k8s-master1 ~]# ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.20:2379,https://192.168.1.21:2379,https://192.168.1.22:2379" endpoint health --write-out=table

如果输出上面信息，就说明集群部署成功。
如果有问题第一步先看日志：/var/log/message 或 journalctl -u etcd