Kubeadm 证书修改

举报
kaliarch 发表于 2021/09/24 13:10:53 2021/09/24
【摘要】 一 kubeadm需改源码kubeadm安装的k8s集群有一个证书问题，证书的有效期为一年，过期的话kubectl命令就会异常。可以对其进行修改编译. 1.1 下载源码# 下载源码[root@master common-service]# git clone https://github.com.cnpmjs.org/kubernetes/kubernetes.git[root@maste...

一 kubeadm需改源码

kubeadm安装的k8s集群有一个证书问题，证书的有效期为一年，过期的话kubectl命令就会异常。可以对其进行修改编译.

1.1 下载源码

# 下载源码
[root@master common-service]# git clone https://github.com.cnpmjs.org/kubernetes/kubernetes.git
[root@master common-service]# cd kubernetes
[root@master kubernetes]# kubectl version
Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.5", GitCommit:"20c265fef0741dd71a66480e35bd69f18351daea", GitTreeState:"clean", BuildDate:"2019-10-15T19:07:57Z", GoVersion:"go1.12.10", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.5", GitCommit:"20c265fef0741dd71a66480e35bd69f18351daea", GitTreeState:"clean", BuildDate:"2019-10-15T19:07:57Z", GoVersion:"go1.12.10", Compiler:"gc", Platform:"linux/amd64"}
[root@master kubernetes]# git checkout release-1.15
Branch release-1.15 set up to track remote branch release-1.15 from origin.
Switched to a new branch 'release-1.15'
# git checkout release-1.16
[root@master kubernetes]# git branch
  master
* release-1.15
[root@master kubernetes]# vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go
[root@master kubernetes]# git diff
diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
index e1d77e1..755c08a 100644
--- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
+++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
@@ -571,7 +571,7 @@ func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certifi
                IPAddresses:  cfg.AltNames.IPs,
                SerialNumber: serial,
                NotBefore:    caCert.NotBefore,
-               NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity).UTC(),
+               NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity * 100).UTC(),
                KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
                ExtKeyUsage:  cfg.Usages,
        }
        
# 编译源码，需要在服务器上运行有go环境

1.2 安装go环境


cd /opt
wget -c https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz
tar -C /usr/local -zxvf go1.12.7.linux-amd64.tar.gz
cat >>/etc/profile <<EOF	
export GOROOT=/usr/local/go
export PATH=\$PATH:\$GOROOT/bin
EOF
source /etc/profile

cat >> /root/.bash_profile <<EOF
export GOPATH=\$HOME/go
EOF

source /root/.bash_profile

1.3 编译kubeadm

[root@VM-16-3-centos kubernetes_bak]# KUBE_BUILD_PLATFORMS=linux/amd64 make all WHAT=cmd/kubeadm GOFLAGS=-v GOGCFLAGS="-N -l"
# 编译完成后在_output/bin目录下

[root@VM-16-3-centos kubernetes]# ls _output/bin/
conversion-gen  deepcopy-gen  defaulter-gen  go2make  go-bindata  kubeadm  openapi-gen
[root@VM-16-3-centos kubernetes]# cd _output/bin/
[root@VM-16-3-centos bin]# ./kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"15+", GitVersion:"v1.15.13-beta.0.1+a34f1e483104bd", GitCommit:"a34f1e483104bd51c3e9a6aec3dbbcf6301789da", GitTreeState:"clean", BuildDate:"2020-08-25T10:45:53Z", GoVersion:"go1.12.7", Compiler:"gc", Platform:"linux/amd64"}

1.4 更新证书

# 将生成好的kubeadm文件上传到k8s集群的各master上
[root@master kubernetes]# /root/kubeadm alpha certs check-expiration
failed to load existing certificate apiserver-etcd-client: open /etc/kubernetes/pki/apiserver-etcd-client.crt: no such file or directory


[root@master kubernetes]# /root/kubeadm alpha certs renew all
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
failed to load existing certificate apiserver-etcd-client: open /etc/kubernetes/pki/apiserver-etcd-client.crt: no such file or directory
# 发现利用kubesphere安装的etcd证书异常，那么，手动更新几个证书

# 1.更新api-server证书
[root@master kubernetes]# /root/kubeadm alpha certs renew apiserver
certificate for serving the Kubernetes API renewed
[root@master kubernetes]# openssl x509 -in ssl/apiserver.crt -noout -enddate 
notAfter=Aug  1 15:35:33 2120 GMT

# 2.更新apiserver-kubelet-client 证书
[root@master kubernetes]# /root/kubeadm alpha certs renew apiserver-kubelet-client             
certificate for the API server to connect to kubelet renewed
[root@master kubernetes]# openssl x509 -in ssl/apiserver-kubelet-client.crt  -noout -text  |grep Not                    
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug  1 15:30:40 2120 GMT

# 3.更新front-proxy-client证书
[root@master kubernetes]# /root/kubeadm alpha certs renew front-proxy-client
certificate for the front proxy client renewed
[root@master kubernetes]# openssl x509 -in ssl/front-proxy-client.crt  -noout -text  |grep Not 
            Not Before: Sep  3 07:13:08 2019 GMT
            Not After : Aug  1 15:36:35 2120 GMT

批量操作

[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done
/etc/kubernetes/pki/apiserver.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug 31 10:55:38 2021 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug 31 10:55:39 2021 GMT
/etc/kubernetes/pki/ca.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug 31 07:13:10 2029 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
            Not Before: Sep  3 07:13:08 2019 GMT
            Not After : Aug 31 07:13:08 2029 GMT
/etc/kubernetes/pki/front-proxy-client.crt
            Not Before: Sep  3 07:13:08 2019 GMT
            Not After : Aug 31 10:55:39 2021 GMT
[root@xuel-ksserver data]# KUBECMD=/data/kubeadm16 
[root@xuel-ksserver data]# for i in apiserver apiserver-kubelet-client front-proxy-client;do ${KUBECMD} alpha certs renew ${i};done             
certificate for serving the Kubernetes API renewed
certificate for the API server to connect to kubelet renewed
certificate for the front proxy client renewed
[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done     
/etc/kubernetes/pki/apiserver.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug  7 11:06:09 2120 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug  7 11:06:10 2120 GMT
/etc/kubernetes/pki/ca.crt
            Not Before: Sep  3 07:13:10 2019 GMT
            Not After : Aug 31 07:13:10 2029 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
            Not Before: Sep  3 07:13:08 2019 GMT
            Not After : Aug 31 07:13:08 2029 GMT
/etc/kubernetes/pki/front-proxy-client.crt
            Not Before: Sep  3 07:13:08 2019 GMT
            Not After : Aug  7 11:06:10 2120 GMT
# 检查etcd证书
  [root@master ~]# for i in `ls /etc/kubernetes/pki/etcd/*.pem`;do echo $i;openssl x509 -in ${i} -noout -dates ;done
/etc/kubernetes/pki/etcd/admin-master-key.pem
unable to load certificate
140633169307552:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/admin-master.pem
notBefore=Sep  3 07:11:37 2019 GMT
notAfter=Aug 10 07:11:37 2119 GMT
/etc/kubernetes/pki/etcd/ca-key.pem
unable to load certificate
140597254109088:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/ca.pem
notBefore=Sep  3 07:11:36 2019 GMT
notAfter=Aug 10 07:11:36 2119 GMT
/etc/kubernetes/pki/etcd/member-master-key.pem
unable to load certificate
140164986681248:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/member-master.pem
notBefore=Sep  3 07:11:36 2019 GMT
notAfter=Aug 10 07:11:36 2119 GMT
/etc/kubernetes/pki/etcd/node-master-key.pem
unable to load certificate
139708689217440:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-master.pem
notBefore=Sep  3 07:11:37 2019 GMT
notAfter=Aug 10 07:11:37 2119 GMT
/etc/kubernetes/pki/etcd/node-node01-key.pem
unable to load certificate
140027733841824:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-node01.pem
notBefore=Sep  3 07:11:38 2019 GMT
notAfter=Aug 10 07:11:38 2119 GMT
/etc/kubernetes/pki/etcd/node-node02-key.pem
unable to load certificate
140489035491232:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-node02.pem
notBefore=Sep  3 07:11:38 2019 GMT
notAfter=Aug 10 07:11:38 2119 GMT

二 全部更新

# 备份相关配置文件和ssl证书
mkdir /etc/kubernetes.bak
cp -r /etc/kubernetes/pki/ /etc/kubernetes.bak
cp /etc/kubernetes/*.conf /etc/kubernetes.bak
cp -r /var/lib/etcd /var/lib/etcd.bak

# 上传已经修改好的kubeadm文件

# 全部替换证书，在所有master节点均操作
[root@master ~]#
[root@master ~]# for file in `./kubeadm alpha certs check-expiration --config=/etc/kubernetes/kubeadm-config.yaml | awk 'NR>1{print $1}'`;do ./kubeadm alpha certs renew $file;done
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
# 检测证书
[root@master ~]# ./kubeadm alpha certs check-expiration --config=/etc/kubernetes/kubeadm-config.yaml 
CERTIFICATE                EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
admin.conf                 Aug 13, 2120 02:52 UTC   99y             no      
apiserver                  Aug 13, 2120 02:52 UTC   99y             no      
apiserver-kubelet-client   Aug 13, 2120 02:52 UTC   99y             no      
controller-manager.conf    Aug 13, 2120 02:52 UTC   99y             no      
front-proxy-client         Aug 13, 2120 02:52 UTC   99y             no      
scheduler.conf             Aug 13, 2120 02:52 UTC   99y             no 

[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done     
/etc/kubernetes/pki/apiserver.crt
            Not Before: Sep  5 03:13:03 2020 GMT
            Not After : Aug 13 02:52:53 2120 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
            Not Before: Sep  5 03:13:03 2020 GMT
            Not After : Aug 13 02:52:54 2120 GMT
/etc/kubernetes/pki/ca.crt
            Not Before: Sep  5 03:13:03 2020 GMT
            Not After : Sep  3 03:13:03 2030 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
            Not Before: Sep  5 03:13:04 2020 GMT
            Not After : Sep  3 03:13:04 2030 GMT
/etc/kubernetes/pki/front-proxy-client.crt
            Not Before: Sep  5 03:13:04 2020 GMT
            Not After : Aug 13 02:52:56 2120 GMT
            
            
# 查看etcd证书
[root@master ~]# for i in `ls /etc/ssl/etcd/ssl/*.pem`;do echo $i;openssl x509 -in ${i} -noout -dates ;done                         
/etc/ssl/etcd/ssl/admin-master-key.pem
unable to load certificate
139850633099168:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/admin-master.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/ca-key.pem
unable to load certificate
140344601663392:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/ca.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/member-master-key.pem
unable to load certificate
140576245700512:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/member-master.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-master-key.pem
unable to load certificate
140665549584288:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-master.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node01-key.pem
unable to load certificate
140005720024992:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node01.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node02-key.pem
unable to load certificate
139824797161376:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node02.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node03-key.pem
unable to load certificate
139721134917536:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node03.pem
notBefore=Sep  5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT



# 重启控制层面
docker ps | grep -v pause | grep -E "etcd|scheduler|controller|apiserver" | awk '{print $1}' | awk '{print "docker","restart",$1}' | bash

#重启
systemctl restart kubelet
# 恢复
cp config config.bak
cp /etc/kubernetes/admin.conf config

注意

  • 编译kubernetes 1.15 对应使用go1.12.7，高版本会存在问题
  • 编译kubernetes 1.16 对应使用go1.13.4，高版本会存在问题
  • 由于ca证书为10年，所以十年后还是需要更新证书
[root@VM-16-3-centos kubernetes_bak]# git branch
  master
* release-1.16
[root@VM-16-3-centos kubernetes_bak]# go version
go version go1.13.6 linux/amd64
  • 证书重新生成后需要重启控制层面生效。

参考链接

【版权声明】本文为华为云社区用户原创内容，转载时必须标注文章的来源（华为云社区）、文章链接、文章作者等基本信息， 否则作者和本社区有权追究责任。如果您发现本社区中有涉嫌抄袭的内容，欢迎发送邮件进行举报，并提供相关证据，一经查实，本社区将立刻删除涉嫌侵权内容，举报邮箱： cloudbbs@huaweicloud.com
  • 点赞
  • 收藏
  • 关注作者

评论（0）

0/1000
抱歉，系统识别当前为高风险访问，暂不支持该操作

全部回复

上滑加载中

设置昵称

在此一键设置昵称，即可参与社区互动！

*长度不超过10个汉字或20个英文字符，设置后3个月内不可修改。

*长度不超过10个汉字或20个英文字符，设置后3个月内不可修改。

确认 取消

加入云驻计划，成为创作者

  • 华为云周边好礼
  • 免费体验产品
  • 特殊身份标识
  • 线下官方门票
  • 内部专家零距离
  • 与10000+优质创作者共同成长
立即加入