- 微信
- 微博
  
  分享文章到微博
- 复制链接
  
  复制链接到剪贴板

Kubernetes二进制手动安装

小源博客发表于 2023/04/11 11:44:19 2023/04/11

【摘要】配置epel和base源。（自行配置阿里云或清华等等）安装基本工具yum -y install wget net-tools bash-completion telnet tree nmap sysstat lrzsz dos2unix bind-utils安装bind（DNS服务,仅局域网生效）hdss7-11:yum -y install bind配置：vi /etc/named.c...

配置epel和base源。（自行配置阿里云或清华等等）

安装基本工具

yum -y install wget net-tools bash-completion telnet tree nmap sysstat lrzsz  dos2unix bind-utils

安装bind（DNS服务,仅局域网生效）

hdss7-11:
yum -y install bind

配置：
vi /etc/named.conf
listen-on port 53 { 10.4.7.11; };
listen-on-v6 port 53 { ::1; };  # 删掉
allow-query     { any; };
forwarders      { 10.4.7.254; };   # 上级DNS，此本机是网卡中配置的DNS
dnssec-enable no;
dnssec-validation no;

检查：named-checkconf （无信息则成功）

区域配置文件（定义了两个主DNS服务器）

vi /etc/named.rfc1912.zones
# 最后行加入
zone "host.com" IN {
        type master;
        file "host.com.zone";
        allow-update { 10.4.7.11; };
};

zone "od.com" IN {
        type master;
        file "od.com.zone";
        allow-update { 10.4.7.11; };
};

配置区域数据文件

vi /var/named/host.com.zone
$TTL 600        ; 10 minutes
@       IN SOA  dns.host.com. dnsadmin.host.com. (
                                2021031701   ; serial       # 01是当前年份时间的第一条记录
                                10800      ; refresh (3 hours)
                                900        ; retry (15 minutes)
                                604800     ; expire (1 week)
                                86400      ; minimum (1 day)
                                )
                         NS   dns.host.com.
$TTL 60 ; 1 minute
dns                A     10.4.7.11
HDSS7-11           A     10.4.7.11
HDSS7-12           A     10.4.7.12
HDSS7-21           A     10.4.7.21
HDSS7-22           A     10.4.7.22
HDSS7-200          A     10.4.7.200

配置业务域数据文件

vi /var/named/od.com.zone
$ORIGIN od.com.
$TTL 600        ; 10 minutes
@               IN SOA dns.od.com. dnsadmin.od.com. (
                               2021031701 ; serial
                               10800      ; refresh (3 hours)
                               900        ; retry (15 minutes)
                               604800     ; expire (1 week)
                               86400      ; minimum (1 day)
                               )
                               NS dns.od.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11

验证以上配置是否有问题
```
named-checkconf 
```

启动并验证

systemctl start named

netstat -luntp | grep 53

dig -t A hdss7-21.host.com @10.4.7.11 +short
10.4.7.21

修改其他主机DNS为 10.4.7.11，我们配置好的DNS服务(仅局域网)

在配置文件中添加配置

cat /etc/resolv.conf
# Generated by NetworkManager
search host.com    # 如果没有添加这行，通过短域名访问

到Windows操作
- 如果不行那么调低跃点数，还不行直接将本地网卡网关设置为 10.4.7.11即可

###

自签证书（重点，因为默认是1年，这里可以自定义）hdss7-200

需要三个文件

签证书**

mkdir certs

cd certs

[root@hdss7-200 certs]# cat ca-csr.json 
{
    "CN": "OldboyEdu",
    "hosts": [
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",          # 国家
            "ST": "beijing",    # 州，省
            "L": "beijing",     # 地区，城市
            "O": "od",          # 组织名称，公司名称
            "OU": "ops"         # 组织单位名称，公司部门
        }
    ],
    "ca": {
        "expiry": "175200h"
    }
}

证书签发

# 查看证书内容
cfssl gencert -initca ca-csr.json

# 生成证书
gencert -initca ca-csr.json | cfssl-json -bare ca
当前目录下会生产 ca.pem(私钥)  ca-key.pem(私钥)  ca.csr 文件

部署docker环境（hdss7-21、hdss7-22、hdss7-200）

# 安装docker依赖
yum install -y yum-utils device-mapper-persistent-data lvm2

yum -y install docker-ce (版本：docker-ce-19.03.1-3.el7.x86_64)

cat /etc/docker/daemon.json 
{
  "graph": "/data/docker",
  "storage-driver": "overlay2",
  "insecure-registries": ["registry.access.redhat.com","quay.io","harbor.od.com"],
  "registry-mirrors": ["https://deckaaa2.mirror.aliyuncs.com"],
  "bip": "172.7.21.1/24",
  "exec-opts": ["native.cgroupdriver=systemd"],
  "live-restore": true
}

systemctl restart docker

Harbor仓库搭建(hdss7-200)

tar xf harbor-offline-installer-v1.8.3.tgz -C /opt/

# 做软连接，便于升级
mv harbor/ harbor-v1.8.3
ln -s harbor-v1.8.3/ /opt/harbor

vi harbor.yml
hostname: harbor.od.com     # 域
port: 180                   # 修改端口
data_volume: /data/harbor
location: /data/harbor/logs

yum -y install docker-compose

sh /opt/harbor/install.sh

yum -y install nginx

cat /etc/nginx/conf.d/harbor.od.com.conf 
server {
    listen          80;
    server_name     harbor.od.com;

    client_max_body_size 1000m;

    location / {
        proxy_pass http://127.0.0.1:180;
    }
}

# 检查可用性
nginx -t
systemctl restart nginx

# 现在 harbor.od.com 不通，在hdss7-11 DNS服务器上做解析
hdss7-11上操作：
vi /var/named/od.com.zone
$ORIGIN od.com.
$TTL 600        ; 10 minutes
@               IN SOA dns.od.com. dnsadmin.od.com. (
                               2021031702 ; serial          # 加一，前滚一个数
                               10800      ; refresh (3 hours)
                               900        ; retry (15 minutes)
                               604800     ; expire (1 week)
                               86400      ; minimum (1 day)
                               )
                               NS dns.od.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11
harbor             A    10.4.7.200      # 添加到DNS

systemctl restart named
# 测试是否成功
dig -t A harbor.od.com +short
10.4.7.200

浏览器访问: harbor.od.com
用户：admin
密码：Harbor12345
新建项目--->public--->公开

docker pull nginx:1.7.9
docker tag 84581e99d807 harbor.od.com/public/nginx:v1.7.9
docker push harbor.od.com/public/nginx:v1.7.9   # push失败，因为没有登陆
docker login harbor.od.com     # 登陆
再push镜像即可

部署master节点服务
Kubernetes架构图

部署etcd集群

hdss-7-200: 
# 给etcd签发证书
 cat /opt/certs/ca-config.json 
{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}



cat /opt/certs/etcd-peer-csr.json 
{
    "CN": "k8s-etcd",
    "hosts": [
        "10.4.7.11",
        "10.4.7.12",
        "10.4.7.21",
        "10.4.7.22"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer

hdss7-12、hdss7-21、hdss7-22 部署etcd。注意hdss7-21和hdss7-22配置文件IP地址要换

 useradd -s /sbin/nologin -M etcd
 
 tar xvf etcd-v3.1.20-linux-amd64.tar.gz -C /opt/
 
 mv etcd-v3.1.20-linux-amd64/ etcd-v3.1.20
 
 ln -s /opt/etcd-v3.1.20/ /opt/etcd
 
 mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
 
 hdss7-200 certs]# scp ca.pem etcd-peer.pem etcd-peer-key.pem root@hdss7-12:/opt/etcd/certs
 
[root@hdss7-12 certs]# ll
-rw-r--r--. 1 root root 1346 Mar 18 23:14 ca.pem
-rw-------. 1 root root 1679 Mar 18 23:14 etcd-peer-key.pem     # 注意权限600
-rw-r--r--. 1 root root 1415 Mar 18 23:14 etcd-peer.pem
[root@hdss7-12 certs]# pwd
/opt/etcd/certs

cat /opt/etcd/etcd-server-startup.sh 
#!/bin/sh
./etcd --name etcd-server-7-12 \
    --data-dir /data/etcd/etcd-server \
    --listen-peer-urls https://10.4.7.12:2380 \
    --listen-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
    --quota-backend-bytes 8000000000 \
    --initial-advertise-peer-urls https://10.4.7.12:2380 \
    --advertise-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
    --initial-cluster  etcd-server-7-12=https://10.4.7.12:2380,etcd-server-7-21=https://10.4.7.21:2380,etcd-server-7-22=https://10.4.7.22:2380 \
    --ca-file ./certs/ca.pem \
    --cert-file ./certs/etcd-peer.pem \
    --key-file ./certs/etcd-peer-key.pem \
    --client-cert-auth  \
    --trusted-ca-file ./certs/ca.pem \
    --peer-ca-file ./certs/ca.pem \
    --peer-cert-file ./certs/etcd-peer.pem \
    --peer-key-file ./certs/etcd-peer-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file ./certs/ca.pem \
    --log-output stdout
       
chmod +x etcd-server-startup.sh
chown -R etcd.etcd /data/etcd/
chown -R etcd.etcd /data/logs/etcd-server/
chown -R etcd.etcd  /opt/etcd-v3.1.20

# 这个软件起到了服务挂掉再次让服务起来
yum -y install supervisor
systemctl start supervisord;systemctl enable supervisord

cat /etc/supervisord.d/etcd-server.ini 
[program:etcd-server-7-12]
command=/opt/etcd/etcd-server-startup.sh              ; the program (relative uses PATH, can take args)
numprocs=1                                            ; number of processes copies to start (def 1)
directory=/opt/etcd                                   ; directory to cwd to before exec (def no cwd)
autostart=true                                        ; start at supervisord start (default: true)
autorestart=true                                      ; retstart at unexpected quit (default: true)
startsecs=30                                          ; number of secs prog must stay running (def. 1)
startretries=3                                        ; max # of serial start failures (default 3)
exitcodes=0,2                                         ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                       ; signal used to kill process (default TERM)
stopwaitsecs=10                                       ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                             ; setuid to this UNIX account to run the program
redirect_stderr=true                                  ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                          ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5                              ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                           ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                           ; emit events on stdout writes (default false)

~]# supervisorctl update
etcd-server-7-12: added process group
~]# supervisorctl status
etcd-server-7-12                 RUNNING   pid 24493, uptime 0:03:14

# 集群健康检查
/opt/etcd/etcdctl cluster-health
member 988139385f78284 is healthy: got healthy result from http://127.0.0.1:2379
member 5a0ef2a004fc4349 is healthy: got healthy result from http://127.0.0.1:2379
member f4a0cb0a765574a8 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy

kube-apiserver部署（hdss7-21、hdss7-22）

下载：https://github.com/kubernetes/kubernetes 使用科学下载方式
- https://github.com/kubernetes/kubernetes/tags 进入tag界面
- https://github.com/kubernetes/kubernetes/releases/tag/v1.15.2 选择需要下载的版本

tar xvf kubernetes-server-linux-amd64-v1.15.2.tar.gz -C /opt/

cd /opt

mv kubernetes/ kubernetes-v1.15.2

ln -s /opt/kubernetes-v1.15.2/ /opt/kubernetes

rm -rf kubernetes-src.tar.gz 删除源码包

[root@hdss7-21 bin]# pwd
/opt/kubernetes/server/bin
rm -rf *.tar  # 不需要docker镜像安装
rm -rf *_tag

mkdir /opt/kubernetes/server/bin/certs

签发client证书（etcd集群与apiserver通信的证书）

vi /opt/certs/client-csr.json
{
    "CN": "k8s-node",
    "hosts": [
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json |cfssl-json -bare client

签发apiserver证书

# 10.4.7.10 是vip地址
vi /opt/certs/apiserver-csr.json
{
    "CN": "k8s-apiserver",
    "hosts": [
        "127.0.0.1",
        "192.168.0.1",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local",
        "10.4.7.10",
        "10.4.7.21",
        "10.4.7.22",
        "10.4.7.23"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssl-json -bare apiserver

发放证书

[root@hdss7-200 certs]# scp ca.pem ca-key.pem client.pem client-key.pem apiserver.pem apiserver-key.pem hdss7-21:/opt/kubernetes/server/bin/certs

配置启动文件

mkdir /opt/kubernetes/server/bin/conf

cd /opt/kubernetes/server/bin/conf

vi audit.yaml
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
  - "RequestReceived"
rules:
  # Log pod changes at RequestResponse level
  - level: RequestResponse
    resources:
    - group: ""
      # Resource "pods" doesn't match requests to any subresource of pods,
      # which is consistent with the RBAC policy.
      resources: ["pods"]
  # Log "pods/log", "pods/status" at Metadata level
  - level: Metadata
    resources:
    - group: ""
      resources: ["pods/log", "pods/status"]

  # Don't log requests to a configmap called "controller-leader"
  - level: None
    resources:
    - group: ""
      resources: ["configmaps"]
      resourceNames: ["controller-leader"]

  # Don't log watch requests by the "system:kube-proxy" on endpoints or services
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
    - group: "" # core API group
      resources: ["endpoints", "services"]

  # Don't log authenticated requests to certain non-resource URL paths.
  - level: None
    userGroups: ["system:authenticated"]
    nonResourceURLs:
    - "/api*" # Wildcard matching.
    - "/version"

  # Log the request body of configmap changes in kube-system.
  - level: Request
    resources:
    - group: "" # core API group
      resources: ["configmaps"]
    # This rule only applies to resources in the "kube-system" namespace.
    # The empty string "" can be used to select non-namespaced resources.
    namespaces: ["kube-system"]

  # Log configmap and secret changes in all other namespaces at the Metadata level.
  - level: Metadata
    resources:
    - group: "" # core API group
      resources: ["secrets", "configmaps"]

  # Log all other resources in core and extensions at the Request level.
  - level: Request
    resources:
    - group: "" # core API group
    - group: "extensions" # Version of group should NOT be included.

  # A catch-all rule to log all other requests at the Metadata level.
  - level: Metadata
    # Long-running requests like watches that fall under this rule will not
    # generate an audit event in RequestReceived.
    omitStages:
      - "RequestReceived"
      
cd ..
vi kube-apiserver.sh
#!/bin/bash
/opt/kubernetes/server/bin/kube-apiserver \
    --apiserver-count 2 \
    --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
    --audit-policy-file ./conf/audit.yaml \
    --authorization-mode RBAC \
    --client-ca-file ./certs/ca.pem \
    --requestheader-client-ca-file ./certs/ca.pem \
    --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
    --etcd-cafile ./certs/ca.pem \
    --etcd-certfile ./certs/client.pem \
    --etcd-keyfile ./certs/client-key.pem \
    --etcd-servers https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
    --service-account-key-file ./certs/ca-key.pem \
    --service-cluster-ip-range 192.168.0.0/16 \
    --service-node-port-range 3000-29999 \
    --target-ram-mb=1024 \
    --kubelet-client-certificate ./certs/client.pem \
    --kubelet-client-key ./certs/client-key.pem \
    --log-dir  /data/logs/kubernetes/kube-apiserver \
    --tls-cert-file ./certs/apiserver.pem \
    --tls-private-key-file ./certs/apiserver-key.pem \
    --v 2

bin]# chmod +x kube-apiserver.sh

supervisor托管程序（会自动重启）

vi /etc/supervisord.d/kube-apiserver.ini
[program:kube-apiserver-7-21]
command=/opt/kubernetes/server/bin/kube-apiserver.sh
numprocs=1
directory=/opt/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

mkdir -p /data/logs/kubernetes/kube-apiserver/

supervisorctl update

@hdss7-21 bin]# netstat -lntup|grep kube-api
tcp        0      0 127.0.0.1:8080          0.0.0.0:*               LISTEN      57215/kube-apiserve 
tcp6       0      0 :::6443                 :::*                    LISTEN      57215/kube-apiserve

配置nginx4层反向代理（hdss7-11、hdss7-12）

# 安装nginx
yum -y install nginx

vi /etc/nginx/nginx.conf  (配置在配置文件中最后，http中是八层代理，这里需要配四层)
stream {
    upstream kube-apiserver {
        server 10.4.7.21:6443     max_fails=3 fail_timeout=30s;
        server 10.4.7.22:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

# 检查
nginx -t
# 启动
systemctl start nginx;systemctl enable nginx

配置keepalived做高可用

# 两个节点
yum -y install keepalived

# 两个节点
vi /etc/keepalived/check_port.sh
#!/bin/bash
if [ $# -eq 1 ] && [[ $1 =~ ^[0-9]+ ]];then
    [ $(netstat -lntp|grep ":$1 " |wc -l) -eq 0 ] && echo "[ERROR] nginx may be not running!" && exit 1 || exit 0
else
    echo "[ERROR] need one port!"
    exit 1
fi

chmod +x /etc/keepalived/check_port.sh

# 删除原有配置
# 主：
[root@hdss7-11 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
   router_id 10.4.7.11
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface ens32
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.4.7.11
    nopreempt               # 非抢占式，在生产上vip地址是绝对不可以乱动的

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        10.4.7.10
    }
}

# 从：
[root@hdss7-12 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
  router_id 10.4.7.12
}
vrrp_script chk_nginx {
  script "/etc/keepalived/check_port.sh 7443"
  interval 2
  weight -20
}
vrrp_instance VI_1 {
  state BACKUP
  interface ens32
  virtual_router_id 251
  mcast_src_ip 10.4.7.12
  priority 90
  advert_int 1
  authentication {
    auth_type PASS
    auth_pass 11111111
  }
  track_script {
    chk_nginx
  }
  virtual_ipaddress {
    10.4.7.10
  }
}

# 两个节点
systemctl start keepalived;systemctl enable keepalived

# 由于配置了非抢占式，所以VIP地址是不会回来的，如果需要将VIP地址回来，那么要检查本地服务过后再配置
[root@hdss7-11 ~]# netstat -lntup|grep 7443
tcp        0      0 0.0.0.0:7443            0.0.0.0:*               LISTEN      29058/nginx: master

# 两个节点重启服务，即可将IP回归
systemctl restart keepalived

配置controller-manager和scheduler（hdss7-21、hdss7-22）

controller-manager

cat /opt/kubernetes/server/bin/kube-controller-manager.sh 
#!/bin/bash
./kube-controller-manager \
    --cluster-cidr 172.7.0.0/16 \
    --leader-elect true \
    --log-dir /data/logs/kubernetes/kube-controller-manager \
    --master http://127.0.0.1:8080 \
    --service-account-private-key-file ./certs/ca-key.pem \
    --service-cluster-ip-range 192.168.0.0/16 \
    --root-ca-file ./certs/ca.pem \
    --v 2

chmod +x kube-controller-manager.sh
mkdir -p /data/logs/kubernetes/kube-controller-manager

[root@hdss7-21 bin]# cat /etc/supervisord.d/kube-controller-manager.ini  (注意序号:22)
[program:kube-controller-manager-7-21]
command=/opt/kubernetes/server/bin/kube-controller-manager.sh
numprocs=1                                                                        ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin                                              ; directory to cwd to before exec (def no cwd)
autostart=true                                                                    ; start at supervisord start (default: true)
autorestart=true                                                                  ; retstart at unexpected quit (default: true)
startsecs=30                                                                      ; number of secs prog must stay running (def. 1)
startretries=3                                                                    ; max # of serial start failures (default 3)
exitcodes=0,2                                                                     ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                                   ; signal used to kill process (default TERM)
stopwaitsecs=10                                                                   ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                                         ; setuid to this UNIX account to run the program
redirect_stderr=true                                                              ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controller.stdout.log  ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                                      ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                                          ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                                       ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                                       ; emit events on stdout writes (default false)

supervisorctl update

scheduler

cat /opt/kubernetes/server/bin/kube-scheduler.sh 
#!/bin/sh
./kube-scheduler \
    --leader-elect  \
    --log-dir /data/logs/kubernetes/kube-scheduler \
    --master http://127.0.0.1:8080 \
    --v 2
    
mkdir /data/logs/kubernetes/kube-scheduler -p
chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh 
 
cat /etc/supervisord.d/kube-scheduler.ini
[program:kube-scheduler-7-21]
command=/opt/kubernetes/server/bin/kube-scheduler.sh
numprocs=1                                                               
directory=/opt/kubernetes/server/bin                                     
autostart=true                                                           
autorestart=true                                                         
startsecs=30                                                             
startretries=3                                                           
exitcodes=0,2                                                            
stopsignal=QUIT                                                          
stopwaitsecs=10                                                          
user=root                                                                
redirect_stderr=true                                                     
stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log 
stdout_logfile_maxbytes=64MB                                             
stdout_logfile_backups=4                                                 
stdout_capture_maxbytes=1MB                                              
stdout_events_enabled=false

supervisorctl update

# 做软链接即可使用命令（两节点）
ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl

# 验证集群
# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok                   
controller-manager   Healthy   ok                   
etcd-0               Healthy   {"health": "true"}   
etcd-2               Healthy   {"health": "true"}   
etcd-1               Healthy   {"health": "true"}

部署计算节点kubelet服务（hdss7-21、hdss7-22）master

签发kubelet证书（签发证书都是在hdss7-200）

# 将所有可能的kubelet机器IP添加到hosts中   （如果需要更换证书，那么将新的证书生成替换老的，再重启即可）
200 certs]# cat kubelet-csr.json 
{
    "CN": "k8s-kubelet",
    "hosts": [
    "127.0.0.1",
    "10.4.7.10",
    "10.4.7.21",
    "10.4.7.22",
    "10.4.7.23",
    "10.4.7.24",
    "10.4.7.25",
    "10.4.7.26",
    "10.4.7.27",
    "10.4.7.28"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

# 生成kubelet证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet

[root@hdss7-200 certs]# pwd
/opt/certs
scp kubelet.pem kubelet-key.pem hdss7-21:/opt/kubernetes/server/bin/certs
scp kubelet.pem kubelet-key.pem hdss7-22:/opt/kubernetes/server/bin/certs

分发证书（bdss7-21、hdss7-22）

# hdss7-21

conf]# pwd
/opt/kubernetes/server/bin/conf/

cd /opt/kubernetes/server/bin/conf/

# set-cluster  创建需要连接的集群信息，可以创建多个k8s集群信息
kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/certs/ca.pem \
--embed-certs=true \
--server=https://10.4.7.10:7443 \
--kubeconfig=kubelet.kubeconfig

# set-credentials  创建用户账号，即用户登陆使用的客户端私有和证书，可以创建多个证书
kubectl config set-credentials k8s-node \
--client-certificate=/opt/kubernetes/server/bin/certs/client.pem \
--client-key=/opt/kubernetes/server/bin/certs/client-key.pem \
--embed-certs=true \
--kubeconfig=kubelet.kubeconfig

# set-context  设置context，即确定账号和集群对应关系
kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=k8s-node \
--kubeconfig=kubelet.kubeconfig

# use-context  设置当前使用哪个context
kubectl config use-context myk8s-context --kubeconfig=kubelet.kubeconfig

授权k8s-node用户（只需在一台节点执行）

# 授权 k8s-node 用户绑定集群角色 system:node ，让 k8s-node 成为具备运算节点的权限
conf]# cat /opt/kubernetes/server/bin/conf/k8s-node.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: k8s-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: k8s-node

# 创建
kubectl create -f k8s-node.yaml
# 查看
kubectl get ClusterRoleBinding k8s-node -oyaml

hdss7-22

# 因为都是用的同一个证书，所以一个节点完成拷贝过来即可
scp hdss7-21:/opt/kubernetes/server/bin/conf/kubelet.kubeconfig .

# k8s-node.yaml 不需要，创建完成存到etcd数据库中，所有节点都能使用

准备pause基础镜像（hdss7-200）

# 做kubelet的第一次工作，类似于初始化
docker image pull kubernetes/pause

准备kubelet启动脚本（hdss7-21、hdss7-22）

cat /opt/kubernetes/server/bin/kubelet.sh 
#!/bin/bash
./kubelet \
    --anonymous-auth=false \
    --cgroup-driver systemd \
    --cluster-dns 192.168.0.2 \
    --cluster-domain cluster.local \
    --runtime-cgroups=/systemd/system.slice \
    --kubelet-cgroups=/systemd/system.slice \
    --fail-swap-on="false" \
    --client-ca-file ./certs/ca.pem \
    --tls-cert-file ./certs/kubelet.pem \
    --tls-private-key-file ./certs/kubelet-key.pem \
    --hostname-override hdss7-21.host.com \
    --image-gc-high-threshold 20 \
    --image-gc-low-threshold 10 \
    --kubeconfig ./conf/kubelet.kubeconfig \
    --log-dir /data/logs/kubernetes/kube-kubelet \
    --pod-infra-container-image harbor.od.com/public/pause:latest \
    --root-dir /data/kubelet
    
chmod +x kubelet.sh 
mkdir -p /data/logs/kubernetes/kube-kubelet /data/kubelet

cat /etc/supervisord.d/kube-kubelet.ini 
[program:kube-kubelet-7-21]
command=/opt/kubernetes/server/bin/kubelet.sh
numprocs=1
directory=/opt/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

supervisorctl update

kubectl get node
NAME                STATUS   ROLES    AGE   VERSION
hdss7-21.host.com   Ready    <none>   30m   v1.15.2
hdss7-22.host.com   Ready    <none>   37m   v1.15.2

kubectl label node hdss7-21.host.com node-role.kubernetes.io/master=
]# kubectl get node
NAME                STATUS   ROLES    AGE   VERSION
hdss7-21.host.com   Ready    master   36m   v1.15.2
hdss7-22.host.com   Ready    <none>   44m   v1.15.2

kubectl label node hdss7-21.host.com node-role.kubernetes.io/master=
kubectl label node hdss7-21.host.com node-role.kubernetes.io/node=
kubectl label node hdss7-22.host.com node-role.kubernetes.io/master=
kubectl label node hdss7-22.host.com node-role.kubernetes.io/node=
]# kubectl get node
NAME                STATUS   ROLES         AGE   VERSION
hdss7-21.host.com   Ready    master,node   42m   v1.15.2
hdss7-22.host.com   Ready    master,node   50m   v1.15.2

# 注意配置hdss7-22名称与IP不一样

部署kube-proxy（hdss7-21、hdss7-22）

签发kube-proxy证书

cat /opt/certs/kube-proxy-csr.json 
{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

# 因为kube-proxy使用的用户是kube-proxy，不能使用client证书，必须要重新签发自己的证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json |cfssl-json -bare kube-proxy-client

[root@hdss7-200 certs]# scp kube-proxy-client.pem kube-proxy-client-key.pem hdss7-21:/opt/kubernetes/server/bin/certs/
[root@hdss7-200 certs]# scp kube-proxy-client.pem kube-proxy-client-key.pem hdss7-22:/opt/kubernetes/server/bin/certs/

分发证书

# hdss7-21做即可，生成的文件复制到hdss7-22
cd /opt/kubernetes/server/bin/conf

kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/certs/ca.pem \
--embed-certs=true \
--server=https://10.4.7.10:7443 \
--kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/server/bin/certs/kube-proxy-client.pem \
--client-key=/opt/kubernetes/server/bin/certs/kube-proxy-client-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

kubectl config use-context myk8s-context --kubeconfig=/opt/kubernetes/server/bin/conf/kube-proxy.kubeconfig

# hdss7-22以下操作即可完成
[root@hdss7-22 conf]# scp hdss7-21:/opt/kubernetes/server/bin/conf/kube-proxy.kubeconfig .

创建启动环境和启动脚本（两边都要操作）

# 加载ipvs模块
kube-proxy 共有3种流量调度模式，分别是 namespace，iptables，ipvs，其中ipvs性能最好。
[root@hdss7-21 ~]# for i in $(ls /usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs|grep -o "^[^.]*");do echo $i; /sbin/modinfo -F filename $i >/dev/null 2>&1 && /sbin/modprobe $i;done
[root@hdss7-21 ~]# lsmod | grep ip_vs  # 查看ipvs模块

cat /opt/kubernetes/server/bin/kube-proxy.sh 
#!/bin/bash
./kube-proxy \
  --cluster-cidr 172.7.0.0/16 \
  --hostname-override hdss7-21.host.com \
  --proxy-mode=ipvs \
  --ipvs-scheduler=nq \
  --kubeconfig ./conf/kube-proxy.kubeconfig
  
chmod +x kube-proxy.sh
mkdir -p /data/logs/kubernetes/kube-proxy

vi /etc/supervisord.d/kube-proxy.ini
[program:kube-proxy-7-21]
command=/opt/kubernetes/server/bin/kube-proxy.sh
numprocs=1                                                      
directory=/opt/kubernetes/server/bin                            
autostart=true                                                  
autorestart=true                                                
startsecs=30                                                    
startretries=3                                                  
exitcodes=0,2                                                   
stopsignal=QUIT                                                 
stopwaitsecs=10                                                 
user=root                                                       
redirect_stderr=true                                            
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log
stdout_logfile_maxbytes=64MB                                    
stdout_logfile_backups=5                                       
stdout_capture_maxbytes=1MB                                     
stdout_events_enabled=false

supervisorctl update

yum -y install ipvsadm

ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.1:443 nq
  -> 10.4.7.21:6443               Masq    1      0          0         
  -> 10.4.7.22:6443               Masq    1      0          0  

kubectl get svc
NAME         TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   192.168.0.1   <none>        443/TCP   21h

验证kubernetes集群

# 任意计算节点创建
cat nginx-ds.yaml 
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nginx-ds
spec:
  template:
    metadata:
      labels:
        app: nginx-ds
    spec:
      containers:
      - name: my-nginx
        image: harbor.od.com/public/nginx:v1.7.9        # 注意这个镜像仓库有没有，没有就先创建镜像
        ports:
        - containerPort: 80

kubectl create -f nginx-ds.yaml 

[root@hdss7-21 ~]# kubectl get pod -owide
NAME             READY   STATUS    RESTARTS   AGE   IP           NODE              
SS GATES
nginx-ds-n97kg   1/1     Running   0          64s   172.7.22.2   hdss7-22.host.com 
nginx-ds-zhgz5   1/1     Running   0          64s   172.7.21.2   hdss7-21.host.com 

curl 172.7.21.2  # 能通

curl 172.7.22.2  # 不能通，docker跨主机不能通信，需要安装其它软件（flannel）实现互通

[root@hdss7-21 ~]# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
controller-manager   Healthy   ok                   
scheduler            Healthy   ok                   
etcd-2               Healthy   {"health": "true"}   
etcd-0               Healthy   {"health": "true"}   
etcd-1               Healthy   {"health": "true"}   
[root@hdss7-21 ~]# kubectl get node
NAME                STATUS   ROLES         AGE    VERSION
hdss7-21.host.com   Ready    master,node   153m   v1.15.2
hdss7-22.host.com   Ready    master,node   161m   v1.15.2
[root@hdss7-21 ~]# kubectl get pods
NAME             READY   STATUS    RESTARTS   AGE
nginx-ds-n97kg   1/1     Running   0          8m5s
nginx-ds-zhgz5   1/1     Running   0          8m5s

关于cfssl工具

cfssl：证书签发的主要工具
cfssl-json：将cfssl生成的证书（json格式）变为文件承载式证书

cfssl-certinfo：验证证书信息

cfssl-certinfo -cert apiserver.pem  # 可以查看证书的详细信息，如证书有效期

cfssl-certinfo -domain www.baidu.com  # 其它网站也行

关于kubeconfig文件：
- 这是一个k8s用户的配置文件
- 它里面含有k8s证书信息
- 证书过期或更换，需要同步替换该文件

kubectl命令行工具使用

陈述式管理方法 - 主要依赖命令行CLI工具进行管理（命令行操作，如create、delete）

# 查看名称空间
kubectl get namespace(ns)

# 查看一个名称空间中所以资源
kubectl -n default get all

# 创建名称空间
kubectl create namespace(ns) app

# 删除名称空间
kubectl delete namespace app

==========================================Deployment=============================================
# 管理Deployment资源(pod控制器)
kubectl -n kube-public create deployment nginx-dp --image=harbor.od.com/public/nginx:v1.8.3

# 查看deployment
kubectl -n kube-public get deployment

# 查看deployment详细信息
kubectl -n kube-public describe deploy nginx-dp

# 进入pod资源(进入容器)
kubectl exec -it nginx-dp-58dcf4d9d5-5vhtc bash

# 删除pod
kubectl -n kube-public delete pod nginx-dp-58dcf4d9d5-5vhtc 后面加上:[--force --grace-period=0] 强制删除

# 删除deployment(pod资源管理器)
kubectl -n kube-public delete deployment nginx-dp

Deployment 部署的副本 Pod 会分布在各个 Node 上，每个 Node 都可能运行好几个副本。DaemonSet 的不同之处在于：每个 Node 上最多只能运行一个副本。

=============================================Service===============================================
# 创建service并暴露80端口
kubectl -n kube-public expose deployment nginx-dp --port=80

# 扩容两个pod
kubectl -n kube-public scale deployment nginx-dp --replicas=2

# 查看service详细信息
 kubectl -n kube-public describe svc nginx-dp
 
命令大全：http://docs.kubernetes.org.cn/

声明式管理方式 - 主要依赖统一资源配置清单（manifest）进行管理（yaml\json）

# 获取pod资源配置清单
kubectl -n kube-public get pod nginx-dp-58dcf4d9d5-mt752 -oyaml

# 获取service资源配置清单
kubectl -n kube-public get svc -oyaml

# 查看metadata中的使用帮助
kubectl explain service.metadata

# 离线式修改
kubectl apply -f xxx.yaml

# 在线式修改
kubctl edit svc xxx

# 将已有配置清单重定向到配置文件
kubectl -n kube-public get svc -oyaml > nginx-dp-svc.yaml

# 根据配置文件删除
kubectl delete -f xxx.yaml

GUI管理方法 - 主要依赖图形化操作界面（web页面）进行管理

Flannel网络插件（hdss7-21、hdss7-22 不同主机容器间的互相通信）

github地址：https://github.com/coreos/flannel/releases

mkdir /opt/flannel-v0.11.0

tar xvf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0/

ln -s /opt/flannel-v0.11.0/ /opt/flannel

mkdir /opt/flannel/certs

# 拷贝证书
scp ca.pem client.pem client-key.pem hdss7-21:/opt/flannel/certs
scp ca.pem client.pem client-key.pem hdss7-22:/opt/flannel/certs

# 创建子网信息，7-22的subnet需要修改(FLANNEL_SUBNET=172.7.22.1/24)
vi /opt/flannel/subnet.env 
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.21.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

# 插件启动脚本，注意hdss7-22不一样(--public-ip=10.4.7.22)
vi /opt/flannel/flanneld.sh 
#!/bin/bash
./flanneld \
    --public-ip=10.4.7.21 \
    --etcd-endpoints=https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
    --etcd-keyfile=./certs/client-key.pem \
    --etcd-certfile=./certs/client.pem \
    --etcd-cafile=./certs/ca.pem \
    --iface=ens32 \
    --subnet-file=./subnet.env \
    --healthz-port=2401

========================= etcd集群中任意一台执行即可（集群查看: ./etcdctl member list）=====================
cd /opt/etcd
./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
./etcdctl get /coreos.com/network/config
======================================================================================================

# 注意hdss7-22不一样([program:flanneld-7-22])
vi /etc/supervisord.d/flannel.ini 
[program:flanneld-7-21]
command=/opt/flannel/flanneld.sh                             ; the program (relative uses PATH, can take args)
numprocs=1                                                   ; number of processes copies to start (def 1)
directory=/opt/flannel                                       ; directory to cwd to before exec (def no cwd)
autostart=true                                               ; start at supervisord start (default: true)
autorestart=true                                             ; retstart at unexpected quit (default: true)
startsecs=30                                                 ; number of secs prog must stay running (def. 1)
startretries=3                                               ; max # of serial start failures (default 3)
exitcodes=0,2                                                ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                              ; signal used to kill process (default TERM)
stopwaitsecs=10                                              ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                    ; setuid to this UNIX account to run the program
redirect_stderr=true                                         ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log       ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                 ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5                                     ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                  ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                  ; emit events on stdout writes (default false)

mkdir -p  /data/logs/flanneld/

supervisorctl update

# 没有报错等一下即可

=====================================额外网络补充，不用做，实验可以===============================
原理其实就是在各个主机上添加了对方IP和docker的静态路由
# hdss7-22添加静态路由
# hdss7-22加上hdss7-21的docker网段和hdss7-21的IP地址作为网关
route add -net 172.17.21.0/24 gw 10.4.7.21 dev ens32
# 到hdss7-21添加iptables规则
iptables -t filter -I FORWARD -d 172.17.21.0/24 -j ACCEPT
# 删除是
iptables -t filter -D FORWARD -d 172.17.21.0/24 -j ACCEPT

# hdss7-21添加静态路由
# hdss7-21加上hdss7-22的docker网段和hdss7-22的IP地址作为网关
router add -net 172.17.22.0/24 gw 10.4.7.22 dev ens32
# 到hdss7-22添加iptables规则
iptables -t filter -I FORWARD -d 172.17.22.0/24 -j ACCEPT

-t 指定类型
-I 写入
-D 删除
-d 目标地址
-j 指定规则

# 删除静态路由
route del -net 172.17.21.0/24 gw 10.4.7.21

# vxlan模型
'{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN"}}'

补充：虽然静态路由添加达到了跨容器之间的访问，但是效率低，还有走真实地址，需要优化，做snet转换，如下

解决pod间IP透传问题，实现IP该走容器走容器，该走主机走主机，默认是绕一圈到真实地址，效率低，需要优化规则，添加snet转换（两边操作）。

# 问题点：
~]# kubectl logs -f nginx-ds-dk9t8
10.4.7.21 - - [22/Mar/2021:03:17:06 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"

# 安装软件
yum -y install iptables-services

# 启动
systemctl start iptables;systemctl enable iptables

# 优化iptables规则
~]# iptables-save |grep -i postrouting
# 优化目标 
-A POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE 

# 删除目标规则
iptables -t nat -D POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE

# 再重新写入优化后的规则
iptables -t nat -I POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
# 解释:  网络走172.7.21.0/24，不走172.7.0.0/16，不是docker0这个设备出网，才做 MASQUERADE 转换

查看： iptables-save |grep -i reject
# 删除原有规则
iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited

# 保存iptables规则
iptables-save > /etc/sysconfig/iptables

# 主机和容器访问一个容器输出结果
~]# kubectl logs -f nginx-ds-pl9km
10.4.7.22 - - [22/Mar/2021:13:52:11 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
172.7.22.2 - - [22/Mar/2021:13:52:37 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.64.0" "-"

CoreDNS(服务发现、hdss7-200)

[root@hdss7-200 ~]# vi /etc/nginx/conf.d/k8s-yaml.od.com.conf
server {
    listen       80;
    server_name  k8s-yaml.od.com;

    location / {
        autoindex on;
        default_type text/plain;
        root /data/k8s-yaml;
    }
}

[root@hdss7-200 ~]# mkdir /data/k8s-yaml
[root@hdss7-200 ~]# nginx -t
[root@hdss7-200 ~]# systemctl restart nginx

配置DNS解析（hdss7-11）

vi /var/named/od.com.zone 
2021031703 ; serial    # 前滚一个数，不是必要的，但是方便记录
k8s-yaml           A    10.4.7.200    # 最后行添加

systemctl restart named

[root@hdss7-11 ~]# dig -t A k8s-yaml.od.com @10.4.7.11 +short
10.4.7.200

部署CoreDNS

# 准备镜像并推入到仓库 hdss7-200
docker load -i coredns-v1.6.1.tar
docker tag c0f6e815079e harbor.od.com/public/coredns:v1.6.1
docker push harbor.od.com/public/coredns:v1.6.1

# 配置资源配置清单，将清单文件放入到 hdss7-200: /data/k8s-yaml/coredns
vi /data/k8s-yaml/coredns/rabc.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
  labels:
      kubernetes.io/cluster-service: "true"
      addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: Reconcile
  name: system:coredns
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  - namespaces
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: EnsureExists
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
 
 
 
vi /data/k8s-yaml/coredns/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        log
        health
        ready
        kubernetes cluster.local 192.168.0.0/16
        forward . 10.4.7.11
        cache 30
        loop
        reload
        loadbalance
    }
    
    
vi /data/k8s-yaml/coredns/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: coredns
    kubernetes.io/name: "CoreDNS"
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: coredns
  template:
    metadata:
      labels:
        k8s-app: coredns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      containers:
      - name: coredns
        image: harbor.od.com/public/coredns:v1.6.1
        args:
        - -conf
        - /etc/coredns/Corefile
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
              
              
              
vi /data/k8s-yaml/coredns/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: coredns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
spec:
  selector:
    k8s-app: coredns
  clusterIP: 192.168.0.2
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
  - name: metrics
    port: 9153
    protocol: TCP

利用配置清单部署coredns（hdss7-21）

kubectl apply -f http://k8s-yaml.od.com/coredns/rabc.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/configmap.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/deployment.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/service.yaml

# 查看
kubectl -n kube-system get all
NAME                           READY   STATUS    RESTARTS   AGE
pod/coredns-6b6c4f9648-ptnc7   1/1     Running   0          82s

NAME              TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)                  AGE
service/coredns   ClusterIP   192.168.0.2   <none>        53/UDP,53/TCP,9153/TCP   46s

NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/coredns   1/1     1            1           83s

NAME                                 DESIRED   CURRENT   READY   AGE
replicaset.apps/coredns-6b6c4f9648   1         1         1       82s

# nginx-dp是deployment里的svc，没有需要创建
]# dig -t A nginx-dp.kube-public.svc.cluster.local. @192.168.0.2 +short
192.168.162.208

# 容器中curl
/# curl nginx-dp.kube-public

# 因为内部是做了解析，达成了服务发现
# cat /etc/resolv.conf 
nameserver 192.168.0.2
search default.svc.cluster.local svc.cluster.local cluster.local host.com
options ndots:5

# 而且只在集群内部生效(容器内)
curl nginx-dp.kube-public.svc.cluster.local
curl: (6) Could not resolve host: nginx-dp.kube-public.svc.cluster.local; Unknown error

Ingress-Controller（k8s服务暴露）

配置traefik资源清单（部署traefik）

# hdss7-200准备镜像和资源配置清单
cd /data/k8s-yaml/coredns
docker load -i traefik-v1.7.2.tar 
docker tag add5fac61ae5 harbor.od.com/public/traefik:v1.7.2
docker push  harbor.od.com/public/traefik:v1.7.2

cd /data/k8s-yaml/coredns/traefik

cat rbac.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: traefik-ingress-controller
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: traefik-ingress-controller
rules:
  - apiGroups:
      - ""
    resources:
      - services
      - endpoints
      - secrets
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik-ingress-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
  name: traefik-ingress-controller
  namespace: kube-system
  


 cat daemonset.yaml 
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: traefik-ingress
  namespace: kube-system
  labels:
    k8s-app: traefik-ingress
spec:
  template:
    metadata:
      labels:
        k8s-app: traefik-ingress
        name: traefik-ingress
    spec:
      serviceAccountName: traefik-ingress-controller
      terminationGracePeriodSeconds: 60
      containers:
      - image: harbor.od.com/public/traefik:v1.7.2
        name: traefik-ingress
        ports:
        - name: controller
          containerPort: 80
          hostPort: 81
        - name: admin-web
          containerPort: 8080
        securityContext:
          capabilities:
            drop:
            - ALL
            add:
            - NET_BIND_SERVICE
        args:
        - --api
        - --kubernetes
        - --logLevel=INFO
        - --insecureskipverify=true
        - --kubernetes.endpoint=https://10.4.7.10:7443
        - --accesslog
        - --accesslog.filepath=/var/log/traefik_access.log
        - --traefiklog
        - --traefiklog.filepath=/var/log/traefik.log
        - --metrics.prometheus
        
        
        
cat service.yaml 
kind: Service
apiVersion: v1
metadata:
  name: traefik-ingress-service
  namespace: kube-system
spec:
  selector:
    k8s-app: traefik-ingress
  ports:
    - protocol: TCP
      port: 80
      name: controller
    - protocol: TCP
      port: 8080
      name: admin-web
      
      
      
cat ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: traefik-web-ui
  namespace: kube-system
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  rules:
  - host: traefik.od.com
    http:
      paths:
        - path: /
          backend:
            serviceName: traefik-ingress-service
            servicePort: 8080
          
          
# 容易计算节点执行（hdss7-21、hdss7-22）
kubectl apply -f http://k8s-yaml.od.com/coredns/traefik/rbac.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/traefik/daemonset.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/traefik/service.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/traefik/ingress.yaml

[root@hdss7-22 ~]# kubectl -n kube-system get pod
NAME                       READY   STATUS    RESTARTS   AGE
coredns-6b6c4f9648-ptnc7   1/1     Running   0          22h
traefik-ingress-z42hs      1/1     Running   0          27m
traefik-ingress-z52pq      1/1     Running   0          27m

# 如果起不来，那么重启docker

配置外部nginx负载均衡（hdss7-11、hdss7-12）

# 在hdss7-11,hdss7-12 配置nginx L7转发(两边都需要配)
vi /etc/nginx/conf.d/od.com.conf
upstream default_backend_traefik {
    server 10.4.7.21:81    max_fails=3 fail_timeout=10s;
    server 10.4.7.22:81    max_fails=3 fail_timeout=10s;
}
server {
    server_name *.od.com;
  
    location / {
        proxy_pass http://default_backend_traefik;
        proxy_set_header Host       $http_host;
        proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
    }
}


# 检查并重启
nginx -t
nginx -s reload


# hdss7-11 配置DNS解析
vi /var/named/od.com.zone
2021031704 ; serial # 前滚一个数
traefik            A    10.4.7.10   # 最后添加解析记录
 
# 重启服务
systemctl restart named

# 浏览器访问
http://traefik.od.com/

Kubernetes-dashboard（仪表盘）

计划：先dashboard-v1.8.3，再dashboard-v1.10.1

docker load -i dashboard-v1.8.3.tar
docker tag fcac9aa03fd6 harbor.od.com/public/dashboard:v1.8.3
docker push harbor.od.com/public/dashboard:v1.8.3

# 资源配置清单
mkdir /data/k8s-yaml/coredns/dashboard
cd /data/k8s-yaml/coredns/dashboard



# rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
  name: kubernetes-dashboard-admin
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard-admin
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kubernetes-dashboard-admin
  namespace: kube-system
  


# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      containers:
      - name: kubernetes-dashboard
        image: harbor.od.com/public/kubernetes-dashboard-amd64:v1.8.3
        resources:
          limits:
            cpu: 100m
            memory: 300Mi
          requests:
            cpu: 50m
            memory: 100Mi
        ports:
        - containerPort: 8443
          protocol: TCP
        args:
          # PLATFORM-SPECIFIC ARGS HERE
          - --auto-generate-certificates
        volumeMounts:
        - name: tmp-volume
          mountPath: /tmp
        livenessProbe:
          httpGet:
            scheme: HTTPS
            path: /
            port: 8443
          initialDelaySeconds: 30
          timeoutSeconds: 30
      volumes:
      - name: tmp-volume
        emptyDir: {}
      serviceAccountName: kubernetes-dashboard-admin
      tolerations:
      - key: "CriticalAddonsOnly"
        operator: "Exists"
        
        
        
# service.yaml
apiVersion: v1
kind: Service
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  selector:
    k8s-app: kubernetes-dashboard
  ports:
  - port: 443
    targetPort: 8443
    
    
    
# ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: kubernetes-dashboard
  namespace: kube-system
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  rules:
  - host: dashboard.od.com
    http:
      paths:
      - backend:
          serviceName: kubernetes-dashboard
          servicePort: 443
          
          
# 任意计算节点(hdss7-21、hdss7-22)
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/rbac.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/deployment.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/service.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/ingress.yaml

配置DNS解析（hdss7-11上DNS服务器）

vi /var/named/od.com.zone
 2021031705 ; serial  # 前滚一个数
 dashboard          A    10.4.7.10  # 最后行添加

# 重启
systemctl restart name

# 浏览器访问dashboard.od.com (界面出来选择跳过)

由于https原因，需要签发dashboard证书，不然无法使用token登陆（hdss7-200）

cd /opt/certs

(umask 077; openssl genrsa -out dashboard.od.com.key 2048)

openssl req -new -key dashboard.od.com.key -out dashboard.od.com.csr -subj "/CN=dashboard.od.com/C=CN/ST=BJ/L=Beijing/O=OldboyEdu/OU=ops"

openssl x509 -req -in dashboard.od.com.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out dashboard.od.com.crt -days 3650

[root@hdss7-200 certs]# ll dashboard*
-rw-r--r-- 1 root root 1196 Mar 25 14:08 dashboard.od.com.crt
-rw-r--r-- 1 root root 1005 Mar 25 14:04 dashboard.od.com.csr
-rw------- 1 root root 1675 Mar 25 14:01 dashboard.od.com.key

配置nginx（hdss7-11、hdss7-12）

mkdir /etc/nginx/certs
cd /etc/nginx/certs/

======================================================================================================
# hdss7-200 拷贝证书
scp dashboard.od.com.crt dashboard.od.com.key root@hdss7-11:/etc/nginx/certs/
======================================================================================================

vi /etc/nginx/conf.d/dashboard.od.com.conf
server {
    listen       80;
    server_name  dashboard.od.com;
    rewrite ^(.*)$ https://${server_name}$1 permanent;
}

server {
    listen       443 ssl;
    server_name  dashboard.od.com;

    ssl_certificate "certs/dashboard.od.com.crt";
    ssl_certificate_key "certs/dashboard.od.com.key";
    ssl_session_cache shared:SSL:1m;
    ssl_session_timeout  10m;
    ssl_ciphers HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers on;

    location / {
        proxy_pass http://default_backend_traefik;
        proxy_set_header Host       $http_host;
        proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
    }
}

nginx -t;nginx -s reload

======================================================================================================
# 任意计算节点查看令牌token
kubectl -n kube-system get secret
'''''''
kubernetes-dashboard-admin-token-krbtj      # 即是令牌
'''''''

# 获得token
kubectl -n kube-system describe secret kubernetes-dashboard-admin-token-krbtj
''''
token:   ''''
''''
# 复制到登陆的令牌下登陆即可
======================================================================================================

升级dashboard版本到dashboard-v1.10.1

# hdss7-200运维主机
docker load -i dashboard-v1.10.1.tar    # 有镜像load，无pull
docker images |grep none
docker tag f9aed6605b81 harbor.od.com/public/dashboard:v1.10.1
docker push harbor.od.com/public/dashboard:v1.10.1

# 修改deployment.yaml文件，将镜像换位新的
vi /data/k8s-yaml/coredns/dashborad/deployment.yaml 
····
image: harbor.od.com/public/dashboard:v1.10.1   # 更换版本
····

# 任意计算节点
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/deployment.yaml

# PS: 也可以在dashboard界面上修改

可利用不同的角色创建不同的token达到进入dashboard的有限操作
- 利用 rbac.yaml 文件进行创建不同的服务账户，绑定不同的角色，赋予角色不同的权限，注意服务账户名（metadata里的name）还有（deployment内的serviceAccountName应和metadata中name名一样）别和有的冲突

通过上图理解进行 rbac 管理、分权管理

Role 角色只对指定名称空间有效
ClusterRole 角色对整个集群有效
用户绑定角色的方式有两种
- RoleBinding：命名空间的角色绑定
- ClusterRoleBinding：集群角色绑定
流程就是: 创建用户---> 定义角色---> 绑定角色

# 查看token，注意，版本不同命令也不同，以下命令是v1.15.4版本，如果报错，那么跟上全称即可
kubectl -n kube-system get pod -owide   # 查看token资源
kubectl -n kube-system  describe secret kubernetes-dashboard

# hdss7-200

[root@hdss7-200 dashborad]# pwd
/data/k8s-yaml/coredns/dashborad

cd /data/k8s-yaml/coredns/dashborad


cat rbac-minimal.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
  name: kubernetes-dashboard
  namespace: kube-system
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
  name: kubernetes-dashboard-minimal
  namespace: kube-system
rules:
- apiGroups: [""]
  resources: ["secrets"]
  resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
  verbs: ["get", "update", "delete"]
- apiGroups: [""]
  resources: ["configmaps"]
  resourceNames: ["kubernetes-dashboard-settings"]
  verbs: ["get", "update"]
- apiGroups: [""]
  resources: ["services"]
  resourceNames: ["heapster"]
  verbs: ["proxy"]
- apiGroups: [""]
  resources: ["services/proxy"]
  resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: kubernetes-dashboard-minimal
  namespace: kube-system
  labels:
    k8s-app: kubernetes-dashboard
    addonmanager.kubernetes.io/mode: Reconcile
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard-minimal
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kube-system



# 更改dashboard默认进入服务账户
vi deployment.yaml
·····
serviceAccountName: kubernetes-dashboard        # 将原来的改掉，原来是admin账户，改了之后dashboard会默认登陆此用户
·····
    

# 任意计算节点（hdss7-21、hdss7-22）
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/rbac-minimal.yaml

kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/deployment.yaml

# 查看令牌
kubectl -n kube-system get secret

复制获得新生成的token登陆浏览器

当然，也可以使用之前token进行登陆

# 获取token在浏览器进行登陆dashboard
kubectl -n kube-system describe secret kubernetes-dashboard-admin-token

如果多个人需要登陆dashboard，那么只需要添加rbac即可，给rbc绑定不同的角色，赋予不同的权限（开始界面报错影响观看，绑定集群角色即可解决问题）

Kubernetes平滑升级技巧

# 通过 kubectl -n kube-system get pod -owide 查看那台压力小，选择小的升级

# 这里选择hdss7-21
kubectl delete node hdss7-21.host.com

# 查看资源全分配到hdss7-22即可继续下一步
kubectl -n kube-system get pod -owide
NAME                                    READY   STATUS    RESTARTS   AGE    IP           NODE                NOMINATED NODE   READINESS GATES
coredns-6b6c4f9648-ptnc7                1/1     Running   0          2d     172.7.22.3   hdss7-22.host.com   <none>           <none>
kubernetes-dashboard-76dcdb4677-w7bpt   1/1     Running   0          2m1s   172.7.22.6   hdss7-22.host.com   <none>           <none>
traefik-ingress-fr2v2                   1/1     Running   0          24h    172.7.22.4   hdss7-22.host.com   <none>           <none>

# 验证集群是否瘫痪
dig -t A kubernetes.default.svc.cluster.local @192.168.0.2 +short
192.168.0.1

# 将4层和7反向代理注释掉
vi /etc/nginx/nginx.conf
#        server 10.4.7.21:6443     max_fails=3 fail_timeout=30s;

vi /etc/nginx/conf.d/od.com.conf 
#    server 10.4.7.21:81    max_fails=3 fail_timeout=10s;

nginx -t;nginx -s reload

升级工作

tar xvf kubernetes-server-linux-amd64-v1.15.4.tar.gz

mv /opt/src/kubernetes /opt/kubernetes-v1.15.4

cd kubernetes-v1.15.4/
rm -rf kubernetes-src.tar.gz 
cd server/bin/
rm -rf *.tar
rm -rf *_tag
mkdir certs conf

# 拷贝证书
cd certs/
cp /opt/kubernetes/server/bin/certs/* .

# 拷贝配置文件
cd ../conf/
cp /opt/kubernetes/server/bin/conf/* .

# 拷贝启动脚本
cd ../bin/
cp /opt/kubernetes/server/bin/*.sh .

# 删掉软连接指向新版本
cd /opt/
rm -rf kubernetes
ln -s /opt/kubernetes-v1.15.4 /opt/kubernetes

# 重启所有服务
supervisorctl status   # 全部重启（先stop再start，别restart，因为服务不会立马停掉而导致报错）一遍，不要all，容易翻车
# 如果中途某个报错，那么把日志清理一下，还不行删掉日志，检查进程是不是还在后台运行

# 将hdss7-21的4层与7层反向代理注释恢复



# 另外一台按照以上步骤进行升级

# 最后将hdss7-22的4层与7层反向代理注释恢复，完成升级！

Kubernetes小彩蛋（heapster）

https://github.com/kubernetes-retired/heapster # 选择相应版本的tag

# hdss7-200 配置资源清单

mkdir /data/k8s-yaml/coredns/dashborad/heapster

cd /data/k8s-yaml/coredns/dashborad/heapster



# vi rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: heapster
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
  
  
  
# deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heapster
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: harbor.od.com/public/heapster:v1.5.4
        imagePullPolicy: IfNotPresent
        command:
        - /opt/bitnami/heapster/bin/heapster
        - --source=kubernetes:https://kubernetes.default
        
        
        
# vi service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
spec:
  ports:
  - port: 80
    targetPort: 8082
  selector:
    k8s-app: heapster

任意计算节点部署heapster

kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/heapster/rbac.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/heapster/deployment.yaml
kubectl apply -f http://k8s-yaml.od.com/coredns/dashborad/heapster/service.yaml

# 创建完成刷新dashboard

来自老男孩视频

到此集群完结

点赞
收藏
关注作者

0/1000

抱歉，系统识别当前为高风险访问，暂不支持该操作

全部回复

上滑加载中

设置昵称

在此一键设置昵称，即可参与社区互动！

*长度不超过10个汉字或20个英文字符，设置后3个月内不可修改。

确认取消

加入云驻计划，成为创作者

华为云周边好礼
免费体验产品
特殊身份标识
线下官方门票
内部专家零距离
与10000+优质创作者共同成长

立即加入