备份和迁移 Kubernetes 利器：Velero

# 备份#!/usr/bin/env bashdate;CACERT="/opt/kubernetes/ssl/ca.pem"CERT="/opt/kubernetes/ssl/server.pem"EKY="/opt/kubernetes/ssl/server-key.pem"ENDPOINTS="192.168.1.36:2379"
ETCDCTL_API=3 etcdctl \--cacert="${CACERT}" --cert="${CERT}" --key="${EKY}" \--endpoints=${ENDPOINTS} \snapshot save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.db
# 备份保留30天find /data/etcd_backup_dir/ -name *.db -mtime +30 -exec rm -f {} \;

Velero is an open source tool to safely backup and restore, perform disaster recovery, andmigrate Kubernetes cluster resources and persistent volumes.

Built from the ground up for Kubernetes, PX-Backup delivers enterprise-grade applicationand data protection with fast recovery at the click of a button

urpose-built for Kubernetes, Kasten K10 provides enterprise operations teams an easy-touse, scalable, and secure system for backup/restore, disaster recovery, and mobility ofKubernetes applications.

Examples: # Create a backup every 6 hours velero create schedule NAME --schedule="0 */6 * * *" # Create a backup every 6 hours with the @every notation velero create schedule NAME --schedule="@every 6h" # Create a daily backup of the web namespace velero create schedule NAME --schedule="@every 24h" --include-namespaces web # Create a weekly backup, each living for 90 days (2160 hours) velero create schedule NAME --schedule="@every 168h" --ttl 2160h0m0s

velero create schedule 360cloud --schedule="@every 24h" --ttl 2160h0m0sSchedule "360cloud" created successfully.[root@xxxxx ~]# kubectl get schedules --all-namespacesNAMESPACE NAME AGEvelero 360cloud 40s[root@xxxxx ~]# kubectl get schedules -n velero 360cloud -o yamlapiVersion: velero.io/v1kind: Schedulemetadata: generation: 3 name: 360cloud namespace: velero resourceVersion: "18164238" selfLink: /apis/velero.io/v1/namespaces/velero/schedules/360cloud uid: 7c04af34-1529-4b48-a3d1-d2f5e98de328spec: schedule: '@every 24h' template: hooks: {} includedNamespaces: - '*' ttl: 2160h0m0sstatus: lastBackup: "2021-03-07T08:18:49Z" phase: Enabled

velero backup create test01 --include-namespaces defaultBackup request "test01" submitted successfully.Run `velero backup describe test01` or `velero backup logs test01` for moredetails.[root@xxxxx ~]# velero backup describe test01Name: test01Namespace: veleroLabels: velero.io/storage-location=defaultAnnotations: velero.io/source-cluster-k8s-gitversion=v1.19.7 velero.io/source-cluster-k8s-major-version=1 velero.io/source-cluster-k8s-minor-version=19Phase: InProgressErrors: 0Warnings: 0Namespaces: Included: default Excluded: <none>Resources: Included: * Excluded: <none> Cluster-scoped: autoLabel selector: <none>Storage Location: defaultVelero-Native Snapshot PVs: autoTTL: 720h0m0sHooks: <none>Backup Format Version: 1.1.0Started: 2021-03-07 16:44:52 +0800 CSTCompleted: <n/a>Expiration: 2021-04-06 16:44:52 +0800 CSTVelero-Native Snapshots: <none included>

[root@xxxxx ~]# velero restore create --from-backup test01Restore request "test01-20210307164809" submitted successfully.Run `velero restore describe test01-20210307164809` or `velero restore logstest01-20210307164809` for more details.[root@xxxxx ~]# kuebctl ^C[root@xxxxx ~]# kubectl get podNAME READY STATUS RESTARTS AGEnginx-6799fc88d8-4bnfg 0/1 ContainerCreating 0 6snginx-6799fc88d8-cq82j 0/1 ContainerCreating 0 6snginx-6799fc88d8-f6qsx 0/1 ContainerCreating 0 6snginx-6799fc88d8-gq2xt 0/1 ContainerCreating 0 6snginx-6799fc88d8-j5fc7 0/1 ContainerCreating 0 6snginx-6799fc88d8-kvvx6 0/1 ContainerCreating 0 5snginx-6799fc88d8-pccc4 0/1 ContainerCreating 0 5snginx-6799fc88d8-q2fnt 0/1 ContainerCreating 0 4snginx-6799fc88d8-r9dqn 0/1 ContainerCreating 0 4snginx-6799fc88d8-zqv6v 0/1 ContainerCreating 0 4s

apiVersion: v1kind: Podmetadata: annotations: backup.velero.io/backup-volumes: mypvc name: rbd-testspec: containers: - name: web-server image: nginx volumeMounts: - name: mypvc mountPath: /var/lib/www/html volumes: - name: mypvc persistentVolumeClaim: claimName: rbd-pvc-zhf readOnly: false

velero backup create testpvc05 --snapshot-volumes=true --include-namespacesdefaultBackup request "testpvc05" submitted successfully.Run `velero backup describe testpvc05` or `velero backup logs testpvc05` formore details.[root@xxxx ceph]# velero backup describe testpvc05Name: testpvc05Namespace: veleroLabels: velero.io/storage-location=defaultAnnotations: velero.io/source-cluster-k8s-gitversion=v1.19.7 velero.io/source-cluster-k8s-major-version=1 velero.io/source-cluster-k8s-minor-version=19Phase: CompletedErrors: 0Warnings: 0Namespaces: Included: default Excluded: <none>Resources: Included: * Excluded: <none> Cluster-scoped: autoLabel selector: <none>Storage Location: defaultVelero-Native Snapshot PVs: trueTTL: 720h0m0sHooks: <none>Backup Format Version: 1.1.0Started: 2021-03-10 15:11:26 +0800 CSTCompleted: 2021-03-10 15:11:36 +0800 CSTExpiration: 2021-04-09 15:11:26 +0800 CSTTotal items to be backed up: 92Items backed up: 92Velero-Native Snapshots: <none included>
Restic Backups (specify --details for more information): Completed: 1

[root@xxxxxx ceph]# kubectl delete pod rbd-testpod "rbd-test" deletedkubectl delete pvc[root@p48453v ceph]# kubectl delete pvc rbd-pvc-zhfpersistentvolumeclaim "rbd-pvc-zhf" deleted

[root@xxxxx ceph]# velero restore create testpvc05 --restore-volumes=true--from-backup testpvc05Restore request "testpvc05" submitted successfully.Run `velero restore describe testpvc05` or `velero restore logs testpvc05` formore details.[root@xxxxxx ceph]#[root@xxxxxx ceph]# kuebctl^C[root@xxxxxx ceph]# kubectl get podNAME READY STATUS RESTARTS AGEnginx-6799fc88d8-4bnfg 1/1 Running 0 2d22hrbd-test 0/1 Init:0/1 0 6s

[root@xxxxxx ceph]# kubectl exec rbd-test sh -- ls -l /var/lib/www/htmltotal 20drwx------ 2 root root 16384 Mar 10 06:31 lost+found-rw-r--r-- 1 root root 13 Mar 10 07:11 zheng.txt[root@xxxxxx ceph]# kubectl exec rbd-test sh -- cat/var/lib/www/html/zheng.txtzhenghongfei[root@xxxxx ceph]#

metadata: name: nginx-deployment namespace: nginx-examplespec: replicas: selector: matchLabels: app: nginx template: metadata: labels: app: nginx annotations: pre.hook.backup.velero.io/container: fsfreeze pre.hook.backup.velero.io/command: '["/sbin/fsfreeze", "--freeze","/var/log/nginx"]' post.hook.backup.velero.io/container: fsfreeze post.hook.backup.velero.io/command: '["/sbin/fsfreeze", "--unfreeze","/var/log/nginx"]'

 collector := &itemCollector{ log: log, backupRequest: backupRequest, discoveryHelper: kb.discoveryHelper, dynamicFactory: kb.dynamicFactory, cohabitatingResources: cohabitatingResources(), dir: tempDir, } items := collector.getAllItems()

func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, grschema.GroupResource, itemBackupper *itemBackupper, unstructured*unstructured.Unstructured, preferredGVR schema.GroupVersionResource) bool { backedUpItem, err := itemBackupper.backupItem(log, unstructured, gr,preferredGVR) if aggregate, ok := err.(kubeerrs.Aggregate); ok { log.WithField("name", unstructured.GetName()).Infof("%d errors encounteredbackup up item", len(aggregate.Errors())) // log each error separately so we get error location info in the log, andan // accurate count of errors for _, err = range aggregate.Errors() { log.WithError(err).WithField("name",unstructured.GetName()).Error("Error backing up item") } return false } if err != nil { log.WithError(err).WithField("name", unstructured.GetName()).Error("Errorbacking up item") return false } return backedUpItem}

client, err :=ib.dynamicFactory.ClientForGroupVersionResource(gvr.GroupVersion(), resource,additionalItem.Namespace) if err != nil { return nil, err } item, err := client.Get(additionalItem.Name, metav1.GetOptions{})

 log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s",groupResource.String(), name, version, preferredVersion) if version == preferredVersion { if namespace != "" { filePath = filepath.Join(velerov1api.ResourcesDir,groupResource.String(), velerov1api.NamespaceScopedDir, namespace,name+".json")PX-Backup kanisterhttps://github.com/vmware-tanzu/velerohttps://portworx.com/https://www.kasten.io/https://github.com/kanisterio/kanisterhttps://duyanghao.github.io/kubernetes-ha-and-bur/https://blog.kubernauts.io/backup-and-restore-of-kubernetes-applications-using-heptios-velerowith-restic-and-rook-ceph-as-2e8df15b1487 } else { filePath = filepath.Join(velerov1api.ResourcesDir,groupResource.String(), velerov1api.ClusterScopedDir, name+".json") } hdr = &tar.Header{ Name: filePath, Size: int64(len(itemBytes)), Typeflag: tar.TypeReg, Mode: 0755, ModTime: time.Now(), } if err := ib.tarWriter.WriteHeader(hdr); err != nil { return false, errors.WithStack(err) } if _, err := ib.tarWriter.Write(itemBytes); err != nil { return false, errors.WithStack(err) }}