一、背景

    由于scheduler-extender方式性能较差且支持的扩展接口有限(Filter、Prioritize和Bind)，开源K8S在1.19版本增加了调度器新特性scheduler-framework(支持PreFilter、Filter、PostFilter、PreScore、Score、Reserve、Permit、PreBind、Bind和PostBind)，并且后续版本主推这种方式实现调度器的扩展。

二、framwork开发示例

     本示例sample-scheduler将指定名称的Pod调度到指定label节点上

2.1 代码结构

2.2 main.go代码示例

package main

import (
   "fmt"
   "github.com/cnych/sample-scheduler-framework/pkg/plugins"
   "k8s.io/component-base/logs"
   "k8s.io/kubernetes/cmd/kube-scheduler/app"
   "math/rand"
   "os"
   "time"
)
func main() {
   rand.Seed(time.Now().UTC().UnixNano())
   //使用k8s提供的sdk注册自定义调度插件
   command := app.NewSchedulerCommand(
      app.WithPlugin(plugins.Name, plugins.New),
   )
   logs.InitLogs()
   defer logs.FlushLogs()
   if err := command.Execute(); err != nil {
      _, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
      os.Exit(1)
   }
}

2.3 plugin代码示例 

package plugins

import (
   "context"
   "fmt"
   v1 "k8s.io/api/core/v1"
   "k8s.io/apimachinery/pkg/runtime"
   "k8s.io/klog/v2"
   "k8s.io/kubernetes/pkg/scheduler/framework"
   runtime2 "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
   "strings"
)
// 插件名称
const Name = "sample-plugin"

type Sample struct {
   handle framework.Handle
   args samplePluginArg
}
type samplePluginArg struct {
   NodeLabel string `json:"node_label,omitempty"`
   WorkloadName string `json:"workload_name,omitempty"`
}
func (s *Sample) Name() string {
   return Name
}

func (s *Sample) PreFilter(ctx context.Context, state  *framework.CycleState, pod *v1.Pod) *framework.Status {
   klog.V(3).Infof("prefilter pod: %v", pod.Name)
   return framework.NewStatus(framework.Success, "")
}
func (s *Sample) PreFilterExtensions() framework.PreFilterExtensions {
   return nil
}

func SplicingKeyValue(k, v string) string {
   return k + "/" + v
}
func (s *Sample) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
   klog.V(3).Infof("filter pod: %v, node: %v", pod.Name, nodeInfo.Node().Name)
   #将指定工作负载的pod与带有特定标签的节点进行绑定
   if strings.Contains(pod.Name, s.args.WorkloadName) {
      match := false
      for k, v := range nodeInfo.Node().Labels {
         if s.args.NodeLabel == SplicingKeyValue(k, v) {
            match = true
         }
      }
      if !match {
         return framework.NewStatus(framework.UnschedulableAndUnresolvable, "no match label")
      }
   }
   return framework.NewStatus(framework.Success, "")
}

func (s *Sample) PreBind(ctx context.Context, state  *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status {
   return framework.NewStatus(framework.Success, fmt.Sprintf("prebind get node : %+v", nodeName))
}

#用于在main函数中调用进行注册
func New(configuration runtime.Object, f framework.Handle) (framework.Plugin, error) {
   args := &samplePluginArg{}
   runtime2.DecodeInto(configuration, args)
   klog.V(3).Infof("Starting: %v plugin, args: %v", Name, args)
   return &Sample{
      handle: f,
      args: *args,
   }, nil
}

2.4 编译组件并构建镜像

GOOS=linux GOARCH=amd64 go build -o sample-scheduler-framework ./main.go 

将二进制文件上传至Node节点，并使用如下dockerfile文件制作镜像；docker build -t sample-scheduler:v1 .

FROM centos:7.6.1810

WORKDIR /

COPY sample-scheduler-framework /usr/local/bin

CMD ["sample-scheduler-framework"]

三、部署调度器

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: sample-scheduler-clusterrole
rules:
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
      - list
      - watch
      - update
  - apiGroups:
      - ""
    resources:
      - bindings
      - pods/binding
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - pods/status
    verbs:
      - patch
      - update
  - apiGroups:
      - ""
    resources:
      - persistentvolumeclaims
      - persistentvolumes
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "storage.k8s.io"
    resources:
      - storageclasses
      - csinodes
      - csistoragecapacities
      - csidrivers
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "coordination.k8s.io"
    resources:
      - leases
    verbs:
      - create
      - get
      - list
      - update
  - apiGroups:
      - "events.k8s.io"
    resources:
      - events
    verbs:
      - create
      - patch
      - update
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: sample-scheduler-sa
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: sample-scheduler-clusterrolebinding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: sample-scheduler-clusterrole
subjects:
  - kind: ServiceAccount
    name: sample-scheduler-sa
    namespace: kube-system

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: sample-scheduler-config   #挂载到sample-scheduler容器
  namespace: kube-system
data:
  scheduler-config.yaml: |
    apiVersion: kubescheduler.config.k8s.io/v1beta1
    kind: KubeSchedulerConfiguration
    leaderElection:
      leaderElect: true
      leaseDuration: 15s
      renewDeadline: 10s
      resourceLock: leases
      resourceName: sample-scheduler
      resourceNamespace: kube-system
      retryPeriod: 2s
    profiles:
    - schedulerName: default-scheduler #如果使用其他名称，则需要在工作负载的schedulerName字段中指明该名称
      pluginConfig:
      - name: sample-plugin
        args:
          node_label: test/test         #业务相关联配置，用于指定节点标签，格式为key/value
          workload_name: nginx     #业务相关联配置，用于指定工作负载名称
      plugins:     #在已实现的阶段开启自己的插件
         filter:
           enabled:
           - name: "sample-plugin"
         preFilter:
           enabled:
           - name: "sample-plugin"
         preBind:
           enabled:
           - name: "sample-plugin"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sample-scheduler
  namespace: kube-system
  labels:
    component: sample-scheduler
spec:
  replicas: 1
  selector:
    matchLabels:
      component: sample-scheduler
  template:
    metadata:
      labels:
        component: sample-scheduler
    spec:
      serviceAccount: sample-scheduler-sa
      priorityClassName: system-cluster-critical
      volumes:
        - name: scheduler-config
          configMap:
            name: sample-scheduler-config
      containers:
        - name: scheduler-ctrl
          image: sample-scheduler:v1
          imagePullPolicy: IfNotPresent
          args:
            - sample-scheduler-framework
            - --config=/etc/kubernetes/scheduler-config.yaml
            - --v=3
          volumeMounts:
            - name: scheduler-config
              mountPath: /etc/kubernetes

创建nginx工作负载时只能调度至带有test=test标签的节点上，其他工作负载不受影响，调度插件工作