k8s1.14.6集群搭建之kube-flannel部署

k8s1.14.6集群部署 - This article is part of a series.

Part 6: This Article

1.Daemonset方式
#

$ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml

$ cat kube-flannel.yml
apiVersion: extensions/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
    - configMap
    - secret
    - emptyDir
    - hostPath
  allowedHostPaths:
    - pathPrefix: "/etc/cni/net.d"
    - pathPrefix: "/etc/kube-flannel"
    - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unsed in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups: ['extensions']
    resources: ['podsecuritypolicies']
    verbs: ['use']
    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-amd64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      hostNetwork: true
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.11.0-amd64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.11.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg

2. 静态Pod方式
#

2.1 master节点上创建flannel 相关的CRD

a) flannel pod 的PodSecurityPolicy，定义资源访问及配置限制 b) 访问apiserver的ServiceAccount, ClusterRole, ClusterRoleBinding

$ kubectl apply -f kube-flannel-CRD.yaml
$ cat kube-flannel-CRD.yaml
apiVersion: extensions/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
    - configMap
    - secret
    - emptyDir
    - hostPath
  allowedHostPaths:
    - pathPrefix: "/etc/cni/net.d"
    - pathPrefix: "/etc/kube-flannel"
    - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unsed in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups: ['extensions']
    resources: ['podsecuritypolicies']
    verbs: ['use']
    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system

2.2 master节点上创建flannel访问apiserver的kubeconfig

ServiceAccount flannel声明后，自动创建secret对象，可获取secret中的token，用于flannel pod 访问apiserver

$ cat create-flannel-kubeconfig.sh
SECRET=$(kubectl get sa flannel -n kube-system -o jsonpath='{.secrets[].name}')
TOKEN=$(kubectl get secret $SECRET -n kube-system -o jsonpath='{.data.token}'|base64 -d)

KUBE_APISERVER="https://192.168.18.142:6443"
kubectl config set-cluster kubernetes \
    --certificate-authority=/root/k8s-1.14.6/ssl/ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=flannel.kubeconfig
kubectl config set-credentials flannel@kubernetes \
    --token=$TOKEN \
    --kubeconfig=flannel.kubeconfig
kubectl config set-context flannel@kubernetes \
    --cluster=kubernetes \
    --user=flannel@kubernetes \
    --kubeconfig=flannel.kubeconfig
kubectl config use-context flannel@kubernetes --kubeconfig=flannel.kubeconfig
$ sh create-flannel-kubeconfig.sh

将生成的flannel.kubeconfig文件分发到node节点192.168.18.160的/root/k8s-1.14.6/conf/flannel目录下

2.3 计算节点192.168.18.160上创建flannel cni插件的配置文件

$ cd /root/k8s-1.14.6/conf/flannel
$ cat <<EOF > net-conf.json
{
  "Network": "10.244.0.0/16",
  "Backend": {
    "Type": "vxlan"
  }
}
EOF

$ cp /etc/cni/net.d/10-flannel.conflist cni-conf.json

2.4 计算节点192.168.18.160上创建kube-flannel的静态pod文件kube-flannel-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  labels:
    tier: node
    app: flannel
  name: kube-flannel
  namespace: kube-system
spec:
  hostNetwork: true
  restartPolicy: Always
  containers:
  - name: kube-flannel
    image: quay.io/coreos/flannel:v0.11.0-amd64
    command:
    - /opt/bin/flanneld
    args:
    - --ip-masq
    - --kube-subnet-mgr
    - --kubeconfig-file=/etc/kube-flannel/flannel.kubeconfig
    resources:
      requests:
        cpu: "100m"
        memory: "50Mi"
      limits:
        cpu: "100m"
        memory: "50Mi"
    securityContext:
      privileged: false
      capabilities:
         add: ["NET_ADMIN"]
    env:
    - name: POD_NAME
      valueFrom:
        fieldRef:
          fieldPath: metadata.name
    - name: POD_NAMESPACE
      valueFrom:
        fieldRef:
          fieldPath: metadata.namespace
    volumeMounts:
    - name: run
      mountPath: /run/flannel
    - name: cni
      mountPath: /etc/kube-flannel/
  volumes:
    - name: run
      hostPath:
        path: /run/flannel
    - name: cni
      hostPath:
        path: /root/k8s-1.14.6/conf/flannel

2.5 查看kube-flannel是否正常启动

$ docker ps
bbc6831c81f9        ff281650a721           "/opt/bin/flanneld -…"   16 seconds ago      Up 15 seconds                           k8s_kube-flannel_kube-flannel-192.168.18.160_kube-system_b61385e9f07935caba0db5f9c2eb1a9e_1
f66c5be28bb1        k8s.gcr.io/pause:3.1   "/pause"                 16 seconds ago      Up 15 seconds                           k8s_POD_kube-flannel-192.168.18.160_kube-system_b61385e9f07935caba0db5f9c2eb1a9e_1

$ docker logs bbc6831c81f9
I1021 07:41:51.567961       1 main.go:514] Determining IP address of default interface
I1021 07:41:51.568823       1 main.go:527] Using interface with name eth0 and address 29.20.18.160
I1021 07:41:51.568846       1 main.go:544] Defaulting external address to interface address (29.20.18.160)
I1021 07:41:51.671121       1 kube.go:126] Waiting 10m0s for node controller to sync
I1021 07:41:51.671194       1 kube.go:309] Starting kube subnet manager
I1021 07:41:52.671328       1 kube.go:133] Node controller sync successful
I1021 07:41:52.671376       1 main.go:244] Created subnet manager: Kubernetes Subnet Manager - 29.20.18.160
I1021 07:41:52.671382       1 main.go:247] Installing signal handlers
I1021 07:41:52.671450       1 main.go:386] Found network config - Backend type: vxlan
I1021 07:41:52.671505       1 vxlan.go:120] VXLAN config: VNI=1 Port=0 GBP=false DirectRouting=false
I1021 07:41:52.673135       1 main.go:317] Wrote subnet file to /run/flannel/subnet.env
I1021 07:41:52.673149       1 main.go:321] Running backend.
I1021 07:41:52.673157       1 main.go:339] Waiting for all goroutines to exit
I1021 07:41:52.673186       1 vxlan_network.go:60] watching for new subnet leases

$ ifconfig flannel.1
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1400
        inet 10.244.4.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::74b1:a2ff:fefd:1e3d  prefixlen 64  scopeid 0x20<link>
        ether 76:b1:a2:fd:1e:3d  txqueuelen 0  (Ethernet)
        RX packets 397  bytes 50538 (49.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 403  bytes 55506 (54.2 KiB)
        TX errors 0  dropped 10 overruns 0  carrier 0  collisions 0