阿里云ECS搭建高可用k8s集群及Terway、CCM、SLB配置

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# 使设置生效
sysctl --system
# 创建 modules 文件，保证在节点重启后能自动加载所需模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_lc
modprobe -- ip_vs_wlc
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_lblc
modprobe -- ip_vs_lblcr
modprobe -- ip_vs_dh
modprobe -- ip_vs_sh
# 3.10 内核不支持
#modprobe -- ip_vs_fo
modprobe -- ip_vs_nq
modprobe -- ip_vs_sed
modprobe -- ip_vs_ftp
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm  -y
# 检查是否安装 成功 
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# haproxy的安装和配置都非常简单
yum install -y haproxy
# cat cat /etc/haproxy/haproxy.cfg
global
    log 127.0.0.1 local0
    log 127.0.0.1 local1 notice
 
    maxconn 4096
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon
defaults
    log global
    mode tcp
    option tcplog
    option dontlognull
    retries 3
    option redispatch
    maxconn 2000
#    contimeout 5000
#    clitimeout 50000
#    srvtimeout 50000
    timeout connect 5000
    timeout client 50000
    timeout server 50000
listen zabbixServer
    bind 127.0.0.1:6443
    server k8s-01 192.168.1.27:5443 check inter 3000 fall 3 rise 3
    server k8s-02 192.168.1.28:5443 check inter 3000 fall 3 rise 3
    server k8s-03 192.168.1.29:5443 check inter 3000 fall 3 rise 3
  
  # 开机启动
  systemctl enable haproxy
  # 启动 haproxy
  systemctl start haproxy
  
  # 检查
  ss -anltp | grep haproxy

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

#!/bin/bash

. /etc/profile

function before_install() {
    getent group etcd >/dev/null || groupadd -r etcd
    getent passwd etcd >/dev/null || useradd -r -g etcd -d /var/lib/etcd -s /sbin/nologin -c "etcd user" etcd

    if [ ! -d "/var/lib/etcd" ]; then
        mkdir /var/lib/etcd
        chown -R etcd:etcd /var/lib/etcd
    fi
}

function get_cfssl_version(){
    local cfssl_version=$(curl -s https://api.github.com/repos/cloudflare/cfssl/releases/latest | grep tag_name | cut -d '"' -f 4| cut -b 2-)
    echo $cfssl_version
}

function install_cfssl(){
    local cfssl_version=$(get_cfssl_version)
    local cfssl_url="https://github.com/cloudflare/cfssl/releases/download/v${cfssl_version}/cfssl_${cfssl_version}_linux_amd64"
    local cfssl_bin="cfssl"
    local cfssl_bin_path="/usr/local/bin/${cfssl_bin}"

    local cfssljson_url="https://github.com/cloudflare/cfssl/releases/download/v${cfssl_version}/cfssljson_${cfssl_version}_linux_amd64"
    local cfssljson_bin="cfssljson"
    local cfssljson_bin_path="/usr/local/bin/${cfssljson_bin}"

    echo "cfssl_version: ${cfssl_version}"

    echo "Downloading cfssl..."
    rm -f ${cfssl_bin_path}
    wget -q -O ${cfssl_bin_path} ${cfssl_url}
    if [ $? -ne 0 ]; then
        echo "Failed to download cfssl."
        exit 1
    fi
    chmod +x ${cfssl_bin_path}
    echo "cfssl installed."
    
    echo "Downloading cfssljson..."
    rm -f ${cfssljson_bin_path}
    wget -q -O ${cfssljson_bin_path} ${cfssljson_url}
    if [ $? -ne 0 ]; then
        echo "Failed to download cfssljson."
        exit 1
    fi
    chmod +x ${cfssljson_bin_path}
    echo "cfssljson installed."
}

function get_etcd_version() {
    local etcd_version=$(curl -s https://api.github.com/repos/etcd-io/etcd/releases/latest | grep tag_name | cut -d '"' -f 4)
    echo $etcd_version
}

function install_etcd() {
    local etcd_version=$(get_etcd_version)
    local etcd_url="https://github.com/etcd-io/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-amd64.tar.gz"
    local etcd_file="etcd-${etcd_version}-linux-amd64.tar.gz"
    local etcd_dir="etcd-${etcd_version}-linux-amd64"

    echo "etcd_version: ${etcd_version}"

    cd /tmp
    echo "Downloading etcd..."
    wget -q -O ${etcd_file} ${etcd_url}
    if [ $? -ne 0 ]; then
        echo "download etcd failed"
        exit 1
    fi

    tar -xzf ${etcd_file}
    rm -f /usr/local/bin/etcd /usr/local/bin/etcdctl /usr/local/bin/etcdutl
    mv ${etcd_dir}/etcd /usr/local/bin/
    mv ${etcd_dir}/etcdctl /usr/local/bin/
    mv ${etcd_dir}/etcdutl /usr/local/bin/
    chmod +x /usr/local/bin/etcd /usr/local/bin/etcdctl /usr/local/bin/etcdutl
    echo "etcd installed successfully"

    echo "clean up..."
    rm -f ${etcd_file}
    rm -rf ${etcd_dir}

    cd -
}

function etcd_systemd() {
    cat > /etc/systemd/system/etcd.service << EOF
[Unit]
Description=etcd - highly-available key value store
Documentation=https://github.com/coreos/etcd
After=network.target
Wants=network-online.target

[Service]
#Type=notify
Type=simple
User=etcd
ExecStart=/usr/local/bin/etcd  --config-file /etc/etcd/etcd.yaml
Restart=on-failure
RestartSec=10s

LimitNOFILE=65536
#MemoryLow=200M
Nice=-10
IOSchedulingClass=best-effort
IOSchedulingPriority=2

[Install]
WantedBy=multi-user.target
EOF
}

function after_install() {
    mkdir -p /data/etcd/{data,wal}
    chown -R etcd:etcd /data/etcd

    mkdir -p /etc/etcd/{cfssl, ssl}
    chown -R etcd:etcd /etc/etcd

    echo "Installing systemd service..."
    etcd_systemd
    systemctl enable etcd
    
    grep 'ETCDCTL_API=3' /etc/profile > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "export ETCDCTL_API=3" >> /etc/profile
        . /etc/profile
    fi
    
    grep '$PATH:/usr/local/bin' /etc/profile > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo 'export PATH=$PATH:/usr/local/bin' >> /etc/profile
    fi
    
    . /etc/profile
}

function main(){
    before_install

    install_etcd
    install_cfssl

    after_install
}

main

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# hosts 中需要根据实际情况修改
# cat etcd-csr.json
{
    "key": {
        "algo": "rsa",
        "size": 2048 
    },
    "names": [
        {
            "O": "etcd",
            "OU": "etcd Security",
            "L": "Chengdu",
            "ST": "Sichuan",
            "C": "CN"
        }
    ],
    "CN": "etcd",
    "hosts": [
        "127.0.0.1",
        "localhost",
        "*.etcd.node",
        "*.kubernetes.node",
        "192.168.1.27",
        "192.168.1.28",
        "192.168.100.29"
    ]
}
# cat etcd-gencert.json
{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "87600h"
    }
  }
}
# etcd-root-ca-csr.json
{
    "CN": "etcd-root-ca",
    "key": {
        "algo": "rsa",
        "size": 4096
    },
    "names": [
        {
            "O": "etcd",
            "OU": "etcd Security",
            "L": "Chengdu",
            "ST": "Sichuan",
            "C": "CN"
        }
    ],
    "ca": {
        "expiry": "87600h"
    }
}

1
2
3
4
5
6

#!/usr/bin/env bash

set -e

cfssl gencert --initca=true etcd-root-ca-csr.json | cfssljson --bare etcd-root-ca
cfssl gencert --ca etcd-root-ca.pem --ca-key etcd-root-ca-key.pem --config etcd-gencert.json etcd-csr.json | cfssljson --bare etcd

1
2
3
4

# 复制证书到etcd目录
cp -rf /etc/etcd/cfssl/*.pem /etc/etcd/ssl
# 更新权限
chown etcd: /etc/etcd/ssl

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

# This is the configuration file for the etcd server.

# Human-readable name for this member.
# 不能重复
name: 'etcd-01'

# Path to the data directory.
data-dir: '/data/etcd/data'

# Path to the dedicated wal directory.
wal-dir: '/data/etcd/wal'

# Number of committed transactions to trigger a snapshot to disk.
#snapshot-count: 10000
snapshot-count: 1000

# Time (in milliseconds) of a heartbeat interval.
heartbeat-interval: 100

# Time (in milliseconds) for an election to timeout.
election-timeout: 1000

# Raise alarms when backend size exceeds the given quota. 0 means use the
# default quota.
#quota-backend-bytes: 0
# 5G
quota-backend-bytes: 5368709120

# List of comma separated URLs to listen on for peer traffic.
listen-peer-urls: https://192.168.1.27:2380

# List of comma separated URLs to listen on for client traffic.
listen-client-urls: https://192.168.1.27:2379,https://127.0.0.1:2379

# Maximum number of snapshot files to retain (0 is unlimited).
max-snapshots: 10

# Maximum number of wal files to retain (0 is unlimited).
max-wals: 10

# Comma-separated white list of origins for CORS (cross-origin resource sharing).
cors:

# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.
initial-advertise-peer-urls: https://192.168.1.27:2380

# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.
advertise-client-urls: https://192.168.1.27:2379

# Discovery URL used to bootstrap the cluster.
discovery:

# Valid values include 'exit', 'proxy'
discovery-fallback: 'proxy'

# HTTP proxy to use for traffic to discovery service.
discovery-proxy:

# DNS domain used to bootstrap initial cluster.
discovery-srv:

# Initial cluster configuration for bootstrapping.
initial-cluster: etcd-01=https://192.168.1.27:2380,etcd-02=https://192.168.1.28:2380,etcd-03=https://192.168.1.29:2380

# Initial cluster token for the etcd cluster during bootstrap.
# 随机字符串，相同集群内，该配置需要一致
# 可使用shell命令生成： openssl rand -base64 15 
initial-cluster-token: 'krMWJC2ha5'

# Initial cluster state ('new' or 'existing').
initial-cluster-state: 'new'

# Reject reconfiguration requests that would cause quorum loss.
strict-reconfig-check: true

# Enable runtime profiling data via HTTP server
enable-pprof: true

# Valid values include 'on', 'readonly', 'off'
proxy: 'off'

# Time (in milliseconds) an endpoint will be held in a failed state.
proxy-failure-wait: 5000

# Time (in milliseconds) of the endpoints refresh interval.
proxy-refresh-interval: 30000

# Time (in milliseconds) for a dial to timeout.
proxy-dial-timeout: 1000

# Time (in milliseconds) for a write to timeout.
proxy-write-timeout: 5000

# Time (in milliseconds) for a read to timeout.
proxy-read-timeout: 0

client-transport-security:
  # Path to the client server TLS cert file.
  cert-file: /etc/etcd/ssl/etcd.pem

  # Path to the client server TLS key file.
  key-file: /etc/etcd/ssl/etcd-key.pem

  # Enable client cert authentication.
  client-cert-auth: true

  # Path to the client server TLS trusted CA cert file.
  trusted-ca-file: /etc/etcd/ssl/etcd-root-ca.pem

  # Client TLS using generated certificates
  auto-tls: true

peer-transport-security:
  # Path to the peer server TLS cert file.
  cert-file: /etc/etcd/ssl/etcd.pem

  # Path to the peer server TLS key file.
  key-file: /etc/etcd/ssl/etcd-key.pem

  # Enable peer client cert authentication.
  client-cert-auth: true

  # Path to the peer server TLS trusted CA cert file.
  trusted-ca-file: /etc/etcd/ssl/etcd-root-ca.pem

  # Peer TLS using generated certificates.
  auto-tls: true

# The validity period of the self-signed certificate, the unit is year.
self-signed-cert-validity: 1

# Enable debug-level logging for etcd.
log-level: debug

logger: zap

# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
log-outputs: [systemd/journal]

# Force to create a new one member cluster.
force-new-cluster: false

auto-compaction-mode: revision
auto-compaction-retention: "300"

1
2
3
4

# 健康检查
etcdctl endpoint health --cacert /etc/etcd/ssl/etcd-root-ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem --endpoints https://192.168.1.27:2379,https://192.168.1.28:2379,https://192.168.1.30:2379
# 清空 ETCD 数据 慎重
etcdctl --cacert /etc/etcd/ssl/etcd-root-ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem --endpoints https://192.168.1.27:2379,https://192.168.1.28:2379,https://192.168.1.30:2379 del "" --prefix

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

# 阿里ECS 可以直接使用阿里云内部的源
# cat /etc/yum.repos.d/docker-ce.repo
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-stable-debuginfo]
name=Docker CE Stable - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/stable
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-stable-source]
name=Docker CE Stable - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/stable
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test]
name=Docker CE Test - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test-debuginfo]
name=Docker CE Test - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test-source]
name=Docker CE Test - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly]
name=Docker CE Nightly - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly-debuginfo]
name=Docker CE Nightly - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly-source]
name=Docker CE Nightly - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# 更新yum 缓存
yum makecache fast
# 安装
yum -y install docker-ce-20.10.14-3.el7
# 配置
# registry-mirror 仅限公司内部使用
# cat /etc/docker/daemon.json 
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirror": [
      "https://e88hruwq.mirror.aliyuncs.com" 
  ],
  "data-root": "/data/docker_data",
  "log-driver": "json-file",
  "log-level": "",
  "log-opts": {
      "max-size": "10m",
      "max-file": "3"
  }
}
# 开机启动
systemctl enable docker
systemctl restart docker

1
2
3
4
5
6
7
8

# cat /etc/yum.repos.d/kubernetes.repo 
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.cloud.aliyuncs.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.cloud.aliyuncs.com/kubernetes/yum/doc/rpm-package-key.gpg

1
2
3
4
5

yum install -y kubelet-1.21.13-0 kubeadm-1.21.13-0 kubectl-1.21.13-0 ebtables ethtool
# 关闭升级，防止引起问题
echo 'exclude=kubelet kubeadm kubectl docker-ce docker-ce-cli' >> /etc/yum.conf

/usr/lib/systemd/system/kubelet.service

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
#ExecStart=/usr/bin/kubelet
ExecStart=/usr/bin/kubelet --cloud-provider=external --hostname-override=xxx.xxx --provider-id=xxx.xxx
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

1
2

systemctl daemon-reload
systemctl enable kubelet

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
#nodeRegistration:
#    criSocket: /run/containerd/containerd.sock
#    name: containerd
localAPIEndpoint:
  # 第一个 master 节点 IP
  advertiseAddress: "192.168.1.27"
  # 6443 留给了 nginx，apiserver 换到 5443
  bindPort: 5443
# 这个 token 使用以下命令生成
# kubeadm certs certificate-key
certificateKey: e8734dd17eaab2d55cfb363037f0d27cdd0f0cf1016d163d07428ac3aa9e19f1
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
# 使用外部 etcd 配置
etcd:
  external:
    endpoints:
    - "https://192.168.1.27:2379"
    - "https://192.168.1.28:2379"
    - "https://192.168.1.29:2379"
    caFile: "/etc/etcd/ssl/etcd-root-ca.pem"
    certFile: "/etc/etcd/ssl/etcd.pem"
    keyFile: "/etc/etcd/ssl/etcd-key.pem"
# 网络配置
networking:
  serviceSubnet: "10.25.0.0/16"
  # flannel
  #podSubnet: "10.30.0.1/16"
  # terway
  # 需要在阿里控制面板创建专门的虚拟交换机给pod使用
  podSubnet: "192.168.32.0/19"
  dnsDomain: "cluster.local"
kubernetesVersion: "v1.21.13"
# 全局 apiserver LB 地址，由于采用了 nginx/haproxy 负载，所以直接指向本地既可
controlPlaneEndpoint: "127.0.0.1:6443"
apiServer:
  # apiserver 的自定义扩展参数
  extraArgs:
    v: "4"
    alsologtostderr: "true"
    # 审计日志相关配置
    audit-log-maxage: "20"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    audit-log-path: "/var/log/kube-audit/audit.log"
    audit-policy-file: "/etc/kubernetes/audit-policy.yaml"
    authorization-mode: "Node,RBAC"
    event-ttl: "720h"
    runtime-config: "api/all=true"
    service-node-port-range: "30000-50000"
    service-cluster-ip-range: "10.25.0.0/16"
  # 由于自行定义了审计日志配置，所以需要将宿主机上的审计配置
  # 挂载到 kube-apiserver 的 pod 容器中
  extraVolumes:
  - name: "audit-config"
    hostPath: "/etc/kubernetes/audit-policy.yaml"
    mountPath: "/etc/kubernetes/audit-policy.yaml"
    readOnly: true
    pathType: "File"
  - name: "audit-log"
    hostPath: "/var/log/kube-audit"
    mountPath: "/var/log/kube-audit"
    pathType: "DirectoryOrCreate"
  # 这里是 apiserver 的证书地址配置
  # 为了防止以后出特殊情况，我增加了一个泛域名
  certSANs:
  - "*.k8s.node"
  - "192.168.1.27"
  - "192.168.1.28"
  - "192.168.1.30"
  - "127.0.0.1"
  timeoutForControlPlane: 5m
controllerManager:
  extraArgs:
    v: "4"
    # 宿主机 ip 掩码
    node-cidr-mask-size: "19"
    deployment-controller-sync-period: "10s"
    experimental-cluster-signing-duration: "87600h"
    node-monitor-grace-period: "20s"
    pod-eviction-timeout: "2m"
    terminated-pod-gc-threshold: "30"
scheduler:
  extraArgs:
    v: "4"
certificatesDir: "/etc/kubernetes/pki"
# gcr.io 被墙
imageRepository: "registry.aliyuncs.com/google_containers"
#imageRepository: "registry.cn-chengdu.aliyuncs.com/archky-k8s"
clusterName: "kuberentes"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
# kubelet specific options here
# 此配置保证了 kubelet 能在 swap 开启的情况下启动
failSwapOn: false
nodeStatusUpdateFrequency: 5s
# 一些驱逐阀值，具体自行查文档修改
evictionSoft:
  "imagefs.available": "15%"
  "memory.available": "512Mi"
  "nodefs.available": "15%"
  "nodefs.inodesFree": "10%"
evictionSoftGracePeriod:
  "imagefs.available": "3m"
  "memory.available": "1m"
  "nodefs.available": "3m"
  "nodefs.inodesFree": "1m"
evictionHard:
  "imagefs.available": "10%"
  "memory.available": "256Mi"
  "nodefs.available": "10%"
  "nodefs.inodesFree": "5%"
evictionMaxPodGracePeriod: 30
imageGCLowThresholdPercent: 70
imageGCHighThresholdPercent: 80
kubeReserved:
  "cpu": "500m"
  "memory": "512Mi"
  "ephemeral-storage": "1Gi"
rotateCertificates: true
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
# kube-proxy specific options here
# 建议与 podSubnet 保持一致
clusterCIDR: "192.168.32.0/19"
# 启用 ipvs 模式
mode: "ipvs"
ipvs:
  minSyncPeriod: 5s
  syncPeriod: 5s
  # ipvs 负载策略
  scheduler: "wrr"

1
2
3
4

# 安装前先检查配置文件
kubeadm init phase preflight --config=kubeadm.yaml
# --v 指定日志级别
kubeadm init --config kubeadm.yaml --upload-certs --v=6

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a Pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  /docs/concepts/cluster-administration/addons/

You can now join any number of machines by running the following on each node
as root:

  kubeadm join 127.0.0.1:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 127.0.0.1:6443 --token <token> \
    --discovery-token-ca-cert-hash sha256:<hash>

1
2
3
4
5
6

#!/bin/bash

rm -rf $HOME/.kube
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

1
2
3
4
5
6

# 查看集群组件状态
kubectl get cs
# 查看 节点状态
kubectl get node
# 查看所有pod状态
kubectl get pod -A

1
2
3
4
5
6
7
8
9

# 删除节点
kubectl drain <node_name> --delete-emptydir-data --force --ignore-daemonsets
# 重置集群
kubeadm reset
# 删除多余的虚拟网卡
ip link del flannel.1
ip link del cni0
# 清空 ETCD 数据 慎重
etcdctl --cacert /etc/etcd/ssl/etcd-root-ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem --endpoints https://192.168.1.27:2379,https://192.168.1.28:2379,https://192.168.1.30:2379 del "" --prefix

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

{
  "Version": "1",
  "Statement": [
    {
      "Action": [
        "ecs:CreateNetworkInterface",
        "ecs:DescribeNetworkInterfaces",
        "ecs:AttachNetworkInterface",
        "ecs:DetachNetworkInterface",
        "ecs:DeleteNetworkInterface",
        "ecs:DescribeInstanceAttribute",
        "ecs:DescribeInstanceTypes",
        "ecs:AssignPrivateIpAddresses",
        "ecs:UnassignPrivateIpAddresses",
        "ecs:DescribeInstances",
        "ecs:ModifyNetworkInterfaceAttribute"
      ],
      "Resource": [
        "*"
      ],
      "Effect": "Allow"
    },
    {
      "Action": [
        "vpc:DescribeVSwitches"
      ],
      "Resource": [
        "*"
      ],
      "Effect": "Allow"
    }
  ]
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368

apiVersion: v1
kind: ServiceAccount
metadata:
  name: terway
  namespace: kube-system

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: terway-pod-reader
  namespace: kube-system
rules:
  - apiGroups: [ "" ]
    resources: [ "pods", "nodes", "namespaces", "configmaps", "serviceaccounts" ]
    verbs: [ "get", "watch", "list", "update" ]
  - apiGroups: [ "" ]
    resources:
      - events
    verbs:
      - create
  - apiGroups: [ "networking.k8s.io" ]
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
  - apiGroups: [ "extensions" ]
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
  - apiGroups: [ "" ]
    resources:
      - pods/status
    verbs:
      - update
  - apiGroups: [ "crd.projectcalico.org" ]
    resources: [ "*" ]
    verbs: [ "*" ]

---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: terway-binding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: terway-pod-reader
subjects:
  - kind: ServiceAccount
    name: terway
    namespace: kube-system

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: eni-config
  namespace: kube-system
data:
  eni_conf: |
    {
      "version": "1",
      "access_key": "ak",
      "access_secret": "ak",
      "security_group": "sg-xxx",
      "service_cidr": "10.25.0.0/16",
      "vswitches": {
        "cn-hongkong-b": ["vsw-xxx"]
      },
      "max_pool_size": 5,
      "min_pool_size": 0
    }    
  10-terway.conf: |
    {
      "cniVersion": "0.3.1",
      "name": "terway",
      "type": "terway",
      "eniip_virtual_type": "Veth",
      "ip_stack": "ipv4"
    }    
  # eniip_virtual_type: virtual type for eni multi ip "Veth" || "IPVlan"
  disable_network_policy: "false"

---

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: terway
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: terway
  template:
    metadata:
      labels:
        app: terway
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      hostPID: true
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
        - operator: "Exists"
      terminationGracePeriodSeconds: 0
      serviceAccountName: terway
      hostNetwork: true
      initContainers:
        - name: terway-init
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          command:
            - 'sh'
            - '-c'
            - 'cp /usr/bin/terway /opt/cni/bin/;
                  chmod +x /opt/cni/bin/terway;
                  cp /etc/eni/10-terway.conf /etc/cni/net.d/;
                  sysctl -w net.ipv4.conf.eth0.rp_filter=0;
                  modprobe sch_htb || true;
                  chroot /host sh -c "systemctl disable eni.service; rm -f /etc/udev/rules.d/75-persistent-net-generator.rules /lib/udev/rules.d/60-net.rules /lib/udev/rules.d/61-eni.rules /lib/udev/write_net_rules && udevadm control --reload-rules && udevadm trigger; true"'
          volumeMounts:
            - name: configvolume
              mountPath: /etc/eni
            - name: cni-bin
              mountPath: /opt/cni/bin/
            - name: cni
              mountPath: /etc/cni/net.d/
            - mountPath: /lib/modules
              name: lib-modules
            - mountPath: /host
              name: host-root
      containers:
        - name: terway
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          command: [ '/usr/bin/terwayd', '-log-level', 'debug', '-daemon-mode', 'ENIMultiIP' ]
          securityContext:
            privileged: true
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          volumeMounts:
            - name: configvolume
              mountPath: /etc/eni
            - mountPath: /var/run/
              name: eni-run
            - mountPath: /opt/cni/bin/
              name: cni-bin
            - mountPath: /lib/modules
              name: lib-modules
            - mountPath: /var/lib/cni/networks
              name: cni-networks
            - mountPath: /var/lib/cni/terway
              name: cni-terway
            - mountPath: /var/lib/kubelet/device-plugins
              name: device-plugin-path
        - name: policy
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          command: [ "/bin/policyinit.sh" ]
          env:
            - name: NODENAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: DISABLE_POLICY
              valueFrom:
                configMapKeyRef:
                  name: eni-config
                  key: disable_network_policy
                  optional: true
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          livenessProbe:
            tcpSocket:
              port: 9099
              host: localhost
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
            tcpSocket:
              port: 9099
              host: localhost
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
      volumes:
        - name: configvolume
          configMap:
            name: eni-config
            items:
              - key: eni_conf
                path: eni.json
              - key: 10-terway.conf
                path: 10-terway.conf
        - name: cni-bin
          hostPath:
            path: /opt/cni/bin
            type: "Directory"
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: eni-run
          hostPath:
            path: /var/run/
            type: "Directory"
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: cni-networks
          hostPath:
            path: /var/lib/cni/networks
        - name: cni-terway
          hostPath:
            path: /var/lib/cni/terway
        - name: device-plugin-path
          hostPath:
            path: /var/lib/kubelet/device-plugins
            type: "Directory"
        - name: host-root
          hostPath:
            path: /
            type: "Directory"

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: felixconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: FelixConfiguration
    plural: felixconfigurations
    singular: felixconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPConfiguration
    plural: bgpconfigurations
    singular: bgpconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ippools.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPPool
    plural: ippools
    singular: ippool

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: hostendpoints.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: HostEndpoint
    plural: hostendpoints
    singular: hostendpoint

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: ClusterInformation
    plural: clusterinformations
    singular: clusterinformation

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworkpolicies.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkPolicy
    plural: globalnetworkpolicies
    singular: globalnetworkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworksets.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkSet
    plural: globalnetworksets
    singular: globalnetworkset

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networkpolicies.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 配置说明
data:
  eni_conf: |
    {
      "version": "1",
      "access_key": "ak",
      "access_secret": "sk",
      "security_group": "ESC的使用的安全组",
      # 与 kubeadm.yaml 的 serviceSubnet 保持一致
      "service_cidr": "10.25.0.0/16",
      # 交换机信息配置，区域和ID
      "vswitches": {
        "cn-hangzhou-k": ["交换机ID"]
      }    

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233

---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
  - configMap
  - secret
  - emptyDir
  - hostPath
  allowedHostPaths:
  - pathPrefix: "/etc/cni/net.d"
  - pathPrefix: "/etc/kube-flannel"
  - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups: ['extensions']
  resources: ['podsecuritypolicies']
  verbs: ['use']
  resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }    
  net-conf.json: |
    {
      "Network": "10.30.0.0/16",
      "Backend": {
        "Type": "ali-vpc"
      }
    }    
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      # 增加DNS服务器223.6.6.6（别的公共服务器也行）
      dnsPolicy: "None"
      dnsConfig:
        nameservers:
        - 223.6.6.6
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        # ali-vpc
        - name: ACCESS_KEY_ID
          value: Ak
        - name: ACCESS_KEY_SECRET
          value: Sk
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg