cat > /etc/sysctl.d/k8s.conf <<-'EOF'
# 以下3个参数是containerd所依赖的内核参数
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv6.conf.all.disable_ipv6 = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

sysctl --system

cat > /etc/modules-load.d/ipvs.conf <<-'EOF'
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

mkdir /script
cat > /script/copy_software.sh <<-'EOF'
#!/bin/bash
# k8s 二进制文件分发脚本（带结果统计）

# 需要分发的节点
WORKER_NODES=("k8s-worker01" "k8s-worker02")

# 目录
SRC_DIR="/usr/local/bin"
DST_DIR="/usr/local/bin"

# 文件
BIN_FILES=("kubelet" "kube-proxy")

# 统计
SUCCESS_COUNT=0
FAIL_COUNT=0
FAILED_NODES=()

echo "开始分发 kubelet / kube-proxy 文件..."

# 循环节点
for node in "${WORKER_NODES[@]}"; do
    echo ">>> 处理 Worker 节点: $node"
    NODE_SUCCESS=true

    for file in "${BIN_FILES[@]}"; do
        if [[ -f "$SRC_DIR/$file" ]]; then
            scp -q "$SRC_DIR/$file" "$node:$DST_DIR/"
            if [[ $? -eq 0 ]]; then
                echo "   [OK] $file 已传输到 $node:$DST_DIR"
                ssh "$node" "chmod +x $DST_DIR/$file"
            else
                echo "   [ERROR] $file 传输到 $node 失败"
                NODE_SUCCESS=false
            fi
        else
            echo "   [WARN] 本地缺少文件: $SRC_DIR/$file"
            NODE_SUCCESS=false
        fi
    done

    if $NODE_SUCCESS; then
        ((SUCCESS_COUNT++))
    else
        ((FAIL_COUNT++))
        FAILED_NODES+=("$node")
    fi
done

# 总结
echo "======================"
echo "分发完成"
echo "成功节点数量: $SUCCESS_COUNT"
echo "失败节点数量: $FAIL_COUNT"

if [[ $FAIL_COUNT -gt 0 ]]; then
    echo "失败节点列表: ${FAILED_NODES[*]}"
fi
echo "======================"
EOF

chmod +x /script/copy_software.sh
/script/copy_software.sh

cat > /script/copy_etcd_certificate.sh <<-'EOF'
#!/bin/bash
# etcd证书分发脚本（带结果统计）

# 需要分发的节点
NODES=("k8s-controller02" "k8s-controller03")

# 需要分发的文件
FILES=(
  "etcd-ca-key.pem"
  "etcd-ca.pem"
  "etcd-key.pem"
  "etcd.pem"
)

SRC_DIR="/etc/etcd/ssl"
DST_DIR="/etc/etcd/ssl"

SUCCESS_COUNT=0
FAIL_COUNT=0
FAILED_NODES=()

echo "开始分发etcd证书..."

# 循环节点
for node in "${NODES[@]}"; do
    echo ">>> 处理节点: $node"

    ssh "$node" "mkdir -p $DST_DIR"

    NODE_SUCCESS=true

    for file in "${FILES[@]}"; do
        if [[ -f "$SRC_DIR/$file" ]]; then
            scp -q "$SRC_DIR/$file" "$node:$DST_DIR/"
            if [[ $? -eq 0 ]]; then
                echo "   [OK] $file 已传输到 $node:$DST_DIR"
            else
                echo "   [ERROR] $file 传输到 $node 失败"
                NODE_SUCCESS=false
            fi
        else
            echo "   [WARN] 本地缺少文件: $SRC_DIR/$file"
            NODE_SUCCESS=false
        fi
    done

    if $NODE_SUCCESS; then
        ((SUCCESS_COUNT++))
    else
        ((FAIL_COUNT++))
        FAILED_NODES+=("$node")
    fi
done

echo "======================"
echo "证书分发完成"
echo "成功节点数量: $SUCCESS_COUNT"
echo "失败节点数量: $FAIL_COUNT"

if [[ $FAIL_COUNT -gt 0 ]]; then
    echo "失败节点列表: ${FAILED_NODES[*]}"
fi
echo "======================"
EOF

chmod +x /script/copy_etcd_certificate.sh
/script/copy_etcd_certificate.sh

cat > /script/copy_kubeconfig_certificate.sh <<-'EOF'
#!/bin/bash
# k8s 证书 + kubeconfig 分发脚本（带结果统计）

# 控制节点（证书 + kubeconfig）
NODES=("k8s-controller02" "k8s-controller03")

# 工作节点（只需要 kube-proxy.kubeconfig）
WORKER_NODES=("k8s-worker01" "k8s-worker02")

# 证书目录
SRC_CERT_DIR="/etc/kubernetes/pki/"
DST_CERT_DIR="/etc/kubernetes/pki/"
CERT_FILES=("$SRC_CERT_DIR"/*)

# kubeconfig 目录（只选取以 .kubeconfig 结尾的文件）
SRC_KUBECONFIG_DIR="/etc/kubernetes"
DST_KUBECONFIG_DIR="/etc/kubernetes"
KUBECONFIG_FILES=("$SRC_KUBECONFIG_DIR"/*.kubeconfig)

# kube-proxy kubeconfig 文件
KUBEPROXY_FILE="$SRC_KUBECONFIG_DIR/kube-proxy.kubeconfig"

# 统计
SUCCESS_COUNT=0
FAIL_COUNT=0
FAILED_NODES=()

echo "开始分发 etcd 证书和 kubeconfig 文件..."

# === 控制节点分发 ===
for node in "${NODES[@]}"; do
    echo ">>> 处理控制节点: $node"
    NODE_SUCCESS=true

    # === 分发证书文件 ===
    ssh "$node" "mkdir -p $DST_CERT_DIR"
    for file in "${CERT_FILES[@]}"; do
        if [[ -f "$file" ]]; then
            scp -q "$file" "$node:$DST_CERT_DIR/"
            if [[ $? -eq 0 ]]; then
                echo "   [OK] $(basename "$file") 已传输到 $node:$DST_CERT_DIR"
            else
                echo "   [ERROR] $(basename "$file") 传输到 $node 失败"
                NODE_SUCCESS=false
            fi
        else
            echo "   [WARN] 本地缺少文件: $file"
            NODE_SUCCESS=false
        fi
    done

    # === 分发 kubeconfig 文件（只处理 .kubeconfig） ===
    ssh "$node" "mkdir -p $DST_KUBECONFIG_DIR"
    for file in "${KUBECONFIG_FILES[@]}"; do
        if [[ -f "$file" ]]; then
            scp -q "$file" "$node:$DST_KUBECONFIG_DIR/"
            if [[ $? -eq 0 ]]; then
                echo "   [OK] $(basename "$file") 已传输到 $node:$DST_KUBECONFIG_DIR"
            else
                echo "   [ERROR] $(basename "$file") 传输到 $node 失败"
                NODE_SUCCESS=false
            fi
        else
            echo "   [WARN] 本地缺少文件: $file"
            NODE_SUCCESS=false
        fi
    done

    # 节点统计
    if $NODE_SUCCESS; then
        ((SUCCESS_COUNT++))
    else
        ((FAIL_COUNT++))
        FAILED_NODES+=("$node")
    fi
done

# === 工作节点分发 kube-proxy.kubeconfig ===
for node in "${WORKER_NODES[@]}"; do
    echo ">>> 处理工作节点: $node"
    NODE_SUCCESS=true

    ssh "$node" "mkdir -p $DST_KUBECONFIG_DIR"
    if [[ -f "$KUBEPROXY_FILE" ]]; then
        scp -q "$KUBEPROXY_FILE" "$node:$DST_KUBECONFIG_DIR/"
        if [[ $? -eq 0 ]]; then
            echo "   [OK] kube-proxy.kubeconfig 已传输到 $node:$DST_KUBECONFIG_DIR"
        else
            echo "   [ERROR] kube-proxy.kubeconfig 传输到 $node 失败"
            NODE_SUCCESS=false
        fi
    else
        echo "   [WARN] 本地缺少文件: $KUBEPROXY_FILE"
        NODE_SUCCESS=false
    fi

    # 节点统计
    if $NODE_SUCCESS; then
        ((SUCCESS_COUNT++))
    else
        ((FAIL_COUNT++))
        FAILED_NODES+=("$node")
    fi
done

# === 总结 ===
echo "======================"
echo "分发完成"
echo "成功节点数量: $SUCCESS_COUNT"
echo "失败节点数量: $FAIL_COUNT"

if [[ $FAIL_COUNT -gt 0 ]]; then
    echo "失败节点列表: ${FAILED_NODES[*]}"
fi
echo "======================"
EOF

chmod +x /script/copy_kubeconfig_certificate.sh
/script/copy_kubeconfig_certificate.sh

cat > /etc/etcd/etcd.config.yml << EOF 
name: 'k8s-controller01'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.109.100:2380'
listen-client-urls: 'https://192.168.109.100:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.109.100:2380'
advertise-client-urls: 'https://192.168.109.100:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-controller01=https://192.168.109.100:2380,k8s-controller02=https://192.168.109.101:2380,k8s-controller03=https://192.168.109.102:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /etc/etcd/etcd.config.yml << EOF 
name: 'k8s-controller02'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.109.101:2380'
listen-client-urls: 'https://192.168.109.101:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.109.101:2380'
advertise-client-urls: 'https://192.168.109.101:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-controller01=https://192.168.109.100:2380,k8s-controller02=https://192.168.109.101:2380,k8s-controller03=https://192.168.109.102:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /etc/etcd/etcd.config.yml << EOF 
name: 'k8s-controller03'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.109.102:2380'
listen-client-urls: 'https://192.168.109.102:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.109.102:2380'
advertise-client-urls: 'https://192.168.109.102:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-controller01=https://192.168.109.100:2380,k8s-controller02=https://192.168.109.101:2380,k8s-controller03=https://192.168.109.102:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/etcd/ssl/etcd.pem'
  key-file: '/etc/etcd/ssl/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /usr/lib/systemd/system/etcd.service << EOF

[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

EOF

cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak
cat >/etc/haproxy/haproxy.cfg<<"EOF"
global
 maxconn 2000
 ulimit-n 16384
 log 127.0.0.1 local0 err
 stats timeout 30s

defaults
 log global
 mode http
 option httplog
 timeout connect 5000
 timeout client 50000
 timeout server 50000
 timeout http-request 15s
 timeout http-keep-alive 15s


frontend monitor-in
 bind *:33305
 mode http
 option httplog
 monitor-uri /monitor

frontend k8s-master
 bind 0.0.0.0:9443
 bind 127.0.0.1:9443
 mode tcp
 option tcplog
 tcp-request inspect-delay 5s
 default_backend k8s-master


backend k8s-master
 mode tcp
 option tcplog
 option tcp-check
 balance roundrobin
 default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
 server  k8s-controller01  192.168.109.100:6443 check
 server  k8s-controller02  192.168.109.101:6443 check
 server  k8s-controller03  192.168.109.102:6443 check
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    # 注意网卡名
    interface ens33 
    mcast_src_ip 192.168.109.100
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.109.108
    }
    track_script {
      chk_apiserver 
} }

EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface ens33
    mcast_src_ip 192.168.109.101
    virtual_router_id 51
    priority 80
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.109.108
    }
    track_script {
      chk_apiserver 
} }

EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface ens33
    mcast_src_ip 192.168.109.102
    virtual_router_id 51
    priority 50
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.109.108
    }
    track_script {
      chk_apiserver 
} }

EOF

cat >  /etc/keepalived/check_apiserver.sh << EOF
#!/bin/bash

err=0
for k in \$(seq 1 3)
do
    check_code=\$(pgrep haproxy)
    if [[ \$check_code == "" ]]; then
        err=\$(expr \$err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ \$err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_apiserver.sh

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.109.100 \\
      --service-cluster-ip-range=10.96.0.0/12  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.109.100:2379,https://192.168.109.101:2379,https://192.168.109.102:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=front-proxy-client  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.109.101 \\
      --service-cluster-ip-range=10.96.0.0/12  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.109.100:2379,https://192.168.109.101:2379,https://192.168.109.102:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \\
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=front-proxy-client  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF

cat > /usr/lib/systemd/system/kube-apiserver.service  << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.109.102 \\
      --service-cluster-ip-range=10.96.0.0/12  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.109.100:2379,https://192.168.109.101:2379,https://192.168.109.102:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \\
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=front-proxy-client  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF

cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
      --v=2 \\
      --bind-address=0.0.0.0 \\
      --root-ca-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \\
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\
      --leader-elect=true \\
      --use-service-account-credentials=true \\
      --node-monitor-grace-period=40s \\
      --node-monitor-period=5s \\
      --controllers=*,bootstrapsigner,tokencleaner \\
      --allocate-node-cidrs=true \\
      --service-cluster-ip-range=10.96.0.0/12 \\
      --cluster-cidr=172.16.0.0/12 \\
      --node-cidr-mask-size-ipv4=24 \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF