今日天气
  1. 博客/

使用 斐讯n1 & openWrt 搭建 k3s 集群

·3283 字·7 分钟· ·
k3s openwrt n1
Johny
作者
Johny
熟练的 云原生搬砖师
Table of Contents

环境说明
#

软件说明
#

  • K3s version : v1.19.7+k3s1

  • Docker version: 19.03.13 (n1),19.03.12 (openwrt)

  • 写盘工具: balenaEtcher

机器说明
#

IP 地址机型配置操作系统角色
192.168.8.1占美 (机型不详)4c 2g ( cpu N2940 )openWrt(X86_64 自编译node/agent
192.168.8.112斐讯 N14c 2gArmbian ( 5.0.2-aml-s905)master/server

openWrt 自编译
#

需要自编译的原因是因为,之前使用 esir 打包提供的高大全 op 固件,部署k3s 时发现,内核没有开启vxlan的特性,导致部署不成功,如你使用的 op 固件已开启了 vxlan 特性可省略此步骤,查看方法:

root@OpenWrt:~# lsmod|grep -i vxlan  # 打印输入表示具有 vxlan 特性
ip6_udp_tunnel         16384  1 vxlan
udp_tunnel             16384  1 vxlan
vxlan                  53248  0 

此处感谢 esir 等大佬 提供的 github actions ci 工作流

仓库地址 视频教程

fork 仓库代码
#

登录 github, 并将 代码 fork 至自己仓库

image-20210126095535833

git clone 代码
#

git clone fork https://github.com.cnpmjs.org/cdryzun/AutoBuild-OpenWrt.git

cd AutoBuild-OpenWrt 

修改配置文件
#

修改工作流文件
#

打开自己心爱的编辑器

且 演示 x86_64 平台

 code .github/workflows/Build_OP_x86_64.yml  # 编辑 ci 构建脚本
 
 ...
   push:
   branches: 
     - master  # 打开注释,注意空格

修改构建文件
#

code x86_64.config

配置参考资料
#

CONFIG_PACKAGE_grub2-efi=y
CONFIG_EFI_IMAGES=y
CONFIG_TARGET_IMAGES_GZIP=y
CONFIG_TARGET_KERNEL_PARTSIZE=16
CONFIG_TARGET_ROOTFS_PARTSIZE=300
# CONFIG_GRUB_CONSOLE is not set
CONFIG_PACKAGE_luci-app-docker=y
CONFIG_PACKAGE_luci-i18n-docker-zh-cn=y
CONFIG_PACKAGE_luci-app-ssr-plus=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Shadowsocks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Simple_obfs=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray_plugin=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Kcptun=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Server=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Socks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_NaiveProxy=y
CONFIG_PACKAGE_luci-theme-argon=y
CONFIG_PACKAGE_luci-app-mwan3=y
CONFIG_PACKAGE_luci-app-mwan3helper=y
CONFIG_PACKAGE_luci-i18n-mwan3-zh-cn=y
CONFIG_PACKAGE_luci-i18n-mwan3helper-zh-cn=y
CONFIG_PACKAGE_luci-app-syncdial=y
CONFIG_PACKAGE_luci-app-sfe=y
CONFIG_PACKAGE_luci-app-ttyd=y
CONFIG_PACKAGE_luci-i18n-ttyd-zh-cn=y
CONFIG_PACKAGE_luci-app-webadmin=y
CONFIG_PACKAGE_luci-i18n-webadmin-zh-cn=y
CONFIG_PACKAGE_luci-app-zerotier=y
CONFIG_PACKAGE_luci-app-frpc=y
CONFIG_PACKAGE_luci-app-kodexplorer=y
CONFIG_PACKAGE_luci-app-nfs=y
CONFIG_PACKAGE_ss=y
CONFIG_PACKAGE_luci-app-nps=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-rclone=y
CONFIG_PACKAGE_luci-app-radicale=y
CONFIG_PACKAGE_luci-app-qbittorrent=y
CONFIG_PACKAGE_luci-app-samba=y
CONFIG_PACKAGE_luci-app-wol=y
CONFIG_PACKAGE_luci-app-xlnetacc=y
CONFIG_PACKAGE_luci-app-bird1-ipv4=y
CONFIG_PACKAGE_luci-app-bird1-ipv6=y
CONFIG_PACKAGE_luci-app-commands=y
CONFIG_PACKAGE_luci-app-diskman=y
CONFIG_PACKAGE_luci-app-dnscrypt-proxy=y
CONFIG_PACKAGE_luci-app-dnsforwarder=y
CONFIG_PACKAGE_luci-app-familycloud=y
CONFIG_PACKAGE_luci-app-hd-idle=y
CONFIG_PACKAGE_luci-app-netdata=y
CONFIG_PACKAGE_ipv6helper=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_curl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_wget=y
CONFIG_PACKAGE_lrzsz=y
CONFIG_PACKAGE_vim=y
CONFIG_PACKAGE_zip=y
CONFIG_PACKAGE_kmod-kvm-amd=y
CONFIG_PACKAGE_kmod-kvm-intel=y
CONFIG_PACKAGE_kmod-kvm-x86=y
CONFIG_OPENSSL_ENGINE_CRYPTO=y
CONFIG_OPENSSL_ENGINE_DIGEST=y
CONFIG_OPENSSL_WITH_CAMELLIA=y
CONFIG_OPENSSL_WITH_COMPRESSION=y
CONFIG_OPENSSL_WITH_DTLS=y
CONFIG_OPENSSL_WITH_EC2M=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_GOST=y
CONFIG_OPENSSL_WITH_IDEA=y
CONFIG_OPENSSL_WITH_MDC2=y
CONFIG_OPENSSL_WITH_RFC3779=y
CONFIG_OPENSSL_WITH_SEED=y
CONFIG_OPENSSL_WITH_WHIRLPOOL=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_KERNEL_BUILD_USER="treeSir Playground"
CONFIG_GRUB_TITLE="OpenWrt AutoBuild by treeSirPlayground"
CONFIG_PACKAGE_kmod-usb-ohci=y
CONFIG_PACKAGE_kmod-usb-ohci-pci=y
CONFIG_PACKAGE_kmod-usb-storage-uas=y
CONFIG_PACKAGE_kmod-usb-uhci=y
CONFIG_PACKAGE_kmod-sdhci=y
CONFIG_PACKAGE_kmod-usb-ehci=y
CONFIG_PACKAGE_kmod-usb2=y
CONFIG_PACKAGE_kmod-usb2-pci=y
CONFIG_PACKAGE_luci-app-passwall=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Brook=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan_GO=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_simple-obfs=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_v2ray-plugin=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_https_dns_proxy=n
# CONFIG_PACKAGE_naiveproxy=y
# CONFIG_PACKAGE_nspr=y
# CONFIG_PACKAGE_shadowsocks-libev-ss-server=y
# CONFIG_PACKAGE_ssocks=y
# CONFIG_PACKAGE_ssocksd=y
# CONFIG_PACKAGE_trojan-go=y
# CONFIG_PACKAGE_trojan-plus=y
# CONFIG_PACKAGE_chinadns-ng=y
CONFIG_TARGET_x86=y
CONFIG_TARGET_x86_64=y
CONFIG_TARGET_x86_64_Generic=y
CONFIG_FEED_luci=y
CONFIG_FEED_packages=y
CONFIG_FEED_routing=y
CONFIG_FEED_telephony=y
CONFIG_IPTABLES_CONNLABEL=y
CONFIG_IPTABLES_NFTABLES=y
CONFIG_KERNEL_BLK_CGROUP=y
CONFIG_KERNEL_CFS_BANDWIDTH=y
CONFIG_KERNEL_CGROUPS=y
CONFIG_KERNEL_CGROUP_CPUACCT=y
CONFIG_KERNEL_CGROUP_DEVICE=y
CONFIG_KERNEL_CGROUP_FREEZER=y
CONFIG_KERNEL_CGROUP_PIDS=y
CONFIG_KERNEL_CGROUP_SCHED=y
CONFIG_KERNEL_CPUSETS=y
CONFIG_KERNEL_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_KERNEL_FAIR_GROUP_SCHED=y
CONFIG_KERNEL_FREEZER=y
CONFIG_KERNEL_IPC_NS=y
CONFIG_KERNEL_KEYS=y
CONFIG_KERNEL_LXC_MISC=y
CONFIG_KERNEL_MEMCG=y
CONFIG_KERNEL_MM_OWNER=y
CONFIG_KERNEL_NAMESPACES=y
CONFIG_KERNEL_NETPRIO_CGROUP=y
CONFIG_KERNEL_NET_CLS_CGROUP=y
CONFIG_KERNEL_NET_NS=y
CONFIG_KERNEL_PID_NS=y
CONFIG_KERNEL_POSIX_MQUEUE=y
CONFIG_KERNEL_PROC_PID_CPUSET=y
CONFIG_KERNEL_RESOURCE_COUNTERS=y
CONFIG_KERNEL_SECCOMP=y
CONFIG_KERNEL_SECCOMP_FILTER=y
CONFIG_KERNEL_USER_NS=y
CONFIG_KERNEL_UTS_NS=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ip-bridge=y
CONFIG_PACKAGE_ip-full=y
CONFIG_PACKAGE_ipset=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_iptables-mod-ipopt=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-br-netfilter=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hw-ccp=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rsa=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-fuse=y
CONFIG_PACKAGE_kmod-ikconfig=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-ipopt=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-leds-gpio=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-ipvs=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-base=y
CONFIG_PACKAGE_kmod-random-core=y
CONFIG_PACKAGE_kmod-softdog=y
CONFIG_PACKAGE_kmod-sound-core=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-core=y
CONFIG_PACKAGE_kmod-usb3=y
CONFIG_PACKAGE_kmod-veth=y
CONFIG_PACKAGE_kmod-vxlan=y
CONFIG_PACKAGE_kmod-ppp=y
CONFIG_PACKAGE_kmod-pppox=y
CONFIG_PACKAGE_libipset=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnftnl=y
CONFIG_TARGET_ROOTFS_PARTSIZE=1024
# CONFIG_PACKAGE_iptables-mod-conntrack-label is not set
# CONFIG_PACKAGE_kmod-ipt-conntrack-label is not set
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y

示例配置且供参考使用

构建固件
#

触发构建
#

提交代码

image-20210126102255870

提交代码前,检查一下 actions 是否是打开状态

git add .github/workflows/Build_OP_x86_64.yml
git add ./x86_64.config

git commit -m "build custom openwrt"

git push

提交完成后,等待构建结束

下载固件
#

构建完成后如何下载 op固件 文件

image-20210126102444462

点击后 往下面拉,点击进行下载

image-20210126102631119

省略 op 写入步骤, 下载固件后解压进入 OpenWrt/targets/x86/64 目录, 就可以看到固件了。

image-20210126104146816

N1 k3s master 部署
#

n1 使用系统为 Armbian,同样省略写入步骤,n1怎么刷入 armbian 系统在网上资料还是非常多的。

docker 配置文件
#

此处为 n1 docker 配置文件,不一定适合你,可截取部分参考使用。

 cat /etc/docker/daemon.json 
{
  "experimental": false,
  "registry-mirrors": [
    "https://7bezldxe.mirror.aliyuncs.com"
  ],
  "oom-score-adjust": -1000,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  },
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "features": {
    "buildkit": true
  },
  "insecure-registries": [
    "idocker.io"
  ]
}

配置项简短说明:

  • “log-driver”: “json-file” 设置json 格式日志
  • “oom-score-adjust”: -1000 防止容器被 内核 oom
  • “log-opts” 设置容器日志大小
  • “max-concurrent-downloads”: 10 并行下载容器数量
  • “max-concurrent-uploads”: 10 并行上传
  • “storage-driver”: “overlay2” 设置存储驱动为 overlay2
  • “bip” 容器默认的网段
  • “registry-mirrors” 配置镜像下载加速这里使用的是阿里云的
  • “insecure-registries” 信任的私服地址

如若修改了配置,记得重启一下服务查看是否生效

service docker restart

docker info

部署前的检查
#

这个打印的内容是 部署完成后打印 的,部署前的检查忘记做记录了。

root@n1:/# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/ad8f0f93ebb9db5c507884fcdec249d73dd348293dac194e01462c57815cca46/bin:
- sha256sum: good
- links: good

System:
- /sbin iptables v1.6.1: older than v1.8
- swap: should be disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled (as module)
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: enabled
- CONFIG_CGROUP_PERF: enabled
- CONFIG_CGROUP_HUGETLB: enabled
- CONFIG_NET_CLS_CGROUP: enabled (as module)
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: enabled
- CONFIG_EXT4_FS_SECURITY: enabled
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: enabled (as module)
      - CONFIG_CRYPTO_SEQIV: enabled
      - CONFIG_CRYPTO_GHASH: enabled (as module)
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: enabled
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

部署 k3s
#

官方提供了部署脚本,部署起来还是非常方便的

export INSTALL_K3S_VERSION=v1.19.7+k3s1

export INSTALL_K3S_EXEC="--docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig  ~/.kube/config --write-kubeconfig-mode 666"

curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -
 
root@n1:~#  kubectl get nodes
NAME   STATUS   ROLES    AGE   VERSION
n1     Ready    master   6s    v1.19.7+k3s1

openWrt k3s agent 部署
#

在 openwrt 中部署 k3s 时就不能使用 官方提供的自动部署脚本了,运行脚本时会提示错误 [ERROR] Can not find systemd or openrc to use as a process supervisor for k3s ,提示说明已经非常清楚了,好在 github 上有位大佬提供了 openwrt k3s 的安装包,谢谢这位大佬。

Docker 配置
#

此处为 openwrt 中 docker 配置,配置项说明请参考上面 n1 中 docker 配置项说明。

 cat /etc/docker/daemon.json 
{
"data-root": "/data/docker-root",
"log-level": "warn",
"oom-score-adjust": -1000,
"log-driver": "json-file",
  "log-opts": {
   "max-size": "100m",
   "max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"insecure-registries": ["idocker.io"],
"registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com"],
"storage-driver": "overlay2",
"storage-opts": [
   "overlay2.override_kernel_check=true"
]
}

⚠️ 注意: 配置项修改后,多需要重启一下 docker 才会生效

/etc/init.d/dockerd restart

k3s agent 部署
#

下载安装包
#

wget https://github.com/discordianfish/k3s-openwrt/releases/download/9/k3s_1.17.4+k3s1_x86_64.opk

opkg install ./k3s_1.17.4+k3s1_x86_64.opk

root@OpenWrt:/opt/tools# /usr/bin/k3s --version
k3s version v1.17.4+k3s1 (3eee8ac3)

升级 k3s 版本
#

可以看的,默认的安装包中,k3s的版本为 v1.17.4+k3s1,与我们的 n1 上的 master 不匹配。

wget https://github.com/k3s-io/k3s/releases/download/v1.19.7%2Bk3s1/k3s

chmod a+x k3s

mv /usr/bin/k3s /usr/bin/k3s.old

cp -a k3s /usr/bin/k3s

root@OpenWrt:/opt/tools# k3s --version
k3s version v1.19.7+k3s1 (5a00e38d)

部署前的检查
#

同样这个打印的内容是 部署完成后打印

root@OpenWrt:~# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/30740d1d67da51fe92b10367ecce4d580e552c634ad4a6c4dd13297ffd1f3edd/bin:
- sha256sum: good
- links: good

System:
- /usr/sbin iptables v1.8.4 (legacy): ok
- swap: disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: missing
- CONFIG_CGROUP_PERF: missing
- CONFIG_CGROUP_HUGETLB: missing
- CONFIG_NET_CLS_CGROUP: enabled
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: missing
- CONFIG_EXT4_FS_SECURITY: missing
    enable these ext4 configs if you are using ext3 or ext4 as backing filesystem
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: missing
      - CONFIG_CRYPTO_SEQIV: missing
      - CONFIG_CRYPTO_GHASH: missing
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: missing
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

第一次部署检查的使用,显示有两项未通过,与 此链接 中描述类似

修改 k3s 服务配置
#

K3s 中文文档

默认安装服务后的配置文件是 /etc/config/k3s, 这个文件默认是没有的,需要我们自己创建

--token 后的配置为在 n1 master 中 cat /var/lib/rancher/k3s/server/node-token 的配置

root@n1:/# cat /var/lib/rancher/k3s/server/node-token
K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85

服务配置文件 /etc/config/k3s

cat /etc/config/k3s
config globals 'globals'
        option opts '--server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666'
        option root '/application/k3s'

配置文件,更具自己的实际使用情况增删即可

服务启动脚本 /etc/init.d/k3s

修改说明: 下面展示的启动脚本中,将 uci_get 替换使用成了 uci get。 将 -- server 修改成了 -- agent 如你部署的是 server 端那么此处你可以不做修改,且 /etc/config/k3s 中也得修改为 server 端的配置项。

root@OpenWrt:~# cat /etc/init.d/k3s
#!/bin/sh /etc/rc.common

START=60
STOP=20

PIDFILE=/var/run/k3s.pid
EXEC="/usr/bin/k3s"

ensure_cgroup_mount() {
  # Unmount /sys/fs/cgroup if mounted as cgroup
  grep -q ' /sys/fs/cgroup cgroup' /proc/self/mounts && umount /sys/fs/cgroup

  grep -q ' /sys/fs/cgroup tmpfs' /proc/self/mounts \
    || mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup

  for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
    mnt="/sys/fs/cgroup/$sys"
    grep -q "cgroup $mnt " /proc/self/mounts && continue
    mkdir -p "$mnt"
    mount -n -t cgroup -o $sys cgroup "$mnt"
  done
}

start() {
  ensure_cgroup_mount
  start-stop-daemon -S -b -x "$EXEC" -m -p "$PIDFILE" \
    -- agent $(uci get k3s.globals.opts) \
    --data-dir $(uci get k3s.globals.root)
}

stop() {
  start-stop-daemon -K -p "$PIDFILE"
}

启动服务
#

/etc/init.d/k3s start  # 启动命令

root@OpenWrt:~# ps -ef|grep k3s
 3451 root      867m S    {k3s-agent} /usr/bin/k3s agent   # 启动成功后,会有进程在后台运行
21059 root      1096 S    grep k3s


root@n1:/# kubectl get nodes
NAME      STATUS   ROLES    AGE    VERSION
n1        Ready    master   2d3h   v1.19.7+k3s1
openwrt   Ready    <none>   2d2h   v1.19.7+k3s1

如你运行启动命令后,没有任何的响应,可先前台手动运行一步一步进行 debug, 如:

 k3s agent --server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666

问题记录
#

启动 agent 时显示 level=error msg="Node password rejected, duplicate hostname or contents of '/etc/rancher/node/password' may not match server nod e-passwd entry, try enabling a unique node name with the --with-node-id flag" 错误 。

此问题的原因是之前部署时部署未成功,但 server 端还留有 agent 端的 server id ,当 agent 端再次注册到 server 端时,server 端通过主机名称进行匹配,发现 id 匹配不对时就导致了冲突。解决方法就是在 server 端删除老的 server id 或 更改 agent 端的主机名称(推荐使用在 server 端删除)。

cat /etc/rancher/node/password  # agent
92a18abb46

cat /var/lib/rancher/k3s/server/cred/node-passwd
898170dfe92a18abb46,n1,n1,
9185b3215d3afcab9,openwrt,openwrt,  # 将此行删除

openWrt 开放防火墙
#

编辑 /etc/config/network 配置文件, 添加如下类容

config interface 'k8s'
	option proto 'none'
	option ifname 'cni0'

编辑 /etc/config/firewall 添加如下类容

config zone
        option name 'k8s'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option network 'k8s'

重启一下 op 系统 ,使配置生效

reboot

部署应用
#

此处我们部署一个 deployment 类型的 nginx 服务,来简单测试一下 k3s 集群完整性

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  ports:
    - protocol: TCP
      name: web
      port: 80
  selector:
    app: nginx

---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 4
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.16.1
          ports:
            - name: web
              containerPort: 80
EOF
watch kubectl get pod -o wide

NAME                    READY   STATUS    RESTARTS   AGE    IP           NODE      NOMINATED NODE   READINESS GATES
nginx-bcf5bbd7d-n2lfx   1/1     Running   0          3m2s   10.42.0.14   n1        <none>           <none>
nginx-bcf5bbd7d-zhj6j   1/1     Running   0          78s    10.42.0.15   n1        <none>           <none>
nginx-bcf5bbd7d-sxlvp   1/1     Running   0          78s    10.42.1.17   openwrt   <none>           <none>
nginx-bcf5bbd7d-bsxtn   1/1     Running   0          78s    10.42.1.18   openwrt   <none>           <none>

可以看到上述打印 n1openwrt 分别启动了 2 个容器,现在我们分别在两台机器上使用 curl 测试一下对方主机中 pod 的 ip,来检查网络是否可以连通。

n1, 测试 curl 10.42.1.17 和 10.42.1.18

image-20210126132703410

openWrt, 测试 curl 10.42.0.14 和 10.42.0.15

image-20210126132810261

经过简单的测试,可以看到 k3s 集群使用起来没有什么问题。

配置使用 dashboard
#

原本打算使用 rancher 来作为 dashboard 的,考虑其资源的占用什么的,还是选择使用了 lens,即将 dashboard 部署在客户端。

Lens Github地址

省略。安装过程。。。。

查看 n1 即 master 中的 kubelet 配置文件

root@n1:/# cat ~/.kube/config

将打印内容复制,粘贴到客户端的电脑上

内容保存在至 ~/.kube/k3s-config

lens 添加主机
#

image-20210126134036320

如若提示证书验证错,这里使用一个小技巧即可,在主机中 hosts 文件中添加对应映射,并修改 kubelet 配置文件中server 地址 为映射的域名即可。

cat /etc/hosts
192.168.8.102 n1

image-20210126134624679

开启监控
#

鼠标移到到集群,右键,然后点击 settings

image-20210126135339985

拉到后面,然后点击进行安装

image-20210126135403341

image-20210126135820673

image-20210126135947717

Done。。。
#

相关文章

Docker 部署的 openWrt 软路由, 并解决无法与宿主机通信问题
·923 字·2 分钟·
fix openwrt n1 docker-compose docker openwrt
Docker 部署多网口 openWrt 软路由
·810 字·2 分钟·
openwrt linux centos7
在 Kubernetes 中部署 nfs storageClass
·1026 字·3 分钟·
devops k8s storage-class nfs
使用 Confluentinc 在 Kubernetes 集群中部署 Kafka 集群
·1133 字·3 分钟·
k8s kafka zookeeper helm
Rancher 单机部署,进行升级后,导致docker无法正常启动
·586 字·2 分钟·
k8s fix docker rancher
记录一次,因误删容器导致的容器恢复过程
·1707 字·4 分钟·
docker docker centos fix