- 博客/
使用 斐讯n1 & openWrt 搭建 k3s 集群
作者
Johny
熟练的 云原生搬砖师
Table of Contents
环境说明#
软件说明#
K3s version : v1.19.7+k3s1
Docker version: 19.03.13 (n1)，19.03.12 (openwrt)
写盘工具: balenaEtcher
机器说明#
| IP 地址 | 机型 | 配置 | 操作系统 | 角色 |
|---|---|---|---|---|
| 192.168.8.1 | 占美 (机型不详) | 4c 2g ( cpu N2940 ) | openWrt（X86_64 自编译） | node/agent |
| 192.168.8.112 | 斐讯 N1 | 4c 2g | Armbian ( 5.0.2-aml-s905) | master/server |
openWrt 自编译#
需要自编译的原因是因为，之前使用 esir 打包提供的高大全 op 固件，部署k3s 时发现，内核没有开启vxlan的特性，导致部署不成功，如你使用的 op 固件已开启了 vxlan 特性可省略此步骤，查看方法:
root@OpenWrt:~# lsmod|grep -i vxlan # 打印输入表示具有 vxlan 特性
ip6_udp_tunnel 16384 1 vxlan
udp_tunnel 16384 1 vxlan
vxlan 53248 0
此处感谢
esir 等大佬提供的github actions ci 工作流
fork 仓库代码#
登录 github, 并将 代码 fork 至自己仓库
git clone 代码#
git clone fork https://github.com.cnpmjs.org/cdryzun/AutoBuild-OpenWrt.git
cd AutoBuild-OpenWrt
修改配置文件#
修改工作流文件#
打开自己心爱的编辑器
且 演示
x86_64 平台
code .github/workflows/Build_OP_x86_64.yml # 编辑 ci 构建脚本
...
push:
branches:
- master # 打开注释，注意空格
修改构建文件#
code x86_64.config
配置参考资料#
CONFIG_PACKAGE_grub2-efi=y
CONFIG_EFI_IMAGES=y
CONFIG_TARGET_IMAGES_GZIP=y
CONFIG_TARGET_KERNEL_PARTSIZE=16
CONFIG_TARGET_ROOTFS_PARTSIZE=300
# CONFIG_GRUB_CONSOLE is not set
CONFIG_PACKAGE_luci-app-docker=y
CONFIG_PACKAGE_luci-i18n-docker-zh-cn=y
CONFIG_PACKAGE_luci-app-ssr-plus=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Shadowsocks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Simple_obfs=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray_plugin=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Kcptun=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Server=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Socks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_NaiveProxy=y
CONFIG_PACKAGE_luci-theme-argon=y
CONFIG_PACKAGE_luci-app-mwan3=y
CONFIG_PACKAGE_luci-app-mwan3helper=y
CONFIG_PACKAGE_luci-i18n-mwan3-zh-cn=y
CONFIG_PACKAGE_luci-i18n-mwan3helper-zh-cn=y
CONFIG_PACKAGE_luci-app-syncdial=y
CONFIG_PACKAGE_luci-app-sfe=y
CONFIG_PACKAGE_luci-app-ttyd=y
CONFIG_PACKAGE_luci-i18n-ttyd-zh-cn=y
CONFIG_PACKAGE_luci-app-webadmin=y
CONFIG_PACKAGE_luci-i18n-webadmin-zh-cn=y
CONFIG_PACKAGE_luci-app-zerotier=y
CONFIG_PACKAGE_luci-app-frpc=y
CONFIG_PACKAGE_luci-app-kodexplorer=y
CONFIG_PACKAGE_luci-app-nfs=y
CONFIG_PACKAGE_ss=y
CONFIG_PACKAGE_luci-app-nps=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-rclone=y
CONFIG_PACKAGE_luci-app-radicale=y
CONFIG_PACKAGE_luci-app-qbittorrent=y
CONFIG_PACKAGE_luci-app-samba=y
CONFIG_PACKAGE_luci-app-wol=y
CONFIG_PACKAGE_luci-app-xlnetacc=y
CONFIG_PACKAGE_luci-app-bird1-ipv4=y
CONFIG_PACKAGE_luci-app-bird1-ipv6=y
CONFIG_PACKAGE_luci-app-commands=y
CONFIG_PACKAGE_luci-app-diskman=y
CONFIG_PACKAGE_luci-app-dnscrypt-proxy=y
CONFIG_PACKAGE_luci-app-dnsforwarder=y
CONFIG_PACKAGE_luci-app-familycloud=y
CONFIG_PACKAGE_luci-app-hd-idle=y
CONFIG_PACKAGE_luci-app-netdata=y
CONFIG_PACKAGE_ipv6helper=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_curl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_wget=y
CONFIG_PACKAGE_lrzsz=y
CONFIG_PACKAGE_vim=y
CONFIG_PACKAGE_zip=y
CONFIG_PACKAGE_kmod-kvm-amd=y
CONFIG_PACKAGE_kmod-kvm-intel=y
CONFIG_PACKAGE_kmod-kvm-x86=y
CONFIG_OPENSSL_ENGINE_CRYPTO=y
CONFIG_OPENSSL_ENGINE_DIGEST=y
CONFIG_OPENSSL_WITH_CAMELLIA=y
CONFIG_OPENSSL_WITH_COMPRESSION=y
CONFIG_OPENSSL_WITH_DTLS=y
CONFIG_OPENSSL_WITH_EC2M=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_GOST=y
CONFIG_OPENSSL_WITH_IDEA=y
CONFIG_OPENSSL_WITH_MDC2=y
CONFIG_OPENSSL_WITH_RFC3779=y
CONFIG_OPENSSL_WITH_SEED=y
CONFIG_OPENSSL_WITH_WHIRLPOOL=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_KERNEL_BUILD_USER="treeSir Playground"
CONFIG_GRUB_TITLE="OpenWrt AutoBuild by treeSirPlayground"
CONFIG_PACKAGE_kmod-usb-ohci=y
CONFIG_PACKAGE_kmod-usb-ohci-pci=y
CONFIG_PACKAGE_kmod-usb-storage-uas=y
CONFIG_PACKAGE_kmod-usb-uhci=y
CONFIG_PACKAGE_kmod-sdhci=y
CONFIG_PACKAGE_kmod-usb-ehci=y
CONFIG_PACKAGE_kmod-usb2=y
CONFIG_PACKAGE_kmod-usb2-pci=y
CONFIG_PACKAGE_luci-app-passwall=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Brook=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan_GO=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_simple-obfs=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_v2ray-plugin=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_https_dns_proxy=n
# CONFIG_PACKAGE_naiveproxy=y
# CONFIG_PACKAGE_nspr=y
# CONFIG_PACKAGE_shadowsocks-libev-ss-server=y
# CONFIG_PACKAGE_ssocks=y
# CONFIG_PACKAGE_ssocksd=y
# CONFIG_PACKAGE_trojan-go=y
# CONFIG_PACKAGE_trojan-plus=y
# CONFIG_PACKAGE_chinadns-ng=y
CONFIG_TARGET_x86=y
CONFIG_TARGET_x86_64=y
CONFIG_TARGET_x86_64_Generic=y
CONFIG_FEED_luci=y
CONFIG_FEED_packages=y
CONFIG_FEED_routing=y
CONFIG_FEED_telephony=y
CONFIG_IPTABLES_CONNLABEL=y
CONFIG_IPTABLES_NFTABLES=y
CONFIG_KERNEL_BLK_CGROUP=y
CONFIG_KERNEL_CFS_BANDWIDTH=y
CONFIG_KERNEL_CGROUPS=y
CONFIG_KERNEL_CGROUP_CPUACCT=y
CONFIG_KERNEL_CGROUP_DEVICE=y
CONFIG_KERNEL_CGROUP_FREEZER=y
CONFIG_KERNEL_CGROUP_PIDS=y
CONFIG_KERNEL_CGROUP_SCHED=y
CONFIG_KERNEL_CPUSETS=y
CONFIG_KERNEL_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_KERNEL_FAIR_GROUP_SCHED=y
CONFIG_KERNEL_FREEZER=y
CONFIG_KERNEL_IPC_NS=y
CONFIG_KERNEL_KEYS=y
CONFIG_KERNEL_LXC_MISC=y
CONFIG_KERNEL_MEMCG=y
CONFIG_KERNEL_MM_OWNER=y
CONFIG_KERNEL_NAMESPACES=y
CONFIG_KERNEL_NETPRIO_CGROUP=y
CONFIG_KERNEL_NET_CLS_CGROUP=y
CONFIG_KERNEL_NET_NS=y
CONFIG_KERNEL_PID_NS=y
CONFIG_KERNEL_POSIX_MQUEUE=y
CONFIG_KERNEL_PROC_PID_CPUSET=y
CONFIG_KERNEL_RESOURCE_COUNTERS=y
CONFIG_KERNEL_SECCOMP=y
CONFIG_KERNEL_SECCOMP_FILTER=y
CONFIG_KERNEL_USER_NS=y
CONFIG_KERNEL_UTS_NS=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ip-bridge=y
CONFIG_PACKAGE_ip-full=y
CONFIG_PACKAGE_ipset=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_iptables-mod-ipopt=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-br-netfilter=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hw-ccp=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rsa=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-fuse=y
CONFIG_PACKAGE_kmod-ikconfig=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-ipopt=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-leds-gpio=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-ipvs=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-base=y
CONFIG_PACKAGE_kmod-random-core=y
CONFIG_PACKAGE_kmod-softdog=y
CONFIG_PACKAGE_kmod-sound-core=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-core=y
CONFIG_PACKAGE_kmod-usb3=y
CONFIG_PACKAGE_kmod-veth=y
CONFIG_PACKAGE_kmod-vxlan=y
CONFIG_PACKAGE_kmod-ppp=y
CONFIG_PACKAGE_kmod-pppox=y
CONFIG_PACKAGE_libipset=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnftnl=y
CONFIG_TARGET_ROOTFS_PARTSIZE=1024
# CONFIG_PACKAGE_iptables-mod-conntrack-label is not set
# CONFIG_PACKAGE_kmod-ipt-conntrack-label is not set
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y
示例配置且供参考使用
构建固件#
触发构建#
提交代码
提交代码前，检查一下
actions是否是打开状态
git add .github/workflows/Build_OP_x86_64.yml
git add ./x86_64.config
git commit -m "build custom openwrt"
git push
提交完成后，等待构建结束
下载固件#
构建完成后如何下载 op固件 文件
点击后 往下面拉，点击进行下载
省略 op 写入步骤, 下载固件后解压进入 OpenWrt/targets/x86/64 目录, 就可以看到固件了。
N1 k3s master 部署#
n1 使用系统为
Armbian，同样省略写入步骤，n1怎么刷入 armbian 系统在网上资料还是非常多的。
docker 配置文件#
此处为
n1docker 配置文件，不一定适合你，可截取部分参考使用。
cat /etc/docker/daemon.json
{
"experimental": false,
"registry-mirrors": [
"https://7bezldxe.mirror.aliyuncs.com"
],
"oom-score-adjust": -1000,
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"features": {
"buildkit": true
},
"insecure-registries": [
"idocker.io"
]
}
配置项简短说明:
- “log-driver”: “json-file” 设置json 格式日志
- “oom-score-adjust”: -1000 防止容器被 内核 oom
- “log-opts” 设置容器日志大小
- “max-concurrent-downloads”: 10 并行下载容器数量
- “max-concurrent-uploads”: 10 并行上传
- “storage-driver”: “overlay2” 设置存储驱动为 overlay2
- “bip” 容器默认的网段
- “registry-mirrors” 配置镜像下载加速这里使用的是阿里云的
- “insecure-registries” 信任的私服地址
如若修改了配置，记得重启一下服务查看是否生效
service docker restart
docker info
部署前的检查#
这个打印的内容是
部署完成后打印的，部署前的检查忘记做记录了。
root@n1:/# k3s check-config
Verifying binaries in /var/lib/rancher/k3s/data/ad8f0f93ebb9db5c507884fcdec249d73dd348293dac194e01462c57815cca46/bin:
- sha256sum: good
- links: good
System:
- /sbin iptables v1.6.1: older than v1.8
- swap: should be disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed
Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000
info: reading kernel config from /proc/config.gz ...
Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled (as module)
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled
Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: enabled
- CONFIG_CGROUP_PERF: enabled
- CONFIG_CGROUP_HUGETLB: enabled
- CONFIG_NET_CLS_CGROUP: enabled (as module)
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: enabled
- CONFIG_EXT4_FS_SECURITY: enabled
- Network Drivers:
- "overlay":
- CONFIG_VXLAN: enabled (as module)
Optional (for encrypted networks):
- CONFIG_CRYPTO: enabled
- CONFIG_CRYPTO_AEAD: enabled
- CONFIG_CRYPTO_GCM: enabled (as module)
- CONFIG_CRYPTO_SEQIV: enabled
- CONFIG_CRYPTO_GHASH: enabled (as module)
- CONFIG_XFRM: enabled
- CONFIG_XFRM_USER: enabled (as module)
- CONFIG_XFRM_ALGO: enabled (as module)
- CONFIG_INET_ESP: enabled (as module)
- CONFIG_INET_XFRM_MODE_TRANSPORT: enabled
- Storage Drivers:
- "overlay":
- CONFIG_OVERLAY_FS: enabled
STATUS: pass
部署 k3s#
官方提供了部署脚本，部署起来还是非常方便的
export INSTALL_K3S_VERSION=v1.19.7+k3s1
export INSTALL_K3S_EXEC="--docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -
root@n1:~# kubectl get nodes
NAME STATUS ROLES AGE VERSION
n1 Ready master 6s v1.19.7+k3s1
openWrt k3s agent 部署#
在 openwrt 中部署 k3s 时就不能使用 官方提供的自动部署脚本了，运行脚本时会提示错误 [ERROR] Can not find systemd or openrc to use as a process supervisor for k3s ,提示说明已经非常清楚了，好在 github 上有位大佬提供了 openwrt k3s 的安装包，谢谢这位大佬。
Docker 配置#
此处为 openwrt 中 docker 配置，配置项说明请参考上面 n1 中 docker 配置项说明。
cat /etc/docker/daemon.json
{
"data-root": "/data/docker-root",
"log-level": "warn",
"oom-score-adjust": -1000,
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"insecure-registries": ["idocker.io"],
"registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com"],
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
]
}
⚠️ 注意: 配置项修改后，多需要重启一下
docker才会生效/etc/init.d/dockerd restart
k3s agent 部署#
下载安装包#
wget https://github.com/discordianfish/k3s-openwrt/releases/download/9/k3s_1.17.4+k3s1_x86_64.opk
opkg install ./k3s_1.17.4+k3s1_x86_64.opk
root@OpenWrt:/opt/tools# /usr/bin/k3s --version
k3s version v1.17.4+k3s1 (3eee8ac3)
升级 k3s 版本#
可以看的，默认的安装包中，k3s的版本为
v1.17.4+k3s1,与我们的 n1 上的 master 不匹配。
wget https://github.com/k3s-io/k3s/releases/download/v1.19.7%2Bk3s1/k3s
chmod a+x k3s
mv /usr/bin/k3s /usr/bin/k3s.old
cp -a k3s /usr/bin/k3s
root@OpenWrt:/opt/tools# k3s --version
k3s version v1.19.7+k3s1 (5a00e38d)
部署前的检查#
同样这个打印的内容是
部署完成后打印的
root@OpenWrt:~# k3s check-config
Verifying binaries in /var/lib/rancher/k3s/data/30740d1d67da51fe92b10367ecce4d580e552c634ad4a6c4dd13297ffd1f3edd/bin:
- sha256sum: good
- links: good
System:
- /usr/sbin iptables v1.8.4 (legacy): ok
- swap: disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed
Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000
info: reading kernel config from /proc/config.gz ...
Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled
Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: missing
- CONFIG_CGROUP_PERF: missing
- CONFIG_CGROUP_HUGETLB: missing
- CONFIG_NET_CLS_CGROUP: enabled
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: missing
- CONFIG_EXT4_FS_SECURITY: missing
enable these ext4 configs if you are using ext3 or ext4 as backing filesystem
- Network Drivers:
- "overlay":
- CONFIG_VXLAN: enabled (as module)
Optional (for encrypted networks):
- CONFIG_CRYPTO: enabled
- CONFIG_CRYPTO_AEAD: enabled
- CONFIG_CRYPTO_GCM: missing
- CONFIG_CRYPTO_SEQIV: missing
- CONFIG_CRYPTO_GHASH: missing
- CONFIG_XFRM: enabled
- CONFIG_XFRM_USER: enabled (as module)
- CONFIG_XFRM_ALGO: enabled (as module)
- CONFIG_INET_ESP: enabled (as module)
- CONFIG_INET_XFRM_MODE_TRANSPORT: missing
- Storage Drivers:
- "overlay":
- CONFIG_OVERLAY_FS: enabled
STATUS: pass
第一次部署检查的使用，显示有两项未通过，与 此链接 中描述类似
修改 k3s 服务配置#
默认安装服务后的配置文件是 /etc/config/k3s， 这个文件默认是没有的，需要我们自己创建
--token 后的配置为在 n1 master 中 cat /var/lib/rancher/k3s/server/node-token 的配置
root@n1:/# cat /var/lib/rancher/k3s/server/node-token
K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85
服务配置文件 /etc/config/k3s
cat /etc/config/k3s
config globals 'globals'
option opts '--server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666'
option root '/application/k3s'
配置文件，更具自己的实际使用情况增删即可
服务启动脚本 /etc/init.d/k3s
修改说明: 下面展示的启动脚本中，将
uci_get替换使用成了uci get。 将-- server修改成了-- agent如你部署的是 server 端那么此处你可以不做修改，且/etc/config/k3s中也得修改为 server 端的配置项。
root@OpenWrt:~# cat /etc/init.d/k3s
#!/bin/sh /etc/rc.common
START=60
STOP=20
PIDFILE=/var/run/k3s.pid
EXEC="/usr/bin/k3s"
ensure_cgroup_mount() {
# Unmount /sys/fs/cgroup if mounted as cgroup
grep -q ' /sys/fs/cgroup cgroup' /proc/self/mounts && umount /sys/fs/cgroup
grep -q ' /sys/fs/cgroup tmpfs' /proc/self/mounts \
|| mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
mnt="/sys/fs/cgroup/$sys"
grep -q "cgroup $mnt " /proc/self/mounts && continue
mkdir -p "$mnt"
mount -n -t cgroup -o $sys cgroup "$mnt"
done
}
start() {
ensure_cgroup_mount
start-stop-daemon -S -b -x "$EXEC" -m -p "$PIDFILE" \
-- agent $(uci get k3s.globals.opts) \
--data-dir $(uci get k3s.globals.root)
}
stop() {
start-stop-daemon -K -p "$PIDFILE"
}
启动服务#
/etc/init.d/k3s start # 启动命令
root@OpenWrt:~# ps -ef|grep k3s
3451 root 867m S {k3s-agent} /usr/bin/k3s agent # 启动成功后，会有进程在后台运行
21059 root 1096 S grep k3s
root@n1:/# kubectl get nodes
NAME STATUS ROLES AGE VERSION
n1 Ready master 2d3h v1.19.7+k3s1
openwrt Ready <none> 2d2h v1.19.7+k3s1
如你运行启动命令后，没有任何的响应，可先前台手动运行一步一步进行
debug, 如：k3s agent --server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666
问题记录#
启动 agent 时显示 level=error msg="Node password rejected, duplicate hostname or contents of '/etc/rancher/node/password' may not match server nod e-passwd entry, try enabling a unique node name with the --with-node-id flag" 错误 。
此问题的原因是之前部署时部署未成功，但 server 端还留有 agent 端的
server id，当 agent 端再次注册到 server 端时，server 端通过主机名称进行匹配，发现 id 匹配不对时就导致了冲突。解决方法就是在 server 端删除老的server id或 更改 agent 端的主机名称（推荐使用在 server 端删除）。cat /etc/rancher/node/password # agent 92a18abb46 cat /var/lib/rancher/k3s/server/cred/node-passwd 898170dfe92a18abb46,n1,n1, 9185b3215d3afcab9,openwrt,openwrt, # 将此行删除
openWrt 开放防火墙#
编辑 /etc/config/network 配置文件, 添加如下类容
config interface 'k8s'
option proto 'none'
option ifname 'cni0'
编辑 /etc/config/firewall 添加如下类容
config zone
option name 'k8s'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option network 'k8s'
重启一下 op 系统 ，使配置生效
reboot
部署应用#
此处我们部署一个
deployment类型的 nginx 服务，来简单测试一下k3s 集群完整性。
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
ports:
- protocol: TCP
name: web
port: 80
selector:
app: nginx
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nginx
labels:
app: nginx
spec:
replicas: 4
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.16.1
ports:
- name: web
containerPort: 80
EOF
watch kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-bcf5bbd7d-n2lfx 1/1 Running 0 3m2s 10.42.0.14 n1 <none> <none>
nginx-bcf5bbd7d-zhj6j 1/1 Running 0 78s 10.42.0.15 n1 <none> <none>
nginx-bcf5bbd7d-sxlvp 1/1 Running 0 78s 10.42.1.17 openwrt <none> <none>
nginx-bcf5bbd7d-bsxtn 1/1 Running 0 78s 10.42.1.18 openwrt <none> <none>
可以看到上述打印
n1和openwrt分别启动了 2 个容器，现在我们分别在两台机器上使用curl测试一下对方主机中 pod 的 ip，来检查网络是否可以连通。
n1, 测试 curl 10.42.1.17 和 10.42.1.18
openWrt, 测试 curl 10.42.0.14 和 10.42.0.15
经过简单的测试，可以看到 k3s 集群使用起来没有什么问题。
配置使用 dashboard#
原本打算使用 rancher 来作为 dashboard 的，考虑其资源的占用什么的，还是选择使用了 lens，即将 dashboard 部署在客户端。
省略。安装过程。。。。
查看 n1 即 master 中的 kubelet 配置文件
root@n1:/# cat ~/.kube/config
将打印内容复制，粘贴到客户端的电脑上
内容保存在至
~/.kube/k3s-config中
lens 添加主机#
如若提示证书验证错，这里使用一个小技巧即可，在主机中
hosts文件中添加对应映射，并修改kubelet 配置文件中的server 地址为映射的域名即可。cat /etc/hosts 192.168.8.102 n1
开启监控#
鼠标移到到集群，右键，然后点击 settings
拉到后面，然后点击进行安装
Done。。。#
相关文章
Docker 部署的 openWrt 软路由, 并解决无法与宿主机通信问题
·923 字·2 分钟·
fix
openwrt
n1
docker-compose
docker
openwrt
Docker 部署多网口 openWrt 软路由
·810 字·2 分钟·
openwrt
linux
centos7
在 Kubernetes 中部署 nfs storageClass
·1026 字·3 分钟·
devops
k8s
storage-class
nfs
使用 Confluentinc 在 Kubernetes 集群中部署 Kafka 集群
·1133 字·3 分钟·
k8s
kafka
zookeeper
helm
Rancher 单机部署，进行升级后，导致docker无法正常启动
·586 字·2 分钟·
k8s
fix
docker
rancher
记录一次，因误删容器导致的容器恢复过程
·1707 字·4 分钟·
docker
docker
centos
fix