环境说明
软件说明
机器说明
| IP 地址 |
机型 |
配置 |
操作系统 |
角色 |
| 192.168.8.1 |
占美 (机型不详) |
4c 2g ( cpu N2940 ) |
openWrt（X86_64 自编译） |
node/agent |
| 192.168.8.112 |
斐讯 N1 |
4c 2g |
Armbian ( 5.0.2-aml-s905) |
master/server |
openWrt 自编译
需要自编译的原因是因为，之前使用 esir 打包提供的高大全 op 固件，部署k3s 时发现，内核没有开启vxlan的特性，导致部署不成功，如你使用的 op 固件已开启了 vxlan 特性可省略此步骤，查看方法:
1
2
3
4
|
root@OpenWrt:~# lsmod|grep -i vxlan # 打印输入表示具有 vxlan 特性
ip6_udp_tunnel 16384 1 vxlan
udp_tunnel 16384 1 vxlan
vxlan 53248 0
|
此处感谢 esir 等大佬 提供的 github actions ci 工作流
仓库地址 视频教程
fork 仓库代码
登录 github, 并将 代码 fork 至自己仓库
git clone 代码
1
2
3
|
git clone fork https://github.com.cnpmjs.org/cdryzun/AutoBuild-OpenWrt.git
cd AutoBuild-OpenWrt
|
修改配置文件
修改工作流文件
打开自己心爱的编辑器
且 演示 x86_64 平台
1
2
3
4
5
6
|
code .github/workflows/Build_OP_x86_64.yml # 编辑 ci 构建脚本
...
push:
branches:
- master # 打开注释，注意空格
|
修改构建文件
配置参考资料
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
CONFIG_PACKAGE_grub2-efi=y
CONFIG_EFI_IMAGES=y
CONFIG_TARGET_IMAGES_GZIP=y
CONFIG_TARGET_KERNEL_PARTSIZE=16
CONFIG_TARGET_ROOTFS_PARTSIZE=300
# CONFIG_GRUB_CONSOLE is not set
CONFIG_PACKAGE_luci-app-docker=y
CONFIG_PACKAGE_luci-i18n-docker-zh-cn=y
CONFIG_PACKAGE_luci-app-ssr-plus=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Shadowsocks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Simple_obfs=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray_plugin=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Kcptun=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Server=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Socks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_NaiveProxy=y
CONFIG_PACKAGE_luci-theme-argon=y
CONFIG_PACKAGE_luci-app-mwan3=y
CONFIG_PACKAGE_luci-app-mwan3helper=y
CONFIG_PACKAGE_luci-i18n-mwan3-zh-cn=y
CONFIG_PACKAGE_luci-i18n-mwan3helper-zh-cn=y
CONFIG_PACKAGE_luci-app-syncdial=y
CONFIG_PACKAGE_luci-app-sfe=y
CONFIG_PACKAGE_luci-app-ttyd=y
CONFIG_PACKAGE_luci-i18n-ttyd-zh-cn=y
CONFIG_PACKAGE_luci-app-webadmin=y
CONFIG_PACKAGE_luci-i18n-webadmin-zh-cn=y
CONFIG_PACKAGE_luci-app-zerotier=y
CONFIG_PACKAGE_luci-app-frpc=y
CONFIG_PACKAGE_luci-app-kodexplorer=y
CONFIG_PACKAGE_luci-app-nfs=y
CONFIG_PACKAGE_ss=y
CONFIG_PACKAGE_luci-app-nps=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-rclone=y
CONFIG_PACKAGE_luci-app-radicale=y
CONFIG_PACKAGE_luci-app-qbittorrent=y
CONFIG_PACKAGE_luci-app-samba=y
CONFIG_PACKAGE_luci-app-wol=y
CONFIG_PACKAGE_luci-app-xlnetacc=y
CONFIG_PACKAGE_luci-app-bird1-ipv4=y
CONFIG_PACKAGE_luci-app-bird1-ipv6=y
CONFIG_PACKAGE_luci-app-commands=y
CONFIG_PACKAGE_luci-app-diskman=y
CONFIG_PACKAGE_luci-app-dnscrypt-proxy=y
CONFIG_PACKAGE_luci-app-dnsforwarder=y
CONFIG_PACKAGE_luci-app-familycloud=y
CONFIG_PACKAGE_luci-app-hd-idle=y
CONFIG_PACKAGE_luci-app-netdata=y
CONFIG_PACKAGE_ipv6helper=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_curl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_wget=y
CONFIG_PACKAGE_lrzsz=y
CONFIG_PACKAGE_vim=y
CONFIG_PACKAGE_zip=y
CONFIG_PACKAGE_kmod-kvm-amd=y
CONFIG_PACKAGE_kmod-kvm-intel=y
CONFIG_PACKAGE_kmod-kvm-x86=y
CONFIG_OPENSSL_ENGINE_CRYPTO=y
CONFIG_OPENSSL_ENGINE_DIGEST=y
CONFIG_OPENSSL_WITH_CAMELLIA=y
CONFIG_OPENSSL_WITH_COMPRESSION=y
CONFIG_OPENSSL_WITH_DTLS=y
CONFIG_OPENSSL_WITH_EC2M=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_GOST=y
CONFIG_OPENSSL_WITH_IDEA=y
CONFIG_OPENSSL_WITH_MDC2=y
CONFIG_OPENSSL_WITH_RFC3779=y
CONFIG_OPENSSL_WITH_SEED=y
CONFIG_OPENSSL_WITH_WHIRLPOOL=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_KERNEL_BUILD_USER="treeSir Playground"
CONFIG_GRUB_TITLE="OpenWrt AutoBuild by treeSirPlayground"
CONFIG_PACKAGE_kmod-usb-ohci=y
CONFIG_PACKAGE_kmod-usb-ohci-pci=y
CONFIG_PACKAGE_kmod-usb-storage-uas=y
CONFIG_PACKAGE_kmod-usb-uhci=y
CONFIG_PACKAGE_kmod-sdhci=y
CONFIG_PACKAGE_kmod-usb-ehci=y
CONFIG_PACKAGE_kmod-usb2=y
CONFIG_PACKAGE_kmod-usb2-pci=y
CONFIG_PACKAGE_luci-app-passwall=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Brook=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan_GO=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_simple-obfs=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_v2ray-plugin=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_https_dns_proxy=n
# CONFIG_PACKAGE_naiveproxy=y
# CONFIG_PACKAGE_nspr=y
# CONFIG_PACKAGE_shadowsocks-libev-ss-server=y
# CONFIG_PACKAGE_ssocks=y
# CONFIG_PACKAGE_ssocksd=y
# CONFIG_PACKAGE_trojan-go=y
# CONFIG_PACKAGE_trojan-plus=y
# CONFIG_PACKAGE_chinadns-ng=y
CONFIG_TARGET_x86=y
CONFIG_TARGET_x86_64=y
CONFIG_TARGET_x86_64_Generic=y
CONFIG_FEED_luci=y
CONFIG_FEED_packages=y
CONFIG_FEED_routing=y
CONFIG_FEED_telephony=y
CONFIG_IPTABLES_CONNLABEL=y
CONFIG_IPTABLES_NFTABLES=y
CONFIG_KERNEL_BLK_CGROUP=y
CONFIG_KERNEL_CFS_BANDWIDTH=y
CONFIG_KERNEL_CGROUPS=y
CONFIG_KERNEL_CGROUP_CPUACCT=y
CONFIG_KERNEL_CGROUP_DEVICE=y
CONFIG_KERNEL_CGROUP_FREEZER=y
CONFIG_KERNEL_CGROUP_PIDS=y
CONFIG_KERNEL_CGROUP_SCHED=y
CONFIG_KERNEL_CPUSETS=y
CONFIG_KERNEL_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_KERNEL_FAIR_GROUP_SCHED=y
CONFIG_KERNEL_FREEZER=y
CONFIG_KERNEL_IPC_NS=y
CONFIG_KERNEL_KEYS=y
CONFIG_KERNEL_LXC_MISC=y
CONFIG_KERNEL_MEMCG=y
CONFIG_KERNEL_MM_OWNER=y
CONFIG_KERNEL_NAMESPACES=y
CONFIG_KERNEL_NETPRIO_CGROUP=y
CONFIG_KERNEL_NET_CLS_CGROUP=y
CONFIG_KERNEL_NET_NS=y
CONFIG_KERNEL_PID_NS=y
CONFIG_KERNEL_POSIX_MQUEUE=y
CONFIG_KERNEL_PROC_PID_CPUSET=y
CONFIG_KERNEL_RESOURCE_COUNTERS=y
CONFIG_KERNEL_SECCOMP=y
CONFIG_KERNEL_SECCOMP_FILTER=y
CONFIG_KERNEL_USER_NS=y
CONFIG_KERNEL_UTS_NS=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ip-bridge=y
CONFIG_PACKAGE_ip-full=y
CONFIG_PACKAGE_ipset=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_iptables-mod-ipopt=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-br-netfilter=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hw-ccp=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rsa=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-fuse=y
CONFIG_PACKAGE_kmod-ikconfig=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-ipopt=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-leds-gpio=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-ipvs=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-base=y
CONFIG_PACKAGE_kmod-random-core=y
CONFIG_PACKAGE_kmod-softdog=y
CONFIG_PACKAGE_kmod-sound-core=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-core=y
CONFIG_PACKAGE_kmod-usb3=y
CONFIG_PACKAGE_kmod-veth=y
CONFIG_PACKAGE_kmod-vxlan=y
CONFIG_PACKAGE_kmod-ppp=y
CONFIG_PACKAGE_kmod-pppox=y
CONFIG_PACKAGE_libipset=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnftnl=y
CONFIG_TARGET_ROOTFS_PARTSIZE=1024
# CONFIG_PACKAGE_iptables-mod-conntrack-label is not set
# CONFIG_PACKAGE_kmod-ipt-conntrack-label is not set
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y
|
示例配置且供参考使用
构建固件
触发构建
提交代码
提交代码前，检查一下 actions 是否是打开状态
1
2
3
4
5
6
|
git add .github/workflows/Build_OP_x86_64.yml
git add ./x86_64.config
git commit -m "build custom openwrt"
git push
|
提交完成后，等待构建结束
下载固件
构建完成后如何下载 op固件 文件
点击后 往下面拉，点击进行下载
省略 op 写入步骤, 下载固件后解压进入 OpenWrt/targets/x86/64 目录, 就可以看到固件了。
N1 k3s master 部署
n1 使用系统为 Armbian，同样省略写入步骤，n1怎么刷入 armbian 系统在网上资料还是非常多的。
docker 配置文件
此处为 n1 docker 配置文件，不一定适合你，可截取部分参考使用。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
cat /etc/docker/daemon.json
{
"experimental": false,
"registry-mirrors": [
"https://7bezldxe.mirror.aliyuncs.com"
],
"oom-score-adjust": -1000,
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"features": {
"buildkit": true
},
"insecure-registries": [
"idocker.io"
]
}
|
配置项简短说明:
- “log-driver”: “json-file” 设置json 格式日志
- “oom-score-adjust”: -1000 防止容器被 内核 oom
- “log-opts” 设置容器日志大小
- “max-concurrent-downloads”: 10 并行下载容器数量
- “max-concurrent-uploads”: 10 并行上传
- “storage-driver”: “overlay2” 设置存储驱动为 overlay2
- “bip” 容器默认的网段
- “registry-mirrors” 配置镜像下载加速这里使用的是阿里云的
- “insecure-registries” 信任的私服地址
如若修改了配置，记得重启一下服务查看是否生效
1
2
3
|
service docker restart
docker info
|
部署前的检查
这个打印的内容是 部署完成后打印 的，部署前的检查忘记做记录了。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
root@n1:/# k3s check-config
Verifying binaries in /var/lib/rancher/k3s/data/ad8f0f93ebb9db5c507884fcdec249d73dd348293dac194e01462c57815cca46/bin:
- sha256sum: good
- links: good
System:
- /sbin iptables v1.6.1: older than v1.8
- swap: should be disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed
Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000
info: reading kernel config from /proc/config.gz ...
Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled (as module)
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled
Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: enabled
- CONFIG_CGROUP_PERF: enabled
- CONFIG_CGROUP_HUGETLB: enabled
- CONFIG_NET_CLS_CGROUP: enabled (as module)
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: enabled
- CONFIG_EXT4_FS_SECURITY: enabled
- Network Drivers:
- "overlay":
- CONFIG_VXLAN: enabled (as module)
Optional (for encrypted networks):
- CONFIG_CRYPTO: enabled
- CONFIG_CRYPTO_AEAD: enabled
- CONFIG_CRYPTO_GCM: enabled (as module)
- CONFIG_CRYPTO_SEQIV: enabled
- CONFIG_CRYPTO_GHASH: enabled (as module)
- CONFIG_XFRM: enabled
- CONFIG_XFRM_USER: enabled (as module)
- CONFIG_XFRM_ALGO: enabled (as module)
- CONFIG_INET_ESP: enabled (as module)
- CONFIG_INET_XFRM_MODE_TRANSPORT: enabled
- Storage Drivers:
- "overlay":
- CONFIG_OVERLAY_FS: enabled
STATUS: pass
|
部署 k3s
官方提供了部署脚本，部署起来还是非常方便的
1
2
3
4
5
6
7
8
9
|
export INSTALL_K3S_VERSION=v1.19.7+k3s1
export INSTALL_K3S_EXEC="--docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"
curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -
root@n1:~# kubectl get nodes
NAME STATUS ROLES AGE VERSION
n1 Ready master 6s v1.19.7+k3s1
|
openWrt k3s agent 部署
在 openwrt 中部署 k3s 时就不能使用 官方提供的自动部署脚本了，运行脚本时会提示错误 [ERROR] Can not find systemd or openrc to use as a process supervisor for k3s ,提示说明已经非常清楚了，好在 github 上有位大佬提供了 openwrt k3s 的安装包，谢谢这位大佬。
Docker 配置
此处为 openwrt 中 docker 配置，配置项说明请参考上面 n1 中 docker 配置项说明。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
cat /etc/docker/daemon.json
{
"data-root": "/data/docker-root",
"log-level": "warn",
"oom-score-adjust": -1000,
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"insecure-registries": ["idocker.io"],
"registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com"],
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
]
}
|
⚠️ 注意: 配置项修改后，多需要重启一下 docker 才会生效
1
|
/etc/init.d/dockerd restart
|
k3s agent 部署
下载安装包
1
2
3
4
5
6
|
wget https://github.com/discordianfish/k3s-openwrt/releases/download/9/k3s_1.17.4+k3s1_x86_64.opk
opkg install ./k3s_1.17.4+k3s1_x86_64.opk
root@OpenWrt:/opt/tools# /usr/bin/k3s --version
k3s version v1.17.4+k3s1 (3eee8ac3)
|
升级 k3s 版本
可以看的，默认的安装包中，k3s的版本为 v1.17.4+k3s1,与我们的 n1 上的 master 不匹配。
1
2
3
4
5
6
7
8
9
10
|
wget https://github.com/k3s-io/k3s/releases/download/v1.19.7%2Bk3s1/k3s
chmod a+x k3s
mv /usr/bin/k3s /usr/bin/k3s.old
cp -a k3s /usr/bin/k3s
root@OpenWrt:/opt/tools# k3s --version
k3s version v1.19.7+k3s1 (5a00e38d)
|
部署前的检查
同样这个打印的内容是 部署完成后打印 的
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
|
root@OpenWrt:~# k3s check-config
Verifying binaries in /var/lib/rancher/k3s/data/30740d1d67da51fe92b10367ecce4d580e552c634ad4a6c4dd13297ffd1f3edd/bin:
- sha256sum: good
- links: good
System:
- /usr/sbin iptables v1.8.4 (legacy): ok
- swap: disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed
Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000
info: reading kernel config from /proc/config.gz ...
Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled
Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: missing
- CONFIG_CGROUP_PERF: missing
- CONFIG_CGROUP_HUGETLB: missing
- CONFIG_NET_CLS_CGROUP: enabled
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: missing
- CONFIG_EXT4_FS_SECURITY: missing
enable these ext4 configs if you are using ext3 or ext4 as backing filesystem
- Network Drivers:
- "overlay":
- CONFIG_VXLAN: enabled (as module)
Optional (for encrypted networks):
- CONFIG_CRYPTO: enabled
- CONFIG_CRYPTO_AEAD: enabled
- CONFIG_CRYPTO_GCM: missing
- CONFIG_CRYPTO_SEQIV: missing
- CONFIG_CRYPTO_GHASH: missing
- CONFIG_XFRM: enabled
- CONFIG_XFRM_USER: enabled (as module)
- CONFIG_XFRM_ALGO: enabled (as module)
- CONFIG_INET_ESP: enabled (as module)
- CONFIG_INET_XFRM_MODE_TRANSPORT: missing
- Storage Drivers:
- "overlay":
- CONFIG_OVERLAY_FS: enabled
STATUS: pass
|
第一次部署检查的使用，显示有两项未通过，与 此链接 中描述类似
修改 k3s 服务配置
K3s 中文文档
默认安装服务后的配置文件是 /etc/config/k3s， 这个文件默认是没有的，需要我们自己创建
--token 后的配置为在 n1 master 中 cat /var/lib/rancher/k3s/server/node-token 的配置
1
2
|
root@n1:/# cat /var/lib/rancher/k3s/server/node-token
K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85
|
服务配置文件 /etc/config/k3s
1
2
3
4
|
cat /etc/config/k3s
config globals 'globals'
option opts '--server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666'
option root '/application/k3s'
|
配置文件，更具自己的实际使用情况增删即可
服务启动脚本 /etc/init.d/k3s
修改说明: 下面展示的启动脚本中，将 uci_get 替换使用成了 uci get。 将 -- server 修改成了 -- agent 如你部署的是 server 端那么此处你可以不做修改，且 /etc/config/k3s 中也得修改为 server 端的配置项。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
root@OpenWrt:~# cat /etc/init.d/k3s
#!/bin/sh /etc/rc.common
START=60
STOP=20
PIDFILE=/var/run/k3s.pid
EXEC="/usr/bin/k3s"
ensure_cgroup_mount() {
# Unmount /sys/fs/cgroup if mounted as cgroup
grep -q ' /sys/fs/cgroup cgroup' /proc/self/mounts && umount /sys/fs/cgroup
grep -q ' /sys/fs/cgroup tmpfs' /proc/self/mounts \
|| mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
mnt="/sys/fs/cgroup/$sys"
grep -q "cgroup $mnt " /proc/self/mounts && continue
mkdir -p "$mnt"
mount -n -t cgroup -o $sys cgroup "$mnt"
done
}
start() {
ensure_cgroup_mount
start-stop-daemon -S -b -x "$EXEC" -m -p "$PIDFILE" \
-- agent $(uci get k3s.globals.opts) \
--data-dir $(uci get k3s.globals.root)
}
stop() {
start-stop-daemon -K -p "$PIDFILE"
}
|
启动服务
1
2
3
4
5
6
7
8
9
10
11
|
/etc/init.d/k3s start # 启动命令
root@OpenWrt:~# ps -ef|grep k3s
3451 root 867m S {k3s-agent} /usr/bin/k3s agent # 启动成功后，会有进程在后台运行
21059 root 1096 S grep k3s
root@n1:/# kubectl get nodes
NAME STATUS ROLES AGE VERSION
n1 Ready master 2d3h v1.19.7+k3s1
openwrt Ready <none> 2d2h v1.19.7+k3s1
|
如你运行启动命令后，没有任何的响应，可先前台手动运行一步一步进行 debug, 如：
1
|
k3s agent --server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666
|
问题记录
启动 agent 时显示 level=error msg="Node password rejected, duplicate hostname or contents of '/etc/rancher/node/password' may not match server nod e-passwd entry, try enabling a unique node name with the --with-node-id flag" 错误 。
此问题的原因是之前部署时部署未成功，但 server 端还留有 agent 端的 server id ，当 agent 端再次注册到 server 端时，server 端通过主机名称进行匹配，发现 id 匹配不对时就导致了冲突。解决方法就是在 server 端删除老的 server id 或 更改 agent 端的主机名称（推荐使用在 server 端删除）。
1
2
3
4
5
6
|
cat /etc/rancher/node/password # agent
92a18abb46
cat /var/lib/rancher/k3s/server/cred/node-passwd
898170dfe92a18abb46,n1,n1,
9185b3215d3afcab9,openwrt,openwrt, # 将此行删除
|
openWrt 开放防火墙
编辑 /etc/config/network 配置文件, 添加如下类容
1
2
3
|
config interface 'k8s'
option proto 'none'
option ifname 'cni0'
|
编辑 /etc/config/firewall 添加如下类容
1
2
3
4
5
6
|
config zone
option name 'k8s'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
option network 'k8s'
|
重启一下 op 系统 ，使配置生效
部署应用
此处我们部署一个 deployment 类型的 nginx 服务，来简单测试一下 k3s 集群完整性。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
|
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
ports:
- protocol: TCP
name: web
port: 80
selector:
app: nginx
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nginx
labels:
app: nginx
spec:
replicas: 4
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.16.1
ports:
- name: web
containerPort: 80
EOF
|
1
2
3
4
5
6
7
|
watch kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-bcf5bbd7d-n2lfx 1/1 Running 0 3m2s 10.42.0.14 n1 <none> <none>
nginx-bcf5bbd7d-zhj6j 1/1 Running 0 78s 10.42.0.15 n1 <none> <none>
nginx-bcf5bbd7d-sxlvp 1/1 Running 0 78s 10.42.1.17 openwrt <none> <none>
nginx-bcf5bbd7d-bsxtn 1/1 Running 0 78s 10.42.1.18 openwrt <none> <none>
|
可以看到上述打印 n1 和 openwrt 分别启动了 2 个容器，现在我们分别在两台机器上使用 curl 测试一下对方主机中 pod 的 ip，来检查网络是否可以连通。
n1, 测试 curl 10.42.1.17 和 10.42.1.18
openWrt, 测试 curl 10.42.0.14 和 10.42.0.15
经过简单的测试，可以看到 k3s 集群使用起来没有什么问题。
配置使用 dashboard
原本打算使用 rancher 来作为 dashboard 的，考虑其资源的占用什么的，还是选择使用了 lens，即将 dashboard 部署在客户端。
Lens Github地址
省略。安装过程。。。。
查看 n1 即 master 中的 kubelet 配置文件
1
|
root@n1:/# cat ~/.kube/config
|
将打印内容复制，粘贴到客户端的电脑上
内容保存在至 ~/.kube/k3s-config 中
lens 添加主机
如若提示证书验证错，这里使用一个小技巧即可，在主机中 hosts 文件中添加对应映射，并修改 kubelet 配置文件中 的 server 地址 为映射的域名即可。
1
2
|
cat /etc/hosts
192.168.8.102 n1
|
开启监控
鼠标移到到集群，右键，然后点击 settings
拉到后面，然后点击进行安装
Done。。。
