环境说明

软件说明

  • K3s version : v1.19.7+k3s1

  • Docker version: 19.03.13 (n1),19.03.12 (openwrt)

  • 写盘工具: balenaEtcher

机器说明

IP 地址 机型 配置 操作系统 角色
192.168.8.1 占美 (机型不详) 4c 2g ( cpu N2940 ) openWrt(X86_64 自编译 node/agent
192.168.8.112 斐讯 N1 4c 2g Armbian ( 5.0.2-aml-s905) master/server

openWrt 自编译

需要自编译的原因是因为,之前使用 esir 打包提供的高大全 op 固件,部署k3s 时发现,内核没有开启vxlan的特性,导致部署不成功,如你使用的 op 固件已开启了 vxlan 特性可省略此步骤,查看方法:

1
2
3
4
root@OpenWrt:~# lsmod|grep -i vxlan  # 打印输入表示具有 vxlan 特性
ip6_udp_tunnel         16384  1 vxlan
udp_tunnel             16384  1 vxlan
vxlan                  53248  0 

此处感谢 esir 等大佬 提供的 github actions ci 工作流

仓库地址 视频教程

fork 仓库代码

登录 github, 并将 代码 fork 至自己仓库 image-20210126095535833

git clone 代码

1
2
3
git clone fork https://github.com.cnpmjs.org/cdryzun/AutoBuild-OpenWrt.git

cd AutoBuild-OpenWrt 

修改配置文件

修改工作流文件

打开自己心爱的编辑器

且 演示 x86_64 平台

1
2
3
4
5
6
 code .github/workflows/Build_OP_x86_64.yml  # 编辑 ci 构建脚本
 
 ...
   push:
   branches: 
     - master  # 打开注释,注意空格

修改构建文件

1
code x86_64.config

配置参考资料

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
CONFIG_PACKAGE_grub2-efi=y
CONFIG_EFI_IMAGES=y
CONFIG_TARGET_IMAGES_GZIP=y
CONFIG_TARGET_KERNEL_PARTSIZE=16
CONFIG_TARGET_ROOTFS_PARTSIZE=300
# CONFIG_GRUB_CONSOLE is not set
CONFIG_PACKAGE_luci-app-docker=y
CONFIG_PACKAGE_luci-i18n-docker-zh-cn=y
CONFIG_PACKAGE_luci-app-ssr-plus=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Shadowsocks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Simple_obfs=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray_plugin=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Kcptun=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Server=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Socks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_NaiveProxy=y
CONFIG_PACKAGE_luci-theme-argon=y
CONFIG_PACKAGE_luci-app-mwan3=y
CONFIG_PACKAGE_luci-app-mwan3helper=y
CONFIG_PACKAGE_luci-i18n-mwan3-zh-cn=y
CONFIG_PACKAGE_luci-i18n-mwan3helper-zh-cn=y
CONFIG_PACKAGE_luci-app-syncdial=y
CONFIG_PACKAGE_luci-app-sfe=y
CONFIG_PACKAGE_luci-app-ttyd=y
CONFIG_PACKAGE_luci-i18n-ttyd-zh-cn=y
CONFIG_PACKAGE_luci-app-webadmin=y
CONFIG_PACKAGE_luci-i18n-webadmin-zh-cn=y
CONFIG_PACKAGE_luci-app-zerotier=y
CONFIG_PACKAGE_luci-app-frpc=y
CONFIG_PACKAGE_luci-app-kodexplorer=y
CONFIG_PACKAGE_luci-app-nfs=y
CONFIG_PACKAGE_ss=y
CONFIG_PACKAGE_luci-app-nps=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-rclone=y
CONFIG_PACKAGE_luci-app-radicale=y
CONFIG_PACKAGE_luci-app-qbittorrent=y
CONFIG_PACKAGE_luci-app-samba=y
CONFIG_PACKAGE_luci-app-wol=y
CONFIG_PACKAGE_luci-app-xlnetacc=y
CONFIG_PACKAGE_luci-app-bird1-ipv4=y
CONFIG_PACKAGE_luci-app-bird1-ipv6=y
CONFIG_PACKAGE_luci-app-commands=y
CONFIG_PACKAGE_luci-app-diskman=y
CONFIG_PACKAGE_luci-app-dnscrypt-proxy=y
CONFIG_PACKAGE_luci-app-dnsforwarder=y
CONFIG_PACKAGE_luci-app-familycloud=y
CONFIG_PACKAGE_luci-app-hd-idle=y
CONFIG_PACKAGE_luci-app-netdata=y
CONFIG_PACKAGE_ipv6helper=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_curl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_wget=y
CONFIG_PACKAGE_lrzsz=y
CONFIG_PACKAGE_vim=y
CONFIG_PACKAGE_zip=y
CONFIG_PACKAGE_kmod-kvm-amd=y
CONFIG_PACKAGE_kmod-kvm-intel=y
CONFIG_PACKAGE_kmod-kvm-x86=y
CONFIG_OPENSSL_ENGINE_CRYPTO=y
CONFIG_OPENSSL_ENGINE_DIGEST=y
CONFIG_OPENSSL_WITH_CAMELLIA=y
CONFIG_OPENSSL_WITH_COMPRESSION=y
CONFIG_OPENSSL_WITH_DTLS=y
CONFIG_OPENSSL_WITH_EC2M=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_GOST=y
CONFIG_OPENSSL_WITH_IDEA=y
CONFIG_OPENSSL_WITH_MDC2=y
CONFIG_OPENSSL_WITH_RFC3779=y
CONFIG_OPENSSL_WITH_SEED=y
CONFIG_OPENSSL_WITH_WHIRLPOOL=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_KERNEL_BUILD_USER="treeSir Playground"
CONFIG_GRUB_TITLE="OpenWrt AutoBuild by treeSirPlayground"
CONFIG_PACKAGE_kmod-usb-ohci=y
CONFIG_PACKAGE_kmod-usb-ohci-pci=y
CONFIG_PACKAGE_kmod-usb-storage-uas=y
CONFIG_PACKAGE_kmod-usb-uhci=y
CONFIG_PACKAGE_kmod-sdhci=y
CONFIG_PACKAGE_kmod-usb-ehci=y
CONFIG_PACKAGE_kmod-usb2=y
CONFIG_PACKAGE_kmod-usb2-pci=y
CONFIG_PACKAGE_luci-app-passwall=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Brook=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan_GO=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_simple-obfs=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_v2ray-plugin=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_https_dns_proxy=n
# CONFIG_PACKAGE_naiveproxy=y
# CONFIG_PACKAGE_nspr=y
# CONFIG_PACKAGE_shadowsocks-libev-ss-server=y
# CONFIG_PACKAGE_ssocks=y
# CONFIG_PACKAGE_ssocksd=y
# CONFIG_PACKAGE_trojan-go=y
# CONFIG_PACKAGE_trojan-plus=y
# CONFIG_PACKAGE_chinadns-ng=y
CONFIG_TARGET_x86=y
CONFIG_TARGET_x86_64=y
CONFIG_TARGET_x86_64_Generic=y
CONFIG_FEED_luci=y
CONFIG_FEED_packages=y
CONFIG_FEED_routing=y
CONFIG_FEED_telephony=y
CONFIG_IPTABLES_CONNLABEL=y
CONFIG_IPTABLES_NFTABLES=y
CONFIG_KERNEL_BLK_CGROUP=y
CONFIG_KERNEL_CFS_BANDWIDTH=y
CONFIG_KERNEL_CGROUPS=y
CONFIG_KERNEL_CGROUP_CPUACCT=y
CONFIG_KERNEL_CGROUP_DEVICE=y
CONFIG_KERNEL_CGROUP_FREEZER=y
CONFIG_KERNEL_CGROUP_PIDS=y
CONFIG_KERNEL_CGROUP_SCHED=y
CONFIG_KERNEL_CPUSETS=y
CONFIG_KERNEL_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_KERNEL_FAIR_GROUP_SCHED=y
CONFIG_KERNEL_FREEZER=y
CONFIG_KERNEL_IPC_NS=y
CONFIG_KERNEL_KEYS=y
CONFIG_KERNEL_LXC_MISC=y
CONFIG_KERNEL_MEMCG=y
CONFIG_KERNEL_MM_OWNER=y
CONFIG_KERNEL_NAMESPACES=y
CONFIG_KERNEL_NETPRIO_CGROUP=y
CONFIG_KERNEL_NET_CLS_CGROUP=y
CONFIG_KERNEL_NET_NS=y
CONFIG_KERNEL_PID_NS=y
CONFIG_KERNEL_POSIX_MQUEUE=y
CONFIG_KERNEL_PROC_PID_CPUSET=y
CONFIG_KERNEL_RESOURCE_COUNTERS=y
CONFIG_KERNEL_SECCOMP=y
CONFIG_KERNEL_SECCOMP_FILTER=y
CONFIG_KERNEL_USER_NS=y
CONFIG_KERNEL_UTS_NS=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ip-bridge=y
CONFIG_PACKAGE_ip-full=y
CONFIG_PACKAGE_ipset=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_iptables-mod-ipopt=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-br-netfilter=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hw-ccp=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rsa=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-fuse=y
CONFIG_PACKAGE_kmod-ikconfig=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-ipopt=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-leds-gpio=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-ipvs=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-base=y
CONFIG_PACKAGE_kmod-random-core=y
CONFIG_PACKAGE_kmod-softdog=y
CONFIG_PACKAGE_kmod-sound-core=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-core=y
CONFIG_PACKAGE_kmod-usb3=y
CONFIG_PACKAGE_kmod-veth=y
CONFIG_PACKAGE_kmod-vxlan=y
CONFIG_PACKAGE_kmod-ppp=y
CONFIG_PACKAGE_kmod-pppox=y
CONFIG_PACKAGE_libipset=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnftnl=y
CONFIG_TARGET_ROOTFS_PARTSIZE=1024
# CONFIG_PACKAGE_iptables-mod-conntrack-label is not set
# CONFIG_PACKAGE_kmod-ipt-conntrack-label is not set
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y

示例配置且供参考使用

构建固件

触发构建

提交代码

image-20210126102255870

提交代码前,检查一下 actions 是否是打开状态

1
2
3
4
5
6
git add .github/workflows/Build_OP_x86_64.yml
git add ./x86_64.config

git commit -m "build custom openwrt"

git push

提交完成后,等待构建结束

下载固件

构建完成后如何下载 op固件 文件

image-20210126102444462

点击后 往下面拉,点击进行下载

image-20210126102631119

省略 op 写入步骤, 下载固件后解压进入 OpenWrt/targets/x86/64 目录, 就可以看到固件了。

image-20210126104146816

N1 k3s master 部署

n1 使用系统为 Armbian,同样省略写入步骤,n1怎么刷入 armbian 系统在网上资料还是非常多的。

docker 配置文件

此处为 n1 docker 配置文件,不一定适合你,可截取部分参考使用。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
 cat /etc/docker/daemon.json 
{
  "experimental": false,
  "registry-mirrors": [
    "https://7bezldxe.mirror.aliyuncs.com"
  ],
  "oom-score-adjust": -1000,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  },
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "features": {
    "buildkit": true
  },
  "insecure-registries": [
    "idocker.io"
  ]
}

配置项简短说明:

  • “log-driver”: “json-file” 设置json 格式日志
  • “oom-score-adjust”: -1000 防止容器被 内核 oom
  • “log-opts” 设置容器日志大小
  • “max-concurrent-downloads”: 10 并行下载容器数量
  • “max-concurrent-uploads”: 10 并行上传
  • “storage-driver”: “overlay2” 设置存储驱动为 overlay2
  • “bip” 容器默认的网段
  • “registry-mirrors” 配置镜像下载加速这里使用的是阿里云的
  • “insecure-registries” 信任的私服地址

如若修改了配置,记得重启一下服务查看是否生效

1
2
3
service docker restart

docker info

部署前的检查

这个打印的内容是 部署完成后打印 的,部署前的检查忘记做记录了。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
root@n1:/# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/ad8f0f93ebb9db5c507884fcdec249d73dd348293dac194e01462c57815cca46/bin:
- sha256sum: good
- links: good

System:
- /sbin iptables v1.6.1: older than v1.8
- swap: should be disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled (as module)
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: enabled
- CONFIG_CGROUP_PERF: enabled
- CONFIG_CGROUP_HUGETLB: enabled
- CONFIG_NET_CLS_CGROUP: enabled (as module)
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: enabled
- CONFIG_EXT4_FS_SECURITY: enabled
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: enabled (as module)
      - CONFIG_CRYPTO_SEQIV: enabled
      - CONFIG_CRYPTO_GHASH: enabled (as module)
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: enabled
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

部署 k3s

官方提供了部署脚本,部署起来还是非常方便的

1
2
3
4
5
6
7
8
9
export INSTALL_K3S_VERSION=v1.19.7+k3s1

export INSTALL_K3S_EXEC="--docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig  ~/.kube/config --write-kubeconfig-mode 666"

curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -
 
root@n1:~#  kubectl get nodes
NAME   STATUS   ROLES    AGE   VERSION
n1     Ready    master   6s    v1.19.7+k3s1

openWrt k3s agent 部署

在 openwrt 中部署 k3s 时就不能使用 官方提供的自动部署脚本了,运行脚本时会提示错误 [ERROR] Can not find systemd or openrc to use as a process supervisor for k3s ,提示说明已经非常清楚了,好在 github 上有位大佬提供了 openwrt k3s 的安装包,谢谢这位大佬。

Docker 配置

此处为 openwrt 中 docker 配置,配置项说明请参考上面 n1 中 docker 配置项说明。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
 cat /etc/docker/daemon.json 
{
"data-root": "/data/docker-root",
"log-level": "warn",
"oom-score-adjust": -1000,
"log-driver": "json-file",
  "log-opts": {
   "max-size": "100m",
   "max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"insecure-registries": ["idocker.io"],
"registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com"],
"storage-driver": "overlay2",
"storage-opts": [
   "overlay2.override_kernel_check=true"
]
}

⚠️ 注意: 配置项修改后,多需要重启一下 docker 才会生效

1
/etc/init.d/dockerd restart

k3s agent 部署

下载安装包

1
2
3
4
5
6
wget https://github.com/discordianfish/k3s-openwrt/releases/download/9/k3s_1.17.4+k3s1_x86_64.opk

opkg install ./k3s_1.17.4+k3s1_x86_64.opk

root@OpenWrt:/opt/tools# /usr/bin/k3s --version
k3s version v1.17.4+k3s1 (3eee8ac3)

升级 k3s 版本

可以看的,默认的安装包中,k3s的版本为 v1.17.4+k3s1,与我们的 n1 上的 master 不匹配。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
wget https://github.com/k3s-io/k3s/releases/download/v1.19.7%2Bk3s1/k3s

chmod a+x k3s

mv /usr/bin/k3s /usr/bin/k3s.old

cp -a k3s /usr/bin/k3s

root@OpenWrt:/opt/tools# k3s --version
k3s version v1.19.7+k3s1 (5a00e38d)

部署前的检查

同样这个打印的内容是 部署完成后打印

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
root@OpenWrt:~# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/30740d1d67da51fe92b10367ecce4d580e552c634ad4a6c4dd13297ffd1f3edd/bin:
- sha256sum: good
- links: good

System:
- /usr/sbin iptables v1.8.4 (legacy): ok
- swap: disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: missing
- CONFIG_CGROUP_PERF: missing
- CONFIG_CGROUP_HUGETLB: missing
- CONFIG_NET_CLS_CGROUP: enabled
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: missing
- CONFIG_EXT4_FS_SECURITY: missing
    enable these ext4 configs if you are using ext3 or ext4 as backing filesystem
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: missing
      - CONFIG_CRYPTO_SEQIV: missing
      - CONFIG_CRYPTO_GHASH: missing
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: missing
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

第一次部署检查的使用,显示有两项未通过,与 此链接 中描述类似

修改 k3s 服务配置

K3s 中文文档

默认安装服务后的配置文件是 /etc/config/k3s, 这个文件默认是没有的,需要我们自己创建

--token 后的配置为在 n1 master 中 cat /var/lib/rancher/k3s/server/node-token 的配置

1
2
root@n1:/# cat /var/lib/rancher/k3s/server/node-token
K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85

服务配置文件 /etc/config/k3s

1
2
3
4
cat /etc/config/k3s
config globals 'globals'
        option opts '--server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666'
        option root '/application/k3s'

配置文件,更具自己的实际使用情况增删即可

服务启动脚本 /etc/init.d/k3s

修改说明: 下面展示的启动脚本中,将 uci_get 替换使用成了 uci get。 将 -- server 修改成了 -- agent 如你部署的是 server 端那么此处你可以不做修改,且 /etc/config/k3s 中也得修改为 server 端的配置项。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
root@OpenWrt:~# cat /etc/init.d/k3s
#!/bin/sh /etc/rc.common

START=60
STOP=20

PIDFILE=/var/run/k3s.pid
EXEC="/usr/bin/k3s"

ensure_cgroup_mount() {
  # Unmount /sys/fs/cgroup if mounted as cgroup
  grep -q ' /sys/fs/cgroup cgroup' /proc/self/mounts && umount /sys/fs/cgroup

  grep -q ' /sys/fs/cgroup tmpfs' /proc/self/mounts \
    || mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup

  for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
    mnt="/sys/fs/cgroup/$sys"
    grep -q "cgroup $mnt " /proc/self/mounts && continue
    mkdir -p "$mnt"
    mount -n -t cgroup -o $sys cgroup "$mnt"
  done
}

start() {
  ensure_cgroup_mount
  start-stop-daemon -S -b -x "$EXEC" -m -p "$PIDFILE" \
    -- agent $(uci get k3s.globals.opts) \
    --data-dir $(uci get k3s.globals.root)
}

stop() {
  start-stop-daemon -K -p "$PIDFILE"
}

启动服务

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
/etc/init.d/k3s start  # 启动命令

root@OpenWrt:~# ps -ef|grep k3s
 3451 root      867m S    {k3s-agent} /usr/bin/k3s agent   # 启动成功后,会有进程在后台运行
21059 root      1096 S    grep k3s


root@n1:/# kubectl get nodes
NAME      STATUS   ROLES    AGE    VERSION
n1        Ready    master   2d3h   v1.19.7+k3s1
openwrt   Ready    <none>   2d2h   v1.19.7+k3s1

如你运行启动命令后,没有任何的响应,可先前台手动运行一步一步进行 debug, 如:

1
 k3s agent --server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666

问题记录

启动 agent 时显示 level=error msg="Node password rejected, duplicate hostname or contents of '/etc/rancher/node/password' may not match server nod e-passwd entry, try enabling a unique node name with the --with-node-id flag" 错误 。

此问题的原因是之前部署时部署未成功,但 server 端还留有 agent 端的 server id ,当 agent 端再次注册到 server 端时,server 端通过主机名称进行匹配,发现 id 匹配不对时就导致了冲突。解决方法就是在 server 端删除老的 server id 或 更改 agent 端的主机名称(推荐使用在 server 端删除)。

1
2
3
4
5
6
cat /etc/rancher/node/password  # agent
92a18abb46

cat /var/lib/rancher/k3s/server/cred/node-passwd
898170dfe92a18abb46,n1,n1,
9185b3215d3afcab9,openwrt,openwrt,  # 将此行删除

openWrt 开放防火墙

编辑 /etc/config/network 配置文件, 添加如下类容

1
2
3
config interface 'k8s'
	option proto 'none'
	option ifname 'cni0'

编辑 /etc/config/firewall 添加如下类容

1
2
3
4
5
6
config zone
        option name 'k8s'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option network 'k8s'

重启一下 op 系统 ,使配置生效

1
reboot

部署应用

此处我们部署一个 deployment 类型的 nginx 服务,来简单测试一下 k3s 集群完整性

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  ports:
    - protocol: TCP
      name: web
      port: 80
  selector:
    app: nginx

---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 4
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.16.1
          ports:
            - name: web
              containerPort: 80
EOF
1
2
3
4
5
6
7
watch kubectl get pod -o wide

NAME                    READY   STATUS    RESTARTS   AGE    IP           NODE      NOMINATED NODE   READINESS GATES
nginx-bcf5bbd7d-n2lfx   1/1     Running   0          3m2s   10.42.0.14   n1        <none>           <none>
nginx-bcf5bbd7d-zhj6j   1/1     Running   0          78s    10.42.0.15   n1        <none>           <none>
nginx-bcf5bbd7d-sxlvp   1/1     Running   0          78s    10.42.1.17   openwrt   <none>           <none>
nginx-bcf5bbd7d-bsxtn   1/1     Running   0          78s    10.42.1.18   openwrt   <none>           <none>

可以看到上述打印 n1openwrt 分别启动了 2 个容器,现在我们分别在两台机器上使用 curl 测试一下对方主机中 pod 的 ip,来检查网络是否可以连通。

n1, 测试 curl 10.42.1.17 和 10.42.1.18

image-20210126132703410

openWrt, 测试 curl 10.42.0.14 和 10.42.0.15

image-20210126132810261

经过简单的测试,可以看到 k3s 集群使用起来没有什么问题。

配置使用 dashboard

原本打算使用 rancher 来作为 dashboard 的,考虑其资源的占用什么的,还是选择使用了 lens,即将 dashboard 部署在客户端。

Lens Github地址

省略。安装过程。。。。

查看 n1 即 master 中的 kubelet 配置文件

1
root@n1:/# cat ~/.kube/config

将打印内容复制,粘贴到客户端的电脑上

内容保存在至 ~/.kube/k3s-config

lens 添加主机

image-20210126134036320

如若提示证书验证错,这里使用一个小技巧即可,在主机中 hosts 文件中添加对应映射,并修改 kubelet 配置文件中server 地址 为映射的域名即可。

1
2
cat /etc/hosts
192.168.8.102 n1

image-20210126134624679

开启监控

鼠标移到到集群,右键,然后点击 settings

image-20210126135339985

拉到后面,然后点击进行安装

image-20210126135403341

image-20210126135820673

image-20210126135947717

Done。。。