Skip to content

Commit ad5d137

Browse files
vadorovskyvadorovsky
committed
vagrant: Deploy Kubernetes with containerd runtime
This commit brings the following changes to the Vagrant ecosystem: - memory increased to 8GB - no restart after provisioning the control plane components - using upstream kubeadm and upstream containerd runtime - adding several useful packages Signed-off-by: Michal Rostecki <[email protected]>
1 parent 36be6a0 commit ad5d137

File tree

8 files changed

+194
-10
lines changed

8 files changed

+194
-10
lines changed

‎Vagrantfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ Vagrant.configure("2") do |config|
1010
cp.vm.hostname = "control-plane.local"
1111
cp.vm.provider :libvirt do |libvirt|
1212
libvirt.cpus = 4
13-
libvirt.memory = 4096
13+
libvirt.memory = 8192
1414
end
1515
cp.vm.provision "shell", path: "contrib/vagrant/vagrant-fix.sh"
1616
cp.vm.provision "shell", path: "contrib/vagrant/base.sh", reboot: true
1717
cp.vm.provision "shell", path: "contrib/vagrant/build.sh", privileged: false
18-
cp.vm.provision "shell", path: "contrib/vagrant/control-plane-base.sh", reboot: true
18+
cp.vm.provision "shell", path: "contrib/vagrant/control-plane-base.sh"
1919
cp.vm.provision "shell", path: "contrib/vagrant/control-plane.sh"
2020
cp.vm.provision "shell", path: "contrib/vagrant/kubeconfig.sh", privileged: false
2121
cp.vm.provision "shell", path: "contrib/vagrant/addons.sh", privileged: false

‎contrib/vagrant/addons.sh‎

100644100755
File mode changed.

‎contrib/vagrant/base.sh‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,36 @@ zypper install -y \
77
bpftool \
88
cargo \
99
clang \
10+
conntrack-tools \
11+
containerd \
12+
docker \
13+
ebtables \
14+
ethtool \
1015
libbpf-devel \
1116
libopenssl-devel \
1217
llvm \
18+
podman \
19+
podman-cni-config \
1320
rust \
14-
rustfmt
21+
rustfmt \
22+
socat \
23+
tmux \
24+
wget
25+
26+
## wget -O /etc/systemd/system/containerd.service https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
27+
## sed -i -e "s|/usr/local/bin/containerd|/sbin/containerd|g" \
28+
## /etc/systemd/system/containerd.service
29+
30+
cat <<EOF > /etc/modules-load.d/k8s.conf
31+
br_netfilter
32+
EOF
33+
34+
cat <<EOF > /etc/sysctl.d/k8s.conf
35+
net.bridge.bridge-nf-call-ip6tables = 1
36+
net.bridge.bridge-nf-call-iptables = 1
37+
net.ipv4.ip_forward = 1
38+
net.ipv4.conf.all.forwarding = 1
39+
EOF
1540

1641
sed -i -e "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=\"lsm=bpf,integrity\"/" \
1742
/etc/default/grub

‎contrib/vagrant/build.sh‎

100644100755
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
cd /home/vagrant/enclave
44

5+
export CLANG=/usr/bin/clang-12
56
cargo install --path .
67
# cargo install --path . --target-dir /usr/local/bin
78

‎contrib/vagrant/control-plane-base.sh‎

100644100755
Lines changed: 159 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,161 @@
11
#!/bin/bash
22

3-
zypper install -y -t pattern kubic_admin
4-
install -D -m 0644 /home/vagrant/enclave/contrib/crio/00-default.conf /etc/crio/crio.conf.d/00-default.conf
5-
systemctl restart crio
6-
systemctl enable kubelet.service
3+
# # zypper install -y -t pattern kubic_admin
4+
# zypper install -y cri-o
5+
# # # install -D -m 0644 /home/vagrant/enclave/contrib/crio/00-default.conf /etc/crio/crio.conf.d/00-default.conf
6+
# mv /etc/crio/crio.conf.d/00-default.conf /etc/crio/crio.conf
7+
# systemctl enable crio
8+
# systemctl start crio
9+
# # systemctl restart crio
10+
# # systemctl enable kubelet.service
11+
12+
cat <<EOF > /etc/containerd/config.toml
13+
root = "/var/lib/containerd"
14+
state = "/run/containerd"
15+
oom_score = 0
16+
17+
[grpc]
18+
address = "/run/containerd/containerd.sock"
19+
uid = 0
20+
gid = 0
21+
max_recv_message_size = 16777216
22+
max_send_message_size = 16777216
23+
24+
[debug]
25+
address = ""
26+
uid = 0
27+
gid = 0
28+
level = ""
29+
30+
[metrics]
31+
address = ""
32+
grpc_histogram = false
33+
34+
[cgroup]
35+
path = ""
36+
37+
[plugins]
38+
[plugins.cgroups]
39+
no_prometheus = false
40+
[plugins.cri]
41+
stream_server_address = ""
42+
stream_server_port = "10010"
43+
enable_selinux = false
44+
sandbox_image = "k8s.gcr.io/pause:3.2"
45+
stats_collect_period = 10
46+
systemd_cgroup = true
47+
enable_tls_streaming = false
48+
max_container_log_line_size = 16384
49+
[plugins.cri.containerd]
50+
snapshotter = "overlayfs"
51+
no_pivot = true
52+
[plugins.cri.containerd.default_runtime]
53+
runtime_type = "io.containerd.runtime.v1.linux"
54+
runtime_engine = ""
55+
runtime_root = ""
56+
[plugins.cri.containerd.untrusted_workload_runtime]
57+
runtime_type = ""
58+
runtime_engine = ""
59+
runtime_root = ""
60+
[plugins.cri.cni]
61+
bin_dir = "/opt/cni/bin"
62+
conf_dir = "/etc/cni/net.d"
63+
conf_template = ""
64+
[plugins.cri.registry]
65+
[plugins.cri.registry.mirrors]
66+
[plugins.cri.registry.mirrors."docker.io"]
67+
endpoint = ["https://registry-1.docker.io"]
68+
[plugins.diff-service]
69+
default = ["walking"]
70+
[plugins.linux]
71+
shim = "containerd-shim"
72+
runtime = "/home/vagrant/.cargo/bin/enclave-runc-wrapper"
73+
runtime_root = ""
74+
no_shim = false
75+
shim_debug = false
76+
[plugins.scheduler]
77+
pause_threshold = 0.02
78+
deletion_threshold = 0
79+
mutation_threshold = 100
80+
schedule_delay = "0s"
81+
startup_delay = "100ms"
82+
EOF
83+
84+
cat <<EOF > /etc/docker/daemon.json
85+
{
86+
"log-level": "warn",
87+
"log-driver": "json-file",
88+
"log-opts": {
89+
"max-size": "10m",
90+
"max-file": "5"
91+
},
92+
"default-runtime": "runc-enclave",
93+
"runtimes": {
94+
"runc-enclave": {
95+
"path": "/home/vagrant/.cargo/bin/enclave-runc-wrapper"
96+
}
97+
}
98+
}
99+
EOF
100+
101+
cat <<EOF > /etc/systemd/system/containerd.service
102+
[Unit]
103+
Description=containerd container runtime
104+
Documentation=https://containerd.io
105+
After=network.target local-fs.target
106+
107+
[Service]
108+
ExecStartPre=-/sbin/modprobe overlay
109+
ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/systemd
110+
ExecStartPre=/usr/bin/mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
111+
ExecStart=/sbin/containerd
112+
113+
Type=notify
114+
Delegate=yes
115+
KillMode=process
116+
Restart=always
117+
RestartSec=5
118+
# Having non-zero Limit*s causes performance problems due to accounting overhead
119+
# in the kernel. We recommend using cgroups to do container-local accounting.
120+
LimitNPROC=infinity
121+
LimitCORE=infinity
122+
LimitNOFILE=infinity
123+
# Comment TasksMax if your systemd version does not supports it.
124+
# Only systemd 226 and above support this version.
125+
TasksMax=infinity
126+
OOMScoreAdjust=-999
127+
128+
[Install]
129+
WantedBy=multi-user.target
130+
EOF
131+
132+
systemctl enable containerd
133+
systemctl start containerd
134+
systemctl enable docker
135+
systemctl start docker
136+
137+
# zypper install -y minikube
138+
139+
CNI_VERSION="v0.9.1"
140+
sudo mkdir -p /opt/cni/bin
141+
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | sudo tar -C /opt/cni/bin -xz
142+
143+
DOWNLOAD_DIR=/usr/local/bin
144+
sudo mkdir -p $DOWNLOAD_DIR
145+
146+
CRI_TOOLS_VERSION="v1.21.0"
147+
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRI_TOOLS_VERSION/crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz
148+
sudo tar zxvf crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz -C /usr/local/bin
149+
rm -f crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz
150+
151+
RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
152+
cd $DOWNLOAD_DIR
153+
sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
154+
sudo chmod +x {kubeadm,kubelet,kubectl}
155+
156+
RELEASE_VERSION="v0.9.0"
157+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service
158+
sudo mkdir -p /etc/systemd/system/kubelet.service.d
159+
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
160+
161+
systemctl enable --now kubelet

‎contrib/vagrant/control-plane.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
kubeadm init --cri-socket /var/run/crio/crio.sock
3+
kubeadm init --cri-socket /run/containerd/containerd.sock

‎contrib/vagrant/kubeconfig.sh‎

100644100755
File mode changed.

‎contrib/vagrant/vagrant-fix.sh‎

100644100755
Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,8 @@
22

33
# Workaround for https://github.com/hashicorp/vagrant/issues/1659
44

5-
echo "" >> /etc/sudoers
6-
echo "vagrant ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
5+
cat <<EOF >> /etc/sudoers
6+
7+
vagrant ALL=(ALL) NOPASSWD:ALL
8+
Defaults:vagrant !requiretty
9+
EOF

0 commit comments

Comments
 (0)