enclave-runc-wrapper: Add runc wrapper as a bridge with k8s

Cargo.toml

@@ -12,11 +12,20 @@ anyhow = "1.0"
 byteorder = "1.4"
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 config = { version = "0.11", default-features = false, features = ["toml"] }
+futures = "0.3"
+kube = "0.57"
+k8s-openapi = { version = "0.12", default-features = false, features = ["v1_21"] }
 lazy_static = "1.4"
+libc = { version = "0.2", features = [ "extra_traits" ] }
 libbpf-rs = "0.10"
+nix = "0.21"
+plain = "0.2"
 regex = { version = "1.5", default-features = false, features = ["perf"] }
 serde = "1.0"
+serde_json = "1.0"
 thiserror = "1.0"
+tokio = { version = "1.7", features = ["macros", "process", "rt-multi-thread"] }
+uuid = { version = "0.8", default-features = false, features = ["v4"] }
 
 [build-dependencies]
 anyhow = "1.0"

Dockerfile

@@ -17,8 +17,9 @@ RUN wget https://apt.llvm.org/llvm-snapshot.gpg.key && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 WORKDIR /usr/local/src
 # Build libbpf and bpftool from the newest stable kernel sources.
+ARG KERNEL_TAG=v5.13.1
 RUN git clone --depth 1 -b \
-        v$(curl -s https://www.kernel.org/ | grep -A1 'stable:' | grep -oP '(?<=strong>).*(?=</strong.*)') \
+        ${KERNEL_TAG} \
         git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git && \
     cd linux && \
     cd tools/lib/bpf && \

Makefile

@@ -1,18 +1,42 @@
 CRUNTIME ?= docker
+KERNEL_TAG ?=
+EXTRA_DOCKER_FLAGS ?=
+
+ifneq ($(KERNEL_TAG),)
+EXTRA_DOCKER_FLAGS += --build-arg KERNEL_TAG=$(KERNEL_TAG)
+endif
 
 .PHONY: all
 all: build fmt lint
 
 .PHONY: build
 build:
-	$(CRUNTIME) build --target artifact --output type=local,dest=out .
+	$(CRUNTIME) build \
+		--target artifact \
+		--output type=local,dest=out \
+		$(EXTRA_DOCKER_FLAGS) \
+		.
 
 .PHONY: fmt
 fmt:
-	$(CRUNTIME) build --target rustfmt --tag enclave-rustfmt .
-	$(CRUNTIME) run --rm -i -v $(shell pwd):/usr/local/src/enclave enclave-rustfmt
+	$(CRUNTIME) build \
+		--target rustfmt \
+		--tag enclave-rustfmt \
+		$(EXTRA_DOCKER_FLAGS) \
+		.
+	$(CRUNTIME) run \
+		--rm -i \
+		-v $(shell pwd):/usr/local/src/enclave \
+		enclave-rustfmt
 
 .PHONY: lint
 lint:
-	$(CRUNTIME) build --target clippy --tag enclave-clippy .
-	$(CRUNTIME) run --rm -i -v $(shell pwd):/usr/local/src/enclave enclave-clippy
+	$(CRUNTIME) build \
+		--target clippy \
+		--tag enclave-clippy \
+		$(EXTRA_DOCKER_FLAGS) \
+		.
+	$(CRUNTIME) run \
+		--rm -i \
+		-v $(shell pwd):/usr/local/src/enclave \
+		enclave-clippy

Vagrantfile

@@ -10,14 +10,14 @@ Vagrant.configure("2") do |config|
     cp.vm.hostname = "control-plane.local"
     cp.vm.provider :libvirt do |libvirt|
       libvirt.cpus = 4
-      libvirt.memory = 4096
+      libvirt.memory = 8192
     end
-    cp.vm.provision "shell", path: "contrib/vagrant/vagrant-fix.sh"
-    cp.vm.provision "shell", path: "contrib/vagrant/base.sh", reboot: true
-    cp.vm.provision "shell", path: "contrib/vagrant/build.sh", privileged: false
-    cp.vm.provision "shell", path: "contrib/vagrant/control-plane-base.sh", reboot: true
-    cp.vm.provision "shell", path: "contrib/vagrant/control-plane.sh"
-    cp.vm.provision "shell", path: "contrib/vagrant/kubeconfig.sh", privileged: false
-    cp.vm.provision "shell", path: "contrib/vagrant/addons.sh", privileged: false
+    cp.vm.provision "shell", path: "contrib/provision/vagrant-fix.sh"
+    cp.vm.provision "shell", path: "contrib/provision/base.sh", reboot: true
+    cp.vm.provision "shell", path: "contrib/provision/build.sh", privileged: false
+    cp.vm.provision "shell", path: "contrib/provision/control-plane-base.sh"
+    cp.vm.provision "shell", path: "contrib/provision/control-plane.sh"
+    cp.vm.provision "shell", path: "contrib/provision/kubeconfig.sh", privileged: false
+    cp.vm.provision "shell", path: "contrib/provision/addons.sh", privileged: false
   end
 end

contrib/etc/containerd/config.toml

@@ -0,0 +1,69 @@
+root = "/var/lib/containerd"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = ""
+    stream_server_port = "10010"
+    enable_selinux = false
+    sandbox_image = "k8s.gcr.io/pause:latest"
+    stats_collect_period = 10
+    systemd_cgroup = true
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = true
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+        [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "containerd-shim"
+    runtime = "/home/vagrant/.cargo/bin/enclave-runc-wrapper"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = false
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"

contrib/etc/docker/daemon.json

@@ -0,0 +1,14 @@
+{
+  "log-level": "warn",
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "10m",
+    "max-file": "5"
+  },
+  "default-runtime": "runc-enclave",
+  "runtimes": {
+    "runc-enclave": {
+      "path": "/home/vagrant/.cargo/bin/enclave-runc-wrapper"
+    }
+  }
+}

contrib/etc/modules-load.d/99-k8s.conf

@@ -0,0 +1 @@
+br_netfilter

contrib/etc/sysctl.d/99-k8s.conf

@@ -0,0 +1,4 @@
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.ip_forward = 1
+net.ipv4.conf.all.forwarding = 1

contrib/provision/base.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+zypper install -y -t pattern \
+       devel_basis \
+       devel_C_C++
+zypper install -y \
+       bpftool \
+       cargo \
+       clang \
+       conntrack-tools \
+       containerd \
+       docker \
+       ebtables \
+       ethtool \
+       libbpf-devel \
+       libopenssl-devel \
+       llvm \
+       podman \
+       podman-cni-config \
+       rust \
+       rustfmt \
+       socat \
+       tmux \
+       wget
+
+pushd /home/vagrant/enclave
+install -D -m 0644 contrib/etc/modules-load.d/99-k8s.conf /etc/modules-load.d/99-k8s.conf
+install -D -m 0644 contrib/etc/sysctl.d/99-k8s.conf /etc/sysctl.d/99-k8s.conf
+popd
+
+sed -i -e "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=\"lsm=bpf,integrity\"/" \
+    /etc/default/grub
+grub2-mkconfig -o /boot/grub2/grub.cfg

contrib/vagrant/build.sh → contrib/provision/build.sh

@@ -2,8 +2,8 @@
 
 cd /home/vagrant/enclave
 
+export CLANG=/usr/bin/clang-12
 cargo install --path .
-# cargo install --path . --target-dir /usr/local/bin
 
 sudo install -D -m 0644 contrib/systemd/enclave.service /etc/systemd/system/enclave.service
 sudo systemctl enable enclave.service

contrib/provision/control-plane-base.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+pushd /home/vagrant/enclave
+install -D -m 0644 contrib/etc/containerd/config.toml /etc/containerd/config.toml
+install -D -m 0644 contrib/etc/docker/daemon.json /etc/docker/daemon.json
+install -D -m 0644 contrib/systemd/containerd.service /etc/systemd/system/containerd.service
+popd
+
+systemctl enable --now containerd
+systemctl enable --now docker
+
+CNI_VERSION="v0.9.1"
+sudo mkdir -p /opt/cni/bin
+curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | sudo tar -C /opt/cni/bin -xz
+
+DOWNLOAD_DIR=/usr/local/bin
+sudo mkdir -p $DOWNLOAD_DIR
+
+CRI_TOOLS_VERSION="v1.21.0"
+wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRI_TOOLS_VERSION/crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz
+sudo tar zxvf crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz -C /usr/local/bin
+rm -f crictl-$CRI_TOOLS_VERSION-linux-amd64.tar.gz
+
+RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
+cd $DOWNLOAD_DIR
+sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+sudo chmod +x {kubeadm,kubelet,kubectl}
+
+RELEASE_VERSION="v0.9.0"
+curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service
+sudo mkdir -p /etc/systemd/system/kubelet.service.d
+curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+systemctl enable --now kubelet

contrib/provision/control-plane.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubeadm init --cri-socket /run/containerd/containerd.sock

contrib/provision/vagrant-fix.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Workaround for https://github.com/hashicorp/vagrant/issues/1659
+
+cat <<EOF >> /etc/sudoers
+
+vagrant ALL=(ALL) NOPASSWD:ALL
+Defaults:vagrant !requiretty
+EOF

contrib/systemd/containerd.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/systemd
+ExecStartPre=/usr/bin/mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
+ExecStart=/sbin/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+LimitNOFILE=infinity
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target