Skip to content

Commit dc2110b

Browse files
committed
bpf: Fix problems with container starts
This change consists of several fixes which were causing random problems with containers getting Ready. * Allowing access to /run directory inside container filesystem. It's used both by popular applications (like nginx) and by kubelet to store the network namespace (/run/netns). * Using uprobes for registering containers and processes in BPF maps. Doing BPF operations from the wrapper had the weird issue - by registering the wrapper process, we were in fact preventing the same lockc-runc-wrapper process from doing any more BPF map modifications. Using uprobes has also an another advantage - it will allow rootless containers to work. * Proper cleanup of old containers and processes. Majority of uprobes code on the userspace side comes from bpfcontain-rs. Fixes: #52 Fixes: #92 Signed-off-by: Michal Rostecki <[email protected]> Signed-off-by: William Findlay <[email protected]>
1 parent c1037c9 commit dc2110b

File tree

15 files changed

+1161
-150
lines changed

15 files changed

+1161
-150
lines changed

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
[workspace]
22
members = [
33
"lockc",
4+
"lockc-uprobes",
45
"xtask",
56
]

contrib/etc/lockc/lockc.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ allowed_paths_access_restricted = [
225225
"/home",
226226
"/lib",
227227
"/proc",
228+
"/run",
228229
"/sys/fs/cgroup",
229230
"/tmp",
230231
"/usr",
@@ -244,6 +245,7 @@ allowed_paths_access_baseline = [
244245
"/home",
245246
"/lib",
246247
"/proc",
248+
"/run",
247249
"/sys/fs/cgroup",
248250
"/tmp",
249251
"/usr",
@@ -252,9 +254,9 @@ allowed_paths_access_baseline = [
252254

253255
denied_paths_access_restricted = [
254256
"/proc/acpi",
257+
"/proc/sys",
255258
]
256259

257260
denied_paths_access_baseline = [
258261
"/proc/acpi",
259-
"/proc/sys",
260262
]

contrib/guestfs/build.sh

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,6 @@ virt-customize -a \
3939
${LOCKC_IMAGE} \
4040
--mkdir /etc/containerd \
4141
--mkdir /etc/docker \
42-
--copy-in provision/etc/containerd/config.toml:/etc/containerd/ \
43-
--copy-in provision/etc/docker/daemon.json:/etc/docker/ \
4442
--copy-in provision/etc/modules-load.d/99-k8s.conf:/etc/modules-load.d/ \
4543
--copy-in provision/etc/sysctl.d/99-k8s.conf:/etc/sysctl.d/ \
4644
--copy-in provision/systemd/containerd.service:/etc/systemd/system/ \

contrib/guestfs/provision/provision.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ EOF
8484
### Rebuild initrd with dracut
8585
mkinitrd
8686

87-
mv /etc/containerd/config.toml.rpmorig /etc/containerd/config.toml
87+
# mv /etc/containerd/config.toml.rpmorig /etc/containerd/config.toml
8888

8989
systemctl enable containerd
9090
systemctl enable docker

contrib/systemd/lockcd.service.in

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ Description=lockc daemon
33
After=network-online.target
44

55
[Service]
6-
Type=oneshot
6+
Type=simple
7+
Restart=always
8+
RestartSec=1
79
ExecStart={{ bindir }}/lockcd
810
StandardOutput=journal
911

lockc-uprobes/Cargo.toml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[package]
2+
name = "lockc-uprobes"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
license = "Apache-2.0"
7+
8+
[dependencies]
9+
libc = "0.2"

lockc-uprobes/src/lib.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
use libc::pid_t;
2+
3+
#[no_mangle]
4+
#[inline(never)]
5+
pub extern "C" fn add_container(_retp: *mut i32, _container_id: u32, _pid: pid_t, _policy: i32) {}
6+
7+
#[no_mangle]
8+
#[inline(never)]
9+
pub extern "C" fn delete_container(_retp: *mut i32, _container_id: u32) {}
10+
11+
#[no_mangle]
12+
#[inline(never)]
13+
pub extern "C" fn add_process(_retp: *mut i32, _container_id: u32, _pid: pid_t) {}

lockc/Cargo.toml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,17 @@ bindgen = "0.59"
2222
byteorder = "1.4"
2323
chrono = { version = "0.4", default-features = false, features = ["clock"] }
2424
config = { version = "0.11", default-features = false, features = ["toml"] }
25+
ctrlc = "3.2"
2526
dirs = "4.0"
27+
fanotify-rs = { git = "https://github.com/Serinalice/fanotify-rs", branch = "master" }
2628
futures = "0.3"
29+
goblin = "0.4"
2730
kube = "0.63"
2831
k8s-openapi = { version = "0.13", default-features = false, features = ["v1_21"] }
2932
lazy_static = "1.4"
3033
libc = { version = "0.2", features = [ "extra_traits" ] }
31-
libbpf-rs = "0.13"
34+
libbpf-rs = { git = "https://github.com/libbpf/libbpf-rs", rev = "ebc55ba" }
35+
lockc-uprobes = { path = "../lockc-uprobes" }
3236
log = "0.4"
3337
log4rs = "1.0"
3438
nix = "0.23"
@@ -41,6 +45,7 @@ sysctl = "0.4"
4145
thiserror = "1.0"
4246
tokio = { version = "1.7", features = ["macros", "process", "rt-multi-thread"] }
4347
uuid = { version = "0.8", default-features = false, features = ["v4"] }
48+
which = "4.2"
4449

4550
[build-dependencies]
4651
anyhow = "1.0"

lockc/src/bin/lockc-runc-wrapper.rs

Lines changed: 131 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ use log4rs::append::file::FileAppender;
66
use log4rs::config::{runtime::ConfigErrors, Appender, Config, Root};
77
use uuid::Uuid;
88

9+
use lockc_uprobes::{add_container, add_process, delete_container};
10+
911
// TODO: To be used for cri-o.
1012
// static ANNOTATION_K8S_LABELS: &str = "io.kubernetes.cri-o.Labels";
1113

@@ -140,13 +142,13 @@ struct Mount {
140142
destination: String,
141143
r#type: String,
142144
source: String,
143-
options: Vec<String>
145+
options: Vec<String>,
144146
}
145147

146148
#[derive(Debug, Deserialize)]
147149
#[serde(rename_all = "camelCase")]
148150
struct Mounts {
149-
mounts: Vec<Mount>
151+
mounts: Vec<Mount>,
150152
}
151153

152154
fn docker_config<P: AsRef<std::path::Path>>(
@@ -161,9 +163,9 @@ fn docker_config<P: AsRef<std::path::Path>>(
161163

162164
for test in m.mounts {
163165
let source: Vec<&str> = test.source.split('/').collect();
164-
if source.len() > 1 && source[ source.len() - 1 ] == "hostname" {
165-
let config_v2= str::replace(&test.source, "hostname", "config.v2.json");
166-
return Ok(std::path::PathBuf::from(config_v2));
166+
if source.len() > 1 && source[source.len() - 1] == "hostname" {
167+
let config_v2 = str::replace(&test.source, "hostname", "config.v2.json");
168+
return Ok(std::path::PathBuf::from(config_v2));
167169
}
168170
}
169171

@@ -185,15 +187,11 @@ fn docker_label<P: AsRef<std::path::Path>>(
185187

186188
match x {
187189
Some(x) => match x {
188-
"restricted" => {
189-
Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED)
190-
}
190+
"restricted" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED),
191191
"baseline" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
192-
"privileged" => {
193-
Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_PRIVILEGED)
194-
}
195-
_ => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE)
196-
}
192+
"privileged" => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_PRIVILEGED),
193+
_ => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
194+
},
197195
None => Ok(lockc::bpfstructs::container_policy_level_POLICY_LEVEL_BASELINE),
198196
}
199197
}
@@ -265,12 +263,33 @@ fn setup_logging() -> Result<(), SetupLoggingError> {
265263
Ok(())
266264
}
267265

266+
#[derive(thiserror::Error, Debug)]
267+
enum UprobeError {
268+
#[error("failed to call into uprobe, BPF programs are most likely not running")]
269+
Call,
270+
271+
#[error("BPF program error")]
272+
BPF,
273+
274+
#[error("unknown error")]
275+
Unknown,
276+
}
277+
278+
fn check_uprobe_ret(ret: i32) -> Result<(), UprobeError> {
279+
match ret {
280+
0 => Ok(()),
281+
n if n == -libc::EAGAIN => Err(UprobeError::Call),
282+
n if n == -libc::EINVAL => Err(UprobeError::BPF),
283+
_ => Err(UprobeError::Unknown),
284+
}
285+
}
286+
268287
#[tokio::main]
269288
async fn main() -> anyhow::Result<()> {
270289
setup_logging()?;
271290

272291
let pid = nix::unistd::getpid();
273-
let pid_u = u32::try_from(libc::pid_t::from(pid))?;
292+
let pid_u = libc::pid_t::from(pid);
274293

275294
let mut opt_parsing_action = OptParsingAction::NoPositional;
276295
let mut arg_parsing_action = ArgParsingAction::None;
@@ -356,22 +375,53 @@ async fn main() -> anyhow::Result<()> {
356375
match container_id_o {
357376
Some(v) => {
358377
let container_key = lockc::hash(&v)?;
359-
lockc::add_process(container_key, pid_u)?;
360-
cmd.status().await?;
361-
lockc::delete_process(pid_u)?;
378+
379+
let mut ret: i32 = -libc::EAGAIN;
380+
add_process(&mut ret as *mut i32, container_key, pid_u);
381+
check_uprobe_ret(ret)?;
382+
383+
let output = cmd.output().await;
384+
match output {
385+
Ok(output) => {
386+
info!("status: {}", output.status);
387+
info!(
388+
"stdout: {}",
389+
std::string::String::from_utf8_lossy(&output.stdout)
390+
);
391+
info!(
392+
"stderr: {}",
393+
std::string::String::from_utf8_lossy(&output.stderr)
394+
);
395+
}
396+
Err(e) => info!("error: {}", e),
397+
}
362398
}
363399
None => {
364400
// The purpose of this fake "container" is only to allow the runc
365401
// subcommand to be detected as wrapped and thus allowed by
366402
// the LSM program to execute. It's only to handle subcommands
367403
// like `init`, `list` or `spec`, so we make it restricted.
368-
lockc::add_container(
369-
0,
370-
pid_u,
371-
lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED,
372-
)?;
373-
cmd.status().await?;
374-
lockc::delete_container(0)?;
404+
// lockc::add_container(
405+
// 0,
406+
// pid_u,
407+
// lockc::bpfstructs::container_policy_level_POLICY_LEVEL_RESTRICTED,
408+
// )?;
409+
let output = cmd.output().await;
410+
match output {
411+
Ok(output) => {
412+
info!("status: {}", output.status);
413+
info!(
414+
"stdout: {}",
415+
std::string::String::from_utf8_lossy(&output.stdout)
416+
);
417+
info!(
418+
"stderr: {}",
419+
std::string::String::from_utf8_lossy(&output.stderr)
420+
);
421+
}
422+
Err(e) => info!("error: {}", e),
423+
}
424+
// lockc::delete_container(0)?;
375425
}
376426
}
377427
}
@@ -392,16 +442,68 @@ async fn main() -> anyhow::Result<()> {
392442
policy = docker_label(docker_conf)?;
393443
}
394444
};
395-
lockc::add_container(container_key, pid_u, policy)?;
396-
cmd.status().await?;
445+
446+
let mut ret: i32 = -libc::EAGAIN;
447+
add_container(&mut ret as *mut i32, container_key, pid_u, policy);
448+
check_uprobe_ret(ret)?;
449+
450+
info!("executing create");
451+
let output = cmd.output().await;
452+
info!("executed create");
453+
match output {
454+
Ok(output) => {
455+
info!("status: {}", output.status);
456+
info!(
457+
"stdout: {}",
458+
std::string::String::from_utf8_lossy(&output.stdout)
459+
);
460+
info!(
461+
"stderr: {}",
462+
std::string::String::from_utf8_lossy(&output.stderr)
463+
);
464+
}
465+
Err(e) => info!("error: {}", e),
466+
}
397467
}
398468
ContainerAction::Delete => {
399469
let container_key = lockc::hash(&container_id_o.unwrap())?;
400-
lockc::delete_container(container_key)?;
401-
cmd.status().await?;
470+
471+
let output = cmd.output().await;
472+
match output {
473+
Ok(output) => {
474+
info!("status: {}", output.status);
475+
info!(
476+
"stdout: {}",
477+
std::string::String::from_utf8_lossy(&output.stdout)
478+
);
479+
info!(
480+
"stderr: {}",
481+
std::string::String::from_utf8_lossy(&output.stderr)
482+
);
483+
}
484+
Err(e) => info!("error: {}", e),
485+
}
486+
487+
let mut ret: i32 = -libc::EAGAIN;
488+
delete_container(&mut ret as *mut i32, container_key);
489+
check_uprobe_ret(ret)?;
402490
}
403491
ContainerAction::Start => {
404-
cmd.status().await?;
492+
let output = cmd.output().await;
493+
match output {
494+
Ok(output) => {
495+
info!("status: {}", output.status);
496+
info!(
497+
"stdout: {}",
498+
std::string::String::from_utf8_lossy(&output.stdout)
499+
);
500+
info!(
501+
"stderr: {}",
502+
std::string::String::from_utf8_lossy(&output.stderr)
503+
);
504+
}
505+
Err(e) => info!("error: {}", e),
506+
}
405507
}
406508
}
407509

lockc/src/bin/lockcd.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@ fn main() -> anyhow::Result<()> {
2020

2121
let path_base_ts = path_base.join(&dirname);
2222

23-
lockc::load_programs(path_base_ts)?;
23+
// lockc::load_programs(path_base_ts)?;
24+
let _skel = lockc::BpfContext::new(path_base_ts)?;
2425
lockc::cleanup(path_base, &dirname)?;
2526

27+
// skel.work_loop()?;
28+
lockc::runc::runc_watcher();
29+
2630
Ok(())
2731
}

0 commit comments

Comments
 (0)