scx_layered: Add tickless layer support

Add a option to enable tickless scheduling on a layer. This may improve
throughput in certain scenarios by reducing the number of context
switches.

Signed-off-by: Daniel Hodges <[email protected]>

Author: hodgesds

scheds/rust/scx_layered/src/bpf/intf.h

@@ -338,6 +338,7 @@ struct layer {
 	bool			allow_node_aligned;
 	bool			skip_remote_node;
 	bool			prev_over_idle_core;
+	bool			tickless;
 	int			growth_algo;
 
 	u64			nr_tasks;

scheds/rust/scx_layered/src/bpf/main.bpf.c

@@ -58,6 +58,7 @@ volatile u64 layer_refresh_seq_avgruntime;
 
 /* Flag to enable or disable antistall feature */
 const volatile bool enable_antistall = true;
+const volatile bool enable_tickless = true;
 const volatile bool enable_match_debug = false;
 const volatile bool enable_gpu_support = false;
 /* Delay permitted, in seconds, before antistall activates */
@@ -184,6 +185,32 @@ static __always_inline bool is_scheduler_task(struct task_struct *p)
 	return (u32)p->tgid == layered_root_tgid;
 }
 
+static void preempt_tickless(struct cpu_ctx *cpuc)
+{
+	struct task_struct *curr;
+	struct layer *layer;
+
+	if (!enable_tickless)
+		return;
+
+	bpf_rcu_read_lock();
+	curr = scx_bpf_cpu_rq(cpuc->cpu)->curr;
+	if (curr->scx.slice == SCX_SLICE_INF) {
+		// If tickless task is running on a unowned layer then preempt it.
+		if (cpuc->layer_id >= nr_layers) {
+			curr->scx.slice = 1;
+			bpf_rcu_read_unlock();
+			return;
+		}
+		if (!(layer = lookup_layer(cpuc->layer_id))) {
+			bpf_rcu_read_unlock();
+			return;
+		}
+		curr->scx.slice = layer->slice_ns;
+	}
+	bpf_rcu_read_unlock();
+}
+
 struct {
 	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
 	__type(key, u32);
@@ -1204,7 +1231,8 @@ s32 BPF_STRUCT_OPS(layered_select_cpu, struct task_struct *p, s32 prev_cpu, u64
 	if (cpu >= 0) {
 		lstat_inc(LSTAT_SEL_LOCAL, layer, cpuc);
 		taskc->dsq_id = SCX_DSQ_LOCAL;
-		scx_bpf_dsq_insert(p, taskc->dsq_id, layer->slice_ns, 0);
+		u64 slice_ns = layer->tickless ? SCX_SLICE_INF : layer->slice_ns;
+		scx_bpf_dsq_insert(p, taskc->dsq_id, slice_ns, 0);
 		return cpu;
 	}
 
@@ -1387,6 +1415,8 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
 	if (!(cpuc = lookup_cpu_ctx(-1)) || !(taskc = lookup_task_ctx(p)))
 		return;
 
+	preempt_tickless(cpuc);
+
 	layer_id = taskc->layer_id;
 	if (!(layer = lookup_layer(layer_id)))
 		return;
@@ -1602,10 +1632,11 @@ void BPF_STRUCT_OPS(layered_enqueue, struct task_struct *p, u64 enq_flags)
 	lstats[LLC_LSTAT_CNT]++;
 
 	taskc->dsq_id = layer_dsq_id(layer_id, llc_id);
+	u64 slice_ns = layer->tickless ? SCX_SLICE_INF : layer->slice_ns;
 	if (layer->fifo)
-		scx_bpf_dsq_insert(p, taskc->dsq_id, layer->slice_ns, enq_flags);
+		scx_bpf_dsq_insert(p, taskc->dsq_id, slice_ns, enq_flags);
 	else
-		scx_bpf_dsq_insert_vtime(p, taskc->dsq_id, layer->slice_ns, vtime, enq_flags);
+		scx_bpf_dsq_insert_vtime(p, taskc->dsq_id, slice_ns, vtime, enq_flags);
 	lstat_inc(LSTAT_ENQ_DSQ, layer, cpuc);
 
 	/*
@@ -3191,6 +3222,7 @@ void BPF_STRUCT_OPS(layered_dump, struct scx_dump_ctx *dctx)
  */
 struct layered_timer layered_timers[MAX_TIMERS] = {
 	{15LLU * NSEC_PER_SEC, CLOCK_BOOTTIME, 0},
+	{1LLU * NSEC_PER_MSEC, CLOCK_BOOTTIME, 0},
 };
 
 /**

scheds/rust/scx_layered/src/bpf/timer.bpf.h

@@ -29,6 +29,7 @@ struct layered_timer {
 
 enum layer_timer_callbacks {
 	ANTISTALL_TIMER,
+	TICKLESS_TIMER,
 	MAX_TIMERS,
 };
 

scheds/rust/scx_layered/src/config.rs

@@ -142,6 +142,8 @@ pub struct LayerCommon {
     pub llcs: Vec<usize>,
     #[serde(default)]
     pub placement: LayerPlacement,
+    #[serde(default)]
+    pub tickless: bool,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]

scheds/rust/scx_layered/src/main.rs

@@ -130,6 +130,7 @@ lazy_static! {
                         nodes: vec![],
                         llcs: vec![],
                         placement: LayerPlacement::Standard,
+                        tickless: false,
                     },
                 },
             },
@@ -165,6 +166,7 @@ lazy_static! {
                         nodes: vec![],
                         llcs: vec![],
                         placement: LayerPlacement::Standard,
+                        tickless: false,
                     },
                 },
             },
@@ -204,6 +206,7 @@ lazy_static! {
                         nodes: vec![],
                         llcs: vec![],
                         placement: LayerPlacement::Standard,
+                        tickless: false,
                     },
                 },
             },
@@ -241,6 +244,7 @@ lazy_static! {
                         nodes: vec![],
                         llcs: vec![],
                         placement: LayerPlacement::Standard,
+                        tickless: false,
                     },
                 },
             },
@@ -1413,6 +1417,7 @@ impl<'a> Scheduler<'a> {
                     disallow_preempt_after_us,
                     xllc_mig_min_us,
                     placement,
+                    tickless,
                     ..
                 } = spec.kind.common();
 
@@ -1435,6 +1440,10 @@ impl<'a> Scheduler<'a> {
                 layer.allow_node_aligned.write(*allow_node_aligned);
                 layer.skip_remote_node.write(*skip_remote_node);
                 layer.prev_over_idle_core.write(*prev_over_idle_core);
+                layer.tickless.write(*tickless);
+                if *tickless {
+                    skel.maps.rodata_data.enable_tickless = *tickless;
+                }
                 layer.growth_algo = growth_algo.as_bpf_enum();
                 layer.weight = *weight;
                 layer.disallow_open_after_ns = match disallow_open_after_us.unwrap() {