scx_p2dq: Add backwards sticky scheduling

Backwards sticky scheduling ensures the task thrashes all CPUs like a
bull in a china shop before it matures and settles down to realize it's
true potential by staying sticky to it's roots. Or some BS like that...

Signed-off-by: Daniel Hodges <[email protected]>

Author: hodgesds

scheds/rust/scx_p2dq/src/bpf/main.bpf.c

@@ -60,6 +60,7 @@ const volatile u32 interactive_ratio = 10;
 const volatile u32 min_nr_queued_pick2 = 10;
 
 const volatile bool autoslice = true;
+const volatile bool backwards_sticky = false;
 const volatile bool dispatch_pick2_disable = false;
 const volatile bool eager_load_balance = true;
 const volatile bool interactive_sticky = false;
@@ -434,6 +435,19 @@ static s32 pick_idle_cpu(struct task_struct *p, task_ctx *taskc,
 		goto found_cpu;
 	}
 
+	if (backwards_sticky &&
+	    ((taskc->bs_mask & nr_cpus) != nr_cpus)) {
+		s32 i;
+		bpf_for(i, 0, nr_cpus)
+			if (taskc->bs_mask == 0 || (taskc->bs_mask & i) != i) {
+				if (i == nr_cpus - 1)
+					taskc->bs_mask |= nr_cpus;
+				cpu = i;
+				*is_idle = scx_bpf_test_and_clear_cpu_idle(i);
+				goto found_cpu;
+			}
+	}
+
 	// First check if last CPU is idle
 	if (taskc->all_cpus &&
 	    bpf_cpumask_test_cpu(prev_cpu, (smt_enabled && !interactive) ?
@@ -770,6 +784,8 @@ static __always_inline int p2dq_running_impl(struct task_struct *p)
 	if (taskc->node_id != cpuc->node_id) {
 		stat_inc(P2DQ_STAT_NODE_MIGRATION);
 	}
+	if (backwards_sticky)
+		taskc->bs_mask |= task_cpu;
 
 	taskc->llc_id = llcx->id;
 	taskc->node_id = llcx->node_id;
@@ -1123,6 +1139,7 @@ static __always_inline s32 p2dq_init_task_impl(struct task_struct *p,
 		return -EINVAL;
 	}
 
+	taskc->bs_mask = 0;
 	taskc->llc_id = cpuc->llc_id;
 	taskc->node_id = cpuc->node_id;
 	taskc->dsq_index = init_dsq_index;

scheds/rust/scx_p2dq/src/bpf/types.h

@@ -58,6 +58,7 @@ struct task_p2dq {
 	u64 			last_run_at;
 	u64			llc_runs; /* how many runs on the current LLC */
 	int			last_dsq_index;
+	s32			bs_mask;
 
 	/* The task is a workqueue worker thread */
 	bool			is_kworker;

scheds/rust/scx_p2dq/src/lib.rs

@@ -45,6 +45,10 @@ pub struct SchedulerOpts {
     #[clap(short = 'e', long, help="DEPRECATED", action = clap::ArgAction::SetTrue)]
     pub eager_load_balance: bool,
 
+    /// Enables backward sticky scheduling.
+    #[clap(long= "bs", action = clap::ArgAction::SetTrue)]
+    pub backwards_sticky: bool,
+
     /// Enables CPU frequency control.
     #[clap(short = 'f', long, action = clap::ArgAction::SetTrue)]
     pub freq_control: bool,
@@ -207,6 +211,7 @@ macro_rules! init_open_skel {
             $skel.maps.rodata_data.lb_slack_factor = opts.lb_slack_factor;
 
             $skel.maps.rodata_data.autoslice = opts.autoslice;
+            $skel.maps.rodata_data.backwards_sticky = opts.backwards_sticky;
             $skel.maps.rodata_data.debug = verbose as u32;
             $skel.maps.rodata_data.dispatch_pick2_disable = opts.dispatch_pick2_disable;
             $skel.maps.rodata_data.dispatch_lb_busy = opts.dispatch_lb_busy;