bpf: Support private stack for struct_ops progs

For struct_ops progs, whether a particular prog will use private stack
or not (prog->aux->use_priv_stack) will be set before actual insn-level
verification for that prog. One particular implementation is to
piggyback on struct_ops->check_member(). The next patch will have an
example for this. The struct_ops->check_member() will set
prog->aux->use_priv_stack to be true which enables private stack
usage with ignoring BPF_PRIV_STACK_MIN_SIZE limit.

If use_priv_stack is true for a particular struct_ops prog, bpf
trampoline will need to do recursion checks (one level at this point)
to avoid stack overwrite.

Signed-off-by: Yonghong Song <[email protected]>

Author: Yonghong Song

include/linux/bpf_verifier.h

@@ -889,6 +889,7 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
 	case BPF_PROG_TYPE_TRACING:
 		return prog->expected_attach_type != BPF_TRACE_ITER;
 	case BPF_PROG_TYPE_STRUCT_OPS:
+		return prog->aux->use_priv_stack;
 	case BPF_PROG_TYPE_LSM:
 		return false;
 	default:

kernel/bpf/verifier.c

@@ -6023,17 +6023,31 @@ static int check_ptr_alignment(struct bpf_verifier_env *env,
 
 static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
 {
+	bool force_priv_stack = env->prog->aux->use_priv_stack;
 	struct bpf_subprog_info *si;
+	int ret;
+
+	if (!bpf_jit_supports_private_stack()) {
+		if (force_priv_stack) {
+			verbose(env, "Private stack not supported by jit\n");
+			return -EACCES;
+		}
 
-	if (!bpf_jit_supports_private_stack())
 		return NO_PRIV_STACK;
+	}
 
+	ret = PRIV_STACK_ADAPTIVE;
 	switch (env->prog->type) {
 	case BPF_PROG_TYPE_KPROBE:
 	case BPF_PROG_TYPE_TRACEPOINT:
 	case BPF_PROG_TYPE_PERF_EVENT:
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 		break;
+	case BPF_PROG_TYPE_STRUCT_OPS:
+		if (!force_priv_stack)
+			return NO_PRIV_STACK;
+		ret = PRIV_STACK_ALWAYS;
+		break;
 	case BPF_PROG_TYPE_TRACING:
 		if (env->prog->expected_attach_type != BPF_TRACE_ITER)
 			break;
@@ -6044,11 +6058,18 @@ static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
 
 	si = env->subprog_info;
 	for (int i = 0; i < env->subprog_cnt; i++) {
-		if (si[i].has_tail_call)
+		if (si[i].has_tail_call) {
+			if (ret == PRIV_STACK_ALWAYS) {
+				verbose(env,
+					"Private stack not supported due to tail call presence\n");
+				return -EACCES;
+			}
+
 			return NO_PRIV_STACK;
+		}
 	}
 
-	return PRIV_STACK_ADAPTIVE;
+	return ret;
 }
 
 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
@@ -6121,7 +6142,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 					idx, subprog_depth);
 				return -EACCES;
 			}
-			if (subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
+			if (priv_stack_supported == PRIV_STACK_ALWAYS ||
+			    subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
 				subprog[idx].use_priv_stack = true;
 				subprog_visited[idx] = 1;
 			}
@@ -6271,6 +6293,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
 				depth_frame, subtree_depth);
 			return -EACCES;
 		}
+		if (orig_priv_stack_supported == PRIV_STACK_ALWAYS) {
+			verbose(env,
+				"Private stack not supported due to possible nested subprog run\n");
+			ret = -EACCES;
+			goto out;
+		}
 		if (orig_priv_stack_supported == PRIV_STACK_ADAPTIVE) {
 			for (int i = 0; i < env->subprog_cnt; i++)
 				si[i].use_priv_stack = false;