diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7805d4f..6b05bdf19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,9 @@ instead `badarg`. - Fixed a bug where empty atom could not be created on some platforms, thus breaking receiving a message for a registered process from an OTP node. - Fix a memory leak in distribution when a BEAM node would monitor a process by name. - Fix `list_to_integer`, it was likely buggy with integers close to INT64_MAX +- Added missing support for supervisor `one_for_all` strategy. +- Supervisor now honors period and intensity options. +- Fix supervisor crash if a child fails to restart. ## [0.6.7] - Unreleased diff --git a/libs/estdlib/src/supervisor.erl b/libs/estdlib/src/supervisor.erl index 260a1644c..2ecd64357 100644 --- a/libs/estdlib/src/supervisor.erl +++ b/libs/estdlib/src/supervisor.erl @@ -88,13 +88,28 @@ start :: {module(), atom(), [any()] | undefined}, restart :: restart(), shutdown :: shutdown(), - type :: child_type, + type :: child_type(), modules = [] :: [module()] | dynamic }). --record(state, {restart_strategy :: strategy(), children = [] :: [#child{}]}). +-record(state, { + restart_strategy = one_for_one :: strategy(), + intensity = 3 :: non_neg_integer(), + period = 5 :: pos_integer(), + restart_count = 0, + restarts = [], + children = [] :: [#child{}] +}). + +%% Used to trim stale restarts when the 'intensity' value is large. +%% The number of restarts before triggering a purge of restarts older +%% than 'period', so stale restarts do not continue to consume ram for +%% the sake of MCUs with limited memory. In the future a function +%% could be used to set a sane default for the platform (OTP uses 1000). +-define(STALE_RESTART_LIMIT, 100). start_link(Module, Args) -> gen_server:start_link(?MODULE, {Module, Args}, []). + start_link(SupName, Module, Args) -> gen_server:start_link(SupName, ?MODULE, {Module, Args}, []). @@ -119,16 +134,27 @@ count_children(Supervisor) -> init({Mod, Args}) -> erlang:process_flag(trap_exit, true), case Mod:init(Args) of - {ok, {{Strategy, _Intensity, _Period}, StartSpec}} -> - State = init_state(StartSpec, #state{restart_strategy = Strategy}), + {ok, {{Strategy, Intensity, Period}, StartSpec}} -> + State = init_state(StartSpec, #state{ + restart_strategy = Strategy, + intensity = Intensity, + period = Period + }), NewChildren = start_children(State#state.children, []), {ok, State#state{children = NewChildren}}; - {ok, {#{strategy := Strategy}, StartSpec}} -> - State = init_state(StartSpec, #state{restart_strategy = Strategy}), + {ok, {#{} = SupSpec, StartSpec}} -> + Strategy = maps:get(strategy, SupSpec, one_for_one), + Intensity = maps:get(intensity, SupSpec, 3), + Period = maps:get(period, SupSpec, 5), + State = init_state(StartSpec, #state{ + restart_strategy = Strategy, + intensity = Intensity, + period = Period + }), NewChildren = start_children(State#state.children, []), {ok, State#state{children = NewChildren}}; Error -> - {stop, {bad_return, {mod, init, Error}}} + {stop, {bad_return, {Mod, init, Error}}} end. -spec child_spec_to_record(child_spec()) -> #child{}. @@ -172,7 +198,7 @@ child_spec_to_record(#{id := ChildId, start := MFA} = ChildMap) -> init_state([ChildSpec | T], State) -> Child = child_spec_to_record(ChildSpec), NewChildren = [Child | State#state.children], - init_state(T, #state{children = NewChildren}); + init_state(T, State#state{children = NewChildren}); init_state([], State) -> State#state{children = lists:reverse(State#state.children)}. @@ -184,7 +210,50 @@ start_children([Child | T], StartedC) -> start_children([], StartedC) -> StartedC. -restart_child(Pid, Reason, State) -> +restart_child(Pid, Reason, #state{restart_strategy = one_for_one} = State) -> + case lists:keyfind(Pid, #child.pid, State#state.children) of + false -> + {ok, State}; + #child{restart = {terminating, temporary, From}} -> + gen_server:reply(From, ok), + NewChildren = lists:keydelete(Pid, #child.pid, State#state.children), + {ok, State#state{children = NewChildren}}; + #child{restart = {terminating, Restart, From}} = Child -> + gen_server:reply(From, ok), + NewChildren = lists:keyreplace(Pid, #child.pid, State#state.children, Child#child{ + pid = undefined, restart = Restart + }), + {ok, State#state{children = NewChildren}}; + #child{} = Child -> + case should_restart(Reason, Child#child.restart) of + true -> + case add_restart(State) of + {ok, State1} -> + case try_start(Child) of + {ok, NewPid, _Result} -> + NewChild = Child#child{pid = NewPid}, + Children = lists:keyreplace( + Pid, #child.pid, State1#state.children, NewChild + ), + {ok, State1#state{children = Children}}; + {error, _} -> + erlang:send_after( + 50, self(), {try_again_restart, Child#child.id} + ), + {ok, State1} + end; + {shutdown, State1} -> + RemainingChildren = lists:keydelete( + Pid, #child.pid, State#state.children + ), + {shutdown, State1#state{children = RemainingChildren}} + end; + false -> + Children = lists:keydelete(Pid, #child.pid, State#state.children), + {ok, State#state{children = Children}} + end + end; +restart_child(Pid, Reason, #state{restart_strategy = one_for_all} = State) -> case lists:keyfind(Pid, #child.pid, State#state.children) of false -> {ok, State}; @@ -201,13 +270,21 @@ restart_child(Pid, Reason, State) -> #child{} = Child -> case should_restart(Reason, Child#child.restart) of true -> - case try_start(Child) of - {ok, NewPid, _Result} -> - NewChild = Child#child{pid = NewPid}, - Children = lists:keyreplace( - Pid, #child.pid, State#state.children, NewChild + case add_restart(State) of + {ok, State1} -> + Siblings = lists:keydelete(Pid, #child.pid, State#state.children), + case restart_many_children(Child, Siblings) of + {ok, NewChildren} -> + {ok, State1#state{children = NewChildren}}; + {ok, NewChildren, RetryID} -> + erlang:send_after(50, self(), {try_again_restart, RetryID}), + {ok, State1#state{children = NewChildren}} + end; + {shutdown, State1} -> + RemainingChildren = lists:keydelete( + Pid, #child.pid, State#state.children ), - {ok, State#state{children = Children}} + {shutdown, State1#state{children = RemainingChildren}} end; false -> Children = lists:keydelete(Pid, #child.pid, State#state.children), @@ -309,6 +386,26 @@ handle_info({ensure_killed, Pid}, State) -> exit(Pid, kill), {noreply, State} end; +handle_info({try_again_restart, Id}, State) -> + Child = lists:keyfind(Id, #child.id, State#state.children), + case add_restart(State) of + {ok, State1} -> + case try_start(Child) of + {ok, NewPid, _Result} -> + UpdatedChildren = lists:keyreplace( + Id, Child#child.id, State#state.children, Child#child{pid = NewPid} + ), + {noreply, State#state{children = UpdatedChildren}}; + {error, {_, _}} -> + erlang:send_after(50, self(), {try_again_restart, Id}), + {noreply, State1} + end; + {shutdown, State1} -> + RemainingChildren = lists:keydelete( + Id, #child.id, State1#state.children + ), + {stop, shutdown, State1#state{children = RemainingChildren}} + end; handle_info(_Msg, State) -> %TODO: log unexpected message {noreply, State}. @@ -321,9 +418,16 @@ terminate(_Reason, #state{children = Children} = State) -> loop_terminate([#child{pid = undefined} | Tail], AccRemaining) -> loop_terminate(Tail, AccRemaining); +loop_terminate([#child{pid = {restarting, _}} | Tail], AccRemaining) -> + loop_terminate(Tail, AccRemaining); loop_terminate([#child{pid = Pid} = Child | Tail], AccRemaining) when is_pid(Pid) -> - do_terminate(Child), - loop_terminate(Tail, [Pid | AccRemaining]); + case is_process_alive(Pid) of + true -> + do_terminate(Child), + loop_terminate(Tail, [Pid | AccRemaining]); + false -> + loop_terminate(Tail, AccRemaining) + end; loop_terminate([], AccRemaining) -> AccRemaining. @@ -364,6 +468,117 @@ try_start(#child{start = {M, F, Args}} = Record) -> {error, {{'EXIT', Error}, Record}} end. +add_restart( + #state{ + intensity = Intensity, period = Period, restart_count = RestartCount, restarts = Restarts + } = State +) -> + Now = erlang:monotonic_time(millisecond), + Threshold = Now - Period * 1000, + case can_restart(Intensity, Threshold, Restarts, RestartCount) of + {true, RestartCount1, Restarts1} -> + {ok, State#state{ + restarts = Restarts1 ++ [Now], restart_count = RestartCount1 + 1 + }}; + {false, RestartCount1, Restarts1} -> + % TODO: log supervisor shutdown due to maximum intensity exceeded + {shutdown, State#state{ + restarts = Restarts1, restart_count = RestartCount1 + }} + end. + +can_restart(0, _, _, _) -> + {false, 0, []}; +can_restart(_, _, _, 0) -> + {true, 0, []}; +can_restart(Intensity, Threshold, Restarts, RestartCount) when + RestartCount >= ?STALE_RESTART_LIMIT +-> + {NewCount, Restarts1} = trim_expired_restarts(Threshold, lists:sort(Restarts)), + can_restart(Intensity, Threshold, Restarts1, NewCount); +can_restart(Intensity, Threshold, [Restart | _] = Restarts, RestartCount) when + RestartCount >= Intensity andalso Restart < Threshold +-> + {NewCount, Restarts1} = trim_expired_restarts(Threshold, lists:sort(Restarts)), + can_restart(Intensity, Threshold, Restarts1, NewCount); +can_restart(Intensity, _, Restarts, RestartCount) when RestartCount >= Intensity -> + {false, RestartCount, Restarts}; +can_restart(Intensity, _, Restarts, RestartCount) when RestartCount < Intensity -> + {true, RestartCount, Restarts}. + +trim_expired_restarts(Threshold, [Restart | Restarts]) when Restart < Threshold -> + trim_expired_restarts(Threshold, Restarts); +trim_expired_restarts(_Threshold, Restarts) -> + {length(Restarts), Restarts}. + +restart_many_children(#child{pid = Pid} = Child, Children) -> + Siblings = lists:keydelete(Pid, #child.pid, Children), + {ok, Children1} = terminate_many_children(Siblings, [Child#child{pid = {restarting, Pid}}]), + do_restart_children(Children1, [], []). + +terminate_many_children([], NewChildren) -> + {ok, lists:reverse(NewChildren)}; +terminate_many_children([Child | Children], NewChildren) -> + case Child of + #child{restart = {terminating, _Restart, From}} = Child when is_pid(From) -> + terminate_many_children(Children, NewChildren); + #child{pid = undefined, restart = temporary} = Child -> + terminate_many_children(Children, NewChildren); + #child{pid = Pid, restart = temporary} = Child when is_pid(Pid) -> + do_terminate(Child), + terminate_many_children(Children, NewChildren); + #child{pid = undefined} = Child -> + terminate_many_children(Children, [Child | NewChildren]); + #child{pid = Pid} = Child when is_pid(Pid) -> + do_terminate(Child), + terminate_many_children(Children, [ + Child#child{pid = {restarting, Pid}} | NewChildren + ]) + end. + +do_restart_children([], NewChildren, []) -> + {ok, lists:reverse(NewChildren)}; +do_restart_children([], NewChildren, [RetryChild | T] = RetryChildren) -> + if + length(T) =:= 0 -> + {ok, {lists:reverse(NewChildren), RetryChild#child.id}}; + true -> + ok = differed_try_again(RetryChildren), + {ok, lists:reverse(NewChildren)} + end; +do_restart_children([#child{pid = Pid} = Child | Children], NewChildren, RetryChildren) -> + case Pid of + {restarting, _} -> + case try_start(Child) of + {ok, Pid1, {ok, Pid1}} -> + do_restart_children( + Children, [Child#child{pid = Pid1} | NewChildren], RetryChildren + ); + {ok, Pid1, {ok, Pid1, _Result}} -> + do_restart_children( + Children, [Child#child{pid = Pid1} | NewChildren], RetryChildren + ); + {ok, undefined, {ok, undefined}} -> + do_restart_children( + Children, [Child#child{pid = undefined} | NewChildren], RetryChildren + ); + {error, _} -> + do_restart_children(Children, NewChildren, [Child | RetryChildren]) + end; + _ -> + % retain previous ignore children without starting them + do_restart_children(Children, [Child | NewChildren], RetryChildren) + end. + +%% Schedules "try again" restarts at 50ms intervals when multiple children have failed to restart +%% on the first attempt. This is an accumulated (reverse start order) list, so the children that +%% should start last get the longest delay before sending the try_again_restart request. +differed_try_again([]) -> + ok; +differed_try_again([Child | Children] = RetryChildren) -> + erlang:send_after(50 * length(RetryChildren), self(), {try_again_restart, Child#child.id}), + differed_try_again(Children). + child_to_info(#child{id = Id, pid = Pid, type = Type, modules = Modules}) -> Child = case Pid of diff --git a/tests/libs/estdlib/test_supervisor.erl b/tests/libs/estdlib/test_supervisor.erl index 552f24076..94e7662e4 100644 --- a/tests/libs/estdlib/test_supervisor.erl +++ b/tests/libs/estdlib/test_supervisor.erl @@ -35,6 +35,9 @@ test() -> ok = test_terminate_timeout(), ok = test_which_children(), ok = test_count_children(), + ok = test_one_for_all(), + ok = test_crash_limits(), + ok = try_again_restart(), ok. test_basic_supervisor() -> @@ -95,7 +98,7 @@ test_count_children() -> [{specs, 0}, {active, 0}, {supervisors, 0}, {workers, 0}] = supervisor:count_children(SupPid), % Add a worker child and verify counts - {ok, _ChildPid} = supervisor:start_child(SupPid, #{ + {ok, ChildPid} = supervisor:start_child(SupPid, #{ id => test_worker, start => {ping_pong_server, start_link, [self()]}, restart => permanent, @@ -103,13 +106,16 @@ test_count_children() -> type => worker }), + % Receive message sent back so it won't be left for another test to receive erroneously. + ChildPid = get_and_test_server(), + % Check count_children with one active worker [{specs, 1}, {active, 1}, {supervisors, 0}, {workers, 1}] = supervisor:count_children(SupPid), - % Add a supervisor child and verify counts + % Add a supervisor child with no children and verify counts {ok, _SupervisorPid} = supervisor:start_child(SupPid, #{ id => test_supervisor, - start => {?MODULE, start_link, [self()]}, + start => {supervisor, start_link, [?MODULE, {test_no_child, self()}]}, restart => permanent, shutdown => infinity, type => supervisor @@ -215,7 +221,50 @@ child_start({trap_exit, Parent}) -> ok -> ok end end), - {ok, Pid}. + {ok, Pid}; +child_start({get_permission, Arbitrator, Parent}) -> + Arbitrator ! {can_start, self()}, + CanStart = + receive + {do_start, Start} -> Start + after 2000 -> + {timeout, arbitrator} + end, + case CanStart of + true -> + Pid = spawn_link(fun() -> + receive + ok -> ok + end + end), + Parent ! Pid, + {ok, Pid}; + false -> + {error, start_denied}; + Error -> + {error, Error} + end. + +arbitrator_start(Deny) when is_integer(Deny) -> + receive + {can_start, From} -> + From ! {do_start, true} + end, + arbitrator(Deny). + +arbitrator(Deny) -> + Allow = + if + Deny =< 0 -> true; + true -> false + end, + receive + {can_start, From} -> + From ! {do_start, Allow}, + arbitrator(Deny - 1); + shutdown -> + ok + end. test_ping_pong(SupPid) -> Pid1 = get_and_test_server(), @@ -234,8 +283,8 @@ test_ping_pong(SupPid) -> end, no_restart = receive - {ping_pong_server_ready, Pid3} -> - Pid3 + {ping_pong_server_ready, Pid4} -> + Pid4 after 100 -> no_restart end, unlink(SupPid), @@ -289,7 +338,59 @@ init({test_supervisor_order, Parent}) -> ], {ok, {{one_for_one, 10000, 3600}, ChildSpecs}}; init({test_no_child, _Parent}) -> - {ok, {#{strategy => one_for_one, intensity => 10000, period => 3600}, []}}. + {ok, {#{strategy => one_for_one, intensity => 10000, period => 3600}, []}}; +init({test_one_for_all, Parent}) -> + ChildSpecs = [ + #{ + id => ping_pong_1, + start => {ping_pong_server, start_link, [Parent]}, + restart => permanent, + shutdown => brutal_kill, + type => worker, + modules => [ping_pong_server] + }, + #{ + id => ping_pong_2, + start => {ping_pong_server, start_link, [Parent]}, + restart => transient, + shutdown => brutal_kill, + type => worker, + modules => [ping_pong_server] + }, + #{ + id => ready_0, + start => {notify_init_server, start_link, [{Parent, ready_0}]}, + restart => temporary, + shutdown => brutal_kill, + type => worker, + modules => [notify_init_server] + } + ], + {ok, {#{strategy => one_for_all, intensity => 10000, period => 3600}, ChildSpecs}}; +init({test_crash_limits, Intensity, Period, Parent}) -> + ChildSpec = [ + #{ + id => test_child, + start => {ping_pong_server, start_link, [Parent]}, + restart => transient, + shutdown => brutal_kill, + type => worker, + modules => [ping_pong_server] + } + ], + {ok, {#{strategy => one_for_one, intensity => Intensity, period => Period}, ChildSpec}}; +init({test_try_again, Arbitrator, Parent}) -> + ChildSpec = [ + #{ + id => finicky_child, + start => {?MODULE, child_start, [{get_permission, Arbitrator, Parent}]}, + restart => permanent, + shutdown => brutal_kill, + type => worker, + modules => [?MODULE] + } + ], + {ok, {#{strategy => one_for_one, intensity => 5, period => 10}, ChildSpec}}. test_supervisor_order() -> {ok, SupPid} = supervisor:start_link(?MODULE, {test_supervisor_order, self()}), @@ -317,7 +418,7 @@ test_which_children() -> [] = supervisor:which_children(SupPid), % Add a child and test - {ok, _ChildPid} = supervisor:start_child(SupPid, #{ + {ok, ChildPid} = supervisor:start_child(SupPid, #{ id => test_child, start => {ping_pong_server, start_link, [self()]}, restart => permanent, @@ -325,6 +426,9 @@ test_which_children() -> type => worker }), + % Receive message sent back so it won't be left for another test to receive erroneously. + ChildPid = get_and_test_server(), + % Check which_children returns the child info [{test_child, ChildPid, worker, [ping_pong_server]}] = supervisor:which_children(SupPid), true = is_pid(ChildPid), @@ -340,3 +444,211 @@ test_which_children() -> unlink(SupPid), exit(SupPid, shutdown), ok. + +test_one_for_all() -> + {ok, SupPid} = supervisor:start_link({local, allforone}, ?MODULE, {test_one_for_all, self()}), + % Collect startup message from permanent ping_pong_server + Server_1 = get_and_test_server(), + % Collect startup message from transient ping_pong_server + Server_2 = get_and_test_server(), + % Collect startup message from temporary notify_init_server + ready_0 = + receive + Msg1 -> Msg1 + after 1000 -> error({timeout, {start, ready_0}}) + end, + + [{specs, 3}, {active, 3}, {supervisors, 0}, {workers, 3}] = supervisor:count_children(SupPid), + + %% Monitor transient Server_2 to make sure it is stopped, and restarted when + %% permanent Server_1 is shutdown. + MonRef = monitor(process, Server_2), + ok = gen_server:call(Server_1, {stop, test_crash}), + %% Server_2 should exit before the first child is restarted, but exit messages from + %% monitored processes may take some time to be received so we may get the message + %% from the first restarted child first. + First = + receive + {'DOWN', MonRef, process, Server_2, killed} -> + down; + {ping_pong_server_ready, Restart1} when is_pid(Restart1) -> + ready + after 1000 -> + error({timeout, restart_after_crash}) + end, + ok = + case First of + down -> + receive + {ping_pong_server_ready, Restart_1} when is_pid(Restart_1) -> ok + after 1000 -> + error({timeout, restart_after_crash}) + end; + ready -> + receive + {'DOWN', MonRef, process, Server_2, killed} -> ok + after 1000 -> + error({timeout, restart_after_crash}) + end + end, + + demonitor(MonRef), + + % Collect startup message from restarted transient ping_pong_server child + _Restart_2 = get_and_test_server(), + % Make sure temporary notify_init_server is not restarted + no_start = + receive + ready_0 -> error({error, restarted_temporary}) + after 1000 -> no_start + end, + + % Ensure correct number of children + [{specs, 2}, {active, 2}, {supervisors, 0}, {workers, 2}] = supervisor:count_children(SupPid), + + unlink(SupPid), + exit(SupPid, shutdown), + ok. + +test_crash_limits() -> + %% Trap exits so this test doesn't shutdown with the supervisor + process_flag(trap_exit, true), + Intensity = 2, + Period = 5, + {ok, SupPid} = supervisor:start_link( + {local, test_crash_limits}, ?MODULE, {test_crash_limits, Intensity, Period, self()} + ), + Pid1 = get_ping_pong_pid(), + gen_server:call(Pid1, {stop, test_crash1}), + Pid2 = get_ping_pong_pid(), + gen_server:cast(Pid2, {crash, test_crash2}), + Pid3 = get_ping_pong_pid(), + %% Wait until period expires so we can test that stale shutdowns are purged from the shutdown list + timer:sleep(6000), + + gen_server:call(Pid3, {stop, test_crash3}), + Pid4 = get_ping_pong_pid(), + gen_server:cast(Pid4, {crash, test_crash4}), + Pid5 = get_ping_pong_pid(), + + %% The next crash will reach the restart threshold and shutdown the supervisor + gen_server:call(Pid5, {stop, test_crash5}), + + %% Test supervisor has exited + ok = + receive + {'EXIT', SupPid, shutdown} -> + ok + after 2000 -> + error({supervisor_not_stopped, reached_max_restart_intensity}) + end, + process_flag(trap_exit, false), + + %% Test child crashed and was not restarted + ok = + try gen_server:call(Pid5, ping) of + pong -> error(not_stopped, Pid5) + catch + exit:{noproc, _MFA} -> ok + end, + ok = + try get_ping_pong_pid() of + Pid6 when is_pid(Pid6) -> + error({child_restarted, reached_max_restart_intensity}) + catch + throw:timeout -> + ok + end, + + ok = + try erlang:process_info(SupPid, links) of + {links, Links} when is_list(Links) -> + error({not_stopped, reached_max_restart_intensity}); + undefined -> + ok + catch + error:badarg -> + ok + end, + ok. + +get_ping_pong_pid() -> + receive + {ping_pong_server_ready, Pid} -> Pid + after 2000 -> throw(timeout) + end. + +try_again_restart() -> + process_flag(trap_exit, true), + timer:sleep(100), + Arbitrator = erlang:spawn(fun() -> arbitrator_start(4) end), + ok = + case is_pid(Arbitrator) of + true -> ok; + false -> {error, no_arbitrator} + end, + {ok, SupPid} = supervisor:start_link( + {local, try_again_test}, ?MODULE, {test_try_again, Arbitrator, self()} + ), + ok = + case is_pid(SupPid) of + true -> ok; + false -> {error, no_supervisor} + end, + ChildPid = + receive + Pid0 when is_pid(Pid0) -> + Pid0 + after 2000 -> + error({timeout, no_child_start}) + end, + ok = + case is_pid(ChildPid) of + true -> ok; + false -> ChildPid + end, + ChildPid ! ok, + ChildPid1 = + receive + Pid1 when is_pid(Pid1) -> + Pid1 + after 2000 -> + error({timeout, no_child_restart}) + end, + ChildPid1 ! ok, + Arbitrator ! shutdown, + exit(SupPid, normal), + ok = + receive + {'EXIT', SupPid, normal} -> + ok + after 2000 -> + error({supervisor_not_stopped, normal}) + end, + + Arbitrator2 = erlang:spawn(fun() -> arbitrator_start(5) end), + {ok, SupPid2} = supervisor:start_link( + {local, test_try_again}, ?MODULE, {test_try_again, Arbitrator2, self()} + ), + ok = + case is_pid(SupPid2) of + true -> ok; + false -> {error, no_supervisor2} + end, + ChildPid2 = + receive + Pid2 when is_pid(Pid2) -> + Pid2 + after 2000 -> + error({timeout, no_child2_start}) + end, + ChildPid2 ! ok, + ok = + receive + {'EXIT', SupPid2, shutdown} -> + ok + after 2000 -> + error({supervisor_not_stopped, restart_try_again_exceeded}) + end, + process_flag(trap_exit, false), + ok.