Improve flakiness of tests (#728)

Author: whatyouhide

.github/workflows/main.yml

@@ -14,9 +14,9 @@ env:
 jobs:
   test:
     name: Test (Elixir ${{ matrix.elixir }}, OTP ${{ matrix.otp }})
-
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         # https://hexdocs.pm/elixir/compatibility-and-deprecations.html#compatibility-between-elixir-and-erlang-otp
         include:

test/logger_backend_test.exs

@@ -134,6 +134,8 @@ defmodule Sentry.LoggerBackendTest do
 
     ref = register_before_send()
 
+    start_supervised!(Sentry.ExamplePlugApplication, restart: :temporary)
+
     :hackney.get("http://127.0.0.1:8003/error_route", [], "", [])
     assert_receive {^ref, _event}, 1000
   after
@@ -273,8 +275,7 @@ defmodule Sentry.LoggerBackendTest do
     Logger.debug("Debug")
 
     assert_receive {^ref, event}
-
-    assert event.message.formatted =~ "Error"
+    assert_formatted_message_matches(event, "Error")
   after
     Logger.configure_backend(Sentry.LoggerBackend, level: :error, capture_log_messages: false)
   end
@@ -308,7 +309,7 @@ defmodule Sentry.LoggerBackendTest do
     Logger.error("Error", callers: [dead_pid, nil])
 
     assert_receive {^ref, event}
-    assert event.message.formatted =~ "Error"
+    assert_formatted_message_matches(event, "Error")
   end
 
   test "doesn't log events with :sentry as a domain" do
@@ -367,4 +368,15 @@ defmodule Sentry.LoggerBackendTest do
   defp test_genserver_invalid_fun(pid) do
     TestGenServer.run_async(pid, fn _state -> apply(NaiveDateTime, :from_erl, [{}, {}, {}]) end)
   end
+
+  defp assert_formatted_message_matches(event, string) do
+    assert %Sentry.Event{} = event
+
+    assert Map.get(event.message, :formatted, "") =~ string, """
+    Expected the event to have a filled-in message containing the word "Error", but
+    instead the whole event was:
+
+    #{inspect(event, pretty: true, limit: :infinity)}
+    """
+  end
 end

test/sentry/logger_handler_test.exs

@@ -91,7 +91,10 @@ defmodule Sentry.LoggerHandlerTest do
     @tag handler_config: %{excluded_domains: []}
     test "sends two errors when a Plug process crashes if cowboy domain is not excluded",
          %{sender_ref: ref} do
+      start_supervised!(Sentry.ExamplePlugApplication, restart: :temporary)
+
       :hackney.get("http://127.0.0.1:8003/error_route", [], "", [])
+
       assert_receive {^ref, _event}, 1000
     end
   end

test/support/example_plug_application.ex

@@ -3,14 +3,19 @@ defmodule Sentry.ExamplePlugApplication do
   use Sentry.PlugCapture
   use Plug.ErrorHandler
 
+  import ExUnit.Assertions
+
   plug Plug.Parsers, parsers: [:multipart, :urlencoded]
   plug Sentry.PlugContext
   plug :match
   plug :dispatch
 
-  get "/error_route" do
-    _ = conn
-    raise RuntimeError, "Error"
+  @spec child_spec(keyword()) :: Supervisor.child_spec()
+  def child_spec([]) do
+    Supervisor.child_spec(
+      {Plug.Cowboy, scheme: :http, plug: __MODULE__, options: [port: 8003]},
+      []
+    )
   end
 
   get "/exit_route" do
@@ -23,16 +28,9 @@ defmodule Sentry.ExamplePlugApplication do
     throw(:test)
   end
 
-  post "/error_route" do
-    _ = conn
-    raise RuntimeError, "Error"
-  end
-
   get "/spawn_error_route" do
-    spawn(fn ->
-      raise "Error"
-    end)
-
+    {_pid, ref} = spawn_monitor(fn -> raise "Error" end)
+    assert_receive {:DOWN, ^ref, _, _, _}
     send_resp(conn, 200, "")
   end
 

test/test_helper.exs

@@ -1,9 +1,7 @@
-ExUnit.start(assert_receive_timeout: 500)
+ExUnit.start(assert_receive_timeout: 1000)
 
 File.rm_rf!(Sentry.Sources.path_of_packaged_source_code())
 
 ExUnit.after_suite(fn _ ->
   File.rm_rf!(Sentry.Sources.path_of_packaged_source_code())
 end)
-
-{:ok, _} = Plug.Cowboy.http(Sentry.ExamplePlugApplication, [], port: 8003)