Errors with HTTP.jl 1.6

[JackDunnNZ]

It looks like JuliaWeb/HTTP.jl@02ba345 in HTTP.jl 1.6 is causing problems here, as I am seeing messages like this:

┌ Error: Internal Error: provided body is empty while the reported content-length is non-zero
└ @ AWS.AWSExceptions ~/.julia/packages/AWS/j8qVm/src/AWSExceptions.jl:98

which is coming from here:

AWS.jl/src/AWSExceptions.jl

Line 97 in ec701b7

# Should only occur streaming a response and error handling is improperly configured

@quinnj do you know what the problem might be?

[quinnj]

Hmmmm, not off the top of my head; do you have a way to reproduce this? Or can do a quick zoom and show me what's going on?

[JackDunnNZ]

Thanks, I see it when running the AWS.jl tests (with the default credentials on my local machine). Here is the first failure:

┌ Error: Internal Error: provided body is empty while the reported content-length is non-zero
└ @ AWS.AWSExceptions ~/.julia/packages/AWS/j8qVm/src/AWSExceptions.jl:98
high-level secrets manager: Error During Test at ~/.julia/packages/AWS/j8qVm/test/AWS.jl:588
  Got exception outside of a @test
  AWSException: 400 -- AWSException

  HTTP.Exceptions.StatusError(400, "POST", "/", HTTP.Messages.Response:
  """
  HTTP/1.1 400 Bad Request
  x-amzn-RequestId: 57855ad7-1d41-4561-8576-0ffd39e8ed70
  Content-Type: application/x-amz-json-1.1
  Content-Length: 275
  Date: Mon, 12 Dec 2022 17:31:06 GMT
  Connection: close

  [Message Body was streamed]""")

  (empty body)

  Stacktrace:
    [1] (::HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}})(stream::HTTP.Streams.Stream{HTTP.Messages.Response, HTTP.ConnectionPool.Connection}; status_exception::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
      @ HTTP.ExceptionRequest ~/.julia/dev/HTTP/src/clientlayers/ExceptionRequest.jl:16
    [2] #1#2
      @ ~/.julia/dev/HTTP/src/clientlayers/TimeoutRequest.jl:17 [inlined]
    [3] (::HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
      @ HTTP.ConnectionRequest ~/.julia/dev/HTTP/src/clientlayers/ConnectionRequest.jl:111
    [4] (::HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; canonicalize_headers::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
      @ HTTP.CanonicalizeRequest ~/.julia/dev/HTTP/src/clientlayers/CanonicalizeRequest.jl:17
    [5] (::HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
      @ HTTP.RetryRequest ~/.julia/dev/HTTP/src/clientlayers/RetryRequest.jl:29
    [6] (::HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
      @ HTTP.CookieRequest ~/.julia/dev/HTTP/src/clientlayers/CookieRequest.jl:42
    [7] (::HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; detect_content_type::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
      @ HTTP.ContentTypeDetection ~/.julia/dev/HTTP/src/clientlayers/ContentTypeRequest.jl:23
    [8] (::HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; basicauth::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
      @ HTTP.BasicAuthRequest ~/.julia/dev/HTTP/src/clientlayers/BasicAuthRequest.jl:22
    [9] (::HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :retry), Tuple{Int64, Bool}}})
      @ HTTP.DefaultHeadersRequest ~/.julia/dev/HTTP/src/clientlayers/DefaultHeadersRequest.jl:47
   [10] (::HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Base.BufferStream, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :retry), Tuple{Int64, Bool}}})
      @ HTTP.RedirectRequest ~/.julia/dev/HTTP/src/clientlayers/RedirectRequest.jl:17
   [11] #1#2
      @ ~/.julia/dev/HTTP/src/clientlayers/DebugRequest.jl:23 [inlined]
   [12] (::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String; response_stream::Base.BufferStream, http_version::VersionNumber, kw::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol}, NamedTuple{(:redirect, :retry), Tuple{Bool, Bool}}})
      @ HTTP.MessageRequest ~/.julia/dev/HTTP/src/clientlayers/MessageRequest.jl:19
   [13] request(stack::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}}, method::String, url::URIs.URI, h::Vector{Pair{SubString{String}, SubString{String}}}, b::String, q::Nothing; headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String, query::Nothing, kw::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:redirect, :retry, :response_stream), Tuple{Bool, Bool, Base.BufferStream}}})
      @ HTTP ~/.julia/dev/HTTP/src/HTTP.jl:420
   [14] #request#16
      @ ~/.julia/dev/HTTP/src/HTTP.jl:280 [inlined]
   [15] macro expansion
      @ ~/.julia/packages/Mocking/HMh91/src/mock.jl:26 [inlined]
   [16] (::AWS.var"#40#42"{Request, LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}})()
      @ AWS ~/.julia/packages/AWS/j8qVm/src/utilities/request.jl:220
   [17] (::Base.var"#88#90"{Base.var"#88#89#91"{AWS.AWSExponentialBackoff, AWS.var"#41#43", AWS.var"#40#42"{Request, LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}}}})(; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
      @ Base ./error.jl:296
   [18] (::Base.var"#88#90"{Base.var"#88#89#91"{AWS.AWSExponentialBackoff, AWS.var"#41#43", AWS.var"#40#42"{Request, LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}}}})()
      @ Base ./error.jl:291
   [19] _http_request(http_backend::AWS.HTTPBackend, request::Request, response_stream::IOBuffer)
      @ AWS ~/.julia/packages/AWS/j8qVm/src/utilities/request.jl:245
   [20] macro expansion
      @ ~/.julia/packages/Mocking/HMh91/src/mock.jl:26 [inlined]
   [21] (::AWS.var"#33#36"{AWSConfig, Request, IOBuffer, Vector{Int64}})()
      @ AWS ~/.julia/packages/AWS/j8qVm/src/utilities/request.jl:129
   [22] (::AWS.var"#34#38"{AWS.var"#33#36"{AWSConfig, Request, IOBuffer, Vector{Int64}}, IOBuffer})()
      @ AWS ~/.julia/packages/AWS/j8qVm/src/utilities/request.jl:144
   [23] (::Base.var"#88#90"{Base.var"#88#89#91"{AWS.AWSExponentialBackoff, AWS.var"#35#39"{AWSConfig, Vector{String}, Vector{String}, Int64}, AWS.var"#34#38"{AWS.var"#33#36"{AWSConfig, Request, IOBuffer, Vector{Int64}}, IOBuffer}}})(; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
      @ Base ./error.jl:296
   [24] (::Base.var"#88#90"{Base.var"#88#89#91"{AWS.AWSExponentialBackoff, AWS.var"#35#39"{AWSConfig, Vector{String}, Vector{String}, Int64}, AWS.var"#34#38"{AWS.var"#33#36"{AWSConfig, Request, IOBuffer, Vector{Int64}}, IOBuffer}}})()
      @ Base ./error.jl:291
   [25] submit_request(aws::AWSConfig, request::Request; return_headers::Nothing)
      @ AWS ~/.julia/packages/AWS/j8qVm/src/utilities/request.jl:195
   [26] (::JSONService)(operation::String, args::Dict{String, Any}; aws_config::AWSConfig, feature_set::AWS.FeatureSet)
      @ AWS ~/.julia/packages/AWS/j8qVm/src/AWS.jl:376
   [27] create_secret(Name::String, params::LittleDict{String, String, Vector{String}, Vector{String}}; aws_config::AWSConfig)
      @ Main.Secrets_Manager ~/.julia/packages/AWS/j8qVm/src/services/secrets_manager.jl:170
   [28] create_secret(Name::String, params::LittleDict{String, String, Vector{String}, Vector{String}})
      @ Main.Secrets_Manager ~/.julia/packages/AWS/j8qVm/src/services/secrets_manager.jl:167
   [29] macro expansion
      @ ~/.julia/packages/AWS/j8qVm/test/AWS.jl:600 [inlined]
   [30] macro expansion
      @ ~/.julia/juliaup/julia-1.8.3+0.x64/share/julia/stdlib/v1.8/Test/src/Test.jl:1360 [inlined]
   [31] macro expansion
      @ ~/.julia/packages/AWS/j8qVm/test/AWS.jl:589 [inlined]
   [32] macro expansion
      @ ~/.julia/juliaup/julia-1.8.3+0.x64/share/julia/stdlib/v1.8/Test/src/Test.jl:1360 [inlined]
   [33] top-level scope
      @ ~/.julia/packages/AWS/j8qVm/test/AWS.jl:588
   [34] include
      @ ./client.jl:476 [inlined]
   [35] macro expansion
      @ ~/.julia/packages/AWS/j8qVm/test/runtests.jl:63 [inlined]
   [36] macro expansion
      @ ~/.julia/juliaup/julia-1.8.3+0.x64/share/julia/stdlib/v1.8/Test/src/Test.jl:1436 [inlined]
   [37] macro expansion
      @ ~/.julia/packages/AWS/j8qVm/test/runtests.jl:61 [inlined]
   [38] macro expansion
      @ ~/.julia/juliaup/julia-1.8.3+0.x64/share/julia/stdlib/v1.8/Test/src/Test.jl:1360 [inlined]
   [39] top-level scope
      @ ~/.julia/packages/AWS/j8qVm/test/runtests.jl:54
   [40] include(fname::String)
      @ Base.MainInclude ./client.jl:476
   [41] top-level scope
      @ none:6
   [42] eval
      @ ./boot.jl:368 [inlined]
   [43] exec_options(opts::Base.JLOptions)
      @ Base ./client.jl:276
   [44] _start()
      @ Base ./client.jl:522

Here's the full log from the test, searching for "internal error" pulls up 14 examples there:

https://gist.github.com/JackDunnNZ/3d1d750c12ea8956e12f1b29f646c030

I saw in the original version of the PR you mentioned something to do with AWS.jl, just in case that's relevant.

[iamed2]

Encountered this in an internal package:

Internal Error: provided body is empty while the reported content-length is non-zero
ERROR: LoadError: AWS.AWSExceptions.AWSException: 400 -- AWSException
HTTP.Exceptions.StatusError(400, "POST", "/", HTTP.Messages.Response:
"""
HTTP/1.1 400 Bad Request
x-amzn-RequestId: d129638d-eeab-45ff-87fa-b70bc098a1a8
Content-Type: application/x-amz-json-1.1
Content-Length: 58
Date: Mon, 12 Dec 2022 09:01:18 GMT
Connection: close
[Message Body was streamed]""")
(empty body)
Stacktrace:
  [1] (::HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}})(stream::HTTP.Streams.Stream{HTTP.Messages.Response, HTTP.ConnectionPool.Connection}; status_exception::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
    @ HTTP.ExceptionRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/ExceptionRequest.jl:16
  [2] #1#2
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/TimeoutRequest.jl:17 [inlined]
  [3] (::HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
    @ HTTP.ConnectionRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/ConnectionRequest.jl:111
  [4] (::HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; canonicalize_headers::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
    @ HTTP.CanonicalizeRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/CanonicalizeRequest.jl:17
  [5] (::HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose), Tuple{Nothing, Nothing, Int64}}})
    @ HTTP.RetryRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/RetryRequest.jl:29
  [6] (::HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
    @ HTTP.CookieRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/CookieRequest.jl:42
  [7] (::HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; detect_content_type::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
    @ HTTP.ContentTypeDetection /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/ContentTypeRequest.jl:23
  [8] (::HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; basicauth::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :retry), Tuple{Nothing, Nothing, Int64, Bool}}})
    @ HTTP.BasicAuthRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/BasicAuthRequest.jl:22
  [9] (::HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, kw::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :retry), Tuple{Int64, Bool}}})
    @ HTTP.DefaultHeadersRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/DefaultHeadersRequest.jl:47
 [10] (::HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Base.BufferStream, kw::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :retry), Tuple{Int64, Bool}}})
    @ HTTP.RedirectRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/RedirectRequest.jl:17
 [11] #1#2
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/DebugRequest.jl:23 [inlined]
 [12] (::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String; response_stream::Base.BufferStream, http_version::VersionNumber, kw::Base.Iterators.Pairs{Symbol, Bool, Tuple{Symbol, Symbol}, NamedTuple{(:redirect, :retry), Tuple{Bool, Bool}}})
    @ HTTP.MessageRequest /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/clientlayers/MessageRequest.jl:19
 [13] request(stack::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}}, method::String, url::URIs.URI, h::Vector{Pair{SubString{String}, SubString{String}}}, b::String, q::Nothing; headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String, query::Nothing, kw::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:redirect, :retry, :response_stream), Tuple{Bool, Bool, Base.BufferStream}}})
    @ HTTP /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/HTTP.jl:420
 [14] #request#16
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/HTTP/6bgtf/src/HTTP.jl:280 [inlined]
 [15] macro expansion
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/Mocking/HMh91/src/mock.jl:29 [inlined]
 [16] (::AWS.var"#40#42"{AWS.Request, OrderedCollections.LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}})()
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/utilities/request.jl:220
 [17] (::Base.var"#70#72"{Base.var"#70#71#73"{AWS.AWSExponentialBackoff, AWS.var"#41#43", AWS.var"#40#42"{AWS.Request, OrderedCollections.LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}}}})(; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./error.jl:288
 [18] (::Base.var"#70#72"{Base.var"#70#71#73"{AWS.AWSExponentialBackoff, AWS.var"#41#43", AWS.var"#40#42"{AWS.Request, OrderedCollections.LittleDict{Symbol, Any, Vector{Symbol}, Vector{Any}}}}})()
    @ Base ./error.jl:284
 [19] _http_request(http_backend::AWS.HTTPBackend, request::AWS.Request, response_stream::IOBuffer)
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/utilities/request.jl:245
 [20] macro expansion
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/Mocking/HMh91/src/mock.jl:29 [inlined]
 [21] (::AWS.var"#33#36"{AWS.AWSConfig, AWS.Request, IOBuffer, Vector{Int64}})()
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/utilities/request.jl:129
 [22] (::AWS.var"#34#38"{AWS.var"#33#36"{AWS.AWSConfig, AWS.Request, IOBuffer, Vector{Int64}}, IOBuffer})()
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/utilities/request.jl:144
 [23] (::Base.var"#70#72"{Base.var"#70#71#73"{AWS.AWSExponentialBackoff, AWS.var"#35#39"{AWS.AWSConfig, Vector{String}, Vector{String}, Int64}, AWS.var"#34#38"{AWS.var"#33#36"{AWS.AWSConfig, AWS.Request, IOBuffer, Vector{Int64}}, IOBuffer}}})(; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./error.jl:288
 [24] (::Base.var"#70#72"{Base.var"#70#71#73"{AWS.AWSExponentialBackoff, AWS.var"#35#39"{AWS.AWSConfig, Vector{String}, Vector{String}, Int64}, AWS.var"#34#38"{AWS.var"#33#36"{AWS.AWSConfig, AWS.Request, IOBuffer, Vector{Int64}}, IOBuffer}}})()
    @ Base ./error.jl:284
 [25] submit_request(aws::AWS.AWSConfig, request::AWS.Request; return_headers::Nothing)
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/utilities/request.jl:195
 [26] (::AWS.JSONService)(operation::String, args::Dict{String, String}; aws_config::AWS.AWSConfig, feature_set::AWS.FeatureSet)
    @ AWS /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/AWS.jl:376
 [27] #describe_log_groups#32
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/AWS/j8qVm/src/services/cloudwatch_logs.jl:674 [inlined]
 [28] macro expansion
    @ /builds/invenia/BackrunBuddy.jl.tmp/depot/packages/Mocking/HMh91/src/mock.jl:29 [inlined]
 [29] _list_log_groups(; prefix::String)
    @ BackrunBuddy /builds/invenia/BackrunBuddy.jl/src/aws/cloudwatch.jl:32

We'll have to switch to Downloads.jl backend in our package to avoid this for now. @JackDunnNZ I recommend trying the Downloads backend to see if that fixes your issue.

[quinnj]

Whoops, sorry about the breakage here; fix is up: JuliaWeb/HTTP.jl#975

[quinnj]

Should be fixed now that HTTP.jl 1.6.1 is out.

[iamed2]

Much appreciated @quinnj !