end-to-end tests for #167

README.md

@@ -16,10 +16,10 @@ Julia 1.3 through 1.5 as well.
 
 The public API of `Downloads` consists of two functions and three types:
 
-- `download` — download a file from a URL, erroring if it can't be downloaded
+- `download` — download a file from a URL, erroring if it can't be downloaded
 - `request` — request a URL, returning a `Response` object indicating success
-- `Response` — a type capturing the status and other metadata about a request
-- `RequestError` — an error type thrown by `download` and `request` on error
+- `Response` — a type capturing the status and other metadata about a request
+- `RequestError` — an error type thrown by `download` and `request` on error
 - `Downloader` — an object encapsulating shared resources for downloading
 
 ### download
@@ -31,6 +31,7 @@ download(url, [ output = tempfile() ];
     [ timeout = <none>, ]
     [ progress = <none>, ]
     [ verbose = false, ]
+    [ debug = <none>, ]
     [ downloader = <default>, ]
 ) -> output
 ```
@@ -41,6 +42,7 @@ download(url, [ output = tempfile() ];
 - `timeout    :: Real`
 - `progress   :: (total::Integer, now::Integer) --> Any`
 - `verbose    :: Bool`
+- `debug      :: (type, message) --> Any`
 - `downloader :: Downloader`
 
 Download a file from the given url, saving it to `output` or if not specified, a
@@ -71,8 +73,14 @@ remains zero until the server gives an indication of the total size of the
 download (e.g. with a `Content-Length` header), which may never happen. So a
 well-behaved progress callback should handle a total size of zero gracefully.
 
-If the `verbose` optoin is set to true, `libcurl`, which is used to implement
-the download functionality will print debugging information to `stderr`.
+If the `verbose` option is set to true, `libcurl`, which is used to implement
+the download functionality will print debugging information to `stderr`. If the
+`debug` option is set to a function accepting two `String` arguments, then the
+verbose option is ignored and instead the data that would have been printed to
+`stderr` is passed to the `debug` callback with `type` and `message` arguments.
+The `type` argument indicates what kind of event has occurred, and is one of:
+`TEXT`, `HEADER IN`, `HEADER OUT`, `DATA IN`, `DATA OUT`, `SSL DATA IN` or `SSL
+DATA OUT`. The `message` argument is the description of the debug event.
 
 ### request
 
@@ -85,6 +93,7 @@ request(url;
     [ timeout = <none>, ]
     [ progress = <none>, ]
     [ verbose = false, ]
+    [ debug = <none>, ]
     [ throw = true, ]
     [ downloader = <default>, ]
 ) -> Union{Response, RequestError}
@@ -97,6 +106,7 @@ request(url;
 - `timeout    :: Real`
 - `progress   :: (dl_total, dl_now, ul_total, ul_now) --> Any`
 - `verbose    :: Bool`
+- `debug      :: (type, message) --> Any`
 - `throw      :: Bool`
 - `downloader :: Downloader`
 

src/Curl/Curl.jl

@@ -6,6 +6,7 @@ export
         set_url,
         set_method,
         set_verbose,
+        set_debug,
         set_body,
         set_upload_size,
         set_seeker,

src/Curl/Easy.jl

@@ -9,6 +9,7 @@ mutable struct Easy
     res_hdrs :: Vector{String}
     code     :: CURLcode
     errbuf   :: Vector{UInt8}
+    debug    :: Union{Function,Nothing}
 end
 
 const EMPTY_BYTE_VECTOR = UInt8[]
@@ -25,6 +26,7 @@ function Easy()
         String[],
         typemax(CURLcode),
         zeros(UInt8, CURL_ERROR_SIZE),
+        nothing,
     )
     finalizer(done!, easy)
     add_callbacks(easy)
@@ -106,6 +108,19 @@ function set_verbose(easy::Easy, verbose::Bool)
     setopt(easy, CURLOPT_VERBOSE, verbose)
 end
 
+function set_debug(easy::Easy, debug::Function)
+    hasmethod(debug, Tuple{String,String}) ||
+        throw(ArgumentError("debug callback must take (::String, ::String)"))
+    easy.debug = debug
+    add_debug_callback(easy)
+    set_verbose(easy, true)
+end
+
+function set_debug(easy::Easy, debug::Nothing)
+    easy.debug = nothing
+    remove_debug_callback(easy)
+end
+
 function set_body(easy::Easy, body::Bool)
     setopt(easy, CURLOPT_NOBODY, !body)
 end
@@ -116,7 +131,7 @@ function set_upload_size(easy::Easy, size::Integer)
 end
 
 function set_seeker(seeker::Function, easy::Easy)
-    add_seek_callbacks(easy)
+    add_seek_callback(easy)
     easy.seeker = seeker
 end
 
@@ -159,7 +174,7 @@ function enable_progress(easy::Easy, on::Bool=true)
 end
 
 function enable_upload(easy::Easy)
-    add_upload_callbacks(easy::Easy)
+    add_upload_callback(easy::Easy)
     setopt(easy, CURLOPT_UPLOAD, true)
 end
 
@@ -229,6 +244,17 @@ function status_ok(proto::AbstractString, status::Integer)
 end
 status_ok(proto::Nothing, status::Integer) = false
 
+function info_type(type::curl_infotype)
+    type == 0 ? "TEXT" :
+    type == 1 ? "HEADER IN" :
+    type == 2 ? "HEADER OUT" :
+    type == 3 ? "DATA IN" :
+    type == 4 ? "DATA OUT" :
+    type == 5 ? "SSL DATA IN" :
+    type == 6 ? "SSL DATA OUT" :
+                "UNKNOWN"
+end
+
 function get_effective_url(easy::Easy)
     url_ref = Ref{Ptr{Cchar}}()
     @check curl_easy_getinfo(easy.handle, CURLINFO_EFFECTIVE_URL, url_ref)
@@ -376,6 +402,19 @@ function progress_callback(
     return 0
 end
 
+function debug_callback(
+    handle :: Ptr{Cvoid},
+    type   :: curl_infotype,
+    data   :: Ptr{Cchar},
+    size   :: Csize_t,
+    easy_p :: Ptr{Cvoid},
+)::Cint
+    easy = unsafe_pointer_to_objref(easy_p)::Easy
+    @assert easy.handle == handle
+    easy.debug(info_type(type), unsafe_string(data, size))
+    return 0
+end
+
 function add_callbacks(easy::Easy)
     # pointer to easy object
     easy_p = pointer_from_objref(easy)
@@ -404,7 +443,7 @@ function add_callbacks(easy::Easy)
     setopt(easy, CURLOPT_XFERINFODATA, easy_p)
 end
 
-function add_upload_callbacks(easy::Easy)
+function add_upload_callback(easy::Easy)
     # pointer to easy object
     easy_p = pointer_from_objref(easy)
 
@@ -415,7 +454,7 @@ function add_upload_callbacks(easy::Easy)
     setopt(easy, CURLOPT_READDATA, easy_p)
 end
 
-function add_seek_callbacks(easy::Easy)
+function add_seek_callback(easy::Easy)
     # pointer to easy object
     easy_p = pointer_from_objref(easy)
 
@@ -425,3 +464,19 @@ function add_seek_callbacks(easy::Easy)
     setopt(easy, CURLOPT_SEEKFUNCTION, seek_cb)
     setopt(easy, CURLOPT_SEEKDATA, easy_p)
 end
+
+function add_debug_callback(easy::Easy)
+    # pointer to easy object
+    easy_p = pointer_from_objref(easy)
+
+    # set debug callback
+    debug_cb = @cfunction(debug_callback,
+        Cint, (Ptr{Cvoid}, curl_infotype, Ptr{Cchar}, Csize_t, Ptr{Cvoid}))
+    setopt(easy, CURLOPT_DEBUGFUNCTION, debug_cb)
+    setopt(easy, CURLOPT_DEBUGDATA, easy_p)
+end
+
+function remove_debug_callback(easy::Easy)
+    setopt(easy, CURLOPT_DEBUGFUNCTION, C_NULL)
+    setopt(easy, CURLOPT_DEBUGDATA, C_NULL)
+end

src/Downloads.jl

@@ -5,10 +5,10 @@ for the `Base.download` function in Julia 1.6 or later.
 
 More generally, the module exports functions and types that provide lower-level control and diagnostic information
 for file downloading:
-- [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded
+- [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded
 - [`request`](@ref) — request a URL, returning a `Response` object indicating success
-- [`Response`](@ref) — a type capturing the status and other metadata about a request
-- [`RequestError`](@ref) — an error type thrown by `download` and `request` on error
+- [`Response`](@ref) — a type capturing the status and other metadata about a request
+- [`RequestError`](@ref) — an error type thrown by `download` and `request` on error
 - [`Downloader`](@ref) — an object encapsulating shared resources for downloading
 """
 module Downloads
@@ -165,6 +165,7 @@ end
         [ timeout = <none>, ]
         [ progress = <none>, ]
         [ verbose = false, ]
+        [ debug = <none>, ]
         [ downloader = <default>, ]
     ) -> output
 
@@ -175,6 +176,7 @@ end
         timeout    :: Real
         progress   :: (total::Integer, now::Integer) --> Any
         verbose    :: Bool
+        debug      :: (type, message) --> Any
         downloader :: Downloader
 
 Download a file from the given url, saving it to `output` or if not specified, a
@@ -206,7 +208,13 @@ download (e.g. with a `Content-Length` header), which may never happen. So a
 well-behaved progress callback should handle a total size of zero gracefully.
 
 If the `verbose` option is set to true, `libcurl`, which is used to implement
-the download functionality will print debugging information to `stderr`.
+the download functionality will print debugging information to `stderr`. If the
+`debug` option is set to a function accepting two `String` arguments, then the
+verbose option is ignored and instead the data that would have been printed to
+`stderr` is passed to the `debug` callback with `type` and `message` arguments.
+The `type` argument indicates what kind of event has occurred, and is one of:
+`TEXT`, `HEADER IN`, `HEADER OUT`, `DATA IN`, `DATA OUT`, `SSL DATA IN` or `SSL
+DATA OUT`. The `message` argument is the description of the debug event.
 """
 function download(
     url        :: AbstractString,
@@ -216,6 +224,7 @@ function download(
     timeout    :: Real = Inf,
     progress   :: Union{Function, Nothing} = nothing,
     verbose    :: Bool = false,
+    debug      :: Union{Function, Nothing} = nothing,
     downloader :: Union{Downloader, Nothing} = nothing,
 ) :: ArgWrite
     arg_write(output) do output
@@ -227,6 +236,7 @@ function download(
             timeout = timeout,
             progress = progress,
             verbose = verbose,
+            debug = debug,
             downloader = downloader,
         )::Response
         status_ok(response.proto, response.status) && return output
@@ -245,6 +255,7 @@ end
         [ timeout = <none>, ]
         [ progress = <none>, ]
         [ verbose = false, ]
+        [ debug = <none>, ]
         [ throw = true, ]
         [ downloader = <default>, ]
     ) -> Union{Response, RequestError}
@@ -257,6 +268,7 @@ end
         timeout    :: Real
         progress   :: (dl_total, dl_now, ul_total, ul_now) --> Any
         verbose    :: Bool
+        debug      :: (type, message) --> Any
         throw      :: Bool
         downloader :: Downloader
 
@@ -287,6 +299,7 @@ function request(
     timeout    :: Real = Inf,
     progress   :: Union{Function, Nothing} = nothing,
     verbose    :: Bool = false,
+    debug      :: Union{Function, Nothing} = nothing,
     throw      :: Bool = true,
     downloader :: Union{Downloader, Nothing} = nothing,
 ) :: Union{Response, RequestError}
@@ -317,6 +330,7 @@ function request(
                 set_url(easy, url)
                 set_timeout(easy, timeout)
                 set_verbose(easy, verbose)
+                set_debug(easy, debug)
                 add_headers(easy, headers)
 
                 # libcurl does not set the default header reliably so set it

test/runtests.jl

@@ -105,6 +105,33 @@ include("setup.jl")
         rm(file)
     end
 
+    @testset "put from io" begin
+        url = "$server/put"
+        file = tempname()
+        write(file, "Hello, world!")
+        len = filesize(file)
+        for headers in [Pair{String,String}[], ["Content-Length" => "$len"]]
+            open(file) do io
+                events = Pair{String,String}[]
+                debug(type, msg) = push!(events, type => msg)
+                resp, json = request_json(url, input=io, debug=debug, headers=headers)
+                @test json["url"] == url
+                @test json["data"] == read(file, String)
+                header_out(hdr::String) = any(events) do (type, msg)
+                    type == "HEADER OUT" && hdr in map(lowercase, split(msg, "\r\n"))
+                end
+                chunked = header_out("transfer-encoding: chunked")
+                content_length = header_out("content-length: $len")
+                if isempty(headers)
+                    @test chunked && !content_length
+                else
+                    @test !chunked && content_length
+                end
+            end
+        end
+        rm(file)
+    end
+
     @testset "redirected get" begin
         url = "$server/get"
         redirect = "$server/redirect-to?url=$(url_escape(url))"
@@ -195,6 +222,22 @@ include("setup.jl")
         end
     end
 
+    @testset "debug callback" begin
+        url = "$server/get"
+        events = Pair{String,String}[]
+        resp = request(url, debug = (type, msg) -> push!(events, type => msg))
+        @test resp isa Response && resp.status == 200
+        @test any(events) do (type, msg)
+            type == "TEXT" && startswith(msg, "Connected to ")
+        end
+        @test any(events) do (type, msg)
+            type == "HEADER OUT" && contains(msg, r"^HEAD /get HTTP/[\d\.+]+\s$"m)
+        end
+        @test any(events) do (type, msg)
+            type == "HEADER IN" && contains(msg, r"^HTTP/[\d\.]+ 200 OK\s*$")
+        end
+    end
+
     @testset "session support" begin
         downloader = Downloader()
 

test/setup.jl

@@ -60,10 +60,10 @@ end
 # URL escape & unescape
 
 function is_url_safe_byte(byte::UInt8)
-    0x2d ≤ byte ≤ 0x2e ||
+    0x2d ≤ byte ≤ 0x2e ||
     0x30 ≤ byte ≤ 0x39 ||
-    0x41 ≤ byte ≤ 0x5a ||
-    0x61 ≤ byte ≤ 0x7a ||
+    0x41 ≤ byte ≤ 0x5a ||
+    0x61 ≤ byte ≤ 0x7a ||
     byte == 0x5f ||
     byte == 0x7e
 end