write multiple bytes of Char at once #27267 #27380

yeonsookimdev · 2018-06-02T09:24:51Z

write() function from 0.7.0-alpha:

function write(io::IO, c::Char) 
    # Let c == '한'
    # v == 0xed959c00
    #v = reinterpret(UInt32, c)
    
    # u = 0x009c95ed
    u = bswap(reinterpret(UInt32, c))
    
    # n |    1 |    2 |    3 |    4  
    #   | 0xed | 0x95 | 0x9c | 0x00
    n = 1
    while true
        # n == 1, write 0xed
        # n == 2, write 0x95
        # n == 3, write 0x9c
        write(io, u % UInt8)
        # return 3
        (u >>= 8) == 0 && return n
    n += 1
    end
end

julia> @time write(stdout, '한') == 3
���  0.000629 seconds (13 allocations: 400 bytes)
true

julia> @time write(stdout, '한') == 3
���  0.000445 seconds (10 allocations: 256 bytes)
true

julia> @time write(stdout, '한') == 3
���  0.000226 seconds (10 allocations: 256 bytes)
true

julia> @time write(stdout, '한') == 3
���  0.000305 seconds (10 allocations: 256 bytes)
true

julia> @time write(stdout, '한') == 3
���  0.000350 seconds (10 allocations: 256 bytes)
true

julia>

write() function from alkorang/display:

function write(io::IO, c::Char)
    # Let c == '한'
    # v == 0xed959c00
    v = reinterpret(UInt32, c)
    u = hton(v) # BIG-endian
    
    # p[0] | p[1] | p[2] | p[3]
    # 0xed | 0x95 | 0x9c | 0x00
    p = unsafe_convert(Ptr{Cvoid}, Ref{UInt32}(u))
    
    # Compare with v, not u
    if      v & 0x000000FF != 0
        unsafe_write(io, p, 4)
    elseif  v & 0x0000FFFF != 0
        # Write 0xed, 0x95, 0x9c at once, return 3
        unsafe_write(io, p, 3)
    elseif  v & 0x00FFFFFF != 0
        unsafe_write(io, p, 2)
    else
        unsafe_write(io, p, 1)
    end
end

julia> @time write(stdout, '한') == 3
한  0.000234 seconds (8 allocations: 320 bytes)
true

julia> @time write(stdout, '한') == 3
한  0.000253 seconds (5 allocations: 176 bytes)
true

julia> @time write(stdout, '한') == 3
한  0.000136 seconds (5 allocations: 176 bytes)
true

julia> @time write(stdout, '한') == 3
한  0.000183 seconds (5 allocations: 176 bytes)
true

julia> @time write(stdout, '한') == 3
한  0.000212 seconds (5 allocations: 176 bytes)
true

julia>

appleparan · 2018-06-02T22:25:45Z

Any tests?

yeonsookimdev · 2018-06-03T00:16:42Z

Any tests?

It passed Base.runtests(["strings/io", "char", "iobuffer"]).

julia> Base.runtests(["strings/io", "char", "iobuffer"]);
Test   (Worker) | Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB)
strings/io  (2) |     4.76 |   0.14 |  3.0 |     326.25 |   239.07
char        (3) |     6.60 |   0.27 |  4.1 |     612.73 |   241.36
iobuffer    (2) |     5.00 |   0.13 |  2.7 |     661.03 |   260.16

Test Summary: |  Pass  Total
  Overall     | 14420  14420
    SUCCESS

julia>

strickek · 2018-06-03T07:51:23Z

Is speed for one byte Chars also better?

yeonsookimdev · 2018-06-03T09:04:07Z

Is speed for one byte Chars also better?

function foo(io::IO, c::Char, n::Int)
    for _ in 1:5
        @time for _ in 1:n
            write(io, c)
        end
    end
end

To make it more clear, I tried with IOBuffer after few warming-up.

0.7.0-alpha:

julia> GC.gc(); io = IOBuffer(); foo(io, 'a', 1000)
  0.000024 seconds (5 allocations: 2.172 KiB)
  0.000025 seconds (1 allocation: 2.125 KiB)
  0.000027 seconds (1 allocation: 2.000 KiB)
  0.000022 seconds
  0.000026 seconds (1 allocation: 4.000 KiB)

julia> GC.gc(); io = IOBuffer(); foo(io, 'δ', 1000)
  0.000041 seconds (6 allocations: 4.297 KiB)
  0.000042 seconds (1 allocation: 2.000 KiB)
  0.000050 seconds (1 allocation: 4.000 KiB)
  0.000043 seconds
  0.000049 seconds (1 allocation: 8.000 KiB)

julia> GC.gc(); io = IOBuffer(); foo(io, '한', 1000)
  0.000060 seconds (7 allocations: 6.297 KiB)
  0.000055 seconds (1 allocation: 4.000 KiB)
  0.000058 seconds (1 allocation: 8.000 KiB)
  0.000051 seconds
  0.000056 seconds

julia>

alkorang/display:

julia> GC.gc(); io = IOBuffer(); foo(io, 'a', 1000)
  0.000030 seconds (5 allocations: 2.172 KiB)
  0.000030 seconds (1 allocation: 2.125 KiB)
  0.000033 seconds (1 allocation: 2.000 KiB)
  0.000028 seconds
  0.000033 seconds (1 allocation: 4.000 KiB)

julia> GC.gc(); io = IOBuffer(); foo(io, 'δ', 1000)
  0.000034 seconds (6 allocations: 4.297 KiB)
  0.000034 seconds (1 allocation: 2.000 KiB)
  0.000033 seconds (1 allocation: 4.000 KiB)
  0.000035 seconds
  0.000043 seconds (1 allocation: 8.000 KiB)

julia> GC.gc(); io = IOBuffer(); foo(io, '한', 1000)
  0.000039 seconds (7 allocations: 6.297 KiB)
  0.000035 seconds (1 allocation: 4.000 KiB)
  0.000036 seconds (1 allocation: 8.000 KiB)
  0.000030 seconds
  0.000030 seconds

julia>

Is there any better way to measure them?

nalimilan · 2018-06-03T09:07:57Z

Is there any better way to measure them?

Use @btime from BenchmarkTools.

yeonsookimdev · 2018-06-03T09:24:17Z

Use @btime from BenchmarkTools.

Great package!

0.7.0-alpha:

julia> GC.gc(); io = IOBuffer(); @btime write(io, 'a');
  47.265 ns (0 allocations: 0 bytes)

julia> GC.gc(); io = IOBuffer(); @btime write(io, 'δ');
  63.404 ns (0 allocations: 0 bytes)

julia> GC.gc(); io = IOBuffer(); @btime write(io, '한');
  79.160 ns (0 allocations: 0 bytes)

julia>

alkorang/display:

julia> GC.gc(); io = IOBuffer(); @btime write(io, 'a');
  55.335 ns (0 allocations: 0 bytes)

julia> GC.gc(); io = IOBuffer(); @btime write(io, 'δ');
  54.950 ns (0 allocations: 0 bytes)

julia> GC.gc(); io = IOBuffer(); @btime write(io, '한');
  54.950 ns (0 allocations: 0 bytes)

julia>

JeffBezanson · 2018-06-03T18:59:52Z

base/io.jl

-        n += 1
+    v = reinterpret(UInt32, c)
+    u = hton(v) # BIG-endian
+    p = unsafe_convert(Ptr{Cvoid}, Ref{UInt32}(u))


I believe this Ref needs a GC.@preserve.

Yes, it requires GC.@preserve for Ref.

function f0(c::Char) u = hton(reinterpret(UInt32, c)) p = unsafe_convert(Ptr{Cvoid}, Ref{UInt32}(u)) show(unsafe_wrap(Array{UInt8,1}, Ptr{UInt8}(p), (4,))) end function f1(c::Char) u = hton(reinterpret(UInt32, c)) GC.@preserve u begin p = unsafe_convert(Ptr{Cvoid}, Ref{UInt32}(u)) show(unsafe_wrap(Array{UInt8,1}, Ptr{UInt8}(p), (4,))) end end function f2(c::Char) u = Ref{UInt32}(hton(reinterpret(UInt32, c))) GC.@preserve u begin p = unsafe_convert(Ptr{Cvoid}, u) show(unsafe_wrap(Array{UInt8,1}, Ptr{UInt8}(p), (4,))) end end

UInt8[0xed, 0x95, 0x9c, 0x00] must be printed:

julia> f0('한') UInt8[0x40, 0x27, 0x3e, 0x3c] julia> f0('한') UInt8[0x01, 0x00, 0x00, 0x00] julia> julia> f1('한') UInt8[0x20, 0x1c, 0x3e, 0x3c] julia> f1('한') UInt8[0x01, 0x00, 0x00, 0x00] julia> julia> f2('한') UInt8[0xed, 0x95, 0x9c, 0x00] julia> f2('한') UInt8[0xed, 0x95, 0x9c, 0x00] julia>

Right – if @time isn't showing the extra allocation of the Ref object, this is broken (as you are demonstrating above)

appleparan · 2018-06-04T05:11:16Z

It passed Base.runtests(["strings/io", "char", "iobuffer"]).

I mean for your issue. e.g. '한'. I think you should create tests for this issue.

yeonsookimdev · 2018-06-04T05:38:41Z

I mean for your issue. e.g. '한'. I think you should create tests for this issue.

julia/test/char.jl

Lines 213 to 239 in 5583ae4

    
           @testset "overlong codes" begin 
        
               function test_overlong(c::Char, n::Integer, rep::String) 
        
                   if isvalid(c) 
        
                       @test Int(c) == n 
        
                   else 
        
                       @test_throws Base.InvalidCharError UInt32(c) 
        
                   end 
        
                   @test sprint(show, c) == rep 
        
                   if Base.isoverlong(c) 
        
                       @test occursin(rep*": [overlong]", sprint(show, "text/plain", c)) 
        
                   end 
        
               end 
        
               # TODO: use char syntax once #25072 is fixed 
        
               test_overlong('\0', 0, "'\\0'") 
        
               test_overlong("\xc0\x80"[1], 0, "'\\xc0\\x80'") 
        
               test_overlong("\xe0\x80\x80"[1], 0, "'\\xe0\\x80\\x80'") 
        
               test_overlong("\xf0\x80\x80\x80"[1], 0, "'\\xf0\\x80\\x80\\x80'") 
        
               test_overlong('\x30', 0x30, "'0'") 
        
               test_overlong("\xc0\xb0"[1], 0x30, "'\\xc0\\xb0'") 
        
               test_overlong("\xe0\x80\xb0"[1], 0x30, "'\\xe0\\x80\\xb0'") 
        
               test_overlong("\xf0\x80\x80\xb0"[1], 0x30, "'\\xf0\\x80\\x80\\xb0'") 
        
               test_overlong('\u8430', 0x8430, "'萰'") 
        
               test_overlong("\xf0\x88\x90\xb0"[1], 0x8430, "'\\xf0\\x88\\x90\\xb0'") 
        
           end

Here are some tests for multibyte cases. test_overlong calls sprint(show, c) which eventually calls write(io::IO, c::Char). If write did not work correctly, it cannot pass these tests.

StefanKarpinski · 2018-06-04T19:39:13Z

base/io.jl

-        (u >>= 8) == 0 && return n
-        n += 1
+    v = reinterpret(UInt32, c)
+    u = Ref{UInt32}(hton(v)) # BIG-endian


This should be an unconditional bswap since UTF-8 is independent of the endianess of the system.

UTF-8 is independent of the endianess of the system.

Yes, it is true, but UInt32 isn't.

StefanKarpinski · 2018-06-04T19:44:58Z

base/io.jl

+            unsafe_write(io, p, 2)
+        else
+            unsafe_write(io, p, 1)
+        end


You can replace this with something along the lines of

unsafe_write(io, p, min(4, (trailing_zeros(v) >>> 3) + 1))

StefanKarpinski

Looks good to me. Shame about AppVeyor timing out. Perhaps we should wait until that's sorted out and rebase since testing Windows seems fairly key here.

yeonsookimdev · 2018-06-18T08:16:07Z

Perhaps we should wait until that's sorted out and rebase since testing Windows seems fairly key here.

@StefanKarpinski @JeffBezanson
Some other tests failed on Windows. I first thought it will be okay if I rebase it, but no.

One of them is from cmdlineargs.jl.

julia/test/cmdlineargs.jl

Lines 544 to 545 in b28496a

    
           write(infile, "(1, 2+3)") 
        
           @test read(pipeline(exename, stdin=infile), String) == "(1, 5)\n"

Error message from 64bit AppVeyor

Error in testset cmdlineargs:
Test Failed at C:\projects\julia\julia-df4bb5401f\share\julia\test\cmdlineargs.jl:545
  Expression: read(pipeline(exename, stdin=infile), String) == "(1, 5)\n"
   Evaluated: "\x001, 5)\n" == "(1, 5)\n"

Error message from 32bit AppVeyor

Test Failed at C:\projects\julia\julia-df4bb5401f\share\julia\test\cmdlineargs.jl:545
  Expression: read(pipeline(exename, stdin=infile), String) == "(1, 5)\n"
   Evaluated: "\xcc1, 5)\n" == "(1, 5)\n"
ERROR: LoadError: Test run finished with errors

I guess the problem is garbage collected Ref, because the first broken Char keep changing like \x00 or \xd0 on my computer. But I don't understand, I protect Ref with GC.@preserve.

Those tests are spawning a process using @cmd, spawn, or pipeline and reading/writing data from/to redirected stdin/stdout.

yeonsookimdev · 2018-07-13T07:37:37Z

@vtjnash
I think this pr won't be available until finding out the reason of broken bytes from unsafe_write. Fixing libuv to use buffering looks more general and nicer.

yeonsookimdev · 2018-07-28T04:20:27Z

#28314 Fixes the problem.

nalimilan added the io Involving the I/O subsystem: libuv, read, write, etc. label Jun 3, 2018

JeffBezanson reviewed Jun 3, 2018

View reviewed changes

StefanKarpinski reviewed Jun 4, 2018

View reviewed changes

StefanKarpinski approved these changes Jun 5, 2018

View reviewed changes

StefanKarpinski mentioned this pull request Jul 10, 2018

Printing of some characters seems broken on Windows #28029

Closed

yeonsookimdev mentioned this pull request Jul 25, 2018

fix #27267, print bytes of multi-byte characters at once on windows #28281

Closed

yeonsookimdev added 3 commits July 26, 2018 09:47

write multiple bytes of Char at once

e854888

add GC.@preserve to avoid broken Ref

ec305dd

use trailing_zeros instead of if-else

e6adf65

yeonsookimdev closed this Jul 28, 2018

Uh oh!

write multiple bytes of Char at once #27267 #27380

write multiple bytes of Char at once #27267 #27380

Uh oh!

Conversation

yeonsookimdev commented Jun 2, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

appleparan commented Jun 2, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

yeonsookimdev commented Jun 3, 2018

Uh oh!

strickek commented Jun 3, 2018

Uh oh!

yeonsookimdev commented Jun 3, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

nalimilan commented Jun 3, 2018

Uh oh!

yeonsookimdev commented Jun 3, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

JeffBezanson Jun 3, 2018

Choose a reason for hiding this comment

Uh oh!

yeonsookimdev Jun 4, 2018

Choose a reason for hiding this comment

Uh oh!

vtjnash Jun 4, 2018

Choose a reason for hiding this comment

Uh oh!

appleparan commented Jun 4, 2018

Uh oh!

yeonsookimdev commented Jun 4, 2018

Uh oh!

StefanKarpinski Jun 4, 2018

Choose a reason for hiding this comment

Uh oh!

yeonsookimdev Jun 5, 2018

Choose a reason for hiding this comment

Uh oh!

StefanKarpinski Jun 4, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

yeonsookimdev Jun 5, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

StefanKarpinski left a comment

Choose a reason for hiding this comment

Uh oh!

yeonsookimdev commented Jun 18, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

yeonsookimdev commented Jul 13, 2018

Uh oh!

yeonsookimdev commented Jul 28, 2018

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

7 participants

yeonsookimdev commented Jun 2, 2018 •

edited

Loading

appleparan commented Jun 2, 2018 •

edited

Loading

yeonsookimdev commented Jun 3, 2018 •

edited

Loading

yeonsookimdev commented Jun 3, 2018 •

edited

Loading

StefanKarpinski Jun 4, 2018 •

edited

Loading

yeonsookimdev Jun 5, 2018 •

edited

Loading

yeonsookimdev commented Jun 18, 2018 •

edited

Loading