From 2b3ede98ab1770e7f31bab2a68ddf1157e851c6b Mon Sep 17 00:00:00 2001 From: Paul Guyot Date: Fri, 24 Oct 2025 08:02:27 +0200 Subject: [PATCH] JIT: Optimize memory usage by patching jump table asap Signed-off-by: Paul Guyot --- libs/estdlib/src/code_server.erl | 2 +- libs/jit/src/jit_aarch64.erl | 30 ++- libs/jit/src/jit_armv6m.erl | 56 +++-- libs/jit/src/jit_x86_64.erl | 33 ++- tests/libs/jit/jit_aarch64_tests.erl | 188 ++++++++-------- tests/libs/jit/jit_armv6m_tests.erl | 321 +++++++++++++++++---------- tests/libs/jit/jit_x86_64_tests.erl | 145 ++++++------ 7 files changed, 468 insertions(+), 307 deletions(-) diff --git a/libs/estdlib/src/code_server.erl b/libs/estdlib/src/code_server.erl index eee061efb..cdd7e8b37 100644 --- a/libs/estdlib/src/code_server.erl +++ b/libs/estdlib/src/code_server.erl @@ -152,7 +152,7 @@ set_native_code(_Module, _LabelsCount, _Stream) -> load(Module) -> case erlang:system_info(emu_flavor) of jit -> - % atomvm_heap_growth, fibonacci divides compilation time by two + % atomvm_heap_growth, fibonacci reduces compilation time {Pid, Ref} = spawn_opt( fun() -> try diff --git a/libs/jit/src/jit_aarch64.erl b/libs/jit/src/jit_aarch64.erl index fefa004d6..cb6504c48 100644 --- a/libs/jit/src/jit_aarch64.erl +++ b/libs/jit/src/jit_aarch64.erl @@ -134,6 +134,7 @@ stream :: stream(), offset :: non_neg_integer(), branches :: [{non_neg_integer(), non_neg_integer(), non_neg_integer()}], + jump_table_start :: non_neg_integer(), available_regs :: [aarch64_register()], used_regs :: [aarch64_register()], labels :: [{integer() | reference(), integer()}], @@ -233,6 +234,7 @@ new(Variant, StreamModule, Stream) -> stream_module = StreamModule, stream = Stream, branches = [], + jump_table_start = 0, offset = StreamModule:offset(Stream), available_regs = ?AVAILABLE_REGS, used_regs = [], @@ -355,22 +357,21 @@ assert_all_native_free(#state{ %% @return Updated backend state %%----------------------------------------------------------------------------- -spec jump_table(state(), pos_integer()) -> state(). -jump_table(State, LabelsCount) -> - jump_table0(State, 0, LabelsCount). +jump_table(#state{stream_module = StreamModule, stream = Stream0} = State, LabelsCount) -> + JumpTableStart = StreamModule:offset(Stream0), + jump_table0(State#state{jump_table_start = JumpTableStart}, 0, LabelsCount). -spec jump_table0(state(), non_neg_integer(), pos_integer()) -> state(). jump_table0(State, N, LabelsCount) when N > LabelsCount -> State; jump_table0( - #state{stream_module = StreamModule, stream = Stream0, branches = Branches} = State, + #state{stream_module = StreamModule, stream = Stream0} = State, N, LabelsCount ) -> - Offset = StreamModule:offset(Stream0), BranchInstr = jit_aarch64_asm:b(0), - Reloc = {N, Offset, b}, Stream1 = StreamModule:append(Stream0, BranchInstr), - jump_table0(State#state{stream = Stream1, branches = [Reloc | Branches]}, N + 1, LabelsCount). + jump_table0(State#state{stream = Stream1}, N + 1, LabelsCount). %%----------------------------------------------------------------------------- %% @doc Rewrite stream to update all branches for labels. @@ -2343,5 +2344,22 @@ add_label(#state{stream_module = StreamModule, stream = Stream} = State, Label) %% @return Updated backend state %%----------------------------------------------------------------------------- -spec add_label(state(), integer() | reference(), integer()) -> state(). +add_label( + #state{ + stream_module = StreamModule, + stream = Stream0, + jump_table_start = JumpTableStart, + labels = Labels + } = State, + Label, + LabelOffset +) when is_integer(Label) -> + % Patch the jump table entry immediately + % Each b instruction is 4 bytes + JumpTableEntryOffset = JumpTableStart + Label * 4, + RelativeOffset = LabelOffset - JumpTableEntryOffset, + BranchInstr = jit_aarch64_asm:b(RelativeOffset), + Stream1 = StreamModule:replace(Stream0, JumpTableEntryOffset, BranchInstr), + State#state{stream = Stream1, labels = [{Label, LabelOffset} | Labels]}; add_label(#state{labels = Labels} = State, Label, Offset) -> State#state{labels = [{Label, Offset} | Labels]}. diff --git a/libs/jit/src/jit_armv6m.erl b/libs/jit/src/jit_armv6m.erl index 2bc174f31..602339f0a 100644 --- a/libs/jit/src/jit_armv6m.erl +++ b/libs/jit/src/jit_armv6m.erl @@ -134,6 +134,7 @@ stream :: stream(), offset :: non_neg_integer(), branches :: [{non_neg_integer(), non_neg_integer(), non_neg_integer()}], + jump_table_start :: non_neg_integer(), available_regs :: [armv6m_register()], used_regs :: [armv6m_register()], labels :: [{integer() | reference(), integer()}], @@ -247,6 +248,7 @@ new(Variant, StreamModule, Stream) -> stream_module = StreamModule, stream = Stream, branches = [], + jump_table_start = 0, offset = StreamModule:offset(Stream), available_regs = ?AVAILABLE_REGS, used_regs = [], @@ -380,13 +382,14 @@ assert_all_native_free(#state{ %% @return Updated backend state %%----------------------------------------------------------------------------- -spec jump_table(state(), pos_integer()) -> state(). -jump_table(State, LabelsCount) -> - jump_table0(State, 0, LabelsCount). +jump_table(#state{stream_module = StreamModule, stream = Stream0} = State, LabelsCount) -> + JumpTableStart = StreamModule:offset(Stream0), + jump_table0(State#state{jump_table_start = JumpTableStart}, 0, LabelsCount). jump_table0(State, N, LabelsCount) when N > LabelsCount -> State; jump_table0( - #state{stream_module = StreamModule, stream = Stream0, branches = Branches} = State, + #state{stream_module = StreamModule, stream = Stream0} = State, N, LabelsCount ) -> @@ -399,15 +402,7 @@ jump_table0( JumpEntry = <>, Stream1 = StreamModule:append(Stream0, JumpEntry), - % Add relocation for the data entry so update_branches/2 can patch the jump target - DataOffset = StreamModule:offset(Stream1) - 4, - % Calculate the offset of the add instruction (3rd instruction, at offset 4 from entry start) - EntryStartOffset = StreamModule:offset(Stream1) - 12, - AddInstrOffset = EntryStartOffset + 4, - DataReloc = {N, DataOffset, {jump_table_data, AddInstrOffset}}, - UpdatedState = State#state{stream = Stream1, branches = [DataReloc | Branches]}, - - jump_table0(UpdatedState, N + 1, LabelsCount). + jump_table0(State#state{stream = Stream1}, N + 1, LabelsCount). %%----------------------------------------------------------------------------- %% @doc Rewrite stream to update all branches for labels. @@ -500,13 +495,7 @@ update_branches( I4 = <>, <> end - end; - {jump_table_data, AddInstrOffset} -> - % Calculate offset from 'add pc, pc, r3' instruction + 4 to target label - % PC when add instruction executes - AddPC = AddInstrOffset + 4, - RelativeOffset = LabelOffset - AddPC, - <> + end end, Stream1 = StreamModule:replace(Stream0, Offset, NewInstr), update_branches(State#state{stream = Stream1, branches = BranchesT}). @@ -3288,5 +3277,34 @@ add_label(#state{stream_module = StreamModule, stream = Stream0} = State0, Label %% @return Updated backend state %%----------------------------------------------------------------------------- -spec add_label(state(), integer() | reference(), integer()) -> state(). +add_label( + #state{ + stream_module = StreamModule, + stream = Stream0, + jump_table_start = JumpTableStart, + labels = Labels + } = State, + Label, + LabelOffset +) when is_integer(Label) -> + % Patch the jump table entry immediately + % Each jump table entry is 12 bytes: + % - ldr r3, [pc, 4] (2 bytes) at offset 0 + % - push {...} (2 bytes) at offset 2 + % - add pc, r3 (2 bytes) at offset 4 + % - nop (2 bytes) at offset 6 + % - data (4 bytes) at offset 8 + JumpTableEntryStart = JumpTableStart + Label * 12, + DataOffset = JumpTableEntryStart + 8, + AddInstrOffset = JumpTableEntryStart + 4, + + % Calculate offset from 'add pc, pc, r3' instruction + 4 to target label + % PC when add instruction executes + AddPC = AddInstrOffset + 4, + RelativeOffset = LabelOffset - AddPC, + DataBytes = <>, + + Stream1 = StreamModule:replace(Stream0, DataOffset, DataBytes), + State#state{stream = Stream1, labels = [{Label, LabelOffset} | Labels]}; add_label(#state{labels = Labels} = State, Label, Offset) -> State#state{labels = [{Label, Offset} | Labels]}. diff --git a/libs/jit/src/jit_x86_64.erl b/libs/jit/src/jit_x86_64.erl index 0c722952b..815dc40d9 100644 --- a/libs/jit/src/jit_x86_64.erl +++ b/libs/jit/src/jit_x86_64.erl @@ -115,6 +115,7 @@ stream :: stream(), offset :: non_neg_integer(), branches :: [{non_neg_integer(), non_neg_integer(), non_neg_integer()}], + jump_table_start :: non_neg_integer(), available_regs :: [x86_64_register()], used_regs :: [x86_64_register()], labels :: [{integer() | reference(), integer()}], @@ -218,6 +219,7 @@ new(Variant, StreamModule, Stream) -> stream_module = StreamModule, stream = Stream, branches = [], + jump_table_start = 0, offset = StreamModule:offset(Stream), available_regs = ?AVAILABLE_REGS, used_regs = [], @@ -340,21 +342,21 @@ assert_all_native_free(State) -> %% @return Updated backend state %%----------------------------------------------------------------------------- -spec jump_table(state(), pos_integer()) -> state(). -jump_table(State, LabelsCount) -> - jump_table0(State, 0, LabelsCount). +jump_table(#state{stream_module = StreamModule, stream = Stream0} = State, LabelsCount) -> + JumpTableStart = StreamModule:offset(Stream0), + jump_table0(State#state{jump_table_start = JumpTableStart}, 0, LabelsCount). jump_table0(State, N, LabelsCount) when N > LabelsCount -> State; jump_table0( - #state{stream_module = StreamModule, stream = Stream0, branches = Branches} = State, + #state{stream_module = StreamModule, stream = Stream0} = State, N, LabelsCount ) -> - Offset = StreamModule:offset(Stream0), - {RelocOffset, I1} = jit_x86_64_asm:jmp_rel32(1), - Reloc = {N, Offset + RelocOffset, 32}, + % Placeholder, encodes with 0xffffffff + {_RelocOffset, I1} = jit_x86_64_asm:jmp_rel32(4), Stream1 = StreamModule:append(Stream0, I1), - jump_table0(State#state{stream = Stream1, branches = [Reloc | Branches]}, N + 1, LabelsCount). + jump_table0(State#state{stream = Stream1}, N + 1, LabelsCount). %%----------------------------------------------------------------------------- %% @doc Rewrite stream to update all branches for labels. @@ -2086,5 +2088,22 @@ add_label(#state{stream_module = StreamModule, stream = Stream} = State, Label) add_label(State, Label, Offset). -spec add_label(state(), integer() | reference(), integer()) -> state(). +add_label( + #state{ + stream_module = StreamModule, + stream = Stream0, + jump_table_start = JumpTableStart, + labels = Labels + } = State, + Label, + LabelOffset +) when is_integer(Label) -> + % Patch the jump table entry immediately + % Each jmp_rel32 instruction is 5 bytes + JumpTableEntryOffset = JumpTableStart + Label * 5, + RelativeOffset = LabelOffset - JumpTableEntryOffset, + {_RelocOffset, JmpInstruction} = jit_x86_64_asm:jmp_rel32(RelativeOffset), + Stream1 = StreamModule:replace(Stream0, JumpTableEntryOffset, JmpInstruction), + State#state{stream = Stream1, labels = [{Label, LabelOffset} | Labels]}; add_label(#state{labels = Labels} = State, Label, Offset) -> State#state{labels = [{Label, Offset} | Labels]}. diff --git a/tests/libs/jit/jit_aarch64_tests.erl b/tests/libs/jit/jit_aarch64_tests.erl index e5887439e..9fdc65f40 100644 --- a/tests/libs/jit/jit_aarch64_tests.erl +++ b/tests/libs/jit/jit_aarch64_tests.erl @@ -1012,11 +1012,12 @@ get_list_test() -> is_integer_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, Arg1 = {x_reg, 0}, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, Arg1), - State2 = ?BACKEND:if_block( - State1, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(MSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, Arg1), + State3 = ?BACKEND:if_block( + State2, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(MSt0) -> MSt1 = ?BACKEND:if_block( MSt0, {Reg, '&', ?TERM_PRIMARY_MASK, '!=', ?TERM_PRIMARY_BOXED}, fun(BSt0) -> ?BACKEND:jump_to_label(BSt0, Label) @@ -1039,28 +1040,30 @@ is_integer_test() -> ) end ), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: f9401807 ldr x7, [x0, #48]\n" - " 4: 92400ce8 and x8, x7, #0xf\n" - " 8: f1003d1f cmp x8, #0xf\n" - " c: 54000180 b.eq 0x3c // b.none\n" - " 10: 924004e8 and x8, x7, #0x3\n" - " 14: f100091f cmp x8, #0x2\n" - " 18: 54000040 b.eq 0x20 // b.none\n" - " 1c: 14000048 b 0x13c\n" - " 20: 927ef4e7 and x7, x7, #0xfffffffffffffffc\n" - " 24: f94000e7 ldr x7, [x7]\n" - " 28: d2800768 mov x8, #0x3b\n" - " 2c: 8a0800e7 and x7, x7, x8\n" - " 30: f10020ff cmp x7, #0x8\n" - " 34: 54000040 b.eq 0x3c // b.none\n" - " 38: 14000041 b 0x13c\n" + " 0: 14000000 b 0x0\n" + " 4: 14000050 b 0x144\n" + " 8: f9401807 ldr x7, [x0, #48]\n" + " c: 92400ce8 and x8, x7, #0xf\n" + " 10: f1003d1f cmp x8, #0xf\n" + " 14: 54000180 b.eq 0x44 // b.none\n" + " 18: 924004e8 and x8, x7, #0x3\n" + " 1c: f100091f cmp x8, #0x2\n" + " 20: 54000040 b.eq 0x28 // b.none\n" + " 24: 14000048 b 0x144\n" + " 28: 927ef4e7 and x7, x7, #0xfffffffffffffffc\n" + " 2c: f94000e7 ldr x7, [x7]\n" + " 30: d2800768 mov x8, #0x3b // #59\n" + " 34: 8a0800e7 and x7, x7, x8\n" + " 38: f10020ff cmp x7, #0x8\n" + " 3c: 54000040 b.eq 0x44 // b.none\n" + " 40: 14000041 b 0x144" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1071,11 +1074,12 @@ cond_jump_to_label(Cond, Label, MMod, MSt0) -> is_number_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, Arg1 = {x_reg, 0}, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, Arg1), - State2 = ?BACKEND:if_block( - State1, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, Arg1), + State3 = ?BACKEND:if_block( + State2, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(BSt0) -> BSt1 = cond_jump_to_label( {Reg, '&', ?TERM_PRIMARY_MASK, '!=', ?TERM_PRIMARY_BOXED}, Label, ?BACKEND, BSt0 ), @@ -1092,57 +1096,61 @@ is_number_test() -> ) end ), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: f9401807 ldr x7, [x0, #48]\n" - " 4: 92400ce8 and x8, x7, #0xf\n" - " 8: f1003d1f cmp x8, #0xf\n" - " c: 540001e0 b.eq 0x48 // b.none\n" - " 10: 924004e8 and x8, x7, #0x3\n" - " 14: f100091f cmp x8, #0x2\n" - " 18: 54000040 b.eq 0x20 // b.none\n" - " 1c: 1400004b b 0x148\n" - " 20: 927ef4e7 and x7, x7, #0xfffffffffffffffc\n" - " 24: f94000e7 ldr x7, [x7]\n" - " 28: d2800768 mov x8, #0x3b\n" - " 2c: 8a0800e8 and x8, x7, x8\n" - " 30: f100211f cmp x8, #0x8\n" - " 34: 540000a0 b.eq 0x48 // b.none\n" - " 38: 924014e7 and x7, x7, #0x3f\n" - " 3c: f10060ff cmp x7, #0x18\n" - " 40: 54000040 b.eq 0x48 // b.none\n" - " 44: 14000041 b 0x148\n" + " 0: 14000000 b 0x0\n" + " 4: 14000053 b 0x150\n" + " 8: f9401807 ldr x7, [x0, #48]\n" + " c: 92400ce8 and x8, x7, #0xf\n" + " 10: f1003d1f cmp x8, #0xf\n" + " 14: 540001e0 b.eq 0x50 // b.none\n" + " 18: 924004e8 and x8, x7, #0x3\n" + " 1c: f100091f cmp x8, #0x2\n" + " 20: 54000040 b.eq 0x28 // b.none\n" + " 24: 1400004b b 0x150\n" + " 28: 927ef4e7 and x7, x7, #0xfffffffffffffffc\n" + " 2c: f94000e7 ldr x7, [x7]\n" + " 30: d2800768 mov x8, #0x3b // #59\n" + " 34: 8a0800e8 and x8, x7, x8\n" + " 38: f100211f cmp x8, #0x8\n" + " 3c: 540000a0 b.eq 0x50 // b.none\n" + " 40: 924014e7 and x7, x7, #0x3f\n" + " 44: f10060ff cmp x7, #0x18\n" + " 48: 54000040 b.eq 0x50 // b.none\n" + " 4c: 14000041 b 0x150" >>, ?assertEqual(dump_to_bin(Dump), Stream). is_boolean_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, {x_reg, 0}), - State2 = ?BACKEND:if_block(State1, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), + State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), - Offset = ?BACKEND:offset(State3), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: f9401807 ldr x7, [x0, #48]\n" - " 4: f1012cff cmp x7, #0x4b\n" - " 8: 54000080 b.eq 0x18 // b.none\n" - " c: f1002cff cmp x7, #0xb\n" - " 10: 54000040 b.eq 0x18 // b.none\n" - " 14: 14000041 b 0x118" + " 0: 14000000 b 0x0\n" + " 4: 14000047 b 0x120\n" + " 8: f9401807 ldr x7, [x0, #48]\n" + " c: f1012cff cmp x7, #0x4b\n" + " 10: 54000080 b.eq 0x20\n" + " 14: f1002cff cmp x7, #0xb\n" + " 18: 54000040 b.eq 0x20\n" + " 1c: 14000041 b 0x120" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1219,7 +1227,7 @@ wait_test() -> Stream = ?BACKEND:stream(State4), Dump = << " 0: 14000000 b 0x0\n" - " 4: 14000000 b 0x4\n" + " 4: 14000005 b 0x18\n" " 8: 14000000 b 0x8\n" " c: 14000000 b 0xc\n" " 10: 14000000 b 0x10\n" @@ -1233,34 +1241,34 @@ wait_test() -> return_labels_and_lines_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 2), % Test return_labels_and_lines with some sample labels and lines - State1 = ?BACKEND:add_label(State0, 2, 32), State2 = ?BACKEND:add_label(State1, 1, 16), - % {Line, Offset} pairs SortedLines = [{10, 16}, {20, 32}], - State3 = ?BACKEND:return_labels_and_lines(State2, SortedLines), - Stream = ?BACKEND:stream(State3), + State3 = ?BACKEND:add_label(State2, 0), - % Should have generated adr + ret + labels table + lines table - % adr = 4 bytes, ret = 4 bytes, labels table = 6*2 = 12 bytes, lines table = 6*2 = 12 bytes - % Total minimum: 36 bytes - ?assert(byte_size(Stream) >= 36), + State4 = ?BACKEND:return_labels_and_lines(State3, SortedLines), + State5 = ?BACKEND:update_branches(State4), + Stream = ?BACKEND:stream(State5), + + ?assert(byte_size(Stream) >= 44), - % Expected: adr x0, #8 + ret + labels table + lines table - % The data tables start at offset 0x8, so we load PC + 8 into x0 Dump = << - " 0: 10000040 adr x0, 0x8\n" - " 4: d65f03c0 ret\n" - " 8: 01000200 .word 0x01000200\n" - " c: 10000000 adr x0, 0xc\n" - " 10: 00000200 .word 0x00000200\n" - " 14: 02002000 .word 0x02002000\n" - " 18: 00000a00 .word 0x00000a00\n" - " 1c: 14001000 .word 0x14001000\n" - " 20: 20000000 .word 0x20000000" + " 0: 14000003 b 0xc\n" + " 4: 14000003 b 0x10\n" + " 8: 14000000 b 0x8\n" + " c: 10000040 adr x0, 0x14\n" + " 10: d65f03c0 ret\n" + " 14: 00000200 .inst 0x00000200\n" + " 18: 0c000000 st4 {v0.8b-v3.8b}, [x0]\n" + " 1c: 00000100 .inst 0x00000100\n" + " 20: 02001000 .inst 0x02001000\n" + " 24: 00000a00 .inst 0x00000a00\n" + " 28: 14001000 b 0x4028\n" + " 2c: 20000000 .inst 0x20000000" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1817,8 +1825,8 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: x_reg to reg[x+offset] ?_test(begin - State1 = setelement(6, State0, ?BACKEND:available_regs(State0) -- [r8, r9]), - State2 = setelement(7, State1, [r8, r9]), + State1 = setelement(7, State0, ?BACKEND:available_regs(State0) -- [r8, r9]), + State2 = setelement(8, State1, [r8, r9]), [r8, r9] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, r8, r9, 1), Stream = ?BACKEND:stream(State3), @@ -1831,8 +1839,8 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: imm to reg[x+offset] ?_test(begin - State1 = setelement(6, State0, ?BACKEND:available_regs(State0) -- [r8, r9]), - State2 = setelement(7, State1, [r8, r9]), + State1 = setelement(7, State0, ?BACKEND:available_regs(State0) -- [r8, r9]), + State2 = setelement(8, State1, [r8, r9]), [r8, r9] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, 42, r8, r9, 1), Stream = ?BACKEND:stream(State3), diff --git a/tests/libs/jit/jit_armv6m_tests.erl b/tests/libs/jit/jit_armv6m_tests.erl index ab791e91b..9c7d3632d 100644 --- a/tests/libs/jit/jit_armv6m_tests.erl +++ b/tests/libs/jit/jit_armv6m_tests.erl @@ -1829,57 +1829,83 @@ is_number_test() -> is_boolean_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, {x_reg, 0}), - State2 = ?BACKEND:if_block(State1, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), + State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - State4 = ?BACKEND:add_label(State3, Label, 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + State5 = ?BACKEND:add_label(State4, Label, 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 6987 ldr r7, [r0, #24]\n" - " 2: 2f4b cmp r7, #75 ; 0x4b\n" - " 4: d006 beq.n 0x14\n" - " 6: 2f0b cmp r7, #11\n" - " 8: d004 beq.n 0x14\n" - " a: e079 b.n 0x100\n" - " c: 46c0 nop ; (mov r8, r8)\n" - " e: 46c0 nop ; (mov r8, r8)\n" - " 10: 46c0 nop ; (mov r8, r8)\n" - " 12: 46c0 nop ; (mov r8, r8)" + " 0: 4b01 ldr r3, [pc, #4]\n" + " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 4: 449f add pc, r3\n" + " 6: 46c0 nop\n" + " 8: ffff .short 0xffff\n" + " a: ffff .short 0xffff\n" + " c: 4b01 ldr r3, [pc, #4]\n" + " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 10: 449f add pc, r3\n" + " 12: 46c0 nop\n" + " 14: 00ec lsls r4, r5, #3\n" + " 16: 0000 movs r0, r0\n" + " 18: 6987 ldr r7, [r0, #24]\n" + " 1a: 2f4b cmp r7, #75\n" + " 1c: d006 beq.n 0x2c\n" + " 1e: 2f0b cmp r7, #11\n" + " 20: d004 beq.n 0x2c\n" + " 22: e06d b.n 0x100\n" + " 24: 46c0 nop\n" + " 26: 46c0 nop\n" + " 28: 46c0 nop\n" + " 2a: 46c0 nop" >>, ?assertEqual(dump_to_bin(Dump), Stream). is_boolean_far_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, {x_reg, 0}), - State2 = ?BACKEND:if_block(State1, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), + State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - State4 = ?BACKEND:add_label(State3, Label, 16#1000), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + State5 = ?BACKEND:add_label(State4, Label, 16#1000), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 6987 ldr r7, [r0, #24]\n" - " 2: 2f4b cmp r7, #75 ; 0x4b\n" - " 4: d006 beq.n 0x14\n" - " 6: 2f0b cmp r7, #11\n" - " 8: d004 beq.n 0x14\n" - " a: 4e01 ldr r6, [pc, #4] ; (0x10)\n" - " c: 447e add r6, pc\n" - " e: 4730 bx r6\n" - " 10: 0ff1 lsrs r0, r6, #31\n" - " 12: 0000 movs r0, r0" + " 0: 4b01 ldr r3, [pc, #4]\n" + " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 4: 449f add pc, r3\n" + " 6: 46c0 nop\n" + " 8: ffff .short 0xffff\n" + " a: ffff .short 0xffff\n" + " c: 4b01 ldr r3, [pc, #4]\n" + " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 10: 449f add pc, r3\n" + " 12: 46c0 nop\n" + " 14: 0fec lsrs r4, r5, #31\n" + " 16: 0000 movs r0, r0\n" + " 18: 6987 ldr r7, [r0, #24]\n" + " 1a: 2f4b cmp r7, #75\n" + " 1c: d006 beq.n 0x2c\n" + " 1e: 2f0b cmp r7, #11\n" + " 20: d004 beq.n 0x2c\n" + " 22: 4e01 ldr r6, [pc, #4]\n" + " 24: 447e add r6, pc\n" + " 26: 4730 bx r6\n" + " 28: 0fd9 lsrs r1, r3, #31\n" + " 2a: 0000 movs r0, r0" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1921,29 +1947,42 @@ is_boolean_far_unaligned_test() -> is_boolean_far_known_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - State1 = ?BACKEND:add_label(State0, Label, 16#1000), - {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), - State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + State2 = ?BACKEND:add_label(State1, Label, 16#1000), + {State3, Reg} = ?BACKEND:move_to_native_register(State2, {x_reg, 0}), + State4 = ?BACKEND:if_block(State3, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State4 = ?BACKEND:free_native_registers(State3, [Reg]), - ?BACKEND:assert_all_native_free(State4), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State5 = ?BACKEND:free_native_registers(State4, [Reg]), + ?BACKEND:assert_all_native_free(State5), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 6987 ldr r7, [r0, #24]\n" - " 2: 2f4b cmp r7, #75 ; 0x4b\n" - " 4: d006 beq.n 0x14\n" - " 6: 2f0b cmp r7, #11\n" - " 8: d004 beq.n 0x14\n" - " a: 4e01 ldr r6, [pc, #4] ; (0x10)\n" - " c: 447e add r6, pc\n" - " e: 4730 bx r6\n" - " 10: 0ff1 lsrs r1, r6, #31\n" - " 12: 0000 movs r0, r0" + " 0: 4b01 ldr r3, [pc, #4]\n" + " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 4: 449f add pc, r3\n" + " 6: 46c0 nop\n" + " 8: ffff .short 0xffff\n" + " a: ffff .short 0xffff\n" + " c: 4b01 ldr r3, [pc, #4]\n" + " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 10: 449f add pc, r3\n" + " 12: 46c0 nop\n" + " 14: 0fec lsrs r4, r5, #31\n" + " 16: 0000 movs r0, r0\n" + " 18: 6987 ldr r7, [r0, #24]\n" + " 1a: 2f4b cmp r7, #75\n" + " 1c: d006 beq.n 0x2c\n" + " 1e: 2f0b cmp r7, #11\n" + " 20: d004 beq.n 0x2c\n" + " 22: 4e01 ldr r6, [pc, #4]\n" + " 24: 447e add r6, pc\n" + " 26: 4730 bx r6\n" + " 28: 0fd9 lsrs r1, r3, #31\n" + " 2a: 0000 movs r0, r0" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1954,32 +1993,45 @@ is_boolean_far_known_unaligned_test() -> TempState = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), TempStream = jit_stream_binary:append(?BACKEND:stream(TempState), PaddingInstruction), State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, TempStream), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - State1 = ?BACKEND:add_label(State0, Label, 16#1000), - {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), - State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + State2 = ?BACKEND:add_label(State1, Label, 16#1000), + {State3, Reg} = ?BACKEND:move_to_native_register(State2, {x_reg, 0}), + State4 = ?BACKEND:if_block(State3, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State4 = ?BACKEND:free_native_registers(State3, [Reg]), - ?BACKEND:assert_all_native_free(State4), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State5 = ?BACKEND:free_native_registers(State4, [Reg]), + ?BACKEND:assert_all_native_free(State5), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << " 0: 4770 bx lr\n" - " 2: 6987 ldr r7, [r0, #24]\n" - " 4: 2f4b cmp r7, #75 ; 0x4b\n" - " 6: d007 beq.n 0x18\n" - " 8: 2f0b cmp r7, #11\n" - " a: d005 beq.n 0x18\n" - " c: 4e01 ldr r6, [pc, #4] ; (0x14)\n" - " e: 447e add r6, pc\n" - " 10: 4730 bx r6\n" - " 12: 46c0 nop ; (mov r8, r8)\n" - " 14: 0fef lsrs r7, r5, #31\n" - " 16: 0000 movs r0, r0" + " 2: 4b01 ldr r3, [pc, #4]\n" + " 4: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 6: 449f add pc, r3\n" + " 8: 46c0 nop\n" + " a: ffff .short 0xffff\n" + " c: ffff .short 0xffff\n" + " e: 4b01 ldr r3, [pc, #4]\n" + " 10: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 12: 449f add pc, r3\n" + " 14: 46c0 nop\n" + " 16: 0fea lsrs r2, r5, #31\n" + " 18: 0000 movs r0, r0\n" + " 1a: 6987 ldr r7, [r0, #24]\n" + " 1c: 2f4b cmp r7, #75\n" + " 1e: d007 beq.n 0x30\n" + " 20: 2f0b cmp r7, #11\n" + " 22: d005 beq.n 0x30\n" + " 24: 4e01 ldr r6, [pc, #4]\n" + " 26: 447e add r6, pc\n" + " 28: 4730 bx r6\n" + " 2a: 46c0 nop\n" + " 2c: 0fd7 lsrs r7, r2, #31\n" + " 2e: 0000 movs r0, r0" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -2104,41 +2156,59 @@ wait_test() -> %% Test return_labels_and_lines/2 function return_labels_and_lines_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 2), % Test return_labels_and_lines with some sample labels and lines - State1 = ?BACKEND:add_label(State0, 2, 32), - State2 = ?BACKEND:add_label(State1, 1, 16), + State2 = ?BACKEND:add_label(State1, 2, 32), + State3 = ?BACKEND:add_label(State2, 1, 16), % {Line, Offset} pairs SortedLines = [{10, 16}, {20, 32}], - State3 = ?BACKEND:return_labels_and_lines(State2, SortedLines), - Stream = ?BACKEND:stream(State3), + State4 = ?BACKEND:return_labels_and_lines(State3, SortedLines), + Stream = ?BACKEND:stream(State4), % Should have generated adr + pop {r1,r4,r5,r6,r7,pc} + labels table + lines table % adr = 4 bytes, pop = 2 bytes, labels table = 6*2 = 12 bytes, lines table = 6*2 = 12 bytes % Total minimum: 30 bytes ?assert(byte_size(Stream) >= 30), - % Expected: adr r0, + pop {r1,r4,r5,r6,r7,pc} + labels table + lines table - % The data tables start at offset 4, so adr should be adr r0, 4 not adr r0, 8 + % Expected: jump table (3 entries) + adr r0, + pop {r1,r4,r5,r6,r7,pc} + labels table + lines table Dump = << - " 0: a000 add r0, pc, #0 ; (adr r0, 0x4)\n" - " 2: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - " 4: 0200 lsls r0, r0, #8\n" - " 6: 0100 lsls r0, r0, #4\n" - " 8: 0000 movs r0, r0\n" - " a: 1000 asrs r0, r0, #32\n" - " c: 0200 lsls r0, r0, #8\n" - " e: 0000 movs r0, r0\n" - " 10: 2000 movs r0, #0\n" - " 12: 0200 lsls r0, r0, #8\n" - " 14: 0a00 lsrs r0, r0, #8\n" - " 16: 0000 movs r0, r0\n" - " 18: 1000 asrs r0, r0, #32\n" - " 1a: 1400 asrs r0, r0, #16\n" - " 1c: 0000 movs r0, r0\n" - " 1e: 2000 movs r0, #0" + " 0: 4b01 ldr r3, [pc, #4]\n" + " 2: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 4: 449f add pc, r3\n" + " 6: 46c0 nop\n" + " 8: ffff .short 0xffff\n" + " a: ffff .short 0xffff\n" + " c: 4b01 ldr r3, [pc, #4]\n" + " e: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 10: 449f add pc, r3\n" + " 12: 46c0 nop\n" + " 14: fffc .short 0xfffc\n" + " 16: ffff .short 0xffff\n" + " 18: 4b01 ldr r3, [pc, #4]\n" + " 1a: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 1c: 449f add pc, r3\n" + " 1e: 46c0 nop\n" + " 20: 0000 movs r0, r0\n" + " 22: 0000 movs r0, r0\n" + " 24: a000 add r0, pc, #0\n" + " 26: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 28: 0200 lsls r0, r0, #8\n" + " 2a: 0100 lsls r0, r0, #4\n" + " 2c: 0000 movs r0, r0\n" + " 2e: 1000 asrs r0, r0, #32\n" + " 30: 0200 lsls r0, r0, #8\n" + " 32: 0000 movs r0, r0\n" + " 34: 2000 movs r0, #0\n" + " 36: 0200 lsls r0, r0, #8\n" + " 38: 0a00 lsrs r0, r0, #8\n" + " 3a: 0000 movs r0, r0\n" + " 3c: 1000 asrs r0, r0, #32\n" + " 3e: 1400 asrs r0, r0, #16\n" + " 40: 0000 movs r0, r0\n" + " 42: 2000 movs r0, #0" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -2150,36 +2220,55 @@ return_labels_and_lines_unaligned_test() -> TempState = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), TempStream = jit_stream_binary:append(?BACKEND:stream(TempState), PaddingInstruction), State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, TempStream), + State1 = ?BACKEND:jump_table(State0, 2), % Test return_labels_and_lines with some sample labels and lines - State1 = ?BACKEND:add_label(State0, 2, 32), - State2 = ?BACKEND:add_label(State1, 1, 16), + State2 = ?BACKEND:add_label(State1, 2, 32), + State3 = ?BACKEND:add_label(State2, 1, 16), % {Line, Offset} pairs SortedLines = [{10, 16}, {20, 32}], - State3 = ?BACKEND:return_labels_and_lines(State2, SortedLines), - Stream = ?BACKEND:stream(State3), + State4 = ?BACKEND:return_labels_and_lines(State3, SortedLines), + Stream = ?BACKEND:stream(State4), Dump = << " 0: 4770 bx lr\n" - "2: a001 add r0, pc, #4 ; (adr r0, 0x8)\n" - "4: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" - "6: 0000 movs r0, r0\n" - "8: 0200 lsls r0, r0, #8\n" - "a: 0100 lsls r0, r0, #4\n" - "c: 0000 movs r0, r0\n" - "e: 1000 asrs r0, r0, #32\n" - "10: 0200 lsls r0, r0, #8\n" - "12: 0000 movs r0, r0\n" - "14: 2000 movs r0, #0\n" - "16: 0200 lsls r0, r0, #8\n" - "18: 0a00 lsrs r0, r0, #8\n" - "1a: 0000 movs r0, r0\n" - "1c: 1000 asrs r0, r0, #32\n" - "1e: 1400 asrs r0, r0, #16\n" - "20: 0000 movs r0, r0\n" - "22: 2000 movs r0, #0" + " 2: 4b01 ldr r3, [pc, #4]\n" + " 4: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 6: 449f add pc, r3\n" + " 8: 46c0 nop\n" + " a: ffff .short 0xffff\n" + " c: ffff .short 0xffff\n" + " e: 4b01 ldr r3, [pc, #4]\n" + " 10: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 12: 449f add pc, r3\n" + " 14: 46c0 nop\n" + " 16: fffa .short 0xfffa\n" + " 18: ffff .short 0xffff\n" + " 1a: 4b01 ldr r3, [pc, #4]\n" + " 1c: b5f2 push {r1, r4, r5, r6, r7, lr}\n" + " 1e: 449f add pc, r3\n" + " 20: 46c0 nop\n" + " 22: fffe .short 0xfffe\n" + " 24: ffff .short 0xffff\n" + " 26: a001 add r0, pc, #4\n" + " 28: bdf2 pop {r1, r4, r5, r6, r7, pc}\n" + " 2a: 0000 movs r0, r0\n" + " 2c: 0200 lsls r0, r0, #8\n" + " 2e: 0100 lsls r0, r0, #4\n" + " 30: 0000 movs r0, r0\n" + " 32: 1000 asrs r0, r0, #32\n" + " 34: 0200 lsls r0, r0, #8\n" + " 36: 0000 movs r0, r0\n" + " 38: 2000 movs r0, #0\n" + " 3a: 0200 lsls r0, r0, #8\n" + " 3c: 0a00 lsrs r0, r0, #8\n" + " 3e: 0000 movs r0, r0\n" + " 40: 1000 asrs r0, r0, #32\n" + " 42: 1400 asrs r0, r0, #16\n" + " 44: 0000 movs r0, r0\n" + " 46: 2000 movs r0, #0" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -2853,8 +2942,8 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: x_reg to reg[x+offset] ?_test(begin - State1 = setelement(6, State0, ?BACKEND:available_regs(State0) -- [r3, r4]), - State2 = setelement(7, State1, [r3, r4]), + State1 = setelement(7, State0, ?BACKEND:available_regs(State0) -- [r3, r4]), + State2 = setelement(8, State1, [r3, r4]), [r3, r4] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, {x_reg, 0}, r3, r4, 1), Stream = ?BACKEND:stream(State3), @@ -2868,8 +2957,8 @@ move_to_array_element_test_() -> end), %% move_to_array_element/5: imm to reg[x+offset] ?_test(begin - State1 = setelement(6, State0, ?BACKEND:available_regs(State0) -- [r3, r4]), - State2 = setelement(7, State1, [r3, r4]), + State1 = setelement(7, State0, ?BACKEND:available_regs(State0) -- [r3, r4]), + State2 = setelement(8, State1, [r3, r4]), [r3, r4] = ?BACKEND:used_regs(State2), State3 = ?BACKEND:move_to_array_element(State2, 42, r3, r4, 1), Stream = ?BACKEND:stream(State3), diff --git a/tests/libs/jit/jit_x86_64_tests.erl b/tests/libs/jit/jit_x86_64_tests.erl index fb751e659..c4a94678e 100644 --- a/tests/libs/jit/jit_x86_64_tests.erl +++ b/tests/libs/jit/jit_x86_64_tests.erl @@ -977,11 +977,12 @@ get_list_test() -> is_integer_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, Arg1 = {x_reg, 0}, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, Arg1), - State2 = ?BACKEND:if_block( - State1, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(MSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, Arg1), + State3 = ?BACKEND:if_block( + State2, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(MSt0) -> MSt1 = ?BACKEND:if_block( MSt0, {Reg, '&', ?TERM_PRIMARY_MASK, '!=', ?TERM_PRIMARY_BOXED}, fun(BSt0) -> ?BACKEND:jump_to_label(BSt0, Label) @@ -1004,29 +1005,31 @@ is_integer_test() -> ) end ), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 48 8b 47 30 mov 0x30(%rdi),%rax\n" - " 4: 49 89 c3 mov %rax,%r11\n" - " 7: 41 80 e3 0f and $0xf,%r11b\n" - " b: 41 80 fb 0f cmp $0xf,%r11b\n" - " f: 74 25 je 0x36\n" - " 11: 49 89 c3 mov %rax,%r11\n" - " 14: 41 80 e3 03 and $0x3,%r11b\n" - " 18: 41 80 fb 02 cmp $0x2,%r11b\n" - " 1c: 74 05 je 0x23\n" - " 1e: e9 13 01 00 00 jmpq 0x136\n" - " 23: 48 83 e0 fc and $0xfffffffffffffffc,%rax\n" - " 27: 48 8b 00 mov (%rax),%rax\n" - " 2a: 24 3b and $0x3b,%al\n" - " 2c: 80 f8 08 cmp $0x8,%al\n" - " 2f: 74 05 je 0x36\n" - " 31: e9 00 01 00 00 jmpq 0x136" + " 0: e9 ff ff ff ff jmpq 0x4\n" + " 5: e9 36 01 00 00 jmpq 0x140\n" + " a: 48 8b 47 30 mov 0x30(%rdi),%rax\n" + " e: 49 89 c3 mov %rax,%r11\n" + " 11: 41 80 e3 0f and $0xf,%r11b\n" + " 15: 41 80 fb 0f cmp $0xf,%r11b\n" + " 19: 74 25 je 0x40\n" + " 1b: 49 89 c3 mov %rax,%r11\n" + " 1e: 41 80 e3 03 and $0x3,%r11b\n" + " 22: 41 80 fb 02 cmp $0x2,%r11b\n" + " 26: 74 05 je 0x2d\n" + " 28: e9 13 01 00 00 jmpq 0x140\n" + " 2d: 48 83 e0 fc and $0xfffffffffffffffc,%rax\n" + " 31: 48 8b 00 mov (%rax),%rax\n" + " 34: 24 3b and $0x3b,%al\n" + " 36: 80 f8 08 cmp $0x8,%al\n" + " 39: 74 05 je 0x40\n" + " 3b: e9 00 01 00 00 jmpq 0x140" >>, ?assertEqual(dump_to_bin(Dump), Stream). @@ -1037,11 +1040,12 @@ cond_jump_to_label(Cond, Label, MMod, MSt0) -> is_number_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, Arg1 = {x_reg, 0}, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, Arg1), - State2 = ?BACKEND:if_block( - State1, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, Arg1), + State3 = ?BACKEND:if_block( + State2, {Reg, '&', ?TERM_IMMED_TAG_MASK, '!=', ?TERM_INTEGER_TAG}, fun(BSt0) -> BSt1 = cond_jump_to_label( {Reg, '&', ?TERM_PRIMARY_MASK, '!=', ?TERM_PRIMARY_BOXED}, Label, ?BACKEND, BSt0 ), @@ -1058,58 +1062,63 @@ is_number_test() -> ) end ), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 48 8b 47 30 mov 0x30(%rdi),%rax\n" - " 4: 49 89 c3 mov %rax,%r11\n" - " 7: 41 80 e3 0f and $0xf,%r11b\n" - " b: 41 80 fb 0f cmp $0xf,%r11b\n" - " f: 74 32 je 0x43\n" - " 11: 49 89 c3 mov %rax,%r11\n" - " 14: 41 80 e3 03 and $0x3,%r11b\n" - " 18: 41 80 fb 02 cmp $0x2,%r11b\n" - " 1c: 74 05 je 0x23\n" - " 1e: e9 20 01 00 00 jmpq 0x143\n" - " 23: 48 83 e0 fc and $0xfffffffffffffffc,%rax\n" - " 27: 48 8b 00 mov (%rax),%rax\n" - " 2a: 49 89 c3 mov %rax,%r11\n" - " 2d: 41 80 e3 3b and $0x3b,%r11b\n" - " 31: 41 80 fb 08 cmp $0x8,%r11b\n" - " 35: 74 0c je 0x43\n" - " 37: 24 3f and $0x3f,%al\n" - " 39: 80 f8 18 cmp $0x18,%al\n" - " 3c: 74 05 je 0x43\n" - " 3e: e9 00 01 00 00 jmpq 0x143" + " 0: e9 ff ff ff ff jmpq 0x4\n" + " 5: e9 43 01 00 00 jmpq 0x14d\n" + " a: 48 8b 47 30 mov 0x30(%rdi),%rax\n" + " e: 49 89 c3 mov %rax,%r11\n" + " 11: 41 80 e3 0f and $0xf,%r11b\n" + " 15: 41 80 fb 0f cmp $0xf,%r11b\n" + " 19: 74 32 je 0x4d\n" + " 1b: 49 89 c3 mov %rax,%r11\n" + " 1e: 41 80 e3 03 and $0x3,%r11b\n" + " 22: 41 80 fb 02 cmp $0x2,%r11b\n" + " 26: 74 05 je 0x2d\n" + " 28: e9 20 01 00 00 jmpq 0x14d\n" + " 2d: 48 83 e0 fc and $0xfffffffffffffffc,%rax\n" + " 31: 48 8b 00 mov (%rax),%rax\n" + " 34: 49 89 c3 mov %rax,%r11\n" + " 37: 41 80 e3 3b and $0x3b,%r11b\n" + " 3b: 41 80 fb 08 cmp $0x8,%r11b\n" + " 3f: 74 0c je 0x4d\n" + " 41: 24 3f and $0x3f,%al\n" + " 43: 80 f8 18 cmp $0x18,%al\n" + " 46: 74 05 je 0x4d\n" + " 48: e9 00 01 00 00 jmpq 0x14d" >>, ?assertEqual(dump_to_bin(Dump), Stream). is_boolean_test() -> State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)), + State1 = ?BACKEND:jump_table(State0, 1), Label = 1, - {State1, Reg} = ?BACKEND:move_to_native_register(State0, {x_reg, 0}), - State2 = ?BACKEND:if_block(State1, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> + {State2, Reg} = ?BACKEND:move_to_native_register(State1, {x_reg, 0}), + State3 = ?BACKEND:if_block(State2, {Reg, '!=', ?TRUE_ATOM}, fun(BSt0) -> ?BACKEND:if_block(BSt0, {Reg, '!=', ?FALSE_ATOM}, fun(BSt1) -> ?BACKEND:jump_to_label(BSt1, Label) end) end), - State3 = ?BACKEND:free_native_registers(State2, [Reg]), - ?BACKEND:assert_all_native_free(State3), - Offset = ?BACKEND:offset(State3), - State4 = ?BACKEND:add_label(State3, Label, Offset + 16#100), - State5 = ?BACKEND:update_branches(State4), - Stream = ?BACKEND:stream(State5), + State4 = ?BACKEND:free_native_registers(State3, [Reg]), + ?BACKEND:assert_all_native_free(State4), + Offset = ?BACKEND:offset(State4), + State5 = ?BACKEND:add_label(State4, Label, Offset + 16#100), + State6 = ?BACKEND:update_branches(State5), + Stream = ?BACKEND:stream(State6), Dump = << - " 0: 48 8b 47 30 mov 0x30(%rdi),%rax\n" - " 4: 48 83 f8 4b cmp $0x4b,%rax\n" - " 8: 74 0b je 0x15\n" - " a: 48 83 f8 0b cmp $0xb,%rax\n" - " e: 74 05 je 0x15\n" - " 10: e9 00 01 00 00 jmpq 0x115\n" + " 0: e9 ff ff ff ff jmpq 0x4\n" + " 5: e9 15 01 00 00 jmpq 0x11f\n" + " a: 48 8b 47 30 mov 0x30(%rdi),%rax\n" + " e: 48 83 f8 4b cmp $0x4b,%rax\n" + " 12: 74 0b je 0x1f\n" + " 14: 48 83 f8 0b cmp $0xb,%rax\n" + " 18: 74 05 je 0x1f\n" + " 1a: e9 00 01 00 00 jmpq 0x11f\n" >>, ?assertEqual(dump_to_bin(Dump), Stream).