Preperations for the new JIT tier

src/asm_writing/assembler.cpp

@@ -790,6 +790,8 @@ void Assembler::cmp(Indirect mem, Register reg) {
 }
 
 void Assembler::lea(Indirect mem, Register reg) {
+    RELEASE_ASSERT(mem.base != RSP && mem.base != R12, "We have to generate the SIB byte...");
+
     int mem_idx = mem.base.regnum;
     int reg_idx = reg.regnum;
 
@@ -886,6 +888,23 @@ void Assembler::jmp(JumpDestination dest) {
     }
 }
 
+void Assembler::jmp(Indirect dest) {
+    int reg_idx = dest.base.regnum;
+
+    assert(reg_idx >= 0 && reg_idx < 8 && "not yet implemented");
+    emitByte(0xFF);
+    if (dest.offset == 0) {
+        emitModRM(0b00, 0b100, reg_idx);
+    } else if (-0x80 <= dest.offset && dest.offset < 0x80) {
+        emitModRM(0b01, 0b100, reg_idx);
+        emitByte(dest.offset);
+    } else {
+        assert((-1L << 31) <= dest.offset && dest.offset < (1L << 31) - 1);
+        emitModRM(0b10, 0b100, reg_idx);
+        emitInt(dest.offset, 4);
+    }
+}
+
 void Assembler::jne(JumpDestination dest) {
     jmp_cond(dest, COND_NOT_EQUAL);
 }
@@ -934,6 +953,10 @@ void Assembler::setne(Register reg) {
     set_cond(reg, COND_NOT_EQUAL);
 }
 
+void Assembler::leave() {
+    emitByte(0xC9);
+}
+
 uint8_t* Assembler::emitCall(void* ptr, Register scratch) {
     mov(Immediate(ptr), scratch);
     callq(scratch);
@@ -1002,5 +1025,19 @@ void Assembler::emitAnnotation(int num) {
     cmp(RAX, Immediate(num));
     nop();
 }
+
+ForwardJump::ForwardJump(Assembler& assembler, ConditionCode condition)
+    : assembler(assembler), condition(condition), jmp_inst(assembler.curInstPointer()) {
+    assembler.jmp_cond(JumpDestination::fromStart(assembler.bytesWritten() + max_jump_size), condition);
+}
+
+ForwardJump::~ForwardJump() {
+    uint8_t* new_pos = assembler.curInstPointer();
+    int offset = new_pos - jmp_inst;
+    RELEASE_ASSERT(offset < max_jump_size, "");
+    assembler.setCurInstPointer(jmp_inst);
+    assembler.jmp_cond(JumpDestination::fromStart(assembler.bytesWritten() + offset), condition);
+    assembler.setCurInstPointer(new_pos);
+}
 }
 }

src/asm_writing/assembler.h

@@ -154,6 +154,7 @@ class Assembler {
 
     void callq(Register reg);
     void retq();
+    void leave();
 
     void cmp(Register reg1, Register reg2);
     void cmp(Register reg, Immediate imm);
@@ -166,6 +167,7 @@ class Assembler {
 
     void jmp_cond(JumpDestination dest, ConditionCode condition);
     void jmp(JumpDestination dest);
+    void jmp(Indirect dest);
     void jmpq(Register dest);
     void je(JumpDestination dest);
     void jne(JumpDestination dest);
@@ -185,9 +187,27 @@ class Assembler {
     void fillWithNopsExcept(int bytes);
     void emitAnnotation(int num);
 
-    int bytesWritten() { return addr - start_addr; }
+    int bytesLeft() const { return end_addr - addr; }
+    int bytesWritten() const { return addr - start_addr; }
     uint8_t* curInstPointer() { return addr; }
-    bool isExactlyFull() { return addr == end_addr; }
+    void setCurInstPointer(uint8_t* ptr) { addr = ptr; }
+    bool isExactlyFull() const { return addr == end_addr; }
+    uint8_t* getStartAddr() { return start_addr; }
+};
+
+// This class helps generating a forward jump with a relative offset.
+// It keeps track of the current assembler offset at construction time and in the destructor patches the
+// generated conditional jump with the correct offset depending on the number of bytes emitted in between.
+class ForwardJump {
+private:
+    const int max_jump_size = 128;
+    Assembler& assembler;
+    ConditionCode condition;
+    uint8_t* jmp_inst;
+
+public:
+    ForwardJump(Assembler& assembler, ConditionCode condition);
+    ~ForwardJump();
 };
 
 uint8_t* initializePatchpoint2(uint8_t* start_addr, uint8_t* slowpath_start, uint8_t* end_addr, StackInfo stack_info,

src/asm_writing/icinfo.cpp

@@ -151,8 +151,8 @@ assembler::GenericRegister ICSlotRewrite::returnRegister() {
 
 
 
-ICSlotRewrite* ICInfo::startRewrite(const char* debug_name) {
-    return new ICSlotRewrite(this, debug_name);
+std::unique_ptr<ICSlotRewrite> ICInfo::startRewrite(const char* debug_name) {
+    return std::unique_ptr<ICSlotRewrite>(new ICSlotRewrite(this, debug_name));
 }
 
 ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
@@ -236,11 +236,11 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
         // writer->emitNop();
         // writer->emitGuardFalse();
 
-        std::unique_ptr<Assembler> writer(new Assembler(start, ic->slot_size));
-        writer->nop();
-        // writer->trap();
-        // writer->jmp(JumpDestination::fromStart(ic->slot_size * (ic->num_slots - i)));
-        writer->jmp(JumpDestination::fromStart(slowpath_start_addr - start));
+        Assembler writer(start, ic->slot_size);
+        writer.nop();
+        // writer.trap();
+        // writer.jmp(JumpDestination::fromStart(ic->slot_size * (ic->num_slots - i)));
+        writer.jmp(JumpDestination::fromStart(slowpath_start_addr - start));
     }
 
     ICInfo* icinfo = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
@@ -272,10 +272,10 @@ void ICInfo::clear(ICSlotInfo* icentry) {
     if (VERBOSITY() >= 4)
         printf("clearing patchpoint %p, slot at %p\n", start_addr, start);
 
-    std::unique_ptr<Assembler> writer(new Assembler(start, getSlotSize()));
-    writer->nop();
-    writer->jmp(JumpDestination::fromStart(getSlotSize()));
-    assert(writer->bytesWritten() <= IC_INVALDITION_HEADER_SIZE);
+    Assembler writer(start, getSlotSize());
+    writer.nop();
+    writer.jmp(JumpDestination::fromStart(getSlotSize()));
+    assert(writer.bytesWritten() <= IC_INVALDITION_HEADER_SIZE);
 
     // std::unique_ptr<MCWriter> writer(createMCWriter(start, getSlotSize(), 0));
     // writer->emitNop();

src/asm_writing/icinfo.h

@@ -119,7 +119,7 @@ class ICInfo {
     llvm::CallingConv::ID getCallingConvention() { return calling_conv; }
     const std::vector<int>& getLiveOuts() { return live_outs; }
 
-    ICSlotRewrite* startRewrite(const char* debug_name);
+    std::unique_ptr<ICSlotRewrite> startRewrite(const char* debug_name);
     void clear(ICSlotInfo* entry);
 
     bool shouldAttempt();

src/asm_writing/rewriter.cpp

@@ -135,10 +135,6 @@ void Location::dump() const {
     RELEASE_ASSERT(0, "%d", type);
 }
 
-static bool isLargeConstant(int64_t val) {
-    return (val < (-1L << 31) || val >= (1L << 31) - 1);
-}
-
 Rewriter::ConstLoader::ConstLoader(Rewriter* rewriter) : rewriter(rewriter) {
 }
 
@@ -554,7 +550,7 @@ void RewriterVar::dump() {
 }
 
 assembler::Immediate RewriterVar::tryGetAsImmediate(bool* is_immediate) {
-    if (this->is_constant && !isLargeConstant(this->constant_value)) {
+    if (this->is_constant && !Rewriter::isLargeConstant(this->constant_value)) {
         *is_immediate = true;
         return assembler::Immediate(this->constant_value);
     } else {
@@ -568,7 +564,7 @@ assembler::Register RewriterVar::getInReg(Location dest, bool allow_constant_in_
 
 #ifndef NDEBUG
     if (!allow_constant_in_reg) {
-        assert(!is_constant || isLargeConstant(constant_value));
+        assert(!is_constant || Rewriter::isLargeConstant(constant_value));
     }
 #endif
 
@@ -753,6 +749,18 @@ RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar*
     return call(has_side_effects, func_addr, args, args_xmm);
 }
 
+RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1,
+                            RewriterVar* arg2, RewriterVar* arg3, RewriterVar* arg4) {
+    RewriterVar::SmallVector args;
+    RewriterVar::SmallVector args_xmm;
+    args.push_back(arg0);
+    args.push_back(arg1);
+    args.push_back(arg2);
+    args.push_back(arg3);
+    args.push_back(arg4);
+    return call(has_side_effects, func_addr, args, args_xmm);
+}
+
 static const Location caller_save_registers[]{
     assembler::RAX,   assembler::RCX,   assembler::RDX,   assembler::RSI,   assembler::RDI,
     assembler::R8,    assembler::R9,    assembler::R10,   assembler::R11,   assembler::XMM0,
@@ -918,10 +926,14 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         if (need_to_spill) {
             if (check_reg.type == Location::Register) {
                 spillRegister(check_reg.asRegister());
+                if (failed)
+                    return;
             } else {
                 assert(check_reg.type == Location::XMMRegister);
                 assert(var->locations.size() == 1);
                 spillRegister(check_reg.asXMMRegister());
+                if (failed)
+                    return;
             }
         } else {
             removeLocationFromVar(var, check_reg);
@@ -1358,7 +1370,8 @@ int Rewriter::_allocate(RewriterVar* result, int n) {
             consec = 0;
         }
     }
-    RELEASE_ASSERT(0, "Using all %d bytes of scratch!", scratch_size);
+    failed = true;
+    return 0;
 }
 
 RewriterVar* Rewriter::allocateAndCopy(RewriterVar* array_ptr, int n) {
@@ -1663,21 +1676,18 @@ TypeRecorder* Rewriter::getTypeRecorder() {
     return rewrite->getTypeRecorder();
 }
 
-Rewriter::Rewriter(ICSlotRewrite* rewrite, int num_args, const std::vector<int>& live_outs)
-    : rewrite(rewrite),
-      assembler(rewrite->getAssembler()),
+Rewriter::Rewriter(std::unique_ptr<ICSlotRewrite> rewrite, int num_args, const std::vector<int>& live_outs)
+    : rewrite(std::move(rewrite)),
+      assembler(this->rewrite->getAssembler()),
       const_loader(this),
-      return_location(rewrite->returnRegister()),
+      return_location(this->rewrite->returnRegister()),
       failed(false),
       added_changing_action(false),
       marked_inside_ic(false),
       last_guard_action(-1),
       done_guarding(false) {
     initPhaseCollecting();
 
-#ifndef NDEBUG
-    start_vars = RewriterVar::nvars;
-#endif
     finished = false;
 
     for (int i = 0; i < num_args; i++) {
@@ -1823,10 +1833,6 @@ Rewriter* Rewriter::createRewriter(void* rtn_addr, int num_args, const char* deb
     return new Rewriter(ic->startRewrite(debug_name), num_args, ic->getLiveOuts());
 }
 
-#ifndef NDEBUG
-int RewriterVar::nvars = 0;
-#endif
-
 static const int INITIAL_CALL_SIZE = 13;
 static const int DWARF_RBP_REGNUM = 6;
 bool spillFrameArgumentIfNecessary(StackMap::Record::Location& l, uint8_t*& inst_addr, uint8_t* inst_end,

src/asm_writing/rewriter.h

@@ -266,22 +266,12 @@ class RewriterVar {
     RewriterVar& operator=(const RewriterVar&) = delete;
 
 public:
-#ifndef NDEBUG
-    static int nvars;
-#endif
-
     RewriterVar(Rewriter* rewriter) : rewriter(rewriter), next_use(0), is_arg(false), is_constant(false) {
-#ifndef NDEBUG
-        nvars++;
-#endif
         assert(rewriter);
     }
 
-#ifndef NDEBUG
-    ~RewriterVar() { nvars--; }
-#endif
-
     friend class Rewriter;
+    friend class JitFragmentWriter;
 };
 
 class RewriterAction {
@@ -297,7 +287,7 @@ enum class ActionType { NORMAL, GUARD, MUTATION };
 #define LOCATION_PLACEHOLDER ((RewriterVar*)1)
 
 class Rewriter : public ICSlotRewrite::CommitHook {
-private:
+protected:
     // Helps generating the best code for loading a const integer value.
     // By keeping track of the last known value of every register and reusing it.
     class ConstLoader {
@@ -335,7 +325,6 @@ class Rewriter : public ICSlotRewrite::CommitHook {
     bool failed;   // if we tried to generate an invalid rewrite.
     bool finished; // committed or aborted
 #ifndef NDEBUG
-    int start_vars;
 
     bool phase_emitting;
     void initPhaseCollecting() { phase_emitting = false; }
@@ -355,7 +344,7 @@ class Rewriter : public ICSlotRewrite::CommitHook {
     std::vector<RewriterVar*> args;
     std::vector<RewriterVar*> live_outs;
 
-    Rewriter(ICSlotRewrite* rewrite, int num_args, const std::vector<int>& live_outs);
+    Rewriter(std::unique_ptr<ICSlotRewrite> rewrite, int num_args, const std::vector<int>& live_outs);
 
     std::vector<RewriterAction> actions;
     void addAction(const std::function<void()>& action, std::vector<RewriterVar*> const& vars, ActionType type) {
@@ -477,11 +466,6 @@ class Rewriter : public ICSlotRewrite::CommitHook {
         for (RewriterVar* var : vars) {
             delete var;
         }
-
-        // This check isn't thread safe and should be fine to remove if it causes
-        // issues (along with the nvars/start_vars accounting)
-        ASSERT(threading::threadWasStarted() || RewriterVar::nvars == start_vars, "%d %d", RewriterVar::nvars,
-               start_vars);
     }
 
     Location getReturnDestination();
@@ -507,6 +491,8 @@ class Rewriter : public ICSlotRewrite::CommitHook {
     RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
     RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2,
                       RewriterVar* arg3);
+    RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2,
+                      RewriterVar* arg3, RewriterVar* arg4);
     RewriterVar* add(RewriterVar* a, int64_t b, Location dest);
     // Allocates n pointer-sized stack slots:
     RewriterVar* allocate(int n);
@@ -521,6 +507,8 @@ class Rewriter : public ICSlotRewrite::CommitHook {
 
     static Rewriter* createRewriter(void* rtn_addr, int num_args, const char* debug_name);
 
+    static bool isLargeConstant(int64_t val) { return (val < (-1L << 31) || val >= (1L << 31) - 1); }
+
     friend class RewriterVar;
 };