@@ -2119,12 +2119,15 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
21192119 FailOrder = AtomicOrdering::Monotonic;
21202120 unsigned nb = isboxed ? sizeof (void *) : jl_datatype_size (jltype);
21212121 AllocaInst *intcast = nullptr ;
2122+ Type *intcast_eltyp = nullptr ;
2123+ bool tracked_pointers = isboxed || CountTrackedPointers (elty).count > 0 ;
21222124 if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy ()) {
2125+ intcast_eltyp = elty;
2126+ elty = Type::getIntNTy (ctx.builder .getContext (), 8 * nb);
21232127 if (!issetfield) {
21242128 intcast = emit_static_alloca (ctx, elty);
21252129 setName (ctx.emission_context , intcast, " atomic_store_box" );
21262130 }
2127- elty = Type::getIntNTy (ctx.builder .getContext (), 8 * nb);
21282131 }
21292132 Type *realelty = elty;
21302133 if (Order != AtomicOrdering::NotAtomic && isa<IntegerType>(elty)) {
@@ -2133,14 +2136,20 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
21332136 elty = Type::getIntNTy (ctx.builder .getContext (), 8 * nb2);
21342137 }
21352138 Value *r = nullptr ;
2136- if (issetfield || isswapfield || isreplacefield || issetfieldonce) {
2137- if (isboxed)
2139+ if (issetfield || isswapfield || isreplacefield || issetfieldonce) { // e.g. !ismodifyfield
2140+ assert (isboxed || rhs.typ == jltype);
2141+ if (isboxed) {
21382142 r = boxed (ctx, rhs);
2139- else if (aliasscope || Order != AtomicOrdering::NotAtomic || CountTrackedPointers (realelty).count ) {
2143+ }
2144+ else if (intcast) {
2145+ emit_unbox_store (ctx, rhs, intcast, ctx.tbaa ().tbaa_stack , intcast->getAlign ());
2146+ r = ctx.builder .CreateLoad (realelty, intcast);
2147+ }
2148+ else if (aliasscope || Order != AtomicOrdering::NotAtomic || tracked_pointers) {
21402149 r = emit_unbox (ctx, realelty, rhs, jltype);
2141- if (realelty != elty)
2142- r = ctx.builder .CreateZExt (r, elty);
21432150 }
2151+ if (realelty != elty)
2152+ r = ctx.builder .CreateZExt (r, elty);
21442153 }
21452154 if (isboxed)
21462155 alignment = sizeof (void *);
@@ -2222,7 +2231,14 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
22222231 Current->addIncoming (instr, SkipBB);
22232232 ctx.builder .SetInsertPoint (BB);
22242233 }
2225- Compare = emit_unbox (ctx, realelty, cmp, jltype);
2234+ cmp = update_julia_type (ctx, cmp, jltype);
2235+ if (intcast) {
2236+ emit_unbox_store (ctx, cmp, intcast, ctx.tbaa ().tbaa_stack , intcast->getAlign ());
2237+ Compare = ctx.builder .CreateLoad (realelty, intcast);
2238+ }
2239+ else {
2240+ Compare = emit_unbox (ctx, realelty, cmp, jltype);
2241+ }
22262242 if (realelty != elty)
22272243 Compare = ctx.builder .CreateZExt (Compare, elty);
22282244 }
@@ -2269,28 +2285,36 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
22692285 if (realelty != elty)
22702286 realCompare = ctx.builder .CreateTrunc (realCompare, realelty);
22712287 if (intcast) {
2288+ assert (!isboxed);
22722289 ctx.builder .CreateStore (realCompare, intcast);
2273- if (maybe_null_if_boxed )
2274- realCompare = ctx.builder .CreateLoad (intcast-> getAllocatedType () , intcast);
2290+ if (tracked_pointers )
2291+ realCompare = ctx.builder .CreateLoad (intcast_eltyp , intcast);
22752292 }
2276- if (maybe_null_if_boxed) {
2277- Value *first_ptr = isboxed ? Compare : extract_first_ptr (ctx, Compare );
2278- if (first_ptr)
2279- null_load_check (ctx, first_ptr, mod, var);
2293+ if (maybe_null_if_boxed && tracked_pointers ) {
2294+ Value *first_ptr = isboxed ? realCompare : extract_first_ptr (ctx, realCompare );
2295+ assert (first_ptr);
2296+ null_load_check (ctx, first_ptr, mod, var);
22802297 }
2281- if (intcast)
2298+ if (intcast && !tracked_pointers )
22822299 oldval = mark_julia_slot (intcast, jltype, NULL , ctx.tbaa ().tbaa_stack );
22832300 else
22842301 oldval = mark_julia_type (ctx, realCompare, isboxed, jltype);
22852302 rhs = newval (oldval);
22862303 if (isboxed) {
22872304 r = boxed (ctx, rhs);
22882305 }
2289- else if (Order != AtomicOrdering::NotAtomic || CountTrackedPointers (realelty).count ) {
2306+ else if (intcast) {
2307+ emit_unbox_store (ctx, rhs, intcast, ctx.tbaa ().tbaa_stack , intcast->getAlign ());
2308+ r = ctx.builder .CreateLoad (realelty, intcast);
2309+ if (!tracked_pointers) // oldval is a slot, so put the oldval back
2310+ ctx.builder .CreateStore (realCompare, intcast);
2311+ }
2312+ else if (Order != AtomicOrdering::NotAtomic) {
2313+ assert (!tracked_pointers);
22902314 r = emit_unbox (ctx, realelty, rhs, jltype);
2291- if (realelty != elty)
2292- r = ctx.builder .CreateZExt (r, elty);
22932315 }
2316+ if (realelty != elty)
2317+ r = ctx.builder .CreateZExt (r, elty);
22942318 if (needlock)
22952319 emit_lockstate_value (ctx, needlock, true );
22962320 cmp = oldval;
@@ -2356,9 +2380,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
23562380 realinstr = ctx.builder .CreateTrunc (realinstr, realelty);
23572381 if (intcast) {
23582382 ctx.builder .CreateStore (realinstr, intcast);
2383+ // n.b. this oldval is only used for emit_f_is in this branch, so we know a priori that it does not need a gc-root
23592384 oldval = mark_julia_slot (intcast, jltype, NULL , ctx.tbaa ().tbaa_stack );
23602385 if (maybe_null_if_boxed)
2361- realinstr = ctx.builder .CreateLoad (intcast-> getAllocatedType () , intcast);
2386+ realinstr = ctx.builder .CreateLoad (intcast_eltyp , intcast);
23622387 }
23632388 else {
23642389 oldval = mark_julia_type (ctx, realinstr, isboxed, jltype);
@@ -2398,20 +2423,23 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
23982423 ctx.builder .SetInsertPoint (DoneBB);
23992424 if (needlock)
24002425 emit_lockstate_value (ctx, needlock, false );
2401- if (parent != NULL ) {
2426+ if (parent != NULL && r && tracked_pointers && (!isboxed || ! type_is_permalloc (rhs. typ )) ) {
24022427 if (isreplacefield || issetfieldonce) {
2403- // TODO: avoid this branch if we aren't making a write barrier
24042428 BasicBlock *BB = BasicBlock::Create (ctx.builder .getContext (), " xchg_wb" , ctx.f );
24052429 DoneBB = BasicBlock::Create (ctx.builder .getContext (), " done_xchg_wb" , ctx.f );
24062430 ctx.builder .CreateCondBr (Success, BB, DoneBB);
24072431 ctx.builder .SetInsertPoint (BB);
24082432 }
2409- if (r) {
2410- if (!isboxed)
2411- emit_write_multibarrier (ctx, parent, r, rhs. typ );
2412- else if (! type_is_permalloc (rhs. typ ))
2413- emit_write_barrier ( ctx, parent, r );
2433+ if (realelty != elty)
2434+ r = ctx. builder . Insert ( CastInst::Create (Instruction::Trunc, r, realelty));
2435+ if (intcast) {
2436+ ctx. builder . CreateStore (r, intcast);
2437+ r = ctx. builder . CreateLoad (intcast_eltyp, intcast );
24142438 }
2439+ if (!isboxed)
2440+ emit_write_multibarrier (ctx, parent, r, rhs.typ );
2441+ else if (!type_is_permalloc (rhs.typ ))
2442+ emit_write_barrier (ctx, parent, r);
24152443 if (isreplacefield || issetfieldonce) {
24162444 ctx.builder .CreateBr (DoneBB);
24172445 ctx.builder .SetInsertPoint (DoneBB);
@@ -2430,21 +2458,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
24302458 instr = ctx.builder .Insert (CastInst::Create (Instruction::Trunc, instr, realelty));
24312459 if (intcast) {
24322460 ctx.builder .CreateStore (instr, intcast);
2433- instr = nullptr ;
2461+ if (tracked_pointers)
2462+ instr = ctx.builder .CreateLoad (intcast_eltyp, intcast);
24342463 }
2435- if (maybe_null_if_boxed) {
2436- if (intcast)
2437- instr = ctx.builder .CreateLoad (intcast->getAllocatedType (), intcast);
2464+ if (maybe_null_if_boxed && tracked_pointers) {
24382465 Value *first_ptr = isboxed ? instr : extract_first_ptr (ctx, instr);
2439- if (first_ptr)
2440- null_load_check (ctx, first_ptr, mod, var);
2441- if (intcast && !first_ptr)
2442- instr = nullptr ;
2466+ assert (first_ptr);
2467+ null_load_check (ctx, first_ptr, mod, var);
24432468 }
2444- if (instr)
2445- oldval = mark_julia_type (ctx, instr, isboxed, jltype);
2446- else
2469+ if (intcast && !tracked_pointers)
24472470 oldval = mark_julia_slot (intcast, jltype, NULL , ctx.tbaa ().tbaa_stack );
2471+ else
2472+ oldval = mark_julia_type (ctx, instr, isboxed, jltype);
24482473 if (isreplacefield) {
24492474 Success = ctx.builder .CreateZExt (Success, getInt8Ty (ctx.builder .getContext ()));
24502475 const jl_cgval_t argv[2 ] = {oldval, mark_julia_type (ctx, Success, false , jl_bool_type)};
0 commit comments