diff --git a/src/gc-interface.h b/src/gc-interface.h index 043aeb019d395..cc9ee947798fe 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -204,7 +204,14 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, // immortal region that is never swept. The second parameter specifies the type of the // object being allocated and will be used to set the object header. struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT; - +// permanently allocates a symbol (jl_sym_t). The object needs to be word aligned, +// and tagged with jl_sym_tag. +// FIXME: Ideally we should merge this with jl_gc_permobj, as symbol is an object. +// Currently there are a few differences between the two functions, and refactoring is needed. +// 1. sz for this function includes the object header, and sz for jl_gc_permobj excludes the header size. +// 2. align for this function is word align, and align for jl_gc_permobj depends on the allocation size. +// 3. ty for this function is jl_symbol_tag << 4, and ty for jl_gc_permobj is a datatype pointer. +struct _jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT; // This function notifies the GC about memory addresses that are set when loading the boot image. // The GC may use that information to, for instance, determine that all objects in that chunk of memory should // be treated as marked and belonged to the old generation in nursery collections. diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 6519c6c34c48d..663995563d6af 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -860,18 +860,28 @@ inline void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uin } } +inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) { + intptr_t addr = (intptr_t) obj; + uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6); + intptr_t shift = (addr >> 3) & 0b111; + while(1) { + uint8_t old_val = *meta_addr; + uint8_t new_val = old_val | (1 << shift); + if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { + break; + } + } +} + inline void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_DEFAULT_IMMIX_ALLOCATOR]; return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0); } -inline void mmtk_immix_post_alloc_slow(MMTkMutatorContext* mutator, void* obj, size_t size) { - mmtk_post_alloc(mutator, obj, size, 0); -} - inline void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - // FIXME: for now, we do nothing - // but when supporting moving, this is where we set the valid object (VO) bit + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); + } } inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { @@ -881,16 +891,11 @@ inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { - intptr_t addr = (intptr_t) obj; - uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6); - intptr_t shift = (addr >> 3) & 0b111; - while(1) { - uint8_t old_val = *meta_addr; - uint8_t new_val = old_val | (1 << shift); - if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { - break; - } - } + mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj); + } + + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); } } @@ -1069,6 +1074,16 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT return jl_valueof(o); } +jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT +{ + jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); + jl_value_t *sym = jl_valueof(tag); + jl_ptls_t ptls = jl_current_task->ptls; + jl_set_typetagof(sym, jl_symbol_tag, 0); // We need to set symbol tag. The GC tag doesnt matter. + mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, sym, sz); + return sym; +} + JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) { jl_ptls_t ptls = jl_current_task->ptls; diff --git a/src/gc-stock.c b/src/gc-stock.c index aa7f2a42c712a..01ad8ea1a725a 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3836,6 +3836,15 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT return jl_valueof(o); } +jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT +{ + jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); + jl_value_t *sym = jl_valueof(tag); + // set to old marked so that we won't look at it in the GC or write barrier. + jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); + return sym; +} + JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void) { if (jl_is_initialized()) { diff --git a/src/julia.h b/src/julia.h index 27d9331944f9d..b81d3bec3889a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2748,7 +2748,13 @@ extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, #define MMTK_DEFAULT_IMMIX_ALLOCATOR (0) #define MMTK_IMMORTAL_BUMP_ALLOCATOR (0) +// VO bit is required to support conservative stack scanning and moving. +#define MMTK_NEEDS_VO_BIT (1) + +void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size); + extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; +extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; // Directly call into MMTk for write barrier (debugging only) STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT diff --git a/src/llvm-late-gc-lowering-mmtk.cpp b/src/llvm-late-gc-lowering-mmtk.cpp index 6342fc9883845..49994c63fbff4 100644 --- a/src/llvm-late-gc-lowering-mmtk.cpp +++ b/src/llvm-late-gc-lowering-mmtk.cpp @@ -132,6 +132,37 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t))); auto v_as_ptr = builder.CreateIntToPtr(v_raw, smallAllocFunc->getReturnType()); + + // Post alloc + if (MMTK_NEEDS_VO_BIT) { + auto intptr_ty = Type::getInt64Ty(target->getContext()); + auto i8_ty = Type::getInt8Ty(F.getContext()); + intptr_t metadata_base_address = reinterpret_cast(MMTK_SIDE_VO_BIT_BASE_ADDRESS); + auto metadata_base_val = ConstantInt::get(intptr_ty, metadata_base_address); + auto metadata_base_ptr = ConstantExpr::getIntToPtr(metadata_base_val, PointerType::get(i8_ty, 0)); + + // intptr_t addr = (intptr_t) v; + auto addr = v_raw; + + // uint8_t* vo_meta_addr = (uint8_t*) (MMTK_SIDE_VO_BIT_BASE_ADDRESS) + (addr >> 6); + auto shr = builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 6)); + auto metadata_ptr = builder.CreateGEP(i8_ty, metadata_base_ptr, shr); + + // intptr_t shift = (addr >> 3) & 0b111; + auto shift = builder.CreateAnd(builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 3)), ConstantInt::get(intptr_ty, 7)); + + // uint8_t byte_val = *vo_meta_addr; + auto byte_val = builder.CreateAlignedLoad(i8_ty, metadata_ptr, Align()); + + // uint8_t new_val = byte_val | (1 << shift); + auto shifted_val = builder.CreateShl(ConstantInt::get(intptr_ty, 1), shift); + auto shifted_val_i8 = builder.CreateTruncOrBitCast(shifted_val, i8_ty); + auto new_val = builder.CreateOr(byte_val, shifted_val_i8); + + // (*vo_meta_addr) = new_val; + builder.CreateStore(new_val, metadata_ptr); + } + builder.CreateBr(next_instr->getParent()); phiNode->addIncoming(new_call, slowpath); diff --git a/src/symbol.c b/src/symbol.c index ef2c11e0842e8..cf53743b60602 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -10,6 +10,7 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "gc-interface.h" #ifdef __cplusplus extern "C" { @@ -34,12 +35,8 @@ static size_t symbol_nbytes(size_t len) JL_NOTSAFEPOINT static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT { - jl_sym_t *sym; size_t nb = symbol_nbytes(len); - jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(nb, 0, sizeof(void*), 0); - sym = (jl_sym_t*)jl_valueof(tag); - // set to old marked so that we won't look at it in the GC or write barrier. - jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); + jl_sym_t *sym = (jl_sym_t*)jl_gc_permsymbol(nb); jl_atomic_store_relaxed(&sym->left, NULL); jl_atomic_store_relaxed(&sym->right, NULL); sym->hash = hash_symbol(str, len);