diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 82d3f1b963e41a..9cf8e16b9e6f41 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2004,7 +2004,46 @@ void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_b GCToEEInterface::StompWriteBarrier(&args); } -void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high) +#ifdef USE_REGIONS +void region_write_barrier_settings (WriteBarrierParameters* args, + gc_heap::region_info* map_region_to_generation_skewed, + uint8_t region_shr) +{ + switch (GCConfig::GetGCWriteBarrier()) + { + default: + case GCConfig::WRITE_BARRIER_DEFAULT: + case GCConfig::WRITE_BARRIER_REGION_BIT: + // bitwise region write barrier is the default now + args->region_to_generation_table = (uint8_t*)map_region_to_generation_skewed; + args->region_shr = region_shr; + args->region_use_bitwise_write_barrier = true; + break; + + case GCConfig::WRITE_BARRIER_REGION_BYTE: + // bytewise region write barrier + args->region_to_generation_table = (uint8_t*)map_region_to_generation_skewed; + args->region_shr = region_shr; + assert (args->region_use_bitwise_write_barrier == false); + break; + + case GCConfig::WRITE_BARRIER_SERVER: + // server write barrier + // args should have been zero initialized + assert (args->region_use_bitwise_write_barrier == false); + assert (args->region_to_generation_table == nullptr); + assert (args->region_shr == 0); + break; + } +} +#endif //USE_REGIONS + +void stomp_write_barrier_ephemeral (uint8_t* ephemeral_low, uint8_t* ephemeral_high +#ifdef USE_REGIONS + , gc_heap::region_info* map_region_to_generation_skewed + , uint8_t region_shr +#endif //USE_REGIONS + ) { initGCShadow(); @@ -2013,10 +2052,18 @@ void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_hi args.is_runtime_suspended = true; args.ephemeral_low = ephemeral_low; args.ephemeral_high = ephemeral_high; +#ifdef USE_REGIONS + region_write_barrier_settings (&args, map_region_to_generation_skewed, region_shr); +#endif //USE_REGIONS GCToEEInterface::StompWriteBarrier(&args); } -void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high) +void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high +#ifdef USE_REGIONS + , gc_heap::region_info* map_region_to_generation_skewed + , uint8_t region_shr +#endif //USE_REGIONS + ) { WriteBarrierParameters args = {}; args.operation = WriteBarrierOp::Initialize; @@ -2032,6 +2079,11 @@ void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_h args.highest_address = g_gc_highest_address; args.ephemeral_low = ephemeral_low; args.ephemeral_high = ephemeral_high; + +#ifdef USE_REGIONS + region_write_barrier_settings (&args, map_region_to_generation_skewed, region_shr); +#endif //USE_REGIONS + GCToEEInterface::StompWriteBarrier(&args); } @@ -2289,6 +2341,9 @@ region_allocator global_region_allocator; uint8_t*(*initial_regions)[total_generation_count][2] = nullptr; size_t gc_heap::region_count = 0; +gc_heap::region_info* gc_heap::map_region_to_generation = nullptr; +gc_heap::region_info* gc_heap::map_region_to_generation_skewed = nullptr; + #endif //USE_REGIONS #ifdef BACKGROUND_GC @@ -2327,6 +2382,15 @@ VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free; VOLATILE(BOOL) gc_heap::gc_background_running = FALSE; #endif //BACKGROUND_GC +#ifdef USE_REGIONS +#ifdef MULTIPLE_HEAPS +uint8_t* gc_heap::gc_low; +uint8_t* gc_heap::gc_high; +#endif //MULTIPLE_HEAPS +VOLATILE(uint8_t*) gc_heap::ephemeral_low; +VOLATILE(uint8_t*) gc_heap::ephemeral_high; +#endif //USE_REGIONS + #ifndef MULTIPLE_HEAPS #ifdef SPINLOCK_HISTORY int gc_heap::spinlock_info_index = 0; @@ -3499,11 +3563,19 @@ sorted_table::clear() #endif //FEATURE_BASICFREEZE #ifdef USE_REGIONS +inline +size_t get_skewed_basic_region_index_for_address (uint8_t* address) +{ + assert ((g_gc_lowest_address <= address) && (address < g_gc_highest_address)); + size_t skewed_basic_region_index = (size_t)address >> gc_heap::min_segment_size_shr; + return skewed_basic_region_index; +} + inline size_t get_basic_region_index_for_address (uint8_t* address) { - size_t basic_region_index = (size_t)address >> gc_heap::min_segment_size_shr; - return (basic_region_index - ((size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr)); + size_t skewed_basic_region_index = get_skewed_basic_region_index_for_address (address); + return (skewed_basic_region_index - get_skewed_basic_region_index_for_address (g_gc_lowest_address)); } // Go from a random address to its region info. The random address could be @@ -4005,6 +4077,13 @@ size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end) return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr); } +size_t size_region_to_generation_table_of (uint8_t* from, uint8_t* end) +{ + dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, + sizeof (uint8_t)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr)))); + return sizeof (uint8_t)*((size_t)(end - from) >> gc_heap::min_segment_size_shr); +} + inline size_t seg_mapping_word_of (uint8_t* add) { @@ -7800,27 +7879,10 @@ bool gc_heap::should_check_brick_for_reloc (uint8_t* o) { assert ((o >= g_gc_lowest_address) && (o < g_gc_highest_address)); - int condemned_gen = settings.condemned_generation; - if (condemned_gen < max_generation) - { - heap_segment* region = region_of (o); - int gen = get_region_gen_num (region); - if ((gen > condemned_gen) || (heap_segment_swept_in_plan (region))) - { - if (heap_segment_swept_in_plan (region)) - { - dprintf (4444, ("-Rsip %Ix", o)); - } - - return false; - } - } - else if (heap_segment_swept_in_plan (region_of (o))) - { - return false; - } + size_t skewed_basic_region_index = get_skewed_basic_region_index_for_address (o); - return true; + // return true if the region is not SIP and the generation is <= condemned generation + return (map_region_to_generation_skewed[skewed_basic_region_index] & (RI_SIP|RI_GEN_MASK)) <= settings.condemned_generation; } #endif //USE_REGIONS @@ -8557,6 +8619,9 @@ void gc_heap::get_card_table_element_sizes (uint8_t* start, uint8_t* end, size_t sizes[software_write_watch_table_element] = SoftwareWriteWatch::GetTableByteSize(start, end); } #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP && BACKGROUND_GC +#ifdef USE_REGIONS + sizes[region_to_generation_table_element] = size_region_to_generation_table_of (start, end); +#endif //USE_REGIONS sizes[seg_mapping_table_element] = size_seg_mapping_table_of (start, end); #ifdef BACKGROUND_GC if (gc_can_use_concurrent) @@ -8581,6 +8646,9 @@ void gc_heap::get_card_table_element_layout (uint8_t* start, uint8_t* end, size_ #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP sizeof(size_t), // software_write_watch_table_element #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +#ifdef USE_REGIONS + sizeof (uint8_t), // region_to_generation_table_element +#endif //USE_REGIONS sizeof (uint8_t*), // seg_mapping_table_element #ifdef BACKGROUND_GC // In order to avoid a dependency between commit_mark_array_by_range and this logic, it is easier to make sure @@ -8859,6 +8927,11 @@ uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end) } #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP && BACKGROUND_GC +#ifdef USE_REGIONS + map_region_to_generation = (region_info*)(mem + card_table_element_layout[region_to_generation_table_element]); + map_region_to_generation_skewed = map_region_to_generation - size_region_to_generation_table_of (0, g_gc_lowest_address); +#endif //USE_REGIONS + seg_mapping_table = (seg_mapping*)(mem + card_table_element_layout[seg_mapping_table_element]); seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table - size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address)))); @@ -11319,17 +11392,28 @@ int gc_heap::get_region_gen_num (heap_segment* region) int gc_heap::get_region_gen_num (uint8_t* obj) { - return heap_segment_gen_num (region_of (obj)); + size_t skewed_basic_region_index = get_skewed_basic_region_index_for_address (obj); + int gen_num = map_region_to_generation_skewed[skewed_basic_region_index] & gc_heap::RI_GEN_MASK; + assert ((soh_gen0 <= gen_num) && (gen_num <= soh_gen2)); + assert (gen_num == heap_segment_gen_num (region_of (obj))); + return gen_num; } int gc_heap::get_region_plan_gen_num (uint8_t* obj) { - return heap_segment_plan_gen_num (region_of (obj)); + size_t skewed_basic_region_index = get_skewed_basic_region_index_for_address (obj); + int plan_gen_num = map_region_to_generation_skewed[skewed_basic_region_index] >> gc_heap::RI_PLAN_GEN_SHR; + assert ((soh_gen0 <= plan_gen_num) && (plan_gen_num <= soh_gen2)); + assert (plan_gen_num == heap_segment_plan_gen_num (region_of (obj))); + return plan_gen_num; } bool gc_heap::is_region_demoted (uint8_t* obj) { - return heap_segment_demoted_p (region_of (obj)); + size_t skewed_basic_region_index = get_skewed_basic_region_index_for_address (obj); + bool demoted_p = (map_region_to_generation_skewed[skewed_basic_region_index] & gc_heap::RI_DEMOTED) != 0; + assert (demoted_p == heap_segment_demoted_p (region_of (obj))); + return demoted_p; } inline @@ -11338,6 +11422,77 @@ void gc_heap::set_region_gen_num (heap_segment* region, int gen_num) assert (gen_num < (1 << (sizeof (uint8_t) * 8))); assert (gen_num >= 0); heap_segment_gen_num (region) = (uint8_t)gen_num; + + uint8_t* region_start = get_region_start (region); + uint8_t* region_end = heap_segment_reserved (region); + + size_t region_index_start = get_basic_region_index_for_address (region_start); + size_t region_index_end = get_basic_region_index_for_address (region_end); + region_info entry = (region_info)((gen_num << RI_PLAN_GEN_SHR) | gen_num); + for (size_t region_index = region_index_start; region_index < region_index_end; region_index++) + { + assert (gen_num <= max_generation); + map_region_to_generation[region_index] = entry; + } + if (gen_num <= soh_gen1) + { + if ((region_start < ephemeral_low) || (ephemeral_high < region_end)) + { + static GCSpinLock write_barrier_spin_lock; + + while (true) + { + if (Interlocked::CompareExchange(&write_barrier_spin_lock.lock, 0, -1) < 0) + break; + + if ((ephemeral_low <= region_start) && (region_end <= ephemeral_high)) + return; + + while (write_barrier_spin_lock.lock >= 0) + { + YieldProcessor(); // indicate to the processor that we are spinning + } + } +#ifdef _DEBUG + write_barrier_spin_lock.holding_thread = GCToEEInterface::GetThread(); +#endif //_DEBUG + + if ((region_start < ephemeral_low) || (ephemeral_high < region_end)) + { + uint8_t* new_ephemeral_low = min (region_start, (uint8_t*)ephemeral_low); + uint8_t* new_ephemeral_high = max (region_end, (uint8_t*)ephemeral_high); + + dprintf (REGIONS_LOG, ("about to set ephemeral_low = %Ix ephemeral_high = %Ix", new_ephemeral_low, new_ephemeral_high)); + + stomp_write_barrier_ephemeral (new_ephemeral_low, new_ephemeral_high, + map_region_to_generation_skewed, (uint8_t)min_segment_size_shr); + + // we should only *decrease* ephemeral_low and only *increase* ephemeral_high + if (ephemeral_low < new_ephemeral_low) + GCToOSInterface::DebugBreak (); + if (new_ephemeral_high < ephemeral_high) + GCToOSInterface::DebugBreak (); + + // only set the globals *after* we have updated the write barrier + ephemeral_low = new_ephemeral_low; + ephemeral_high = new_ephemeral_high; + + dprintf (REGIONS_LOG, ("set ephemeral_low = %Ix ephemeral_high = %Ix", new_ephemeral_low, new_ephemeral_high)); + } + else + { + dprintf (REGIONS_LOG, ("leaving lock - no need to update ephemeral range [%Ix,%Ix[ for region [%Ix,%Ix]", (uint8_t*)ephemeral_low, (uint8_t*)ephemeral_high, region_start, region_end)); + } +#ifdef _DEBUG + write_barrier_spin_lock.holding_thread = (Thread*)-1; +#endif //_DEBUG + write_barrier_spin_lock.lock = -1; + } + else + { + dprintf (REGIONS_LOG, ("no need to update ephemeral range [%Ix,%Ix[ for region [%Ix,%Ix]", (uint8_t*)ephemeral_low, (uint8_t*)ephemeral_high, region_start, region_end)); + } + } } inline @@ -11351,7 +11506,8 @@ void gc_heap::set_region_plan_gen_num (heap_segment* region, int plan_gen_num) gen_num, plan_gen_num, supposed_plan_gen_num, ((plan_gen_num < supposed_plan_gen_num) ? "DEMOTED" : "ND"))); - if (plan_gen_num < supposed_plan_gen_num) + region_info region_info_bits_to_set = (region_info)(plan_gen_num << RI_PLAN_GEN_SHR); + if ((plan_gen_num < supposed_plan_gen_num) && (heap_segment_pinned_survived (region) != 0)) { if (!settings.demotion) { @@ -11359,6 +11515,7 @@ void gc_heap::set_region_plan_gen_num (heap_segment* region, int plan_gen_num) } get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit); region->flags |= heap_segment_flags_demoted; + region_info_bits_to_set = (region_info)(region_info_bits_to_set | RI_DEMOTED); } else { @@ -11366,6 +11523,17 @@ void gc_heap::set_region_plan_gen_num (heap_segment* region, int plan_gen_num) } heap_segment_plan_gen_num (region) = plan_gen_num; + + uint8_t* region_start = get_region_start (region); + uint8_t* region_end = heap_segment_reserved (region); + + size_t region_index_start = get_basic_region_index_for_address (region_start); + size_t region_index_end = get_basic_region_index_for_address (region_end); + for (size_t region_index = region_index_start; region_index < region_index_end; region_index++) + { + assert (plan_gen_num <= max_generation); + map_region_to_generation[region_index] = (region_info)(region_info_bits_to_set | (map_region_to_generation[region_index] & ~(RI_PLAN_GEN_MASK|RI_DEMOTED))); + } } inline @@ -11376,6 +11544,42 @@ void gc_heap::set_region_plan_gen_num_sip (heap_segment* region, int plan_gen_nu set_region_plan_gen_num (region, plan_gen_num); } } + +void gc_heap::set_region_sweep_in_plan (heap_segment*region) +{ + heap_segment_swept_in_plan (region) = true; + + // this should be a basic region + assert (get_region_size (region) == global_region_allocator.get_region_alignment()); + + uint8_t* region_start = get_region_start (region); + size_t region_index = get_basic_region_index_for_address (region_start); + map_region_to_generation[region_index] = (region_info)(map_region_to_generation[region_index] | RI_SIP); +} + +void gc_heap::clear_region_sweep_in_plan (heap_segment*region) +{ + heap_segment_swept_in_plan (region) = false; + + // this should be a basic region + assert (get_region_size (region) == global_region_allocator.get_region_alignment()); + + uint8_t* region_start = get_region_start (region); + size_t region_index = get_basic_region_index_for_address (region_start); + map_region_to_generation[region_index] = (region_info)(map_region_to_generation[region_index] & ~RI_SIP); +} + +void gc_heap::clear_region_demoted (heap_segment* region) +{ + region->flags &= ~heap_segment_flags_demoted; + + // this should be a basic region + assert (get_region_size (region) == global_region_allocator.get_region_alignment()); + + uint8_t* region_start = get_region_start (region); + size_t region_index = get_basic_region_index_for_address (region_start); + map_region_to_generation[region_index] = (region_info)(map_region_to_generation[region_index] & ~RI_DEMOTED); +} #endif //USE_REGIONS int gc_heap::get_plan_gen_num (int gen_number) @@ -11514,7 +11718,7 @@ void gc_heap::init_heap_segment (heap_segment* seg, gc_heap* hp #ifdef USE_REGIONS int gen_num_for_region = min (gen_num, max_generation); - heap_segment_gen_num (seg) = (uint8_t)gen_num_for_region; + set_region_gen_num (seg, gen_num_for_region); heap_segment_plan_gen_num (seg) = gen_num_for_region; heap_segment_swept_in_plan (seg) = false; #endif //USE_REGIONS @@ -14119,11 +14323,14 @@ gc_heap::init_gc_heap (int h_number) if (heap_number == 0) { stomp_write_barrier_initialize( -#if defined(MULTIPLE_HEAPS) || defined(USE_REGIONS) +#if defined(USE_REGIONS) + ephemeral_low, ephemeral_high, + map_region_to_generation_skewed, (uint8_t)min_segment_size_shr +#elif defined(MULTIPLE_HEAPS) reinterpret_cast(1), reinterpret_cast(~0) #else ephemeral_low, ephemeral_high -#endif //!MULTIPLE_HEAPS || USE_REGIONS +#endif //MULTIPLE_HEAPS || USE_REGIONS ); } @@ -21170,6 +21377,10 @@ void gc_heap::gc1() } #ifdef USE_REGIONS distribute_free_regions(); + verify_region_to_generation_map (); + compute_gc_and_ephemeral_range (settings.condemned_generation, true); + stomp_write_barrier_ephemeral (ephemeral_low, ephemeral_high, + map_region_to_generation_skewed, (uint8_t)min_segment_size_shr); #endif //USE_REGIONS #ifdef FEATURE_LOH_COMPACTION @@ -21234,6 +21445,10 @@ void gc_heap::gc1() { #ifdef USE_REGIONS distribute_free_regions(); + verify_region_to_generation_map (); + compute_gc_and_ephemeral_range (settings.condemned_generation, true); + stomp_write_barrier_ephemeral (ephemeral_low, ephemeral_high, + map_region_to_generation_skewed, (uint8_t)min_segment_size_shr); if (settings.condemned_generation == max_generation) { // age and print all kinds of free regions @@ -23197,30 +23412,40 @@ inline bool is_in_heap_range (uint8_t* o) return (o != nullptr); #endif //FEATURE_BASICFREEZE } + +inline bool gc_heap::is_in_gc_range (uint8_t* o) +{ +#ifdef FEATURE_BASICFREEZE + // we may have frozen objects in read only segments + // outside of the reserved address range of the gc heap + assert (((g_gc_lowest_address <= o) && (o < g_gc_highest_address)) || + (o == nullptr) || (ro_segment_lookup (o) != nullptr)); +#else //FEATURE_BASICFREEZE + // without frozen objects, every non-null pointer must be + // within the heap + assert ((o == nullptr) || (g_gc_lowest_address <= o) && (o < g_gc_highest_address)); +#endif //FEATURE_BASICFREEZE + return ((gc_low <= o) && (o < gc_high)); +} #endif //USE_REGIONS inline BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high, int condemned_gen) { #ifdef USE_REGIONS - assert (low == 0); - assert (high == 0); - if (is_in_heap_range (o)) + if ((o >= low) && (o < high)) { - BOOL already_marked = marked (o); - if (already_marked) - return FALSE; - if (condemned_gen == max_generation) + if (condemned_gen != max_generation && get_region_gen_num (o) > condemned_gen) { - set_marked (o); - return TRUE; + return FALSE; } - int gen = get_region_gen_num (o); - if (gen <= condemned_gen) + BOOL already_marked = marked (o); + if (already_marked) { - set_marked (o); - return TRUE; + return FALSE; } + set_marked(o); + return TRUE; } return FALSE; #else //USE_REGIONS @@ -24231,7 +24456,7 @@ inline void gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL) { #ifdef USE_REGIONS - if (is_in_heap_range (o) && is_in_condemned_gc (o)) + if (is_in_gc_range (o) && is_in_condemned_gc (o)) { mark_object_simple (&o THREAD_NUMBER_ARG); } @@ -25619,6 +25844,113 @@ void gc_heap::record_mark_time (uint64_t& mark_time, } #endif // FEATURE_EVENT_TRACE +#ifdef USE_REGIONS +void gc_heap::verify_region_to_generation_map() +{ +#ifdef _DEBUG + uint8_t* local_ephemeral_low = MAX_PTR; + uint8_t* local_ephemeral_high = nullptr; + for (int gen_number = soh_gen0; gen_number < total_generation_count; gen_number++) + { +#ifdef MULTIPLE_HEAPS + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; +#else //MULTIPLE_HEAPS + { + gc_heap* hp = pGenGCHeap; +#endif //MULTIPLE_HEAPS + generation *gen = hp->generation_of (gen_number); + for (heap_segment *region = generation_start_segment (gen); region != nullptr; region = heap_segment_next (region)) + { + size_t region_index_start = get_basic_region_index_for_address (get_region_start (region)); + size_t region_index_end = get_basic_region_index_for_address (heap_segment_reserved (region)); + int gen_num = min (gen_number, soh_gen2); + assert (gen_num == heap_segment_gen_num (region)); + int plan_gen_num = heap_segment_plan_gen_num (region); + bool is_demoted = (region->flags & heap_segment_flags_demoted) != 0; + bool is_sweep_in_plan = heap_segment_swept_in_plan (region); + for (size_t region_index = region_index_start; region_index < region_index_end; region_index++) + { + region_info region_info_bits = map_region_to_generation[region_index]; + assert ((region_info_bits & RI_GEN_MASK) == gen_num); + assert ((region_info_bits >> RI_PLAN_GEN_SHR) == plan_gen_num); + assert (((region_info_bits & RI_SIP) != 0) == is_sweep_in_plan); + assert (((region_info_bits & RI_DEMOTED) != 0) == is_demoted); + } + } + } + } +#endif //_DEBUG +} + +// recompute ephemeral range - it may have become too large because of temporary allocation +// and deallocation of regions +void gc_heap::compute_gc_and_ephemeral_range (int condemned_gen_number, bool end_of_gc_p) +{ + ephemeral_low = MAX_PTR; + ephemeral_high = nullptr; + gc_low = MAX_PTR; + gc_high = nullptr; + if (condemned_gen_number >= soh_gen2 || end_of_gc_p) + { + gc_low = g_gc_lowest_address; + gc_high = g_gc_highest_address; + } + if (end_of_gc_p) + { +#if 1 + // simple and safe value + ephemeral_low = g_gc_lowest_address; +#else + // conservative value - should still avoid changing + // ephemeral bounds in the write barrier while app is running + // scan our address space for a region that is either free + // or in an ephemeral generation + uint8_t* addr = g_gc_lowest_address; + while (true) + { + heap_segment* region = get_region_info (addr); + if (is_free_region (region)) + break; + if (heap_segment_gen_num (region) <= soh_gen1) + break; + addr += ((size_t)1) << min_segment_size_shr; + } + ephemeral_low = addr; +#endif + ephemeral_high = g_gc_highest_address; + } + else + { + for (int gen_number = soh_gen0; gen_number <= soh_gen1; gen_number++) + { + #ifdef MULTIPLE_HEAPS + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + #else //MULTIPLE_HEAPS + { + gc_heap* hp = pGenGCHeap; + #endif //MULTIPLE_HEAPS + generation *gen = hp->generation_of (gen_number); + for (heap_segment *region = generation_start_segment (gen); region != nullptr; region = heap_segment_next (region)) + { + ephemeral_low = min ((uint8_t*)ephemeral_low, get_region_start (region)); + ephemeral_high = max ((uint8_t*)ephemeral_high, heap_segment_reserved (region)); + if (gen_number <= condemned_gen_number) + { + gc_low = min (gc_low, get_region_start (region)); + gc_high = max (gc_high, heap_segment_reserved (region)); + } + } + } + } + } + dprintf (2, ("ephemeral_low = %Ix, ephemeral_high = %Ix, gc_low = %Ix, gc_high = %Ix", (uint8_t*)ephemeral_low, (uint8_t*)ephemeral_high, gc_low, gc_high)); +} +#endif //USE_REGIONS + void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p) { assert (settings.concurrent == FALSE); @@ -25752,6 +26084,8 @@ void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p) special_sweep_p = false; region_count = global_region_allocator.get_used_region_count(); grow_mark_list_piece(); + verify_region_to_generation_map(); + compute_gc_and_ephemeral_range (condemned_gen_number, false); #endif //USE_REGIONS GCToEEInterface::BeforeGcScanRoots(condemned_gen_number, /* is_bgc */ false, /* is_concurrent */ false); @@ -30220,6 +30554,10 @@ void gc_heap::plan_phase (int condemned_gen_number) } #endif //!USE_REGIONS +#ifdef USE_REGIONS + verify_region_to_generation_map (); +#endif //USE_REGIONS + #ifdef MULTIPLE_HEAPS //join all threads to make sure they are synchronized dprintf(3, ("Restarting after Promotion granted")); @@ -30523,8 +30861,8 @@ heap_segment* gc_heap::find_first_valid_region (heap_segment* region, bool compa } // Take this opportunity to make sure all the regions left with flags only for this GC are reset. - heap_segment_swept_in_plan (current_region) = false; - current_region->flags &= ~heap_segment_flags_demoted; + clear_region_sweep_in_plan (current_region); + clear_region_demoted (current_region); return current_region; } @@ -30937,7 +31275,7 @@ void gc_heap::sweep_region_in_plan (heap_segment* region, uint8_t**& mark_list_next, uint8_t** mark_list_index) { - heap_segment_swept_in_plan (region) = true; + set_region_sweep_in_plan (region); region->init_free_list(); @@ -31070,7 +31408,7 @@ void gc_heap::check_demotion_helper_sip (uint8_t** pval, int parent_gen_num, uin uint8_t* child_object = *pval; if (!is_in_heap_range (child_object)) return; - if (!child_object) return; + assert (child_object != nullptr); int child_object_plan_gen = get_region_plan_gen_num (child_object); if (child_object_plan_gen < parent_gen_num) @@ -31614,7 +31952,7 @@ void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL) { uint8_t* old_address = *pold_address; #ifdef USE_REGIONS - if (!is_in_heap_range (old_address) || !should_check_brick_for_reloc (old_address)) + if (!is_in_gc_range (old_address) || !should_check_brick_for_reloc (old_address)) { return; } @@ -32585,6 +32923,10 @@ void gc_heap::relocate_phase (int condemned_gen_number, } #endif //FEATURE_EVENT_TRACE +#ifdef USE_REGIONS + verify_region_to_generation_map(); +#endif //USE_REGIONS + #ifdef MULTIPLE_HEAPS //join all threads to make sure they are synchronized dprintf(3, ("Restarting for relocation")); @@ -36838,11 +37180,21 @@ BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end) while (1) { // Find a non-zero bundle - while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0)) + while (cardb < end_cardb) { - cardb++; + uint32_t cbw = card_bundle_table[card_bundle_word(cardb)] >> card_bundle_bit (cardb); + DWORD bit_index; + if (BitScanForward (&bit_index, cbw)) + { + cardb += bit_index; + break; + } + else + { + cardb += sizeof(cbw)*8 - card_bundle_bit (cardb); + } } - if (cardb == end_cardb) + if (cardb >= end_cardb) return FALSE; uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)]; @@ -36857,8 +37209,19 @@ BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end) cardw = (card_word - &card_table[0]); return TRUE; } - else if ((cardw <= card_bundle_cardw (cardb)) && - (card_word == &card_table [card_bundle_cardw (cardb+1)])) + // explore the beginning of the card bundle so we can possibly clear it + if (cardw == (card_bundle_cardw (cardb) + 1) && !card_table[cardw-1]) + { + cardw--; + } + // explore the end of the card bundle so we can possibly clear it + card_word_end = &card_table[card_bundle_cardw (cardb+1)]; + while ((card_word < card_word_end) && !(*card_word)) + { + card_word++; + } + if ((cardw <= card_bundle_cardw (cardb)) && + (card_word == card_word_end)) { // a whole bundle was explored and is empty dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[", @@ -36914,14 +37277,23 @@ BOOL gc_heap::find_card(uint32_t* card_table, // Find the first card which is set last_card_word = &card_table [card_word (card)]; bit_position = card_bit (card); - card_word_value = (*last_card_word) >> bit_position; +#ifdef CARD_BUNDLE + // if we have card bundles, consult them before fetching a new card word + if (bit_position == 0) + { + card_word_value = 0; + } + else +#endif + { + card_word_value = (*last_card_word) >> bit_position; + } if (!card_word_value) { - bit_position = 0; #ifdef CARD_BUNDLE // Using the card bundle, go through the remaining card words between here and // card_word_end until we find one that is non-zero. - size_t lcw = card_word(card) + 1; + size_t lcw = card_word(card) + (bit_position != 0); if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE) { return FALSE; @@ -36931,6 +37303,7 @@ BOOL gc_heap::find_card(uint32_t* card_table, last_card_word = &card_table [lcw]; card_word_value = *last_card_word; } + bit_position = 0; #else //CARD_BUNDLE // Go through the remaining card words between here and card_word_end until we find // one that is non-zero. @@ -36955,11 +37328,11 @@ BOOL gc_heap::find_card(uint32_t* card_table, // Look for the lowest bit set if (card_word_value) { - while (!(card_word_value & 1)) - { - bit_position++; - card_word_value = card_word_value / 2; - } + DWORD bit_index; + uint8_t res = BitScanForward (&bit_index, card_word_value); + assert (res != 0); + card_word_value >>= bit_index; + bit_position += bit_index; } // card is the card word index * card size + the bit index within the card @@ -37054,8 +37427,9 @@ gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen, assert (nhigh == 0); assert (next_boundary == 0); uint8_t* child_object = *poo; - if (!is_in_heap_range (child_object)) + if ((child_object < ephemeral_low) || (ephemeral_high <= child_object)) return; + int child_object_gen = get_region_gen_num (child_object); int saved_child_object_gen = child_object_gen; uint8_t* saved_child_object = child_object; @@ -37075,7 +37449,7 @@ gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen, { cg_pointers_found++; dprintf (4, ("cg pointer %Ix found, %Id so far", - (size_t)*poo, cg_pointers_found )); + (size_t)*poo, cg_pointers_found )); } #else //USE_REGIONS assert (condemned_gen == -1); @@ -44416,6 +44790,12 @@ HRESULT GCHeap::Initialize() initGCShadow(); // If we are debugging write barriers, initialize heap shadow +#ifdef USE_REGIONS + gc_heap::ephemeral_low = MAX_PTR; + + gc_heap::ephemeral_high = nullptr; +#endif //!USE_REGIONS + #ifdef MULTIPLE_HEAPS for (uint32_t i = 0; i < nhp; i++) @@ -44572,7 +44952,7 @@ bool GCHeap::IsPromoted(Object* object) else { #ifdef USE_REGIONS - return (is_in_heap_range (o) ? (gc_heap::is_in_condemned_gc (o) ? gc_heap::is_mark_set (o) : true) : true); + return (gc_heap::is_in_gc_range (o) ? (gc_heap::is_in_condemned_gc (o) ? gc_heap::is_mark_set (o) : true) : true); #else gc_heap* hp = gc_heap::heap_of (o); return (!((o < hp->gc_high) && (o >= hp->gc_low)) diff --git a/src/coreclr/gc/gcconfig.h b/src/coreclr/gc/gcconfig.h index e99a9ecded330c..50eda8e110e8ba 100644 --- a/src/coreclr/gc/gcconfig.h +++ b/src/coreclr/gc/gcconfig.h @@ -134,7 +134,7 @@ class GCConfigStringHolder INT_CONFIG (GCHeapHardLimitPOHPercent, "GCHeapHardLimitPOHPercent", "System.GC.HeapHardLimitPOHPercent", 0, "Specifies the GC heap POH usage as a percentage of the total memory") \ INT_CONFIG (GCEnabledInstructionSets, "GCEnabledInstructionSets", NULL, -1, "Specifies whether GC can use AVX2 or AVX512F - 0 for neither, 1 for AVX2, 3 for AVX512F")\ INT_CONFIG (GCConserveMem, "GCConserveMemory", "System.GC.ConserveMemory", 0, "Specifies how hard GC should try to conserve memory - values 0-9") \ - + INT_CONFIG (GCWriteBarrier, "GCWriteBarrier", NULL, 0, "Specifies whether GC should use more precise but slower write barrier") // This class is responsible for retreiving configuration information // for how the GC should operate. class GCConfig @@ -182,6 +182,14 @@ enum HeapVerifyFlags { HEAPVERIFY_DEEP_ON_COMPACT = 0x80 // Performs deep object verfication only on compacting GCs. }; +enum WriteBarrierFlavor +{ + WRITE_BARRIER_DEFAULT = 0, + WRITE_BARRIER_REGION_BIT = 1, + WRITE_BARRIER_REGION_BYTE = 2, + WRITE_BARRIER_SERVER = 3, +}; + // Initializes the GCConfig subsystem. Must be called before accessing any // configuration information. static void Initialize(); diff --git a/src/coreclr/gc/gcinterface.h b/src/coreclr/gc/gcinterface.h index 8bdcf0c4aa22b8..023f257d10fa58 100644 --- a/src/coreclr/gc/gcinterface.h +++ b/src/coreclr/gc/gcinterface.h @@ -108,6 +108,15 @@ struct WriteBarrierParameters // The new write watch table, if we are using our own write watch // implementation. Used for WriteBarrierOp::SwitchToWriteWatch only. uint8_t* write_watch_table; + + // mapping table from region index to generation + uint8_t* region_to_generation_table; + + // shift count - how many bits to shift right to obtain region index from address + uint8_t region_shr; + + // whether to use the more precise but slower write barrier + bool region_use_bitwise_write_barrier; }; struct EtwGCSettingsInfo diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 2c18c93681e028..4c56fd886b6cc3 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -1206,6 +1206,9 @@ enum bookkeeping_element #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP software_write_watch_table_element, #endif +#ifdef USE_REGIONS + region_to_generation_table_element, +#endif //USE_REGIONS seg_mapping_table_element, #ifdef BACKGROUND_GC mark_array_element, @@ -1378,6 +1381,12 @@ class gc_heap PER_HEAP void set_region_plan_gen_num_sip (heap_segment* region, int plan_gen_num); PER_HEAP + void set_region_sweep_in_plan (heap_segment* region); + PER_HEAP + void clear_region_sweep_in_plan (heap_segment* region); + PER_HEAP + void clear_region_demoted (heap_segment* region); + PER_HEAP void decide_on_demotion_pin_surv (heap_segment* region); PER_HEAP void skip_pins_in_alloc_region (generation* consing_gen, int plan_gen_num); @@ -1445,6 +1454,12 @@ class gc_heap // This relocates the SIP regions and return the next non SIP region. PER_HEAP heap_segment* relocate_advance_to_non_sip (heap_segment* region); + + PER_HEAP_ISOLATED + void verify_region_to_generation_map(); + + PER_HEAP_ISOLATED + void compute_gc_and_ephemeral_range (int condemned_gen_number, bool end_of_gc_p); #ifdef STRESS_REGIONS PER_HEAP void pin_by_gc (uint8_t* object); @@ -3083,6 +3098,8 @@ class gc_heap #endif //BACKGROUND_GC #ifdef USE_REGIONS + PER_HEAP_ISOLATED + bool is_in_gc_range (uint8_t* o); // o is guaranteed to be in the heap range. PER_HEAP_ISOLATED bool is_in_condemned_gc (uint8_t* o); @@ -3673,6 +3690,39 @@ class gc_heap size_t* old_card_survived_per_region; PER_HEAP_ISOLATED size_t region_count; + + // table mapping region number to generation + // there are actually two generation numbers per entry: + // - the region's current generation + // - the region's planned generation, i.e. after the GC + // and there are flags + // - whether the region is sweep in plan + // - and whether the region is demoted + enum region_info : uint8_t + { + // lowest 2 bits are current generation number + RI_GEN_0 = 0x0, + RI_GEN_1 = 0x1, + RI_GEN_2 = 0x2, + RI_GEN_MASK = 0x3, + + // we have 4 bits available for flags, of which 2 are used + RI_SIP = 0x4, + RI_DEMOTED = 0x8, + + // top 2 bits are planned generation number + RI_PLAN_GEN_SHR = 0x6, // how much to shift the value right to obtain plan gen + RI_PLAN_GEN_0 = 0x00, + RI_PLAN_GEN_1 = 0x40, + RI_PLAN_GEN_2 = 0x80, + RI_PLAN_GEN_MASK= 0xC0, + }; + PER_HEAP_ISOLATED + region_info* map_region_to_generation; + // same table as above, but skewed so that we can index + // directly with address >> min_segment_size_shr + PER_HEAP_ISOLATED + region_info* map_region_to_generation_skewed; #endif //USE_REGIONS #define max_oom_history_count 4 @@ -3723,7 +3773,13 @@ class gc_heap PER_HEAP void exit_gc_done_event_lock(); -#ifndef USE_REGIONS +#ifdef USE_REGIONS + PER_HEAP_ISOLATED + VOLATILE(uint8_t*) ephemeral_low; //lowest ephemeral address + + PER_HEAP_ISOLATED + VOLATILE(uint8_t*) ephemeral_high; //highest ephemeral address +#else //!USE_REGIONS PER_HEAP uint8_t* ephemeral_low; //lowest ephemeral address @@ -4089,13 +4145,19 @@ class gc_heap PER_HEAP uint64_t time_bgc_last; -//#ifndef USE_REGIONS +#ifdef USE_REGIONS + PER_HEAP_ISOLATED + uint8_t* gc_low; // low end of the lowest region being condemned + + PER_HEAP_ISOLATED + uint8_t* gc_high; // high end of the highest region being condemned +#else // USE_REGIONS PER_HEAP uint8_t* gc_low; // lowest address being condemned PER_HEAP uint8_t* gc_high; // highest address being condemned -//#endif //USE_REGIONS +#endif //USE_REGIONS PER_HEAP size_t mark_stack_tos; diff --git a/src/coreclr/vm/amd64/JitHelpers_Fast.asm b/src/coreclr/vm/amd64/JitHelpers_Fast.asm index 9ec3a8617dac3b..5c654ab72a0438 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Fast.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Fast.asm @@ -20,6 +20,9 @@ EXTERN g_ephemeral_high:QWORD EXTERN g_lowest_address:QWORD EXTERN g_highest_address:QWORD EXTERN g_card_table:QWORD +EXTERN g_region_shr:BYTE +EXTERN g_region_use_bitwise_write_barrier:BYTE +EXTERN g_region_to_generation_table:QWORD ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN g_card_bundle_table:QWORD @@ -135,6 +138,31 @@ endif align 16 Exit: REPRET + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + else ; JIT_WriteBarrier_PostGrow64 @@ -305,6 +333,60 @@ endif cmp rcx, [g_ephemeral_high] jnb Exit + ; do the following checks only if we are allowed to trash rax + ; otherwise we don't have enough registers +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + mov rax, rcx + + mov cl, [g_region_shr] + test cl, cl + je SkipCheck + + ; check if the source is in gen 2 - then it's not an ephemeral pointer + shr rax, cl + add rax, [g_region_to_generation_table] + cmp byte ptr [rax], 82h + je Exit + + ; check if the destination happens to be in gen 0 + mov rax, rdi + shr rax, cl + add rax, [g_region_to_generation_table] + cmp byte ptr [rax], 0 + je Exit + SkipCheck: + + cmp [g_region_use_bitwise_write_barrier], 0 + je CheckCardTableByte + + ; compute card table bit + mov rcx, rdi + mov al, 1 + shr rcx, 8 + and cl, 7 + shl al, cl + + ; move current rdi value into rcx and then increment the pointers + mov rcx, rdi + add rsi, 8h + add rdi, 8h + + ; Check if we need to update the card table + ; Calc pCardByte + shr rcx, 0Bh + add rcx, [g_card_table] + + ; Check if this card table bit is already set + test byte ptr [rcx], al + je SetCardTableBit + REPRET + + SetCardTableBit: + lock or byte ptr [rcx], al + jmp CheckCardBundle +endif +CheckCardTableByte: + ; move current rdi value into rcx and then increment the pointers mov rcx, rdi add rsi, 8h @@ -322,6 +404,9 @@ endif UpdateCardTable: mov byte ptr [rcx], 0FFh + + CheckCardBundle: + ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ; check if we need to update the card bundle table ; restore destination address from rdi - rdi has been incremented by 8 already diff --git a/src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm b/src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm index 1bd47cae1fd8a2..68ab221278442b 100644 --- a/src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm +++ b/src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm @@ -196,6 +196,152 @@ endif ret LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + align 8 + + ; Do the move into the GC . It is correct to take an AV here, the EH code + ; figures out that this came from a WriteBarrier and correctly maps it back + ; to the managed method which called the WriteBarrier (see setup in + ; InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rcx], rdx + + mov r8, rcx + +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration + mov rax, 0F0F0F0F0F0F0F0F0h + +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest + shr rcx, 16h ; compute region index + + ; Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rcx + rax], 0 + jne NotGen0 + REPRET + + NOP_2_BYTE ; padding for alignment of constant + + NotGen0: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jae NotLow + ret + NotLow: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_Upper + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jb NotHigh + REPRET + NotHigh: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrSrc + shr rdx, 16h ; compute region index + mov dl, [rdx + rax] + cmp dl, [rcx + rax] + jb isOldToYoung + REPRET + nop + + IsOldToYoung: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable + mov rax, 0F0F0F0F0F0F0F0F0h + + shr r8, 0Bh + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardTable + REPRET + + UpdateCardTable: + mov byte ptr [r8 + rax], 0FFh +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + shr r8, 0Ah +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable + mov rax, 0F0F0F0F0F0F0F0F0h + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardBundleTable + REPRET + + UpdateCardBundleTable: + mov byte ptr [r8 + rax], 0FFh +endif + ret +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + align 8 + + ; Do the move into the GC . It is correct to take an AV here, the EH code + ; figures out that this came from a WriteBarrier and correctly maps it back + ; to the managed method which called the WriteBarrier (see setup in + ; InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rcx], rdx + + mov r8, rcx + +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration + mov rax, 0F0F0F0F0F0F0F0F0h + +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest + shr rcx, 16h ; compute region index + + ; Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rcx + rax], 0 + jne NotGen0 + REPRET + + NOP_2_BYTE ; padding for alignment of constant + + NotGen0: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jae NotLow + ret + NotLow: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_Upper + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jb NotHigh + REPRET + NotHigh: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrSrc + shr rdx, 16h ; compute region index + mov dl, [rdx + rax] + cmp dl, [rcx + rax] + jb isOldToYoung + REPRET + nop + + IsOldToYoung: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable + mov rax, 0F0F0F0F0F0F0F0F0h + + mov ecx, r8d + shr r8, 0Bh + shr ecx, 8 + and ecx, 7 + mov dl, 1 + shl dl, cl + test byte ptr [r8 + rax], dl + je UpdateCardTable + REPRET + + UpdateCardTable: + lock or byte ptr [r8 + rax], dl + +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable + mov rax, 0F0F0F0F0F0F0F0F0h + shr r8, 0Ah + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardBundleTable + REPRET + + UpdateCardBundleTable: + mov byte ptr [r8 + rax], 0FFh +endif + ret +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT + endif @@ -405,6 +551,172 @@ endif LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT endif + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + align 8 + + ; Do the move into the GC . It is correct to take an AV here, the EH code + ; figures out that this came from a WriteBarrier and correctly maps it back + ; to the managed method which called the WriteBarrier (see setup in + ; InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rcx], rdx + + ; Update the write watch table if necessary + mov rax, rcx +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable + mov r8, 0F0F0F0F0F0F0F0F0h + shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift + add rax, r8 + mov r8, rcx +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest + shr rcx, 16h ; compute region index + cmp byte ptr [rax], 0h + jne JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration + mov byte ptr [rax], 0FFh + +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration + mov rax, 0F0F0F0F0F0F0F0F0h + + ; Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rcx + rax], 0 + jne NotGen0 + REPRET + + NOP_2_BYTE ; padding for alignment of constant + NOP_2_BYTE ; padding for alignment of constant + NOP_2_BYTE ; padding for alignment of constant + + NotGen0: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Lower + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jae NotLow + ret + NotLow: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Upper + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jb NotHigh + REPRET + NotHigh: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrSrc + shr rdx, 16h ; compute region index + mov dl, [rdx + rax] + cmp dl, [rcx + rax] + jb isOldToYoung + REPRET + nop + + IsOldToYoung: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable + mov rax, 0F0F0F0F0F0F0F0F0h + + shr r8, 0Bh + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardTable + REPRET + + UpdateCardTable: + mov byte ptr [r8 + rax], 0FFh +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + shr r8, 0Ah +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable + mov rax, 0F0F0F0F0F0F0F0F0h + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardBundleTable + REPRET + + UpdateCardBundleTable: + mov byte ptr [r8 + rax], 0FFh +endif + ret +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + align 8 + + ; Do the move into the GC . It is correct to take an AV here, the EH code + ; figures out that this came from a WriteBarrier and correctly maps it back + ; to the managed method which called the WriteBarrier (see setup in + ; InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rcx], rdx + + ; Update the write watch table if necessary + mov rax, rcx +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable + mov r8, 0F0F0F0F0F0F0F0F0h + shr rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift + add rax, r8 + mov r8, rcx +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest + shr rcx, 16h ; compute region index + cmp byte ptr [rax], 0h + jne JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration + mov byte ptr [rax], 0FFh + +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration + mov rax, 0F0F0F0F0F0F0F0F0h + + ; Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rcx + rax], 0 + jne NotGen0 + REPRET + + NOP_2_BYTE ; padding for alignment of constant + NOP_2_BYTE ; padding for alignment of constant + NOP_2_BYTE ; padding for alignment of constant + + NotGen0: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Lower + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jae NotLow + ret + NotLow: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Upper + mov r9, 0F0F0F0F0F0F0F0F0h + cmp rdx, r9 + jb NotHigh + REPRET + NotHigh: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrSrc + shr rdx, 16h ; compute region index + mov dl, [rdx + rax] + cmp dl, [rcx + rax] + jb isOldToYoung + REPRET + nop + + IsOldToYoung: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable + mov rax, 0F0F0F0F0F0F0F0F0h + + mov ecx, r8d + shr r8, 0Bh + shr ecx, 8 + and ecx, 7 + mov dl, 1 + shl dl, cl + test byte ptr [r8 + rax], dl + je UpdateCardTable + REPRET + + UpdateCardTable: + lock or byte ptr [r8 + rax], dl +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable + mov rax, 0F0F0F0F0F0F0F0F0h + shr r8, 0Ah + cmp byte ptr [r8 + rax], 0FFh + jne UpdateCardBundleTable + REPRET + + UpdateCardBundleTable: + mov byte ptr [r8 + rax], 0FFh +endif + ret +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + endif diff --git a/src/coreclr/vm/amd64/jithelpers_fast.S b/src/coreclr/vm/amd64/jithelpers_fast.S index 5d7cb379ce96d1..9b582103509c2a 100644 --- a/src/coreclr/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/vm/amd64/jithelpers_fast.S @@ -138,6 +138,31 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT .balign 16 Exit: REPRET + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + NOP_3_BYTE + #else // JIT_WriteBarrier_PostGrow64 @@ -330,6 +355,41 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT cmp rcx, [rax] jnb Exit_ByRefWriteBarrier + mov rax, rcx + + PREPARE_EXTERNAL_VAR g_region_shr, rcx + mov cl, [rcx] + test cl, cl + je SkipCheck_ByRefWriteBarrier + + // check if the source is in gen 2 - then it's not an ephemeral pointer + shr rax, cl + PREPARE_EXTERNAL_VAR g_region_to_generation_table, r10 + mov r10, [r10] + cmp byte ptr [rax + r10], 0x82 + je Exit_ByRefWriteBarrier + + // check if the destination happens to be in gen 0 + mov rax, rdi + shr rax, cl + cmp byte ptr [rax + r10], 0 + je Exit_ByRefWriteBarrier + SkipCheck_ByRefWriteBarrier: + + PREPARE_EXTERNAL_VAR g_card_table, r10 + mov r10, [r10] + + PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, rax + cmp byte ptr [rax], 0 + je CheckCardTableByte_ByRefWriteBarrier + + // compute card table bit + mov ecx, edi + mov al, 1 + shr ecx, 8 + and cl, 7 + shl al, cl + // move current rdi value into rcx and then increment the pointers mov rcx, rdi add rsi, 0x8 @@ -337,19 +397,31 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT // Check if we need to update the card table // Calc pCardByte - shr rcx, 0x0B + shr rcx, 0xB + // Check if this card table bit is already set + test byte ptr [rcx + r10], al + je SetCardTableBit_ByRefWriteBarrier + REPRET + + SetCardTableBit_ByRefWriteBarrier: + lock or byte ptr [rcx + r10], al - PREPARE_EXTERNAL_VAR g_card_table, rax - mov rax, [rax] + jmp CheckCardBundle_ByRefWriteBarrier - // Check if this card is dirty - cmp byte ptr [rcx + rax], 0xFF + CheckCardTableByte_ByRefWriteBarrier: + // move current rdi value into rcx and then increment the pointers + mov rcx, rdi + add rsi, 0x8 + add rdi, 0x8 - jne UpdateCardTable_ByRefWriteBarrier + shr rcx, 0xB + cmp byte ptr [rcx + r10], 0xFF + jne SetCardTableByte_ByRefWriteBarrier REPRET + SetCardTableByte_ByRefWriteBarrier: + mov byte ptr [rcx + r10], 0xFF - UpdateCardTable_ByRefWriteBarrier: - mov byte ptr [rcx + rax], 0xFF + CheckCardBundle_ByRefWriteBarrier: #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already) diff --git a/src/coreclr/vm/amd64/jithelpers_fastwritebarriers.S b/src/coreclr/vm/amd64/jithelpers_fastwritebarriers.S index 70d2f2072aa51a..f987751bdcb358 100644 --- a/src/coreclr/vm/amd64/jithelpers_fastwritebarriers.S +++ b/src/coreclr/vm/amd64/jithelpers_fastwritebarriers.S @@ -5,7 +5,7 @@ #include "unixasmmacros.inc" - .balign 8 + .balign 16 LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT // Do the move into the GC . It is correct to take an AV here, the EH code // figures out that this came from a WriteBarrier and correctly maps it back @@ -26,7 +26,7 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Lower #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES .byte 0x72, 0x4B #else - .byte 0x72, 0x23 + .byte 0x72, 0x2b #endif // jb Exit_PreGrow64 @@ -72,7 +72,7 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT - .balign 8 + .balign 16 // See comments for JIT_WriteBarrier_PreGrow (above). LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT // Do the move into the GC . It is correct to take an AV here, the EH code @@ -95,9 +95,9 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Lower cmp rsi, rax #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - .byte 0x72, 0x53 + .byte 0x72, 0x5b #else - .byte 0x72, 0x33 + .byte 0x72, 0x3b #endif // jb Exit_PostGrow64 @@ -109,9 +109,9 @@ PATCH_LABEL JIT_WriteBarrier_PostGrow64_Patch_Label_Upper cmp rsi, r8 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - .byte 0x73, 0x43 + .byte 0x73, 0x4b #else - .byte 0x73, 0x23 + .byte 0x73, 0x2b #endif // jae Exit_PostGrow64 @@ -159,7 +159,7 @@ LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT #ifdef FEATURE_SVR_GC - .balign 8 + .balign 16 LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT // // SVR GC has multiple heaps, so it cannot provide one single @@ -213,10 +213,165 @@ LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT #endif + .balign 16 +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + + // Do the move into the GC . It is correct to take an AV here, the EH code + // figures out that this came from a WriteBarrier and correctly maps it back + // to the managed method which called the WriteBarrier (see setup in + // InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rdi], rsi + + mov r8, rdi + +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration + movabs rax, 0xF0F0F0F0F0F0F0F0 + +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest + shr rdi, 0x16 // compute region index + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rdi + rax], 0 + .byte 0x75, 0x04 + //jne NotGen0_Byte_Region64 + REPRET + + NOP_2_BYTE // padding for alignment of constant + + NotGen0_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x73, 0x01 + // jae NotLow_Byte_Region64 + ret + NotLow_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_Upper + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x72, 0x02 + // jb NotHigh_Byte_Region64 + REPRET + NotHigh_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrSrc + shr rsi, 0x16 // compute region index + mov dl, [rsi + rax] + cmp dl, [rdi + rax] + .byte 0x72, 0x03 + // jb IsOldToYoung_Byte_Region64 + REPRET + nop + + IsOldToYoung_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable + movabs rax, 0xF0F0F0F0F0F0F0F0 + + shr r8, 0xB + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardTable_Byte_Region64 + REPRET + + UpdateCardTable_Byte_Region64: + mov byte ptr [r8 + rax], 0xFF +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + shr r8, 0x0A +PATCH_LABEL JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable + movabs rax, 0xF0F0F0F0F0F0F0F0 + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardBundleTable_Byte_Region64 + REPRET + + UpdateCardBundleTable_Byte_Region64: + mov byte ptr [r8 + rax], 0xFF +#endif + ret +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + .balign 16 +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + + // Do the move into the GC . It is correct to take an AV here, the EH code + // figures out that this came from a WriteBarrier and correctly maps it back + // to the managed method which called the WriteBarrier (see setup in + // InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rdi], rsi + + mov r8, rdi + +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration + movabs rax, 0xF0F0F0F0F0F0F0F0 + +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest + shr rdi, 0x16 // compute region index + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rdi + rax], 0 + .byte 0x75, 0x04 + //jne NotGen0_Bit_Region64 + REPRET + + NOP_2_BYTE // padding for alignment of constant + + NotGen0_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x73, 0x01 + // jae NotLow_Bit_Region64 + ret + NotLow_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_Upper + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x72, 0x02 + // jb NotHigh_Bit_Region64 + REPRET + NotHigh_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrSrc + shr rsi, 0x16 // compute region index + mov dl, [rsi + rax] + cmp dl, [rdi + rax] + .byte 0x72, 0x03 + // jb IsOldToYoung_Bit_Region64 + REPRET + nop + + IsOldToYoung_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable + movabs rax, 0xF0F0F0F0F0F0F0F0 + + mov ecx, r8d + shr r8, 0xB + shr ecx, 8 + and ecx, 7 + mov dl, 1 + shl dl, cl + test byte ptr [r8 + rax], dl + .byte 0x74, 0x02 + // je UpdateCardTable_Bit_Region64 + REPRET + + UpdateCardTable_Bit_Region64: + lock or byte ptr [r8 + rax], dl +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +PATCH_LABEL JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable + movabs rax, 0xF0F0F0F0F0F0F0F0 + shr r8, 0x0A + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardBundleTable_Bit_Region64 + REPRET + + UpdateCardBundleTable_Bit_Region64: + mov byte ptr [r8 + rax], 0xFF +#endif + ret +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - .balign 8 + .balign 16 LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT // Regarding patchable constants: // - 64-bit constants have to be loaded into a register @@ -294,7 +449,7 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT - .balign 8 + .balign 16 LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT // Regarding patchable constants: // - 64-bit constants have to be loaded into a register @@ -330,7 +485,7 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower cmp rsi, r11 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - .byte 0x72, 0x55 + .byte 0x72, 0x4d #else .byte 0x72, 0x3d #endif @@ -344,7 +499,7 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper cmp rsi, r10 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - .byte 0x73, 0x43 + .byte 0x73, 0x3b #else .byte 0x73, 0x2b #endif @@ -390,7 +545,7 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT #ifdef FEATURE_SVR_GC - .balign 8 + .balign 16 LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT // Regarding patchable constants: // - 64-bit constants have to be loaded into a register @@ -457,4 +612,182 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT #endif + .balign 16 +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + + // Do the move into the GC . It is correct to take an AV here, the EH code + // figures out that this came from a WriteBarrier and correctly maps it back + // to the managed method which called the WriteBarrier (see setup in + // InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rdi], rsi + + // Update the write watch table if necessary + mov rax, rdi +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable + movabs r8, 0xF0F0F0F0F0F0F0F0 + shr rax, 0x0C // SoftwareWriteWatch::AddressToTableByteIndexShift + add rax, r8 + mov r8, rdi +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest + shr rdi, 0x16 // compute region index + cmp byte ptr [rax], 0x0 + .byte 0x75, 0x03 + // jne CheckGen0_WriteWatch_Byte_Region64 + mov byte ptr [rax], 0xFF + CheckGen0_WriteWatch_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration + mov rax, 0xF0F0F0F0F0F0F0F0 + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rdi + rax], 0 + .byte 0x75, 0x08 + // jne NotGen0_WriteWatch_Byte_Region64 + REPRET + + NOP_2_BYTE // padding for alignment of constant + NOP_2_BYTE // padding for alignment of constant + NOP_2_BYTE // padding for alignment of constant + + NotGen0_WriteWatch_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Lower + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x73, 0x01 + // jae NotLow_WriteWatch_Byte_Region64 + ret + NotLow_WriteWatch_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Upper + mov r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x72, 0x02 + // jb NotHigh_WriteWatch_Byte_Region64 + REPRET + NotHigh_WriteWatch_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrSrc + shr rsi, 0x16 // compute region index + mov dl, [rsi + rax] + cmp dl, [rdi + rax] + .byte 0x72, 0x03 + // jb IsOldToYoung_WriteWatch_Byte_Region64 + REPRET + nop + + IsOldToYoung_WriteWatch_Byte_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable + mov rax, 0xF0F0F0F0F0F0F0F0 + + shr r8, 0xB + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardTable_WriteWatch_Byte_Region64 + REPRET + + UpdateCardTable_WriteWatch_Byte_Region64: + mov byte ptr [r8 + rax], 0xFF +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + shr r8, 0x0A +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable + mov rax, 0xF0F0F0F0F0F0F0F0 + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardBundleTable_WriteWatch_Byte_Region64 + REPRET + + UpdateCardBundleTable_WriteWatch_Byte_Region64: + mov byte ptr [r8 + rax], 0xFF +#endif + ret +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + .balign 16 +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + + // Do the move into the GC . It is correct to take an AV here, the EH code + // figures out that this came from a WriteBarrier and correctly maps it back + // to the managed method which called the WriteBarrier (see setup in + // InitializeExceptionHandling, vm\exceptionhandling.cpp). + mov [rdi], rsi + + // Update the write watch table if necessary + mov rax, rdi +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable + movabs r8, 0xF0F0F0F0F0F0F0F0 + shr rax, 0x0C // SoftwareWriteWatch::AddressToTableByteIndexShift + add rax, r8 + mov r8, rdi +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest + shr rdi, 0x16 // compute region index + cmp byte ptr [rax], 0x0 + .byte 0x75, 0x03 + // jne CheckGen0_WriteWatch_Bit_Region64 + mov byte ptr [rax], 0xFF + CheckGen0_WriteWatch_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration + mov rax, 0xF0F0F0F0F0F0F0F0 + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + cmp byte ptr [rdi + rax], 0 + .byte 0x75, 0x08 + // jne NotGen0_WriteWatch_Bit_Region64 + REPRET + + NOP_2_BYTE // padding for alignment of constant + NOP_2_BYTE // padding for alignment of constant + NOP_2_BYTE // padding for alignment of constant + + NotGen0_WriteWatch_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Lower + movabs r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x73, 0x01 + // jae NotLow_WriteWatch_Bit_Region64 + ret + NotLow_WriteWatch_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Upper + mov r9, 0xF0F0F0F0F0F0F0F0 + cmp rsi, r9 + .byte 0x72, 0x02 + // jb NotHigh_WriteWatch_Bit_Region64 + REPRET + NotHigh_WriteWatch_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrSrc + shr rsi, 0x16 // compute region index + mov dl, [rsi + rax] + cmp dl, [rdi + rax] + .byte 0x72, 0x03 + // jb IsOldToYoung_WriteWatch_Bit_Region64 + REPRET + nop + + IsOldToYoung_WriteWatch_Bit_Region64: +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable + mov rax, 0xF0F0F0F0F0F0F0F0 + + mov ecx, r8d + shr r8, 0xB + shr ecx, 8 + and ecx, 7 + mov dl, 1 + shl dl, cl + test byte ptr [r8 + rax], dl + .byte 0x74, 0x02 + // je UpdateCardTable_WriteWatch_Bit_Region64 + REPRET + + UpdateCardTable_WriteWatch_Bit_Region64: + lock or byte ptr [r8 + rax], dl +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +PATCH_LABEL JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable + mov rax, 0xF0F0F0F0F0F0F0F0 + shr r8, 0x0A + cmp byte ptr [r8 + rax], 0xFF + .byte 0x75, 0x02 + // jne UpdateCardBundleTable_WriteWatch_Bit_Region64 + REPRET + + UpdateCardBundleTable_WriteWatch_Bit_Region64: + mov byte ptr [r8 + rax], 0xFF +#endif + ret +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + #endif diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index 29f2bb7a5afb18..08d5e9493f6c22 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -50,6 +50,31 @@ EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable(); EXTERN_C void JIT_WriteBarrier_SVR64_End(); #endif // FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable(); +#endif +EXTERN_C void JIT_WriteBarrier_Byte_Region64_End(); + +EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable(); +#endif +EXTERN_C void JIT_WriteBarrier_Bit_Region64_End(); + + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable(); @@ -79,6 +104,33 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable(); #endif EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End(); #endif // FEATURE_SVR_GC + +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable(); +#endif +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_End(); + +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable(); +#endif +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); + #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP WriteBarrierManager g_WriteBarrierManager; @@ -152,6 +204,36 @@ void WriteBarrierManager::Validate() #endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES #endif // FEATURE_SVR_GC + PBYTE pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP PBYTE pWriteWatchTableImmediate; @@ -194,6 +276,37 @@ void WriteBarrierManager::Validate() _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); #endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES #endif // FEATURE_SVR_GC + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP } @@ -214,6 +327,10 @@ PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() case WRITE_BARRIER_SVR64: return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64); #endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_Byte_Region64); + case WRITE_BARRIER_BIT_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_Bit_Region64); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_PREGROW64: return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64); @@ -223,6 +340,10 @@ PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() case WRITE_BARRIER_WRITE_WATCH_SVR64: return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64); #endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Byte_Region64); + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Bit_Region64); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP default: UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); @@ -246,6 +367,10 @@ size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBa case WRITE_BARRIER_SVR64: return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_SVR64); #endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Byte_Region64); + case WRITE_BARRIER_BIT_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Bit_Region64); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_PREGROW64: return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PreGrow64); @@ -255,6 +380,10 @@ size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBa case WRITE_BARRIER_WRITE_WATCH_SVR64: return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_SVR64); #endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Byte_Region64); + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Bit_Region64); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_BUFFER: return MARKED_FUNCTION_SIZE(JIT_WriteBarrier); @@ -351,6 +480,50 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, } #endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_BIT_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_PREGROW64: { @@ -407,6 +580,56 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, break; } #endif // FEATURE_SVR_GC + + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP default: @@ -441,12 +664,16 @@ void WriteBarrierManager::Initialize() #ifdef FEATURE_SVR_GC _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_SVR64)); #endif // FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BYTE_REGIONS64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BIT_REGIONS64)); #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_PREGROW64)); _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_POSTGROW64)); #ifdef FEATURE_SVR_GC _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_SVR64)); #endif // FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64)); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP #if !defined(CODECOVERAGE) @@ -454,7 +681,7 @@ void WriteBarrierManager::Initialize() #endif } -bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, WriteBarrierType* pNewWriteBarrierType) +bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType) { // Init code for the JIT_WriteBarrier assembly routine. Since it will be bashed everytime the GC Heap // changes size, we want to do most of the work just once. @@ -476,8 +703,14 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, W break; } #endif - - writeBarrierType = GCHeapUtilities::IsServerHeap() ? WRITE_BARRIER_SVR64 : WRITE_BARRIER_PREGROW64; + if (g_region_shr != 0) + { + writeBarrierType = bUseBitwiseWriteBarrier ? WRITE_BARRIER_BIT_REGIONS64: WRITE_BARRIER_BYTE_REGIONS64; + } + else + { + writeBarrierType = GCHeapUtilities::IsServerHeap() ? WRITE_BARRIER_SVR64 : WRITE_BARRIER_PREGROW64; + } continue; case WRITE_BARRIER_PREGROW64: @@ -495,6 +728,10 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, W break; #endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: + break; + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_PREGROW64: if (bReqUpperBoundsCheck) @@ -510,6 +747,9 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, W case WRITE_BARRIER_WRITE_WATCH_SVR64: break; #endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + break; #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP default: @@ -525,7 +765,7 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, W int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) { WriteBarrierType newType; - if (NeedDifferentWriteBarrier(false, &newType)) + if (NeedDifferentWriteBarrier(false, g_region_use_bitwise_write_barrier, &newType)) { return ChangeWriteBarrierTo(newType, isRuntimeSuspended); } @@ -541,8 +781,12 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) switch (m_currentWriteBarrier) { case WRITE_BARRIER_POSTGROW64: + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP { // Change immediate if different from new g_ephermeral_high. @@ -592,7 +836,7 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus // we need to switch to the WriteBarrier_PostGrow function for good. WriteBarrierType newType; - if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, &newType)) + if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, g_region_use_bitwise_write_barrier, &newType)) { return ChangeWriteBarrierTo(newType, isRuntimeSuspended); } @@ -613,6 +857,8 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #ifdef FEATURE_SVR_GC case WRITE_BARRIER_WRITE_WATCH_SVR64: #endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) { ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pWriteWatchTableImmediate, sizeof(UINT64)); @@ -626,6 +872,36 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + if (*(UINT64*)m_pRegionToGenTableImmediate != (size_t)g_region_to_generation_table) + { + ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pRegionToGenTableImmediate, sizeof(UINT64)); + *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_region_to_generation_table; + stompWBCompleteActions |= SWB_ICACHE_FLUSH; + } + if (*m_pRegionShrDest != g_region_shr) + { + ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrDest, sizeof(UINT8)); + *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; + stompWBCompleteActions |= SWB_ICACHE_FLUSH; + } + if (*m_pRegionShrSrc != g_region_shr) + { + ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrSrc, sizeof(UINT8)); + *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; + stompWBCompleteActions |= SWB_ICACHE_FLUSH; + } + break; + + default: + break; // clang seems to require all enum values to be covered for some reason + } + if (*(UINT64*)m_pCardTableImmediate != (size_t)g_card_table) { ExecutableWriterHolder cardTableImmediateWriterHolder((UINT64*)m_pCardTableImmediate, sizeof(UINT64)); @@ -669,6 +945,14 @@ int WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended) break; #endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64; + break; + + case WRITE_BARRIER_BIT_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64; + break; + default: UNREACHABLE(); } @@ -699,6 +983,14 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) break; #endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_BYTE_REGIONS64; + break; + + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_BIT_REGIONS64; + break; + default: UNREACHABLE(); } diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index 02537f0dd7f545..d12c9253a8e6cc 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1002,6 +1002,9 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(args->ephemeral_high != nullptr); g_ephemeral_low = args->ephemeral_low; g_ephemeral_high = args->ephemeral_high; + g_region_to_generation_table = args->region_to_generation_table; + g_region_shr = args->region_shr; + g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended); break; case WriteBarrierOp::Initialize: @@ -1026,6 +1029,9 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_lowest_address = args->lowest_address; g_highest_address = args->highest_address; + g_region_to_generation_table = args->region_to_generation_table; + g_region_shr = args->region_shr; + g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); // StompWriteBarrierResize does not necessarily bash g_ephemeral_low diff --git a/src/coreclr/vm/gcheaputilities.cpp b/src/coreclr/vm/gcheaputilities.cpp index 088af916fc5bd0..d3301df9ff51fa 100644 --- a/src/coreclr/vm/gcheaputilities.cpp +++ b/src/coreclr/vm/gcheaputilities.cpp @@ -17,6 +17,9 @@ GPTR_IMPL_INIT(uint8_t, g_highest_address, nullptr); GVAL_IMPL_INIT(GCHeapType, g_heap_type, GC_HEAP_INVALID); uint8_t* g_ephemeral_low = (uint8_t*)1; uint8_t* g_ephemeral_high = (uint8_t*)~0; +uint8_t* g_region_to_generation_table = nullptr; +uint8_t g_region_shr = 0; +bool g_region_use_bitwise_write_barrier = false; #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES uint32_t* g_card_bundle_table = nullptr; diff --git a/src/coreclr/vm/gcheaputilities.h b/src/coreclr/vm/gcheaputilities.h index c6dbb54631b2b0..c652cc52bf417c 100644 --- a/src/coreclr/vm/gcheaputilities.h +++ b/src/coreclr/vm/gcheaputilities.h @@ -29,6 +29,10 @@ GVAL_DECL(gc_alloc_context, g_global_alloc_context); extern "C" uint32_t* g_card_bundle_table; extern "C" uint8_t* g_ephemeral_low; extern "C" uint8_t* g_ephemeral_high; +extern "C" uint8_t* g_region_to_generation_table; +extern "C" uint8_t g_region_shr; +extern "C" bool g_region_use_bitwise_write_barrier; + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index d0a415a1f5f1d7..efb57186571eb3 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -262,12 +262,16 @@ class WriteBarrierManager #ifdef FEATURE_SVR_GC WRITE_BARRIER_SVR64, #endif // FEATURE_SVR_GC + WRITE_BARRIER_BYTE_REGIONS64, + WRITE_BARRIER_BIT_REGIONS64, #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP WRITE_BARRIER_WRITE_WATCH_PREGROW64, WRITE_BARRIER_WRITE_WATCH_POSTGROW64, #ifdef FEATURE_SVR_GC WRITE_BARRIER_WRITE_WATCH_SVR64, #endif // FEATURE_SVR_GC + WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64, + WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64, #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP WRITE_BARRIER_BUFFER }; @@ -289,18 +293,21 @@ class WriteBarrierManager PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); PCODE GetCurrentWriteBarrierCode(); int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); - bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, WriteBarrierType* pNewWriteBarrierType); + bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); private: void Validate(); WriteBarrierType m_currentWriteBarrier; - PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | - PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | - PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | - PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | - PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | + PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION }; #endif // TARGET_AMD64