From 5acfc4a2dfaae101ccc8087882a9447538b34eda Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 3 Nov 2025 17:14:49 +0800 Subject: [PATCH 01/13] Fix LOS object enumeration and refactor TreadMill We fix a bug when enumerating objects in LOS. During Prepare of ConcurrentImmix, we flip the from and to space of the TreadMill, and then enumerate objects in the LOS to set their unmark bits. However, the object enumeration code only enumerates objects in the to-space. Because we have flipped all objects to the from-space, object enumeration visited no objects at all. Consequently, no objects in LOS have unlog bits during concurrent marking, and the SATB barriers will not be applied to any objects in the LOS. Now `TreadMill::enumerate_objects` take an additional argument `gc`. When it is set to true, we enumerate objects in both from-spaces and to-spaces. We do this when setting and clearing unlog bits, and fix the bug. We also refactored the `TreadMill` so that all of its `HashSet` fields are protected by one single `Mutex`, because it is not safe to enumerate one `HashSet` while another thread modifies another. We also renamed the hash sets to `nursery.from_space`, `nursery.to_space`, `mature.from_space` and `mature.to_space`, respectively, for consistency. --- src/policy/largeobjectspace.rs | 16 ++-- src/util/treadmill.rs | 160 ++++++++++++++++++++++----------- 2 files changed, 119 insertions(+), 57 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index ddef8d6bc1..5533b438b8 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -219,21 +219,21 @@ impl Space for LargeObjectSpace { } fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - self.treadmill.enumerate_objects(enumerator); + self.treadmill.enumerate_objects(enumerator, false); } fn clear_side_log_bits(&self) { let mut enumator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumator); + self.treadmill.enumerate_objects(&mut enumator, true); } fn set_side_log_bits(&self) { let mut enumator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumator); + self.treadmill.enumerate_objects(&mut enumator, true); } } @@ -297,10 +297,16 @@ impl LargeObjectSpace { } pub fn release(&mut self, full_heap: bool) { + // We swapped the from/to spaces during Prepare, and the nursery to-space should have + // remained empty for the whole duration of the collection. + debug_assert!(self.treadmill.is_nursery_to_space_empty()); + self.sweep_large_pages(true); - debug_assert!(self.treadmill.is_nursery_empty()); + debug_assert!(self.treadmill.is_nursery_from_space_empty()); + if full_heap { self.sweep_large_pages(false); + debug_assert!(self.treadmill.is_mature_from_space_empty()); } } @@ -364,7 +370,7 @@ impl LargeObjectSpace { sweep(object); } } else { - for object in self.treadmill.collect() { + for object in self.treadmill.collect_mature() { sweep(object) } } diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 96b5eb9e7d..2a3441ae79 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -6,20 +6,38 @@ use crate::util::ObjectReference; use super::object_enum::ObjectEnumerator; +/// A data structure for recording objects in the LOS. +/// +/// It is divided into the nursery and the mature space, and each of them is further divided into +/// the from-space and the to-space. +/// +/// All operations are protected by a single mutex [`TreadMill::sync`]. pub struct TreadMill { - from_space: Mutex>, - to_space: Mutex>, - collect_nursery: Mutex>, - alloc_nursery: Mutex>, + sync: Mutex, +} + +/// The synchronized part of [`TreadMill`] +#[derive(Default)] +struct TreadMillSync { + nursery: FromToSpace, + mature: FromToSpace, +} + +/// A pair of from and two spaces. +#[derive(Default)] +struct FromToSpace { + from_space: HashSet, + to_space: HashSet, } impl std::fmt::Debug for TreadMill { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let sync = self.sync.lock().unwrap(); f.debug_struct("TreadMill") - .field("from", &self.from_space.lock().unwrap()) - .field("to", &self.to_space.lock().unwrap()) - .field("collect_nursery", &self.collect_nursery.lock().unwrap()) - .field("alloc_nursery", &self.alloc_nursery.lock().unwrap()) + .field("nursery.from", &sync.nursery.from_space) + .field("nursery.to", &sync.nursery.to_space) + .field("mature.from", &sync.mature.from_space) + .field("mature.to", &sync.mature.to_space) .finish() } } @@ -27,90 +45,128 @@ impl std::fmt::Debug for TreadMill { impl TreadMill { pub fn new() -> Self { TreadMill { - from_space: Mutex::new(HashSet::new()), - to_space: Mutex::new(HashSet::new()), - collect_nursery: Mutex::new(HashSet::new()), - alloc_nursery: Mutex::new(HashSet::new()), + sync: Mutex::new(Default::default()), } } + /// Add an object to the treadmill. + /// + /// New objects are normally added to `nursery.to_space`. But when allocatin as live (e.g. when + /// concurrent marking is active), we directly add into `mature.to_space`. pub fn add_to_treadmill(&self, object: ObjectReference, nursery: bool) { + let mut sync = self.sync.lock().unwrap(); if nursery { - trace!("Adding {} to nursery", object); - self.alloc_nursery.lock().unwrap().insert(object); + trace!("Adding {} to nursery.to_space", object); + sync.nursery.to_space.insert(object); } else { - trace!("Adding {} to to_space", object); - self.to_space.lock().unwrap().insert(object); + trace!("Adding {} to mature.to_space", object); + sync.mature.to_space.insert(object); } } - pub fn collect_nursery(&self) -> Vec { - let mut guard = self.collect_nursery.lock().unwrap(); - let vals = guard.iter().copied().collect(); - guard.clear(); - drop(guard); - vals + /// Take all objects from the `nursery.from_space`. This is called during sweeping at which time + /// all objects in the from space are unreachable. + pub fn collect_nursery(&self) -> impl IntoIterator { + let mut sync = self.sync.lock().unwrap(); + std::mem::take(&mut sync.nursery.from_space) } - pub fn collect(&self) -> Vec { - let mut guard = self.from_space.lock().unwrap(); - let vals = guard.iter().copied().collect(); - guard.clear(); - drop(guard); - vals + /// Take all objects from the `mature.from_space`. This is called during sweeping at which time + /// all objects in the from space are unreachable. + pub fn collect_mature(&self) -> impl IntoIterator { + let mut sync = self.sync.lock().unwrap(); + std::mem::take(&mut sync.mature.from_space) } + /// Move an object to `mature.to_space`. Called when an object is determined to be reachable. pub fn copy(&self, object: ObjectReference, is_in_nursery: bool) { + let mut sync = self.sync.lock().unwrap(); if is_in_nursery { - let mut guard = self.collect_nursery.lock().unwrap(); debug_assert!( - guard.contains(&object), - "copy source object ({}) must be in collect_nursery", + sync.nursery.from_space.contains(&object), + "copy source object ({}) must be in nursery.from_space", object ); - guard.remove(&object); + sync.nursery.from_space.remove(&object); } else { - let mut guard = self.from_space.lock().unwrap(); debug_assert!( - guard.contains(&object), - "copy source object ({}) must be in from_space", + sync.mature.from_space.contains(&object), + "copy source object ({}) must be in mature.from_space", object ); - guard.remove(&object); + sync.mature.from_space.remove(&object); } - self.to_space.lock().unwrap().insert(object); + sync.mature.to_space.insert(object); } - pub fn is_to_space_empty(&self) -> bool { - self.to_space.lock().unwrap().is_empty() + /// Return true if the nursery from-space is empty. + pub fn is_nursery_from_space_empty(&self) -> bool { + let sync = self.sync.lock().unwrap(); + sync.nursery.from_space.is_empty() } - pub fn is_from_space_empty(&self) -> bool { - self.from_space.lock().unwrap().is_empty() + /// Return true if the nursery to-space is empty. + pub fn is_nursery_to_space_empty(&self) -> bool { + let sync = self.sync.lock().unwrap(); + sync.nursery.to_space.is_empty() } - pub fn is_nursery_empty(&self) -> bool { - self.collect_nursery.lock().unwrap().is_empty() + /// Return true if the mature from-space is empty. + pub fn is_mature_from_space_empty(&self) -> bool { + let sync = self.sync.lock().unwrap(); + sync.mature.from_space.is_empty() } + /// Flip the from and to spaces. + /// + /// `full_heap` is true during full-heap GC, or false during nursery GC. pub fn flip(&mut self, full_heap: bool) { - swap(&mut self.alloc_nursery, &mut self.collect_nursery); - trace!("Flipped alloc_nursery and collect_nursery"); + let sync = self.sync.get_mut().unwrap(); + swap(&mut sync.nursery.from_space, &mut sync.nursery.to_space); + trace!("Flipped nursery.from_space and nursery.to_space"); if full_heap { - swap(&mut self.from_space, &mut self.to_space); - trace!("Flipped from_space and to_space"); + swap(&mut sync.mature.from_space, &mut sync.mature.to_space); + trace!("Flipped mature.from_space and mature.to_space"); } } - pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - let mut visit_objects = |set: &Mutex>| { - let set = set.lock().unwrap(); + /// Enumerate objects. + /// + /// `gc` is true if called as part of GC activity. It will enumerate objects in both + /// from-spaces and to-spaces because we may have flipped the from/to spaces. + /// + /// `gc` is false if called by mutator. It will only enumerate objects in to-spaces. + pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator, gc: bool) { + let sync = self.sync.lock().unwrap(); + let mut enumerated = 0usize; + let mut visit_objects = |set: &HashSet| { for object in set.iter() { enumerator.visit_object(*object); + enumerated += 1; } }; - visit_objects(&self.alloc_nursery); - visit_objects(&self.to_space); + visit_objects(&sync.nursery.to_space); + visit_objects(&sync.mature.to_space); + if gc { + visit_objects(&sync.nursery.from_space); + visit_objects(&sync.mature.from_space); + } else { + // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved to + // from-spaces, and GC workers are tracing objects concurrently, moving object to + // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during concurrent + // GC, the assertions below will fail. That's expected because we currently disallow + // the VM binding to call `MMTK::enumerate_objects` during any GC activities, including + // concurrent GC. + assert!( + sync.nursery.from_space.is_empty(), + "nursery.from_space is not empty" + ); + assert!( + sync.mature.from_space.is_empty(), + "mature.from_space is not empty" + ); + } + debug!("Enumerated {enumerated} objects in LOS. gc={gc}"); } } From 98d7163f2a30af66403aeec0c01f3062d8919114 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 3 Nov 2025 17:31:25 +0800 Subject: [PATCH 02/13] Fix spelling --- src/policy/largeobjectspace.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 5533b438b8..553bb62fb8 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -223,17 +223,17 @@ impl Space for LargeObjectSpace { } fn clear_side_log_bits(&self) { - let mut enumator = ClosureObjectEnumerator::<_, VM>::new(|object| { + let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumator, true); + self.treadmill.enumerate_objects(&mut enumerator, true); } fn set_side_log_bits(&self) { - let mut enumator = ClosureObjectEnumerator::<_, VM>::new(|object| { + let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumator, true); + self.treadmill.enumerate_objects(&mut enumerator, true); } } From 3c213ec508d8e0def642d0cd6f27844b59b28e0b Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 11:27:26 +0800 Subject: [PATCH 03/13] Rename `FromToSpace` to `SpacePair` --- src/util/treadmill.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 2a3441ae79..dd9dfd6799 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -19,13 +19,13 @@ pub struct TreadMill { /// The synchronized part of [`TreadMill`] #[derive(Default)] struct TreadMillSync { - nursery: FromToSpace, - mature: FromToSpace, + nursery: SpacePair, + mature: SpacePair, } /// A pair of from and two spaces. #[derive(Default)] -struct FromToSpace { +struct SpacePair { from_space: HashSet, to_space: HashSet, } From b84a67ee52935fbf71cb5ab95db24559a16cc21c Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 13:08:09 +0800 Subject: [PATCH 04/13] Introduce FromSpacePolicy --- src/policy/compressor/compressorspace.rs | 8 +-- src/policy/largeobjectspace.rs | 16 +++++- src/util/treadmill.rs | 73 ++++++++++++++++-------- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/src/policy/compressor/compressorspace.rs b/src/policy/compressor/compressorspace.rs index fca2987caa..cc4cf4dcc0 100644 --- a/src/policy/compressor/compressorspace.rs +++ b/src/policy/compressor/compressorspace.rs @@ -405,11 +405,11 @@ impl CompressorSpace { pub fn after_compact(&self, worker: &mut GCWorker, los: &LargeObjectSpace) { self.pr.reset_allocator(); // Update references from the LOS to Compressor too. - los.enumerate_objects(&mut object_enum::ClosureObjectEnumerator::<_, VM>::new( - &mut |o: ObjectReference| { + los.enumerate_objects_for_forwarding( + &mut object_enum::ClosureObjectEnumerator::<_, VM>::new(&mut |o: ObjectReference| { self.update_references(worker, o); - }, - )); + }), + ); } } diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 553bb62fb8..5c24ab0de4 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -12,6 +12,7 @@ use crate::util::metadata; use crate::util::object_enum::ClosureObjectEnumerator; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; +use crate::util::treadmill::FromSpacePolicy; use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; @@ -219,21 +220,24 @@ impl Space for LargeObjectSpace { } fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - self.treadmill.enumerate_objects(enumerator, false); + self.treadmill + .enumerate_objects(enumerator, FromSpacePolicy::ExpectEmpty); } fn clear_side_log_bits(&self) { let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumerator, true); + self.treadmill + .enumerate_objects(&mut enumerator, FromSpacePolicy::Include); } fn set_side_log_bits(&self) { let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); - self.treadmill.enumerate_objects(&mut enumerator, true); + self.treadmill + .enumerate_objects(&mut enumerator, FromSpacePolicy::Include); } } @@ -376,6 +380,12 @@ impl LargeObjectSpace { } } + /// Enumerate objects for forwarding references. Used by certain mark-compact plans. + pub(crate) fn enumerate_objects_for_forwarding(&self, enumerator: &mut dyn ObjectEnumerator) { + self.treadmill + .enumerate_objects(enumerator, FromSpacePolicy::Skip); + } + /// Allocate an object pub fn allocate_pages( &self, diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index dd9dfd6799..34557eb428 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -30,6 +30,22 @@ struct SpacePair { to_space: HashSet, } +/// Used by [`TreadMill::enumerate_objects`] to determine what to do to objects in the from-spaces. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FromSpacePolicy { + /// Also enumerate objects in the from-spaces. Useful when setting object metadata during + /// `Prepare`, at which time we may have swapped the from- and to-spaces. + Include, + /// Silently skip objects in the from-spaces. Useful when enumerating live objects after the + /// liveness of objects is determined and live objects have been moved to the to-spaces. One + /// use case is for forwarding references in some mark-compact GC algorithms. + Skip, + /// Assert that from-spaces must be empty. Useful when the mutator calls + /// `MMTK::enumerate_objects`, at which time GC must not be in progress and the from-spaces must + /// be empty. + ExpectEmpty, +} + impl std::fmt::Debug for TreadMill { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let sync = self.sync.lock().unwrap(); @@ -132,11 +148,13 @@ impl TreadMill { /// Enumerate objects. /// - /// `gc` is true if called as part of GC activity. It will enumerate objects in both - /// from-spaces and to-spaces because we may have flipped the from/to spaces. - /// - /// `gc` is false if called by mutator. It will only enumerate objects in to-spaces. - pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator, gc: bool) { + /// Objects in the to-spaces are always enumerated. `from_space_policy` determines the action + /// for objects in the nursery and mature from-spaces. + pub(crate) fn enumerate_objects( + &self, + enumerator: &mut dyn ObjectEnumerator, + from_space_policy: FromSpacePolicy, + ) { let sync = self.sync.lock().unwrap(); let mut enumerated = 0usize; let mut visit_objects = |set: &HashSet| { @@ -147,26 +165,33 @@ impl TreadMill { }; visit_objects(&sync.nursery.to_space); visit_objects(&sync.mature.to_space); - if gc { - visit_objects(&sync.nursery.from_space); - visit_objects(&sync.mature.from_space); - } else { - // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved to - // from-spaces, and GC workers are tracing objects concurrently, moving object to - // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during concurrent - // GC, the assertions below will fail. That's expected because we currently disallow - // the VM binding to call `MMTK::enumerate_objects` during any GC activities, including - // concurrent GC. - assert!( - sync.nursery.from_space.is_empty(), - "nursery.from_space is not empty" - ); - assert!( - sync.mature.from_space.is_empty(), - "mature.from_space is not empty" - ); + + match from_space_policy { + FromSpacePolicy::Include => { + visit_objects(&sync.nursery.from_space); + visit_objects(&sync.mature.from_space); + } + FromSpacePolicy::Skip => { + // Do nothing. + } + FromSpacePolicy::ExpectEmpty => { + // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved + // to from-spaces, and GC workers are tracing objects concurrently, moving object to + // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during + // concurrent GC, the assertions below will fail. That's expected because we + // currently disallow the VM binding to call `MMTK::enumerate_objects` during any GC + // activities, including concurrent GC. + assert!( + sync.nursery.from_space.is_empty(), + "nursery.from_space is not empty" + ); + assert!( + sync.mature.from_space.is_empty(), + "mature.from_space is not empty" + ); + } } - debug!("Enumerated {enumerated} objects in LOS. gc={gc}"); + debug!("Enumerated {enumerated} objects in LOS. from_space_policy={from_space_policy:?}"); } } From 39c39aaefb3b3e2fd4c6580a9379caeebb2927d3 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 13:09:57 +0800 Subject: [PATCH 05/13] Use from-space and to-space consistently. The GC Handbook uses "fromspace" and "tospace" (no hyphen). --- src/util/treadmill.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 34557eb428..51c148c3ea 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -81,14 +81,14 @@ impl TreadMill { } /// Take all objects from the `nursery.from_space`. This is called during sweeping at which time - /// all objects in the from space are unreachable. + /// all objects in the from-space are unreachable. pub fn collect_nursery(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); std::mem::take(&mut sync.nursery.from_space) } /// Take all objects from the `mature.from_space`. This is called during sweeping at which time - /// all objects in the from space are unreachable. + /// all objects in the from-space are unreachable. pub fn collect_mature(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); std::mem::take(&mut sync.mature.from_space) @@ -133,7 +133,7 @@ impl TreadMill { sync.mature.from_space.is_empty() } - /// Flip the from and to spaces. + /// Flip the from- and to-spaces. /// /// `full_heap` is true during full-heap GC, or false during nursery GC. pub fn flip(&mut self, full_heap: bool) { From 7cb8718f0c254fba085b3fff00f63426397f7f87 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 14:53:40 +0800 Subject: [PATCH 06/13] Remove "nursery to-space". We now have only nursery + from-space + to-space. --- src/policy/largeobjectspace.rs | 10 +-- src/util/treadmill.rs | 110 ++++++++++++++------------------- 2 files changed, 49 insertions(+), 71 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 5c24ab0de4..b7322c6031 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -120,7 +120,7 @@ impl SFT for LargeObjectSpace { } // Add to the treadmill. Nursery and mature objects need to be added to different sets. - self.treadmill.add_to_treadmill(object, into_nursery); + self.treadmill.add_to_treadmill(object, allocate_as_live); } #[cfg(feature = "is_mmtk_object")] @@ -301,16 +301,12 @@ impl LargeObjectSpace { } pub fn release(&mut self, full_heap: bool) { - // We swapped the from/to spaces during Prepare, and the nursery to-space should have - // remained empty for the whole duration of the collection. - debug_assert!(self.treadmill.is_nursery_to_space_empty()); - self.sweep_large_pages(true); - debug_assert!(self.treadmill.is_nursery_from_space_empty()); + debug_assert!(self.treadmill.is_nursery_empty()); if full_heap { self.sweep_large_pages(false); - debug_assert!(self.treadmill.is_mature_from_space_empty()); + debug_assert!(self.treadmill.is_from_space_empty()); } } diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 51c148c3ea..e56b763a7f 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -8,9 +8,6 @@ use super::object_enum::ObjectEnumerator; /// A data structure for recording objects in the LOS. /// -/// It is divided into the nursery and the mature space, and each of them is further divided into -/// the from-space and the to-space. -/// /// All operations are protected by a single mutex [`TreadMill::sync`]. pub struct TreadMill { sync: Mutex, @@ -19,14 +16,15 @@ pub struct TreadMill { /// The synchronized part of [`TreadMill`] #[derive(Default)] struct TreadMillSync { - nursery: SpacePair, - mature: SpacePair, -} - -/// A pair of from and two spaces. -#[derive(Default)] -struct SpacePair { + /// The nursery. During mutator time, newly allocated objects are added to the nursery. After + /// a GC, the nursery will be evacuated. + nursery: HashSet, + /// The from-space. During GC, old objects whose liveness are not yet determined are kept in + /// the from-space. After GC, the from-space will be evacuated. from_space: HashSet, + /// The to-space. It holds old objects during mutator time. Objects in the to-space are moved + /// to the from-space at the beginning of GC, and objects are moved to the to-space once they + /// are determined to be live. to_space: HashSet, } @@ -50,10 +48,9 @@ impl std::fmt::Debug for TreadMill { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let sync = self.sync.lock().unwrap(); f.debug_struct("TreadMill") - .field("nursery.from", &sync.nursery.from_space) - .field("nursery.to", &sync.nursery.to_space) - .field("mature.from", &sync.mature.from_space) - .field("mature.to", &sync.mature.to_space) + .field("nursery", &sync.nursery) + .field("from", &sync.from_space) + .field("to", &sync.to_space) .finish() } } @@ -67,70 +64,64 @@ impl TreadMill { /// Add an object to the treadmill. /// - /// New objects are normally added to `nursery.to_space`. But when allocatin as live (e.g. when - /// concurrent marking is active), we directly add into `mature.to_space`. - pub fn add_to_treadmill(&self, object: ObjectReference, nursery: bool) { + /// New objects are normally added to `nursery.to_space`. But when allocating as live (e.g. + /// when concurrent marking is active), we directly add into the `to_space`. + pub fn add_to_treadmill(&self, object: ObjectReference, allocate_as_live: bool) { let mut sync = self.sync.lock().unwrap(); - if nursery { - trace!("Adding {} to nursery.to_space", object); - sync.nursery.to_space.insert(object); + if allocate_as_live { + trace!("Adding {} to to_space", object); + sync.to_space.insert(object); } else { - trace!("Adding {} to mature.to_space", object); - sync.mature.to_space.insert(object); + trace!("Adding {} to nursery", object); + sync.nursery.insert(object); } } - /// Take all objects from the `nursery.from_space`. This is called during sweeping at which time - /// all objects in the from-space are unreachable. + /// Take all objects from the `nursery`. This is called during sweeping at which time all + /// objects in the nursery are unreachable. pub fn collect_nursery(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); - std::mem::take(&mut sync.nursery.from_space) + std::mem::take(&mut sync.nursery) } - /// Take all objects from the `mature.from_space`. This is called during sweeping at which time - /// all objects in the from-space are unreachable. + /// Take all objects from the `from_space`. This is called during sweeping at which time all + /// objects in the from-space are unreachable. pub fn collect_mature(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); - std::mem::take(&mut sync.mature.from_space) + std::mem::take(&mut sync.from_space) } - /// Move an object to `mature.to_space`. Called when an object is determined to be reachable. + /// Move an object to `to_space`. Called when an object is determined to be reachable. pub fn copy(&self, object: ObjectReference, is_in_nursery: bool) { let mut sync = self.sync.lock().unwrap(); if is_in_nursery { debug_assert!( - sync.nursery.from_space.contains(&object), - "copy source object ({}) must be in nursery.from_space", + sync.nursery.contains(&object), + "copy source object ({}) must be in nursery", object ); - sync.nursery.from_space.remove(&object); + sync.nursery.remove(&object); } else { debug_assert!( - sync.mature.from_space.contains(&object), - "copy source object ({}) must be in mature.from_space", + sync.from_space.contains(&object), + "copy source object ({}) must be in from_space", object ); - sync.mature.from_space.remove(&object); + sync.from_space.remove(&object); } - sync.mature.to_space.insert(object); - } - - /// Return true if the nursery from-space is empty. - pub fn is_nursery_from_space_empty(&self) -> bool { - let sync = self.sync.lock().unwrap(); - sync.nursery.from_space.is_empty() + sync.to_space.insert(object); } - /// Return true if the nursery to-space is empty. - pub fn is_nursery_to_space_empty(&self) -> bool { + /// Return true if the nursery is empty. + pub fn is_nursery_empty(&self) -> bool { let sync = self.sync.lock().unwrap(); - sync.nursery.to_space.is_empty() + sync.nursery.is_empty() } - /// Return true if the mature from-space is empty. - pub fn is_mature_from_space_empty(&self) -> bool { + /// Return true if the from-space is empty. + pub fn is_from_space_empty(&self) -> bool { let sync = self.sync.lock().unwrap(); - sync.mature.from_space.is_empty() + sync.from_space.is_empty() } /// Flip the from- and to-spaces. @@ -138,11 +129,9 @@ impl TreadMill { /// `full_heap` is true during full-heap GC, or false during nursery GC. pub fn flip(&mut self, full_heap: bool) { let sync = self.sync.get_mut().unwrap(); - swap(&mut sync.nursery.from_space, &mut sync.nursery.to_space); - trace!("Flipped nursery.from_space and nursery.to_space"); if full_heap { - swap(&mut sync.mature.from_space, &mut sync.mature.to_space); - trace!("Flipped mature.from_space and mature.to_space"); + swap(&mut sync.from_space, &mut sync.to_space); + trace!("Flipped from_space and to_space"); } } @@ -163,13 +152,12 @@ impl TreadMill { enumerated += 1; } }; - visit_objects(&sync.nursery.to_space); - visit_objects(&sync.mature.to_space); + visit_objects(&sync.to_space); match from_space_policy { FromSpacePolicy::Include => { - visit_objects(&sync.nursery.from_space); - visit_objects(&sync.mature.from_space); + visit_objects(&sync.nursery); + visit_objects(&sync.from_space); } FromSpacePolicy::Skip => { // Do nothing. @@ -181,14 +169,8 @@ impl TreadMill { // concurrent GC, the assertions below will fail. That's expected because we // currently disallow the VM binding to call `MMTK::enumerate_objects` during any GC // activities, including concurrent GC. - assert!( - sync.nursery.from_space.is_empty(), - "nursery.from_space is not empty" - ); - assert!( - sync.mature.from_space.is_empty(), - "mature.from_space is not empty" - ); + assert!(sync.nursery.is_empty(), "nursery is not empty"); + assert!(sync.from_space.is_empty(), "from_space is not empty"); } } debug!("Enumerated {enumerated} objects in LOS. from_space_policy={from_space_policy:?}"); From cd681ca20bf6b89489b0f0893a95fec800d1b74f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 15:37:37 +0800 Subject: [PATCH 07/13] Remove FromSpacePolicy --- src/policy/largeobjectspace.rs | 11 +++---- src/util/treadmill.rs | 60 +++++++++++++--------------------- 2 files changed, 26 insertions(+), 45 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index b7322c6031..b26252a947 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -12,7 +12,6 @@ use crate::util::metadata; use crate::util::object_enum::ClosureObjectEnumerator; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; -use crate::util::treadmill::FromSpacePolicy; use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; @@ -220,8 +219,7 @@ impl Space for LargeObjectSpace { } fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - self.treadmill - .enumerate_objects(enumerator, FromSpacePolicy::ExpectEmpty); + self.treadmill.enumerate_objects(enumerator, false, true); } fn clear_side_log_bits(&self) { @@ -229,7 +227,7 @@ impl Space for LargeObjectSpace { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); }); self.treadmill - .enumerate_objects(&mut enumerator, FromSpacePolicy::Include); + .enumerate_objects(&mut enumerator, true, false); } fn set_side_log_bits(&self) { @@ -237,7 +235,7 @@ impl Space for LargeObjectSpace { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); self.treadmill - .enumerate_objects(&mut enumerator, FromSpacePolicy::Include); + .enumerate_objects(&mut enumerator, true, false); } } @@ -378,8 +376,7 @@ impl LargeObjectSpace { /// Enumerate objects for forwarding references. Used by certain mark-compact plans. pub(crate) fn enumerate_objects_for_forwarding(&self, enumerator: &mut dyn ObjectEnumerator) { - self.treadmill - .enumerate_objects(enumerator, FromSpacePolicy::Skip); + self.treadmill.enumerate_objects(enumerator, false, false); } /// Allocate an object diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index e56b763a7f..222be8e84a 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -28,22 +28,6 @@ struct TreadMillSync { to_space: HashSet, } -/// Used by [`TreadMill::enumerate_objects`] to determine what to do to objects in the from-spaces. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum FromSpacePolicy { - /// Also enumerate objects in the from-spaces. Useful when setting object metadata during - /// `Prepare`, at which time we may have swapped the from- and to-spaces. - Include, - /// Silently skip objects in the from-spaces. Useful when enumerating live objects after the - /// liveness of objects is determined and live objects have been moved to the to-spaces. One - /// use case is for forwarding references in some mark-compact GC algorithms. - Skip, - /// Assert that from-spaces must be empty. Useful when the mutator calls - /// `MMTK::enumerate_objects`, at which time GC must not be in progress and the from-spaces must - /// be empty. - ExpectEmpty, -} - impl std::fmt::Debug for TreadMill { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let sync = self.sync.lock().unwrap(); @@ -137,12 +121,17 @@ impl TreadMill { /// Enumerate objects. /// - /// Objects in the to-spaces are always enumerated. `from_space_policy` determines the action - /// for objects in the nursery and mature from-spaces. + /// Objects in the to-spaces are always enumerated. If `all` is true, it will enumerate objects + /// in the nursery and from-spaces as well. + /// + /// If `assert_empty` is true, it will assert that spaces not enumerated are empty. It is + /// useful for asserting that the nursery and the from-space are both empty when enumerating + /// objects for mutators. pub(crate) fn enumerate_objects( &self, enumerator: &mut dyn ObjectEnumerator, - from_space_policy: FromSpacePolicy, + all: bool, + assert_empty: bool, ) { let sync = self.sync.lock().unwrap(); let mut enumerated = 0usize; @@ -154,26 +143,21 @@ impl TreadMill { }; visit_objects(&sync.to_space); - match from_space_policy { - FromSpacePolicy::Include => { - visit_objects(&sync.nursery); - visit_objects(&sync.from_space); - } - FromSpacePolicy::Skip => { - // Do nothing. - } - FromSpacePolicy::ExpectEmpty => { - // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved - // to from-spaces, and GC workers are tracing objects concurrently, moving object to - // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during - // concurrent GC, the assertions below will fail. That's expected because we - // currently disallow the VM binding to call `MMTK::enumerate_objects` during any GC - // activities, including concurrent GC. - assert!(sync.nursery.is_empty(), "nursery is not empty"); - assert!(sync.from_space.is_empty(), "from_space is not empty"); - } + if all { + visit_objects(&sync.nursery); + visit_objects(&sync.from_space); + } else if assert_empty { + // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved to + // from-spaces, and GC workers are tracing objects concurrently, moving object to + // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during concurrent + // GC, the assertions below will fail. That's expected because we currently disallow + // the VM binding to call `MMTK::enumerate_objects` during any GC activities, including + // concurrent GC. + assert!(sync.nursery.is_empty(), "nursery is not empty"); + assert!(sync.from_space.is_empty(), "from_space is not empty"); } - debug!("Enumerated {enumerated} objects in LOS. from_space_policy={from_space_policy:?}"); + + debug!("Enumerated {enumerated} objects in LOS. all={all}, assert_empty={assert_empty}"); } } From b47a8fb9ffe478c9f443ebae3388d39b00dd7c5a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 15:42:53 +0800 Subject: [PATCH 08/13] Minor style and comment changes --- src/policy/largeobjectspace.rs | 1 - src/util/treadmill.rs | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index b26252a947..8763484a58 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -301,7 +301,6 @@ impl LargeObjectSpace { pub fn release(&mut self, full_heap: bool) { self.sweep_large_pages(true); debug_assert!(self.treadmill.is_nursery_empty()); - if full_heap { self.sweep_large_pages(false); debug_assert!(self.treadmill.is_from_space_empty()); diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 222be8e84a..db14cc68dc 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -108,9 +108,10 @@ impl TreadMill { sync.from_space.is_empty() } - /// Flip the from- and to-spaces. + /// Flip the from-space and to-space. /// - /// `full_heap` is true during full-heap GC, or false during nursery GC. + /// `full_heap` is true during full-heap GC, or false during nursery GC. It does nothing during + /// nursery GC. pub fn flip(&mut self, full_heap: bool) { let sync = self.sync.get_mut().unwrap(); if full_heap { From 4c526f99fcf5ce78a9667a314d4a0e36c324e0ca Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 16:19:49 +0800 Subject: [PATCH 09/13] Fix mutator obj enum Allow controlling whether `TreadMill::enumerate_objects` visits the nursery and the from-space separately. Mutator visits nursery + to_space; Compressor visits only to_space; ConcurrentImmix visits all spaces. It off-loads the assertion to its caller. --- src/policy/largeobjectspace.rs | 20 +++++++++++++++++--- src/util/treadmill.rs | 28 +++++++++------------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 8763484a58..992647bc7d 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -219,23 +219,36 @@ impl Space for LargeObjectSpace { } fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - self.treadmill.enumerate_objects(enumerator, false, true); + // `MMTK::enumerate_objects` is not allowed during GC, so the from space must be empty. In + // `ConcurrentImmix`, mutators may run during GC and call `MMTK::enumerate_objects`. It has + // undefined behavior according to the current API, so the assertion failure is expected. + assert!( + self.treadmill.is_from_space_empty(), + "From-space is not empty" + ); + + // Visit objects in the nursery and the to-space, which contain young and old objects, + // respectively, during mutator time. + self.treadmill.enumerate_objects(enumerator, true, false); } fn clear_side_log_bits(&self) { let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); }); + // Visit all objects. It can be ordered arbitrarily with `Self::Release` which sweeps dead + // objects (removing them from the treadmill) and clears their unlog bits, too. self.treadmill - .enumerate_objects(&mut enumerator, true, false); + .enumerate_objects(&mut enumerator, true, true); } fn set_side_log_bits(&self) { let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); + // Visit all objects. self.treadmill - .enumerate_objects(&mut enumerator, true, false); + .enumerate_objects(&mut enumerator, true, true); } } @@ -375,6 +388,7 @@ impl LargeObjectSpace { /// Enumerate objects for forwarding references. Used by certain mark-compact plans. pub(crate) fn enumerate_objects_for_forwarding(&self, enumerator: &mut dyn ObjectEnumerator) { + // Only visit the `to_space` which should contain all objects determined to be live. self.treadmill.enumerate_objects(enumerator, false, false); } diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index db14cc68dc..e8d5199c9f 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -122,17 +122,13 @@ impl TreadMill { /// Enumerate objects. /// - /// Objects in the to-spaces are always enumerated. If `all` is true, it will enumerate objects - /// in the nursery and from-spaces as well. - /// - /// If `assert_empty` is true, it will assert that spaces not enumerated are empty. It is - /// useful for asserting that the nursery and the from-space are both empty when enumerating - /// objects for mutators. + /// Objects in the to-spaces are always enumerated. `nursery` and `from` controls whether to + /// enumerate objects in the nursery and from-spaces, respectively. pub(crate) fn enumerate_objects( &self, enumerator: &mut dyn ObjectEnumerator, - all: bool, - assert_empty: bool, + nursery: bool, + from: bool, ) { let sync = self.sync.lock().unwrap(); let mut enumerated = 0usize; @@ -144,21 +140,15 @@ impl TreadMill { }; visit_objects(&sync.to_space); - if all { + if nursery { visit_objects(&sync.nursery); + } + + if from { visit_objects(&sync.from_space); - } else if assert_empty { - // Note that during concurrent GC (e.g. in ConcurrentImmix), object have been moved to - // from-spaces, and GC workers are tracing objects concurrently, moving object to - // `mature.to_space`. If a mutator calls `MMTK::enumerate_objects` during concurrent - // GC, the assertions below will fail. That's expected because we currently disallow - // the VM binding to call `MMTK::enumerate_objects` during any GC activities, including - // concurrent GC. - assert!(sync.nursery.is_empty(), "nursery is not empty"); - assert!(sync.from_space.is_empty(), "from_space is not empty"); } - debug!("Enumerated {enumerated} objects in LOS. all={all}, assert_empty={assert_empty}"); + debug!("Enumerated {enumerated} objects in LOS. nursery: {nursery}, from: {from}"); } } From bfd75095e8054a73613e9212dc657f667600002f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 19:51:33 +0800 Subject: [PATCH 10/13] Revert naming changes We revert back to "alloc_nursery" and "collect_nursery", which were the names used back in the JikesRVM MMTk. --- src/policy/largeobjectspace.rs | 38 +++++++++------ src/util/treadmill.rs | 88 ++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 51 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 992647bc7d..8c7864abe7 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -119,7 +119,7 @@ impl SFT for LargeObjectSpace { } // Add to the treadmill. Nursery and mature objects need to be added to different sets. - self.treadmill.add_to_treadmill(object, allocate_as_live); + self.treadmill.add_to_treadmill(object, into_nursery); } #[cfg(feature = "is_mmtk_object")] @@ -219,17 +219,22 @@ impl Space for LargeObjectSpace { } fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { - // `MMTK::enumerate_objects` is not allowed during GC, so the from space must be empty. In - // `ConcurrentImmix`, mutators may run during GC and call `MMTK::enumerate_objects`. It has - // undefined behavior according to the current API, so the assertion failure is expected. + // `MMTK::enumerate_objects` is not allowed during GC, so the collection nursery and the + // from space must be empty. In `ConcurrentImmix`, mutators may run during GC and call + // `MMTK::enumerate_objects`. It has undefined behavior according to the current API, so + // the assertion failure is expected. + assert!( + self.treadmill.is_collect_nursery_empty(), + "Collection nursery is not empty" + ); assert!( self.treadmill.is_from_space_empty(), "From-space is not empty" ); - // Visit objects in the nursery and the to-space, which contain young and old objects, - // respectively, during mutator time. - self.treadmill.enumerate_objects(enumerator, true, false); + // Visit objects in the allocation nursery and the to-space, which contain young and old + // objects, respectively, during mutator time. + self.treadmill.enumerate_objects(enumerator, false); } fn clear_side_log_bits(&self) { @@ -238,8 +243,7 @@ impl Space for LargeObjectSpace { }); // Visit all objects. It can be ordered arbitrarily with `Self::Release` which sweeps dead // objects (removing them from the treadmill) and clears their unlog bits, too. - self.treadmill - .enumerate_objects(&mut enumerator, true, true); + self.treadmill.enumerate_objects(&mut enumerator, true); } fn set_side_log_bits(&self) { @@ -247,8 +251,7 @@ impl Space for LargeObjectSpace { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); }); // Visit all objects. - self.treadmill - .enumerate_objects(&mut enumerator, true, true); + self.treadmill.enumerate_objects(&mut enumerator, true); } } @@ -312,8 +315,13 @@ impl LargeObjectSpace { } pub fn release(&mut self, full_heap: bool) { + // We swapped the allocation nursery and the collection nursery when GC starts, and we don't + // add objects to the allocation nursery during GC. It should have remained empty during + // the whole GC. + debug_assert!(self.treadmill.is_alloc_nursery_empty()); + self.sweep_large_pages(true); - debug_assert!(self.treadmill.is_nursery_empty()); + debug_assert!(self.treadmill.is_collect_nursery_empty()); if full_heap { self.sweep_large_pages(false); debug_assert!(self.treadmill.is_from_space_empty()); @@ -388,8 +396,10 @@ impl LargeObjectSpace { /// Enumerate objects for forwarding references. Used by certain mark-compact plans. pub(crate) fn enumerate_objects_for_forwarding(&self, enumerator: &mut dyn ObjectEnumerator) { - // Only visit the `to_space` which should contain all objects determined to be live. - self.treadmill.enumerate_objects(enumerator, false, false); + // The alloc nursery should have remained empty during the GC. + debug_assert!(self.treadmill.is_alloc_nursery_empty()); + // We only need to visit the to_space, which contains all objects determined to be live. + self.treadmill.enumerate_objects(enumerator, false); } /// Allocate an object diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index e8d5199c9f..28b3849ec7 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -16,9 +16,6 @@ pub struct TreadMill { /// The synchronized part of [`TreadMill`] #[derive(Default)] struct TreadMillSync { - /// The nursery. During mutator time, newly allocated objects are added to the nursery. After - /// a GC, the nursery will be evacuated. - nursery: HashSet, /// The from-space. During GC, old objects whose liveness are not yet determined are kept in /// the from-space. After GC, the from-space will be evacuated. from_space: HashSet, @@ -26,15 +23,22 @@ struct TreadMillSync { /// to the from-space at the beginning of GC, and objects are moved to the to-space once they /// are determined to be live. to_space: HashSet, + /// The collection nursery. During GC, young objects whose liveness are not yet determined are + /// kept in the collection nursery. After GC, the collection nursery will be evacuated. + collect_nursery: HashSet, + /// The allocation nursery. It holds newly allocated objects during mutator time. Objects in + /// the allocation nursery are moved to the collection nursery at the beginning of a GC. + alloc_nursery: HashSet, } impl std::fmt::Debug for TreadMill { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let sync = self.sync.lock().unwrap(); f.debug_struct("TreadMill") - .field("nursery", &sync.nursery) - .field("from", &sync.from_space) - .field("to", &sync.to_space) + .field("from_space", &sync.from_space) + .field("to_space", &sync.to_space) + .field("collect_nursery", &sync.collect_nursery) + .field("alloc_nursery", &sync.alloc_nursery) .finish() } } @@ -50,22 +54,22 @@ impl TreadMill { /// /// New objects are normally added to `nursery.to_space`. But when allocating as live (e.g. /// when concurrent marking is active), we directly add into the `to_space`. - pub fn add_to_treadmill(&self, object: ObjectReference, allocate_as_live: bool) { + pub fn add_to_treadmill(&self, object: ObjectReference, nursery: bool) { let mut sync = self.sync.lock().unwrap(); - if allocate_as_live { + if nursery { + trace!("Adding {} to alloc_nursery", object); + sync.alloc_nursery.insert(object); + } else { trace!("Adding {} to to_space", object); sync.to_space.insert(object); - } else { - trace!("Adding {} to nursery", object); - sync.nursery.insert(object); } } - /// Take all objects from the `nursery`. This is called during sweeping at which time all - /// objects in the nursery are unreachable. + /// Take all objects from the `collect_nursery`. This is called during sweeping at which time + /// all objects in the collection nursery are unreachable. pub fn collect_nursery(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); - std::mem::take(&mut sync.nursery) + std::mem::take(&mut sync.collect_nursery) } /// Take all objects from the `from_space`. This is called during sweeping at which time all @@ -80,11 +84,11 @@ impl TreadMill { let mut sync = self.sync.lock().unwrap(); if is_in_nursery { debug_assert!( - sync.nursery.contains(&object), - "copy source object ({}) must be in nursery", + sync.collect_nursery.contains(&object), + "copy source object ({}) must be in collect_nursery", object ); - sync.nursery.remove(&object); + sync.collect_nursery.remove(&object); } else { debug_assert!( sync.from_space.contains(&object), @@ -96,10 +100,10 @@ impl TreadMill { sync.to_space.insert(object); } - /// Return true if the nursery is empty. - pub fn is_nursery_empty(&self) -> bool { + /// Return true if the to-space is empty. + pub fn is_to_space_empty(&self) -> bool { let sync = self.sync.lock().unwrap(); - sync.nursery.is_empty() + sync.to_space.is_empty() } /// Return true if the from-space is empty. @@ -108,12 +112,27 @@ impl TreadMill { sync.from_space.is_empty() } - /// Flip the from-space and to-space. + /// Return true if the allocation nursery is empty. + pub fn is_alloc_nursery_empty(&self) -> bool { + let sync = self.sync.lock().unwrap(); + sync.alloc_nursery.is_empty() + } + + /// Return true if the collection nursery is empty. + pub fn is_collect_nursery_empty(&self) -> bool { + let sync = self.sync.lock().unwrap(); + sync.collect_nursery.is_empty() + } + + /// Flip object sets. /// - /// `full_heap` is true during full-heap GC, or false during nursery GC. It does nothing during - /// nursery GC. + /// It will flip the allocation nursery and the collection nursery. + /// + /// If `full_heap` is true, it will also flip the from-space and the to-space. pub fn flip(&mut self, full_heap: bool) { let sync = self.sync.get_mut().unwrap(); + swap(&mut sync.alloc_nursery, &mut sync.collect_nursery); + trace!("Flipped alloc_nursery and collect_nursery"); if full_heap { swap(&mut sync.from_space, &mut sync.to_space); trace!("Flipped from_space and to_space"); @@ -122,14 +141,11 @@ impl TreadMill { /// Enumerate objects. /// - /// Objects in the to-spaces are always enumerated. `nursery` and `from` controls whether to - /// enumerate objects in the nursery and from-spaces, respectively. - pub(crate) fn enumerate_objects( - &self, - enumerator: &mut dyn ObjectEnumerator, - nursery: bool, - from: bool, - ) { + /// Objects in the allocation nursery and the to-spaces are always enumerated. They include + /// live objects during mutator time, and objects determined to be live during a GC. + /// + /// If `all` is true, it will enumerate the collection nursery and the from-space, too. + pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator, all: bool) { let sync = self.sync.lock().unwrap(); let mut enumerated = 0usize; let mut visit_objects = |set: &HashSet| { @@ -138,17 +154,15 @@ impl TreadMill { enumerated += 1; } }; + visit_objects(&sync.alloc_nursery); visit_objects(&sync.to_space); - if nursery { - visit_objects(&sync.nursery); - } - - if from { + if all { + visit_objects(&sync.collect_nursery); visit_objects(&sync.from_space); } - debug!("Enumerated {enumerated} objects in LOS. nursery: {nursery}, from: {from}"); + debug!("Enumerated {enumerated} objects in LOS. all: {all}"); } } From 3b48be5077ad7aecefbea2877b3b1b40e5cc0935 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 4 Nov 2025 20:02:53 +0800 Subject: [PATCH 11/13] Update comments --- src/util/treadmill.rs | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 28b3849ec7..e7e91ef9f5 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -16,18 +16,15 @@ pub struct TreadMill { /// The synchronized part of [`TreadMill`] #[derive(Default)] struct TreadMillSync { - /// The from-space. During GC, old objects whose liveness are not yet determined are kept in - /// the from-space. After GC, the from-space will be evacuated. + /// The from-space. During GC, it contains old objects with unknown liveness. from_space: HashSet, - /// The to-space. It holds old objects during mutator time. Objects in the to-space are moved - /// to the from-space at the beginning of GC, and objects are moved to the to-space once they - /// are determined to be live. + /// The to-space. During mutator time, it contains old objects; during GC, it contains objects + /// determined to be live. to_space: HashSet, - /// The collection nursery. During GC, young objects whose liveness are not yet determined are - /// kept in the collection nursery. After GC, the collection nursery will be evacuated. + /// The collection nursery. During GC, it contains young objects with unknown liveness. collect_nursery: HashSet, - /// The allocation nursery. It holds newly allocated objects during mutator time. Objects in - /// the allocation nursery are moved to the collection nursery at the beginning of a GC. + /// The allocation nursery. During mutator time, it contains young objects; during GC, it + /// remains empty. alloc_nursery: HashSet, } @@ -52,8 +49,8 @@ impl TreadMill { /// Add an object to the treadmill. /// - /// New objects are normally added to `nursery.to_space`. But when allocating as live (e.g. - /// when concurrent marking is active), we directly add into the `to_space`. + /// New objects are normally added to `alloc_nursery`. But when allocating as live (e.g. when + /// concurrent marking is active), we directly add into the `to_space`. pub fn add_to_treadmill(&self, object: ObjectReference, nursery: bool) { let mut sync = self.sync.lock().unwrap(); if nursery { @@ -66,14 +63,14 @@ impl TreadMill { } /// Take all objects from the `collect_nursery`. This is called during sweeping at which time - /// all objects in the collection nursery are unreachable. + /// all unreachable young objects are in the collection nursery. pub fn collect_nursery(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); std::mem::take(&mut sync.collect_nursery) } /// Take all objects from the `from_space`. This is called during sweeping at which time all - /// objects in the from-space are unreachable. + /// unreachable old objects are in the from-space. pub fn collect_mature(&self) -> impl IntoIterator { let mut sync = self.sync.lock().unwrap(); std::mem::take(&mut sync.from_space) @@ -141,8 +138,8 @@ impl TreadMill { /// Enumerate objects. /// - /// Objects in the allocation nursery and the to-spaces are always enumerated. They include - /// live objects during mutator time, and objects determined to be live during a GC. + /// Objects in the allocation nursery and the to-spaces are always enumerated. They include all + /// objects during mutator time, and objects determined to be live during a GC. /// /// If `all` is true, it will enumerate the collection nursery and the from-space, too. pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator, all: bool) { From 14168358c9aec669628ec9d5c5138173aad8a5fc Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 5 Nov 2025 10:30:24 +0800 Subject: [PATCH 12/13] Rename function We no longer mention "forwarding" in the function name, but we add a comment saying that it is a workaround for Compressor. --- src/policy/compressor/compressorspace.rs | 8 ++++---- src/policy/largeobjectspace.rs | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/policy/compressor/compressorspace.rs b/src/policy/compressor/compressorspace.rs index cc4cf4dcc0..adf5746b6e 100644 --- a/src/policy/compressor/compressorspace.rs +++ b/src/policy/compressor/compressorspace.rs @@ -405,11 +405,11 @@ impl CompressorSpace { pub fn after_compact(&self, worker: &mut GCWorker, los: &LargeObjectSpace) { self.pr.reset_allocator(); // Update references from the LOS to Compressor too. - los.enumerate_objects_for_forwarding( - &mut object_enum::ClosureObjectEnumerator::<_, VM>::new(&mut |o: ObjectReference| { + los.enumerate_to_space_objects(&mut object_enum::ClosureObjectEnumerator::<_, VM>::new( + &mut |o: ObjectReference| { self.update_references(worker, o); - }), - ); + }, + )); } } diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 8c7864abe7..e30c12e18a 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -394,8 +394,10 @@ impl LargeObjectSpace { } } - /// Enumerate objects for forwarding references. Used by certain mark-compact plans. - pub(crate) fn enumerate_objects_for_forwarding(&self, enumerator: &mut dyn ObjectEnumerator) { + /// Enumerate objects in the to-space. It is a workaround for Compressor which currently needs + /// to enumerate reachable objects for during reference forwarding. + pub(crate) fn enumerate_to_space_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + // This function is intended to enumerate objects in the to-space. // The alloc nursery should have remained empty during the GC. debug_assert!(self.treadmill.is_alloc_nursery_empty()); // We only need to visit the to_space, which contains all objects determined to be live. From 3aaab7cf54328a6dfc874ab303cb3ef8267727ab Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 5 Nov 2025 10:56:27 +0800 Subject: [PATCH 13/13] More detailed logging for enumerate_objects --- src/util/treadmill.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index e7e91ef9f5..8ae409a006 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -159,7 +159,12 @@ impl TreadMill { visit_objects(&sync.from_space); } - debug!("Enumerated {enumerated} objects in LOS. all: {all}"); + debug!("Enumerated {enumerated} objects in LOS. all: {all}. from_space: {fs}, to_space: {ts}, collect_nursery: {cn}, alloc_nursery: {an}", + fs=sync.from_space.len(), + ts=sync.to_space.len(), + cn=sync.collect_nursery.len(), + an=sync.alloc_nursery.len(), + ); } }