Auto merge of #37972 - bluss:iter-find-is-on-a-roll, r=sfackler

Improve the slice iterator's searching methods

Improve all, any, find, position, rposition by explicitly unrolling the loop for the slice iterators.

- Introduce a few extension methods and functions for raw pointers make the new code easy to express
- Introduce helper methods `search_while, rsearch_while` that generalize all the searching methods

LLVM doesn't unroll the loop in `.find()` by default (clang is the same), so performance benefits a lot from explicit unrolling here. An iterator method without conditional exits (like `.fold()`) does not need this on the other hand.

One of the raw pointer extension methods is `fn post_inc(&mut self) -> Self` which is the rustic equivalent of “`ptr++`”, and it is a nice way to express the raw pointer loop (see commit 3).

Specific development notes about `search_while`: I tried both computing an end pointer "rounded" to 4, as well as the `ptrdistance >= 4` loop condition, ptrdistance was better. I tried handling the last 0-3 elements unrolled or with a while loop, the loop was better.

Author: bors

src/libcore/slice.rs

@@ -172,21 +172,35 @@ macro_rules! slice_offset {
     ($ptr:expr, $by:expr) => {{
         let ptr = $ptr;
         if size_from_ptr(ptr) == 0 {
-            ::intrinsics::arith_offset(ptr as *mut i8, $by) as *mut _
+            (ptr as *mut i8).wrapping_offset($by) as _
         } else {
             ptr.offset($by)
         }
     }};
 }
 
-macro_rules! slice_ref {
+// make a &T from a *const T
+macro_rules! make_ref {
+    ($ptr:expr) => {{
+        let ptr = $ptr;
+        if size_from_ptr(ptr) == 0 {
+            // Use a non-null pointer value
+            &*(1 as *mut _)
+        } else {
+            &*ptr
+        }
+    }};
+}
+
+// make a &mut T from a *mut T
+macro_rules! make_ref_mut {
     ($ptr:expr) => {{
         let ptr = $ptr;
         if size_from_ptr(ptr) == 0 {
             // Use a non-null pointer value
             &mut *(1 as *mut _)
         } else {
-            mem::transmute(ptr)
+            &mut *ptr
         }
     }};
 }
@@ -963,7 +977,7 @@ fn size_from_ptr<T>(_: *const T) -> usize {
 
 // The shared definition of the `Iter` and `IterMut` iterators
 macro_rules! iterator {
-    (struct $name:ident -> $ptr:ty, $elem:ty) => {
+    (struct $name:ident -> $ptr:ty, $elem:ty, $mkref:ident) => {
         #[stable(feature = "rust1", since = "1.0.0")]
         impl<'a, T> Iterator for $name<'a, T> {
             type Item = $elem;
@@ -979,18 +993,14 @@ macro_rules! iterator {
                     if self.ptr == self.end {
                         None
                     } else {
-                        let old = self.ptr;
-                        self.ptr = slice_offset!(self.ptr, 1);
-                        Some(slice_ref!(old))
+                        Some($mkref!(self.ptr.post_inc()))
                     }
                 }
             }
 
             #[inline]
             fn size_hint(&self) -> (usize, Option<usize>) {
-                let diff = (self.end as usize).wrapping_sub(self.ptr as usize);
-                let size = mem::size_of::<T>();
-                let exact = diff / (if size == 0 {1} else {size});
+                let exact = ptrdistance(self.ptr, self.end);
                 (exact, Some(exact))
             }
 
@@ -1009,6 +1019,64 @@ macro_rules! iterator {
             fn last(mut self) -> Option<$elem> {
                 self.next_back()
             }
+
+            fn all<F>(&mut self, mut predicate: F) -> bool
+                where F: FnMut(Self::Item) -> bool,
+            {
+                self.search_while(true, move |elt| {
+                    if predicate(elt) {
+                        SearchWhile::Continue
+                    } else {
+                        SearchWhile::Done(false)
+                    }
+                })
+            }
+
+            fn any<F>(&mut self, mut predicate: F) -> bool
+                where F: FnMut(Self::Item) -> bool,
+            {
+                !self.all(move |elt| !predicate(elt))
+            }
+
+            fn find<F>(&mut self, mut predicate: F) -> Option<Self::Item>
+                where F: FnMut(&Self::Item) -> bool,
+            {
+                self.search_while(None, move |elt| {
+                    if predicate(&elt) {
+                        SearchWhile::Done(Some(elt))
+                    } else {
+                        SearchWhile::Continue
+                    }
+                })
+            }
+
+            fn position<F>(&mut self, mut predicate: F) -> Option<usize>
+                where F: FnMut(Self::Item) -> bool,
+            {
+                let mut index = 0;
+                self.search_while(None, move |elt| {
+                    if predicate(elt) {
+                        SearchWhile::Done(Some(index))
+                    } else {
+                        index += 1;
+                        SearchWhile::Continue
+                    }
+                })
+            }
+
+            fn rposition<F>(&mut self, mut predicate: F) -> Option<usize>
+                where F: FnMut(Self::Item) -> bool,
+            {
+                let mut index = self.len();
+                self.rsearch_while(None, move |elt| {
+                    index -= 1;
+                    if predicate(elt) {
+                        SearchWhile::Done(Some(index))
+                    } else {
+                        SearchWhile::Continue
+                    }
+                })
+            }
         }
 
         #[stable(feature = "rust1", since = "1.0.0")]
@@ -1024,10 +1092,51 @@ macro_rules! iterator {
                     if self.end == self.ptr {
                         None
                     } else {
-                        self.end = slice_offset!(self.end, -1);
-                        Some(slice_ref!(self.end))
+                        Some($mkref!(self.end.pre_dec()))
+                    }
+                }
+            }
+        }
+
+        // search_while is a generalization of the internal iteration methods.
+        impl<'a, T> $name<'a, T> {
+            // search through the iterator's element using the closure `g`.
+            // if no element was found, return `default`.
+            fn search_while<Acc, G>(&mut self, default: Acc, mut g: G) -> Acc
+                where Self: Sized,
+                      G: FnMut($elem) -> SearchWhile<Acc>
+            {
+                // manual unrolling is needed when there are conditional exits from the loop
+                unsafe {
+                    while ptrdistance(self.ptr, self.end) >= 4 {
+                        search_while!(g($mkref!(self.ptr.post_inc())));
+                        search_while!(g($mkref!(self.ptr.post_inc())));
+                        search_while!(g($mkref!(self.ptr.post_inc())));
+                        search_while!(g($mkref!(self.ptr.post_inc())));
+                    }
+                    while self.ptr != self.end {
+                        search_while!(g($mkref!(self.ptr.post_inc())));
+                    }
+                }
+                default
+            }
+
+            fn rsearch_while<Acc, G>(&mut self, default: Acc, mut g: G) -> Acc
+                where Self: Sized,
+                      G: FnMut($elem) -> SearchWhile<Acc>
+            {
+                unsafe {
+                    while ptrdistance(self.ptr, self.end) >= 4 {
+                        search_while!(g($mkref!(self.end.pre_dec())));
+                        search_while!(g($mkref!(self.end.pre_dec())));
+                        search_while!(g($mkref!(self.end.pre_dec())));
+                        search_while!(g($mkref!(self.end.pre_dec())));
+                    }
+                    while self.ptr != self.end {
+                        search_while!(g($mkref!(self.end.pre_dec())));
                     }
                 }
+                default
             }
         }
     }
@@ -1061,6 +1170,24 @@ macro_rules! make_mut_slice {
     }}
 }
 
+// An enum used for controlling the execution of `.search_while()`.
+enum SearchWhile<T> {
+    // Continue searching
+    Continue,
+    // Fold is complete and will return this value
+    Done(T),
+}
+
+// helper macro for search while's control flow
+macro_rules! search_while {
+    ($e:expr) => {
+        match $e {
+            SearchWhile::Continue => { }
+            SearchWhile::Done(done) => return done,
+        }
+    }
+}
+
 /// Immutable slice iterator
 ///
 /// This struct is created by the [`iter`] method on [slices].
@@ -1147,7 +1274,7 @@ impl<'a, T> Iter<'a, T> {
     }
 }
 
-iterator!{struct Iter -> *const T, &'a T}
+iterator!{struct Iter -> *const T, &'a T, make_ref}
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<'a, T> ExactSizeIterator for Iter<'a, T> {
@@ -1275,7 +1402,7 @@ impl<'a, T> IterMut<'a, T> {
     }
 }
 
-iterator!{struct IterMut -> *mut T, &'a mut T}
+iterator!{struct IterMut -> *mut T, &'a mut T, make_ref_mut}
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<'a, T> ExactSizeIterator for IterMut<'a, T> {
@@ -1290,6 +1417,50 @@ impl<'a, T> FusedIterator for IterMut<'a, T> {}
 #[unstable(feature = "trusted_len", issue = "37572")]
 unsafe impl<'a, T> TrustedLen for IterMut<'a, T> {}
 
+
+// Return the number of elements of `T` from `start` to `end`.
+// Return the arithmetic difference if `T` is zero size.
+#[inline(always)]
+fn ptrdistance<T>(start: *const T, end: *const T) -> usize {
+    let diff = (end as usize).wrapping_sub(start as usize);
+    let size = mem::size_of::<T>();
+    diff / (if size == 0 { 1 } else { size })
+}
+
+// Extension methods for raw pointers, used by the iterators
+trait PointerExt : Copy {
+    unsafe fn slice_offset(self, i: isize) -> Self;
+
+    /// Increment self by 1, but return the old value
+    #[inline(always)]
+    unsafe fn post_inc(&mut self) -> Self {
+        let current = *self;
+        *self = self.slice_offset(1);
+        current
+    }
+
+    /// Decrement self by 1, and return the new value
+    #[inline(always)]
+    unsafe fn pre_dec(&mut self) -> Self {
+        *self = self.slice_offset(-1);
+        *self
+    }
+}
+
+impl<T> PointerExt for *const T {
+    #[inline(always)]
+    unsafe fn slice_offset(self, i: isize) -> Self {
+        slice_offset!(self, i)
+    }
+}
+
+impl<T> PointerExt for *mut T {
+    #[inline(always)]
+    unsafe fn slice_offset(self, i: isize) -> Self {
+        slice_offset!(self, i)
+    }
+}
+
 /// An internal abstraction over the splitting iterators, so that
 /// splitn, splitn_mut etc can be implemented once.
 #[doc(hidden)]