don't execute long operations more than once per iteration (#760), fixes #736

* don't execute long operations more than once per iteration, #736

* generate the right C# code for #736

* EngineParameters.Resolver was always null or ignored ;), #736

* don't forget to JIT idle, #736

* do the math right for unroll factor for JIT, #736

* generate the right IL code, #736

* Setup and Cleanup are jitted together with benchmark, #736

* engine factory is now supposed to create an engine which is ready to run (hence the method name change), #736

* addressing PR feedback, #736

* bring back the calls to DummyActions, #736

* align iteration mode

* don't measure the overhead for time consuming benchmarks, don't run pilot if jitting gives the answer, #736

* fix the Linux build

Author: adamsitnik

samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs

@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Order;
+using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Toolchains.InProcess;
 
 namespace BenchmarkDotNet.Samples.Intro
@@ -18,7 +19,7 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                var wrongPlatform = IntPtr.Size == sizeof(int)
+                var wrongPlatform = RuntimeInformation.GetCurrentPlatform() == Platform.X86
                     ? Platform.X64
                     : Platform.X86;
 

src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs

@@ -156,7 +156,7 @@ public static bool Is64Bit(Process process)
                     return !isWow64;
                 }
 
-                return IntPtr.Size == 8; // todo: find the way to cover all scenarios for .NET Core
+                return Portability.RuntimeInformation.GetCurrentPlatform() == Platform.X64; // todo: find the way to cover all scenarios for .NET Core
             }
 
             [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)]

src/BenchmarkDotNet/Engines/Engine.cs

@@ -41,10 +41,10 @@ public class Engine : IEngine
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineTargetStage targetStage;
         private readonly bool includeMemoryStats;
-        private bool isJitted;
 
         internal Engine(
             IHost host,
+            IResolver resolver,
             Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> idleAction, Action<long> mainAction, Job targetJob,
             Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke,
             bool includeMemoryStats)
@@ -64,7 +64,7 @@ internal Engine(
             OperationsPerInvoke = operationsPerInvoke;
             this.includeMemoryStats = includeMemoryStats;
 
-            Resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance);
+            Resolver = resolver;
 
             Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver);
             ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
@@ -78,22 +78,10 @@ internal Engine(
             targetStage = new EngineTargetStage(this);
         }
 
-        public void Jitting()
-        {
-            // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt
-            Dummy1Action.Invoke();
-            MainAction.Invoke(1);
-            Dummy2Action.Invoke();
-            IdleAction.Invoke(1);
-            Dummy3Action.Invoke();
-            isJitted = true;
-        }
+        public void Dispose() => GlobalCleanupAction?.Invoke();
 
         public RunResults Run()
         {
-            if (Strategy.NeedsJitting() != isJitted)
-                throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!");
-
             long invokeCount = InvocationCount;
             IReadOnlyList<Measurement> idle = null;
 

src/BenchmarkDotNet/Engines/EngineFactory.cs

@@ -1,39 +1,114 @@
 using System;
+using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 
 namespace BenchmarkDotNet.Engines
 {
-    // TODO: Default instance?
     public class EngineFactory : IEngineFactory
     {
-        public IEngine Create(EngineParameters engineParameters)
+        public IEngine CreateReadyToRun(EngineParameters engineParameters)
         {
-            if (engineParameters.MainAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.MainAction));
+            if (engineParameters.MainSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainSingleAction));
+            if (engineParameters.MainMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainMultiAction));
             if (engineParameters.Dummy1Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy1Action));
             if (engineParameters.Dummy2Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy2Action));
             if (engineParameters.Dummy3Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy3Action));
-            if (engineParameters.IdleAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.IdleAction));
+            if (engineParameters.IdleSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleSingleAction));
+            if (engineParameters.IdleMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction));
             if(engineParameters.TargetJob == null)
                 throw new ArgumentNullException(nameof(engineParameters.TargetJob));
 
-            return new Engine(
+            engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose
+            
+            if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit
+                return CreateMultiActionEngine(engineParameters);
+
+            int jitIndex = 0;
+            
+            if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it
+            {
+                var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters);
+                
+                DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor));
+
+                return warmedUpMultiActionEngine;
+            }
+            
+            var singleActionEngine = CreateSingleActionEngine(engineParameters);
+            if (Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1) > engineParameters.IterationTime)
+                return singleActionEngine; // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead
+
+            var multiActionEngine = CreateMultiActionEngine(engineParameters);
+            int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver);
+
+            if (Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor) > engineParameters.IterationTime) 
+            {   // executing defaultUnrollFactor times takes longer than iteration time => medium running benchmark, needs no pilot and no overhead
+                var defaultUnrollFactorTimesPerIterationNoPilotNoOverhead = CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, 
+                    invocationCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor); // run the benchmark exactly once per iteration
+                
+                return CreateEngine(engineParameters, defaultUnrollFactorTimesPerIterationNoPilotNoOverhead, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+            }
+            
+            return multiActionEngine;
+        }
+
+        /// <returns>the time it took to run the benchmark</returns>
+        private static TimeInterval Jit(Engine engine, int jitIndex, int invokeCount, int unrollFactor)
+        {
+            engine.Dummy1Action.Invoke();
+
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, jitIndex, invokeCount, unrollFactor))); // don't forget to JIT idle
+            
+            engine.Dummy2Action.Invoke();
+
+            var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, jitIndex, invokeCount, unrollFactor));
+
+            engine.Dummy3Action.Invoke();
+
+            engine.WriteLine();
+            
+            return TimeInterval.FromNanoseconds(result.Nanoseconds);
+        }
+
+        private static Engine CreateMultiActionEngine(EngineParameters engineParameters) 
+            => CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+
+        private static Engine CreateSingleActionEngine(EngineParameters engineParameters) 
+            => CreateEngine(engineParameters,
+                CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, invocationCount: 1, unrollFactor: 1), // run the benchmark exactly once per iteration
+                engineParameters.IdleSingleAction, 
+                engineParameters.MainSingleAction);
+        
+        private static Job CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(Job sourceJob, int invocationCount, int unrollFactor)
+            => sourceJob
+                .WithInvocationCount(invocationCount).WithUnrollFactor(unrollFactor) 
+                .WithEvaluateOverhead(false); // it's very time consuming, don't evaluate the overhead which would be 0,000025% of the target run or even less
+                // todo: consider if we should set the warmup count to 2
+
+        private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action<long> idle, Action<long> main)
+            => new Engine(
                 engineParameters.Host,
+                EngineParameters.DefaultResolver,
                 engineParameters.Dummy1Action,
                 engineParameters.Dummy2Action,
                 engineParameters.Dummy3Action,
-                engineParameters.IdleAction,
-                engineParameters.MainAction,
-                engineParameters.TargetJob,
+                idle,
+                main,
+                job,
                 engineParameters.GlobalSetupAction,
                 engineParameters.GlobalCleanupAction,
                 engineParameters.IterationSetupAction,
                 engineParameters.IterationCleanupAction,
                 engineParameters.OperationsPerInvoke,
                 engineParameters.MeasureGcStats);
-        }
     }
 }

src/BenchmarkDotNet/Engines/EngineParameters.cs

@@ -1,24 +1,39 @@
 using System;
 using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Horology;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
 
 namespace BenchmarkDotNet.Engines
 {
     public class EngineParameters
     {
+        public static readonly IResolver DefaultResolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance);
+        
         public IHost Host { get; set; }
-        public Action<long> MainAction { get; set; }
+        public Action<long> MainSingleAction { get; set; }
+        public Action<long> MainMultiAction { get; set; }
         public Action Dummy1Action { get; set; }
         public Action Dummy2Action { get; set; }
         public Action Dummy3Action { get; set; }
-        public Action<long> IdleAction { get; set; }
+        public Action<long> IdleSingleAction { get; set; }
+        public Action<long> IdleMultiAction { get; set; }
         public Job TargetJob { get; set; } = Job.Default;
         public long OperationsPerInvoke { get; set; } = 1;
-        public Action GlobalSetupAction { get; set; } = null;
-        public Action GlobalCleanupAction { get; set; } = null;
-        public Action IterationSetupAction { get; set; } = null;
-        public Action IterationCleanupAction { get; set; } = null;
-        public IResolver Resolver { get; set; }
+        public Action GlobalSetupAction { get; set; }
+        public Action GlobalCleanupAction { get; set; }
+        public Action IterationSetupAction { get; set; }
+        public Action IterationCleanupAction { get; set; }
         public bool MeasureGcStats { get; set; }
+        
+        public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting();
+
+        public bool HasInvocationCount => TargetJob.HasValue(RunMode.InvocationCountCharacteristic);
+        
+        public bool HasUnrollFactor => TargetJob.HasValue(RunMode.UnrollFactorCharacteristic);
+
+        public int UnrollFactor => TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver);
+        
+        public TimeInterval IterationTime => TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, DefaultResolver);
     }
 }

src/BenchmarkDotNet/Engines/IEngine.cs

@@ -6,7 +6,7 @@
 
 namespace BenchmarkDotNet.Engines
 {
-    public interface IEngine
+    public interface IEngine : IDisposable
     {
         [NotNull]
         IHost Host { get; }
@@ -36,12 +36,6 @@ public interface IEngine
 
         Measurement RunIteration(IterationData data);
 
-        /// <summary>
-        /// must perform jitting via warmup calls
-        /// <remarks>is called after first call to GlobalSetup, from the auto-generated benchmark process</remarks>
-        /// </summary>
-        void Jitting();
-
         RunResults Run();
     }
 }

src/BenchmarkDotNet/Engines/IEngineFactory.cs

@@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines
 {
     public interface IEngineFactory
     {
-        IEngine Create(EngineParameters engineParameters);
+        IEngine CreateReadyToRun(EngineParameters engineParameters);
     }
 }

src/BenchmarkDotNet/Engines/IterationMode.cs

@@ -35,6 +35,11 @@ public enum IterationMode
         /// <summary>
         /// Unknown 
         /// </summary>
-        Unknown
+        Unknown,
+        
+        /// <summary>
+        /// executing benchmark for the purpose of JIT wamup
+        /// </summary>
+        MainJitting, IdleJitting
     }
 }

src/BenchmarkDotNet/Engines/IterationModeExtensions.cs

@@ -3,6 +3,6 @@
     public static class IterationModeExtensions
     {
         public static bool IsIdle(this IterationMode mode) 
-            => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget;
+            => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJitting;
     }
 }

src/BenchmarkDotNet/Extensions/ProcessExtensions.cs

@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Running;
 using JetBrains.Annotations;
 
@@ -31,7 +32,7 @@ private static IntPtr FixAffinity(IntPtr processorAffinity)
         {
             int cpuMask = (1 << Environment.ProcessorCount) - 1;
 
-            return IntPtr.Size == sizeof(Int64) 
+            return RuntimeInformation.GetCurrentPlatform() == Platform.X64
                 ? new IntPtr(processorAffinity.ToInt64() & cpuMask)
                 : new IntPtr(processorAffinity.ToInt32() & cpuMask);
         }