⚡ Add vernier profiler for SequenceSet tests and benchmarks

Gemfile

@@ -16,6 +16,7 @@ gem "test-unit-ruby-core", git: "https://github.com/ruby/test-unit-ruby-core"
 
 gem "benchmark", require: false
 gem "benchmark-driver", require: false
+gem "vernier", require: false, platform: :mri
 
 group :test do
   gem "simplecov",        require: false

benchmarks/sequence_set-new.yml

@@ -3,6 +3,9 @@ prelude: |
   require "net/imap"
   SeqSet = Net::IMAP::SequenceSet
 
+  require "./test/lib/profiling_helper"
+  include ProfilingHelper
+
   N_RAND = 100
 
   def rand_nums(n, min: 1, max: (n * 1.25).to_i) = Array.new(n) { rand(1..max) }
@@ -11,10 +14,12 @@ prelude: |
 
   def build_string_inputs(n, n_rand, **)
     Array.new(n_rand) { rand_string(n, **) }
+      .tap { maybe_profile("seqset-new-#{n}-string") }
   end
 
   def build_int_inputs(n, n_rand, **)
     Array.new(n_rand) { rand_entries(n, **) }
+      .tap { maybe_profile("seqset-new-#{n}-int") }
   end
 
   inputs = nil
@@ -82,6 +87,7 @@ contexts:
   - name: local
     prelude: |
       $LOAD_PATH.unshift "./lib"
+      $allowed_to_profile = true # only profile local code
     require: false
   - name: v0.5.9
     gems:

benchmarks/sequence_set-normalize.yml

@@ -7,6 +7,9 @@ prelude: |
   MAX_INPUT   = Integer ENV.fetch("BENCHMARK_MAX_INPUT",   1400)
   WARMUP_RUNS = Integer ENV.fetch("BENCHMARK_WARMUP_RUNS",  200)
 
+  require "./test/lib/profiling_helper"
+  include ProfilingHelper
+
   def init_sets(count: 100, set_size: INPUT_COUNT, max: MAX_INPUT)
     Array.new(count) {
       Net::IMAP::SequenceSet.new(Array.new(set_size) { rand(1..max) })
@@ -15,11 +18,13 @@ prelude: |
 
   def init_normal_sets(...)
     init_sets(...)
+      .tap do maybe_profile("seqset-normalize-normal") end
   end
 
   def init_frozen_normal_sets(...)
     init_sets(...)
       .map(&:freeze)
+      .tap do maybe_profile("seqset-normalize-frozen") end
   end
 
   def init_unsorted_sets(...)
@@ -31,6 +36,7 @@ prelude: |
           seqset.append entry
         end
       end
+      .tap do maybe_profile("seqset-normalize-unsorted") end
   end
 
   def init_abnormal_sets(...)
@@ -48,6 +54,7 @@ prelude: |
           end
         end
       end
+      .tap do maybe_profile("seqset-normalize-abnormal") end
   end
 
   # warmup (esp. for JIT)
@@ -80,6 +87,7 @@ contexts:
   - name: local
     prelude: |
       $LOAD_PATH.unshift "./lib"
+      $allowed_to_profile = true # only profile local code
     require: false
   - name: v0.5.0
     gems:

benchmarks/sequence_set-predicates.yml

@@ -7,6 +7,9 @@ prelude: |
   MAX_INPUT   = Integer ENV.fetch("BENCHMARK_MAX_INPUT",   1400)
   WARMUP_RUNS = Integer ENV.fetch("BENCHMARK_WARMUP_RUNS",  200)
 
+  require "./test/lib/profiling_helper"
+  include ProfilingHelper
+
   SETS = Array.new(1000) {
     Net::IMAP::SequenceSet[Array.new(INPUT_COUNT) { rand(1..MAX_INPUT) }]
   }
@@ -28,16 +31,20 @@ prelude: |
 
 benchmark:
   - name:   "intersect?"
+    prelude: 'maybe_profile("seqset-intersect-p")'
     script:  l, r = sets; l.intersect? r
   - name: "disjoint?"
+    prelude: 'maybe_profile("seqset-disjoint-p")'
     script:  l, r = sets; l.disjoint? r
   - name: "cover?"
+    prelude: 'maybe_profile("seqset-cover-p")'
     script:  l, r = sets; l.cover? r
 
 contexts:
   - name: local
     prelude: |
       $LOAD_PATH.unshift "./lib"
+      $allowed_to_profile = true # only profile local code
     require: false
   - name: v0.5.9
     gems:

benchmarks/sequence_set-slice.yml

@@ -2,7 +2,10 @@
 prelude: |
   require "net/imap"
 
-  def init(n:, d:)
+  require "./test/lib/profiling_helper"
+  include ProfilingHelper
+
+  def init(label = nil, n:, d: 1.5)
     n       = n.to_int
     samples = (1e5.to_i / n).ceil.clamp(1..100)
     domain  = 1..(n*d).floor
@@ -11,13 +14,14 @@ prelude: |
       n.times do set << rand(domain) end
       set
     }
-    $idxs   = Array.new([10_000, 2 * n].min) { rand(0..n - 1) }
+    $idxs   = Array.new([10_000, 2 * n].min) { rand(-n..n - 1) }
     $lens   = Array.new([10_000,     n].min) { rand(1..n) }
     $ranges = Array.new([10_000, n * n].min) {
       start = idx
       stop  = start.negative? ? rand(start..-1) : rand(start...n)
       start..stop
     }
+    maybe_profile("seqset-slice-#{label}-#{n}") if label
   end
 
   def set   = $sets.sample
@@ -26,7 +30,7 @@ prelude: |
   def range = $ranges.sample
 
   # warmup
-  init n: 100, d: 2
+  init n: 100
   2000.times do
     set[idx]
     set[range]
@@ -35,31 +39,32 @@ prelude: |
 
 benchmark:
 
-  - { name: "(N= 10 )      set[idx]", prelude: "init(n: 1e1, d: 4.0)", script: "set[idx]" }
-  - { name: "(N=100 )      set[idx]", prelude: "init(n: 1e2, d: 4.0)", script: "set[idx]" }
-  - { name: "(N=  1K)      set[idx]", prelude: "init(n: 1e3, d: 2.0)", script: "set[idx]" }
-  - { name: "(N= 10K)      set[idx]", prelude: "init(n: 1e4, d: 2.0)", script: "set[idx]" }
-  - { name: "(N=100K)      set[idx]", prelude: "init(n: 1e5, d: 2.0)", script: "set[idx]" }
-  - { name: "(N=  1M)      set[idx]", prelude: "init(n: 1e6, d: 1.5)", script: "set[idx]" }
+  - { name: "(N= 10 )      set[idx]", prelude: "init(:at, n: 1e1)", script: "set[idx]" }
+  - { name: "(N=100 )      set[idx]", prelude: "init(:at, n: 1e2)", script: "set[idx]" }
+  - { name: "(N=  1K)      set[idx]", prelude: "init(:at, n: 1e3)", script: "set[idx]" }
+  - { name: "(N= 10K)      set[idx]", prelude: "init(:at, n: 1e4)", script: "set[idx]" }
+  - { name: "(N=100K)      set[idx]", prelude: "init(:at, n: 1e5)", script: "set[idx]" }
+  - { name: "(N=  1M)      set[idx]", prelude: "init(:at, n: 1e6)", script: "set[idx]" }
 
-  - { name: "(N= 10 ) set[idx, len]", prelude: "init(n: 1e1, d: 4.0)", script: "set[idx, len]" }
-  - { name: "(N=100 ) set[idx, len]", prelude: "init(n: 1e2, d: 4.0)", script: "set[idx, len]" }
-  - { name: "(N=  1K) set[idx, len]", prelude: "init(n: 1e3, d: 2.0)", script: "set[idx, len]" }
-  - { name: "(N= 10K) set[idx, len]", prelude: "init(n: 1e4, d: 2.0)", script: "set[idx, len]" }
-  - { name: "(N=100K) set[idx, len]", prelude: "init(n: 1e5, d: 2.0)", script: "set[idx, len]" }
-  - { name: "(N=  1M) set[idx, len]", prelude: "init(n: 1e6, d: 1.5)", script: "set[idx, len]" }
+  - { name: "(N= 10 ) set[idx, len]", prelude: "init(:len, n: 1e1)", script: "set[idx, len]" }
+  - { name: "(N=100 ) set[idx, len]", prelude: "init(:len, n: 1e2)", script: "set[idx, len]" }
+  - { name: "(N=  1K) set[idx, len]", prelude: "init(:len, n: 1e3)", script: "set[idx, len]" }
+  - { name: "(N= 10K) set[idx, len]", prelude: "init(:len, n: 1e4)", script: "set[idx, len]" }
+  - { name: "(N=100K) set[idx, len]", prelude: "init(:len, n: 1e5)", script: "set[idx, len]" }
+  - { name: "(N=  1M) set[idx, len]", prelude: "init(:len, n: 1e6)", script: "set[idx, len]" }
 
-  - { name: "(N= 10 )    set[range]", prelude: "init(n: 1e1, d: 4.0)", script: "set[range]" }
-  - { name: "(N=100 )    set[range]", prelude: "init(n: 1e2, d: 4.0)", script: "set[range]" }
-  - { name: "(N=  1K)    set[range]", prelude: "init(n: 1e3, d: 2.0)", script: "set[range]" }
-  - { name: "(N= 10K)    set[range]", prelude: "init(n: 1e4, d: 2.0)", script: "set[range]" }
-  - { name: "(N=100K)    set[range]", prelude: "init(n: 1e5, d: 2.0)", script: "set[range]" }
-  - { name: "(N=  1M)    set[range]", prelude: "init(n: 1e6, d: 1.5)", script: "set[range]" }
+  - { name: "(N= 10 )    set[range]", prelude: "init(:range, n: 1e1)", script: "set[range]" }
+  - { name: "(N=100 )    set[range]", prelude: "init(:range, n: 1e2)", script: "set[range]" }
+  - { name: "(N=  1K)    set[range]", prelude: "init(:range, n: 1e3)", script: "set[range]" }
+  - { name: "(N= 10K)    set[range]", prelude: "init(:range, n: 1e4)", script: "set[range]" }
+  - { name: "(N=100K)    set[range]", prelude: "init(:range, n: 1e5)", script: "set[range]" }
+  - { name: "(N=  1M)    set[range]", prelude: "init(:range, n: 1e6)", script: "set[range]" }
 
 contexts:
   - name: local
     prelude: |
       $LOAD_PATH.unshift "./lib"
+      $allowed_to_profile = true # only profile local code
     require: false
   - name: v0.5.8 # fixes several bugs
     gems:

test/lib/helper.rb

@@ -1,31 +1,34 @@
-require "simplecov"
-
-# Cannot use ".simplecov" file: simplecov-json triggers a circular require.
-require "simplecov-json"
-SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new([
-  SimpleCov::Formatter::HTMLFormatter,
-  SimpleCov::Formatter::JSONFormatter,
-])
-
-SimpleCov.start do
-  command_name "Net::IMAP tests"
-  enable_coverage  :branch
-  primary_coverage :branch
-  enable_coverage_for_eval
-
-  add_filter "/test/"
-  add_filter "/rakelib/"
-
-  add_group "Parser", %w[lib/net/imap/response_parser.rb
-                         lib/net/imap/response_parser]
-  add_group "Config", %w[lib/net/imap/config.rb
-                         lib/net/imap/config]
-  add_group "SASL", %w[lib/net/imap/sasl.rb
-                       lib/net/imap/sasl
-                       lib/net/imap/authenticators.rb]
-  add_group "StringPrep", %w[lib/net/imap/stringprep.rb
-                             lib/net/imap/stringprep]
+unless ENV["SIMPLECOV_DISABLE"] in /\A(1|y(es)?|t(rue)?)\z/i
+  require "simplecov"
+
+  # Cannot use ".simplecov" file: simplecov-json triggers a circular require.
+  require "simplecov-json"
+  SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new([
+    SimpleCov::Formatter::HTMLFormatter,
+    SimpleCov::Formatter::JSONFormatter,
+  ])
+
+  SimpleCov.start do
+    command_name "Net::IMAP tests"
+    enable_coverage  :branch
+    primary_coverage :branch
+    enable_coverage_for_eval
+
+    add_filter "/test/"
+    add_filter "/rakelib/"
+
+    add_group "Parser", %w[lib/net/imap/response_parser.rb
+                          lib/net/imap/response_parser]
+    add_group "Config", %w[lib/net/imap/config.rb
+                          lib/net/imap/config]
+    add_group "SASL", %w[lib/net/imap/sasl.rb
+                        lib/net/imap/sasl
+                        lib/net/imap/authenticators.rb]
+    add_group "StringPrep", %w[lib/net/imap/stringprep.rb
+                              lib/net/imap/stringprep]
+  end
 end
+
 require "test/unit"
 require "core_assertions"
 
@@ -55,3 +58,5 @@ def assert_pattern
   end
 
 end
+
+require_relative "profiling_helper"

test/lib/profiling_helper.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ProfilingHelper
+  module_function
+
+  def profile?
+    require "vernier"
+    ENV["NET_IMAP_PROFILE"] in /\A(1|t|true|on|yes)\z/i
+  rescue LoadError
+    def profile? = false
+    false
+  end
+
+  def start_profiler(...) = if profile? then start_profiler!(...) end
+  def stop_profiler       = if profile? then stop_profiler!       end
+  def stop_profiler!      = Vernier.stop_profile
+
+  def maybe_profile(...)
+    return unless $allowed_to_profile # set in benchmark context prelude
+    profile_until_exit(...)
+  end
+
+  def profile_until_exit(...)
+    return unless profile?
+    start_profiler!(...)
+    at_exit { stop_profiler }
+  end
+
+  def start_profiler!(name, **opts)
+    require "vernier"
+    require "fileutils"
+
+    outdir = ENV.fetch("NET_IMAP_PROFILE_OUTDIR") {
+      cache_dir = ENV.fetch("XDG_CACHE_HOME", File.join(Dir.home, ".cache"))
+      File.join(cache_dir, "net-imap")
+    }
+    outfile = "prof-#{name}-#{Time.now.iso8601}.json.gz"
+    out = File.join(outdir, outfile)
+
+    FileUtils.mkdir_p outdir
+    Vernier.start_profile(out:, **opts)
+  end
+end

test/net/imap/test_sequence_set.rb

@@ -8,6 +8,24 @@ class IMAPSequenceSetTest < Test::Unit::TestCase
   SequenceSet     = Net::IMAP::SequenceSet
   DataFormatError = Net::IMAP::DataFormatError
 
+  if ProfilingHelper.profile?
+    def self.startup  = ProfilingHelper.start_profiler("seqset-tests")
+    def self.shutdown = ProfilingHelper.stop_profiler
+  end
+
+  if ENV["PROFILE_ALLOCATIONS"] =~ /\A(1|y(es)?|t(rue)?)\z/i
+    module ProfileAllocations
+      def setup = @allocated = GC.stat(:total_allocated_objects)
+
+      def teardown
+        return unless @allocated
+        allocated = GC.stat(:total_allocated_objects)
+        $stderr.puts "Allocated: %6d in %p" % [allocated - @allocated, local_name]
+      end
+    end
+    include ProfileAllocations
+  end
+
   def compare_to_reference_set(nums, set, seqset)
     set.merge nums
     seqset.merge nums