Improve benchmark drivers

.github/workflows/benchmark.yml

@@ -0,0 +1,30 @@
+name: Test benchmarks
+
+on:
+  push:
+    branches:
+      - master
+    tags: '*'
+  pull_request:
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        julia-version:
+          - '1.7'
+          - 'nightly'
+      fail-fast: false
+    name: Benchmark Julia ${{ matrix.julia-version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup julia
+        uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia-version }}
+      # Use `JULIA_PKG_SERVER` mitigation implemented in julia-buildpkg:
+      - uses: julia-actions/julia-buildpkg@v1
+      - run: make -C benchmark all
+        env:
+          BENCHMARK_NUM_THREADS: "2"

benchmark/.gitignore

@@ -0,0 +1 @@
+/config.mk

benchmark/Makefile

@@ -0,0 +1,44 @@
+include config.mk
+
+JULIA = julia
+JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no
+
+export JULIA_LOAD_PATH ?= @
+export JULIA_PROJECT ?= $(shell pwd)
+
+SUB_MAKEFILES = $(wildcard */Makefile)
+BENCHMARK_TARGETS = $(patsubst %/Makefile, benchmark-%, $(SUB_MAKEFILES))
+PLOTS_TARGETS = $(patsubst %/Makefile, plots-%, $(SUB_MAKEFILES))
+DUMP_CONFIG_TARGETS = $(patsubst %/Makefile, dump-config-%, $(SUB_MAKEFILES))
+
+.PHONY: benchmark* reinstantiate repl
+
+all: dump-config
+	$(MAKE) benchmark
+	$(MAKE) plots
+
+benchmark: $(BENCHMARK_TARGETS)
+$(BENCHMARK_TARGETS): benchmark-%: Manifest.toml
+	$(MAKE) -C $* benchmark
+
+plots: $(PLOTS_TARGETS)
+$(PLOTS_TARGETS): plots-%: Manifest.toml
+	$(MAKE) -C $* plots
+
+dump-config: $(DUMP_CONFIG_TARGETS)
+$(DUMP_CONFIG_TARGETS): dump-config-%:
+	$(MAKE) -C $* dump-config
+
+Manifest.toml:
+	JULIA_LOAD_PATH=@:@stdlib $(JULIA_CMD) --project=. instantiate.jl
+
+reinstantiate:
+	rm -f Manifest.toml
+	$(MAKE) Manifest.toml
+
+repl:
+	JULIA_LOAD_PATH=: $(JULIA)
+
+config.mk:
+	touch $@
+#	ln -s default-config.mk $@

benchmark/Project.toml

@@ -0,0 +1,10 @@
+[deps]
+BenchmarkConfigSweeps = "5dead3b6-c9e4-435a-a292-80e2716ed6d9"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ConcurrentCollections = "5060bff5-0b44-40c5-b522-fcd3ca5cecdd"
+ConcurrentCollectionsBenchmarks = "172d371b-ba13-4e7e-a6ad-f169f5735fdf"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+VegaLite = "112f6efa-9a02-5b7d-90c0-432ed331239a"

benchmark/dict_histogram/.gitignore

@@ -1,3 +1,4 @@
 /backup
 /build
+/config.mk
 /tmp

benchmark/dict_histogram/Makefile

@@ -1,25 +1,52 @@
-JULIA = julia1.7
+include config.mk
+
+JULIA ?= julia
 JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no
 
-export JULIA_PROJECT = $(shell pwd)/../../test/ConcurrentCollectionsTests
-# export JULIA_LOAD_PATH = @
+export JULIA_LOAD_PATH ?= @
+export JULIA_PROJECT ?= $(shell pwd)/..
+
+BENCHMARK_NUM_THREADS ?= $(shell ../nthreads.sh)
+BENCHMARK_EXCLUSIVE ?= 0
 
-.PHONY: benchmark clean backup
+.PHONY: all benchmark plots clean backup print-* dump-*
 
-BUILD = build
+all: dump-config
+	$(MAKE) benchmark
+	$(MAKE) plots
 
-benchmark: $(BUILD)/results.json
+benchmark: build/results.json
+build/results.json:
+	JULIA_NUM_THREADS=$(BENCHMARK_NUM_THREADS) \
+	JULIA_EXCLUSIVE=$(BENCHMARK_EXCLUSIVE) \
+		$(JULIA_CMD) run.jl
 
-$(BUILD)/results.json:
-	$(JULIA_CMD) -t16 run.jl
+plots: build/results.png
+build/results.png:
+	$(JULIA_CMD) plot.jl
 
 clean:
-	rm -fv $(BUILD)/*.json
+	rm -fv build/*.*
 
 backup:
-	test -e $(BUILD)/results.json
+	test -e build/results.json
 	mkdir -pv backup
 	rm -rf tmp/backup
 	mkdir -pv tmp/backup/build
-	mv $(BUILD)/* tmp/backup/build/
+	mv build/* tmp/backup/build/
 	mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S)
+
+print-%:
+	@echo "    "$*=${$*}
+
+dump-config: \
+print-JULIA \
+print-JULIA_CMD \
+print-JULIA_LOAD_PATH \
+print-JULIA_PROJECT \
+print-BENCHMARK_NUM_THREADS \
+print-BENCHMARK_EXCLUSIVE
+
+config.mk:
+	touch $@
+#	ln -s default-config.mk $@

benchmark/dict_histogram/run.jl

@@ -2,38 +2,13 @@ import BenchmarkTools
 import ConcurrentCollectionsBenchmarks
 import JSON
 
-function git_info(dir = @__DIR__)
-    git(cmd) = strip(read(setenv(`git $cmd`; dir), String))
-    return (;
-        revision = git(`rev-parse HEAD`),
-        status = git(`status --short --untracked-files=no --porcelain`),
-    )
-end
-
-function julia_info()
-    return (
-        version = string(VERSION),
-        git = (
-            commit = Base.GIT_VERSION_INFO.commit,
-            branch = Base.GIT_VERSION_INFO.branch,
-        ),
-        is_debugbuild = ccall(:jl_is_debugbuild, Cint, ()) != 0,
-        libllvm_version = string(Base.libllvm_version),
-        Sys = (
-            WORD_SIZE = Sys.WORD_SIZE,
-            JIT = Sys.JIT,
-            # CPU_NAME = Sys.CPU_NAME,
-            # CPU_THREADS = Sys.CPU_THREADS,
-        ),
-        env = Dict(k => v for (k, v) in ENV if startswith(k, "JULIA_")),
-    )
-end
+include("../info_dump.jl")
 
 function main(args = ARGS)
     output = get(args, 1, joinpath(@__DIR__, "build", "results.json"))
     mkpath(dirname(output))
 
-    info = (; git = git_info(), julia = julia_info())
+    info = InfoDump.info()
     open(joinpath(dirname(output), "info.json"), write = true) do io
         JSON.print(io, info)
     end

benchmark/hotpotato/.gitignore

@@ -1 +1,4 @@
+/backup
 /build
+/config.mk
+/tmp

benchmark/hotpotato/Makefile

@@ -0,0 +1,52 @@
+include config.mk
+
+JULIA ?= julia
+JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no
+
+export JULIA_LOAD_PATH ?= @
+export JULIA_PROJECT ?= $(shell pwd)/..
+
+BENCHMARK_NUM_THREADS ?= $(shell ../nthreads.sh)
+BENCHMARK_EXCLUSIVE ?= 0
+
+.PHONY: all benchmark plots print-* dump-*
+
+all: dump-config
+	$(MAKE) benchmark
+	$(MAKE) plots
+
+benchmark: build/results.json
+build/results.json:
+	JULIA_NUM_THREADS=$(BENCHMARK_NUM_THREADS) \
+	JULIA_EXCLUSIVE=$(BENCHMARK_EXCLUSIVE) \
+		$(JULIA_CMD) run.jl
+
+plots: build/results.png
+build/results.png:
+	$(JULIA_CMD) plot.jl
+
+clean:
+	rm -fv build/*.*
+
+backup:
+	test -e build/results.json
+	mkdir -pv backup
+	rm -rf tmp/backup
+	mkdir -pv tmp/backup/build
+	mv build/* tmp/backup/build/
+	mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S)
+
+print-%:
+	@echo "    "$*=${$*}
+
+dump-config: \
+print-JULIA \
+print-JULIA_CMD \
+print-JULIA_LOAD_PATH \
+print-JULIA_PROJECT \
+print-BENCHMARK_NUM_THREADS \
+print-BENCHMARK_EXCLUSIVE
+
+config.mk:
+	touch $@
+#	ln -s default-config.mk $@

benchmark/hotpotato/plot.jl

@@ -4,7 +4,8 @@ using JSON
 using Statistics
 using VegaLite
 
-rawdata = JSON.parsefile(joinpath(@__DIR__, "build/results.json"))
+resultsdata = JSON.parsefile(joinpath(@__DIR__, "build/results.json"))
+rawdata = resultsdata["results"]
 
 potatos = map(rawdata["potatos"]) do info
     result = info["result"]

benchmark/hotpotato/run.jl

@@ -2,6 +2,8 @@ using ConcurrentCollections: DualLinkedConcurrentRingQueue
 using ConcurrentCollectionsBenchmarks.BenchQueueHotPotato: hotpotato!, fai_stats
 using JSON
 
+include("../info_dump.jl")
+
 function sweep(; repeat = 10, duration = 1, maxntasks = Threads.nthreads())
     # cooldown() = sleep(0.1)
     # cooldown() = GC.gc()
@@ -27,23 +29,15 @@ function sweep(; repeat = 10, duration = 1, maxntasks = Threads.nthreads())
     return (; potatos, fais, repeat, duration)
 end
 
-function git_info(dir = @__DIR__)
-    git(cmd) = strip(read(setenv(`git $cmd`; dir), String))
-    return (;
-        revision = git(`rev-parse HEAD`),
-        status = git(`status --short --untracked-files=no --porcelain`),
-    )
-end
-
 function main(args = ARGS)
     output = get(args, 1, joinpath(@__DIR__, "build", "results.json"))
     mkpath(dirname(output))
-    git = git_info()
+    info = InfoDump.info()
     @info "Warmup..."
     sweep(; repeat = 1, duration = 0.1, maxntasks = 1)
     @info "Benchmarking..."
     results = sweep()
-    results = (; results..., git)
+    results = (; results, info)
     open(output, write = true) do io
         JSON.print(io, results)
     end

benchmark/info_dump.jl

@@ -0,0 +1,47 @@
+#!/bin/bash
+# -*- mode: julia -*-
+#=
+JULIA="${JULIA:-julia}"
+JULIA_CMD="${JULIA_CMD:-$JULIA --color=yes --startup-file=no --compile=min -O0}"
+export JULIA_LOAD_PATH=@
+export JULIA_PROJECT="$(dirname "${BASH_SOURCE[0]}")"
+exec $JULIA_CMD "${BASH_SOURCE[0]}" "$@"
+=#
+
+module InfoDump
+
+function git_info(dir = @__DIR__)
+    git(cmd) = strip(read(setenv(`git $cmd`; dir), String))
+    return (;
+        revision = git(`rev-parse HEAD`),
+        status = git(`status --short --untracked-files=no --porcelain`),
+    )
+end
+
+function julia_info()
+    return (
+        version = string(VERSION),
+        git = (
+            commit = Base.GIT_VERSION_INFO.commit,
+            branch = Base.GIT_VERSION_INFO.branch,
+        ),
+        is_debugbuild = ccall(:jl_is_debugbuild, Cint, ()) != 0,
+        libllvm_version = string(Base.libllvm_version),
+        Sys = (
+            WORD_SIZE = Sys.WORD_SIZE,
+            JIT = Sys.JIT,
+            # CPU_NAME = Sys.CPU_NAME,
+            # CPU_THREADS = Sys.CPU_THREADS,
+        ),
+        env = Dict(k => v for (k, v) in ENV if startswith(k, "JULIA_")),
+    )
+end
+
+info() = (; git = git_info(), julia = julia_info())
+
+end  # module
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    import JSON
+    JSON.print(stdout, InfoDump.info(), 4)
+end

benchmark/instantiate.jl

@@ -0,0 +1,20 @@
+using Pkg
+
+packages = [
+    PackageSpec(
+        name = "ConcurrentCollections",
+        path = dirname(@__DIR__),
+        # url = "https://github.com/JuliaConcurrent/ConcurrentCollections.jl.git",
+    ),
+    PackageSpec(
+        name = "ConcurrentCollectionsBenchmarks",
+        path = joinpath(@__DIR__, "ConcurrentCollectionsBenchmarks"),
+        # url = "https://github.com/JuliaConcurrent/ConcurrentCollections.jl.git",
+        # subdir = "benchmark/ConcurrentCollectionsBenchmarks",
+    ),
+]
+
+Pkg.develop(packages)
+# Pkg.add(packages)
+
+Pkg.instantiate()

benchmark/nthreads.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if lscpu --parse >/dev/null 2>/dev/null
+then
+    lscpu --parse | grep -v '^#' | sort --unique --field-separator=, --key=2 | wc -l
+elif nproc >/dev/null 2>/dev/null
+then
+    exec nproc
+else
+    echo auto
+fi