From bc03853ee32a70d63ec1b67b7363108133fc0dc1 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Dec 2021 17:11:51 -0500 Subject: [PATCH 1/3] Automate benchmark drivers --- benchmark/.gitignore | 1 + benchmark/Makefile | 39 ++++++++++++++++++++++ benchmark/Project.toml | 10 ++++++ benchmark/dict_histogram/.gitignore | 1 + benchmark/dict_histogram/Makefile | 49 +++++++++++++++++++++------ benchmark/dict_histogram/run.jl | 29 ++-------------- benchmark/hotpotato/.gitignore | 3 ++ benchmark/hotpotato/Makefile | 52 +++++++++++++++++++++++++++++ benchmark/hotpotato/plot.jl | 3 +- benchmark/hotpotato/run.jl | 14 +++----- benchmark/info_dump.jl | 47 ++++++++++++++++++++++++++ benchmark/instantiate.jl | 20 +++++++++++ benchmark/nthreads.sh | 11 ++++++ 13 files changed, 230 insertions(+), 49 deletions(-) create mode 100644 benchmark/.gitignore create mode 100644 benchmark/Makefile create mode 100644 benchmark/Project.toml create mode 100644 benchmark/hotpotato/Makefile create mode 100755 benchmark/info_dump.jl create mode 100644 benchmark/instantiate.jl create mode 100755 benchmark/nthreads.sh diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..6896ef6 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1 @@ +/config.mk diff --git a/benchmark/Makefile b/benchmark/Makefile new file mode 100644 index 0000000..be1b6f9 --- /dev/null +++ b/benchmark/Makefile @@ -0,0 +1,39 @@ +include config.mk + +JULIA = julia +JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no + +export JULIA_LOAD_PATH ?= @ +export JULIA_PROJECT ?= $(shell pwd) + +SUB_MAKEFILES = $(wildcard */Makefile) +BENCHMARK_TARGETS = $(patsubst %/Makefile, benchmark-%, $(SUB_MAKEFILES)) +PLOTS_TARGETS = $(patsubst %/Makefile, plots-%, $(SUB_MAKEFILES)) + +.PHONY: benchmark* reinstantiate repl + +all: + $(MAKE) benchmark + $(MAKE) plots + +benchmark: $(BENCHMARK_TARGETS) +$(BENCHMARK_TARGETS): benchmark-%: Manifest.toml + $(MAKE) -C $* benchmark + +plots: $(PLOTS_TARGETS) +$(PLOTS_TARGETS): plots-%: Manifest.toml + $(MAKE) -C $* plots + +Manifest.toml: + JULIA_LOAD_PATH=@:@stdlib $(JULIA_CMD) --project=. instantiate.jl + +reinstantiate: + rm -f Manifest.toml + $(MAKE) Manifest.toml + +repl: + JULIA_LOAD_PATH=: $(JULIA) + +config.mk: + touch $@ +# ln -s default-config.mk $@ diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 0000000..dbc44cd --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,10 @@ +[deps] +BenchmarkConfigSweeps = "5dead3b6-c9e4-435a-a292-80e2716ed6d9" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ConcurrentCollections = "5060bff5-0b44-40c5-b522-fcd3ca5cecdd" +ConcurrentCollectionsBenchmarks = "172d371b-ba13-4e7e-a6ad-f169f5735fdf" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +VegaLite = "112f6efa-9a02-5b7d-90c0-432ed331239a" diff --git a/benchmark/dict_histogram/.gitignore b/benchmark/dict_histogram/.gitignore index 87e4091..2aa0c6b 100644 --- a/benchmark/dict_histogram/.gitignore +++ b/benchmark/dict_histogram/.gitignore @@ -1,3 +1,4 @@ /backup /build +/config.mk /tmp diff --git a/benchmark/dict_histogram/Makefile b/benchmark/dict_histogram/Makefile index c86300d..b707cf1 100644 --- a/benchmark/dict_histogram/Makefile +++ b/benchmark/dict_histogram/Makefile @@ -1,25 +1,52 @@ -JULIA = julia1.7 +include config.mk + +JULIA ?= julia JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no -export JULIA_PROJECT = $(shell pwd)/../../test/ConcurrentCollectionsTests -# export JULIA_LOAD_PATH = @ +export JULIA_LOAD_PATH ?= @ +export JULIA_PROJECT ?= $(shell pwd)/.. + +BENCHMARK_NUM_THREADS ?= $(shell ../nthreads.sh) +BENCHMARK_EXCLUSIVE ?= 0 -.PHONY: benchmark clean backup +.PHONY: all benchmark plots clean backup print-* dump-* -BUILD = build +all: + $(MAKE) benchmark + $(MAKE) plots -benchmark: $(BUILD)/results.json +benchmark: build/results.json +build/results.json: + JULIA_NUM_THREADS=$(BENCHMARK_NUM_THREADS) \ + JULIA_EXCLUSIVE=$(BENCHMARK_EXCLUSIVE) \ + $(JULIA_CMD) run.jl -$(BUILD)/results.json: - $(JULIA_CMD) -t16 run.jl +plots: build/results.png +build/results.png: + $(JULIA_CMD) plot.jl clean: - rm -fv $(BUILD)/*.json + rm -fv build/*.* backup: - test -e $(BUILD)/results.json + test -e build/results.json mkdir -pv backup rm -rf tmp/backup mkdir -pv tmp/backup/build - mv $(BUILD)/* tmp/backup/build/ + mv build/* tmp/backup/build/ mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S) + +print-%: + @echo $*=${$*} + +dump-config: \ +print-JULIA \ +print-JULIA_CMD \ +print-JULIA_LOAD_PATH \ +print-JULIA_PROJECT \ +print-BENCHMARK_NUM_THREADS \ +print-BENCHMARK_EXCLUSIVE + +config.mk: + touch $@ +# ln -s default-config.mk $@ diff --git a/benchmark/dict_histogram/run.jl b/benchmark/dict_histogram/run.jl index e075e33..eabc0b1 100644 --- a/benchmark/dict_histogram/run.jl +++ b/benchmark/dict_histogram/run.jl @@ -2,38 +2,13 @@ import BenchmarkTools import ConcurrentCollectionsBenchmarks import JSON -function git_info(dir = @__DIR__) - git(cmd) = strip(read(setenv(`git $cmd`; dir), String)) - return (; - revision = git(`rev-parse HEAD`), - status = git(`status --short --untracked-files=no --porcelain`), - ) -end - -function julia_info() - return ( - version = string(VERSION), - git = ( - commit = Base.GIT_VERSION_INFO.commit, - branch = Base.GIT_VERSION_INFO.branch, - ), - is_debugbuild = ccall(:jl_is_debugbuild, Cint, ()) != 0, - libllvm_version = string(Base.libllvm_version), - Sys = ( - WORD_SIZE = Sys.WORD_SIZE, - JIT = Sys.JIT, - # CPU_NAME = Sys.CPU_NAME, - # CPU_THREADS = Sys.CPU_THREADS, - ), - env = Dict(k => v for (k, v) in ENV if startswith(k, "JULIA_")), - ) -end +include("../info_dump.jl") function main(args = ARGS) output = get(args, 1, joinpath(@__DIR__, "build", "results.json")) mkpath(dirname(output)) - info = (; git = git_info(), julia = julia_info()) + info = InfoDump.info() open(joinpath(dirname(output), "info.json"), write = true) do io JSON.print(io, info) end diff --git a/benchmark/hotpotato/.gitignore b/benchmark/hotpotato/.gitignore index 796b96d..2aa0c6b 100644 --- a/benchmark/hotpotato/.gitignore +++ b/benchmark/hotpotato/.gitignore @@ -1 +1,4 @@ +/backup /build +/config.mk +/tmp diff --git a/benchmark/hotpotato/Makefile b/benchmark/hotpotato/Makefile new file mode 100644 index 0000000..3b33f14 --- /dev/null +++ b/benchmark/hotpotato/Makefile @@ -0,0 +1,52 @@ +include config.mk + +JULIA ?= julia +JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no + +export JULIA_LOAD_PATH ?= @ +export JULIA_PROJECT ?= $(shell pwd)/.. + +BENCHMARK_NUM_THREADS ?= $(shell ../nthreads.sh) +BENCHMARK_EXCLUSIVE ?= 0 + +.PHONY: all benchmark plots print-* dump-* + +all: + $(MAKE) benchmark + $(MAKE) plots + +benchmark: build/results.json +build/results.json: + JULIA_NUM_THREADS=$(BENCHMARK_NUM_THREADS) \ + JULIA_EXCLUSIVE=$(BENCHMARK_EXCLUSIVE) \ + $(JULIA_CMD) run.jl + +plots: build/results.png +build/results.png: + $(JULIA_CMD) plot.jl + +clean: + rm -fv build/*.* + +backup: + test -e build/results.json + mkdir -pv backup + rm -rf tmp/backup + mkdir -pv tmp/backup/build + mv build/* tmp/backup/build/ + mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S) + +print-%: + @echo $*=${$*} + +dump-config: \ +print-JULIA \ +print-JULIA_CMD \ +print-JULIA_LOAD_PATH \ +print-JULIA_PROJECT \ +print-BENCHMARK_NUM_THREADS \ +print-BENCHMARK_EXCLUSIVE + +config.mk: + touch $@ +# ln -s default-config.mk $@ diff --git a/benchmark/hotpotato/plot.jl b/benchmark/hotpotato/plot.jl index e0c24cc..cb5a6c9 100644 --- a/benchmark/hotpotato/plot.jl +++ b/benchmark/hotpotato/plot.jl @@ -4,7 +4,8 @@ using JSON using Statistics using VegaLite -rawdata = JSON.parsefile(joinpath(@__DIR__, "build/results.json")) +resultsdata = JSON.parsefile(joinpath(@__DIR__, "build/results.json")) +rawdata = resultsdata["results"] potatos = map(rawdata["potatos"]) do info result = info["result"] diff --git a/benchmark/hotpotato/run.jl b/benchmark/hotpotato/run.jl index c479267..27670a1 100644 --- a/benchmark/hotpotato/run.jl +++ b/benchmark/hotpotato/run.jl @@ -2,6 +2,8 @@ using ConcurrentCollections: DualLinkedConcurrentRingQueue using ConcurrentCollectionsBenchmarks.BenchQueueHotPotato: hotpotato!, fai_stats using JSON +include("../info_dump.jl") + function sweep(; repeat = 10, duration = 1, maxntasks = Threads.nthreads()) # cooldown() = sleep(0.1) # cooldown() = GC.gc() @@ -27,23 +29,15 @@ function sweep(; repeat = 10, duration = 1, maxntasks = Threads.nthreads()) return (; potatos, fais, repeat, duration) end -function git_info(dir = @__DIR__) - git(cmd) = strip(read(setenv(`git $cmd`; dir), String)) - return (; - revision = git(`rev-parse HEAD`), - status = git(`status --short --untracked-files=no --porcelain`), - ) -end - function main(args = ARGS) output = get(args, 1, joinpath(@__DIR__, "build", "results.json")) mkpath(dirname(output)) - git = git_info() + info = InfoDump.info() @info "Warmup..." sweep(; repeat = 1, duration = 0.1, maxntasks = 1) @info "Benchmarking..." results = sweep() - results = (; results..., git) + results = (; results, info) open(output, write = true) do io JSON.print(io, results) end diff --git a/benchmark/info_dump.jl b/benchmark/info_dump.jl new file mode 100755 index 0000000..2940954 --- /dev/null +++ b/benchmark/info_dump.jl @@ -0,0 +1,47 @@ +#!/bin/bash +# -*- mode: julia -*- +#= +JULIA="${JULIA:-julia}" +JULIA_CMD="${JULIA_CMD:-$JULIA --color=yes --startup-file=no --compile=min -O0}" +export JULIA_LOAD_PATH=@ +export JULIA_PROJECT="$(dirname "${BASH_SOURCE[0]}")" +exec $JULIA_CMD "${BASH_SOURCE[0]}" "$@" +=# + +module InfoDump + +function git_info(dir = @__DIR__) + git(cmd) = strip(read(setenv(`git $cmd`; dir), String)) + return (; + revision = git(`rev-parse HEAD`), + status = git(`status --short --untracked-files=no --porcelain`), + ) +end + +function julia_info() + return ( + version = string(VERSION), + git = ( + commit = Base.GIT_VERSION_INFO.commit, + branch = Base.GIT_VERSION_INFO.branch, + ), + is_debugbuild = ccall(:jl_is_debugbuild, Cint, ()) != 0, + libllvm_version = string(Base.libllvm_version), + Sys = ( + WORD_SIZE = Sys.WORD_SIZE, + JIT = Sys.JIT, + # CPU_NAME = Sys.CPU_NAME, + # CPU_THREADS = Sys.CPU_THREADS, + ), + env = Dict(k => v for (k, v) in ENV if startswith(k, "JULIA_")), + ) +end + +info() = (; git = git_info(), julia = julia_info()) + +end # module + +if abspath(PROGRAM_FILE) == @__FILE__ + import JSON + JSON.print(stdout, InfoDump.info(), 4) +end diff --git a/benchmark/instantiate.jl b/benchmark/instantiate.jl new file mode 100644 index 0000000..22ab24e --- /dev/null +++ b/benchmark/instantiate.jl @@ -0,0 +1,20 @@ +using Pkg + +packages = [ + PackageSpec( + name = "ConcurrentCollections", + path = dirname(@__DIR__), + # url = "https://github.com/JuliaConcurrent/ConcurrentCollections.jl.git", + ), + PackageSpec( + name = "ConcurrentCollectionsBenchmarks", + path = joinpath(@__DIR__, "ConcurrentCollectionsBenchmarks"), + # url = "https://github.com/JuliaConcurrent/ConcurrentCollections.jl.git", + # subdir = "benchmark/ConcurrentCollectionsBenchmarks", + ), +] + +Pkg.develop(packages) +# Pkg.add(packages) + +Pkg.instantiate() diff --git a/benchmark/nthreads.sh b/benchmark/nthreads.sh new file mode 100755 index 0000000..0abf609 --- /dev/null +++ b/benchmark/nthreads.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if lscpu --parse >/dev/null 2>/dev/null +then + lscpu --parse | grep -v '^#' | sort --unique --field-separator=, --key=2 | wc -l +elif nproc >/dev/null 2>/dev/null +then + exec nproc +else + echo auto +fi From 35e30e3a8b74dfdfc655981585af8eef1dd24944 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Dec 2021 19:29:54 -0500 Subject: [PATCH 2/3] Test benchmarks in CI --- .github/workflows/benchmark.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..035e6ef --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,30 @@ +name: Test benchmarks + +on: + push: + branches: + - master + tags: '*' + pull_request: + +jobs: + benchmark: + runs-on: ubuntu-latest + strategy: + matrix: + julia-version: + - '1.7' + - 'nightly' + fail-fast: false + name: Benchmark Julia ${{ matrix.julia-version }} + steps: + - uses: actions/checkout@v2 + - name: Setup julia + uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.julia-version }} + # Use `JULIA_PKG_SERVER` mitigation implemented in julia-buildpkg: + - uses: julia-actions/julia-buildpkg@v1 + - run: make -C benchmark all + env: + BENCHMARK_NUM_THREADS: "2" From 6e76cac4809faa53d21326990e138abe3123906f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Dec 2021 14:49:24 -0500 Subject: [PATCH 3/3] Run `dump-config` via `make -C benchmark` --- benchmark/Makefile | 7 ++++++- benchmark/dict_histogram/Makefile | 4 ++-- benchmark/hotpotato/Makefile | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index be1b6f9..41854df 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -9,10 +9,11 @@ export JULIA_PROJECT ?= $(shell pwd) SUB_MAKEFILES = $(wildcard */Makefile) BENCHMARK_TARGETS = $(patsubst %/Makefile, benchmark-%, $(SUB_MAKEFILES)) PLOTS_TARGETS = $(patsubst %/Makefile, plots-%, $(SUB_MAKEFILES)) +DUMP_CONFIG_TARGETS = $(patsubst %/Makefile, dump-config-%, $(SUB_MAKEFILES)) .PHONY: benchmark* reinstantiate repl -all: +all: dump-config $(MAKE) benchmark $(MAKE) plots @@ -24,6 +25,10 @@ plots: $(PLOTS_TARGETS) $(PLOTS_TARGETS): plots-%: Manifest.toml $(MAKE) -C $* plots +dump-config: $(DUMP_CONFIG_TARGETS) +$(DUMP_CONFIG_TARGETS): dump-config-%: + $(MAKE) -C $* dump-config + Manifest.toml: JULIA_LOAD_PATH=@:@stdlib $(JULIA_CMD) --project=. instantiate.jl diff --git a/benchmark/dict_histogram/Makefile b/benchmark/dict_histogram/Makefile index b707cf1..278d9f8 100644 --- a/benchmark/dict_histogram/Makefile +++ b/benchmark/dict_histogram/Makefile @@ -11,7 +11,7 @@ BENCHMARK_EXCLUSIVE ?= 0 .PHONY: all benchmark plots clean backup print-* dump-* -all: +all: dump-config $(MAKE) benchmark $(MAKE) plots @@ -37,7 +37,7 @@ backup: mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S) print-%: - @echo $*=${$*} + @echo " "$*=${$*} dump-config: \ print-JULIA \ diff --git a/benchmark/hotpotato/Makefile b/benchmark/hotpotato/Makefile index 3b33f14..cc257a5 100644 --- a/benchmark/hotpotato/Makefile +++ b/benchmark/hotpotato/Makefile @@ -11,7 +11,7 @@ BENCHMARK_EXCLUSIVE ?= 0 .PHONY: all benchmark plots print-* dump-* -all: +all: dump-config $(MAKE) benchmark $(MAKE) plots @@ -37,7 +37,7 @@ backup: mv tmp/backup backup/backup-$$(date +%Y-%m-%d-%H%M%S) print-%: - @echo $*=${$*} + @echo " "$*=${$*} dump-config: \ print-JULIA \