CI (Buildkite): Add package_linux and tester_linux for more Linux architectures

DilumAluthge · DilumAluthge · commit ddc2bf79d9e8 · 2021-09-26T10:03:22.000-04:00
diff --git a/.buildkite/pipelines/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml
@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:
@@ -16,9 +15,8 @@ steps:
           # pipelines.
           unsigned_pipelines:
             - .buildkite/pipelines/experimental/launch_unsigned_builders.yml
-
           # Our signed pipelines must have a `signature` or `signature_file` parameter that
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           # signed_pipelines:
           #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
-          #     signature: "my_signature"
+          #     signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature
diff --git a/.buildkite/pipelines/experimental/misc/sanitizers.yml b/.buildkite/pipelines/experimental/misc/sanitizers.yml
@@ -3,10 +3,9 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "asan"
-    key: asan
+    key: "asan"
     plugins:
       - JuliaCI/julia#v1:
           version: 1.6
@@ -17,13 +16,10 @@ steps:
           gid: 1000
           workspaces:
             - "/cache/repos:/cache/repos"
-      # `contrib/check-asan.jl` needs a `julia` binary:
+      # `contrib/check-asan.jl` needs a `julia` binary inside the inner sandbox:
       - JuliaCI/julia#v1:
           version: 1.6
     timeout_in_minutes: 120
-    # notify:
-    #   - github_commit_status:
-    #       context: "asan"
     commands: |
       echo "--- Build julia-debug with ASAN"
-      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_NUM_CORES} debug
+      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
diff --git a/.buildkite/pipelines/main/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml
@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:
diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml
@@ -15,15 +15,16 @@
 steps:
   - label: ":buildkite: Launch unsigned builders"
     commands: |
-      # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible.
+      # Launch the `whitespace` builder first, because we want that builder to finish as quickly as possible.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
 
-      # Next, we launch the miscellaneous builders in alphabetical order.
+      # Launch the miscellaneous builders in alphabetical order.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
 
-      # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
+      # Launch all of the platform builders.
+      bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
+      bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
     agents:
       queue: julia
diff --git a/.buildkite/pipelines/main/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "doctest"
     key: doctest
@@ -31,6 +30,3 @@ steps:
       echo "--- Run Julia doctests"
       JULIA_NUM_THREADS=1 make -C doc doctest=true
     timeout_in_minutes: 45
-    notify:
-      - github_commit_status:
-          context: "doctest"
diff --git a/.buildkite/pipelines/main/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "embedding"
     key: "embedding"
@@ -20,15 +19,11 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       prefix="/tmp/prefix"
-      echo "+++ Build julia, deploy to $${prefix}"
-      make -j$${JULIA_NUM_CORES} JULIA_PRECOMPILE=0 prefix=$${prefix} install
+      echo "+++ Build julia, deploy to $${prefix:?}"
+      make -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
 
       embedding_output="/tmp/embedding-test"
-      echo "+++ Run embedding tests, deploy to $${embedding_output}"
-      mkdir -p "$${embedding_output}"
-      make -j$${JULIA_NUM_CORES} -C test/embedding JULIA="$${prefix}/bin/julia" BIN="$${embedding_output}"
-
+      echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
+      mkdir -p "$${embedding_output:?}"
+      make -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "embedding"
diff --git a/.buildkite/pipelines/main/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "analyzegc"
     key: "analyzegc"
@@ -18,15 +17,11 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- Install in-tree LLVM dependencies"
-      make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
+      make -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
       echo "+++ run clangsa/analyzegc"
-      make -j$${JULIA_NUM_CORES} -C test/clangsa
-      make -j$${JULIA_NUM_CORES} -C src analyzegc
+      make -j$${JULIA_CPU_THREADS:?} -C test/clangsa
+      make -j$${JULIA_CPU_THREADS:?} -C src analyzegc
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "analyzegc"
-
   - label: "llvmpasses"
     key: "llvmpasses"
     plugins:
@@ -41,12 +36,9 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- make release"
-      make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0
+      make -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0
       echo "--- make src/install-analysis-deps"
-      make -j$${JULIA_NUM_CORES} -C src install-analysis-deps
+      make -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps
       echo "+++ make test/llvmpasses"
-      make -j$${JULIA_NUM_CORES} -C test/llvmpasses
+      make -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "llvmpasses"
diff --git a/.buildkite/pipelines/main/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "whitespace"
     key: "whitespace"
@@ -15,9 +14,9 @@ steps:
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           workspaces:
             - "/cache/repos:/cache/repos"
-    commands: |
-      make -j$${JULIA_NUM_CORES} check-whitespace
     timeout_in_minutes: 10
     notify:
       - github_commit_status:
           context: "whitespace"
+    commands: |
+      make -j$${JULIA_CPU_THREADS:?} check-whitespace
diff --git a/.buildkite/pipelines/main/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml
diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches
@@ -0,0 +1,6 @@
+# ARCH      ARCH_LABEL      ROOTFS_ARCH      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
+# aarch64   aarch64         aarch64          60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# armv7l    armv7l          armv7l           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+32          32              i686             60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# ppc64le   ppc64le         powerpc64le      60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
+64          64              x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml
@@ -0,0 +1,49 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "package_linux${ARCH_LABEL?}"
+    key: package_linux${ARCH_LABEL?}
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
+          rootfs_treehash: "${ROOTFS_TREE?}"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: ${TIMEOUT?}
+    commands: |
+      echo "--- Print the full and short commit hashes"
+      SHORT_COMMIT_LENGTH=10
+      SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
+      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
+      ARTIFACT_FILE_EXTENSION="tar.gz"
+      ARTIFACT_FILENAME="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
+      echo "The full commit is:                     $${BUILDKITE_COMMIT:?}"
+      echo "The short commit is:                    $${SHORT_COMMIT:?}"
+      echo "The artifact filename will be:    $${ARTIFACT_FILENAME:?}"
+
+      echo "--- Build Julia from source"
+      rm -rf $${ARTIFACT_FILENAME:?}
+      make -j 8
+
+      echo "--- Make sure that the working directory is clean"
+      if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
+
+      echo "--- Print Julia version info"
+      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+
+      echo "--- Create build artifacts"
+      make -j 8 binary-dist
+      ls -l $${ARTIFACT_FILENAME:?}
+
+      echo "--- Upload build artifacts"
+      buildkite-agent artifact upload $${ARTIFACT_FILENAME:?}
diff --git a/.buildkite/pipelines/main/platforms/platforms.sh b/.buildkite/pipelines/main/platforms/platforms.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+PLATFORM="$1"
+
+cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do
+  # Remove whitespace from the beginning and end of each line
+  line=`echo $_line | tr -s ' '`
+
+  # Skip all lines that begin with `#`
+  if [[ $line == \#* ]]; then
+    continue
+  fi
+
+  export ARCH=`echo $line        | cut -d ' ' -f 1`
+  export ARCH_LABEL=`echo $line  | cut -d ' ' -f 2`
+  export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 3`
+  export TIMEOUT=`echo $line     | cut -d ' ' -f 4`
+  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 5`
+  export ROOTFS_TREE=`echo $line | cut -d ' ' -f 6`
+  echo "Launching: $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $TIMEOUT"
+  buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml"
+done
diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches
@@ -0,0 +1,8 @@
+# ARCH      ARCH_LABEL      ROOTFS_ARCH      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
+# aarch64   aarch64         aarch64          60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# armv7l    armv7l          armv7l           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+32          32              i686             60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# ppc64le   ppc64le         powerpc64le      60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
+64          64_rr           x86_64           180          v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+64          64_st           x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+64          64_mt           x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml
diff --git a/.buildkite/pipelines/scheduled/0_webui.yml b/.buildkite/pipelines/scheduled/0_webui.yml
diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml
diff --git a/.buildkite/utilities/rr/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl
