diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1023b4cd4..6ace49dbb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,10 @@ jobs: platform: "i686" - policy: "manylinux_2_24" platform: "x86_64" + - policy: "musllinux_1_1" + platform: "x86_64" + - policy: "musllinux_1_1" + platform: "i686" env: POLICY: ${{ matrix.policy }} diff --git a/.travis.yml b/.travis.yml index 93a737d98..ef78b7714 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,14 @@ jobs: env: POLICY="manylinux_2_24" PLATFORM="s390x" - arch: ppc64le env: POLICY="manylinux_2_24" PLATFORM="ppc64le" MANYLINUX_BUILD_FRONTEND="buildkit" + - arch: arm64-graviton2 + virt: vm + group: edge + env: POLICY="musllinux_1_1" PLATFORM="aarch64" + - arch: s390x + env: POLICY="musllinux_1_1" PLATFORM="s390x" + - arch: ppc64le + env: POLICY="musllinux_1_1" PLATFORM="ppc64le" MANYLINUX_BUILD_FRONTEND="buildkit" before_install: - if [ -d "${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM}" ]; then cp -rlf ${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM} ./; fi diff --git a/build.sh b/build.sh index e4f58e702..6d65ec345 100755 --- a/build.sh +++ b/build.sh @@ -62,6 +62,11 @@ elif [ "${POLICY}" == "manylinux_2_24" ]; then DEVTOOLSET_ROOTPATH= PREPEND_PATH= LD_LIBRARY_PATH_ARG= +elif [ "${POLICY}" == "musllinux_1_1" ]; then + BASEIMAGE="${MULTIARCH_PREFIX}alpine:3.12" + DEVTOOLSET_ROOTPATH= + PREPEND_PATH= + LD_LIBRARY_PATH_ARG= else echo "Unsupported policy: '${POLICY}'" exit 1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 747749706..27ebadb46 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,12 +25,19 @@ ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh build_scripts/update-system-packages.sh /build_scripts/ -RUN bash /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +COPY build_scripts/install-entrypoint.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ + +RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] -COPY build_scripts/install-runtime-packages.sh build_scripts/update-system-packages.sh /build_scripts/ +COPY build_scripts/install-runtime-packages.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ COPY build_scripts/build_utils.sh /build_scripts/ @@ -68,7 +75,6 @@ RUN export LIBXCRYPT_VERSION=4.4.26 && \ export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ manylinux-entrypoint /build_scripts/install-libxcrypt.sh - FROM runtime_base AS build_base COPY build_scripts/install-build-packages.sh /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-build-packages.sh diff --git a/docker/build_scripts/build-git.sh b/docker/build_scripts/build-git.sh index 588d3a92d..fac047435 100755 --- a/docker/build_scripts/build-git.sh +++ b/docker/build_scripts/build-git.sh @@ -7,6 +7,10 @@ set -exuo pipefail # Get script directory MY_DIR=$(dirname "${BASH_SOURCE[0]}") +if [ "${POLICY}" == "musllinux_1_1" ]; then + export NO_REGEX=NeedsStartEnd +fi + # Get build utilities source $MY_DIR/build_utils.sh diff --git a/docker/build_scripts/build-openssl.sh b/docker/build_scripts/build-openssl.sh index e39fbf0ab..c2393ad9d 100755 --- a/docker/build_scripts/build-openssl.sh +++ b/docker/build_scripts/build-openssl.sh @@ -27,6 +27,8 @@ fi if which yum; then yum erase -y openssl-devel +elif which apk; then + apk del openssl-dev else apt-get remove -y libssl-dev fi diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh index dd5f92687..41c3efc75 100755 --- a/docker/build_scripts/build-sqlite3.sh +++ b/docker/build_scripts/build-sqlite3.sh @@ -30,7 +30,7 @@ strip_ /manylinux-rootfs # Install cp -rlf /manylinux-rootfs/* / -ldconfig +ldconfig / # Clean-up for runtime rm -rf /manylinux-rootfs/usr/local/share diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 45f593166..9e93254aa 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -10,6 +10,10 @@ MANYLINUX_CFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong - MANYLINUX_CXXFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security" MANYLINUX_LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now" +export BASE_POLICY=manylinux +if [ "${POLICY:0:9}" == "musllinux" ]; then + export BASE_POLICY=musllinux +fi function check_var { if [ -z "$1" ]; then diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index e35618254..a9e9ec91a 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -75,7 +75,13 @@ deactivate pushd $MY_DIR/requirements-tools for TOOL_PATH in $(find . -type f); do TOOL=$(basename ${TOOL_PATH}) - pipx install --pip-args="--require-hashes -r" ${TOOL} + if [ "${TOOL}" == "cmake" ] && [ "${POLICY}" == "musllinux_1_1" ]; then + # TODO remove this exception once https://github.com/scikit-build/cmake-python-distributions + # provides a musllinux wheel + apk add --no-cache cmake + else + pipx install --pip-args="--require-hashes -r" ${TOOL} + fi done popd diff --git a/docker/build_scripts/install-build-packages.sh b/docker/build_scripts/install-build-packages.sh index 65707a675..a24e81ecc 100755 --- a/docker/build_scripts/install-build-packages.sh +++ b/docker/build_scripts/install-build-packages.sh @@ -19,6 +19,9 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then PACKAGE_MANAGER=apt COMPILE_DEPS="libz-dev libbz2-dev libexpat1-dev libncurses5-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-kernel-headers" +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + PACKAGE_MANAGER=apk + COMPILE_DEPS="zlib-dev bzip2-dev expat-dev ncurses-dev readline-dev tk-dev gdbm-dev libpcap-dev xz-dev openssl openssl-dev keyutils-dev krb5-dev libcom_err libidn-dev curl-dev util-linux-dev libffi-dev linux-headers" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -35,6 +38,8 @@ elif [ "${PACKAGE_MANAGER}" == "apt" ]; then apt-get install -qq -y --no-install-recommends ${COMPILE_DEPS} apt-get clean -qq rm -rf /var/lib/apt/lists/* +elif [ "${PACKAGE_MANAGER}" == "apk" ]; then + apk add --no-cache ${COMPILE_DEPS} else echo "Not implemented" exit 1 diff --git a/docker/build_scripts/install-entrypoint.sh b/docker/build_scripts/install-entrypoint.sh index 3eb0a00e8..964fa5ce9 100755 --- a/docker/build_scripts/install-entrypoint.sh +++ b/docker/build_scripts/install-entrypoint.sh @@ -1,15 +1,16 @@ -#!/bin/bash +#!/bin/sh # Install entrypoint: -# make sure yum is configured correctly and linux32 is available on i686 +# - make sure yum is configured correctly and linux32 is available on i686 +# - install bash on Alpine as most scripts require it # Stop at any error, show all commands -set -exuo pipefail +set -exu # Set build environment variables -MY_DIR=$(dirname "${BASH_SOURCE[0]}") +MY_DIR=$(dirname "$0") -if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then +if [ "${AUDITWHEEL_PLAT}" = "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" = "manylinux2014_i686" ]; then echo "i386" > /etc/yum/vars/basearch fixup-mirrors yum -y update @@ -19,5 +20,9 @@ if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == # the following script takes care of cleaning-up some things # and since it's also needed in the finalize step, everything's # centralized in this script to avoid code duplication - LC_ALL=C ${MY_DIR}/update-system-packages.sh + LC_ALL=C "${MY_DIR}/update-system-packages.sh" +fi + +if [ "${AUDITWHEEL_POLICY}" = "musllinux_1_1" ]; then + apk add --no-cache bash fi diff --git a/docker/build_scripts/install-libxcrypt.sh b/docker/build_scripts/install-libxcrypt.sh index 5361eae32..424e31460 100755 --- a/docker/build_scripts/install-libxcrypt.sh +++ b/docker/build_scripts/install-libxcrypt.sh @@ -10,6 +10,11 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh +if [ "$BASE_POLICY" == "musllinux" ]; then + echo "Skip libxcrypt installation on musllinux" + exit 0 +fi + # We need perl 5.14+ if ! perl -e 'use 5.14.0' &> /dev/null; then check_var ${PERL_ROOT} diff --git a/docker/build_scripts/install-pypy.sh b/docker/build_scripts/install-pypy.sh index ddc0c0ac1..5869fb515 100755 --- a/docker/build_scripts/install-pypy.sh +++ b/docker/build_scripts/install-pypy.sh @@ -9,6 +9,10 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh +if [ "${BASE_POLICY}" == "musllinux" ]; then + echo "Skip PyPy build on musllinux" + exit 0 +fi PYTHON_VERSION=$1 PYPY_VERSION=$2 diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index db2a5744d..7d3ba8863 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -7,6 +7,9 @@ set -exuo pipefail # Set build environment variables MY_DIR=$(dirname "${BASH_SOURCE[0]}") +# Get build utilities +source $MY_DIR/build_utils.sh + # Libraries that are allowed as part of the manylinux2014 profile # Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy # On RPM-based systems, they are provided by these packages: @@ -28,6 +31,8 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev" +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + MANYLINUX_DEPS="musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -43,15 +48,17 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == fi elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then RUNTIME_DEPS="zlib1g libbz2-1.0 libexpat1 libncurses5 libreadline7 tk libgdbm3 libdb5.3 libpcap0.8 liblzma5 libssl1.1 libkeyutils1 libkrb5-3 libcomerr2 libidn2-0 libcurl3 uuid libffi6" +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + RUNTIME_DEPS="zlib bzip2 expat ncurses5-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 fi -BASETOOLS="autoconf automake bison bzip2 diffutils file hardlink make patch unzip" +BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip" if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then PACKAGE_MANAGER=yum - BASETOOLS="${BASETOOLS} which" + BASETOOLS="${BASETOOLS} hardlink which" # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages echo "multilib_policy=best" >> /etc/yum.conf fixup-mirrors @@ -72,7 +79,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then fi elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then PACKAGE_MANAGER=yum - BASETOOLS="${BASETOOLS} hostname which" + BASETOOLS="${BASETOOLS} hardlink hostname which" # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages echo "multilib_policy=best" >> /etc/yum.conf # Error out if requested packages do not exist @@ -101,13 +108,18 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then fi elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then PACKAGE_MANAGER=apt - BASETOOLS="${BASETOOLS} hostname" + BASETOOLS="${BASETOOLS} hardlink hostname" export DEBIAN_FRONTEND=noninteractive sed -i 's/none/en_US/g' /etc/apt/apt.conf.d/docker-no-languages apt-get update -qq apt-get upgrade -qq -y apt-get install -qq -y --no-install-recommends ca-certificates gpg curl locales TOOLCHAIN_DEPS="binutils gcc g++ gfortran" +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + TOOLCHAIN_DEPS="binutils gcc g++ gfortran" + BASETOOLS="${BASETOOLS} curl util-linux" + PACKAGE_MANAGER=apk + apk add --no-cache ca-certificates gnupg else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -117,6 +129,8 @@ if [ "${PACKAGE_MANAGER}" == "yum" ]; then yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} elif [ "${PACKAGE_MANAGER}" == "apt" ]; then apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} +elif [ "${PACKAGE_MANAGER}" == "apk" ]; then + apk add --no-cache ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} else echo "Not implemented" exit 1 @@ -128,9 +142,11 @@ fi # centralized in this script to avoid code duplication LC_ALL=C ${MY_DIR}/update-system-packages.sh -# we'll be removing libcrypt.so.1 later on -# this is needed to ensure the new one will be found -# as LD_LIBRARY_PATH does not seem enough. -# c.f. https://github.com/pypa/manylinux/issues/1022 -echo "/usr/local/lib" > /etc/ld.so.conf.d/00-manylinux.conf -ldconfig +if [ "${BASE_POLICY}" == "manylinux" ]; then + # we'll be removing libcrypt.so.1 later on + # this is needed to ensure the new one will be found + # as LD_LIBRARY_PATH does not seem enough. + # c.f. https://github.com/pypa/manylinux/issues/1022 + echo "/usr/local/lib" > /etc/ld.so.conf.d/00-manylinux.conf + ldconfig +fi diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index c57d4bb61..1e6d496b0 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -4,6 +4,11 @@ # Stop at any error, show all commands set -exuo pipefail +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh fixup-mirrors if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then @@ -21,6 +26,8 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then apt-get upgrade -qq -y apt-get clean -qq rm -rf /var/lib/apt/lists/* +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + apk upgrade --no-cache else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -28,25 +35,27 @@ fi fixup-mirrors # do we want to update locales ? -LOCALE_ARCHIVE=/usr/lib/locale/locale-archive -TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp -if [ ! -f ${TIMESTAMP_FILE} ] || [ ${LOCALE_ARCHIVE} -nt ${TIMESTAMP_FILE} ]; then - # upgrading glibc-common can end with removal on en_US.UTF-8 locale - localedef -i en_US -f UTF-8 en_US.UTF-8 - - # if we updated glibc, we need to strip locales again... - if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then - localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive - fi - if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then - mv -f ${LOCALE_ARCHIVE} ${LOCALE_ARCHIVE}.tmpl - build-locale-archive --install-langs="en_US.utf8" - elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then - rm ${LOCALE_ARCHIVE} +if [ "${BASE_POLICY}" == "manylinux" ]; then + LOCALE_ARCHIVE=/usr/lib/locale/locale-archive + TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp + if [ ! -f ${TIMESTAMP_FILE} ] || [ ${LOCALE_ARCHIVE} -nt ${TIMESTAMP_FILE} ]; then + # upgrading glibc-common can end with removal on en_US.UTF-8 locale localedef -i en_US -f UTF-8 en_US.UTF-8 - update-locale LANG=en_US.UTF-8 + + # if we updated glibc, we need to strip locales again... + if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then + localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive + fi + if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + mv -f ${LOCALE_ARCHIVE} ${LOCALE_ARCHIVE}.tmpl + build-locale-archive --install-langs="en_US.utf8" + elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + rm ${LOCALE_ARCHIVE} + localedef -i en_US -f UTF-8 en_US.UTF-8 + update-locale LANG=en_US.UTF-8 + fi + touch ${TIMESTAMP_FILE} fi - touch ${TIMESTAMP_FILE} fi if [ -d /usr/share/locale ]; then diff --git a/tests/run_tests.sh b/tests/run_tests.sh index d86628c9e..6f59eea8d 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -12,6 +12,8 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then export DEBIAN_FRONTEND=noninteractive PACKAGE_MANAGER=apt apt-get update -qq +elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then + PACKAGE_MANAGER=apk else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -61,6 +63,8 @@ if [ "${PACKAGE_MANAGER}" == "yum" ]; then yum -y install openssh-clients elif [ "${PACKAGE_MANAGER}" == "apt" ]; then apt-get install -qq -y --no-install-recommends openssh-client +elif [ "${PACKAGE_MANAGER}" == "apk" ]; then + apk add --no-cache openssh-client else echo "Unsupported package manager: '${PACKAGE_MANAGER}'" exit 1