diff --git a/scripts/dockcross-manylinux-build-module-wheels.sh b/scripts/dockcross-manylinux-build-module-wheels.sh index 9807879c..ae98545e 100755 --- a/scripts/dockcross-manylinux-build-module-wheels.sh +++ b/scripts/dockcross-manylinux-build-module-wheels.sh @@ -16,9 +16,19 @@ # export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2" # scripts/dockcross-manylinux-build-module-wheels.sh cp39 # +# A specialized manylinux image and tag can be used by exporting +# MANYLINUX_VERSION and IMAGE_TAG before running this script. +# See https://github.com/dockcross/dockcross for available versions and tags. +# +# For example, +# +# export MANYLINUX_VERSION=2014 +# export IMAGE_TAG=20221108-102ebcc +# scripts/dockcross-manylinux-build-module-wheels.sh cp39 +# -MANYLINUX_VERSION=_2_28 -IMAGE_TAG=20221128-2024e4b +MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28} +IMAGE_TAG=${IMAGE_TAG:=20221108-102ebcc} # Generate dockcross scripts docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64 @@ -31,6 +41,7 @@ chmod 777 $(pwd)/tools # Build wheels mkdir -p dist DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/..:/ITKPythonPackage -v $(pwd)/tools:/tools" +DOCKER_ARGS+=" -e MANYLINUX_VERSION" # Mount any shared libraries if [[ -n ${LD_LIBRARY_PATH} ]]; then for libpath in ${LD_LIBRARY_PATH//:/ }; do diff --git a/scripts/dockcross-manylinux-build-wheels.sh b/scripts/dockcross-manylinux-build-wheels.sh index 4e70d7d7..eb81d624 100755 --- a/scripts/dockcross-manylinux-build-wheels.sh +++ b/scripts/dockcross-manylinux-build-wheels.sh @@ -6,9 +6,20 @@ # For example, # # scripts/dockcross-manylinux-build-wheels.sh cp39 +# +# A specialized manylinux image and tag can be used by exporting to +# MANYLINUX_VERSION and IMAGE_TAG before running this script. +# See https://github.com/dockcross/dockcross for available versions and tags. +# +# For example, +# +# export MANYLINUX_VERSION=2014 +# export IMAGE_TAG=20221108-102ebcc +# scripts/dockcross-manylinux-build-module-wheels.sh cp39 +# -MANYLINUX_VERSION=_2_28 -IMAGE_TAG=20221128-2024e4b +MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28} +IMAGE_TAG=${IMAGE_TAG:=20221108-102ebcc} # Generate dockcross scripts docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64 @@ -20,6 +31,7 @@ script_dir=$(cd $(dirname $0) || exit 1; pwd) pushd $script_dir/.. mkdir -p dist DOCKER_ARGS="-v $(pwd)/dist:/work/dist/" +DOCKER_ARGS+=" -e MANYLINUX_VERSION" /tmp/dockcross-manylinux-x64 \ -a "$DOCKER_ARGS" \ ./scripts/internal/manylinux-build-wheels.sh "$@" diff --git a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh index d0ef8d3d..6bd4add5 100755 --- a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh +++ b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh @@ -2,6 +2,21 @@ # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module, # it is used by the azure-pipeline.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate +# +# Exported variables used in this script: +# - ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use +# Examples: "v5.3.0", "v5.2.1.post1" +# See available tags at https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/tags +# - MANYLINUX_VERSION: manylinux specialization to use +# Examples: "_2_28", "2014", "_2_28_aarch64" +# See https://github.com/dockcross/dockcross +# - ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use. +# If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed +# with the ITKPythonBuilds archive will be used. +# - ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage +# build script source. Default is InsightSoftwareConsortium. +# Ignored if ITKPYTHONPACKAGE_TAG is empty. +# # ----------------------------------------------------------------------- # Script argument parsing @@ -46,9 +61,14 @@ fi # Expect unzstd > v1.3.2, see discussion in `dockcross-manylinux-build-tarball.sh` ${unzstd_exe} --version +# ----------------------------------------------------------------------- +# Fetch build archive + +TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION:=_2_28}" TARBALL_NAME="ITKPythonBuilds-linux${TARBALL_SPECIALIZATION}.tar" if [[ ! -f ${TARBALL_NAME}.zst ]]; then + echo "Fetching https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/${TARBALL_NAME}.zst" curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/${TARBALL_NAME}.zst -O fi if [[ ! -f ./${TARBALL_NAME}.zst ]]; then @@ -67,14 +87,31 @@ else tar xf ${TARBALL_NAME} --wildcards ITKPythonPackage/ITK-$1* fi rm ${TARBALL_NAME} + +# Optional: Update build scripts +if [[ -n ${ITKPYTHONPACKAGE_TAG} ]]; then + echo "Updating build scripts to ${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}/ITKPythonPackage@${ITKPYTHONPACKAGE_TAG}" + git clone "https://github.com/${ITKPYTHONPACKAGE_ORG}/ITKPythonPackage.git" "IPP-tmp" + pushd IPP-tmp/ + git checkout "${ITKPYTHONPACKAGE_TAG}" + git status + popd + + rm -rf ITKPythonPackage/scripts/ + cp -r IPP-tmp/scripts ITKPythonPackage/ + rm -rf IPP-tmp/ +fi + if [[ ! -f ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh ]]; then echo "ERROR: can not find required binary './ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh'" exit 255 fi cp -a ITKPythonPackage/oneTBB-prefix ./ +# ----------------------------------------------------------------------- + set -- "${FORWARD_ARGS[@]}"; # Restore initial argument list -if [[ "${TARBALL_SPECIALIZATION}" = "-manylinux_2_28_aarch64" ]]; then +if [[ "${MANYLINUX_VERSION}" = "_2_28_aarch64" ]]; then ./ITKPythonPackage/scripts/manylinux_2_28_aarch64-build-module-wheels.sh "$@" else ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@" diff --git a/scripts/internal/manylinux-build-common.sh b/scripts/internal/manylinux-build-common.sh index bd133882..f87ca18d 100644 --- a/scripts/internal/manylinux-build-common.sh +++ b/scripts/internal/manylinux-build-common.sh @@ -80,6 +80,6 @@ if ! type ninja > /dev/null 2>&1; then popd fi -MANYLINUX_VERSION=_2_28 +MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28} echo "Building wheels for $ARCH using manylinux${MANYLINUX_VERSION}" diff --git a/scripts/internal/manylinux-build-module-wheels.sh b/scripts/internal/manylinux-build-module-wheels.sh index da004646..0a089752 100755 --- a/scripts/internal/manylinux-build-module-wheels.sh +++ b/scripts/internal/manylinux-build-module-wheels.sh @@ -15,6 +15,16 @@ # DOCKER_ARGS="-v /path/to/lib.so:/usr/local/lib64/lib.so" # /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-module-wheels.sh # +# The specialized manylinux container version should be set prior to running this script. +# See https://github.com/dockcross/dockcross for available versions and tags. +# +# For example, `docker run -e ` can be used to set an environment variable when launching a container: +# +# export MANYLINUX_VERSION=2014 +# docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64 +# chmod u+x /tmp/dockcross-manylinux-x64 +# /tmp/dockcross-manylinux-x64 -e MANYLINUX_VERSION manylinux-build-module-wheels.sh cp39 +# # ----------------------------------------------------------------------- # Script argument parsing diff --git a/scripts/internal/manylinux-build-wheels.sh b/scripts/internal/manylinux-build-wheels.sh index 2a8054b5..76ed3ee2 100755 --- a/scripts/internal/manylinux-build-wheels.sh +++ b/scripts/internal/manylinux-build-wheels.sh @@ -15,6 +15,16 @@ # DOCKER_ARGS="-v /path/to/lib.so:/usr/local/lib64/lib.so" # /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-wheels.sh # +# The specialized manylinux container version should be set prior to running this script. +# See https://github.com/dockcross/dockcross for available versions and tags. +# +# For example, `docker run -e ` can be used to set an environment variable when launching a container: +# +# export MANYLINUX_VERSION=2014 +# docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64 +# chmod u+x /tmp/dockcross-manylinux-x64 +# /tmp/dockcross-manylinux-x64 -e MANYLINUX_VERSION manylinux-build-module-wheels.sh cp39 +# # ----------------------------------------------------------------------- # These variables are set in common script: diff --git a/scripts/macpython-download-cache-and-build-module-wheels.sh b/scripts/macpython-download-cache-and-build-module-wheels.sh index a8fb5260..b42c2a9d 100755 --- a/scripts/macpython-download-cache-and-build-module-wheels.sh +++ b/scripts/macpython-download-cache-and-build-module-wheels.sh @@ -2,15 +2,43 @@ # This module should be pull and run from an ITKModule root directory to generate the Mac python wheels of this module, # it is used by the .travis.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate +# +# Exported variables used in this script: +# - ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use +# - ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use. +# If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed +# with the ITKPythonBuilds archive will be used. +# - ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage +# build script source. Default is InsightSoftwareConsortium. +# Ignored if ITKPYTHONPACKAGE_TAG is empty. +# Install dependencies brew update brew install zstd aria2 gnu-tar doxygen ninja brew upgrade cmake + +# Fetch ITKPythonBuilds archive containing ITK build artifacts aria2c -c --file-allocation=none -o ITKPythonBuilds-macosx.tar.zst -s 10 -x 10 https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/ITKPythonBuilds-macosx.tar.zst unzstd --long=31 ITKPythonBuilds-macosx.tar.zst -o ITKPythonBuilds-macosx.tar PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" tar xf ITKPythonBuilds-macosx.tar --checkpoint=10000 --checkpoint-action=dot rm ITKPythonBuilds-macosx.tar + +# Optional: Update build scripts +if [[ -n ${ITKPYTHONPACKAGE_TAG} ]]; then + echo "Updating build scripts to ${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}/ITKPythonPackage@${ITKPYTHONPACKAGE_TAG}" + git clone "https://github.com/${ITKPYTHONPACKAGE_ORG}/ITKPythonPackage.git" "IPP-tmp" + pushd IPP-tmp/ + git checkout "${ITKPYTHONPACKAGE_TAG}" + git status + popd + + rm -rf ITKPythonPackage/scripts/ + cp -r IPP-tmp/scripts ITKPythonPackage/ + rm -rf IPP-tmp/ +fi + +# Run build scripts sudo mkdir -p /Users/svc-dashboard/D/P && sudo chown $UID:$GID /Users/svc-dashboard/D/P && mv ITKPythonPackage /Users/svc-dashboard/D/P/ sudo rm -rf /Library/Frameworks/Python.framework/Versions/* /Users/svc-dashboard/D/P/ITKPythonPackage/scripts/macpython-install-python.sh diff --git a/scripts/manylinux_2_28_aarch64-build-module-wheels.sh b/scripts/manylinux_2_28_aarch64-build-module-wheels.sh index 6f15c537..9cf470ad 100755 --- a/scripts/manylinux_2_28_aarch64-build-module-wheels.sh +++ b/scripts/manylinux_2_28_aarch64-build-module-wheels.sh @@ -16,9 +16,17 @@ # export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2" # scripts/dockcross-manylinux-build-module-wheels.sh cp39 # +# A specialized manylinux image and tag can be used by exporting to MANYLINUX_VERSION before +# running this script. Default is _2_28_aarch64. +# +# For example, +# +# export MANYLINUX_VERSION=2014 +# export IMAGE_TAG=20221108-102ebcc +# scripts/dockcross-manylinux-build-module-wheels.sh cp39 -MANYLINUX_VERSION=_2_28 -IMAGE_TAG=latest +MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28_aarch64} +IMAGE_TAG=${IMAGE_TAG:=latest} script_dir=$(cd $(dirname $0) || exit 1; pwd) @@ -35,4 +43,4 @@ if [[ -n ${LD_LIBRARY_PATH} ]]; then fi docker run --privileged --rm tonistiigi/binfmt --install all -docker run --rm -it $DOCKER_ARGS -v $(pwd):/work/ quay.io/pypa/manylinux${MANYLINUX_VERSION}_aarch64:${IMAGE_TAG} "/ITKPythonPackage/scripts/internal/manylinux-aarch64-build-module-wheels.sh" "$@" +docker run --rm -it $DOCKER_ARGS -v $(pwd):/work/ quay.io/pypa/manylinux${MANYLINUX_VERSION}:${IMAGE_TAG} "/ITKPythonPackage/scripts/internal/manylinux-aarch64-build-module-wheels.sh" "$@" diff --git a/scripts/windows-download-cache-and-build-module-wheels.ps1 b/scripts/windows-download-cache-and-build-module-wheels.ps1 index 3a46e32e..a55d9b30 100644 --- a/scripts/windows-download-cache-and-build-module-wheels.ps1 +++ b/scripts/windows-download-cache-and-build-module-wheels.ps1 @@ -1,7 +1,19 @@ +# Pulls build dependencies and build an ITK remote module. +# +# Variables to set before calling the script: +# - $env:ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use +# - $env:ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use. +# If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed +# with the ITKPythonBuilds archive will be used. +# - $env:ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage +# build script source. Default is InsightSoftwareConsortium. +# Ignored if ITKPYTHONPACKAGE_TAG is empty. + trap { Write-Error $_; Exit 1 } $AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12' [System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols +# Get ITKPythonBuilds archive with ITK build artifacts set-alias sz "$env:ProgramFiles\7-Zip\7z.exe" if (-not (Test-Path env:APPVEYOR)) { $pythonArch = "64" @@ -10,9 +22,34 @@ if (-not (Test-Path env:APPVEYOR)) { if (-not (Test-Path env:ITK_PACKAGE_VERSION)) { $env:ITK_PACKAGE_VERSION = 'v5.3.0' } Invoke-WebRequest -Uri "https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/$env:ITK_PACKAGE_VERSION/ITKPythonBuilds-windows.zip" -OutFile "ITKPythonBuilds-windows.zip" sz x ITKPythonBuilds-windows.zip -oC:\P -aoa -r + +# Optional: Update ITKPythonPackage build scripts +if ($env:ITKPYTHONPACKAGE_TAG) { + if(!$env:ITKPYTHONPACKAGE_ORG) { + $env:ITKPYTHONPACKAGE_ORG="InsightSoftwareConsortium" + } + + echo "Updating build scripts to ($env:ITKPYTHONPACKAGE_ORG)/ITKPythonPackage@($env:ITKPYTHONPACKAGE_TAG)" + + pushd C:\P + git clone "https://github.com/($env:ITKPYTHONPACKAGE_ORG)/ITKPythonPackage.git" "IPP-tmp" + pushd "IPP-tmp" + git checkout "($env:ITKPYTHONPACKAGE_TAG)" + git status + popd + + Remove-Item -Recurse -Force IPP/scripts/ + Copy-Item -Recurse IPP-tmp/scripts IPP/ + Remove-Item -Recurse -Force IPP-tmp/ + popd +} + +# Get other build dependencies Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5c0ad59d8d777f2179dd3e9c/download" -OutFile "doxygen-1.8.11.windows.bin.zip" sz x doxygen-1.8.11.windows.bin.zip -oC:\P\doxygen -aoa -r Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5bbf87ba8d777f06b91f27d6/download/grep-win.zip" -OutFile "grep-win.zip" sz x grep-win.zip -oC:\P\grep -aoa -r $env:Path += ";C:\P\grep" + +# Run build scripts C:\Python37-x64\python.exe C:\P\IPP\scripts\windows_build_module_wheels.py