ENH: Add build script update step to download script

scripts/dockcross-manylinux-build-module-wheels.sh

@@ -16,9 +16,19 @@
 #   export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2"
 #   scripts/dockcross-manylinux-build-module-wheels.sh cp39
 #
+# A specialized manylinux image and tag can be used by exporting
+# MANYLINUX_VERSION and IMAGE_TAG before running this script.
+# See https://github.com/dockcross/dockcross for available versions and tags.
+#
+# For example,
+#
+#   export MANYLINUX_VERSION=2014
+#   export IMAGE_TAG=20221108-102ebcc
+#   scripts/dockcross-manylinux-build-module-wheels.sh cp39
+#
 
-MANYLINUX_VERSION=_2_28
-IMAGE_TAG=20221128-2024e4b
+MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
+IMAGE_TAG=${IMAGE_TAG:=20221108-102ebcc}
 
 # Generate dockcross scripts
 docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
@@ -31,6 +41,7 @@ chmod 777 $(pwd)/tools
 # Build wheels
 mkdir -p dist
 DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
+DOCKER_ARGS+=" -e MANYLINUX_VERSION"
 # Mount any shared libraries
 if [[ -n ${LD_LIBRARY_PATH} ]]; then
   for libpath in ${LD_LIBRARY_PATH//:/ }; do

scripts/dockcross-manylinux-build-wheels.sh

@@ -6,9 +6,20 @@
 # For example,
 #
 #   scripts/dockcross-manylinux-build-wheels.sh cp39
+#
+# A specialized manylinux image and tag can be used by exporting to 
+# MANYLINUX_VERSION and IMAGE_TAG before running this script.
+# See https://github.com/dockcross/dockcross for available versions and tags.
+#
+# For example,
+#
+#   export MANYLINUX_VERSION=2014
+#   export IMAGE_TAG=20221108-102ebcc
+#   scripts/dockcross-manylinux-build-module-wheels.sh cp39
+#
 
-MANYLINUX_VERSION=_2_28
-IMAGE_TAG=20221128-2024e4b
+MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
+IMAGE_TAG=${IMAGE_TAG:=20221108-102ebcc}
 
 # Generate dockcross scripts
 docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
@@ -20,6 +31,7 @@ script_dir=$(cd $(dirname $0) || exit 1; pwd)
 pushd $script_dir/..
 mkdir -p dist
 DOCKER_ARGS="-v $(pwd)/dist:/work/dist/"
+DOCKER_ARGS+=" -e MANYLINUX_VERSION"
 /tmp/dockcross-manylinux-x64 \
   -a "$DOCKER_ARGS" \
   ./scripts/internal/manylinux-build-wheels.sh "$@"

scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh

@@ -2,6 +2,21 @@
 
 # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module,
 # it is used by the azure-pipeline.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate
+#
+# Exported variables used in this script:
+# - ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use
+#     Examples: "v5.3.0", "v5.2.1.post1"
+#     See available tags at https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/tags
+# - MANYLINUX_VERSION: manylinux specialization to use
+#     Examples: "_2_28", "2014", "_2_28_aarch64"
+#     See https://github.com/dockcross/dockcross
+# - ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use.
+#     If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed
+#     with the ITKPythonBuilds archive will be used.
+# - ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage
+#     build script source. Default is InsightSoftwareConsortium.
+#     Ignored if ITKPYTHONPACKAGE_TAG is empty.
+#
 
 # -----------------------------------------------------------------------
 # Script argument parsing
@@ -46,9 +61,14 @@ fi
 # Expect unzstd > v1.3.2, see discussion in `dockcross-manylinux-build-tarball.sh`
 ${unzstd_exe} --version
 
+# -----------------------------------------------------------------------
+# Fetch build archive
+
+TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION:=_2_28}"
 TARBALL_NAME="ITKPythonBuilds-linux${TARBALL_SPECIALIZATION}.tar"
 
 if [[ ! -f ${TARBALL_NAME}.zst ]]; then
+  echo "Fetching https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/${TARBALL_NAME}.zst"
   curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/${TARBALL_NAME}.zst -O
 fi
 if [[ ! -f ./${TARBALL_NAME}.zst ]]; then
@@ -67,14 +87,31 @@ else
   tar xf ${TARBALL_NAME} --wildcards ITKPythonPackage/ITK-$1*
 fi
 rm ${TARBALL_NAME}
+
+# Optional: Update build scripts
+if [[ -n ${ITKPYTHONPACKAGE_TAG} ]]; then
+  echo "Updating build scripts to ${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}/ITKPythonPackage@${ITKPYTHONPACKAGE_TAG}"
+  git clone "https://github.com/${ITKPYTHONPACKAGE_ORG}/ITKPythonPackage.git" "IPP-tmp"
+  pushd IPP-tmp/
+  git checkout "${ITKPYTHONPACKAGE_TAG}"
+  git status
+  popd
+
+  rm -rf ITKPythonPackage/scripts/
+  cp -r IPP-tmp/scripts ITKPythonPackage/
+  rm -rf IPP-tmp/
+fi
+
 if [[ ! -f ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh ]]; then
   echo "ERROR: can not find required binary './ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh'"
   exit 255
 fi
 cp -a ITKPythonPackage/oneTBB-prefix ./
 
+# -----------------------------------------------------------------------
+
 set -- "${FORWARD_ARGS[@]}"; # Restore initial argument list
-if [[ "${TARBALL_SPECIALIZATION}" = "-manylinux_2_28_aarch64" ]]; then
+if [[ "${MANYLINUX_VERSION}" = "_2_28_aarch64" ]]; then
   ./ITKPythonPackage/scripts/manylinux_2_28_aarch64-build-module-wheels.sh "$@"
 else
   ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@"

scripts/internal/manylinux-build-common.sh

@@ -80,6 +80,6 @@ if ! type ninja > /dev/null 2>&1; then
   popd
 fi
 
-MANYLINUX_VERSION=_2_28
+MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
 
 echo "Building wheels for $ARCH using manylinux${MANYLINUX_VERSION}"

scripts/internal/manylinux-build-module-wheels.sh

@@ -15,6 +15,16 @@
 #   DOCKER_ARGS="-v /path/to/lib.so:/usr/local/lib64/lib.so"
 #   /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-module-wheels.sh
 #
+# The specialized manylinux container version should be set prior to running this script.
+# See https://github.com/dockcross/dockcross for available versions and tags.
+#
+# For example, `docker run -e <var>` can be used to set an environment variable when launching a container:
+#
+#   export MANYLINUX_VERSION=2014
+#   docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
+#   chmod u+x /tmp/dockcross-manylinux-x64
+#   /tmp/dockcross-manylinux-x64 -e MANYLINUX_VERSION manylinux-build-module-wheels.sh cp39
+#
 
 # -----------------------------------------------------------------------
 # Script argument parsing

scripts/internal/manylinux-build-wheels.sh

@@ -15,6 +15,16 @@
 #   DOCKER_ARGS="-v /path/to/lib.so:/usr/local/lib64/lib.so"
 #   /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-wheels.sh
 #
+# The specialized manylinux container version should be set prior to running this script.
+# See https://github.com/dockcross/dockcross for available versions and tags.
+#
+# For example, `docker run -e <var>` can be used to set an environment variable when launching a container:
+#
+#   export MANYLINUX_VERSION=2014
+#   docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
+#   chmod u+x /tmp/dockcross-manylinux-x64
+#   /tmp/dockcross-manylinux-x64 -e MANYLINUX_VERSION manylinux-build-module-wheels.sh cp39
+#
 
 # -----------------------------------------------------------------------
 # These variables are set in common script:

scripts/macpython-download-cache-and-build-module-wheels.sh

@@ -2,15 +2,43 @@
 
 # This module should be pull and run from an ITKModule root directory to generate the Mac python wheels of this module,
 # it is used by the .travis.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate
+# 
+# Exported variables used in this script:
+# - ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use
+# - ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use.
+#     If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed
+#     with the ITKPythonBuilds archive will be used.
+# - ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage
+#     build script source. Default is InsightSoftwareConsortium.
+#     Ignored if ITKPYTHONPACKAGE_TAG is empty.
 
+# Install dependencies
 brew update
 brew install zstd aria2 gnu-tar doxygen ninja
 brew upgrade cmake
+
+# Fetch ITKPythonBuilds archive containing ITK build artifacts
 aria2c -c --file-allocation=none -o ITKPythonBuilds-macosx.tar.zst -s 10 -x 10 https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/ITKPythonBuilds-macosx.tar.zst
 unzstd --long=31 ITKPythonBuilds-macosx.tar.zst -o ITKPythonBuilds-macosx.tar
 PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
 tar xf ITKPythonBuilds-macosx.tar --checkpoint=10000 --checkpoint-action=dot
 rm ITKPythonBuilds-macosx.tar
+
+# Optional: Update build scripts
+if [[ -n ${ITKPYTHONPACKAGE_TAG} ]]; then
+  echo "Updating build scripts to ${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}/ITKPythonPackage@${ITKPYTHONPACKAGE_TAG}"
+  git clone "https://github.com/${ITKPYTHONPACKAGE_ORG}/ITKPythonPackage.git" "IPP-tmp"
+  pushd IPP-tmp/
+  git checkout "${ITKPYTHONPACKAGE_TAG}"
+  git status
+  popd
+
+  rm -rf ITKPythonPackage/scripts/
+  cp -r IPP-tmp/scripts ITKPythonPackage/
+  rm -rf IPP-tmp/
+fi
+
+# Run build scripts
 sudo mkdir -p /Users/svc-dashboard/D/P && sudo chown $UID:$GID /Users/svc-dashboard/D/P && mv ITKPythonPackage /Users/svc-dashboard/D/P/
 sudo rm -rf /Library/Frameworks/Python.framework/Versions/*
 /Users/svc-dashboard/D/P/ITKPythonPackage/scripts/macpython-install-python.sh

scripts/manylinux_2_28_aarch64-build-module-wheels.sh

@@ -16,9 +16,17 @@
 #   export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2"
 #   scripts/dockcross-manylinux-build-module-wheels.sh cp39
 #
+# A specialized manylinux image and tag can be used by exporting to MANYLINUX_VERSION before
+# running this script. Default is _2_28_aarch64.
+#
+# For example,
+#
+#   export MANYLINUX_VERSION=2014
+#   export IMAGE_TAG=20221108-102ebcc
+#   scripts/dockcross-manylinux-build-module-wheels.sh cp39
 
-MANYLINUX_VERSION=_2_28
-IMAGE_TAG=latest
+MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28_aarch64}
+IMAGE_TAG=${IMAGE_TAG:=latest}
 
 script_dir=$(cd $(dirname $0) || exit 1; pwd)
 
@@ -35,4 +43,4 @@ if [[ -n ${LD_LIBRARY_PATH} ]]; then
 fi
 
 docker run --privileged --rm tonistiigi/binfmt --install all
-docker run --rm -it $DOCKER_ARGS -v $(pwd):/work/ quay.io/pypa/manylinux${MANYLINUX_VERSION}_aarch64:${IMAGE_TAG} "/ITKPythonPackage/scripts/internal/manylinux-aarch64-build-module-wheels.sh" "$@"
+docker run --rm -it $DOCKER_ARGS -v $(pwd):/work/ quay.io/pypa/manylinux${MANYLINUX_VERSION}:${IMAGE_TAG} "/ITKPythonPackage/scripts/internal/manylinux-aarch64-build-module-wheels.sh" "$@"

scripts/windows-download-cache-and-build-module-wheels.ps1

@@ -1,7 +1,19 @@
+# Pulls build dependencies and build an ITK remote module.
+#
+# Variables to set before calling the script:
+# - $env:ITK_PACKAGE_VERSION: Tag for ITKPythonBuilds build archive to use
+# - $env:ITKPYTHONPACKAGE_TAG: Tag for ITKPythonPackage build scripts to use.
+#     If ITKPYTHONPACKAGE_TAG is empty then the default scripts distributed
+#     with the ITKPythonBuilds archive will be used.
+# - $env:ITKPYTHONPACKAGE_ORG: Github organization or user to use for ITKPythonPackage
+#     build script source. Default is InsightSoftwareConsortium.
+#     Ignored if ITKPYTHONPACKAGE_TAG is empty.
+
 trap { Write-Error $_; Exit 1 }
 $AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12'
 [System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols
 
+# Get ITKPythonBuilds archive with ITK build artifacts
 set-alias sz "$env:ProgramFiles\7-Zip\7z.exe"
 if (-not (Test-Path env:APPVEYOR)) {
   $pythonArch = "64"
@@ -10,9 +22,34 @@ if (-not (Test-Path env:APPVEYOR)) {
 if (-not (Test-Path env:ITK_PACKAGE_VERSION)) { $env:ITK_PACKAGE_VERSION = 'v5.3.0' }
 Invoke-WebRequest -Uri "https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/$env:ITK_PACKAGE_VERSION/ITKPythonBuilds-windows.zip" -OutFile "ITKPythonBuilds-windows.zip"
 sz x ITKPythonBuilds-windows.zip -oC:\P -aoa -r
+
+# Optional: Update ITKPythonPackage build scripts
+if ($env:ITKPYTHONPACKAGE_TAG) {
+  if(!$env:ITKPYTHONPACKAGE_ORG) {
+    $env:ITKPYTHONPACKAGE_ORG="InsightSoftwareConsortium"
+  }
+
+  echo "Updating build scripts to ($env:ITKPYTHONPACKAGE_ORG)/ITKPythonPackage@($env:ITKPYTHONPACKAGE_TAG)"
+
+  pushd C:\P
+  git clone "https://github.com/($env:ITKPYTHONPACKAGE_ORG)/ITKPythonPackage.git" "IPP-tmp"
+  pushd "IPP-tmp"
+  git checkout "($env:ITKPYTHONPACKAGE_TAG)"
+  git status
+  popd
+
+  Remove-Item -Recurse -Force IPP/scripts/
+  Copy-Item -Recurse IPP-tmp/scripts IPP/
+  Remove-Item -Recurse -Force IPP-tmp/
+  popd
+}
+
+# Get other build dependencies
 Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5c0ad59d8d777f2179dd3e9c/download" -OutFile "doxygen-1.8.11.windows.bin.zip"
 sz x doxygen-1.8.11.windows.bin.zip -oC:\P\doxygen -aoa -r
 Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5bbf87ba8d777f06b91f27d6/download/grep-win.zip" -OutFile "grep-win.zip"
 sz x grep-win.zip -oC:\P\grep -aoa -r
 $env:Path += ";C:\P\grep"
+
+# Run build scripts
 C:\Python37-x64\python.exe C:\P\IPP\scripts\windows_build_module_wheels.py