From 2f85a2c2e9393e1b779a117d1ff92862228ae307 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 27 Jan 2023 22:38:32 +0530 Subject: [PATCH 1/5] Build app archives for all python versions * Builds app archives for python 3.7-3.11 for linux/mac/windows * Modifies app archive creation script to pass python version as variable * Publishes python 3.11 wheel * smoke test install and scan for all python versions in linux/mac/windows Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/scancode-release.yml | 63 ++++++++++--------- .../scancode-create-release-app-linux.sh | 4 +- .../scancode-create-release-app-macos.sh | 4 +- .../scancode-create-release-app-windows.sh | 4 +- 4 files changed, 41 insertions(+), 34 deletions(-) diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml index 60dca1ab48e..b6eda8e0150 100644 --- a/.github/workflows/scancode-release.yml +++ b/.github/workflows/scancode-release.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: true matrix: - pyver: ["3.7", "3.8", "3.9", "3.10"] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -99,22 +99,24 @@ jobs: strategy: fail-fast: true + matrix: + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 - - name: Set up Python + - name: Set up Python on ${{ matrix.pyver }} uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: ${{ matrix.pyver }} - name: Build linux app archive - run: etc/release/scancode-create-release-app-linux.sh + run: etc/release/scancode-create-release-app-linux.sh ${{ matrix.pyver }} - - name: Collect built linux app + - name: Collect built linux app for python {{ matrix.pyver }} uses: actions/upload-artifact@v3 with: - name: linux_app + name: linux_app_py_${{ matrix.pyver }} path: release/* @@ -131,22 +133,24 @@ jobs: strategy: fail-fast: true + matrix: + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 - - name: Set up Python + - name: Set up Python on ${{ matrix.pyver }} uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: ${{ matrix.pyver }} - name: Build mac app archive - run: etc/release/scancode-create-release-app-macos.sh + run: etc/release/scancode-create-release-app-macos.sh ${{ matrix.pyver }} - - name: Collect built mac app + - name: Collect built mac app for python {{ matrix.pyver }} uses: actions/upload-artifact@v3 with: - name: macos_app + name: macos_app_py_${{ matrix.pyver }} path: release/* @@ -160,24 +164,27 @@ jobs: defaults: run: shell: bash + strategy: fail-fast: true + matrix: + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 - - name: Set up Python + - name: Set up Python on ${{ matrix.pyver }} uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: ${{ matrix.pyver }} - name: Build windows app archive - run: etc/release/scancode-create-release-app-windows.sh + run: etc/release/scancode-create-release-app-windows.sh ${{ matrix.pyver }} - - name: Collect built windows app + - name: Collect built windows app for python {{ matrix.pyver }} uses: actions/upload-artifact@v3 with: - name: windows_app + name: windows_app_py_${{ matrix.pyver }} path: release/* @@ -229,7 +236,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12] - pyver: ["3.7", "3.8", "3.9", "3.10"] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -275,7 +282,7 @@ jobs: fail-fast: true matrix: os: [windows-2019, windows-2022] - pyver: ["3.7", "3.8", "3.9", "3.10"] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -320,7 +327,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04] - pyver: [3.8] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -330,10 +337,10 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Download a single artifact linux_app + - name: Download a single artifact linux_app_py_${{ matrix.pyver }} uses: actions/download-artifact@v3 with: - name: linux_app + name: linux_app_py_${{ matrix.pyver }} path: dist - name: test install app archive @@ -361,7 +368,7 @@ jobs: fail-fast: true matrix: os: [macos-11, macos-12] - pyver: [3.8] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -371,10 +378,10 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Download a single artifact macos_app + - name: Download a single artifact macos_app_py_${{ matrix.pyver }} uses: actions/download-artifact@v3 with: - name: macos_app + name: macos_app_py_${{ matrix.pyver }} path: dist - name: test install app archive @@ -402,7 +409,7 @@ jobs: fail-fast: true matrix: os: [windows-2019, windows-2022] - pyver: [3.8] + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -412,10 +419,10 @@ jobs: with: python-version: ${{ matrix.pyver }} - - name: Download a single artifact windows_app + - name: Download a single artifact windows_app_py_${{ matrix.pyver }} uses: actions/download-artifact@v3 with: - name: windows_app + name: windows_app_py_${{ matrix.pyver }} path: dist - name: test install app archive @@ -491,7 +498,7 @@ jobs: strategy: fail-fast: true matrix: - dist_names: ["wheels-3.7", "wheels-3.8", "wheels-3.9", "wheels-3.10", sdists] + dist_names: ["wheels-3.7", "wheels-3.8", "wheels-3.9", "wheels-3.10", "wheels-3.11", sdists] steps: - name: Set up Python diff --git a/etc/release/scancode-create-release-app-linux.sh b/etc/release/scancode-create-release-app-linux.sh index 329182a2a18..94aa847ae47 100755 --- a/etc/release/scancode-create-release-app-linux.sh +++ b/etc/release/scancode-create-release-app-linux.sh @@ -18,8 +18,8 @@ set -e operating_system=linux -python_dot_version=3.8 -python_version=38 +python_dot_version=$1 +python_version=${python_dot_version//.} python_exe="python$python_dot_version" release_dir=scancode-toolkit-$(git describe --tags) diff --git a/etc/release/scancode-create-release-app-macos.sh b/etc/release/scancode-create-release-app-macos.sh index 99885234297..e689f1f1b04 100755 --- a/etc/release/scancode-create-release-app-macos.sh +++ b/etc/release/scancode-create-release-app-macos.sh @@ -18,8 +18,8 @@ set -e operating_system=macos -python_dot_version=3.8 -python_version=38 +python_dot_version=$1 +python_version=${python_dot_version//.} python_exe="python3" release_dir=scancode-toolkit-$(git describe --tags) diff --git a/etc/release/scancode-create-release-app-windows.sh b/etc/release/scancode-create-release-app-windows.sh index bed0743b0b9..a7957ae15bb 100755 --- a/etc/release/scancode-create-release-app-windows.sh +++ b/etc/release/scancode-create-release-app-windows.sh @@ -17,8 +17,8 @@ set -e #set -x operating_system=windows -python_dot_version=3.8 -python_version=38 +python_dot_version=$1 +python_version=${python_dot_version//.} python_exe="py -$python_dot_version" release_dir=scancode-toolkit-$(git describe --tags) From ac6b311197f206641f1ceb1f9485872020f8b24a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 30 Jan 2023 23:46:23 +0530 Subject: [PATCH 2/5] Update utils_thirdparty.py with python 3.11 support Signed-off-by: Ayan Sinha Mahapatra --- etc/scripts/utils_thirdparty.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index addf8e5e106..14c4dc2c3b8 100755 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -115,13 +115,14 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "37", "38", "39", "310" +PYTHON_VERSIONS = "37", "38", "39", "310", "311" PYTHON_DOT_VERSIONS_BY_VER = { "37": "3.7", "38": "3.8", "39": "3.9", "310": "3.10", + "311": "3.11", } @@ -137,6 +138,7 @@ def get_python_dot_version(version): "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], } PLATFORMS_BY_OS = { From 589ce829c16c62d4e1110e63517e912f18c11391 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 30 Jan 2023 23:47:01 +0530 Subject: [PATCH 3/5] Bump MarkupSafe version Signed-off-by: Ayan Sinha Mahapatra --- requirements.txt | 2 +- setup-mini.cfg | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index c4c9cf47864..3d0054b4a26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ Jinja2==3.1.2 jsonstreams==0.6.0 license-expression==30.0.0 lxml==4.9.2 -MarkupSafe==2.1.1 +MarkupSafe==2.1.2 more-itertools==8.13.0 normality==2.3.3 packageurl-python==0.10.0 diff --git a/setup-mini.cfg b/setup-mini.cfg index 08309ca2d6a..e818b33947b 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -87,7 +87,7 @@ install_requires = jsonstreams >= 0.5.0 license_expression >= 30.0.0 lxml >= 4.9.2 - MarkupSafe >= 1.0 + MarkupSafe >= 2.1.2 packageurl_python >= 0.9.0 packvers >= 21.0.0 # use temp advanced patched release diff --git a/setup.cfg b/setup.cfg index 7356bb6f640..fda90e5e4f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,7 +87,7 @@ install_requires = jsonstreams >= 0.5.0 license_expression >= 30.0.0 lxml >= 4.9.2 - MarkupSafe >= 1.0 + MarkupSafe >= 2.1.2 packageurl_python >= 0.9.0 packvers >= 21.0.0 # use temp advanced patched release From aef205d9bd2f953bbec09504839160fafc1a5e2c Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 31 Jan 2023 00:42:18 +0530 Subject: [PATCH 4/5] Update github artifact download action Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/scancode-release.yml | 87 ++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml index b6eda8e0150..50a8785f2ac 100644 --- a/.github/workflows/scancode-release.yml +++ b/.github/workflows/scancode-release.yml @@ -439,6 +439,9 @@ jobs: - smoke_test_install_and_run_app_archives_on_windows - smoke_test_install_and_run_app_archives_on_macos - build_scancode_for_release_source + - build_scancode_for_release_windows + - build_scancode_for_release_linux + - build_scancode_for_release_macos runs-on: ubuntu-20.04 defaults: @@ -454,22 +457,94 @@ jobs: name: source_app path: dist - - name: Download a single artifact macos_app + - name: Download a single artifact macos_app for python 3.7 + uses: actions/download-artifact@v3 + with: + name: macos_app_py_3.7 + path: dist + + - name: Download a single artifact macos_app for python 3.8 + uses: actions/download-artifact@v3 + with: + name: macos_app_py_3.8 + path: dist + + - name: Download a single artifact macos_app for python 3.9 + uses: actions/download-artifact@v3 + with: + name: macos_app_py_3.9 + path: dist + + - name: Download a single artifact macos_app for python 3.10 + uses: actions/download-artifact@v3 + with: + name: macos_app_py_3.10 + path: dist + + - name: Download a single artifact macos_app for python 3.11 + uses: actions/download-artifact@v3 + with: + name: macos_app_py_3.11 + path: dist + + - name: Download a single artifact linux_app for python 3.7 + uses: actions/download-artifact@v3 + with: + name: linux_app_py_3.7 + path: dist + + - name: Download a single artifact linux_app for python 3.8 + uses: actions/download-artifact@v3 + with: + name: linux_app_py_3.8 + path: dist + + - name: Download a single artifact linux_app for python 3.9 + uses: actions/download-artifact@v3 + with: + name: linux_app_py_3.9 + path: dist + + - name: Download a single artifact linux_app for python 3.10 + uses: actions/download-artifact@v3 + with: + name: linux_app_py_3.10 + path: dist + + - name: Download a single artifact linux_app for python 3.11 + uses: actions/download-artifact@v3 + with: + name: linux_app_py_3.11 + path: dist + + - name: Download a single artifact windows_app for python 3.7 + uses: actions/download-artifact@v3 + with: + name: windows_app_py_3.7 + path: dist + + - name: Download a single artifact windows_app for python 3.8 + uses: actions/download-artifact@v3 + with: + name: windows_app_py_3.8 + path: dist + + - name: Download a single artifact windows_app for python 3.9 uses: actions/download-artifact@v3 with: - name: macos_app + name: windows_app_py_3.9 path: dist - - name: Download a single artifact linux_app + - name: Download a single artifact windows_app for python 3.10 uses: actions/download-artifact@v3 with: - name: linux_app + name: windows_app_py_3.10 path: dist - - name: Download a single artifact windows_app + - name: Download a single artifact windows_app for python 3.11 uses: actions/download-artifact@v3 with: - name: windows_app + name: windows_app_py_3.11 path: dist - name: Mock GH release From 6b0105aa882c2ecf3f761f2e5e79f71ec73c78f9 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 17 Feb 2023 22:48:28 +0530 Subject: [PATCH 5/5] Add missing $ on release script for captions Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/scancode-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml index 50a8785f2ac..20f25d99089 100644 --- a/.github/workflows/scancode-release.yml +++ b/.github/workflows/scancode-release.yml @@ -113,7 +113,7 @@ jobs: - name: Build linux app archive run: etc/release/scancode-create-release-app-linux.sh ${{ matrix.pyver }} - - name: Collect built linux app for python {{ matrix.pyver }} + - name: Collect built linux app for python ${{ matrix.pyver }} uses: actions/upload-artifact@v3 with: name: linux_app_py_${{ matrix.pyver }} @@ -147,7 +147,7 @@ jobs: - name: Build mac app archive run: etc/release/scancode-create-release-app-macos.sh ${{ matrix.pyver }} - - name: Collect built mac app for python {{ matrix.pyver }} + - name: Collect built mac app for python ${{ matrix.pyver }} uses: actions/upload-artifact@v3 with: name: macos_app_py_${{ matrix.pyver }} @@ -181,7 +181,7 @@ jobs: - name: Build windows app archive run: etc/release/scancode-create-release-app-windows.sh ${{ matrix.pyver }} - - name: Collect built windows app for python {{ matrix.pyver }} + - name: Collect built windows app for python ${{ matrix.pyver }} uses: actions/upload-artifact@v3 with: name: windows_app_py_${{ matrix.pyver }}